From d4cec15b47e7524a0765f0296f8178978ef1c06c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:20:54 -0700 Subject: [PATCH] refactor(tools): first-wave simplification of tools/ (file ops split, lazy_deps, code_exec, approval, browser, delegate, mcp, skills, terminal, voice, media) Behavior-neutral structural pass over tools/*: god-file extractions into sibling modules (file_operations_common/lint/search, file_tools_paths/ read_tracking/write, code_execution_env/rpc, tool_search_catalog/names/ validation, tts_command_provider, ...), duplicate helper unification, if/elif -> dispatch tables, dead-code removal, docstring compaction. Tool schemas (get_tool_definitions) verified byte-identical to base. --- tests/tools/test_async_delegation.py | 32 - tests/tools/test_bot_mode_dm.py | 2 +- tests/tools/test_clarify_gateway.py | 43 +- tests/tools/test_debug_helpers.py | 9 - tests/tools/test_image_generation.py | 2 - tests/tools/test_managed_tool_gateway.py | 277 --- tests/tools/test_read_extract.py | 15 +- tests/tools/test_send_message_tool.py | 46 +- tests/tools/test_tool_result_storage.py | 16 - tests/tools/test_transcription_tools.py | 2 +- tests/tools/test_tts_command_providers.py | 6 +- tools/__init__.py | 17 +- tools/ansi_strip.py | 66 +- tools/apply_layout_tool.py | 40 +- tools/approval.py | 2131 +---------------- tools/approval_detection.py | 2059 ++++++++++++++++ tools/async_delegation.py | 1291 ++++------- tools/binary_extensions.py | 44 +- tools/blueprints.py | 113 +- tools/bot_mode_dm.py | 337 +-- tools/bot_mode_probe.py | 326 ++- tools/bot_relay.py | 331 +-- tools/browser_camofox.py | 554 ++--- tools/browser_cdp_tool.py | 277 +-- tools/browser_dialog_tool.py | 25 +- tools/browser_extension_router.py | 176 +- tools/browser_lightpanda.py | 44 +- tools/browser_supervisor.py | 622 ++--- tools/clarify_gateway.py | 457 +--- tools/clarify_tool.py | 270 +-- tools/close_preview_tool.py | 57 +- tools/code_execution_env.py | 342 +++ tools/code_execution_rpc.py | 270 +++ tools/credential_files.py | 390 +--- tools/cronjob_job_args.py | 468 ++++ tools/cronjob_prompt_scan.py | 174 ++ tools/cronjob_tools.py | 2067 +++++------------ tools/debug_helpers.py | 49 +- tools/delegation_live_log.py | 232 +- tools/desktop_ui.py | 48 +- tools/discord_tool.py | 621 ++--- tools/drive_preview_tool.py | 48 +- tools/fal_common.py | 57 +- tools/feishu_drive_tool.py | 201 +- tools/file_operations_common.py | 296 +++ tools/file_operations_lint.py | 369 +++ tools/file_operations_search.py | 638 +++++ tools/file_state.py | 163 +- tools/file_tools.py | 2580 +++++---------------- tools/file_tools_paths.py | 270 +++ tools/file_tools_read_tracking.py | 329 +++ tools/file_tools_write_guards.py | 530 +++++ tools/focus_pane_tool.py | 28 +- tools/fuzzy_match.py | 1120 ++------- tools/fuzzy_match_strategies.py | 334 +++ tools/homeassistant_tool.py | 246 +- tools/hook_output_spill.py | 95 +- tools/image_generation_catalog.py | 567 +++++ tools/image_generation_tool.py | 1446 +++--------- tools/kanban_tools.py | 2549 ++++++-------------- tools/kanban_tools_schemas.py | 511 ++++ tools/lazy_deps.py | 930 ++------ tools/managed_tool_gateway.py | 279 +-- tools/open_preview_tool.py | 68 +- tools/openrouter_client.py | 34 +- tools/osv_check.py | 45 +- tools/patch_parser.py | 690 ++---- tools/path_security.py | 31 +- tools/plugin_guard.py | 238 +- tools/preview_tool.py | 51 +- tools/process_registry_notifications.py | 369 +++ tools/read_extract.py | 429 ++-- tools/read_preview_tool.py | 58 +- tools/read_window_tool.py | 18 +- tools/self_repo_guard.py | 370 ++- tools/send_message_senders.py | 891 +++++++ tools/send_message_targets.py | 326 +++ tools/send_message_tool.py | 2348 +++---------------- tools/shell_heredoc.py | 149 +- tools/skill_linter.py | 302 +-- tools/skill_provenance.py | 54 +- tools/skillevaluator_scan.py | 81 +- tools/skills_ast_audit.py | 21 +- tools/slash_confirm.py | 103 +- tools/spill_safety.py | 59 +- tools/terminal_hints.py | 217 +- tools/terminal_scope.py | 171 +- tools/threat_patterns.py | 160 +- tools/tip_tool.py | 23 +- tools/tirith_security.py | 668 ++---- tools/todo_tool.py | 270 +-- tools/tool_backend_helpers.py | 255 +- tools/tool_output_limits.py | 66 +- tools/tool_result_storage.py | 189 +- tools/tool_search.py | 1067 ++------- tools/tool_search_catalog.py | 307 +++ tools/tool_search_names.py | 7 + tools/tool_search_validation.py | 173 ++ tools/tour_tool.py | 60 +- tools/transcription_tools.py | 314 +-- tools/tts_command_provider.py | 275 +++ tools/tts_tool.py | 281 +-- tools/url_safety.py | 507 ++-- tools/video_generation_tool.py | 236 +- tools/web_result_cache.py | 193 +- tools/web_tools.py | 1524 ++++-------- tools/web_tools_rescue.py | 142 ++ tools/website_policy.py | 149 +- tools/working_diff.py | 60 +- tools/write_approval.py | 350 +-- tools/x_search_tool.py | 389 ++-- tools/xai_http.py | 213 +- tools/xai_video_tools.py | 233 +- tools/yuanbao_tools.py | 292 +-- 114 files changed, 18160 insertions(+), 25300 deletions(-) create mode 100644 tools/approval_detection.py create mode 100644 tools/code_execution_env.py create mode 100644 tools/code_execution_rpc.py create mode 100644 tools/cronjob_job_args.py create mode 100644 tools/cronjob_prompt_scan.py create mode 100644 tools/file_operations_common.py create mode 100644 tools/file_operations_lint.py create mode 100644 tools/file_operations_search.py create mode 100644 tools/file_tools_paths.py create mode 100644 tools/file_tools_read_tracking.py create mode 100644 tools/file_tools_write_guards.py create mode 100644 tools/fuzzy_match_strategies.py create mode 100644 tools/image_generation_catalog.py create mode 100644 tools/kanban_tools_schemas.py create mode 100644 tools/process_registry_notifications.py create mode 100644 tools/send_message_senders.py create mode 100644 tools/send_message_targets.py create mode 100644 tools/tool_search_catalog.py create mode 100644 tools/tool_search_names.py create mode 100644 tools/tool_search_validation.py create mode 100644 tools/tts_command_provider.py create mode 100644 tools/web_tools_rescue.py diff --git a/tests/tools/test_async_delegation.py b/tests/tools/test_async_delegation.py index 5d0a01c22a..9c9766a7a1 100644 --- a/tests/tools/test_async_delegation.py +++ b/tests/tools/test_async_delegation.py @@ -121,38 +121,6 @@ def test_connect_preserves_wal_and_applies_macos_durability_barriers( conn.close() -def test_active_for_session_counts_every_live_delegation_state(): - with ad._records_lock: - ad._records.update( - { - "running": { - "status": "running", - "origin_ui_session_id": "desktop-sid", - }, - "stalling": { - "status": "stalling", - "origin_ui_session_id": "desktop-sid", - }, - "finalizing": { - "status": "finalizing", - "origin_ui_session_id": "desktop-sid", - }, - "completed": { - "status": "completed", - "origin_ui_session_id": "desktop-sid", - }, - "other-session": { - "status": "running", - "origin_ui_session_id": "other-sid", - }, - } - ) - - assert ad.active_for_session("desktop-sid") == 3 - assert ad.active_for_session("other-sid") == 1 - assert ad.active_for_session("") == 0 - - def test_dispatch_returns_immediately_without_blocking(): gate = threading.Event() diff --git a/tests/tools/test_bot_mode_dm.py b/tests/tools/test_bot_mode_dm.py index ed0936f4c9..7a5de3b96d 100644 --- a/tests/tools/test_bot_mode_dm.py +++ b/tests/tools/test_bot_mode_dm.py @@ -681,7 +681,7 @@ def test_sweeper_removes_only_stale_dm_files(tmp_path, monkeypatch): old = now - bot_mode_dm._DM_STALE_SECONDS - 1 os.utime(legacy_stale, (old, old)) os.utime(stale, (old, old)) - bot_mode_dm._sweep_stale_dm_files(now=now) + bot_mode_dm.cleanup_bot_dm_cache(now=now) assert not legacy_stale.exists() assert not stale.exists() diff --git a/tests/tools/test_clarify_gateway.py b/tests/tools/test_clarify_gateway.py index e06bf9d608..218ab3ce96 100644 --- a/tests/tools/test_clarify_gateway.py +++ b/tests/tools/test_clarify_gateway.py @@ -130,26 +130,6 @@ class TestClarifyPrimitive: assert result == "B" - def test_notify_register_unregister_clears_pending(self): - """unregister_notify cancels any pending clarify so threads unwind.""" - from tools import clarify_gateway as cm - - cm.register("id9", "sk9", "Q?", ["A"]) - - def waiter(): - return cm.wait_for_response("id9", timeout=10.0) - - with ThreadPoolExecutor(1) as pool: - fut = pool.submit(waiter) - time.sleep(0.05) - - cm.register_notify("sk9", lambda entry: None) - cm.unregister_notify("sk9") - - # unregister_notify calls clear_session; thread unwinds - result = fut.result(timeout=10.0) - assert result == "" - def test_session_index_isolation(self): """Entries from different sessions don't leak across get_pending lookups.""" from tools import clarify_gateway as cm @@ -223,24 +203,11 @@ class TestGatewayTextIntercept: class TestCoverageGaps: - """Cover remaining branches: signature(), get_entry miss, find_awaiting - with deleted entry, cancel with None entry, timeout exception, get_notify.""" + """Cover remaining branches: unknown-id wait, timeout config exception.""" def setup_method(self): _clear_clarify_state() - def test_entry_signature(self): - """_ClarifyEntry.signature() returns the expected dict.""" - from tools import clarify_gateway as cm - - entry = cm.register("sig1", "sk", "Q?", ["A", "B"]) - sig = entry.signature() - assert sig["clarify_id"] == "sig1" - assert sig["session_key"] == "sk" - assert sig["question"] == "Q?" - assert sig["choices"] == ["A", "B"] - - def test_wait_for_response_unknown_id_returns_none(self): """wait_for_response on a non-existent id returns None immediately.""" from tools import clarify_gateway as cm @@ -257,13 +224,6 @@ class TestCoverageGaps: assert cm.get_clarify_timeout() == 3600 - def test_get_notify_returns_none_when_not_registered(self): - """get_notify returns None for an unregistered session.""" - from tools import clarify_gateway as cm - - assert cm.get_notify("unregistered") is None - - class TestClarifyTimeoutResolution: """resolve_clarify_timeout is the single source of truth for the clarify timeout, shared by the CLI, TUI/desktop, and messaging-gateway paths.""" @@ -335,7 +295,6 @@ class TestMultiSelectTextFallback: def test_register_stores_multi_select_flag(self): entry = self._register_multi() assert entry.multi_select is True - assert entry.signature()["multi_select"] is True def test_multi_select_without_choices_is_ignored(self): diff --git a/tests/tools/test_debug_helpers.py b/tests/tools/test_debug_helpers.py index 3d4fbfca4f..19e456e103 100644 --- a/tests/tools/test_debug_helpers.py +++ b/tests/tools/test_debug_helpers.py @@ -16,15 +16,6 @@ class TestDebugSessionDisabled: assert ds.enabled is False - def test_get_session_info_disabled(self): - ds = DebugSession("test_tool", env_var="FAKE_DEBUG_VAR_XYZ") - info = ds.get_session_info() - assert info["enabled"] is False - assert info["session_id"] is None - assert info["log_path"] is None - assert info["total_calls"] == 0 - - class TestDebugSessionEnabled: """When the env var is set to 'true', DebugSession records and saves.""" diff --git a/tests/tools/test_image_generation.py b/tests/tools/test_image_generation.py index d39d121480..6631388491 100644 --- a/tests/tools/test_image_generation.py +++ b/tests/tools/test_image_generation.py @@ -453,13 +453,11 @@ class TestKreaModelNormalization: def test_native_models_detected(self, image_tool): for mid in ("krea-2-medium", "krea-2-large", "krea-2-medium-turbo"): - assert image_tool.is_krea_model(mid) is True assert image_tool._normalize_krea_model(mid) == mid def test_non_krea_models_are_not_krea(self, image_tool): for mid in ("fal-ai/flux-2/klein/9b", "fal-ai/nano-banana-pro", None, "", 123): - assert image_tool.is_krea_model(mid) is False assert image_tool._normalize_krea_model(mid) is None diff --git a/tests/tools/test_managed_tool_gateway.py b/tests/tools/test_managed_tool_gateway.py index f690cf8dc2..7f5d788c14 100644 --- a/tests/tools/test_managed_tool_gateway.py +++ b/tests/tools/test_managed_tool_gateway.py @@ -1,4 +1,3 @@ -import asyncio import os import json from datetime import datetime, timedelta, timezone @@ -7,7 +6,6 @@ from pathlib import Path import sys from unittest.mock import patch -import pytest MODULE_PATH = Path(__file__).resolve().parents[2] / "tools" / "managed_tool_gateway.py" MODULE_SPEC = spec_from_file_location("managed_tool_gateway_test_module", MODULE_PATH) @@ -103,281 +101,6 @@ def test_read_nous_access_token_refreshes_expiring_cached_token(tmp_path, monkey assert managed_tool_gateway.read_nous_access_token() == "fresh-token" -def test_managed_vendor_endpoints_pin_the_deployed_gateway_url(): - """The exact URL an agent may connect to is a code fact, not a lookup. - - Exercises the real ``build_vendor_gateway_url`` (which once resolved a - typo'd pseudo-vendor to a non-existent host while every other test stubbed - it): default builder, real deployed host, pinned vendor path. - """ - with patch.dict( - os.environ, - {"TOOL_GATEWAY_DOMAIN": "nousresearch.com", "TOOL_GATEWAY_SCHEME": "https"}, - clear=False, - ): - os.environ.pop("TOOL_GATEWAY_URL", None) - endpoints = managed_tool_gateway.managed_vendor_endpoints("vendorx") - - assert endpoints == { - "origin": "https://tool-gateway.nousresearch.com", - "base_url": "https://tool-gateway.nousresearch.com/api/vendorx", - "upload_path": "/api/uploads/vendorx", - } - - -def test_managed_vendor_endpoints_do_not_consult_entitlement(): - """Address resolution, not a policy decision. - - What an account may spend is the gateway's ruling, stated in its refusals. - Guessing at it here would hide the address from a caller the server would - have served, so entitlement must not be read on this path at all. - """ - with patch.dict(os.environ, {"TOOL_GATEWAY_DOMAIN": "nousresearch.com"}, clear=False), \ - patch.object( - managed_tool_gateway, - "managed_nous_tools_enabled", - side_effect=AssertionError("entitlement must not gate address resolution"), - ): - os.environ.pop("TOOL_GATEWAY_URL", None) - endpoints = managed_tool_gateway.managed_vendor_endpoints("vendorx") - - assert endpoints is not None - assert endpoints["base_url"] == "https://tool-gateway.nousresearch.com/api/vendorx" - - -def test_managed_vendor_endpoints_are_none_when_no_origin_resolves(): - # A misconfigured scheme leaves nothing to call, and the caller reports - # that rather than building a URL out of a broken setting. - with patch.dict(os.environ, {"TOOL_GATEWAY_SCHEME": "ftp"}, clear=False): - os.environ.pop("TOOL_GATEWAY_URL", None) - assert managed_tool_gateway.managed_vendor_endpoints("vendorx") is None - - -def test_managed_gateway_auth_headers_carry_the_bearer(): - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - headers = managed_tool_gateway.managed_gateway_auth_headers( - "https://tool-gateway.example.com/api/vendorx/generations", - gateway_builder=lambda vendor: f"https://{vendor}-gateway.example.com", - token_reader=lambda: "nous-token", - ) - - assert headers == {"Authorization": "Bearer nous-token"} - - -def test_managed_gateway_auth_headers_reflect_a_rotated_token(): - # Read fresh on every call: a Nous access token expires within the hour, - # and a long session must not keep presenting a dead bearer. - tokens = iter(["first-token", "second-token"]) - builder = lambda vendor: f"https://{vendor}-gateway.example.com" - url = "https://tool-gateway.example.com/api/vendorx/generations" - - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - first = managed_tool_gateway.managed_gateway_auth_headers(url, builder, lambda: next(tokens)) - second = managed_tool_gateway.managed_gateway_auth_headers(url, builder, lambda: next(tokens)) - - assert first["Authorization"] == "Bearer first-token" - assert second["Authorization"] == "Bearer second-token" - - -def test_managed_gateway_auth_headers_refuse_a_url_off_the_gateway_origin(): - # Gated on the URL, never a name: our bearer must never be handed to a - # host that merely looks managed. - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - assert managed_tool_gateway.managed_gateway_auth_headers( - "https://attacker.example/api/vendorx/generations", - gateway_builder=lambda vendor: f"https://{vendor}-gateway.example.com", - token_reader=lambda: "nous-token", - ) == {} - - -def test_managed_gateway_auth_headers_empty_without_a_token(): - # Empty rather than raising, so a caller can say "sign in" instead of - # sending an unauthenticated request. - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - assert managed_tool_gateway.managed_gateway_auth_headers( - "https://tool-gateway.example.com/api/vendorx/generations", - gateway_builder=lambda vendor: f"https://{vendor}-gateway.example.com", - token_reader=lambda: None, - ) == {} - - -class TestManagedMediaUploader: - """The presign -> PUT -> ``nous-upload:`` protocol. - - This is the only way a local image or video reaches a managed vendor, and - the pieces it gets right are not incidental: the presigned URL signs the - content type and byte length, so a PUT that disagrees with the presign is - rejected by storage rather than by us. - """ - - GATEWAY = "https://tool-gateway.example.com" - BASE_URL = f"{GATEWAY}/api/vendorx" - UPLOAD_PATH = "/api/uploads/vendorx" - - def _uploader(self, **kwargs): - return managed_tool_gateway.build_managed_media_uploader( - kwargs.pop("server_url", self.BASE_URL), - kwargs.pop("upload_path", self.UPLOAD_PATH), - gateway_builder=lambda vendor: self.GATEWAY, - token_reader=kwargs.pop("token_reader", lambda: "nous-token"), - ) - - @staticmethod - def _response(status_code=200, payload=None): - class _R: - def __init__(self): - self.status_code = status_code - - def json(self): - if payload is None: - raise ValueError("no json") - return payload - - return _R() - - def _run(self, uploader, data=b"bytes", mime="image/png", presign=None, put=None): - """Drive one upload with both HTTP legs stubbed; returns the calls made.""" - import httpx - - from tools import url_safety - - calls = {"presign": [], "put": []} - presign = presign if presign is not None else self._response( - 200, {"uploadUrl": "https://storage.example/put?sig=abc", "token": "tok-1"} - ) - put = put if put is not None else self._response(200) - - class _PresignClient: - def __init__(self, **_kw): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *_exc): - return False - - async def post(self, url, headers=None, json=None): - calls["presign"].append({"url": url, "headers": headers, "json": json}) - return presign - - class _PutClient: - async def __aenter__(self): - return self - - async def __aexit__(self, *_exc): - return False - - async def put(self, url, content=None, headers=None): - calls["put"].append({"url": url, "content": content, "headers": headers}) - return put - - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True), \ - patch.object(httpx, "AsyncClient", _PresignClient), \ - patch.object(url_safety, "create_ssrf_safe_async_client", lambda **_kw: _PutClient()): - calls["result"] = asyncio.run(uploader(data, mime)) - return calls - - def test_presign_declares_the_exact_type_and_length_the_put_then_sends(self): - # Storage validates the PUT against what was signed, so a mismatch - # between these two is a rejection with no useful error. - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - uploader = self._uploader() - data = b"\x89PNG\r\n\x1a\n" + b"payload" * 100 - - calls = self._run(uploader, data=data, mime="image/png") - - assert calls["presign"][0]["url"] == f"{self.GATEWAY}{self.UPLOAD_PATH}" - assert calls["presign"][0]["json"] == { - "contentType": "image/png", - "contentLength": len(data), - } - assert calls["presign"][0]["headers"]["Authorization"] == "Bearer nous-token" - assert calls["put"][0]["url"] == "https://storage.example/put?sig=abc" - assert calls["put"][0]["content"] == data - assert calls["put"][0]["headers"] == {"Content-Type": "image/png"} - assert calls["result"] == "nous-upload:tok-1" - - def test_the_bytes_go_to_storage_and_never_through_the_gateway(self): - # The whole point of presigning is that the gateway's request-size - # ceiling does not apply to a 50MB clip. - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - uploader = self._uploader() - - calls = self._run(uploader, data=b"v" * 4096, mime="video/mp4") - - assert len(calls["presign"]) == 1 and len(calls["put"]) == 1 - assert self.GATEWAY not in calls["put"][0]["url"] - assert calls["presign"][0]["json"]["contentType"] == "video/mp4" - - def test_no_uploader_when_the_url_is_not_a_managed_gateway(self): - # Refusing to build is what makes the caller say "pass a URL instead" - # rather than forwarding a raw local path to a third party. - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - assert self._uploader(server_url="https://attacker.example/api/vendorx") is None - - @pytest.mark.parametrize("upload_path", [None, "", "api/uploads/vendorx", 42]) - def test_no_uploader_without_a_rooted_upload_path(self, upload_path): - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - assert self._uploader(upload_path=upload_path) is None - - def test_a_missing_credential_fails_before_any_request(self): - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - uploader = self._uploader() - - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True), \ - patch.object(managed_tool_gateway, "managed_gateway_auth_headers", return_value={}): - with pytest.raises(RuntimeError, match="no Nous credential"): - asyncio.run(uploader(b"x", "image/png")) - - def test_a_gateway_refusal_surfaces_its_own_message(self): - # Quota and size refusals carry guidance written for the model; a bare - # status code would throw that away. - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - uploader = self._uploader() - refusal = self._response( - 413, {"error": {"message": "That file is 82MB; the limit for video is 50MB."}} - ) - - with pytest.raises(RuntimeError, match="the limit for video is 50MB"): - self._run(uploader, presign=refusal) - - def test_an_unreadable_refusal_still_reports_the_status(self): - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - uploader = self._uploader() - - with pytest.raises(RuntimeError, match="HTTP 502"): - self._run(uploader, presign=self._response(502, None)) - - @pytest.mark.parametrize( - "payload", - [ - {}, - {"uploadUrl": "https://storage.example/put"}, - {"token": "tok-1"}, - {"uploadUrl": "", "token": "tok-1"}, - {"uploadUrl": "https://storage.example/put", "token": ""}, - ], - ) - def test_a_malformed_presign_response_is_refused_rather_than_guessed(self, payload): - # Half a presign must not become a PUT to nowhere or an empty token - # that later reads as a valid reference. - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - uploader = self._uploader() - - with pytest.raises(RuntimeError, match="malformed"): - self._run(uploader, presign=self._response(200, payload)) - - def test_a_storage_rejection_is_not_reported_as_a_successful_upload(self): - # A signature mismatch answers non-200 with an XML body; returning a - # token here would hand the vendor a reference to nothing. - with patch.object(managed_tool_gateway, "managed_nous_tools_enabled", return_value=True): - uploader = self._uploader() - - with pytest.raises(RuntimeError, match="storage refused the upload"): - self._run(uploader, put=self._response(403)) - - def test_is_managed_tool_gateway_ready_skips_refresh_for_expired_cached_token(tmp_path, monkeypatch): monkeypatch.delenv("TOOL_GATEWAY_USER_TOKEN", raising=False) monkeypatch.setenv("HERMES_HOME", str(tmp_path)) diff --git a/tests/tools/test_read_extract.py b/tests/tools/test_read_extract.py index de2069c8cd..e44881728e 100644 --- a/tests/tools/test_read_extract.py +++ b/tests/tools/test_read_extract.py @@ -805,27 +805,22 @@ class TestPdfCoverageNote(unittest.TestCase): self.assertEqual(self._note_with_counts(None), "") self.assertEqual(self._note_with_counts([0]), "") # single page - def test_page_ranges_compact(self): - from tools.read_extract import _page_ranges - self.assertEqual(_page_ranges([2, 3, 4, 7, 9, 10]), "2-4, 7, 9-10") - self.assertEqual(_page_ranges([5]), "5") - - def test_page_char_counts_missing_pdftotext(self): + def test_page_texts_missing_pdftotext(self): from tools import read_extract with mock.patch.object(read_extract.shutil, "which", return_value=None): - self.assertIsNone(read_extract._pdf_page_char_counts("/x/doc.pdf")) + self.assertIsNone(read_extract._pdf_page_texts("/x/doc.pdf")) - def test_page_char_counts_parses_formfeeds(self): + def test_page_texts_parses_formfeeds(self): from tools import read_extract fake = mock.Mock(returncode=0, stdout=b"alpha beta\fgamma\f\f") with mock.patch.object(read_extract.shutil, "which", return_value="/usr/bin/pdftotext"), \ mock.patch.object(read_extract.subprocess, "run", return_value=fake): - counts = read_extract._pdf_page_char_counts("/x/doc.pdf") + pages = read_extract._pdf_page_texts("/x/doc.pdf") # Trailing empty segment after the final \f is dropped; the real # empty page between the two \f markers is preserved. - self.assertEqual(counts, [len("alpha beta"), len("gamma"), 0]) + self.assertEqual(pages, ["alpha beta", "gamma", ""]) def test_extract_anydoc_prepends_note_for_pdf(self): """The warning leads the extracted text for .pdf inputs (a trailing diff --git a/tests/tools/test_send_message_tool.py b/tests/tools/test_send_message_tool.py index 5aaffdd590..0632fb3acb 100644 --- a/tests/tools/test_send_message_tool.py +++ b/tests/tools/test_send_message_tool.py @@ -1741,50 +1741,6 @@ class TestSendViaAdapterStandaloneFallback: assert result == {"error": "Plugin standalone send failed: boom!"} -# --------------------------------------------------------------------------- -# _check_send_message — availability gating -# --------------------------------------------------------------------------- - -class TestCheckSendMessage: - """The tool's check_fn governs whether the model sees ``send_message`` as - callable for a given session. The four passing conditions are: - - 1. ``HERMES_KANBAN_TASK`` is set (worker spawned by the kanban dispatcher - — parent gateway is by definition running, but the worker's - ``HERMES_HOME`` may be a profile dir without a ``gateway.pid``). - 2. ``HERMES_SESSION_PLATFORM`` resolves to a non-empty, non-``local`` value - (the session is wired to a messaging platform like Telegram). - 3. ``is_gateway_running()`` returns True (CLI / orchestrator profile with - a live gateway colocated under the same ``HERMES_HOME``). - 4. None of the above → False, tool is hidden. - """ - - def test_kanban_task_env_grants_access(self, monkeypatch): - """Workers spawned by the dispatcher (HERMES_KANBAN_TASK set) must be - allowed regardless of session_platform / gateway-pid state.""" - from tools.send_message_tool import _check_send_message - - monkeypatch.setenv("HERMES_KANBAN_TASK", "t_abc12345") - monkeypatch.delenv("HERMES_SESSION_PLATFORM", raising=False) - - with patch("gateway.session_context.get_session_env", return_value=""), \ - patch("gateway.status.is_gateway_running", return_value=False): - assert _check_send_message() is True - - - def test_gateway_status_import_error_is_swallowed(self, monkeypatch): - """If gateway.status can't be imported (unusual deployment / partial - install), the check returns False rather than raising.""" - from tools.send_message_tool import _check_send_message - - monkeypatch.delenv("HERMES_KANBAN_TASK", raising=False) - - with patch("gateway.session_context.get_session_env", return_value=""), \ - patch("gateway.status.is_gateway_running", - side_effect=ImportError("simulated")): - assert _check_send_message() is False - - class TestSendTelegramThreadNotFoundRetry: """Tests for thread-not-found retry behaviour in _send_telegram (#27012).""" @@ -1801,7 +1757,7 @@ class TestSendTelegramThreadNotFoundRetry: async def run_test(): with patch( - "tools.send_message_tool._send_telegram_message_with_retry", + "tools.send_message_senders._send_telegram_message_with_retry", fake_retry, ): # _send_telegram imports Bot locally; we only need to mock diff --git a/tests/tools/test_tool_result_storage.py b/tests/tools/test_tool_result_storage.py index c72d8f933f..4e04ffc47c 100644 --- a/tests/tools/test_tool_result_storage.py +++ b/tests/tools/test_tool_result_storage.py @@ -9,12 +9,10 @@ from tools.budget_config import ( BudgetConfig, ) from tools.tool_result_storage import ( - HEREDOC_MARKER, PERSISTED_OUTPUT_TAG, PERSISTED_OUTPUT_CLOSING_TAG, STORAGE_DIR, _build_persisted_message, - _heredoc_marker, _resolve_storage_dir, _safe_result_filename, _write_to_sandbox, @@ -43,20 +41,6 @@ class TestGeneratePreview: assert has_more is False -# ── _heredoc_marker ─────────────────────────────────────────────────── - -class TestHeredocMarker: - def test_default_marker_when_no_collision(self): - assert _heredoc_marker("normal content") == HEREDOC_MARKER - - def test_uuid_marker_on_collision(self): - content = f"some text with {HEREDOC_MARKER} embedded" - marker = _heredoc_marker(content) - assert marker != HEREDOC_MARKER - assert marker.startswith("HERMES_PERSIST_") - assert marker not in content - - # ── _write_to_sandbox ───────────────────────────────────────────────── class TestWriteToSandbox: diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index 30b66429b6..9c452fe582 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -246,7 +246,7 @@ class TestTranscribeLocalCommand: captured["env"] = kwargs["env"] return Proc() - monkeypatch.setattr("tools.transcription_tools.subprocess.Popen", fake_popen) + monkeypatch.setattr("tools.tts_command_provider.subprocess.Popen", fake_popen) from tools.transcription_tools import _run_command_stt diff --git a/tests/tools/test_tts_command_providers.py b/tests/tools/test_tts_command_providers.py index 8072ab45d6..631d7d183a 100644 --- a/tests/tools/test_tts_command_providers.py +++ b/tests/tools/test_tts_command_providers.py @@ -128,7 +128,7 @@ class TestCommandTtsEnv: captured["env"] = kwargs["env"] return Proc() - monkeypatch.setattr("tools.tts_tool.subprocess.Popen", fake_popen) + monkeypatch.setattr("tools.tts_command_provider.subprocess.Popen", fake_popen) result = _run_command_tts("echo hi", timeout=1) @@ -347,7 +347,7 @@ class TestRunCommandTts: def wait(self, timeout=None): return self.returncode - with patch("tools.tts_tool.subprocess.Popen", return_value=FakeProcess()): + with patch("tools.tts_command_provider.subprocess.Popen", return_value=FakeProcess()): result = _run_command_tts("fake tts", timeout=0.25) assert result.returncode == 0 @@ -524,7 +524,7 @@ class TestCommandTtsEnvPassthrough: captured["env"] = kwargs["env"] return Proc() - monkeypatch.setattr("tools.tts_tool.subprocess.Popen", fake_popen) + monkeypatch.setattr("tools.tts_command_provider.subprocess.Popen", fake_popen) result = _run_command_tts( "echo hi", timeout=1, env_passthrough=["MY_TTS_API_KEY"] diff --git a/tools/__init__.py b/tools/__init__.py index 3214b979e5..8fac3d7c81 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -1,18 +1,7 @@ #!/usr/bin/env python3 -"""Tools package namespace. - -Keep package import side effects minimal. Importing ``tools`` should not -eagerly import the full tool stack, because several subsystems load tools while -``hermes_cli.config`` is still initializing. - -Callers should import concrete submodules directly, for example: - - import tools.web_tools - from tools import browser_tool - -Python will resolve those submodules via the package path without needing them -to be re-exported here. -""" +"""Tools package namespace. Kept side-effect free: importing ``tools`` must not +load the tool stack, since some subsystems import tools while +``hermes_cli.config`` is still initializing. Import concrete submodules directly.""" def check_file_requirements(): diff --git a/tools/ansi_strip.py b/tools/ansi_strip.py index c344c1aafb..3e880363f3 100644 --- a/tools/ansi_strip.py +++ b/tools/ansi_strip.py @@ -1,9 +1,8 @@ """Strip ANSI escape sequences from subprocess output. -Used by terminal_tool, code_execution_tool, and process_registry to clean -command output before returning it to the model. This prevents ANSI codes -from entering the model's context — which is the root cause of models -copying escape sequences into file writes. +Used by terminal_tool, code_execution_tool, and process_registry so ANSI codes +never enter the model's context (the root cause of models copying escape +sequences into file writes). Covers the full ECMA-48 spec: CSI (including private-mode ``?`` prefix, colon-separated params, intermediate bytes), OSC (BEL and ST terminators), @@ -31,28 +30,20 @@ _ANSI_ESCAPE_RE = re.compile( # Fast-path check — skip full regex when no escape-like bytes are present. _HAS_ESCAPE = re.compile(r"[\x1b\x80-\x9f]") -# C0 control characters (minus tab/newline/carriage-return, handled -# separately) plus DEL. These survive strip_ansi() — it only removes -# well-formed escape *sequences* — but are still dangerous or garbled -# when echoed back to a terminal (BEL rings, backspace/DEL overwrite, -# NUL truncates in some terminals). +# C0 controls (minus tab/newline/CR, handled separately) plus DEL. They survive +# strip_ansi() — it only removes well-formed *sequences* — but are dangerous when +# echoed to a terminal (BEL rings, backspace/DEL overwrite, NUL truncates). _CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") # Fast-path check for sanitize_display_text — any C0 control (except # tab/newline), CR, DEL, ESC, or C1 byte triggers the slow path. _HAS_CONTROL = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]") -# Unicode TAG characters (U+E0000–U+E007F). Deprecated as language tags, -# these render as nothing in every terminal and chat UI but are perfectly -# visible to an LLM tokenizer — the classic "ASCII smuggling" prompt-injection -# channel (hide `\u{E0069}\u{E0067}\u{E006E}...` = invisible instructions -# inside otherwise benign tool output). Ported from block/goose#10746. -# -# The ONLY legitimate modern use is emoji tag sequences (Unicode TR51): -# a U+1F3F4 black-flag base followed by tag spec characters and the -# U+E007F CANCEL TAG terminator (e.g. the flags of Scotland/Wales/England). -# goose strips those too; we preserve them — same rationale as keeping ZWJ -# inside emoji sequences. +# Unicode TAG characters (U+E0000–U+E007F) render as nothing in terminals and +# chat UIs but are visible to LLM tokenizers — the "ASCII smuggling" injection +# channel. The only legitimate modern use is emoji tag sequences (TR51: U+1F3F4 +# base + tag spec + U+E007F CANCEL TAG, e.g. the Scotland/Wales flags); those +# are preserved, same rationale as keeping ZWJ inside emoji sequences. _UNICODE_TAG_SUB_RE = re.compile( r"(\U0001F3F4[\U000E0020-\U000E007E]+\U000E007F)" # valid emoji tag seq (kept) r"|[\U000E0000-\U000E007F]" # any other tag char (stripped) @@ -63,12 +54,7 @@ _HAS_UNICODE_TAG = re.compile(r"[\U000E0000-\U000E007F]") def strip_ansi(text: str) -> str: - """Remove ANSI escape sequences from text. - - Returns the input unchanged (fast path) when no ESC or C1 bytes are - present. Safe to call on any string — clean text passes through - with negligible overhead. - """ + """Remove ANSI escape sequences; clean text passes through unchanged (fast path).""" if not text or not _HAS_ESCAPE.search(text): return text return _ANSI_ESCAPE_RE.sub("", text) @@ -77,18 +63,11 @@ def strip_ansi(text: str) -> str: def sanitize_display_text(text: str) -> str: """Sanitize stored/untrusted text before echoing it to a terminal. - Removes ANSI/ECMA-48 escape sequences AND bare control characters, - preserving only newlines and tabs (carriage returns are normalized - to newlines so ``\\r``-overwrite spoofing can't hide content). - - Use this when re-rendering conversation history or other persisted - text in a terminal UI (e.g. the ``/resume`` recap): a message that - arrived with embedded escapes — pasted content, gateway-origin - text, or model output echoing injected tool results — must not be - able to clear the screen, retitle the window, move the cursor, or - restyle adjacent UI when replayed. Rich's ``Text()`` does NOT - neutralize raw escape bytes, so sanitization has to happen before - display. Mirrors openai/codex#31494 (``sanitize_user_text``). + Removes ANSI/ECMA-48 sequences AND bare control characters, keeping only + newlines and tabs (CRs become newlines so ``\\r``-overwrite spoofing can't + hide content). Use when re-rendering persisted text (e.g. the ``/resume`` + recap): Rich's ``Text()`` does NOT neutralize raw escape bytes, so a replayed + message must not be able to clear the screen, retitle the window, or restyle UI. """ if not text or not _HAS_CONTROL.search(text): return text @@ -101,14 +80,9 @@ def sanitize_display_text(text: str) -> str: def strip_unicode_tags(text: str) -> str: """Remove invisible Unicode TAG characters (U+E0000–U+E007F) from text. - Tag characters are invisible in terminals and chat UIs but fully visible - to LLM tokenizers, making them a prompt-injection smuggling channel for - untrusted tool output (MCP servers, web content). Valid emoji tag - sequences (U+1F3F4 base + tag spec + U+E007F CANCEL TAG — regional - flags like Scotland/Wales) are preserved. - - Returns the input unchanged (fast path) when no plane-14 tag characters - are present. Ported from block/goose#10746. + A prompt-injection smuggling channel for untrusted tool output (MCP servers, + web content). Valid emoji tag sequences (regional flags) are preserved; + tag-free input is returned unchanged (fast path). """ if not text or not _HAS_UNICODE_TAG.search(text): return text diff --git a/tools/apply_layout_tool.py b/tools/apply_layout_tool.py index a6f8c89edc..1b7cae8dba 100644 --- a/tools/apply_layout_tool.py +++ b/tools/apply_layout_tool.py @@ -1,27 +1,17 @@ #!/usr/bin/env python3 -"""Apply a layout preset in the Hermes desktop GUI. +"""Apply a layout preset in the Hermes desktop GUI (``layout.apply`` via ``desktop_ui``). -Lives in the ``desktop_ui`` toolset (like ``focus_pane``), which the GUI -gateway enables only for desktop-sourced sessions. Emits ``layout.apply`` -through the shared ``desktop_ui`` bridge; the renderer resolves the preset id -against its layouts registry (core presets, plugin presets, and user-saved -presets are all the same list) and applies the tree through the exact code -path the layout picker uses. Only the active window's session may act — a -background turn never rearranges the user's desktop. - -Preset ids are free-form on purpose: plugins and users mint their own. The -renderer answers with the applied preset's id/title on success and the list -of available ids when the id is unknown, so the model can self-correct -without a second registry-listing tool. +The renderer resolves the id against its layouts registry (core, plugin, and user +presets are one list) and applies it through the layout picker's own code path; only +the active window's session acts, so a background turn never rearranges the desktop. +Preset ids are free-form on purpose: plugins and users mint their own. The renderer +answers with the applied id/title, or the list of available ids when unknown, so the +model can self-correct without a registry-listing tool. """ -import json - from tools import desktop_ui from tools.registry import registry, tool_error -# Renderer answer arrives via the blocking-prompt bridge with this timeout; -# applying a layout is synchronous in the renderer, so this is generous. _TIMEOUT_NOTE = "Layout apply is only available in the Hermes desktop app." @@ -30,15 +20,13 @@ def apply_layout_tool(preset: str) -> str: name = (preset or "").strip() if not name: return tool_error("preset is required — a layout preset id, e.g. 'default' or 'focus'.") - - try: - ok = desktop_ui.emit("layout.apply", {"preset": name}) - except Exception as exc: - return tool_error(f"Failed to apply layout '{name}': {exc}") - if not ok: - return tool_error(_TIMEOUT_NOTE) - - return json.dumps({"success": True, "preset": name}, ensure_ascii=False) + return desktop_ui.emit_or_error( + "layout.apply", + {"preset": name}, + f"Failed to apply layout '{name}': ", + _TIMEOUT_NOTE, + {"success": True, "preset": name}, + ) APPLY_LAYOUT_SCHEMA = { diff --git a/tools/approval.py b/tools/approval.py index 15867a188c..81a993d9a3 100644 --- a/tools/approval.py +++ b/tools/approval.py @@ -378,417 +378,101 @@ def _should_fall_through_to_cli_approval( """ return bool(is_cli and approval_callback is not None and notify_cb is None) -# Sensitive write targets that should trigger approval even when referenced -# via shell expansions like $HOME or $HERMES_HOME, or by the resolved absolute -# active profile home path such as /home/hermes/.hermes/config.yaml. The -# resolved-absolute form is folded into the ~/.hermes/ patterns at detection -# time by _normalize_command_for_detection() — see the rewrite step there — so -# these static patterns stay free of any import-time path snapshot (which would -# go stale when HERMES_HOME is set after this module is imported, e.g. under the -# hermetic test conftest or any deferred-profile-resolution path). -_SSH_SENSITIVE_PATH = r'(?:~|\$home|\$\{home\})/\.ssh(?:/|$)' -_HERMES_ENV_PATH = ( - r'(?:~\/\.hermes/|' - r'(?:\$home|\$\{home\})/\.hermes/|' - r'(?:\$hermes_home|\$\{hermes_home\})/)' - r'\.env\b' +from tools.approval_detection import ( # noqa: F401 -- re-exported for callers/tests + _SSH_SENSITIVE_PATH, + _HERMES_ENV_PATH, + _HERMES_CONFIG_PATH, + _PROJECT_ENV_PATH, + _PROJECT_CONFIG_PATH, + _SHELL_RC_FILES, + _CREDENTIAL_FILES, + _MACOS_PRIVATE_SYSTEM_PATH, + _SYSTEM_CONFIG_PATH, + _SENSITIVE_WRITE_TARGET, + _USER_SENSITIVE_WRITE_TARGET, + _PROJECT_SENSITIVE_WRITE_TARGET, + _COMMAND_TAIL, + _WRITE_TARGET_BOUNDARY, + _CMDPOS, + _hardline_rm_path, + _HARDLINE_SYSTEM_DIRS, + _RM_FLAG_PREFIX, + HARDLINE_PATTERNS, + _RE_FLAGS, + _QUOTE_MASKED_HARDLINE_DESCRIPTIONS, + HARDLINE_PATTERNS_COMPILED, + _SHELL_CARRIER_NAMES, + _contains_shell_carrier, + _mask_quoted_prose, + _SUDO_STDIN_RE, + _check_sudo_stdin_guard, + detect_hardline_command, + DANGEROUS_PATTERNS, + DANGEROUS_PATTERNS_COMPILED, + _legacy_pattern_key, + _REMOVED_PATTERN_KEY_ALIASES, + _approval_key_aliases, + _normalize_command_for_detection, + _PATH_TOKEN_STOP, + _PATH_TAIL, + _home_prefix_fold_regex, + _fold_home_prefixes, + _rewrite_resolved_user_home, + _rewrite_resolved_hermes_home, + _PARAM_REPLACEMENT_RE, + _PARAM_DEFAULT_RE, + _SIMPLE_SHELL_LITERAL_RE, + _ENV_ASSIGNMENT_RE, + _COMMAND_WRAPPER_WORDS, + _SUDO_OPTIONS_WITH_ARG, + _INTERPRETER_EXEC_FLAGS, + _INTERPRETER_WITH_ARG, + _READ_TOOL_EXEC_FLAGS, + _READ_TOOL_LONG_OPTIONS_WITH_ARG, + _READ_TOOL_SHORT_OPTIONS_WITH_ARG, + _SHELL_PUNCTUATION, + _MAX_DETECTION_COMMAND_CHARS, + _MAX_SEPARATOR_FREE_COMMAND_CHARS, + _MAX_DETECTION_SEGMENTS, + _PARSER_LIMIT_DESCRIPTION, + _MALFORMED_EXEC_DESCRIPTION, + _command_parser_limit_exceeded, + _shell_tokens_with_spans, + _GREP_OPTIONS_WITH_ARG, + _GREP_SHORT_OPTIONS_WITH_ARG, + _quoted_grep_pattern_spans, + _grep_safe_detection_variant, + _interpreter_family, + _shell_segment_tokens, + _iter_top_level_shell_segments, + _split_option, + _interpreter_exec_flag, + _BASH_OPTIONS_WITH_ARG, + _BASH_SHORT_OPTION_LETTERS, + _bash_exec_payload, + _read_tool_exec_flag, + _execution_flag_findings, + _skip_shell_whitespace, + _scan_dollar_paren_end, + _scan_backtick_end, + _read_shell_word, + _strip_optional_shell_quotes, + _is_simple_shell_literal, + _literal_command_substitution_output, + _replace_simple_command_substitutions, + _replace_simple_shell_expansions, + _strip_shell_word_syntax, + _deobfuscate_shell_word_for_detection, + _iter_shell_command_starts, + _mark_command_starts, + _mask_quoted_newlines, + _iter_shell_command_word_spans, + _command_detection_variants, + _is_verification_artifact_cleanup, + _GATEWAY_LIFECYCLE_SPLICE_DESCRIPTION, + _is_shell_token_spliced_gateway_lifecycle, + detect_dangerous_command, ) -# ~/.hermes/config.yaml IS the security policy: approvals.mode, yolo, and the -# permanent-approval allowlist live here, and the config cache is mtime-keyed -# so a write takes effect mid-session (the agent could flip approvals.mode=off -# and immediately bypass the gate). Pair the write_file/patch deny (file_tools -# _check_sensitive_path) with terminal-side coverage so `sed -i`, `tee`, `>`, -# `cp`, etc. targeting it are gated too — otherwise the deny is unpaired -# theater. Mirrors _HERMES_ENV_PATH; matches the HERMES_HOME override form as -# well as ~/.hermes/. -_HERMES_CONFIG_PATH = ( - r'(?:~\/\.hermes/|' - r'(?:\$home|\$\{home\})/\.hermes/|' - r'(?:\$hermes_home|\$\{hermes_home\})/)' - r'config\.yaml\b' -) -_PROJECT_ENV_PATH = r'(?:(?:/|\.{1,2}/)?(?:[^\s/"\'`]+/)*\.env(?:\.[^/\s"\'`]+)*)' -_PROJECT_CONFIG_PATH = r'(?:(?:/|\.{1,2}/)?(?:[^\s/"\'`]+/)*config\.yaml)' -_SHELL_RC_FILES = ( - r'(?:~|\$home|\$\{home\})/\.' - r'(?:bashrc|zshrc|profile|bash_profile|zprofile)\b' -) -_CREDENTIAL_FILES = ( - r'(?:~|\$home|\$\{home\})/\.' - r'(?:netrc|pgpass|npmrc|pypirc)\b' -) -# macOS: /etc, /var, /tmp, /home are symlinks to /private/{etc,var,tmp,home}. -# A command written to target /private/etc/sudoers works identically to -# /etc/sudoers on macOS but bypasses a plain "/etc/" pattern check. Match -# both forms. Inspired by Claude Code 2.1.113's "dangerous path protection". -_MACOS_PRIVATE_SYSTEM_PATH = r'/private/(?:etc|var|tmp|home)/' -# System-config paths that should trigger approval for any write/edit, -# collapsing /etc, its macOS /private/etc mirror, and /etc/sudoers.d/ into -# one shared fragment so new DANGEROUS_PATTERNS stay consistent. -_SYSTEM_CONFIG_PATH = ( - rf'(?:/etc/|{_MACOS_PRIVATE_SYSTEM_PATH})' -) -_SENSITIVE_WRITE_TARGET = ( - rf'(?:{_SYSTEM_CONFIG_PATH}|/dev/sd|' - rf'{_SSH_SENSITIVE_PATH}|' - rf'{_HERMES_ENV_PATH}|' - rf'{_HERMES_CONFIG_PATH}|' - rf'{_SHELL_RC_FILES}|' - rf'{_CREDENTIAL_FILES})' -) -_USER_SENSITIVE_WRITE_TARGET = ( - rf'(?:{_SSH_SENSITIVE_PATH}|' - rf'{_SHELL_RC_FILES}|' - rf'{_CREDENTIAL_FILES})' -) -_PROJECT_SENSITIVE_WRITE_TARGET = rf'(?:{_PROJECT_ENV_PATH}|{_PROJECT_CONFIG_PATH})' -# Anchor for the cp/mv/install rule, where the sensitive path is only a write -# target when it is the LAST argument (the destination). Requiring end-of-line -# (or a command separator) keeps `cp config.yaml backup.yaml` — config.yaml as -# the SOURCE — out of the deny. -_COMMAND_TAIL = r'(?:\s*(?:&&|\|\||;).*)?$' -# Boundary for stream-write rules (`>`/`>>` redirection and `tee`), where the -# sensitive path is ALWAYS a write target no matter what follows it. We only -# need the path token to END at a shell word boundary — whitespace, a quote, a -# command separator, a redirection operator, or end-of-line. -# Using _COMMAND_TAIL here was too strict: it required the rest of the line to -# be empty or a command separator, so `echo x > .env extra` (extra arg to echo) -# and `echo x > .env # note` (trailing comment) slipped past the deny even -# though the shell still overwrites `.env`. Mirrors the looser system-path -# redirection rule, which never had this restriction. -# -# `#` is deliberately NOT a boundary char: a real trailing comment always has -# whitespace before the `#` (already covered by `\s`), whereas a `#` glued to -# the path is part of the filename. `echo x > .env#backup` writes to the -# distinct file `.env#backup`, not `.env`, so it must stay OUT of the deny — -# the same reasoning that keeps `config.yaml.bak` safe. -_WRITE_TARGET_BOUNDARY = r'(?=[\s;&|<>"\']|$)' - -# ========================================================================= -# Hardline (unconditional) blocklist -# ========================================================================= -# -# Commands so catastrophic they should NEVER run via the agent, regardless -# of --yolo, /yolo, approvals.mode=off, or cron approve mode. This is a -# floor below yolo: opting into yolo is the user trusting the agent with -# their files and services, not trusting it to wipe the disk or power the -# box off. -# -# Hardline only applies to environments that can actually damage the host -# (local, ssh, container-host cron). Containerized backends (docker, -# singularity, modal, daytona) already bypass the dangerous-command layer -# because nothing they do can touch the host, so we leave that behavior -# alone. -# -# The list is deliberately tiny — only things with no recovery path: -# filesystem destruction rooted at /, raw block device overwrites, kernel -# shutdown/reboot, and denial-of-service commands that take the host down. -# Recoverable-but-costly operations (git reset --hard, rm -rf /tmp/x, -# chmod -R 777, curl|sh) stay in DANGEROUS_PATTERNS where yolo can pass -# them through — that's what yolo is for. -# -# Inspired by Mercury Agent's permission-hardened blocklist -# (https://github.com/cosmicstack-labs/mercury-agent). - -# Regex fragment matching the *start* of a command (i.e. positions where -# a shell would begin parsing a new command). Used by shutdown/reboot -# patterns so they don't fire on "echo reboot" or "grep 'shutdown' log". -# Matches: start of string, after command separators (; && || | newline), -# after subshell openers ( `$(` or backtick ), optionally consuming -# leading wrapper commands (sudo, env VAR=VAL, exec, nohup, setsid). -_CMDPOS = ( - # Real ;/&/| separators are converted to newlines by the quote-aware - # _mark_command_starts pass. Keeping them in this flat regex mistakes - # quoted regex/data (for example grep '(safe|rm -rf /)') for commands. - r'(?:^|[\n`]|\$\()' # start position - r'\s*' # optional whitespace - r'(?:sudo\s+(?:-[^\s]+\s+)*)?' # optional sudo with flags - r'(?:env\s+(?:\w+=\S*\s+)*)?' # optional env with VAR=VAL pairs - r'(?:(?:exec|nohup|setsid|time)\s+)*' # optional wrapper commands - r'\s*' -) - -# Destructive-path argument matcher for the rm hardline rules. -# -# The path token in `rm -rf /` is almost always written quoted in real -# shells — `rm -rf "/"`, `rm -rf "$HOME"` — and `${HOME}` is the universal -# brace form. A bare-token anchor (`(/...)(\s|$)`) silently misses all of -# these: the surrounding quote breaks both the leading position (the flag -# group can't consume `"`) and the trailing `(\s|$)` terminator, letting -# `rm -rf "/"` slip past the unconditional floor entirely. -# -# Accept the path either fully wrapped in a matching quote pair OR bare with -# a terminator. The matching-quote branch catches `rm -rf "/"` (path quoted -# on its own). The bare branch's terminator accepts whitespace, end-of-string -# OR a shell metacharacter (`) ` ; | &`) so a real root wipe inside a command -# substitution — `$(rm -rf /)`, `` `rm -rf /` `` — whose `/` is terminated by -# `)`/backtick is still caught. -def _hardline_rm_path(path_alt: str, tail: str = r'(?:\s|$|[)`;|&])') -> str: - return rf'(?:["\'](?:{path_alt})["\']|(?:{path_alt}){tail})' - - -# Protected system roots whose recursive deletion has no recovery path. -_HARDLINE_SYSTEM_DIRS = ( - r'/home|/home/\*|/root|/root/\*|/etc|/etc/\*|/usr|/usr/\*|' - r'/var|/var/\*|/bin|/bin/\*|/sbin|/sbin/\*|/boot|/boot/\*|/lib|/lib/\*' -) - -# `rm` plus its flag group, shared by the three rm hardline rules. Kept as a -# plain concatenation (not an f-string) so the regex backslashes never live -# inside an f-string replacement field — unsupported on the Python 3.11 floor. -# -# Anchored to _CMDPOS (start of line, after a command separator ; && || |, -# after a subshell opener $(/backtick, or after sudo/env/exec wrappers) so the -# rule fires only when `rm` is an actual command word — not when the literal -# string "rm -rf /" appears as DATA inside another command's argument, e.g. -# `gh pr create --title "block rm -rf / spellings"` or `git commit -m "…rm -rf -# /…"`. Those tripped the unconditional floor and could not run at all before -# the anchor. A real wipe at any command position (bare, chained, in $()/`…`, -# under sudo) still matches; the quoted-path branch in _hardline_rm_path keeps -# catching `rm -rf "/"`. -_RM_FLAG_PREFIX = _CMDPOS + r'rm\s+(-[^\s]*\s+)*' - -HARDLINE_PATTERNS = [ - # rm recursive targeting the root filesystem or protected roots. - # `${HOME}` brace form and quoted paths (`rm -rf "/"`, `rm -rf "$HOME"`) - # are handled via _hardline_rm_path so the floor cannot be bypassed with - # the ordinary quoting/brace shell idioms. - # - # The path token matches any root-anchored path whose components collapse - # back to "/" in the shell: a bare "/", repeated slashes ("//"), and - # "."/".." current/parent segments ("/.", "/./", "/..", "/../..") all - # resolve to root, optionally followed by a trailing glob ("/*", "//*"). - # Each inter-slash segment must be exactly "." or "..", so a longer dot - # run or any real name is a literal directory, NOT root — "/tmp", "/home", - # "/.ssh", "/.config" and even "/..." (a dir literally named "...") fall - # through to the softer DANGEROUS_PATTERNS / system-directory rules - # instead of being unconditionally hardline-blocked. The explicit "/ \*" - # alt preserves the slash-space-glob spelling (`rm -rf / *`, which the - # shell sees as two args: "/" plus the "*" glob). - (_RM_FLAG_PREFIX + _hardline_rm_path(r'/(?:(?:\.\.?)?/)*(?:\.\.?)?\**|/ \*'), "recursive delete of root filesystem"), - (_RM_FLAG_PREFIX + _hardline_rm_path(_HARDLINE_SYSTEM_DIRS), "recursive delete of system directory"), - (_RM_FLAG_PREFIX + _hardline_rm_path(r'(?:~|\$\{?HOME\}?)(?:/?|/\*)?'), "recursive delete of home directory"), - # Filesystem format — anchor to command position like every other - # hardline entry so quoted prose ("echo \"does this workflow use mkfs - # anywhere?\"") does not trip the unconditional floor (#93392). - (_CMDPOS + r'mkfs(\.[a-z0-9]+)?\b', "format filesystem (mkfs)"), - # Raw block device overwrites (dd + redirection). `dd` is a command-name - # token, so anchor it to command position like mkfs/rm/shutdown (#93392): - # quoted prose such as `git commit -m "never dd of=/dev/sda"` is an - # argument, not a command. The argument tail ([^\n]*of=/dev/...) is kept - # so flag order doesn't matter. - (_CMDPOS + r'dd\b[^\n]*\bof=/dev/(sd|nvme|hd|mmcblk|vd|xvd)[a-z0-9]*', "dd to raw block device"), - # The redirect rule has no command-name token to anchor (`>` appears - # mid-command: `cat f > /dev/sda`), so command-position anchoring is the - # wrong tool. It is instead matched against a QUOTE-MASKED variant of the - # command (see _QUOTE_MASKED_HARDLINE / _mask_quoted_strings) so quoted - # prose (`echo "cat f > /dev/sda"`) cannot trip it, while shell-carrying - # wrappers (sh -c / bash -c / eval) still surface their payload as a raw - # detection variant — quoting is not a bypass (#93392). - (r'>\s*/dev/(sd|nvme|hd|mmcblk|vd|xvd)[a-z0-9]*\b', "redirect to raw block device"), - # Fork bomb (classic shell form). Also positionless (the trigger is the - # function definition itself, valid anywhere in a command line), so it is - # quote-masked like the redirect rule above rather than _CMDPOS-anchored. - (r':\(\)\s*\{\s*:\s*\|\s*:\s*&\s*\}\s*;\s*:', "fork bomb"), - # Kill every process on the system — anchor the command-name token so - # `echo "kill -1 sends SIGHUP to everything"` doesn't trip (#93392). - (_CMDPOS + r'kill\s+(-[^\s]+\s+)*-1\b', "kill all processes"), - # System shutdown / reboot — anchor to command position (start of line, - # after a command separator, or after sudo/env wrappers) so we don't - # false-positive on "echo reboot" or "grep 'shutdown' logs". - # _CMDPOS matches start-of-command positions. - (_CMDPOS + r'(shutdown|reboot|halt|poweroff)\b', "system shutdown/reboot"), - (_CMDPOS + r'init\s+[06]\b', "init 0/6 (shutdown/reboot)"), - (_CMDPOS + r'systemctl\s+(poweroff|reboot|halt|kexec)\b', "systemctl poweroff/reboot"), - (_CMDPOS + r'telinit\s+[06]\b', "telinit 0/6 (shutdown/reboot)"), -] - -# Pre-compiled variant used by the hot-path matcher. Building these at module -# load eliminates the ~2.6 ms cold-cache re.compile fan-out on the first -# terminal() call per process (12 HARDLINE + 47 DANGEROUS patterns, each -# potentially evicted from Python's 512-entry ``re._cache`` by unrelated -# regex work elsewhere in the agent). DANGEROUS_PATTERNS_COMPILED is built -# at the end of this module after DANGEROUS_PATTERNS is defined. -_RE_FLAGS = re.IGNORECASE | re.DOTALL - -# Hardline rules whose trigger has no command-name token to anchor (the -# redirect target / fork-bomb definition are valid anywhere in a command -# line). These are matched against QUOTE-MASKED variants of the command so -# quoted prose (`echo "cat f > /dev/sda"`, `git commit -m "fork bomb -# :(){ :|:& };:"`) cannot trip the unconditional floor, while the raw -# payloads of shell-carrying wrappers (sh -c, bash -c, eval) are still -# scanned unmasked — quoting is not a bypass (#93392). -_QUOTE_MASKED_HARDLINE_DESCRIPTIONS = frozenset({ - "redirect to raw block device", - "fork bomb", -}) - -HARDLINE_PATTERNS_COMPILED = [ - ( - re.compile(pattern, _RE_FLAGS), - description, - description in _QUOTE_MASKED_HARDLINE_DESCRIPTIONS, - ) - for pattern, description in HARDLINE_PATTERNS -] - - -# Command names that hand a quoted argument to another shell/parser to -# EXECUTE. For these, quoted text is code, not prose, so the quote-masked -# hardline rules must scan the raw string (see detect_hardline_command). -_SHELL_CARRIER_NAMES = frozenset({ - "eval", "sh", "bash", "zsh", "ksh", "dash", "source", ".", -}) - - -def _contains_shell_carrier(command: str) -> bool: - """Return whether any command-position word is a shell-carrying command.""" - for _, _, word in _iter_shell_command_word_spans(command): - name = os.path.basename( - _deobfuscate_shell_word_for_detection(word) - ).lower() - if name in _SHELL_CARRIER_NAMES: - return True - return False - - -def _mask_quoted_prose(command: str) -> str: - """Blank out quoted string CONTENT for positionless hardline matching. - - Detection-only rewrite used by the quote-masked hardline rules - (redirect-to-block-device, fork bomb): text inside single or double - quotes is data the shell passes as an argument, so `echo "cat f > - /dev/sda"` must not trip the unconditional floor (#93392). Structure is - preserved: the quote characters themselves stay, and inside double - quotes `$(...)` command substitutions and backtick spans are kept RAW - because the shell really executes them (`echo "$(cat f > /dev/sda)"` - remains a true positive). Unquoted text is untouched. Quote tracking - mirrors _mask_quoted_newlines; an unclosed quote masks to end-of-string, - which cannot hide a runnable command (the shell would not run it - either). - """ - out: list[str] = [] - quote: str | None = None - i = 0 - n = len(command) - while i < n: - ch = command[i] - if quote == "'": - if ch == "'": - quote = None - out.append(ch) - else: - out.append(" ") - i += 1 - continue - if quote == '"': - if ch == "\\" and i + 1 < n: - out.append(" ") - i += 2 - continue - if ch == '"': - quote = None - out.append(ch) - i += 1 - continue - if ch == "$" and i + 1 < n and command[i + 1] == "(": - end = _scan_dollar_paren_end(command, i) - if end is not None: - out.append(command[i:end]) - i = end - continue - if ch == "`": - close = command.find("`", i + 1) - if close != -1: - out.append(command[i:close + 1]) - i = close + 1 - continue - out.append(" ") - i += 1 - continue - if ch == "\\" and i + 1 < n: - out.append(command[i:i + 2]) - i += 2 - continue - if ch in ("'", '"'): - quote = ch - out.append(ch) - i += 1 - return "".join(out) - - -# ========================================================================= -# Sudo stdin guard — block password guessing via "sudo -S" -# ========================================================================= -# When SUDO_PASSWORD is not configured, any explicit "sudo -S" in the -# command is the LLM piping a guessed password via stdin. This is a -# brute-force attack vector: the model iterates through candidate -# passwords, inspects sudo's "Sorry, try again" output, and refines. -# Treat this as an unconditional block — there is never a legitimate -# reason for the agent to pipe passwords to sudo -S when no password -# has been configured. -_SUDO_STDIN_RE = re.compile( - r'(?:^|[;&|`\n]|&&|\|\||\$\()\s*sudo\s+-S\b', - re.IGNORECASE) - - -def _check_sudo_stdin_guard(command: str) -> tuple: - """Detect ``sudo -S`` (stdin password) without configured SUDO_PASSWORD. - - When SUDO_PASSWORD is set, ``_transform_sudo_command`` injects ``-S`` - internally — that path is legitimate and handled elsewhere. This guard - only fires when SUDO_PASSWORD is *not* set, meaning the LLM explicitly - wrote ``sudo -S`` to pipe a guessed password. - - Returns: - (is_blocked: bool, description: str | None) - """ - if "SUDO_PASSWORD" in os.environ: - return (False, None) - normalized = _normalize_command_for_detection(command).lower() - if _SUDO_STDIN_RE.search(normalized): - return (True, "sudo password guessing via stdin (sudo -S)") - return (False, None) - - -def detect_hardline_command(command: str) -> tuple: - """Check if a command matches hardline blocklist patterns. - - Hardline patterns are NEVER bypassable, even in YOLO mode. - - Returns: - (is_hardline, description) or (False, None) - """ - if _command_parser_limit_exceeded(command): - return (True, _PARSER_LIMIT_DESCRIPTION) - normalized = _normalize_command_for_detection(command) - _, malformed_grep = _grep_safe_detection_variant(normalized) - if malformed_grep: - return (True, _MALFORMED_EXEC_DESCRIPTION) - for command_variant in _command_detection_variants(command): - variant_lower = command_variant.lower() - masked_lower: str | None = None - for pattern_re, description, quote_masked in HARDLINE_PATTERNS_COMPILED: - if quote_masked: - # Positionless rules (redirect-to-block-device, fork bomb) - # match a quote-masked variant so quoted prose in echo / - # git commit -m / gh --body arguments is DATA (#93392). - # Shell-carrying commands (sh/bash -c, eval, source) hand - # their quoted argument to another parser, so those scan - # the raw variant — quoting is not a bypass. bash/sh -c - # payloads additionally surface as their own raw variants - # via _execution_flag_findings. - if masked_lower is None: - if _contains_shell_carrier(command_variant): - masked_lower = variant_lower - else: - masked_lower = _mask_quoted_prose(command_variant).lower() - haystack = masked_lower - else: - haystack = variant_lower - if pattern_re.search(haystack): - return (True, description) - return (False, None) def _match_user_deny_rule(command: str) -> str | None: @@ -938,1633 +622,6 @@ def _sudo_stdin_block_result(description: str) -> dict: } -# ========================================================================= -# Dangerous command patterns -# ========================================================================= - -DANGEROUS_PATTERNS = [ - (r'\brm\s+(-[^\s]*\s+)*/', "delete in root path"), - (r'\brm\s+-[^\s]*r', "recursive delete"), - (r'\brm\s+--recursive\b', "recursive delete (long flag)"), - # GNU rm permutes options, so a recursive flag group may legally FOLLOW - # the operands: `rm build/ -rf`, `rm build/ -r -f`, and `rm build/ - # --recursive --force` are all equivalent to the flags-first spellings the - # two patterns above catch — without this rule they run with no approval - # prompt at all. The operand run is tempered: it cannot cross a command - # separator (`;`, `|`, `&`, newline — so a later pipeline segment's flags, - # e.g. `rm foo | grep -r bar`, are not attributed to `rm`), cannot cross a - # quote (so `git commit -m "rm x" --amend` style data can't bridge an `rm` - # word to an unrelated dash token), and cannot cross a bare ` -- ` - # end-of-options separator (after `--`, POSIX rm treats `-rf` as a literal - # filename, not flags; guarded both leading and mid-run). The flag token - # itself must start right after whitespace so the `r` inside long options - # like `--registry` (preceded by `-`, not whitespace) does not count. - # Port of openai/codex#33464 ("recognize force options when they follow - # operands"). - (r'\brm\s+(?!--(?:\s|$))(?:(?!\s--(?:\s|$))[^\n"\';|&])*\s' - r'(?:-[a-z]*r[a-z]*\b|--recursive\b)', - "recursive delete (flags after operands)"), - # Windows shell front-ends have destructive built-ins that do not look like - # Unix `rm`. Gate only when they are executed through cmd/powershell so - # ordinary prose or filenames containing "del"/"rd" do not trip the guard. - (r'\bcmd(?:\.exe)?\s+/(?:c|k)\s+.*\b(?:del|erase|rd|rmdir)\b', "Windows cmd destructive delete"), - # PowerShell/pwsh: the destructive verb runs as the default positional - # argument, so `powershell Remove-Item ...` needs NO explicit -Command. - # Anchor the verb to the command position (right after the shell name, - # after any leading `-Flag` switches, and optionally after -Command/-c) - # so bare invocations are caught while a benign path arg containing - # "del"/"rm" (e.g. `-File c:\del-logs\run.ps1`) is not. - (r'\b(?:powershell|pwsh)(?:\.exe)?\b(?:\s+-\S+)*\s+(?:-(?:command|c)\s+)?["\']?(?:remove-item|rmdir|erase|del|rd|ri|rm)\b', "Windows PowerShell destructive delete"), - (r'\b(?:powershell|pwsh)(?:\.exe)?\b.*\s-(?:encodedcommand|enc|e)\b', "PowerShell encoded command execution"), - # ── Windows destructive tier (#69472) ──────────────────────────────── - # These are native Windows EXEs / cmdlets reachable from ANY Hermes - # terminal backend on a Windows host — including the default git-bash - # backend (taskkill.exe, icacls.exe, reg.exe, vssadmin.exe, bcdedit.exe, - # cipher.exe are ordinary PATH executables there). Detection input is - # lowercased by the variant loop, so patterns are written lowercase. - # Each pattern requires the destructive flag/verb so benign usage - # (`taskkill /IM app.exe` graceful kill, `reg query`, `icacls file`) - # does NOT prompt. - # Bare PowerShell destructive delete: Remove-Item/ri with -Recurse or - # -Force. The cmd/powershell-prefixed forms are covered above; this - # catches the bare form (ACP clients, pwsh-default SSH hosts, or - # `powershell` invoked earlier in a compound command). - (r'\bremove-item\b[^\n;|&]*\s-(?:recurse|force)\b', "PowerShell destructive delete (Remove-Item)"), - # cmd builtins with destructive switches, bare form: del/erase/rd/rmdir - # with /s (recurse) or /q (quiet). Requires the switch so `del file.txt` - # inside a cmd /c string stays covered by the prefixed rule only. - (r'\b(?:del|erase|rd|rmdir)\s+(?:/[a-z]\s+)*/[sq]\b', "Windows destructive delete (recursive/quiet switch)"), - # Remote content piped to Invoke-Expression — PowerShell's `curl | sh`. - (r'\b(?:iwr|invoke-webrequest|invoke-restmethod|irm|curl|wget)\b[^\n]*\|\s*(?:iex|invoke-expression)\b', "pipe remote content to PowerShell (iwr | iex)"), - (r'\b(?:iex|invoke-expression)\s*\(\s*(?:iwr|invoke-webrequest|invoke-restmethod|irm)\b', "execute remote content via Invoke-Expression"), - # Force process kills — Windows analogue of pkill -9. - (r'\btaskkill\b[^\n]*\s/f\b', "force kill processes (taskkill /F)"), - (r'\bstop-process\b[^\n]*\s-force\b', "force kill processes (Stop-Process -Force)"), - # Volume/disk destruction — Windows analogue of mkfs / dd. - (r'\bformat-volume\b', "format filesystem (Format-Volume)"), - (r'\bclear-disk\b', "wipe disk (Clear-Disk)"), - (r'\bdiskpart\b', "disk partitioning (diskpart)"), - (r'\bformat(?:\.com)?\s+[a-z]:', "format drive (format.com)"), - (r'\bcipher\s+/w\b', "wipe free space (cipher /w)"), - # ACL destruction — Windows analogue of chmod 777. - (r'\bicacls\b[^\n]*\s/grant\b[^\n]*\b(?:everyone|todos|jeder|tout\s+le\s+monde|\*s-1-1-0)\b', "grant Everyone access (icacls)"), - (r'\bicacls\b[^\n]*\s/reset\b', "reset ACLs recursively (icacls /reset)"), - # Backup/recovery destruction — classic ransomware prep, no benign - # agent use case. - (r'\bvssadmin\b[^\n]*\bdelete\s+shadows\b', "delete volume shadow copies (vssadmin)"), - (r'\bwbadmin\b[^\n]*\bdelete\b', "delete backups (wbadmin)"), - (r'\bbcdedit\b[^\n]*\s/set\b', "modify boot configuration (bcdedit /set)"), - # Registry deletion with force flag. - (r'\breg(?:\.exe)?\s+delete\b', "registry delete (reg delete)"), - (r'\bremove-itemproperty\b[^\n]*\s-force\b', "registry value delete (Remove-ItemProperty -Force)"), - # Windows service/system stop — analogue of systemctl stop. - (r'\bstop-service\b[^\n]*\s-force\b', "force stop service (Stop-Service -Force)"), - (r'\bsc(?:\.exe)?\s+(?:stop|delete)\b', "stop/delete service (sc)"), - # Credential/key paths in Windows form — the POSIX ~/.ssh patterns never - # match drive-letter or backslash spellings. Match both separators. - (r'\busers[\\/][^\\/\s]+[\\/]\.ssh\b', "access to SSH keys (Windows path)"), - (r'\bappdata[\\/](?:local|roaming)[\\/]hermes[^\n]*\.env\b', "access to Hermes secrets (Windows path)"), - # ───────────────────────────────────────────────────────────────────── - (r'\bchmod\s+(-[^\s]*\s+)*(777|666|o\+[rwx]*w|a\+[rwx]*w)\b', "world/other-writable permissions"), - (r'\bchmod\s+--recursive\b.*(777|666|o\+[rwx]*w|a\+[rwx]*w)', "recursive world/other-writable (long flag)"), - (r'\bchown\s+(-[^\s]*)?R\s+root', "recursive chown to root"), - (r'\bchown\s+--recur[a-z]*\b.*root', "recursive chown to root (long flag)"), - # Anchored to command position like the hardline twins (#93392): - # quoted prose mentioning mkfs/dd must not require approval to echo. - (_CMDPOS + r'mkfs\b', "format filesystem"), - (_CMDPOS + r'dd\s+.*if=', "disk copy"), - (r'>\s*/dev/sd', "write to block device"), - (r'\bDROP\s+(TABLE|DATABASE)\b', "SQL DROP"), - # Use [^\n]* instead of .* so DOTALL mode does not cause a WHERE clause on the - # *next* line to satisfy the negative lookahead, silently allowing DELETE without WHERE. - (r'\bDELETE\s+FROM\b(?![^\n]*\bWHERE\b)', "SQL DELETE without WHERE"), - (r'\bTRUNCATE\s+(TABLE)?\s*\w', "SQL TRUNCATE"), - (rf'>\s*{_SYSTEM_CONFIG_PATH}', "overwrite system config"), - (r'\bsystemctl\s+(-[^\s]+\s+)*(stop|restart|disable|mask)\b', "stop/restart system service"), - (r'\bkill\s+-9\s+-1\b', "kill all processes"), - (r'\bpkill\s+-9\b', "force kill processes"), - # killall with SIGKILL (parallel to pkill -9). Catches -9 / -KILL / - # -s KILL / -SIGKILL forms, and also `killall -r ` broad sweeps - # that can wipe out unrelated processes by accident. - # Inspired by Claude Code 2.1.113 expanded deny rules. - (r'\bkillall\s+(-[^\s]*\s+)*-(9|KILL|SIGKILL)\b', "force kill processes (killall -KILL)"), - (r'\bkillall\s+(-[^\s]*\s+)*-s\s+(KILL|SIGKILL|9)\b', "force kill processes (killall -s KILL)"), - (r'\bkillall\s+(-[^\s]*\s+)*-r\b', "kill processes by regex (killall -r)"), - (r':\(\)\s*\{\s*:\s*\|\s*:\s*&\s*\}\s*;\s*:', "fork bomb"), - # Shell -c is parsed structurally by _execution_flag_findings(). A regex - # that merely searched a dash-token for "c" also matched --norc, - # --rcfile, and --restricted. - (r'\b(curl|wget)\b.*\|\s*(?:[/\w]*/)?(?:ba)?sh(?:\s|$|-c)', "pipe remote content to shell"), - (r'\b(bash|sh|zsh|ksh)\s+<\s* | base64 -d | bash` silently runs `rm -rf /` or any - # other command because the raw text carries no dangerous keywords. - (r'\b(base64|base32|base16)\s+(?:-[dD]|--decode)\b.*\|\s*\b(bash|sh|zsh|ksh|dash)\b', - "pipe decoded content to shell (possible command obfuscation)"), - # xxd reverse hex dump to shell (xxd uses -r for decode, not -d). - (r'\bxxd\s+-r\b.*\|\s*\b(bash|sh|zsh|ksh|dash)\b', - "pipe xxd-decoded content to shell (possible command obfuscation)"), - # Character transformation via tr piped to shell: - # `echo 'eq -pe v/' | tr 'eqv' 'rmf' | bash` decodes to `rm -rf /`. - (r'\becho\b[^|]*\|\s*\btr\b[^|]*\|\s*\b(bash|sh|zsh|ksh|dash)\b', - "pipe tr-transformed output to shell (possible command obfuscation)"), - # openssl decode piped to shell: - # `echo | openssl base64 -d | bash` decodes arbitrary commands. - (r'\bopenssl\b.*\b(?:base64|enc)\b[^|]*\s+-[dD]\b[^|]*\|\s*\b(bash|sh|zsh|ksh|dash)\b', - "pipe openssl-decoded content to shell (possible command obfuscation)"), - (rf'\btee\b.*["\']?{_SENSITIVE_WRITE_TARGET}', "overwrite system file via tee"), - (rf'>>?\s*["\']?{_SENSITIVE_WRITE_TARGET}', "overwrite system file via redirection"), - (rf'\btee\b.*["\']?{_PROJECT_SENSITIVE_WRITE_TARGET}["\']?{_WRITE_TARGET_BOUNDARY}', "overwrite project env/config via tee"), - (rf'>>?\s*["\']?{_PROJECT_SENSITIVE_WRITE_TARGET}["\']?{_WRITE_TARGET_BOUNDARY}', "overwrite project env/config via redirection"), - (r'\bxargs\s+.*\brm\b', "xargs with rm"), - # find -exec rm / -execdir rm — the -execdir variant (same semantics, - # runs in the directory of each match) was previously missed. Claude - # Code 2.1.113 tightened their equivalent find rule to stop auto- - # approving -exec / -delete flags. - (r'\bfind\b.*-exec(?:dir)?\s+(/\S*/)?rm\b', "find -exec/-execdir rm"), - (r'\bfind\b.*-delete\b', "find -delete"), - # Gateway lifecycle protection: prevent the agent from killing its own - # gateway process. These commands trigger a gateway restart/stop that - # terminates all running agents mid-work. Allow global flags between - # `hermes` and `gateway` (e.g. `hermes -p ade gateway restart`) so a - # profile flag can't slip the agent past the guard. - (r'\bhermes\s+(?:-{1,2}\S+(?:\s+\S+)?\s+)*gateway\s+(stop|restart)\b', "stop/restart hermes gateway (kills running agents)"), - (r'\bhermes\s+update\b', "hermes update (restarts gateway, kills running agents)"), - # Docker container lifecycle — any user with docker.sock mounted (a common - # Docker Compose pattern) gives the agent the ability to restart/stop/kill - # containers without approval. These are agent-initiated lifecycle operations - # that should always require user consent, just like `hermes gateway restart` - # already does for the gateway process. - # Docker/Podman daemon redirect — global flags or env prefixes that point - # the CLI at a DIFFERENT daemon, often a remote host over ssh/tcp. A - # command that looks local (`docker -H ssh://prod stop app`) silently - # operates on remote infrastructure, so any docker/podman invocation - # carrying a redirect requires approval regardless of subcommand. The - # redirect flag must appear in the global-flag position (before the - # subcommand) and -H/--host/--context must carry a value, which keeps - # `docker -h` (help) and subcommand flags like `docker run -h ` - # out of the deny. Listed BEFORE the lifecycle rules so a redirected - # lifecycle command surfaces the more specific "remote daemon" reason. - # Inspired by Claude Code 2.1.214, which added permission prompts for - # docker/podman commands carrying daemon-redirect flags (--url, - # --connection, --identity, remote mode). - (r'\bdocker\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(?:-h|--host)[=\s]+\S+', - "docker with remote daemon redirect (-H/--host)"), - (r'\bdocker\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(?:-c|--context)[=\s]+\S+', - "docker with daemon redirect (--context: alternate daemon)"), - (r'\bdocker\s+context\s+use\b', - "docker context use (switches default daemon for future commands)"), - (r'\bpodman\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(?:--url|--connection|--identity)[=\s]+\S+', - "podman with remote daemon redirect (--url/--connection/--identity)"), - (r'\bpodman\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(?:-r\b|--remote\b)', - "podman remote mode (-r/--remote: remote daemon)"), - (r'\b(?:docker_host|docker_context|container_host|container_connection)=\S+', - "docker/podman daemon redirect via environment (DOCKER_HOST/CONTAINER_HOST)"), - # Allow global flags between `docker`/`compose` and the verb (e.g. - # `docker compose -f prod.yml down`, `docker --log-level debug stop app`) - # and the legacy hyphenated `docker-compose` binary, so a flag can't slip - # a lifecycle command past the guard — same treatment as the `hermes ... - # gateway` pattern above. - (r'\bdocker(?:-compose|\s+compose)\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(restart|stop|kill|down)\b', - "docker compose restart/stop/kill/down (container lifecycle)"), - (r'\bdocker\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(restart|stop|kill)\b', - "docker restart/stop/kill (container lifecycle)"), - # Gateway protection: never start gateway outside systemd management - (r'gateway\s+run\b.*(&\s*$|&\s*;|\bdisown\b|\bsetsid\b)', "start gateway outside systemd (use 'systemctl --user restart hermes-gateway')"), - (r'\bnohup\b.*gateway\s+run\b', "start gateway outside systemd (use 'systemctl --user restart hermes-gateway')"), - # Self-termination protection: prevent agent from killing its own process - (r'\b(pkill|killall)\b.*\b(hermes|gateway|cli\.py)\b', "kill hermes/gateway process (self-termination)"), - # Self-termination via kill + command substitution (pgrep/pidof). - # The name-based pattern above catches `pkill hermes` but not - # `kill -9 $(pgrep -f hermes)` because the substitution is opaque - # to regex at detection time. Catch the structural pattern instead. - # `pidof` is the BSD/Linux alternative to `pgrep` and is equally - # opaque, so include it in the same alternation. - (r'\bkill\b.*\$\(\s*(pgrep|pidof)\b', "kill process via pgrep/pidof expansion (self-termination)"), - (r'\bkill\b.*`\s*(pgrep|pidof)\b', "kill process via backtick pgrep/pidof expansion (self-termination)"), - # launchctl-driven gateway stop/restart on macOS. The agent can bypass - # the `hermes gateway stop|restart` pattern above by driving launchd - # directly against the service label (commonly `ai.hermes.gateway`). - # Catch the operations that stop, restart, or unload it. - # - # Order-independent (2026-08-02 incident): the previous version required - # "hermes"/"ai.hermes" to appear AFTER the launchctl verb in the same - # string (`.*` only scans forward). A shell for-loop that builds the - # label from a list defined earlier in the command — e.g. `for item in - # 'ai.hermes.gateway-apollo:...' ...; do label=${item%%:*}; launchctl - # bootout "$label"; done` — never has the literal text "hermes" appear - # after "bootout" (only the expanded variable does), so it slipped past - # undetected and restarted 4 gateways with zero approval. Two - # independent lookaheads instead of one sequential match: both - # substrings must appear SOMEWHERE in the command, in either order. - # This is intentionally broader (a launchctl-verb command anywhere near - # an unrelated "hermes" mention now also matches) — for an approval gate - # that's the correct direction to err: an extra approval prompt is - # cheap, a missed one took down the whole gateway fleet. - (r'(?=[\s\S]*\blaunchctl\s+(?:stop|kickstart|bootout|unload|kill|disable|remove)\b)(?=[\s\S]*\b(?:hermes|ai\.hermes)\b)', "stop/restart hermes launchd service (kills running agents)"), - # File copy/move/edit into sensitive system paths (/etc/ and macOS - # /private/etc/ mirror). - (rf'\b(cp|mv|install)\b.*\s{_SYSTEM_CONFIG_PATH}', "copy/move file into system config path"), - (rf'\b(cp|mv|install)\b.*\s["\']?{_PROJECT_SENSITIVE_WRITE_TARGET}["\']?{_COMMAND_TAIL}', "overwrite project env/config file"), - # cp/mv/install OVERWRITING a sensitive credential/SSH/shell-rc/Hermes file. - # The tee/redirection patterns above already gate _SENSITIVE_WRITE_TARGET - # (~/.ssh/*, ~/.netrc/.pgpass/.npmrc/.pypirc, shell rc files, - # ~/.hermes/config.yaml/.env), but cp/mv/install was only paired for /etc and - # project-relative env/config — so `cp evil ~/.ssh/authorized_keys` (key - # implant), `cp creds ~/.netrc`, and `cp evil ~/.bashrc` (login-time command - # injection) slipped through with auto-approve. Same unpaired-door rationale - # as #14639 / the sed-tee-redirect pairing on these targets. - # Anchor the sensitive target to the command tail so this fires on the - # DESTINATION (last arg) only — `cp evil ~/.ssh/authorized_keys` is gated, - # but reading OUT of a sensitive path (`cp ~/.ssh/config /tmp/x`) stays safe. - # The trailing `[^\s"\']*` consumes the rest of the destination filename - # (e.g. `authorized_keys` after the `~/.ssh/` fragment). - (rf'\b(cp|mv|install)\b.*\s["\']?{_SENSITIVE_WRITE_TARGET}[^\s"\']*["\']?{_COMMAND_TAIL}', "copy/move file into sensitive credential/SSH/shell-rc path"), - # In-place edits mutate the target file directly, bypassing redirection, - # tee, and copy/move/install coverage. Gate the same user-controlled - # startup/credential files so `sed -i ... ~/.bashrc` and `perl -i ... - # ~/.ssh/authorized_keys` cannot silently plant login commands or keys. - (rf'\bsed\s+-[^\s]*i.*(?:{_USER_SENSITIVE_WRITE_TARGET})[^\s"\']*', "in-place edit of sensitive credential/SSH/shell-rc path"), - (rf'\bsed\s+--in-place\b.*(?:{_USER_SENSITIVE_WRITE_TARGET})[^\s"\']*', "in-place edit of sensitive credential/SSH/shell-rc path (long flag)"), - (rf'\b(?:perl|ruby)\b.*(?:^|\s)-[^\s]*i\b.*(?:{_USER_SENSITIVE_WRITE_TARGET})[^\s"\']*', "in-place edit of sensitive credential/SSH/shell-rc path (perl/ruby)"), - (rf'\bsed\s+-[^\s]*i.*\s{_SYSTEM_CONFIG_PATH}', "in-place edit of system config"), - (rf'\bsed\s+--in-place\b.*\s{_SYSTEM_CONFIG_PATH}', "in-place edit of system config (long flag)"), - # In-place edit of a Hermes-managed security file (~/.hermes/config.yaml or - # .env). sed -i bypasses the redirection/tee patterns above because it - # mutates the file directly. Pairs the file_tools write_file/patch deny so - # the terminal side is not an open door. See #14639. - (rf'\bsed\s+-[^\s]*i.*(?:{_HERMES_CONFIG_PATH}|{_HERMES_ENV_PATH})', "in-place edit of Hermes config/env"), - (rf'\bsed\s+--in-place\b.*(?:{_HERMES_CONFIG_PATH}|{_HERMES_ENV_PATH})', "in-place edit of Hermes config/env (long flag)"), - # perl -i and ruby -i perform the same in-place mutation as sed -i but are - # not caught by the -e/-c script-execution pattern above (which targets code - # evaluation, not file mutation). Pairs the sed -i coverage from #14639. - # The -i flag can appear as its own token after other flags - # (`perl -p -i -e ... config.yaml`), combined (`perl -pi -e`), or with a - # backup suffix (`perl -i.bak`). Match any flag token containing `i` - # anywhere in the args, not just the first token — `perl -e '...'` (code - # eval, no -i) does not trip because it has no `-...i` flag token. - (rf'\b(?:perl|ruby)\b.*(?:^|\s)-[^\s]*i\b.*(?:{_HERMES_CONFIG_PATH}|{_HERMES_ENV_PATH})', "in-place edit of Hermes config/env (perl/ruby)"), - # Interpreter heredocs are handled by _execution_flag_findings() alongside - # inline-exec flags; keep only shell heredocs regex-based here. - # Shell execution via heredoc — `bash <<'EOF' ... EOF` runs arbitrary - # shell commands without triggering the `bash -c` pattern above. The - # inner commands may not individually match any dangerous pattern (e.g. - # data-exfiltration pipelines using curl/cat) yet are still executed in - # a full shell context. - (r'\b(bash|sh|zsh|ksh)\s+<<', "shell execution via heredoc"), - # Git destructive operations that can lose uncommitted work or rewrite - # shared history. Not captured by rm/chmod/etc patterns. - # `git reset --hard` accepts any unambiguous long-flag prefix (--h, - # --ha, --har, --hard) because git's own option parser resolves - # abbreviated long flags -- `--hard` is the only `git reset` mode - # starting with "h" (siblings are --soft/--mixed/--merge/--keep), so - # this cannot collide with another reset mode. It also does not match - # `--help`, which git special-cases before mode resolution. - (r'\bgit\s+reset\s+--h(?:a(?:r(?:d)?)?)?\b', "git reset --hard (destroys uncommitted changes)"), - (r'\bgit\s+push\b.*--forc[a-z]*\b', "git force push (rewrites remote history)"), - (r'\bgit\s+push\b.*-f\b', "git force push short flag (rewrites remote history)"), - (r'\bgit\s+clean\s+-[^\s]*f', "git clean with force (deletes untracked files)"), - (r'\bgit\s+branch\s+-D\b', "git branch force delete"), - # `-D` is shorthand for `-d --force`; the long-flag spellings - # (`--delete`, `--force`) are different tokens entirely, so they slip - # past the `-D\b` pattern above even though `git branch -d --force` - # and `git branch --delete --force` delete an unmerged branch exactly - # like `-D` does. Match delete+force in either order, bounded to the - # same command segment (not spanning `;`/`|`/`&`/newline) the same - # way the sudo patterns below do, to avoid contaminating an unrelated - # later command in the same script. - (r'\bgit\s+branch\b[^;|&\n]*?(?:-d\b|--delete\b)[^;|&\n]*?(?:-f\b|--force\b)', "git branch force delete (long flags)"), - (r'\bgit\s+branch\b[^;|&\n]*?(?:-f\b|--force\b)[^;|&\n]*?(?:-d\b|--delete\b)', "git branch force delete (long flags, force-first)"), - # Script execution after chmod +x — catches the two-step pattern where - # a script is first made executable then immediately run. The script - # content may contain dangerous commands that individual patterns miss. - (r'\bchmod\s+\+x\b.*[;&|]+\s*\./', "chmod +x followed by immediate execution"), - # Sudo with stdin / askpass / shell / list-privs flags. An LLM-driven - # agent has no TTY, so sudo invocations that succeed without human - # interaction are those reading the password from stdin (-S/--stdin) - # or via an askpass helper (-A/--askpass). The shell-launch (-s) and - # list-privileges (-a) flags are also gated since they are - # privilege-relevant invocations the agent can chain after acquiring - # the password (e.g. read SUDO_PASSWORD from .env -> sudo -S -s -> - # root shell). Plain `sudo cmd` (no flag) is TTY-bound and excluded. - # `_normalize_command_for_detection` lowercases input before pattern - # matching, so case variants of S/s and A/a collapse — both forms - # are gated below. Lazy `[^;|&\n]*?` allows flag arguments (e.g. - # `sudo -u root -S whoami`) without spanning command separators. See - # #17873 category 4. - # sudo's own option parser (like git's) resolves unambiguous - # long-flag prefixes, so `sudo --stdi` runs identically to - # `sudo --stdin` and `sudo --ask` to `sudo --askpass` -- confirmed - # against a live sudo binary. `--st[a-z]*` and `--a[a-z]*` are safe - # to match broadly: per `man sudo`, `--stdin` is the only long option - # starting with "st" (siblings are --shell/--set-home) and - # `--askpass` is the only one starting with "a" at all. - (r'\bsudo\b[^;|&\n]*?\s+(?:-s\b|--st[a-z]*\b|-a\b|--a[a-z]*\b)', - "sudo with privilege flag (stdin/askpass/shell/list)"), - # Combined short-flag form: -nS, -ns, -sa, -las — sudo flags packed - # into a single -X token. Catches the same threat class. - (r'\bsudo\b[^;|&\n]*?\s+-[a-z]*[sa][a-z]*\b', - "sudo with combined-flag privilege escalation"), -] - - -# Pre-compiled variant (same rationale as HARDLINE_PATTERNS_COMPILED above). -DANGEROUS_PATTERNS_COMPILED = [ - (re.compile(pattern, _RE_FLAGS), description) - for pattern, description in DANGEROUS_PATTERNS -] - - -def _legacy_pattern_key(pattern: str) -> str: - """Reproduce the old regex-derived approval key for backwards compatibility.""" - return pattern.split(r'\b')[1] if r'\b' in pattern else pattern[:20] - - -_PATTERN_KEY_ALIASES: dict[str, set[str]] = {} -for _pattern, _description in DANGEROUS_PATTERNS: - _legacy_key = _legacy_pattern_key(_pattern) - _canonical_key = _description - _PATTERN_KEY_ALIASES.setdefault(_canonical_key, set()).update({_canonical_key, _legacy_key}) - _PATTERN_KEY_ALIASES.setdefault(_legacy_key, set()).update({_legacy_key, _canonical_key}) - -# Preserve approvals stored under the removed interpreter regex rules. -_REMOVED_PATTERN_KEY_ALIASES = { - "script execution via -e/-c flag": "(python[23]?|perl|ruby|node)\\s+-[ec]\\s+", - "script execution via heredoc": "(python[23]?|perl|ruby|node)\\s+<<", -} -for _canonical_key, _legacy_key in _REMOVED_PATTERN_KEY_ALIASES.items(): - _PATTERN_KEY_ALIASES.setdefault(_canonical_key, set()).update( - {_canonical_key, _legacy_key} - ) - _PATTERN_KEY_ALIASES.setdefault(_legacy_key, set()).update( - {_legacy_key, _canonical_key} - ) - - -def _approval_key_aliases(pattern_key: str) -> set[str]: - """Return all approval keys that should match this pattern. - - New approvals use the human-readable description string, but older - command_allowlist entries and session approvals may still contain the - historical regex-derived key. - """ - return _PATTERN_KEY_ALIASES.get(pattern_key, {pattern_key}) - - -# ========================================================================= -# Detection -# ========================================================================= - -def _normalize_command_for_detection(command: str) -> str: - """Normalize a command string before dangerous-pattern matching. - - Strips ANSI escape sequences (full ECMA-48 via tools.ansi_strip), - null bytes, and normalizes Unicode fullwidth characters so that - obfuscation techniques cannot bypass the pattern-based detection. - """ - from tools.ansi_strip import strip_ansi - - # Strip all ANSI escape sequences (CSI, OSC, DCS, 8-bit C1, etc.) - command = strip_ansi(command) - # Strip null bytes - command = command.replace('\x00', '') - # Normalize Unicode (fullwidth Latin, halfwidth Katakana, etc.) - command = unicodedata.normalize('NFKC', command) - # Collapse shell line continuations (backslash-newline). The shell removes - # BOTH characters and joins the tokens, so `rm -rf \/` executes as - # `rm -rf /`. This must run BEFORE the generic backslash-escape strip below, - # whose [^\n] class deliberately skips newlines and would otherwise leave - # the dangling backslash wedged between tokens — defeating the structured - # rm/mkfs/dd patterns (notably the HARDLINE root-delete floor, which cannot - # be bypassed even with yolo). Handles both \n and \r\n line endings. Line - # continuations carry no path separator, so this is a no-op on the Windows - # home-prefix folds below (which match C:\Users\alice\... — no newline). - command = re.sub(r'\\\r?\n', '', command) - # Fold absolute home / active-profile-home prefixes into their canonical - # ~/ and ~/.hermes/ forms so static user-sensitive patterns catch - # /home/alice/.bashrc and C:\Users\alice\.bashrc the same way they catch - # ~/.bashrc. Resolve at detection time (not via an import-time snapshot) so - # it tracks HOME / HERMES_HOME even when those are set after this module is - # imported — as the hermetic test conftest and profile/session launchers do. - # - # This MUST run before the backslash-escape strip below: on Windows the home - # prefix is separated by backslashes (C:\Users\alice\...), which that strip - # would otherwise dissolve (-> C:Usersalice) and make the fold impossible. - # The fold matches either separator, so POSIX paths are unaffected by order. - # - # Fold the (more specific) Hermes home first: on Windows it nests under the - # user home (C:\Users\alice\AppData\...\hermes), so folding the user home - # first would eat the prefix the Hermes-home fold needs. - command = _rewrite_resolved_hermes_home(command) - command = _rewrite_resolved_user_home(command) - # Strip shell backslash-escapes: r\m → rm. Prevents \-injection bypass. - command = re.sub(r'\\([^\n])', r'\1', command) - # Strip empty-string literals that split tokens: r''m → rm, r"\"m → rm. - command = re.sub(r"''|\"\"", '', command) - # Collapse $IFS / ${IFS} word-separator expansions to a literal space. - # In any POSIX shell the IFS variable defaults to , - # so `rm${IFS}-rf${IFS}/` is executed as `rm -rf /`. Because the dangerous - # and hardline patterns anchor on literal whitespace (\s) between a command - # and its arguments, leaving the unexpanded `${IFS}` token in place lets an - # attacker slip past EVERY pattern — including the unconditional hardline - # floor (rm -rf /, mkfs, dd to raw device, shutdown/reboot). Substituting a - # space here mirrors the shell's own expansion so the patterns fire. The - # brace form also covers bash substring expansions like `${IFS:0:1}` (a - # single space). Same de-obfuscation class as the backslash/empty-quote - # handling above. - command = re.sub(r'\$\{IFS\b[^}]*\}|\$IFS\b', ' ', command) - return command - - -# Shell metacharacters, quotes, and whitespace that terminate a filesystem -# path token on a command line. Used to bound the path tail we normalize. -_PATH_TOKEN_STOP = r"""\s'"`;|&<>()""" -# One path segment (no separators, no terminators) preceded by a separator. -_PATH_TAIL = r"(?P(?:[/\\][^/\\" + _PATH_TOKEN_STOP + r"]*)+)" - - -@functools.lru_cache(maxsize=64) -def _home_prefix_fold_regex(path: str): - """Compile a regex matching *path* used as an absolute directory prefix. - - The home components are matched with either separator (``/`` or ``\\``) - between them, followed by the rest of the path token (the ``tail`` group), - so a Windows native path (``C:\\Users\\alice\\.ssh\\authorized_keys``), its - forward-slash form, and mixed-separator forms all fold — and the tail's - backslashes get normalized to ``/`` by the caller so multi-segment static - patterns (``~/.ssh/authorized_keys``) still match. The trailing tail is - required (``+``), so a bare home with no path under it is not folded. - - Returns ``None`` for an unset or degenerate path — one with fewer than two - components below the root — so a stray HOME / HERMES_HOME such as ``/``, - ``C:\\`` or ``""`` cannot rewrite unrelated filesystem prefixes. Cached - because the resolved home is stable across calls on this hot path. - """ - if not path: - return None - components = [c for c in re.split(r"[/\\]+", path) if c] - # Require at least two non-empty components below the root. For POSIX this - # mirrors the historical ``count("/") >= 2`` guard (``/home/alice`` folds, - # ``/home`` does not); for Windows it rejects a bare drive root (``C:\\``) - # while accepting a real home (``C:\\Users\\alice``). - if len(components) < 2: - return None - body = r"[/\\]+".join(re.escape(c) for c in components) - # Optional leading root separator (POSIX ``/`` or UNC ``\\``); a Windows - # drive letter is captured as the first component. - return re.compile(r"[/\\]*" + body + _PATH_TAIL) - - -def _fold_home_prefixes(command: str, paths, replacement: str) -> str: - """Fold each resolved home *path* prefix in *command* to *replacement*. - - *replacement* has no trailing separator (``~`` / ``~/.hermes``); the matched - path tail (with its backslashes normalized to ``/``) supplies it. Longest - candidate first so a deeper home (e.g. an explicit HOME under USERPROFILE) - folds before a shorter overlapping one that would otherwise clobber it. - """ - seen: set[str] = set() - for path in sorted((p for p in paths if p), key=len, reverse=True): - if path in seen: - continue - seen.add(path) - pattern = _home_prefix_fold_regex(path) - if pattern is not None: - command = pattern.sub( - lambda m: replacement + m.group("tail").replace("\\", "/"), - command, - ) - return command - - -def _rewrite_resolved_user_home(command: str) -> str: - """Rewrite the current user's absolute home prefix to ``~/``. - - Resolves the home at detection time — its expanduser form, symlink-resolved - form, and an explicitly set ``HOME`` — so absolute home paths are checked by - the same static patterns as tilde and ``$HOME`` forms. ``HOME`` is consulted - directly because Windows' ``os.path.expanduser`` resolves ``~`` from - ``USERPROFILE`` and ignores ``HOME``, unlike POSIX. Matches both POSIX - (``/home/alice``) and Windows (``C:\\Users\\alice`` or ``C:/Users/alice``) - separators. No-op when the home is unset or degenerate. - """ - try: - home = os.path.expanduser("~") - candidates = [ - home, - os.path.realpath(home), - os.environ.get("HOME", ""), - ] - except Exception: - return command - return _fold_home_prefixes(command, candidates, "~") - - -def _rewrite_resolved_hermes_home(command: str) -> str: - """Rewrite the resolved absolute Hermes home prefix to ``~/.hermes/``. - - Resolves the active ``HERMES_HOME`` at call time (and its symlink-resolved - form) and folds an occurrence of ``/`` in *command* into - ``~/.hermes/`` so the static ``_HERMES_CONFIG_PATH`` / ``_HERMES_ENV_PATH`` - patterns match. In Docker and gateway deployments the agent often references - the resolved absolute path directly (e.g. ``sed -i ... - /home/hermes/.hermes/config.yaml``) rather than ``~``, ``$HOME``, or - ``$HERMES_HOME``. Matches both POSIX and Windows separators. No-op when the - path can't be resolved or doesn't appear. - """ - try: - from hermes_constants import get_hermes_home - home = get_hermes_home().expanduser() - candidates = [ - str(home), - str(home.resolve(strict=False)), - ] - except Exception: - return command - return _fold_home_prefixes(command, candidates, "~/.hermes") - - -_PARAM_REPLACEMENT_RE = re.compile(r"\$\{[^}/\s]+/[^}/]*/(?P[^}]*)\}") -_PARAM_DEFAULT_RE = re.compile(r"\$\{[^}:}\s]+:-(?P[^}]*)\}") -_SIMPLE_SHELL_LITERAL_RE = re.compile(r"^[A-Za-z0-9_./:@%+=,-]+$") -_ENV_ASSIGNMENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*=.*") -_COMMAND_WRAPPER_WORDS = { - "sudo", - "env", - "exec", - "nohup", - "setsid", - "time", - "command", - "builtin", -} -_SUDO_OPTIONS_WITH_ARG = { - "-c", "--close-from", - "-g", "--group", - "-h", "--host", - "-p", "--prompt", - "-u", "--user", -} - -_INTERPRETER_EXEC_FLAGS = { - "python": {"-c"}, - "node": {"-e", "--eval", "-p", "--print"}, - "perl": {"-e", "--eval"}, - "ruby": {"-e"}, - "php": {"-r"}, - "powershell": {"-command", "-c", "-file", "-f"}, -} -_INTERPRETER_WITH_ARG = { - "python": {"-W", "-X", "--check-hash-based-pycs"}, - "node": {"-C", "--conditions", "--cpu-prof-dir", "--diagnostic-dir", "--icu-data-dir", "--import", "--loader", "--openssl-config", "--require", "--title"}, - "perl": {"-0", "-F", "-I", "-M", "-m", "-x"}, - "ruby": {"-C", "-E", "-F", "-I", "-K", "-r"}, - "php": {"-c", "-d", "-z"}, - "powershell": {"-configurationname", "-custompipename", "-executionpolicy", "-inputformat", "-outputformat", "-settingsfile", "-version", "-windowstyle", "-workingdirectory"}, -} -_READ_TOOL_EXEC_FLAGS = { - "sort": {"--compress-program"}, - "rg": {"--pre", "--hostname-bin"}, - "ag": {"--pager"}, - "man": {"--pager", "--html", "-P", "-H"}, -} -# Required-argument options are ownership boundaries: an option-looking next -# token is data, not another option. These sets mirror the invocation grammar -# of the supported binaries (ripgrep 14, GNU sort, man-db, and ag 2.2). -_READ_TOOL_LONG_OPTIONS_WITH_ARG = { - "rg": { - "--after-context", "--before-context", "--color", "--colors", - "--context", "--context-separator", "--dfa-size-limit", "--encoding", - "--engine", "--field-context-separator", "--field-match-separator", - "--file", "--generate", "--glob", "--hostname-bin", - "--hyperlink-format", "--iglob", "--ignore-file", "--max-columns", - "--max-count", "--max-depth", "--max-filesize", "--path-separator", - "--pre", "--pre-glob", "--regex-size-limit", "--regexp", "--replace", - "--sort", "--sortr", "--threads", "--type", "--type-add", - "--type-clear", "--type-not", - }, - "sort": { - "--batch-size", "--buffer-size", "--compress-program", - "--field-separator", "--files0-from", "--key", "--output", - "--parallel", "--random-source", "--sort", "--temporary-directory", - }, - "man": { - "--config-file", "--encoding", "--extension", "--locale", - "--manpath", "--pager", "--preprocessor", "--prompt", "--recode", - "--sections", "--systems", - }, - "ag": { - "--ackmate-dir-filter", "--color-line-number", "--color-match", - "--color-path", "--depth", "--filename-pattern", "--file-search-regex", - "--ignore", "--ignore-dir", "--max-count", "--pager", - "--path-to-ignore", "--width", "--workers", - }, -} -_READ_TOOL_SHORT_OPTIONS_WITH_ARG = { - "rg": frozenset("efEmjgdtTABCMr"), - "sort": frozenset("koStT"), - "man": frozenset("CRLmMSserEPp"), - "ag": frozenset("gGmpW"), -} -_SHELL_PUNCTUATION = {";", "&", "&&", "|", "||", "(", ")", "{", "}"} -_MAX_DETECTION_COMMAND_CHARS = 128_000 -_MAX_SEPARATOR_FREE_COMMAND_CHARS = 4_096 -_MAX_DETECTION_SEGMENTS = 25_000 -_PARSER_LIMIT_DESCRIPTION = "command parser limit exceeded" -_MALFORMED_EXEC_DESCRIPTION = "command parser limit or malformed executable payload" - - - -def _command_parser_limit_exceeded(command: str) -> bool: - """Bound all parser work before normalization/tokenization. - - Counting separator characters is deliberately conservative: quoted - separators can over-count, but crossing this very high ceiling fails - closed rather than allowing an uninspected suffix to execute. - """ - if len(command) > _MAX_DETECTION_COMMAND_CHARS: - return True - # Long separator-free input has no compound-command utility and otherwise - # makes every legacy regex inspect one giant token. Reject it before any - # normalization, tokenization, or regex work. - if ( - len(command) > _MAX_SEPARATOR_FREE_COMMAND_CHARS - and not any(char in command for char in ";&|\n") - ): - return True - separators = 0 - for char in command: - if char in ";&|\n": - separators += 1 - if separators >= _MAX_DETECTION_SEGMENTS: - return True - return False - - -def _shell_tokens_with_spans(segment: str, start: int): - """Return shell words as ``(value, start, end, quoted)`` or ``None``. - - This deliberately small lexer never expands shell syntax. It exists to - preserve source spans, which ``shlex`` does not expose, while deciding - which *quoted* grep operand is data rather than another command. - """ - tokens = [] - i = start - while i < len(segment): - while i < len(segment) and segment[i].isspace(): - i += 1 - if i >= len(segment): - break - token_start = i - value = [] - quote = None - while i < len(segment) and (quote or not segment[i].isspace()): - char = segment[i] - if quote: - if char == quote: - quote = None - i += 1 - elif char == "\\" and quote == '"' and i + 1 < len(segment): - value.append(segment[i + 1]) - i += 2 - else: - value.append(char) - i += 1 - elif char in {"'", '"'}: - quote = char - i += 1 - elif char == "\\": - if i + 1 >= len(segment): - return None - value.append(segment[i + 1]) - i += 2 - else: - value.append(char) - i += 1 - if quote: - return None - raw = segment[token_start:i] - # Only a wholly single-quoted operand is inert shell data. Double - # quotes still execute $() and backticks; unquoted substitutions do too. - inert_single_quoted = ( - (raw.startswith("'") and raw.endswith("'")) - or ("='" in raw and raw.endswith("'")) - ) - tokens.append(("".join(value), token_start, i, inert_single_quoted)) - return tokens - - -_GREP_OPTIONS_WITH_ARG = { - "--after-context", "--before-context", "--binary-files", "--context", - "--directories", "--devices", "--exclude", "--exclude-dir", - "--exclude-from", "--include", "--label", "--max-count", - "--regexp", "--file", -} -_GREP_SHORT_OPTIONS_WITH_ARG = {"A", "B", "C", "D", "d", "e", "f", "m"} - - -def _quoted_grep_pattern_spans(command: str) -> tuple[list[tuple[int, int]], bool]: - """Structurally locate quoted grep PCRE operands. - - The returned boolean means the grep parse was ambiguous or malformed. In - that case callers fail closed and, critically, use the original command: - no text is hidden on an uncertain parse. - """ - spans: list[tuple[int, int]] = [] - offset = 0 - for segment in _iter_top_level_shell_segments(command): - segment_at = command.find(segment, offset) - offset = segment_at + len(segment) - for start, _, word in _iter_shell_command_word_spans(segment): - if os.path.basename(_deobfuscate_shell_word_for_detection(word)).lower() not in { - "grep", "egrep", - }: - continue - tokens = _shell_tokens_with_spans(segment, start) - if tokens is None: - return [], True - args = tokens[1:] - pcre = False - explicit_patterns = False - pattern_indexes: list[int] = [] - operand_index = None - i = 0 - options = True - while i < len(args): - token = args[i][0] - if options and token == "--": - options = False - i += 1 - continue - if options and token.startswith("--"): - option, equals, _ = token.partition("=") - if option == "--perl-regexp": - pcre = True - if option in {"--regexp", "--file"}: - explicit_patterns = True - if option in _GREP_OPTIONS_WITH_ARG and not equals: - if i + 1 >= len(args): - return [], True - if option == "--regexp": - pattern_indexes.append(i + 1) - i += 2 - continue - if option == "--regexp" and equals: - pattern_indexes.append(i) - i += 1 - continue - if options and token.startswith("-") and token != "-": - chars = token[1:] - j = 0 - while j < len(chars): - char = chars[j] - if char == "P": - pcre = True - if char in {"e", "f"}: - explicit_patterns = True - if char in _GREP_SHORT_OPTIONS_WITH_ARG: - if j + 1 < len(chars): - if char == "e": - pattern_indexes.append(i) - else: - if i + 1 >= len(args): - return [], True - if char == "e": - pattern_indexes.append(i + 1) - i += 1 - break - j += 1 - i += 1 - continue - if operand_index is None: - operand_index = i - i += 1 - if not explicit_patterns: - if operand_index is None: - return [], bool(pcre) - pattern_indexes.append(operand_index) - if pcre: - for index in pattern_indexes: - _, token_start, token_end, quoted = args[index] - if quoted: - spans.append((segment_at + token_start, segment_at + token_end)) - return spans, False - - -def _grep_safe_detection_variant(command: str) -> tuple[str, bool]: - spans, malformed = _quoted_grep_pattern_spans(command) - if malformed or not spans: - return command, malformed - parts = [] - previous = 0 - for start, end in spans: - parts.extend((command[previous:start], " " * (end - start))) - previous = end - parts.append(command[previous:]) - return "".join(parts), False - - -def _interpreter_family(executable: str) -> str | None: - name = os.path.basename(executable).lower() - if re.fullmatch(r"py(?:\.exe)?|python[23]?(?:\.\d+)*(?:\.exe)?", name): - return "python" - if re.fullmatch(r"node(?:js)?(?:\.exe)?", name): - return "node" - if re.fullmatch(r"perl[0-9]*(?:\.\d+)*(?:\.exe)?", name): - return "perl" - if re.fullmatch(r"ruby[0-9.]*(?:\.exe)?", name): - return "ruby" - if re.fullmatch(r"php(?:\.exe)?", name): - return "php" - if re.fullmatch(r"powershell(?:\.exe)?|pwsh(?:\.exe)?", name): - return "powershell" - return None - - -def _shell_segment_tokens(segment: str, start: int) -> list[str] | None: - """Tokenize an already-bounded command segment. - - ``None`` distinguishes malformed quoting from an empty segment so callers - can fail closed for a program-bearing option rather than silently skip it. - """ - try: - lexer = shlex.shlex(segment[start:], posix=True, punctuation_chars="<>") - lexer.whitespace_split = True - lexer.commenters = "" - return list(lexer) - except ValueError: - return None - - -def _iter_top_level_shell_segments(command: str): - """Yield top-level command segments in one left-to-right pass.""" - start = 0 - quote: str | None = None - escaped = False - index = 0 - while index < len(command): - char = command[index] - if escaped: - escaped = False - elif char == "\\" and quote != "'": - escaped = True - elif quote: - if char == quote: - quote = None - elif char in {"'", '"'}: - quote = char - elif char in ";&|\n": - if start < index: - yield command[start:index] - # Consume a doubled && / || separator as one boundary. - if char in "&|" and index + 1 < len(command) and command[index + 1] == char: - index += 1 - start = index + 1 - index += 1 - if start < len(command): - yield command[start:] - - -def _split_option(token: str) -> tuple[str, str | None]: - if "=" in token: - option, value = token.split("=", 1) - return option, value - return token, None - - -def _interpreter_exec_flag(family: str, args: list[str]) -> str | None: - """Return an execution-bearing interpreter option, if present.""" - flags = _INTERPRETER_EXEC_FLAGS[family] - skip_value = False - for token in args: - if skip_value: - skip_value = False - continue - if token == "--": - break - if family != "powershell" and not token.startswith("-"): - break - option, attached = _split_option(token) - comparable = option.lower() if family == "powershell" else option - if comparable in flags: - return comparable - with_arg = _INTERPRETER_WITH_ARG[family] - # `-Wonce` and `ruby -rjson` attach an option value; they are not - # short-option bundles containing an execution flag. PowerShell's - # normal long options also use one dash, so bundle parsing never - # applies to that family. - has_attached_option_value = any( - option.startswith(short) and len(option) > len(short) - for short in with_arg - if short.startswith("-") and not short.startswith("--") - ) - if ( - family != "powershell" - and not option.startswith("--") - and len(option) > 2 - and not has_attached_option_value - ): - for char in option[1:]: - short = f"-{char}" - if short in flags: - return short - if comparable in with_arg and attached is None: - skip_value = True - return None - - -_BASH_OPTIONS_WITH_ARG = {"-O", "+O", "-o", "+o", "--init-file", "--rcfile"} -_BASH_SHORT_OPTION_LETTERS = frozenset("ilrsDcabefhkmnptuvxBCEHPTOo") - - -def _bash_exec_payload(args: list[str]) -> tuple[bool, str | None]: - """Return whether Bash ``-c`` occurs and the command string it owns. - - Bash's O/o invocation options consume the following argument even when - they precede a later ``-c`` or occur in the same short-option bundle. - Likewise, the two startup-file long options own their next token. Parsing - those operands first prevents both missed payloads and false ``-c`` hits. - """ - index = 0 - while index < len(args): - token = args[index] - if token == "--" or not token.startswith(("-", "+")): - break - if token in _BASH_OPTIONS_WITH_ARG: - index += 2 - continue - if token.startswith("--"): - index += 1 - continue - - chars = token[1:] - # Bash option letters are case-sensitive. Restricting this to its - # documented alphabet preserves invalid controls such as `-Wc`. - if not set(chars) <= _BASH_SHORT_OPTION_LETTERS: - index += 1 - continue - consumed_option_arg = "O" in chars or "o" in chars - if "c" not in chars: - index += 1 + int(consumed_option_arg) - continue - payload_index = index + 1 + int(consumed_option_arg) - payload = args[payload_index] if payload_index < len(args) else None - return True, payload - return False, None - - -def _read_tool_exec_flag(tool: str, args: list[str]) -> tuple[str, str] | None: - """Return (option, program) for a read-only tool's program-running flag.""" - flags = _READ_TOOL_EXEC_FLAGS[tool] - index = 0 - while index < len(args): - token = args[index] - if token == "--": - break - option, payload = _split_option(token) - matched = option if option in flags else None - if tool == "man" and token.startswith(("-P", "-H")) and len(token) > 2: - matched, payload = token[:2], token[2:] - if matched: - if payload is None and index + 1 < len(args): - payload = args[index + 1] - # This option owns its program argument regardless of spelling. - # The real binaries execute a payload beginning with '-' rather - # than reparsing it as one of the tool's later options. - if payload: - return matched, payload - index += 2 if payload is not None and "=" not in token else 1 - continue - - if option in _READ_TOOL_LONG_OPTIONS_WITH_ARG[tool] and payload is None: - index += 2 - continue - - # In a short bundle, the first argument-taking option owns the rest of - # the token, or the following token when it occurs last. - if token.startswith("-") and not token.startswith("--") and len(token) > 1: - for short_index, char in enumerate(token[1:], start=1): - if char in _READ_TOOL_SHORT_OPTIONS_WITH_ARG[tool]: - index += 2 if short_index == len(token) - 1 else 1 - break - else: - index += 1 - continue - index += 1 - return None - - -def _execution_flag_findings(command: str): - """Yield scoped execution mechanisms and any executable payloads.""" - for segment in _iter_top_level_shell_segments(command): - for start, _, word in _iter_shell_command_word_spans(segment): - executable = _deobfuscate_shell_word_for_detection(word) - tokens = _shell_segment_tokens(segment, start) - executable_name = os.path.basename(executable).lower() - family = _interpreter_family(executable) - is_program_bearing = ( - family is not None or executable_name in _READ_TOOL_EXEC_FLAGS - ) - if tokens is None: - if is_program_bearing: - yield (_MALFORMED_EXEC_DESCRIPTION, None) - continue - if not tokens: - continue - if family: - flag = _interpreter_exec_flag(family, tokens[1:]) - if flag: - yield ("script execution via -e/-c flag", None) - continue - if any(token.startswith("<<") for token in tokens[1:]): - yield ("script execution via heredoc", None) - continue - if executable_name in {"bash", "sh", "zsh", "ksh"}: - found, payload = _bash_exec_payload(tokens[1:]) - if found: - yield ("shell command via -c/-lc flag", payload) - tool = executable_name - if tool in _READ_TOOL_EXEC_FLAGS: - finding = _read_tool_exec_flag(tool, tokens[1:]) - if finding: - option, payload = finding - yield (f"arbitrary program execution via {tool} {option}", payload) - - -def _skip_shell_whitespace(command: str, pos: int) -> int: - while pos < len(command) and command[pos].isspace(): - pos += 1 - return pos - - -def _scan_dollar_paren_end(command: str, start: int) -> int | None: - """Return the offset after a balanced ``$(...)`` command substitution.""" - depth = 1 - quote: str | None = None - i = start + 2 - while i < len(command): - ch = command[i] - if quote: - if ch == "\\" and quote == '"' and i + 1 < len(command): - i += 2 - continue - if ch == quote: - quote = None - i += 1 - continue - if ch in ("'", '"'): - quote = ch - i += 1 - continue - if ch == "\\" and i + 1 < len(command): - i += 2 - continue - if command.startswith("$(", i): - depth += 1 - i += 2 - continue - if ch == ")": - depth -= 1 - i += 1 - if depth == 0: - return i - continue - i += 1 - return None - - -def _scan_backtick_end(command: str, start: int) -> int | None: - i = start + 1 - while i < len(command): - if command[i] == "\\" and i + 1 < len(command): - i += 2 - continue - if command[i] == "`": - return i + 1 - i += 1 - return None - - -def _read_shell_word(command: str, pos: int) -> tuple[int, int, str]: - """Read one shell word without executing expansions.""" - start = _skip_shell_whitespace(command, pos) - i = start - quote: str | None = None - while i < len(command): - ch = command[i] - if quote: - if ch == "\\" and quote == '"' and i + 1 < len(command): - i += 2 - continue - if ch == quote: - quote = None - i += 1 - continue - if ch in ("'", '"'): - quote = ch - i += 1 - continue - if ch == "\\" and i + 1 < len(command): - i += 2 - continue - if command.startswith("$(", i): - end = _scan_dollar_paren_end(command, i) - if end is None: - i += 2 - else: - i = end - continue - if command.startswith("${", i): - end = command.find("}", i + 2) - if end == -1: - i += 2 - else: - i = end + 1 - continue - if ch == "`": - end = _scan_backtick_end(command, i) - if end is None: - i += 1 - else: - i = end - continue - if ch.isspace() or ch in ";&|": - break - i += 1 - return (start, i, command[start:i]) - - -def _strip_optional_shell_quotes(word: str) -> str: - if len(word) >= 2 and word[0] == word[-1] and word[0] in ("'", '"'): - return word[1:-1] - return word - - -def _is_simple_shell_literal(value: str) -> bool: - return bool(value and _SIMPLE_SHELL_LITERAL_RE.fullmatch(value)) - - -def _literal_command_substitution_output(script: str) -> str | None: - """Resolve tiny literal command substitutions without executing a shell.""" - try: - tokens = shlex.split(script, posix=True) - except ValueError: - return None - if not tokens: - return None - - command = tokens[0].lower() - args = tokens[1:] - if command == "echo": - while args and re.fullmatch(r"-[nEe]+", args[0]): - args = args[1:] - if len(args) == 1 and _is_simple_shell_literal(args[0]): - return args[0] - return None - - if command == "printf": - if len(args) == 1 and _is_simple_shell_literal(args[0]): - return args[0] - if ( - len(args) == 2 - and args[0] == "%s" - and _is_simple_shell_literal(args[1]) - ): - return args[1] - return None - - -def _replace_simple_command_substitutions(word: str) -> str: - chars: list[str] = [] - i = 0 - while i < len(word): - if word.startswith("$(", i): - end = _scan_dollar_paren_end(word, i) - if end is not None: - replacement = _literal_command_substitution_output(word[i + 2:end - 1]) - if replacement is not None: - chars.append(replacement) - i = end - continue - if word[i] == "`": - end = _scan_backtick_end(word, i) - if end is not None: - replacement = _literal_command_substitution_output(word[i + 1:end - 1]) - if replacement is not None: - chars.append(replacement) - i = end - continue - chars.append(word[i]) - i += 1 - return "".join(chars) - - -def _replace_simple_shell_expansions(word: str) -> str: - word = _replace_simple_command_substitutions(word) - word = _PARAM_REPLACEMENT_RE.sub(lambda match: match.group("replacement"), word) - return _PARAM_DEFAULT_RE.sub(lambda match: match.group("default"), word) - - -def _strip_shell_word_syntax(word: str) -> str: - chars: list[str] = [] - quote: str | None = None - i = 0 - while i < len(word): - ch = word[i] - if quote: - if ch == "\\" and quote == '"' and i + 1 < len(word): - chars.append(word[i + 1]) - i += 2 - continue - if ch == quote: - quote = None - i += 1 - continue - chars.append(ch) - i += 1 - continue - if ch in ("'", '"'): - quote = ch - i += 1 - continue - if ch == "\\" and i + 1 < len(word): - chars.append(word[i + 1]) - i += 2 - continue - chars.append(ch) - i += 1 - return "".join(chars) - - -def _deobfuscate_shell_word_for_detection(word: str) -> str: - """Approximate how shell syntax can spell a command word. - - This is intentionally narrow and non-executing: it only collapses shell - quoting/escaping plus simple literal command substitutions that appear in - the command word itself. - """ - deobfuscated = word - for _ in range(2): - previous = deobfuscated - deobfuscated = _replace_simple_shell_expansions(deobfuscated) - deobfuscated = _strip_shell_word_syntax(deobfuscated) - if deobfuscated == previous: - break - return deobfuscated - - -def _iter_shell_command_starts(command: str): - starts = [0] - - def scan(start: int, end: int) -> None: - quote: str | None = None - i = start - while i < end: - ch = command[i] - if quote == "'": - if ch == "'": - quote = None - i += 1 - continue - if quote == '"': - if ch == "\\" and i + 1 < end: - i += 2 - continue - if ch == '"': - quote = None - i += 1 - continue - if command.startswith("$(", i): - nested_end = _scan_dollar_paren_end(command, i) - starts.append(i + 2) - scan(i + 2, nested_end - 1 if nested_end is not None else end) - i = nested_end if nested_end is not None else end - continue - if ch == "`": - nested_end = _scan_backtick_end(command, i) - starts.append(i + 1) - scan(i + 1, nested_end - 1 if nested_end is not None else end) - i = nested_end if nested_end is not None else end - continue - i += 1 - continue - if ch in ("'", '"'): - quote = ch - i += 1 - continue - if ch == "\\" and i + 1 < end: - i += 2 - continue - if command.startswith("$(", i): - nested_end = _scan_dollar_paren_end(command, i) - starts.append(i + 2) - scan(i + 2, nested_end - 1 if nested_end is not None else end) - i = nested_end if nested_end is not None else end - continue - if ch == "`": - nested_end = _scan_backtick_end(command, i) - starts.append(i + 1) - scan(i + 1, nested_end - 1 if nested_end is not None else end) - i = nested_end if nested_end is not None else end - continue - if ch in ("(", "{"): - starts.append(i + 1) - elif ch in ";\n": - starts.append(i + 1) - elif ch in "&|": - repeated = i + 1 < end and command[i + 1] == ch - starts.append(i + 2 if repeated else i + 1) - if repeated: - i += 1 - i += 1 - - scan(0, len(command)) - - seen: set[int] = set() - for start in starts: - start = _skip_shell_whitespace(command, start) - if start < len(command) and start not in seen: - seen.add(start) - yield start - - -def _mark_command_starts(command: str) -> str: - """Insert a newline before each real (quote-aware) command start. - - ``\\n`` is already a ``_CMDPOS`` separator, so this rewrites subshell - ``(cmd)`` and brace-group ``{ cmd; }`` openers — which the flat pattern - class deliberately omits — into a form the anchored hardline/dangerous - patterns recognize, WITHOUT the quoted-prose false positives that adding - ``(`` / ``{`` to ``_CMDPOS`` would cause. Starts inside quotes are never - produced by ``_iter_shell_command_starts``, so quoted arguments such as - ``--title "block (reboot)"`` are left exactly as-is. - """ - # Collect the (whitespace-skipped) start offsets, drop 0 (already anchored - # by ``^``), and splice a newline in front of each — right-to-left so the - # earlier offsets stay valid as we mutate. - offsets = sorted(o for o in _iter_shell_command_starts(command) if o > 0) - if not offsets: - return command - # Build once instead of repeatedly slicing and copying the full command for - # every segment (quadratic at 10k+ compound-command segments). - parts: list[str] = [] - previous = 0 - for offset in offsets: - parts.extend((command[previous:offset], "\n")) - previous = offset - parts.append(command[previous:]) - return "".join(parts) - - -def _mask_quoted_newlines(command: str) -> str: - """Replace raw newlines inside single/double quotes with a space. - - Detection-only rewrite. A newline inside a quoted string is DATA to the - shell — part of the argument, not a command separator — yet the flat - ``_CMDPOS`` start-position class treats every raw ``\\n`` as a command - start. That made any multi-line quoted argument (``hermes send`` message - bodies, ``git commit -m`` messages, heredoc text) trip the hardline - blocklist when a data line began with e.g. ``sudo reboot``. - - Quote tracking mirrors ``_iter_shell_command_starts``: single quotes are - literal until the closing quote; inside double quotes a backslash escapes - the next character. Real command boundaries are unaffected: unquoted - newlines pass through untouched, ``$(``/backtick remain ``_CMDPOS`` - anchors independent of newlines, and ``_mark_command_starts`` still - re-inserts newlines at every genuine quote-aware command start. An - unclosed quote absorbs following newlines exactly as the shell would - (the quoted word continues across the line break), so masking them - cannot hide a runnable command. - """ - if "\n" not in command: - return command - out: list[str] = [] - quote: str | None = None - i = 0 - while i < len(command): - ch = command[i] - if quote: - if ch == "\\" and quote == '"' and i + 1 < len(command): - out.append(command[i:i + 2]) - i += 2 - continue - if ch == quote: - quote = None - out.append(" " if ch == "\n" else ch) - i += 1 - continue - if ch in ("'", '"'): - quote = ch - elif ch == "\\" and i + 1 < len(command): - out.append(command[i:i + 2]) - i += 2 - continue - out.append(ch) - i += 1 - return "".join(out) - - -def _iter_shell_command_word_spans(command: str): - """Yield command-position words that may be executable names.""" - for command_start in _iter_shell_command_starts(command): - pos = command_start - prefix_words = 0 - skip_wrapper_options = False - skip_next_wrapper_arg = False - while prefix_words < 12: - word_start, word_end, word = _read_shell_word(command, pos) - if word_start == word_end: - break - deobfuscated = _deobfuscate_shell_word_for_detection(word) - lower_word = deobfuscated.lower() - if skip_next_wrapper_arg: - skip_next_wrapper_arg = False - pos = word_end - prefix_words += 1 - continue - if skip_wrapper_options and lower_word.startswith("-"): - option_name = lower_word.split("=", 1)[0] - skip_next_wrapper_arg = ( - "=" not in lower_word - and option_name in _SUDO_OPTIONS_WITH_ARG - ) - pos = word_end - prefix_words += 1 - continue - - yield (word_start, word_end, word) - prefix_words += 1 - - if lower_word in _COMMAND_WRAPPER_WORDS: - skip_wrapper_options = lower_word in {"sudo", "env"} - pos = word_end - continue - if _ENV_ASSIGNMENT_RE.fullmatch(deobfuscated): - skip_wrapper_options = False - pos = word_end - continue - break - - -def _command_detection_variants(command: str): - # Mask quoted newlines BEFORE normalization: normalization strips - # backslash-escapes (\" -> ") and empty-string pairs (""), which would - # corrupt quote tracking — e.g. `echo "a\""` normalizes to `echo "a` (an - # unterminated quote), so masking the normalized text could swallow a - # REAL unquoted newline separator that follows. The raw command carries - # faithful shell quote state. - normalized = _normalize_command_for_detection(_mask_quoted_newlines(command)) - # Quote-aware grep parsing hides only structurally identified pattern - # operands. Malformed/ambiguous input remains byte-for-byte intact. - grep_safe, _ = _grep_safe_detection_variant(normalized) - seen = {grep_safe} - yield grep_safe - # Windows-path variant (#69472): normalization treats backslashes as - # shell escapes and strips them, so `del C:\Users\me\.ssh\id_rsa` - # reaches the patterns as `del C:Usersme.sshid_rsa` — no path rule can - # ever match a backslash Windows path. When the RAW command contains a - # drive-letter or UNC backslash path, also yield a variant with - # backslashes flattened to forward slashes BEFORE normalization eats - # them. Gated on a real path shape (letter, colon, backslash — or - # double backslash UNC) so POSIX escape semantics (`echo a\"b`) are - # untouched on every other command. - if re.search(r"(?:[A-Za-z]:|\\\\)[\\\\]", command) or re.search(r"[A-Za-z]:\\", command): - win_variant = _normalize_command_for_detection( - _mask_quoted_newlines(command.replace("\\", "/")) - ) - if win_variant not in seen: - seen.add(win_variant) - yield win_variant - # Program-bearing options are parsed in their owning command's context. - # Surfacing only their payload lets the hardline floor inspect the command - # that will actually run without promoting similar flags or quoted prose. - pending = [normalized] - while pending: - variant = pending.pop() - for _, payload in _execution_flag_findings(variant): - if payload and payload not in seen: - seen.add(payload) - yield payload - # A payload can begin with an option-looking program and then - # invoke a hardline command after a separator. Mark its real - # command starts just as we do for the outer command. - marked_payload = _mark_command_starts(payload) - if marked_payload != payload and marked_payload not in seen: - seen.add(marked_payload) - yield marked_payload - pending.append(payload) - # Subshell `(cmd)` and brace-group `{ cmd; }` openers put `cmd` at a real - # command position, but the flat `_CMDPOS`-anchored patterns can't see it: - # their start-position class deliberately omits `(`/`{` because a bare - # regex cannot tell `(reboot)` (real subshell) from `--title "(reboot)"` - # (quoted prose) — adding them there regresses ordinary quoted arguments. - # Instead, reconstruct the command with a newline (already a `_CMDPOS` - # separator) inserted at each command start the QUOTE-AWARE tokenizer - # found. Openers inside quotes never yield a start, so quoted prose is - # untouched, while `(reboot)` / `{ shutdown -h now; }` now anchor. This - # covers every `_CMDPOS` rule (shutdown/reboot/init/systemctl/telinit and - # the rm root/home/system floor) in one place. - marked = _mark_command_starts(grep_safe) - if marked != grep_safe and marked not in seen: - seen.add(marked) - yield marked - # Shell quoting/escaping can spell a dangerous executable name in pieces - # (for example r\m or r''m). Keep that deobfuscation scoped to command - # words so similarly shaped arguments do not become false positives. - for word_start, word_end, word in _iter_shell_command_word_spans(normalized): - deobfuscated = _deobfuscate_shell_word_for_detection(word) - if not deobfuscated or deobfuscated == word: - continue - variant = normalized[:word_start] + deobfuscated + normalized[word_end:] - if variant in seen: - continue - seen.add(variant) - yield variant - - -def _is_verification_artifact_cleanup(command: str) -> bool: - """Return whether *command* only removes one Hermes ad-hoc temp script.""" - try: - argv = shlex.split(command, posix=True) - except ValueError: - return False - if len(argv) != 3 or argv[0] != "rm" or argv[1] != "-f": - return False - - operand = argv[2] - temp_dir = os.path.realpath(tempfile.gettempdir()) - basename = os.path.basename(operand) - if operand != os.path.join(temp_dir, basename): - return False - - target = os.path.realpath(operand) - if os.path.dirname(target) != temp_dir: - return False - return re.fullmatch(r"hermes-(?:verify|ad-hoc)-[A-Za-z0-9_.-]+", basename) is not None - - -_GATEWAY_LIFECYCLE_SPLICE_DESCRIPTION = ( - "stop/restart hermes gateway via shell-spliced verb (kills running agents)" -) - - -def _is_shell_token_spliced_gateway_lifecycle(command: str) -> bool: - """Catch gateway-lifecycle verbs spelled with quote/backslash splicing. - - ``_normalize_command_for_detection`` strips backslash escapes, so - ``kick\\start`` already reaches the launchctl pattern above. Quote - splicing does not: ``_deobfuscate_shell_word_for_detection`` is - deliberately scoped to command-position words (widening it would let - quoted prose like ``git commit -m "rm -rf /"`` match the destructive - patterns), and the spliced verb sits in an ARGUMENT position. So - ``launchctl kick"start" -k gui/501/ai.hermes.gateway`` auto-approved - while executing exactly as the gated ``kickstart`` form (#80269). - - Delegate to ``cron.lifecycle_guard``, which tokenizes with shlex and is - anchored on a hermes-gateway identifier — reusing its prose - false-positive coverage instead of loosening the generic pattern - engine. This runs last, so an ordinary pattern match still wins and - keeps its more specific reason string. Unlike the guard's use inside - ``terminal_tool``, this layer only raises an approval prompt; the - non-bypassable block still lives in ``cron.lifecycle_guard``. - """ - try: - from cron.lifecycle_guard import contains_gateway_lifecycle_command - except Exception: - return False - return contains_gateway_lifecycle_command(command) - - -def detect_dangerous_command(command: str) -> tuple: - """Check if a command matches any dangerous patterns. - - Returns: - (is_dangerous, pattern_key, description) or (False, None, None) - """ - if _command_parser_limit_exceeded(command): - return (True, _PARSER_LIMIT_DESCRIPTION, _PARSER_LIMIT_DESCRIPTION) - if _is_verification_artifact_cleanup(command): - return (False, None, None) - - for command_variant in _command_detection_variants(command): - command_lower = command_variant.lower() - for pattern_re, description in DANGEROUS_PATTERNS_COMPILED: - if pattern_re.search(command_lower): - pattern_key = description - return (True, pattern_key, description) - normalized = _normalize_command_for_detection(command) - for description, _ in _execution_flag_findings(normalized): - return (True, description, description) - if _is_shell_token_spliced_gateway_lifecycle(command): - return ( - True, - _GATEWAY_LIFECYCLE_SPLICE_DESCRIPTION, - _GATEWAY_LIFECYCLE_SPLICE_DESCRIPTION, - ) - return (False, None, None) # ========================================================================= diff --git a/tools/approval_detection.py b/tools/approval_detection.py new file mode 100644 index 0000000000..0764ea2725 --- /dev/null +++ b/tools/approval_detection.py @@ -0,0 +1,2059 @@ +"""Dangerous-command detection: normalization, tokenizing, and pattern tables. + +Pure command classification for :mod:`tools.approval` — no approval state, +config reads, or prompting live here. ``tools.approval`` re-exports every +public and private name so ``from tools.approval import X`` and +``patch("tools.approval.X")`` keep working. +""" + +import functools +import logging +import os +import re +import shlex +import tempfile +import unicodedata +from typing import Optional + +logger = logging.getLogger("tools.approval") + +# Sensitive write targets that should trigger approval even when referenced +# via shell expansions like $HOME or $HERMES_HOME, or by the resolved absolute +# active profile home path such as /home/hermes/.hermes/config.yaml. The +# resolved-absolute form is folded into the ~/.hermes/ patterns at detection +# time by _normalize_command_for_detection() — see the rewrite step there — so +# these static patterns stay free of any import-time path snapshot (which would +# go stale when HERMES_HOME is set after this module is imported, e.g. under the +# hermetic test conftest or any deferred-profile-resolution path). +_SSH_SENSITIVE_PATH = r'(?:~|\$home|\$\{home\})/\.ssh(?:/|$)' +_HERMES_ENV_PATH = ( + r'(?:~\/\.hermes/|' + r'(?:\$home|\$\{home\})/\.hermes/|' + r'(?:\$hermes_home|\$\{hermes_home\})/)' + r'\.env\b' +) +# ~/.hermes/config.yaml IS the security policy: approvals.mode, yolo, and the +# permanent-approval allowlist live here, and the config cache is mtime-keyed +# so a write takes effect mid-session (the agent could flip approvals.mode=off +# and immediately bypass the gate). Pair the write_file/patch deny (file_tools +# _check_sensitive_path) with terminal-side coverage so `sed -i`, `tee`, `>`, +# `cp`, etc. targeting it are gated too — otherwise the deny is unpaired +# theater. Mirrors _HERMES_ENV_PATH; matches the HERMES_HOME override form as +# well as ~/.hermes/. +_HERMES_CONFIG_PATH = ( + r'(?:~\/\.hermes/|' + r'(?:\$home|\$\{home\})/\.hermes/|' + r'(?:\$hermes_home|\$\{hermes_home\})/)' + r'config\.yaml\b' +) +_PROJECT_ENV_PATH = r'(?:(?:/|\.{1,2}/)?(?:[^\s/"\'`]+/)*\.env(?:\.[^/\s"\'`]+)*)' +_PROJECT_CONFIG_PATH = r'(?:(?:/|\.{1,2}/)?(?:[^\s/"\'`]+/)*config\.yaml)' +_SHELL_RC_FILES = ( + r'(?:~|\$home|\$\{home\})/\.' + r'(?:bashrc|zshrc|profile|bash_profile|zprofile)\b' +) +_CREDENTIAL_FILES = ( + r'(?:~|\$home|\$\{home\})/\.' + r'(?:netrc|pgpass|npmrc|pypirc)\b' +) +# macOS: /etc, /var, /tmp, /home are symlinks to /private/{etc,var,tmp,home}. +# A command written to target /private/etc/sudoers works identically to +# /etc/sudoers on macOS but bypasses a plain "/etc/" pattern check. Match +# both forms. Inspired by Claude Code 2.1.113's "dangerous path protection". +_MACOS_PRIVATE_SYSTEM_PATH = r'/private/(?:etc|var|tmp|home)/' +# System-config paths that should trigger approval for any write/edit, +# collapsing /etc, its macOS /private/etc mirror, and /etc/sudoers.d/ into +# one shared fragment so new DANGEROUS_PATTERNS stay consistent. +_SYSTEM_CONFIG_PATH = ( + rf'(?:/etc/|{_MACOS_PRIVATE_SYSTEM_PATH})' +) +_SENSITIVE_WRITE_TARGET = ( + rf'(?:{_SYSTEM_CONFIG_PATH}|/dev/sd|' + rf'{_SSH_SENSITIVE_PATH}|' + rf'{_HERMES_ENV_PATH}|' + rf'{_HERMES_CONFIG_PATH}|' + rf'{_SHELL_RC_FILES}|' + rf'{_CREDENTIAL_FILES})' +) +_USER_SENSITIVE_WRITE_TARGET = ( + rf'(?:{_SSH_SENSITIVE_PATH}|' + rf'{_SHELL_RC_FILES}|' + rf'{_CREDENTIAL_FILES})' +) +_PROJECT_SENSITIVE_WRITE_TARGET = rf'(?:{_PROJECT_ENV_PATH}|{_PROJECT_CONFIG_PATH})' +# Anchor for the cp/mv/install rule, where the sensitive path is only a write +# target when it is the LAST argument (the destination). Requiring end-of-line +# (or a command separator) keeps `cp config.yaml backup.yaml` — config.yaml as +# the SOURCE — out of the deny. +_COMMAND_TAIL = r'(?:\s*(?:&&|\|\||;).*)?$' +# Boundary for stream-write rules (`>`/`>>` redirection and `tee`), where the +# sensitive path is ALWAYS a write target no matter what follows it. We only +# need the path token to END at a shell word boundary — whitespace, a quote, a +# command separator, a redirection operator, or end-of-line. +# Using _COMMAND_TAIL here was too strict: it required the rest of the line to +# be empty or a command separator, so `echo x > .env extra` (extra arg to echo) +# and `echo x > .env # note` (trailing comment) slipped past the deny even +# though the shell still overwrites `.env`. Mirrors the looser system-path +# redirection rule, which never had this restriction. +# +# `#` is deliberately NOT a boundary char: a real trailing comment always has +# whitespace before the `#` (already covered by `\s`), whereas a `#` glued to +# the path is part of the filename. `echo x > .env#backup` writes to the +# distinct file `.env#backup`, not `.env`, so it must stay OUT of the deny — +# the same reasoning that keeps `config.yaml.bak` safe. +_WRITE_TARGET_BOUNDARY = r'(?=[\s;&|<>"\']|$)' + +# ========================================================================= +# Hardline (unconditional) blocklist +# ========================================================================= +# +# Commands so catastrophic they should NEVER run via the agent, regardless +# of --yolo, /yolo, approvals.mode=off, or cron approve mode. This is a +# floor below yolo: opting into yolo is the user trusting the agent with +# their files and services, not trusting it to wipe the disk or power the +# box off. +# +# Hardline only applies to environments that can actually damage the host +# (local, ssh, container-host cron). Containerized backends (docker, +# singularity, modal, daytona) already bypass the dangerous-command layer +# because nothing they do can touch the host, so we leave that behavior +# alone. +# +# The list is deliberately tiny — only things with no recovery path: +# filesystem destruction rooted at /, raw block device overwrites, kernel +# shutdown/reboot, and denial-of-service commands that take the host down. +# Recoverable-but-costly operations (git reset --hard, rm -rf /tmp/x, +# chmod -R 777, curl|sh) stay in DANGEROUS_PATTERNS where yolo can pass +# them through — that's what yolo is for. +# +# Inspired by Mercury Agent's permission-hardened blocklist +# (https://github.com/cosmicstack-labs/mercury-agent). + +# Regex fragment matching the *start* of a command (i.e. positions where +# a shell would begin parsing a new command). Used by shutdown/reboot +# patterns so they don't fire on "echo reboot" or "grep 'shutdown' log". +# Matches: start of string, after command separators (; && || | newline), +# after subshell openers ( `$(` or backtick ), optionally consuming +# leading wrapper commands (sudo, env VAR=VAL, exec, nohup, setsid). +_CMDPOS = ( + # Real ;/&/| separators are converted to newlines by the quote-aware + # _mark_command_starts pass. Keeping them in this flat regex mistakes + # quoted regex/data (for example grep '(safe|rm -rf /)') for commands. + r'(?:^|[\n`]|\$\()' # start position + r'\s*' # optional whitespace + r'(?:sudo\s+(?:-[^\s]+\s+)*)?' # optional sudo with flags + r'(?:env\s+(?:\w+=\S*\s+)*)?' # optional env with VAR=VAL pairs + r'(?:(?:exec|nohup|setsid|time)\s+)*' # optional wrapper commands + r'\s*' +) + +# Destructive-path argument matcher for the rm hardline rules. +# +# The path token in `rm -rf /` is almost always written quoted in real +# shells — `rm -rf "/"`, `rm -rf "$HOME"` — and `${HOME}` is the universal +# brace form. A bare-token anchor (`(/...)(\s|$)`) silently misses all of +# these: the surrounding quote breaks both the leading position (the flag +# group can't consume `"`) and the trailing `(\s|$)` terminator, letting +# `rm -rf "/"` slip past the unconditional floor entirely. +# +# Accept the path either fully wrapped in a matching quote pair OR bare with +# a terminator. The matching-quote branch catches `rm -rf "/"` (path quoted +# on its own). The bare branch's terminator accepts whitespace, end-of-string +# OR a shell metacharacter (`) ` ; | &`) so a real root wipe inside a command +# substitution — `$(rm -rf /)`, `` `rm -rf /` `` — whose `/` is terminated by +# `)`/backtick is still caught. +def _hardline_rm_path(path_alt: str, tail: str = r'(?:\s|$|[)`;|&])') -> str: + return rf'(?:["\'](?:{path_alt})["\']|(?:{path_alt}){tail})' + + +# Protected system roots whose recursive deletion has no recovery path. +_HARDLINE_SYSTEM_DIRS = ( + r'/home|/home/\*|/root|/root/\*|/etc|/etc/\*|/usr|/usr/\*|' + r'/var|/var/\*|/bin|/bin/\*|/sbin|/sbin/\*|/boot|/boot/\*|/lib|/lib/\*' +) + +# `rm` plus its flag group, shared by the three rm hardline rules. Kept as a +# plain concatenation (not an f-string) so the regex backslashes never live +# inside an f-string replacement field — unsupported on the Python 3.11 floor. +# +# Anchored to _CMDPOS (start of line, after a command separator ; && || |, +# after a subshell opener $(/backtick, or after sudo/env/exec wrappers) so the +# rule fires only when `rm` is an actual command word — not when the literal +# string "rm -rf /" appears as DATA inside another command's argument, e.g. +# `gh pr create --title "block rm -rf / spellings"` or `git commit -m "…rm -rf +# /…"`. Those tripped the unconditional floor and could not run at all before +# the anchor. A real wipe at any command position (bare, chained, in $()/`…`, +# under sudo) still matches; the quoted-path branch in _hardline_rm_path keeps +# catching `rm -rf "/"`. +_RM_FLAG_PREFIX = _CMDPOS + r'rm\s+(-[^\s]*\s+)*' + +HARDLINE_PATTERNS = [ + # rm recursive targeting the root filesystem or protected roots. + # `${HOME}` brace form and quoted paths (`rm -rf "/"`, `rm -rf "$HOME"`) + # are handled via _hardline_rm_path so the floor cannot be bypassed with + # the ordinary quoting/brace shell idioms. + # + # The path token matches any root-anchored path whose components collapse + # back to "/" in the shell: a bare "/", repeated slashes ("//"), and + # "."/".." current/parent segments ("/.", "/./", "/..", "/../..") all + # resolve to root, optionally followed by a trailing glob ("/*", "//*"). + # Each inter-slash segment must be exactly "." or "..", so a longer dot + # run or any real name is a literal directory, NOT root — "/tmp", "/home", + # "/.ssh", "/.config" and even "/..." (a dir literally named "...") fall + # through to the softer DANGEROUS_PATTERNS / system-directory rules + # instead of being unconditionally hardline-blocked. The explicit "/ \*" + # alt preserves the slash-space-glob spelling (`rm -rf / *`, which the + # shell sees as two args: "/" plus the "*" glob). + (_RM_FLAG_PREFIX + _hardline_rm_path(r'/(?:(?:\.\.?)?/)*(?:\.\.?)?\**|/ \*'), "recursive delete of root filesystem"), + (_RM_FLAG_PREFIX + _hardline_rm_path(_HARDLINE_SYSTEM_DIRS), "recursive delete of system directory"), + (_RM_FLAG_PREFIX + _hardline_rm_path(r'(?:~|\$\{?HOME\}?)(?:/?|/\*)?'), "recursive delete of home directory"), + # Filesystem format — anchor to command position like every other + # hardline entry so quoted prose ("echo \"does this workflow use mkfs + # anywhere?\"") does not trip the unconditional floor (#93392). + (_CMDPOS + r'mkfs(\.[a-z0-9]+)?\b', "format filesystem (mkfs)"), + # Raw block device overwrites (dd + redirection). `dd` is a command-name + # token, so anchor it to command position like mkfs/rm/shutdown (#93392): + # quoted prose such as `git commit -m "never dd of=/dev/sda"` is an + # argument, not a command. The argument tail ([^\n]*of=/dev/...) is kept + # so flag order doesn't matter. + (_CMDPOS + r'dd\b[^\n]*\bof=/dev/(sd|nvme|hd|mmcblk|vd|xvd)[a-z0-9]*', "dd to raw block device"), + # The redirect rule has no command-name token to anchor (`>` appears + # mid-command: `cat f > /dev/sda`), so command-position anchoring is the + # wrong tool. It is instead matched against a QUOTE-MASKED variant of the + # command (see _QUOTE_MASKED_HARDLINE / _mask_quoted_strings) so quoted + # prose (`echo "cat f > /dev/sda"`) cannot trip it, while shell-carrying + # wrappers (sh -c / bash -c / eval) still surface their payload as a raw + # detection variant — quoting is not a bypass (#93392). + (r'>\s*/dev/(sd|nvme|hd|mmcblk|vd|xvd)[a-z0-9]*\b', "redirect to raw block device"), + # Fork bomb (classic shell form). Also positionless (the trigger is the + # function definition itself, valid anywhere in a command line), so it is + # quote-masked like the redirect rule above rather than _CMDPOS-anchored. + (r':\(\)\s*\{\s*:\s*\|\s*:\s*&\s*\}\s*;\s*:', "fork bomb"), + # Kill every process on the system — anchor the command-name token so + # `echo "kill -1 sends SIGHUP to everything"` doesn't trip (#93392). + (_CMDPOS + r'kill\s+(-[^\s]+\s+)*-1\b', "kill all processes"), + # System shutdown / reboot — anchor to command position (start of line, + # after a command separator, or after sudo/env wrappers) so we don't + # false-positive on "echo reboot" or "grep 'shutdown' logs". + # _CMDPOS matches start-of-command positions. + (_CMDPOS + r'(shutdown|reboot|halt|poweroff)\b', "system shutdown/reboot"), + (_CMDPOS + r'init\s+[06]\b', "init 0/6 (shutdown/reboot)"), + (_CMDPOS + r'systemctl\s+(poweroff|reboot|halt|kexec)\b', "systemctl poweroff/reboot"), + (_CMDPOS + r'telinit\s+[06]\b', "telinit 0/6 (shutdown/reboot)"), +] + +# Pre-compiled variant used by the hot-path matcher. Building these at module +# load eliminates the ~2.6 ms cold-cache re.compile fan-out on the first +# terminal() call per process (12 HARDLINE + 47 DANGEROUS patterns, each +# potentially evicted from Python's 512-entry ``re._cache`` by unrelated +# regex work elsewhere in the agent). DANGEROUS_PATTERNS_COMPILED is built +# at the end of this module after DANGEROUS_PATTERNS is defined. +_RE_FLAGS = re.IGNORECASE | re.DOTALL + +# Hardline rules whose trigger has no command-name token to anchor (the +# redirect target / fork-bomb definition are valid anywhere in a command +# line). These are matched against QUOTE-MASKED variants of the command so +# quoted prose (`echo "cat f > /dev/sda"`, `git commit -m "fork bomb +# :(){ :|:& };:"`) cannot trip the unconditional floor, while the raw +# payloads of shell-carrying wrappers (sh -c, bash -c, eval) are still +# scanned unmasked — quoting is not a bypass (#93392). +_QUOTE_MASKED_HARDLINE_DESCRIPTIONS = frozenset({ + "redirect to raw block device", + "fork bomb", +}) + +HARDLINE_PATTERNS_COMPILED = [ + ( + re.compile(pattern, _RE_FLAGS), + description, + description in _QUOTE_MASKED_HARDLINE_DESCRIPTIONS, + ) + for pattern, description in HARDLINE_PATTERNS +] + + +# Command names that hand a quoted argument to another shell/parser to +# EXECUTE. For these, quoted text is code, not prose, so the quote-masked +# hardline rules must scan the raw string (see detect_hardline_command). +_SHELL_CARRIER_NAMES = frozenset({ + "eval", "sh", "bash", "zsh", "ksh", "dash", "source", ".", +}) + + +def _contains_shell_carrier(command: str) -> bool: + """Return whether any command-position word is a shell-carrying command.""" + for _, _, word in _iter_shell_command_word_spans(command): + name = os.path.basename( + _deobfuscate_shell_word_for_detection(word) + ).lower() + if name in _SHELL_CARRIER_NAMES: + return True + return False + + +def _mask_quoted_prose(command: str) -> str: + """Blank out quoted string CONTENT for positionless hardline matching. + + Detection-only rewrite used by the quote-masked hardline rules + (redirect-to-block-device, fork bomb): text inside single or double + quotes is data the shell passes as an argument, so `echo "cat f > + /dev/sda"` must not trip the unconditional floor (#93392). Structure is + preserved: the quote characters themselves stay, and inside double + quotes `$(...)` command substitutions and backtick spans are kept RAW + because the shell really executes them (`echo "$(cat f > /dev/sda)"` + remains a true positive). Unquoted text is untouched. Quote tracking + mirrors _mask_quoted_newlines; an unclosed quote masks to end-of-string, + which cannot hide a runnable command (the shell would not run it + either). + """ + out: list[str] = [] + quote: str | None = None + i = 0 + n = len(command) + while i < n: + ch = command[i] + if quote == "'": + if ch == "'": + quote = None + out.append(ch) + else: + out.append(" ") + i += 1 + continue + if quote == '"': + if ch == "\\" and i + 1 < n: + out.append(" ") + i += 2 + continue + if ch == '"': + quote = None + out.append(ch) + i += 1 + continue + if ch == "$" and i + 1 < n and command[i + 1] == "(": + end = _scan_dollar_paren_end(command, i) + if end is not None: + out.append(command[i:end]) + i = end + continue + if ch == "`": + close = command.find("`", i + 1) + if close != -1: + out.append(command[i:close + 1]) + i = close + 1 + continue + out.append(" ") + i += 1 + continue + if ch == "\\" and i + 1 < n: + out.append(command[i:i + 2]) + i += 2 + continue + if ch in ("'", '"'): + quote = ch + out.append(ch) + i += 1 + return "".join(out) + + +# ========================================================================= +# Sudo stdin guard — block password guessing via "sudo -S" +# ========================================================================= +# When SUDO_PASSWORD is not configured, any explicit "sudo -S" in the +# command is the LLM piping a guessed password via stdin. This is a +# brute-force attack vector: the model iterates through candidate +# passwords, inspects sudo's "Sorry, try again" output, and refines. +# Treat this as an unconditional block — there is never a legitimate +# reason for the agent to pipe passwords to sudo -S when no password +# has been configured. +_SUDO_STDIN_RE = re.compile( + r'(?:^|[;&|`\n]|&&|\|\||\$\()\s*sudo\s+-S\b', + re.IGNORECASE) + + +def _check_sudo_stdin_guard(command: str) -> tuple: + """Detect ``sudo -S`` (stdin password) without configured SUDO_PASSWORD. + + When SUDO_PASSWORD is set, ``_transform_sudo_command`` injects ``-S`` + internally — that path is legitimate and handled elsewhere. This guard + only fires when SUDO_PASSWORD is *not* set, meaning the LLM explicitly + wrote ``sudo -S`` to pipe a guessed password. + + Returns: + (is_blocked: bool, description: str | None) + """ + if "SUDO_PASSWORD" in os.environ: + return (False, None) + normalized = _normalize_command_for_detection(command).lower() + if _SUDO_STDIN_RE.search(normalized): + return (True, "sudo password guessing via stdin (sudo -S)") + return (False, None) + + +def detect_hardline_command(command: str) -> tuple: + """Check if a command matches hardline blocklist patterns. + + Hardline patterns are NEVER bypassable, even in YOLO mode. + + Returns: + (is_hardline, description) or (False, None) + """ + if _command_parser_limit_exceeded(command): + return (True, _PARSER_LIMIT_DESCRIPTION) + normalized = _normalize_command_for_detection(command) + _, malformed_grep = _grep_safe_detection_variant(normalized) + if malformed_grep: + return (True, _MALFORMED_EXEC_DESCRIPTION) + for command_variant in _command_detection_variants(command): + variant_lower = command_variant.lower() + masked_lower: str | None = None + for pattern_re, description, quote_masked in HARDLINE_PATTERNS_COMPILED: + if quote_masked: + # Positionless rules (redirect-to-block-device, fork bomb) + # match a quote-masked variant so quoted prose in echo / + # git commit -m / gh --body arguments is DATA (#93392). + # Shell-carrying commands (sh/bash -c, eval, source) hand + # their quoted argument to another parser, so those scan + # the raw variant — quoting is not a bypass. bash/sh -c + # payloads additionally surface as their own raw variants + # via _execution_flag_findings. + if masked_lower is None: + if _contains_shell_carrier(command_variant): + masked_lower = variant_lower + else: + masked_lower = _mask_quoted_prose(command_variant).lower() + haystack = masked_lower + else: + haystack = variant_lower + if pattern_re.search(haystack): + return (True, description) + return (False, None) + + +# ========================================================================= +# Dangerous command patterns +# ========================================================================= + +DANGEROUS_PATTERNS = [ + (r'\brm\s+(-[^\s]*\s+)*/', "delete in root path"), + (r'\brm\s+-[^\s]*r', "recursive delete"), + (r'\brm\s+--recursive\b', "recursive delete (long flag)"), + # GNU rm permutes options, so a recursive flag group may legally FOLLOW + # the operands: `rm build/ -rf`, `rm build/ -r -f`, and `rm build/ + # --recursive --force` are all equivalent to the flags-first spellings the + # two patterns above catch — without this rule they run with no approval + # prompt at all. The operand run is tempered: it cannot cross a command + # separator (`;`, `|`, `&`, newline — so a later pipeline segment's flags, + # e.g. `rm foo | grep -r bar`, are not attributed to `rm`), cannot cross a + # quote (so `git commit -m "rm x" --amend` style data can't bridge an `rm` + # word to an unrelated dash token), and cannot cross a bare ` -- ` + # end-of-options separator (after `--`, POSIX rm treats `-rf` as a literal + # filename, not flags; guarded both leading and mid-run). The flag token + # itself must start right after whitespace so the `r` inside long options + # like `--registry` (preceded by `-`, not whitespace) does not count. + # Port of openai/codex#33464 ("recognize force options when they follow + # operands"). + (r'\brm\s+(?!--(?:\s|$))(?:(?!\s--(?:\s|$))[^\n"\';|&])*\s' + r'(?:-[a-z]*r[a-z]*\b|--recursive\b)', + "recursive delete (flags after operands)"), + # Windows shell front-ends have destructive built-ins that do not look like + # Unix `rm`. Gate only when they are executed through cmd/powershell so + # ordinary prose or filenames containing "del"/"rd" do not trip the guard. + (r'\bcmd(?:\.exe)?\s+/(?:c|k)\s+.*\b(?:del|erase|rd|rmdir)\b', "Windows cmd destructive delete"), + # PowerShell/pwsh: the destructive verb runs as the default positional + # argument, so `powershell Remove-Item ...` needs NO explicit -Command. + # Anchor the verb to the command position (right after the shell name, + # after any leading `-Flag` switches, and optionally after -Command/-c) + # so bare invocations are caught while a benign path arg containing + # "del"/"rm" (e.g. `-File c:\del-logs\run.ps1`) is not. + (r'\b(?:powershell|pwsh)(?:\.exe)?\b(?:\s+-\S+)*\s+(?:-(?:command|c)\s+)?["\']?(?:remove-item|rmdir|erase|del|rd|ri|rm)\b', "Windows PowerShell destructive delete"), + (r'\b(?:powershell|pwsh)(?:\.exe)?\b.*\s-(?:encodedcommand|enc|e)\b', "PowerShell encoded command execution"), + # ── Windows destructive tier (#69472) ──────────────────────────────── + # These are native Windows EXEs / cmdlets reachable from ANY Hermes + # terminal backend on a Windows host — including the default git-bash + # backend (taskkill.exe, icacls.exe, reg.exe, vssadmin.exe, bcdedit.exe, + # cipher.exe are ordinary PATH executables there). Detection input is + # lowercased by the variant loop, so patterns are written lowercase. + # Each pattern requires the destructive flag/verb so benign usage + # (`taskkill /IM app.exe` graceful kill, `reg query`, `icacls file`) + # does NOT prompt. + # Bare PowerShell destructive delete: Remove-Item/ri with -Recurse or + # -Force. The cmd/powershell-prefixed forms are covered above; this + # catches the bare form (ACP clients, pwsh-default SSH hosts, or + # `powershell` invoked earlier in a compound command). + (r'\bremove-item\b[^\n;|&]*\s-(?:recurse|force)\b', "PowerShell destructive delete (Remove-Item)"), + # cmd builtins with destructive switches, bare form: del/erase/rd/rmdir + # with /s (recurse) or /q (quiet). Requires the switch so `del file.txt` + # inside a cmd /c string stays covered by the prefixed rule only. + (r'\b(?:del|erase|rd|rmdir)\s+(?:/[a-z]\s+)*/[sq]\b', "Windows destructive delete (recursive/quiet switch)"), + # Remote content piped to Invoke-Expression — PowerShell's `curl | sh`. + (r'\b(?:iwr|invoke-webrequest|invoke-restmethod|irm|curl|wget)\b[^\n]*\|\s*(?:iex|invoke-expression)\b', "pipe remote content to PowerShell (iwr | iex)"), + (r'\b(?:iex|invoke-expression)\s*\(\s*(?:iwr|invoke-webrequest|invoke-restmethod|irm)\b', "execute remote content via Invoke-Expression"), + # Force process kills — Windows analogue of pkill -9. + (r'\btaskkill\b[^\n]*\s/f\b', "force kill processes (taskkill /F)"), + (r'\bstop-process\b[^\n]*\s-force\b', "force kill processes (Stop-Process -Force)"), + # Volume/disk destruction — Windows analogue of mkfs / dd. + (r'\bformat-volume\b', "format filesystem (Format-Volume)"), + (r'\bclear-disk\b', "wipe disk (Clear-Disk)"), + (r'\bdiskpart\b', "disk partitioning (diskpart)"), + (r'\bformat(?:\.com)?\s+[a-z]:', "format drive (format.com)"), + (r'\bcipher\s+/w\b', "wipe free space (cipher /w)"), + # ACL destruction — Windows analogue of chmod 777. + (r'\bicacls\b[^\n]*\s/grant\b[^\n]*\b(?:everyone|todos|jeder|tout\s+le\s+monde|\*s-1-1-0)\b', "grant Everyone access (icacls)"), + (r'\bicacls\b[^\n]*\s/reset\b', "reset ACLs recursively (icacls /reset)"), + # Backup/recovery destruction — classic ransomware prep, no benign + # agent use case. + (r'\bvssadmin\b[^\n]*\bdelete\s+shadows\b', "delete volume shadow copies (vssadmin)"), + (r'\bwbadmin\b[^\n]*\bdelete\b', "delete backups (wbadmin)"), + (r'\bbcdedit\b[^\n]*\s/set\b', "modify boot configuration (bcdedit /set)"), + # Registry deletion with force flag. + (r'\breg(?:\.exe)?\s+delete\b', "registry delete (reg delete)"), + (r'\bremove-itemproperty\b[^\n]*\s-force\b', "registry value delete (Remove-ItemProperty -Force)"), + # Windows service/system stop — analogue of systemctl stop. + (r'\bstop-service\b[^\n]*\s-force\b', "force stop service (Stop-Service -Force)"), + (r'\bsc(?:\.exe)?\s+(?:stop|delete)\b', "stop/delete service (sc)"), + # Credential/key paths in Windows form — the POSIX ~/.ssh patterns never + # match drive-letter or backslash spellings. Match both separators. + (r'\busers[\\/][^\\/\s]+[\\/]\.ssh\b', "access to SSH keys (Windows path)"), + (r'\bappdata[\\/](?:local|roaming)[\\/]hermes[^\n]*\.env\b', "access to Hermes secrets (Windows path)"), + # ───────────────────────────────────────────────────────────────────── + (r'\bchmod\s+(-[^\s]*\s+)*(777|666|o\+[rwx]*w|a\+[rwx]*w)\b', "world/other-writable permissions"), + (r'\bchmod\s+--recursive\b.*(777|666|o\+[rwx]*w|a\+[rwx]*w)', "recursive world/other-writable (long flag)"), + (r'\bchown\s+(-[^\s]*)?R\s+root', "recursive chown to root"), + (r'\bchown\s+--recur[a-z]*\b.*root', "recursive chown to root (long flag)"), + # Anchored to command position like the hardline twins (#93392): + # quoted prose mentioning mkfs/dd must not require approval to echo. + (_CMDPOS + r'mkfs\b', "format filesystem"), + (_CMDPOS + r'dd\s+.*if=', "disk copy"), + (r'>\s*/dev/sd', "write to block device"), + (r'\bDROP\s+(TABLE|DATABASE)\b', "SQL DROP"), + # Use [^\n]* instead of .* so DOTALL mode does not cause a WHERE clause on the + # *next* line to satisfy the negative lookahead, silently allowing DELETE without WHERE. + (r'\bDELETE\s+FROM\b(?![^\n]*\bWHERE\b)', "SQL DELETE without WHERE"), + (r'\bTRUNCATE\s+(TABLE)?\s*\w', "SQL TRUNCATE"), + (rf'>\s*{_SYSTEM_CONFIG_PATH}', "overwrite system config"), + (r'\bsystemctl\s+(-[^\s]+\s+)*(stop|restart|disable|mask)\b', "stop/restart system service"), + (r'\bkill\s+-9\s+-1\b', "kill all processes"), + (r'\bpkill\s+-9\b', "force kill processes"), + # killall with SIGKILL (parallel to pkill -9). Catches -9 / -KILL / + # -s KILL / -SIGKILL forms, and also `killall -r ` broad sweeps + # that can wipe out unrelated processes by accident. + # Inspired by Claude Code 2.1.113 expanded deny rules. + (r'\bkillall\s+(-[^\s]*\s+)*-(9|KILL|SIGKILL)\b', "force kill processes (killall -KILL)"), + (r'\bkillall\s+(-[^\s]*\s+)*-s\s+(KILL|SIGKILL|9)\b', "force kill processes (killall -s KILL)"), + (r'\bkillall\s+(-[^\s]*\s+)*-r\b', "kill processes by regex (killall -r)"), + (r':\(\)\s*\{\s*:\s*\|\s*:\s*&\s*\}\s*;\s*:', "fork bomb"), + # Shell -c is parsed structurally by _execution_flag_findings(). A regex + # that merely searched a dash-token for "c" also matched --norc, + # --rcfile, and --restricted. + (r'\b(curl|wget)\b.*\|\s*(?:[/\w]*/)?(?:ba)?sh(?:\s|$|-c)', "pipe remote content to shell"), + (r'\b(bash|sh|zsh|ksh)\s+<\s* | base64 -d | bash` silently runs `rm -rf /` or any + # other command because the raw text carries no dangerous keywords. + (r'\b(base64|base32|base16)\s+(?:-[dD]|--decode)\b.*\|\s*\b(bash|sh|zsh|ksh|dash)\b', + "pipe decoded content to shell (possible command obfuscation)"), + # xxd reverse hex dump to shell (xxd uses -r for decode, not -d). + (r'\bxxd\s+-r\b.*\|\s*\b(bash|sh|zsh|ksh|dash)\b', + "pipe xxd-decoded content to shell (possible command obfuscation)"), + # Character transformation via tr piped to shell: + # `echo 'eq -pe v/' | tr 'eqv' 'rmf' | bash` decodes to `rm -rf /`. + (r'\becho\b[^|]*\|\s*\btr\b[^|]*\|\s*\b(bash|sh|zsh|ksh|dash)\b', + "pipe tr-transformed output to shell (possible command obfuscation)"), + # openssl decode piped to shell: + # `echo | openssl base64 -d | bash` decodes arbitrary commands. + (r'\bopenssl\b.*\b(?:base64|enc)\b[^|]*\s+-[dD]\b[^|]*\|\s*\b(bash|sh|zsh|ksh|dash)\b', + "pipe openssl-decoded content to shell (possible command obfuscation)"), + (rf'\btee\b.*["\']?{_SENSITIVE_WRITE_TARGET}', "overwrite system file via tee"), + (rf'>>?\s*["\']?{_SENSITIVE_WRITE_TARGET}', "overwrite system file via redirection"), + (rf'\btee\b.*["\']?{_PROJECT_SENSITIVE_WRITE_TARGET}["\']?{_WRITE_TARGET_BOUNDARY}', "overwrite project env/config via tee"), + (rf'>>?\s*["\']?{_PROJECT_SENSITIVE_WRITE_TARGET}["\']?{_WRITE_TARGET_BOUNDARY}', "overwrite project env/config via redirection"), + (r'\bxargs\s+.*\brm\b', "xargs with rm"), + # find -exec rm / -execdir rm — the -execdir variant (same semantics, + # runs in the directory of each match) was previously missed. Claude + # Code 2.1.113 tightened their equivalent find rule to stop auto- + # approving -exec / -delete flags. + (r'\bfind\b.*-exec(?:dir)?\s+(/\S*/)?rm\b', "find -exec/-execdir rm"), + (r'\bfind\b.*-delete\b', "find -delete"), + # Gateway lifecycle protection: prevent the agent from killing its own + # gateway process. These commands trigger a gateway restart/stop that + # terminates all running agents mid-work. Allow global flags between + # `hermes` and `gateway` (e.g. `hermes -p ade gateway restart`) so a + # profile flag can't slip the agent past the guard. + (r'\bhermes\s+(?:-{1,2}\S+(?:\s+\S+)?\s+)*gateway\s+(stop|restart)\b', "stop/restart hermes gateway (kills running agents)"), + (r'\bhermes\s+update\b', "hermes update (restarts gateway, kills running agents)"), + # Docker container lifecycle — any user with docker.sock mounted (a common + # Docker Compose pattern) gives the agent the ability to restart/stop/kill + # containers without approval. These are agent-initiated lifecycle operations + # that should always require user consent, just like `hermes gateway restart` + # already does for the gateway process. + # Docker/Podman daemon redirect — global flags or env prefixes that point + # the CLI at a DIFFERENT daemon, often a remote host over ssh/tcp. A + # command that looks local (`docker -H ssh://prod stop app`) silently + # operates on remote infrastructure, so any docker/podman invocation + # carrying a redirect requires approval regardless of subcommand. The + # redirect flag must appear in the global-flag position (before the + # subcommand) and -H/--host/--context must carry a value, which keeps + # `docker -h` (help) and subcommand flags like `docker run -h ` + # out of the deny. Listed BEFORE the lifecycle rules so a redirected + # lifecycle command surfaces the more specific "remote daemon" reason. + # Inspired by Claude Code 2.1.214, which added permission prompts for + # docker/podman commands carrying daemon-redirect flags (--url, + # --connection, --identity, remote mode). + (r'\bdocker\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(?:-h|--host)[=\s]+\S+', + "docker with remote daemon redirect (-H/--host)"), + (r'\bdocker\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(?:-c|--context)[=\s]+\S+', + "docker with daemon redirect (--context: alternate daemon)"), + (r'\bdocker\s+context\s+use\b', + "docker context use (switches default daemon for future commands)"), + (r'\bpodman\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(?:--url|--connection|--identity)[=\s]+\S+', + "podman with remote daemon redirect (--url/--connection/--identity)"), + (r'\bpodman\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(?:-r\b|--remote\b)', + "podman remote mode (-r/--remote: remote daemon)"), + (r'\b(?:docker_host|docker_context|container_host|container_connection)=\S+', + "docker/podman daemon redirect via environment (DOCKER_HOST/CONTAINER_HOST)"), + # Allow global flags between `docker`/`compose` and the verb (e.g. + # `docker compose -f prod.yml down`, `docker --log-level debug stop app`) + # and the legacy hyphenated `docker-compose` binary, so a flag can't slip + # a lifecycle command past the guard — same treatment as the `hermes ... + # gateway` pattern above. + (r'\bdocker(?:-compose|\s+compose)\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(restart|stop|kill|down)\b', + "docker compose restart/stop/kill/down (container lifecycle)"), + (r'\bdocker\s+(?:-{1,2}\S+(?:[=\s]\S+)?\s+)*(restart|stop|kill)\b', + "docker restart/stop/kill (container lifecycle)"), + # Gateway protection: never start gateway outside systemd management + (r'gateway\s+run\b.*(&\s*$|&\s*;|\bdisown\b|\bsetsid\b)', "start gateway outside systemd (use 'systemctl --user restart hermes-gateway')"), + (r'\bnohup\b.*gateway\s+run\b', "start gateway outside systemd (use 'systemctl --user restart hermes-gateway')"), + # Self-termination protection: prevent agent from killing its own process + (r'\b(pkill|killall)\b.*\b(hermes|gateway|cli\.py)\b', "kill hermes/gateway process (self-termination)"), + # Self-termination via kill + command substitution (pgrep/pidof). + # The name-based pattern above catches `pkill hermes` but not + # `kill -9 $(pgrep -f hermes)` because the substitution is opaque + # to regex at detection time. Catch the structural pattern instead. + # `pidof` is the BSD/Linux alternative to `pgrep` and is equally + # opaque, so include it in the same alternation. + (r'\bkill\b.*\$\(\s*(pgrep|pidof)\b', "kill process via pgrep/pidof expansion (self-termination)"), + (r'\bkill\b.*`\s*(pgrep|pidof)\b', "kill process via backtick pgrep/pidof expansion (self-termination)"), + # launchctl-driven gateway stop/restart on macOS. The agent can bypass + # the `hermes gateway stop|restart` pattern above by driving launchd + # directly against the service label (commonly `ai.hermes.gateway`). + # Catch the operations that stop, restart, or unload it. + # + # Order-independent (2026-08-02 incident): the previous version required + # "hermes"/"ai.hermes" to appear AFTER the launchctl verb in the same + # string (`.*` only scans forward). A shell for-loop that builds the + # label from a list defined earlier in the command — e.g. `for item in + # 'ai.hermes.gateway-apollo:...' ...; do label=${item%%:*}; launchctl + # bootout "$label"; done` — never has the literal text "hermes" appear + # after "bootout" (only the expanded variable does), so it slipped past + # undetected and restarted 4 gateways with zero approval. Two + # independent lookaheads instead of one sequential match: both + # substrings must appear SOMEWHERE in the command, in either order. + # This is intentionally broader (a launchctl-verb command anywhere near + # an unrelated "hermes" mention now also matches) — for an approval gate + # that's the correct direction to err: an extra approval prompt is + # cheap, a missed one took down the whole gateway fleet. + (r'(?=[\s\S]*\blaunchctl\s+(?:stop|kickstart|bootout|unload|kill|disable|remove)\b)(?=[\s\S]*\b(?:hermes|ai\.hermes)\b)', "stop/restart hermes launchd service (kills running agents)"), + # File copy/move/edit into sensitive system paths (/etc/ and macOS + # /private/etc/ mirror). + (rf'\b(cp|mv|install)\b.*\s{_SYSTEM_CONFIG_PATH}', "copy/move file into system config path"), + (rf'\b(cp|mv|install)\b.*\s["\']?{_PROJECT_SENSITIVE_WRITE_TARGET}["\']?{_COMMAND_TAIL}', "overwrite project env/config file"), + # cp/mv/install OVERWRITING a sensitive credential/SSH/shell-rc/Hermes file. + # The tee/redirection patterns above already gate _SENSITIVE_WRITE_TARGET + # (~/.ssh/*, ~/.netrc/.pgpass/.npmrc/.pypirc, shell rc files, + # ~/.hermes/config.yaml/.env), but cp/mv/install was only paired for /etc and + # project-relative env/config — so `cp evil ~/.ssh/authorized_keys` (key + # implant), `cp creds ~/.netrc`, and `cp evil ~/.bashrc` (login-time command + # injection) slipped through with auto-approve. Same unpaired-door rationale + # as #14639 / the sed-tee-redirect pairing on these targets. + # Anchor the sensitive target to the command tail so this fires on the + # DESTINATION (last arg) only — `cp evil ~/.ssh/authorized_keys` is gated, + # but reading OUT of a sensitive path (`cp ~/.ssh/config /tmp/x`) stays safe. + # The trailing `[^\s"\']*` consumes the rest of the destination filename + # (e.g. `authorized_keys` after the `~/.ssh/` fragment). + (rf'\b(cp|mv|install)\b.*\s["\']?{_SENSITIVE_WRITE_TARGET}[^\s"\']*["\']?{_COMMAND_TAIL}', "copy/move file into sensitive credential/SSH/shell-rc path"), + # In-place edits mutate the target file directly, bypassing redirection, + # tee, and copy/move/install coverage. Gate the same user-controlled + # startup/credential files so `sed -i ... ~/.bashrc` and `perl -i ... + # ~/.ssh/authorized_keys` cannot silently plant login commands or keys. + (rf'\bsed\s+-[^\s]*i.*(?:{_USER_SENSITIVE_WRITE_TARGET})[^\s"\']*', "in-place edit of sensitive credential/SSH/shell-rc path"), + (rf'\bsed\s+--in-place\b.*(?:{_USER_SENSITIVE_WRITE_TARGET})[^\s"\']*', "in-place edit of sensitive credential/SSH/shell-rc path (long flag)"), + (rf'\b(?:perl|ruby)\b.*(?:^|\s)-[^\s]*i\b.*(?:{_USER_SENSITIVE_WRITE_TARGET})[^\s"\']*', "in-place edit of sensitive credential/SSH/shell-rc path (perl/ruby)"), + (rf'\bsed\s+-[^\s]*i.*\s{_SYSTEM_CONFIG_PATH}', "in-place edit of system config"), + (rf'\bsed\s+--in-place\b.*\s{_SYSTEM_CONFIG_PATH}', "in-place edit of system config (long flag)"), + # In-place edit of a Hermes-managed security file (~/.hermes/config.yaml or + # .env). sed -i bypasses the redirection/tee patterns above because it + # mutates the file directly. Pairs the file_tools write_file/patch deny so + # the terminal side is not an open door. See #14639. + (rf'\bsed\s+-[^\s]*i.*(?:{_HERMES_CONFIG_PATH}|{_HERMES_ENV_PATH})', "in-place edit of Hermes config/env"), + (rf'\bsed\s+--in-place\b.*(?:{_HERMES_CONFIG_PATH}|{_HERMES_ENV_PATH})', "in-place edit of Hermes config/env (long flag)"), + # perl -i and ruby -i perform the same in-place mutation as sed -i but are + # not caught by the -e/-c script-execution pattern above (which targets code + # evaluation, not file mutation). Pairs the sed -i coverage from #14639. + # The -i flag can appear as its own token after other flags + # (`perl -p -i -e ... config.yaml`), combined (`perl -pi -e`), or with a + # backup suffix (`perl -i.bak`). Match any flag token containing `i` + # anywhere in the args, not just the first token — `perl -e '...'` (code + # eval, no -i) does not trip because it has no `-...i` flag token. + (rf'\b(?:perl|ruby)\b.*(?:^|\s)-[^\s]*i\b.*(?:{_HERMES_CONFIG_PATH}|{_HERMES_ENV_PATH})', "in-place edit of Hermes config/env (perl/ruby)"), + # Interpreter heredocs are handled by _execution_flag_findings() alongside + # inline-exec flags; keep only shell heredocs regex-based here. + # Shell execution via heredoc — `bash <<'EOF' ... EOF` runs arbitrary + # shell commands without triggering the `bash -c` pattern above. The + # inner commands may not individually match any dangerous pattern (e.g. + # data-exfiltration pipelines using curl/cat) yet are still executed in + # a full shell context. + (r'\b(bash|sh|zsh|ksh)\s+<<', "shell execution via heredoc"), + # Git destructive operations that can lose uncommitted work or rewrite + # shared history. Not captured by rm/chmod/etc patterns. + # `git reset --hard` accepts any unambiguous long-flag prefix (--h, + # --ha, --har, --hard) because git's own option parser resolves + # abbreviated long flags -- `--hard` is the only `git reset` mode + # starting with "h" (siblings are --soft/--mixed/--merge/--keep), so + # this cannot collide with another reset mode. It also does not match + # `--help`, which git special-cases before mode resolution. + (r'\bgit\s+reset\s+--h(?:a(?:r(?:d)?)?)?\b', "git reset --hard (destroys uncommitted changes)"), + (r'\bgit\s+push\b.*--forc[a-z]*\b', "git force push (rewrites remote history)"), + (r'\bgit\s+push\b.*-f\b', "git force push short flag (rewrites remote history)"), + (r'\bgit\s+clean\s+-[^\s]*f', "git clean with force (deletes untracked files)"), + (r'\bgit\s+branch\s+-D\b', "git branch force delete"), + # `-D` is shorthand for `-d --force`; the long-flag spellings + # (`--delete`, `--force`) are different tokens entirely, so they slip + # past the `-D\b` pattern above even though `git branch -d --force` + # and `git branch --delete --force` delete an unmerged branch exactly + # like `-D` does. Match delete+force in either order, bounded to the + # same command segment (not spanning `;`/`|`/`&`/newline) the same + # way the sudo patterns below do, to avoid contaminating an unrelated + # later command in the same script. + (r'\bgit\s+branch\b[^;|&\n]*?(?:-d\b|--delete\b)[^;|&\n]*?(?:-f\b|--force\b)', "git branch force delete (long flags)"), + (r'\bgit\s+branch\b[^;|&\n]*?(?:-f\b|--force\b)[^;|&\n]*?(?:-d\b|--delete\b)', "git branch force delete (long flags, force-first)"), + # Script execution after chmod +x — catches the two-step pattern where + # a script is first made executable then immediately run. The script + # content may contain dangerous commands that individual patterns miss. + (r'\bchmod\s+\+x\b.*[;&|]+\s*\./', "chmod +x followed by immediate execution"), + # Sudo with stdin / askpass / shell / list-privs flags. An LLM-driven + # agent has no TTY, so sudo invocations that succeed without human + # interaction are those reading the password from stdin (-S/--stdin) + # or via an askpass helper (-A/--askpass). The shell-launch (-s) and + # list-privileges (-a) flags are also gated since they are + # privilege-relevant invocations the agent can chain after acquiring + # the password (e.g. read SUDO_PASSWORD from .env -> sudo -S -s -> + # root shell). Plain `sudo cmd` (no flag) is TTY-bound and excluded. + # `_normalize_command_for_detection` lowercases input before pattern + # matching, so case variants of S/s and A/a collapse — both forms + # are gated below. Lazy `[^;|&\n]*?` allows flag arguments (e.g. + # `sudo -u root -S whoami`) without spanning command separators. See + # #17873 category 4. + # sudo's own option parser (like git's) resolves unambiguous + # long-flag prefixes, so `sudo --stdi` runs identically to + # `sudo --stdin` and `sudo --ask` to `sudo --askpass` -- confirmed + # against a live sudo binary. `--st[a-z]*` and `--a[a-z]*` are safe + # to match broadly: per `man sudo`, `--stdin` is the only long option + # starting with "st" (siblings are --shell/--set-home) and + # `--askpass` is the only one starting with "a" at all. + (r'\bsudo\b[^;|&\n]*?\s+(?:-s\b|--st[a-z]*\b|-a\b|--a[a-z]*\b)', + "sudo with privilege flag (stdin/askpass/shell/list)"), + # Combined short-flag form: -nS, -ns, -sa, -las — sudo flags packed + # into a single -X token. Catches the same threat class. + (r'\bsudo\b[^;|&\n]*?\s+-[a-z]*[sa][a-z]*\b', + "sudo with combined-flag privilege escalation"), +] + + +# Pre-compiled variant (same rationale as HARDLINE_PATTERNS_COMPILED above). +DANGEROUS_PATTERNS_COMPILED = [ + (re.compile(pattern, _RE_FLAGS), description) + for pattern, description in DANGEROUS_PATTERNS +] + + +def _legacy_pattern_key(pattern: str) -> str: + """Reproduce the old regex-derived approval key for backwards compatibility.""" + return pattern.split(r'\b')[1] if r'\b' in pattern else pattern[:20] + + +_PATTERN_KEY_ALIASES: dict[str, set[str]] = {} +for _pattern, _description in DANGEROUS_PATTERNS: + _legacy_key = _legacy_pattern_key(_pattern) + _canonical_key = _description + _PATTERN_KEY_ALIASES.setdefault(_canonical_key, set()).update({_canonical_key, _legacy_key}) + _PATTERN_KEY_ALIASES.setdefault(_legacy_key, set()).update({_legacy_key, _canonical_key}) + +# Preserve approvals stored under the removed interpreter regex rules. +_REMOVED_PATTERN_KEY_ALIASES = { + "script execution via -e/-c flag": "(python[23]?|perl|ruby|node)\\s+-[ec]\\s+", + "script execution via heredoc": "(python[23]?|perl|ruby|node)\\s+<<", +} +for _canonical_key, _legacy_key in _REMOVED_PATTERN_KEY_ALIASES.items(): + _PATTERN_KEY_ALIASES.setdefault(_canonical_key, set()).update( + {_canonical_key, _legacy_key} + ) + _PATTERN_KEY_ALIASES.setdefault(_legacy_key, set()).update( + {_legacy_key, _canonical_key} + ) + + +def _approval_key_aliases(pattern_key: str) -> set[str]: + """Return all approval keys that should match this pattern. + + New approvals use the human-readable description string, but older + command_allowlist entries and session approvals may still contain the + historical regex-derived key. + """ + return _PATTERN_KEY_ALIASES.get(pattern_key, {pattern_key}) + + +# ========================================================================= +# Detection +# ========================================================================= + +def _normalize_command_for_detection(command: str) -> str: + """Normalize a command string before dangerous-pattern matching. + + Strips ANSI escape sequences (full ECMA-48 via tools.ansi_strip), + null bytes, and normalizes Unicode fullwidth characters so that + obfuscation techniques cannot bypass the pattern-based detection. + """ + from tools.ansi_strip import strip_ansi + + # Strip all ANSI escape sequences (CSI, OSC, DCS, 8-bit C1, etc.) + command = strip_ansi(command) + # Strip null bytes + command = command.replace('\x00', '') + # Normalize Unicode (fullwidth Latin, halfwidth Katakana, etc.) + command = unicodedata.normalize('NFKC', command) + # Collapse shell line continuations (backslash-newline). The shell removes + # BOTH characters and joins the tokens, so `rm -rf \/` executes as + # `rm -rf /`. This must run BEFORE the generic backslash-escape strip below, + # whose [^\n] class deliberately skips newlines and would otherwise leave + # the dangling backslash wedged between tokens — defeating the structured + # rm/mkfs/dd patterns (notably the HARDLINE root-delete floor, which cannot + # be bypassed even with yolo). Handles both \n and \r\n line endings. Line + # continuations carry no path separator, so this is a no-op on the Windows + # home-prefix folds below (which match C:\Users\alice\... — no newline). + command = re.sub(r'\\\r?\n', '', command) + # Fold absolute home / active-profile-home prefixes into their canonical + # ~/ and ~/.hermes/ forms so static user-sensitive patterns catch + # /home/alice/.bashrc and C:\Users\alice\.bashrc the same way they catch + # ~/.bashrc. Resolve at detection time (not via an import-time snapshot) so + # it tracks HOME / HERMES_HOME even when those are set after this module is + # imported — as the hermetic test conftest and profile/session launchers do. + # + # This MUST run before the backslash-escape strip below: on Windows the home + # prefix is separated by backslashes (C:\Users\alice\...), which that strip + # would otherwise dissolve (-> C:Usersalice) and make the fold impossible. + # The fold matches either separator, so POSIX paths are unaffected by order. + # + # Fold the (more specific) Hermes home first: on Windows it nests under the + # user home (C:\Users\alice\AppData\...\hermes), so folding the user home + # first would eat the prefix the Hermes-home fold needs. + command = _rewrite_resolved_hermes_home(command) + command = _rewrite_resolved_user_home(command) + # Strip shell backslash-escapes: r\m → rm. Prevents \-injection bypass. + command = re.sub(r'\\([^\n])', r'\1', command) + # Strip empty-string literals that split tokens: r''m → rm, r"\"m → rm. + command = re.sub(r"''|\"\"", '', command) + # Collapse $IFS / ${IFS} word-separator expansions to a literal space. + # In any POSIX shell the IFS variable defaults to , + # so `rm${IFS}-rf${IFS}/` is executed as `rm -rf /`. Because the dangerous + # and hardline patterns anchor on literal whitespace (\s) between a command + # and its arguments, leaving the unexpanded `${IFS}` token in place lets an + # attacker slip past EVERY pattern — including the unconditional hardline + # floor (rm -rf /, mkfs, dd to raw device, shutdown/reboot). Substituting a + # space here mirrors the shell's own expansion so the patterns fire. The + # brace form also covers bash substring expansions like `${IFS:0:1}` (a + # single space). Same de-obfuscation class as the backslash/empty-quote + # handling above. + command = re.sub(r'\$\{IFS\b[^}]*\}|\$IFS\b', ' ', command) + return command + + +# Shell metacharacters, quotes, and whitespace that terminate a filesystem +# path token on a command line. Used to bound the path tail we normalize. +_PATH_TOKEN_STOP = r"""\s'"`;|&<>()""" +# One path segment (no separators, no terminators) preceded by a separator. +_PATH_TAIL = r"(?P(?:[/\\][^/\\" + _PATH_TOKEN_STOP + r"]*)+)" + + +@functools.lru_cache(maxsize=64) +def _home_prefix_fold_regex(path: str): + """Compile a regex matching *path* used as an absolute directory prefix. + + The home components are matched with either separator (``/`` or ``\\``) + between them, followed by the rest of the path token (the ``tail`` group), + so a Windows native path (``C:\\Users\\alice\\.ssh\\authorized_keys``), its + forward-slash form, and mixed-separator forms all fold — and the tail's + backslashes get normalized to ``/`` by the caller so multi-segment static + patterns (``~/.ssh/authorized_keys``) still match. The trailing tail is + required (``+``), so a bare home with no path under it is not folded. + + Returns ``None`` for an unset or degenerate path — one with fewer than two + components below the root — so a stray HOME / HERMES_HOME such as ``/``, + ``C:\\`` or ``""`` cannot rewrite unrelated filesystem prefixes. Cached + because the resolved home is stable across calls on this hot path. + """ + if not path: + return None + components = [c for c in re.split(r"[/\\]+", path) if c] + # Require at least two non-empty components below the root. For POSIX this + # mirrors the historical ``count("/") >= 2`` guard (``/home/alice`` folds, + # ``/home`` does not); for Windows it rejects a bare drive root (``C:\\``) + # while accepting a real home (``C:\\Users\\alice``). + if len(components) < 2: + return None + body = r"[/\\]+".join(re.escape(c) for c in components) + # Optional leading root separator (POSIX ``/`` or UNC ``\\``); a Windows + # drive letter is captured as the first component. + return re.compile(r"[/\\]*" + body + _PATH_TAIL) + + +def _fold_home_prefixes(command: str, paths, replacement: str) -> str: + """Fold each resolved home *path* prefix in *command* to *replacement*. + + *replacement* has no trailing separator (``~`` / ``~/.hermes``); the matched + path tail (with its backslashes normalized to ``/``) supplies it. Longest + candidate first so a deeper home (e.g. an explicit HOME under USERPROFILE) + folds before a shorter overlapping one that would otherwise clobber it. + """ + seen: set[str] = set() + for path in sorted((p for p in paths if p), key=len, reverse=True): + if path in seen: + continue + seen.add(path) + pattern = _home_prefix_fold_regex(path) + if pattern is not None: + command = pattern.sub( + lambda m: replacement + m.group("tail").replace("\\", "/"), + command, + ) + return command + + +def _rewrite_resolved_user_home(command: str) -> str: + """Rewrite the current user's absolute home prefix to ``~/``. + + Resolves the home at detection time — its expanduser form, symlink-resolved + form, and an explicitly set ``HOME`` — so absolute home paths are checked by + the same static patterns as tilde and ``$HOME`` forms. ``HOME`` is consulted + directly because Windows' ``os.path.expanduser`` resolves ``~`` from + ``USERPROFILE`` and ignores ``HOME``, unlike POSIX. Matches both POSIX + (``/home/alice``) and Windows (``C:\\Users\\alice`` or ``C:/Users/alice``) + separators. No-op when the home is unset or degenerate. + """ + try: + home = os.path.expanduser("~") + candidates = [ + home, + os.path.realpath(home), + os.environ.get("HOME", ""), + ] + except Exception: + return command + return _fold_home_prefixes(command, candidates, "~") + + +def _rewrite_resolved_hermes_home(command: str) -> str: + """Rewrite the resolved absolute Hermes home prefix to ``~/.hermes/``. + + Resolves the active ``HERMES_HOME`` at call time (and its symlink-resolved + form) and folds an occurrence of ``/`` in *command* into + ``~/.hermes/`` so the static ``_HERMES_CONFIG_PATH`` / ``_HERMES_ENV_PATH`` + patterns match. In Docker and gateway deployments the agent often references + the resolved absolute path directly (e.g. ``sed -i ... + /home/hermes/.hermes/config.yaml``) rather than ``~``, ``$HOME``, or + ``$HERMES_HOME``. Matches both POSIX and Windows separators. No-op when the + path can't be resolved or doesn't appear. + """ + try: + from hermes_constants import get_hermes_home + home = get_hermes_home().expanduser() + candidates = [ + str(home), + str(home.resolve(strict=False)), + ] + except Exception: + return command + return _fold_home_prefixes(command, candidates, "~/.hermes") + + +_PARAM_REPLACEMENT_RE = re.compile(r"\$\{[^}/\s]+/[^}/]*/(?P[^}]*)\}") +_PARAM_DEFAULT_RE = re.compile(r"\$\{[^}:}\s]+:-(?P[^}]*)\}") +_SIMPLE_SHELL_LITERAL_RE = re.compile(r"^[A-Za-z0-9_./:@%+=,-]+$") +_ENV_ASSIGNMENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*=.*") +_COMMAND_WRAPPER_WORDS = { + "sudo", + "env", + "exec", + "nohup", + "setsid", + "time", + "command", + "builtin", +} +_SUDO_OPTIONS_WITH_ARG = { + "-c", "--close-from", + "-g", "--group", + "-h", "--host", + "-p", "--prompt", + "-u", "--user", +} + +_INTERPRETER_EXEC_FLAGS = { + "python": {"-c"}, + "node": {"-e", "--eval", "-p", "--print"}, + "perl": {"-e", "--eval"}, + "ruby": {"-e"}, + "php": {"-r"}, + "powershell": {"-command", "-c", "-file", "-f"}, +} +_INTERPRETER_WITH_ARG = { + "python": {"-W", "-X", "--check-hash-based-pycs"}, + "node": {"-C", "--conditions", "--cpu-prof-dir", "--diagnostic-dir", "--icu-data-dir", "--import", "--loader", "--openssl-config", "--require", "--title"}, + "perl": {"-0", "-F", "-I", "-M", "-m", "-x"}, + "ruby": {"-C", "-E", "-F", "-I", "-K", "-r"}, + "php": {"-c", "-d", "-z"}, + "powershell": {"-configurationname", "-custompipename", "-executionpolicy", "-inputformat", "-outputformat", "-settingsfile", "-version", "-windowstyle", "-workingdirectory"}, +} +_READ_TOOL_EXEC_FLAGS = { + "sort": {"--compress-program"}, + "rg": {"--pre", "--hostname-bin"}, + "ag": {"--pager"}, + "man": {"--pager", "--html", "-P", "-H"}, +} +# Required-argument options are ownership boundaries: an option-looking next +# token is data, not another option. These sets mirror the invocation grammar +# of the supported binaries (ripgrep 14, GNU sort, man-db, and ag 2.2). +_READ_TOOL_LONG_OPTIONS_WITH_ARG = { + "rg": { + "--after-context", "--before-context", "--color", "--colors", + "--context", "--context-separator", "--dfa-size-limit", "--encoding", + "--engine", "--field-context-separator", "--field-match-separator", + "--file", "--generate", "--glob", "--hostname-bin", + "--hyperlink-format", "--iglob", "--ignore-file", "--max-columns", + "--max-count", "--max-depth", "--max-filesize", "--path-separator", + "--pre", "--pre-glob", "--regex-size-limit", "--regexp", "--replace", + "--sort", "--sortr", "--threads", "--type", "--type-add", + "--type-clear", "--type-not", + }, + "sort": { + "--batch-size", "--buffer-size", "--compress-program", + "--field-separator", "--files0-from", "--key", "--output", + "--parallel", "--random-source", "--sort", "--temporary-directory", + }, + "man": { + "--config-file", "--encoding", "--extension", "--locale", + "--manpath", "--pager", "--preprocessor", "--prompt", "--recode", + "--sections", "--systems", + }, + "ag": { + "--ackmate-dir-filter", "--color-line-number", "--color-match", + "--color-path", "--depth", "--filename-pattern", "--file-search-regex", + "--ignore", "--ignore-dir", "--max-count", "--pager", + "--path-to-ignore", "--width", "--workers", + }, +} +_READ_TOOL_SHORT_OPTIONS_WITH_ARG = { + "rg": frozenset("efEmjgdtTABCMr"), + "sort": frozenset("koStT"), + "man": frozenset("CRLmMSserEPp"), + "ag": frozenset("gGmpW"), +} +_SHELL_PUNCTUATION = {";", "&", "&&", "|", "||", "(", ")", "{", "}"} +_MAX_DETECTION_COMMAND_CHARS = 128_000 +_MAX_SEPARATOR_FREE_COMMAND_CHARS = 4_096 +_MAX_DETECTION_SEGMENTS = 25_000 +_PARSER_LIMIT_DESCRIPTION = "command parser limit exceeded" +_MALFORMED_EXEC_DESCRIPTION = "command parser limit or malformed executable payload" + + + +def _command_parser_limit_exceeded(command: str) -> bool: + """Bound all parser work before normalization/tokenization. + + Counting separator characters is deliberately conservative: quoted + separators can over-count, but crossing this very high ceiling fails + closed rather than allowing an uninspected suffix to execute. + """ + if len(command) > _MAX_DETECTION_COMMAND_CHARS: + return True + # Long separator-free input has no compound-command utility and otherwise + # makes every legacy regex inspect one giant token. Reject it before any + # normalization, tokenization, or regex work. + if ( + len(command) > _MAX_SEPARATOR_FREE_COMMAND_CHARS + and not any(char in command for char in ";&|\n") + ): + return True + separators = 0 + for char in command: + if char in ";&|\n": + separators += 1 + if separators >= _MAX_DETECTION_SEGMENTS: + return True + return False + + +def _shell_tokens_with_spans(segment: str, start: int): + """Return shell words as ``(value, start, end, quoted)`` or ``None``. + + This deliberately small lexer never expands shell syntax. It exists to + preserve source spans, which ``shlex`` does not expose, while deciding + which *quoted* grep operand is data rather than another command. + """ + tokens = [] + i = start + while i < len(segment): + while i < len(segment) and segment[i].isspace(): + i += 1 + if i >= len(segment): + break + token_start = i + value = [] + quote = None + while i < len(segment) and (quote or not segment[i].isspace()): + char = segment[i] + if quote: + if char == quote: + quote = None + i += 1 + elif char == "\\" and quote == '"' and i + 1 < len(segment): + value.append(segment[i + 1]) + i += 2 + else: + value.append(char) + i += 1 + elif char in {"'", '"'}: + quote = char + i += 1 + elif char == "\\": + if i + 1 >= len(segment): + return None + value.append(segment[i + 1]) + i += 2 + else: + value.append(char) + i += 1 + if quote: + return None + raw = segment[token_start:i] + # Only a wholly single-quoted operand is inert shell data. Double + # quotes still execute $() and backticks; unquoted substitutions do too. + inert_single_quoted = ( + (raw.startswith("'") and raw.endswith("'")) + or ("='" in raw and raw.endswith("'")) + ) + tokens.append(("".join(value), token_start, i, inert_single_quoted)) + return tokens + + +_GREP_OPTIONS_WITH_ARG = { + "--after-context", "--before-context", "--binary-files", "--context", + "--directories", "--devices", "--exclude", "--exclude-dir", + "--exclude-from", "--include", "--label", "--max-count", + "--regexp", "--file", +} +_GREP_SHORT_OPTIONS_WITH_ARG = {"A", "B", "C", "D", "d", "e", "f", "m"} + + +def _quoted_grep_pattern_spans(command: str) -> tuple[list[tuple[int, int]], bool]: + """Structurally locate quoted grep PCRE operands. + + The returned boolean means the grep parse was ambiguous or malformed. In + that case callers fail closed and, critically, use the original command: + no text is hidden on an uncertain parse. + """ + spans: list[tuple[int, int]] = [] + offset = 0 + for segment in _iter_top_level_shell_segments(command): + segment_at = command.find(segment, offset) + offset = segment_at + len(segment) + for start, _, word in _iter_shell_command_word_spans(segment): + if os.path.basename(_deobfuscate_shell_word_for_detection(word)).lower() not in { + "grep", "egrep", + }: + continue + tokens = _shell_tokens_with_spans(segment, start) + if tokens is None: + return [], True + args = tokens[1:] + pcre = False + explicit_patterns = False + pattern_indexes: list[int] = [] + operand_index = None + i = 0 + options = True + while i < len(args): + token = args[i][0] + if options and token == "--": + options = False + i += 1 + continue + if options and token.startswith("--"): + option, equals, _ = token.partition("=") + if option == "--perl-regexp": + pcre = True + if option in {"--regexp", "--file"}: + explicit_patterns = True + if option in _GREP_OPTIONS_WITH_ARG and not equals: + if i + 1 >= len(args): + return [], True + if option == "--regexp": + pattern_indexes.append(i + 1) + i += 2 + continue + if option == "--regexp" and equals: + pattern_indexes.append(i) + i += 1 + continue + if options and token.startswith("-") and token != "-": + chars = token[1:] + j = 0 + while j < len(chars): + char = chars[j] + if char == "P": + pcre = True + if char in {"e", "f"}: + explicit_patterns = True + if char in _GREP_SHORT_OPTIONS_WITH_ARG: + if j + 1 < len(chars): + if char == "e": + pattern_indexes.append(i) + else: + if i + 1 >= len(args): + return [], True + if char == "e": + pattern_indexes.append(i + 1) + i += 1 + break + j += 1 + i += 1 + continue + if operand_index is None: + operand_index = i + i += 1 + if not explicit_patterns: + if operand_index is None: + return [], bool(pcre) + pattern_indexes.append(operand_index) + if pcre: + for index in pattern_indexes: + _, token_start, token_end, quoted = args[index] + if quoted: + spans.append((segment_at + token_start, segment_at + token_end)) + return spans, False + + +def _grep_safe_detection_variant(command: str) -> tuple[str, bool]: + spans, malformed = _quoted_grep_pattern_spans(command) + if malformed or not spans: + return command, malformed + parts = [] + previous = 0 + for start, end in spans: + parts.extend((command[previous:start], " " * (end - start))) + previous = end + parts.append(command[previous:]) + return "".join(parts), False + + +def _interpreter_family(executable: str) -> str | None: + name = os.path.basename(executable).lower() + if re.fullmatch(r"py(?:\.exe)?|python[23]?(?:\.\d+)*(?:\.exe)?", name): + return "python" + if re.fullmatch(r"node(?:js)?(?:\.exe)?", name): + return "node" + if re.fullmatch(r"perl[0-9]*(?:\.\d+)*(?:\.exe)?", name): + return "perl" + if re.fullmatch(r"ruby[0-9.]*(?:\.exe)?", name): + return "ruby" + if re.fullmatch(r"php(?:\.exe)?", name): + return "php" + if re.fullmatch(r"powershell(?:\.exe)?|pwsh(?:\.exe)?", name): + return "powershell" + return None + + +def _shell_segment_tokens(segment: str, start: int) -> list[str] | None: + """Tokenize an already-bounded command segment. + + ``None`` distinguishes malformed quoting from an empty segment so callers + can fail closed for a program-bearing option rather than silently skip it. + """ + try: + lexer = shlex.shlex(segment[start:], posix=True, punctuation_chars="<>") + lexer.whitespace_split = True + lexer.commenters = "" + return list(lexer) + except ValueError: + return None + + +def _iter_top_level_shell_segments(command: str): + """Yield top-level command segments in one left-to-right pass.""" + start = 0 + quote: str | None = None + escaped = False + index = 0 + while index < len(command): + char = command[index] + if escaped: + escaped = False + elif char == "\\" and quote != "'": + escaped = True + elif quote: + if char == quote: + quote = None + elif char in {"'", '"'}: + quote = char + elif char in ";&|\n": + if start < index: + yield command[start:index] + # Consume a doubled && / || separator as one boundary. + if char in "&|" and index + 1 < len(command) and command[index + 1] == char: + index += 1 + start = index + 1 + index += 1 + if start < len(command): + yield command[start:] + + +def _split_option(token: str) -> tuple[str, str | None]: + if "=" in token: + option, value = token.split("=", 1) + return option, value + return token, None + + +def _interpreter_exec_flag(family: str, args: list[str]) -> str | None: + """Return an execution-bearing interpreter option, if present.""" + flags = _INTERPRETER_EXEC_FLAGS[family] + skip_value = False + for token in args: + if skip_value: + skip_value = False + continue + if token == "--": + break + if family != "powershell" and not token.startswith("-"): + break + option, attached = _split_option(token) + comparable = option.lower() if family == "powershell" else option + if comparable in flags: + return comparable + with_arg = _INTERPRETER_WITH_ARG[family] + # `-Wonce` and `ruby -rjson` attach an option value; they are not + # short-option bundles containing an execution flag. PowerShell's + # normal long options also use one dash, so bundle parsing never + # applies to that family. + has_attached_option_value = any( + option.startswith(short) and len(option) > len(short) + for short in with_arg + if short.startswith("-") and not short.startswith("--") + ) + if ( + family != "powershell" + and not option.startswith("--") + and len(option) > 2 + and not has_attached_option_value + ): + for char in option[1:]: + short = f"-{char}" + if short in flags: + return short + if comparable in with_arg and attached is None: + skip_value = True + return None + + +_BASH_OPTIONS_WITH_ARG = {"-O", "+O", "-o", "+o", "--init-file", "--rcfile"} +_BASH_SHORT_OPTION_LETTERS = frozenset("ilrsDcabefhkmnptuvxBCEHPTOo") + + +def _bash_exec_payload(args: list[str]) -> tuple[bool, str | None]: + """Return whether Bash ``-c`` occurs and the command string it owns. + + Bash's O/o invocation options consume the following argument even when + they precede a later ``-c`` or occur in the same short-option bundle. + Likewise, the two startup-file long options own their next token. Parsing + those operands first prevents both missed payloads and false ``-c`` hits. + """ + index = 0 + while index < len(args): + token = args[index] + if token == "--" or not token.startswith(("-", "+")): + break + if token in _BASH_OPTIONS_WITH_ARG: + index += 2 + continue + if token.startswith("--"): + index += 1 + continue + + chars = token[1:] + # Bash option letters are case-sensitive. Restricting this to its + # documented alphabet preserves invalid controls such as `-Wc`. + if not set(chars) <= _BASH_SHORT_OPTION_LETTERS: + index += 1 + continue + consumed_option_arg = "O" in chars or "o" in chars + if "c" not in chars: + index += 1 + int(consumed_option_arg) + continue + payload_index = index + 1 + int(consumed_option_arg) + payload = args[payload_index] if payload_index < len(args) else None + return True, payload + return False, None + + +def _read_tool_exec_flag(tool: str, args: list[str]) -> tuple[str, str] | None: + """Return (option, program) for a read-only tool's program-running flag.""" + flags = _READ_TOOL_EXEC_FLAGS[tool] + index = 0 + while index < len(args): + token = args[index] + if token == "--": + break + option, payload = _split_option(token) + matched = option if option in flags else None + if tool == "man" and token.startswith(("-P", "-H")) and len(token) > 2: + matched, payload = token[:2], token[2:] + if matched: + if payload is None and index + 1 < len(args): + payload = args[index + 1] + # This option owns its program argument regardless of spelling. + # The real binaries execute a payload beginning with '-' rather + # than reparsing it as one of the tool's later options. + if payload: + return matched, payload + index += 2 if payload is not None and "=" not in token else 1 + continue + + if option in _READ_TOOL_LONG_OPTIONS_WITH_ARG[tool] and payload is None: + index += 2 + continue + + # In a short bundle, the first argument-taking option owns the rest of + # the token, or the following token when it occurs last. + if token.startswith("-") and not token.startswith("--") and len(token) > 1: + for short_index, char in enumerate(token[1:], start=1): + if char in _READ_TOOL_SHORT_OPTIONS_WITH_ARG[tool]: + index += 2 if short_index == len(token) - 1 else 1 + break + else: + index += 1 + continue + index += 1 + return None + + +def _execution_flag_findings(command: str): + """Yield scoped execution mechanisms and any executable payloads.""" + for segment in _iter_top_level_shell_segments(command): + for start, _, word in _iter_shell_command_word_spans(segment): + executable = _deobfuscate_shell_word_for_detection(word) + tokens = _shell_segment_tokens(segment, start) + executable_name = os.path.basename(executable).lower() + family = _interpreter_family(executable) + is_program_bearing = ( + family is not None or executable_name in _READ_TOOL_EXEC_FLAGS + ) + if tokens is None: + if is_program_bearing: + yield (_MALFORMED_EXEC_DESCRIPTION, None) + continue + if not tokens: + continue + if family: + flag = _interpreter_exec_flag(family, tokens[1:]) + if flag: + yield ("script execution via -e/-c flag", None) + continue + if any(token.startswith("<<") for token in tokens[1:]): + yield ("script execution via heredoc", None) + continue + if executable_name in {"bash", "sh", "zsh", "ksh"}: + found, payload = _bash_exec_payload(tokens[1:]) + if found: + yield ("shell command via -c/-lc flag", payload) + tool = executable_name + if tool in _READ_TOOL_EXEC_FLAGS: + finding = _read_tool_exec_flag(tool, tokens[1:]) + if finding: + option, payload = finding + yield (f"arbitrary program execution via {tool} {option}", payload) + + +def _skip_shell_whitespace(command: str, pos: int) -> int: + while pos < len(command) and command[pos].isspace(): + pos += 1 + return pos + + +def _scan_dollar_paren_end(command: str, start: int) -> int | None: + """Return the offset after a balanced ``$(...)`` command substitution.""" + depth = 1 + quote: str | None = None + i = start + 2 + while i < len(command): + ch = command[i] + if quote: + if ch == "\\" and quote == '"' and i + 1 < len(command): + i += 2 + continue + if ch == quote: + quote = None + i += 1 + continue + if ch in ("'", '"'): + quote = ch + i += 1 + continue + if ch == "\\" and i + 1 < len(command): + i += 2 + continue + if command.startswith("$(", i): + depth += 1 + i += 2 + continue + if ch == ")": + depth -= 1 + i += 1 + if depth == 0: + return i + continue + i += 1 + return None + + +def _scan_backtick_end(command: str, start: int) -> int | None: + i = start + 1 + while i < len(command): + if command[i] == "\\" and i + 1 < len(command): + i += 2 + continue + if command[i] == "`": + return i + 1 + i += 1 + return None + + +def _read_shell_word(command: str, pos: int) -> tuple[int, int, str]: + """Read one shell word without executing expansions.""" + start = _skip_shell_whitespace(command, pos) + i = start + quote: str | None = None + while i < len(command): + ch = command[i] + if quote: + if ch == "\\" and quote == '"' and i + 1 < len(command): + i += 2 + continue + if ch == quote: + quote = None + i += 1 + continue + if ch in ("'", '"'): + quote = ch + i += 1 + continue + if ch == "\\" and i + 1 < len(command): + i += 2 + continue + if command.startswith("$(", i): + end = _scan_dollar_paren_end(command, i) + if end is None: + i += 2 + else: + i = end + continue + if command.startswith("${", i): + end = command.find("}", i + 2) + if end == -1: + i += 2 + else: + i = end + 1 + continue + if ch == "`": + end = _scan_backtick_end(command, i) + if end is None: + i += 1 + else: + i = end + continue + if ch.isspace() or ch in ";&|": + break + i += 1 + return (start, i, command[start:i]) + + +def _strip_optional_shell_quotes(word: str) -> str: + if len(word) >= 2 and word[0] == word[-1] and word[0] in ("'", '"'): + return word[1:-1] + return word + + +def _is_simple_shell_literal(value: str) -> bool: + return bool(value and _SIMPLE_SHELL_LITERAL_RE.fullmatch(value)) + + +def _literal_command_substitution_output(script: str) -> str | None: + """Resolve tiny literal command substitutions without executing a shell.""" + try: + tokens = shlex.split(script, posix=True) + except ValueError: + return None + if not tokens: + return None + + command = tokens[0].lower() + args = tokens[1:] + if command == "echo": + while args and re.fullmatch(r"-[nEe]+", args[0]): + args = args[1:] + if len(args) == 1 and _is_simple_shell_literal(args[0]): + return args[0] + return None + + if command == "printf": + if len(args) == 1 and _is_simple_shell_literal(args[0]): + return args[0] + if ( + len(args) == 2 + and args[0] == "%s" + and _is_simple_shell_literal(args[1]) + ): + return args[1] + return None + + +def _replace_simple_command_substitutions(word: str) -> str: + chars: list[str] = [] + i = 0 + while i < len(word): + if word.startswith("$(", i): + end = _scan_dollar_paren_end(word, i) + if end is not None: + replacement = _literal_command_substitution_output(word[i + 2:end - 1]) + if replacement is not None: + chars.append(replacement) + i = end + continue + if word[i] == "`": + end = _scan_backtick_end(word, i) + if end is not None: + replacement = _literal_command_substitution_output(word[i + 1:end - 1]) + if replacement is not None: + chars.append(replacement) + i = end + continue + chars.append(word[i]) + i += 1 + return "".join(chars) + + +def _replace_simple_shell_expansions(word: str) -> str: + word = _replace_simple_command_substitutions(word) + word = _PARAM_REPLACEMENT_RE.sub(lambda match: match.group("replacement"), word) + return _PARAM_DEFAULT_RE.sub(lambda match: match.group("default"), word) + + +def _strip_shell_word_syntax(word: str) -> str: + chars: list[str] = [] + quote: str | None = None + i = 0 + while i < len(word): + ch = word[i] + if quote: + if ch == "\\" and quote == '"' and i + 1 < len(word): + chars.append(word[i + 1]) + i += 2 + continue + if ch == quote: + quote = None + i += 1 + continue + chars.append(ch) + i += 1 + continue + if ch in ("'", '"'): + quote = ch + i += 1 + continue + if ch == "\\" and i + 1 < len(word): + chars.append(word[i + 1]) + i += 2 + continue + chars.append(ch) + i += 1 + return "".join(chars) + + +def _deobfuscate_shell_word_for_detection(word: str) -> str: + """Approximate how shell syntax can spell a command word. + + This is intentionally narrow and non-executing: it only collapses shell + quoting/escaping plus simple literal command substitutions that appear in + the command word itself. + """ + deobfuscated = word + for _ in range(2): + previous = deobfuscated + deobfuscated = _replace_simple_shell_expansions(deobfuscated) + deobfuscated = _strip_shell_word_syntax(deobfuscated) + if deobfuscated == previous: + break + return deobfuscated + + +def _iter_shell_command_starts(command: str): + starts = [0] + + def scan(start: int, end: int) -> None: + quote: str | None = None + i = start + while i < end: + ch = command[i] + if quote == "'": + if ch == "'": + quote = None + i += 1 + continue + if quote == '"': + if ch == "\\" and i + 1 < end: + i += 2 + continue + if ch == '"': + quote = None + i += 1 + continue + if command.startswith("$(", i): + nested_end = _scan_dollar_paren_end(command, i) + starts.append(i + 2) + scan(i + 2, nested_end - 1 if nested_end is not None else end) + i = nested_end if nested_end is not None else end + continue + if ch == "`": + nested_end = _scan_backtick_end(command, i) + starts.append(i + 1) + scan(i + 1, nested_end - 1 if nested_end is not None else end) + i = nested_end if nested_end is not None else end + continue + i += 1 + continue + if ch in ("'", '"'): + quote = ch + i += 1 + continue + if ch == "\\" and i + 1 < end: + i += 2 + continue + if command.startswith("$(", i): + nested_end = _scan_dollar_paren_end(command, i) + starts.append(i + 2) + scan(i + 2, nested_end - 1 if nested_end is not None else end) + i = nested_end if nested_end is not None else end + continue + if ch == "`": + nested_end = _scan_backtick_end(command, i) + starts.append(i + 1) + scan(i + 1, nested_end - 1 if nested_end is not None else end) + i = nested_end if nested_end is not None else end + continue + if ch in ("(", "{"): + starts.append(i + 1) + elif ch in ";\n": + starts.append(i + 1) + elif ch in "&|": + repeated = i + 1 < end and command[i + 1] == ch + starts.append(i + 2 if repeated else i + 1) + if repeated: + i += 1 + i += 1 + + scan(0, len(command)) + + seen: set[int] = set() + for start in starts: + start = _skip_shell_whitespace(command, start) + if start < len(command) and start not in seen: + seen.add(start) + yield start + + +def _mark_command_starts(command: str) -> str: + """Insert a newline before each real (quote-aware) command start. + + ``\\n`` is already a ``_CMDPOS`` separator, so this rewrites subshell + ``(cmd)`` and brace-group ``{ cmd; }`` openers — which the flat pattern + class deliberately omits — into a form the anchored hardline/dangerous + patterns recognize, WITHOUT the quoted-prose false positives that adding + ``(`` / ``{`` to ``_CMDPOS`` would cause. Starts inside quotes are never + produced by ``_iter_shell_command_starts``, so quoted arguments such as + ``--title "block (reboot)"`` are left exactly as-is. + """ + # Collect the (whitespace-skipped) start offsets, drop 0 (already anchored + # by ``^``), and splice a newline in front of each — right-to-left so the + # earlier offsets stay valid as we mutate. + offsets = sorted(o for o in _iter_shell_command_starts(command) if o > 0) + if not offsets: + return command + # Build once instead of repeatedly slicing and copying the full command for + # every segment (quadratic at 10k+ compound-command segments). + parts: list[str] = [] + previous = 0 + for offset in offsets: + parts.extend((command[previous:offset], "\n")) + previous = offset + parts.append(command[previous:]) + return "".join(parts) + + +def _mask_quoted_newlines(command: str) -> str: + """Replace raw newlines inside single/double quotes with a space. + + Detection-only rewrite. A newline inside a quoted string is DATA to the + shell — part of the argument, not a command separator — yet the flat + ``_CMDPOS`` start-position class treats every raw ``\\n`` as a command + start. That made any multi-line quoted argument (``hermes send`` message + bodies, ``git commit -m`` messages, heredoc text) trip the hardline + blocklist when a data line began with e.g. ``sudo reboot``. + + Quote tracking mirrors ``_iter_shell_command_starts``: single quotes are + literal until the closing quote; inside double quotes a backslash escapes + the next character. Real command boundaries are unaffected: unquoted + newlines pass through untouched, ``$(``/backtick remain ``_CMDPOS`` + anchors independent of newlines, and ``_mark_command_starts`` still + re-inserts newlines at every genuine quote-aware command start. An + unclosed quote absorbs following newlines exactly as the shell would + (the quoted word continues across the line break), so masking them + cannot hide a runnable command. + """ + if "\n" not in command: + return command + out: list[str] = [] + quote: str | None = None + i = 0 + while i < len(command): + ch = command[i] + if quote: + if ch == "\\" and quote == '"' and i + 1 < len(command): + out.append(command[i:i + 2]) + i += 2 + continue + if ch == quote: + quote = None + out.append(" " if ch == "\n" else ch) + i += 1 + continue + if ch in ("'", '"'): + quote = ch + elif ch == "\\" and i + 1 < len(command): + out.append(command[i:i + 2]) + i += 2 + continue + out.append(ch) + i += 1 + return "".join(out) + + +def _iter_shell_command_word_spans(command: str): + """Yield command-position words that may be executable names.""" + for command_start in _iter_shell_command_starts(command): + pos = command_start + prefix_words = 0 + skip_wrapper_options = False + skip_next_wrapper_arg = False + while prefix_words < 12: + word_start, word_end, word = _read_shell_word(command, pos) + if word_start == word_end: + break + deobfuscated = _deobfuscate_shell_word_for_detection(word) + lower_word = deobfuscated.lower() + if skip_next_wrapper_arg: + skip_next_wrapper_arg = False + pos = word_end + prefix_words += 1 + continue + if skip_wrapper_options and lower_word.startswith("-"): + option_name = lower_word.split("=", 1)[0] + skip_next_wrapper_arg = ( + "=" not in lower_word + and option_name in _SUDO_OPTIONS_WITH_ARG + ) + pos = word_end + prefix_words += 1 + continue + + yield (word_start, word_end, word) + prefix_words += 1 + + if lower_word in _COMMAND_WRAPPER_WORDS: + skip_wrapper_options = lower_word in {"sudo", "env"} + pos = word_end + continue + if _ENV_ASSIGNMENT_RE.fullmatch(deobfuscated): + skip_wrapper_options = False + pos = word_end + continue + break + + +def _command_detection_variants(command: str): + # Mask quoted newlines BEFORE normalization: normalization strips + # backslash-escapes (\" -> ") and empty-string pairs (""), which would + # corrupt quote tracking — e.g. `echo "a\""` normalizes to `echo "a` (an + # unterminated quote), so masking the normalized text could swallow a + # REAL unquoted newline separator that follows. The raw command carries + # faithful shell quote state. + normalized = _normalize_command_for_detection(_mask_quoted_newlines(command)) + # Quote-aware grep parsing hides only structurally identified pattern + # operands. Malformed/ambiguous input remains byte-for-byte intact. + grep_safe, _ = _grep_safe_detection_variant(normalized) + seen = {grep_safe} + yield grep_safe + # Windows-path variant (#69472): normalization treats backslashes as + # shell escapes and strips them, so `del C:\Users\me\.ssh\id_rsa` + # reaches the patterns as `del C:Usersme.sshid_rsa` — no path rule can + # ever match a backslash Windows path. When the RAW command contains a + # drive-letter or UNC backslash path, also yield a variant with + # backslashes flattened to forward slashes BEFORE normalization eats + # them. Gated on a real path shape (letter, colon, backslash — or + # double backslash UNC) so POSIX escape semantics (`echo a\"b`) are + # untouched on every other command. + if re.search(r"(?:[A-Za-z]:|\\\\)[\\\\]", command) or re.search(r"[A-Za-z]:\\", command): + win_variant = _normalize_command_for_detection( + _mask_quoted_newlines(command.replace("\\", "/")) + ) + if win_variant not in seen: + seen.add(win_variant) + yield win_variant + # Program-bearing options are parsed in their owning command's context. + # Surfacing only their payload lets the hardline floor inspect the command + # that will actually run without promoting similar flags or quoted prose. + pending = [normalized] + while pending: + variant = pending.pop() + for _, payload in _execution_flag_findings(variant): + if payload and payload not in seen: + seen.add(payload) + yield payload + # A payload can begin with an option-looking program and then + # invoke a hardline command after a separator. Mark its real + # command starts just as we do for the outer command. + marked_payload = _mark_command_starts(payload) + if marked_payload != payload and marked_payload not in seen: + seen.add(marked_payload) + yield marked_payload + pending.append(payload) + # Subshell `(cmd)` and brace-group `{ cmd; }` openers put `cmd` at a real + # command position, but the flat `_CMDPOS`-anchored patterns can't see it: + # their start-position class deliberately omits `(`/`{` because a bare + # regex cannot tell `(reboot)` (real subshell) from `--title "(reboot)"` + # (quoted prose) — adding them there regresses ordinary quoted arguments. + # Instead, reconstruct the command with a newline (already a `_CMDPOS` + # separator) inserted at each command start the QUOTE-AWARE tokenizer + # found. Openers inside quotes never yield a start, so quoted prose is + # untouched, while `(reboot)` / `{ shutdown -h now; }` now anchor. This + # covers every `_CMDPOS` rule (shutdown/reboot/init/systemctl/telinit and + # the rm root/home/system floor) in one place. + marked = _mark_command_starts(grep_safe) + if marked != grep_safe and marked not in seen: + seen.add(marked) + yield marked + # Shell quoting/escaping can spell a dangerous executable name in pieces + # (for example r\m or r''m). Keep that deobfuscation scoped to command + # words so similarly shaped arguments do not become false positives. + for word_start, word_end, word in _iter_shell_command_word_spans(normalized): + deobfuscated = _deobfuscate_shell_word_for_detection(word) + if not deobfuscated or deobfuscated == word: + continue + variant = normalized[:word_start] + deobfuscated + normalized[word_end:] + if variant in seen: + continue + seen.add(variant) + yield variant + + +def _is_verification_artifact_cleanup(command: str) -> bool: + """Return whether *command* only removes one Hermes ad-hoc temp script.""" + try: + argv = shlex.split(command, posix=True) + except ValueError: + return False + if len(argv) != 3 or argv[0] != "rm" or argv[1] != "-f": + return False + + operand = argv[2] + temp_dir = os.path.realpath(tempfile.gettempdir()) + basename = os.path.basename(operand) + if operand != os.path.join(temp_dir, basename): + return False + + target = os.path.realpath(operand) + if os.path.dirname(target) != temp_dir: + return False + return re.fullmatch(r"hermes-(?:verify|ad-hoc)-[A-Za-z0-9_.-]+", basename) is not None + + +_GATEWAY_LIFECYCLE_SPLICE_DESCRIPTION = ( + "stop/restart hermes gateway via shell-spliced verb (kills running agents)" +) + + +def _is_shell_token_spliced_gateway_lifecycle(command: str) -> bool: + """Catch gateway-lifecycle verbs spelled with quote/backslash splicing. + + ``_normalize_command_for_detection`` strips backslash escapes, so + ``kick\\start`` already reaches the launchctl pattern above. Quote + splicing does not: ``_deobfuscate_shell_word_for_detection`` is + deliberately scoped to command-position words (widening it would let + quoted prose like ``git commit -m "rm -rf /"`` match the destructive + patterns), and the spliced verb sits in an ARGUMENT position. So + ``launchctl kick"start" -k gui/501/ai.hermes.gateway`` auto-approved + while executing exactly as the gated ``kickstart`` form (#80269). + + Delegate to ``cron.lifecycle_guard``, which tokenizes with shlex and is + anchored on a hermes-gateway identifier — reusing its prose + false-positive coverage instead of loosening the generic pattern + engine. This runs last, so an ordinary pattern match still wins and + keeps its more specific reason string. Unlike the guard's use inside + ``terminal_tool``, this layer only raises an approval prompt; the + non-bypassable block still lives in ``cron.lifecycle_guard``. + """ + try: + from cron.lifecycle_guard import contains_gateway_lifecycle_command + except Exception: + return False + return contains_gateway_lifecycle_command(command) + + +def detect_dangerous_command(command: str) -> tuple: + """Check if a command matches any dangerous patterns. + + Returns: + (is_dangerous, pattern_key, description) or (False, None, None) + """ + if _command_parser_limit_exceeded(command): + return (True, _PARSER_LIMIT_DESCRIPTION, _PARSER_LIMIT_DESCRIPTION) + if _is_verification_artifact_cleanup(command): + return (False, None, None) + + for command_variant in _command_detection_variants(command): + command_lower = command_variant.lower() + for pattern_re, description in DANGEROUS_PATTERNS_COMPILED: + if pattern_re.search(command_lower): + pattern_key = description + return (True, pattern_key, description) + normalized = _normalize_command_for_detection(command) + for description, _ in _execution_flag_findings(normalized): + return (True, description, description) + if _is_shell_token_spliced_gateway_lifecycle(command): + return ( + True, + _GATEWAY_LIFECYCLE_SPLICE_DESCRIPTION, + _GATEWAY_LIFECYCLE_SPLICE_DESCRIPTION, + ) + return (False, None, None) diff --git a/tools/async_delegation.py b/tools/async_delegation.py index ea0757143d..160da36296 100644 --- a/tools/async_delegation.py +++ b/tools/async_delegation.py @@ -2,42 +2,27 @@ """ Async (background) delegation registry. -Backs ``delegate_task(background=true)``: the parent agent dispatches a -subagent that runs on a module-level daemon executor and returns a handle -immediately, so the user and the model can keep working while the child runs. +Backs ``delegate_task(background=true)``: the parent dispatches a subagent on +a module-level daemon executor and returns a handle immediately. When the +child finishes, a ``type="async_delegation"`` event is pushed onto the SHARED +``process_registry.completion_queue``, which the CLI and gateway already drain +while the agent is idle. Reusing that rail means completions surface as a NEW +turn (never spliced mid-turn, so role alternation and the prompt cache stay +intact) and inherit the queue's de-dup, crash-recovery and drain wiring. -When the child finishes, a completion event is pushed onto the SHARED -``process_registry.completion_queue`` with ``type="async_delegation"``. The -CLI (``cli.py`` process_loop) and gateway (``_run_process_watcher`` / -``completion_queue`` drain) already poll that queue while the agent is idle -and forge a fresh user/internal turn from each event. We deliberately reuse -that rail rather than reaching into a running agent loop: +The completion payload carries a self-contained task-source block (goal, +context, toolsets, model, dispatch time, status, result) so a parent deep in +unrelated context can use the result or re-dispatch. - - completions surface as a NEW turn when the agent is idle, never spliced - between a tool result and an assistant message. That keeps strict - message-role alternation legal and the prompt cache intact (hard - invariant: never mutate past context). - - we inherit the queue's de-dup, crash-recovery checkpoint, and the - existing CLI + gateway drain wiring for free — no new drain loops in the - two largest files in the repo. - -The completion payload carries a RICH, self-contained task-source block (the -original goal, the context the parent supplied, toolsets, model, dispatch -time, status, and the full result summary). When the result re-enters the -conversation the parent may be deep in unrelated context and won't remember -why the subagent existed; the block lets it either use the result or -re-dispatch if the world has moved on. - -This module owns ONLY the async lifecycle. The actual child build + run is -delegated back to ``delegate_tool._run_single_child`` via an injected -runner, so all the credential leasing, heartbeat, timeout, and result-shaping -logic stays in one place. +This module owns ONLY the async lifecycle; the child build + run is injected +as a ``runner`` (``delegate_tool._run_single_child``). """ from __future__ import annotations import json import logging +import os import sqlite3 import threading import time @@ -52,75 +37,79 @@ from tools.thread_context import propagate_context_to_thread logger = logging.getLogger(__name__) -# Back-compat alias — the daemon executor now lives in tools.daemon_pool so -# other subsystems (tool_executor, memory_manager, delegate_tool, skills_hub) -# can share it. Existing imports of ``_DaemonThreadPoolExecutor`` keep working. -_DaemonThreadPoolExecutor = DaemonThreadPoolExecutor - - -# --------------------------------------------------------------------------- -# Module-level state -# --------------------------------------------------------------------------- -# A persistent daemon executor (NOT a `with ThreadPoolExecutor()` block, which -# would join on exit and defeat the whole point of async). Workers are daemon -# threads so a hard process exit doesn't hang on an in-flight child. +# ── Module-level state ────────────────────────────────────────────────────── +# Persistent daemon executor (never a `with ThreadPoolExecutor()` block, which +# would join on exit and defeat async); daemon workers can't hang a hard exit. _executor: Optional[ThreadPoolExecutor] = None _executor_lock = threading.Lock() _executor_max_workers: int = 0 _records_lock = threading.Lock() -# delegation_id -> record dict. Kept for the lifetime of the run plus a short -# tail after completion so `list_async_delegations()` can show recent results. +# delegation_id -> record dict; kept for the run plus a short completed tail. _records: Dict[str, Dict[str, Any]] = {} _DEFAULT_MAX_ASYNC_CHILDREN = 3 -# How many completed records to retain for status queries before pruning. +# Completed records retained (in memory and in the ledger) for status queries. _MAX_RETAINED_COMPLETED = 50 _DURABLE_RETENTION_SECONDS = 7 * 24 * 60 * 60 _MAX_DURABLE_PENDING = 1000 -# A pending completion whose delivery keeps failing is retried across claim -# cycles (and across restarts via restore_undelivered_completions). Cap the -# attempts so an unroutable row converges to a terminal 'dropped' state +# Cap retried deliveries so an unroutable row converges to terminal 'dropped' # instead of replaying on every restart forever. _MAX_DELIVERY_ATTEMPTS = 8 -# Staleness cap for restart replay: a pending completion older than this is -# terminally dropped instead of re-run as a fresh full-context turn (see -# restore_undelivered_completions). 48h keeps overnight/weekend results -# deliverable while stopping weeks-old sessions from replaying after upgrades. +# Pending completions older than this are dropped on restart replay instead +# of re-run as a full-context turn; 48h keeps weekend results deliverable. _MAX_COMPLETION_REPLAY_AGE_S = 48 * 3600.0 _DB_LOCK = threading.Lock() -# --------------------------------------------------------------------------- -# Stale-delegation detection (progress-based, on by default) -# --------------------------------------------------------------------------- -# A detached runner that wedges before returning (e.g. stuck inside its first -# model API call — #60203) never reaches its ``finally`` finalizer, so no -# completion event is ever published: the delegation shows "dispatched" -# forever and the owning session looks silent until a process restart. We do -# NOT fix this with a wall-clock timeout — legitimate heavy subagent work -# (deep reviews, research fan-outs, slow reasoning models) must never be -# killed for taking long (see delegate_tool.DEFAULT_CHILD_TIMEOUT rationale). -# Instead a single monitor thread watches per-dispatch PROGRESS (api-call -# count + current tool, via an injected ``progress_fn``): a child that is -# advancing is left alone forever; a child with NO progress past the stale -# threshold is interrupted, given a grace window to unwind and deliver its -# partial results through the normal finalize path, and only force-finalized -# with a terminal ``stalled`` event if it never returns. -# -# Thresholds mirror the sync-path heartbeat staleness monitor in -# delegate_tool: idle (not inside a tool) stays tight so a wedged first API -# call is caught quickly; in-tool is much higher so legitimately slow tools -# (long terminal commands, big fetches) get time to finish. -_STALE_CHECK_INTERVAL = 30.0 # seconds between monitor sweeps -_STALE_IDLE_SECONDS = 450.0 # no progress, no current tool → stalled -_STALE_IN_TOOL_SECONDS = 1200.0 # no progress while inside a tool → stalled -_STALL_GRACE_SECONDS = 120.0 # after interrupt, time for the runner to return +# ── Stale-delegation detection (progress-based, on by default) ────────────── +# A runner wedged before returning (e.g. stuck in its first API call) never +# reaches its finalizer, so the delegation shows "dispatched" forever. We do +# NOT use a wall-clock timeout (legitimate heavy work must never be killed for +# taking long); a single monitor thread samples per-dispatch PROGRESS via an +# injected ``progress_fn``. Advancing children are left alone; a frozen one is +# interrupted, given a grace window to unwind through the normal finalize path, +# and only force-finalized with a terminal ``stalled`` event if it never returns. +# Thresholds mirror delegate_tool's sync heartbeat monitor: idle stays tight, +# in-tool is much higher so legitimately slow tools can finish. +_STALE_CHECK_INTERVAL = 30.0 +_STALE_IDLE_SECONDS = 450.0 +_STALE_IN_TOOL_SECONDS = 1200.0 +_STALL_GRACE_SECONDS = 120.0 _monitor_lock = threading.Lock() _monitor_thread: Optional[threading.Thread] = None _monitor_stop = threading.Event() +_LIVE_STATES = {"running", "stalling", "finalizing"} +_ACTIVE_STATES = ("running", "stalling") +# Routing origin persisted at dispatch so a restart-recovered completion can +# reconstruct a full SessionSource (scope_id drives relay tenant egress). +_ROUTING_KEYS = ("scope_id", "user_id", "user_name") +# Structured stall metadata — additive, present only on stall finalizations. +_STALL_META_KEYS = ( + "stalled_after_quiet_seconds", "stall_threshold_seconds", "stall_phase", "stall_grace_seconds", +) +# Private stall bookkeeping on the record -> public field in list_async_delegations(). +_STALL_FIELD_MAP = ( + ("_stall_quiet_seconds", "stalled_after_quiet_seconds"), + ("_stall_threshold_seconds", "stall_threshold_seconds"), + ("_stall_in_tool", "stall_in_tool"), +) + +def _copy_truthy(src: Dict[str, Any], dst: Dict[str, Any], keys) -> None: + for k in keys: + if src.get(k): + dst[k] = src[k] + + +def _copy_present(src: Dict[str, Any], dst: Dict[str, Any], keys) -> None: + for k in keys: + if k in src: + dst[k] = src[k] + + +# ── Durable ledger (state.db / async_delegations) ─────────────────────────── def _db_path(): return get_hermes_home() / "state.db" @@ -132,9 +121,7 @@ def _connect() -> sqlite3.Connection: try: _initialize_schema(conn) except Exception: - # A PRAGMA/DDL failure after a successful connect() must not leak the - # just-opened connection back to the caller. - conn.close() + conn.close() # don't leak the connection on PRAGMA/DDL failure raise return conn @@ -142,14 +129,9 @@ def _connect() -> sqlite3.Connection: def _initialize_schema(conn: sqlite3.Connection) -> None: from hermes_state import apply_durability_barriers - # state.db's owning SessionDB connection establishes the configured journal - # mode. This secondary durability ledger must preserve that mode: applying - # WAL here on every short-lived connection requires an exclusive lock when - # the file is not already WAL and can collide with live transcript/FTS - # writers. The ledger works in either WAL or DELETE mode; if it opens a new - # file first, the default rollback journal remains valid until SessionDB - # establishes the configured mode. sqlite3.connect(timeout=10) above also - # gives its small transactions a busy handler for ordinary contention. + # Preserve the journal mode SessionDB configured on state.db: forcing WAL + # from every short-lived connection needs an exclusive lock and collides + # with live transcript/FTS writers. The ledger works in WAL or DELETE mode. apply_durability_barriers(conn) conn.execute( """CREATE TABLE IF NOT EXISTS async_delegations ( @@ -181,10 +163,8 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: ("task_json", "TEXT"), ("delivery_claim", "TEXT"), ("delivery_claimed_at", "REAL"), - # Raw api_server session id (X-Hermes-Session-Id) of the ORIGINATING - # request — the wake self-post target. Without persisting it, - # completions recovered after a process restart are unroutable on - # api_server (the in-memory record that carried it is gone). + # Raw api_server session id of the ORIGINATING request (wake target); + # without it restart-recovered completions are unroutable there. ("origin_session_id", "TEXT"), ): if name not in columns: @@ -195,12 +175,8 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: def _transaction() -> Iterator[sqlite3.Connection]: """Open a connection, commit/rollback on exit, and ALWAYS close it. - ``sqlite3.Connection.__enter__``/``__exit__`` only commit or roll back the - transaction; they do not close the connection. Using ``with _connect()`` - alone therefore leaks a connection — and its WAL/SHM file descriptors — on - every durable dispatch, completion, and delivery-claim, deferring the close - to the garbage collector. On a long-running gateway that exhausts - ``RLIMIT_NOFILE`` (the cron-ledger sibling of this bug was #69567 / PR #69594). + ``with conn:`` only commits/rolls back; without the explicit close every + durable write leaks a connection (and WAL/SHM fds) until GC. """ conn = _connect() try: @@ -211,31 +187,20 @@ def _transaction() -> Iterator[sqlite3.Connection]: def _capture_routing_origin() -> Dict[str, Any]: - """Snapshot the dispatching turn's routing origin for the completion event. + """Snapshot scope_id/user_id/user_name from the dispatching turn. - Captured on the PARENT thread at dispatch time (the daemon worker doesn't - carry the contextvars) and persisted with the durable record, so a - completion replayed after a restart can reconstruct a full SessionSource - even when the session-store origin and in-memory source cache are gone. - scope_id matters most: on a relay-fronted deployment the connector's - fail-closed egress guard needs the tenant discriminator (or a user - binding) to route a scoped reply; without it, post-restart scoped - completions bounce with "target not routed to an onboarded tenant" - (staging 2026-08-09 defect #4). Best-effort — empty values are simply - omitted so CLI/contextvar-unaware paths persist nothing new. + Captured on the PARENT thread (the daemon worker has no contextvars) and + persisted so a restart-replayed completion can rebuild a SessionSource. + Best-effort: empty values are omitted. """ origin: Dict[str, Any] = {} try: from gateway.session_context import get_session_env - for evt_key, env_name in ( - ("scope_id", "HERMES_SESSION_SCOPE_ID"), - ("user_id", "HERMES_SESSION_USER_ID"), - ("user_name", "HERMES_SESSION_USER_NAME"), - ): - value = get_session_env(env_name, "") + for key in _ROUTING_KEYS: + value = get_session_env(f"HERMES_SESSION_{key.upper()}", "") if value: - origin[evt_key] = value + origin[key] = value except Exception: # noqa: BLE001 - routing origin is additive, never fatal pass return origin @@ -245,17 +210,14 @@ def _persist_dispatch(record: Dict[str, Any]) -> None: now = time.time() try: from gateway.status import get_process_start_time - owner_started_at = get_process_start_time(__import__("os").getpid()) + owner_started_at = get_process_start_time(os.getpid()) except Exception: owner_started_at = None task_payload = { key: record.get(key) for key in ( "goal", "goals", "context", "toolsets", "role", "model", "is_batch", - # Routing origin (scope_id/user_id/user_name): persisted so a - # restart-recovered completion can reconstruct a full - # SessionSource — see _capture_routing_origin. - "scope_id", "user_id", "user_name", + *_ROUTING_KEYS, ) if key in record } @@ -269,7 +231,7 @@ def _persist_dispatch(record: Dict[str, Any]) -> None: VALUES (?, ?, ?, ?, 'running', ?, ?, 'pending', 0, ?, ?, ?, ?)""", (record["delegation_id"], record.get("session_key", ""), record.get("origin_ui_session_id", ""), record.get("parent_session_id"), - record["dispatched_at"], now, __import__("os").getpid(), + record["dispatched_at"], now, os.getpid(), owner_started_at, json.dumps(task_payload), record.get("origin_session_id", "")), ) @@ -283,8 +245,7 @@ def _delete_durable_delegation(delegation_id: str) -> None: def _prune_durable_records() -> None: """Bound terminal history, preferring delivered records for deletion.""" - now = time.time() - cutoff = now - _DURABLE_RETENTION_SECONDS + cutoff = time.time() - _DURABLE_RETENTION_SECONDS with _DB_LOCK, _transaction() as conn: conn.execute( "DELETE FROM async_delegations WHERE delivery_state='delivered' AND updated_at < ?", @@ -332,14 +293,6 @@ def _persist_completion(event: Dict[str, Any], result: Dict[str, Any]) -> None: ) -def _note_delivery_attempt(delegation_id: str) -> None: - with _DB_LOCK, _transaction() as conn: - conn.execute( - "UPDATE async_delegations SET delivery_attempts=delivery_attempts+1, updated_at=? WHERE delegation_id=?", - (time.time(), delegation_id), - ) - - def recover_abandoned_delegations() -> int: """Classify records whose owning process disappeared as outcome unknown.""" try: @@ -369,8 +322,6 @@ def recover_abandoned_delegations() -> int: event = { "type": "async_delegation", "delegation_id": delegation_id, "session_key": session_key, "origin_ui_session_id": origin_ui, - # Restore the durable wake target so completions recovered - # after a restart remain routable to api_server sessions. "origin_session_id": origin_session_id or "", "parent_session_id": parent_id, "goal": task.get("goal", ""), "goals": task.get("goals"), "context": task.get("context"), @@ -380,12 +331,7 @@ def recover_abandoned_delegations() -> int: "error": "Delegation owner exited before recording a terminal result; outcome unknown.", "dispatched_at": dispatched_at, "completed_at": now, } - # Routing origin persisted at dispatch (see _capture_routing_origin): - # restores scope_id/user_id for the reconstructed SessionSource so - # relay egress priming works after a restart. - for _k in ("scope_id", "user_id", "user_name"): - if task.get(_k): - event[_k] = task[_k] + _copy_truthy(task, event, _ROUTING_KEYS) result = {"status": "unknown", "summary": None, "error": event["error"]} conn.execute( """UPDATE async_delegations SET state='unknown', completed_at=?, @@ -400,21 +346,11 @@ def recover_abandoned_delegations() -> int: def restore_undelivered_completions(target_queue) -> int: """Enqueue durable pending completions as fresh turns after process start. - Every restored event is stamped ``restored=True`` (in-memory only — the - stamp is added after the durable payload is deserialized and is never - persisted). Restored events originate from a *previous* process, so no - consumer in THIS process implicitly owns them: drain paths that run - without an ownership filter (the legacy single-session behavior) must - leave them queued for a consumer that can positively prove ownership, - otherwise a brand-new session adopts a dead session's delegation - results seconds after boot (#64484). - - Staleness cap: a pending completion older than - ``_MAX_COMPLETION_REPLAY_AGE_S`` is terminally dropped instead of - replayed. Replaying a weeks-old completion re-runs its parent session as - a full-context turn (a July session replayed in August burned a - 102K-token context on the staging fleet) for a result nobody is waiting - on anymore; the payload stays queryable on the dropped row. + Restored events are stamped ``restored=True`` in memory only: they came + from a PREVIOUS process, so drains without an ownership filter must leave + them for a consumer that can prove ownership. Rows older than + ``_MAX_COMPLETION_REPLAY_AGE_S`` are terminally dropped instead of + replaying a full-context turn nobody is waiting on. """ recover_abandoned_delegations() now = time.time() @@ -440,8 +376,7 @@ def restore_undelivered_completions(target_queue) -> int: "Async delegation %s: pending completion is %.1fh old " "(cap %.1fh); terminally dropping the replay (result " "remains queryable).", - delegation_id, (now - age_basis) / 3600.0, - _MAX_COMPLETION_REPLAY_AGE_S / 3600.0, + delegation_id, (now - age_basis) / 3600.0, _MAX_COMPLETION_REPLAY_AGE_S / 3600.0, ) continue evt = json.loads(payload) @@ -452,16 +387,20 @@ def restore_undelivered_completions(target_queue) -> int: return restored +def _update_delivery(sql: str, params: tuple) -> bool: + """Run one UPDATE on the ledger; True iff exactly one row changed.""" + with _DB_LOCK, _transaction() as conn: + return conn.execute(sql, params).rowcount == 1 + + def mark_completion_delivered(delegation_id: str) -> bool: """Atomically acknowledge successful injection of a durable completion.""" now = time.time() - with _DB_LOCK, _transaction() as conn: - cur = conn.execute( - """UPDATE async_delegations SET delivery_state='delivered', delivered_at=?, updated_at=? - WHERE delegation_id=? AND delivery_state!='delivered'""", - (now, now, delegation_id), - ) - return cur.rowcount == 1 + return _update_delivery( + """UPDATE async_delegations SET delivery_state='delivered', delivered_at=?, updated_at=? + WHERE delegation_id=? AND delivery_state!='delivered'""", + (now, now, delegation_id), + ) def claim_completion_delivery(delegation_id: str, claim_id: str) -> bool: @@ -491,19 +430,15 @@ def claim_event_delivery(evt: Dict[str, Any], consumer: str) -> Optional[str]: delegation_id = str(evt.get("delegation_id") or "") if not delegation_id: return "" - claim_id = f"{consumer}:{__import__('os').getpid()}:{uuid.uuid4().hex}" + claim_id = f"{consumer}:{os.getpid()}:{uuid.uuid4().hex}" return claim_id if claim_completion_delivery(delegation_id, claim_id) else None def release_completion_delivery(delegation_id: str, claim_id: str) -> bool: """Release a failed delivery claim so another consumer may retry. - Attempts are counted at claim time, so a row that keeps being claimed and - released has burned real delivery attempts. Once the budget is exhausted - the row converges to a terminal ``dropped`` state instead of returning to - ``pending`` — otherwise an undeliverable completion replays on every - gateway restart forever (restore_undelivered_completions only restores - pending rows). + Attempts are counted at claim time; once the budget is exhausted the row + converges to terminal ``dropped`` (only pending rows replay on restart). """ now = time.time() with _DB_LOCK, _transaction() as conn: @@ -515,11 +450,9 @@ def release_completion_delivery(delegation_id: str, claim_id: str) -> bool: (now, delegation_id, claim_id, _MAX_DELIVERY_ATTEMPTS), ) if capped.rowcount == 1: - logger.warning( - "Async delegation %s exhausted its %d delivery attempts; " - "marking terminally dropped (result remains queryable).", - delegation_id, _MAX_DELIVERY_ATTEMPTS, - ) + logger.warning("Async delegation %s exhausted its %d delivery attempts; " + "marking terminally dropped (result remains queryable).", + delegation_id, _MAX_DELIVERY_ATTEMPTS) return True cur = conn.execute( """UPDATE async_delegations SET delivery_claim=NULL, @@ -532,40 +465,34 @@ def release_completion_delivery(delegation_id: str, claim_id: str) -> bool: def drop_completion_delivery(delegation_id: str, claim_id: str) -> bool: - """Terminally drop a claimed completion that can never be delivered. + """Terminally drop a claimed completion whose target is permanently gone. - Used when the delivery target is permanently gone — the spawning session - ended at an explicit user boundary (/new, reset) rather than a compression - rotation. Marking the row ``dropped`` (not ``delivered``) keeps the ack - honest, and (not ``pending``) keeps restart recovery from replaying a - completion that will be fail-closed dropped again every time. + Used when the spawning session ended at an explicit user boundary (/new, + reset) rather than a compression rotation. ``dropped`` (not ``delivered``) + keeps the ack honest; (not ``pending``) + keeps restart recovery from replaying it into a fail-closed drop forever. """ - now = time.time() - with _DB_LOCK, _transaction() as conn: - cur = conn.execute( - """UPDATE async_delegations SET delivery_state='dropped', - updated_at=?, delivery_claim=NULL, - delivery_claimed_at=NULL - WHERE delegation_id=? AND delivery_state='pending' - AND delivery_claim=?""", - (now, delegation_id, claim_id), - ) - return cur.rowcount == 1 + return _update_delivery( + """UPDATE async_delegations SET delivery_state='dropped', + updated_at=?, delivery_claim=NULL, + delivery_claimed_at=NULL + WHERE delegation_id=? AND delivery_state='pending' + AND delivery_claim=?""", + (time.time(), delegation_id, claim_id), + ) def complete_completion_delivery(delegation_id: str, claim_id: str) -> bool: """Acknowledge acceptance for the consumer holding this claim.""" now = time.time() - with _DB_LOCK, _transaction() as conn: - cur = conn.execute( - """UPDATE async_delegations SET delivery_state='delivered', - delivered_at=?, updated_at=?, delivery_claim=NULL, - delivery_claimed_at=NULL - WHERE delegation_id=? AND delivery_state='pending' - AND delivery_claim=?""", - (now, now, delegation_id, claim_id), - ) - return cur.rowcount == 1 + return _update_delivery( + """UPDATE async_delegations SET delivery_state='delivered', + delivered_at=?, updated_at=?, delivery_claim=NULL, + delivery_claimed_at=NULL + WHERE delegation_id=? AND delivery_state='pending' + AND delivery_claim=?""", + (now, now, delegation_id, claim_id), + ) def complete_event_delivery(evt: Dict[str, Any], claim_id: str) -> None: @@ -592,120 +519,82 @@ def get_durable_delegation(delegation_id: str) -> Optional[Dict[str, Any]]: "delegation_id": delegation_id, "origin_session": row[0], "state": row[1], "dispatched_at": row[2], "completed_at": row[3], "result": json.loads(row[4]) if row[4] else None, - "delivery_state": row[5], "delivery_attempts": row[6], - "origin_session_id": row[7] or "", + "delivery_state": row[5], "delivery_attempts": row[6], "origin_session_id": row[7] or "", } +# ── In-memory registry queries ────────────────────────────────────────────── def _get_executor(max_workers: int) -> ThreadPoolExecutor: - """Lazily create (or grow) the shared daemon executor. + """Lazily create (or grow, never shrink) the shared daemon executor. - We never shrink — ThreadPoolExecutor can't resize — but if the configured - cap grows between calls we rebuild a larger pool. Existing in-flight - futures keep running on the old pool until it's garbage collected. + In-flight futures keep running on a replaced pool until it is collected. """ global _executor, _executor_max_workers with _executor_lock: if _executor is None or max_workers > _executor_max_workers: - # Daemon threads: thread_name_prefix aids debugging in stack dumps. - _executor = _DaemonThreadPoolExecutor( - max_workers=max_workers, - thread_name_prefix="async-delegate", + _executor = DaemonThreadPoolExecutor( + max_workers=max_workers, thread_name_prefix="async-delegate", ) _executor_max_workers = max_workers return _executor def active_count() -> int: - """Number of async delegation UNITS currently running. - - A unit is one dispatch: a single subagent OR a whole fan-out batch. A batch - counts as ONE here because it occupies one async-pool slot (the capacity - semantics ``dispatch_async_delegation_batch`` relies on). For the count of - actual concurrent child subagents (batch expanded), use - ``active_task_count()``. - """ + """Number of live async delegation UNITS (a whole batch counts as ONE slot).""" with _records_lock: - return sum( - 1 for r in _records.values() - if r.get("status") in {"running", "stalling", "finalizing"} - ) - - -def active_for_session(origin_ui_session_id: str) -> int: - """Number of live async delegations owned by one UI session.""" - if not origin_ui_session_id: - return 0 - with _records_lock: - return sum( - 1 - for r in _records.values() - if r.get("status") in {"running", "stalling", "finalizing"} - and str(r.get("origin_ui_session_id") or "") - == origin_ui_session_id - ) + return sum(1 for r in _records.values() if r.get("status") in _LIVE_STATES) def active_task_count() -> int: - """Number of async delegation TASKS (child subagents) currently running. + """Number of running child subagents (a batch of N contributes N). - Unlike ``active_count()`` (units/slots), this expands a batch to its child - count: a running batch of N tasks contributes N, a single subagent - contributes 1. This is the truthful "how many subagents are actually - working right now" figure for observability, where a 3-task batch shown as - "1" undercounts real concurrent work. Falls back to counting a batch as 1 - if its goal list is missing. + The truthful observability figure, unlike ``active_count()`` slots; a + batch with no goal list falls back to 1. """ + def _size(r: Dict[str, Any]) -> int: + goals = r.get("goals") if r.get("is_batch") else None + return len(goals) if isinstance(goals, (list, tuple)) and goals else 1 + with _records_lock: - total = 0 - for r in _records.values(): - if r.get("status") not in {"running", "finalizing"}: - continue - if r.get("is_batch"): - goals = r.get("goals") - total += len(goals) if isinstance(goals, (list, tuple)) and goals else 1 - else: - total += 1 - return total + return sum(_size(r) for r in _records.values() if r.get("status") in {"running", "finalizing"}) def _matches_session_selectors( - record: Dict[str, Any], - *, - session_key: str = "", - origin_ui_session_id: str = "", + record: Dict[str, Any], *, session_key: str = "", origin_ui_session_id: str = "", parent_session_id: str = "", ) -> bool: - return ( - (origin_ui_session_id and str(record.get("origin_ui_session_id") or "") == origin_ui_session_id) - or (session_key and str(record.get("session_key") or "") == session_key) - or (parent_session_id and str(record.get("parent_session_id") or "") == parent_session_id) + return any( + wanted and str(record.get(field) or "") == wanted + for field, wanted in ( + ("origin_ui_session_id", origin_ui_session_id), + ("session_key", session_key), + ("parent_session_id", parent_session_id), + ) ) -def has_live_for_session( - session_key: str = "", - origin_ui_session_id: str = "", - parent_session_id: str = "", -) -> bool: - """Whether a session still owns any live async delegation. +def _session_records_locked(statuses, session_key, origin_ui_session_id, parent_session_id): + """Records in ``statuses`` matching any selector. Caller holds ``_records_lock``.""" + return [ + r for r in _records.values() + if r.get("status") in statuses + and _matches_session_selectors( + r, + session_key=session_key, + origin_ui_session_id=origin_ui_session_id, + parent_session_id=parent_session_id, + ) + ] - Live = running / stalling / finalizing — the same states the reapers' - keepalive treats as active work. - """ + +def has_live_for_session( + session_key: str = "", origin_ui_session_id: str = "", parent_session_id: str = "", +) -> bool: + """Whether a session still owns any live (running/stalling/finalizing) delegation.""" if not session_key and not origin_ui_session_id and not parent_session_id: return False with _records_lock: - return any( - r.get("status") in {"running", "stalling", "finalizing"} - and _matches_session_selectors( - r, - session_key=session_key, - origin_ui_session_id=origin_ui_session_id, - parent_session_id=parent_session_id, - ) - for r in _records.values() - ) + return bool(_session_records_locked(_LIVE_STATES, session_key, origin_ui_session_id, parent_session_id)) def _new_delegation_id() -> str: @@ -713,18 +602,10 @@ def _new_delegation_id() -> str: def _prune_completed_locked() -> None: - """Drop the oldest completed records beyond the retention cap. - - Caller must hold ``_records_lock``. - """ - completed = [ - (rid, r) - for rid, r in _records.items() - if r.get("status") != "running" - ] + """Drop the oldest completed records beyond the cap. Caller holds ``_records_lock``.""" + completed = [(rid, r) for rid, r in _records.items() if r.get("status") != "running"] if len(completed) <= _MAX_RETAINED_COMPLETED: return - # Oldest-first by completion time (fall back to dispatch time). completed.sort(key=lambda kv: kv[1].get("completed_at") or kv[1].get("dispatched_at") or 0) for rid, _ in completed[: len(completed) - _MAX_RETAINED_COMPLETED]: _records.pop(rid, None) @@ -733,20 +614,12 @@ def _prune_completed_locked() -> None: def _current_origin_session_id() -> str: """Raw session id of the ORIGINATING api_server request, or ``""``. - The obvious source — ``HERMES_SESSION_ID`` via ``get_session_env`` — is - NOT safe to read at dispatch time: constructing a child agent - (``agent/agent_init.py``) calls ``set_current_session_id(child.session_id)``, - clobbering that ContextVar *and* ``os.environ`` with the subagent's - internal ``{timestamp}_{uuid}`` id moments before the dispatch code reads - it, so the completion wake would self-post into the subagent's own - (unread) session instead of the spawner's. - - The request-scoped ``HERMES_SESSION_CHAT_ID`` binding survives child - construction: ``_bind_api_server_session`` binds ``chat_id`` to the raw - ``X-Hermes-Session-Id``, and its only writer is ``set_session_vars`` — - ``set_current_session_id`` never touches it. Gate on the platform: on - push platforms ``chat_id`` is a chat, not a session, so yield ``""`` - there. + ``HERMES_SESSION_ID`` is unsafe here: building the child agent calls + ``set_current_session_id(child.session_id)`` moments before dispatch, so + the wake would self-post into the subagent's own session. The + request-scoped ``HERMES_SESSION_CHAT_ID`` binding (raw X-Hermes-Session-Id + on api_server) survives child construction; on push platforms chat_id is + a chat, not a session, so yield ``""`` there. """ try: from gateway.session_context import get_session_env @@ -758,69 +631,50 @@ def _current_origin_session_id() -> str: return "" -def dispatch_async_delegation( - *, - goal: str, - context: Optional[str], - toolsets: Optional[List[str]], - role: str, - model: Optional[str], - session_key: str, - parent_session_id: Optional[str] = None, - runner: Callable[[], Dict[str, Any]], - origin_ui_session_id: str = "", - origin_session_id: str = "", - interrupt_fn: Optional[Callable[[], None]] = None, - max_async_children: int = _DEFAULT_MAX_ASYNC_CHILDREN, - progress_fn: Optional[Callable[[], tuple]] = None, +# ── Dispatch ──────────────────────────────────────────────────────────────── +def _single_crash(error: str, duration: float) -> Dict[str, Any]: + return {"status": "error", "summary": None, "error": error, "api_calls": 0, + "duration_seconds": duration} + + +def _batch_crash(error: str, duration: float) -> Dict[str, Any]: + return {"results": [], "error": error, "total_duration_seconds": duration} + + +def _batch_status(combined: Dict[str, Any]) -> str: + """Batch status: completed unless every child errored/was interrupted.""" + child_results = combined.get("results") or [] + if child_results and all( + (r.get("status") not in ("completed", "success")) for r in child_results + ): + return "error" + return "completed" + + +def _dispatch( + *, delegation_id: str, goal: str, goals: Optional[List[str]], context: Optional[str], + toolsets: Optional[List[str]], role: str, model: Optional[str], session_key: str, + parent_session_id: Optional[str], runner: Callable[[], Dict[str, Any]], + origin_ui_session_id: str, origin_session_id: str, + interrupt_fn: Optional[Callable[[], None]], max_async_children: int, + progress_fn: Optional[Callable[[], tuple]], capacity_error: str, ) -> Dict[str, Any]: - """Spawn ``runner`` on the daemon executor and return a handle immediately. + """Shared dispatch core for single (``goals is None``) and batch units. - Parameters - ---------- - goal, context, toolsets, role, model - The dispatch-time task spec, captured verbatim for the rich - completion block. - session_key - The gateway session_key (from ``tools.approval.get_current_session_key``) - captured on the parent thread BEFORE dispatch, because the daemon - worker thread won't carry the contextvar. Used to route the - completion back to the originating session. - parent_session_id - The durable ``state.db`` session id of the parent agent that spawned - the delegation. Carried on the completion event so the gateway can - pin routing to the spawning session instead of recovering the latest - ``ended_at IS NULL`` row for the peer tuple (#57498). - runner - Zero-arg callable that builds + runs the child and returns the same - result dict ``_run_single_child`` produces. Runs on the worker thread. - interrupt_fn - Optional callable to signal the child to stop (used on shutdown / - explicit cancel). - progress_fn - Optional zero-arg callable returning ``(token, in_tool)`` where - ``token`` is any comparable snapshot of the child's progress (api - call count + current tool) and ``in_tool`` says whether the child is - currently inside a tool call. Sampled by the stale monitor; a frozen - token past the stale threshold marks the delegation stuck (see the - stale-detection block at the top of this module). When omitted, the - delegation is not monitored. - max_async_children - Concurrency cap. When at capacity the dispatch is REJECTED (the caller - should fall back to sync or tell the user) rather than queued, so a - runaway model can't pile up unbounded background work. - - Returns - ------- - dict - ``{"status": "dispatched", "delegation_id": ...}`` on success, or - ``{"status": "rejected", "error": ...}`` when at capacity. + Capacity check + record insert happen under ONE lock hold so concurrent + dispatches can't both pass the check and exceed the cap. At capacity the + dispatch is REJECTED (never queued) so a runaway model can't pile up + unbounded background work. """ - delegation_id = _new_delegation_id() + is_batch = goals is not None + label = " batch" if is_batch else "" + classify = _batch_status if is_batch else (lambda r: r.get("status") or "completed") + crash_result = _batch_crash if is_batch else _single_crash dispatched_at = time.time() record: Dict[str, Any] = { "delegation_id": delegation_id, "goal": goal, + **({"goals": list(goals)} if is_batch else {}), "context": context, "toolsets": list(toolsets) if toolsets else None, "role": role, @@ -834,31 +688,17 @@ def dispatch_async_delegation( "dispatched_at": dispatched_at, "completed_at": None, "interrupt_fn": interrupt_fn, + **({"is_batch": True} if is_batch else {}), "progress_fn": progress_fn, # Stale-monitor bookkeeping (see _stale_monitor_loop). "_progress_token": None, "_progress_ts": dispatched_at, "_interrupted_at": None, } - # Capacity check and record insert under ONE lock hold — checking - # active_count() separately would let two concurrent dispatches (e.g. - # from different gateway sessions) both pass the check and exceed the cap. with _records_lock: - running = sum( - 1 for r in _records.values() - if r.get("status") in ("running", "stalling") - ) + running = sum(1 for r in _records.values() if r.get("status") in _ACTIVE_STATES) if running >= max_async_children: - return { - "status": "rejected", - "error": ( - f"Async delegation capacity reached ({max_async_children} " - f"running). Wait for one to finish (its result will re-enter " - f"the chat), or run this task synchronously " - f"(background=false). Raise delegation.max_concurrent_children in " - f"config.yaml to allow more concurrent background subagents." - ), - } + return {"status": "rejected", "error": capacity_error} _records[delegation_id] = record _persist_dispatch(record) @@ -869,16 +709,10 @@ def dispatch_async_delegation( status = "error" try: result = runner() or {} - status = result.get("status") or "completed" + status = classify(result) except Exception as exc: # noqa: BLE001 — must never crash the worker - logger.exception("Async delegation %s crashed", delegation_id) - result = { - "status": "error", - "summary": None, - "error": f"{type(exc).__name__}: {exc}", - "api_calls": 0, - "duration_seconds": round(time.time() - dispatched_at, 2), - } + logger.exception(f"Async delegation{label} %s crashed", delegation_id) + result = crash_result(f"{type(exc).__name__}: {exc}", round(time.time() - dispatched_at, 2)) status = "error" finally: _finalize(delegation_id, result, status) @@ -891,50 +725,114 @@ def dispatch_async_delegation( with _records_lock: _records.pop(delegation_id, None) _delete_durable_delegation(delegation_id) - return { - "status": "rejected", - "error": f"Failed to schedule async delegation: {exc}", - } + return {"status": "rejected", "error": f"Failed to schedule async delegation{label}: {exc}"} if progress_fn is not None: _ensure_stale_monitor() - - logger.info( - "Dispatched async delegation %s (session_key=%s): %s", - delegation_id, session_key or "", (goal or "")[:80], - ) return {"status": "dispatched", "delegation_id": delegation_id} -def _finalize(delegation_id: str, result: Dict[str, Any], status: str) -> None: - """Mark a record complete and push the completion event onto the queue.""" - claimed = _begin_finalization(delegation_id) - if claimed is None: - return - event_record, _interrupt_fn = claimed +def dispatch_async_delegation( + *, goal: str, context: Optional[str], toolsets: Optional[List[str]], role: str, + model: Optional[str], session_key: str, parent_session_id: Optional[str] = None, + runner: Callable[[], Dict[str, Any]], origin_ui_session_id: str = "", + origin_session_id: str = "", interrupt_fn: Optional[Callable[[], None]] = None, + max_async_children: int = _DEFAULT_MAX_ASYNC_CHILDREN, + progress_fn: Optional[Callable[[], tuple]] = None, +) -> Dict[str, Any]: + """Spawn ``runner`` on the daemon executor and return a handle immediately. + ``session_key``/``parent_session_id`` are captured on the parent thread + (the worker carries no contextvars) and route the completion back to the + spawning session. ``progress_fn() -> (token, in_tool)`` enables stale + monitoring; omitted = unmonitored. Returns ``{"status": "dispatched", + "delegation_id"}`` or ``{"status": "rejected", "error"}`` at capacity. + """ + delegation_id = _new_delegation_id() + handle = _dispatch( + delegation_id=delegation_id, goal=goal, goals=None, context=context, + toolsets=toolsets, role=role, model=model, session_key=session_key, + parent_session_id=parent_session_id, runner=runner, + origin_ui_session_id=origin_ui_session_id, origin_session_id=origin_session_id, + interrupt_fn=interrupt_fn, max_async_children=max_async_children, + progress_fn=progress_fn, + capacity_error=( + f"Async delegation capacity reached ({max_async_children} " + f"running). Wait for one to finish (its result will re-enter " + f"the chat), or run this task synchronously " + f"(background=false). Raise delegation.max_concurrent_children in " + f"config.yaml to allow more concurrent background subagents." + ), + ) + if handle["status"] == "dispatched": + logger.info("Dispatched async delegation %s (session_key=%s): %s", + delegation_id, session_key or "", (goal or "")[:80]) + return handle + + +def dispatch_async_delegation_batch( + *, goals: List[str], context: Optional[str], toolsets: Optional[List[str]], role: str, + model: Optional[str], session_key: str, parent_session_id: Optional[str] = None, + runner: Callable[[], Dict[str, Any]], origin_ui_session_id: str = "", + origin_session_id: str = "", interrupt_fn: Optional[Callable[[], None]] = None, + max_async_children: int = _DEFAULT_MAX_ASYNC_CHILDREN, + delegation_id: Optional[str] = None, progress_fn: Optional[Callable[[], tuple]] = None, +) -> Dict[str, Any]: + """Dispatch a WHOLE fan-out batch as ONE background unit. + + ``runner`` runs the entire batch and returns the combined + ``{"results": [...], "total_duration_seconds": N}`` dict. The batch + occupies ONE async slot (in-batch parallelism is bounded separately) and + produces a SINGLE completion event carrying the per-task ``results``. + """ + delegation_id = delegation_id or _new_delegation_id() + n = len(goals) + combined_goal = ( + goals[0] if n == 1 else f"{n} parallel subagents: " + "; ".join(g[:40] for g in goals) + ) + handle = _dispatch( + delegation_id=delegation_id, goal=combined_goal, goals=goals, context=context, + toolsets=toolsets, role=role, model=model, session_key=session_key, + parent_session_id=parent_session_id, runner=runner, + origin_ui_session_id=origin_ui_session_id, origin_session_id=origin_session_id, + interrupt_fn=interrupt_fn, max_async_children=max_async_children, + progress_fn=progress_fn, + capacity_error=( + f"Async delegation capacity reached ({max_async_children} " + f"running). Wait for one to finish (its result will re-enter " + f"the chat), or raise delegation.max_concurrent_children in " + f"config.yaml to allow more concurrent background units." + ), + ) + if handle["status"] == "dispatched": + logger.info("Dispatched async delegation batch %s (%d task(s), session_key=%s)", + delegation_id, n, session_key or "") + return handle + + +# ── Finalization + completion events ──────────────────────────────────────── +def _finalize(delegation_id: str, result: Dict[str, Any], status: str) -> None: + """Mark a record complete and push its completion event onto the queue.""" + event_record = _begin_finalization(delegation_id) + if event_record is None: + return _push_completion_event(event_record, result, status) _finish_finalization(delegation_id, status) -def _begin_finalization( - delegation_id: str, -) -> Optional[tuple[Dict[str, Any], Optional[Callable[[], None]]]]: - """Atomically claim terminal delivery while keeping the record active.""" +def _begin_finalization(delegation_id: str) -> Optional[Dict[str, Any]]: + """Atomically claim terminal delivery; returns a record snapshot or None.""" with _records_lock: record = _records.get(delegation_id) - if record is None or record.get("status") not in ("running", "stalling"): - return + if record is None or record.get("status") not in _ACTIVE_STATES: + return None # Stay active until durable persistence and queue publication finish; - # otherwise process shutdown can kill this daemon worker in the narrow - # gap after status flips but before SQLite is committed. + # otherwise process shutdown can kill this daemon worker in the gap + # after status flips but before SQLite is committed. record["status"] = "finalizing" record["completed_at"] = time.time() - interrupt_fn = record.get("interrupt_fn") record["interrupt_fn"] = None # drop the closure; child is done record["progress_fn"] = None # stop stale-monitor sampling - event_record = dict(record) - - return event_record, interrupt_fn + return dict(record) def _finish_finalization(delegation_id: str, status: str) -> None: @@ -945,301 +843,80 @@ def _finish_finalization(delegation_id: str, status: str) -> None: _prune_completed_locked() -def _push_completion_event( - record: Dict[str, Any], result: Dict[str, Any], status: str -) -> None: +def _push_completion_event(record: Dict[str, Any], result: Dict[str, Any], status: str) -> None: """Push a type='async_delegation' event onto the shared completion queue. - Best-effort: a failure here must not crash the worker, but it WOULD mean a - silently-lost result, so we log loudly. + Batch records (``is_batch``) carry the per-task ``results`` list instead of + a single summary. Best-effort: failure must not crash the worker, but it + WOULD mean a silently-lost result, so we log loudly. """ + is_batch = bool(record.get("is_batch")) + label = " batch" if is_batch else "" try: from tools.process_registry import process_registry except Exception as exc: # pragma: no cover - logger.error( - "Async delegation %s finished but process_registry import failed; " - "result lost: %s", - record.get("delegation_id"), exc, - ) + logger.error(f"Async delegation{label} %s finished but process_registry import failed; " + "result lost: %s", record.get("delegation_id"), exc) return - summary = result.get("summary") - error = result.get("error") dispatched_at = record.get("dispatched_at") or time.time() completed_at = record.get("completed_at") or time.time() - + if is_batch: + # Batch payload: the full per-task results list (the formatter renders a + # consolidated block) plus live transcript paths, which persist after + # completion as the full-fidelity record of each child's run. + payload = { + "is_batch": True, + "results": result.get("results") or [], + "live_transcripts": result.get("live_transcripts"), + "error": result.get("error"), + "total_duration_seconds": result.get("total_duration_seconds"), + } + else: + payload = { + "summary": result.get("summary"), + "error": result.get("error"), + "api_calls": result.get("api_calls", 0), + "duration_seconds": result.get( + "duration_seconds", round(completed_at - dispatched_at, 2) + ), + } evt = { "type": "async_delegation", "delegation_id": record.get("delegation_id"), - # session_key routes the completion back to the originating gateway - # session; empty string => CLI (single-session) path. + # session_key routes back to the originating gateway session; "" => CLI. "session_key": record.get("session_key", ""), "origin_ui_session_id": record.get("origin_ui_session_id", ""), "origin_session_id": record.get("origin_session_id", ""), "parent_session_id": record.get("parent_session_id"), "goal": record.get("goal", ""), + **({"goals": record.get("goals")} if is_batch else {}), "context": record.get("context"), "toolsets": record.get("toolsets"), "role": record.get("role"), - "model": result.get("model") or record.get("model"), + "model": record.get("model") if is_batch else (result.get("model") or record.get("model")), "status": status, - "summary": summary, - "error": error, - "api_calls": result.get("api_calls", 0), - "duration_seconds": result.get( - "duration_seconds", round(completed_at - dispatched_at, 2) - ), + **payload, "dispatched_at": dispatched_at, "completed_at": completed_at, - "exit_reason": result.get("exit_reason"), + **({} if is_batch else {"exit_reason": result.get("exit_reason")}), } - # Routing origin captured at dispatch (see _capture_routing_origin): - # additive, lets the gateway reconstruct a full SessionSource (incl. - # scope_id for relay tenant egress) when its own caches are cold. - for _k in ("scope_id", "user_id", "user_name"): - if record.get(_k): - evt[_k] = record[_k] - # Structured stall metadata (#51690) — additive, present only on - # stall-monitor finalizations. - for _k in ( - "stalled_after_quiet_seconds", - "stall_threshold_seconds", - "stall_phase", - "stall_grace_seconds", - ): - if _k in result: - evt[_k] = result[_k] + _copy_truthy(record, evt, _ROUTING_KEYS) + _copy_present(result, evt, _STALL_META_KEYS) _persist_completion(evt, result) try: process_registry.completion_queue.put(evt) except Exception as exc: # pragma: no cover - logger.error( - "Async delegation %s: failed to enqueue completion event; " - "result lost: %s", - record.get("delegation_id"), exc, - ) - - -def dispatch_async_delegation_batch( - *, - goals: List[str], - context: Optional[str], - toolsets: Optional[List[str]], - role: str, - model: Optional[str], - session_key: str, - parent_session_id: Optional[str] = None, - runner: Callable[[], Dict[str, Any]], - origin_ui_session_id: str = "", - origin_session_id: str = "", - interrupt_fn: Optional[Callable[[], None]] = None, - max_async_children: int = _DEFAULT_MAX_ASYNC_CHILDREN, - delegation_id: Optional[str] = None, - progress_fn: Optional[Callable[[], tuple]] = None, -) -> Dict[str, Any]: - """Dispatch a WHOLE fan-out batch as ONE background unit. - - Unlike ``dispatch_async_delegation`` (which backs a single subagent), - ``runner`` here runs the entire batch — it builds and joins on every child - in parallel and returns the combined ``{"results": [...], - "total_duration_seconds": N}`` dict that the synchronous path would have - returned. We occupy ONE async slot for the whole batch (the in-batch - parallelism is bounded separately by ``max_concurrent_children``), so a - single ``delegate_task`` fan-out never exhausts the async pool by itself. - - When the batch finishes, a SINGLE completion event is pushed onto the - shared ``process_registry.completion_queue`` carrying the full per-task - ``results`` list, so the consolidated summaries re-enter the conversation - as one message once every child is done — the chat is never blocked while - they run. - - Returns ``{"status": "dispatched", "delegation_id": ...}`` on success or - ``{"status": "rejected", "error": ...}`` when the async pool is at - capacity. - """ - delegation_id = delegation_id or _new_delegation_id() - dispatched_at = time.time() - n = len(goals) - # A combined goal label for status listings / the completion header. - combined_goal = ( - goals[0] if n == 1 else f"{n} parallel subagents: " + "; ".join(g[:40] for g in goals) - ) - record: Dict[str, Any] = { - "delegation_id": delegation_id, - "goal": combined_goal, - "goals": list(goals), - "context": context, - "toolsets": list(toolsets) if toolsets else None, - "role": role, - "model": model, - "session_key": session_key, - "origin_ui_session_id": origin_ui_session_id, - "origin_session_id": origin_session_id, - "parent_session_id": parent_session_id, - **_capture_routing_origin(), - "status": "running", - "dispatched_at": dispatched_at, - "completed_at": None, - "interrupt_fn": interrupt_fn, - "is_batch": True, - "progress_fn": progress_fn, - "_progress_token": None, - "_progress_ts": dispatched_at, - "_interrupted_at": None, - } - with _records_lock: - running = sum( - 1 for r in _records.values() - if r.get("status") in ("running", "stalling") - ) - if running >= max_async_children: - return { - "status": "rejected", - "error": ( - f"Async delegation capacity reached ({max_async_children} " - f"running). Wait for one to finish (its result will re-enter " - f"the chat), or raise delegation.max_concurrent_children in " - f"config.yaml to allow more concurrent background units." - ), - } - _records[delegation_id] = record - - _persist_dispatch(record) - executor = _get_executor(max_async_children) - - def _worker() -> None: - combined: Dict[str, Any] = {} - status = "error" - try: - combined = runner() or {} - # Batch status: completed unless every child errored/was interrupted. - child_results = combined.get("results") or [] - if child_results and all( - (r.get("status") not in ("completed", "success")) - for r in child_results - ): - status = "error" - else: - status = "completed" - except Exception as exc: # noqa: BLE001 — must never crash the worker - logger.exception("Async delegation batch %s crashed", delegation_id) - combined = { - "results": [], - "error": f"{type(exc).__name__}: {exc}", - "total_duration_seconds": round(time.time() - dispatched_at, 2), - } - status = "error" - finally: - _finalize_batch(delegation_id, combined, status) - - try: - # Propagate the dispatching profile to the detached batch children. - executor.submit(propagate_context_to_thread(_worker)) - except Exception as exc: # pragma: no cover - with _records_lock: - _records.pop(delegation_id, None) - _delete_durable_delegation(delegation_id) - return { - "status": "rejected", - "error": f"Failed to schedule async delegation batch: {exc}", - } - if progress_fn is not None: - _ensure_stale_monitor() - - logger.info( - "Dispatched async delegation batch %s (%d task(s), session_key=%s)", - delegation_id, n, session_key or "", - ) - return {"status": "dispatched", "delegation_id": delegation_id} - - -def _finalize_batch( - delegation_id: str, combined: Dict[str, Any], status: str -) -> None: - """Mark a batch record complete and push ONE combined completion event.""" - claimed = _begin_finalization(delegation_id) - if claimed is None: - return - event_record, _interrupt_fn = claimed - - _push_batch_completion_event(event_record, combined, status) - _finish_finalization(delegation_id, status) - - -def _push_batch_completion_event( - event_record: Dict[str, Any], combined: Dict[str, Any], status: str -) -> None: - """Push a combined async-delegation batch completion event.""" - try: - from tools.process_registry import process_registry - except Exception as exc: # pragma: no cover - logger.error( - "Async delegation batch %s finished but process_registry import " - "failed; result lost: %s", - event_record.get("delegation_id"), exc, - ) - return - - dispatched_at = event_record.get("dispatched_at") or time.time() - completed_at = event_record.get("completed_at") or time.time() - evt = { - "type": "async_delegation", - "delegation_id": event_record.get("delegation_id"), - "session_key": event_record.get("session_key", ""), - "origin_ui_session_id": event_record.get("origin_ui_session_id", ""), - "origin_session_id": event_record.get("origin_session_id", ""), - "parent_session_id": event_record.get("parent_session_id"), - "goal": event_record.get("goal", ""), - "goals": event_record.get("goals"), - "context": event_record.get("context"), - "toolsets": event_record.get("toolsets"), - "role": event_record.get("role"), - "model": event_record.get("model"), - "status": status, - "is_batch": True, - # The full per-task results list — the formatter renders a - # consolidated multi-task block from this. - "results": combined.get("results") or [], - # Per-task live transcript log paths (cache/delegation/live/...). - # They persist after completion and double as the full-fidelity - # operational record of each child's run. - "live_transcripts": combined.get("live_transcripts"), - "error": combined.get("error"), - "total_duration_seconds": combined.get("total_duration_seconds"), - "dispatched_at": dispatched_at, - "completed_at": completed_at, - } - # Routing origin captured at dispatch (see _capture_routing_origin). - for _k in ("scope_id", "user_id", "user_name"): - if event_record.get(_k): - evt[_k] = event_record[_k] - # Structured stall metadata (#51690) — additive, present only on - # stall-monitor finalizations. - for _k in ( - "stalled_after_quiet_seconds", - "stall_threshold_seconds", - "stall_phase", - "stall_grace_seconds", - ): - if _k in combined: - evt[_k] = combined[_k] - _persist_completion(evt, combined) - try: - process_registry.completion_queue.put(evt) - except Exception as exc: # pragma: no cover - logger.error( - "Async delegation batch %s: failed to enqueue completion event; " - "result lost: %s", - event_record.get("delegation_id"), exc, - ) + logger.error(f"Async delegation{label} %s: failed to enqueue completion event; " + "result lost: %s", record.get("delegation_id"), exc) +# ── Stale monitor ─────────────────────────────────────────────────────────── def _ensure_stale_monitor() -> None: - """Start (once) the module-level stale-delegation monitor thread. + """Start (once) the stale-delegation monitor thread. - One daemon thread serves every dispatch; it exits on its own when no - monitorable records remain, and is restarted by the next dispatch that - carries a ``progress_fn``. + One daemon thread serves every dispatch; it exits when no monitorable + records remain and is restarted by the next dispatch with a ``progress_fn``. """ global _monitor_thread with _monitor_lock: @@ -1247,9 +924,7 @@ def _ensure_stale_monitor() -> None: return _monitor_stop.clear() _monitor_thread = threading.Thread( - target=_stale_monitor_loop, - name="async-delegate-stale-monitor", - daemon=True, + target=_stale_monitor_loop, name="async-delegate-stale-monitor", daemon=True, ) _monitor_thread.start() @@ -1257,23 +932,15 @@ def _ensure_stale_monitor() -> None: def _stale_monitor_loop() -> None: """Sweep running delegations for stalled progress. - Per sweep, for every running record with a ``progress_fn``: - - - Sample ``(token, in_tool)``. A changed token refreshes the record's - progress timestamp — a child that keeps advancing is never touched, no - matter how long it runs. - - A frozen token past the idle/in-tool threshold marks the record - ``stalling``: we call ``interrupt_fn`` so a responsive-but-slow child - can unwind and deliver its (partial) result through the normal - ``_finalize`` path with full fidelity. - - A ``stalling`` record whose runner still hasn't returned after the - grace window is force-finalized with one terminal ``stalled`` event so - the owning session hears an outcome and the async slot frees. A late - runner return after that is ignored by ``_begin_finalization``. + A changed progress token refreshes the record's timestamp. A frozen token + past the idle/in-tool threshold marks the record ``stalling`` and calls + ``interrupt_fn``; a ``stalling`` record still unreturned after the grace + window is force-finalized with a terminal ``stalled`` event (a late + runner return is then ignored by ``_begin_finalization``). """ while not _monitor_stop.wait(_STALE_CHECK_INTERVAL): now = time.time() - stalled: List[tuple] = [] # (delegation_id, is_batch, quiet_for, in_tool) + stalled: List[tuple] = [] # (delegation_id, quiet_for, in_tool) expired: List[str] = [] # stalling past grace → force-finalize any_monitorable = False with _records_lock: @@ -1298,37 +965,22 @@ def _stale_monitor_loop() -> None: # keep the last timestamp running instead of refreshing it. token, in_tool = record.get("_progress_token"), False if token != record.get("_progress_token"): - record["_progress_token"] = token - record["_progress_ts"] = now + record.update(_progress_token=token, _progress_ts=now) continue quiet_for = now - (record.get("_progress_ts") or now) - limit = ( - _STALE_IN_TOOL_SECONDS if in_tool else _STALE_IDLE_SECONDS - ) + limit = _STALE_IN_TOOL_SECONDS if in_tool else _STALE_IDLE_SECONDS if quiet_for >= limit: - record["status"] = "stalling" - record["_interrupted_at"] = now - # Structured stall context for the terminal event and - # status listings (#51690): how long progress was frozen, - # which threshold applied, and whether the child was - # inside a tool when it went quiet. - record["_stall_quiet_seconds"] = round(quiet_for, 2) - record["_stall_threshold_seconds"] = limit - record["_stall_in_tool"] = bool(in_tool) - stalled.append( - ( - record["delegation_id"], - bool(record.get("is_batch")), - quiet_for, - in_tool, - ) + # Stall context feeds the terminal event and status listings. + record.update( + status="stalling", _interrupted_at=now, + _stall_quiet_seconds=round(quiet_for, 2), + _stall_threshold_seconds=limit, _stall_in_tool=bool(in_tool), ) - for delegation_id, _is_batch, quiet_for, in_tool in stalled: - logger.warning( - "Async delegation %s made no progress for %.0fs " - "(in_tool=%s) — interrupting; grace window %.0fs", - delegation_id, quiet_for, in_tool, _STALL_GRACE_SECONDS, - ) + stalled.append((record["delegation_id"], quiet_for, in_tool)) + for delegation_id, quiet_for, in_tool in stalled: + logger.warning("Async delegation %s made no progress for %.0fs " + "(in_tool=%s) — interrupting; grace window %.0fs", + delegation_id, quiet_for, in_tool, _STALL_GRACE_SECONDS) with _records_lock: record = _records.get(delegation_id) fn = record.get("interrupt_fn") if record else None @@ -1336,10 +988,7 @@ def _stale_monitor_loop() -> None: try: fn() except Exception as exc: - logger.debug( - "Async delegation %s stall interrupt failed: %s", - delegation_id, exc, - ) + logger.debug("Async delegation %s stall interrupt failed: %s", delegation_id, exc) for delegation_id in expired: _finalize_stalled(delegation_id) if not any_monitorable: @@ -1348,19 +997,11 @@ def _stale_monitor_loop() -> None: def _finalize_stalled(delegation_id: str) -> None: """Force-finalize a stalling delegation whose runner never returned.""" - claimed = _begin_finalization(delegation_id) - if claimed is None: + event_record = _begin_finalization(delegation_id) + if event_record is None: return - event_record, _interrupt_fn = claimed - completed_at = event_record.get("completed_at") or time.time() - duration = round( - completed_at - (event_record.get("dispatched_at") or completed_at), - 2, - ) - quiet_seconds = event_record.get("_stall_quiet_seconds") - threshold_seconds = event_record.get("_stall_threshold_seconds") - stall_in_tool = event_record.get("_stall_in_tool") + duration = round(completed_at - (event_record.get("dispatched_at") or completed_at), 2) error = ( f"Async delegation {delegation_id} stalled: the detached subagent " "stopped making progress (no new API calls, tool activity, or " @@ -1370,59 +1011,30 @@ def _finalize_stalled(delegation_id: str) -> None: "gateway processes (#60203). Re-dispatch the task if it is still " "needed." ) - logger.error( - "Async delegation %s force-finalized as stalled after %.0fs", - delegation_id, duration, - ) - # Structured stall metadata (#51690): lets parents and UIs distinguish - # a stall-monitor kill from other failures without parsing the error - # string, mirroring the sync path's timeout_seconds/timed_out_after_ - # seconds/timeout_phase fields. + logger.error("Async delegation %s force-finalized as stalled after %.0fs", delegation_id, duration) + # Structured stall metadata lets parents/UIs distinguish a stall-monitor + # kill from other failures without parsing the error string. + stall_in_tool = event_record.get("_stall_in_tool") stall_meta = { - "stalled_after_quiet_seconds": quiet_seconds, - "stall_threshold_seconds": threshold_seconds, - "stall_phase": ( - "in_tool" if stall_in_tool - else "idle" if stall_in_tool is not None - else None - ), + "stalled_after_quiet_seconds": event_record.get("_stall_quiet_seconds"), + "stall_threshold_seconds": event_record.get("_stall_threshold_seconds"), + "stall_phase": "in_tool" if stall_in_tool else "idle" if stall_in_tool is not None else None, "stall_grace_seconds": _STALL_GRACE_SECONDS, } if event_record.get("is_batch"): - _push_batch_completion_event( - event_record, - { - "results": [], - "error": error, - "total_duration_seconds": duration, - **stall_meta, - }, - "stalled", - ) + result = {**_batch_crash(error, duration), **stall_meta} else: - _push_completion_event( - event_record, - { - "status": "stalled", - "summary": None, - "error": error, - "api_calls": 0, - "duration_seconds": duration, - "exit_reason": "stalled", - **stall_meta, - }, - "stalled", - ) + result = {**_single_crash(error, duration), "status": "stalled", "exit_reason": "stalled", **stall_meta} + _push_completion_event(event_record, result, "stalled") _finish_finalization(delegation_id, "stalled") +# ── Observability + control ───────────────────────────────────────────────── def _children_activity_from_token(token: Any, now: float) -> Optional[List]: """Parse a progress token into per-child activity dicts (best-effort). - delegate_tool's ``_batch_progress`` emits one ``(api_call_count, - current_tool, last_activity_ts)`` tuple per child. Foreign token shapes - (custom dispatchers) degrade to ``None`` entries rather than raising — - the token contract is intentionally opaque to the registry. + delegate_tool emits one ``(api_call_count, current_tool, last_activity_ts)`` + tuple per child; foreign token shapes degrade to ``None`` entries. """ try: parts = list(token) @@ -1431,14 +1043,9 @@ def _children_activity_from_token(token: Any, now: float) -> Optional[List]: out: List[Optional[Dict[str, Any]]] = [] for part in parts: if isinstance(part, (list, tuple)) and len(part) >= 2: - entry: Dict[str, Any] = { - "api_calls": part[0], - "current_tool": part[1], - } + entry: Dict[str, Any] = {"api_calls": part[0], "current_tool": part[1]} if len(part) >= 3 and isinstance(part[2], (int, float)): - entry["seconds_since_activity"] = round( - max(0.0, now - float(part[2])), 1 - ) + entry["seconds_since_activity"] = round(max(0.0, now - float(part[2])), 1) out.append(entry) else: out.append(None) @@ -1448,17 +1055,10 @@ def _children_activity_from_token(token: Any, now: float) -> Optional[List]: def list_async_delegations() -> List[Dict[str, Any]]: """Snapshot of async delegations (running + recently completed). - Safe to call from any thread. Excludes the non-serialisable callables - and private monitor bookkeeping, but exposes computed live-status - fields for UIs (#51690): - - - ``seconds_since_progress``: how long the stale monitor has seen a - frozen progress token (running/stalling records). - - ``children_activity``: per-child ``{api_calls, current_tool, - seconds_since_activity}`` sampled live from the dispatch's - ``progress_fn``. - - ``stalled_after_quiet_seconds`` / ``stall_threshold_seconds`` / - ``stall_in_tool``: stall context once the monitor has tripped. + Excludes callables and private monitor bookkeeping; adds computed live + fields for UIs: ``seconds_since_progress``, ``children_activity`` / + ``in_tool`` (sampled from ``progress_fn``), and stall context once the + monitor has tripped. """ now = time.time() samplers: Dict[str, Callable] = {} @@ -1468,11 +1068,10 @@ def list_async_delegations() -> List[Dict[str, Any]]: item = { k: v for k, v in r.items() - if k not in {"interrupt_fn", "progress_fn"} - and not k.startswith("_") + if k not in {"interrupt_fn", "progress_fn"} and not k.startswith("_") } status = r.get("status") - if status in ("running", "stalling"): + if status in _ACTIVE_STATES: ts = r.get("_progress_ts") if ts: item["seconds_since_progress"] = round(now - ts, 1) @@ -1480,18 +1079,13 @@ def list_async_delegations() -> List[Dict[str, Any]]: if callable(fn): samplers[r["delegation_id"]] = fn if status in ("stalling", "stalled"): - for src, dst in ( - ("_stall_quiet_seconds", "stalled_after_quiet_seconds"), - ("_stall_threshold_seconds", "stall_threshold_seconds"), - ("_stall_in_tool", "stall_in_tool"), - ): + for src, dst in _STALL_FIELD_MAP: if r.get(src) is not None: item[dst] = r.get(src) items.append(item) - # Sample live activity OUTSIDE the lock — progress_fn reads child-agent - # attributes and must never run under _records_lock (a slow or broken - # sampler would block every dispatch/finalize in the process). + # Sample OUTSIDE the lock — progress_fn reads child-agent attributes and a + # slow/broken sampler must not block every dispatch/finalize. for item in items: fn = samplers.get(item.get("delegation_id")) if fn is None: @@ -1507,19 +1101,9 @@ def list_async_delegations() -> List[Dict[str, Any]]: return items -def interrupt_all(reason: str = "shutdown") -> int: - """Signal every running async delegation to stop. Returns how many. - - Used on ``/stop`` and gateway shutdown so a dangling background subagent - can't keep burning tokens with no one listening. The child still emits a - completion event (status='interrupted') via the normal finalize path. - """ +def _interrupt_records(targets: List[Dict[str, Any]], caller: str) -> int: + """Call ``interrupt_fn`` on each record; returns how many succeeded.""" count = 0 - with _records_lock: - targets = [ - r for r in _records.values() - if r.get("status") in ("running", "stalling") - ] for r in targets: fn = r.get("interrupt_fn") if callable(fn): @@ -1527,69 +1111,42 @@ def interrupt_all(reason: str = "shutdown") -> int: fn() count += 1 except Exception as exc: - logger.debug( - "interrupt_all: %s interrupt failed: %s", - r.get("delegation_id"), exc, - ) + logger.debug("%s: %s interrupt failed: %s", caller, r.get("delegation_id"), exc) + return count + + +def interrupt_all(reason: str = "shutdown") -> int: + """Signal every running async delegation to stop (``/stop``, shutdown). Returns how many. + + The child still emits a completion event (status='interrupted') via the + normal finalize path. + """ + with _records_lock: + targets = [r for r in _records.values() if r.get("status") in _ACTIVE_STATES] + count = _interrupt_records(targets, "interrupt_all") if count: logger.info("Interrupted %d async delegation(s) (%s)", count, reason) return count def interrupt_for_session( - session_key: str = "", - origin_ui_session_id: str = "", - parent_session_id: str = "", + session_key: str = "", origin_ui_session_id: str = "", parent_session_id: str = "", reason: str = "session_end", ) -> int: - """Signal running async delegations owned by ONE session to stop. + """Signal running async delegations owned by ONE ending session to stop. - A delegation's lifecycle is bound to the session that spawned it: when - that session ends, its in-flight background subagents must end with it — - a completed orphan would otherwise sit on the shared completion queue - with no live owner, either leaking into another chat or burning tokens - with no one listening (#55578). - - Selectors (any matching field claims the record): - - ``origin_ui_session_id``: the live TUI tab/window that commissioned it. - - ``session_key``: the durable routing key captured at dispatch. - - ``parent_session_id``: the spawning agent's durable session-db id — - the right selector for gateway chats, whose ``session_key`` (the - platform conversation key) SURVIVES a ``/new`` reset while the - session id rotates. - - Returns how many were interrupted. + Any matching selector claims the record: ``origin_ui_session_id`` (TUI + tab), ``session_key`` (routing key at dispatch), or ``parent_session_id`` + (spawner's durable id — the right one for gateway chats, whose session_key + survives ``/new`` while the session id rotates). Returns how many. """ if not session_key and not origin_ui_session_id and not parent_session_id: return 0 - count = 0 with _records_lock: - targets = [ - r for r in _records.values() - if r.get("status") in ("running", "stalling") - and _matches_session_selectors( - r, - session_key=session_key, - origin_ui_session_id=origin_ui_session_id, - parent_session_id=parent_session_id, - ) - ] - for r in targets: - fn = r.get("interrupt_fn") - if callable(fn): - try: - fn() - count += 1 - except Exception as exc: - logger.debug( - "interrupt_for_session: %s interrupt failed: %s", - r.get("delegation_id"), exc, - ) + targets = _session_records_locked(_ACTIVE_STATES, session_key, origin_ui_session_id, parent_session_id) + count = _interrupt_records(targets, "interrupt_for_session") if count: - logger.info( - "Interrupted %d async delegation(s) for ending session (%s)", - count, reason, - ) + logger.info("Interrupted %d async delegation(s) for ending session (%s)", count, reason) return count diff --git a/tools/binary_extensions.py b/tools/binary_extensions.py index 5eac3d0d75..d38a6f48da 100644 --- a/tools/binary_extensions.py +++ b/tools/binary_extensions.py @@ -33,23 +33,12 @@ BINARY_EXTENSIONS = frozenset({ ".lockb", ".dat", ".data", }) - -def has_binary_extension(path: str) -> bool: - """Check if a file path has a binary extension. Pure string check, no I/O.""" - dot = path.rfind(".") - if dot == -1: - return False - return path[dot:].lower() in BINARY_EXTENSIONS - - -# Container document formats (OOXML zip / OLE compound / ODF zip / EPUB zip / RTF) -# that a plain-text write can NEVER produce validly. read_file auto-extracts -# these to readable text (via anydoc for the non-built-in formats), so a model -# that "read" report.docx and then writes the edited text back via -# write_file/patch silently destroys the document. -# PDF is intentionally NOT here: raw PDF syntax is text-authorable, so -# new-file creation is legitimate — only overwrites are dangerous (handled -# separately by the write guard). +# Container document formats (OOXML/ODF/EPUB zips, OLE compound, RTF) that a +# plain-text write can NEVER produce validly. read_file auto-extracts these to +# text, so a model that "read" report.docx and writes the text back via +# write_file/patch silently destroys the document. PDF is deliberately absent: +# raw PDF syntax is text-authorable, so only overwrites are dangerous (handled +# by the write guard via is_pdf_path). OPAQUE_DOCUMENT_EXTENSIONS = frozenset({ ".doc", ".docx", ".docm", ".xls", ".xlsx", ".xlsm", ".xlsb", @@ -59,17 +48,20 @@ OPAQUE_DOCUMENT_EXTENSIONS = frozenset({ }) -def has_opaque_document_extension(path: str) -> bool: - """True when the path names an opaque container document (.docx etc.). - - Pure string check, no I/O. - """ +def _has_extension_in(path: str, extensions: frozenset) -> bool: + """Pure string check on the final ``.suffix`` (case-insensitive), no I/O.""" dot = path.rfind(".") - if dot == -1: - return False - return path[dot:].lower() in OPAQUE_DOCUMENT_EXTENSIONS + return dot != -1 and path[dot:].lower() in extensions + + +def has_binary_extension(path: str) -> bool: + return _has_extension_in(path, BINARY_EXTENSIONS) + + +def has_opaque_document_extension(path: str) -> bool: + """True when the path names an opaque container document (.docx etc.).""" + return _has_extension_in(path, OPAQUE_DOCUMENT_EXTENSIONS) def is_pdf_path(path: str) -> bool: - """True when the path has a .pdf extension. Pure string check, no I/O.""" return path.lower().endswith(".pdf") diff --git a/tools/blueprints.py b/tools/blueprints.py index c236a32a8b..526c5a93e5 100644 --- a/tools/blueprints.py +++ b/tools/blueprints.py @@ -12,21 +12,10 @@ frontmatter: prompt: "..." # optional task instruction for the run no_agent: false # optional -Because a blueprint is just a skill, it flows through the ENTIRE existing -skills-hub pipeline for free — search, inspect, quarantine, security scan, -install, lock-file provenance, audit log, taps, the centralized index, and -`hermes skills publish` for sharing. No new source type, no new store, no new -transport. This module is the thin bridge between that skill metadata and the -existing cron `create_job()` API: - - * ``parse_blueprint(skill_md_text)`` -> BlueprintSpec | None - * ``blueprint_spec_for_installed(name)`` -> BlueprintSpec | None - * ``create_blueprint_job(spec, ...)`` -> the created cron job dict - * ``export_blueprint(job, body)`` -> a shareable SKILL.md string - -The dev guide's "Extend, Don't Duplicate" rule is the whole design: the blueprint -is a skill, the schedule is a cron job, sharing is the existing publish/tap/ -index path. +Because a blueprint is just a skill it rides the whole skills-hub pipeline +(search, scan, install, provenance, publish) for free; this module is only the +bridge from that frontmatter to the cron ``create_job()`` API, plus the inverse +(``export_blueprint``) that renders a cron job back into a shareable SKILL.md. """ from __future__ import annotations @@ -117,11 +106,7 @@ def parse_blueprint(skill_md_text: str) -> Optional[BlueprintSpec]: if not schedule: raise BlueprintError("blueprint.schedule is required and must be non-empty") - deliver = str(blueprint.get("deliver", "origin")).strip() or "origin" prompt = blueprint.get("prompt") - if prompt is not None: - prompt = str(prompt) - no_agent = bool(blueprint.get("no_agent", False)) model = blueprint.get("model") provider = blueprint.get("provider") toolsets = blueprint.get("enabled_toolsets") @@ -131,9 +116,9 @@ def parse_blueprint(skill_md_text: str) -> Optional[BlueprintSpec]: return BlueprintSpec( skill_name=name, schedule=schedule, - deliver=deliver, - prompt=prompt, - no_agent=no_agent, + deliver=str(blueprint.get("deliver", "origin")).strip() or "origin", + prompt=str(prompt) if prompt is not None else None, + no_agent=bool(blueprint.get("no_agent", False)), model=str(model).strip() if model else None, provider=str(provider).strip() if provider else None, enabled_toolsets=[str(t) for t in toolsets] if toolsets else None, @@ -142,20 +127,15 @@ def parse_blueprint(skill_md_text: str) -> Optional[BlueprintSpec]: def blueprint_spec_for_installed(skill_name: str) -> Optional[BlueprintSpec]: - """Locate an installed skill's SKILL.md and parse its blueprint block. - - Searches the standard skills tree for ``/SKILL.md``. Returns - None if the skill isn't found or isn't a blueprint. - """ + """Find ``/SKILL.md`` anywhere in the skills tree and parse its + blueprint block; None when not found or not a blueprint.""" try: from tools.skills_hub import SKILLS_DIR except Exception: # pragma: no cover - import guard return None - base = Path(SKILLS_DIR) # Skills live at skills///SKILL.md or skills//SKILL.md. - candidates = list(base.glob(f"**/{skill_name}/SKILL.md")) - for path in candidates: + for path in Path(SKILLS_DIR).glob(f"**/{skill_name}/SKILL.md"): try: text = path.read_text(encoding="utf-8") except OSError: @@ -174,13 +154,8 @@ def blueprint_to_job_spec( *, name: Optional[str] = None, ) -> Dict[str, Any]: - """Build the ``cron.jobs.create_job`` kwargs dict for a BlueprintSpec. - - This is the single source of truth for translating a blueprint into a job. - Both the direct ``create_blueprint_job`` path and the suggestion path - (``register_blueprint_suggestion``) build on it, so a blueprint scheduled now and - a blueprint accepted from a suggestion produce an identical job. - """ + """``cron.jobs.create_job`` kwargs for a spec — the single translation used by + both ``create_blueprint_job`` and the suggestion path so they never drift.""" return { "prompt": spec.prompt, "schedule": spec.schedule, @@ -200,12 +175,7 @@ def create_blueprint_job( origin: Optional[Dict[str, Any]] = None, name: Optional[str] = None, ) -> Dict[str, Any]: - """Create the cron job described by a BlueprintSpec via the existing cron API. - - The blueprint's skill is loaded before the run (cron ``skills=[name]``); the - optional ``prompt`` becomes the task instruction. Delivery, model, and - toolsets carry through. Returns the created job dict. - """ + """Create the cron job for a spec (skill preloaded via ``skills=[name]``); returns the job dict.""" from cron.scheduler import create_job_with_scheduler_registration job_spec = blueprint_to_job_spec(spec, name=name) @@ -215,14 +185,8 @@ def create_blueprint_job( def register_blueprint_suggestion(spec: BlueprintSpec) -> Optional[Dict[str, Any]]: - """Turn an installed blueprint into a pending Suggested Cron Job. - - Blueprints are source ``blueprint`` of the unified suggestion surface: installing - a skill that carries a ``blueprint:`` block does NOT auto-schedule it — it - registers a suggestion the user accepts (or dismisses) like any other. - Returns the suggestion record, or None if it was skipped (already - seen/dismissed, backlog full, etc.). - """ + """Register an installed blueprint as a Suggested Cron Job (never auto-scheduled; + the user accepts or dismisses it). None when skipped (seen/dismissed/backlog full).""" if not spec.skill_name: return None try: @@ -244,13 +208,9 @@ def register_blueprint_suggestion(spec: BlueprintSpec) -> Optional[Dict[str, Any def export_blueprint(job: Dict[str, Any], body: str, *, blueprint_name: Optional[str] = None) -> str: - """Render a shareable blueprint SKILL.md from an existing cron job dict. - - The inverse of ``create_blueprint_job``: take a cron job a user already built - and emit a SKILL.md (with a ``metadata.hermes.blueprint`` block) they can hand - to ``hermes skills publish`` to share. ``body`` is the plain-language - description / instructions that become the SKILL.md body. - """ + """Inverse of ``create_blueprint_job``: render a cron job as a SKILL.md (with a + ``metadata.hermes.blueprint`` block) ready for ``hermes skills publish``. + ``body`` becomes the SKILL.md body; its first line is the description.""" import yaml name = blueprint_name or job.get("name") or "shared-blueprint" @@ -258,28 +218,21 @@ def export_blueprint(job: Dict[str, Any], body: str, *, blueprint_name: Optional name = "".join(c if (c.isalnum() or c in "-_") else "-" for c in str(name).lower()) name = name.strip("-_") or "shared-blueprint" - schedule = job.get("schedule_display") or _schedule_to_string(job.get("schedule")) - - blueprint_block: Dict[str, Any] = {"schedule": schedule} - deliver = job.get("deliver") - if deliver and deliver != "origin": - blueprint_block["deliver"] = deliver + blueprint_block: Dict[str, Any] = { + "schedule": job.get("schedule_display") or _schedule_to_string(job.get("schedule")), + } + if job.get("deliver") and job["deliver"] != "origin": + blueprint_block["deliver"] = job["deliver"] if job.get("prompt"): blueprint_block["prompt"] = job["prompt"] if job.get("no_agent"): blueprint_block["no_agent"] = True - if job.get("model"): - blueprint_block["model"] = job["model"] - if job.get("provider"): - blueprint_block["provider"] = job["provider"] - if job.get("enabled_toolsets"): - blueprint_block["enabled_toolsets"] = job["enabled_toolsets"] + for key in ("model", "provider", "enabled_toolsets"): + if job.get(key): + blueprint_block[key] = job[key] - description = ( - (body.strip().splitlines() or ["Shared automation blueprint."])[0][:200] - if body.strip() - else "Shared automation blueprint." - ) + body = body.strip() + description = body.splitlines()[0][:200] if body else "Shared automation blueprint." frontmatter = { "name": name, @@ -294,7 +247,7 @@ def export_blueprint(job: Dict[str, Any], body: str, *, blueprint_name: Optional }, } fm_yaml = yaml.safe_dump(frontmatter, sort_keys=False, allow_unicode=True).strip() - body_text = body.strip() or f"# {name}\n\nShared automation blueprint." + body_text = body or f"# {name}\n\nShared automation blueprint." return f"---\n{fm_yaml}\n---\n\n{body_text}\n" @@ -311,14 +264,10 @@ def _schedule_to_string(schedule: Any) -> str: # legacy/foreign "seconds" form too. if schedule.get("minutes"): mins = int(schedule["minutes"]) - if mins % 60 == 0: - return f"every {mins // 60}h" - return f"every {mins}m" + return f"every {mins // 60}h" if mins % 60 == 0 else f"every {mins}m" if schedule.get("seconds"): secs = int(schedule["seconds"]) if secs % 3600 == 0: return f"every {secs // 3600}h" - if secs % 60 == 0: - return f"every {secs // 60}m" - return f"every {secs}s" + return f"every {secs // 60}m" if secs % 60 == 0 else f"every {secs}s" return "0 9 * * *" # safe daily fallback diff --git a/tools/bot_mode_dm.py b/tools/bot_mode_dm.py index cde6eec992..2c44316c38 100644 --- a/tools/bot_mode_dm.py +++ b/tools/bot_mode_dm.py @@ -1,41 +1,26 @@ """Bot Mode agent-to-agent DM tool — ``message_agent``. -A structured, Bot-Chat-only tool that lets a Bot Mode agent message a -teammate agent (another Hermes profile on this install, or an agent on a -registered peer gateway) WITHOUT hand-assembling shell commands. +Lets a Bot Mode agent message a teammate (another profile on this install, an +agent on a registered peer gateway, or one on another Desktop-connected machine) +as a real tool call instead of a hand-assembled ``hermes -p …`` shellout: the +message is a parameter, the target is validated against the live roster, the +attribution prefix is applied server-side, and the reply arrives later through +the background-process completion notification (fire-and-forget). -Why this exists (Aug 2026): the Bot Mode teammate protocol taught agents to -DM each other via a prompt-injected ``hermes -p chat ...`` shellout. -That transport works, but the *invocation* was fragile — quoting traps -(#91339/#91304), temp-file choreography, dead-profile races — and the -Desktop's remote-mention path forwarded raw user text verbatim (#91397). -``message_agent`` replaces the invocation with a real tool call: the message -is a parameter, the target is validated against the live roster, the -attribution prefix is applied server-side, and the reply arrives through the -existing background-process notification path (fire-and-forget, never -blocks the sender's turn). - -Containment contract (MUST hold — reviewers check all three): -- The tool schema is injected ONLY into a bot's canonical "Bot Chat" - session on Bot-Mode-managed installs — the exact same gate as the - protocol section in ``tools/bot_mode_probe.py``. It is NOT registered in - the global tool registry, is NOT part of any toolset, and never appears - in CLI sessions, ordinary gateway chats, group-room member sessions - (titled "Group: …"), cron agents, or subagents. -- Dispatch is title-gated again at execution time (defense in depth): a - forged call from a session that shouldn't have the tool returns a +Containment contract (all three MUST hold): +- The schema is injected ONLY into a bot's canonical "Bot Chat" session on a + Bot-Mode-managed install — the same gate as ``tools/bot_mode_probe.py``. It is + not in the registry or any toolset; CLI, ordinary gateway, group-room, cron and + subagent sessions never see it. +- Dispatch re-checks the title gate (defense in depth): a forged call returns a structured error instead of delivering. -- Everything here is additive. The legacy protocol transports - (``hermes -p`` / ``hermes peer dm``) keep working for older prompts. +- Everything is additive; the legacy ``hermes -p`` / ``hermes peer dm`` + transports keep working for older prompts. -The transports themselves are unchanged and proven: -- local teammate → ``hermes -p chat --in ~ -c "Bot Chat" - --create-if-missing -Q --query-file `` (one turn, reply on stdout) -- peer teammate → ``hermes peer dm [/] < `` - -Both run through ``terminal_tool(background=True, notify_on_complete=True)`` -so the reply lands as a completion notification on the sender's NEXT turn — -the same wake shape every Bot Mode agent already knows. +Transports: local → ``hermes -p chat --in ~ -c "Bot Chat" +--create-if-missing -Q --query-file ``; peer → ``hermes peer dm +[/] < ``. Both run via ``terminal_tool(background=True, +notify_on_complete=True)`` so the reply wakes the sender on its next turn. """ from __future__ import annotations @@ -54,6 +39,17 @@ import time from pathlib import Path from typing import Any, Optional +from tools.bot_mode_probe import ( + BOT_CHAT_TITLE, + _handle, + _hermes_root, + _peers, + _profile_name as _self_profile_name, + _roster, + is_bot_mode_managed, +) +from tools.bot_relay import _HANDLE_RE as _LOCAL_TARGET_RE + logger = logging.getLogger(__name__) MESSAGE_AGENT_TOOL_NAME = "message_agent" @@ -62,14 +58,16 @@ MESSAGE_AGENT_TOOL_NAME = "message_agent" # runaway paste can't turn one DM into a context bomb on the recipient. MESSAGE_MAX_CHARS = 16000 -# A runner normally owns and removes each file. This bounds the residual -# plaintext lifetime if the machine dies after background-spawn acknowledgement -# but before the runner reaches its ``finally`` block. +# A runner normally owns and removes each DM file. This bounds the residual +# plaintext lifetime if the machine dies between spawn ack and the runner's finally. _DM_DIR_NAME = "hermes-dm" _DM_STALE_SECONDS = 24 * 60 * 60 +def _default_home() -> str: + return os.getenv("HERMES_HOME") or os.path.expanduser("~/.hermes") + + _PEER_TARGET_RE = re.compile(r"^([a-z0-9][a-z0-9_-]{0,63})/([a-zA-Z0-9][a-zA-Z0-9_-]{0,63})$") -_LOCAL_TARGET_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$") def message_agent_tool_schema() -> dict: @@ -129,12 +127,9 @@ def message_agent_tool_schema() -> dict: def ensure_message_agent_tool(agent: Any) -> bool: """Inject the ``message_agent`` schema into a Bot Chat agent's tool list. - Called once per turn from the conversation loop. Idempotent and - deterministic for the life of a session: the gate (canonical Bot Chat - title on a Bot-Mode-managed install) is stable from the session's first - turn, so the tool list is byte-identical across turns — prompt-cache - safe. Every non-Bot-Chat session fails the gate on every turn and never - sees the schema. Never raises. + Called once per turn. Idempotent and deterministic for the life of a session + (the gate is stable from the first turn), so the tool list is byte-identical + across turns — prompt-cache safe. Never raises. """ try: if not getattr(agent, "_bot_mode_protocol", True): @@ -147,14 +142,10 @@ def ensure_message_agent_tool(agent: Any) -> bool: and tool.get("function", {}).get("name") == MESSAGE_AGENT_TOOL_NAME ): return True - from tools.bot_mode_probe import BOT_CHAT_TITLE, is_bot_mode_managed - if _session_title(agent) != BOT_CHAT_TITLE: return False - # Managed-install check, NOT section non-emptiness: a profile whose - # SOUL.md carries the legacy plugin-appended protocol text gets an - # empty section (dedupe) but must still receive the tool — otherwise - # upgraded installs silently lose A2A messaging (Aug 2026). + # Managed-install check, NOT section non-emptiness: a SOUL.md carrying the + # legacy protocol text gets an empty section but must still get the tool. if not is_bot_mode_managed(_agent_home(agent)): return False if agent.tools is None: @@ -172,43 +163,9 @@ def ensure_message_agent_tool(agent: Any) -> bool: # ── roster resolution ──────────────────────────────────────────────────────── -def _hermes_root(home: Path) -> Path: - if home.parent.name == "profiles": - return home.parent.parent - return home - - -def _self_profile_name(home: Path) -> str: - if home.parent.name == "profiles": - return home.name - return "default" - - def _local_roster(root: Path) -> list[str]: """Profile names on this install: default + every named profile.""" - names = ["default"] - try: - profiles = root / "profiles" - if profiles.is_dir(): - for child in sorted(profiles.iterdir()): - if child.is_dir(): - names.append(child.name) - except Exception: - pass - return names - - -def _peers(root: Path) -> list[str]: - try: - from tools.bot_mode_probe import _peers as _probe_peers - - return _probe_peers(root) - except Exception: - return [] - - -def _handle(name: str) -> str: - return "hermes" if name == "default" else name + return [name for name, _dir in _roster(root)] def _resolve_local_name(target: str, roster: list[str]) -> Optional[str]: @@ -246,17 +203,13 @@ def message_agent_tool( ) -> str: """Deliver ``message`` to ``target``'s Bot Chat. Returns a JSON ack/error. - ``agent`` is the calling AIAgent (threaded by the executor) — used for - the Bot Chat gate, the sender identity, and the session key so the - spawned transport is tracked against the right session. + ``agent`` is the calling AIAgent (threaded by the executor) — used for the + Bot Chat gate, the sender identity, and the session key. """ - # ── defense-in-depth gate: only a canonical Bot Chat may deliver ── + # Defense-in-depth gate: only a canonical Bot Chat may deliver. home = _agent_home(agent) try: - from tools.bot_mode_probe import BOT_CHAT_TITLE, is_bot_mode_managed - - title = _session_title(agent) - if title != BOT_CHAT_TITLE: + if _session_title(agent) != BOT_CHAT_TITLE: return _err( "message_agent is only available in a Bot Mode 'Bot Chat' session. " "This session is not one; do not retry." @@ -291,7 +244,7 @@ def message_agent_tool( sender_handle = _handle(me) prefix = f"Message from 🤖 {sender_handle} (@{sender_handle}): " - # ── peer target: '/' or a bare registered peer name ── + # Peer target: '/' or a bare registered peer name. peer_match = _PEER_TARGET_RE.match(raw_target) bare_peer = raw_target.lower() if raw_target.lower() in peers else None if peer_match or bare_peer: @@ -303,23 +256,12 @@ def message_agent_tool( ) dm_target = f"{peer_name}/{peer_profile}" if peer_profile else peer_name label = f"@{peer_profile or peer_name} on peer '{peer_name}'" - # Pin the registry-owning profile (#93935): `hermes peer` resolves - # bot_peers through load_config(), which is profile-scoped — an - # unpinned subprocess inherits THIS gateway's profile context, so a - # secondary-profile bot's peer DM ran against an empty registry and - # died with "No peer named". The tool-side roster above reads the - # machine-root config (the default profile's home), so the CLI must - # run in that same profile to see the same registry. Mirrors the - # local-teammate path's `-p ` pin below. + # Pin the registry-owning profile: `hermes peer` resolves bot_peers via + # the profile-scoped load_config(), while the roster above reads the + # machine-root config — the CLI must run in that same profile or a + # secondary-profile bot sees an empty registry ("No peer named"). return _start_delivery( - [ - "hermes", - "-p", - _self_profile_name(root), - "peer", - "dm", - dm_target, - ], + ["hermes", "-p", _self_profile_name(root), "peer", "dm", dm_target], prefix + body, label, stdin_file=True, @@ -327,20 +269,21 @@ def message_agent_tool( agent=agent, ) - # ── local teammate ── + # Local teammate. if not _LOCAL_TARGET_RE.match(raw_target) and "@" not in raw_target: return _err(f"Invalid target: {raw_target!r}.", roster=teammates, peers=peers) resolved = _resolve_local_name(raw_target, roster) if _LOCAL_TARGET_RE.match(raw_target) else None - if resolved is None: - # ── cross-connection teammate (Desktop relay) ── - # Every gateway connected to the user's Desktop is reachable: the - # relay roster lists agents on the other connections; delivery rides - # the Desktop's own persistent socket to that gateway. + if resolved is None or resolved == me: + # Unknown locally, or same-name target on ANOTHER connection (this + # gateway's 'default' messaging the cloud 'default'): try the Desktop + # relay before reporting a resolution failure / self-message. relayed = _try_relay_delivery( root, raw_target, body, me, sender_handle, task_id=task_id, agent=agent ) if relayed is not None: return relayed + if resolved == me: + return _err("You can't message yourself. Pick a teammate from the roster.") return _err( f"No teammate named '{raw_target}' on this install, on a connected " "machine, or on a registered peer. Pick a name from the roster " @@ -348,30 +291,9 @@ def message_agent_tool( roster=teammates, peers=peers, ) - if resolved == me: - # Same-name target on ANOTHER connection (e.g. this gateway's - # 'default' messaging the cloud 'default') — try the relay before - # calling it a self-message. - relayed = _try_relay_delivery( - root, raw_target, body, me, sender_handle, task_id=task_id, agent=agent - ) - if relayed is not None: - return relayed - return _err("You can't message yourself. Pick a teammate from the roster.") return _start_delivery( - [ - "hermes", - "-p", - resolved, - "chat", - "--in", - "~", - "-c", - "Bot Chat", - "--create-if-missing", - "-Q", - ], + ["hermes", "-p", resolved, "chat", "--in", "~", "-c", "Bot Chat", "--create-if-missing", "-Q"], prefix + body, f"@{_handle(resolved)}", stdin_file=False, @@ -390,12 +312,11 @@ def _try_relay_delivery( task_id: Optional[str], agent: Any, ) -> Optional[str]: - """Cross-connection delivery via the Desktop relay, or None if the - target doesn't resolve against the relay roster. + """Cross-connection delivery via the Desktop relay; None when the target + doesn't resolve against the relay roster. - The envelope is queued on disk; the Desktop drains it over RPC and - delivers on the target connection's own socket. A background waiter is - spawned immediately so the relayed reply wakes the sender through the + The envelope is queued on disk for the Desktop to drain; a background waiter + is spawned immediately so the relayed reply wakes the sender through the standard completion-notification path — identical UX to a local DM. """ try: @@ -433,8 +354,7 @@ def _try_relay_delivery( ) except EnvelopeRefusedError as exc: # Fail fast: target definitively offline — nothing was queued. - # Structured refusal so the agent can distinguish it from a - # resolution error ('runtime_offline' per the #93091 reason enum). + # Structured 'runtime_offline' refusal, distinct from a resolution error. return json.dumps({"error": str(exc), "reason": exc.reason}) label = f"@{match['handle']} on {match['connection_label'] or match['connection_id']}" return _spawn_delivery( @@ -467,18 +387,16 @@ def _dm_dir() -> Path: def cleanup_bot_dm_cache( max_age_hours: float = _DM_STALE_SECONDS / 3600, *, now: float | None = None ) -> int: - """Delete orphaned DM payload files older than *max_age_hours*. + """Delete orphaned DM payload files older than *max_age_hours*; returns count. - Same contract as the other ``cleanup_*_cache`` helpers — returns the - number of files removed — so the gateway housekeeping loop can prune - this cache on the same hourly cadence as the media caches, even on - installs that never send another DM (the in-band sweep in - ``_write_dm_file`` only runs when a DM is written). + Same contract as the other ``cleanup_*_cache`` helpers so the gateway + housekeeping loop can prune hourly even on installs that never send another + DM (the in-band sweep in ``_write_dm_file`` only runs when a DM is written). """ cutoff = (time.time() if now is None else now) - max_age_hours * 3600 removed = 0 - # Include the legacy temp-root locations so upgrades clean files created - # by versions predating the dedicated directory. + # Legacy temp-root locations too, so upgrades clean files from versions + # predating the dedicated directory. temp_root = Path(tempfile.gettempdir()) locations: list[tuple[Path, str]] = [ (temp_root, "hermes-dm-*.txt"), @@ -502,14 +420,9 @@ def cleanup_bot_dm_cache( return removed -def _sweep_stale_dm_files(*, now: float | None = None) -> None: - """Best-effort cleanup for files orphaned before their runner started.""" - cleanup_bot_dm_cache(now=now) - - def _write_dm_file(content: str) -> str: """The message rides a temp file — never inline shell text.""" - _sweep_stale_dm_files() + cleanup_bot_dm_cache() fd, path = tempfile.mkstemp(prefix="dm-", suffix=".txt", dir=_dm_dir(), text=True) try: with os.fdopen(fd, "w", encoding="utf-8") as f: @@ -534,20 +447,17 @@ def _unlink_dm_file(path: str) -> None: def _delivery_lock(argv: list[str], *, stdin_file: bool): - """Per-profile turn lock context for a LOCAL teammate delivery (#93091). + """Per-profile turn lock context for a LOCAL teammate delivery. Local deliveries (``hermes -p chat …``) collide with relay - deliveries into the same profile — both run a Bot Chat turn on this - install — so the turn window is serialized on the shared cross-process - lock in ``tools.bot_relay``. Peer transports (stdin mode) run on the - remote gateway; their turn is locked THERE by its own deliver path. + deliveries into the same profile — both run a Bot Chat turn on this install — + so the turn window is serialized on the cross-process lock in + ``tools.bot_relay``. Peer transports (stdin mode) are locked on the remote + gateway by its own deliver path. """ - # The CLI element is matched by basename: local_delivery_command now - # resolves the venv-relative hermes next to this gateway's interpreter - # (#93590 — service contexts lack PATH), so argv[0] may be an absolute - # path (and on Windows carries the .exe suffix). Split on both - # separators so the shape matches regardless of which platform built - # the argv. + # Match the CLI element by basename: argv[0] may be an absolute venv path + # (service contexts lack PATH) and carries .exe on Windows; split on both + # separators so the shape matches whichever platform built the argv. cli = (argv[0] if argv else "").rsplit("\\", 1)[-1].rsplit("/", 1)[-1] if ( stdin_file @@ -558,8 +468,7 @@ def _delivery_lock(argv: list[str], *, stdin_file: bool): return contextlib.nullcontext() from tools.bot_relay import acquire_turn_lock - home = Path(os.getenv("HERMES_HOME") or os.path.expanduser("~/.hermes")) - return acquire_turn_lock(_hermes_root(home), argv[2]) + return acquire_turn_lock(_hermes_root(Path(_default_home())), argv[2]) def _run_delivery(argv: list[str], dm_file: str, *, stdin_file: bool) -> int: @@ -569,12 +478,11 @@ def _run_delivery(argv: list[str], dm_file: str, *, stdin_file: bool) -> int: cross-process lock, so two deliveries into one profile queue instead of racing; a bounded wait ends in a structured 'target_busy' refusal. - Local (query-file) turns get one policy-gated retry (#93091 item 5): - transient failures re-run the same session; a context_overflow re-run - lets the retried turn's pre-API compaction pass compact the Bot Chat - transcript first (agent/conversation_loop.py) — the sanctioned - compression lever; no fresh session is ever minted. Auth/quota/config - failures never retry. Peer transports (stdin mode) retry on their own + Local (query-file) turns get one policy-gated retry: transient failures + re-run the same session, and a context_overflow re-run lets the retried + turn's pre-API compaction compact the Bot Chat transcript first — the + sanctioned compression lever; no fresh session is ever minted. + Auth/quota/config failures never retry. Peer transports retry on their own gateway's deliver path, not here. """ try: @@ -584,13 +492,17 @@ def _run_delivery(argv: list[str], dm_file: str, *, stdin_file: bool) -> int: # after subprocess.run returns, not merely after stdin reaches EOF. with open(dm_file, "r", encoding="utf-8") as stream: return subprocess.run(argv, stdin=stream, check=False).returncode - proc = subprocess.run( - [*argv, "--query-file", dm_file], - check=False, - stdin=subprocess.DEVNULL, - capture_output=True, - text=True, - ) + + def _turn(): + return subprocess.run( + [*argv, "--query-file", dm_file], + check=False, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + ) + + proc = _turn() if proc.returncode != 0: from tools.bot_failure_reasons import ( RETRY_NONE, @@ -600,19 +512,11 @@ def _run_delivery(argv: list[str], dm_file: str, *, stdin_file: bool) -> int: detail = (proc.stderr or proc.stdout or "").strip()[-500:] if retry_action(classify_agent_error(detail)) != RETRY_NONE: - proc = subprocess.run( - [*argv, "--query-file", dm_file], - check=False, - stdin=subprocess.DEVNULL, - capture_output=True, - text=True, - ) - # Re-emit the transport's streams: stdout is the reply text the - # completion notification carries back to the sending agent. + proc = _turn() if proc.returncode != 0 and "already has a live owner" in (proc.stderr or ""): - # #100523: the target's Bot Chat is held live by another - # surface (Desktop). The turn never ran, so tell the sender - # plainly instead of leaking a raw lease error + exit code. + # The target's Bot Chat is held live by another surface (Desktop). + # The turn never ran — tell the sender plainly instead of leaking + # a raw lease error + exit code. who = argv[argv.index("-p") + 1] if "-p" in argv[:-1] else "the teammate" print(json.dumps({ "error": f"Delivery failed: @{who}'s Bot Chat is open on another " @@ -620,6 +524,8 @@ def _run_delivery(argv: list[str], dm_file: str, *, stdin_file: bool) -> int: "reason": "target_busy", })) return 1 + # Re-emit the transport's streams: stdout is the reply text the + # completion notification carries back to the sending agent. if proc.stdout: sys.stdout.write(proc.stdout) sys.stdout.flush() @@ -642,10 +548,9 @@ def _delivery_command(argv: list[str], dm_file: str, *, stdin_file: bool) -> str *argv, ] if sys.platform == "win32": - # The tracked local backend uses Git Bash on native Windows. Forward - # slashes preserve native drive paths while remaining executable by - # that shell; backslash-form paths are parsed as command names and die - # with exit 127 before this runner starts. + # The tracked local backend uses Git Bash on native Windows: forward + # slashes keep native drive paths executable there; backslash paths are + # parsed as command names and die with exit 127 before the runner starts. runner_argv = [part.replace("\\", "/") for part in runner_argv] return shlex.join(runner_argv) @@ -666,13 +571,7 @@ def _start_delivery( except BaseException: _unlink_dm_file(dm_file) raise - return _spawn_delivery( - command, - label, - dm_file=dm_file, - task_id=task_id, - agent=agent, - ) + return _spawn_delivery(command, label, dm_file=dm_file, task_id=task_id, agent=agent) def _spawn_delivery( @@ -685,9 +584,8 @@ def _spawn_delivery( ) -> str: """Launch the cleanup-owning runner and transfer file ownership on ack. - ``dm_file`` is None for relay deliveries: the waiter command watches a - reply file, and the envelope artifacts are owned and swept by - ``tools/bot_relay.py`` — there is no plaintext DM tempfile to reclaim. + ``dm_file`` is None for relay deliveries: the waiter watches a reply file and + the envelope artifacts are owned/swept by ``tools/bot_relay.py``. """ transferred = False try: @@ -710,8 +608,8 @@ def _spawn_delivery( return _err(f"Delivery to {label} failed to start: {parsed['error']}") if not proc_id: return _err(f"Delivery to {label} failed to start: no process id returned") - # From this point the background runner owns the file and removes it - # only after the local query-file or peer stdin consumer has finished. + # From here the background runner owns the file and removes it only + # after the local query-file or peer stdin consumer has finished. transferred = True return json.dumps( { @@ -745,12 +643,10 @@ def _delivery_main(args: list[str]) -> int: try: return _run_delivery(args[3:], dm_file, stdin_file=stdin_file) except Exception as exc: - # 'target_busy' extends the #93091 item-1 structured refusal enum: - # the queued delivery gave up after its bounded wait — surface the - # structured payload on stdout so the completion notification carries - # it back to the sending agent. - reason = getattr(exc, "reason", "") - if reason == "target_busy": + # 'target_busy': the queued delivery gave up after its bounded wait — + # surface the structured payload on stdout so the completion + # notification carries it back to the sending agent. + if getattr(exc, "reason", "") == "target_busy": print(json.dumps({"error": str(exc), "reason": "target_busy"})) return 1 print( @@ -766,13 +662,12 @@ def _delivery_main(args: list[str]) -> int: def _agent_home(agent: Any) -> str: """The calling agent's OWN home (session-db derived), not ambient env.""" try: - sdb = getattr(agent, "_session_db", None) - db_path = getattr(sdb, "db_path", None) + db_path = getattr(getattr(agent, "_session_db", None), "db_path", None) if db_path: return str(Path(db_path).parent) except Exception: pass - return os.getenv("HERMES_HOME") or os.path.expanduser("~/.hermes") + return _default_home() def _session_title(agent: Any) -> str: diff --git a/tools/bot_mode_probe.py b/tools/bot_mode_probe.py index ca92c6f26a..0d84ef454c 100644 --- a/tools/bot_mode_probe.py +++ b/tools/bot_mode_probe.py @@ -1,30 +1,15 @@ """Bot Mode roster probe — canonical Bot Chat system prompt section. -When the desktop's Bot Mode manages this install (any profile carries a -``ui_meta['hermes-bots']`` block in its profile.yaml), a bot's canonical -"Bot Chat" session — and ONLY that session — gets a short "Messaging other -agents" section so the bot can receive teammate DMs, reply with attribution, -and hand off @mentions. Regular sessions never carry the section; the -desktop's composer middleware owns the @mention send path there. +When any profile on this install carries ``ui_meta['hermes-bots']`` in its +profile.yaml (Bot-Mode-managed), a bot's canonical "Bot Chat" session — and ONLY +that session (the caller, agent/system_prompt.py, enforces the title gate against +``BOT_CHAT_TITLE``) — gets a "Messaging other agents" section. Silent (``""``) +when no profile is managed, when the profile's SOUL.md already carries the +heading (legacy plugin-appended text must never double up), or on any error — +a prompt build must never crash. Cached per (process, home) so compression- +triggered rebuilds produce identical bytes. Toggle: ``agent.bot_mode_protocol``. -The caller (agent/system_prompt.py) enforces the session-title gate against -``BOT_CHAT_TITLE``; this module answers "is this install Bot-Mode-managed, -and what should the section say for this profile". - -This replaces the plugin-side SOUL.md backfill: the protocol is injected by -the core at prompt-build time instead of appended to user-authored SOUL -files. If the profile's SOUL.md already carries the section (created by an -older plugin version), the probe stays silent so the text never doubles up. - -Silent (returns ``""``) when: -- no profile on this install is Bot-Mode-managed (the dominant case), -- the current profile's SOUL.md already contains the protocol heading, -- anything at all goes wrong (never crash a prompt build). - -Deterministic within a process: the result is computed once and cached, so -compression-triggered prompt rebuilds produce identical bytes. - -Toggle via ``agent.bot_mode_protocol`` in config.yaml (default True). +Also hosts the path/roster helpers shared by ``bot_mode_dm`` and ``bot_relay``. """ from __future__ import annotations @@ -35,76 +20,94 @@ from pathlib import Path _PROTOCOL_HEADING = "## Messaging other agents" -# The canonical per-bot conversation title — the only session shape that -# receives the protocol section. Must match the desktop plugin's -# createCanonicalChat title and the `-c "Bot Chat"` resume target. +# The only session title that receives the protocol section. Must match the +# desktop plugin's createCanonicalChat title and the `-c "Bot Chat"` resume target. BOT_CHAT_TITLE = "Bot Chat" _lock = threading.Lock() _cached: dict[str, str] = {} +# ── shared path / roster helpers ───────────────────────────────────────────── + + +def _default_home() -> str: + """Ambient HERMES_HOME (env, else ~/.hermes) as a string.""" + return os.getenv("HERMES_HOME") or os.path.expanduser("~/.hermes") + + +def _resolve_home(home: str | os.PathLike | None) -> Path: + return Path(str(home) if home else _default_home()) + + def _hermes_root(home: Path) -> Path: """Root ~/.hermes for both the default profile and named profiles.""" - if home.parent.name == "profiles": - return home.parent.parent - return home + return home.parent.parent if home.parent.name == "profiles" else home def _profile_name(home: Path) -> str: - if home.parent.name == "profiles": - return home.name - return "default" + return home.name if home.parent.name == "profiles" else "default" -def _is_bot_managed(profile_dir: Path) -> bool: - """True when profile.yaml carries a ui_meta['hermes-bots'] block. - - Cheap substring check before the YAML parse keeps the silent path fast. - """ - meta = profile_dir / "profile.yaml" - try: - if not meta.is_file(): - return False - raw = meta.read_text(encoding="utf-8", errors="replace") - if "hermes-bots" not in raw: - return False - import yaml - - data = yaml.safe_load(raw) - ui_meta = data.get("ui_meta") if isinstance(data, dict) else None - return isinstance(ui_meta, dict) and isinstance(ui_meta.get("hermes-bots"), dict) - except Exception: - return False +def _handle(name: str) -> str: + # The mention middleware aliases the default profile as @hermes. + return "hermes" if name == "default" else name def _roster(root: Path) -> list[tuple[str, Path]]: - """(name, dir) for the default profile + every named profile.""" + """(name, dir) for the default profile + every named profile, sorted.""" entries: list[tuple[str, Path]] = [("default", root)] try: profiles = root / "profiles" if profiles.is_dir(): - for child in sorted(profiles.iterdir()): - if child.is_dir(): - entries.append((child.name, child)) + entries.extend((c.name, c) for c in sorted(profiles.iterdir()) if c.is_dir()) except Exception: pass return entries +def _read_yaml_dict(path: Path, needle: str | None = None) -> dict | None: + """YAML mapping at ``path``, or None when missing / not a mapping / unreadable. + + ``needle``: cheap substring precheck that skips the YAML parse on the + dominant (unmanaged) path — the key is absent from most installs. + """ + try: + if not path.is_file(): + return None + raw = path.read_text(encoding="utf-8", errors="replace") + if needle is not None and needle not in raw: + return None + import yaml + + data = yaml.safe_load(raw) + return data if isinstance(data, dict) else None + except Exception: + return None + + +def _bots_meta(data: dict | None) -> dict | None: + """The ``ui_meta['hermes-bots']`` block of a parsed profile.yaml, if a dict.""" + ui_meta = data.get("ui_meta") if data else None + bots = ui_meta.get("hermes-bots") if isinstance(ui_meta, dict) else None + return bots if isinstance(bots, dict) else None + + +def _is_bot_managed(profile_dir: Path) -> bool: + """True when profile.yaml carries a ui_meta['hermes-bots'] block.""" + return _bots_meta(_read_yaml_dict(profile_dir / "profile.yaml", "hermes-bots")) is not None + + def is_bot_mode_managed(home: str | os.PathLike | None = None) -> bool: """True when ANY profile on this install is Bot-Mode-managed. - The tool-injection gate for ``message_agent`` — deliberately independent - of :func:`get_bot_mode_protocol_section`'s emptiness: a profile whose - SOUL.md carries the legacy plugin-appended protocol gets an empty - section (text dedupe) but must still get the tool. Never raises. + The tool-injection gate for ``message_agent`` — deliberately independent of + :func:`get_bot_mode_protocol_section`'s emptiness: a profile whose SOUL.md + carries the legacy protocol gets an empty section but must still get the + tool. Never raises. """ try: - resolved = Path( - str(home) if home else (os.getenv("HERMES_HOME") or os.path.expanduser("~/.hermes")) - ) - root = _hermes_root(resolved) + root = _hermes_root(_resolve_home(home)) return any(_is_bot_managed(d) for _n, d in _roster(root)) except Exception: return False @@ -118,40 +121,20 @@ def _soul_has_protocol(profile_dir: Path) -> bool: return False -def _handle(name: str) -> str: - # The mention middleware aliases the default profile as @hermes. - return "hermes" if name == "default" else name - - def _profile_role(profile_dir: Path) -> str: - """A teammate's role line: Bot Mode title, else profile description. - - The ui_meta['hermes-bots'].title is the name the user gave the bot in - Bot Mode; profile.yaml's description is the profile's stated purpose. - Either one tells a teammate WHO to message for a given job. Bounded and - single-line; empty when neither exists. Never raises. - """ - meta = profile_dir / "profile.yaml" + """Teammate role line: Bot Mode title — profile description (either may be + absent). Single-line, ≤160 chars, "" when neither exists. Never raises.""" try: - if not meta.is_file(): + data = _read_yaml_dict(profile_dir / "profile.yaml") + if not data: return "" - raw = meta.read_text(encoding="utf-8", errors="replace") - import yaml - - data = yaml.safe_load(raw) - if not isinstance(data, dict): - return "" - parts = [] - ui_meta = data.get("ui_meta") - if isinstance(ui_meta, dict) and isinstance(ui_meta.get("hermes-bots"), dict): - title = str(ui_meta["hermes-bots"].get("title") or "").strip() - if title: - parts.append(title) - description = str(data.get("description") or "").strip() - if description: - parts.append(description) - line = " — ".join(parts) - return " ".join(line.split())[:160] + bots = _bots_meta(data) or {} + parts = [ + p + for p in (str(bots.get("title") or "").strip(), str(data.get("description") or "").strip()) + if p + ] + return " ".join(" — ".join(parts).split())[:160] except Exception: return "" @@ -163,28 +146,19 @@ def _roster_lines(root: Path, me: str) -> list[str]: if name == me: continue role = _profile_role(profile_dir) - handle = _handle(name) - lines.append(f"- `@{handle}`" + (f" — {role}" if role else "")) + lines.append(f"- `@{_handle(name)}`" + (f" — {role}" if role else "")) return lines def _peers(root: Path) -> list[str]: - """Registered peer gateway names (``hermes peer``), for the protocol text. + """Registered peer gateway names (``hermes peer``) from config.yaml. - Reads config.yaml directly (cheap, no config-loader import) — the section - is optional and absent on most installs. Never raises. + Read directly (no config-loader import — cheap; the section is absent on + most installs). Never raises. """ try: - cfg_path = root / "config.yaml" - if not cfg_path.is_file(): - return [] - raw = cfg_path.read_text(encoding="utf-8", errors="replace") - if "bot_peers" not in raw: - return [] - import yaml - - data = yaml.safe_load(raw) - peers = data.get("bot_peers") if isinstance(data, dict) else None + data = _read_yaml_dict(root / "config.yaml", "bot_peers") + peers = data.get("bot_peers") if data else None if not isinstance(peers, dict): return [] return sorted(str(name) for name in peers if str(name).strip()) @@ -193,14 +167,8 @@ def _peers(root: Path) -> list[str]: def _remote_paragraph(root: Path) -> str: - """Protocol addendum for agents on OTHER connected machines. - - Fed by the Desktop relay roster (``tools/bot_relay.py``) — every gateway - connected to the user's Desktop (local, remote URL, SSH, Hermes Cloud, - docker) syncs its agents here, so bots can DM across machines with the - same message_agent tool. Only rendered when the relay roster is - non-empty. - """ + """Addendum for agents on OTHER connected machines (Desktop relay roster, + ``tools/bot_relay.py``). Rendered only when the relay roster is non-empty.""" try: from tools.bot_relay import read_remote_roster, remote_target_forms @@ -225,7 +193,7 @@ def _remote_paragraph(root: Path) -> str: def _peer_paragraph(root: Path) -> str: - """Protocol addendum for cross-machine DMs — only when peers exist.""" + """Addendum for cross-machine DMs — only when peers exist.""" peers = _peers(root) if not peers: return "" @@ -243,12 +211,10 @@ def _build_section(home: Path) -> str: root = _hermes_root(home) me = _profile_name(home) - roster = _roster(root) - if not any(_is_bot_managed(d) for _n, d in roster): + if not any(_is_bot_managed(d) for _n, d in _roster(root)): return "" - # An older plugin build may have appended the protocol to SOUL.md - # already — never double it up. + # An older plugin build may have appended the protocol to SOUL.md — never double it. my_dir = home if me == "default" else root / "profiles" / me if _soul_has_protocol(my_dir): return "" @@ -288,11 +254,11 @@ def _build_section(home: Path) -> str: def get_bot_mode_protocol_section(home: str | os.PathLike | None = None, *, force_refresh: bool = False) -> str: """Cached probe entry point — one filesystem pass per (process, home). - ``home`` should be the AGENT'S OWN resolved home (session-db derived), - not the ambient HERMES_HOME — build threads can lose the ContextVar - override and the env var would then name the wrong profile. + ``home`` should be the AGENT'S OWN resolved home (session-db derived), not + the ambient HERMES_HOME — build threads can lose the ContextVar override + and the env var would then name the wrong profile. """ - resolved = str(home) if home else (os.getenv("HERMES_HOME") or os.path.expanduser("~/.hermes")) + resolved = str(_resolve_home(home)) with _lock: if force_refresh or resolved not in _cached: try: @@ -304,17 +270,12 @@ def get_bot_mode_protocol_section(home: str | os.PathLike | None = None, *, forc # ── capability epoch ───────────────────────────────────────────────────────── # -# Bot Chat sessions are effectively eternal — the "new sessions come along -# often" assumption behind build-once system prompts does not hold. When the -# user changes a bot's capabilities (skills, toolsets, MCP servers, SOUL) or -# the teammate roster changes, they expect the change to work on the NEXT -# message. The fingerprint below hashes exactly that capability surface; the -# built Bot Chat prompt embeds it, and the restore path in -# agent/conversation_loop.py rebuilds the prompt when the stored epoch no -# longer matches the disk state. This is the /model exception applied to -# capabilities: a LOUD, USER-INITIATED, once-per-change cache break — never -# a per-turn drift (unchanged state hashes identically and the stored bytes -# are reused verbatim). +# Bot Chat sessions are effectively eternal, so "build the prompt once" would +# strand capability changes (skills, toolsets, MCP, SOUL, roster, peers) forever. +# The fingerprint hashes exactly that surface; the built Bot Chat prompt embeds +# it and agent/conversation_loop.py rebuilds only when the stored epoch differs +# from disk — a loud, user-initiated, once-per-change cache break, never per-turn +# drift (unchanged state hashes identically; stored bytes are reused verbatim). _EPOCH_PREFIX = "Capability epoch: " _EPOCH_RE_TEXT = r"Capability epoch: ([0-9a-f]{12})" @@ -323,16 +284,13 @@ _EPOCH_RE_TEXT = r"Capability epoch: ([0-9a-f]{12})" def capability_fingerprint(home: str | os.PathLike | None = None) -> str: """12-hex digest of the capability surface for ``home``'s profile. - Sources: the profile's disabled skills + enabled toolsets + MCP server - config (config.yaml), SOUL.md bytes, installed skill names, and the - Bot-Mode roster (managed profile names). Deliberately NOT cached — the - whole point is detecting on-disk drift; callers compare it against the - epoch embedded in a stored prompt. Never raises. + Deliberately NOT cached — the point is detecting on-disk drift against the + epoch embedded in a stored prompt. Never raises ("unavailable" on failure). """ import hashlib import json - resolved = Path(str(home) if home else (os.getenv("HERMES_HOME") or os.path.expanduser("~/.hermes"))) + resolved = _resolve_home(home) surface: dict = {} try: # Canonical loader (managed overlay + env expansion + normalization), @@ -347,8 +305,6 @@ def capability_fingerprint(home: str | os.PathLike | None = None) -> str: reset_hermes_home_override(token) skills_cfg = cfg.get("skills") if isinstance(cfg.get("skills"), dict) else {} tools_cfg = cfg.get("tools") if isinstance(cfg.get("tools"), dict) else {} - skills_cfg = skills_cfg or {} - tools_cfg = tools_cfg or {} surface["disabled_skills"] = sorted(str(s).lower() for s in (skills_cfg.get("disabled") or [])) surface["enabled_toolsets"] = sorted(str(t) for t in (tools_cfg.get("enabled_toolsets") or [])) mcp = cfg.get("mcp_servers") @@ -361,45 +317,39 @@ def capability_fingerprint(home: str | os.PathLike | None = None) -> str: except Exception: surface["soul"] = "" try: - names = [] skills_root = resolved / "skills" - if skills_root.is_dir(): - for skill_md in skills_root.glob("**/SKILL.md"): - names.append(str(skill_md.parent.relative_to(skills_root))) + names = ( + [str(p.parent.relative_to(skills_root)) for p in skills_root.glob("**/SKILL.md")] + if skills_root.is_dir() + else [] + ) surface["skills"] = sorted(names) except Exception: surface["skills"] = [] + root = _hermes_root(resolved) try: - root = _hermes_root(resolved) - surface["roster"] = sorted(n for n, d in _roster(root) if _is_bot_managed(d)) - # Roles are part of the messaging surface: renaming a bot or editing - # a profile description must refresh eternal Bot Chat prompts so the - # roster block teammates pick recipients from stays current. - surface["roster_roles"] = sorted( - f"{n}:{_profile_role(d)}" for n, d in _roster(root) - ) + roster = _roster(root) + surface["roster"] = sorted(n for n, d in roster if _is_bot_managed(d)) + # Roles are part of the messaging surface: renaming a bot or editing a + # description must refresh the roster block teammates pick recipients from. + surface["roster_roles"] = sorted(f"{n}:{_profile_role(d)}" for n, d in roster) except Exception: surface["roster"] = [] - # Protocol-text version salt: bumping this refreshes every eternal Bot - # Chat prompt ONCE so existing bots adopt a new protocol section (e.g. - # the v2 message_agent tool replacing the shellout instructions). + # Protocol-text version salt: bumping it refreshes every eternal Bot Chat + # prompt ONCE so existing bots adopt a new protocol section. surface["protocol_version"] = 2 + # Peer gateways and the Desktop relay roster are part of the messaging + # surface too: registering a peer or (dis)connecting a machine must show up + # on the next message. try: - # Peer gateways are part of the messaging surface: registering one - # must refresh eternal Bot Chat prompts so the cross-machine DM - # paragraph appears on the next message. - surface["peers"] = _peers(_hermes_root(resolved)) + surface["peers"] = _peers(root) except Exception: surface["peers"] = [] try: - # The Desktop relay roster is part of the messaging surface too: - # connecting/disconnecting a machine, or agents appearing on one, - # must refresh eternal Bot Chat prompts the same way. from tools.bot_relay import read_remote_roster surface["remote_roster"] = sorted( - f"{r['connection_id']}:{r['profile']}:{r['title']}" - for r in read_remote_roster(_hermes_root(resolved)) + f"{r['connection_id']}:{r['profile']}:{r['title']}" for r in read_remote_roster(root) ) except Exception: surface["remote_roster"] = [] @@ -416,13 +366,9 @@ def epoch_line(home: str | os.PathLike | None = None) -> str: def stored_prompt_capability_stale(stored_prompt: str, home: str | os.PathLike | None = None) -> bool: - """True when ``stored_prompt`` is a Bot Chat prompt whose embedded - capability epoch no longer matches the current disk state. - - Non-Bot-Chat prompts (no epoch stamp) are never stale by this check. - Fails closed to "not stale" — a broken probe must never turn into a - rebuild-every-turn cache burner. - """ + """True when ``stored_prompt`` is a Bot Chat prompt whose embedded epoch no + longer matches disk. Unstamped prompts are never stale. Fails closed to + "not stale" — a broken probe must not become a rebuild-every-turn cache burner.""" import re try: @@ -438,27 +384,19 @@ def stored_prompt_capability_stale(stored_prompt: str, home: str | os.PathLike | def stored_bot_chat_prompt_needs_upgrade(stored_prompt: str, home: str | os.PathLike | None = None) -> bool: - """True when a Bot Chat session's stored prompt PREDATES this feature. + """True when a Bot Chat session's stored prompt PREDATES the epoch mechanism. - Legacy Bot Chats (created before bundling / this epoch mechanism) - persisted prompts with no protocol section and no epoch stamp; without - an explicit upgrade they would be stranded forever — the staleness check - above only fires on stamped prompts. This is a one-time migration per - legacy session: the caller must only invoke it for sessions titled - "Bot Chat", and only rebuilds when the probe would actually emit a - section (a profile whose SOUL.md already carries the legacy plugin-side - append keeps its protocol-free prompt — rebuilding those would loop, - since the probe stays silent and the rebuilt prompt would be unstamped - again). Fails closed to "no upgrade". + Legacy prompts carry neither protocol section nor epoch stamp, so the + staleness check (stamped prompts only) would strand them forever. One-time + migration: the caller must only ask for sessions titled "Bot Chat", and we + rebuild only when the probe would actually emit a section — a SOUL.md that + already carries the legacy protocol yields an empty section, and rebuilding + would produce another unstamped prompt and loop. Fails closed to "no upgrade". """ try: - if _EPOCH_PREFIX in (stored_prompt or ""): + text = stored_prompt or "" + if _EPOCH_PREFIX in text or _PROTOCOL_HEADING in text: return False - if _PROTOCOL_HEADING in (stored_prompt or ""): - return False - # Only upgrade when the rebuild would actually add the section — - # this is what guarantees the rebuilt prompt carries a stamp and - # the upgrade can never re-fire. return bool(get_bot_mode_protocol_section(home)) except Exception: return False diff --git a/tools/bot_relay.py b/tools/bot_relay.py index bc74a5952e..e4d4272d84 100644 --- a/tools/bot_relay.py +++ b/tools/bot_relay.py @@ -1,33 +1,22 @@ """Bot Mode cross-connection relay — connections ARE the peer set. -Every gateway connected to the user's Desktop (local, remote URL, SSH, -Hermes Cloud, docker) is a persistent line. This module is the gateway-side -half of the relay that rides those lines so agents on ANY connected gateway -can find and message agents on ANY other, with `message_agent` as the one -send path (Teknium ruling, Aug 2026 — the peers-vs-connections split was -itself the bug). +Gateway-side half of the relay that lets agents on ANY Desktop-connected +gateway message agents on ANY other, with ``message_agent`` as the one send +path. Plain file plumbing under ``/bot_relay/`` — no network; the Desktop +owns every socket and does all cross-connection I/O: -How the relay works (three files under ``/bot_relay/``): +- ``roster.json`` — union roster of agents on OTHER connections, pushed by the + Desktop (``bot_relay.roster.sync``); folded into the Bot Chat protocol section + and used to resolve cross-connection targets. +- ``outbox/`` — envelopes queued by ``message_agent``; the Desktop drains them + (``bot_relay.outbox.drain``) and delivers on the target connection. +- ``replies/`` — one JSON per envelope (``bot_relay.reply``); a background + waiter spawned at send time watches it so the reply wakes the sender via the + same completion-notification path local DMs use. -- ``roster.json`` — the union roster of agents on OTHER connections, pushed - by the Desktop over each connection's WebSocket (``bot_relay.roster.sync``). - ``tools/bot_mode_probe.py`` folds it into the Bot Chat protocol section so - every bot knows every reachable teammate, and ``message_agent`` resolves - cross-connection targets against it. -- ``outbox/`` — envelopes queued by ``message_agent`` for targets that live - on another connection. The Desktop drains them (``bot_relay.outbox.drain``) - and delivers each to the target connection (``bot_relay.deliver``). -- ``replies/`` — one JSON per envelope, written when the Desktop relays the - target agent's reply back (``bot_relay.reply``). A background waiter - spawned at send time watches for it, so the reply wakes the sender through - the exact same completion-notification path local DMs already use. - -The gateway never holds another connection's credentials; the Desktop owns -every socket and does all cross-connection I/O. Everything here is plain -file plumbing on the gateway's own HERMES root — no network. The public -helpers never raise, with one deliberate exception: ``enqueue_envelope`` -raises ``EnvelopeRefusedError`` when the target is definitively offline, so -the sender fails fast instead of queueing a DM nobody will drain (#93091). +Public helpers never raise, except ``enqueue_envelope`` → ``EnvelopeRefusedError`` +when the target is definitively offline (fail fast instead of queueing a DM +nobody will drain). """ from __future__ import annotations @@ -46,6 +35,8 @@ import uuid from pathlib import Path from typing import Any, Iterator, Optional +from tools.bot_mode_probe import _default_home, _hermes_root + logger = logging.getLogger(__name__) RELAY_DIR_NAME = "bot_relay" @@ -55,36 +46,34 @@ CLAIMED_DIR = "claimed" REPLIES_DIR = "replies" LOCKS_DIR = "locks" -# Fallback wait budget for a queued delivery turn when config is unreadable. -# The real knob is ``bot_mode.turn_wait_seconds`` in config.yaml. +# Fallback wait budget for a queued delivery turn when config is unreadable +# (real knob: ``bot_mode.turn_wait_seconds``). TURN_WAIT_SECONDS_FALLBACK = 120 -# A reply must arrive before the waiter gives up. Cross-connection turns can -# be slow (remote model, cold gateway) — generous, but bounded. +# Waiter give-up budget. Cross-connection turns can be slow (remote model, +# cold gateway) — generous, but bounded. REPLY_WAIT_SECONDS = 900 -# Envelopes and replies older than this are stale artifacts (Desktop was -# closed, connection died) and are swept opportunistically. +# Envelopes/replies older than this are stale artifacts (Desktop closed, +# connection died) and are swept opportunistically. STALE_AFTER_SECONDS = 6 * 3600 # Fallback envelope TTL when config is unreachable — mirrors the -# ``bot_mode.envelope_ttl_seconds`` default in hermes_cli/config_defaults.py. -# Envelopes older than the TTL are refused at drain time with a -# 'queued_expired' error reply instead of being delivered late. +# ``bot_mode.envelope_ttl_seconds`` default. Older envelopes are refused at +# drain time with a 'queued_expired' error reply instead of delivered late. DEFAULT_ENVELOPE_TTL_SECONDS = 900 # A roster older than this proves nothing about who is offline: the Desktop -# pushes roster.sync on connection-state changes, so only a recently-written -# roster is treated as an authoritative view for the fail-fast check. +# re-pushes roster.sync on connection-state changes, so only a recent roster +# is authoritative for the fail-fast check. ROSTER_FRESH_SECONDS = 600 class EnvelopeRefusedError(RuntimeError): """``enqueue_envelope`` refused to queue — nothing was written to disk. - ``reason`` is a stable machine code; ``str(exc)`` is the human text. - 'runtime_offline' matches the #93091 item-1 failure-reason enum (plain - literal here so the branches merge cleanly). + ``reason`` is a stable machine code ('runtime_offline'); ``str(exc)`` is the + human text. """ def __init__(self, reason: str, message: str): @@ -105,16 +94,29 @@ def _ensure_dirs(root: Path | str) -> Path: return base +def _atomic_write_json(target: Path, payload: Any, *, prefix: str, sort_keys: bool = False) -> None: + """Write ``payload`` to ``target`` via tempfile + os.replace (readers never see + a partial file). The tempfile is removed if the write fails.""" + fd, tmp = tempfile.mkstemp(dir=str(target.parent), prefix=prefix, suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False, sort_keys=sort_keys) + os.replace(tmp, target) + except Exception: + try: + os.unlink(tmp) + except OSError: + pass + raise + + # ── remote roster ──────────────────────────────────────────────────────────── def _normalize_roster_row(row: Any) -> Optional[dict]: """Validated, minimal roster row or None. - Rows come from the Desktop over RPC — treat as untrusted input. A row - names an agent on another connection: profile name, taggable handle, - the connection id/label of the gateway that owns it, and optional - friendly title/description for the protocol section. + Rows come from the Desktop over RPC — treat as untrusted input. """ if not isinstance(row, dict): return None @@ -139,9 +141,8 @@ def _normalize_roster_row(row: Any) -> Optional[dict]: "title": str(row.get("title") or "").strip()[:120], "description": " ".join(str(row.get("description") or "").split())[:160], } - # Optional explicit liveness flag (additive — the Desktop may push it). - # Preserved only when it is a real bool so absent stays distinguishable - # from false: absent == liveness unknown == fail-open on enqueue. + # Optional liveness flag, kept only when a real bool so absent stays + # distinguishable from false: absent == unknown == fail-open on enqueue. if isinstance(row.get("online"), bool): out["online"] = row["online"] return out @@ -150,31 +151,14 @@ def _normalize_roster_row(row: Any) -> Optional[dict]: def write_remote_roster(root: Path | str, rows: Any) -> int: """Atomically persist the Desktop-pushed remote roster. Returns count.""" base = _ensure_dirs(root) - cleaned: list[dict] = [] - seen: set[tuple[str, str]] = set() + by_key: dict[tuple[str, str], dict] = {} for row in rows if isinstance(rows, list) else []: norm = _normalize_roster_row(row) - if not norm: - continue - key = (norm["connection_id"], norm["profile"]) - if key in seen: - continue - seen.add(key) - cleaned.append(norm) - cleaned.sort(key=lambda r: (r["connection_id"], r["profile"])) + if norm: + by_key.setdefault((norm["connection_id"], norm["profile"]), norm) + cleaned = [by_key[k] for k in sorted(by_key)] payload = {"updated_at": int(time.time()), "agents": cleaned} - target = base / ROSTER_FILE - fd, tmp = tempfile.mkstemp(dir=str(base), prefix=".roster-", suffix=".tmp") - try: - with os.fdopen(fd, "w", encoding="utf-8") as f: - json.dump(payload, f, ensure_ascii=False, sort_keys=True) - os.replace(tmp, target) - except Exception: - try: - os.unlink(tmp) - except OSError: - pass - raise + _atomic_write_json(base / ROSTER_FILE, payload, prefix=".roster-", sort_keys=True) return len(cleaned) @@ -214,18 +198,15 @@ def resolve_remote_target(raw_target: str, roster: list[dict]) -> Any: conn = conn.strip() if not want or not conn: return None - matches = [] - for row in roster: - if want.lower() not in (row["handle"].lower(), row["profile"].lower()): - continue - if conn and row["connection_id"].lower() != conn.lower(): - continue - matches.append(row) + matches = [ + row + for row in roster + if want.lower() in (row["handle"].lower(), row["profile"].lower()) + and (not conn or row["connection_id"].lower() == conn.lower()) + ] if not matches: return None - if len(matches) > 1: - return "ambiguous" - return matches[0] + return matches[0] if len(matches) == 1 else "ambiguous" def remote_target_forms(roster: list[dict]) -> list[str]: @@ -233,26 +214,19 @@ def remote_target_forms(roster: list[dict]) -> list[str]: by_handle: dict[str, int] = {} for row in roster: by_handle[row["handle"].lower()] = by_handle.get(row["handle"].lower(), 0) + 1 - forms = [] - for row in roster: - if by_handle[row["handle"].lower()] > 1: - forms.append(f"{row['handle']}@{row['connection_id']}") - else: - forms.append(row["handle"]) - return forms + return [ + f"{row['handle']}@{row['connection_id']}" if by_handle[row["handle"].lower()] > 1 else row["handle"] + for row in roster + ] # ── outbox / replies ───────────────────────────────────────────────────────── def _envelope_ttl_seconds() -> int: - """Configured drain TTL (``bot_mode.envelope_ttl_seconds``), lazily read. - - tools/ must not pull heavy CLI config at import time, so the read happens - per-drain and falls back to ``DEFAULT_ENVELOPE_TTL_SECONDS`` when config - is unavailable (tests, stripped installs). ``0`` (or negative) disables - drain-time expiry. - """ + """Configured drain TTL (``bot_mode.envelope_ttl_seconds``), read per-drain + (tools/ must not import CLI config at import time); falls back to + ``DEFAULT_ENVELOPE_TTL_SECONDS``. ``0`` (or negative) disables expiry.""" try: from hermes_cli.config import load_config_readonly @@ -268,18 +242,10 @@ def _envelope_ttl_seconds() -> int: def _target_liveness(root: Path | str, target: dict) -> Optional[bool]: """Tri-state liveness for ``target``: True / False / None (unknown). - Roster rows carry no heartbeat today, so 'definitively offline' is keyed - off the two signals roster.json actually gives us: - - - an explicit ``online: false`` on the target's row (additive field, - honored when the Desktop starts pushing it); - - the target's (connection_id, profile) being ABSENT from a *fresh* - roster — the Desktop re-pushes the whole roster on connection-state - changes, so a recently-synced roster that dropped the target means its - connection is gone. - - A missing, unreadable, or stale (older than ``ROSTER_FRESH_SECONDS``) - roster proves nothing → None, and callers fail open. Never raises. + 'Definitively offline' = explicit ``online: false`` on the row, or the + target ABSENT from a *fresh* roster (the Desktop re-pushes the whole roster + on connection-state changes). A missing, unreadable, empty or stale roster + proves nothing → None, and callers fail open. Never raises. """ try: roster_path = relay_root(root) / ROSTER_FILE @@ -315,16 +281,14 @@ def enqueue_envelope( ) -> dict: """Queue a cross-connection DM for the Desktop relay. Returns envelope. - Raises ``EnvelopeRefusedError`` (reason ``'runtime_offline'``) instead of - writing the outbox file when the target is definitively offline per - ``_target_liveness``. Unknown liveness enqueues as before (fail-open). + Raises ``EnvelopeRefusedError`` ('runtime_offline') without writing when the + target is definitively offline; unknown liveness enqueues (fail-open). """ if _target_liveness(root, target) is False: label = ( f"@{target.get('handle') or target.get('profile') or '?'} on " f"{target.get('connection_label') or target.get('connection_id') or '?'}" ) - # 'runtime_offline' matches the #93091 item-1 reason enum. raise EnvelopeRefusedError( "runtime_offline", f"{label} is offline right now — the message was NOT queued. " @@ -341,11 +305,7 @@ def enqueue_envelope( "target_handle": target["handle"], "message": message, } - path = base / OUTBOX_DIR / f"{envelope['id']}.json" - fd, tmp = tempfile.mkstemp(dir=str(base / OUTBOX_DIR), prefix=".env-", suffix=".tmp") - with os.fdopen(fd, "w", encoding="utf-8") as f: - json.dump(envelope, f, ensure_ascii=False) - os.replace(tmp, path) + _atomic_write_json(base / OUTBOX_DIR / f"{envelope['id']}.json", envelope, prefix=".env-") return envelope @@ -354,8 +314,8 @@ def claim_pending_envelopes(root: Path | str) -> list[dict]: deliver). Sweeps stale claimed/reply artifacts opportunistically. Envelopes older than ``bot_mode.envelope_ttl_seconds`` are NOT delivered: - each gets an error reply (reason ``'queued_expired'``) so the sender's - waiter resolves, and its outbox file is removed (#93091 item 2). + each gets a 'queued_expired' error reply (so the sender's waiter resolves) + and its outbox file is removed. """ base = _ensure_dirs(root) _sweep_stale(base) @@ -373,7 +333,6 @@ def claim_pending_envelopes(root: Path | str) -> list[dict]: expired = True handle = str(env.get("target_handle") or "?") conn = str(env.get("target_connection") or "?") - # 'queued_expired' matches the #93091 item-1 reason enum. write_reply( root, str(env.get("id") or ""), @@ -385,9 +344,8 @@ def claim_pending_envelopes(root: Path | str) -> list[dict]: reason="queued_expired", ) except (OSError, ValueError): - # Unreadable envelope or invalid id: if it already counted as - # expired, still remove it below; otherwise let the normal - # claim attempt below deal with it. + # Unreadable envelope / invalid id: still removed below if it + # already counted as expired, else the claim attempt handles it. pass if expired: try: @@ -409,10 +367,8 @@ def write_reply( ) -> Path: """Persist the relayed reply (or delivery error) for the waiter. - ``reason`` is an optional typed failure code (see - ``tools.bot_failure_reasons``, e.g. 'queued_expired'); when omitted and - ``error`` is non-empty it is classified from the error text. The waiter - only surfaces the human ``error``. + ``reason`` is an optional typed failure code (``tools.bot_failure_reasons``); + when omitted and ``error`` is non-empty it is classified from the text. """ base = _ensure_dirs(root) safe = str(envelope_id or "").strip() @@ -425,17 +381,8 @@ def write_reply( code = classify_agent_error(err) path = base / REPLIES_DIR / f"{safe}.json" - payload = { - "id": safe, - "at": int(time.time()), - "reply": str(reply or ""), - "error": err, - "reason": code, - } - fd, tmp = tempfile.mkstemp(dir=str(base / REPLIES_DIR), prefix=".rep-", suffix=".tmp") - with os.fdopen(fd, "w", encoding="utf-8") as f: - json.dump(payload, f, ensure_ascii=False) - os.replace(tmp, path) + payload = {"id": safe, "at": int(time.time()), "reply": str(reply or ""), "error": err, "reason": code} + _atomic_write_json(path, payload, prefix=".rep-") return path @@ -459,18 +406,14 @@ def _sweep_stale(base: Path, *, now: float | None = None) -> int: def cleanup_bot_relay_artifacts(max_age_hours: float | None = None) -> int: """Sweep stale relay artifacts (envelopes/replies hold DM plaintext). - ``_sweep_stale`` otherwise runs only when the Desktop drains the outbox - (``claim_pending_envelopes``) — if the Desktop never reconnects, queued - plaintext envelopes would sit on disk forever. Same contract as the - ``cleanup_*_cache`` helpers so the gateway housekeeping loop can call it - hourly. ``max_age_hours`` is accepted for signature compatibility but the - relay's own ``STALE_AFTER_SECONDS`` governs staleness. + ``_sweep_stale`` otherwise runs only on Desktop drains — if the Desktop never + reconnects, plaintext would sit on disk forever. Same contract as the + ``cleanup_*_cache`` helpers (hourly housekeeping). ``max_age_hours`` is + accepted for signature compatibility; ``STALE_AFTER_SECONDS`` governs. """ - del max_age_hours # relay staleness is governed by STALE_AFTER_SECONDS + del max_age_hours try: - home = Path(os.getenv("HERMES_HOME") or os.path.expanduser("~/.hermes")) - root = home.parent.parent if home.parent.name == "profiles" else home - base = relay_root(root) + base = relay_root(_hermes_root(Path(_default_home()))) if not base.is_dir(): return 0 return _sweep_stale(base) @@ -485,26 +428,20 @@ def cleanup_bot_relay_artifacts(max_age_hours: float | None = None) -> int: def waiter_command(root: Path | str, envelope: dict) -> str: """Shell command that blocks until the reply file appears, then prints it. - Spawned with ``terminal_tool(background=True, notify_on_complete=True)`` - so its stdout — the teammate's reply — arrives as the same completion - notification local DMs use. Stdlib-only; runs under the sender gateway's - interpreter. + Spawned via ``terminal_tool(background=True, notify_on_complete=True)`` so + its stdout — the reply — arrives as the same completion notification local + DMs use. Stdlib-only; runs under the sender gateway's interpreter. """ reply_path = str(relay_root(root) / REPLIES_DIR / f"{envelope['id']}.json") label = ( f"@{envelope.get('target_handle', '')} " f"on {envelope.get('target_connection', '')}" ) - # Encode label with !r so roster fields cannot break out of the generated - # python -c source (quotes, parens, or extra statements in connection_id). - # The raw-string prefix keeps Windows paths viable: repr escapes each - # backslash ("C:\\Users\\..."), but the Windows execution layer the - # waiter runs under folds "\\" back to "\", which turns "\U" into an - # invalid unicode escape and SyntaxErrors the whole script (#93590). - # With the r prefix the folded single backslash parses as a literal. - # POSIX paths contain no backslashes, so the prefix is a no-op there, - # and \' inside a raw literal still cannot terminate the string, so - # the injection defense above is unchanged. + # !r keeps roster fields from breaking out of the generated python -c source. + # The r-prefix keeps Windows paths viable: the Windows execution layer folds + # repr's "\\" back to "\", turning "\U" into an invalid unicode escape; a + # raw literal parses the folded backslash literally. No-op on POSIX, and \' + # still cannot terminate a raw literal, so the injection defense holds. code = ( "import json,os,sys,time\n" f"p = r{reply_path!r}\n" @@ -514,9 +451,8 @@ def waiter_command(root: Path | str, envelope: dict) -> str: " if os.path.exists(p):\n" " d = json.load(open(p, encoding='utf-8'))\n" " if d.get('error'):\n" - # The typed reason code (#93091) rides ahead of the free text so the - # sending agent can branch on it (auth vs rate limit vs offline) - # without parsing provider prose. + # Typed reason code rides ahead of the free text so the sender can + # branch on it without parsing provider prose. " code = str(d.get('reason') or '').strip()\n" " tag = ' [reason: ' + code + ']' if code else ''\n" " print('Delivery to ' + label + ' failed' + tag + ': ' + d['error'])\n" @@ -524,9 +460,7 @@ def waiter_command(root: Path | str, envelope: dict) -> str: " print('Reply from ' + label + ':')\n" " print(d.get('reply') or '(empty reply)')\n" " sys.exit(0)\n" - # 250ms cadence: the reply file is written once by the target - # gateway's deliver path; a 2s sleep here added up to 2s of dead - # air to every cross-machine reply for no benefit (stat is cheap). + # 250ms cadence: stat is cheap and a longer sleep is pure dead air. " time.sleep(0.25)\n" f"print('No reply from ' + label + ' within {REPLY_WAIT_SECONDS}s. The message may " "still be delivered when the Desktop reconnects; do not resend blindly.')\n" @@ -541,63 +475,38 @@ def waiter_command(root: Path | str, envelope: dict) -> str: def _hermes_cli() -> str: """Resolve the hermes CLI beside this gateway's own interpreter. - The deliver RPC runs on the target gateway, whose process is the venv - python — its bin/Scripts directory holds the matching ``hermes`` - entrypoint. A bare ``"hermes"`` relies on PATH, which is exactly what - service contexts (systemd units, desktop launchers, non-login SSH - shells) do not provide, so delivery died with ENOENT there (#93590). - When no sibling exists (e.g. running from a source tree without an - installed script), a ``shutil.which`` lookup runs next — it honors - whatever PATH the process does have — before falling back to the bare - name, preserving today's behavior for interactive shells. + Service contexts (systemd, desktop launchers, non-login SSH) lack PATH, so a + bare "hermes" died with ENOENT; the venv sibling wins, then ``shutil.which`` + (honors whatever PATH exists), then the bare name. """ exe = Path(sys.executable or "") sibling = exe.parent / ("hermes.exe" if sys.platform == "win32" else "hermes") if sibling.is_file(): return str(sibling) - found = shutil.which("hermes") - if found: - return found - return "hermes" + return shutil.which("hermes") or "hermes" def local_delivery_command(profile: str, query_file: str) -> list[str]: """argv that delivers a DM into ``profile``'s Bot Chat on THIS gateway.""" - return [ - _hermes_cli(), - "-p", - profile, - "chat", - "--in", - "~", - "-c", - "Bot Chat", - "--create-if-missing", - "-Q", - "--query-file", - query_file, - ] + return [_hermes_cli(), "-p", profile, "chat", "--in", "~", "-c", "Bot Chat", + "--create-if-missing", "-Q", "--query-file", query_file] -# ── per-profile turn lock (#93091) ─────────────────────────────────────────── +# ── per-profile turn lock ──────────────────────────────────────────────────── # -# Two deliveries into the SAME target profile must never run their Bot Chat -# turns concurrently: deliveries spawn separate ``hermes`` subprocesses, so -# an in-memory mutex is useless — the lock is a per-profile lockfile under -# ``/bot_relay/locks/`` held with ``fcntl.flock`` for exactly the turn -# execution window. flock is released by the kernel when the holder's fd -# closes (including process death), so a crashed turn can never wedge the -# profile. A queued delivery waits up to ``bot_mode.turn_wait_seconds`` and -# then fails with a structured 'target_busy' refusal instead of blocking -# forever. +# Two deliveries into the SAME profile must never run Bot Chat turns +# concurrently. Deliveries are separate ``hermes`` subprocesses, so the lock is +# a per-profile lockfile under ``/bot_relay/locks/`` held with +# ``fcntl.flock`` for exactly the turn window; the kernel releases it on fd +# close (including process death), so a crashed turn can never wedge the +# profile. Waiters are bounded by ``bot_mode.turn_wait_seconds`` and then fail +# with a structured 'target_busy' refusal. class TurnBusyError(RuntimeError): """A delivery turn is already running for the target profile. - ``reason`` is 'target_busy' — extends the #93091 item-1 structured - refusal enum. ``waited_seconds`` is roughly how long the caller queued - behind the current turn before giving up. + ``waited_seconds``: roughly how long the caller queued before giving up. """ reason = "target_busy" @@ -637,13 +546,11 @@ def acquire_turn_lock( ) -> Iterator[Path]: """Hold ``profile``'s cross-process turn lock for the ``with`` body. - Non-blocking flock probe + short-sleep retry loop up to the budget - (``bot_mode.turn_wait_seconds`` unless ``timeout_seconds`` is given). - No ordering guarantee among waiters — whichever probe lands first after - release wins — but every waiter is bounded by the budget, so no - deadlock. Raises :class:`TurnBusyError` when the budget is exhausted. - On platforms without ``fcntl`` (Windows) the lock degrades to a no-op — - those installs never had this race path in production. + Non-blocking flock probe + short-sleep retry up to the budget + (``bot_mode.turn_wait_seconds`` unless ``timeout_seconds`` is given). No + ordering among waiters, but every waiter is bounded — no deadlock. Raises + :class:`TurnBusyError` when the budget is exhausted. Without ``fcntl`` + (Windows) the lock is a no-op. """ try: import fcntl diff --git a/tools/browser_camofox.py b/tools/browser_camofox.py index 4fad1eb0b7..b451422072 100644 --- a/tools/browser_camofox.py +++ b/tools/browser_camofox.py @@ -1,37 +1,28 @@ """Camofox browser backend — local anti-detection browser via REST API. -Camofox-browser is a self-hosted Node.js server wrapping Camoufox (Firefox -fork with C++ fingerprint spoofing). It exposes a REST API that maps 1:1 -to our browser tool interface: accessibility snapshots with element refs, -click/type/scroll by ref, screenshots, etc. +Camofox-browser (https://github.com/jo-inc/camofox-browser) is a self-hosted +Node.js server wrapping Camoufox (Firefox fork with C++ fingerprint spoofing). +Its REST API maps 1:1 to our browser tool interface: accessibility snapshots +with element refs, click/type/scroll by ref, screenshots. -When ``CAMOFOX_URL`` is set (e.g. ``http://localhost:9377``), the browser -tools route through this module instead of the ``agent-browser`` CLI. - -Setup:: - - # Option 1: npm - git clone https://github.com/jo-inc/camofox-browser && cd camofox-browser - npm install && npm start # downloads Camoufox (~300MB) on first run - - # Option 2: Docker - docker run -p 9377:9377 -e CAMOFOX_PORT=9377 jo-inc/camofox-browser - -Then set ``CAMOFOX_URL=http://localhost:9377`` in ``~/.hermes/.env``. -For Docker Camofox, optionally set ``CAMOFOX_REWRITE_LOOPBACK_URLS=true`` -so page URLs like ``http://127.0.0.1:3000`` are opened inside the -container as ``http://host.docker.internal:3000``. +Setup: ``npm install && npm start`` in a camofox-browser checkout, or +``docker run -p 9377:9377 -e CAMOFOX_PORT=9377 jo-inc/camofox-browser``; then set +``CAMOFOX_URL=http://localhost:9377`` in ``~/.hermes/.env``. For Docker Camofox, +``CAMOFOX_REWRITE_LOOPBACK_URLS=true`` opens page URLs like ``http://127.0.0.1:3000`` +inside the container as ``http://host.docker.internal:3000``. """ from __future__ import annotations import base64 +import ipaddress import json import logging import os +import re import threading import uuid -from typing import Any, Dict, Optional +from typing import Any, Callable, Dict, Optional from urllib.parse import SplitResult, urlsplit, urlunsplit import requests @@ -48,7 +39,7 @@ logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- _DEFAULT_TIMEOUT = 30 # fallback when config is unreadable -_SNAPSHOT_MAX_CHARS = 80_000 # camofox paginates at this limit +_NO_SESSION_ERROR = "No browser session. Call browser_navigate first." _vnc_url: Optional[str] = None # cached from /health response _vnc_url_checked = False # only probe once per process @@ -58,11 +49,10 @@ _cmd_timeout_resolved = False def _get_command_timeout() -> int: - """Return ``browser.command_timeout`` from config, falling back to 30s. + """Return ``browser.command_timeout`` (floored at 5s, default 30s), cached after first read. - Mirrors :func:`tools.browser_tool._get_command_timeout` so both the - local browser path and the Camofox path honour the same config knob. - Result is cached after the first call. + Mirrors :func:`tools.browser_tool._get_command_timeout` so both backends honour + the same config knob. """ global _cached_cmd_timeout, _cmd_timeout_resolved if _cmd_timeout_resolved: @@ -71,10 +61,9 @@ def _get_command_timeout() -> int: _cmd_timeout_resolved = True result = _DEFAULT_TIMEOUT try: - cfg = read_raw_config() - val = cfg_get(cfg, "browser", "command_timeout") + val = cfg_get(read_raw_config(), "browser", "command_timeout") if val is not None: - result = max(int(val), 5) # floor at 5s + result = max(int(val), 5) except Exception as exc: logger.debug("Could not read browser.command_timeout: %s", exc) _cached_cmd_timeout = result @@ -84,9 +73,7 @@ def _get_command_timeout() -> int: def _auth_headers() -> Dict[str, str]: """Return Authorization header when CAMOFOX_API_KEY is set.""" key = (get_secret("CAMOFOX_API_KEY", "") or "").strip() - if key: - return {"Authorization": f"Bearer {key}"} - return {} + return {"Authorization": f"Bearer {key}"} if key else {} def get_camofox_url() -> str: @@ -97,9 +84,8 @@ def get_camofox_url() -> str: def _config_cdp_url() -> str: """Persistent ``browser.cdp_url`` from config.yaml, or empty string. - Read here (instead of importing ``browser_tool._get_cdp_override`` to avoid - a circular import) so Camofox can yield to a config-based CDP override the - same way it already yields to the ``BROWSER_CDP_URL`` env override. + Read here rather than via ``browser_tool._get_cdp_override`` (circular import) + so Camofox yields to a config CDP override like it yields to ``BROWSER_CDP_URL``. """ try: from hermes_cli.config import read_raw_config @@ -115,24 +101,14 @@ def _config_cdp_url() -> str: def is_camofox_mode() -> bool: """True when the Camofox backend is selected and no CDP override is active. - Camofox is a selection: ``browser.cloud_provider: camofox`` (set via - ``hermes tools``). ``CAMOFOX_URL`` is the server ADDRESS only — its - presence no longer selects the backend when a different - ``browser.cloud_provider`` is stored. Legacy read-time interpretation: - when NO cloud provider selection was ever written, a set ``CAMOFOX_URL`` - keeps activating Camofox exactly as before (nothing is migrated/written - to config). - - A CDP override takes priority over Camofox so the browser tools operate on - the real CDP browser (and a CDP backend is treated as non-local for SSRF - checks) instead of being silently routed to Camofox. The override may come - from the ``BROWSER_CDP_URL`` env var (set by ``/browser connect``) OR a - persistent ``browser.cdp_url`` in config.yaml — both are honored, matching - ``browser_tool._get_cdp_override()``'s precedence. + Selection is ``browser.cloud_provider: camofox``; ``CAMOFOX_URL`` is only the + server address and does not override a different stored selection. Legacy: when + no selection was ever written, a set ``CAMOFOX_URL`` still activates Camofox. + A CDP override (``BROWSER_CDP_URL`` env or ``browser.cdp_url`` config, matching + ``browser_tool._get_cdp_override()`` precedence) wins so tools drive the real + CDP browser instead of being silently routed to Camofox. """ - if os.getenv("BROWSER_CDP_URL", "").strip(): - return False - if _config_cdp_url(): + if os.getenv("BROWSER_CDP_URL", "").strip() or _config_cdp_url(): return False try: from tools.tool_backend_helpers import read_selection @@ -140,17 +116,13 @@ def is_camofox_mode() -> bool: selected = read_selection("browser") except Exception: # pragma: no cover — helpers are in-repo selected = None - if selected == "camofox": - return True if selected is not None: - # An explicit different browser selection wins: CAMOFOX_URL is just - # an address, not a choice. - return False + return selected == "camofox" return bool(get_camofox_url()) def check_camofox_available() -> bool: - """Verify the Camofox server is reachable.""" + """Verify the Camofox server is reachable (and cache its VNC URL once).""" global _vnc_url, _vnc_url_checked url = get_camofox_url() if not url: @@ -159,12 +131,9 @@ def check_camofox_available() -> bool: resp = requests.get(f"{url}/health", timeout=5) if resp.status_code == 200 and not _vnc_url_checked: try: - data = resp.json() - vnc_port = data.get("vncPort") + vnc_port = resp.json().get("vncPort") if isinstance(vnc_port, int) and 1 <= vnc_port <= 65535: - from urllib.parse import urlparse - parsed = urlparse(url) - host = parsed.hostname or "localhost" + host = urlsplit(url).hostname or "localhost" _vnc_url = f"http://{host}:{vnc_port}" except (ValueError, KeyError): pass @@ -191,35 +160,31 @@ def _get_camofox_config() -> Dict[str, Any]: return camofox_cfg if isinstance(camofox_cfg, dict) else {} -def _managed_persistence_enabled() -> bool: - """Return whether Hermes-managed persistence is enabled for Camofox. +def _managed_persistence_enabled(camofox_cfg: Optional[Dict[str, Any]] = None) -> bool: + """``browser.camofox.managed_persistence``: stable profile-scoped userId vs random per session.""" + if camofox_cfg is None: + camofox_cfg = _get_camofox_config() + return bool(camofox_cfg.get("managed_persistence")) - When enabled, sessions use a stable profile-scoped userId so the - Camofox server can map it to a persistent browser profile directory. - When disabled (default), each session gets a random userId (ephemeral). - Controlled by ``browser.camofox.managed_persistence`` in config.yaml. - """ - return bool(_get_camofox_config().get("managed_persistence")) +def _secret_or_cfg(secret_name: str, camofox_cfg: Dict[str, Any], cfg_key: str) -> str: + return ( + (get_secret(secret_name, "") or "").strip() + or str(camofox_cfg.get(cfg_key) or "").strip() + ) def _camofox_identity_override(task_id: Optional[str], camofox_cfg: Dict[str, Any]) -> Optional[Dict[str, str]]: """Return an externally configured Camofox identity, if one is set. - Integrations that own the visible Camofox browser can set a shared user ID - so Hermes operates in the same browser profile instead of creating a - separate private session. + Integrations that own the visible Camofox browser share a user ID so Hermes + operates in the same profile instead of a separate private session. """ - user_id = ( - (get_secret("CAMOFOX_USER_ID", "") or "").strip() - or str(camofox_cfg.get("user_id") or "").strip() - ) + user_id = _secret_or_cfg("CAMOFOX_USER_ID", camofox_cfg, "user_id") if not user_id: return None - session_key = ( - (get_secret("CAMOFOX_SESSION_KEY", "") or "").strip() - or str(camofox_cfg.get("session_key") or "").strip() + _secret_or_cfg("CAMOFOX_SESSION_KEY", camofox_cfg, "session_key") or f"task_{(task_id or 'default')[:16]}" ) return {"user_id": user_id, "session_key": session_key} @@ -237,30 +202,25 @@ def _env_flag(name: str) -> Optional[bool]: return None +def _flag(env_name: str, camofox_cfg: Dict[str, Any], cfg_key: str) -> bool: + """Boolean toggle: env var wins when set to a valid value, else config key.""" + env_value = _env_flag(env_name) + return env_value if env_value is not None else bool(camofox_cfg.get(cfg_key)) + + def _adopt_existing_tab_enabled(camofox_cfg: Dict[str, Any]) -> bool: """Return whether Hermes should recover an existing Camofox tab ID.""" - env_value = _env_flag("CAMOFOX_ADOPT_EXISTING_TAB") - if env_value is not None: - return env_value - return bool(camofox_cfg.get("adopt_existing_tab")) + return _flag("CAMOFOX_ADOPT_EXISTING_TAB", camofox_cfg, "adopt_existing_tab") def _loopback_rewrite_enabled(camofox_cfg: Dict[str, Any]) -> bool: - """Return whether loopback navigation URLs should be rewritten for Docker. + """Return whether loopback page URLs should be rewritten for Docker-hosted Camofox. - ``CAMOFOX_URL`` itself often points at a host-published Docker port such as - ``http://127.0.0.1:9377``. That is correct for Hermes talking to the - Camofox control API, but a page URL like ``http://127.0.0.1:3000`` is opened - by the browser *inside* the Docker container. In that context loopback - points at the container, not the host running the web app. - - The rewrite is opt-in because non-Docker Camofox installs run the browser on - the host, where loopback URLs are already correct. + ``CAMOFOX_URL`` may point at a host-published Docker port, but page URLs are + opened by the browser *inside* the container, where loopback is the container, + not the host. Opt-in because non-Docker installs run the browser on the host. """ - env_value = _env_flag("CAMOFOX_REWRITE_LOOPBACK_URLS") - if env_value is not None: - return env_value - return bool(camofox_cfg.get("rewrite_loopback_urls")) + return _flag("CAMOFOX_REWRITE_LOOPBACK_URLS", camofox_cfg, "rewrite_loopback_urls") def _loopback_rewrite_host(camofox_cfg: Dict[str, Any]) -> str: @@ -280,8 +240,6 @@ def _is_loopback_hostname(hostname: Optional[str]) -> bool: if host in {"localhost", "localhost.localdomain"}: return True try: - import ipaddress - return ipaddress.ip_address(host).is_loopback except ValueError: return False @@ -290,7 +248,7 @@ def _is_loopback_hostname(hostname: Optional[str]) -> bool: def _rewrite_loopback_url_for_camofox(url: str) -> tuple[str, Optional[Dict[str, str]]]: """Rewrite loopback page URLs for Docker-hosted Camofox, if configured. - Returns ``(rewritten_url, metadata)``. ``metadata`` is present only when a + Returns ``(rewritten_url, metadata)``; ``metadata`` is present only when a rewrite happened so the tool result can disclose the change to the model. """ camofox_cfg = _get_camofox_config() @@ -331,7 +289,7 @@ def _rewrite_loopback_url_for_camofox(url: str) -> tuple[str, Optional[Dict[str, # --------------------------------------------------------------------------- # Session management # --------------------------------------------------------------------------- -# Maps task_id -> {"user_id": str, "tab_id": str|None} +# Maps task_id -> {"user_id": str, "tab_id": str|None, ...} _sessions: Dict[str, Dict[str, Any]] = {} _sessions_lock = threading.Lock() @@ -339,14 +297,10 @@ _sessions_lock = threading.Lock() def _adopt_existing_tab(session: Dict[str, Any]) -> Dict[str, Any]: """Attach process-local state to an already-open managed Camofox tab. - Some integrations own the visible Camofox tab outside Hermes. Gateway - restarts can leave this module's in-memory session cache empty even though - Camofox still has that tab, so rehydrate tab_id before creating a new tab. + Gateway restarts empty this module's in-memory cache while Camofox still has + the integration-owned tab, so rehydrate tab_id before creating a new one. """ - if session.get("tab_id") or not session.get("adopt_existing_tab"): - return session - - if not get_camofox_url(): + if session.get("tab_id") or not session.get("adopt_existing_tab") or not get_camofox_url(): return session try: @@ -359,14 +313,9 @@ def _adopt_existing_tab(session: Dict[str, Any]) -> Dict[str, Any]: return session session_key = session.get("session_key") - matching_tabs = [ - tab - for tab in tabs - if isinstance(tab, dict) and tab.get("listItemId") == session_key - ] - candidates = matching_tabs or [tab for tab in tabs if isinstance(tab, dict)] - latest = candidates[-1] if candidates else None - tab_id = latest.get("tabId") if isinstance(latest, dict) else None + dict_tabs = [tab for tab in tabs if isinstance(tab, dict)] + candidates = [tab for tab in dict_tabs if tab.get("listItemId") == session_key] or dict_tabs + tab_id = candidates[-1].get("tabId") if candidates else None if isinstance(tab_id, str) and tab_id: session["tab_id"] = tab_id logger.debug("Adopted existing Camofox tab %s for %s", tab_id, session.get("user_id")) @@ -377,9 +326,9 @@ def _adopt_existing_tab(session: Dict[str, Any]) -> Dict[str, Any]: def _get_session(task_id: Optional[str]) -> Dict[str, Any]: """Get or create a camofox session for the given task. - When managed persistence is enabled, uses a deterministic userId - derived from the Hermes profile so the Camofox server can map it - to the same persistent browser profile across restarts. + Identity precedence: external override (CAMOFOX_USER_ID / config), then the + deterministic profile-scoped identity when managed persistence is on, else a + random ephemeral userId. """ task_id = task_id or "default" with _sessions_lock: @@ -387,17 +336,10 @@ def _get_session(task_id: Optional[str]) -> Dict[str, Any]: return _adopt_existing_tab(_sessions[task_id]) camofox_cfg = _get_camofox_config() - identity_override = _camofox_identity_override(task_id, camofox_cfg) - if identity_override: - session = { - "user_id": identity_override["user_id"], - "tab_id": None, - "session_key": identity_override["session_key"], - "managed": True, - "adopt_existing_tab": _adopt_existing_tab_enabled(camofox_cfg), - } - elif bool(camofox_cfg.get("managed_persistence")): + identity = _camofox_identity_override(task_id, camofox_cfg) + if identity is None and _managed_persistence_enabled(camofox_cfg): identity = get_camofox_identity(task_id) + if identity is not None: session = { "user_id": identity["user_id"], "tab_id": None, @@ -422,41 +364,34 @@ def _ensure_tab(task_id: Optional[str], url: str = "about:blank") -> Dict[str, A session = _get_session(task_id) if session["tab_id"]: return session - base = get_camofox_url() - resp = requests.post( - f"{base}/tabs", + data = _request( + "post", + "/tabs", json={ "userId": session["user_id"], "listItemId": session["session_key"], "url": url, }, - timeout=_get_command_timeout(), - headers=_auth_headers(), - ) - resp.raise_for_status() - data = resp.json() + ).json() session["tab_id"] = data.get("tabId") return session def _drop_session(task_id: Optional[str]) -> Optional[Dict[str, Any]]: """Remove and return session info.""" - task_id = task_id or "default" with _sessions_lock: - return _sessions.pop(task_id, None) + return _sessions.pop(task_id or "default", None) def camofox_soft_cleanup(task_id: Optional[str] = None) -> bool: """Release the in-memory session without destroying the server-side context. - When managed persistence is enabled the browser profile (and its cookies) - must survive across agent tasks. This helper drops only the local tracking - entry and returns ``True``. When managed persistence is *not* enabled it - does nothing and returns ``False`` so the caller can fall back to - :func:`camofox_close`. + Managed (persistent or externally-owned) profiles must survive across agent + tasks, so only the local tracking entry is dropped (returns ``True``). For + ephemeral sessions returns ``False`` so the caller falls back to :func:`camofox_close`. """ camofox_cfg = _get_camofox_config() - if bool(camofox_cfg.get("managed_persistence")) or _camofox_identity_override(task_id, camofox_cfg): + if _managed_persistence_enabled(camofox_cfg) or _camofox_identity_override(task_id, camofox_cfg): _drop_session(task_id) logger.debug("Camofox soft cleanup for task %s (managed persistence)", task_id) return True @@ -467,68 +402,85 @@ def camofox_soft_cleanup(task_id: Optional[str] = None) -> bool: # HTTP helpers # --------------------------------------------------------------------------- -def _post(path: str, body: dict, timeout: Optional[int] = None) -> dict: - """POST JSON to camofox and return parsed response.""" +def _request(method: str, path: str, timeout: Optional[int] = None, **kwargs: Any) -> requests.Response: + """Issue an authenticated request to camofox and return the raised-for-status response.""" if timeout is None: timeout = _get_command_timeout() - url = f"{get_camofox_url()}{path}" - resp = requests.post(url, json=body, timeout=timeout, headers=_auth_headers()) - resp.raise_for_status() - return resp.json() - - -def _get(path: str, params: dict = None, timeout: Optional[int] = None) -> dict: - """GET from camofox and return parsed response.""" - if timeout is None: - timeout = _get_command_timeout() - url = f"{get_camofox_url()}{path}" - resp = requests.get(url, params=params, timeout=timeout, headers=_auth_headers()) - resp.raise_for_status() - return resp.json() - - -def _get_raw(path: str, params: dict = None, timeout: Optional[int] = None) -> requests.Response: - """GET from camofox and return raw response (for binary data).""" - if timeout is None: - timeout = _get_command_timeout() - url = f"{get_camofox_url()}{path}" - resp = requests.get(url, params=params, timeout=timeout, headers=_auth_headers()) + resp = getattr(requests, method)( + f"{get_camofox_url()}{path}", timeout=timeout, headers=_auth_headers(), **kwargs + ) resp.raise_for_status() return resp +def _post(path: str, body: dict, timeout: Optional[int] = None) -> dict: + """POST JSON to camofox and return parsed response.""" + return _request("post", path, timeout, json=body).json() + + +def _get(path: str, params: dict = None, timeout: Optional[int] = None) -> dict: + """GET from camofox and return parsed response.""" + return _request("get", path, timeout, params=params).json() + + +def _get_raw(path: str, params: dict = None, timeout: Optional[int] = None) -> requests.Response: + """GET from camofox and return raw response (for binary data).""" + return _request("get", path, timeout, params=params) + + def _delete(path: str, body: dict = None, timeout: Optional[int] = None) -> dict: """DELETE to camofox and return parsed response.""" - if timeout is None: - timeout = _get_command_timeout() - url = f"{get_camofox_url()}{path}" - resp = requests.delete(url, json=body, timeout=timeout, headers=_auth_headers()) - resp.raise_for_status() - return resp.json() + return _request("delete", path, timeout, json=body).json() # --------------------------------------------------------------------------- # Tool implementations # --------------------------------------------------------------------------- +def _tab_path(session: Dict[str, Any], suffix: str) -> str: + return f"/tabs/{session['tab_id']}/{suffix}" + + +def _user_params(session: Dict[str, Any]) -> Dict[str, str]: + return {"userId": session["user_id"]} + + +def _fetch_snapshot(session: Dict[str, Any]) -> tuple[str, int]: + """Return ``(snapshot_text, refs_count)`` truncated like the main browser tool. + + Cuts at line boundaries, stores the full tree to cache/web, and appends a + read_file pointer. ``browser_tool`` imports this module, so import lazily. + """ + data = _get(_tab_path(session, "snapshot"), params=_user_params(session)) + snapshot = data.get("snapshot", "") + from tools.browser_tool import ( + get_browser_snapshot_threshold, + _truncate_snapshot, + ) + + threshold = get_browser_snapshot_threshold() + if len(snapshot) > threshold: + snapshot = _truncate_snapshot(snapshot, max_chars=threshold) + return snapshot, data.get("refsCount", 0) + + def camofox_navigate(url: str, task_id: Optional[str] = None) -> str: """Navigate to a URL via Camofox.""" try: browser_url, rewrite_info = _rewrite_loopback_url_for_camofox(url) session = _get_session(task_id) if not session["tab_id"]: - # Create tab with the target URL directly session = _ensure_tab(task_id, browser_url) data = {"ok": True, "url": browser_url} else: - # Navigate existing tab — recover from stale tab 404 try: data = _post( - f"/tabs/{session['tab_id']}/navigate", + _tab_path(session, "navigate"), {"userId": session["user_id"], "url": browser_url}, timeout=60, ) except requests.HTTPError as e: + # Stale tab (garbage collected server-side) — recreate it. if e.response is not None and e.response.status_code == 404: logger.warning( "Camofox tab %s returned 404 — tab was garbage collected. " @@ -560,22 +512,9 @@ def camofox_navigate(url: str, task_id: Optional[str] = None) -> str: "Share this link with the user so they can watch the browser live." ) - # Auto-take a compact snapshot so the model can act immediately + # Auto-take a compact snapshot so the model can act immediately. try: - snap_data = _get( - f"/tabs/{session['tab_id']}/snapshot", - params={"userId": session["user_id"]}, - ) - snapshot_text = snap_data.get("snapshot", "") - from tools.browser_tool import ( - get_browser_snapshot_threshold, - _truncate_snapshot, - ) - threshold = get_browser_snapshot_threshold() - if len(snapshot_text) > threshold: - snapshot_text = _truncate_snapshot(snapshot_text, max_chars=threshold) - result["snapshot"] = snapshot_text - result["element_count"] = snap_data.get("refsCount", 0) + result["snapshot"], result["element_count"] = _fetch_snapshot(session) except Exception: pass # Navigation succeeded; snapshot is a bonus @@ -596,16 +535,12 @@ def camofox_navigate(url: str, task_id: Optional[str] = None) -> str: def _camofox_private_page_block(session: Dict[str, Any], task_id: Optional[str], action: str) -> Optional[str]: """Return a blocked payload when the current Camofox page is private/internal. - Mirrors the eval-path guard added for ``_camofox_eval`` (browser_tool.py): - Camofox snapshot / vision / image-extraction all read current page state, so - on a non-local backend they can leak the content of an intranet/metadata - page the terminal itself can't reach. The gate matches ``browser_snapshot`` - / ``browser_vision`` — only active when the SSRF guard applies (non-local - backend, not a local sidecar, ``allow_private_urls`` unset). Fail-open on - probe failure, matching the sibling guards. - - Imports are deferred to call time because ``browser_tool`` imports this - module; importing it at module load would create a circular import. + Mirrors the ``_camofox_eval`` guard in browser_tool.py: snapshot / vision / + image-extraction read current page state, so on a non-local backend they can + leak an intranet/metadata page the terminal itself can't reach. Only active + when the SSRF guard applies (non-local backend, not a local sidecar, + ``allow_private_urls`` unset); fail-open on probe failure like sibling guards. + ``browser_tool`` imports this module, so import lazily. """ from tools.browser_tool import ( _camofox_current_page_private_url, @@ -627,6 +562,16 @@ def _camofox_private_page_block(session: Dict[str, Any], task_id: Optional[str], }, ensure_ascii=False) +def _require_tab(task_id: Optional[str], action: Optional[str] = None) -> tuple[Dict[str, Any], Optional[str]]: + """Return ``(session, error_payload)``: error when no tab exists or, if ``action`` given, the page is private.""" + session = _get_session(task_id) + if not session["tab_id"]: + return session, tool_error(_NO_SESSION_ERROR, success=False) + if action is not None: + return session, _camofox_private_page_block(session, task_id, action) + return session, None + + def camofox_snapshot(full: bool = False, task_id: Optional[str] = None, user_task: Optional[str] = None) -> str: """Get accessibility tree snapshot from Camofox. @@ -635,34 +580,10 @@ def camofox_snapshot(full: bool = False, task_id: Optional[str] = None, truncate-and-store (no LLM summarization), same as the main browser tool. """ try: - session = _get_session(task_id) - if not session["tab_id"]: - return tool_error("No browser session. Call browser_navigate first.", success=False) - - blocked = _camofox_private_page_block(session, task_id, "read a page snapshot") + session, blocked = _require_tab(task_id, "read a page snapshot") if blocked: return blocked - - data = _get( - f"/tabs/{session['tab_id']}/snapshot", - params={"userId": session["user_id"]}, - ) - - snapshot = data.get("snapshot", "") - refs_count = data.get("refsCount", 0) - - # Same truncate-and-store handling as the main browser tool: cut at - # line boundaries, store the full tree to cache/web, append a - # read_file pointer. - from tools.browser_tool import ( - get_browser_snapshot_threshold, - _truncate_snapshot, - ) - - threshold = get_browser_snapshot_threshold() - if len(snapshot) > threshold: - snapshot = _truncate_snapshot(snapshot, max_chars=threshold) - + snapshot, refs_count = _fetch_snapshot(session) return json.dumps({ "success": True, "snapshot": snapshot, @@ -672,29 +593,29 @@ def camofox_snapshot(full: bool = False, task_id: Optional[str] = None, return tool_error(str(e), success=False) +def _tab_action(task_id: Optional[str], guard_action: Optional[str], suffix: str, + body: Dict[str, Any], result: Callable[[dict], dict]) -> str: + """Shared shape of the simple tab actions: require a tab (+ private-page guard + when ``guard_action`` is set), POST ``body`` to ``/tabs//``, build result.""" + try: + session, blocked = _require_tab(task_id, guard_action) + if blocked: + return blocked + data = _post(_tab_path(session, suffix), {"userId": session["user_id"], **body}) + return json.dumps(result(data)) + except Exception as e: + return tool_error(str(e), success=False) + + def camofox_click(ref: str, task_id: Optional[str] = None) -> str: """Click an element by ref via Camofox.""" try: - session = _get_session(task_id) - if not session["tab_id"]: - return tool_error("No browser session. Call browser_navigate first.", success=False) - - blocked = _camofox_private_page_block(session, task_id, "click") + session, blocked = _require_tab(task_id, "click") if blocked: return blocked - - # Strip @ prefix if present (our tool convention) - clean_ref = ref.lstrip("@") - - data = _post( - f"/tabs/{session['tab_id']}/click", - {"userId": session["user_id"], "ref": clean_ref}, - ) - return json.dumps({ - "success": True, - "clicked": clean_ref, - "url": data.get("url", ""), - }) + clean_ref = ref.lstrip("@") # our tool convention prefixes refs with @ + data = _post(_tab_path(session, "click"), {"userId": session["user_id"], "ref": clean_ref}) + return json.dumps({"success": True, "clicked": clean_ref, "url": data.get("url", "")}) except Exception as e: return tool_error(str(e), success=False) @@ -702,18 +623,12 @@ def camofox_click(ref: str, task_id: Optional[str] = None) -> str: def camofox_type(ref: str, text: str, task_id: Optional[str] = None) -> str: """Type text into an element by ref via Camofox.""" try: - session = _get_session(task_id) - if not session["tab_id"]: - return tool_error("No browser session. Call browser_navigate first.", success=False) - - blocked = _camofox_private_page_block(session, task_id, "type") + session, blocked = _require_tab(task_id, "type") if blocked: return blocked - clean_ref = ref.lstrip("@") - _post( - f"/tabs/{session['tab_id']}/type", + _tab_path(session, "type"), {"userId": session["user_id"], "ref": clean_ref, "text": text}, ) from agent.display import ( @@ -721,14 +636,12 @@ def camofox_type(ref: str, text: str, task_id: Optional[str] = None) -> str: redact_tool_args_for_display, ) + # Match browser_tool.browser_type: the raw text is typed into the page, but + # the returned display value is run through the secret-pattern redactor so + # API keys / tokens don't leak into tool progress or chat history. display_text = (redact_tool_args_for_display("browser_type", {"text": text}) or {})["text"] - response = { "success": True, - # Match browser_tool.browser_type: run typed text through the - # secret-pattern redactor so API keys / tokens don't leak into - # tool progress or chat history. The raw text is still typed into - # the page; only the returned display value is redacted. "typed": display_text, "element": clean_ref, } @@ -742,66 +655,34 @@ def camofox_type(ref: str, text: str, task_id: Optional[str] = None) -> str: def camofox_scroll(direction: str, task_id: Optional[str] = None) -> str: """Scroll the page via Camofox.""" - try: - session = _get_session(task_id) - if not session["tab_id"]: - return tool_error("No browser session. Call browser_navigate first.", success=False) - - _post( - f"/tabs/{session['tab_id']}/scroll", - {"userId": session["user_id"], "direction": direction}, - ) - return json.dumps({"success": True, "scrolled": direction}) - except Exception as e: - return tool_error(str(e), success=False) + return _tab_action( + task_id, None, "scroll", {"direction": direction}, + lambda data: {"success": True, "scrolled": direction}, + ) def camofox_back(task_id: Optional[str] = None) -> str: """Navigate back via Camofox.""" - try: - session = _get_session(task_id) - if not session["tab_id"]: - return tool_error("No browser session. Call browser_navigate first.", success=False) - - data = _post( - f"/tabs/{session['tab_id']}/back", - {"userId": session["user_id"]}, - ) - return json.dumps({"success": True, "url": data.get("url", "")}) - except Exception as e: - return tool_error(str(e), success=False) + return _tab_action( + task_id, None, "back", {}, + lambda data: {"success": True, "url": data.get("url", "")}, + ) def camofox_press(key: str, task_id: Optional[str] = None) -> str: """Press a keyboard key via Camofox.""" - try: - session = _get_session(task_id) - if not session["tab_id"]: - return tool_error("No browser session. Call browser_navigate first.", success=False) - - blocked = _camofox_private_page_block(session, task_id, "press") - if blocked: - return blocked - - _post( - f"/tabs/{session['tab_id']}/press", - {"userId": session["user_id"], "key": key}, - ) - return json.dumps({"success": True, "pressed": key}) - except Exception as e: - return tool_error(str(e), success=False) + return _tab_action( + task_id, "press", "press", {"key": key}, + lambda data: {"success": True, "pressed": key}, + ) def camofox_close(task_id: Optional[str] = None) -> str: """Close the browser session via Camofox.""" try: session = _drop_session(task_id) - if not session: - return json.dumps({"success": True, "closed": True}) - - _delete( - f"/sessions/{session['user_id']}", - ) + if session: + _delete(f"/sessions/{session['user_id']}") return json.dumps({"success": True, "closed": True}) except Exception as e: return json.dumps({"success": True, "closed": True, "warning": str(e)}) @@ -810,29 +691,17 @@ def camofox_close(task_id: Optional[str] = None) -> str: def camofox_get_images(task_id: Optional[str] = None) -> str: """Get images on the current page via Camofox. - Extracts image information from the accessibility tree snapshot, - since Camofox does not expose a dedicated /images endpoint. + Parsed from the accessibility tree snapshot (``img "alt" [eN]`` entries with + the URL on the following ``/url:`` line) — Camofox has no /images endpoint. """ try: - session = _get_session(task_id) - if not session["tab_id"]: - return tool_error("No browser session. Call browser_navigate first.", success=False) - - blocked = _camofox_private_page_block(session, task_id, "extract page images") + session, blocked = _require_tab(task_id, "extract page images") if blocked: return blocked - import re - - data = _get( - f"/tabs/{session['tab_id']}/snapshot", - params={"userId": session["user_id"]}, - ) + data = _get(_tab_path(session, "snapshot"), params=_user_params(session)) snapshot = data.get("snapshot", "") - # Parse img elements from the accessibility tree. - # Format: img "alt text" or img "alt text" [eN] - # URLs appear on /url: lines following img entries images = [] lines = snapshot.split("\n") for i, line in enumerate(lines): @@ -840,7 +709,6 @@ def camofox_get_images(task_id: Optional[str] = None) -> str: if stripped.startswith(("- img ", "img ")): alt_match = re.search(r'img\s+"([^"]*)"', stripped) alt = alt_match.group(1) if alt_match else "" - # Look for URL on the next line src = "" if i + 1 < len(lines): url_match = re.search(r'/url:\s*(\S+)', lines[i + 1].strip()) @@ -862,21 +730,12 @@ def camofox_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> str: """Take a screenshot and analyze it with vision AI via Camofox.""" try: - session = _get_session(task_id) - if not session["tab_id"]: - return tool_error("No browser session. Call browser_navigate first.", success=False) - - blocked = _camofox_private_page_block(session, task_id, "capture a screenshot") + session, blocked = _require_tab(task_id, "capture a screenshot") if blocked: return blocked - # Get screenshot as binary PNG - resp = _get_raw( - f"/tabs/{session['tab_id']}/screenshot", - params={"userId": session["user_id"]}, - ) + resp = _get_raw(_tab_path(session, "screenshot"), params=_user_params(session)) - # Save screenshot to cache from hermes_constants import get_hermes_home screenshots_dir = get_hermes_home() / "browser_screenshots" screenshots_dir.mkdir(parents=True, exist_ok=True) @@ -885,28 +744,21 @@ def camofox_vision(question: str, annotate: bool = False, with open(screenshot_path, "wb") as f: f.write(resp.content) - # Encode for vision LLM img_b64 = base64.b64encode(resp.content).decode("utf-8") - # Also get annotated snapshot if requested annotation_context = "" if annotate: try: - snap_data = _get( - f"/tabs/{session['tab_id']}/snapshot", - params={"userId": session["user_id"]}, - ) + snap_data = _get(_tab_path(session, "snapshot"), params=_user_params(session)) annotation_context = f"\n\nAccessibility tree (element refs for interaction):\n{snap_data.get('snapshot', '')[:3000]}" except Exception: pass - # Redact secrets from annotation context before sending to vision LLM. - # The screenshot image itself cannot be redacted, but at least the - # text-based accessibility tree snippet won't leak secret values. + # The screenshot itself cannot be redacted, but the text-based accessibility + # snippet sent alongside it must not leak secret values. from agent.redact import redact_sensitive_text annotation_context = redact_sensitive_text(annotation_context) - # Send to vision LLM from agent.auxiliary_client import call_llm vision_prompt = ( @@ -915,8 +767,7 @@ def camofox_vision(question: str, annotate: bool = False, ) try: - _cfg = load_config() - _vision_cfg = cfg_get(_cfg, "auxiliary", "vision", default={}) + _vision_cfg = cfg_get(load_config(), "auxiliary", "vision", default={}) _vision_timeout = float(_vision_cfg.get("timeout", 120)) _vision_temperature = float(_vision_cfg.get("temperature", 0.1)) except Exception: @@ -943,7 +794,6 @@ def camofox_vision(question: str, annotate: bool = False, analysis = (response.choices[0].message.content or "").strip() if response.choices else "" # Redact secrets the vision LLM may have read from the screenshot. - from agent.redact import redact_sensitive_text analysis = redact_sensitive_text(analysis) return json.dumps({ @@ -956,11 +806,7 @@ def camofox_vision(question: str, annotate: bool = False, def camofox_console(clear: bool = False, task_id: Optional[str] = None) -> str: - """Get console output — limited support in Camofox. - - Camofox does not expose browser console logs via its REST API. - Returns an empty result with a note. - """ + """Console output is not exposed by the Camofox REST API; return an empty result with a note.""" return json.dumps({ "success": True, "console_messages": [], @@ -970,5 +816,3 @@ def camofox_console(clear: bool = False, task_id: Optional[str] = None) -> str: "note": "Console log capture is not available with the Camofox backend. " "Use browser_snapshot or browser_vision to inspect page state.", }) - - diff --git a/tools/browser_cdp_tool.py b/tools/browser_cdp_tool.py index c5ccd874f5..e5acb58283 100644 --- a/tools/browser_cdp_tool.py +++ b/tools/browser_cdp_tool.py @@ -1,19 +1,10 @@ #!/usr/bin/env python3 -""" -Raw Chrome DevTools Protocol (CDP) passthrough tool. +"""Raw Chrome DevTools Protocol (CDP) passthrough tool ``browser_cdp``. -Exposes a single tool, ``browser_cdp``, that sends arbitrary CDP commands to -the browser's DevTools WebSocket endpoint. Works when a CDP URL is -configured — either via ``/browser connect`` (sets ``BROWSER_CDP_URL``) or -``browser.cdp_url`` in ``config.yaml`` — or when a CDP-backed cloud provider -session is active. - -This is the escape hatch for browser operations not covered by the main -browser tool surface (``browser_navigate``, ``browser_click``, -``browser_console``, etc.) — handling native dialogs, iframe-scoped -evaluation, cookie/network control, low-level tab management, etc. - -Method reference: https://chromedevtools.github.io/devtools-protocol/ +Sends arbitrary CDP commands to the browser's DevTools WebSocket when a CDP URL +is configured (``/browser connect`` → ``BROWSER_CDP_URL``, ``browser.cdp_url``, +or a CDP-backed cloud session). Escape hatch for operations the main browser +tools don't cover. Method reference: https://chromedevtools.github.io/devtools-protocol/ """ from __future__ import annotations @@ -29,10 +20,9 @@ logger = logging.getLogger(__name__) CDP_DOCS_URL = "https://chromedevtools.github.io/devtools-protocol/" +# Browser/target inspection that never reads page body/cookies/DOM/storage — +# stays usable so the model can list tabs or navigate away from a blocked page. _CDP_PRIVATE_PAGE_ALLOWED_METHODS = { - # Browser/target inspection does not read the current page body, cookies, - # DOM, storage, or screenshots. Keep these working so the model can list - # tabs or navigate away from a blocked page. "Browser.getVersion", "Target.getTargets", "Target.attachToTarget", @@ -43,14 +33,11 @@ _CDP_PRIVATE_PAGE_ALLOWED_METHODS = { } +# method → result paths that are ALWAYS opaque base64 (protocol-declared binary). +# redact_sensitive_text's Fernet pattern ("gAAAA" + base64 alphabet) can match +# arbitrary spans inside such payloads and corrupt the decoded bytes; the payload +# is not free text the model reads, so redaction protects no secret there. _CDP_ALWAYS_BINARY_PATHS: Dict[str, tuple] = { - # method → result paths that are ALWAYS opaque base64 payloads (the - # protocol declares them binary with no flag of their own). - # redact_sensitive_text's Fernet pattern ("gAAAA" + base64 alphabet) can - # match arbitrary spans inside such payloads — collapsing them to - # "first6...last4" and corrupting the decoded bytes (#94138). The payload - # is binary, not free text the model reads, so redaction has no secret to - # protect there. "Page.captureScreenshot": (("data",),), "Page.printToPDF": (("data",),), "Network.streamResourceContent": (("bufferedData",),), @@ -58,12 +45,9 @@ _CDP_ALWAYS_BINARY_PATHS: Dict[str, tuple] = { "CacheStorage.requestCachedResponse": (("response", "body"),), } +# method → result paths that are opaque base64 ONLY when the carrying dict has a +# ``base64Encoded`` sibling that is exactly ``True``; otherwise text → redacted. _CDP_FLAGGED_BINARY_PATHS: Dict[str, tuple] = { - # method → result paths that are opaque base64 ONLY when the dict that - # carries the final field has a ``base64Encoded`` sibling that is exactly - # ``True``. The discriminator is type information only at these - # protocol-defined paths; ``base64Encoded: false`` or absent means text, - # which is redacted. "Network.getResponseBody": (("body",),), "Fetch.getResponseBody": (("body",),), "IO.read": (("data",),), @@ -77,18 +61,12 @@ def _redact_cdp_output( always_paths: tuple = (), flagged_paths: tuple = (), ) -> Any: - """Redact browser-originated CDP result data before returning it. + """Redact browser-originated CDP result text; opaque bytes stay byte-identical. - Policy: semantic text is redacted; opaque bytes stay byte-identical - (#94138). Exemptions come ONLY from the calling method's spec - (``_CDP_ALWAYS_BINARY_PATHS`` / ``_CDP_FLAGGED_BINARY_PATHS``) as exact - result paths — every other string in every result keeps full - ``redact_sensitive_text(force=True)``. Path suffixes are propagated only - into the matching subtree, so ``base64Encoded`` is honored solely as a - sibling on the trusted carrier object, never as ambient trust in - arbitrary nested JSON (a ``Runtime.evaluate`` by-value object could - otherwise spoof ``{"base64Encoded": true, "data": ""}`` past the - redactor — second review on #94142). + Exemptions come ONLY from the calling method's spec as exact result paths. + Path suffixes propagate only into the matching subtree, so ``base64Encoded`` + is honored solely as a sibling on the trusted carrier object — never as + ambient trust a ``Runtime.evaluate`` by-value object could spoof (#94142). """ from agent.redact import redact_sensitive_text @@ -125,9 +103,7 @@ def _redact_cdp_output( return redacted return value -# ``websockets`` is a direct hermes-agent dependency because the browser CDP -# supervisor and browser_dialog tool import it during tool discovery. Wrap the -# import so a clean error surfaces if an environment is stale or incomplete. +# ``websockets`` is a direct dependency; wrap so a stale env yields a clean error. try: import websockets from websockets.exceptions import WebSocketException @@ -139,11 +115,6 @@ except ImportError: _WS_AVAILABLE = False -# --------------------------------------------------------------------------- -# Async-from-sync bridge (matches the pattern in homeassistant_tool.py) -# --------------------------------------------------------------------------- - - def _run_async(coro): """Run an async coroutine from a sync handler, safe inside or outside a loop.""" try: @@ -160,20 +131,8 @@ def _run_async(coro): return asyncio.run(coro) -# --------------------------------------------------------------------------- -# Endpoint resolution -# --------------------------------------------------------------------------- - - def _resolve_cdp_endpoint() -> str: - """Return the normalized CDP WebSocket URL, or empty string if unavailable. - - Delegates to ``tools.browser_tool._get_cdp_override`` so precedence stays - consistent with the rest of the browser tool surface: - - 1. ``BROWSER_CDP_URL`` env var (live override from ``/browser connect``) - 2. ``browser.cdp_url`` in ``config.yaml`` - """ + """Normalized CDP WebSocket URL via ``browser_tool._get_cdp_override``, or "".""" try: from tools.browser_tool import _get_cdp_override # type: ignore[import-not-found] @@ -201,12 +160,8 @@ def _browser_cdp_private_guard( ) -> Optional[str]: """Apply the browser SSRF/private-page guard to raw CDP calls. - ``browser_cdp`` is intentionally an escape hatch, but it still shares the - same cloud/private-network boundary as ``browser_snapshot``, - ``browser_console`` and ``browser_eval``. If a cloud browser has landed on - a private/internal URL (for example via a prior eval navigation), raw CDP - calls like ``Runtime.evaluate`` or ``DOM.getDocument`` must not become the - sibling bypass for the guarded browser tools. + Raw CDP shares the cloud/private-network boundary of ``browser_snapshot`` / + ``browser_console`` / ``browser_eval`` and must not become their bypass. """ try: from tools import browser_tool as bt # type: ignore[import-not-found] @@ -243,17 +198,11 @@ def _browser_cdp_private_guard( if blocked_url: return _private_page_guard_error(blocked_url, method) except Exception as exc: # noqa: BLE001 - # Match the existing browser guards' posture: guard probes are - # best-effort and should not break local/custom CDP workflows. + # Guard probes are best-effort; never break local/custom CDP workflows. logger.debug("browser_cdp: private-page guard probe failed: %s", exc) return None -# --------------------------------------------------------------------------- -# Core CDP call -# --------------------------------------------------------------------------- - - async def _cdp_call( ws_url: str, method: str, @@ -263,12 +212,9 @@ async def _cdp_call( ) -> Dict[str, Any]: """Make a single CDP call, optionally attaching to a target first. - When ``target_id`` is provided, we call ``Target.attachToTarget`` with - ``flatten=True`` to multiplex a page-level session over the same - browser-level WebSocket, then send ``method`` with that ``sessionId``. - When ``target_id`` is None, ``method`` is sent at browser level — which - works for ``Target.*``, ``Browser.*``, ``Storage.*`` and a few other - globally-scoped domains. + With ``target_id``, ``Target.attachToTarget(flatten=True)`` multiplexes a + page-level session over the browser-level WebSocket; without it ``method`` + runs at browser level (``Target.*``, ``Browser.*``, ``Storage.*`` …). """ assert websockets is not None # guarded by _WS_AVAILABLE at call-site @@ -280,74 +226,44 @@ async def _cdp_call( ping_interval=None, # CDP server doesn't expect pings ) as ws: next_id = 1 - session_id: Optional[str] = None - # --- Step 1: attach to target if requested --- - if target_id: - attach_id = next_id + async def _send(req: Dict[str, Any], what: str) -> Dict[str, Any]: + nonlocal next_id + call_id = next_id next_id += 1 - await ws.send( - json.dumps( - { - "id": attach_id, - "method": "Target.attachToTarget", - "params": {"targetId": target_id, "flatten": True}, - } - ) - ) + await ws.send(json.dumps({"id": call_id, **req})) deadline = asyncio.get_running_loop().time() + timeout while True: remaining = deadline - asyncio.get_running_loop().time() if remaining <= 0: - raise TimeoutError( - f"Timed out attaching to target {target_id}" - ) - raw = await asyncio.wait_for(ws.recv(), timeout=remaining) - msg = json.loads(raw) - if msg.get("id") == attach_id: - if "error" in msg: - raise RuntimeError( - f"Target.attachToTarget failed: {msg['error']}" - ) - session_id = msg.get("result", {}).get("sessionId") - if not session_id: - raise RuntimeError( - "Target.attachToTarget did not return a sessionId" - ) - break - # Ignore events (messages without "id") while waiting + raise TimeoutError(f"Timed out {what}") + msg = json.loads(await asyncio.wait_for(ws.recv(), timeout=remaining)) + if msg.get("id") == call_id: + return msg + # Ignore events / out-of-order responses - # --- Step 2: dispatch the real method --- - call_id = next_id - next_id += 1 - req: Dict[str, Any] = { - "id": call_id, - "method": method, - "params": params or {}, - } + session_id: Optional[str] = None + if target_id: + msg = await _send( + { + "method": "Target.attachToTarget", + "params": {"targetId": target_id, "flatten": True}, + }, + f"attaching to target {target_id}", + ) + if "error" in msg: + raise RuntimeError(f"Target.attachToTarget failed: {msg['error']}") + session_id = msg.get("result", {}).get("sessionId") + if not session_id: + raise RuntimeError("Target.attachToTarget did not return a sessionId") + + req: Dict[str, Any] = {"method": method, "params": params or {}} if session_id: req["sessionId"] = session_id - await ws.send(json.dumps(req)) - - deadline = asyncio.get_running_loop().time() + timeout - while True: - remaining = deadline - asyncio.get_running_loop().time() - if remaining <= 0: - raise TimeoutError( - f"Timed out waiting for response to {method}" - ) - raw = await asyncio.wait_for(ws.recv(), timeout=remaining) - msg = json.loads(raw) - if msg.get("id") == call_id: - if "error" in msg: - raise RuntimeError(f"CDP error: {msg['error']}") - return msg.get("result", {}) - # Ignore events / out-of-order responses - - -# --------------------------------------------------------------------------- -# Public tool function -# --------------------------------------------------------------------------- + msg = await _send(req, f"waiting for response to {method}") + if "error" in msg: + raise RuntimeError(f"CDP error: {msg['error']}") + return msg.get("result", {}) def _browser_cdp_via_supervisor( @@ -357,13 +273,7 @@ def _browser_cdp_via_supervisor( params: Optional[Dict[str, Any]], timeout: float, ) -> str: - """Route a CDP call through the live supervisor session for an OOPIF frame. - - Looks up the frame in the supervisor's snapshot, extracts its child - ``cdp_session_id``, and dispatches ``method`` with that sessionId via - the supervisor's already-connected WebSocket (using - ``asyncio.run_coroutine_threadsafe`` onto the supervisor loop). - """ + """Route a CDP call through the live supervisor session for an OOPIF frame.""" try: from tools.browser_supervisor import SUPERVISOR_REGISTRY # type: ignore[import-not-found] except Exception as exc: # pragma: no cover — defensive @@ -383,7 +293,6 @@ def _browser_cdp_via_supervisor( ) snap = supervisor.snapshot() - # Search both the top frame and the children for the requested id. top = snap.frame_tree.get("top") frame_info: Optional[Dict[str, Any]] = None if top and top.get("frame_id") == frame_id: @@ -394,7 +303,7 @@ def _browser_cdp_via_supervisor( frame_info = child break if frame_info is None: - # Check the raw frames dict too (frame_tree is capped at 30 entries) + # frame_tree is capped at 30 entries — check the raw frames dict too. with supervisor._state_lock: # type: ignore[attr-defined] raw = supervisor._frames.get(frame_id) # type: ignore[attr-defined] if raw is not None: @@ -408,9 +317,8 @@ def _browser_cdp_via_supervisor( child_sid = frame_info.get("session_id") if not child_sid: - # Not an OOPIF — fall back to top-level session (evaluating at page - # scope). Same-origin iframes don't get their own sessionId; the - # agent can still use contentWindow/contentDocument from the parent. + # Same-origin iframes have no dedicated session; the agent reaches them + # via contentWindow/contentDocument from the parent instead. return tool_error( f"frame_id {frame_id!r} is not an out-of-process iframe (no " f"dedicated CDP session). For same-origin iframes, use " @@ -419,7 +327,6 @@ def _browser_cdp_via_supervisor( f"at the top-level page instead." ) - # Dispatch onto the supervisor's loop. loop = supervisor._loop # type: ignore[attr-defined] if loop is None or not loop.is_running(): return tool_error( @@ -468,37 +375,19 @@ def browser_cdp( timeout: float = 30.0, task_id: Optional[str] = None, ) -> str: - """Send a raw CDP command. See ``CDP_DOCS_URL`` for method documentation. + """Send a raw CDP command (see ``CDP_DOCS_URL``). - Args: - method: CDP method name, e.g. ``"Target.getTargets"``. - params: Method-specific parameters; defaults to ``{}``. - target_id: Optional target/tab ID for page-level methods. When set, - we first attach to the target (``flatten=True``) and send - ``method`` with the resulting ``sessionId``. Uses a fresh - stateless CDP connection. - frame_id: Optional cross-origin (OOPIF) iframe ``frame_id`` from - ``browser_snapshot.frame_tree.children[]``. When set (and the - frame is an OOPIF with a live session tracked by the CDP - supervisor), routes the call through the supervisor's existing - WebSocket — which is how you Runtime.evaluate *inside* an - iframe on backends where per-call fresh CDP connections would - hit signed-URL expiry (Browserbase) or expensive reattach. - timeout: Seconds to wait for the call to complete. - task_id: Task identifier for supervisor lookup. When ``frame_id`` - is set, this identifies which task's supervisor to use; the - handler will default to ``"default"`` otherwise. - - Returns: - JSON string ``{"success": True, "method": ..., "result": {...}}`` on - success, or ``{"error": "..."}`` on failure. + ``target_id`` attaches a fresh stateless connection to a tab; ``frame_id`` + (OOPIF from ``browser_snapshot.frame_tree``) routes through the supervisor's + live WebSocket instead — the only reliable way to evaluate inside an iframe + on backends where fresh per-call connections hit signed-URL expiry + (Browserbase). Returns JSON ``{"success": True, "method", "result"}`` or + ``{"error": ...}``. """ effective_task_id = task_id or "default" - # --- Route iframe-scoped calls through the supervisor --------------- if frame_id: - # Same private-page/SSRF boundary as the stateless path below — - # frame_id routing must not become the sibling bypass for it. + # Same private-page/SSRF boundary as the stateless path below. blocked = _browser_cdp_private_guard( task_id=effective_task_id, method=method, @@ -573,9 +462,7 @@ def browser_cdp( f"CDP call timed out after {safe_timeout}s: {exc}", method=method, ) - except TimeoutError as exc: - return tool_error(str(exc), method=method) - except RuntimeError as exc: + except (TimeoutError, RuntimeError) as exc: return tool_error(str(exc), method=method) except WebSocketException as exc: return tool_error( @@ -604,11 +491,6 @@ def browser_cdp( return json.dumps(payload, ensure_ascii=False) -# --------------------------------------------------------------------------- -# Registry -# --------------------------------------------------------------------------- - - BROWSER_CDP_SCHEMA: Dict[str, Any] = { "name": "browser_cdp", "description": ( @@ -708,21 +590,11 @@ BROWSER_CDP_SCHEMA: Dict[str, Any] = { def _browser_cdp_check() -> bool: - """Availability check for browser_cdp. + """Availability check: offered only when a static CDP URL is set. - The tool is only offered when the Python side can actually reach a CDP - endpoint right now — meaning a static URL is set via ``/browser connect`` - (``BROWSER_CDP_URL``) or ``browser.cdp_url`` in ``config.yaml``. - - Backends that do *not* currently expose CDP to us — Camofox (REST-only), - the default local agent-browser mode (Playwright hides its internal CDP - port), and cloud providers whose per-session ``cdp_url`` is not yet - surfaced — are gated out so the model doesn't see a tool that would - reliably fail. Cloud-provider CDP routing is a follow-up. - - Kept in a thin wrapper so the registration statement stays at module top - level (the tool-discovery AST scan only picks up top-level - ``registry.register(...)`` calls). + Camofox (REST-only), default local agent-browser (hidden CDP port) and cloud + providers whose per-session ``cdp_url`` isn't surfaced are gated out. Thin + wrapper so ``registry.register`` stays a top-level statement (AST scan). """ try: from tools.browser_tool import ( # type: ignore[import-not-found] @@ -734,9 +606,8 @@ def _browser_cdp_check() -> bool: return False if not check_browser_requirements(): return False - # Raw (no-I/O) gate: check_fns run during tool-schema assembly at every - # startup; resolving the endpoint over HTTP here would block launch when - # the configured endpoint is stale/unreachable. + # Raw (no-I/O) gate: check_fns run at every startup; resolving the + # endpoint over HTTP here would block launch on a stale endpoint. return bool(_get_cdp_override_raw()) diff --git a/tools/browser_dialog_tool.py b/tools/browser_dialog_tool.py index e37337b9bb..ac519489da 100644 --- a/tools/browser_dialog_tool.py +++ b/tools/browser_dialog_tool.py @@ -1,16 +1,8 @@ """Agent-facing tool: respond to a native JS dialog captured by the CDP supervisor. -This tool is response-only — the agent first reads ``pending_dialogs`` from -``browser_snapshot`` output, then calls ``browser_dialog(action=...)`` to -accept or dismiss. - -Gated on the same ``_browser_cdp_check`` as ``browser_cdp`` so it only -appears when a CDP endpoint is reachable (Browserbase with a -``connectUrl``, local Chromium-family browser via ``/browser connect``, or -``browser.cdp_url`` set in config). - -See ``website/docs/developer-guide/browser-supervisor.md`` for the full -design. +Response-only: the agent reads ``pending_dialogs`` from ``browser_snapshot``, +then calls ``browser_dialog(action=...)``. Gated on ``_browser_cdp_check`` so it +appears together with ``browser_cdp``. Design: ``website/docs/developer-guide/browser-supervisor.md``. """ from __future__ import annotations @@ -86,8 +78,7 @@ def browser_dialog( task_id: Optional[str] = None, ) -> str: """Respond to a pending dialog on the active task's CDP supervisor.""" - effective_task_id = task_id or "default" - supervisor = SUPERVISOR_REGISTRY.get(effective_task_id) + supervisor = SUPERVISOR_REGISTRY.get(task_id or "default") if supervisor is None: return json.dumps( { @@ -118,13 +109,7 @@ def browser_dialog( def _browser_dialog_check() -> bool: - """Gate: same as ``browser_cdp`` — only offered when CDP is reachable. - - Kept identical so the two tools appear and disappear together. The - supervisor itself is started lazily by ``browser_navigate`` / - ``/browser connect`` / Browserbase session creation, so a reachable - CDP URL is enough to commit to showing the tool. - """ + """Gate: same as ``browser_cdp`` so the two tools appear/disappear together.""" try: from tools.browser_cdp_tool import _browser_cdp_check # type: ignore[import-not-found] except Exception as exc: # pragma: no cover — defensive diff --git a/tools/browser_extension_router.py b/tools/browser_extension_router.py index f9cbecc305..b5c1c27a85 100644 --- a/tools/browser_extension_router.py +++ b/tools/browser_extension_router.py @@ -1,74 +1,64 @@ """Registry-level browser extension router. -This module is the *agent-side* half of the browser-extension-control -feature: it decides, for one registry ``browser_*`` handler invocation, -whether the command is executed by an attached extension controller (via -the :mod:`gateway.browser_control_broker`) or by the existing legacy -browser backend. +Agent-side half of browser-extension-control: decides, per ``browser_*`` handler +invocation, whether an attached extension controller (via +:mod:`gateway.browser_control_broker`) or the legacy backend executes it. -Routing contract (exercised by ``tests/tools/test_browser_extension_router.py``): +Routing contract (see ``tests/tools/test_browser_extension_router.py``): -- **Feature off ⇒ legacy, untouched.** When ``enabled`` is false the broker - is never touched and ``fallback()`` is called exactly once. This is the - default: ``browser.extension_control.enabled`` is false unless explicitly - configured, so every real browser action keeps its exact legacy path. +- Feature off ⇒ legacy, broker never touched, ``fallback()`` called exactly once. +- No server-bound identity ⇒ legacy. +- Bound identity ⇒ authoritative extension lane; missing/ambiguous scope, + disconnect, or capability mismatch fail closed (never jump to another browser). +- Selected controller ⇒ authoritative; its errors propagate, legacy never retried. +- ``args`` is never mutated. -- **No server-bound identity ⇒ legacy.** Generic Hermes callers keep the - existing backend when no authenticated browser-controller identity is bound. - -- **Bound identity ⇒ authoritative extension lane.** Once the gateway binds a - browser-controller principal and transport family, missing/ambiguous scope, - disconnect, or capability mismatch fail closed. A "control this tab" turn - must never jump to an unrelated local/cloud browser backend. - -- **Selected controller ⇒ authoritative.** Once a controller is selected the - command is dispatched to it and its result returned; the legacy backend - is *never* retried, even when the controller fails (timeout, cancellation, - rejection, transport error all propagate to the caller). - -- **Arguments are never mutated.** ``args`` is passed through untouched; - the broker copies arguments into its command frame itself. - -The lazy wrapper :func:`routed_browser_handler` is what the ``browser_*`` -registry handlers call. It resolves the feature flag and the process-local -broker lazily on every invocation so importing this module (or -``tools.browser_tool``) never pulls in the gateway, and so a mid-process -config change is honored without restart. +:func:`routed_browser_handler` resolves the flag and broker lazily on every call +so importing this module never pulls in the gateway and config changes apply +without restart. """ from __future__ import annotations import json import logging -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, Optional, Tuple logger = logging.getLogger(__name__) +def _bound_identity() -> Tuple[Optional[str], Optional[str], Optional[str]]: + """(session_id, principal_id, transport_family) from the session context.""" + from gateway.session_context import get_session_env + + return ( + get_session_env("HERMES_SESSION_ID", "") or None, + get_session_env("HERMES_BROWSER_CONTROL_PRINCIPAL", "") or None, + get_session_env("HERMES_BROWSER_CONTROL_TRANSPORT_FAMILY", "") or None, + ) + + +def _controller_unavailable(message: str) -> Exception: + from gateway.browser_control_broker import ControllerUnavailable + + return ControllerUnavailable(message) + + def extension_controller_available(action: str) -> bool: """Whether this request owns one exact controller capable of ``action``. - Tool-schema assembly runs inside the API request's session context, before - a model can call a browser tool. The legacy browser backend's availability - probe cannot decide whether the extension route is usable, so routeable - tools consult the process-local broker directly. Missing server-bound - identity, ambiguous scope, a detached controller, or a capability mismatch - all fail closed. + Runs during tool-schema assembly inside the request's session context; + consults the process-local broker directly and fails closed on any gap. """ try: from gateway.browser_control_broker import ( browser_control_enabled, get_browser_control_broker, ) - from gateway.session_context import get_session_env if not browser_control_enabled(): return False - session_id = get_session_env("HERMES_SESSION_ID", "") or None - principal_id = get_session_env("HERMES_BROWSER_CONTROL_PRINCIPAL", "") or None - transport_family = get_session_env( - "HERMES_BROWSER_CONTROL_TRANSPORT_FAMILY", "" - ) or None + session_id, principal_id, transport_family = _bound_identity() if not session_id or not principal_id or not transport_family: return False broker = get_browser_control_broker() @@ -102,37 +92,10 @@ def route_browser_tool( ) -> Any: """Route one browser action through the extension-control broker. - Parameters - ---------- - action: - Registry tool name / controller capability, e.g. ``"browser_navigate"``. - args: - Tool arguments as received from the model. Never mutated. - fallback: - The existing backend handler, called exactly once when the feature is - off or no server-bound controller identity exists. Must be a - zero-argument callable. - broker: - Object exposing ``scope_for_session(**identity) -> scope|None``, - ``select(scope, capability) -> controller|None`` and - ``dispatch(scope, *, action, arguments, tool_call_id)``. The real - implementation is ``gateway.browser_control_broker``. - enabled: - Feature flag; false bypasses the broker entirely. - session_id/task_id: - Caller session hints forwarded to ``scope_for_session``. - principal_id/transport_family: - Server-bound caller identity. Both are mandatory when the feature is - enabled; missing values preserve the existing backend for generic - Hermes callers. - tool_call_id: - Caller tool-call id forwarded verbatim to ``dispatch``. - - Returns - ------- - The legacy backend's return value when falling back, or the controller's - completion result when routed. Exceptions from a selected controller are - propagated — the legacy backend is never retried after selection. + ``broker`` exposes ``scope_for_session(**identity)``, ``select(scope, cap)`` + and ``dispatch(scope, *, action, arguments, tool_call_id)``. ``fallback`` is + called exactly once when the feature is off or no server-bound identity + exists; once a controller is selected its result/exception is final. """ if not enabled: return fallback() @@ -140,47 +103,33 @@ def route_browser_tool( if not str(principal_id or "").strip() or not str(transport_family or "").strip(): return fallback() - scope = broker.scope_for_session( + identity = dict( session_id=session_id, task_id=task_id, principal_id=principal_id, transport_family=transport_family, ) + scope = broker.scope_for_session(**identity) if scope is None: - # A stamped identity alone does not make the extension lane - # authoritative — authentication happens at transport auth, but the - # lane only BINDS when a controller actually registers for it. If no - # controller ever registered, generic callers keep the legacy - # backend. Once a lane registered (even if the controller is - # currently offline/ambiguous), fail closed: a "control this tab" - # session must never silently jump to an unrelated browser. + # A stamped identity only becomes authoritative once a controller has + # registered for the lane; unregistered lanes keep the legacy backend, + # registered-but-offline lanes fail closed. lane_bound = getattr(broker, "lane_registered", None) - if callable(lane_bound) and not lane_bound( - session_id=session_id, - task_id=task_id, - principal_id=principal_id, - transport_family=transport_family, - ): + if callable(lane_bound) and not lane_bound(**identity): return fallback() - from gateway.browser_control_broker import ControllerUnavailable - - raise ControllerUnavailable( + raise _controller_unavailable( f"bound browser controller unavailable for {action}" ) controller = broker.select(scope, action) if controller is None: - from gateway.browser_control_broker import ControllerUnavailable - - raise ControllerUnavailable( + raise _controller_unavailable( f"bound browser controller cannot execute {action}" ) - # A controller was selected: it is authoritative. Never retry through the - # existing backend, whatever happens here. Registry handlers must return a - # string (or the dedicated multimodal envelope), while controller transports - # naturally complete with decoded JSON values. Preserve existing string - # results byte-for-byte and serialize decoded values at this boundary. + # Controller is authoritative: never retry the legacy backend. Registry + # handlers must return a string; keep string results byte-identical and + # serialize decoded JSON values at this boundary. result = broker.dispatch( scope, action=action, arguments=args, tool_call_id=tool_call_id ) @@ -190,13 +139,7 @@ def route_browser_tool( def current_tool_call_id() -> str: - """Return the active tool_call_id, or ``""`` when none is bound. - - The agent executor binds the id via - ``tools.approval.set_current_observability_context`` immediately before - registry dispatch, so the registry handler (and this router) can read it - back from the same context. Bare/offline callers have no binding. - """ + """Active tool_call_id bound by the agent executor, or ``""`` when none.""" try: from tools.approval import _approval_tool_call_id @@ -218,11 +161,7 @@ def routed_browser_handler( ) -> Any: """Lazy registry-handler route wrapper for ``browser_*`` tools. - Resolves the feature flag and process-local broker lazily so the - default (feature off) path costs one cached config read and an immediate - fallback, and so importing ``tools.browser_tool`` never imports the - gateway. When the gateway cannot be imported or the feature is off, the - legacy handler runs unchanged. + Feature off (or gateway unimportable) ⇒ the legacy handler runs unchanged. """ try: from gateway.browser_control_broker import ( @@ -243,15 +182,10 @@ def routed_browser_handler( tool_call_id = current_tool_call_id() try: - from gateway.session_context import get_session_env - - session_id = session_id or get_session_env("HERMES_SESSION_ID", "") or None - principal_id = principal_id or get_session_env( - "HERMES_BROWSER_CONTROL_PRINCIPAL", "" - ) or None - transport_family = transport_family or get_session_env( - "HERMES_BROWSER_CONTROL_TRANSPORT_FAMILY", "" - ) or None + env_session, env_principal, env_transport = _bound_identity() + session_id = session_id or env_session + principal_id = principal_id or env_principal + transport_family = transport_family or env_transport except Exception: pass diff --git a/tools/browser_lightpanda.py b/tools/browser_lightpanda.py index bb64d2c82a..3bb8faf79c 100644 --- a/tools/browser_lightpanda.py +++ b/tools/browser_lightpanda.py @@ -1,15 +1,12 @@ """Lightpanda local engine for Browser Use mode. -With ``browser.engine: lightpanda``, Browser Use mode spawns one -``lightpanda serve`` per browser session and points ``browser_exec`` at its -CDP endpoint (``BU_CDP_URL``). The built-in ``browser_*`` tools keep driving -Lightpanda through ``agent-browser --engine lightpanda``; this module is the -launcher for the path where no agent-browser daemon is involved. - -Lifecycle: ``tools.browser_tool`` owns the session cache, the inactivity -reaper and the atexit sweep; it calls :func:`launch_lightpanda` / -:func:`stop_lightpanda` and :func:`reap_orphaned_lightpanda` for processes -left behind by a crashed Hermes. +With ``browser.engine: lightpanda``, Browser Use mode spawns one ``lightpanda +serve`` per browser session and points ``browser_exec`` at its CDP endpoint +(``BU_CDP_URL``); the built-in ``browser_*`` tools keep going through +``agent-browser --engine lightpanda``. ``tools.browser_tool`` owns the session +cache, inactivity reaper and atexit sweep, calling :func:`launch_lightpanda`, +:func:`stop_lightpanda` and :func:`reap_orphaned_lightpanda` (for processes a +crashed Hermes left behind). """ import json @@ -50,8 +47,7 @@ class LightpandaServer: @property def cdp_url(self) -> str: - # The http discovery URL: the browser-use harness resolves - # /json/version itself on every daemon start (BU_CDP_URL). + # HTTP discovery URL; the browser-use harness resolves /json/version itself. return f"http://127.0.0.1:{self.port}" def is_alive(self) -> bool: @@ -171,6 +167,13 @@ def _safe_start_time(pid: int) -> Optional[int]: return None +def _tree_kill(pid: int, expected_start) -> None: + """Tree-kill ``pid`` via ProcessRegistry, verifying its start time first.""" + from tools.process_registry import ProcessRegistry + + ProcessRegistry._terminate_host_pid(pid, expected_start=expected_start) + + def _write_record(server: LightpandaServer) -> None: record = { "pid": server.proc.pid, @@ -221,10 +224,7 @@ def launch_lightpanda( argv.append("--block-private-networks") log_path = str(_state_dir() / f"{session_name}.log") - # No Windows branch here: find_lightpanda_binary() returns None on nt, - # so launch always errors out above before reaching the spawn. - popen_kwargs = {"start_new_session": True} - + # find_lightpanda_binary() returns None on nt, so no Windows spawn branch is needed. try: with open(log_path, "wb") as log_file: proc = subprocess.Popen( @@ -233,7 +233,7 @@ def launch_lightpanda( stdout=subprocess.DEVNULL, stderr=log_file, env=_browser_env(), - **popen_kwargs, + start_new_session=True, ) except (OSError, subprocess.SubprocessError) as e: return None, f"Failed to launch lightpanda serve ({binary}): {e}" @@ -292,11 +292,7 @@ def stop_lightpanda(session_name: str) -> None: return if server.is_alive(): try: - from tools.process_registry import ProcessRegistry - - ProcessRegistry._terminate_host_pid( - server.proc.pid, expected_start=server.start_time - ) + _tree_kill(server.proc.pid, server.start_time) except Exception as e: logger.debug("lightpanda tree-kill failed for %s: %s", session_name, e) _terminate(server.proc) @@ -380,9 +376,7 @@ def reap_orphaned_lightpanda() -> int: record_path.unlink(missing_ok=True) continue try: - from tools.process_registry import ProcessRegistry - - ProcessRegistry._terminate_host_pid(int(pid), expected_start=record.get("start_time")) + _tree_kill(int(pid), record.get("start_time")) reaped += 1 logger.info("Reaped orphaned lightpanda serve pid %s (session %s)", pid, session_name) except Exception as e: diff --git a/tools/browser_supervisor.py b/tools/browser_supervisor.py index d17806f4b2..86e6aaf41d 100644 --- a/tools/browser_supervisor.py +++ b/tools/browser_supervisor.py @@ -1,20 +1,14 @@ """Persistent CDP supervisor for browser dialog + frame detection. -One ``CDPSupervisor`` runs per Hermes ``task_id`` that has a reachable CDP -endpoint. It holds a single persistent WebSocket to the backend, subscribes -to ``Page`` / ``Runtime`` / ``Target`` events on every attached session -(top-level page and every OOPIF / worker target that auto-attaches), and -surfaces observable state — pending dialogs and frame tree — through a -thread-safe snapshot object that tool handlers consume synchronously. - -The supervisor is NOT in the agent's tool schema. Its output reaches the -agent via two channels: - -1. ``browser_snapshot`` merges supervisor state into its return payload - (see ``tools/browser_tool.py``). -2. ``browser_dialog`` tool responds to a pending dialog by calling - ``respond_to_dialog()`` on the active supervisor. +One ``CDPSupervisor`` runs per Hermes ``task_id`` with a reachable CDP endpoint. +It holds one persistent WebSocket, subscribes to ``Page`` / ``Runtime`` / +``Target`` events on every attached session (top page + auto-attached OOPIF / +worker targets), and exposes pending dialogs + frame tree through a +thread-safe snapshot that tool handlers read synchronously. +Not in the agent's tool schema. Output reaches the agent via +``browser_snapshot`` (merges supervisor state, see ``tools/browser_tool.py``) +and ``browser_dialog`` (calls ``respond_to_dialog()``). Design spec: ``website/docs/developer-guide/browser-supervisor.md``. """ @@ -25,13 +19,11 @@ import json import logging import threading import time -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING +from dataclasses import dataclass, replace +from typing import Any, Callable, Coroutine, Dict, List, Optional, Tuple, TYPE_CHECKING -# ``websockets`` costs ~22 ms at import and is only needed when a supervisor -# actually connects to a CDP endpoint (``_connect_ws``). With -# ``from __future__ import annotations`` in force the ``ClientConnection`` -# annotation is string-only, so the type import stays under TYPE_CHECKING. +# ``websockets`` costs ~22 ms at import and is only needed once a supervisor +# connects; with postponed annotations the type import stays under TYPE_CHECKING. if TYPE_CHECKING: from websockets.asyncio.client import ClientConnection @@ -39,14 +31,11 @@ logger = logging.getLogger(__name__) def _redact_cdp_error_text(exc: object) -> str: - """Redact any CDP endpoint credentials from an error's string form. + """Redact CDP endpoint credentials from an exception's string form. - ``websockets`` bakes the raw target URL into its exception messages - (``InvalidURI``, connection errors, TLS failures all embed the full - ``self.cdp_url`` — including a ``?token=`` query credential or - ``user:pass@`` userinfo). Every supervisor egress point that turns such an - exception into log text or a re-raised message MUST route through here so - those credentials never reach Hermes logs or tracebacks. Falls back to a + ``websockets`` bakes the raw target URL (``?token=`` / ``user:pass@``) into + its exception messages. Every egress point that turns such an exception into + log text or a re-raised message MUST route through here; falls back to a fixed sentinel if redaction itself raises, erring toward masking. """ try: @@ -81,25 +70,23 @@ DEFAULT_DIALOG_TIMEOUT_S = 300.0 FRAME_TREE_MAX_ENTRIES = 30 FRAME_TREE_MAX_OOPIF_DEPTH = 2 -# Ring buffer of recent console-level events (used later by PR 2 diagnostics). +# Ring buffer of recent console-level events. CONSOLE_HISTORY_MAX = 50 -# Keep the last N closed dialogs in ``recent_dialogs`` so agents on backends -# that auto-dismiss server-side (e.g. Browserbase) can still observe that a -# dialog fired, even if they couldn't respond to it in time. +# Last N closed dialogs kept in ``recent_dialogs`` so agents on backends that +# auto-dismiss server-side (Browserbase) can still observe that a dialog fired. RECENT_DIALOGS_MAX = 20 -# Magic host the injected dialog bridge XHRs to. Intercepted via the CDP -# Fetch domain before any network resolution happens, so the hostname never -# has to exist. Keep this ASCII + URL-safe; we also gate Fetch patterns on it. +# Magic host the injected dialog bridge XHRs to. Intercepted via the CDP Fetch +# domain before any network resolution, so it never has to exist. Keep ASCII + +# URL-safe; Fetch patterns are gated on it. DIALOG_BRIDGE_HOST = "hermes-dialog-bridge.invalid" DIALOG_BRIDGE_URL_PATTERN = f"http://{DIALOG_BRIDGE_HOST}/*" -# Script injected into every frame via Page.addScriptToEvaluateOnNewDocument. -# Overrides alert/confirm/prompt to round-trip through a sync XHR that we -# intercept via Fetch.requestPaused. Works on Browserbase (whose CDP proxy -# auto-dismisses REAL native dialogs) because the native dialogs never fire -# in the first place — the overrides take precedence. +# Injected into every frame via Page.addScriptToEvaluateOnNewDocument. Overrides +# alert/confirm/prompt to round-trip through a sync XHR we intercept via +# Fetch.requestPaused. Works on Browserbase (whose CDP proxy auto-dismisses REAL +# native dialogs) because the native dialogs never fire. _DIALOG_BRIDGE_SCRIPT = r""" (() => { if (window.__hermesDialogBridgeInstalled) return; @@ -168,9 +155,8 @@ class PendingDialog: opened_at: float cdp_session_id: str # which attached CDP session the dialog fired in frame_id: Optional[str] = None - # When set, the dialog was captured via the bridge XHR path (Fetch domain). - # Response must be delivered via Fetch.fulfillRequest, NOT - # Page.handleJavaScriptDialog — the native dialog never fired. + # Set when captured via the bridge XHR path: respond via Fetch.fulfillRequest, + # NOT Page.handleJavaScriptDialog — the native dialog never fired. bridge_request_id: Optional[str] = None def to_dict(self) -> Dict[str, Any]: @@ -186,12 +172,7 @@ class PendingDialog: @dataclass class DialogRecord: - """A historical record of a dialog that was opened and then handled. - - Retained in ``recent_dialogs`` for a short window so agents on backends - that auto-dismiss dialogs server-side (Browserbase) can still observe - that a dialog fired, even though they couldn't respond to it. - """ + """A dialog that was opened and then handled (kept briefly in ``recent_dialogs``).""" id: str type: str @@ -217,9 +198,9 @@ class DialogRecord: class FrameInfo: """One frame in the page's frame tree. - ``is_oopif`` means the frame has its own CDP target (separate process, - reachable via ``cdp_session_id``). Same-origin / srcdoc iframes share - the parent process and have ``is_oopif=False`` + ``cdp_session_id=None``. + ``is_oopif`` frames have their own CDP target (reachable via + ``cdp_session_id``); same-origin / srcdoc iframes share the parent process + and have ``is_oopif=False`` + ``cdp_session_id=None``. """ frame_id: str @@ -258,11 +239,7 @@ class ConsoleEvent: @dataclass(frozen=True) class SupervisorSnapshot: - """Read-only snapshot of supervisor state. - - Frozen dataclass so tool handlers can freely dereference without - worrying about mutation under their feet. - """ + """Read-only (frozen) snapshot of supervisor state for tool handlers.""" pending_dialogs: Tuple[PendingDialog, ...] recent_dialogs: Tuple[DialogRecord, ...] @@ -289,18 +266,11 @@ class SupervisorSnapshot: class CDPSupervisor: """One supervisor per (task_id, cdp_url) pair. - Lifecycle: - * ``start()`` — kicked off by ``SupervisorRegistry.get_or_start``; spawns - a daemon thread running its own asyncio loop, connects the WebSocket, - attaches to the first page target, enables domains, starts - auto-attaching to child targets. - * ``snapshot()`` — sync, thread-safe, called from tool handlers. - * ``respond_to_dialog(action, ...)`` — sync bridge; schedules a coroutine - on the supervisor's loop and waits (with timeout) for the CDP ack. - * ``stop()`` — cancels task, closes WebSocket, joins thread. - - All CDP I/O lives on the supervisor's own loop. External callers never - touch the loop directly; they go through the sync API above. + ``start()`` spawns a daemon thread running its own asyncio loop, connects, + attaches to the first page target, enables domains and auto-attach. + ``snapshot()`` / ``respond_to_dialog()`` / ``evaluate_runtime()`` are sync, + thread-safe bridges onto that loop; ``stop()`` tears it down. All CDP I/O + lives on the supervisor's own loop. """ def __init__( @@ -343,20 +313,17 @@ class CDPSupervisor: self._page_session_id: Optional[str] = None self._child_sessions: Dict[str, Dict[str, Any]] = {} # session_id -> info - # Dialog auto-dismiss watchdog handles (per dialog id). + # Dialog auto-dismiss watchdog handles (per dialog id) + id generator. self._dialog_watchdogs: Dict[str, asyncio.TimerHandle] = {} - # Monotonic id generator for dialogs (human-readable in snapshots). self._dialog_seq = 0 # ── Public sync API ────────────────────────────────────────────────────── def start(self, timeout: float = 15.0) -> None: - """Launch the background loop and wait until attachment is complete. + """Launch the background loop and block until attachment completes. - Raises whatever exception attach failed with (connect error, bad - WebSocket URL, CDP domain enable failure, etc.). On success, the - supervisor is fully wired up — pending-dialog events will be captured - as of the moment ``start()`` returns. + Raises whatever attach failed with (redacted). On return, dialog events + are already being captured. """ if self._thread and self._thread.is_alive(): return @@ -383,11 +350,9 @@ class CDPSupervisor: if self._start_error is not None: err = self._start_error self.stop() - # ``err`` is a raw ``websockets`` exception whose message embeds the - # full cdp_url (token / userinfo). Re-raise a redacted RuntimeError - # and suppress the raw cause (``from None``) so no credential leaks - # via the message OR the traceback chain. Type is not load-bearing: - # the sole caller (_ensure_cdp_supervisor) only logs it. + # ``err`` is a raw ``websockets`` exception embedding the full cdp_url + # (token / userinfo). Re-raise redacted and suppress the cause + # (``from None``) so nothing leaks via message OR traceback chain. raise RuntimeError( f"CDP supervisor failed to start: {_redact_cdp_error_text(err)}" ) from None @@ -397,21 +362,12 @@ class CDPSupervisor: self._stop_requested = True loop = self._loop if loop is not None and loop.is_running(): - # Close the WebSocket from inside the loop — this makes ``async for - # raw in self._ws`` return cleanly, ``_run`` hits its ``finally``, - # pending tasks get cancelled in order, THEN the thread exits. - async def _close_ws(): - ws = self._ws - self._ws = None - if ws is not None: - try: - await ws.close() - except Exception: - pass - + # Close the WebSocket from inside the loop so ``async for raw in + # self._ws`` returns cleanly, ``_run`` hits its ``finally``, pending + # tasks cancel in order, THEN the thread exits. try: from agent.async_utils import safe_schedule_threadsafe - fut = safe_schedule_threadsafe(_close_ws(), loop) + fut = safe_schedule_threadsafe(self._close_ws(), loop) if fut is not None: try: fut.result(timeout=2.0) @@ -450,11 +406,10 @@ class CDPSupervisor: dialog_id: Optional[str] = None, timeout: float = 10.0, ) -> Dict[str, Any]: - """Accept/dismiss a pending dialog. Sync bridge onto the supervisor loop. + """Accept/dismiss a pending dialog (sync bridge onto the supervisor loop). - Returns ``{"ok": True, "dialog": {...}}`` on success, - ``{"ok": False, "error": "..."}`` on a recoverable error (no dialog, - ambiguous dialog_id, supervisor inactive). + Returns ``{"ok": True, "dialog": {...}}`` or ``{"ok": False, "error": ...}`` + for recoverable errors (no dialog, ambiguous dialog_id, inactive). """ if action not in {"accept", "dismiss"}: return {"ok": False, "error": f"action must be 'accept' or 'dismiss', got {action!r}"} @@ -514,17 +469,11 @@ class CDPSupervisor: ) -> Dict[str, Any]: """Evaluate ``expression`` in the page's Runtime context over the live WS. - Reuses the supervisor's already-connected WebSocket — zero subprocess - startup cost vs the agent-browser CLI ``eval`` command (which does - fork+exec+Node-startup+CDP-setup on every call). - - Returns a dict shaped like ``{"ok": True, "result": , "result_type": "..."}`` - on success, or ``{"ok": False, "error": "..."}`` on failure. - - ``return_by_value=True`` asks the browser to JSON-serialize the result - before sending it back, matching DevTools-console semantics for - primitive / plain-object expressions. For DOM nodes or non-serializable - objects, the browser returns a description string in ``result_type``. + Zero subprocess cost vs the agent-browser CLI ``eval``. Returns + ``{"ok": True, "result": , "result_type": ...}`` or + ``{"ok": False, "error": ...}``. ``return_by_value=True`` JSON-serializes + the result (DevTools-console semantics); non-serializable objects come + back as a description string. """ loop = self._loop if loop is None or not loop.is_running(): @@ -545,8 +494,7 @@ class CDPSupervisor: "expression": expression, "returnByValue": by_value, "awaitPromise": await_promise, - # userGesture matters for things like clipboard / fullscreen - # APIs that require a user-activation context. + # userGesture: clipboard / fullscreen APIs need user activation. "userGesture": True, }, session_id=session_id, @@ -564,14 +512,10 @@ class CDPSupervisor: try: response = _run_eval(return_by_value) except Exception as exc: - # ``returnByValue=True`` asks Chrome to deep-serialize the result. - # For live DOM nodes / NodeLists / Window that serialization can - # blow past CDP's recursion guard and fail the whole call with - # ``Object reference chain is too long`` (a protocol-level error, - # not a JS exception). Retry once with ``returnByValue=False`` so - # Chrome returns the object's description string instead — the same - # graceful degradation path used for ``document.querySelector(...)`` - # results — rather than crashing the eval. + # Deep-serializing live DOM nodes / NodeLists / Window can blow past + # CDP's recursion guard with the protocol-level error ``Object + # reference chain is too long``. Retry once with returnByValue=False + # so Chrome returns the description string instead of failing. if return_by_value and "reference chain is too long" in str(exc).lower(): try: response = _run_eval(False) @@ -580,13 +524,10 @@ class CDPSupervisor: else: return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} - # Runtime.evaluate response shape: - # {"id": N, "result": {"result": {"type": "...", "value": ..., ...}, - # "exceptionDetails": {...} (only on error)}} + # Response: {"result": {"result": {"type", "value", ...}, "exceptionDetails"?}} result_payload = response.get("result", {}) if isinstance(response, dict) else {} exception_details = result_payload.get("exceptionDetails") if exception_details: - # Surface the JS-side exception with a clean message. exc_text = exception_details.get("text") or "JavaScript exception" exc_obj = exception_details.get("exception") or {} description = exc_obj.get("description") @@ -602,8 +543,8 @@ class CDPSupervisor: elif result_type == "undefined": value = None else: - # Non-serializable (functions, DOM nodes, etc.) — return the - # browser's string description so the model gets *something*. + # Non-serializable (functions, DOM nodes…) — give the model the + # browser's description so it gets *something*. value = result_obj.get("description") or result_obj.get("unserializableValue") return {"ok": True, "result": value, "result_type": result_type} @@ -624,8 +565,8 @@ class CDPSupervisor: else: logger.warning("CDP supervisor %s crashed: %s", self.task_id, e) finally: - # Flush any remaining tasks before closing the loop so we don't - # emit "Task was destroyed but it is pending" warnings. + # Flush remaining tasks before closing the loop to avoid + # "Task was destroyed but it is pending" warnings. try: pending = [t for t in asyncio.all_tasks(loop) if not t.done()] for t in pending: @@ -641,14 +582,22 @@ class CDPSupervisor: with self._state_lock: self._active = False - async def _run(self) -> None: - """Top-level supervisor coroutine. + async def _close_ws(self) -> None: + """Detach and close the current WebSocket, swallowing close errors.""" + ws = self._ws + self._ws = None + if ws is not None: + try: + await ws.close() + except Exception: + pass - Holds a reconnecting loop so we survive the remote closing the - WebSocket — Browserbase in particular tears down the CDP socket - every time a short-lived client (e.g. agent-browser's per-command - CDP client) disconnects. We drop our state snapshot keys that - depend on specific CDP session ids, re-attach, and keep going. + async def _run(self) -> None: + """Top-level reconnecting supervisor coroutine. + + Browserbase tears down the CDP socket every time a short-lived client + (e.g. agent-browser's per-command CDP client) disconnects, so on drop we + reset per-session ids, re-attach, and keep going. """ attempt = 0 last_success_at = 0.0 @@ -677,16 +626,12 @@ class CDPSupervisor: reader_task = asyncio.create_task(self._read_loop(), name="cdp-reader") try: - # Reset per-connection session state so stale ids don't hang - # around after a reconnect. + # Reset per-connection session ids. ``_pending_dialogs`` and + # ``_frames`` are deliberately kept — they reconcile as fresh + # events arrive; worst case a stale dialog entry is rejected + # with "no dialog is showing" (logged, not surfaced). self._page_session_id = None self._child_sessions.clear() - # We deliberately keep `_pending_dialogs` and `_frames` — - # they're reconciled as the supervisor resubscribes and - # receives fresh events. Worst case: an agent sees a stale - # dialog entry that the new session's handleJavaScriptDialog - # call rejects with "no dialog is showing" (logged, not - # surfaced). await self._attach_initial_page() with self._state_lock: self._active = True @@ -694,7 +639,6 @@ class CDPSupervisor: backoff = 0.5 # reset after a successful attach if not self._ready_event.is_set(): self._ready_event.set() - # Run until the reader returns. await reader_task except BaseException as e: if not self._ready_event.is_set(): @@ -720,18 +664,11 @@ class CDPSupervisor: for handle in list(self._dialog_watchdogs.values()): handle.cancel() self._dialog_watchdogs.clear() - ws = self._ws - self._ws = None - if ws is not None: - try: - await ws.close() - except Exception: - pass + await self._close_ws() if self._stop_requested: return - # Reconnect: brief backoff, then reattach. logger.debug( "CDP supervisor %s: reconnecting in %.1fs...", self.task_id, backoff, ) @@ -761,25 +698,16 @@ class CDPSupervisor: {"autoAttach": True, "waitForDebuggerOnStart": False, "flatten": True}, session_id=self._page_session_id, ) - # Install the dialog bridge — overrides native alert/confirm/prompt with - # a synchronous XHR we intercept via Fetch domain. This is how we make - # dialog response work on Browserbase (whose CDP proxy auto-dismisses - # real native dialogs before we can call handleJavaScriptDialog). await self._install_dialog_bridge(self._page_session_id) async def _install_dialog_bridge(self, session_id: str) -> None: """Install the dialog-bridge init script + Fetch interceptor on a session. - Two CDP calls: - 1. ``Page.addScriptToEvaluateOnNewDocument`` — the JS override runs - in every frame before any page script. Replaces alert/confirm/ - prompt with a sync XHR to our bridge URL. - 2. ``Fetch.enable`` scoped to the bridge URL — we catch those XHRs, - surface them as pending dialogs, then fulfill once the agent - responds. - - Idempotent at the CDP level: Chromium de-duplicates identical - add-script calls by source, and Fetch.enable replaces prior patterns. + The JS override runs in every frame before page scripts; Fetch.enable + scoped to the bridge URL catches the XHRs, which surface as pending + dialogs and are fulfilled when the agent responds. Idempotent at the CDP + level (Chromium de-dupes identical add-script calls; Fetch.enable + replaces prior patterns). """ try: await self._cdp( @@ -813,8 +741,8 @@ class CDPSupervisor: "dialog bridge: Fetch.enable failed on sid=%s: %s", (session_id or "")[:16], e, ) - # Also try to inject into the already-loaded document so existing - # pages pick up the override on reconnect. Best-effort. + # Best-effort inject into the already-loaded document so existing pages + # pick up the override on reconnect. try: await self._cdp( "Runtime.evaluate", @@ -882,57 +810,66 @@ class CDPSupervisor: async def _on_event( self, method: str, params: Dict[str, Any], session_id: Optional[str] ) -> None: - if method == "Page.javascriptDialogOpening": - await self._on_dialog_opening(params, session_id) - elif method == "Page.javascriptDialogClosed": - await self._on_dialog_closed(params, session_id) - elif method == "Fetch.requestPaused": - await self._on_fetch_paused(params, session_id) - elif method == "Page.frameAttached": - self._on_frame_attached(params, session_id) - elif method == "Page.frameNavigated": - self._on_frame_navigated(params, session_id) - elif method == "Page.frameDetached": - self._on_frame_detached(params, session_id) - elif method == "Target.attachedToTarget": - await self._on_target_attached(params) - elif method == "Target.detachedFromTarget": - self._on_target_detached(params) - elif method == "Runtime.consoleAPICalled": - self._on_console(params, level_from="api") - elif method == "Runtime.exceptionThrown": - self._on_console(params, level_from="exception") + handler = self._EVENT_HANDLERS.get(method) + if handler is None: + return + result = handler(self, params, session_id) + if result is not None: + await result async def _on_dialog_opening( self, params: Dict[str, Any], session_id: Optional[str] ) -> None: - self._dialog_seq += 1 - dialog = PendingDialog( - id=f"d-{self._dialog_seq}", + dialog = self._new_dialog( type=str(params.get("type") or ""), message=str(params.get("message") or ""), default_prompt=str(params.get("defaultPrompt") or ""), - opened_at=time.time(), - cdp_session_id=session_id or self._page_session_id or "", + session_id=session_id, frame_id=params.get("frameId"), ) + self._admit_dialog(dialog, self._auto_handle_dialog) + def _new_dialog( + self, + *, + type: str, + message: str, + default_prompt: str, + session_id: Optional[str], + frame_id: Optional[str], + bridge_request_id: Optional[str] = None, + ) -> PendingDialog: + self._dialog_seq += 1 + return PendingDialog( + id=f"d-{self._dialog_seq}", + type=type, + message=message, + default_prompt=default_prompt, + opened_at=time.time(), + cdp_session_id=session_id or self._page_session_id or "", + frame_id=frame_id, + bridge_request_id=bridge_request_id, + ) + + def _admit_dialog( + self, + dialog: PendingDialog, + responder: Callable[..., Coroutine[Any, Any, None]], + ) -> None: + """Apply the dialog policy: auto-respond via ``responder`` or queue + arm watchdog. + + Auto policies archive FIRST (tagged ``auto_policy``) so the ``closed`` + event that follows our own response isn't re-archived as ``remote``. + """ if self.dialog_policy == DIALOG_POLICY_AUTO_DISMISS: - # Archive immediately with the policy tag so the ``closed`` event - # arriving right after our handleJavaScriptDialog call doesn't - # re-archive it as "remote". with self._state_lock: self._archive_dialog_locked(dialog, "auto_policy") - asyncio.create_task( - self._auto_handle_dialog(dialog, accept=False, prompt_text="") - ) + asyncio.create_task(responder(dialog, accept=False, prompt_text="")) elif self.dialog_policy == DIALOG_POLICY_AUTO_ACCEPT: with self._state_lock: self._archive_dialog_locked(dialog, "auto_policy") asyncio.create_task( - self._auto_handle_dialog( - dialog, accept=True, prompt_text=dialog.default_prompt - ) + responder(dialog, accept=True, prompt_text=dialog.default_prompt) ) else: # must_respond → add to pending and arm watchdog. @@ -945,27 +882,40 @@ class CDPSupervisor: ) self._dialog_watchdogs[dialog.id] = handle + async def _native_handle_dialog( + self, dialog: PendingDialog, *, accept: bool, prompt_text: Optional[str] + ) -> None: + """Page.handleJavaScriptDialog; ``promptText`` sent only for prompt dialogs + when ``prompt_text`` is given. Raises on CDP failure.""" + params: Dict[str, Any] = {"accept": accept} + if prompt_text is not None and dialog.type == "prompt": + params["promptText"] = prompt_text + await self._cdp( + "Page.handleJavaScriptDialog", + params, + session_id=dialog.cdp_session_id or None, + timeout=5.0, + ) + async def _auto_handle_dialog( self, dialog: PendingDialog, *, accept: bool, prompt_text: str ) -> None: - """Send handleJavaScriptDialog for auto_dismiss/auto_accept. - - Dialog has already been archived by the caller (``_on_dialog_opening``); - this just fires the CDP call so the page unblocks. - """ - params: Dict[str, Any] = {"accept": accept} - if dialog.type == "prompt": - params["promptText"] = prompt_text + """Auto-policy response for a native dialog (already archived by the caller).""" try: - await self._cdp( - "Page.handleJavaScriptDialog", - params, - session_id=dialog.cdp_session_id or None, - timeout=5.0, - ) + await self._native_handle_dialog(dialog, accept=accept, prompt_text=prompt_text) except Exception as e: logger.debug("auto-handle CDP call failed for %s: %s", dialog.id, e) + def _retire_dialog(self, dialog_id: str, closed_by: str) -> None: + """Remove a pending dialog (archiving it with ``closed_by``) and cancel its watchdog.""" + with self._state_lock: + dialog = self._pending_dialogs.pop(dialog_id, None) + if dialog is not None: + self._archive_dialog_locked(dialog, closed_by) + handle = self._dialog_watchdogs.pop(dialog_id, None) + if handle is not None: + handle.cancel() + async def _dialog_timeout_expired(self, dialog_id: str) -> None: with self._state_lock: dialog = self._pending_dialogs.get(dialog_id) @@ -979,22 +929,14 @@ class CDPSupervisor: self.dialog_timeout_s, ) try: - # Archive with watchdog tag BEFORE fulfilling / dismissing. + # Archive with watchdog tag BEFORE unblocking the page. with self._state_lock: - if dialog_id in self._pending_dialogs: - self._pending_dialogs.pop(dialog_id, None) + if self._pending_dialogs.pop(dialog_id, None) is not None: self._archive_dialog_locked(dialog, "watchdog") - # Unblock the page — via bridge Fetch fulfill for bridge dialogs, - # else native Page.handleJavaScriptDialog for real dialogs. if dialog.bridge_request_id: await self._fulfill_bridge_request(dialog, accept=False, prompt_text="") else: - await self._cdp( - "Page.handleJavaScriptDialog", - {"accept": False}, - session_id=dialog.cdp_session_id or None, - timeout=5.0, - ) + await self._native_handle_dialog(dialog, accept=False, prompt_text=None) except Exception as e: logger.debug("auto-dismiss failed for %s: %s", dialog_id, e) @@ -1016,94 +958,49 @@ class CDPSupervisor: async def _handle_dialog_cdp( self, dialog: PendingDialog, *, accept: bool, prompt_text: str ) -> None: - """Send the Page.handleJavaScriptDialog CDP command (agent path only). + """Agent response path: bridge-fulfill for XHR-captured dialogs, else native CDP. - Routes to the bridge-fulfill path when the dialog was captured via - the injected XHR override (see ``_on_fetch_paused``). + The dialog is retired regardless of outcome — a CDP error usually means + it already closed (browser auto-dismissed after navigation, etc.). """ - if dialog.bridge_request_id: - try: + try: + if dialog.bridge_request_id: await self._fulfill_bridge_request( dialog, accept=accept, prompt_text=prompt_text ) - finally: - with self._state_lock: - if dialog.id in self._pending_dialogs: - self._pending_dialogs.pop(dialog.id, None) - self._archive_dialog_locked(dialog, "agent") - handle = self._dialog_watchdogs.pop(dialog.id, None) - if handle is not None: - handle.cancel() - return - - params: Dict[str, Any] = {"accept": accept} - if dialog.type == "prompt": - params["promptText"] = prompt_text - try: - await self._cdp( - "Page.handleJavaScriptDialog", - params, - session_id=dialog.cdp_session_id or None, - timeout=5.0, - ) + else: + await self._native_handle_dialog(dialog, accept=accept, prompt_text=prompt_text) finally: - # Clear regardless — the CDP error path usually means the dialog - # already closed (browser auto-dismissed after navigation, etc.). - with self._state_lock: - if dialog.id in self._pending_dialogs: - self._pending_dialogs.pop(dialog.id, None) - self._archive_dialog_locked(dialog, "agent") - handle = self._dialog_watchdogs.pop(dialog.id, None) - if handle is not None: - handle.cancel() + self._retire_dialog(dialog.id, "agent") async def _on_dialog_closed( self, params: Dict[str, Any], session_id: Optional[str] ) -> None: - # ``Page.javascriptDialogClosed`` spec has only ``result`` (bool) and - # ``userInput`` (string), not the original ``message``. Match by - # session id and clear the oldest dialog on that session — if Chrome - # closed one on us (e.g. our disconnect auto-dismissed it, or the - # browser navigated, or Browserbase's CDP proxy auto-dismissed), there - # shouldn't be more than one in flight per session anyway because the - # JS thread is blocked while a dialog is up. + # ``Page.javascriptDialogClosed`` carries only ``result``/``userInput``, not + # the message. Match by session id and clear the oldest native dialog on + # it — the JS thread blocks while a dialog is up, so at most one is in + # flight per session. Bridge dialogs resolve via Fetch.fulfillRequest. with self._state_lock: candidate_ids = [ d.id for d in self._pending_dialogs.values() - if d.cdp_session_id == session_id - # Bridge-captured dialogs aren't cleared by native close events; - # they're resolved via Fetch.fulfillRequest instead. Only the - # real-native-dialog path uses Page.javascriptDialogClosed. - and d.bridge_request_id is None + if d.cdp_session_id == session_id and d.bridge_request_id is None ] - if candidate_ids: - did = candidate_ids[0] - dialog = self._pending_dialogs.pop(did, None) - if dialog is not None: - self._archive_dialog_locked(dialog, "remote") - handle = self._dialog_watchdogs.pop(did, None) - if handle is not None: - handle.cancel() + if candidate_ids: + self._retire_dialog(candidate_ids[0], "remote") async def _on_fetch_paused( self, params: Dict[str, Any], session_id: Optional[str] ) -> None: """Bridge XHR captured mid-flight — materialize as a pending dialog. - The injected script (``_DIALOG_BRIDGE_SCRIPT``) fires a synchronous - XHR to ``DIALOG_BRIDGE_HOST`` whenever page code calls alert/confirm/ - prompt. We catch it via Fetch.enable pattern; the page's JS thread - is blocked on the XHR's response until we call Fetch.fulfillRequest - (which happens from ``respond_to_dialog``) or until the watchdog - fires (at which point we fulfill with a cancel response). + The page's JS thread is blocked on the XHR until we Fetch.fulfillRequest + (from ``respond_to_dialog`` or the watchdog). """ url = str(params.get("request", {}).get("url") or "") request_id = params.get("requestId") if not request_id: return - # Only care about our bridge URLs. Fetch can still deliver other - # intercepted requests if patterns were ever broadened. if DIALOG_BRIDGE_HOST not in url: # Not ours — forward unchanged so the page sees its own request. try: @@ -1115,7 +1012,6 @@ class CDPSupervisor: pass return - # Parse query string for dialog metadata. Use urllib to be robust. from urllib.parse import urlparse, parse_qs q = parse_qs(urlparse(url).query) @@ -1123,47 +1019,15 @@ class CDPSupervisor: v = q.get(name, [""]) return v[0] if v else "" - kind = _q("kind") or "alert" - message = _q("message") - default_prompt = _q("default_prompt") - - self._dialog_seq += 1 - dialog = PendingDialog( - id=f"d-{self._dialog_seq}", - type=kind, - message=message, - default_prompt=default_prompt, - opened_at=time.time(), - cdp_session_id=session_id or self._page_session_id or "", + dialog = self._new_dialog( + type=_q("kind") or "alert", + message=_q("message"), + default_prompt=_q("default_prompt"), + session_id=session_id, frame_id=params.get("frameId"), bridge_request_id=str(request_id), ) - - # Apply policy exactly as for native dialogs. - if self.dialog_policy == DIALOG_POLICY_AUTO_DISMISS: - with self._state_lock: - self._archive_dialog_locked(dialog, "auto_policy") - asyncio.create_task( - self._fulfill_bridge_request(dialog, accept=False, prompt_text="") - ) - elif self.dialog_policy == DIALOG_POLICY_AUTO_ACCEPT: - with self._state_lock: - self._archive_dialog_locked(dialog, "auto_policy") - asyncio.create_task( - self._fulfill_bridge_request( - dialog, accept=True, prompt_text=default_prompt - ) - ) - else: - # must_respond — add to pending + arm watchdog. - with self._state_lock: - self._pending_dialogs[dialog.id] = dialog - loop = asyncio.get_running_loop() - handle = loop.call_later( - self.dialog_timeout_s, - lambda: asyncio.create_task(self._dialog_timeout_expired(dialog.id)), - ) - self._dialog_watchdogs[dialog.id] = handle + self._admit_dialog(dialog, self._fulfill_bridge_request) async def _fulfill_bridge_request( self, dialog: PendingDialog, *, accept: bool, prompt_text: str @@ -1223,7 +1087,7 @@ class CDPSupervisor: return with self._state_lock: existing = self._frames.get(frame_id) - info = FrameInfo( + self._frames[frame_id] = FrameInfo( frame_id=frame_id, url=str(frame.get("url") or ""), origin=str(frame.get("securityOrigin") or frame.get("origin") or ""), @@ -1232,27 +1096,17 @@ class CDPSupervisor: cdp_session_id=existing.cdp_session_id if existing else session_id, name=str(frame.get("name") or (existing.name if existing else "")), ) - self._frames[frame_id] = info def _on_frame_detached( self, params: Dict[str, Any], session_id: Optional[str] ) -> None: - """Remove a frame from our state only when it's truly gone. + """Drop a frame only when it's truly gone. - CDP emits ``Page.frameDetached`` with a ``reason`` of either - ``"remove"`` (the frame is actually gone from the DOM) or ``"swap"`` - (the frame is migrating to a new process — typical when a - same-process iframe becomes an OOPIF, or when history navigates). - Dropping on ``swap`` would hide OOPIFs from the agent the moment - Chromium promotes them to their own process, so treat swap as a - no-op. - - Even with ``reason=remove``, the parent page's perspective is - "the child frame left MY process tree" — which is what happens - when a same-origin iframe gets promoted to an OOPIF. If we - already have a live child CDP session attached for that frame_id, - the frame is still very much alive; only drop it when we have - no session record. + ``reason="swap"`` means the frame is migrating processes (e.g. promoted + to an OOPIF) — dropping it would hide the iframe, so it's a no-op. Even + with ``reason="remove"`` the parent only knows the child left ITS + process; if we hold a live child session for that frame_id it is still + alive, so keep it until Target.detached + a later frameDetached clear it. """ frame_id = params.get("frameId") if not frame_id: @@ -1262,16 +1116,11 @@ class CDPSupervisor: return with self._state_lock: existing = self._frames.get(frame_id) - # Keep OOPIF records even when the parent says the frame was - # "removed" — the iframe is still visible, just in a different - # process. If the frame truly goes away later, Target.detached - # + the next Page.frameDetached without a live session will - # clear it. if existing and existing.is_oopif and existing.cdp_session_id: return self._frames.pop(frame_id, None) - async def _on_target_attached(self, params: Dict[str, Any]) -> None: + async def _on_target_attached(self, params: Dict[str, Any], session_id: Optional[str] = None) -> None: info = params.get("targetInfo") or {} sid = params.get("sessionId") target_type = info.get("type") @@ -1294,17 +1143,12 @@ class CDPSupervisor: name=str(info.get("title") or (existing.name if existing else "")), ) - # Enable domains on the child off-loop so the reader keeps pumping. - # Awaiting the CDP replies here would deadlock because only the - # reader can resolve those replies' Futures. + # Enable child domains off-loop: awaiting the replies here would deadlock + # because only the reader can resolve those Futures. asyncio.create_task(self._enable_child_domains(sid)) async def _enable_child_domains(self, sid: str) -> None: - """Enable Page+Runtime (+nested setAutoAttach) on a child CDP session. - - Also installs the dialog bridge so iframe-scoped alert/confirm/prompt - calls round-trip through Fetch too. - """ + """Enable Page+Runtime (+nested setAutoAttach) and the dialog bridge on a child session.""" try: await self._cdp("Page.enable", session_id=sid, timeout=3.0) await self._cdp("Runtime.enable", session_id=sid, timeout=3.0) @@ -1316,19 +1160,16 @@ class CDPSupervisor: ) except Exception as e: logger.debug("child session %s setup failed: %s", sid[:16], e) - # Install the dialog bridge on the child so iframe dialogs are captured. await self._install_dialog_bridge(sid) - def _on_target_detached(self, params: Dict[str, Any]) -> None: - """Handle a child CDP session detaching. + def _on_target_detached(self, params: Dict[str, Any], session_id: Optional[str] = None) -> None: + """Clear the session binding of frames on a detached child session. - We deliberately DO NOT drop frames from ``_frames`` here — Browserbase - fires transient detach events during page transitions even while the - iframe is still visible to the user, and dropping the record hides - OOPIFs from the agent between the detach and the next - ``Target.attachedToTarget``. Instead, we just clear the session - binding so stale ``cdp_session_id`` values aren't used for routing. - If the iframe truly goes away, ``Page.frameDetached`` will clean up. + Frames are deliberately NOT dropped: Browserbase fires transient detaches + during page transitions while the iframe is still visible, and dropping + would hide OOPIFs until the next ``Target.attachedToTarget``. Clearing + ``cdp_session_id`` just stops stale routing; ``Page.frameDetached`` + cleans up if the iframe truly goes away. """ sid = params.get("sessionId") if not sid: @@ -1337,17 +1178,7 @@ class CDPSupervisor: with self._state_lock: for fid, frame in list(self._frames.items()): if frame.cdp_session_id == sid: - # Replace with a copy that has cdp_session_id cleared so - # routing falls back to top-level page session if retried. - self._frames[fid] = FrameInfo( - frame_id=frame.frame_id, - url=frame.url, - origin=frame.origin, - parent_frame_id=frame.parent_frame_id, - is_oopif=frame.is_oopif, - cdp_session_id=None, - name=frame.name, - ) + self._frames[fid] = replace(frame, cdp_session_id=None) # ── Console / exception ring buffer ───────────────────────────────────── @@ -1371,28 +1202,43 @@ class CDPSupervisor: with self._state_lock: self._console_events.append(event) if len(self._console_events) > CONSOLE_HISTORY_MAX * 2: - # Keep last CONSOLE_HISTORY_MAX; allow 2x slack to reduce churn. + # Keep last CONSOLE_HISTORY_MAX; 2x slack reduces churn. self._console_events = self._console_events[-CONSOLE_HISTORY_MAX:] + # CDP event → handler(self, params, session_id). Async handlers return an + # awaitable that ``_on_event`` awaits; sync handlers return None. + _EVENT_HANDLERS: Dict[str, Callable[..., Any]] = { + "Page.javascriptDialogOpening": _on_dialog_opening, + "Page.javascriptDialogClosed": _on_dialog_closed, + "Fetch.requestPaused": _on_fetch_paused, + "Page.frameAttached": _on_frame_attached, + "Page.frameNavigated": _on_frame_navigated, + "Page.frameDetached": _on_frame_detached, + "Target.attachedToTarget": _on_target_attached, + "Target.detachedFromTarget": _on_target_detached, + "Runtime.consoleAPICalled": lambda self, p, _sid: self._on_console(p, level_from="api"), + "Runtime.exceptionThrown": lambda self, p, _sid: self._on_console(p, level_from="exception"), + } + # ── Frame tree building (bounded) ─────────────────────────────────────── def _build_frame_tree_locked(self) -> Dict[str, Any]: """Build the capped frame_tree payload. Must be called under state lock.""" frames = self._frames + empty = {"top": None, "children": [], "truncated": False} if not frames: - return {"top": None, "children": [], "truncated": False} + return empty - # Identify a top frame — one with no parent, preferring oopif=False. + # Top frame: one with no parent, preferring oopif=False. tops = [f for f in frames.values() if not f.parent_frame_id] top = next((f for f in tops if not f.is_oopif), tops[0] if tops else None) + if top is None: + return empty # BFS from top, capped by FRAME_TREE_MAX_ENTRIES and # FRAME_TREE_MAX_OOPIF_DEPTH for OOPIF branches. children: List[Dict[str, Any]] = [] truncated = False - if top is None: - return {"top": None, "children": [], "truncated": False} - queue: List[Tuple[FrameInfo, int]] = [ (f, 1) for f in frames.values() if f.parent_frame_id == top.frame_id ] @@ -1425,8 +1271,7 @@ class CDPSupervisor: class _SupervisorRegistry: """Process-global (task_id → supervisor) map with idempotent start/stop. - One instance, exposed as ``SUPERVISOR_REGISTRY``. Safe to call from any - thread — mutations go through ``_lock``. + One instance, exposed as ``SUPERVISOR_REGISTRY``; mutations go through ``_lock``. """ def __init__(self) -> None: @@ -1449,8 +1294,8 @@ class _SupervisorRegistry: ) -> CDPSupervisor: """Idempotently ensure a supervisor is running for ``(task_id, cdp_url)``. - If a supervisor exists for this task but was bound to a different - ``cdp_url``, the old one is stopped and a fresh one is started. + An existing supervisor bound to a different ``cdp_url`` (or unhealthy) + is stopped and replaced. """ with self._lock: existing = self._by_task.get(task_id) @@ -1460,7 +1305,6 @@ class _SupervisorRegistry: loop_ok = existing._loop is not None and existing._loop.is_running() if thread_ok and loop_ok: return existing - # Unhealthy — tear down and recreate. # URL changed or unhealthy — tear down, fall through to re-create. self._by_task.pop(task_id, None) if existing is not None: diff --git a/tools/clarify_gateway.py b/tools/clarify_gateway.py index e9bec3e95c..6f8e138f0f 100644 --- a/tools/clarify_gateway.py +++ b/tools/clarify_gateway.py @@ -1,36 +1,16 @@ """Gateway-side clarify primitive (blocking event-based queue). -The ``clarify`` tool needs to ask the user a question and block the agent -thread until they respond. In CLI mode this is trivial — ``input()`` is -synchronous. In gateway mode the agent runs on a worker thread while the -event loop handles the user's reply, so we need a thread-safe primitive -that: - - * stores a pending clarify request (with a generated ``clarify_id``), - * blocks the agent thread on an ``Event``, - * resolves the wait when the gateway's button-callback or text-intercept - fires ``resolve_gateway_clarify(clarify_id, response)``, - * supports timeouts so a user who never responds does NOT hang the agent - thread forever (which would also pin the gateway's running-agent guard). - -State is module-level (same shape as ``tools.approval``) so platform -adapters can call ``resolve_gateway_clarify`` without holding a back- -reference to the ``GatewayRunner`` instance. - -Two delivery paths from the adapter: - - 1. **Button UI** — adapters override ``send_clarify`` to render inline - buttons (e.g. Telegram ``InlineKeyboardMarkup``). The button - callback resolves with the chosen string. A final "Other (type - answer)" button enters text-capture mode for free-form responses. - - 2. **Text fallback** — adapters without rich UI render a numbered list. - The user replies with a number ("2") or with free text; the gateway's - ``_handle_message`` intercepts the reply and resolves directly. +In gateway mode the agent runs on a worker thread while the event loop handles +the user's reply, so a pending clarify is stored module-level (same shape as +``tools.approval``) and the agent thread blocks on an ``Event`` until an +adapter button callback or the gateway text-intercept resolves it, or the +timeout fires. Adapters render either inline buttons (with an "Other" row that +flips the entry into text-capture mode) or a numbered-list text fallback. """ from __future__ import annotations +import json import logging import threading import time @@ -40,10 +20,6 @@ from typing import Callable, Dict, List, Optional logger = logging.getLogger(__name__) -# ========================================================================= -# Module-level state -# ========================================================================= - @dataclass class _ClarifyEntry: """One pending clarify request inside a gateway session.""" @@ -56,26 +32,22 @@ class _ClarifyEntry: response: Optional[str] = None awaiting_text: bool = False # set when user picked "Other" or clarify is open-ended - def signature(self) -> Dict[str, object]: - return { - "clarify_id": self.clarify_id, - "session_key": self.session_key, - "question": self.question, - "choices": list(self.choices) if self.choices else None, - "multi_select": bool(self.multi_select), - } - _lock = threading.RLock() -# clarify_id → _ClarifyEntry (primary lookup for button callbacks) -_entries: Dict[str, _ClarifyEntry] = {} -# session_key → list[clarify_id] (FIFO; for text-fallback intercept and session cleanup) -_session_index: Dict[str, List[str]] = {} +_entries: Dict[str, _ClarifyEntry] = {} # clarify_id -> entry (button callbacks) +_session_index: Dict[str, List[str]] = {} # session_key -> [clarify_id] FIFO (text intercept, cleanup) +# Per-session notify callbacks (gateway -> adapter bridge); mirrors tools.approval. +_notify_cbs: Dict[str, Callable[[_ClarifyEntry], None]] = {} + +# Outcomes for typed clarify replies. Gateway cancels the pending prompt on +# free prose (deadlock break) but keeps it armed for a retryable bad selection. +TEXT_RESOLVED = "resolved" +TEXT_REJECTED_PROSE = "rejected_prose" +TEXT_REJECTED_SELECTION = "rejected_selection" +TEXT_NO_PENDING = "no_pending" -# ========================================================================= -# Public API — agent-thread side -# ========================================================================= +# --- agent-thread side --------------------------------------------------- def register( clarify_id: str, @@ -84,18 +56,14 @@ def register( choices: Optional[List[str]], multi_select: bool = False, ) -> _ClarifyEntry: - """Register a pending clarify request and return the entry. - - The caller (gateway clarify_callback) will then send the prompt to the - user and block on ``wait_for_response(clarify_id, timeout)``. - """ + """Register a pending clarify request; caller then blocks on ``wait_for_response``.""" entry = _ClarifyEntry( clarify_id=clarify_id, session_key=session_key, question=question, choices=list(choices) if choices else None, multi_select=bool(multi_select) and bool(choices), - # Open-ended (no choices) → next message IS the response, no buttons needed. + # Open-ended (no choices) -> next message IS the response, no buttons needed. awaiting_text=not bool(choices), ) with _lock: @@ -105,18 +73,11 @@ def register( def wait_for_response(clarify_id: str, timeout: float) -> Optional[str]: - """Block on the entry's event until resolved or timeout fires. + """Block until the entry resolves or ``timeout`` elapses; None on timeout/unknown id. - Polls in 1-second slices so the agent's inactivity heartbeat keeps - firing — without this, ``Event.wait(timeout=600)`` blocks the thread - for 10 minutes with zero activity touches and the gateway's inactivity - watchdog kills the agent while the user is still typing. - - ``timeout <= 0`` means an unlimited wait (never auto-skip mid-think); the - heartbeat still fires each slice so inactivity watchdogs don't kill a live - prompt. - - Returns the resolved response string, or ``None`` on timeout. + Polls in 1s slices so the inactivity heartbeat keeps firing (a single long + ``Event.wait`` would let the gateway watchdog kill a live prompt). + ``timeout <= 0`` means an unlimited wait. """ with _lock: entry = _entries.get(clarify_id) @@ -128,7 +89,6 @@ def wait_for_response(clarify_id: str, timeout: float) -> Optional[str]: except Exception: # pragma: no cover - optional touch_activity_if_due = None - # 0 / negative → unlimited: no deadline, poll forever in 1s slices. unlimited = timeout is None or float(timeout) <= 0.0 deadline = None if unlimited else time.monotonic() + float(timeout) activity_state = {"last_touch": time.monotonic(), "start": time.monotonic()} @@ -146,8 +106,7 @@ def wait_for_response(clarify_id: str, timeout: float) -> Optional[str]: touch_activity_if_due(activity_state, "waiting for user clarify response") with _lock: - # Remove from indices regardless of resolution outcome. - _entries.pop(clarify_id, None) + _entries.pop(clarify_id, None) # regardless of outcome ids = _session_index.get(entry.session_key) if ids and clarify_id in ids: ids.remove(clarify_id) @@ -157,16 +116,10 @@ def wait_for_response(clarify_id: str, timeout: float) -> Optional[str]: return entry.response -# ========================================================================= -# Public API — gateway / adapter side -# ========================================================================= +# --- gateway / adapter side ----------------------------------------------- def resolve_gateway_clarify(clarify_id: str, response: str) -> bool: - """Unblock the agent thread waiting on ``clarify_id``. - - Returns True if an entry was found and resolved, False otherwise - (already resolved, expired, or never existed). - """ + """Unblock the waiter on ``clarify_id``; False if already resolved/expired/unknown.""" with _lock: entry = _entries.get(clarify_id) if entry is None or entry.event.is_set(): @@ -181,101 +134,68 @@ def get_pending_for_session( *, include_choice_prompts: bool = False, ) -> Optional[_ClarifyEntry]: - """Return the oldest pending clarify entry for a session, or None. + """Oldest pending entry awaiting free text (open-ended, or after "Other"). - By default this only returns entries awaiting free-form text (open-ended - clarifies, or a multi-choice clarify after the user picked ``Other``). - Gateways may pass ``include_choice_prompts=True`` when the user has typed - directly in response to an active multi-choice prompt; in that case the - oldest unresolved clarify is returned so the text can resolve it instead - of being queued as an unrelated follow-up turn. + ``include_choice_prompts=True`` returns the oldest unresolved entry of any + kind — for when the user typed directly at an active multi-choice prompt, + so the text resolves it instead of queueing as an unrelated follow-up turn. """ with _lock: - ids = _session_index.get(session_key) or [] - for cid in ids: + for cid in _session_index.get(session_key) or []: entry = _entries.get(cid) - if entry is None: - continue - if include_choice_prompts or entry.awaiting_text: + if entry is not None and (include_choice_prompts or entry.awaiting_text): return entry return None def _label_matches(text: str, choice: object) -> bool: - """Case-insensitive label match that ignores the '(Recommended)' suffix. - - The first choice reaches adapters already decorated (see - ``tools.clarify_tool.mark_recommended``), so a user who types the option - text as the agent worded it — without the label — must still resolve the - prompt. - """ + """Case-insensitive label match ignoring the '(Recommended)' suffix the + first choice carries by the time it reaches adapters.""" from tools.clarify_tool import strip_recommended return strip_recommended(text).casefold() == strip_recommended(str(choice)).casefold() -# Outcomes for typed clarify replies. Gateway uses these to decide whether to -# cancel a pending prompt (free prose deadlock break) or keep it armed so the -# user can retry a selection-like invalid reply (out-of-range / bad list). -TEXT_RESOLVED = "resolved" -TEXT_REJECTED_PROSE = "rejected_prose" -TEXT_REJECTED_SELECTION = "rejected_selection" -TEXT_NO_PENDING = "no_pending" +def _match_label(text: str, choices: List[str]) -> Optional[str]: + """Canonical (stripped) choice text matching ``text``, or None.""" + for choice in choices: + if _label_matches(text, choice): + return str(choice).strip() + return None + + +def _split_tokens(text: str) -> Optional[List[str]]: + """Comma-separated tokens, or space-separated all-numeric tokens ("1 3"); else None.""" + if "," in text: + return [t.strip() for t in text.split(",") if t.strip()] + parts = text.split() + if len(parts) > 1 and all(p.isdigit() for p in parts): + return parts + return None def _selection_attempt_tokens( text: str, choices: Optional[List[str]] = None, ) -> Optional[List[str]]: - """Return tokens when ``text`` looks like a typed selection attempt. - - Selection-shaped input includes: - - a bare integer ("2", "99") - - comma-separated numbers/labels ("1,3", "staging, prod", "1,99") - - space-separated all-numeric lists ("1 3") - - Free prose ("just checking the visual UI, no need to pass any data") returns - None even when it contains commas, so the gateway can release the clarify - and continue normal routing instead of forcing a retry. - - Multi-word choice labels are allowed in comma-lists up to the longest - choice's word count (e.g. "Send to SOL, Keep with Enoch"). - """ + """Tokens when ``text`` looks like a typed selection (bare int, comma list, + all-numeric space list); None for free prose so the gateway can release the + clarify. Comma-list labels may span up to the longest choice's word count.""" stripped = str(text).strip() if not stripped: return None - max_choice_words = 1 - if choices: - max_choice_words = max( - (len(str(choice).split()) for choice in choices), - default=1, - ) - max_choice_words = max(1, max_choice_words) + tokens = _split_tokens(stripped) + if tokens is not None: + if "," not in stripped or not tokens: + return tokens or None + max_words = max(1, max((len(str(c).split()) for c in choices or []), default=1)) + if all(t.isdigit() or len(t.split()) <= max_words for t in tokens): + return tokens + return None - if "," in stripped: - tokens = [t.strip() for t in stripped.split(",") if t.strip()] - if not tokens: - return None - # Natural-language clauses with commas are not selection lists. - # Each selection token is either a number or at most as many words - # as the longest configured choice label. - for token in tokens: - if token.isdigit(): - continue - words = token.split() - if len(words) == 0 or len(words) > max_choice_words: - return None - return tokens - - parts = stripped.split() - if len(parts) > 1 and all(p.strip().isdigit() for p in parts): - return [p.strip() for p in parts] - - # Bare integer (in-range or out-of-range) is always a selection attempt. if stripped.isdigit() or (stripped.startswith("-") and stripped[1:].isdigit()): return [stripped] - try: int(stripped) return [stripped] @@ -284,69 +204,38 @@ def _selection_attempt_tokens( def _coerce_text_response(entry: _ClarifyEntry, response: str) -> Optional[str]: - """Map typed choice replies to canonical choice text, otherwise keep or reject custom text. - - Thin wrapper over :func:`_coerce_text_response_detailed` for callers that - only need the accepted value (or ``None`` on any rejection). - """ - coerced, _reason = _coerce_text_response_detailed(entry, response) - return coerced + """Accepted value for a typed reply, or None on any rejection.""" + return _coerce_text_response_detailed(entry, response)[0] def _coerce_text_response_detailed( entry: _ClarifyEntry, response: str, ) -> tuple[Optional[str], Optional[str]]: - """Map typed replies and classify rejections. + """Map a typed reply to ``(value, None)`` or ``(None, reason)``. - Returns ``(value, None)`` when the reply is accepted. - - Returns ``(None, reason)`` when rejected: - - ``"invalid_selection"`` — selection-shaped but unusable (out-of-range - number, unrecognised comma-list). Keep the pending clarify so the - user can retry. - - ``"prose"`` — free text that is not a selection attempt. Gateway may - cancel the clarify and continue normal busy-message routing so a - redirect-to-steer path cannot deadlock behind the waiting tool. - - For native interactive multi-choice clarifies (button UI, awaiting_text=False): - - Accept numeric selections ("2" → choice[1]) - - Accept exact choice label matches (case-insensitive) - - Reject arbitrary prose so the message can continue as a normal turn - - For multi-select clarifies (entry.multi_select=True): - - Accept several numbers separated by commas and/or spaces ("1,3" / "1 3") - - Accept exact choice label matches (single or comma-separated) - - Out-of-range numbers / unrecognised lists reject the whole reply so the - user can retry instead of silently getting a partial selection - - Selections are returned as a JSON array string, which the clarify - tool's ``_parse_multi_select_response`` decodes back into a list - - For text fallback or awaiting_text mode: - - Accept any text (numeric/label/custom) after passing through coercion - - For open-ended clarifies (no choices): - - Accept any text + Reasons: ``"invalid_selection"`` (selection-shaped but out of range / + unrecognised — keep the clarify armed for a retry) or ``"prose"`` (free + text on a native choice prompt — gateway may cancel and route normally). + Open-ended entries and ``awaiting_text`` mode accept any text; numeric + picks and exact labels always resolve; multi-select returns a JSON array + string that ``_parse_multi_select_response`` decodes on the tool side. """ text = str(response).strip() if not entry.choices: - # Open-ended: accept any text return text, None if entry.multi_select: coerced = _coerce_multi_select_text(entry, text) if coerced is not None: return coerced, None - # Not a parseable selection — accept as custom text only in - # awaiting_text mode (the "Other" path); otherwise classify reject. if entry.awaiting_text: return text, None if _selection_attempt_tokens(text, entry.choices) is not None: return None, "invalid_selection" return None, "prose" - # Try numeric selection first (always valid for multi-choice) try: idx = int(text) - 1 is_int = True @@ -357,94 +246,53 @@ def _coerce_text_response_detailed( if is_int and 0 <= idx < len(entry.choices): return entry.choices[idx], None - # Try exact choice label match (always valid for multi-choice) - for choice in entry.choices: - if _label_matches(text, choice): - return str(choice).strip(), None + matched = _match_label(text, entry.choices) + if matched is not None: + return matched, None - # For text fallback or awaiting_text mode, accept custom text - # For native interactive multi-choice mode, reject with a reason if entry.awaiting_text: return text, None # Out-of-range / non-canonical integer is a failed selection, not prose. - if is_int: - return None, "invalid_selection" - return None, "prose" + return None, "invalid_selection" if is_int else "prose" def _coerce_multi_select_text(entry: _ClarifyEntry, text: str) -> Optional[str]: - """Parse a typed multi-select reply into a JSON array of choice labels. - - Accepts numbers and/or exact labels separated by commas (and, for - all-numeric replies, bare spaces): "1,3", "1 3", "staging, prod". - Returns ``None`` when any token is out of range or unrecognised so the - caller can reject the reply cleanly instead of resolving a partial or - wrong selection. - """ - import json as _json - + """Parse "1,3" / "1 3" / "staging, prod" into a JSON array of choice labels; + None when any token is out of range or unrecognised (reject the whole reply).""" if not text: return None choices = entry.choices or [] - - # Split on commas first; if no commas and every whitespace-separated - # token is numeric, treat spaces as separators too ("1 3"). - if "," in text: - tokens = [t.strip() for t in text.split(",") if t.strip()] - else: - parts = text.split() - if len(parts) > 1 and all(p.strip().isdigit() for p in parts): - tokens = [p.strip() for p in parts] - else: - tokens = [text] + tokens = _split_tokens(text) + if tokens is None: + tokens = [text] selected: List[str] = [] for token in tokens: if token.isdigit(): idx = int(token) - 1 - if 0 <= idx < len(choices): - label = str(choices[idx]).strip() - if label not in selected: - selected.append(label) - continue - return None # out-of-range number → reject whole reply - # Exact label match (case-insensitive) - matched = None - for choice in choices: - if _label_matches(token, choice): - matched = str(choice).strip() - break - if matched is None: - return None - if matched not in selected: - selected.append(matched) + if not 0 <= idx < len(choices): + return None + label = str(choices[idx]).strip() + else: + label = _match_label(token, choices) + if label is None: + return None + if label not in selected: + selected.append(label) - if not selected: - return None - return _json.dumps(selected, ensure_ascii=False) + return json.dumps(selected, ensure_ascii=False) if selected else None def attempt_text_response_for_session(session_key: str, response: str) -> str: - """Try to resolve the oldest pending clarify in ``session_key`` from typed text. - - Returns one of: - - ``TEXT_RESOLVED`` — accepted; waiter unblocked - - ``TEXT_REJECTED_PROSE`` — free prose on a native choice prompt; caller - may cancel the clarify and continue ordinary message routing - - ``TEXT_REJECTED_SELECTION`` — selection-shaped but invalid; leave the - pending clarify armed so the user can retry - - ``TEXT_NO_PENDING`` — no interceptable clarify for this session - """ + """Try to resolve the oldest pending clarify from typed text; returns a TEXT_* outcome.""" entry = get_pending_for_session(session_key, include_choice_prompts=True) if entry is None: return TEXT_NO_PENDING coerced, reason = _coerce_text_response_detailed(entry, response) if coerced is None: - if reason == "invalid_selection": - return TEXT_REJECTED_SELECTION - return TEXT_REJECTED_PROSE + return TEXT_REJECTED_SELECTION if reason == "invalid_selection" else TEXT_REJECTED_PROSE if resolve_gateway_clarify(entry.clarify_id, coerced): return TEXT_RESOLVED @@ -453,21 +301,12 @@ def attempt_text_response_for_session(session_key: str, response: str) -> str: def resolve_text_response_for_session(session_key: str, response: str) -> bool: - """Resolve the oldest pending clarify in ``session_key`` from typed text. - - Returns True only when the reply was accepted and the waiter unblocked. - Rejected prose, rejected selections, and missing prompts all return False; - use :func:`attempt_text_response_for_session` when the caller must - distinguish those cases (gateway deadlock vs multi-select retry). - """ + """True only when the typed reply was accepted and the waiter unblocked.""" return attempt_text_response_for_session(session_key, response) == TEXT_RESOLVED def mark_awaiting_text(clarify_id: str) -> bool: - """Flip an entry into text-capture mode (user picked the 'Other' button). - - Returns True if the entry exists and was flipped, False otherwise. - """ + """Flip an entry into text-capture mode (user picked 'Other'); False if unknown.""" with _lock: entry = _entries.get(clarify_id) if entry is None: @@ -477,71 +316,39 @@ def mark_awaiting_text(clarify_id: str) -> bool: def has_pending(session_key: str) -> bool: - """Return True when this session has at least one pending clarify entry.""" + """True when this session has at least one pending clarify entry.""" with _lock: ids = _session_index.get(session_key) or [] return any(_entries.get(cid) is not None for cid in ids) def clear_session(session_key: str) -> int: - """Resolve and drop every pending clarify for a session. + """Drop every pending clarify for a session; returns how many were cancelled. - Used by session-boundary cleanup (e.g. ``/new``, gateway shutdown, - cached-agent eviction) so blocked agent threads don't hang past the - end of their session. Returns the number of entries actually - cancelled (i.e. whose event had not yet been set). Already-resolved - entries are dropped from the registry but their response is preserved. - - First-writer-wins: an entry whose event is already set has been resolved - by a real response (button callback or text intercept). Session cleanup - must NOT overwrite that response with the empty cancellation sentinel — - the waiting agent thread would observe a cancelled prompt even though the - user answered. Only unresolved entries are cancelled here. + Session-boundary cleanup (``/new``, shutdown, cached-agent eviction) so + blocked agent threads don't outlive their session; cancelled waiters see "". + First-writer-wins: an entry whose event is already set was answered for + real, so it is dropped from the registry but its response is preserved + rather than overwritten with the "" cancellation sentinel. The whole loop + stays inside the lock so a button callback cannot slip between pop and check. """ with _lock: ids = list(_session_index.pop(session_key, []) or []) - entries = [_entries.pop(cid, None) for cid in ids] - # The mutation loop must stay inside the lock: the pop above and the - # event.is_set() check below have to be atomic with respect to - # resolve_gateway_clarify, or a button callback could win between the - # pop and the check and have its answer clobbered by the sentinel. cancelled = 0 - for entry in entries: - if entry is None: + for entry in (_entries.pop(cid, None) for cid in ids): + if entry is None or entry.event.is_set(): continue - # Entry is removed from the global registry regardless of its - # state — a cleared session must not be resurrected by late - # callbacks — but a resolved entry keeps its real response. - if entry.event.is_set(): - continue - # Empty string sentinel — agent code can distinguish from a real - # response by inspecting the wait_for_response return value - # alongside its own timeout deadline. Most callers just treat any - # falsy result as "user did not respond". entry.response = "" entry.event.set() cancelled += 1 return cancelled -# ========================================================================= -# Config -# ========================================================================= +# --- config --------------------------------------------------------------- def resolve_clarify_timeout(config: dict) -> int: - """Resolve the clarify timeout (seconds) from an already-loaded config dict. - - Single source of truth shared by every surface (messaging gateway, CLI, - TUI/desktop) so the timeout can't drift between them. Resolution order: - - 1. legacy top-level ``clarify.timeout`` if a user explicitly set it, - 2. else the canonical ``agent.clarify_timeout``, - 3. else 3600 (1 hour). - - ``<= 0`` is preserved verbatim and means *unlimited* to callers (never - auto-skip while the user is still deciding); the waiting loops translate - that into a null deadline. A non-numeric value falls back to 3600. - """ + """Clarify timeout (seconds) from a loaded config: legacy ``clarify.timeout``, + else ``agent.clarify_timeout``, else 3600. ``<= 0`` is preserved (unlimited).""" raw = (config.get("clarify") or {}).get("timeout") if raw is None: raw = (config.get("agent") or {}).get("clarify_timeout", 3600) @@ -552,54 +359,14 @@ def resolve_clarify_timeout(config: dict) -> int: def get_clarify_timeout() -> int: - """Read the clarify response timeout (seconds) from config. + """Read the clarify timeout from config.yaml; 0/negative = unlimited. - Defaults to 3600 (1 hour) — long enough that a user who steps away - (meeting, AFK, slow to read) still finds a live entry when they tap - the button, short enough that a genuinely abandoned prompt eventually - unblocks the agent thread instead of pinning the running-agent guard - forever. The old 600s default evicted the entry mid-think, so a late - tap landed on a dead entry and the agent hung on ``running: clarify`` - (#32762). - - Reads ``agent.clarify_timeout`` from config.yaml (see - :func:`resolve_clarify_timeout` for the full resolution order). Set to - ``0`` (or negative) for an unlimited wait — never auto-skip while the user - is still deciding. + Default 3600: long enough that a user who stepped away still finds a live + entry when they tap, short enough that an abandoned prompt eventually + unblocks the agent thread instead of pinning the running-agent guard. """ try: from hermes_cli.config import load_config return resolve_clarify_timeout(load_config() or {}) except Exception: return 3600 - - -# ========================================================================= -# Per-session notify hook (gateway → adapter bridge) -# ========================================================================= -# Mirrors tools.approval's _gateway_notify_cbs: the gateway registers a -# per-session callback that sends the clarify prompt to the user. The -# callback bridges sync→async (runs on the agent thread; schedules the -# adapter ``send_clarify`` call on the event loop). - -_notify_cbs: Dict[str, Callable[[_ClarifyEntry], None]] = {} - - -def register_notify(session_key: str, cb: Callable[[_ClarifyEntry], None]) -> None: - """Register a per-session notify callback used by ``clarify_callback``.""" - with _lock: - _notify_cbs[session_key] = cb - - -def unregister_notify(session_key: str) -> None: - """Drop the per-session notify callback and cancel any pending clarify entries.""" - with _lock: - _notify_cbs.pop(session_key, None) - # Cancel any pending entries so blocked threads unwind when the run - # ends (interrupt, completion, gateway shutdown). - clear_session(session_key) - - -def get_notify(session_key: str) -> Optional[Callable[[_ClarifyEntry], None]]: - with _lock: - return _notify_cbs.get(session_key) diff --git a/tools/clarify_tool.py b/tools/clarify_tool.py index 9d9b4deb1c..fd2cd26d2b 100644 --- a/tools/clarify_tool.py +++ b/tools/clarify_tool.py @@ -1,62 +1,37 @@ #!/usr/bin/env python3 -""" -Clarify Tool Module - Interactive Clarifying Questions +"""Clarify tool: structured multiple-choice / open-ended questions to the user. -Allows the agent to present structured multiple-choice questions or open-ended -prompts to the user. In CLI mode, choices are navigable with arrow keys. On -messaging platforms, choices are rendered as a numbered list. - -Supports both single-select (radio) and multi-select (checkbox) modes via the -``multi_select`` parameter. - -The actual user-interaction logic lives in the platform layer (cli.py for CLI, -gateway/run.py for messaging). This module defines the schema, validation, and -a thin dispatcher that delegates to a platform-provided callback. +Schema, validation and a thin dispatcher; the UI lives in a platform-provided +callback (cli.py, gateway/run.py, tui_gateway). """ +import inspect import json from typing import Dict, List, Optional, Callable -# Maximum number of predefined choices the agent can offer. -# A 5th "Other (type your answer)" option is always appended by the UI. -MAX_CHOICES = 4 +MAX_CHOICES = 4 # the UI always appends an "Other (type your answer)" row +MAX_QUESTIONS = 5 # independent questions per batch call -# Maximum number of independent questions in one batch clarify call. -MAX_QUESTIONS = 5 - -# Canonical timeout sentinel returned to the agent when the user never -# answers. The CLI has always returned this exact text; the batch fallback -# loop also recognises it (alongside ``None``) as "the user walked away", -# which aborts the remaining questions instead of pestering one by one. +# Canonical timeout sentinel. The CLI returns this exact text; the batch loop +# treats it (like ``None``) as "the user walked away" and aborts remaining questions. TIMEOUT_RESPONSE = ( "The user did not provide a response within the time limit. " "Use your best judgement to make the choice and proceed." ) -# Suffix appended to the first choice so the user can see, at a glance, which -# option the agent actually recommends. Applied here rather than per-surface so -# CLI, TUI, desktop, and messaging adapters all render the same label. +# Applied to the first choice here (not per-surface) so every adapter renders it identically. RECOMMENDED_LABEL = "(Recommended)" def _flatten_choice(c) -> str: - """Coerce a single choice into its user-facing display string. + """Coerce one choice to display text. - The schema declares choices as bare strings, but LLMs sometimes emit - dict-shaped choices like ``[{"description": "..."}]``. A naive ``str(c)`` - turns the whole dict into its Python repr — ``{'description': '...'}`` — - which then leaks onto every surface that renders the choice (CLI panel, - Discord buttons, Telegram numbered list) AND is returned verbatim as the - user's answer. Normalising here, at the one platform-agnostic entry point, - fixes the whole class in one place instead of per-adapter. - - Dict unwrap order is the canonical LLM tool-call user-facing keys: - ``label`` → ``description`` → ``text`` → ``title``. ``name`` and ``value`` - are deliberately excluded — they're component-shaped fields that could - carry raw enum values or short identifiers, not human-readable labels. A - dict with none of the canonical keys is dropped (returns ""), since a - garbage label is worse than no choice at all. + LLMs sometimes emit dict-shaped choices; ``str(c)`` would leak the dict repr + onto every surface and back as the answer, so normalise once here. Unwrap + order ``label`` > ``description`` > ``text`` > ``title``; ``name``/``value`` + are excluded (component fields carrying raw enums, not labels). A dict with + none of these becomes "" and is dropped — no choice beats a garbage label. """ if c is None: return "" @@ -74,18 +49,9 @@ def _flatten_choice(c) -> str: def mark_recommended(choices: List[str]) -> List[str]: - """Label the first choice as the agent's recommendation. + """Suffix the first choice (the schema says best-first) with RECOMMENDED_LABEL. - The schema tells the model to order ``choices`` best-first, so element 0 is - always the option it would pick itself. Tagging it here — the one - platform-agnostic entry point — means every surface (CLI panel, TUI, - desktop card, Telegram buttons) reads the same way without four copies of - the same string concatenation, and the label can never drift between them. - - Idempotent: a model that writes its own "(recommended)" into the choice is - left alone rather than getting the suffix twice. A lone choice isn't a - recommendation — there's nothing to prefer it over — so single-choice lists - pass through untouched. + Idempotent, and a lone choice is left untouched (nothing to prefer it over). """ if len(choices) < 2: return choices @@ -96,60 +62,40 @@ def mark_recommended(choices: List[str]) -> List[str]: def strip_recommended(text: str) -> str: - """Remove the recommendation label from a resolved answer. - - The user picks the decorated string, but the agent asked about the bare - option — returning "Rebase onto main (Recommended)" as ``user_response`` - would leak presentation into the answer the model reasons about and into - anything it echoes back. - """ + """Remove the recommendation label so presentation never leaks into ``user_response``.""" stripped = str(text).strip() if stripped.casefold().endswith(RECOMMENDED_LABEL.casefold()): return stripped[: -len(RECOMMENDED_LABEL)].strip() return stripped -def _invoke_callback(callback, question, choices, multi_select): - """Invoke the platform callback, passing multi_select if supported. - - Uses signature inspection (not a ``TypeError`` retry) to decide whether - the callback accepts the ``multi_select`` keyword — a retry-on-TypeError - approach would re-invoke a *compatible* callback that raised TypeError - internally, potentially prompting the user twice. - """ - import inspect - - accepts_multi = False +def _accepts_kwarg(callback, name: str) -> bool: + """Signature-inspect (never a TypeError retry, which could re-prompt the user) + whether ``callback`` takes ``name`` or ``**kwargs``. Non-introspectable + callables are conservatively treated as legacy.""" try: - sig = inspect.signature(callback) - params = sig.parameters - accepts_multi = "multi_select" in params or any( - p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values() - ) + params = inspect.signature(callback).parameters except (TypeError, ValueError): - # Builtins / C callables without introspectable signatures: - # be conservative and use the legacy 2-arg form. - accepts_multi = False + return False + return name in params or any( + p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values() + ) - if accepts_multi: + +def _invoke_callback(callback, question, choices, multi_select): + """Invoke the platform callback, passing multi_select if supported.""" + if _accepts_kwarg(callback, "multi_select"): return callback(question, choices, multi_select=multi_select) return callback(question, choices) def _parse_multi_select_response(raw_response) -> List[str]: - """Parse a multi-select response into a list of cleaned choice strings. - - Handles three forms: - - Already a list → stringify + strip each element - - JSON array → parse and strip - - Comma-separated → split, strip, drop empties - """ + """Parse a list / JSON array / comma-separated reply into stripped non-empty strings.""" if isinstance(raw_response, list): return [str(r).strip() for r in raw_response if str(r).strip()] raw = str(raw_response).strip() - # Try JSON array if raw.startswith("["): try: parsed = json.loads(raw) @@ -157,33 +103,29 @@ def _parse_multi_select_response(raw_response) -> List[str]: return [str(p).strip() for p in parsed if str(p).strip()] except json.JSONDecodeError: pass - - # Fall back to comma-separated return [s.strip() for s in raw.split(",") if s.strip()] -# ============================================================================= -# Batch (multi-question) support — issue #18450 -# ============================================================================= +def _clean_choices(choices: list) -> Optional[List[str]]: + """Flatten, drop empties, cap at MAX_CHOICES; None when nothing survives (open-ended).""" + cleaned = [s for s in (_flatten_choice(c) for c in choices) if s] + return cleaned[:MAX_CHOICES] or None + + +def _is_timeout(raw) -> bool: + return raw is None or (isinstance(raw, str) and raw.strip() == TIMEOUT_RESPONSE) + + +# --- batch (multi-question) support ----------------------------------------- def _normalize_questions(questions) -> tuple: - """Validate and normalize the ``questions`` batch parameter. + """Validate the ``questions`` batch param -> ``(normalized, error)``. - Returns ``(normalized, error)`` where exactly one is non-None, except the - empty-list case which returns ``(None, None)`` — an empty array is not an - error, it just means "no batch here" and the caller falls back to the - single-question path. - - Each normalized entry carries: - - ``qid``: stable wire id (``q0``..``qN``, index order). Surfaces key - their per-question answers by this; a model-supplied ``id`` is NOT - used on the wire (it's unvalidated text) and only echoed in results. - - ``id``: the model's optional identifier, or None. - - ``question``: stripped question text. - - ``choices``: decorated choice list (recommended label applied), or - None for open-ended. - - ``choices_offered``: the bare list as offered, for the result JSON. - - ``multi_select``: honored only when choices exist. + An empty list returns ``(None, None)`` (fall back to the single-question + path). Each entry carries ``qid`` (stable wire id ``q`` surfaces key + answers by; the model's ``id`` is unvalidated text so it is only echoed in + results), ``question``, decorated ``choices``, bare ``choices_offered``, + and ``multi_select`` (honored only with choices). """ if not isinstance(questions, list): return None, "questions must be an array of question objects." @@ -208,11 +150,7 @@ def _normalize_questions(questions) -> tuple: if choices is not None: if not isinstance(choices, list): return None, f"questions[{index}].choices must be a list." - choices = [s for s in (_flatten_choice(c) for c in choices) if s] - if len(choices) > MAX_CHOICES: - choices = choices[:MAX_CHOICES] - if not choices: - choices = None + choices = _clean_choices(choices) model_id = str(item.get("id") or "").strip() or None @@ -228,23 +166,6 @@ def _normalize_questions(questions) -> tuple: return normalized, None -def _callback_accepts_questions(callback) -> bool: - """True when the platform callback understands the ``questions`` kwarg. - - Same signature-inspection approach as ``_invoke_callback`` (never a - TypeError retry — that would re-prompt the user on an internal bug). - """ - import inspect - - try: - params = inspect.signature(callback).parameters - return "questions" in params or any( - p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values() - ) - except (TypeError, ValueError): - return False - - def _clean_batch_answer(entry: dict, raw) -> object: """Strip presentation from one locked answer (label, multi-select JSON).""" if entry["multi_select"]: @@ -253,12 +174,7 @@ def _clean_batch_answer(entry: dict, raw) -> object: def _batch_result(normalized: List[dict], answers: dict, timed_out: bool) -> str: - """Assemble the batch result JSON from per-qid answers. - - Unanswered questions surface as empty ``user_response`` — with the - top-level ``timed_out`` flag (present only when true) telling the agent - whether those blanks are deliberate skips or the user walking away. - """ + """Batch result JSON; unanswered -> "" and ``timed_out`` present only when true.""" responses = [] for entry in normalized: row = {} @@ -279,24 +195,16 @@ def _batch_result(normalized: List[dict], answers: dict, timed_out: bool) -> str def _run_batch(normalized: List[dict], callback, question: str) -> str: """Dispatch a validated batch to the platform callback. - Batch-capable callbacks (a ``questions`` kwarg, detected by signature) - get the whole list once and reply with ``{"answers": {qid: raw}}`` plus - an optional ``timed_out`` flag — as a dict or a JSON string (the - tui_gateway ``_block`` bridge can only carry strings). - - Legacy callbacks are looped one question at a time (messaging adapters, - older plugins). An explicit empty answer is a skip and the loop - continues; a timeout (``None`` or the ``TIMEOUT_RESPONSE`` sentinel) - means the user walked away, so the loop aborts instead of pestering - them with the remaining questions. Answers collected before the abort - are kept either way. + Batch-capable callbacks (``questions`` kwarg) get the whole list once and + reply ``{"answers": {qid: raw}, "timed_out"?}`` as a dict or JSON string + (the tui_gateway bridge only carries strings). Legacy callbacks are looped + per question; an empty answer is a skip, a timeout aborts the rest. """ - if _callback_accepts_questions(callback): + answers: dict = {} + timed_out = False + if _accepts_kwarg(callback, "questions"): raw = callback(question, None, questions=normalized) - - answers: dict = {} - timed_out = False - if raw is None or (isinstance(raw, str) and raw.strip() == TIMEOUT_RESPONSE): + if _is_timeout(raw): timed_out = True elif isinstance(raw, dict): answers = dict(raw.get("answers") or {}) @@ -309,17 +217,14 @@ def _run_batch(normalized: List[dict], callback, question: str) -> str: if isinstance(parsed, dict): answers = dict(parsed.get("answers") or {}) timed_out = bool(parsed.get("timed_out")) - # Any other falsy/unparseable reply is a cancel-all: every answer - # empty, no timeout flag (mirrors the single-question skip). + # Any other falsy/unparseable reply is a cancel-all (mirrors the single-question skip). return _batch_result(normalized, answers, timed_out) - answers = {} - timed_out = False for entry in normalized: raw = _invoke_callback( callback, entry["question"], entry["choices"], entry["multi_select"], ) - if raw is None or (isinstance(raw, str) and raw.strip() == TIMEOUT_RESPONSE): + if _is_timeout(raw): timed_out = True break answers[entry["qid"]] = raw @@ -333,35 +238,12 @@ def clarify_tool( questions: Optional[List[dict]] = None, callback: Optional[Callable] = None, ) -> str: - """ - Ask the user a question, optionally with multiple-choice options. + """Ask the user one question (``question``/``choices``/``multi_select``) or a + batch (``questions``, which takes precedence when non-empty). - Args: - question: The question text to present. - choices: Up to 4 predefined answer choices. When omitted the - question is purely open-ended. - multi_select: When True, the user can select multiple choices - (checkboxes). The ``user_response`` in the output JSON - will be a list of strings instead of a single string. - Has no effect when ``choices`` is omitted. - questions: Up to 5 independent questions asked as one batch - (issue #18450). Each item: ``{id?, question, choices?, - multi_select?}``. When present (non-empty), the single - ``question``/``choices``/``multi_select`` parameters - are ignored and the result JSON is ``{"responses": - [...]}`` (plus ``"timed_out": true`` when the user - stopped answering partway). - callback: Platform-provided function that handles the actual UI - interaction. Signature: - ``callback(question, choices, multi_select=False) -> str``. - Batch-capable platforms additionally accept a - ``questions`` keyword and receive the normalized list - in one call; platforms without it are looped one - question at a time. - Injected by the agent runner (cli.py / gateway). - - Returns: - JSON string with the user's response(s). + ``callback(question, choices, multi_select=False) -> str`` is platform + injected; batch-capable callbacks also accept ``questions=``. Returns the + result JSON (``{"responses": [...]}`` for batches). """ if questions is not None: normalized, error = _normalize_questions(questions) @@ -387,27 +269,15 @@ def clarify_tool( question = question.strip() - # Validate and trim choices if choices is not None: if not isinstance(choices, list): return tool_error("choices must be a list of strings.") - # LLMs sometimes emit dict-shaped choices (e.g. [{"description": "..."}]) - # instead of bare strings. _flatten_choice unwraps them to their - # user-facing text here — the single platform-agnostic entry point — - # so the CLI panel, Discord buttons, and Telegram list all render clean - # text and the resolved answer is never a raw Python dict repr. - choices = [s for s in (_flatten_choice(c) for c in choices) if s] - if len(choices) > MAX_CHOICES: - choices = choices[:MAX_CHOICES] - if not choices: - choices = None # empty list → open-ended + choices = _clean_choices(choices) if callback is None: return tool_error("Clarify tool is not available in this execution context.") - # The first choice is the agent's pick (the schema says order best-first), - # so it reaches every surface carrying the "(Recommended)" label. The bare - # list is what goes back to the agent — the label is presentation only. + # The bare list goes back to the agent; the "(Recommended)" label is presentation only. offered = choices if choices is not None: choices = mark_recommended(choices) @@ -434,10 +304,6 @@ def check_clarify_requirements() -> bool: return True -# ============================================================================= -# OpenAI Function-Calling Schema -# ============================================================================= - CLARIFY_SCHEMA = { "name": "clarify", "description": ( diff --git a/tools/close_preview_tool.py b/tools/close_preview_tool.py index d9727155dd..5b3cdc5410 100644 --- a/tools/close_preview_tool.py +++ b/tools/close_preview_tool.py @@ -1,59 +1,22 @@ #!/usr/bin/env python3 """Close the Hermes desktop GUI's preview pane, or one of its tabs. -Lives in the ``desktop_ui`` toolset (same as ``open_preview``), which the GUI -gateway enables only for a session whose source is the desktop app. Emits -``preview.close`` through the shared ``desktop_ui`` bridge; the renderer drops -the matching tab — or the whole pane when no url is given — for the window -that asked and never steals a background session's view. +Registration moved into `desktop_preview`; kept for its ``preview.close`` action. The +renderer drops the matching tab — or the whole pane when no url is given — for the +window that asked, never a background session's view. """ -import json - from tools import desktop_ui from tools.open_preview_tool import _normalize_target -from tools.registry import registry, tool_error def close_preview_tool(url: str = "") -> str: """Ask the desktop GUI to close the preview pane, or the tab for ``url``.""" target = _normalize_target(url or "") - - try: - ok = desktop_ui.emit("preview.close", {"url": target}) - except Exception as exc: - return tool_error(f"Failed to close the preview pane: {exc}") - if not ok: - return tool_error("The preview pane is only available in the Hermes desktop app.") - - return json.dumps({"success": True, "url": target}, ensure_ascii=False) - - -CLOSE_PREVIEW_SCHEMA = { - "name": "close_preview", - "description": ( - "Close the preview pane beside the chat in the Hermes desktop app, or one " - "tab inside it. Use this when the user asks to close, hide, or dismiss the " - "preview — e.g. \"close the preview pane\", \"close cnn.com\", \"hide the " - "preview\". Omit url to close the whole pane (every tab). Pass a web URL, " - "localhost address, or file path to close only that tab. Counterpart of " - "open_preview." - ), - "parameters": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": ( - "Optional. The tab to close: a web URL (https://… or a bare " - "domain), a localhost URL, or a file path. Omit to close the " - "whole preview pane." - ), - }, - }, - }, -} - - -# Registration removed: consolidated into the `preview` tool (#95681); -# this module keeps its functions for the preview_tool. + return desktop_ui.emit_or_error( + "preview.close", + {"url": target}, + "Failed to close the preview pane: ", + "The preview pane is only available in the Hermes desktop app.", + {"success": True, "url": target}, + ) diff --git a/tools/code_execution_env.py b/tools/code_execution_env.py new file mode 100644 index 0000000000..e70eb874e3 --- /dev/null +++ b/tools/code_execution_env.py @@ -0,0 +1,342 @@ +"""Child-process environment for execute_code: env scrubbing, interpreter and cwd resolution. + +Both the per-call remote path and the local session kernel build their child +env through ``_build_child_env`` so the security rules (secret scrubbing, +PYTHONPATH hygiene, UTF-8 forcing, TZ) cannot drift between them. +""" + +import logging +import os +import platform +import subprocess +import sys +from typing import Dict + +# Logger name kept as the origin module's so existing log expectations hold. +logger = logging.getLogger("tools.code_execution_tool") + +_IS_WINDOWS = platform.system() == "Windows" + +# Scrub order: secret-substring block first; whatever is left must match a safe +# prefix, the exact-name HERMES_ allowlist, or (Windows) an OS-essential name. +# The broad "HERMES_" prefix is deliberately NOT safe — it leaked config vars +# without a secret substring (HERMES_BASE_URL, HERMES_KANBAN_DB, *_WEBHOOK). +# HERMES_RPC_SOCKET / HERMES_RPC_DIR / TZ / HOME are injected after scrubbing. +_SAFE_ENV_PREFIXES = ("PATH", "HOME", "USER", "LANG", "LC_", "TERM", + "TMPDIR", "TMP", "TEMP", "SHELL", "LOGNAME", + "XDG_", "PYTHONPATH", "VIRTUAL_ENV", "CONDA") +# "PASS" is intentionally absent: it false-positives on BYPASS_CACHE / +# COMPASS_DIR / PASSENGER_HOST while PASSWORD/PASSWD already cover credentials. +_SECRET_SUBSTRINGS = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL", + "PASSWD", "AUTH", "DSN", "WEBHOOK", + "CREDS", "BEARER", "APIKEY") + +# Non-secret runtime-location flags that repo-root modules a sandbox script +# imports may read at import time. HERMES_DELEGATED_CHILD_CONTEXT must ride +# along or a child that imports Hermes code loses the Kanban mutation guard +# while still inheriting HERMES_HOME. +_HERMES_CHILD_ALLOWED = frozenset({ + "HERMES_HOME", + "HERMES_PROFILE", + "HERMES_CONFIG", + "HERMES_ENV", + "HERMES_DELEGATED_CHILD_CONTEXT", +}) + +# Windows-only: without these the CRT itself fails — socket.socket() raises +# WinError 10106 (Winsock can't find mswsock.dll) and subprocess can't resolve +# cmd.exe. Well-known OS paths, not secrets; the substring block still runs. +_WINDOWS_ESSENTIAL_ENV_VARS = frozenset({ + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "OS", + "PROCESSOR_ARCHITECTURE", + "NUMBER_OF_PROCESSORS", + "PUBLIC", + "ALLUSERSPROFILE", + "PROGRAMDATA", + "PROGRAMFILES", + "PROGRAMFILES(X86)", + "PROGRAMW6432", + "APPDATA", + "LOCALAPPDATA", + "USERPROFILE", + "USERDOMAIN", + "USERNAME", + "HOMEDRIVE", + "HOMEPATH", + "COMPUTERNAME", +}) + + +def _scrub_child_env(source_env, is_passthrough=None, is_windows=None): + """Produce the scrubbed child-process env for execute_code. + + Rules, in order: (1) passthrough vars (skill/config-declared) resolve + through the active profile secret scope — an absent scoped value is + omitted; (2) secret-substring names are blocked; (3) safe prefixes pass; + (4) operational HERMES_* pass by exact name; (5) on Windows the + OS-essential allowlist passes by exact name. + """ + try: + from tools.env_passthrough import is_env_passthrough, resolve_passthrough_value + except Exception: + is_env_passthrough = lambda _: False # noqa: E731 + resolve_passthrough_value = lambda _name, _fallback: None # noqa: E731 + if is_passthrough is None: + is_passthrough = is_env_passthrough + if is_windows is None: + is_windows = _IS_WINDOWS + + scrubbed = {} + # Non-secret HERMES_* vars that no allowlist admits are dropped on purpose, + # but a script importing a repo module that reads one at import time would + # otherwise see it silently unset — log the drop once, pointing at the + # env_passthrough opt-in. + _dropped_hermes = [] + for k, v in source_env.items(): + if is_passthrough(k): + resolved = resolve_passthrough_value(k, v) + if resolved is not None: + scrubbed[k] = resolved + continue + if any(s in k.upper() for s in _SECRET_SUBSTRINGS): + continue + if (any(k.startswith(p) for p in _SAFE_ENV_PREFIXES) + or k in _HERMES_CHILD_ALLOWED + or (is_windows and k.upper() in _WINDOWS_ESSENTIAL_ENV_VARS)): + scrubbed[k] = v + elif k.startswith("HERMES_"): + _dropped_hermes.append(k) + if _dropped_hermes: + logger.debug( + "execute_code: dropped %d non-allowlisted HERMES_* var(s) from the " + "sandbox child env (%s). This is intentional hardening (#27303); if " + "a sandbox script legitimately needs one, declare it via " + "env_passthrough in the skill/config so it passes by explicit opt-in.", + len(_dropped_hermes), + ", ".join(sorted(_dropped_hermes)), + ) + + # delegate_task children are marked by a ContextVar, not os.environ, and the + # sandbox crosses a process boundary: bridge the marker and strip + # dispatcher-owned Kanban vars AFTER the scrub so an explicit passthrough + # cannot re-grant a delegated child the parent's board mutation capability. + try: + from agent.delegation_context import ( + is_delegated_child_process_context, + scrub_kanban_env, + ) + + if is_delegated_child_process_context(): + scrubbed = scrub_kanban_env(scrubbed) + except Exception: + pass + return scrubbed + + +def _build_child_env(*, rpc_endpoint: str, rpc_token: str, tmpdir: str, + child_python: str) -> Dict[str, str]: + """Build the scrubbed child environment both execution paths share.""" + from hermes_constants import apply_subprocess_home_env + child_env = _scrub_child_env(os.environ) + child_env["HERMES_RPC_SOCKET"] = rpc_endpoint + child_env["HERMES_RPC_TOKEN"] = rpc_token + child_env["PYTHONDONTWRITEBYTECODE"] = "1" + # Force UTF-8 stdio and default file encoding: on Windows sys.stdout is + # bound to the console code page (cp1252) and print("→") raises + # UnicodeEncodeError; PYTHONUTF8 also makes open()'s default UTF-8. + # Harmless belt-and-suspenders under a C/POSIX locale (minimal containers). + child_env["PYTHONIOENCODING"] = "utf-8" + child_env["PYTHONUTF8"] = "1" + # Only TZ reaches the child; HERMES_TIMEZONE is an internal setting. + _tz_name = os.getenv("HERMES_TIMEZONE", "").strip() + if _tz_name: + child_env["TZ"] = _tz_name + child_env.pop("HERMES_TIMEZONE", None) + + apply_subprocess_home_env(child_env) + # PYTHONPATH: the staging dir (hermes_tools.py lives there) must always be + # importable, even when project mode changes CWD. Hermes's own root is + # added ONLY when the child runs in Hermes's Python environment — exposing + # Hermes's site-packages to an external project interpreter can mix + # incompatible compiled extensions (3.12 NumPy under a 3.9 venv). Inherited + # Hermes-owned entries (PYTHONPATH passes the scrub) are stripped first so + # they never shadow the child's sys.path. + from tools.environments.local import _strip_hermes_owned_pythonpath + _strip_hermes_owned_pythonpath(child_env) + _hermes_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + _existing_pp = child_env.get("PYTHONPATH", "") + _pp_parts = [tmpdir] + if _uses_hermes_python_environment(child_python): + _pp_parts.append(_hermes_root) + elif child_python not in _external_env_logged: + # Import behavior changes silently otherwise — surface it once per + # interpreter path so "import hermes_constants fails" is diagnosable. + _external_env_logged.add(child_python) + logger.info( + "execute_code: child interpreter %s is outside the Hermes " + "environment; hermes root omitted from PYTHONPATH", + child_python, + ) + if _existing_pp: + _pp_parts.append(_existing_pp) + child_env["PYTHONPATH"] = os.pathsep.join(_pp_parts) + return child_env + + +# Interpreter-probe caches: success-only dicts (FIFO-evicted at the cap) rather +# than lru_cache — a transient probe failure (fork pressure, 5s timeout on a +# loaded host) must not stick for the process lifetime. +_PROBE_CACHE_MAX = 32 +_usable_python_cache: dict = {} +_python_prefix_cache: dict = {} + +# Interpreter paths already reported as outside the Hermes environment. +_external_env_logged: set = set() + + +def _cache_probe_result(cache: dict, key: str, value): + """Insert into a bounded probe cache, FIFO-evicting at the cap.""" + if len(cache) >= _PROBE_CACHE_MAX: + cache.pop(next(iter(cache))) + cache[key] = value + + +def _probe_python(python_path: str, code: str, *, text: bool = False): + """Run ``python_path -c code``; None if missing, unspawnable, or past the 5s timeout.""" + try: + from agent.delegation_context import delegated_child_subprocess_env + + return subprocess.run( + [python_path, "-c", code], + timeout=5, + capture_output=True, + text=text, + creationflags=subprocess.CREATE_NO_WINDOW if _IS_WINDOWS else 0, + stdin=subprocess.DEVNULL, + env=delegated_child_subprocess_env(), + ) + except (OSError, subprocess.TimeoutExpired, subprocess.SubprocessError): + return None + + +def _is_usable_python(python_path: str) -> bool: + """Whether the interpreter is Python 3.8+ (what the RPC stubs need); success cached, failure retried.""" + cached = _usable_python_cache.get(python_path) + if cached is not None: + return cached + result = _probe_python( + python_path, + "import sys; sys.exit(0 if sys.version_info >= (3, 8) else 1)", + ) + if result is None: + return False + usable = result.returncode == 0 + _cache_probe_result(_usable_python_cache, python_path, usable) + return usable + + +def _python_environment_prefix(python_path: str) -> str: + """Resolved ``sys.prefix`` reported by *python_path* ("" on failure; failures are not cached).""" + cached = _python_prefix_cache.get(python_path) + if cached is not None: + return cached + result = _probe_python(python_path, "import sys; print(sys.prefix)", text=True) + if result is not None and result.returncode == 0 and result.stdout.strip(): + prefix = os.path.realpath(result.stdout.strip()) + _cache_probe_result(_python_prefix_cache, python_path, prefix) + return prefix + return "" + + +def _uses_hermes_python_environment(python_path: str) -> bool: + """Whether *python_path* belongs to Hermes's active Python environment. + + Short-circuits when it IS the running interpreter (by path or realpath) so + no probe runs on the default strict path and a flaky probe of + sys.executable can never drop the hermes root; the realpath leg also covers + venvs whose bin/python resolves to the same binary (``uv run``). + """ + if python_path == sys.executable or ( + os.path.realpath(python_path) == os.path.realpath(sys.executable) + ): + return True + return _python_environment_prefix(python_path) == os.path.realpath(sys.prefix) + + +def _resolve_child_python(mode: str) -> str: + """Interpreter for the child: ``sys.executable`` in strict mode; in project + mode the active VIRTUAL_ENV/CONDA_PREFIX python if it exists and passes the + 3.8+ probe, else ``sys.executable``.""" + if mode != "project": + return sys.executable + + if _IS_WINDOWS: + exe_names = ("python.exe", "python3.exe") + subdirs = ("Scripts",) + else: + exe_names = ("python", "python3") + subdirs = ("bin",) + + for var in ("VIRTUAL_ENV", "CONDA_PREFIX"): + root = os.environ.get(var, "").strip() + if not root: + continue + for subdir in subdirs: + for exe in exe_names: + candidate = os.path.join(root, subdir, exe) + if not (os.path.isfile(candidate) and os.access(candidate, os.X_OK)): + continue + if _is_usable_python(candidate): + return candidate + logger.info( + "execute_code: skipping %s=%s (Python version < 3.8 or broken). " + "Using sys.executable instead.", var, candidate, + ) + return sys.executable + + return sys.executable + + +def _resolve_child_cwd(mode: str, staging_dir: str, task_id: str = "") -> str: + """Working directory for the child. + + Strict mode: the staging dir. Project mode mirrors the terminal/file-tool + ladder so every file-writing path in a session agrees: the session's cwd + record (its `cd` state) → registered ``session.cwd.set`` override → + TERMINAL_CWD → os.getcwd() → staging dir (never Popen on a missing cwd). + """ + if mode != "project": + return staging_dir + if task_id: + try: + from tools.terminal_tool import get_session_cwd + + recorded = get_session_cwd(task_id) + except Exception: + recorded = None + if recorded and os.path.isdir(recorded): + return recorded + try: + from tools.file_tools import _registered_task_cwd_override + + session_cwd = _registered_task_cwd_override(task_id) + except Exception: + session_cwd = None + if session_cwd and os.path.isdir(session_cwd): + return session_cwd + from agent.runtime_cwd import scope_terminal_cwd + + raw = scope_terminal_cwd().strip() + if raw: + expanded = os.path.expanduser(raw) + if os.path.isdir(expanded): + return expanded + here = os.getcwd() + if os.path.isdir(here): + return here + return staging_dir diff --git a/tools/code_execution_rpc.py b/tools/code_execution_rpc.py new file mode 100644 index 0000000000..1a49a96f98 --- /dev/null +++ b/tools/code_execution_rpc.py @@ -0,0 +1,270 @@ +"""Host-side RPC servers for execute_code sandboxes. + +Two transports share one request pipeline (token check → allow-list → call +budget → dispatch under output silence → log): ``_rpc_server_loop`` serves the +local UDS/TCP socket, ``_rpc_poll_loop`` polls a remote filesystem for request +files via ``env.execute()``. +""" + +import base64 +import json +import logging +import secrets +import shlex +import socket +import threading +import time + +from agent.thread_scoped_output import thread_scoped_silence +from tools.registry import tool_error + +# Logger name kept as the origin module's so existing log expectations hold. +logger = logging.getLogger("tools.code_execution_tool") + +# Terminal parameters that must not be used from ephemeral sandbox scripts. +_TERMINAL_BLOCKED_PARAMS = {"background", "pty", "notify", "notify_on_complete", "watch_patterns"} + + +def _default_dispatch(task_id): + from model_tools import handle_function_call + + def dispatch(tool_name, tool_args): + return handle_function_call(tool_name, tool_args, task_id=task_id) + + return dispatch + + +def _rpc_token_ok(request: dict, rpc_token: str) -> bool: + """Constant-time token check; an empty server token fails closed.""" + # Compare as bytes: compare_digest raises TypeError on a str with + # non-ASCII characters, and the token comes from script-supplied JSON. + return bool(rpc_token) and secrets.compare_digest( + str(request.get("token") or "").encode(), rpc_token.encode() + ) + + +def _handle_rpc_request( + request: dict, + *, + allowed_tools: frozenset, + tool_call_counter: list, + max_tool_calls: int, + dispatch, + tool_call_log: list, + call_start: float, + where: str, +) -> str: + """Enforce allow-list + budget, then dispatch one authenticated request. + + Only a dispatched call consumes budget and is logged; refusals are free. + """ + tool_name = request.get("tool", "") + tool_args = request.get("args", {}) + + if tool_name not in allowed_tools: + available = ", ".join(sorted(allowed_tools)) + return tool_error( + f"Tool '{tool_name}' is not available in execute_code. " + f"Available: {available}" + ) + if tool_call_counter[0] >= max_tool_calls: + return tool_error( + f"Tool call limit reached ({max_tool_calls}). " + "No more tool calls allowed in this execution." + ) + + if tool_name == "terminal" and isinstance(tool_args, dict): + for param in _TERMINAL_BLOCKED_PARAMS: + tool_args.pop(param, None) + + # Silence handler status prints so they don't leak into the CLI spinner. + try: + with thread_scoped_silence(): + result = dispatch(tool_name, tool_args) + except Exception as exc: + logger.error("Tool call failed in %s: %s", where, exc, exc_info=True) + result = tool_error(str(exc)) + + tool_call_counter[0] += 1 + tool_call_log.append({ + "tool": tool_name, + "args_preview": str(tool_args)[:80], + "duration": round(time.monotonic() - call_start, 2), + }) + return result + + +def _rpc_server_loop( + server_sock: socket.socket, + task_id: str, + tool_call_log: list, + tool_call_counter: list, # mutable [int] so the thread can increment + max_tool_calls: int, + allowed_tools: frozenset, + stop_event: threading.Event, + rpc_token: str, + dispatch=None, +): + """Accept one client and serve newline-delimited JSON requests until it + disconnects, idles 300s, or the call limit is reached. + + ``dispatch`` overrides how an allowed, budgeted call runs: per-call + sandboxes use the default (the thread already carries the cell's context), + while session kernels pass a dispatcher that rebinds each call to the + CURRENT cell's authority — the serving thread outlives many cells there. + """ + if dispatch is None: + dispatch = _default_dispatch(task_id) + + conn = None + try: + server_sock.settimeout(0.05) + while not stop_event.is_set(): + try: + conn, _ = server_sock.accept() + break + except socket.timeout: + continue + if conn is None: + return + conn.settimeout(300) + + buf = b"" + while True: + try: + chunk = conn.recv(65536) + except socket.timeout: + break + if not chunk: + break + buf += chunk + + while b"\n" in buf: + line, buf = buf.split(b"\n", 1) + line = line.strip() + if not line: + continue + + call_start = time.monotonic() + try: + request = json.loads(line.decode()) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + resp = tool_error(f"Invalid RPC request: {exc}") + else: + if not _rpc_token_ok(request, rpc_token): + resp = tool_error("Unauthorized RPC request") + else: + resp = _handle_rpc_request( + request, + allowed_tools=allowed_tools, + tool_call_counter=tool_call_counter, + max_tool_calls=max_tool_calls, + dispatch=dispatch, + tool_call_log=tool_call_log, + call_start=call_start, + where="sandbox", + ) + conn.sendall((resp + "\n").encode()) + + except socket.timeout: + logger.debug("RPC listener socket timeout") + except OSError as e: + logger.debug("RPC listener socket error: %s", e, exc_info=True) + finally: + if conn: + try: + conn.close() + except OSError as e: + logger.debug("RPC conn close error: %s", e) + + +def _rpc_poll_loop( + env, + rpc_dir: str, + task_id: str, + tool_call_log: list, + tool_call_counter: list, + max_tool_calls: int, + allowed_tools: frozenset, + stop_event: threading.Event, + rpc_token: str, +): + """Poll the remote filesystem for request files and answer them. + + Runs in a background thread; each ``env.execute()`` is an independent + process, so this is safe alongside the script-execution thread. Malformed + or unauthorized requests are removed without a response. + """ + dispatch = _default_dispatch(task_id) + poll_interval = 0.1 + + quoted_rpc_dir = shlex.quote(rpc_dir) + while not stop_event.is_set(): + try: + ls_result = env.execute( + f"ls -1 {quoted_rpc_dir}/req_* 2>/dev/null || true", + cwd="/", + timeout=10, + ) + output = ls_result.get("output", "").strip() + if not output: + stop_event.wait(poll_interval) + continue + + req_files = sorted([ + f.strip() for f in output.split("\n") + if f.strip() + and not f.strip().endswith(".tmp") + and "/req_" in f.strip() + ]) + + for req_file in req_files: + if stop_event.is_set(): + break + + call_start = time.monotonic() + quoted_req_file = shlex.quote(req_file) + read_result = env.execute(f"cat {quoted_req_file}", cwd="/", timeout=10) + try: + request = json.loads(read_result.get("output", "")) + except (json.JSONDecodeError, ValueError): + logger.debug("Malformed RPC request in %s", req_file) + env.execute(f"rm -f {quoted_req_file}", cwd="/", timeout=5) + continue + if not _rpc_token_ok(request, rpc_token): + logger.debug("Unauthorized RPC request in %s", req_file) + env.execute(f"rm -f {quoted_req_file}", cwd="/", timeout=5) + continue + + tool_result = _handle_rpc_request( + request, + allowed_tools=allowed_tools, + tool_call_counter=tool_call_counter, + max_tool_calls=max_tool_calls, + dispatch=dispatch, + tool_call_log=tool_call_log, + call_start=call_start, + where="remote sandbox", + ) + + # Write the response atomically (tmp + rename) via echo piping — + # Modal doesn't reliably deliver stdin_data to chained commands. + res_file = f"{rpc_dir}/res_{request.get('seq', 0):06d}" + quoted_res_file = shlex.quote(res_file) + encoded_result = base64.b64encode( + tool_result.encode("utf-8") + ).decode("ascii") + env.execute( + f"echo '{encoded_result}' | base64 -d > {quoted_res_file}.tmp" + f" && mv {quoted_res_file}.tmp {quoted_res_file}", + cwd="/", + timeout=60, + ) + env.execute(f"rm -f {quoted_req_file}", cwd="/", timeout=5) + + except Exception as e: + if not stop_event.is_set(): + logger.debug("RPC poll error: %s", e, exc_info=True) + + if not stop_event.is_set(): + stop_event.wait(poll_interval) diff --git a/tools/credential_files.py b/tools/credential_files.py index 631eb03710..83e44f9ff8 100644 --- a/tools/credential_files.py +++ b/tools/credential_files.py @@ -1,21 +1,10 @@ """File passthrough registry for remote terminal backends. Remote backends (Docker, Modal, SSH) create sandboxes with no host files. -This module ensures that credential files, skill directories, and host-side -cache directories (documents, images, audio, screenshots) are mounted or -synced into those sandboxes so the agent can access them. - -**Credentials and skills** — session-scoped registry fed by skill declarations -(``required_credential_files``) and user config (``terminal.credential_files``). - -**Cache directories** — gateway-cached uploads, browser screenshots, TTS -audio, and processed images. Mounted read-only so the remote terminal can -reference files the host side created (e.g. ``unzip`` an uploaded archive). - -Remote backends call :func:`get_credential_file_mounts`, -:func:`get_skills_directory_mount` / :func:`iter_skills_files`, and -:func:`get_cache_directory_mounts` / :func:`iter_cache_files` at sandbox -creation time and before each command (for resync on Modal). +This module tells them which credential files (skill ``required_credential_files`` ++ ``terminal.credential_files`` config), skill directories, and host-side cache +directories (documents, images, audio, screenshots, uploads) to mount or sync +in, at sandbox creation and before each command (resync on Modal). """ from __future__ import annotations @@ -25,7 +14,7 @@ import os import posixpath from contextvars import ContextVar from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, Iterator, List, Optional, Tuple from hermes_cli.config import cfg_get from agent.skill_utils import EXCLUDED_SKILL_DIRS @@ -37,13 +26,11 @@ except ImportError: # noqa: F401 - sentinel consumed in register_credential_fil logger = logging.getLogger(__name__) -# Session-scoped list of credential files to mount. -# Backed by ContextVar to prevent cross-session data bleed in the gateway pipeline. +# Session-scoped registry; ContextVar prevents cross-session bleed in the gateway. _registered_files_var: ContextVar[Dict[str, str]] = ContextVar("_registered_files") def _get_registered() -> Dict[str, str]: - """Get or create the registered credential files dict for the current context/session.""" try: return _registered_files_var.get() except LookupError: @@ -61,66 +48,54 @@ def _resolve_hermes_home() -> Path: return get_hermes_home() -def register_credential_file( - relative_path: str, - container_base: str = "/root/.hermes", -) -> bool: - """Register a credential file for mounting into remote sandboxes. - - *relative_path* is relative to ``HERMES_HOME`` (e.g. ``google_token.json``). - Returns True if the file exists on the host and was registered. - - Security: rejects absolute paths and path traversal sequences (``..``). - The resolved host path must remain inside HERMES_HOME so that a malicious - skill cannot declare ``required_credential_files: ['../../.ssh/id_rsa']`` - and exfiltrate sensitive host files into a container sandbox. - - Containment alone is not sufficient, because HERMES_HOME is exactly where - the MASTER credential stores live. A skill legitimately needs its own - service token (``google_token.json``); it never needs ``.env`` (every - provider key), ``auth.json`` (all provider tokens and OAuth grants), - ``mcp-tokens/`` or the Bitwarden plaintext cache. Those are refused via - the canonical read deny-list (``agent.file_safety.get_read_block_error``) - — the same guard that stops the agent reading them with ``read_file``, so - the mount surface cannot hand a skill what the read surface denies it. - """ - hermes_home = _resolve_hermes_home() - - # Reject absolute paths — they bypass the HERMES_HOME sandbox entirely. - if os.path.isabs(relative_path): - logger.warning( - "credential_files: rejected absolute path %r (must be relative to HERMES_HOME)", - relative_path, - ) - return False - - host_path = hermes_home / relative_path - - # Resolve symlinks and normalise ``..`` before the containment check so - # that traversal like ``../. ssh/id_rsa`` cannot escape HERMES_HOME. +def _contained_host_path( + rel: str, hermes_home: Path, abs_msg: str, traversal_msg: str +) -> Optional[Path]: + """Resolve *rel* under HERMES_HOME, refusing absolute paths and escapes.""" + if os.path.isabs(rel): + logger.warning(abs_msg, rel) + return None + host_path = hermes_home / rel + # Resolve symlinks and ``..`` before the containment check. from tools.path_security import validate_within_dir containment_error = validate_within_dir(host_path, hermes_home) if containment_error: - logger.warning( - "credential_files: rejected path traversal %r (%s)", - relative_path, - containment_error, - ) - return False + logger.warning(traversal_msg, rel, containment_error) + return None + return host_path.resolve() - resolved = host_path.resolve() + +def register_credential_file( + relative_path: str, + container_base: str = "/root/.hermes", +) -> bool: + """Register a HERMES_HOME-relative credential file for mounting. + + Returns True if the file exists on the host and was registered. Rejects + absolute paths and traversal out of HERMES_HOME. Containment alone is not + enough because HERMES_HOME holds the MASTER stores (``.env``, ``auth.json``, + ``mcp-tokens/``): those are refused via the canonical read deny-list + (``agent.file_safety.get_read_block_error``), so the mount surface cannot + hand a skill what the read surface denies it. + """ + hermes_home = _resolve_hermes_home() + resolved = _contained_host_path( + relative_path, + hermes_home, + "credential_files: rejected absolute path %r (must be relative to HERMES_HOME)", + "credential_files: rejected path traversal %r (%s)", + ) + if resolved is None: + return False if not resolved.is_file(): logger.debug("credential_files: skipping %s (not found)", resolved) return False - # Master credential stores are never mountable, even though they sit - # inside HERMES_HOME and therefore pass the containment check above. - # Fails CLOSED: if the canonical guard can't be consulted we refuse the - # mount rather than risk bind-mounting auth.json into a sandbox. The - # import lives at module top (no circular-import concern — file_safety is - # stdlib-only); the sentinel + logger.exception keep guard failures - # debuggable instead of silently swallowed (#67665). + # Master stores pass the containment check above, so the deny-list is the + # real gate. Fails CLOSED: if the guard can't be consulted, refuse rather + # than risk bind-mounting auth.json into a sandbox; the import sentinel + + # logger.exception keep guard failures debuggable, not silently swallowed. if get_read_block_error is None: logger.error( "credential_files: refusing %r — agent.file_safety could not be " @@ -154,12 +129,7 @@ def register_credential_files( entries: list, container_base: str = "/root/.hermes", ) -> List[str]: - """Register multiple credential files from skill frontmatter entries. - - Each entry is either a string (relative path) or a dict with a ``path`` - key. Returns the list of relative paths that were NOT found on the host - (i.e. missing files). - """ + """Register skill-frontmatter entries (str or dict with ``path``); return missing paths.""" missing = [] for entry in entries: if isinstance(entry, str): @@ -188,30 +158,19 @@ def _load_config_files() -> List[Dict[str, str]]: cfg = read_raw_config() cred_files = cfg_get(cfg, "terminal", "credential_files") if isinstance(cred_files, list): - from tools.path_security import validate_within_dir - for item in cred_files: if isinstance(item, str) and item.strip(): rel = item.strip() - if os.path.isabs(rel): - logger.warning( - "credential_files: rejected absolute config path %r", rel, - ) - continue - host_path = hermes_home / rel - containment_error = validate_within_dir(host_path, hermes_home) - if containment_error: - logger.warning( - "credential_files: rejected config path traversal %r (%s)", - rel, containment_error, - ) - continue - resolved_path = host_path.resolve() - if resolved_path.is_file(): - container_path = f"/root/.hermes/{rel}" + resolved_path = _contained_host_path( + rel, + hermes_home, + "credential_files: rejected absolute config path %r", + "credential_files: rejected config path traversal %r (%s)", + ) + if resolved_path is not None and resolved_path.is_file(): result.append({ "host_path": str(resolved_path), - "container_path": container_path, + "container_path": f"/root/.hermes/{rel}", }) except Exception as e: logger.warning("Could not read terminal.credential_files from config: %s", e) @@ -221,20 +180,14 @@ def _load_config_files() -> List[Dict[str, str]]: def get_credential_file_mounts() -> List[Dict[str, str]]: - """Return all credential files that should be mounted into remote sandboxes. - - Each item has ``host_path`` and ``container_path`` keys. - Combines skill-registered files and user config. - """ + """Skill-registered + config credential files as ``host_path``/``container_path`` dicts.""" mounts: Dict[str, str] = {} - # Skill-registered files + # Re-check existence (file may have been deleted since registration). for container_path, host_path in _get_registered().items(): - # Re-check existence (file may have been deleted since registration) if Path(host_path).is_file(): mounts[container_path] = host_path - # Config-based files for entry in _load_config_files(): cp = entry["container_path"] if cp not in mounts and Path(entry["host_path"]).is_file(): @@ -246,58 +199,52 @@ def get_credential_file_mounts() -> List[Dict[str, str]]: ] -def get_skills_directory_mount( - container_base: str = "/root/.hermes", -) -> list[Dict[str, str]]: - """Return mount info for all skill directories (local + external). +def _skill_dir_roots(container_base: str) -> Iterator[Tuple[Path, str]]: + """Yield ``(host_dir, container_root)`` for every existing skills directory. - Skills may include ``scripts/``, ``templates/``, and ``references/`` - subdirectories that the agent needs to execute inside remote sandboxes. - - **Security:** Bind mounts follow symlinks, so a malicious symlink inside - the skills tree could expose arbitrary host files to the container. When - symlinks are detected, this function creates a sanitized copy (regular - files only) in a temp directory and returns that path instead. When no - symlinks are present (the common case), the original directory is returned - directly with zero overhead. - - Returns a list of dicts with ``host_path`` and ``container_path`` keys. - The local skills dir mounts at ``/skills``, external dirs - at ``/external_skills/``. + Local skills mount at ``/skills``, external dirs at + ``/external_skills/``, trusted project-local dirs at + ``/project_skills/`` (separate namespace so container paths stay + stable if external_dirs change). """ - mounts = [] - hermes_home = _resolve_hermes_home() - skills_dir = hermes_home / "skills" + base = container_base.rstrip("/") + skills_dir = _resolve_hermes_home() / "skills" if skills_dir.is_dir(): - host_path = _safe_skills_path(skills_dir) - mounts.append({ - "host_path": host_path, - "container_path": f"{container_base.rstrip('/')}/skills", - }) - - # Mount external skill dirs + yield skills_dir, f"{base}/skills" try: from agent.skill_utils import get_external_skills_dirs, get_project_skills_dirs for idx, ext_dir in enumerate(get_external_skills_dirs()): if ext_dir.is_dir(): - host_path = _safe_skills_path(ext_dir) - mounts.append({ - "host_path": host_path, - "container_path": f"{container_base.rstrip('/')}/external_skills/{idx}", - }) - # Trusted project-local skill dirs (repo checkouts). Separate - # namespace so container paths stay stable if external_dirs change. + yield ext_dir, f"{base}/external_skills/{idx}" for idx, proj_dir in enumerate(get_project_skills_dirs()): if proj_dir.is_dir(): - host_path = _safe_skills_path(proj_dir) - mounts.append({ - "host_path": host_path, - "container_path": f"{container_base.rstrip('/')}/project_skills/{idx}", - }) + yield proj_dir, f"{base}/project_skills/{idx}" except ImportError: pass - return mounts + +def _iter_regular_files(host_dir: Path, container_root: str) -> Iterator[Dict[str, str]]: + """Per-file mount entries under *host_dir*, skipping symlinks.""" + for item in host_dir.rglob("*"): + if item.is_symlink() or not item.is_file(): + continue + rel = item.relative_to(host_dir) + yield {"host_path": str(item), "container_path": f"{container_root}/{rel}"} + + +def get_skills_directory_mount( + container_base: str = "/root/.hermes", +) -> list[Dict[str, str]]: + """Directory mount entries for all skill dirs (local + external + project). + + Bind mounts follow symlinks, so a dir containing any symlink is replaced by + a sanitized temp copy (regular files only); symlink-free dirs are returned + directly with zero overhead. + """ + return [ + {"host_path": _safe_skills_path(host_dir), "container_path": container_path} + for host_dir, container_path in _skill_dir_roots(container_base) + ] _safe_skills_tempdir: Path | None = None @@ -377,58 +324,22 @@ def _iter_syncable_files(root: Path): def iter_skills_files( container_base: str = "/root/.hermes", ) -> List[Dict[str, str]]: - """Yield individual (host_path, container_path) entries for skills files. + """Per-file entries for all skills files (for backends that upload individually). - Includes both the local skills dir and any external dirs configured via - skills.external_dirs. Skips symlinks and anything under - EXCLUDED_SKILL_DIRS entirely. Preferred for backends that upload files - individually (Daytona, Modal) rather than mounting a directory. + Skips symlinks and anything under EXCLUDED_SKILL_DIRS (see _iter_syncable_files). """ - result: List[Dict[str, str]] = [] - - hermes_home = _resolve_hermes_home() - skills_dir = hermes_home / "skills" - if skills_dir.is_dir(): - container_root = f"{container_base.rstrip('/')}/skills" - for item, rel in _iter_syncable_files(skills_dir): - result.append({ - "host_path": str(item), - "container_path": f"{container_root}/{rel}", - }) - - # Include external skill dirs - try: - from agent.skill_utils import get_external_skills_dirs, get_project_skills_dirs - for idx, ext_dir in enumerate(get_external_skills_dirs()): - if not ext_dir.is_dir(): - continue - container_root = f"{container_base.rstrip('/')}/external_skills/{idx}" - for item, rel in _iter_syncable_files(ext_dir): - result.append({ - "host_path": str(item), - "container_path": f"{container_root}/{rel}", - }) - for idx, proj_dir in enumerate(get_project_skills_dirs()): - if not proj_dir.is_dir(): - continue - container_root = f"{container_base.rstrip('/')}/project_skills/{idx}" - for item, rel in _iter_syncable_files(proj_dir): - result.append({ - "host_path": str(item), - "container_path": f"{container_root}/{rel}", - }) - except ImportError: - pass - - return result + return [ + {"host_path": str(item), "container_path": f"{container_root}/{rel}"} + for host_dir, container_root in _skill_dir_roots(container_base) + for item, rel in _iter_syncable_files(host_dir) + ] # --------------------------------------------------------------------------- # Cache directory mounts (documents, images, audio, videos, screenshots) # --------------------------------------------------------------------------- -# The cache subdirectories that should be mirrored into remote backends. -# Each tuple is (new_subpath, old_name) matching hermes_constants.get_hermes_dir(). +# (new_subpath, old_name) pairs matching hermes_constants.get_hermes_dir(). _CACHE_DIRS: list[tuple[str, str]] = [ ("cache/documents", "document_cache"), ("cache/images", "image_cache"), @@ -437,20 +348,12 @@ _CACHE_DIRS: list[tuple[str, str]] = [ ("cache/screenshots", "browser_screenshots"), ("cache/web", "web_cache"), ("cache/delegation", "delegation_cache"), - # Oversized tool results (tools/tool_result_storage.py). Host-side is the - # single canonical location; mounting/syncing it lets remote backends - # read spilled results at the translated path instead of needing a - # separate in-sandbox copy. + # Oversized tool results (tools/tool_result_storage.py); host side is the + # single canonical location. ("cache/spillover", "cache/spillover"), - # Desktop/clipboard/PDF uploads land in the flat top-level ``images/`` dir - # (tui_gateway attach RPCs), not under ``cache/``. Mount it so vision can - # reach uploads inside sandbox containers (#69575). No legacy alias exists, - # so both tuple slots are ``images``. + # Flat top-level desktop staging dirs (tui_gateway attach RPCs), not under + # cache/; no legacy alias, so both slots match (#69575, #76577). ("images", "images"), - # Desktop non-image file attachments (tui_gateway ``file.attach`` staging) - # land in the flat top-level ``attachments/`` dir. Mount it so the agent's - # file tools can read dropped binaries (zip/pdf/...) from inside sandbox - # containers instead of dangling host paths (#76577). ("attachments", "attachments"), ] @@ -458,35 +361,24 @@ _CACHE_DIRS: list[tuple[str, str]] = [ def get_cache_directory_mounts( container_base: str = "/root/.hermes", ) -> List[Dict[str, str]]: - """Return mount entries for each cache directory that exists on disk. - - Used by Docker to create bind mounts. Each entry has ``host_path`` and - ``container_path`` keys. The host path is resolved via - ``get_hermes_dir()`` for backward compatibility with old directory layouts. - """ + """Bind-mount entries for each cache directory (host layout via ``get_hermes_dir``).""" from hermes_constants import get_hermes_dir mounts: List[Dict[str, str]] = [] for new_subpath, old_name in _CACHE_DIRS: host_dir = get_hermes_dir(new_subpath, old_name) if not host_dir.is_dir(): - # Create missing staging dirs instead of skipping them: Docker - # snapshots this mount list at container CREATION, so a dir that - # appears later (first desktop attachment, first clipboard image) - # would dangle for the whole life of a persistent container - # (#76577). An empty bind-mounted dir costs nothing; a missing - # mount costs the feature. get_hermes_dir() already resolved - # new-vs-legacy layout, so creating its answer cannot shadow a - # populated legacy dir. + # Docker snapshots this list at container CREATION, so a dir that + # appears later would dangle for the container's life (#76577): + # create it now; an empty bind mount costs nothing. try: host_dir.mkdir(parents=True, exist_ok=True) except OSError: continue # unwritable home (tests, RO mounts) — skip as before # Always map to the *new* container layout regardless of host layout. - container_path = f"{container_base.rstrip('/')}/{new_subpath}" mounts.append({ "host_path": str(host_dir), - "container_path": container_path, + "container_path": f"{container_base.rstrip('/')}/{new_subpath}", }) return mounts @@ -495,15 +387,7 @@ def map_cache_path_to_container( host_path: str, container_base: str = "/root/.hermes", ) -> Optional[str]: - """Map a host cache path to its mounted path under *container_base*. - - Returns the POSIX container path when *host_path* lives under one of the - auto-mounted cache directories, otherwise ``None``. Backend-agnostic: the - caller decides which ``container_base`` applies (Docker ``/root/.hermes``, - SSH ``/.hermes``, etc.) and whether translation is wanted. - Always joins with ``posixpath`` because container/remote paths are POSIX - regardless of the host OS. - """ + """POSIX container path for a host path under an auto-mounted cache dir, else None.""" path = Path(host_path) for mount in get_cache_directory_mounts(container_base=container_base): host_dir = Path(mount["host_path"]) @@ -519,13 +403,7 @@ def from_agent_visible_cache_path( container_path: str, container_base: str = "/root/.hermes", ) -> str: - """Translate a sandbox/container cache path back to its host path. - - Inverse of :func:`to_agent_visible_cache_path`. Returns the input unchanged - when the active backend is not Docker, or when the path is not under any - auto-mounted cache directory — the caller then treats a still-container - path as "no host file" and falls back to an in-container read. - """ + """Inverse of :func:`to_agent_visible_cache_path`; unchanged unless Docker + cache dir.""" if os.environ.get("TERMINAL_ENV", "local") != "docker": return container_path @@ -543,29 +421,16 @@ def to_agent_visible_cache_path( host_path: str, container_base: str = "/root/.hermes", ) -> str: - """Translate a host cache path to its mounted path inside the sandbox. - - Returns the input unchanged if it is not under any auto-mounted cache - directory, or if the active terminal backend does not require path - translation (local). + """Translate a host cache path to where the active backend sees it. Per-backend base (mirrors ``_agent_cache_base_for_env`` in - tools/image_generation_tool.py, the proven heuristics for where each - backend's Hermes cache lands): - - * docker / modal — bind-mounted (docker) or per-file-synced (modal) at - ``/root/.hermes`` (the *container_base* default). - * ssh / daytona / vercel_sandbox — file-synced under the remote user's - home; ``~/.hermes`` is shell-expanded by the remote shell, so tool - commands resolve it regardless of the actual remote home. Previously - these backends synced the bytes but still rendered the dangling host - path (#76577 gap). - * singularity — NOT translated: Apptainer auto-binds the host home, so - the host path is directly readable and translation would dangle - (cache dirs are not remapped into that sandbox). - - Backend is identified by TERMINAL_ENV (same env var - tools/terminal_tool.py reads in _get_environment_config). + tools/image_generation_tool.py): docker/modal mount/sync at + ``/root/.hermes``; ssh/daytona/vercel_sandbox file-sync under the remote + home, so ``~/.hermes`` (expanded by the remote shell) resolves regardless of + the actual remote home; plugin backends declare ``cache_path_base`` (None = + host paths remain correct); local/singularity/unknown stay unchanged + (Apptainer auto-binds the host home, so translation would dangle). + Backend comes from TERMINAL_ENV, as in terminal_tool._get_environment_config. """ backend = (os.environ.get("TERMINAL_ENV") or "local").strip().lower() if backend in ("docker", "modal"): @@ -573,8 +438,6 @@ def to_agent_visible_cache_path( elif backend in ("ssh", "daytona", "vercel_sandbox"): container_base = "~/.hermes" else: - # Plugin-registered backends declare where synced cache files land - # via ``cache_path_base``; None means host paths remain correct. plugin_base = None try: from agent.terminal_env_registry import provider_flag @@ -583,7 +446,7 @@ def to_agent_visible_cache_path( except Exception: plugin_base = None if not plugin_base: - return host_path # local, singularity, unknown: host path is correct + return host_path container_base = str(plugin_base) mapped = map_cache_path_to_container(host_path, container_base=container_base) @@ -593,11 +456,7 @@ def to_agent_visible_cache_path( def iter_cache_files( container_base: str = "/root/.hermes", ) -> List[Dict[str, str]]: - """Return individual (host_path, container_path) entries for cache files. - - Used by Modal to upload files individually and resync before each command. - Skips symlinks. The container paths use the new ``cache/`` layout. - """ + """Per-file cache entries (Modal upload/resync); skips symlinks.""" from hermes_constants import get_hermes_dir result: List[Dict[str, str]] = [] @@ -606,19 +465,10 @@ def iter_cache_files( if not host_dir.is_dir(): continue container_root = f"{container_base.rstrip('/')}/{new_subpath}" - for item in host_dir.rglob("*"): - if item.is_symlink() or not item.is_file(): - continue - rel = item.relative_to(host_dir) - result.append({ - "host_path": str(item), - "container_path": f"{container_root}/{rel}", - }) + result.extend(_iter_regular_files(host_dir, container_root)) return result def clear_credential_files() -> None: """Reset the skill-scoped registry (e.g. on session reset).""" _get_registered().clear() - - diff --git a/tools/cronjob_job_args.py b/tools/cronjob_job_args.py new file mode 100644 index 0000000000..094061e8a3 --- /dev/null +++ b/tools/cronjob_job_args.py @@ -0,0 +1,468 @@ +"""Cron job argument normalization, validation and result shaping +(extracted from tools/cronjob_tools.py; re-exported there).""" + +import logging +from typing import Any, Dict, List, Optional, Union + +from cron.jobs import effective_job_state + +# Logger parity with the origin module. +logger = logging.getLogger("tools.cronjob_tools") + + +def _origin_from_env() -> Optional[Dict[str, str]]: + from gateway.session_context import get_session_env + origin_platform = get_session_env("HERMES_SESSION_PLATFORM") + origin_chat_id = get_session_env("HERMES_SESSION_CHAT_ID") + if not (origin_platform and origin_chat_id): + return None + thread_id = get_session_env("HERMES_SESSION_THREAD_ID") or None + # Slack stamps every TOP-LEVEL message's own id as the session thread (a + # per-message session KEY, not a conversation location). Persisting it + # would pin all future deliveries inside an ephemeral thread, so a thread + # id equal to the creating message's id is synthetic and dropped; a real + # in-thread creation (thread == parent's id != this message) keeps it. + if thread_id and origin_platform == "slack": + message_id = get_session_env("HERMES_SESSION_MESSAGE_ID") or None + if message_id and str(thread_id) == str(message_id): + logger.debug( + "Cron origin: dropping synthetic per-message Slack " + "thread_id=%s (== creation message id)", thread_id, + ) + thread_id = None + if thread_id: + logger.debug( + "Cron origin captured thread_id=%s for %s:%s", + thread_id, origin_platform, origin_chat_id, + ) + return { + "platform": origin_platform, + "chat_id": origin_chat_id, + "chat_name": get_session_env("HERMES_SESSION_CHAT_NAME") or None, + "thread_id": thread_id, + # Lets an opt-in delivery mirror resolve the exact participant's + # session in per-user-isolated group chats (parity with send_message). + "user_id": get_session_env("HERMES_SESSION_USER_ID") or None, + # Workspace/server scope (Slack team, Discord guild...). Slack session + # keys embed it, so a continuable cron seed built without it would + # create a row no scoped reply ever resolves to. + "scope_id": get_session_env("HERMES_SESSION_SCOPE_ID") or None, + } + + +def _local_delivery_notice(job: Dict[str, Any], user_deliver: Optional[str]) -> Optional[str]: + """Notice when a created job won't deliver anywhere. + + CLI/TUI sessions have no capturable origin, so deliver='origin' (or an + omitted deliver) yields a job whose output is saved but never delivered. + Surface that at create time rather than silently dropping the user's + "tell me when it runs" intent. None when the user explicitly asked for + ``local`` or the job resolves to a real target. + """ + if (user_deliver or "").strip().lower() == "local": + return None + try: + from cron.scheduler import _resolve_delivery_targets + + if _resolve_delivery_targets(job): + return None + except Exception: + # Resolution unavailable — fall back to the origin signal. + if job.get("origin"): + return None + return ( + "This is a local-only cron job: its output is saved (view it with " + "cronjob(action='list')) but will NOT be delivered back into this " + "session — CLI/TUI sessions have no live-delivery channel. To be " + "notified when it runs, recreate or update the job with deliver set to " + "a gateway-connected platform, e.g. deliver='telegram' or deliver='all'." + ) + + +def _mode_guidance_notes(job: Dict[str, Any], user_deliver: Optional[str]) -> List[str]: + """Mode-specific guidance echoed once in the create/update response + (instead of in the schema, which is paid for on every API call).""" + notes: List[str] = [] + if job.get("monitor_script") or job.get("monitor_url"): + notes.append( + "Monitor mode: the source runs first each tick and its output is " + "hashed as exact bytes — unchanged output suppresses the agent run " + "(silent no_change tick), changed output injects a MONITOR CHANGE " + "DETECTED diff into the prompt. The first tick always runs as " + "baseline. The source must emit STABLE output (no timestamps, no " + "random ordering) or every tick will look changed." + ) + if job.get("no_agent"): + notes.append( + "no_agent mode: stdout is delivered verbatim; EMPTY stdout sends " + "nothing at all (watchdog pattern — script should stay quiet when " + "there is nothing to report). Non-zero exit or timeout sends an " + "error alert. prompt/skills are ignored." + ) + _deliver = (user_deliver or "").strip().lower() + if _deliver: + if "all" in _deliver.split(","): + notes.append( + "deliver='all' resolves at fire time and never includes " + "bot-chat targets — channels connected later are picked up " + "automatically." + ) + if _deliver.startswith("bot-chat:"): + notes.append( + "Targeting another profile's Bot Chat costs that bot an agent " + "turn per run." + ) + # platform:chat_id with no thread segment loses topic targeting. + for target in _deliver.split(","): + parts = target.strip().split(":") + if ( + len(parts) == 2 + and parts[0] not in ("bot-chat", "sms") + and parts[1] + and not parts[1].startswith("#") + ): + notes.append( + f"deliver target '{target.strip()}' has no :thread_id " + "segment — on thread/topic platforms the delivery lands in " + "the main chat, not a topic." + ) + break + return notes + + +def _split_monitor_arg( + monitor: Optional[str], + monitor_script: Optional[str], + monitor_url: Optional[str], +) -> tuple: + """Resolve the single model-facing ``monitor`` field into the stored + ``(monitor_script, monitor_url)`` pair. + + Shape decides transport: http(s):// is a URL, anything else a script path. + Update semantics: None = unchanged, '' = clear; setting one source clears + the other so switching transports never trips mutual exclusion. An + explicit ``monitor`` wins over the legacy alias fields. + """ + if monitor is None: + return monitor_script, monitor_url + value = monitor.strip() + if not value: + return "", "" + if value.lower().startswith(("http://", "https://")): + return "", value + return value, "" + + +def _repeat_display(job: Dict[str, Any]) -> str: + times = (job.get("repeat") or {}).get("times") + completed = (job.get("repeat") or {}).get("completed", 0) + if times is None: + return "forever" + if times == 1: + return "once" if completed == 0 else "1/1" + return f"{completed}/{times}" if completed else f"{times} times" + + +def _clean_str_list(items: Any) -> List[str]: + """Stripped, non-empty ``str(item)`` values from a str-or-iterable (order kept).""" + if items is None: + return [] + if isinstance(items, str): + items = [items] + return [s for s in (str(i).strip() for i in items) if s] + + +def _canonical_skills(skill: Optional[str] = None, skills: Optional[Any] = None) -> List[str]: + if skills is None: + raw_items = [skill] if skill else [] + elif isinstance(skills, str): + raw_items = [skills] + else: + raw_items = list(skills) + return list(dict.fromkeys(_clean_str_list(item or "" for item in raw_items))) + + +def _normalize_optional_job_value(value: Optional[Any], *, strip_trailing_slash: bool = False) -> Optional[str]: + if value is None: + return None + text = str(value).strip() + if strip_trailing_slash: + text = text.rstrip("/") + return text or None + + +def _normalize_deliver_param(value: Any) -> Optional[str]: + """Canonical string form of ``deliver``; None for None/empty. + + MCP clients / scripts may pass a list (``["telegram"]``); stored as-is the + scheduler's ``str(deliver).split(",")`` would yield the literal + ``"['telegram']"``. Flatten at the API boundary. + """ + if value is None: + return None + if isinstance(value, (list, tuple)): + return ",".join(_clean_str_list(value)) or None + return str(value).strip() or None + + +def _validate_bot_chat_deliver(deliver: Optional[str]) -> Optional[str]: + """Validate ``bot-chat[:]`` deliver elements at create time. + + Bot Chat delivery is machine-local: the profile must exist where the + scheduler fires. Fail loudly here rather than as a per-run delivery error. + Returns an error string or None. + """ + if not deliver: + return None + try: + from cron.scheduler import parse_bot_chat_deliver_token + from hermes_cli.profiles import normalize_profile_name, profile_exists + except Exception: + return None # best-effort; resolution re-checks at fire time + for part in str(deliver).split(","): + profile_arg = parse_bot_chat_deliver_token(part.strip()) + if not profile_arg: + continue # not a bot-chat token, or bare token (own profile) + try: + canon = normalize_profile_name(profile_arg) + except Exception: + return f"invalid bot-chat profile name '{profile_arg}'" + if not profile_exists(canon): + return ( + f"bot-chat delivery profile '{profile_arg}' not found on this " + "gateway's machine. Bot Chat delivery is machine-local — use a " + "profile that exists here (hermes profile list), or omit the " + "name (deliver='bot-chat') for the job's own profile." + ) + return None + + +def _resolve_cron_context_deliver(deliver: Optional[str]) -> Optional[str]: + """Resolve ``origin`` to a concrete target for creates made FROM a cron run. + + The creating session is ephemeral, so by fire time there is no origin to + resolve. Non-cron sessions: returned unchanged. Cron sessions: ``origin`` + (or an omitted value) becomes the creating run's ``platform:chat_id[:thread]`` + from the HERMES_CRON_AUTO_DELIVER_* contextvars, or ``local`` when the + creating run has no concrete target; other elements pass through verbatim. + """ + from gateway.session_context import get_session_env + from utils import is_truthy_value + + if not is_truthy_value(get_session_env("HERMES_CRON_SESSION", "")): + return deliver + + def _creator_target() -> str: + platform = get_session_env("HERMES_CRON_AUTO_DELIVER_PLATFORM", "").strip() + chat_id = get_session_env("HERMES_CRON_AUTO_DELIVER_CHAT_ID", "").strip() + if not platform or not chat_id: + return "local" + thread_id = get_session_env("HERMES_CRON_AUTO_DELIVER_THREAD_ID", "").strip() + return f"{platform}:{chat_id}:{thread_id}" if thread_id else f"{platform}:{chat_id}" + + if deliver is None: + return _creator_target() + resolved = [_creator_target() if p.lower() == "origin" else p for p in _clean_str_list(str(deliver).split(","))] + # Order-preserving de-dup: 'origin,local' with a local creator -> 'local'. + return ",".join(dict.fromkeys(resolved)) or None + + +def _validate_cron_base_url( + provider: Optional[Any], base_url: Optional[Any] +) -> Optional[str]: + """Reject pairing a named provider's stored credential with an off-host base_url. + + A prompt-injected job could name a real provider plus an attacker + base_url; at fire time the provider's stored key would be sent there + (credential exfil). Allowed: no override; bare 'custom' (pure BYOK, key + derived from the base_url itself); an override whose host matches the + named provider's own endpoint. Everything else fails closed. + Returns an error string if blocked, else None. + """ + bu = _normalize_optional_job_value(base_url, strip_trailing_slash=True) + if not bu: + return None + prov = _normalize_optional_job_value(provider) + if not prov: + # No provider inherits the default provider's stored key — same primitive. + return ( + "base_url override requires an explicit provider. Set provider to a " + "configured custom provider to use a custom endpoint." + ) + try: + from hermes_cli.runtime_provider import ( + has_named_custom_provider, + resolve_requested_provider, + _get_named_custom_provider, + ) + from hermes_cli.auth import PROVIDER_REGISTRY + from utils import base_url_host_matches, base_url_hostname + except Exception: + return f"Unable to validate base_url override for provider {prov!r}; refused." + + if prov.lower() == "custom": + return None + if has_named_custom_provider(prov): + # A NAMED custom provider carries a STORED key that the runtime still + # sends to an override base_url — require the configured host. + try: + cp = _get_named_custom_provider(prov) + except Exception: + cp = None + cfg_host = base_url_hostname((cp or {}).get("base_url", "")) if cp else "" + if cfg_host and base_url_host_matches(bu, cfg_host): + return None + return ( + f"base_url {bu!r} is not allowed for provider {prov!r}. A named " + f"custom provider's stored credential may only be sent to its own " + f"configured endpoint ({cfg_host or 'unknown'})." + ) + try: + resolved = resolve_requested_provider(prov) + except Exception: + resolved = prov + pconfig = PROVIDER_REGISTRY.get(resolved) if isinstance(resolved, str) else None + known_host = base_url_hostname(getattr(pconfig, "inference_base_url", "") if pconfig else "") + if known_host and base_url_host_matches(bu, known_host): + return None + # Fail closed: covers named providers with stored credentials AND + # aliases/unknown names we cannot host-match. + return ( + f"base_url {bu!r} is not allowed for provider {prov!r}. A named " + f"provider's stored credential may only be sent to its own endpoint; " + f'use a configured custom provider (provider="custom") for a custom base_url.' + ) + + +def _validate_cron_script_path(script: Optional[str]) -> Optional[str]: + """Scripts must be relative paths resolving within HERMES_HOME/scripts/ + (absolute / ~ / drive-letter paths rejected — prompt-injection guard). + Returns an error string if blocked, else None; empty = clearing, OK.""" + if not script or not script.strip(): + return None + + from hermes_constants import get_hermes_home + + raw = script.strip() + if raw.startswith(("/", "~")) or (len(raw) >= 2 and raw[1] == ":"): + return ( + f"Script path must be relative to ~/.hermes/scripts/. " + f"Got absolute or home-relative path: {raw!r}. " + f"Place scripts in ~/.hermes/scripts/ and use just the filename." + ) + + from tools.path_security import validate_within_dir + + scripts_dir = get_hermes_home() / "scripts" + scripts_dir.mkdir(parents=True, exist_ok=True) + if validate_within_dir(scripts_dir / raw, scripts_dir): + return f"Script path escapes the scripts directory via traversal: {raw!r}" + return None + + +def _apply_continuity( + context_from: Optional[Union[str, List[str]]], + continuity: bool, +) -> Optional[List[str]]: + """continuity=True ensures "self" is in context_from; False removes any + "self" entry. Other entries are preserved untouched.""" + refs = _clean_str_list(context_from) + has_self = any(r.lower() == "self" for r in refs) + if continuity and not has_self: + refs.append("self") + elif not continuity and has_self: + refs = [r for r in refs if r.lower() != "self"] + return refs or None + + +def _validate_context_from_refs(refs: List[Any]) -> Optional[str]: + """Error string if any non-"self" ref names a job that doesn't exist. + ("self" resolves to the job's own id at run time.)""" + from cron.jobs import get_job as _get_job + for ref_id in refs: + if isinstance(ref_id, str) and ref_id.strip().lower() == "self": + continue + if not _get_job(ref_id): + return ( + f"context_from job '{ref_id}' not found. " + "Use cronjob(action='list') to see available jobs." + ) + return None + + +# Optional fields echoed by _format_job only when truthy on the job record +# (order matters: it is the JSON key order). +_FORMAT_JOB_OPTIONAL_KEYS = ( + "script", "reasoning_effort", "monitor_script", "monitor_url", + "monitor_state", "no_agent", "enabled_toolsets", "workdir", +) + + +def _format_job(job: Dict[str, Any]) -> Dict[str, Any]: + prompt = str(job.get("prompt") or "") + skills = _canonical_skills(job.get("skill"), job.get("skills")) + job_id = str(job.get("id") or "unknown") + name = str(job.get("name") or prompt[:50] or (skills[0] if skills else "") or job_id or "cron job") + result = { + "job_id": job_id, + "name": name, + "skill": skills[0] if skills else None, + "skills": skills, + "prompt_preview": prompt[:100] + "..." if len(prompt) > 100 else prompt, + "model": job.get("model"), + "provider": job.get("provider"), + "base_url": job.get("base_url"), + "schedule": job.get("schedule_display") or "?", + "repeat": _repeat_display(job), + "deliver": job.get("deliver", "local"), + "next_run_at": job.get("next_run_at"), + "last_run_at": job.get("last_run_at"), + "last_status": job.get("last_status"), + "last_delivery_error": job.get("last_delivery_error"), + "last_delivery_unverified": job.get("last_delivery_unverified"), + "last_fire_error": job.get("last_fire_error"), + "enabled": job.get("enabled", True), + # Derive from enabled so half-paused records never render as paused. + "state": effective_job_state(job), + "paused_at": job.get("paused_at"), + "paused_reason": job.get("paused_reason"), + } + for key in _FORMAT_JOB_OPTIONAL_KEYS: + if job.get(key): + result[key] = True if key == "no_agent" else job[key] + stored_refs = job.get("context_from") or [] + if isinstance(stored_refs, str): + stored_refs = [stored_refs] + is_self = lambda r: str(r).strip().lower() == "self" or r == job.get("id") # noqa: E731 + if any(is_self(r) for r in stored_refs): + result["continuity"] = True + external_refs = [r for r in stored_refs if not is_self(r)] + if external_refs: + result["context_from"] = external_refs + if isinstance(job.get("attach_to_session"), bool): + result["attach_to_session"] = job["attach_to_session"] + return result + + +def _gateway_liveness_notice(plural: bool = False) -> dict: + """``gateway_running``/``warning`` payload via the shared CLI helper so the + CLI and this tool agree on what "scheduler active" means. False -> warning + (builtin ticker has no gateway process), None -> probe failed.""" + try: + from hermes_cli.cron import _builtin_gateway_liveness + + _gw = _builtin_gateway_liveness() + except Exception: + return {"gateway_running": None} + if _gw is False: + subject = "these jobs are saved" if plural else "this job is saved" + return { + "gateway_running": False, + "warning": ( + f"The Hermes gateway is not running — {subject} " + "but will NOT fire until the gateway is started " + "(hermes gateway install / hermes gateway start). " + "Tell the user the task is scheduled but not active yet." + ), + } + return {"gateway_running": None if _gw is None else True} diff --git a/tools/cronjob_prompt_scan.py b/tools/cronjob_prompt_scan.py new file mode 100644 index 0000000000..8ae1beff6f --- /dev/null +++ b/tools/cronjob_prompt_scan.py @@ -0,0 +1,174 @@ +"""Cron prompt threat scanning (extracted from tools/cronjob_tools.py). + +Two surfaces, two pattern sets: the small user-authored prompt gets the strict +set; the assembled prompt (with skill bodies) gets only prose-proof directives. +""" + +import logging +import re + +# Single source of truth shared with the install-time scanner (skills_guard): +# a narrower cron-local copy once let obfuscated directives (invisible math +# operators, directional isolates) slip past this runtime tripwire. +from tools.threat_patterns import INVISIBLE_CHARS as _CRON_INVISIBLE_CHARS + +# Logger parity with the origin module (these functions used to log there). +logger = logging.getLogger("tools.cronjob_tools") + +# Strict patterns — user prompt only. A directive-shaped cron prompt has no +# business containing `cat ~/.hermes/.env` or `rm -rf /`; there it is a +# smoking gun, not prose. +_CRON_THREAT_PATTERNS = [ + (r'ignore\s+(?:\w+\s+)*(?:previous|all|above|prior)\s+(?:\w+\s+)*instructions', "prompt_injection"), + (r'do\s+not\s+tell\s+the\s+user', "deception_hide"), + (r'system\s+prompt\s+override', "sys_prompt_override"), + (r'disregard\s+(your|all|any)\s+(instructions|rules|guidelines)', "disregard_rules"), + (r'cat\s+[^\n]*(\.env|credentials|\.netrc|\.pgpass|id_rsa|id_ed25519|id_ecdsa)', "read_secrets"), + (r'authorized_keys', "ssh_backdoor"), + (r'/etc/sudoers|visudo', "sudoers_mod"), + (r'rm\s+-rf\s+/', "destructive_root_rm"), +] + +# Looser set — assembled prompt with skills attached. Command-shape patterns are +# dropped because skill markdown (security postmortems, runbooks) legitimately +# *describes* those commands; skill bodies are already vetted at install time, +# so this is only a tripwire for unambiguous injection directives. +_CRON_SKILL_ASSEMBLED_PATTERNS = _CRON_THREAT_PATTERNS[:4] + +_CRON_SECRET_VAR_RE = r'\$\{?\w*(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)\w*\}?' +# Obvious leak paths only: secret in the destination URL, in a POST/form body, +# or in an Authorization header to an arbitrary host. +_CRON_EXFIL_COMMAND_PATTERNS = [ + (rf'curl\s+[^\n]*https?://[^\s"\'`]*{_CRON_SECRET_VAR_RE}', "exfil_curl_url"), + (rf'wget\s+[^\n]*https?://[^\s"\'`]*{_CRON_SECRET_VAR_RE}', "exfil_wget_url"), + (rf'curl\s+[^\n]*(?:--data(?:-raw|-binary|-urlencode)?|-d|--form|-F)\s+[^\n]*{_CRON_SECRET_VAR_RE}', "exfil_curl_data"), + (rf'wget\s+[^\n]*--post-(?:data|file)=[^\n]*{_CRON_SECRET_VAR_RE}', "exfil_wget_post"), + (rf'curl\s+[^\n]*(?:-H|--header)\s+["\']Authorization:\s*(?:Bearer|token)\s+{_CRON_SECRET_VAR_RE}["\']', "exfil_curl_auth_header"), +] + +_BLOCKED_PATTERN_MSG = ( + "Blocked: prompt matches threat pattern '{pid}'. Cron prompts must not " + "contain injection or exfiltration payloads." +) + +# U+200D (ZWJ) is a required part of many emoji sequences (👨‍👩‍👧, 🏳️‍🌈). +# Block it between plain text, allow it inside an emoji grapheme cluster. +_EMOJI_NEIGHBOUR_CP_RANGES = ( + (0x1F000, 0x1FFFF), + (0x2600, 0x27BF), + (0x2300, 0x23FF), + (0x1F1E6, 0x1F1FF), + (0x20E3, 0x20E3), +) +_VARIATION_SELECTOR_CP = 0xFE0F + + +def _is_emoji_cp(cp: int) -> bool: + return any(lo <= cp <= hi for lo, hi in _EMOJI_NEIGHBOUR_CP_RANGES) + + +def _zwj_has_emoji_neighbour(text: str, idx: int) -> bool: + """True when the ZWJ at text[idx] sits between emoji codepoints (skipping VS16).""" + left = idx - 1 + while left >= 0 and ord(text[left]) == _VARIATION_SELECTOR_CP: + left -= 1 + right = idx + 1 + while right < len(text) and ord(text[right]) == _VARIATION_SELECTOR_CP: + right += 1 + return ( + left >= 0 and right < len(text) + and _is_emoji_cp(ord(text[left])) + and _is_emoji_cp(ord(text[right])) + ) + + +def _strip_legitimate_emoji_zwj(prompt: str) -> str: + if '\u200d' not in prompt: + return prompt + return ''.join( + ch for idx, ch in enumerate(prompt) + if not (ch == '\u200d' and _zwj_has_emoji_neighbour(prompt, idx)) + ) + + +def _strip_cron_safe_constructs(prompt: str) -> str: + """Scrub the bundled GitHub skill's `Authorization: token $GITHUB_TOKEN` + + api.github.com curl so it doesn't trip the auth-header exfil rule. + + re.sub scrubs EVERY occurrence (a job loading several GitHub skills has + many). The trailing ``[^\\s;&|$`]*`` consumes only the URL path — never + separators or subshell openers — so a payload smuggled onto the same line + still gets scanned. Host must be exactly api.github.com followed by ``/``, + whitespace, quote, or end: lookalike authorities (api.github.com.evil.com, + api.github.com@evil.com) fall through to the exfil detectors. + """ + return re.sub( + rf'curl\s+[^\n;&|$`]*(?:-H|--header)\s+["\']Authorization:\s*token\s+{_CRON_SECRET_VAR_RE}["\']' + r'\s+["\']?https://api\.github\.com(?::\d+)?(?:/|\s|$|["\'])[^\s;&|$`]*', + 'curl https://api.github.com/user', + prompt, + flags=re.IGNORECASE, + ) + + +def _check_invisible_unicode(prompt: str) -> str: + """Error string if the prompt holds invisible-unicode markers (emoji ZWJ allowed).""" + prompt_for_invisible_scan = _strip_legitimate_emoji_zwj(prompt) + for char in _CRON_INVISIBLE_CHARS: + if char in prompt_for_invisible_scan: + return f"Blocked: prompt contains invisible unicode U+{ord(char):04X} (possible injection)." + return "" + + +def _strip_invisible_unicode(prompt: str) -> tuple[str, list[str]]: + """Strip invisible-unicode chars, keeping ZWJ inside legitimate emoji. + + Returns ``(cleaned, sorted U+XXXX labels removed)``. Used for the + skills-attached path, where a stray zero-width space in vetted skill + content should be sanitized rather than permanently kill the job. + """ + if not prompt: + return prompt, [] + removed: set[str] = set() + cleaned: list[str] = [] + for idx, ch in enumerate(prompt): + if ch in _CRON_INVISIBLE_CHARS and not (ch == '\u200d' and _zwj_has_emoji_neighbour(prompt, idx)): + removed.add(f"U+{ord(ch):04X}") + continue + cleaned.append(ch) + return ''.join(cleaned), sorted(removed) + + +def _first_pattern_error(text: str, *pattern_sets) -> str: + for patterns in pattern_sets: + for pattern, pid in patterns: + if re.search(pattern, text, re.IGNORECASE): + return _BLOCKED_PATTERN_MSG.format(pid=pid) + return "" + + +def _scan_cron_prompt(prompt: str) -> str: + """Strict scan of the USER-SUPPLIED prompt (create/update + runtime + defense-in-depth). Returns an error string when blocked, else "".""" + prompt_to_scan = _strip_cron_safe_constructs(prompt) + return _check_invisible_unicode(prompt_to_scan) or _first_pattern_error( + prompt_to_scan, _CRON_THREAT_PATTERNS, _CRON_EXFIL_COMMAND_PATTERNS + ) + + +def _scan_cron_skill_assembled(assembled: str) -> tuple[str, str]: + """Loose scan of the ASSEMBLED prompt (skill content included). + + Invisible unicode is SANITIZED (stripped + logged), not blocked — the hard + block stays on raw user prompts, the actual injection surface. Returns + ``(cleaned_prompt, error)`` with ``error`` empty when it passed. + """ + cleaned, removed = _strip_invisible_unicode(assembled) + if removed: + logger.warning( + "Cron skill-assembled prompt: stripped %d invisible-unicode " + "char(s) (%s) from vetted skill content", + len(removed), ", ".join(removed), + ) + error = _first_pattern_error(_strip_cron_safe_constructs(cleaned), _CRON_SKILL_ASSEMBLED_PATTERNS) + return cleaned, error diff --git a/tools/cronjob_tools.py b/tools/cronjob_tools.py index cdcdc6cea5..0b9c8ae7c3 100644 --- a/tools/cronjob_tools.py +++ b/tools/cronjob_tools.py @@ -7,7 +7,6 @@ Compatibility wrappers remain for direct Python callers and legacy tests. import json import logging -import re import sys import threading import time @@ -18,29 +17,21 @@ from hermes_constants import display_hermes_home logger = logging.getLogger(__name__) -# Cadence for the heartbeat that keeps the calling agent's inactivity watchdog -# at bay while a manual `cronjob(action="run")` executes the job synchronously -# in-process (#76502). Mirrors the 10s cadence of -# tools/environments/base.py::touch_activity_if_due (delegate_task's heartbeat -# uses 30s) — comfortably below the 1800s default HERMES_AGENT_TIMEOUT. +# Heartbeat cadence that keeps the calling agent's inactivity watchdog at bay +# while a manual `cronjob(action="run")` executes synchronously in-process +# (mirrors tools/environments/base.py::touch_activity_if_due). _CRON_RUN_HEARTBEAT_INTERVAL = 10.0 -# Hard ceiling on how long the heartbeat keeps the parent watchdog at bay. -# The child cron run has its own inactivity watchdog (HERMES_CRON_TIMEOUT, -# default 600s) that bounds a wedged job, but with HERMES_CRON_TIMEOUT=0 -# (explicit "unlimited") a truly hung run_one_job would otherwise mask the -# gateway watchdog forever — pre-#76502 the parent was at least reaped at -# ~1800s. After this ceiling the heartbeat stops and the gateway watchdog -# regains authority over the turn. +# Hard ceiling on the heartbeat: with HERMES_CRON_TIMEOUT=0 (unlimited) a truly +# hung run would otherwise mask the gateway watchdog forever. Past this, the +# heartbeat stops and the gateway watchdog regains authority over the turn. _CRON_RUN_HEARTBEAT_CEILING = 6 * 3600.0 -# Import from cron module (will be available when properly installed) sys.path.insert(0, str(Path(__file__).parent.parent)) from cron.jobs import ( AmbiguousJobReference, claim_job_for_fire, - effective_job_state, get_job, is_job_runnable, list_jobs, @@ -52,6 +43,32 @@ from cron.jobs import ( resume_job, update_job, ) +from tools.cronjob_prompt_scan import ( # noqa: F401 (re-exported) + _CRON_EXFIL_COMMAND_PATTERNS, + _CRON_INVISIBLE_CHARS, + _CRON_SKILL_ASSEMBLED_PATTERNS, + _CRON_THREAT_PATTERNS, + _scan_cron_prompt, + _scan_cron_skill_assembled, +) +from tools.cronjob_job_args import ( # noqa: F401 (re-exported) + _apply_continuity, + _canonical_skills, + _format_job, + _gateway_liveness_notice, + _local_delivery_notice, + _mode_guidance_notes, + _normalize_deliver_param, + _normalize_optional_job_value, + _origin_from_env, + _repeat_display, + _resolve_cron_context_deliver, + _split_monitor_arg, + _validate_bot_chat_deliver, + _validate_context_from_refs, + _validate_cron_base_url, + _validate_cron_script_path, +) def _notify_provider_jobs_changed_safe() -> None: @@ -65,750 +82,8 @@ def _notify_provider_jobs_changed_safe() -> None: # --------------------------------------------------------------------------- -# Cron prompt scanning +# Manual run execution (claim -> run_one_job -> report) # --------------------------------------------------------------------------- -# -# Two threat surfaces, two scanners: -# -# 1. User-supplied cron prompt (small, written as a directive). -# Strict scanning is appropriate — a legit cron prompt has no business -# saying "cat ~/.hermes/.env" or "rm -rf /". `_scan_cron_prompt()` runs -# against this at create/update time and as a runtime defense-in-depth. -# -# 2. Assembled prompt that includes loaded skill content (large markdown -# bodies, often security docs, postmortems, runbooks discussing attack -# patterns in PROSE). Reusing the strict patterns here false-positives -# every time a skill *describes* a command — see #3968 follow-up: the -# `hermes-agent-dev` skill contains a security postmortem mentioning -# `cat ~/.hermes/.env`, which tripped `read_secrets` and silently -# killed all PR-scout jobs. -# -# Skill bodies are user-curated and scanned at install time by -# `skills_guard.py`. The runtime cron scan only needs to catch the -# patterns whose phrasing does NOT survive normal English prose: -# classic prompt-injection directives ("ignore previous instructions", -# "disregard your rules"), deception directives, and invisible -# unicode. `_scan_cron_skill_assembled()` runs against the assembled -# prompt with this tighter pattern set. -# -# Both scanners share the invisible-unicode check and the GitHub Authorization -# header exemption. - -# Strict patterns — applied to the user prompt only. -_CRON_THREAT_PATTERNS = [ - (r'ignore\s+(?:\w+\s+)*(?:previous|all|above|prior)\s+(?:\w+\s+)*instructions', "prompt_injection"), - (r'do\s+not\s+tell\s+the\s+user', "deception_hide"), - (r'system\s+prompt\s+override', "sys_prompt_override"), - (r'disregard\s+(your|all|any)\s+(instructions|rules|guidelines)', "disregard_rules"), - (r'cat\s+[^\n]*(\.env|credentials|\.netrc|\.pgpass|id_rsa|id_ed25519|id_ecdsa)', "read_secrets"), - (r'authorized_keys', "ssh_backdoor"), - (r'/etc/sudoers|visudo', "sudoers_mod"), - (r'rm\s+-rf\s+/', "destructive_root_rm"), -] - -# Looser pattern set — applied to the assembled prompt when skills are -# attached. Only patterns whose phrasing is unambiguous in any context; -# command-shape patterns are dropped because they false-positive on prose -# in security docs / postmortems. Skill bodies are scanned at install time -# by `skills_guard.py`, so the runtime cron scan is purely a tripwire for -# obvious injection directives surviving a malicious skill that slipped -# through install. -_CRON_SKILL_ASSEMBLED_PATTERNS = [ - (r'ignore\s+(?:\w+\s+)*(?:previous|all|above|prior)\s+(?:\w+\s+)*instructions', "prompt_injection"), - (r'do\s+not\s+tell\s+the\s+user', "deception_hide"), - (r'system\s+prompt\s+override', "sys_prompt_override"), - (r'disregard\s+(your|all|any)\s+(instructions|rules|guidelines)', "disregard_rules"), -] - -_CRON_SECRET_VAR_RE = r'\$\{?\w*(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|API)\w*\}?' -_CRON_EXFIL_COMMAND_PATTERNS = [ - # Tighten exfil detection to obvious leak paths: embedding a secret - # directly in the destination URL, sending it in POST/FORM payloads, - # or shipping it via Authorization headers to arbitrary hosts. The - # only intended allowlist exception today is the bundled GitHub skill - # pattern that talks to api.github.com. - (rf'curl\s+[^\n]*https?://[^\s"\'`]*{_CRON_SECRET_VAR_RE}', "exfil_curl_url"), - (rf'wget\s+[^\n]*https?://[^\s"\'`]*{_CRON_SECRET_VAR_RE}', "exfil_wget_url"), - (rf'curl\s+[^\n]*(?:--data(?:-raw|-binary|-urlencode)?|-d|--form|-F)\s+[^\n]*{_CRON_SECRET_VAR_RE}', "exfil_curl_data"), - (rf'wget\s+[^\n]*--post-(?:data|file)=[^\n]*{_CRON_SECRET_VAR_RE}', "exfil_wget_post"), - (rf'curl\s+[^\n]*(?:-H|--header)\s+["\']Authorization:\s*(?:Bearer|token)\s+{_CRON_SECRET_VAR_RE}["\']', "exfil_curl_auth_header"), -] - -# Single source of truth, shared with the install-time scanner -# (threat_patterns.INVISIBLE_CHARS / skills_guard). Keeping a separate, narrower -# copy here let an obfuscated injection directive slip past this runtime cron -# tripwire while being caught at install time (or vice versa): U+2062-U+2064 -# (invisible math operators) and U+2066-U+2069 (directional isolates) are real -# attack tools and were missing from the cron-local set. Importing the canonical -# set keeps the cron tripwire and the install scanner from drifting apart. -from tools.threat_patterns import INVISIBLE_CHARS as _CRON_INVISIBLE_CHARS - -# U+200D Zero-Width Joiner is also a legitimate, required part of many -# Unicode emoji sequences (for example 👨‍👩‍👧, 🏳️‍🌈, ❤️‍🩹, 🧑‍💻). -# We should still block ZWJ when it is hiding between plain text characters, -# but not when it is clearly part of an emoji grapheme cluster. -_EMOJI_NEIGHBOUR_CP_RANGES = ( - (0x1F000, 0x1FFFF), - (0x2600, 0x27BF), - (0x2300, 0x23FF), - (0x1F1E6, 0x1F1FF), - (0x20E3, 0x20E3), -) -_VARIATION_SELECTOR_CP = 0xFE0F - - -def _is_emoji_cp(cp: int) -> bool: - return any(lo <= cp <= hi for lo, hi in _EMOJI_NEIGHBOUR_CP_RANGES) - - -def _zwj_has_emoji_neighbour(text: str, idx: int) -> bool: - """Return True when the ZWJ at text[idx] appears inside an emoji sequence.""" - left = idx - 1 - while left >= 0 and ord(text[left]) == _VARIATION_SELECTOR_CP: - left -= 1 - right = idx + 1 - while right < len(text) and ord(text[right]) == _VARIATION_SELECTOR_CP: - right += 1 - return ( - left >= 0 and right < len(text) - and _is_emoji_cp(ord(text[left])) - and _is_emoji_cp(ord(text[right])) - ) - - -def _strip_legitimate_emoji_zwj(prompt: str) -> str: - if '\u200d' not in prompt: - return prompt - cleaned: list[str] = [] - for idx, ch in enumerate(prompt): - if ch == '\u200d' and _zwj_has_emoji_neighbour(prompt, idx): - continue - cleaned.append(ch) - return ''.join(cleaned) - - -def _strip_cron_safe_constructs(prompt: str) -> str: - """Strip the GitHub `Authorization: token $GITHUB_TOKEN` auth-header - pattern so it doesn't trip the broader curl-auth-header exfil rule. - - Allows the bundled GitHub skill fallback without opening a blanket - exemption for arbitrary Authorization-header exfiltration. - - Uses ``re.sub`` so EVERY occurrence is scrubbed, not just the first — a - cron job that loads 2+ GitHub skills (e.g. github-issues + - github-pr-workflow + github-code-review) contains several such blocks, - and the old ``re.search`` + single ``str.replace`` left the rest to trip - the exfil_curl_auth_header detector on every run. The trailing - ``[^\\s;&|$`]*`` consumes only the URL path — never whitespace, command - separators, or subshell openers — so a payload smuggled onto the same - line (``;``, ``&&``, ``|``, ``$(...)``, backticks) survives the strip - and is still scanned. The host must be exactly ``api.github.com`` - followed by ``/``, whitespace, quote, or end: lookalike authorities - (``api.github.com.evil.com``, ``api.github.com@evil.com``) are not the - trusted construct and fall through to the exfil detectors, while - legitimately quoted bare-host URLs stay exempt. - """ - return re.sub( - rf'curl\s+[^\n;&|$`]*(?:-H|--header)\s+["\']Authorization:\s*token\s+{_CRON_SECRET_VAR_RE}["\']' - r'\s+["\']?https://api\.github\.com(?::\d+)?(?:/|\s|$|["\'])[^\s;&|$`]*', - 'curl https://api.github.com/user', - prompt, - flags=re.IGNORECASE, - ) - - -def _check_invisible_unicode(prompt: str) -> str: - """Return an error string if the prompt contains invisible-unicode - injection markers (ZWJ inside legitimate emoji sequences is allowed). - """ - prompt_for_invisible_scan = _strip_legitimate_emoji_zwj(prompt) - for char in _CRON_INVISIBLE_CHARS: - if char in prompt_for_invisible_scan: - return f"Blocked: prompt contains invisible unicode U+{ord(char):04X} (possible injection)." - return "" - - -def _strip_invisible_unicode(prompt: str) -> tuple[str, list[str]]: - """Strip invisible-unicode characters from *prompt*, preserving the ZWJ - that lives inside legitimate emoji sequences. - - Returns ``(cleaned_prompt, removed_codepoints)`` where ``removed_codepoints`` - is the sorted list of ``U+XXXX`` labels that were stripped (empty when the - prompt was already clean). Used by the skills-attached cron path, where the - skill body is already vetted at install time by ``skills_guard.py`` — a - stray zero-width space in a code example should be sanitized, not turned - into a hard block that permanently kills the job. - """ - if not prompt: - return prompt, [] - # Keep emoji-ZWJ: temporarily remove the legitimate joiners, scan/strip the - # rest, then the legitimate joiners survive because we operate on the - # original string and only drop chars that are NOT part of an emoji cluster. - removed: set[str] = set() - cleaned: list[str] = [] - for idx, ch in enumerate(prompt): - if ch in _CRON_INVISIBLE_CHARS: - if ch == '\u200d' and _zwj_has_emoji_neighbour(prompt, idx): - cleaned.append(ch) # legitimate emoji joiner — keep - continue - removed.add(f"U+{ord(ch):04X}") - continue - cleaned.append(ch) - return ''.join(cleaned), sorted(removed) - - -def _scan_cron_prompt(prompt: str) -> str: - """Scan the USER-SUPPLIED cron prompt for critical threats. - - Strict pattern set — used at job create/update time and as a runtime - defense-in-depth for prompts authored before the scanner existed. - The user prompt is small and directive; bare `cat .env` or `rm -rf /` - there is a smoking gun, not prose. Returns an error string when - blocked, else empty string. - """ - prompt_to_scan = _strip_cron_safe_constructs(prompt) - invisible_err = _check_invisible_unicode(prompt_to_scan) - if invisible_err: - return invisible_err - for pattern, pid in _CRON_THREAT_PATTERNS: - if re.search(pattern, prompt_to_scan, re.IGNORECASE): - return f"Blocked: prompt matches threat pattern '{pid}'. Cron prompts must not contain injection or exfiltration payloads." - for pattern, pid in _CRON_EXFIL_COMMAND_PATTERNS: - if re.search(pattern, prompt_to_scan, re.IGNORECASE): - return f"Blocked: prompt matches threat pattern '{pid}'. Cron prompts must not contain injection or exfiltration payloads." - return "" - - -def _scan_cron_skill_assembled(assembled: str) -> tuple[str, str]: - """Scan an ASSEMBLED cron prompt that includes loaded skill content. - - Looser pattern set — only catches unambiguous prompt-injection - directives. Drops command-shape patterns (cat .env, rm -rf /, - authorized_keys, /etc/sudoers) because they false-positive on - legitimate skill markdown that *describes* attack commands in - security postmortems and runbooks. - - Invisible unicode is SANITIZED, not blocked. Skill bodies are - user-curated and already scanned at install time by - ``skills_guard.py``; a stray zero-width space in a code example - (common in copy-pasted unicode docs) should not permanently kill the - job. The offending codepoints are stripped and logged, the cleaned - prompt is returned. The hard block remains for raw user prompts via - ``_scan_cron_prompt`` — that path is the actual injection surface. - - Returns ``(cleaned_prompt, error)``; ``error`` is empty when the - prompt passed (after sanitization). - """ - cleaned, removed = _strip_invisible_unicode(assembled) - if removed: - logger.warning( - "Cron skill-assembled prompt: stripped %d invisible-unicode " - "char(s) (%s) from vetted skill content", - len(removed), ", ".join(removed), - ) - prompt_to_scan = _strip_cron_safe_constructs(cleaned) - for pattern, pid in _CRON_SKILL_ASSEMBLED_PATTERNS: - if re.search(pattern, prompt_to_scan, re.IGNORECASE): - return cleaned, f"Blocked: prompt matches threat pattern '{pid}'. Cron prompts must not contain injection or exfiltration payloads." - return cleaned, "" - - -def _origin_from_env() -> Optional[Dict[str, str]]: - from gateway.session_context import get_session_env - origin_platform = get_session_env("HERMES_SESSION_PLATFORM") - origin_chat_id = get_session_env("HERMES_SESSION_CHAT_ID") - if origin_platform and origin_chat_id: - thread_id = get_session_env("HERMES_SESSION_THREAD_ID") or None - # Slack thread-per-message session keying (native parity: thread_ts = - # event.thread_ts or ts) stamps every TOP-LEVEL message's own id as - # the session thread. That stamp is a per-message session KEY, not a - # durable conversation location — persisting it as origin routing - # pins every future delivery inside the ephemeral thread spawned - # around the creation message. Recognize it at the source: a Slack - # thread id equal to the triggering message's own id is synthetic. - # A genuine in-thread creation (thread == the parent's id != this - # message's id) keeps its thread. - if thread_id and origin_platform == "slack": - message_id = get_session_env("HERMES_SESSION_MESSAGE_ID") or None - if message_id and str(thread_id) == str(message_id): - logger.debug( - "Cron origin: dropping synthetic per-message Slack " - "thread_id=%s (== creation message id)", thread_id, - ) - thread_id = None - if thread_id: - logger.debug( - "Cron origin captured thread_id=%s for %s:%s", - thread_id, origin_platform, origin_chat_id, - ) - return { - "platform": origin_platform, - "chat_id": origin_chat_id, - "chat_name": get_session_env("HERMES_SESSION_CHAT_NAME") or None, - "thread_id": thread_id, - # Captured so an opt-in delivery mirror (cron.mirror_delivery / - # attach_to_session) can resolve the exact participant's session in - # per-user-isolated group chats — parity with interactive - # send_message, which passes HERMES_SESSION_USER_ID to - # gateway.mirror.mirror_to_session. Harmless for DMs/shared sessions. - "user_id": get_session_env("HERMES_SESSION_USER_ID") or None, - # Workspace/server scope (Slack team, Discord guild, Matrix - # server). build_session_key embeds it in every Slack session key - # (dm/group/thread alike), so a continuable cron seed built - # WITHOUT it creates a row no scoped reply ever resolves to — - # the seeded key is agent:main:slack:dm:: while the - # reply keys agent:main:slack:dm:::. Captured - # here so the scheduler's seed helpers can reproduce the reply's - # exact key. Same session-context var async_delegation already - # snapshots; None for platforms without scope. - "scope_id": get_session_env("HERMES_SESSION_SCOPE_ID") or None, - } - return None - - -def _local_delivery_notice(job: Dict[str, Any], user_deliver: Optional[str]) -> Optional[str]: - """Return an informational notice when a created job won't deliver anywhere. - - TUI/CLI sessions cannot be captured as a cron ``origin`` (no - ``HERMES_SESSION_PLATFORM``/``CHAT_ID`` is set for them), so a - ``deliver="origin"`` request — or an omitted ``deliver`` that defaults to - origin-or-local — produces a job that runs and saves output to - ``last_output`` but is never delivered back into the session. This is by - design (there is no live-delivery channel for local sessions), but silently - dropping the user's "tell me when it runs" intent is the trap reported in - #51568. Surface it at create time so the agent can relay it instead of - promising a delivery that never happens. - - Returns ``None`` when the user explicitly asked for ``local`` (no surprise), - or when the job resolves to a real delivery target. - """ - # An explicit local request is exactly what the user asked for — no notice. - if (user_deliver or "").strip().lower() == "local": - return None - try: - from cron.scheduler import _resolve_delivery_targets - - if _resolve_delivery_targets(job): - return None # Will actually deliver somewhere — nothing to flag. - except Exception: - # If resolution can't be evaluated, fall back to the origin signal. - if job.get("origin"): - return None - return ( - "This is a local-only cron job: its output is saved (view it with " - "cronjob(action='list')) but will NOT be delivered back into this " - "session — CLI/TUI sessions have no live-delivery channel. To be " - "notified when it runs, recreate or update the job with deliver set to " - "a gateway-connected platform, e.g. deliver='telegram' or deliver='all'." - ) - - -def _mode_guidance_notes(job: Dict[str, Any], user_deliver: Optional[str]) -> List[str]: - """Mode-specific guidance echoed in the create/update response. - - The teaching that used to live in CRONJOB_SCHEMA parameter descriptions - (paid for on every API call of every session) is delivered here instead — - once, in the tool result, at the moment the model actually created a job - in that mode. Keep each note short and actionable; only fire notes for - modes the job actually uses. - """ - notes: List[str] = [] - if job.get("monitor_script") or job.get("monitor_url"): - notes.append( - "Monitor mode: the source runs first each tick and its output is " - "hashed as exact bytes — unchanged output suppresses the agent run " - "(silent no_change tick), changed output injects a MONITOR CHANGE " - "DETECTED diff into the prompt. The first tick always runs as " - "baseline. The source must emit STABLE output (no timestamps, no " - "random ordering) or every tick will look changed." - ) - if job.get("no_agent"): - notes.append( - "no_agent mode: stdout is delivered verbatim; EMPTY stdout sends " - "nothing at all (watchdog pattern — script should stay quiet when " - "there is nothing to report). Non-zero exit or timeout sends an " - "error alert. prompt/skills are ignored." - ) - _deliver = (user_deliver or "").strip().lower() - if _deliver: - if "all" in _deliver.split(","): - notes.append( - "deliver='all' resolves at fire time and never includes " - "bot-chat targets — channels connected later are picked up " - "automatically." - ) - if _deliver.startswith("bot-chat:"): - notes.append( - "Targeting another profile's Bot Chat costs that bot an agent " - "turn per run." - ) - # platform:chat_id with no thread segment loses topic targeting — - # warn once here instead of carrying the warning in the schema. - for target in _deliver.split(","): - parts = target.strip().split(":") - if ( - len(parts) == 2 - and parts[0] not in ("bot-chat", "sms") - and parts[1] - and not parts[1].startswith("#") - ): - notes.append( - f"deliver target '{target.strip()}' has no :thread_id " - "segment — on thread/topic platforms the delivery lands in " - "the main chat, not a topic." - ) - break - return notes - - -def _split_monitor_arg( - monitor: Optional[str], - monitor_script: Optional[str], - monitor_url: Optional[str], -) -> tuple: - """Resolve the model-facing ``monitor`` field into the stored pair. - - The schema advertises ONE ``monitor`` field; the value's shape decides the - transport: ``http(s)://...`` is a URL source, anything else is a script - path (a legal script path can never start with a URL scheme). Jobs keep - storing ``monitor_script``/``monitor_url`` separately — this is an - interface merge, not a storage migration — and the legacy field names are - still accepted as aliases so older transcripts/replays keep working. - - Returns ``(monitor_script, monitor_url)`` with update semantics: - ``None`` = leave unchanged, ``''`` = clear. Setting one source via - ``monitor`` clears the other, so switching transports in one call never - trips the mutual-exclusion invariant. An explicit ``monitor`` wins over - the legacy aliases. - """ - if monitor is None: - return monitor_script, monitor_url - value = monitor.strip() - if not value: - return "", "" # clear both sources - if value.lower().startswith(("http://", "https://")): - return "", value - return value, "" - - -def _repeat_display(job: Dict[str, Any]) -> str: - times = (job.get("repeat") or {}).get("times") - completed = (job.get("repeat") or {}).get("completed", 0) - if times is None: - return "forever" - if times == 1: - return "once" if completed == 0 else "1/1" - return f"{completed}/{times}" if completed else f"{times} times" - - -def _canonical_skills(skill: Optional[str] = None, skills: Optional[Any] = None) -> List[str]: - if skills is None: - raw_items = [skill] if skill else [] - elif isinstance(skills, str): - raw_items = [skills] - else: - raw_items = list(skills) - - normalized: List[str] = [] - for item in raw_items: - text = str(item or "").strip() - if text and text not in normalized: - normalized.append(text) - return normalized - - - - -def _normalize_optional_job_value(value: Optional[Any], *, strip_trailing_slash: bool = False) -> Optional[str]: - if value is None: - return None - text = str(value).strip() - if strip_trailing_slash: - text = text.rstrip("/") - return text or None - - -def _normalize_deliver_param(value: Any) -> Optional[str]: - """Normalize a user-supplied ``deliver`` value to the canonical string form. - - The cron schema documents ``deliver`` as a string (``"local"``, ``"origin"``, - ``"telegram"``, ``"telegram:chat_id[:thread_id]"``, or comma-separated combos). - Some callers — MCP clients passing arrays, scripts building the payload as a - list — supply ``["telegram"]``. ``create_job``/``update_job`` store it as-is, - and the scheduler's ``str(deliver).split(",")`` then serializes the list to - the literal ``"['telegram']"`` which is not a known platform. Flatten lists - / tuples at the API boundary so storage is always a string. Returns ``None`` - for ``None``/empty so callers can treat it as "not supplied". - """ - if value is None: - return None - if isinstance(value, (list, tuple)): - parts = [str(p).strip() for p in value if str(p).strip()] - return ",".join(parts) if parts else None - text = str(value).strip() - return text or None - - -def _validate_bot_chat_deliver(deliver: Optional[str]) -> Optional[str]: - """Validate any ``bot-chat[:]`` deliver elements at create time. - - Bot Chat delivery is machine-local: the named profile must exist on THIS - machine (the one whose scheduler will fire the job). Failing loudly here - beats a per-run ``last_delivery_error`` at 3am — especially for Desktop - clients whose merged multi-gateway rosters may show same-named profiles - from other machines. Returns an error string or None. - """ - if not deliver: - return None - try: - from cron.scheduler import parse_bot_chat_deliver_token - from hermes_cli.profiles import normalize_profile_name, profile_exists - except Exception: - return None # validation is best-effort; resolution re-checks at fire time - for part in str(deliver).split(","): - profile_arg = parse_bot_chat_deliver_token(part.strip()) - if profile_arg is None or not profile_arg: - continue # not a bot-chat token, or bare token (own profile) - try: - canon = normalize_profile_name(profile_arg) - except Exception: - return f"invalid bot-chat profile name '{profile_arg}'" - if not profile_exists(canon): - return ( - f"bot-chat delivery profile '{profile_arg}' not found on this " - "gateway's machine. Bot Chat delivery is machine-local — use a " - "profile that exists here (hermes profile list), or omit the " - "name (deliver='bot-chat') for the job's own profile." - ) - return None - - -def _resolve_cron_context_deliver(deliver: Optional[str]) -> Optional[str]: - """Resolve ``origin`` to a concrete target for cron-context creates. - - A job created FROM a cron run must never store the literal ``origin``: - the creating session is ephemeral, so by fire time there is no origin to - resolve and the scheduler would fall back to guessing a home channel. - Resolve at create time instead, using the creating run's own concrete - delivery target — the ``HERMES_CRON_AUTO_DELIVER_*`` contextvars that - ``run_job`` publishes per run (already per-job-safe under the parallel - pool). Rules: - - * Not a cron-context session → returned unchanged (chat/CLI creates keep - today's fire-time ``origin`` semantics, byte-identical). - * ``origin`` element (or an omitted value, which the scheduler treats as - origin) → replaced with ``platform:chat_id[:thread_id]`` from the - creating run's target; ``local`` when the creating run has no concrete - target (e.g. its own deliver is ``local``). - * Every other element (``local``, ``all``, explicit ``platform:...``) - passes through verbatim, including inside comma lists. - """ - from gateway.session_context import get_session_env - from utils import is_truthy_value - - if not is_truthy_value(get_session_env("HERMES_CRON_SESSION", "")): - return deliver - - def _creator_target() -> str: - platform = get_session_env("HERMES_CRON_AUTO_DELIVER_PLATFORM", "").strip() - chat_id = get_session_env("HERMES_CRON_AUTO_DELIVER_CHAT_ID", "").strip() - if not platform or not chat_id: - return "local" - thread_id = get_session_env("HERMES_CRON_AUTO_DELIVER_THREAD_ID", "").strip() - if thread_id: - return f"{platform}:{chat_id}:{thread_id}" - return f"{platform}:{chat_id}" - - if deliver is None: - return _creator_target() - parts = [p.strip() for p in str(deliver).split(",") if p.strip()] - resolved = [_creator_target() if p.lower() == "origin" else p for p in parts] - # De-dup while preserving order: 'origin,local' with a local-target - # creator would otherwise store 'local,local'. - seen: set = set() - unique = [p for p in resolved if not (p in seen or seen.add(p))] - return ",".join(unique) if unique else None - - -def _validate_cron_base_url( - provider: Optional[Any], base_url: Optional[Any] -) -> Optional[str]: - """Reject pairing a named provider's stored credential with an off-host base_url. - - The cron tool is model-callable, so a prompt-injected job could set a real - provider plus an attacker ``base_url``; on fire the scheduler resolves that - provider's stored API key and sends it to the URL, exfiltrating the - credential (CWE-200/CWE-522). Allow a ``base_url`` override only when it - cannot leak a stored secret: no override at all, a configured custom/byok - provider that carries its own endpoint+key, or an override whose host - matches the named provider's own endpoint. - - Returns an error string if blocked, else None (valid). - """ - bu = _normalize_optional_job_value(base_url, strip_trailing_slash=True) - if not bu: - return None - prov = _normalize_optional_job_value(provider) - if not prov: - # A base_url with no explicit provider inherits the default/session - # provider's stored key — the same exfil primitive without naming a - # provider. Require an explicit (custom) provider for custom endpoints. - return ( - "base_url override requires an explicit provider. Set provider to a " - "configured custom provider to use a custom endpoint." - ) - try: - from hermes_cli.runtime_provider import ( - has_named_custom_provider, - resolve_requested_provider, - _get_named_custom_provider, - ) - from hermes_cli.auth import PROVIDER_REGISTRY - from utils import base_url_host_matches, base_url_hostname - except Exception: - # Can't resolve provider metadata -> fail closed. - return f"Unable to validate base_url override for provider {prov!r}; refused." - - if prov.lower() == "custom": - # Bare/inline 'custom' (and aliases that resolve to it) is pure BYOK: the - # runtime derives the key from a pool keyed by THIS base_url or from - # host-gated env vars, never an arbitrary stored secret. Safe to allow. - return None - if has_named_custom_provider(prov): - # A NAMED custom provider carries a STORED key, and - # _resolve_named_custom_runtime prefers the override base_url while still - # sending that stored key — so an off-host override exfiltrates it. - # Require the override host to match the provider's CONFIGURED endpoint. - try: - cp = _get_named_custom_provider(prov) - except Exception: - cp = None - cfg_host = base_url_hostname((cp or {}).get("base_url", "")) if cp else "" - if cfg_host and base_url_host_matches(bu, cfg_host): - return None - return ( - f"base_url {bu!r} is not allowed for provider {prov!r}. A named " - f"custom provider's stored credential may only be sent to its own " - f"configured endpoint ({cfg_host or 'unknown'})." - ) - try: - resolved = resolve_requested_provider(prov) - except Exception: - resolved = prov - pconfig = PROVIDER_REGISTRY.get(resolved) if isinstance(resolved, str) else None - known_host = base_url_hostname(getattr(pconfig, "inference_base_url", "") if pconfig else "") - if known_host and base_url_host_matches(bu, known_host): - return None - # Fail closed: any non-custom provider we cannot host-match to its own - # endpoint is refused. This covers named providers with a stored credential - # AND aliases/unknown names we can't resolve to a known host (e.g. "openai", - # "google"), which would otherwise pair a stored key with the override URL. - return ( - f"base_url {bu!r} is not allowed for provider {prov!r}. A named " - f"provider's stored credential may only be sent to its own endpoint; " - f'use a configured custom provider (provider="custom") for a custom base_url.' - ) - - -def _validate_cron_script_path(script: Optional[str]) -> Optional[str]: - """Validate a cron job script path at the API boundary. - - Scripts must be relative paths that resolve within HERMES_HOME/scripts/. - Absolute paths and ~ expansion are rejected to prevent arbitrary script - execution via prompt injection. - - Returns an error string if blocked, else None (valid). - """ - if not script or not script.strip(): - return None # empty/None = clearing the field, always OK - - from hermes_constants import get_hermes_home - - raw = script.strip() - - # Reject absolute paths and ~ expansion at the API boundary. - # Only relative paths within ~/.hermes/scripts/ are allowed. - if raw.startswith(("/", "~")) or (len(raw) >= 2 and raw[1] == ":"): - return ( - f"Script path must be relative to ~/.hermes/scripts/. " - f"Got absolute or home-relative path: {raw!r}. " - f"Place scripts in ~/.hermes/scripts/ and use just the filename." - ) - - # Validate containment after resolution - from tools.path_security import validate_within_dir - - scripts_dir = get_hermes_home() / "scripts" - scripts_dir.mkdir(parents=True, exist_ok=True) - containment_error = validate_within_dir(scripts_dir / raw, scripts_dir) - if containment_error: - return ( - f"Script path escapes the scripts directory via traversal: {raw!r}" - ) - - return None - - -def _format_job(job: Dict[str, Any]) -> Dict[str, Any]: - prompt = str(job.get("prompt") or "") - skills = _canonical_skills(job.get("skill"), job.get("skills")) - job_id = str(job.get("id") or "unknown") - name = str(job.get("name") or prompt[:50] or (skills[0] if skills else "") or job_id or "cron job") - result = { - "job_id": job_id, - "name": name, - "skill": skills[0] if skills else None, - "skills": skills, - "prompt_preview": prompt[:100] + "..." if len(prompt) > 100 else prompt, - "model": job.get("model"), - "provider": job.get("provider"), - "base_url": job.get("base_url"), - "schedule": job.get("schedule_display") or "?", - "repeat": _repeat_display(job), - "deliver": job.get("deliver", "local"), - "next_run_at": job.get("next_run_at"), - "last_run_at": job.get("last_run_at"), - "last_status": job.get("last_status"), - "last_delivery_error": job.get("last_delivery_error"), - "last_delivery_unverified": job.get("last_delivery_unverified"), - "last_fire_error": job.get("last_fire_error"), - "enabled": job.get("enabled", True), - # Derive from enabled so half-paused records never render as paused. - "state": effective_job_state(job), - "paused_at": job.get("paused_at"), - "paused_reason": job.get("paused_reason"), - } - if job.get("script"): - result["script"] = job["script"] - if job.get("reasoning_effort"): - result["reasoning_effort"] = job["reasoning_effort"] - if job.get("monitor_script"): - result["monitor_script"] = job["monitor_script"] - if job.get("monitor_url"): - result["monitor_url"] = job["monitor_url"] - if job.get("monitor_state"): - result["monitor_state"] = job["monitor_state"] - if job.get("no_agent"): - result["no_agent"] = True - if job.get("enabled_toolsets"): - result["enabled_toolsets"] = job["enabled_toolsets"] - if job.get("workdir"): - result["workdir"] = job["workdir"] - stored_refs = job.get("context_from") or [] - if isinstance(stored_refs, str): - stored_refs = [stored_refs] - if any(str(r).strip().lower() == "self" or r == job.get("id") for r in stored_refs): - result["continuity"] = True - external_refs = [ - r for r in stored_refs - if str(r).strip().lower() != "self" and r != job.get("id") - ] - if external_refs: - result["context_from"] = external_refs - if isinstance(job.get("attach_to_session"), bool): - result["attach_to_session"] = job["attach_to_session"] - return result - def _relay_fronted_delivery_platforms(job: Dict[str, Any]) -> set: """Delivery-platform names for this job that the relay connector fronts.""" @@ -825,8 +100,7 @@ def _relay_fronted_delivery_platforms(job: Dict[str, Any]) -> set: targets = _resolve_delivery_targets(job) or [] except Exception: return set() - theirs = {t.get("platform") for t in targets if t.get("platform")} - return theirs & fronted + return {t.get("platform") for t in targets if t.get("platform")} & fronted def _forward_relay_fronted_run( @@ -835,14 +109,11 @@ def _forward_relay_fronted_run( """Forward a manual run to the gateway when it targets a relay-fronted platform and this process has no live relay adapter. - Relay-fronted delivery has no standalone sender: the connector owns the - credential and the gateway's live relay adapter is the only path. The - gateway api_server's ``POST /api/jobs/{id}/run`` marks the job due for its - own ticker, which fires it with the live adapter. ``extra_prompt`` - (transient per-run context) rides in the request body so the forwarded - fire keeps it. Returns a JSON result string when forwarding engages - (dispatch or the accurate error), else None to fall through to the normal - in-process run. + Relay-fronted delivery has no standalone sender — the gateway's live relay + adapter is the only path, reached via ``POST /api/jobs/{id}/run`` (which + marks the job due for the gateway ticker; ``extra_prompt`` rides in the + body). Returns a JSON result string when forwarding engages, else None to + fall through to the normal in-process run. """ if not _relay_fronted_delivery_platforms(job): return None @@ -854,9 +125,8 @@ def _forward_relay_fronted_run( port = int(port_raw) if port_raw else 8642 except ValueError: port = 8642 - # Mirror the api_server's own bind resolution (adapter reads - # extra.host -> API_SERVER_HOST -> 127.0.0.1). A wildcard bind - # (0.0.0.0/::) listens on loopback too, so dial loopback for those. + # Mirror the api_server's bind resolution (extra.host -> API_SERVER_HOST + # -> 127.0.0.1); a wildcard bind listens on loopback too. host = "" try: from hermes_cli.config import cfg_get, load_config_readonly @@ -922,21 +192,12 @@ def _forward_relay_fronted_run( def _manual_run_delivery_note(deliver: str, refreshed: Dict[str, Any]) -> str: - """Parenthetical delivery note for a manual run's completion summary. - - Follows the refreshed job record (#83993): ``run_one_job`` writes - ``last_delivery_error`` via ``mark_job_run`` when the post-run delivery - (telegram/discord/…) failed, and the summary must not claim success over - that record — the calling agent relays this line to the user. Local jobs - never deliver; an empty/missing error keeps the legacy wording - byte-for-byte. - """ - # Falsy deliver ("", stored JSON null) means no delivery target — the - # fire-time path normalizes it to "local" (no delivery, output persisted - # in last_output, no delivery error), so it must read as saved-locally, - # not as a delivered remote target. Whitespace-only values are NOT folded - # in here: they keep falling through to the error check, where the - # fire-time "no delivery target resolved" error gets surfaced. + """Parenthetical delivery note for a manual run's completion summary, + following the refreshed record's ``last_delivery_error`` so the summary + never claims success over a failed post-run delivery.""" + # Falsy deliver ("", stored JSON null) is normalized to "local" at fire + # time -> read as saved-locally. Whitespace-only values fall through to + # the error check so the fire-time "no delivery target" error surfaces. if not deliver or deliver == "local": return " (output saved locally only)" err = str(refreshed.get("last_delivery_error") or "").strip() @@ -945,50 +206,55 @@ def _manual_run_delivery_note(deliver: str, refreshed: Dict[str, Any]) -> str: return f" (⚠ delivery FAILED: {err[:200]})" +_ALREADY_RUNNING_ERROR = ( + "Job is already running (a scheduler tick or another " + "manual run is executing it); not started again." +) + + +def _claim_for_manual_run(job_id: str, log_label: str): + """At-most-once claim shared by the sync and background run paths. + + Returns ``(claimed_job, None)`` on success, else ``(None, error_dict)`` + where the dict has the ``_execute_job_now`` result shape. A lost claim is + labelled precisely: claim_job_for_fire also returns False for paused / + disabled / missing jobs, which must not read as "already being fired". + """ + try: + claimed_job = claim_job_for_fire(job_id, return_job=True) + if isinstance(claimed_job, dict): + return claimed_job, None + refreshed = get_job(job_id) + if refreshed is None: + reason = "Job no longer exists; nothing to run." + elif not is_job_runnable(refreshed): + reason = "Job is paused/disabled; resume it before running." + else: + reason = "Job is already being fired by the scheduler; not run again." + return None, {"claimed": False, "success": False, "error": reason} + except Exception as e: + logger.error("Failed to claim cron job %s for %s: %s", job_id, log_label, e) + try: + mark_job_run(job_id, False, str(e)) + except Exception: + pass + return None, {"claimed": True, "success": False, "error": str(e)} + + def _execute_job_now( job: Dict[str, Any], extra_prompt: Optional[str] = None ) -> Dict[str, Any]: """Execute a cron job immediately, outside the scheduler tick. - Atomically claims the job first via ``claim_job_for_fire`` — the same - at-most-once CAS the scheduler/external-provider fire path uses — so a - concurrently-running gateway ticker cannot also fire it (the claim both - blocks a duplicate fire and advances ``next_run_at`` for recurring jobs). - If the claim is lost (another fire is in flight), this is a no-op. - - The actual firing is delegated to ``run_one_job`` — the single shared - execute→save→deliver→mark body the ticker and external providers use — so - failure delivery, ``[SILENT]`` handling, and live-adapter delivery stay - identical across paths and can't drift. - + Claims first via ``claim_job_for_fire`` (the same CAS the ticker uses, so a + concurrent tick cannot double-fire and next_run_at advances), then fires + through the shared ``run_one_job`` body so delivery / [SILENT] handling + cannot drift between paths. Returns {"claimed": bool, "success": bool, "error": str|None}. """ - job_id = job["id"] - claimed_job = None - try: - # At-most-once claim: bail without running if a tick/other fire owns it. - claimed_job = claim_job_for_fire(job_id, return_job=True) - if not isinstance(claimed_job, dict): - # claim_job_for_fire returns False for paused/disabled/missing - # jobs too — don't mislabel those as "already being fired" - # (#60703): that message sends the user chasing a phantom - # in-flight run when the job simply isn't runnable. - refreshed = get_job(job_id) - if refreshed is None: - reason = "Job no longer exists; nothing to run." - elif not is_job_runnable(refreshed): - reason = "Job is paused/disabled; resume it before running." - else: - reason = "Job is already being fired by the scheduler; not run again." - return {"claimed": False, "success": False, "error": reason} - except Exception as e: - logger.error("Failed to claim cron job %s for immediate run: %s", job_id, e) - try: - mark_job_run(job_id, False, str(e)) - except Exception: - pass - return {"claimed": True, "success": False, "error": str(e)} - + claimed_job, err = _claim_for_manual_run(job["id"], "immediate run") + if err is not None: + return err return _run_claimed_job(claimed_job, extra_prompt=extra_prompt) @@ -997,12 +263,9 @@ def _run_claimed_job( ) -> Dict[str, Any]: """Fire an already-claimed job through the shared ``run_one_job`` body. - Split out of ``_execute_job_now`` so the background dispatch path - (``_try_dispatch_background_run``) can take the claim synchronously — so - the tool response can report "paused"/"already firing" immediately — and - hand the actual run to a daemon worker. - - Returns {"claimed": True, "success": bool, "error": str|None}. + Split from ``_execute_job_now`` so the background path can take the claim + synchronously (reporting paused/already-firing immediately) and hand the + run to a worker. Returns {"claimed": True, "success": bool, "error": ...}. """ job_id = job["id"] _registered = False @@ -1014,48 +277,24 @@ def _run_claimed_job( try_register_running_job, ) - # In-flight dedupe (idea from #53395 by @izumi0uu): the fire claim's - # TTL (300s) is routinely outlived by real jobs, so it alone cannot - # stop a manual run from double-firing a job the ticker (or another - # manual run) is still executing. Register in the scheduler's shared - # running set — the same guard _submit_with_guard uses — which also - # makes this run visible to the gateway shutdown drain - # (get_running_job_ids, #60432) and mark_running_jobs_interrupted. + # In-flight dedupe: the fire claim's TTL is routinely outlived by real + # jobs, so register in the scheduler's shared running set (same guard + # the ticker uses; also visible to the gateway shutdown drain). if not try_register_running_job(job_id): - return { - "claimed": True, - "success": False, - "error": ( - "Job is already running (a scheduler tick or another " - "manual run is executing it); not started again." - ), - } + return {"claimed": True, "success": False, "error": _ALREADY_RUNNING_ERROR} _registered = True claim = job.get("fire_claim") fire_owner = str(claim.get("by") or "") if isinstance(claim, dict) else None - # run_one_job records last_run_at/last_status via mark_job_run (which - # also clears the fire claim) and returns True iff it processed the job. - # ``job`` here is the exact claimed snapshot (owner-bearing), so the - # shared body fences every terminal write by that owner. - # - # A manual `run` executes the job synchronously on the caller's thread, - # and a cron job is itself a full agent run that routinely takes - # minutes. The calling turn emits no tool activity for that entire - # window, so the gateway inactivity watchdog concludes the agent is - # hung and kills the parent turn (#76502). Fire a heartbeat into the - # caller's activity tracker (the same signal tool progress uses) while - # the job runs, so the watchdog sees a working tool instead of a - # silent one — mirrors the delegate_task heartbeat pattern. Best-effort: - # if no activity callback is registered (direct Python callers, tests), - # behavior is unchanged. + # A manual run executes synchronously on the caller's thread and can + # take minutes; without tool activity the gateway inactivity watchdog + # would kill the parent turn. Heartbeat into the caller's activity + # tracker while the job runs (best-effort: no callback -> unchanged). try: from tools.environments.base import get_activity_callback - # Capture on THIS thread: the callback is thread-local (installed - # by the tool executor as the calling agent's _touch_activity), so - # a freshly spawned thread cannot read it back. + # Capture on THIS thread: the callback is thread-local. activity_cb = get_activity_callback() except Exception: activity_cb = None @@ -1071,8 +310,6 @@ def _run_claimed_job( while not _heartbeat_stop.wait(_CRON_RUN_HEARTBEAT_INTERVAL): elapsed = time.monotonic() - started if elapsed > _CRON_RUN_HEARTBEAT_CEILING: - # Stop masking the gateway watchdog — a run this long - # with an unlimited child watchdog is likely wedged. logger.warning( "cronjob run heartbeat ceiling reached for job " "'%s' (%.0fs) — stopping heartbeat; gateway " @@ -1085,10 +322,7 @@ def _run_claimed_job( f"cronjob: running job '{job_name}' ({int(elapsed)}s elapsed)" ) except Exception: - # Never break the job run; keep heartbeating — one - # transient callback error must not silently drop - # watchdog protection for the rest of a long job. - continue + continue # one transient callback error must not drop protection _heartbeat_thread = threading.Thread( target=_heartbeat_loop, @@ -1097,14 +331,10 @@ def _run_claimed_job( ) _heartbeat_thread.start() - # Manual runs invoked from a gateway agent execute outside the scheduler - # ticker, but they still share the process with the live platform - # adapters. Pass the gateway-owned adapter map and event loop through - # to run_one_job so delivery is scheduled on the loop that owns clients - # such as Matrix/aiohttp. Calling those clients from run_one_job's - # standalone asyncio.run() loop raises errors like "Timeout context - # manager should be used inside a task" and can break encrypted Matrix - # delivery (#61495 — salvaged from #63586 by @Fly-onlyone). + # Manual runs from a gateway agent share the process with live platform + # adapters: pass the gateway adapter map + event loop so delivery runs + # on the loop that owns clients such as Matrix/aiohttp (a standalone + # asyncio.run() loop breaks them). gateway_module = sys.modules.get("gateway.run") runner_ref = getattr(gateway_module, "_gateway_runner_ref", None) runner = runner_ref() if callable(runner_ref) else None @@ -1113,6 +343,9 @@ def _run_claimed_job( try: try: + # run_one_job records last_run_at/last_status via mark_job_run; + # `job` is the owner-bearing claimed snapshot, so terminal writes + # stay fenced by that owner. processed = run_one_job( job, adapters=adapters, loop=gateway_loop, extra_prompt=extra_prompt, @@ -1126,29 +359,22 @@ def _run_claimed_job( release_running_job(job_id) refreshed = get_job(job_id) or {} last_status = refreshed.get("last_status") - # "delivery_failed" (#83993): the agent run itself succeeded but the - # output never reached the user. That is NOT a success for the caller - # — the calling agent relays this result — so report it as failed - # and surface the delivery error, which lives in last_delivery_error - # (last_error is None for these runs, and a bare success=False with - # error=None reads as an unexplained failure). - ok = last_status == "ok" + # "delivery_failed": the run succeeded but output never reached the + # user — not a success for the caller; surface last_delivery_error. run_error = refreshed.get("last_error") if last_status == "delivery_failed" and not run_error: run_error = refreshed.get("last_delivery_error") return { "claimed": True, - "success": bool(processed and ok), + "success": bool(processed and last_status == "ok"), "error": run_error, } except Exception as e: logger.error("Failed to execute cron job %s immediately: %s", job_id, e) if _registered: - # Registration succeeded but we raised before the run's own - # release ran (e.g. heartbeat setup) — don't leave the job - # permanently marked in-flight. Only release registrations WE - # took: a bare discard here could erase a ticker-owned entry. + # Only release registrations WE took — a bare discard could erase + # a ticker-owned entry. try: from cron.scheduler import release_running_job as _release @@ -1164,20 +390,11 @@ def _run_claimed_job( ) except Exception: pass - return { - "claimed": True, - "success": False, - "error": str(e), - } + return {"claimed": True, "success": False, "error": str(e)} def _latest_job_output_excerpt(job_id: str, max_chars: int = 2000) -> Optional[str]: - """Best-effort excerpt of the job's most recent saved output file. - - Included in the background-run completion block so the parent agent sees - what the job actually produced without having to dig through - ``~/.hermes/cron/output/``. Never raises. - """ + """Best-effort excerpt of the job's most recent saved output file. Never raises.""" try: from cron.jobs import get_cron_output_dir @@ -1201,41 +418,20 @@ def _try_dispatch_background_run( ) -> Optional[Dict[str, Any]]: """Claim ``job`` now, then fire it on the async-delegation daemon executor. - A manual ``cronjob(action='run')`` used to execute the job synchronously - on the calling agent's tool thread. A cron job is a full agent run that - routinely takes minutes-to-hours, so the parent turn sat inside ONE tool - call the whole time: uninterruptible (the interrupt flag is only checked - between loop iterations) and serial (a batch of runs executed one by one). + A cron job is a full agent run (minutes to hours); running it inline made + the parent turn uninterruptible and serialized batches. This dispatches + like ``delegate_task``'s background mode: the tool returns a handle, and a + ``type="async_delegation"`` completion event re-enters the conversation as + a fresh turn (keeps role alternation legal and the prompt cache intact). + The claim is taken SYNCHRONOUSLY so unrunnable jobs report immediately. - This dispatches the run like ``delegate_task``'s background mode: the tool - returns immediately with a handle, the run executes on the shared async - daemon executor, and a ``type="async_delegation"`` completion event - re-enters the conversation as a fresh turn when the job finishes — riding - the existing completion-queue rail (CLI drain + gateway watcher), which - keeps message-role alternation legal and the prompt cache intact. - - The at-most-once claim is taken SYNCHRONOUSLY before dispatch so - unrunnable jobs (paused / missing / already firing) report in the tool - response immediately instead of as a delayed completion event. - - Returns - ------- - None - Background delivery unavailable on this session runtime (one-shot - ``hermes -z``, stateless HTTP, Kanban worker, nested cron run). - Caller falls back to the synchronous path unchanged. - dict - ``{"claimed": False, "success": False, "error": ...}`` — claim lost; - same shape as ``_execute_job_now`` so the caller's existing response - formatting applies. - ``{"claimed": True, "dispatched": True, "delegation_id": ...}`` — - run is executing in the background. - ``{"claimed": True, "dispatched": False, "success": ..., "error": ...}`` - — dispatch pool was at capacity; the run executed inline (the claim - was already taken and must not be stranded). + Returns None when background delivery is unavailable on this runtime + (caller falls back to the sync path); ``{"claimed": False, ...}`` on a lost + claim; ``{"claimed": True, "dispatched": True, "delegation_id": ...}`` when + running in the background; ``{"claimed": True, "dispatched": False, ...}`` + when the pool was full and the run executed inline (claim already taken). """ - # Finite sessions cannot route a detached result back after the turn - # ends — mirror delegate_task's gate and fall back to sync execution. + # Finite sessions cannot route a detached result back after the turn ends. try: from gateway.session_context import async_delivery_supported @@ -1247,18 +443,11 @@ def _try_dispatch_background_run( job_id = job["id"] job_name = str(job.get("name") or job_id) - # Reap any execution row this job (or any job) left stranded 'claimed'/ - # 'running' by a dead owner process -- e.g. a PRIOR one-shot `hermes - # cron run` invocation whose dispatched runner died with the exiting - # process before writing a terminal status (issue #86721). The - # long-lived scheduler ticker already does this once at its own - # startup (cron/scheduler.py's self.recover_interrupted()); a one-shot - # CLI invocation has no equivalent "startup" moment of its own, so it - # never got this self-heal -- leaving a permanently-stale claim that - # blocked every subsequent manual run on the same job. Safe and cheap: - # only provably-dead owners (PID gone, or PID reused by a different - # process per its start time) are reaped; a genuinely live owner's row - # is left untouched. + # Reap execution rows left 'claimed'/'running' by a dead owner process + # (e.g. a prior one-shot `hermes cron run` that exited mid-run). The + # ticker does this at its own startup; one-shot invocations have no such + # moment, so a stale claim would block every later manual run. Only + # provably-dead owners are reaped. try: from cron.executions import recover_interrupted_executions @@ -1271,13 +460,11 @@ def _try_dispatch_background_run( job_name, ) except Exception as _reap_exc: - # Best-effort self-heal; a failure here must not block dispatch — - # but stay diagnosable (mirrors the scheduler tick's reap handling). logger.debug("Stale execution reclaim failed: %s", _reap_exc) - # ---- routing capture (on THIS thread; contextvars don't cross the pool) ---- - # Resolved BEFORE the claim: with no routable session there is no durable - # consumer for a detached completion, so we must not claim-and-dispatch. + # Routing capture on THIS thread (contextvars don't cross the pool), and + # BEFORE the claim: with no routable session there is no durable consumer + # for a detached completion, so we must not claim-and-dispatch. try: from tools.approval import get_current_session_key @@ -1286,56 +473,29 @@ def _try_dispatch_background_run( session_key = "" if not session_key and session_id: # CLI path: the approval contextvar is only bound during gateway/TUI - # turns. The CLI drain filters completions by the durable agent - # session id (#64240), so stamp it as the key — an empty key would - # fail closed and the completion could never be claimed. + # turns; the CLI drain filters completions by the durable session id. session_key = str(session_id) if not session_key: - # Direct Python callers (`hermes cron run`, tests) have no agent - # session to deliver a completion to — the process exits right after - # the tool returns. Run synchronously. + # Direct Python callers (`hermes cron run`, tests): process exits right + # after the tool returns, so run synchronously. return None - # ---- synchronous claim (same semantics as _execute_job_now) ---- + # Best-effort early dedupe so a mid-run job reports in THIS tool response + # instead of as a delayed error completion. The authoritative (atomic) + # check is try_register_running_job inside _run_claimed_job. try: - # Best-effort early dedupe so a mid-run job reports in THIS tool - # response instead of as a delayed error completion event. The - # authoritative (atomic) check is try_register_running_job inside - # _run_claimed_job on the worker. - try: - from cron.scheduler import get_running_job_ids + from cron.scheduler import get_running_job_ids - if job_id in get_running_job_ids(): - return { - "claimed": False, - "success": False, - "error": ( - "Job is already running (a scheduler tick or another " - "manual run is executing it); not started again." - ), - } - except Exception: - pass + if job_id in get_running_job_ids(): + return {"claimed": False, "success": False, "error": _ALREADY_RUNNING_ERROR} + except Exception: + pass - # Same snapshot claim as _execute_job_now: carry the owner-bearing - # record into the run so terminal writes stay fenced by this owner. - claimed_job = claim_job_for_fire(job_id, return_job=True) - if not isinstance(claimed_job, dict): - refreshed = get_job(job_id) - if refreshed is None: - reason = "Job no longer exists; nothing to run." - elif not is_job_runnable(refreshed): - reason = "Job is paused/disabled; resume it before running." - else: - reason = "Job is already being fired by the scheduler; not run again." - return {"claimed": False, "success": False, "error": reason} - except Exception as e: - logger.error("Failed to claim cron job %s for background run: %s", job_id, e) - try: - mark_job_run(job_id, False, str(e)) - except Exception: - pass - return {"claimed": True, "dispatched": False, "success": False, "error": str(e)} + claimed_job, err = _claim_for_manual_run(job_id, "background run") + if err is not None: + if err["claimed"]: + err["dispatched"] = False + return err origin_ui_session_id = "" try: @@ -1369,11 +529,8 @@ def _try_dispatch_background_run( max_async = 3 started_at = time.time() - # Canonicalize with the scheduler's own normalizer so the summary states - # the same target fire time will use: falsy ("", stored JSON null) reads - # "local", legacy list-form deliver flattens to its comma string. Read - # from the claimed snapshot — the owner-bearing record the run actually - # executes — not the pre-claim `job` the tool loaded. + # Canonicalize with the scheduler's own normalizer (falsy -> "local", list + # -> comma string), reading the claimed snapshot the run actually executes. from cron.scheduler import _normalize_deliver_value deliver = _normalize_deliver_value(claimed_job.get("deliver", "local")) @@ -1438,61 +595,362 @@ def _try_dispatch_background_run( return result -def _apply_continuity( - context_from: Optional[Union[str, List[str]]], - continuity: bool, -) -> Optional[List[str]]: - """Translate the ``continuity`` flag into the ``context_from`` list. +# --------------------------------------------------------------------------- +# Tool actions. Each takes the cronjob() argument dict `a` (and the resolved +# job record for job-bound actions) and returns the JSON result string. +# --------------------------------------------------------------------------- - ``continuity=True`` ensures ``"self"`` is present (the job's own previous - output is injected each run); ``continuity=False`` removes any - ``"self"``/own-id entry. Other entries are preserved untouched. - """ - if isinstance(context_from, str): - refs = [context_from.strip()] if context_from.strip() else [] - elif context_from: - refs = [str(j).strip() for j in context_from if str(j).strip()] - else: - refs = [] - has_self = any(r.lower() == "self" for r in refs) - if continuity and not has_self: - refs.append("self") - elif not continuity and has_self: - refs = [r for r in refs if r.lower() != "self"] - return refs or None +def _dumps(payload: Dict[str, Any]) -> str: + return json.dumps(payload, indent=2) -def _gateway_liveness_notice(plural: bool = False) -> dict: - """Build the ``gateway_running``/``warning`` payload for tool results. +def _action_create(a: Dict[str, Any]) -> str: + prompt, script, deliver = a["prompt"], a["script"], a["deliver"] + if not a["schedule"]: + return tool_error("schedule is required for create", success=False) + canonical_skills = _canonical_skills(a["skill"], a["skills"]) + _no_agent = bool(a["no_agent"]) + # no_agent=True -> the script IS the job (prompt/skills optional); + # otherwise at least one of prompt/skills is required. + if _no_agent: + if not script: + return tool_error( + "create with no_agent=True requires a script — " + "the script is the job. In no_agent mode the LLM is " + "skipped entirely: prompt and skills are ignored, " + "non-empty stdout is delivered verbatim, empty stdout " + "sends nothing (watchdog pattern), and a non-zero " + "exit or timeout sends an error alert.", + success=False, + ) + elif not prompt and not canonical_skills: + return tool_error("create requires either prompt or at least one skill", success=False) + error = ( + (prompt and _scan_cron_prompt(prompt)) + or (script and _validate_cron_script_path(script)) + or (a["monitor_script"] and _validate_cron_script_path(a["monitor_script"])) + # A model-supplied base_url must not route a named provider's stored + # credential to an attacker endpoint. + or _validate_cron_base_url(a["provider"], a["base_url"]) + # bot-chat targets are machine-local: fail the CREATE, not the run. + or _validate_bot_chat_deliver(_normalize_deliver_param(deliver)) + # failure_deliver shares deliver's grammar and validators (NS-788). + or _validate_bot_chat_deliver(_normalize_deliver_param(a["failure_deliver"])) + or (a["context_from"] and _validate_context_from_refs( + [a["context_from"]] if isinstance(a["context_from"], str) else a["context_from"] + )) + ) + if error: + return tool_error(error, success=False) + + context_from = a["context_from"] + if a["continuity"] is not None: + context_from = _apply_continuity(context_from, a["continuity"]) + + from cron.scheduler import ( + CronSchedulerRegistrationError, + create_job_with_scheduler_registration, + ) - Thin adapter over the shared CLI helper ``hermes_cli.cron._builtin_gateway_liveness`` - (#87033) so the CLI and this tool can never disagree about what "scheduler - active" means. Returns ``{"gateway_running": False, "warning": ...}`` when - the builtin ticker has no gateway process to run it, ``{"gateway_running": - None}`` when the probe failed, and ``{"gateway_running": True}`` when the - scheduler is active. ``plural`` rewords the warning for multi-job results - (the ``list`` action). - """ try: - from hermes_cli.cron import _builtin_gateway_liveness - - _gw = _builtin_gateway_liveness() - except Exception: - return {"gateway_running": None} - subject = "these jobs are saved" if plural else "this job is saved" - if _gw is False: - return { - "gateway_running": False, - "warning": ( - f"The Hermes gateway is not running — {subject} " - "but will NOT fire until the gateway is started " - "(hermes gateway install / hermes gateway start). " - "Tell the user the task is scheduled but not active yet." + job = create_job_with_scheduler_registration( + prompt=prompt or "", + schedule=a["schedule"], + name=a["name"], + repeat=a["repeat"], + deliver=_resolve_cron_context_deliver(_normalize_deliver_param(deliver)), + origin=_origin_from_env(), + skills=canonical_skills, + model=_normalize_optional_job_value(a["model"]), + provider=_normalize_optional_job_value(a["provider"]), + base_url=_normalize_optional_job_value(a["base_url"], strip_trailing_slash=True), + script=_normalize_optional_job_value(script), + context_from=context_from, + enabled_toolsets=a["enabled_toolsets"] or None, + workdir=_normalize_optional_job_value(a["workdir"]), + no_agent=_no_agent, + attach_to_session=a["attach_to_session"], + monitor_script=_normalize_optional_job_value(a["monitor_script"]), + monitor_url=_normalize_optional_job_value(a["monitor_url"]), + # CLI-only lane: deliberately absent from CRONJOB_SCHEMA and the + # model dispatch — models do not make model-config decisions. + reasoning_effort=a["reasoning_effort"], + failure_deliver=_resolve_cron_context_deliver( + _normalize_deliver_param(a["failure_deliver"]) ), - } - if _gw is None: - return {"gateway_running": None} - return {"gateway_running": True} + ) + except CronSchedulerRegistrationError as exc: + _partial = exc.to_dict() + return tool_error(_partial.pop("error"), success=False, **_partial) + _create_message = f"Cron job '{job['name']}' created." + _local_notice = _local_delivery_notice(job, _normalize_deliver_param(deliver)) + if _local_notice: + _create_message = f"{_create_message} {_local_notice}" + # A job created with no gateway running is stored but never fires — tell + # the model, which otherwise reports a clean success. + _result = { + "success": True, + "job_id": job["id"], + "name": job["name"], + "skill": job.get("skill"), + "skills": job.get("skills", []), + "schedule": job["schedule_display"], + "repeat": _repeat_display(job), + "deliver": job.get("deliver", "local"), + "next_run_at": job["next_run_at"], + "job": _format_job(job), + "message": _create_message, + **_gateway_liveness_notice(), + } + _notes = _mode_guidance_notes(job, _normalize_deliver_param(deliver)) + if _notes: + _result["guidance"] = _notes + return _dumps(_result) + + +def _action_list(a: Dict[str, Any]) -> str: + jobs = [_format_job(job) for job in list_jobs(include_disabled=a["include_disabled"])] + _result = {"success": True, "count": len(jobs), "jobs": jobs} + # Same inert-job class as create; an empty list has nothing inert. + if jobs: + _result.update(_gateway_liveness_notice(plural=True)) + return _dumps(_result) + + +def _action_remove(job: Dict[str, Any], a: Dict[str, Any]) -> str: + job_id = job["id"] + if not remove_job(job_id): + return tool_error(f"Failed to remove job '{job_id}'", success=False) + _notify_provider_jobs_changed_safe() + return _dumps({ + "success": True, + "message": f"Cron job '{job['name']}' removed.", + "removed_job": { + "id": job_id, + "name": job["name"], + "schedule": job.get("schedule_display"), + }, + }) + + +def _action_pause(job: Dict[str, Any], a: Dict[str, Any]) -> str: + updated = pause_job(job["id"], reason=a["reason"]) + _notify_provider_jobs_changed_safe() + return _dumps({"success": True, "job": _format_job(updated)}) + + +def _action_resume(job: Dict[str, Any], a: Dict[str, Any]) -> str: + updated = resume_job(job["id"]) + _notify_provider_jobs_changed_safe() + return _dumps({"success": True, "job": _format_job(updated)}) + + +def _action_run(job: Dict[str, Any], a: Dict[str, Any]) -> str: + job_id = job["id"] + # `prompt` on run is transient per-fire context appended to the stored + # prompt, never persisted; same strict scan as stored prompts. + extra_prompt = a["prompt"] or None + if extra_prompt: + scan_error = _scan_cron_prompt(extra_prompt) + if scan_error: + return tool_error(scan_error, success=False) + # A manual run must actually run even with no ticker active. Preferred: + # background dispatch (handle now, outcome as a completion event); falls + # back to inline execution when the runtime can't receive completions. + bg = _try_dispatch_background_run( + job, session_id=a["session_id"], extra_prompt=extra_prompt + ) + if bg is not None and bg.get("dispatched"): + _notify_provider_jobs_changed_safe() + result = _format_job(get_job(job_id) or {"id": job_id}) + result["executed"] = True + result["execution_mode"] = "background" + result["delegation_id"] = bg.get("delegation_id") + return _dumps({ + "success": True, + "job": result, + "note": ( + "The job is running in the background. You and the " + "user can keep working; its outcome re-enters the " + "conversation as a new message when it finishes. " + "Do not wait or poll — just continue." + ), + }) + if bg is not None: + exec_result = bg # terminal result: claim lost or inline fallback + else: + # Relay-fronted manual run: no live adapter here — forward to the + # running gateway, whose adapter owns that delivery. + forwarded = _forward_relay_fronted_run(job, extra_prompt=extra_prompt) + if forwarded is not None: + return forwarded + exec_result = _execute_job_now(job, extra_prompt=extra_prompt) + # A claimed direct run advances next_run_at and may race an external + # provider's one-shot for the same occurrence — reconcile after the run. + claimed = exec_result.get("claimed", False) + if claimed: + _notify_provider_jobs_changed_safe() + # Re-read so the response reflects the post-run last_run_at/last_status. + result = _format_job(get_job(job_id) or {"id": job_id}) + result["executed"] = claimed + result["execution_success"] = exec_result.get("success", False) + if not claimed: + result["execution_skipped"] = exec_result.get("error") or ( + "Already being fired by the scheduler; not run again." + ) + elif exec_result.get("error"): + result["execution_error"] = exec_result["error"] + return _dumps({"success": True, "job": result}) + + +def _action_update(job: Dict[str, Any], a: Dict[str, Any]) -> str: + job_id = job["id"] + updates: Dict[str, Any] = {} + prompt, deliver, skill, skills = a["prompt"], a["deliver"], a["skill"], a["skills"] + script, monitor_script, monitor_url = a["script"], a["monitor_script"], a["monitor_url"] + context_from, continuity, no_agent = a["context_from"], a["continuity"], a["no_agent"] + if prompt is not None: + scan_error = _scan_cron_prompt(prompt) + if scan_error: + return tool_error(scan_error, success=False) + updates["prompt"] = prompt + if a["name"] is not None and a["name"].strip(): + # Blank name is a no-op, not a clear: a model re-sending the whole + # schema with type-default empties must not wipe untouched fields. + updates["name"] = a["name"] + if deliver is not None: + bot_chat_error = _validate_bot_chat_deliver(_normalize_deliver_param(deliver)) + if bot_chat_error: + return tool_error(bot_chat_error, success=False) + updates["deliver"] = _resolve_cron_context_deliver(_normalize_deliver_param(deliver)) + if a["failure_deliver"] is not None: + # '' clears the override (job falls back to deliver on failures); + # non-empty values share deliver's validation AND its cron-context + # origin resolution (a job created from inside a cron run must never + # store literal 'origin' — same rule as deliver). + _norm_fd = _normalize_deliver_param(a["failure_deliver"]) + if _norm_fd: + bot_chat_error = _validate_bot_chat_deliver(_norm_fd) + if bot_chat_error: + return tool_error(bot_chat_error, success=False) + _norm_fd = _resolve_cron_context_deliver(_norm_fd) + updates["failure_deliver"] = _norm_fd + if skills is not None or skill is not None: + canonical_skills = _canonical_skills(skill, skills) + updates["skills"] = canonical_skills + updates["skill"] = canonical_skills[0] if canonical_skills else None + if a["model"] is not None: + updates["model"] = _normalize_optional_job_value(a["model"]) + if a["provider"] is not None: + updates["provider"] = _normalize_optional_job_value(a["provider"]) + if a["base_url"] is not None: + updates["base_url"] = _normalize_optional_job_value(a["base_url"], strip_trailing_slash=True) + if a["reasoning_effort"] is not None: + # CLI-only lane; update_job validates, empty string clears the pin. + updates["reasoning_effort"] = a["reasoning_effort"] + # Re-validate the EFFECTIVE provider/base_url on EVERY update: a job + # persisted before this guard may already hold an unsafe pair, and editing + # an unrelated field must not leave it schedulable. + base_url_error = _validate_cron_base_url( + updates["provider"] if "provider" in updates else job.get("provider"), + updates["base_url"] if "base_url" in updates else job.get("base_url"), + ) + if base_url_error: + return tool_error(base_url_error, success=False) + # Empty string clears script / monitor fields. + for field, value in (("script", script), ("monitor_script", monitor_script)): + if value is not None: + if value: + path_error = _validate_cron_script_path(value) + if path_error: + return tool_error(path_error, success=False) + updates[field] = _normalize_optional_job_value(value) if value else None + if monitor_url is not None: + updates["monitor_url"] = _normalize_optional_job_value(monitor_url) if monitor_url else None + if monitor_script is not None or monitor_url is not None: + eff_mon_script = updates["monitor_script"] if "monitor_script" in updates else job.get("monitor_script") + eff_mon_url = updates["monitor_url"] if "monitor_url" in updates else job.get("monitor_url") + if eff_mon_script and eff_mon_url: + return tool_error( + "monitor_script and monitor_url are mutually exclusive — " + "clear one before setting the other.", + success=False, + ) + if context_from is not None or continuity is not None: + # Empty string / list clears; otherwise every ref must exist. Stored + # as a list (or None) to match create_job(). + if context_from is None: + # continuity-only update: start from the job's stored refs. + existing = job.get("context_from") or [] + refs = [str(j).strip() for j in existing if str(j).strip()] + elif isinstance(context_from, str): + refs = [context_from.strip()] if context_from.strip() else [] + else: + refs = [str(j).strip() for j in context_from if str(j).strip()] + if continuity is not None: + refs = _apply_continuity(refs, continuity) or [] + if refs: + ref_error = _validate_context_from_refs(refs) + if ref_error: + return tool_error(ref_error, success=False) + updates["context_from"] = refs or None + if a["enabled_toolsets"] is not None: + updates["enabled_toolsets"] = a["enabled_toolsets"] or None + if a["attach_to_session"] is not None: + updates["attach_to_session"] = bool(a["attach_to_session"]) + if a["workdir"] is not None: + # Empty string clears; otherwise update_job() validates/normalizes. + updates["workdir"] = _normalize_optional_job_value(a["workdir"]) or None + if no_agent is not None: + # Flipping to True needs a script on the job or in this same update, + # otherwise the next tick would error out. + target_no_agent = bool(no_agent) + if target_no_agent: + effective_script = updates.get("script") if "script" in updates else job.get("script") + if not effective_script: + return tool_error( + "Cannot set no_agent=True on a job without a script. " + "Set `script` in the same update, or on the job first.", + success=False, + ) + updates["no_agent"] = target_no_agent + if a["repeat"] is not None: + # Shared chokepoint coerces string forms ('forever'/'once'/'3') and + # 0/negative values. + from cron.jobs import normalize_repeat_value + repeat_state = dict(job.get("repeat") or {}) + repeat_state["times"] = normalize_repeat_value(a["repeat"]) + updates["repeat"] = repeat_state + if a["schedule"] is not None: + parsed_schedule = parse_schedule(a["schedule"]) + updates["schedule"] = parsed_schedule + updates["schedule_display"] = parsed_schedule.get("display", a["schedule"]) + if job.get("state") != "paused": + updates["state"] = "scheduled" + updates["enabled"] = True + if not updates: + return tool_error("No updates provided.", success=False) + updated = update_job(job_id, updates) + _notify_provider_jobs_changed_safe() + _upd_result: Dict[str, Any] = {"success": True, "job": _format_job(updated)} + # An update can switch modes or delivery — echo the same guidance as create. + _upd_notes = _mode_guidance_notes(updated, _normalize_deliver_param(deliver)) + if _upd_notes: + _upd_result["guidance"] = _upd_notes + return _dumps(_upd_result) + + +# Actions that need no job_id, and job-bound actions (job resolved first). +_JOBLESS_ACTIONS = {"create": _action_create, "list": _action_list} +_JOB_ACTIONS = { + "remove": _action_remove, + "pause": _action_pause, + "resume": _action_resume, + "run": _action_run, + "run_now": _action_run, + "trigger": _action_run, + "update": _action_update, +} def cronjob( @@ -1525,172 +983,15 @@ def cronjob( session_id: Optional[str] = None, ) -> str: """Unified cron job management tool.""" - del task_id # unused but kept for handler signature compatibility + a = dict(locals()) + del a["task_id"] # unused but kept for handler signature compatibility try: normalized = (action or "").strip().lower() - if normalized == "create": - if not schedule: - return tool_error("schedule is required for create", success=False) - canonical_skills = _canonical_skills(skill, skills) - _no_agent = bool(no_agent) - # Job-shape validation differs by mode: - # - no_agent=True → script is the job; prompt/skills are optional - # (and irrelevant to execution). - # - no_agent=False (default) → at least one of prompt/skills must - # be set, same as before. - if _no_agent: - if not script: - return tool_error( - "create with no_agent=True requires a script — " - "the script is the job. In no_agent mode the LLM is " - "skipped entirely: prompt and skills are ignored, " - "non-empty stdout is delivered verbatim, empty stdout " - "sends nothing (watchdog pattern), and a non-zero " - "exit or timeout sends an error alert.", - success=False, - ) - elif not prompt and not canonical_skills: - return tool_error("create requires either prompt or at least one skill", success=False) - if prompt: - scan_error = _scan_cron_prompt(prompt) - if scan_error: - return tool_error(scan_error, success=False) - - # Validate script path before storing - if script: - script_error = _validate_cron_script_path(script) - if script_error: - return tool_error(script_error, success=False) - - # Validate monitor source (same containment rules as script). - if monitor_script: - monitor_error = _validate_cron_script_path(monitor_script) - if monitor_error: - return tool_error(monitor_error, success=False) - - # Reject a model-supplied base_url that would route a named - # provider's stored credential to an attacker endpoint (F8). - base_url_error = _validate_cron_base_url(provider, base_url) - if base_url_error: - return tool_error(base_url_error, success=False) - - # bot-chat deliver targets are machine-local: named profiles must - # exist here, and a bad name should fail the CREATE, not the run. - bot_chat_error = _validate_bot_chat_deliver(_normalize_deliver_param(deliver)) - if bot_chat_error: - return tool_error(bot_chat_error, success=False) - # failure_deliver shares deliver's grammar and validators (NS-788). - bot_chat_error = _validate_bot_chat_deliver( - _normalize_deliver_param(failure_deliver) - ) - if bot_chat_error: - return tool_error(bot_chat_error, success=False) - - # Validate context_from references existing jobs - if context_from: - from cron.jobs import get_job as _get_job - refs = [context_from] if isinstance(context_from, str) else context_from - for ref_id in refs: - # "self" is resolved to the job's own id at run time — - # it can't be validated against the store (the job does - # not exist yet at create time). - if isinstance(ref_id, str) and ref_id.strip().lower() == "self": - continue - if not _get_job(ref_id): - return tool_error( - f"context_from job '{ref_id}' not found. " - "Use cronjob(action='list') to see available jobs.", - success=False, - ) - - # continuity=True is sugar for context_from including "self": - # the job wakes up with its own previous run's output injected. - if continuity is not None: - context_from = _apply_continuity(context_from, continuity) - - from cron.scheduler import ( - CronSchedulerRegistrationError, - create_job_with_scheduler_registration, - ) - - try: - job = create_job_with_scheduler_registration( - prompt=prompt or "", - schedule=schedule, - name=name, - repeat=repeat, - deliver=_resolve_cron_context_deliver( - _normalize_deliver_param(deliver) - ), - origin=_origin_from_env(), - skills=canonical_skills, - model=_normalize_optional_job_value(model), - provider=_normalize_optional_job_value(provider), - base_url=_normalize_optional_job_value(base_url, strip_trailing_slash=True), - script=_normalize_optional_job_value(script), - context_from=context_from, - enabled_toolsets=enabled_toolsets or None, - workdir=_normalize_optional_job_value(workdir), - no_agent=_no_agent, - attach_to_session=attach_to_session, - monitor_script=_normalize_optional_job_value(monitor_script), - monitor_url=_normalize_optional_job_value(monitor_url), - # reasoning_effort reaches here from the CLI - # (hermes cron create --reasoning-effort) ONLY — it is - # deliberately absent from CRONJOB_SCHEMA and the model - # dispatch below: models do not make model-config - # decisions (standing policy). - reasoning_effort=reasoning_effort, - failure_deliver=_resolve_cron_context_deliver( - _normalize_deliver_param(failure_deliver) - ), - ) - except CronSchedulerRegistrationError as exc: - _partial = exc.to_dict() - return tool_error(_partial.pop("error"), success=False, **_partial) - _create_message = f"Cron job '{job['name']}' created." - _local_notice = _local_delivery_notice(job, _normalize_deliver_param(deliver)) - if _local_notice: - _create_message = f"{_create_message} {_local_notice}" - # Gateway liveness surfacing (#87033): the builtin scheduler's - # ticker lives in the gateway process, so a job created with no - # gateway running is stored but will never fire. Tell the model - # here — the CLI already warns, but the agent path saw only a - # clean success and confidently told the user it was scheduled. - _result = { - "success": True, - "job_id": job["id"], - "name": job["name"], - "skill": job.get("skill"), - "skills": job.get("skills", []), - "schedule": job["schedule_display"], - "repeat": _repeat_display(job), - "deliver": job.get("deliver", "local"), - "next_run_at": job["next_run_at"], - "job": _format_job(job), - "message": _create_message, - **_gateway_liveness_notice(), - } - # Mode-specific guidance rides in the create response (once, when - # relevant) instead of in the schema (every API call). See - # _mode_guidance_notes. - _notes = _mode_guidance_notes(job, _normalize_deliver_param(deliver)) - if _notes: - _result["guidance"] = _notes - return json.dumps(_result, indent=2) - - if normalized == "list": - jobs = [_format_job(job) for job in list_jobs(include_disabled=include_disabled)] - _result = {"success": True, "count": len(jobs), "jobs": jobs} - # Same silent-inert-job class as create (#87033): an agent - # inspecting existing jobs in a gateway-less environment must - # learn they are not firing, not just see a clean list. An empty - # list has nothing inert — stay quiet (and skip the probe). - if jobs: - _result.update(_gateway_liveness_notice(plural=True)) - return json.dumps(_result, indent=2) + handler = _JOBLESS_ACTIONS.get(normalized) + if handler is not None: + return handler(a) if not job_id: return tool_error(f"job_id is required for action '{normalized}'", success=False) @@ -1698,321 +999,33 @@ def cronjob( try: job = resolve_job_ref(job_id) except AmbiguousJobReference as exc: - return json.dumps( - { - "success": False, - "error": str(exc), - "matches": [ - { - "id": m["id"], - "name": m.get("name"), - "schedule": m.get("schedule_display"), - "next_run_at": m.get("next_run_at"), - } - for m in exc.matches - ], - }, - indent=2, - ) - if not job: - return json.dumps( - {"success": False, "error": f"Job with ID or name '{job_id}' not found. Use cronjob(action='list') to inspect jobs."}, - indent=2, - ) - # Resolve to canonical ID (supports name-based lookup) - job_id = job["id"] - - if normalized == "remove": - removed = remove_job(job_id) - if not removed: - return tool_error(f"Failed to remove job '{job_id}'", success=False) - _notify_provider_jobs_changed_safe() - return json.dumps( - { - "success": True, - "message": f"Cron job '{job['name']}' removed.", - "removed_job": { - "id": job_id, - "name": job["name"], - "schedule": job.get("schedule_display"), - }, - }, - indent=2, - ) - - if normalized == "pause": - updated = pause_job(job_id, reason=reason) - _notify_provider_jobs_changed_safe() - return json.dumps({"success": True, "job": _format_job(updated)}, indent=2) - - if normalized == "resume": - updated = resume_job(job_id) - _notify_provider_jobs_changed_safe() - return json.dumps({"success": True, "job": _format_job(updated)}, indent=2) - - if normalized in {"run", "run_now", "trigger"}: - # Per-run context (#57331, salvaged from #57342/@liuhao1024 and - # #57360/@ghedeselmabot): `prompt` on the run action is transient - # context appended to the stored prompt for THIS fire only, never - # persisted. It goes through the same strict injection scan as - # stored prompts before firing. - extra_prompt = prompt or None - if extra_prompt: - scan_error = _scan_cron_prompt(extra_prompt) - if scan_error: - return tool_error(scan_error, success=False) - # Execute the job immediately rather than only scheduling it for the - # next scheduler tick — a manual `run` should actually run, even when - # no gateway/ticker is active (the #41037 case). The claim (taken - # inside both paths below) advances next_run_at and blocks a - # concurrent tick from double-firing. - # - # Preferred path: dispatch the run to the background like - # delegate_task — the tool returns a handle immediately and the - # job's outcome re-enters the conversation as a completion event. - # A cron job is a full agent run (minutes to hours); executing it - # inline made the parent turn uninterruptible and serialized - # batches of manual runs (#80xxx — the "stuck Telegram session" - # incident). Falls back to inline execution when the session - # runtime can't receive detached completions. - bg = _try_dispatch_background_run( - job, session_id=session_id, extra_prompt=extra_prompt - ) - if bg is not None and bg.get("dispatched"): - _notify_provider_jobs_changed_safe() - result = _format_job(get_job(job_id) or {"id": job_id}) - result["executed"] = True - result["execution_mode"] = "background" - result["delegation_id"] = bg.get("delegation_id") - return json.dumps( + return _dumps({ + "success": False, + "error": str(exc), + "matches": [ { - "success": True, - "job": result, - "note": ( - "The job is running in the background. You and the " - "user can keep working; its outcome re-enters the " - "conversation as a new message when it finishes. " - "Do not wait or poll — just continue." - ), - }, - indent=2, - ) - # bg carries a terminal result (claim lost, or inline fallback - # after pool rejection); None means background delivery is - # unsupported here — run synchronously as before. - if bg is not None: - exec_result = bg - else: - # Relay-fronted manual run: a standalone process has no live - # relay adapter and no standalone sender, so forward to the - # running gateway (its live adapter owns that delivery). - forwarded = _forward_relay_fronted_run(job, extra_prompt=extra_prompt) - if forwarded is not None: - return forwarded - exec_result = _execute_job_now(job, extra_prompt=extra_prompt) - # A claimed direct run advances next_run_at and may race the - # external one-shot for the same occurrence. If Chronos loses that - # claim, its consumed fire cannot re-arm itself; reconcile from the - # winning direct path after the run has persisted its final state. - if exec_result.get("claimed", False): - _notify_provider_jobs_changed_safe() - # Re-read so the response reflects the post-run last_run_at/last_status. - result = _format_job(get_job(job_id) or {"id": job_id}) - result["executed"] = exec_result.get("claimed", False) - result["execution_success"] = exec_result.get("success", False) - if not exec_result.get("claimed", False): - result["execution_skipped"] = exec_result.get("error") or ( - "Already being fired by the scheduler; not run again." - ) - elif exec_result.get("error"): - result["execution_error"] = exec_result["error"] - return json.dumps({"success": True, "job": result}, indent=2) - - if normalized == "update": - updates: Dict[str, Any] = {} - if prompt is not None: - scan_error = _scan_cron_prompt(prompt) - if scan_error: - return tool_error(scan_error, success=False) - updates["prompt"] = prompt - if name is not None and name.strip(): - # Blank name is a no-op, not a clear. The `is not None` sentinel - # treats every supplied field as an explicit edit, and a model - # that re-sends the whole schema with type-default empties ("", [], 0) - # then wipes fields it never meant to touch. - updates["name"] = name - if deliver is not None: - bot_chat_error = _validate_bot_chat_deliver(_normalize_deliver_param(deliver)) - if bot_chat_error: - return tool_error(bot_chat_error, success=False) - updates["deliver"] = _resolve_cron_context_deliver( - _normalize_deliver_param(deliver) - ) - if failure_deliver is not None: - # '' clears the override (job falls back to deliver on - # failures); non-empty values share deliver's validation - # AND its cron-context origin resolution (a job created - # from inside a cron run must never store literal - # 'origin' — same rule as deliver). - _norm_fd = _normalize_deliver_param(failure_deliver) - if _norm_fd: - bot_chat_error = _validate_bot_chat_deliver(_norm_fd) - if bot_chat_error: - return tool_error(bot_chat_error, success=False) - _norm_fd = _resolve_cron_context_deliver(_norm_fd) - updates["failure_deliver"] = _norm_fd - if skills is not None or skill is not None: - canonical_skills = _canonical_skills(skill, skills) - updates["skills"] = canonical_skills - updates["skill"] = canonical_skills[0] if canonical_skills else None - if model is not None: - updates["model"] = _normalize_optional_job_value(model) - if provider is not None: - updates["provider"] = _normalize_optional_job_value(provider) - if base_url is not None: - updates["base_url"] = _normalize_optional_job_value(base_url, strip_trailing_slash=True) - if reasoning_effort is not None: - # CLI-only lane (see create above): update_job validates - # against the canonical grammar; empty string clears the pin. - updates["reasoning_effort"] = reasoning_effort - # Re-validate the EFFECTIVE provider/base_url on EVERY update, not - # only when this update supplies provider/base_url. A job persisted - # before this guard (or written directly to the jobs store) may - # already hold an unsafe named-provider + off-host base_url pair; - # if we only checked when the update touches those axes, editing any - # unrelated field (name, schedule, ...) would succeed and leave that - # exfil-capable pair active and schedulable (F8). The effective pair - # merges this update's normalized values over the stored job; an - # operator can still remediate in the same update by clearing - # base_url or pointing provider/base_url at a safe pair. - eff_provider = ( - updates["provider"] if "provider" in updates else job.get("provider") + "id": m["id"], + "name": m.get("name"), + "schedule": m.get("schedule_display"), + "next_run_at": m.get("next_run_at"), + } + for m in exc.matches + ], + }) + if not job: + return _dumps( + {"success": False, "error": f"Job with ID or name '{job_id}' not found. Use cronjob(action='list') to inspect jobs."}, ) - eff_base_url = ( - updates["base_url"] if "base_url" in updates else job.get("base_url") - ) - base_url_error = _validate_cron_base_url(eff_provider, eff_base_url) - if base_url_error: - return tool_error(base_url_error, success=False) - if script is not None: - # Pass empty string to clear an existing script - if script: - script_error = _validate_cron_script_path(script) - if script_error: - return tool_error(script_error, success=False) - updates["script"] = _normalize_optional_job_value(script) if script else None - if monitor_script is not None: - # Pass empty string to clear an existing monitor_script - if monitor_script: - monitor_error = _validate_cron_script_path(monitor_script) - if monitor_error: - return tool_error(monitor_error, success=False) - updates["monitor_script"] = ( - _normalize_optional_job_value(monitor_script) if monitor_script else None - ) - if monitor_url is not None: - # Pass empty string to clear an existing monitor_url - updates["monitor_url"] = ( - _normalize_optional_job_value(monitor_url) if monitor_url else None - ) - if monitor_script is not None or monitor_url is not None: - eff_mon_script = ( - updates["monitor_script"] if "monitor_script" in updates else job.get("monitor_script") - ) - eff_mon_url = ( - updates["monitor_url"] if "monitor_url" in updates else job.get("monitor_url") - ) - if eff_mon_script and eff_mon_url: - return tool_error( - "monitor_script and monitor_url are mutually exclusive — " - "clear one before setting the other.", - success=False, - ) - if context_from is not None or continuity is not None: - # Empty string / empty list clears the field; otherwise validate - # each referenced job exists before storing. Normalized to a list - # (or None) to match the shape stored by create_job(). - if context_from is None: - # continuity-only update: start from the job's stored refs. - existing = job.get("context_from") or [] - refs = [str(j).strip() for j in existing if str(j).strip()] - elif isinstance(context_from, str): - refs = [context_from.strip()] if context_from.strip() else [] - else: - refs = [str(j).strip() for j in context_from if str(j).strip()] - if continuity is not None: - refs = _apply_continuity(refs, continuity) or [] - if refs: - from cron.jobs import get_job as _get_job - for ref_id in refs: - # "self" resolves to the job's own id at run time. - if ref_id.lower() == "self": - continue - if not _get_job(ref_id): - return tool_error( - f"context_from job '{ref_id}' not found. " - "Use cronjob(action='list') to see available jobs.", - success=False, - ) - updates["context_from"] = refs or None - if enabled_toolsets is not None: - updates["enabled_toolsets"] = enabled_toolsets or None - if attach_to_session is not None: - updates["attach_to_session"] = bool(attach_to_session) - if workdir is not None: - # Empty string clears the field (restores old behaviour); - # otherwise pass raw — update_job() validates / normalizes. - updates["workdir"] = _normalize_optional_job_value(workdir) or None - if no_agent is not None: - # Toggling no_agent on/off at update time. If flipping to True, - # we need a script to already exist on the job (or be part of - # the same update) — otherwise the next tick would error out. - target_no_agent = bool(no_agent) - if target_no_agent: - effective_script = updates.get("script") if "script" in updates else job.get("script") - if not effective_script: - return tool_error( - "Cannot set no_agent=True on a job without a script. " - "Set `script` in the same update, or on the job first.", - success=False, - ) - updates["no_agent"] = target_no_agent - if repeat is not None: - # Coerce string forms ('forever'/'once'/'3') and 0/negative - # via the shared chokepoint — a bare `repeat <= 0` here - # raised TypeError for string repeats on the UPDATE path - # (create was fixed first; same class). - from cron.jobs import normalize_repeat_value - normalized_repeat = normalize_repeat_value(repeat) - repeat_state = dict(job.get("repeat") or {}) - repeat_state["times"] = normalized_repeat - updates["repeat"] = repeat_state - if schedule is not None: - parsed_schedule = parse_schedule(schedule) - updates["schedule"] = parsed_schedule - updates["schedule_display"] = parsed_schedule.get("display", schedule) - if job.get("state") != "paused": - updates["state"] = "scheduled" - updates["enabled"] = True - if not updates: - return tool_error("No updates provided.", success=False) - updated = update_job(job_id, updates) - _notify_provider_jobs_changed_safe() - _upd_result: Dict[str, Any] = {"success": True, "job": _format_job(updated)} - # An update can switch a job into monitor / no_agent mode or - # change its delivery — echo the same mode guidance as create. - _upd_notes = _mode_guidance_notes(updated, _normalize_deliver_param(deliver)) - if _upd_notes: - _upd_result["guidance"] = _upd_notes - return json.dumps(_upd_result, indent=2) - return tool_error(f"Unknown cron action '{action}'", success=False) + handler = _JOB_ACTIONS.get(normalized) + if handler is None: + return tool_error(f"Unknown cron action '{action}'", success=False) + return handler(job, a) except Exception as e: return tool_error(str(e), success=False) - CRONJOB_SCHEMA = { "name": "cronjob_manage", "description": """Manage scheduled cron jobs: action='create' schedules a job from a prompt and/or skills; 'list' inspects jobs; 'update'/'pause'/'resume'/'remove' manage one by job_id (always list first — never guess job IDs); 'run' fires a job immediately in the BACKGROUND (returns a handle at once, outcome re-enters the conversation when done — do not wait or poll; optional 'prompt' adds transient context for that fire only). @@ -2101,18 +1114,9 @@ Jobs run in a fresh session with no current-chat context, so prompts must be sel def check_cronjob_requirements() -> bool: - """ - Check if cronjob tools can be used. - - Available in interactive CLI mode and gateway/messaging platforms. - The cron system is internal (JSON file-based scheduler ticked by the gateway), - so no external crontab executable is required. - - Session env vars must hold an explicit truthy string (``1``, ``true``, - ``yes``, ``on``) — false-like values (``0``, ``false``, ``no``, ``off``) - leave the tool disabled. Uses the shared ``env_var_enabled`` helper so - every consumer of these flags agrees on the truthy set. - """ + """Available in interactive CLI mode and gateway/messaging platforms (the + scheduler is internal; no crontab needed). Flags must be explicitly truthy + via the shared ``env_var_enabled`` helper.""" from utils import env_var_enabled return ( @@ -2127,12 +1131,9 @@ from tools.registry import registry, tool_error def _cronjob_handler(args, **kw): - """Model-tool dispatch for ``cronjob``. - - Resolves the one model-facing ``monitor`` field into the stored - ``monitor_script``/``monitor_url`` pair (legacy field names still accepted - as aliases so older transcripts/replays keep working). - """ + """Model-tool dispatch for ``cronjob``: resolves the one model-facing + ``monitor`` field into the stored ``monitor_script``/``monitor_url`` pair + (legacy field names still accepted for older transcripts).""" _mon_script, _mon_url = _split_monitor_arg( args.get("monitor"), args.get("monitor_script"), args.get("monitor_url") ) diff --git a/tools/debug_helpers.py b/tools/debug_helpers.py index bb124b45eb..16a97dad42 100644 --- a/tools/debug_helpers.py +++ b/tools/debug_helpers.py @@ -1,25 +1,5 @@ -"""Shared debug session infrastructure for Hermes tools. - -Replaces the identical DEBUG_MODE / _log_debug_call / _save_debug_log / -get_debug_session_info boilerplate previously duplicated across web_tools, -vision_tools, and image_generation_tool. - -Usage in a tool module: - - from tools.debug_helpers import DebugSession - - _debug = DebugSession("web_tools", env_var="WEB_TOOLS_DEBUG") - - # Log a call (no-op when debug mode is off) - _debug.log_call("web_search", {"query": q, "results": len(r)}) - - # Save the debug log (no-op when debug mode is off) - _debug.save() - - # Expose debug info to external callers - def get_debug_session_info(): - return _debug.get_session_info() -""" +"""Shared per-tool debug session: records tool calls to a JSON log when a +tool-specific env var (e.g. WEB_TOOLS_DEBUG=true) is set; no-ops otherwise.""" import datetime import json @@ -34,11 +14,7 @@ logger = logging.getLogger(__name__) class DebugSession: - """Per-tool debug session that records tool calls to a JSON log file. - - Activated by a tool-specific environment variable (e.g. WEB_TOOLS_DEBUG=true). - When disabled, all methods are cheap no-ops. - """ + """Per-tool debug session that records tool calls to a JSON log file.""" def __init__(self, tool_name: str, *, env_var: str) -> None: self.tool_name = tool_name @@ -72,8 +48,7 @@ class DebugSession: if not self.enabled: return try: - filename = f"{self.tool_name}_debug_{self.session_id}.json" - filepath = self.log_dir / filename + filepath = self.log_dir / f"{self.tool_name}_debug_{self.session_id}.json" payload = { "session_id": self.session_id, "start_time": self._start_time, @@ -87,19 +62,3 @@ class DebugSession: logger.debug("%s debug log saved: %s", self.tool_name, filepath) except Exception as e: logger.error("Error saving %s debug log: %s", self.tool_name, e) - - def get_session_info(self) -> Dict[str, Any]: - """Return a summary dict suitable for returning from get_debug_session_info().""" - if not self.enabled: - return { - "enabled": False, - "session_id": None, - "log_path": None, - "total_calls": 0, - } - return { - "enabled": True, - "session_id": self.session_id, - "log_path": str(self.log_dir / f"{self.tool_name}_debug_{self.session_id}.json"), - "total_calls": len(self._calls), - } diff --git a/tools/delegation_live_log.py b/tools/delegation_live_log.py index d355740be6..9e17d8c34c 100644 --- a/tools/delegation_live_log.py +++ b/tools/delegation_live_log.py @@ -1,30 +1,19 @@ """Live, tail-able transcripts for delegated subagents. Every ``delegate_task`` dispatch creates one append-only, human-readable log -per child under:: +per child under ``/cache/delegation/live//task-.log``. +Files are pre-created with a header at dispatch (so ``tail -f`` attaches +immediately) and stream one line per child event; the paths are returned from +``delegate_task`` so the parent or user can watch a child work. - /cache/delegation/live//task-.log +``cache/delegation`` is mounted read-only into remote terminal backends +(``credential_files._CACHE_DIRS``), so the logs are readable from any backend — +and every line written here must therefore be credential-redacted. -The files are pre-created with a header at dispatch time (so ``tail -f`` -attaches immediately) and then stream one line per child event: assistant -text, thinking, tool calls, tool results, and lifecycle markers. The paths -are returned from ``delegate_task`` so the parent agent (or the user) can -watch a child work instead of waiting blind for the consolidated summary. - -Placement under ``cache/delegation`` is deliberate: that directory is -mounted read-only into remote terminal backends (Docker/Modal/SSH) via -``credential_files._CACHE_DIRS``, so the logs are readable from any backend. - -Design constraints: - -* **Never raise into the agent loop.** Every write is wrapped; the first - failure disables the writer and degrades to a debug log. -* **Survive child crashes.** Files are opened in append mode per write — - no long-lived handle to lose, every event is flushed when written. -* **Side-channel only.** Nothing here touches message content, so prompt - caching is unaffected. -* **No config knobs.** Retention is a module constant (7 days), pruned - opportunistically on each new dispatch. +Design constraints: never raise into the agent loop (first write failure +disables the writer); open in append mode per write (no handle to lose on a +child crash); side-channel only (prompt cache unaffected); no config knobs +(retention is a 7-day module constant pruned on each dispatch). """ from __future__ import annotations @@ -40,12 +29,10 @@ from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) -# Live transcript directories older than this are pruned on new dispatches. LIVE_RETENTION_DAYS = 7 -# Per-line truncation budgets (chars). The .log is a compact operational -# view, not the full-fidelity record — the child's SessionDB transcript and -# the summary spill files carry complete text. +# Per-line truncation budgets (chars). The .log is a compact operational view; +# the child's SessionDB transcript and summary spill files carry full text. _ASSISTANT_MAX = 600 _THINKING_MAX = 300 _ARGS_MAX = 220 @@ -53,10 +40,11 @@ _RESULT_MAX = 400 _KICKOFF_MAX = 500 # Stream deltas are buffered and flushed as one assistant line when another -# event type arrives (or on completion). Cap the buffer so a huge streamed -# reply can't hold memory hostage. +# event type arrives (or on completion); capped so a huge reply can't hold memory. _STREAM_BUFFER_FLUSH_CHARS = 4000 +_TIME_FMT = "%Y-%m-%d %H:%M:%S" + def live_transcript_root() -> Path: """Root directory for live transcripts (profile-safe, never ~/.hermes).""" @@ -72,32 +60,19 @@ def new_live_delegation_id() -> str: def _one_line(text: Any, limit: int) -> str: """Collapse to a single line and truncate with an elided-chars note.""" - s = str(text or "") - s = " ".join(s.split()) # collapse newlines/runs of whitespace + s = " ".join(str(text or "").split()) if len(s) > limit: - omitted = len(s) - limit - s = s[:limit] + f" …(+{omitted} chars)" + s = s[:limit] + f" …(+{len(s) - limit} chars)" return s def _redact(text: str) -> str: - """Mask credentials before anything reaches the transcript file. + """Mask credentials before anything reaches the sandbox-readable transcript. - These logs live under ``cache/delegation``, which ``delegate_tool`` mounts - READ-ONLY into remote terminal backends — so every line written here is - readable from inside the sandbox. The events rendered here carry exactly - the data that tends to hold secrets: tool args (a bearer header on a - curl), tool results (a ``.env`` dump, a provider error echoing the key - back) and streamed assistant text. Every other sink for that data already - routes through this same redactor — search results via - ``redact_sensitive_text``, terminal output via ``redact_terminal_output`` - — so a transcript that skipped it is the one place the operator's keys - land in plaintext. - - ``force=True``: this is a safety boundary, so it must redact even when the - global toggle is off. Withholds the line rather than emitting raw text if - the redactor is somehow unavailable — losing a debug line costs less than - writing a live credential into a sandbox-readable file. + Tool args, results and streamed text are exactly the data that carries + secrets. ``force=True`` because this is a safety boundary (redact even + when the global toggle is off); if the redactor is unavailable, withhold + the line rather than write a live credential into a mounted file. """ if not text: return text @@ -109,11 +84,15 @@ def _redact(text: str) -> str: return "[line withheld: redaction unavailable]" +def _dump_json(path: Path, payload: Dict[str, Any]) -> None: + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") + + class LiveTranscriptWriter: """Append-only human-readable event log for ONE subagent task. - All methods are best-effort: the first write failure flips ``_ok`` off - and subsequent calls become no-ops (debug-logged). Never raises. + Best-effort: the first write failure flips ``_ok`` off and later calls + become debug-logged no-ops. Never raises. """ def __init__(self, delegation_id: str, task_index: int, goal: str, @@ -125,17 +104,15 @@ class LiveTranscriptWriter: self._stream_buf: List[str] = [] self._stream_len = 0 try: - base = (root if root is not None else live_transcript_root()) - d = base / delegation_id + d = (root if root is not None else live_transcript_root()) / delegation_id d.mkdir(parents=True, exist_ok=True) self.path: Optional[Path] = d / f"task-{task_index}.log" header = [ "=== Hermes subagent live transcript ===", f"delegation: {delegation_id} task: {task_index}", - # Header bypasses event(), so redact here too — a goal string - # can carry a key the caller pasted into the task. + # Header bypasses event(), so redact here too. f"goal: {_redact(_one_line(goal, _KICKOFF_MAX))}", - f"started: {time.strftime('%Y-%m-%d %H:%M:%S')}", + f"started: {time.strftime(_TIME_FMT)}", "(append-only; streams while the subagent runs — tail -f me)", "=" * 40, ] @@ -148,21 +125,18 @@ class LiveTranscriptWriter: self._ok = False self.path = None - # ── low-level ──────────────────────────────────────────────────────── def event(self, role: str, text: str) -> None: - """Append one ``HH:MM:SS role ⟩ text`` line. Flushed per event.""" + """Append one ``HH:MM:SS role | text`` line. Flushed per event. + + Single choke point: every typed helper funnels through here, so one + redaction covers args, results, thinking and streamed text. + """ if not self._ok or self.path is None: return - # Single choke point: every typed helper funnels through here, so - # redacting once covers args, results, thinking and streamed text — - # and a helper added later can't bypass it. line = f"{time.strftime('%H:%M:%S')} {role:<9}| {_redact(text)}\n" try: - with self._lock: - # Append mode per write: no held handle, survives child crash, - # and the close() acts as the flush. - with open(self.path, "a", encoding="utf-8") as fh: - fh.write(line) + with self._lock, open(self.path, "a", encoding="utf-8") as fh: + fh.write(line) except Exception as exc: self._ok = False logger.debug("Live transcript write failed (%s): %s", self.path, exc) @@ -180,8 +154,7 @@ class LiveTranscriptWriter: def tool_start(self, name: str, args_preview: Any = None) -> None: self.flush_stream() - args = _one_line(args_preview, _ARGS_MAX) - self.event("tool", f"-> {name or '?'}({args})") + self.event("tool", f"-> {name or '?'}({_one_line(args_preview, _ARGS_MAX)})") def tool_result(self, name: str, result: Any = None, duration: Any = None, is_error: bool = False) -> None: @@ -219,53 +192,66 @@ class LiveTranscriptWriter: self.assistant_text(text) # ── event demux (the tool_progress_callback surface) ───────────────── + def _on_tool_started(self, tool_name, preview, args, kwargs): + self.tool_start(str(tool_name or ""), preview if preview else args) + + def _on_tool_completed(self, tool_name, preview, args, kwargs): + self.tool_result( + str(tool_name or ""), + result=kwargs.get("result"), + duration=kwargs.get("duration"), + is_error=bool(kwargs.get("is_error")), + ) + + def _on_thinking(self, tool_name, preview, args, kwargs): + # Fired as cb("_thinking", ) — text rides in the tool_name slot. + self.thinking(str(tool_name or preview or "")) + + def _on_reasoning(self, tool_name, preview, args, kwargs): + # cb("reasoning.available", "_thinking", , None) + self.thinking(str(preview or "")) + + def _on_text(self, tool_name, preview, args, kwargs): + self.add_stream_delta(str(preview or "")) + + def _on_start(self, tool_name, preview, args, kwargs): + self.event("start", _one_line(preview, _KICKOFF_MAX)) + + def _on_complete(self, tool_name, preview, args, kwargs): + self.flush_stream() + parts = [f"status={kwargs.get('status', '?')}"] + dur = kwargs.get("duration_seconds") + if dur is not None: + parts.append(f"duration={dur}s") + summary = kwargs.get("summary") or preview + if summary: + parts.append(f"summary: {_one_line(summary, _RESULT_MAX)}") + self.marker(" ".join(parts)) + + _OBSERVERS = { + "tool.started": _on_tool_started, + "tool.completed": _on_tool_completed, + "_thinking": _on_thinking, + "reasoning.available": _on_reasoning, + "subagent.text": _on_text, + "subagent.start": _on_start, + "subagent.complete": _on_complete, + } + def observe(self, event_type: Any, tool_name: Any = None, preview: Any = None, args: Any = None, **kwargs: Any) -> None: """Map a child tool_progress_callback event onto transcript lines. Mirrors the shapes emitted by agent/tool_executor.py, - agent/conversation_loop.py, and tools/delegate_tool._run_single_child. + agent/conversation_loop.py and delegate_tool._run_single_child. Unknown events are ignored. Never raises (event() swallows I/O). """ - et = str(event_type or "") - if et == "tool.started": - self.tool_start(str(tool_name or ""), preview if preview else args) - elif et == "tool.completed": - self.tool_result( - str(tool_name or ""), - result=kwargs.get("result"), - duration=kwargs.get("duration"), - is_error=bool(kwargs.get("is_error")), - ) - elif et == "_thinking": - # Fired as cb("_thinking", ) — the text rides in the - # tool_name positional slot (see conversation_loop.py). - self.thinking(str(tool_name or preview or "")) - elif et == "reasoning.available": - # cb("reasoning.available", "_thinking", , None) - self.thinking(str(preview or "")) - elif et == "subagent.text": - self.add_stream_delta(str(preview or "")) - elif et == "subagent.start": - self.event("start", _one_line(preview, _KICKOFF_MAX)) - elif et == "subagent.complete": - self.flush_stream() - status = kwargs.get("status", "?") - dur = kwargs.get("duration_seconds") - parts = [f"status={status}"] - if dur is not None: - parts.append(f"duration={dur}s") - summary = kwargs.get("summary") or preview - if summary: - parts.append(f"summary: {_one_line(summary, _RESULT_MAX)}") - self.marker(" ".join(parts)) + handler = self._OBSERVERS.get(str(event_type or "")) + if handler is not None: + handler(self, tool_name, preview, args, kwargs) def finalize(self, entry: Dict[str, Any]) -> None: - """Terminal marker from the aggregated result entry. - - Adds exit-reason detail the subagent.complete event doesn't carry - (budget exhaustion via exit_reason=max_iterations, errors, etc.). - """ + """Terminal marker with exit-reason detail subagent.complete lacks.""" parts = [f"end status={entry.get('status', '?')}"] exit_reason = entry.get("exit_reason") if exit_reason: @@ -280,10 +266,9 @@ class LiveTranscriptWriter: def wrap_progress_callback(inner_cb, writer: LiveTranscriptWriter): """Wrap a child's tool_progress_callback so events also land in the log. - ``inner_cb`` may be None (no parent display) — the wrapper still records. - Writer failures never propagate; inner callback behavior is unchanged - (its own exceptions are handled by callers exactly as before). - Preserves the ``_flush`` attribute contract used by _run_single_child. + ``inner_cb`` may be None; writer failures never propagate and the inner + callback's behavior is unchanged. Preserves the ``_flush`` attribute + contract used by _run_single_child. """ def _cb(event_type, tool_name=None, preview=None, args=None, **kwargs): @@ -318,9 +303,9 @@ def create_live_transcripts( ) -> tuple[Optional[str], List[Optional[LiveTranscriptWriter]], List[str]]: """Create one pre-headered writer per task + a manifest.json. - Returns ``(delegation_id, writers, paths)``. On any top-level failure - returns ``(None, [None]*n, [])`` so delegation proceeds untouched. - Also opportunistically prunes stale live dirs (retention). + Returns ``(delegation_id, writers, paths)``; on any top-level failure + ``(None, [None]*n, [])`` so delegation proceeds untouched. Also prunes + stale live dirs opportunistically. """ n = len(task_list) try: @@ -358,18 +343,15 @@ def _write_manifest(delegation_id: str, task_list: List[Dict[str, Any]], try: manifest = { "delegation_id": delegation_id, - "started": time.strftime("%Y-%m-%d %H:%M:%S"), + "started": time.strftime(_TIME_FMT), "task_count": len(task_list), "model": model, "provider": provider, "tasks": [ { "index": i, - # manifest.json sits in the same mounted - # cache/delegation/live// directory as the .log files, - # so it needs the same treatment — redacting the header - # while serialising the goal verbatim here would leave the - # credential exposed one file over. + # Same mounted directory as the .log files, so the goal + # needs the same redaction here. "goal": _redact(str(t.get("goal", ""))[:500]), "log": paths[i] if i < len(paths) else None, "status": "running", @@ -377,9 +359,7 @@ def _write_manifest(delegation_id: str, task_list: List[Dict[str, Any]], for i, t in enumerate(task_list) ], } - _manifest_path(delegation_id).write_text( - json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8" - ) + _dump_json(_manifest_path(delegation_id), manifest) except Exception as exc: logger.debug("Live transcript manifest write failed: %s", exc) @@ -399,18 +379,14 @@ def update_manifest_statuses(delegation_id: Optional[str], task["status"] = r.get("status", task.get("status")) if r.get("exit_reason"): task["exit_reason"] = r["exit_reason"] - manifest["completed"] = time.strftime("%Y-%m-%d %H:%M:%S") - mp.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), - encoding="utf-8") + manifest["completed"] = time.strftime(_TIME_FMT) + _dump_json(mp, manifest) except Exception as exc: logger.debug("Live transcript manifest update failed: %s", exc) def prune_stale_live_dirs(max_age_days: int = LIVE_RETENTION_DAYS) -> int: - """Remove live/ dirs older than the retention window. - - Returns how many were removed. Fully best-effort. - """ + """Remove live/ dirs older than the retention window. Best-effort.""" removed = 0 try: root = live_transcript_root() diff --git a/tools/desktop_ui.py b/tools/desktop_ui.py index 7179e3ea16..4ec636aa7d 100644 --- a/tools/desktop_ui.py +++ b/tools/desktop_ui.py @@ -1,18 +1,16 @@ #!/usr/bin/env python3 """Bridge desktop-only tools to Hermes-desktop renderer events. -The preview pane, pane focus, and friends live in the desktop renderer, so -desktop-gated tools reach them through an emitter the desktop ``tui_gateway`` -installs at session start via :func:`set_emitter`. Everywhere else it stays -``None`` and the tools report "desktop only". Routing keys off -``HERMES_UI_SESSION_ID`` so the event lands on the window that owns the turn -(``_emit``/``write_json`` is ``_stdout_lock``-guarded, so emitting from the -tool's thread is safe). +The desktop ``tui_gateway`` installs an emitter via :func:`set_emitter`; elsewhere it +stays ``None`` and tools report "desktop only". Routing keys off ``HERMES_UI_SESSION_ID`` +so the event lands on the window that owns the turn (the sink is lock-guarded). """ +import json from typing import Callable, Optional from gateway.session_context import get_session_env +from tools.registry import tool_error # (sid, event, payload) sink, installed by the desktop gateway. _emit: Optional[Callable[[str, str, dict], None]] = None @@ -32,15 +30,10 @@ def available() -> bool: def user_enabled(setting: str, default: bool) -> bool: """Read one of the desktop's Appearance switches from ``display.``. - The renderer owns these toggles and mirrors them onto the CONNECTED - gateway's config (``config.set``), so this reads the user's real answer - whether that gateway is local, SSH, URL, or cloud — where an env var would - only ever describe the process. Tool ``check_fn``s call it to withdraw - themselves from the schema when the user has switched the feature off: - Hermes should not be told about a surface it isn't allowed to use. - - An unreadable config falls back to ``default``, which is how a feature that - ships on stays on rather than disappearing on a transient read error. + The renderer mirrors these toggles onto the CONNECTED gateway's config, so this + reads the user's real answer for local/SSH/URL/cloud gateways alike (an env var + would only describe the process). ``check_fn``s use it to withdraw a tool from the + schema when the user switched the feature off. Unreadable config -> ``default``. """ try: from hermes_cli.config import load_config_readonly @@ -62,3 +55,26 @@ def emit(event: str, payload: dict) -> bool: return False fn(get_session_env("HERMES_UI_SESSION_ID", ""), event, payload) return True + + +def emit_or_error(event: str, payload: dict, fail_prefix: str, desktop_only: str, result: dict) -> str: + """Emit ``event``; return ``tool_error`` text on failure, else ``result`` as JSON. + + ``fail_prefix`` is prepended to the exception text; ``desktop_only`` is the error + when no emitter is wired. Looked up as ``desktop_ui.emit`` so tests can patch it. + """ + try: + ok = emit(event, payload) + except Exception as exc: + return tool_error(f"{fail_prefix}{exc}") + if not ok: + return tool_error(desktop_only) + return json.dumps(result, ensure_ascii=False) + + +def passthrough_json(raw) -> str: + """Desktop answers with a JSON object; pass it through, else wrap the raw text.""" + try: + return json.dumps(json.loads(raw), ensure_ascii=False) + except (TypeError, ValueError): + return json.dumps({"text": str(raw)}, ensure_ascii=False) diff --git a/tools/discord_tool.py b/tools/discord_tool.py index db979a21d8..e20cd06d47 100644 --- a/tools/discord_tool.py +++ b/tools/discord_tool.py @@ -1,44 +1,28 @@ -"""Discord server introspection and management tool. +"""Discord server introspection and management tool (REST API + bot token). -Provides the agent with the ability to interact with Discord servers -when running on the Discord gateway. Uses Discord REST API directly -with the bot token — no dependency on the gateway adapter's client. - -Only included in the hermes-discord toolset, so it has zero cost -for users on other platforms. - -The schema exposed to the model is filtered by two gates: - -1. Privileged intents detected from GET /applications/@me at schema - build time. Actions that require an intent the bot doesn't have - (search_members / member_info → GUILD_MEMBERS intent) are hidden. - fetch_messages is kept regardless of MESSAGE_CONTENT intent, but - its description is annotated when the intent is missing. - -2. User config allowlist at ``discord.server_actions``. If the user - sets a comma-separated list (or YAML list) of action names, only - those appear in the schema. Empty/unset means all intent-available - actions are exposed. - -Per-guild permissions (MANAGE_ROLES etc.) are NOT pre-checked — Discord -returns a 403 at call time and :func:`_enrich_403` maps it to -actionable guidance the model can relay to the user. +Only in the hermes-discord toolset. The model-visible schema is filtered by two +gates: (1) privileged intents from GET /applications/@me — actions needing an +intent the bot lacks (search_members / member_info → GUILD_MEMBERS) are hidden, +and fetch_messages/list_pins are annotated when MESSAGE_CONTENT is missing; +(2) the ``discord.server_actions`` config allowlist (comma string or YAML list; +empty/unset = all). Per-guild permissions are NOT pre-checked — a call-time 403 +is mapped to actionable guidance by :func:`_enrich_403`. """ +import hashlib import json import logging import threading +import time import urllib.error import urllib.parse import urllib.request -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple from agent.secret_scope import get_secret from tools.registry import registry, tool_error -if TYPE_CHECKING: - from pathlib import Path - logger = logging.getLogger(__name__) DISCORD_API_BASE = "https://discord.com/api/v10" @@ -52,10 +36,6 @@ _FLAG_GATEWAY_GUILD_MEMBERS_LIMITED = 1 << 15 _FLAG_GATEWAY_MESSAGE_CONTENT = 1 << 18 _FLAG_GATEWAY_MESSAGE_CONTENT_LIMITED = 1 << 19 -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - class DiscordAPIError(Exception): """Raised when a Discord API call fails.""" def __init__(self, status: int, body: str): @@ -88,14 +68,9 @@ def _discord_request( url = f"{DISCORD_API_BASE}{path}" if params: url += "?" + urllib.parse.urlencode(params) - - data = None - if body is not None: - data = json.dumps(body).encode("utf-8") - req = urllib.request.Request( url, - data=data, + data=None if body is None else json.dumps(body).encode("utf-8"), method=method, headers={ "Authorization": f"Bot {token}", @@ -117,12 +92,9 @@ def _discord_request( except urllib.error.HTTPError as e: error_body = "" try: - raw_error_body = _read_limited_response_body( - e, - _DISCORD_ERROR_BODY_MAX_BYTES, - label="error body", - ) - error_body = raw_error_body.decode("utf-8", errors="replace") + error_body = _read_limited_response_body( + e, _DISCORD_ERROR_BODY_MAX_BYTES, label="error body" + ).decode("utf-8", errors="replace") except DiscordAPIError as too_large: error_body = too_large.body except Exception: @@ -130,10 +102,6 @@ def _discord_request( raise DiscordAPIError(e.code, error_body) from e -# --------------------------------------------------------------------------- -# Channel type mapping -# --------------------------------------------------------------------------- - _CHANNEL_TYPE_NAMES = { 0: "text", 2: "voice", @@ -152,27 +120,26 @@ def _channel_type_name(type_id: int) -> str: return _CHANNEL_TYPE_NAMES.get(type_id, f"unknown({type_id})") -# --------------------------------------------------------------------------- -# Capability detection (application intents) -# --------------------------------------------------------------------------- +# ── capability detection (application intents) ────────────────────────────── -# Module-level cache so the app/me endpoint is hit at most once per process. +# Per-token in-process cache: the app/me endpoint is hit at most once per process. _capability_cache: Dict[str, Dict[str, Any]] = {} -# Disk-cache TTL for detected capabilities. Privileged intents change only -# when the user flips them in the Discord Developer Portal, so 24h staleness -# is harmless — and a stale value only affects which actions appear in the -# schema (a hidden action re-appears on the next refresh; an exposed action -# the bot lost fails at call time with an enriched 403). +# Privileged intents change only when the user flips them in the Developer +# Portal, so 24h disk staleness is harmless: a hidden action re-appears on the +# next refresh; an exposed action the bot lost fails at call time with an +# enriched 403. _CAPABILITY_DISK_TTL_SECONDS = 24 * 3600 -# One background detection per process at most. +# One background detection per (process, token) at most. _capability_bg_started: set = set() _capability_bg_lock = threading.Lock() +# Permissive default: all actions exposed, call-time 403s mapped to guidance. +_PERMISSIVE_CAPS = {"has_members_intent": True, "has_message_content": True, "detected": False} -def _capability_disk_cache_path() -> "Path": +def _capability_disk_cache_path() -> Path: from hermes_constants import get_hermes_home return get_hermes_home() / "cache" / "discord_capabilities.json" @@ -180,15 +147,11 @@ def _capability_disk_cache_path() -> "Path": def _token_cache_key(token: str) -> str: """Stable non-reversible cache key for a bot token.""" - import hashlib - return hashlib.sha256(token.encode("utf-8")).hexdigest()[:16] def _load_caps_from_disk(token: str) -> Optional[Dict[str, Any]]: """Return fresh disk-cached capabilities for *token*, or None.""" - import time - try: path = _capability_disk_cache_path() with path.open("r", encoding="utf-8") as f: @@ -207,8 +170,6 @@ def _load_caps_from_disk(token: str) -> Optional[Dict[str, Any]]: def _save_caps_to_disk(token: str, caps: Dict[str, Any]) -> None: - import time - try: path = _capability_disk_cache_path() path.parent.mkdir(parents=True, exist_ok=True) @@ -231,19 +192,10 @@ def _save_caps_to_disk(token: str, caps: Dict[str, Any]) -> None: def _detect_capabilities_nonblocking(token: str) -> Dict[str, Any]: """Non-blocking capability lookup for schema builds. - Resolution order: - 1. In-process memory cache (populated by a previous sync/bg detection). - 2. Fresh disk cache (populated by a previous process). - 3. Permissive default + fire-and-forget background detection that - populates both caches for the next schema build / process. - - Rationale: ``_detect_capabilities`` makes a blocking HTTPS call to - discord.com (measured ~2s, up to 5s on the timeout) and used to run - inside ``get_tool_definitions`` → ``AIAgent.__init__`` — i.e. on the - critical path of the FIRST TOKEN of every cold process for any user - with DISCORD_BOT_TOKEN set, on every platform. The permissive default - mirrors the existing detection-failure fallback: all actions exposed, - call-time 403s mapped to guidance by ``_enrich_403``. + Order: in-process cache → fresh disk cache → permissive default plus a + fire-and-forget background detection that fills the disk cache for the + NEXT process. The blocking HTTPS call (~2-5s) used to sit on the first-token + critical path of every cold process with DISCORD_BOT_TOKEN set. """ cached = _capability_cache.get(token) if cached is not None: @@ -254,15 +206,10 @@ def _detect_capabilities_nonblocking(token: str) -> Dict[str, Any]: _capability_cache[token] = disk return disk - # Cold start — pin the permissive default for THIS process (schema - # stability: tool schemas must not change between agent inits within a - # live process, or the per-conversation prompt cache breaks) and detect - # in the background for the NEXT process via the disk cache. - caps_default = { - "has_members_intent": True, - "has_message_content": True, - "detected": False, - } + # Cold start — pin the permissive default for THIS process: schemas must not + # change between agent inits within a live process or the per-conversation + # prompt cache breaks. + caps_default = dict(_PERMISSIVE_CAPS) _capability_cache[token] = caps_default with _capability_bg_lock: @@ -285,15 +232,10 @@ def _detect_capabilities_nonblocking(token: str) -> Dict[str, Any]: def _fetch_capabilities(token: str) -> Dict[str, Any]: - """Fetch capabilities from GET /applications/@me. Pure network fetch — - does NOT read or write the in-process cache (background detection must - not mutate schemas mid-process).""" - caps: Dict[str, Any] = { - "has_members_intent": True, - "has_message_content": True, - "detected": False, - } - + """Fetch capabilities from GET /applications/@me. Pure network fetch — never + touches the in-process cache (background detection must not mutate schemas + mid-process). Detection failure is permissive.""" + caps: Dict[str, Any] = dict(_PERMISSIVE_CAPS) try: app = _discord_request("GET", "/applications/@me", token, timeout=5) flags = int(app.get("flags", 0) or 0) @@ -313,21 +255,10 @@ def _fetch_capabilities(token: str) -> Dict[str, Any]: def _detect_capabilities(token: str, *, force: bool = False) -> Dict[str, Any]: - """Detect the bot's app-wide capabilities via GET /applications/@me. - - Returns a dict with keys: - - - ``has_members_intent``: GUILD_MEMBERS intent is enabled - - ``has_message_content``: MESSAGE_CONTENT intent is enabled - - ``detected``: detection succeeded (False means exposing everything - and letting runtime errors handle it) - - Cached in a module-global. Pass ``force=True`` to re-fetch. - """ - global _capability_cache + """Blocking detection via GET /applications/@me, cached per token (the + warm-up path; schema builds use the non-blocking variant). ``force`` re-fetches.""" if token in _capability_cache and not force: return _capability_cache[token] - caps = _fetch_capabilities(token) _capability_cache[token] = caps return caps @@ -341,23 +272,55 @@ def _reset_capability_cache() -> None: _capability_bg_started = set() -# --------------------------------------------------------------------------- -# Action implementations -# --------------------------------------------------------------------------- +# ── action implementations ─────────────────────────────────────────────────── + + +def _listing(key: str, items: List[Dict[str, Any]]) -> str: + return json.dumps({key: items, "count": len(items)}) + + +def _ok(message: str) -> str: + return json.dumps({"success": True, "message": message}) + + +def _member_summary(m: Dict[str, Any], *, full: bool) -> Dict[str, Any]: + user = m.get("user", {}) + out = { + "user_id": user.get("id"), + "username": user.get("username"), + "display_name": user.get("global_name"), + "nickname": m.get("nick"), + } + if full: + out["avatar"] = user.get("avatar") + out["bot"] = user.get("bot", False) + out["roles"] = m.get("roles", []) + if full: + out["joined_at"] = m.get("joined_at") + out["premium_since"] = m.get("premium_since") + return out + + +def _int_or(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + def _list_guilds(token: str, **_kwargs: Any) -> str: """List all guilds the bot is a member of.""" guilds = _discord_request("GET", "/users/@me/guilds", token) - result = [] - for g in guilds: - result.append({ + return _listing("guilds", [ + { "id": g["id"], "name": g["name"], "icon": g.get("icon"), "owner": g.get("owner", False), "permissions": g.get("permissions"), - }) - return json.dumps({"guilds": result, "count": len(result)}) + } + for g in guilds + ]) def _server_info(token: str, guild_id: str, **_kwargs: Any) -> str: @@ -381,22 +344,12 @@ def _server_info(token: str, guild_id: str, **_kwargs: Any) -> str: def _list_channels(token: str, guild_id: str, **_kwargs: Any) -> str: """List all channels in a guild, organized by category.""" channels = _discord_request("GET", f"/guilds/{guild_id}/channels", token) - - # Organize: categories first, then channels under each - categories: Dict[Optional[str], Dict[str, Any]] = {} + categories: Dict[Optional[str], Dict[str, Any]] = { + ch["id"]: {"id": ch["id"], "name": ch["name"], "position": ch.get("position", 0), "channels": []} + for ch in channels + if ch["type"] == 4 # category + } uncategorized: List[Dict[str, Any]] = [] - - # First pass: collect categories - for ch in channels: - if ch["type"] == 4: # category - categories[ch["id"]] = { - "id": ch["id"], - "name": ch["name"], - "position": ch.get("position", 0), - "channels": [], - } - - # Second pass: assign channels to categories for ch in channels: if ch["type"] == 4: continue @@ -414,7 +367,6 @@ def _list_channels(token: str, guild_id: str, **_kwargs: Any) -> str: else: uncategorized.append(entry) - # Sort sorted_cats = sorted(categories.values(), key=lambda c: c["position"]) for cat in sorted_cats: cat["channels"].sort(key=lambda c: c["position"]) @@ -423,12 +375,10 @@ def _list_channels(token: str, guild_id: str, **_kwargs: Any) -> str: result: List[Dict[str, Any]] = [] if uncategorized: result.append({"category": None, "channels": uncategorized}) - for cat in sorted_cats: - result.append({ - "category": {"id": cat["id"], "name": cat["name"]}, - "channels": cat["channels"], - }) - + result.extend( + {"category": {"id": cat["id"], "name": cat["name"]}, "channels": cat["channels"]} + for cat in sorted_cats + ) total = sum(len(group["channels"]) for group in result) return json.dumps({"channel_groups": result, "total_channels": total}) @@ -453,9 +403,8 @@ def _channel_info(token: str, channel_id: str, **_kwargs: Any) -> str: def _list_roles(token: str, guild_id: str, **_kwargs: Any) -> str: """List all roles in a guild.""" roles = _discord_request("GET", f"/guilds/{guild_id}/roles", token) - result = [] - for r in sorted(roles, key=lambda r: r.get("position", 0), reverse=True): - result.append({ + return _listing("roles", [ + { "id": r["id"], "name": r["name"], "color": f"#{r.get('color', 0):06x}" if r.get("color") else None, @@ -464,47 +413,22 @@ def _list_roles(token: str, guild_id: str, **_kwargs: Any) -> str: "managed": r.get("managed", False), "member_count": r.get("member_count"), "hoist": r.get("hoist", False), - }) - return json.dumps({"roles": result, "count": len(result)}) + } + for r in sorted(roles, key=lambda r: r.get("position", 0), reverse=True) + ]) def _member_info(token: str, guild_id: str, user_id: str, **_kwargs: Any) -> str: """Get info about a specific guild member.""" m = _discord_request("GET", f"/guilds/{guild_id}/members/{user_id}", token) - user = m.get("user", {}) - return json.dumps({ - "user_id": user.get("id"), - "username": user.get("username"), - "display_name": user.get("global_name"), - "nickname": m.get("nick"), - "avatar": user.get("avatar"), - "bot": user.get("bot", False), - "roles": m.get("roles", []), - "joined_at": m.get("joined_at"), - "premium_since": m.get("premium_since"), - }) + return json.dumps(_member_summary(m, full=True)) def _search_members(token: str, guild_id: str, query: str, limit: int = 20, **_kwargs: Any) -> str: """Search for guild members by name.""" - try: - limit = int(limit) - except (TypeError, ValueError): - limit = 20 - params = {"query": query, "limit": str(min(limit, 100))} + params = {"query": query, "limit": str(min(_int_or(limit, 20), 100))} members = _discord_request("GET", f"/guilds/{guild_id}/members/search", token, params=params) - result = [] - for m in members: - user = m.get("user", {}) - result.append({ - "user_id": user.get("id"), - "username": user.get("username"), - "display_name": user.get("global_name"), - "nickname": m.get("nick"), - "bot": user.get("bot", False), - "roles": m.get("roles", []), - }) - return json.dumps({"members": result, "count": len(result)}) + return _listing("members", [_member_summary(m, full=False) for m in members]) def _fetch_messages( @@ -513,11 +437,7 @@ def _fetch_messages( **_kwargs: Any, ) -> str: """Fetch recent messages from a channel.""" - try: - limit = int(limit) - except (TypeError, ValueError): - limit = 50 - params: Dict[str, str] = {"limit": str(min(limit, 100))} + params: Dict[str, str] = {"limit": str(min(_int_or(limit, 50), 100))} if before: params["before"] = before if after: @@ -547,40 +467,36 @@ def _fetch_messages( ] if msg.get("reactions") else [], "pinned": msg.get("pinned", False), }) - return json.dumps({"messages": result, "count": len(result)}) + return _listing("messages", result) def _list_pins(token: str, channel_id: str, **_kwargs: Any) -> str: - """List pinned messages in a channel.""" + """List pinned messages in a channel (content truncated for overview).""" messages = _discord_request("GET", f"/channels/{channel_id}/pins", token) - result = [] - for msg in messages: - author = msg.get("author", {}) - result.append({ + return _listing("pinned_messages", [ + { "id": msg["id"], - "content": msg.get("content", "")[:200], # Truncate for overview - "author": author.get("username"), + "content": msg.get("content", "")[:200], + "author": msg.get("author", {}).get("username"), "timestamp": msg.get("timestamp"), - }) - return json.dumps({"pinned_messages": result, "count": len(result)}) + } + for msg in messages + ]) def _pin_message(token: str, channel_id: str, message_id: str, **_kwargs: Any) -> str: - """Pin a message in a channel.""" _discord_request("PUT", f"/channels/{channel_id}/pins/{message_id}", token) - return json.dumps({"success": True, "message": f"Message {message_id} pinned."}) + return _ok(f"Message {message_id} pinned.") def _unpin_message(token: str, channel_id: str, message_id: str, **_kwargs: Any) -> str: - """Unpin a message from a channel.""" _discord_request("DELETE", f"/channels/{channel_id}/pins/{message_id}", token) - return json.dumps({"success": True, "message": f"Message {message_id} unpinned."}) + return _ok(f"Message {message_id} unpinned.") def _delete_message(token: str, channel_id: str, message_id: str, **_kwargs: Any) -> str: - """Delete a message from a channel or thread.""" _discord_request("DELETE", f"/channels/{channel_id}/messages/{message_id}", token) - return json.dumps({"success": True, "message": f"Message {message_id} deleted."}) + return _ok(f"Message {message_id} deleted.") def _create_thread( @@ -589,45 +505,28 @@ def _create_thread( auto_archive_duration: int = 1440, **_kwargs: Any, ) -> str: - """Create a thread in a channel.""" + """Create a thread — anchored to ``message_id`` when given, else standalone public.""" + body: Dict[str, Any] = {"name": name, "auto_archive_duration": auto_archive_duration} if message_id: - # Create thread from an existing message path = f"/channels/{channel_id}/messages/{message_id}/threads" - body: Dict[str, Any] = { - "name": name, - "auto_archive_duration": auto_archive_duration, - } else: - # Create a standalone thread path = f"/channels/{channel_id}/threads" - body = { - "name": name, - "auto_archive_duration": auto_archive_duration, - "type": 11, # PUBLIC_THREAD - } + body["type"] = 11 # PUBLIC_THREAD thread = _discord_request("POST", path, token, body=body) - return json.dumps({ - "success": True, - "thread_id": thread["id"], - "name": thread.get("name"), - }) + return json.dumps({"success": True, "thread_id": thread["id"], "name": thread.get("name")}) def _add_role(token: str, guild_id: str, user_id: str, role_id: str, **_kwargs: Any) -> str: - """Add a role to a guild member.""" _discord_request("PUT", f"/guilds/{guild_id}/members/{user_id}/roles/{role_id}", token) - return json.dumps({"success": True, "message": f"Role {role_id} added to user {user_id}."}) + return _ok(f"Role {role_id} added to user {user_id}.") def _remove_role(token: str, guild_id: str, user_id: str, role_id: str, **_kwargs: Any) -> str: - """Remove a role from a guild member.""" _discord_request("DELETE", f"/guilds/{guild_id}/members/{user_id}/roles/{role_id}", token) - return json.dumps({"success": True, "message": f"Role {role_id} removed from user {user_id}."}) + return _ok(f"Role {role_id} removed from user {user_id}.") -# --------------------------------------------------------------------------- -# Action dispatch + metadata -# --------------------------------------------------------------------------- +# ── action dispatch + metadata ─────────────────────────────────────────────── _ACTIONS = { "list_guilds": _list_guilds, @@ -653,9 +552,8 @@ _ADMIN_ACTION_NAMES = frozenset(_ACTIONS.keys()) - _CORE_ACTION_NAMES _CORE_ACTIONS = {k: v for k, v in _ACTIONS.items() if k in _CORE_ACTION_NAMES} _ADMIN_ACTIONS = {k: v for k, v in _ACTIONS.items() if k in _ADMIN_ACTION_NAMES} -# Single-source-of-truth manifest: action → (signature, one-line description). -# Consumed by :func:`_build_schema` so the schema's top-level description -# always matches the registered action set. +# Single source of truth: action → (required-param signature, description). +# Drives the schema description AND runtime required-param validation. _ACTION_MANIFEST: List[Tuple[str, str, str]] = [ ("list_guilds", "()", "list servers the bot is in"), ("server_info", "(guild_id)", "server details + member counts"), @@ -677,38 +575,16 @@ _ACTION_MANIFEST: List[Tuple[str, str, str]] = [ # Actions that require the GUILD_MEMBERS privileged intent. _INTENT_GATED_MEMBERS = frozenset({"member_info", "search_members"}) -# Per-action required params for runtime validation. +# Per-action required params for runtime validation, parsed from the manifest. _REQUIRED_PARAMS: Dict[str, List[str]] = { - "server_info": ["guild_id"], - "list_channels": ["guild_id"], - "list_roles": ["guild_id"], - "member_info": ["guild_id", "user_id"], - "search_members": ["guild_id", "query"], - "channel_info": ["channel_id"], - "fetch_messages": ["channel_id"], - "list_pins": ["channel_id"], - "pin_message": ["channel_id", "message_id"], - "unpin_message": ["channel_id", "message_id"], - "delete_message": ["channel_id", "message_id"], - "create_thread": ["channel_id", "name"], - "add_role": ["guild_id", "user_id", "role_id"], - "remove_role": ["guild_id", "user_id", "role_id"], + name: [p.strip() for p in sig.strip("()").split(",") if p.strip()] + for name, sig, _desc in _ACTION_MANIFEST } -# --------------------------------------------------------------------------- -# Config-based action allowlist -# --------------------------------------------------------------------------- - def _load_allowed_actions_config() -> Optional[List[str]]: - """Read ``discord.server_actions`` from user config. - - Returns a list of allowed action names, or ``None`` if the user - hasn't restricted the set (default: all actions allowed). - - Accepts either a comma-separated string or a YAML list. - Unknown action names are dropped with a log warning. - """ + """``discord.server_actions`` allowlist (comma string or YAML list), or + ``None`` when unrestricted. Unknown names are dropped with a warning.""" try: from hermes_cli.config import load_config cfg = load_config() @@ -745,47 +621,69 @@ def _available_actions( caps: Dict[str, Any], allowlist: Optional[List[str]], ) -> List[str]: - """Compute the visible action list from intents + config allowlist. - - Preserves the canonical order from :data:`_ACTIONS`. - """ - actions: List[str] = [] - for name in _ACTIONS: - # Intent filter - if not caps.get("has_members_intent", True) and name in _INTENT_GATED_MEMBERS: - continue - # Config allowlist filter - if allowlist is not None and name not in allowlist: - continue - actions.append(name) - return actions + """Visible actions from intents + config allowlist, in :data:`_ACTIONS` order.""" + members_ok = caps.get("has_members_intent", True) + return [ + name + for name in _ACTIONS + if (members_ok or name not in _INTENT_GATED_MEMBERS) + and (allowlist is None or name in allowlist) + ] -# --------------------------------------------------------------------------- -# Schema construction -# --------------------------------------------------------------------------- +# ── schema construction ────────────────────────────────────────────────────── + +_TOOL_DESCRIPTIONS = { + "discord_admin": ( + "Manage a Discord server via the REST API.", + "Call list_guilds first to discover guild_ids, then list_channels for " + "channel_ids. Runtime errors will tell you if the bot lacks a specific " + "per-guild permission (e.g. MANAGE_ROLES for add_role).", + ), + "discord": ( + "Read and participate in a Discord server.", + "Use the channel_id from the current conversation context. " + "Use search_members to look up user IDs by name prefix.", + ), +} + +_SCHEMA_PROPERTIES: Dict[str, Any] = { + "guild_id": {"type": "string", "description": "Discord server (guild) ID."}, + "channel_id": {"type": "string", "description": "Discord channel ID."}, + "user_id": {"type": "string", "description": "Discord user ID."}, + "role_id": {"type": "string", "description": "Discord role ID."}, + "message_id": {"type": "string", "description": "Discord message ID."}, + "query": {"type": "string", "description": "Member name prefix to search for (search_members)."}, + "name": {"type": "string", "description": "New thread name (create_thread)."}, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "description": "Max results (default 50). Applies to fetch_messages, search_members.", + }, + "before": {"type": "string", "description": "Snowflake ID for reverse pagination (fetch_messages)."}, + "after": {"type": "string", "description": "Snowflake ID for forward pagination (fetch_messages)."}, + "auto_archive_duration": { + "type": "integer", + "enum": [60, 1440, 4320, 10080], + "description": "Thread archive duration in minutes (create_thread, default 1440).", + }, +} + def _build_schema( actions: List[str], caps: Optional[Dict[str, Any]] = None, tool_name: str = "discord", ) -> Optional[Dict[str, Any]]: - """Build the tool schema for the given filtered action list. - - Returns ``None`` when *actions* is empty — callers should drop the - tool from registration in that case. - """ + """Tool schema for the filtered action list; ``None`` when empty (drop the tool).""" caps = caps or {} if not actions: return None - # Action manifest lines (action-first, parameter-scoped). - manifest_lines = [ - f" {name}{sig} — {desc}" - for name, sig, desc in _ACTION_MANIFEST - if name in actions - ] - manifest_block = "\n".join(manifest_lines) + manifest_block = "\n".join( + f" {name}{sig} — {desc}" for name, sig, desc in _ACTION_MANIFEST if name in actions + ) content_note = "" affected_actions = {"fetch_messages", "list_pins"} & set(actions) @@ -799,88 +697,13 @@ def _build_schema( "Enable the intent in the Discord Developer Portal to see all content." ) - if tool_name == "discord_admin": - description = ( - "Manage a Discord server via the REST API.\n\n" - "Available actions:\n" - f"{manifest_block}\n\n" - "Call list_guilds first to discover guild_ids, then list_channels for " - "channel_ids. Runtime errors will tell you if the bot lacks a specific " - "per-guild permission (e.g. MANAGE_ROLES for add_role)." - f"{content_note}" - ) - else: - description = ( - "Read and participate in a Discord server.\n\n" - "Available actions:\n" - f"{manifest_block}\n\n" - "Use the channel_id from the current conversation context. " - "Use search_members to look up user IDs by name prefix." - f"{content_note}" - ) - - properties: Dict[str, Any] = { - "action": { - "type": "string", - "enum": actions, - }, - "guild_id": { - "type": "string", - "description": "Discord server (guild) ID.", - }, - "channel_id": { - "type": "string", - "description": "Discord channel ID.", - }, - "user_id": { - "type": "string", - "description": "Discord user ID.", - }, - "role_id": { - "type": "string", - "description": "Discord role ID.", - }, - "message_id": { - "type": "string", - "description": "Discord message ID.", - }, - "query": { - "type": "string", - "description": "Member name prefix to search for (search_members).", - }, - "name": { - "type": "string", - "description": "New thread name (create_thread).", - }, - "limit": { - "type": "integer", - "minimum": 1, - "maximum": 100, - "description": "Max results (default 50). Applies to fetch_messages, search_members.", - }, - "before": { - "type": "string", - "description": "Snowflake ID for reverse pagination (fetch_messages).", - }, - "after": { - "type": "string", - "description": "Snowflake ID for forward pagination (fetch_messages).", - }, - "auto_archive_duration": { - "type": "integer", - "enum": [60, 1440, 4320, 10080], - "description": "Thread archive duration in minutes (create_thread, default 1440).", - }, - } - + lead, guidance = _TOOL_DESCRIPTIONS["discord_admin" if tool_name == "discord_admin" else "discord"] + description = f"{lead}\n\nAvailable actions:\n{manifest_block}\n\n{guidance}{content_note}" + properties: Dict[str, Any] = {"action": {"type": "string", "enum": actions}, **_SCHEMA_PROPERTIES} return { "name": tool_name, "description": description, - "parameters": { - "type": "object", - "properties": properties, - "required": ["action"], - }, + "parameters": {"type": "object", "properties": properties, "required": ["action"]}, } @@ -908,14 +731,7 @@ def get_dynamic_schema_admin() -> Optional[Dict[str, Any]]: return _get_dynamic_schema(_ADMIN_ACTIONS, "discord_admin") -def get_dynamic_schema() -> Optional[Dict[str, Any]]: - """Backward-compat wrapper — returns core schema.""" - return get_dynamic_schema_core() - - -# --------------------------------------------------------------------------- -# 403 error enrichment -# --------------------------------------------------------------------------- +# ── 403 error enrichment ───────────────────────────────────────────────────── _ACTION_403_HINT = { "pin_message": ( @@ -965,41 +781,31 @@ def _enrich_403(action: str, body: str) -> str: """Return a user-friendly guidance string for a 403 on ``action``.""" hint = _ACTION_403_HINT.get(action) base = f"Discord API 403 (forbidden) on '{action}'." - if hint: - return f"{base} {hint} (Raw: {body})" - return f"{base} (Raw: {body})" + return f"{base} {hint} (Raw: {body})" if hint else f"{base} (Raw: {body})" -# --------------------------------------------------------------------------- -# Check function -# --------------------------------------------------------------------------- - def check_discord_tool_requirements() -> bool: """Tool is available only when a Discord bot token is configured.""" return bool(_get_bot_token()) -# --------------------------------------------------------------------------- -# Handlers -# --------------------------------------------------------------------------- +# ── handlers ───────────────────────────────────────────────────────────────── + +_HANDLER_DEFAULTS = { + "action": "", "guild_id": "", "channel_id": "", "user_id": "", + "role_id": "", "message_id": "", "query": "", "name": "", + "limit": 50, "before": "", "after": "", "auto_archive_duration": 1440, +} + def _run_discord_action( action: str, valid_actions: Dict[str, Any], tool_label: str, - guild_id: str = "", - channel_id: str = "", - user_id: str = "", - role_id: str = "", - message_id: str = "", - query: str = "", - name: str = "", - limit: int = 50, - before: str = "", - after: str = "", - auto_archive_duration: int = 1440, + **params: Any, ) -> str: - """Shared handler logic for both discord tools.""" + """Shared handler logic for both discord tools (``params`` default per + :data:`_HANDLER_DEFAULTS`).""" token = _get_bot_token() if not token: return tool_error("DISCORD_BOT_TOKEN not configured.") @@ -1011,9 +817,8 @@ def _run_discord_action( available_actions=list(valid_actions.keys()), ) - # Config-level allowlist gate (defense in depth — schema already filtered, - # but a stale cached schema from a prior config should not let denied - # actions through). + # Config-level allowlist gate (defense in depth): a stale cached schema from + # a prior config must not let denied actions through. allowlist = _load_allowed_actions_config() if allowlist is not None and action not in allowlist: return tool_error( @@ -1021,37 +826,15 @@ def _run_discord_action( f"Allowed: {', '.join(allowlist) if allowlist else ''}" ) - local_vars = { - "guild_id": guild_id, - "channel_id": channel_id, - "user_id": user_id, - "role_id": role_id, - "message_id": message_id, - "query": query, - "name": name, - } - - missing = [p for p in _REQUIRED_PARAMS.get(action, []) if not local_vars.get(p)] + kwargs = {k: params.get(k, v) for k, v in _HANDLER_DEFAULTS.items() if k != "action"} + missing = [p for p in _REQUIRED_PARAMS.get(action, []) if not kwargs.get(p)] if missing: return tool_error( f"Missing required parameters for '{action}': {', '.join(missing)}" ) try: - return action_fn( - token=token, - guild_id=guild_id, - channel_id=channel_id, - user_id=user_id, - role_id=role_id, - message_id=message_id, - query=query, - name=name, - limit=limit, - before=before, - after=after, - auto_archive_duration=auto_archive_duration, - ) + return action_fn(token=token, **kwargs) except DiscordAPIError as e: logger.warning("Discord API error in %s action '%s': %s", tool_label, action, e) if e.status == 403: @@ -1072,15 +855,7 @@ def discord_admin_handler(action: str, **kwargs) -> str: return _run_discord_action(action, _ADMIN_ACTIONS, "discord_admin", **kwargs) -# --------------------------------------------------------------------------- -# Tool registration -# --------------------------------------------------------------------------- - -_HANDLER_DEFAULTS = { - "action": "", "guild_id": "", "channel_id": "", "user_id": "", - "role_id": "", "message_id": "", "query": "", "name": "", - "limit": 50, "before": "", "after": "", "auto_archive_duration": 1440, -} +# ── tool registration ──────────────────────────────────────────────────────── def _make_handler(handler_fn): diff --git a/tools/drive_preview_tool.py b/tools/drive_preview_tool.py index 653819db34..ff11397a59 100644 --- a/tools/drive_preview_tool.py +++ b/tools/drive_preview_tool.py @@ -1,35 +1,19 @@ #!/usr/bin/env python3 """Interact with the in-app browser / preview pane in the Hermes desktop GUI. -``open_preview`` shows a page and ``read_preview`` reads it; this tool is the -third leg — clicking, typing, scrolling, and history — so the agent can drive -the same page the user is looking at instead of narrating from the outside. - -Elements are addressed by refs from ``action="elements"`` that say what they -are: ``btn-sign-in``, ``inp-email``. A ref lasts as long as the page is open, -including across a re-render that destroys and rebuilds the element, and only a -navigation retires it — the renderer says so rather than acting on whatever now -occupies the spot. - -Because the refs hold, the renderer answers with a *delta* — what appeared, -what went, what changed, and what was rebound — instead of re-sending the whole -inventory after every click. That is the cheap half of the arrangement, and it -only works because the refs are legible enough to read on their own three turns -later. - -Round-trips through the gateway's blocking-prompt bridge like ``read_preview``: -tui_gateway emits ``preview.act.request``, the renderer injects the interaction -engine into the pane's webview and answers ``preview.act.respond`` with the -outcome plus whatever moved. This module is just schema + a thin dispatcher -over the platform-injected callback. - -Lives in the ``desktop_ui`` toolset, which the GUI gateway enables only for -desktop-sourced sessions. +Clicking, typing, scrolling, history — so the agent drives the page the user sees. +Elements are addressed by legible refs from ``action="elements"`` (``btn-sign-in``, +``inp-email``). A ref lasts while the page is open — including across a re-render that +rebuilds the element — and only a navigation retires it (the renderer says so rather +than acting on whatever now occupies the spot). Because refs hold, the renderer answers +with a *delta* (appeared/went/changed/rebound) instead of re-sending the inventory. +Round-trips through the gateway's blocking-prompt bridge (``preview.act.request`` -> +``preview.act.respond``); this module is schema + a thin dispatcher over the callback. """ -import json from typing import Callable, Optional +from tools.desktop_ui import passthrough_json from tools.registry import registry, tool_error ACTIONS = ( @@ -116,20 +100,14 @@ def drive_preview_tool( "The action timed out, or no GUI window answered. " "Open a page with open_preview first." ) - - # The renderer answers with a JSON object; pass it through, else wrap it. - try: - return json.dumps(json.loads(raw), ensure_ascii=False) - except (TypeError, ValueError): - return json.dumps({"text": str(raw)}, ensure_ascii=False) + return passthrough_json(raw) ACT_PREVIEW_SCHEMA = { "name": "drive_preview", - # Dieted (#95681): world-building compressed; response-shape teaching - # kept only where skipping it causes wasted calls (delta semantics, - # rebound refs, strobe's burst) — those are pre-effect: a model that - # doesn't know them re-reads pages or loops strobe. + # Response-shape teaching kept only where skipping it wastes calls (delta + # semantics, rebound refs, strobe's burst): a model that doesn't know them + # re-reads pages or loops strobe. "description": ( "Use the web page open in the desktop preview pane (the one " "`desktop_preview` opens): log in, fill forms, click through flows. ALWAYS " diff --git a/tools/fal_common.py b/tools/fal_common.py index 27636f9038..61feb31ed4 100644 --- a/tools/fal_common.py +++ b/tools/fal_common.py @@ -1,27 +1,10 @@ -"""Shared FAL.ai SDK plumbing. +"""Shared FAL.ai SDK plumbing: lazy import, managed-gateway sync client, small helpers. -Holds the stateless atoms that every FAL-backed tool needs: - -* :func:`import_fal_client` — lazy import + ``lazy_deps`` integration so - ``fal_client`` isn't pulled at cold start (it added ~64 ms per CLI - invocation when imported eagerly). -* :class:`_ManagedFalSyncClient` — wrapper that drives a Nous-managed - fal-queue gateway through the standard ``fal_client.SyncClient`` - primitives. -* :func:`_normalize_fal_queue_url_format`, :func:`_extract_http_status` - — small helpers used by both the managed client wrapper and - ``_submit_fal_request``. - -Stateful pieces (cache globals, ``_managed_fal_client*`` selectors, -``_submit_fal_request``) intentionally stay on -:mod:`tools.image_generation_tool`. That module is the patch target for -existing test suites (``tests/tools/test_image_generation.py``, -``tests/tools/test_managed_media_gateways.py``) and for the -``plugins/image_gen/fal/`` plugin's ``_it`` indirection — moving the -caches here would silently defeat ``monkeypatch.setattr(image_tool, -"_managed_fal_client", None)`` because the lookups would go against -``fal_common``'s namespace instead. See the per-rule walkthrough at -issue #26241 for details. +Stateful pieces (cache globals, ``_managed_fal_client*``, ``_submit_fal_request``) +intentionally stay on :mod:`tools.image_generation_tool`: it is the patch target +for the test suites and for ``plugins/image_gen/fal/``'s ``_it`` indirection, so +moving the caches here would silently defeat +``monkeypatch.setattr(image_tool, "_managed_fal_client", None)``. """ from __future__ import annotations @@ -31,15 +14,11 @@ from urllib.parse import urlencode def import_fal_client() -> Any: - """Import ``fal_client`` (via ``lazy_deps`` when available) and return - the module reference. + """Import ``fal_client`` (via ``lazy_deps`` when available); raises ImportError if unavailable. - Callers are responsible for caching the result on their own module - global — keeping per-module globals lets tests monkey-patch the - target module's ``fal_client`` attribute and have the patched value - stick for that module's call sites. - - Raises :class:`ImportError` if the package is genuinely unavailable. + Not imported at cold start (it cost ~64 ms per CLI invocation). Callers + cache the result on their own module global so tests can monkeypatch that + module's ``fal_client`` attribute and have it stick for its call sites. """ try: from tools.lazy_deps import ensure as _lazy_ensure @@ -60,12 +39,7 @@ def _normalize_fal_queue_url_format(queue_run_origin: str) -> str: def _extract_http_status(exc: BaseException) -> Optional[int]: - """Return an HTTP status code from httpx/fal exceptions, else None. - - Defensive across exception shapes — httpx.HTTPStatusError exposes - ``.response.status_code`` while fal_client wrappers may expose - ``.status_code`` directly. - """ + """HTTP status from httpx (``.response.status_code``) or fal_client (``.status_code``) exceptions, else None.""" response = getattr(exc, "response", None) if response is not None: status = getattr(response, "status_code", None) @@ -78,13 +52,10 @@ def _extract_http_status(exc: BaseException) -> Optional[int]: class _ManagedFalSyncClient: - """Small per-instance wrapper around ``fal_client.SyncClient`` for - managed queue hosts. + """Per-instance wrapper driving a Nous-managed fal-queue gateway via ``fal_client.SyncClient`` primitives. - The wrapper carries its own ``fal_client`` module reference instead - of reaching into a module global, so callers stay in control of - which module's ``fal_client`` is in scope (matters for the test - patches that swap the legacy module's ``fal_client`` attribute). + Carries its own ``fal_client`` reference instead of a module global so the + caller decides which module's (possibly test-patched) ``fal_client`` is used. """ def __init__(self, fal_client: Any, *, key: str, queue_run_origin: str): diff --git a/tools/feishu_drive_tool.py b/tools/feishu_drive_tool.py index 76e50ca800..ff4cf14eb5 100644 --- a/tools/feishu_drive_tool.py +++ b/tools/feishu_drive_tool.py @@ -1,8 +1,7 @@ """Feishu Drive Tools -- document comment operations via Feishu/Lark API. -Provides tools for listing, replying to, and adding document comments. -Uses the same lazy-import + BaseRequest pattern as feishu_comment.py. -The lark client is injected per-thread by the comment event handler. +List / reply-to / add document comments through the generic BaseRequest path (lazy SDK +import). The lark client is injected per-thread by the feishu_comment event handler. """ import json @@ -28,8 +27,8 @@ def get_client(): def _check_feishu(): - # See ``tools/feishu_doc_tool.py::_check_feishu`` — ``find_spec`` keeps - # CLI startup fast (the SDK itself takes ~5s to import eagerly). + # find_spec avoids executing lark_oapi's ~5s __init__ at every hermes startup; + # the handlers do the real import when invoked. import importlib.util try: return importlib.util.find_spec("lark_oapi") is not None @@ -43,11 +42,9 @@ def _do_request(client, method, uri, paths=None, queries=None, body=None): from lark_oapi.core.enum import HttpMethod from lark_oapi.core.model.base_request import BaseRequest - http_method = HttpMethod.GET if method == "GET" else HttpMethod.POST - builder = ( BaseRequest.builder() - .http_method(http_method) + .http_method(HttpMethod.GET if method == "GET" else HttpMethod.POST) .uri(uri) .token_types({AccessTokenType.TENANT}) ) @@ -58,22 +55,19 @@ def _do_request(client, method, uri, paths=None, queries=None, body=None): if body is not None: builder = builder.body(body) - request = builder.build() - # Tool handlers run synchronously in a worker thread (no running event # loop), so call the blocking lark client directly. - response = client.request(request) + response = client.request(builder.build()) code = getattr(response, "code", None) msg = getattr(response, "msg", "") - # Parse response data + # Prefer the raw JSON body's "data"; fall back to the typed response.data. data = {} raw = getattr(response, "raw", None) if raw and hasattr(raw, "content"): try: - body_json = json.loads(raw.content) - data = body_json.get("data", {}) + data = json.loads(raw.content).get("data", {}) except (json.JSONDecodeError, AttributeError): pass if not data: @@ -86,6 +80,20 @@ def _do_request(client, method, uri, paths=None, queries=None, body=None): return code, msg, data +def _paged_queries(args: dict) -> list: + """Common query params for comment/reply listing endpoints.""" + return [ + ("file_type", args.get("file_type", "docx") or "docx"), + ("user_id_type", "open_id"), + ("page_size", str(args.get("page_size", 100))), + ] + + +_FILE_TOKEN_PROP = {"type": "string", "description": "The document file token."} +_FILE_TYPE_PROP = {"type": "string", "description": "File type (default: docx).", "default": "docx"} +_PAGE_TOKEN_PROP = {"type": "string", "description": "Pagination token for next page."} + + # --------------------------------------------------------------------------- # feishu_drive_list_comments # --------------------------------------------------------------------------- @@ -101,15 +109,8 @@ FEISHU_DRIVE_LIST_COMMENTS_SCHEMA = { "parameters": { "type": "object", "properties": { - "file_token": { - "type": "string", - "description": "The document file token.", - }, - "file_type": { - "type": "string", - "description": "File type (default: docx).", - "default": "docx", - }, + "file_token": _FILE_TOKEN_PROP, + "file_type": _FILE_TYPE_PROP, "is_whole": { "type": "boolean", "description": "If true, only return whole-document comments.", @@ -120,10 +121,7 @@ FEISHU_DRIVE_LIST_COMMENTS_SCHEMA = { "description": "Number of comments per page (max 100).", "default": 100, }, - "page_token": { - "type": "string", - "description": "Pagination token for next page.", - }, + "page_token": _PAGE_TOKEN_PROP, }, "required": ["file_token"], }, @@ -139,18 +137,10 @@ def _handle_list_comments(args: dict, **kwargs) -> str: if not file_token: return tool_error("file_token is required") - file_type = args.get("file_type", "docx") or "docx" - is_whole = args.get("is_whole", False) - page_size = args.get("page_size", 100) - page_token = args.get("page_token", "") - - queries = [ - ("file_type", file_type), - ("user_id_type", "open_id"), - ("page_size", str(page_size)), - ] - if is_whole: + queries = _paged_queries(args) + if args.get("is_whole", False): queries.append(("is_whole", "true")) + page_token = args.get("page_token", "") if page_token: queries.append(("page_token", page_token)) @@ -177,28 +167,18 @@ FEISHU_DRIVE_LIST_REPLIES_SCHEMA = { "parameters": { "type": "object", "properties": { - "file_token": { - "type": "string", - "description": "The document file token.", - }, + "file_token": _FILE_TOKEN_PROP, "comment_id": { "type": "string", "description": "The comment ID to list replies for.", }, - "file_type": { - "type": "string", - "description": "File type (default: docx).", - "default": "docx", - }, + "file_type": _FILE_TYPE_PROP, "page_size": { "type": "integer", "description": "Number of replies per page (max 100).", "default": 100, }, - "page_token": { - "type": "string", - "description": "Pagination token for next page.", - }, + "page_token": _PAGE_TOKEN_PROP, }, "required": ["file_token", "comment_id"], }, @@ -215,15 +195,8 @@ def _handle_list_replies(args: dict, **kwargs) -> str: if not file_token or not comment_id: return tool_error("file_token and comment_id are required") - file_type = args.get("file_type", "docx") or "docx" - page_size = args.get("page_size", 100) + queries = _paged_queries(args) page_token = args.get("page_token", "") - - queries = [ - ("file_type", file_type), - ("user_id_type", "open_id"), - ("page_size", str(page_size)), - ] if page_token: queries.append(("page_token", page_token)) @@ -254,10 +227,7 @@ FEISHU_DRIVE_REPLY_SCHEMA = { "parameters": { "type": "object", "properties": { - "file_token": { - "type": "string", - "description": "The document file token.", - }, + "file_token": _FILE_TOKEN_PROP, "comment_id": { "type": "string", "description": "The comment ID to reply to.", @@ -266,11 +236,7 @@ FEISHU_DRIVE_REPLY_SCHEMA = { "type": "string", "description": "The reply text content (plain text only, no markdown).", }, - "file_type": { - "type": "string", - "description": "File type (default: docx).", - "default": "docx", - }, + "file_type": _FILE_TYPE_PROP, }, "required": ["file_token", "comment_id", "content"], }, @@ -289,17 +255,7 @@ def _handle_reply_comment(args: dict, **kwargs) -> str: return tool_error("file_token, comment_id, and content are required") file_type = args.get("file_type", "docx") or "docx" - - body = { - "content": { - "elements": [ - { - "type": "text_run", - "text_run": {"text": content}, - } - ] - } - } + body = {"content": {"elements": [{"type": "text_run", "text_run": {"text": content}}]}} code, msg, data = _do_request( client, "POST", _REPLY_COMMENT_URI, @@ -329,19 +285,12 @@ FEISHU_DRIVE_ADD_COMMENT_SCHEMA = { "parameters": { "type": "object", "properties": { - "file_token": { - "type": "string", - "description": "The document file token.", - }, + "file_token": _FILE_TOKEN_PROP, "content": { "type": "string", "description": "The comment text content (plain text only, no markdown).", }, - "file_type": { - "type": "string", - "description": "File type (default: docx).", - "default": "docx", - }, + "file_type": _FILE_TYPE_PROP, }, "required": ["file_token", "content"], }, @@ -359,13 +308,7 @@ def _handle_add_comment(args: dict, **kwargs) -> str: return tool_error("file_token and content are required") file_type = args.get("file_type", "docx") or "docx" - - body = { - "file_type": file_type, - "reply_elements": [ - {"type": "text", "text": content}, - ], - } + body = {"file_type": file_type, "reply_elements": [{"type": "text", "text": content}]} code, msg, data = _do_request( client, "POST", _ADD_COMMENT_URI, @@ -382,50 +325,24 @@ def _handle_add_comment(args: dict, **kwargs) -> str: # Registration # --------------------------------------------------------------------------- -registry.register( - name="feishu_drive_list_comments", - toolset="feishu_drive", - schema=FEISHU_DRIVE_LIST_COMMENTS_SCHEMA, - handler=_handle_list_comments, - check_fn=_check_feishu, - requires_env=[], - is_async=False, - description="List document comments", - emoji="\U0001f4ac", -) - -registry.register( - name="feishu_drive_list_comment_replies", - toolset="feishu_drive", - schema=FEISHU_DRIVE_LIST_REPLIES_SCHEMA, - handler=_handle_list_replies, - check_fn=_check_feishu, - requires_env=[], - is_async=False, - description="List comment replies", - emoji="\U0001f4ac", -) - -registry.register( - name="feishu_drive_reply_comment", - toolset="feishu_drive", - schema=FEISHU_DRIVE_REPLY_SCHEMA, - handler=_handle_reply_comment, - check_fn=_check_feishu, - requires_env=[], - is_async=False, - description="Reply to a document comment", - emoji="\u2709\ufe0f", -) - -registry.register( - name="feishu_drive_add_comment", - toolset="feishu_drive", - schema=FEISHU_DRIVE_ADD_COMMENT_SCHEMA, - handler=_handle_add_comment, - check_fn=_check_feishu, - requires_env=[], - is_async=False, - description="Add a whole-document comment", - emoji="\u2709\ufe0f", -) +for _name, _schema, _handler, _desc, _emoji in ( + ("feishu_drive_list_comments", FEISHU_DRIVE_LIST_COMMENTS_SCHEMA, _handle_list_comments, + "List document comments", "\U0001f4ac"), + ("feishu_drive_list_comment_replies", FEISHU_DRIVE_LIST_REPLIES_SCHEMA, _handle_list_replies, + "List comment replies", "\U0001f4ac"), + ("feishu_drive_reply_comment", FEISHU_DRIVE_REPLY_SCHEMA, _handle_reply_comment, + "Reply to a document comment", "\u2709\ufe0f"), + ("feishu_drive_add_comment", FEISHU_DRIVE_ADD_COMMENT_SCHEMA, _handle_add_comment, + "Add a whole-document comment", "\u2709\ufe0f"), +): + registry.register( + name=_name, + toolset="feishu_drive", + schema=_schema, + handler=_handler, + check_fn=_check_feishu, + requires_env=[], + is_async=False, + description=_desc, + emoji=_emoji, + ) diff --git a/tools/file_operations_common.py b/tools/file_operations_common.py new file mode 100644 index 0000000000..64e200a6d2 --- /dev/null +++ b/tools/file_operations_common.py @@ -0,0 +1,296 @@ +"""Result dataclasses and pure text helpers shared by ``tools.file_operations`` +and its search/lint mixins. + +Leaf module (imports nothing from ``tools`` at module scope) so the mixins and +the origin module can all depend on it without an import cycle. The +``to_dict`` output of every class here IS tool behavior — key names, key +order, and omission rules are pinned by tests and read by the model. +""" + +import re +from dataclasses import dataclass, field +from typing import Any, ClassVar, Dict, List, Optional + + +@dataclass +class ReadResult: + """Result from reading a file.""" + content: str = "" + total_lines: int = 0 + file_size: int = 0 + truncated: bool = False + hint: Optional[str] = None + is_binary: bool = False + is_image: bool = False + base64_content: Optional[str] = None + mime_type: Optional[str] = None + dimensions: Optional[str] = None # For images: "WIDTHxHEIGHT" + error: Optional[str] = None + similar_files: List[str] = field(default_factory=list) + + def to_dict(self) -> dict: + return {k: v for k, v in self.__dict__.items() if v is not None and v != []} + + +@dataclass +class WriteResult: + """Result from writing a file.""" + bytes_written: int = 0 + dirs_created: bool = False + # True when the on-disk sha256 matched the intended content; None when the + # backend couldn't verify (no sha256sum). A mismatch is a hard error, never a flag. + verified: Optional[bool] = None + lint: Optional[Dict[str, Any]] = None + # LSP semantic diagnostics, kept separate from ``lint`` (syntax) so the model + # reads the two as independent signals. None when LSP is off/inapplicable. + lsp_diagnostics: Optional[str] = None + error: Optional[str] = None + warning: Optional[str] = None + + def to_dict(self) -> dict: + return {k: v for k, v in self.__dict__.items() if v is not None} + + +@dataclass +class PatchResult: + """Result from patching a file.""" + success: bool = False + diff: str = "" + files_modified: List[str] = field(default_factory=list) + files_created: List[str] = field(default_factory=list) + files_deleted: List[str] = field(default_factory=list) + lint: Optional[Dict[str, Any]] = None + lsp_diagnostics: Optional[str] = None # see WriteResult.lsp_diagnostics + error: Optional[str] = None + # Success-shaped no-op: the edit was already present, nothing written; ``note`` says why. + no_change: bool = False + note: Optional[str] = None + + # Emission order is part of the output contract. + _DICT_FIELDS: ClassVar[tuple] = ( + "diff", "files_modified", "files_created", "files_deleted", + "lint", "lsp_diagnostics", "error", + ) + + def to_dict(self) -> dict: + result: Dict[str, Any] = {"success": self.success} + if self.no_change: + result["no_change"] = True + if self.note: + result["note"] = self.note + for key in self._DICT_FIELDS: + value = getattr(self, key) + if value: + result[key] = value + return result + + +@dataclass +class SearchMatch: + """A single search match.""" + path: str + line_number: int + content: str + mtime: float = 0.0 # Modification time for sorting + + +@dataclass +class SearchResult: + """Result from searching.""" + matches: List[SearchMatch] = field(default_factory=list) + files: List[str] = field(default_factory=list) + counts: Dict[str, int] = field(default_factory=dict) + total_count: int = 0 + truncated: bool = False + limit_reason: Optional[str] = None + warning: Optional[str] = None + error: Optional[str] = None + + # Below this many matches the verbose array is already compact enough that + # a path-grouping header would cost more tokens than it saves. + _DENSIFY_MIN_MATCHES: ClassVar[int] = 5 + + def _densify_matches(self) -> Optional[str]: + """Render content matches as a lossless, path-grouped text block. + + Path printed once, then `` : `` rows. Relies on rg/grep + emitting a file's hits consecutively, so grouping on path change needs + no reordering. Returns None when too few matches to be worth it. + """ + if len(self.matches) < self._DENSIFY_MIN_MATCHES: + return None + lines: list[str] = [] + current_path: Optional[str] = None + for m in self.matches: + if m.path != current_path: + lines.append(m.path) + current_path = m.path + # rstrip only: leading indentation is meaningful code and kept verbatim. + lines.append(f" {m.line_number}: {m.content.rstrip()}") + return "\n".join(lines) + + def to_dict(self, densify: bool = False) -> dict: + result: dict[str, object] = {"total_count": self.total_count} + if self.matches: + dense = self._densify_matches() if densify else None + if dense is not None: + # Self-describing so the model never guesses the block's shape. + result["matches_format"] = ( + "path-grouped: each file path on its own line, followed by " + "indented ': ' rows for matches in that file" + ) + result["matches_text"] = dense + else: + result["matches"] = [ + {"path": m.path, "line": m.line_number, "content": m.content} + for m in self.matches + ] + if self.files: + result["files"] = self.files + if self.counts: + result["counts"] = self.counts + if self.truncated: + result["truncated"] = True + for key in ("limit_reason", "warning", "error"): + value = getattr(self, key) + if value: + result[key] = value + return result + + +@dataclass +class LintResult: + """Result from linting a file.""" + success: bool = True + skipped: bool = False + output: str = "" + message: str = "" + + def to_dict(self) -> dict: + if self.skipped: + return {"status": "skipped", "message": self.message} + result = {"status": "ok" if self.success else "error", "output": self.output} + if self.message: + result["message"] = self.message + return result + + +@dataclass +class ExecuteResult: + """Result from executing a shell command.""" + stdout: str = "" + exit_code: int = 0 + + +# --------------------------------------------------------------------------- +# Pure text helpers (no I/O) +# --------------------------------------------------------------------------- + +_OSC_SEQUENCE_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)") +_FENCE_MARKER_RE = re.compile(r"'?\x07?__HERMES_FENCE_[A-Za-z0-9]+__\x07?'?") + + +def _strip_terminal_fence_leaks(text: str) -> str: + """Strip leaked terminal fence wrappers (OSC sequences, fence markers) from + command output; drops lines that were nothing but wrapper.""" + if not text: + return text + cleaned_lines: List[str] = [] + for line in text.splitlines(keepends=True): + had_terminal_wrapper = "__HERMES_FENCE_" in line or "\x1b]" in line + cleaned = _OSC_SEQUENCE_RE.sub("", line) + cleaned = _FENCE_MARKER_RE.sub("", cleaned) + cleaned = cleaned.replace("\x07", "") + if had_terminal_wrapper and cleaned.strip("'\r\n\t ") == "": + continue + cleaned_lines.append(cleaned) + return "".join(cleaned_lines) + + +def _detect_line_ending(sample: str) -> Optional[str]: + """Dominant line ending of ``sample`` (``\\r\\n`` if any CRLF in the first 4KB, + else ``\\n``), or None for empty/single-line content. + + Used to preserve a file's endings across write_file/patch: the agent's bare-LF + tool args would otherwise silently normalize CRLF files, and patch would + produce mixed endings when only the substituted region changes. + """ + if not sample: + return None + head = sample[:4096] + if "\r\n" in head: + return "\r\n" + if "\n" in head: + return "\n" + return None + + +def _normalize_line_endings(text: str, target: str) -> str: + """Convert every line ending (CRLF, lone CR, LF) in ``text`` to ``target``. + Idempotent. Collapses to LF first — separate replacements would + double-convert CRLF → LFLF.""" + lf_normalized = text.replace("\r\n", "\n").replace("\r", "\n") + if target == "\n": + return lf_normalized + if target == "\r\n": + return lf_normalized.replace("\n", "\r\n") + return text + + +# UTF-8 BOM (EF BB BF == U+FEFF), prepended by some Windows editors. Stripped on +# read so the model never sees a phantom first character (and patch's first-line +# match works), restored on write when the on-disk file had one — mirroring the +# line-ending preservation above. +_UTF8_BOM = "\ufeff" + + +def _strip_bom(text: str) -> tuple[str, bool]: + """Return (text-without-leading-BOM, had_bom). Only a leading BOM is + stripped; mid-content U+FEFF is legitimate data.""" + if text and text.startswith(_UTF8_BOM): + return text[len(_UTF8_BOM):], True + return text, False + + +def _has_bom(text: Optional[str]) -> bool: + """True if ``text`` begins with a UTF-8 BOM.""" + return bool(text) and text.startswith(_UTF8_BOM) + + +# --------------------------------------------------------------------------- +# Pagination clamps +# --------------------------------------------------------------------------- + +DEFAULT_READ_OFFSET = 1 +DEFAULT_READ_LIMIT = 2000 +DEFAULT_SEARCH_OFFSET = 0 +DEFAULT_SEARCH_LIMIT = 50 + + +def _coerce_int(value: Any, default: int) -> int: + """Best-effort integer coercion for tool pagination inputs.""" + try: + return int(value) + except (TypeError, ValueError): + return default + + +def normalize_read_pagination(offset: Any = DEFAULT_READ_OFFSET, + limit: Any = DEFAULT_READ_LIMIT) -> tuple[int, int]: + """Clamp read_file pagination so invalid values can never reach a sed range + like ``0,-1p`` (schemas declare bounds, but not every caller enforces them). + The ``limit`` ceiling is ``tool_output.max_lines`` from config.yaml.""" + from tools.tool_output_limits import get_max_lines + max_lines = get_max_lines() + normalized_offset = max(1, _coerce_int(offset, DEFAULT_READ_OFFSET)) + normalized_limit = _coerce_int(limit, DEFAULT_READ_LIMIT) + normalized_limit = max(1, min(normalized_limit, max_lines)) + return normalized_offset, normalized_limit + + +def normalize_search_pagination(offset: Any = DEFAULT_SEARCH_OFFSET, + limit: Any = DEFAULT_SEARCH_LIMIT) -> tuple[int, int]: + """Return safe search pagination bounds for shell head/tail pipelines.""" + normalized_offset = max(0, _coerce_int(offset, DEFAULT_SEARCH_OFFSET)) + normalized_limit = max(1, _coerce_int(limit, DEFAULT_SEARCH_LIMIT)) + return normalized_offset, normalized_limit diff --git a/tools/file_operations_lint.py b/tools/file_operations_lint.py new file mode 100644 index 0000000000..feb58b4512 --- /dev/null +++ b/tools/file_operations_lint.py @@ -0,0 +1,369 @@ +"""Syntax-lint and LSP-diagnostics tier for ``tools.file_operations``. + +Extracted from ``ShellFileOperations`` as ``LintMixin``; the class inherits it +so every ``self._check_lint(...)`` call resolves unchanged via the MRO. Module +constants are re-imported into ``tools.file_operations`` for back-compat. +""" + +import ast +import json +import os +import tomllib +from typing import Callable, Dict, Optional + +from tools.file_operations_types import LintResult + +# Shell linters by extension, run via _exec() for languages whose check needs an +# external toolchain. ``.tsx`` is deliberately absent: it has never had a shell +# linter (it hits the "No linter" skip) and LSP covers it when enabled. +LINTERS = { + '.py': 'python -m py_compile {file} 2>&1', + '.js': 'node --check {file} 2>&1', + '.ts': 'npx tsc --noEmit {file} 2>&1', + '.go': 'go vet {file} 2>&1', + '.rs': 'rustfmt --check {file} 2>&1', +} + +# Extensions whose per-file shell linter is structurally weaker than a real LSP +# server and floods phantom errors on real projects: single-file ``tsc`` ignores +# tsconfig (no-lib/ES5 → every ES2015+ stdlib name "missing"), ``go vet`` fails +# outside a module, ``rustfmt --check`` is style-only and rejects non-Cargo files. +# When an LSP server claims the file, ``_check_lint`` skips the shell linter for +# these; py_compile / node --check are file-local and correct so always run. +_SHELL_LINTER_LSP_REDUNDANT = frozenset({'.ts', '.go', '.rs'}) + +# Output substrings meaning the linter binary exists but could not actually run +# (tooling gap, not a lint failure). ``_check_lint`` then returns ``skipped`` so +# the write isn't flagged and the LSP tier (which gates on ok/skipped) still runs. +# Matched case-insensitively. +_LINTER_UNUSABLE_PATTERNS = { + 'npx': ( + 'this is not the tsc command you are looking for', # tsc not installed locally + 'could not determine executable to run', + 'not found in npm registry', + ), + 'rustfmt': ( + 'no input filename given', # outside a Cargo project + 'error: not a workspace', + ), + 'go': ( + 'cannot find package', # outside a module / GOPATH + 'go: cannot find main module', + ), +} + + +def _looks_like_linter_unusable(base_cmd: str, output: str) -> bool: + """True iff ``output`` from ``base_cmd`` (first word of the linter cmd) says the tool itself couldn't run.""" + patterns = _LINTER_UNUSABLE_PATTERNS.get(base_cmd) + if not patterns: + return False + lower = output.lower() + return any(p in lower for p in patterns) + + +def _lint_json_inproc(content: str) -> tuple[bool, str]: + """In-process JSON syntax check. Returns (ok, error_message).""" + try: + json.loads(content) + return True, "" + except json.JSONDecodeError as e: + return False, f"JSONDecodeError: {e.msg} (line {e.lineno}, column {e.colno})" + except Exception as e: # noqa: BLE001 — any parse failure is a lint failure + return False, f"{type(e).__name__}: {e}" + + +def _lint_yaml_inproc(content: str) -> tuple[bool, str]: + """In-process YAML syntax check; ``__SKIP__`` when PyYAML is missing. + + Syntax-only (``yaml.parse``), NOT ``safe_load``: loading rejects valid YAML + that isn't one plain document — multi-doc ``---`` streams (ComposerError) and + app tags like CloudFormation ``!Sub`` / Ansible ``!vault`` (ConstructorError). + This verdict is a fail-closed WRITE gate, so a false positive refuses a + legitimate write; ``parse`` still catches real scanner/parser errors. + """ + try: + import yaml as _yaml + except ImportError: + return True, "__SKIP__" + try: + for _event in _yaml.parse(content): + pass + return True, "" + except _yaml.YAMLError as e: + return False, f"YAMLError: {e}" + except Exception as e: # noqa: BLE001 + return False, f"{type(e).__name__}: {e}" + + +def _lint_toml_inproc(content: str) -> tuple[bool, str]: + """In-process TOML syntax check (stdlib tomllib).""" + try: + tomllib.loads(content) + return True, "" + except Exception as e: # TOMLDecodeError is a ValueError subclass + return False, f"{type(e).__name__}: {e}" + + +def _lint_python_inproc(content: str) -> tuple[bool, str]: + """In-process Python syntax check via ast.parse (py_compile's scope, no subprocess).""" + try: + ast.parse(content) + return True, "" + except SyntaxError as e: + loc = f" (line {e.lineno}, column {e.offset})" if e.lineno else "" + return False, f"{type(e).__name__}: {e.msg}{loc}" + except Exception as e: # noqa: BLE001 + return False, f"{type(e).__name__}: {e}" + + +# In-process linters, preferred over shell linters (microseconds, no subprocess). +# Each takes content and returns (ok, error); error ``"__SKIP__"`` means the +# linter is unavailable (missing dependency) and counts as "no linter". +LINTERS_INPROC: Dict[str, Callable[[str], tuple[bool, str]]] = { + '.py': _lint_python_inproc, + '.json': _lint_json_inproc, + '.yaml': _lint_yaml_inproc, + '.yml': _lint_yaml_inproc, + '.toml': _lint_toml_inproc, +} + +# Extensions where write_file REFUSES on a parse failure (fail-closed gate) rather +# than merely reporting. ``.py`` is excluded on purpose: test fixtures use ``*.py`` +# paths as a generic stand-in for arbitrary text, so hard-refusing invalid Python +# would break exercised patterns. Python keeps the non-blocking lint-delta report. +_FAIL_CLOSED_INPROC_EXTS = frozenset({'.json', '.yaml', '.yml', '.toml'}) + + +class LintMixin: + """Post-write syntax lint + LSP diagnostics. Requires ``_exec``, + ``_has_command``, ``_escape_shell_arg``, ``_escape_native_tool_arg`` and + ``env`` from the host class.""" + + def _check_lint(self, path: str, content: Optional[str] = None) -> LintResult: + """Syntax-check ``path``: in-process linter when one matches the + extension (``content`` avoids a re-read), else the shell linter table.""" + ext = os.path.splitext(path)[1].lower() + + inproc = LINTERS_INPROC.get(ext) + if inproc is not None: + if content is None: + read_result = self._exec(f"cat {self._escape_shell_arg(path)} 2>/dev/null") + if read_result.exit_code != 0: + return LintResult(skipped=True, message=f"Failed to read {path} for lint") + content = read_result.stdout + ok, err = inproc(content) + if err == "__SKIP__": + return LintResult(skipped=True, message=f"No linter available for {ext} (missing dependency)") + return LintResult(success=ok, output="" if ok else err) + + if ext not in LINTERS: + return LintResult(skipped=True, message=f"No linter for {ext} files") + + # Single-file tsc can't read the project's tsconfig.json, so for project + # .ts files it floods phantom TS2307/TS2339 errors the delta filter then + # misreports as "pre-existing"; skip and let the LSP tier speak. + if ext == '.ts' and self._has_ancestor_tsconfig(path): + return LintResult( + skipped=True, + message=( + "Project tsconfig.json detected — per-file tsc skipped " + "(single-file tsc can't resolve project aliases/globals; " + "use the LSP tier or `tsc -p tsconfig.json` for real " + "diagnostics)." + ), + ) + + if ext in _SHELL_LINTER_LSP_REDUNDANT and self._lsp_will_handle(path): + return LintResult( + skipped=True, + message=f"LSP server handles {ext} — shell linter skipped", + ) + + linter_cmd = LINTERS[ext] + base_cmd = linter_cmd.split()[0] + if not self._has_command(base_cmd): + return LintResult(skipped=True, message=f"{base_cmd} not available") + + # Linters are native Windows binaries on Windows: they need the C:/... + # form, not MSYS /c/... (node would resolve it as C:\c\Users\... → phantom ENOENT). + cmd = linter_cmd.replace("{file}", self._escape_native_tool_arg(path)) + result = self._exec(cmd, timeout=30) + + if result.exit_code != 0 and _looks_like_linter_unusable(base_cmd, result.stdout): + from tools.ansi_strip import strip_ansi + cleaned = strip_ansi(result.stdout).strip() + # Collapse to one line — the npx banner is multi-line ASCII art. + first_line = next( + (ln.strip() for ln in cleaned.splitlines() if ln.strip()), + cleaned[:120], + ) + return LintResult( + skipped=True, + message=f"{base_cmd} not usable: {first_line[:200]}", + ) + + return LintResult( + success=result.exit_code == 0, + output=result.stdout.strip() if result.stdout.strip() else "" + ) + + def _check_lint_delta(self, path: str, pre_content: Optional[str], + post_content: Optional[str] = None) -> LintResult: + """Post-write lint; when it fails and ``pre_content`` is known, report + only errors this edit introduced (pre-existing lines filtered out). + + Semantic (LSP) diagnostics are a separate channel — see + ``_maybe_lsp_diagnostics`` — so syntax and semantic signals stay distinct. + """ + post = self._check_lint(path, content=post_content) + if post.success or post.skipped or pre_content is None: + return post + + pre = self._check_lint(path, content=pre_content) + if pre.success or pre.skipped or not pre.output: + return post # pre-write was clean (or unlintable): all post errors are new + + # Single-error parsers (ast.parse, json.loads) stop at the first error, so + # if every post error already existed we can't prove the edit is clean — + # report the file as still broken but say nothing new was introduced. + pre_lines = {ln.strip() for ln in pre.output.splitlines() if ln.strip()} + post_lines = [ln for ln in post.output.splitlines() if ln.strip() and ln.strip() not in pre_lines] + + if not post_lines: + return LintResult( + success=False, + output=post.output, + message="Pre-existing lint errors — this edit didn't introduce new ones but the file is still broken.", + ) + + return LintResult( + success=False, + output=( + "New lint errors introduced by this edit " + "(pre-existing errors filtered out):\n" + "\n".join(post_lines) + ) + ) + + def _lsp_local_only(self) -> bool: + """True iff wired to a local backend. LSP servers run on the host and + can't see files inside Docker/Modal/SSH/Daytona sandboxes.""" + env = getattr(self, "env", None) # tests may build via __new__ without __init__ + if env is None: + return False + try: + from tools.environments.local import LocalEnvironment + except Exception: # noqa: BLE001 + return False + return isinstance(env, LocalEnvironment) + + def _lsp_service(self): + """The active LSPService, or None on a non-local backend / any failure. + + Shared best-effort probe: LSP is an enrichment layer and must never + break a write, so every failure path collapses to None. + """ + if not self._lsp_local_only(): + return None + try: + from agent.lsp import get_service + return get_service() + except Exception: # noqa: BLE001 + return None + + def _lsp_handles_extension(self, ext: str) -> bool: + """True iff some registered LSP server claims ``ext`` (static registry + only; safe on remote backends). Decides whether pre-write content is + worth capturing for the line-shift map.""" + if not ext: + return False + try: + from agent.lsp.servers import SERVERS + except Exception: # noqa: BLE001 + return False + ext_lower = ext.lower() + return any(ext_lower in srv.extensions for srv in SERVERS) + + def _has_ancestor_tsconfig(self, path: str) -> bool: + """True iff a tsconfig.json exists in ``path``'s directory or any ancestor. + + Host-side walk, local backend only: on a remote backend the tree isn't + here, so this answers False and the shell linter runs as before — never + suppress lint based on a probe that couldn't answer. + """ + if not self._lsp_local_only(): + return False + try: + d = os.path.dirname(os.path.abspath(path)) + while True: + if os.path.isfile(os.path.join(d, "tsconfig.json")): + return True + parent = os.path.dirname(d) + if parent == d: + return False + d = parent + except Exception: # noqa: BLE001 + return False + + def _lsp_will_handle(self, path: str) -> bool: + """True iff the LSP service is active AND ``enabled_for(path)`` (workspace + detection, disabled-server set, broken-pair short-circuit). Any failure → + False so the shell linter still runs.""" + svc = self._lsp_service() + if svc is None: + return False + try: + return bool(svc.enabled_for(path)) + except Exception: # noqa: BLE001 + return False + + def _snapshot_lsp_baseline(self, path: str) -> None: + """Capture pre-edit LSP diagnostics so the post-write delta is correct. Silent on failure.""" + svc = self._lsp_service() + if svc is None: + return + try: + svc.snapshot_baseline(path) + except Exception: # noqa: BLE001 + pass + + def _maybe_lsp_diagnostics( + self, + path: str, + *, + pre_content: Optional[str] = None, + post_content: Optional[str] = None, + ) -> str: + """Formatted LSP diagnostics introduced by this edit, or "" when LSP is + unavailable/disabled/clean. Never raises past the service probe. + + With both pre and post content a line-shift map remaps baseline + diagnostics into post-edit coordinates; otherwise every pre-existing + diagnostic below an inserted/deleted line would look newly introduced. + """ + svc = self._lsp_service() + if svc is None or not svc.enabled_for(path): + return "" + + line_shift = None + if pre_content is not None and post_content is not None and pre_content != post_content: + try: + from agent.lsp.range_shift import build_line_shift + line_shift = build_line_shift(pre_content, post_content) + except Exception: # noqa: BLE001 + line_shift = None + + try: + diagnostics = svc.get_diagnostics_sync(path, delta=True, line_shift=line_shift) + except Exception: # noqa: BLE001 + return "" + if not diagnostics: + return "" + try: + from agent.lsp.reporter import report_for_file, truncate + block = report_for_file(path, diagnostics) + if not block: + return "" + return truncate("LSP diagnostics introduced by this edit:\n" + block) + except Exception: # noqa: BLE001 + return "" diff --git a/tools/file_operations_search.py b/tools/file_operations_search.py new file mode 100644 index 0000000000..cfbbfb9ef2 --- /dev/null +++ b/tools/file_operations_search.py @@ -0,0 +1,638 @@ +"""Content/file search tier for ``tools.file_operations``. + +Extracted from ``ShellFileOperations`` as ``SearchMixin``; the class inherits it +so every ``self._search_*`` call resolves unchanged via the MRO. Module-level +helpers are re-imported into ``tools.file_operations`` for back-compat. +""" + +import os +import re +import sys +from pathlib import Path +from typing import List, Optional + +from tools.file_operations_common import ExecuteResult, SearchMatch, SearchResult + +_MACOS_TCC_PROTECTED_HOME_DIRS = ( + "Desktop", "Documents", "Downloads", "Library", "Movies", "Music", "Pictures", +) + + +def _macos_protected_search_exclusions( + path: str, + *, + cwd: Optional[str] = None, + home: Optional[str] = None, + platform: Optional[str] = None, +) -> List[str]: + """Protected home dirs (relative to ``path``) below a broad macOS search root. + + Only an ANCESTOR search (``$HOME``, ``/Users``) gets exclusions, so recursive + tools never trigger unattended TCC prompts; a search rooted inside a + protected dir stays allowed. + """ + if (platform or sys.platform) != "darwin": + return [] + + home_path = Path(home or Path.home()).expanduser() + root = Path(path).expanduser() + if not root.is_absolute(): + root = Path(cwd or os.getcwd()) / root + root = Path(os.path.normpath(str(root))) + home_path = Path(os.path.normpath(str(home_path))) + + exclusions: List[str] = [] + for dirname in _MACOS_TCC_PROTECTED_HOME_DIRS: + try: + relative = (home_path / dirname).relative_to(root) + except ValueError: + continue + if relative.parts: + exclusions.append(relative.as_posix()) + return exclusions + + +_SEARCH_TIMEOUT_MARKER_RE = re.compile(r"\n?\[Command timed out after \d+s\]\s*$") + + +def _search_stdout_and_limit(result: ExecuteResult) -> tuple[str, Optional[str]]: + """Return stdout cleaned for parsing and a limit reason for search timeouts.""" + if result.exit_code == 124: + return _SEARCH_TIMEOUT_MARKER_RE.sub("", result.stdout), "search_timeout" + return result.stdout, None + + +# A real rg/grep output line is a whitespace-free path token followed by ``:`` +# (match/count), ``-`` (context), or nothing (files_only). Tool diagnostics +# ("rg: ...", "error: ...", indented carets) never match: the leading token +# forbids whitespace and a tool prefix is followed by ": " (space). +_SEARCH_OUTPUT_RE = re.compile(r'^([A-Za-z]:)?[^\s:][^\n]*?[:\-]\d|^[^\s:][^\s]*$') + + +def _split_tool_diagnostics(output: str) -> tuple[str, str]: + """Separate rg/grep diagnostic lines from real match output. + + ``_exec`` merges stderr into stdout, so tool errors interleave with matches. + Returns ``(diagnostics, payload)``. Classifying by SHAPE (not error prefix) + lets the exit-2 guard tell a pure failure (no payload → surface the error) + from a partial one (one unreadable file, others matched → keep matches), and + guarantees error text is never parsed as a match. + """ + diagnostics: list[str] = [] + payload: list[str] = [] + for line in output.split('\n'): + if not line.strip(): + continue + # Prefix check first: a real match path can contain "-" (e.g. + # ".../pytest-686/..."), which the shape regex would accept as a match. + stripped = line.lstrip() + if stripped.startswith("rg: ") or stripped.startswith("grep: "): + diagnostics.append(line) + continue + if line == "--" or _SEARCH_OUTPUT_RE.match(line): + payload.append(line) + else: + diagnostics.append(line) + return '\n'.join(diagnostics), '\n'.join(payload) + + +def _parse_search_context_line(line: str) -> tuple[str, int, str] | None: + """Parse a ``path-line-content`` context line. + + Filenames may contain ``--`` segments, so use the RIGHTMOST numeric + separator: ``dir/file-12-name.py-8-context`` → (``dir/file-12-name.py``, 8). + """ + if not line or line == "--": + return None + match = None + for candidate in re.finditer(r'-(\d+)-', line): + match = candidate + if match is None: + return None + path = line[:match.start()] + if not path: + return None + return path, int(match.group(1)), line[match.end():] + + +_REGEX_NEWLINE_ESCAPE_RE = re.compile(r"(? bool: + """True when a content regex wants to match a newline: a literal newline or a + ``\\n`` escape with an ODD number of backslashes (``\\\\n`` is a literal + backslash+n and must not count).""" + return "\n" in pattern or bool(_REGEX_NEWLINE_ESCAPE_RE.search(pattern)) + + +def _is_line_oriented_newline_error(error: Optional[str]) -> bool: + """Return True for rg's hard error when multiline mode is required.""" + if not error: + return False + return "literal \"\\n\" is not allowed" in error and "--multiline" in error + + +def _maybe_warn_line_oriented_newline_pattern(result: SearchResult, pattern: str) -> SearchResult: + """Attach a newline-regex warning only when search found no usable results.""" + if result.total_count != 0 or not _pattern_has_regex_newline(pattern): + return result + if result.error and not _is_line_oriented_newline_error(result.error): + return result + result.error = None + result.warning = ( + "0 results found. Note: search_files content search is line-oriented " + "and does not run ripgrep with -U/--multiline, so `\\n` in the regex " + "does not match line breaks. Use context=N to inspect neighboring " + "lines, or escape as `\\\\n` when searching for a literal backslash+n." + ) + return result + + +# Match lines are "file:lineno:content". Windows paths carry a drive letter +# ("C:\path"), so a naive split(":") breaks — the regex handles both. +_MATCH_LINE_RE = re.compile(r'^([A-Za-z]:)?(.*?):(\d+):(.*)$') + +# Output-mode → engine flag (identical for rg and grep). +_OUTPUT_MODE_FLAGS = {"files_only": "-l", "count": "-c"} + + +def _parse_search_output(result, output_mode: str, limit: int, offset: int, + context: int, warning: Optional[str] = None) -> SearchResult: + """Parse rg/grep ``| head`` output into a SearchResult (shared by both engines). + + Exit codes: 0=matches, 1=none, 2=error — but both tools return 2 on PARTIAL + errors (one unreadable file in a tree that otherwise matched), so an error is + surfaced only when exit==2 AND no usable payload remains. + ``warning`` is attached to files_only/content results (rg's multiline note). + """ + stdout, limit_reason = _search_stdout_and_limit(result) + diagnostics, payload = _split_tool_diagnostics(stdout) + if result.exit_code == 2 and not payload.strip(): + error_msg = diagnostics.strip() or result.stdout.strip() or "Search error" + return SearchResult(error=f"Search failed: {error_msg}", total_count=0) + + lines = [ln for ln in payload.strip().split('\n') if ln] + if output_mode == "files_only": + return SearchResult( + files=lines[offset:offset + limit], + total_count=len(lines), + truncated=bool(limit_reason), + limit_reason=limit_reason, + warning=warning, + ) + + if output_mode == "count": + counts = {} + for line in lines: + if ':' in line: + path, n = line.rsplit(':', 1) + try: + counts[path] = int(n) + except ValueError: + pass + return SearchResult( + counts=counts, + total_count=sum(counts.values()), + truncated=bool(limit_reason), + limit_reason=limit_reason, + ) + + matches = [] + for line in lines: + if line == "--": + continue + m = _MATCH_LINE_RE.match(line) + if m: + matches.append(SearchMatch( + path=(m.group(1) or '') + m.group(2), + line_number=int(m.group(3)), + content=m.group(4)[:500], + )) + continue + # Context lines ("file-line-content") only when context was requested, + # to avoid false positives on dash-heavy paths. + if context > 0: + parsed = _parse_search_context_line(line) + if parsed: + matches.append(SearchMatch( + path=parsed[0], line_number=parsed[1], content=parsed[2][:500], + )) + total = len(matches) + return SearchResult( + matches=matches[offset:offset + limit], + total_count=total, + truncated=total > offset + limit or bool(limit_reason), + limit_reason=limit_reason, + warning=warning, + ) + + +class SearchMixin: + """File-name and content search via rg with find/grep fallbacks. Requires + ``_exec``, ``_has_command``, ``_expand_path``, ``_escape_shell_arg``, + ``_escape_native_tool_arg``, ``env`` and ``cwd`` from the host class.""" + + def _macos_search_exclusions(self, path: str) -> List[str]: + """Protected descendants to prune for this search root, if any. + + Gated on ``env.is_local``: ``sys.platform``/``_HOME`` describe the + CONTROLLER, but the search runs on ``env``'s host — a macOS controller + driving a Linux container must not prune the remote's Downloads. Envs + without the flag (fakes, plugins) default to local semantics; pruning is + a warning-carrying skip, never data loss. + """ + env = getattr(self, "env", None) + if env is not None and getattr(env, "is_local", True) is False: + return [] + from tools import file_operations as _fo # lazy: _HOME is monkeypatched there + cwd = getattr(self.env, "cwd", None) or self.cwd + return _macos_protected_search_exclusions( + path, cwd=cwd, home=_fo._HOME, platform=sys.platform + ) + + def _protected_prune_paths(self, path: str) -> List[str]: + """Absolute-ish protected paths for find's ``-path ... -prune``.""" + return [ + os.path.normpath(os.path.join(path, item)) + for item in self._macos_search_exclusions(path) + ] + + def _path_exists_probe(self, path: str) -> str: + """Stdout of the existence probe: contains "exists" or "not_found".""" + return self._exec( + f"test -e {self._escape_shell_arg(path)} && echo exists || echo not_found" + ).stdout + + def _dispatch_search(self, pattern: str, path: str, target: str, + file_glob: Optional[str], limit: int, offset: int, + output_mode: str, context: int) -> SearchResult: + if target == "files": + return self._search_files(pattern, path, limit, offset) + return self._search_content(pattern, path, file_glob, limit, offset, + output_mode, context) + + def _path_not_found_result(self, path: str) -> SearchResult: + """Error result for a missing search root, with nearby-entry suggestions.""" + parent = os.path.dirname(path) or "." + basename_query = os.path.basename(path) + hint_parts = [f"Path not found: {path}"] + parent_check = self._exec( + f"test -d {self._escape_shell_arg(parent)} && echo yes || echo no" + ) + if "yes" in parent_check.stdout and basename_query: + ls_result = self._exec( + f"ls -1 {self._escape_shell_arg(parent)} 2>/dev/null | head -20" + ) + if ls_result.exit_code == 0 and ls_result.stdout.strip(): + lower_q = basename_query.lower() + candidates = [] + for entry in ls_result.stdout.strip().split('\n'): + if not entry: + continue + le = entry.lower() + if lower_q in le or le in lower_q or le.startswith(lower_q[:3]): + candidates.append(os.path.join(parent, entry)) + if candidates: + hint_parts.append("Similar paths: " + ", ".join(candidates[:5])) + return SearchResult(error=". ".join(hint_parts), total_count=0) + + def _try_multi_path_search(self, pattern: str, path: str, target: str, + file_glob: Optional[str], limit: int, offset: int, + output_mode: str, context: int) -> Optional[SearchResult]: + """Recover a not-found ``path`` that is really several paths in one string + ("dir1 dir2" or comma-separated): search every existing part, merge, and + note skipped parts. None when it doesn't look like a multi-path string.""" + parts = [p for chunk in path.split(",") for p in chunk.split() if p.strip()] + if len(parts) < 2: + return None + existing, missing = [], [] + for p in parts: + expanded = self._expand_path(p) + (existing if self._path_exists(expanded) else missing).append(expanded) + if not existing: + return None + + merged = SearchResult() + for p in existing: + sub = self._dispatch_search(pattern, p, target, file_glob, limit, offset, + output_mode, context) + if sub.error: + continue + merged.matches.extend(sub.matches) + merged.files.extend(sub.files) + merged.counts.update(sub.counts) + merged.total_count += sub.total_count + merged.truncated = merged.truncated or sub.truncated + merged.matches = merged.matches[:limit] + merged.files = merged.files[:limit] + note = f"path contained {len(parts)} entries; searched {len(existing)} that exist" + if missing: + note += "; skipped missing: " + ", ".join(missing[:3]) + if len(missing) > 3: + note += f" (+{len(missing) - 3} more)" + merged.warning = note + return merged + + # (rg flags, message template) probes for a 0-match content search, in order. + # The fixed-string probe only runs when the pattern has regex metacharacters. + _ZERO_MATCH_PROBES = ( + ("-i", "0 exact matches, but {total} case-insensitive match(es) in {n} file(s): " + "{paths} — the pattern's casing may be wrong."), + # rg skips dotdirs and .gitignore'd files by default; say so instead of a bare zero. + ("--hidden --no-ignore", "0 matches in visible files, but {total} match(es) in {n} " + "hidden or gitignored file(s): {paths} — these are excluded by default."), + ("-F", "0 regex matches, but {total} literal match(es) in {n} file(s): {paths} — the " + "pattern contains regex metacharacters that likely need escaping " + "(or pass a simpler substring)."), + ) + + def _zero_match_probe(self, pattern: str, path: str, + file_glob: Optional[str]) -> Optional[str]: + """Steering hint for a 0-match content search, or None. + + A bare zero gives the model nothing to act on, so run cheap count-only rg + probes (case-insensitive, hidden/ignored, fixed-string) and report the + first that hits. Bounded to three rg invocations. + """ + if not self._has_command('rg'): + return None + has_meta = bool(re.search(r"[.\[\](){}?*+^$\\|]", pattern)) + glob_expr = f" --glob {self._escape_shell_arg(file_glob)}" if file_glob else "" + for flags, template in self._ZERO_MATCH_PROBES: + if flags == "-F" and not has_meta: + continue + probe = self._exec( + f"rg {flags} --count-matches{glob_expr} " + f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} " + f"2>/dev/null | head -50", + timeout=30, + ) + total, per_file = 0, [] + for line in (probe.stdout or "").strip().splitlines(): + p, _sep, n = line.rpartition(":") + if n.isdigit(): + total += int(n) + per_file.append(p) + if total > 0: + extra = len(per_file) - 5 + paths = ", ".join(per_file[:5]) + (f" (+{extra} more)" if extra > 0 else "") + return template.format(total=total, n=len(per_file), paths=paths) + return None + + def _search_files(self, pattern: str, path: str, limit: int, offset: int) -> SearchResult: + """Search for files by name (glob-like): rg --files, else find.""" + search_pattern = pattern if (not pattern.startswith('**/') and '/' not in pattern) \ + else pattern.split('/')[-1] + + search_root = Path(path) + has_hidden_path_ancestor = any( + part not in {".", ".."} and part.startswith(".") + for part in search_root.parts + ) + + # rg respects .gitignore, skips hidden dirs, and walks in parallel (~200x find). + if self._has_command('rg'): + return self._search_files_rg(search_pattern, path, limit, offset) + if not self._has_command('find'): + return SearchResult( + error="File search requires 'rg' (ripgrep) or 'find'. " + "Install ripgrep for best results: " + "https://github.com/BurntSushi/ripgrep#installation" + ) + + # Hidden roots: find's path filter would exclude everything under the root, + # so gather full output and filter descendants in Python (pagination too). + hidden_filter_expr = "" if has_hidden_path_ancestor else " -not -path '*/.*'" + pagination_expr = "" if has_hidden_path_ancestor else f" | tail -n +{offset + 1} | head -n {limit}" + + # Prune protected dirs BEFORE traversal so macOS never sees an access attempt. + protected_paths = self._protected_prune_paths(path) + prune_expr = "" + if protected_paths: + prune_terms = " -o ".join( + f"-path {self._escape_shell_arg(item)}" for item in protected_paths + ) + prune_expr = f" \\( {prune_terms} \\) -prune -o" + + base = (f"find {self._escape_shell_arg(path)}{prune_expr}{hidden_filter_expr} " + f"-type f -name {self._escape_shell_arg(search_pattern)} ") + result = self._exec(f"{base}-printf '%T@ %p\\n' 2>/dev/null | sort -rn{pagination_expr}", timeout=60) + stdout, limit_reason = _search_stdout_and_limit(result) + if not stdout.strip() and not limit_reason: + # BSD find (macOS) has no -printf. + result = self._exec(f"{base}2>/dev/null | sort -rn{pagination_expr}", timeout=60) + stdout, limit_reason = _search_stdout_and_limit(result) + + files = [] + for line in stdout.strip().split('\n'): + if not line: + continue + parts = line.split(' ', 1) + files.append(parts[1] if len(parts) == 2 and parts[0].replace('.', '').isdigit() else line) + + if has_hidden_path_ancestor: + normalized_root = search_root.resolve() + filtered_files = [] + for file_path in files: + try: + rel_parts = Path(file_path).resolve().relative_to(normalized_root).parts + except ValueError: + rel_parts = Path(file_path).parts + if any(part not in {".", ".."} and part.startswith(".") for part in rel_parts): + continue + filtered_files.append(file_path) + files = filtered_files[offset:offset + limit] + + return SearchResult( + files=files, + total_count=len(files), + truncated=bool(limit_reason), + limit_reason=limit_reason, + ) + + def _search_files_rg(self, pattern: str, path: str, limit: int, offset: int) -> SearchResult: + """File-name search via ``rg --files``, mtime-sorted when rg >= 13 supports --sortr.""" + # Wrap bare names so -g matches at any depth (equivalent to find -name). + glob_pattern = f"*{pattern}" if ('/' not in pattern and not pattern.startswith('*')) else pattern + + fetch_limit = limit + offset + exclusion_globs = " ".join( + f"--glob {self._escape_shell_arg(f'!{item}/**')}" + for item in self._macos_search_exclusions(path) + ) + exclusion_args = f" {exclusion_globs}" if exclusion_globs else "" + tail = (f"-g {self._escape_shell_arg(glob_pattern)}{exclusion_args} " + f"{self._escape_native_tool_arg(path)} 2>/dev/null | head -n {fetch_limit}") + result = self._exec(f"rg --files --sortr=modified {tail}", timeout=60) + stdout, limit_reason = _search_stdout_and_limit(result) + all_files = [f for f in stdout.strip().split('\n') if f] + + if not all_files and not limit_reason: + # --sortr may have failed on older rg; retry without it. + result = self._exec(f"rg --files {tail}", timeout=60) + stdout, limit_reason = _search_stdout_and_limit(result) + all_files = [f for f in stdout.strip().split('\n') if f] + + return SearchResult( + files=all_files[offset:offset + limit], + total_count=len(all_files), + truncated=len(all_files) >= fetch_limit or bool(limit_reason), + limit_reason=limit_reason, + ) + + def _search_content(self, pattern: str, path: str, file_glob: Optional[str], + limit: int, offset: int, output_mode: str, context: int) -> SearchResult: + """Content search: rg, else grep; attaches zero-match steering hints.""" + used_rg = False + if self._has_command('rg'): + used_rg = True + result = self._search_with_rg(pattern, path, file_glob, limit, offset, + output_mode, context) + elif self._has_command('grep'): + result = self._search_with_grep(pattern, path, file_glob, limit, offset, + output_mode, context) + else: + return SearchResult( + error="Content search requires ripgrep (rg) or grep. " + "Install ripgrep: https://github.com/BurntSushi/ripgrep#installation" + ) + + if (not result.error and result.total_count == 0 + and not result.matches and not result.files and not result.counts): + try: + hint = self._zero_match_probe(pattern, path, file_glob) + except Exception: + hint = None + if hint: + result.warning = hint if not result.warning else f"{result.warning} {hint}" + + # rg auto-enables --multiline for \n patterns, so the line-oriented + # explanation only applies to the grep fallback. + if used_rg: + return result + return _maybe_warn_line_oriented_newline_pattern(result, pattern) + + def _search_with_rg(self, pattern: str, path: str, file_glob: Optional[str], + limit: int, offset: int, output_mode: str, context: int) -> SearchResult: + """Search using ripgrep.""" + cmd_parts = ["rg", "--line-number", "--no-heading", "--with-filename"] + + # A regex \n can't match in line-oriented mode (rg hard-errors); enable -U + # up front when the pattern clearly wants to cross lines, and say so. + multiline = _pattern_has_regex_newline(pattern) + if multiline: + cmd_parts.append("--multiline") + if context > 0: + cmd_parts.extend(["-C", str(context)]) + for item in self._macos_search_exclusions(path): + cmd_parts.extend(["--glob", self._escape_shell_arg(f"!{item}/**")]) + if file_glob: + cmd_parts.extend(["--glob", self._escape_shell_arg(file_glob)]) + if output_mode in _OUTPUT_MODE_FLAGS: + cmd_parts.append(_OUTPUT_MODE_FLAGS[output_mode]) + cmd_parts.append(self._escape_shell_arg(pattern)) + # rg is a native Windows binary (winget/cargo/choco): needs C:/... not MSYS /c/... + cmd_parts.append(self._escape_native_tool_arg(path)) + + # Fetch extra rows to report the true total; context mode also emits "--" + # separators, so grab generously and filter in Python. + fetch_limit = limit + offset + 200 if context > 0 else limit + offset + cmd_parts.extend(["|", "head", "-n", str(fetch_limit)]) + + # pipefail so rg's exit 2 survives `| head` (else head's 0 masks it). rg + # exits 0 on SIGPIPE from a truncating head, so no false errors. + cmd = "set -o pipefail; " + " ".join(cmd_parts) + result = self._exec(cmd, timeout=60) + ml_note = ( + "Pattern contains \\n — multiline mode (-U) was enabled automatically " + "so the regex can match across line boundaries." + ) if multiline else None + return _parse_search_output(result, output_mode, limit, offset, context, warning=ml_note) + + def _search_with_grep(self, pattern: str, path: str, file_glob: Optional[str], + limit: int, offset: int, output_mode: str, context: int) -> SearchResult: + """Fallback search using grep.""" + # -H forces filenames; -E matches rg regex behavior; --exclude-dir='.*' + # mirrors rg's hidden-dir default (.git/, .hub/index-cache/, ...). + cmd_parts = ["grep", "-rnHE", "--exclude-dir='.*'"] + + # grep's --exclude-dir matches BASENAMES anywhere in the tree, so it can't + # express "only the home-level Downloads"; route protected-dir pruning + # through find's path-scoped -prune instead. + protected_paths = self._protected_prune_paths(path) + if protected_paths: + return self._search_with_grep_pruned( + pattern, path, file_glob, limit, offset, output_mode, context, + protected_paths, + ) + + if context > 0: + cmd_parts.extend(["-C", str(context)]) + if file_glob: + cmd_parts.extend(["--include", self._escape_shell_arg(file_glob)]) + if output_mode in _OUTPUT_MODE_FLAGS: + cmd_parts.append(_OUTPUT_MODE_FLAGS[output_mode]) + cmd_parts.append(self._escape_shell_arg(pattern)) + + # grep applies --exclude-dir to the search root too, so a relative root + # "." would be excluded by '.*'. Anchor relative paths at the shell's + # live $PWD (quoted separately so user paths stay escaped). + is_absolute = path.startswith(("/", "\\\\")) or bool( + re.match(r"^[A-Za-z]:[\\/]", path) + ) + if is_absolute: + search_root = self._escape_shell_arg(path) + else: + relative_path = path[2:] if path.startswith("./") else path + search_root = '"$PWD"' + if relative_path not in {"", "."}: + search_root += f"/{self._escape_shell_arg(relative_path)}" + cmd_parts.append(search_root) + + fetch_limit = limit + offset + (200 if context > 0 else 0) + cmd_parts.extend(["|", "head", "-n", str(fetch_limit)]) + + # pipefail so grep's exit 2 survives `| head`; a truncating head makes + # grep exit 141 (SIGPIPE), which the strict ==2 guard ignores. + cmd = "set -o pipefail; " + " ".join(cmd_parts) + result = self._exec(cmd, timeout=60) + return _parse_search_output(result, output_mode, limit, offset, context) + + def _search_with_grep_pruned(self, pattern: str, path: str, file_glob: Optional[str], + limit: int, offset: int, output_mode: str, context: int, + protected_paths: List[str]) -> SearchResult: + """grep fallback with PATH-scoped protected-dir pruning. + + ``find ... -prune`` enumerates files (traversal never enters protected + dirs, so macOS never sees an access attempt) and hands them to grep via + ``-exec {} +``; hidden dirs are pruned to mirror ``--exclude-dir='.*'``. + Trade-off: find folds grep's exit code into its own generic non-zero, so + a hard grep error surfaces as an empty result rather than exit 2 — + acceptable for this darwin-local-broad-search-only branch. + """ + grep_parts = ["grep", "-nHE"] + if context > 0: + grep_parts.extend(["-C", str(context)]) + if output_mode in _OUTPUT_MODE_FLAGS: + grep_parts.append(_OUTPUT_MODE_FLAGS[output_mode]) + grep_parts.append(self._escape_shell_arg(pattern)) + + prune_terms = " -o ".join( + f"-path {self._escape_shell_arg(item)}" for item in protected_paths + ) + find_parts = [ + "find", self._escape_shell_arg(path or "."), + f"\\( {prune_terms} \\) -prune", "-o", + "\\( -type d -name '.*' \\) -prune", "-o", + "-type f", + ] + if file_glob: + find_parts.extend(["-name", self._escape_shell_arg(file_glob)]) + find_parts.extend(["-exec", *grep_parts, "{}", "+"]) + fetch_limit = limit + offset + (200 if context > 0 else 0) + cmd = ( + "set -o pipefail; " + " ".join(find_parts) + + f" 2>/dev/null | head -n {fetch_limit}" + ) + result = self._exec(cmd, timeout=60) + return _parse_search_output(result, output_mode, limit, offset, context) diff --git a/tools/file_state.py b/tools/file_state.py index f22a966e1d..893a1f1a95 100644 --- a/tools/file_state.py +++ b/tools/file_state.py @@ -1,34 +1,24 @@ """Cross-agent file state coordination. Prevents mangled edits when concurrent subagents (same process, same -filesystem) touch the same file. Complements the single-agent path-overlap -check in ``run_agent._should_parallelize_tool_batch`` — this module catches -the case where subagent B writes a file that subagent A already read, so -A's next write would overwrite B's changes with stale content. +filesystem) touch the same file: subagent B writes a file that subagent A +already read, so A's next write would clobber B's changes with stale content. +Complements the single-agent path-overlap check in +``run_agent._should_parallelize_tool_batch``. -Design ------- -A process-wide singleton ``FileStateRegistry`` tracks, per resolved path: +A process-wide ``FileStateRegistry`` tracks, per resolved path: + * per-agent read stamps {task_id: {path: (mtime, read_ts, partial)}} + * last writer globally {path: (task_id, write_ts)} + * a per-path ``threading.Lock`` for read->modify->write sections - * per-agent read stamps: {task_id: {path: (mtime, read_ts, partial)}} - * last writer globally: {path: (task_id, write_ts)} - * per-path ``threading.Lock`` for read→modify→write critical sections +Hooks used by the file tools: ``record_read`` (read_file), ``note_write`` +(after write_file/patch), ``check_stale`` (BEFORE write_file/patch), +``lock_path`` (wrap the whole read->modify->write block) and ``writes_since`` +(delegate_tool's subagent-completion reminder). -Three public hooks are used by the file tools: - - * ``record_read(task_id, path, *, partial)`` — called by read_file - * ``note_write(task_id, path)`` — called after write_file / patch - * ``check_stale(task_id, path)`` — called BEFORE write_file / patch - -Plus ``lock_path(path)`` — a context-manager returning a per-path lock to -wrap the whole read→modify→write block. And ``writes_since(task_id, -since_ts, paths)`` for the subagent-completion reminder in delegate_tool. - -All methods are no-ops when ``HERMES_DISABLE_FILE_STATE_GUARD=1`` is set. - -This module is intentionally separate from ``_read_tracker`` in -``file_tools.py`` — that tracker is per-task and handles consecutive-read -loop detection, which is a different concern. +All methods are no-ops when ``HERMES_DISABLE_FILE_STATE_GUARD=1``. This is +separate from ``file_tools._read_tracker``, which handles per-task +consecutive-read loop detection. """ from __future__ import annotations @@ -40,19 +30,14 @@ from contextlib import contextmanager from pathlib import Path from typing import Dict, Iterable, List, Optional, Tuple - -# ── Public stamp type ──────────────────────────────────────────────── -# (mtime, read_ts, partial). partial=True when read_file returned a -# windowed view (offset > 1 or limit < total_lines) — writes that happen -# after a partial read should still warn so the model re-reads in full. +# (mtime, read_ts, partial). partial=True when read_file returned a windowed +# view (offset > 1 or limit < total_lines) — a later write should still warn +# so the model re-reads in full. ReadStamp = Tuple[float, float, bool] -# Number of resolved-path entries retained per agent. Bounded to keep -# long sessions from accumulating unbounded state. On overflow we drop -# the oldest entries by insertion order. +# Bounded so long sessions don't accumulate unbounded state; oldest by +# insertion order are dropped on overflow. _MAX_PATHS_PER_AGENT = 4096 - -# Global last-writer map cap. Same policy. _MAX_GLOBAL_WRITERS = 4096 @@ -66,7 +51,6 @@ class FileStateRegistry: self._meta_lock = threading.Lock() # guards _path_locks self._state_lock = threading.Lock() # guards _reads + _last_writer - # ── Path lock management ──────────────────────────────────────── def _lock_for(self, resolved: str) -> threading.Lock: with self._meta_lock: lock = self._path_locks.get(resolved) @@ -77,11 +61,7 @@ class FileStateRegistry: @contextmanager def lock_path(self, resolved: str): - """Acquire the per-path lock for a read→modify→write section. - - Same process, same filesystem — threads on the same path serialize. - Different paths proceed in parallel. - """ + """Per-path lock: threads on the same path serialize, different paths proceed.""" lock = self._lock_for(resolved) lock.acquire() try: @@ -89,7 +69,6 @@ class FileStateRegistry: finally: lock.release() - # ── Read/write accounting ─────────────────────────────────────── def record_read( self, task_id: str, @@ -100,11 +79,9 @@ class FileStateRegistry: ) -> None: if _disabled(): return + mtime = _mtime_or_none(resolved) if mtime is None else mtime if mtime is None: - try: - mtime = os.path.getmtime(resolved) - except OSError: - return + return now = time.time() with self._state_lock: agent_reads = self._reads[task_id] @@ -118,38 +95,27 @@ class FileStateRegistry: *, mtime: Optional[float] = None, ) -> None: - """Record a successful write. - - Updates the global last-writer map AND this agent's own read stamp - (a write is an implicit read — the agent now knows the current - content). - """ + """Record a successful write: global last-writer AND this agent's own + read stamp (a write is an implicit read of the current content).""" if _disabled(): return + mtime = _mtime_or_none(resolved) if mtime is None else mtime if mtime is None: - try: - mtime = os.path.getmtime(resolved) - except OSError: - return + return now = time.time() with self._state_lock: self._last_writer[resolved] = (task_id, now) _cap_dict(self._last_writer, _MAX_GLOBAL_WRITERS) - # Writer's own view is now up-to-date. self._reads[task_id][resolved] = (float(mtime), now, False) _cap_dict(self._reads[task_id], _MAX_PATHS_PER_AGENT) def check_stale(self, task_id: str, resolved: str) -> Optional[str]: - """Return a model-facing warning if this write would be stale. + """Model-facing warning if this write would be stale, else ``None``. - Three staleness classes, in order of severity: - - 1. Sibling subagent wrote this file after this agent's last read. - 2. External/unknown change (mtime differs from our last read). - 3. Agent never read the file (write-without-read). - - Returns ``None`` when the write is safe. Does not raise — callers - decide whether to block or warn. + Checked in severity order: (1) a sibling subagent wrote after this + agent's last read; (2) mtime drifted since our read (external edit) or + the read was partial; (3) this agent never read the file. Never raises + — callers decide whether to block or warn. """ if _disabled(): return None @@ -157,19 +123,14 @@ class FileStateRegistry: stamp = self._reads.get(task_id, {}).get(resolved) last_writer = self._last_writer.get(resolved) - # Case 3: never read AND we have no write record — net-new file or - # first touch by this agent. Let existing _check_sensitive_path - # and file-exists logic handle it; nothing to warn about here. + # Never read and no write record: net-new file or first touch — + # existing sensitive-path / file-exists logic handles it. if stamp is None and last_writer is None: return None + current_mtime = _mtime_or_none(resolved) + if current_mtime is None: + return None # file doesn't exist — write creates it; not stale - try: - current_mtime = os.path.getmtime(resolved) - except OSError: - # File doesn't exist — write will create it; not stale. - return None - - # Case 1: sibling subagent modified after our last read. if last_writer is not None: writer_tid, writer_ts = last_writer if writer_tid != task_id: @@ -189,7 +150,6 @@ class FileStateRegistry: "Re-read the file before writing." ) - # Case 2: external / unknown modification (mtime drifted). if stamp is not None: read_mtime, _read_ts, partial = stamp if current_mtime != read_mtime: @@ -204,53 +164,41 @@ class FileStateRegistry: "(partial view). Re-read the whole file before " "overwriting it." ) + return None - # Case 3b: agent truly never read the file. - if stamp is None: - return ( - f"{resolved} was not read by this agent. " - "Read the file first so you can write an informed edit." - ) + return ( + f"{resolved} was not read by this agent. " + "Read the file first so you can write an informed edit." + ) - return None - - # ── Reminder helper for delegate_tool ─────────────────────────── def writes_since( self, exclude_task_id: str, since_ts: float, paths: Iterable[str], ) -> Dict[str, List[str]]: - """Return ``{writer_task_id: [paths]}`` for writes done after - ``since_ts`` by agents OTHER than ``exclude_task_id``. - - Used by delegate_task to append a "subagent modified files the - parent previously read" reminder to the delegation result. - """ + """``{writer_task_id: [paths]}`` for writes after ``since_ts`` by agents + other than ``exclude_task_id`` (delegate_task's "subagent modified files + you previously read" reminder).""" if _disabled(): return {} paths_set = set(paths) out: Dict[str, List[str]] = defaultdict(list) with self._state_lock: for p, (writer_tid, ts) in self._last_writer.items(): - if writer_tid == exclude_task_id: - continue - if ts < since_ts: - continue - if p in paths_set: + if writer_tid != exclude_task_id and ts >= since_ts and p in paths_set: out[writer_tid].append(p) return dict(out) def known_reads(self, task_id: str) -> List[str]: - """Return the list of resolved paths this agent has read.""" + """Resolved paths this agent has read.""" if _disabled(): return [] with self._state_lock: return list(self._reads.get(task_id, {}).keys()) - # ── Testing hooks ─────────────────────────────────────────────── def clear(self) -> None: - """Reset all state. Intended for tests only.""" + """Reset all state. Intended for tests only.""" with self._state_lock: self._reads.clear() self._last_writer.clear() @@ -258,7 +206,6 @@ class FileStateRegistry: self._path_locks.clear() -# ── Module-level singleton + helpers ───────────────────────────────── _registry = FileStateRegistry() @@ -271,18 +218,22 @@ def _disabled() -> bool: return os.environ.get("HERMES_DISABLE_FILE_STATE_GUARD", "").strip() == "1" +def _mtime_or_none(resolved: str) -> Optional[float]: + try: + return os.path.getmtime(resolved) + except OSError: + return None + + def _fmt_ts(ts: float) -> str: - # Short relative wall-clock for error messages; avoids pulling in - # datetime formatting overhead on the hot path. return time.strftime("%H:%M:%S", time.localtime(ts)) def _cap_dict(d: dict, limit: int) -> None: - """Trim a dict to ``limit`` entries by dropping insertion-order oldest.""" + """Trim ``d`` to ``limit`` entries by dropping the insertion-order oldest.""" over = len(d) - limit if over <= 0: return - # dict preserves insertion order (PY>=3.7) — pop the oldest keys. it = iter(d) for _ in range(over): try: @@ -291,7 +242,7 @@ def _cap_dict(d: dict, limit: int) -> None: break -# ── Convenience wrappers (short names used at call sites) ──────────── +# Convenience wrappers (short names used at call sites). def record_read(task_id: str, resolved_or_path: str | Path, *, partial: bool = False) -> None: _registry.record_read(task_id, str(resolved_or_path), partial=partial) diff --git a/tools/file_tools.py b/tools/file_tools.py index a2a55236f3..c34801f8fd 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -1,22 +1,26 @@ #!/usr/bin/env python3 -"""File Tools Module - LLM agent file manipulation tools.""" +"""File Tools Module - LLM agent file manipulation tools. + +Companion modules (every name is re-imported here so ``tools.file_tools.X`` +keeps working for callers and test patches): + * ``file_tools_paths`` — task-aware path resolution / ``~`` expansion. + * ``file_tools_write_guards`` — sensitive-path, protected-instruction, + approval, mirror and binary-document guards. + * ``file_tools_read_tracking`` — per-task dedup / loop-detection / staleness state. +""" import base64 import errno import json import logging import os -import posixpath -import sys +import re +import stat import threading -from pathlib import Path, PurePosixPath +from pathlib import Path from agent.file_safety import get_read_block_error -from tools.binary_extensions import ( - has_binary_extension, - has_opaque_document_extension, - is_pdf_path, -) +from tools.binary_extensions import has_binary_extension from tools.file_operations import ( ShellFileOperations, normalize_read_pagination, @@ -24,55 +28,82 @@ from tools.file_operations import ( ) from tools import file_state from agent.redact import redact_sensitive_text +from tools.file_tools_paths import ( # noqa: F401 (re-exported) + _CONTAINER_PATH_BACKENDS_FALLBACK, + _TERMINAL_CWD_SENTINELS, + _authoritative_workspace_root, + _configured_terminal_cwd, + _expand_tilde, + _normalize_without_host_deref, + _path_resolution_warning, + _registered_task_cwd_override, + _resolve_base_dir, + _resolve_path, + _resolve_path_for_task, + _sentinel_free_abs_cwd, + _terminal_env_type_for_task, + _uses_container_paths, +) +from tools.file_tools_write_guards import ( # noqa: F401 (re-exported) + _PROTECTED_INSTRUCTION_BASENAMES, + _READ_DEDUP_STATUS_MESSAGE, + _SENSITIVE_EXACT_PATHS, + _SENSITIVE_PATH_PREFIXES, + _check_approval_required_write, + _check_binary_document_write, + _check_cross_profile_path, + _check_protected_instruction_write, + _check_sensitive_path, + _get_container_mirror_prefix_for_task, + _get_hermes_config_resolved, + _get_real_hermes_home, + _is_internal_file_status_text, + _is_internal_file_tool_content, + _looks_like_read_file_line_numbered_content, + _protected_instruction_config, + _protected_instruction_reason, + _request_protected_instruction_approval, +) +from tools.file_tools_read_tracking import ( # noqa: F401 (re-exported) + _DEDUP_CAP, + _NOT_FOUND_CAP, + _NOT_FOUND_TTL_SECONDS, + _READ_HISTORY_CAP, + _READ_TIMESTAMPS_CAP, + _bump_consecutive, + _cap_read_tracker_data, + _check_file_staleness, + _check_not_found_cache, + _invalidate_dedup_for_path, + _mark_verification_stale, + _patch_failure_lock, + _patch_failure_tracker, + _read_tracker, + _read_tracker_lock, + _record_not_found, + _record_patch_failure, + _reset_patch_failures, + _task_data, + _update_read_timestamp, + notify_other_tool_call, + reset_file_dedup, +) logger = logging.getLogger(__name__) _EXPECTED_WRITE_ERRNOS = {errno.EACCES, errno.EPERM, errno.EROFS} - -def _expand_tilde(path: str) -> str: - """Expand ``~`` using the effective profile home when available. - - In-process file tools share the gateway process's HOME, which may differ - from the profile-specific HOME that interactive CLI sessions use. This - mirrors ``hermes_constants.get_subprocess_home()`` so that ``~`` resolves - consistently regardless of whether the tool runs interactively or inside a - gateway-driven cron job (#48552). - """ - if not path or "~" not in path: - return path - try: - from hermes_constants import get_subprocess_home - - home = get_subprocess_home() - except Exception: - home = None - if home and (path == "~" or path.startswith("~/")): - return home if path == "~" else os.path.join(home, path[2:]) - return os.path.expanduser(path) - - # --------------------------------------------------------------------------- -# Read-size guard: cap the character count returned to the model. -# We're model-agnostic so we can't count tokens; characters are a safe proxy. -# 100K chars ≈ 25–35K tokens across typical tokenisers. Files larger than -# this in a single read are a context-window hazard — the model should use -# offset+limit to read the relevant section. -# -# Configurable via config.yaml: file_read_max_chars: 200000 +# Read-size guard. Model-agnostic, so characters proxy tokens: 100K chars is +# ~25-35K tokens across typical tokenisers. Configurable: file_read_max_chars. # --------------------------------------------------------------------------- _DEFAULT_MAX_READ_CHARS = 100_000 _max_read_chars_cached: int | None = None def _get_max_read_chars() -> int: - """Return the configured max characters per file read. - - Reads ``file_read_max_chars`` from config.yaml on first call, caches - the result for the lifetime of the process. Falls back to the - built-in default if the config is missing or invalid. - """ + """Return ``file_read_max_chars`` from config.yaml (cached per process; default on missing/invalid).""" global _max_read_chars_cached if _max_read_chars_cached is not None: return _max_read_chars_cached @@ -90,26 +121,14 @@ def _get_max_read_chars() -> int: def _truncate_to_char_budget(content: str, max_chars: int) -> tuple[str, int, bool]: - """Trim line-numbered ``read_file`` content to fit a char budget. + """Trim line-numbered ``read_file`` content to the last COMPLETE line within *max_chars*. - Ported in spirit from nearai/ironclaw#5029 (dual line/byte cap on - ``read_file``). Where hermes previously hard-rejected an oversized read - (forcing the model to guess a smaller ``limit`` and burn a round-trip - returning nothing), this trims the content to the last *complete line* - that fits within ``max_chars`` and reports how many lines were kept so - the caller can offer a ``next_offset`` continuation. - - ``content`` is the gutter-rendered text (``LINE_NUM|CONTENT`` joined by - ``\\n``). Individual lines are already clamped to ``get_max_line_length()`` - upstream, so a single line never blows the whole budget on its own; the - overflow this handles is the *accumulation* of many lines under the - line-count limit (logs, wide CSV rows, minified data). - - Returns ``(kept_text, lines_kept, truncated)``. When ``content`` already - fits, returns it unchanged with ``truncated=False``. If not even the - first line fits, that single line is clamped on a code-point boundary - (Python ``str`` slicing never splits a code point) so the read never - returns empty and the cursor can still advance. + Returns ``(kept_text, lines_kept, truncated)`` so the caller can offer a + ``next_offset`` instead of rejecting the read. Lines are already clamped to + ``get_max_line_length()`` upstream; the overflow handled here is the + accumulation of many lines under the line-count limit (logs, wide CSV). + If not even the first line fits it is clamped mid-line (Python slicing + never splits a code point) so the read is never empty and the cursor advances. """ if len(content) <= max_chars: return content, (content.count("\n") + 1 if content else 0), False @@ -118,346 +137,67 @@ def _truncate_to_char_budget(content: str, max_chars: int) -> tuple[str, int, bo kept: list[str] = [] running = 0 for line in lines: - # +1 for the "\n" that rejoins this line to the previous one. - addition = len(line) + (1 if kept else 0) + addition = len(line) + (1 if kept else 0) # +1 for the rejoining "\n" if running + addition > max_chars: break kept.append(line) running += addition if not kept: - # First line alone exceeds the budget. Clamp on a code-point - # boundary rather than emitting nothing. kept.append(lines[0][:max_chars]) return "\n".join(kept), len(kept), True -# If the total file size exceeds this AND the caller didn't specify a narrow -# range (limit <= 200), we include a hint encouraging targeted reads. -_LARGE_FILE_HINT_BYTES = 512_000 # 512 KB - -# --------------------------------------------------------------------------- -# Device path blocklist — reading these hangs the process (infinite output -# or blocking on input). Checked by path only (no I/O). -# --------------------------------------------------------------------------- -_BLOCKED_DEVICE_PATHS = frozenset({ - # Infinite output — never reach EOF - "/dev/zero", "/dev/random", "/dev/urandom", "/dev/full", - # Blocks waiting for input - "/dev/stdin", "/dev/tty", "/dev/console", - # Nonsensical to read - "/dev/stdout", "/dev/stderr", - # fd aliases - "/dev/fd/0", "/dev/fd/1", "/dev/fd/2", -}) - - -def _resolve_path(filepath: str, task_id: str = "default") -> Path | PurePosixPath: - """Resolve a path relative to TERMINAL_CWD (the worktree base directory) - instead of the main repository root. - """ - return _resolve_path_for_task(filepath, task_id) - - -# Sentinel ``TERMINAL_CWD`` values that mean "not configured", NOT a literal -# directory to resolve against. A stale config / .env commonly leaves the -# literal "." here; "auto"/"cwd" are setup-wizard placeholders. Treating any of -# these as a real relative base silently anchors edits to the agent PROCESS cwd -# (e.g. the main repo while a worktree session is active), routing writes to the -# wrong checkout. The gateway sanitizes the same set at import time -# (gateway/run.py); the file/terminal-tool layer must do likewise so CLI -# sessions get the same protection. See references/worktree-cwd-discipline.md. -_TERMINAL_CWD_SENTINELS = frozenset({"", ".", "./", "auto", "cwd"}) -_CONTAINER_PATH_BACKENDS_FALLBACK = frozenset({"docker", "singularity", "modal", "daytona", "vercel_sandbox"}) - - -def _terminal_env_type_for_task(task_id: str = "default") -> str: - """Best-effort terminal backend type for path-resolution decisions.""" - try: - from tools.terminal_tool import ( - _active_environments, - _env_lock, - _get_env_config, - _resolve_container_task_id, +def _apply_char_budget(result_dict: dict, content: str, offset: int, total_lines, max_chars: int) -> str: + """Trim *content* to the char budget, annotate *result_dict* with the + continuation hint, and return the trimmed text.""" + trimmed, lines_kept, _ = _truncate_to_char_budget(content, max_chars) + next_offset = offset + lines_kept + shown_end = offset + lines_kept - 1 + result_dict["content"] = trimmed + result_dict["truncated"] = True + result_dict["truncated_by"] = "bytes" + result_dict["next_offset"] = next_offset + result_dict["hint"] = ( + f"Output truncated at the {max_chars:,}-char read budget after " + f"{lines_kept} line(s) (showing lines {offset}-{shown_end} of " + f"{total_lines}). Use offset={next_offset} to continue." + ) + if len(trimmed.split("\n", 1)[0]) >= max_chars: + result_dict["hint"] += ( + " Note: the first line alone exceeded the budget and was " + "clamped mid-line; its remainder is not retrievable via " + "offset." ) - - try: - container_key = _resolve_container_task_id(task_id) - except Exception: - container_key = task_id - with _env_lock: - env = _active_environments.get(container_key) or _active_environments.get(task_id) - if env is not None: - name = env.__class__.__name__.lower() - if "local" in name: - return "local" - if "ssh" in name: - return "ssh" - if "docker" in name: - return "docker" - if "singularity" in name: - return "singularity" - if "modal" in name: - return "modal" - if "daytona" in name: - return "daytona" - stamped = getattr(env, "_hermes_backend_name", None) - if isinstance(stamped, str) and stamped: - return stamped - cfg = _get_env_config() - return str(cfg.get("env_type") or os.getenv("TERMINAL_ENV") or "local").lower() - except Exception: - return str(os.getenv("TERMINAL_ENV") or "local").lower() + return trimmed -def _uses_container_paths(task_id: str = "default") -> bool: - env_type = _terminal_env_type_for_task(task_id) - try: - from tools.terminal_tool import _is_container_backend +# Above this size, a wide read (limit > 200) gets a hint toward targeted reads. +_LARGE_FILE_HINT_BYTES = 512_000 - return _is_container_backend(env_type) - except Exception: - return env_type in _CONTAINER_PATH_BACKENDS_FALLBACK - - -def _normalize_without_host_deref(path: str | Path | PurePosixPath) -> PurePosixPath: - """Normalize path syntax without following host symlinks. - - Container backends use paths that are meaningful inside the sandbox. Calling - ``Path.resolve()`` on the host can dereference a host-side symlink such as - ``/workspace`` and rewrite the path before Docker sees it. - """ - return PurePosixPath(posixpath.normpath(str(path))) - - -def _sentinel_free_abs_cwd(raw: str | None) -> str | None: - """Normalize a cwd candidate to an absolute, sentinel-free anchor. - - Returns the expanded path only when *raw* is non-empty, not a sentinel (see - ``_TERMINAL_CWD_SENTINELS``), and absolute. A relative anchor is meaningless - without knowing which cwd it is relative to — exactly the ambiguity that - misroutes worktree edits — so relative/sentinel/empty values yield ``None``. - """ - raw = str(raw or "").strip() - if raw.lower() in _TERMINAL_CWD_SENTINELS: - return None - expanded = _expand_tilde(raw) - if not os.path.isabs(expanded): - return None - return expanded - - -def _configured_terminal_cwd() -> str | None: - """Return ``$TERMINAL_CWD`` only when it names a real directory anchor. - - Sentinel values (see ``_TERMINAL_CWD_SENTINELS``) and relative paths are - rejected — a relative anchor is meaningless without knowing which cwd it is - relative to, which is exactly the ambiguity that misroutes worktree edits. - Only an absolute, sentinel-free value is honored. - """ - # Scope-aware: under gateway multiplexing the routed profile's cwd lives in - # the per-turn terminal scope, not the process env (#68559). - from agent.runtime_cwd import scope_terminal_cwd - - return _sentinel_free_abs_cwd(scope_terminal_cwd() or None) - - -def _registered_task_cwd_override(task_id: str = "default") -> str | None: - """Return a registered cwd override for the raw task id, when available. - - ``terminal_tool`` intentionally collapses CWD-only task overrides to the - shared ``"default"`` environment so TUI/dashboard/ACP sessions do not spin - up isolated sandboxes just because they have different workspaces. The cwd - value itself is still keyed by the raw session/task id, so file tools must - read that raw override before falling back to the collapsed container key. - """ - try: - from tools.terminal_tool import resolve_task_overrides - - overrides = resolve_task_overrides(task_id) - except Exception: - return None - - return _sentinel_free_abs_cwd(overrides.get("cwd")) - - -def _authoritative_workspace_root(task_id: str = "default") -> str | None: - """Best-effort absolute workspace root for divergence checks. - - Resolution: - - 1. The session's own cwd RECORD (``terminal_tool.get_session_cwd``) — - written on every completed terminal command and seeded by workspace - registration, keyed by the raw session id. Because the record is - per-session, one session's ``cd`` can never leak into another - session's resolution. - 2. A registered task/session cwd override (TUI/Desktop/ACP sessions - register a raw-keyed cwd before any tool runs). Normally already - mirrored into the record at registration; kept as a direct fallback - so a cleared/never-written record still resolves the workspace. - 3. A sentinel-free absolute ``$TERMINAL_CWD`` (the worktree path set by - ``cli.py``/``main.py`` for ``-w`` sessions). - - Returns ``None`` only when there is genuinely no reliable anchor, in which - case callers fall back to the process cwd. - """ - try: - from tools.terminal_tool import get_session_cwd - - recorded = get_session_cwd(task_id) - except Exception: - recorded = None - if recorded: - return recorded - registered = _registered_task_cwd_override(task_id) - if registered: - return registered - return _configured_terminal_cwd() - - -def _resolve_base_dir( - task_id: str = "default", - *, - container_paths: bool | None = None, -) -> Path | PurePosixPath: - """Return the ABSOLUTE base directory for resolving relative paths. - - Resolution order: - 1. The task's live terminal cwd (the directory the agent is actually - working in — e.g. a git worktree). Authoritative when known. - 2. A registered task/session cwd override (TUI/Desktop/ACP sessions - register a raw-keyed workspace cwd before any terminal command runs). - 3. A sentinel-free, absolute ``$TERMINAL_CWD`` (the worktree path set by - ``cli.py``/``main.py`` for ``-w`` sessions). Used even before any - terminal command has populated the live cwd registry. - 4. The process cwd. - - The returned base is ALWAYS absolute. This is the core invariant that - prevents the worktree-cwd divergence bug: a relative or sentinel - ``TERMINAL_CWD`` (commonly the literal ``"."`` from a stale config) is - meaningless as a resolution anchor — left to ``Path.resolve()`` it silently - resolves against whatever the agent PROCESS cwd happens to be (e.g. the main - repo while the terminal is in a worktree), routing edits to the wrong - checkout. We therefore reject sentinel/relative ``TERMINAL_CWD`` values - outright (rather than anchoring them to the process cwd) and fall through to - the process cwd only as a last resort, deterministically. - """ - root = _authoritative_workspace_root(task_id) - if container_paths is None: - container_paths = _uses_container_paths(task_id) - if root: - base_text = _expand_tilde(root) - else: - base_text = os.getcwd() - if container_paths: - if not posixpath.isabs(base_text): - base_text = posixpath.join(os.getcwd(), base_text) - return _normalize_without_host_deref(base_text) - # Git Bash ``pwd -P`` reports ``/c/Users/...``; translate before Path so - # relative file-tool paths don't anchor under a nonexistent ``\\c\\Users``. - from tools.environments.local import _msys_to_windows_path - - base_text = _msys_to_windows_path(base_text) - if sys.platform == "win32": - import ntpath - - if not ntpath.isabs(base_text): - base_text = ntpath.join(os.getcwd(), base_text) - return Path(ntpath.normpath(base_text)) - base = Path(base_text) - if not base.is_absolute(): - # Last-resort anchoring: a live cwd should already be absolute, but if a - # terminal backend ever reports a relative cwd, anchor it to the process - # cwd once, here, so the result no longer depends on cwd at resolve(). - base = Path(os.getcwd()) / base - return base.resolve() - - -def _resolve_path_for_task(filepath: str, task_id: str = "default") -> Path | PurePosixPath: - """Resolve *filepath* against the task's absolute base directory. - - See :func:`_resolve_base_dir` for how the base is chosen. Absolute input - paths are returned resolved-but-unanchored. - - On native Windows, Git Bash / MSYS drive paths (``/c/Users/...``) are - translated to ``C:\\Users\\...`` before resolution so file tools don't - treat them as relative ``\\c\\Users\\...`` under the process cwd. - """ - container_paths = _uses_container_paths(task_id) - if container_paths: - expanded = _expand_tilde(filepath) - if posixpath.isabs(expanded): - return _normalize_without_host_deref(expanded) - resolved = _resolve_base_dir(task_id, container_paths=True) / expanded - return _normalize_without_host_deref(resolved) - - # Host paths only — never rewrite Linux paths inside a container/WSL env. - from tools.environments.local import _msys_to_windows_path - - expanded = _expand_tilde(_msys_to_windows_path(filepath)) - if sys.platform == "win32": - import ntpath - - if ntpath.isabs(expanded): - return Path(ntpath.normpath(expanded)) - joined = ntpath.join(str(_resolve_base_dir(task_id, container_paths=False)), expanded) - return Path(ntpath.normpath(joined)) - - p = Path(expanded) - if p.is_absolute(): - return p.resolve() - resolved = _resolve_base_dir(task_id, container_paths=False) / p - return resolved.resolve() - - -def _path_resolution_warning(filepath: str, resolved: Path, task_id: str = "default") -> str | None: - """Warn when a relative path resolved OUTSIDE the task's workspace root. - - Surfaces the worktree-cwd divergence the moment it would matter: if the - agent passes a relative path but it resolves under a directory that is not - the workspace root (i.e. the edit is about to land in a different checkout - than the one the agent is working in), return a message naming the absolute - target. ``None`` when the path is absolute, the base is unknown, or the - resolved path is correctly under the workspace root. - - The workspace root is the live terminal cwd when known, else a registered - task/session cwd override, else a sentinel-free absolute ``$TERMINAL_CWD`` - — so a worktree or Desktop session whose terminal registry is still empty - (no ``cd`` run yet) is warned on the very first write. - """ - try: - if Path(_expand_tilde(filepath)).is_absolute(): - return None - workspace_root = _authoritative_workspace_root(task_id) - if not workspace_root: - return None # No authoritative workspace root to compare against. - if _uses_container_paths(task_id): - root = _normalize_without_host_deref(Path(_expand_tilde(workspace_root))) - else: - root = Path(_expand_tilde(workspace_root)).resolve() - # Is `resolved` inside `root`? - try: - resolved.relative_to(root) - return None # Inside the workspace — expected. - except ValueError: - return ( - f"Relative path {filepath!r} resolved to {str(resolved)!r}, which is " - f"OUTSIDE the active workspace ({str(root)!r}). The edit will land in " - f"a different directory than the terminal's cwd. If this is not " - f"intended (e.g. a git-worktree session writing into the main " - f"checkout), pass an absolute path under the workspace instead." - ) - except Exception: - return None +# Device/fd paths whose reads hang the process (infinite output or blocking on +# input). Checked by path only — no I/O. +_BLOCKED_DEVICE_PATHS = frozenset({ + "/dev/zero", "/dev/random", "/dev/urandom", "/dev/full", # never reach EOF + "/dev/stdin", "/dev/tty", "/dev/console", # block on input + "/dev/stdout", "/dev/stderr", # nonsensical to read + "/dev/fd/0", "/dev/fd/1", "/dev/fd/2", # fd aliases +}) +# /proc//... (and /proc//task//...) files that leak secrets, +# argv, memory layout (ASLR oracle: maps family, auxv, pagemap) or raw memory. +_BLOCKED_PROC_SUFFIXES = ( + "/fd/0", "/fd/1", "/fd/2", # stdio aliases + "/environ", "/cmdline", "/maps", "/smaps", "/smaps_rollup", "/numa_maps", + "/mem", "/auxv", "/pagemap", +) def _file_ops_uses_host_paths(file_ops) -> bool: - """Return True when *file_ops* targets the same host filesystem as Hermes. + """True when *file_ops* targets the same host filesystem as Hermes. - Only then may we rewrite V4A header paths to resolved host-absolute - paths: a container/remote backend has its own filesystem namespace where - a host-absolute path would be meaningless. + Only then may we stat paths or rewrite V4A headers to host-absolute paths; + a container/remote backend has its own filesystem namespace. """ env = getattr(file_ops, "env", None) if env is None: @@ -469,104 +209,55 @@ def _file_ops_uses_host_paths(file_ops) -> bool: return isinstance(env, LocalEnvironment) +# V4A file headers. ``\s*`` after ``***`` mirrors patch_parser's leniency +# (``***Update File:`` with no space parses and applies, so it must be checked). +_V4A_SINGLE_HEADER_RE = re.compile(r'^(\*\*\*\s*(?:Update|Add|Delete)\s+File:\s*)(.+)$', re.MULTILINE) +_V4A_SINGLE_HEADER_OP_RE = re.compile(r'^\*\*\*\s*(Update|Add|Delete)\s+File:\s*(.+)$', re.MULTILINE) +_V4A_MOVE_HEADER_RE = re.compile(r'^(\*\*\*\s*Move\s+File:\s*)(.+?)\s*->\s*(.+)$', re.MULTILINE) + + def _rewrite_v4a_patch_paths_for_host( patch: str, path_to_resolved: dict, file_ops, ) -> str: - """Rewrite V4A file headers to the exact host paths the tool layer resolved. + """Rewrite V4A ``*** Update/Add/Delete/Move File:`` headers to the resolved host paths. - ``patch_tool`` resolves every header path against the task's workspace for - locking, staleness, and reporting, but historically handed the *original* - patch text to ``file_ops.patch_v4a`` — so the shell layer re-resolved the - (often relative) header against its own cwd, which can differ from the - tool layer's workspace (the git-worktree cwd bug). That made a relative - header land in a different directory than everything else the tool - reported. This rewrites ``*** Update/Add/Delete/Move File:`` headers to the - resolved absolute paths so both layers agree on the target. - - Header patterns mirror ``patch_parser`` (``\\s*`` after ``***`` accepts the - no-space ``***Update File:`` form) and cover ``Move File: src -> dst``. - Only applied when *file_ops* targets the host filesystem. + ``patch_tool`` resolves every header against the task's workspace for + locking/staleness/reporting; the shell layer must patch those SAME files + rather than re-resolving a relative header against its own cwd (which can + differ — the git-worktree cwd bug). Only applied for host-filesystem backends. """ if not _file_ops_uses_host_paths(file_ops): return patch - import re as _re - def _resolved_or_original(raw: str) -> str: raw = raw.strip() return path_to_resolved.get(raw) or raw - def _replace_single(match): - prefix = match.group(1) - resolved = _resolved_or_original(match.group(2)) - return f"{prefix}{resolved}" - - patch = _re.sub( - r'^(\*\*\*\s*(?:Update|Add|Delete)\s+File:\s*)(.+)$', - _replace_single, - patch, - flags=_re.MULTILINE, + patch = _V4A_SINGLE_HEADER_RE.sub( + lambda m: f"{m.group(1)}{_resolved_or_original(m.group(2))}", patch, ) - - def _replace_move(match): - prefix = match.group(1) - src = _resolved_or_original(match.group(2)) - dst = _resolved_or_original(match.group(3)) - return f"{prefix}{src} -> {dst}" - - patch = _re.sub( - r'^(\*\*\*\s*Move\s+File:\s*)(.+?)\s*->\s*(.+)$', - _replace_move, + return _V4A_MOVE_HEADER_RE.sub( + lambda m: f"{m.group(1)}{_resolved_or_original(m.group(2))} -> {_resolved_or_original(m.group(3))}", patch, - flags=_re.MULTILINE, ) - return patch def _is_blocked_device_path(path: str) -> bool: - """Return True for concrete device/fd paths that can hang reads.""" + """Return True for concrete device/fd/proc paths that can hang reads or leak process state.""" normalized = os.path.normpath(_expand_tilde(path)) if normalized in _BLOCKED_DEVICE_PATHS: return True - # /proc/self/fd/0-2 and /proc//fd/0-2 are Linux aliases for stdio - if normalized.startswith("/proc/") and normalized.endswith( - ("/fd/0", "/fd/1", "/fd/2") - ): - return True - # /proc/*/environ, /proc/*/cmdline, /proc/*/maps (and the maps variants - # smaps, smaps_rollup, numa_maps) can leak secrets, command-line args, and - # memory layout (ASLR bypass) from the host process (issue #4427). - # /proc/*/mem exposes raw process memory; block it as defense-in-depth even - # though it requires address knowledge to exploit usefully. - # /proc/*/auxv leaks AT_RANDOM (stack canary seed) plus AT_BASE/AT_PHDR - # load addresses — an ASLR oracle on par with maps. /proc/*/pagemap exposes - # virtual->physical translation. Both are blocked alongside the maps family. - # endswith matches both /proc//X and /proc//task//X. - if normalized.startswith("/proc/") and normalized.endswith( - ( - "/environ", - "/cmdline", - "/maps", - "/smaps", - "/smaps_rollup", - "/numa_maps", - "/mem", - "/auxv", - "/pagemap", - ) - ): - return True - return False + return normalized.startswith("/proc/") and normalized.endswith(_BLOCKED_PROC_SUFFIXES) def _is_blocked_device(filepath: str, base_dir: str | Path | None = None) -> bool: - """Return True if the path would hang the process (infinite output or blocking input). + """True if the path (literal, any symlink hop, or final realpath) is a blocked device. - Check the literal path first so aliases like /dev/stdin are caught before - they resolve to terminal-specific paths. Then check each symlink hop before - the final resolved path so aliases to devices cannot bypass the guard. + The literal path is checked first so aliases like /dev/stdin are caught + before they resolve to terminal-specific paths; each symlink hop is checked + so an alias to a device cannot bypass the guard. """ expanded = _expand_tilde(filepath) if base_dir is not None and not os.path.isabs(expanded): @@ -596,19 +287,12 @@ def _is_blocked_device(filepath: str, base_dir: str | Path | None = None) -> boo resolved = os.path.normpath(os.path.realpath(normalized)) except (OSError, ValueError): return False - if _is_blocked_device_path(resolved): - return True - return False + return _is_blocked_device_path(resolved) def _search_result_read_block_error(path: str, task_id: str = "default") -> str | None: - """Return the read-safety error for a search result path. - - Search backends may return paths relative to the task cwd, while - ``get_read_block_error`` expects an already-resolved path when the task cwd - can differ from the Python process cwd. Mirror ``read_file_tool``'s path - resolution before applying the shared read guard. - """ + """Read-safety error for a search result path, resolved against the task cwd + (search backends may return cwd-relative paths; the process cwd can differ).""" try: resolved = _resolve_path_for_task(path, task_id) except (OSError, ValueError, RuntimeError): @@ -617,818 +301,161 @@ def _search_result_read_block_error(path: str, task_id: str = "default") -> str def _filter_read_blocked_search_results(result, task_id: str = "default") -> int: - """Remove credential/cache/env paths from a SearchResult in-place.""" + """Remove credential/cache/env paths from a SearchResult in-place; return the omitted count.""" omitted = 0 - if hasattr(result, "matches") and result.matches: - allowed_matches = [] - for match in result.matches: - if _search_result_read_block_error(match.path, task_id): - omitted += 1 - continue - allowed_matches.append(match) - result.matches = allowed_matches - - if hasattr(result, "files") and result.files: - allowed_files = [] - for file_path in result.files: - if _search_result_read_block_error(file_path, task_id): - omitted += 1 - continue - allowed_files.append(file_path) - result.files = allowed_files - - if hasattr(result, "counts") and result.counts: - allowed_counts = {} - for file_path, count in result.counts.items(): - if _search_result_read_block_error(file_path, task_id): - omitted += 1 - continue - allowed_counts[file_path] = count - result.counts = allowed_counts + def _blocked(path: str) -> bool: + nonlocal omitted + if _search_result_read_block_error(path, task_id): + omitted += 1 + return True + return False + if getattr(result, "matches", None): + result.matches = [m for m in result.matches if not _blocked(m.path)] + if getattr(result, "files", None): + result.files = [f for f in result.files if not _blocked(f)] + if getattr(result, "counts", None): + result.counts = {f: c for f, c in result.counts.items() if not _blocked(f)} return omitted -# Paths that file tools should refuse to write to without going through the -# terminal tool's approval system. These match prefixes after os.path.realpath. -_SENSITIVE_PATH_PREFIXES = ( - "/etc/", "/boot/", "/usr/lib/systemd/", - "/private/etc/", - # macOS: /private/var mirrors /var. Block the sensitive subtrees, NOT the - # whole thing — a blanket "/private/var/" refused every legitimate temp-file - # write, because $TMPDIR, /tmp, and /var/folders all realpath() into - # /private/var/folders/... on macOS (and _resolve_path_for_task resolves - # symlinks), and /private/var/tmp is a normal temp dir. - "/private/var/db/", "/private/var/root/", -) -_SENSITIVE_EXACT_PATHS = {"/var/run/docker.sock", "/run/docker.sock"} - -_hermes_config_resolved: str | None = None -_hermes_config_resolved_loaded = False - - -def _get_hermes_config_resolved() -> str | None: - """Return the resolved absolute path of the Hermes config file (cached).""" - global _hermes_config_resolved, _hermes_config_resolved_loaded - if _hermes_config_resolved_loaded: - return _hermes_config_resolved - _hermes_config_resolved_loaded = True - try: - from hermes_cli.config import get_config_path - _hermes_config_resolved = str(get_config_path().resolve()) - except Exception: - try: - _hermes_config_resolved = str(Path(_expand_tilde("~/.hermes/config.yaml")).resolve()) - except Exception: - _hermes_config_resolved = None - return _hermes_config_resolved - - -def _check_sensitive_path(filepath: str, task_id: str = "default") -> str | None: - """Return an error message if the path targets a sensitive system location.""" - try: - resolved = str(_resolve_path_for_task(filepath, task_id)) - except (OSError, ValueError): - resolved = filepath - normalized = os.path.normpath(_expand_tilde(filepath)) - _err = ( - f"Refusing to write to sensitive system path: {filepath}\n" - "Use the terminal tool with sudo if you need to modify system files." - ) - for prefix in _SENSITIVE_PATH_PREFIXES: - if resolved.startswith(prefix) or normalized.startswith(prefix): - return _err - if resolved in _SENSITIVE_EXACT_PATHS or normalized in _SENSITIVE_EXACT_PATHS: - return _err - # Prevent agents from modifying the Hermes config file directly. - # approvals.mode and other security settings live here; a malicious or - # prompt-injected agent could silently disable exec approval by writing to - # this file. - hermes_config = _get_hermes_config_resolved() - if hermes_config and (resolved == hermes_config or normalized == hermes_config): - return ( - f"Refusing to write to Hermes config file: {filepath}\n" - "Agent cannot modify security-sensitive configuration. " - "Edit ~/.hermes/config.yaml directly or use 'hermes config' instead." - ) - return None - - -# --------------------------------------------------------------------------- -# Protected agent-instruction files (always-ask approval gate) -# --------------------------------------------------------------------------- -# Files that steer FUTURE agent behavior are a prompt-injection persistence -# vector: an injected instruction that edits AGENTS.md / CLAUDE.md / SOUL.md / -# .cursorrules (or a project-local .hermes config tree) outlives the current -# turn and poisons every later session that loads it. Writes to these files -# therefore ALWAYS require human approval — even under --yolo / auto-approve — -# and fail closed when no human channel exists. -# -# Ported from: RooCodeInc/Roo-Code RooProtectedController (Apache-2.0). -# Companion: the terminal-tool vector is covered separately (#58631); this -# gate covers the write_file/patch vector. Symlink lesson from #41351: -# always realpath before matching. -# -# Scope decision (documented): basenames match in ANY directory, because -# project-context instruction files are loaded from cwd trees — an -# AGENTS.md anywhere the agent might later run from is a live target. -# Basenames match case-insensitively so case-variant spellings on -# case-insensitive filesystems (macOS/Windows) cannot slip past; on -# case-sensitive filesystems most loaders probe common case variants too, -# so the stricter behavior is kept uniform. -_PROTECTED_INSTRUCTION_BASENAMES = frozenset({ - "agents.md", "claude.md", "soul.md", ".cursorrules", -}) - -_real_hermes_home_cached: str | None = None -_real_hermes_home_loaded = False - - -def _get_real_hermes_home() -> str | None: - """Return the realpath of the authoritative Hermes home (cached).""" - global _real_hermes_home_cached, _real_hermes_home_loaded - if _real_hermes_home_loaded: - return _real_hermes_home_cached - _real_hermes_home_loaded = True - try: - from hermes_constants import get_hermes_home - _real_hermes_home_cached = os.path.realpath(str(get_hermes_home())) - except Exception: - try: - _real_hermes_home_cached = os.path.realpath(_expand_tilde("~/.hermes")) - except Exception: - _real_hermes_home_cached = None - return _real_hermes_home_cached - - -def _protected_instruction_config() -> tuple[bool, list[str]]: - """Read the protected-instruction-files gate config. - - Returns ``(enabled, extra_patterns)``. Defaults to enabled with no extra - patterns; config read failures keep the gate ON (fail-safe for a - security boundary). - - Config keys (config.yaml):: - - security: - protected_instruction_files: true # default - protected_instruction_extra_patterns: [] # fnmatch on basename - """ - try: - from hermes_cli.config import load_config, cfg_get - cfg = load_config() - enabled = cfg_get(cfg, "security", "protected_instruction_files", - default=True) - extra = cfg_get(cfg, "security", "protected_instruction_extra_patterns", - default=[]) - except Exception: - return True, [] - if not isinstance(enabled, bool): - enabled = True - if not isinstance(extra, list): - extra = [] - return enabled, [str(p) for p in extra if p] - - -def _protected_instruction_reason(filepath: str, task_id: str = "default", - *, enabled: bool | None = None, - extra_patterns: list[str] | None = None) -> str | None: - """Return a short label when ``filepath`` targets a protected - agent-instruction file, else ``None``. - - Matching runs on BOTH the normalized input path and its realpath so - neither a symlink pointing AT a protected file (#41351) nor a protected - name that is itself a symlink escapes the gate. ``..`` traversal is - neutralized by normpath/realpath before the basename compare. - """ - if enabled is None or extra_patterns is None: - enabled, extra_patterns = _protected_instruction_config() - if not enabled: - return None - - normalized = os.path.normpath(_expand_tilde(filepath)) - try: - resolved = os.path.realpath(str(_resolve_path_for_task(filepath, task_id))) - except (OSError, ValueError, RuntimeError): - resolved = os.path.realpath(normalized) - - # The authoritative ~/.hermes home is governed by its own guards - # (config.yaml hard-block, cross-profile guard, write_approval); this - # gate targets PROJECT-LOCAL instruction files only. Checked before the - # ``.hermes`` component rule below, which would otherwise match the - # home directory itself. - real_home = _get_real_hermes_home() - if real_home and (resolved == real_home - or resolved.startswith(real_home + os.sep)): - return None - - import fnmatch - for candidate in (normalized, resolved): - base = os.path.basename(candidate) - base_lower = base.lower() - if base_lower in _PROTECTED_INSTRUCTION_BASENAMES: - return base - for pattern in extra_patterns: - if fnmatch.fnmatch(base_lower, pattern.lower()): - return base - # Project-local .hermes config dirs (e.g. /.hermes/config.yaml) - # are loaded as project context and steer behavior the same way. - # Scope: the file's IMMEDIATE parent must be ``.hermes`` — matching - # any ancestor named .hermes would gate every write inside a - # checkout that happens to live under ~/.hermes (e.g. the - # hermes-agent repo itself at ~/.hermes/hermes-agent). - parts = candidate.replace("\\", "/").rstrip("/").split("/") - if len(parts) >= 2 and parts[-2] == ".hermes": - return candidate - return None - - -def _request_protected_instruction_approval( - reasons: list[str], task_id: str = "default") -> str | None: - """Ask the human to approve a write to protected instruction file(s). - - Returns ``None`` when approved, or a BLOCKED error string. This gate - intentionally does NOT route through ``_run_approval_gate``: that gate - honors --yolo and session/permanent allowlists, and the entire point - here is one-operation approval EVERY time, with no persistent scope - and no yolo bypass. Fail-closed when no human channel exists. - """ - targets = ", ".join(dict.fromkeys(reasons)) - description = ( - f"Write to protected agent-instruction file(s): {targets}. " - "These files steer future agent behavior; approval is always " - "required (not bypassed by auto-approve)." - ) - display = f"" - blocked = ( - f"BLOCKED: write to protected agent-instruction file(s) ({targets}) " - "{why} The user has NOT consented to this write. Do NOT retry it or " - "attempt the same edit via another path (terminal, execute_code, " - "etc.)." - ) - - try: - import tools.approval as _approval - except Exception: - return blocked.format(why="requires approval but the approval " - "subsystem is unavailable.") - - # Gateway surface: block on the button round-trip when a notify callback - # is registered for this session (Telegram/Discord/Slack). One-operation - # only — no session/permanent buttons are offered. - session_key = _approval.get_current_session_key() - notify_cb = None - try: - with _approval._lock: - notify_cb = _approval._gateway_notify_cbs.get(session_key) - except Exception: - notify_cb = None - - if notify_cb is not None: - approval_data = { - "command": display, - "pattern_key": "protected_instruction_file", - "pattern_keys": ["protected_instruction_file"], - "description": description, - "allow_permanent": False, - "allow_session": False, - } - decision = _approval._await_gateway_decision( - session_key, notify_cb, approval_data, surface="gateway", - ) - if decision.get("notify_failed"): - return blocked.format( - why="requires approval but the approval request could not " - "be delivered.") - choice = decision.get("choice") - if decision.get("resolved") and choice in {"once", "session", "always"}: - # One-operation grant regardless of the tapped scope — nothing - # is persisted for this gate. - return None - if not decision.get("resolved"): - return blocked.format( - why="approval prompt timed out without a user response. " - "Silence is not consent.") - return blocked.format(why="was denied by the user.") - - # CLI surface: per-thread approval callback (prompt_toolkit panel). - callback = None - try: - from tools.terminal_tool import _get_approval_callback - callback = _get_approval_callback() - except Exception: - callback = None - - if callback is not None: - choice = _approval.prompt_dangerous_approval( - display, description, - allow_permanent=False, - allow_session=False, - approval_callback=callback, - ) - if choice in {"once", "session", "always"}: - # One-operation grant; never persisted (see docstring). - return None - if choice == "timeout": - return blocked.format( - why="approval prompt timed out without a user response. " - "Silence is not consent.") - return blocked.format(why="was denied by the user.") - - # No human channel at all (script, cron, background thread): fail - # closed. Auto-approving here would recreate the persistence vector. - return blocked.format( - why="requires approval but no interactive user or gateway is " - "present to approve it.") - - -def _check_protected_instruction_write(paths: list[str], - task_id: str = "default") -> str | None: - """Gate a write/patch touching protected instruction files. - - Returns ``None`` when no target is protected or the human approved; - otherwise a BLOCKED error string. For multi-file V4A patches, ONE - protected file gates the ENTIRE patch: a single prompt lists every - protected target, and a deny applies nothing (including innocent - files) — partial application of an approved-in-part patch would be - more surprising than an atomic all-or-nothing outcome. - """ - enabled, extra = _protected_instruction_config() - if not enabled: - return None - reasons: list[str] = [] - for p in paths: - reason = _protected_instruction_reason( - p, task_id, enabled=enabled, extra_patterns=extra) - if reason: - reasons.append(reason) - if not reasons: - return None - return _request_protected_instruction_approval(reasons, task_id) - - -def _check_approval_required_write(paths: list[str], - task_id: str = "default") -> str | None: - """Gate a write/patch touching an approval-required path (``~/.ssh/config``). - - These paths are NOT credentials and NOT hard-denied, but a write must - be confirmed by a human because they can steer process execution - (an SSH ``ProxyCommand`` / ``Match exec``). Unlike the protected- - instruction gate this is a routine, user-initiated edit, so the prompt - offers once/session/always scopes and honors --yolo (the historical - dangerous-command semantics) rather than always re-asking. - - Returns ``None`` when no target is approval-gated or the human - approved; otherwise a BLOCKED error string. Fail-closed when no - interactive/gateway channel exists (a background/ACP caller cannot - consent on the user's behalf). - """ - try: - from agent.file_safety import is_write_approval_required - except Exception: - return None - - targets = [p for p in paths if is_write_approval_required(p)] - if not targets: - return None - - display_targets = ", ".join(dict.fromkeys(targets)) - description = ( - f"Write to SSH client config file(s): {display_targets}. " - "The SSH config can carry ProxyCommand / Match exec directives that " - "run commands, so writes require your approval." - ) - blocked = ( - f"BLOCKED: write to SSH config file(s) ({display_targets}) " - "{why} Do NOT retry it via another path (terminal, execute_code) " - "without the user's explicit consent." - ) - - try: - import tools.approval as _approval - except Exception: - return blocked.format(why="requires approval but the approval " - "subsystem is unavailable.") - - result = _approval._run_approval_gate( - pattern_key="ssh_config_write", - description=description, - display_target=f"", - cron_deny_message=blocked.format( - why="requires approval but this cron session denies it."), - single_query_deny_message=blocked.format( - why="requires approval but single-query (-q) sessions run " - "without a user present to approve it. To allow flagged " - "actions in single-query mode, set approvals.single_query_mode: " - "approve in config.yaml."), - autoapprove_log_prefix="ssh_config_write", - fail_closed_when_no_human=True, - no_human_block_message=blocked.format( - why="requires approval but no interactive user or gateway is " - "present to approve it."), - ) - if result.get("approved"): - return None - return result.get("message") or blocked.format(why="was denied.") - - -def _get_container_mirror_prefix_for_task(task_id: str = "default") -> str | None: - """Return the container-side Hermes mirror prefix for Docker file tools.""" - try: - from tools.terminal_tool import ( - _active_environments, - _env_lock, - _get_env_config, - _resolve_container_task_id, - ) - - container_key = _resolve_container_task_id(task_id) - except Exception: - return None - - try: - with _env_lock: - env = _active_environments.get(container_key) or _active_environments.get(task_id) - - if env is not None: - if env.__class__.__name__ == "DockerEnvironment" and bool( - getattr(env, "_persistent", False) - ): - return "/root/.hermes" - return None - - config = _get_env_config() - except Exception: - return None - - if config.get("env_type") == "docker" and config.get("container_persistent", True): - return "/root/.hermes" - return None - - -def _check_cross_profile_path(filepath: str, task_id: str = "default") -> str | None: - """Return a soft-guard warning when ``filepath`` lands on a host-side - sandbox-mirror of authoritative profile state, or the Docker - container's sandbox mirror of Hermes state. - - Two detectors (both #32049): these catch writes that would be - SILENTLY LOST — the host Hermes process never reads the mirror, so - the write succeeds but changes nothing. That is a lost-work guard, - not profile isolation. - - NOTE: the third detector this shared check used to run — the - cross-PROFILE write guard (another profile's skills/plugins/cron/ - memories) — was removed by maintainer decision: profiles were never - isolated (same OS user; terminal writes anywhere), so the guard was - ceremony. The system prompt's profile hint remains the only - steering. ``cross_profile=True`` still bypasses the mirror guards - (name kept for replay/transcript compat). - - Returns ``None`` when the write is in-scope or outside Hermes scope. - """ - try: - from agent.file_safety import ( - get_container_mirror_warning, - get_sandbox_mirror_warning, - ) - except Exception: - # Fail open on import error — the existing sensitive-path guard - # plus the write_denied list still apply. - return None - - # Resolve via the task's cwd so a relative path in a session that - # cd'd elsewhere is classified against the right base. - try: - resolved = str(_resolve_path_for_task(filepath, task_id)) - except (OSError, ValueError): - resolved = filepath - - warning = get_sandbox_mirror_warning(resolved) - if warning is not None: - return warning - - return get_container_mirror_warning( - resolved, - mirror_prefix=_get_container_mirror_prefix_for_task(task_id), - ) - - def _is_expected_write_exception(exc: Exception) -> bool: """Return True for expected write denials that should not hit error logs.""" if isinstance(exc, PermissionError): return True - if isinstance(exc, OSError) and exc.errno in _EXPECTED_WRITE_ERRNOS: - return True - return False + return isinstance(exc, OSError) and exc.errno in _EXPECTED_WRITE_ERRNOS +# --------------------------------------------------------------------------- +# ShellFileOperations per terminal environment +# --------------------------------------------------------------------------- _file_ops_lock = threading.Lock() _file_ops_cache: dict = {} -# Track files read per task to detect re-read loops and deduplicate reads. -# Per task_id we store: -# "last_key": the key of the most recent read/search call (or None) -# "consecutive": how many times that exact call has been repeated in a row -# "read_history": set of (path, offset, limit) tuples for get_read_files_summary -# "dedup": dict mapping (resolved_path, offset, limit) → mtime float -# Used to skip re-reads of unchanged files. Reset on -# context compression (the original content is summarised -# away so the model needs the full content again). -# "read_timestamps": dict mapping resolved_path → modification-time float -# recorded when the file was last read (or written) by -# this task. Used by write_file and patch to detect -# external changes between the agent's read and write. -# Updated after successful writes so consecutive edits -# by the same task don't trigger false warnings. -_read_tracker_lock = threading.Lock() -_read_tracker: dict = {} - -# Track consecutive patch failures per (task_id, resolved_path). Used to -# escalate the hint when the model repeatedly fails to patch the same file -# (typical cause: stale view of file contents, ambiguous old_string, or -# the file was modified externally between the agent's read and patch -# attempt). Reset on a successful patch to that path. -_patch_failure_lock = threading.Lock() -_patch_failure_tracker: dict = {} # {task_id: {resolved_path: count}} - - -def _record_patch_failure(task_id: str, resolved_path: str) -> int: - """Increment and return the consecutive-failure count for this path.""" - with _patch_failure_lock: - task_failures = _patch_failure_tracker.setdefault(task_id, {}) - # Cap dict size per task to avoid unbounded growth in long sessions - # where the agent fails on many distinct files. 64 distinct - # failing files per task is generous; older entries get evicted. - if len(task_failures) >= 64 and resolved_path not in task_failures: - try: - first_key = next(iter(task_failures)) - del task_failures[first_key] - except StopIteration: - pass - task_failures[resolved_path] = task_failures.get(resolved_path, 0) + 1 - return task_failures[resolved_path] - - -def _reset_patch_failures(task_id: str, resolved_paths: list) -> None: - """Clear consecutive-failure counts for the given paths.""" - if not resolved_paths: - return - with _patch_failure_lock: - task_failures = _patch_failure_tracker.get(task_id) - if not task_failures: - return - for rp in resolved_paths: - task_failures.pop(rp, None) - -# Per-task bounds for the containers inside each _read_tracker[task_id]. -# A CLI session uses one stable task_id for its lifetime; without these -# caps, a 10k-read session would accumulate ~1.5MB of dict/set state that -# is never referenced again (only the most recent reads matter for dedup, -# loop detection, and external-edit warnings). Hard caps bound the -# accretion to a few hundred KB regardless of session length. -_READ_HISTORY_CAP = 500 # set; used only by get_read_files_summary -_DEDUP_CAP = 1000 # dict; skip-identical-reread guard -_READ_TIMESTAMPS_CAP = 1000 # dict; external-edit detection for write/patch -_NOT_FOUND_CAP = 500 # dict; per-task negative-result cache for missing paths -_NOT_FOUND_TTL_SECONDS = 60.0 # short TTL — a path that didn't exist may be created soon -_READ_DEDUP_STATUS_MESSAGE = ( - "File unchanged since last read. The content from " - "the earlier read_file result in this conversation is " - "still current — refer to that instead of re-reading." +# Per-backend config key that names the sandbox image (overridable per task). +_ENV_IMAGE_KEYS = { + "docker": "docker_image", + "singularity": "singularity_image", + "modal": "modal_image", + "daytona": "daytona_image", +} +_CONTAINER_CONFIG_DEFAULTS = ( + ("container_cpu", 1), + ("container_memory", 5120), + ("container_disk", 51200), + ("container_persistent", True), + ("vercel_runtime", ""), + ("docker_volumes", []), + ("docker_mount_cwd_to_workspace", False), + ("docker_forward_env", []), + ("docker_run_as_host_user", False), + ("docker_network", True), ) -def _cap_read_tracker_data(task_data: dict) -> None: - """Enforce size caps on the per-task read-tracker sub-containers. +def _create_terminal_env_for_file_ops(raw_task_id: str, task_id: str): + """Build the terminal environment for *task_id* from config + per-task overrides. - Must be called with ``_read_tracker_lock`` held. Eviction policy: - - * ``read_history`` (set): pop arbitrary entries on overflow. This - is fine because the set only feeds diagnostic summaries; losing - old entries just trims the summary's tail. - * ``dedup`` / ``read_timestamps`` (dict): pop oldest by insertion - order (Python 3.7+ dicts). Evicted entries lose their dedup - skip on a future re-read (the file gets re-sent once) and - external-edit mtime comparison (the write/patch falls back to - a non-mtime check). Both are graceful degradations, not bugs. + Mirrors terminal_tool's own environment construction so a file tool that + runs before any terminal command uses the configured backend (docker, + modal, ...) rather than always defaulting to local. """ - rh = task_data.get("read_history") - if rh is not None and len(rh) > _READ_HISTORY_CAP: - excess = len(rh) - _READ_HISTORY_CAP - for _ in range(excess): - try: - rh.pop() - except KeyError: - break - - dedup = task_data.get("dedup") - if dedup is not None and len(dedup) > _DEDUP_CAP: - excess = len(dedup) - _DEDUP_CAP - for _ in range(excess): - try: - dedup.pop(next(iter(dedup))) - except (StopIteration, KeyError): - break - - dedup_hits = task_data.get("dedup_hits") - if dedup_hits is not None and len(dedup_hits) > _DEDUP_CAP: - excess = len(dedup_hits) - _DEDUP_CAP - for _ in range(excess): - try: - dedup_hits.pop(next(iter(dedup_hits))) - except (StopIteration, KeyError): - break - - ts = task_data.get("read_timestamps") - if ts is not None and len(ts) > _READ_TIMESTAMPS_CAP: - excess = len(ts) - _READ_TIMESTAMPS_CAP - for _ in range(excess): - try: - ts.pop(next(iter(ts))) - except (StopIteration, KeyError): - break - - nf = task_data.get("not_found") - if nf is not None and len(nf) > _NOT_FOUND_CAP: - excess = len(nf) - _NOT_FOUND_CAP - for _ in range(excess): - try: - nf.pop(next(iter(nf))) - except (StopIteration, KeyError): - break - - -def _check_not_found_cache(op: str, resolved_str: str, task_id: str) -> str | None: - """Return cached not-found JSON for *(op, resolved_str)* if still fresh. - - Skips the expensive subprocess + suggestion walk when the model retries - the same missing path. Observed in agent.log: a single typo'd path was - retried 13 times — each retry forked a shell to walk the parent directory - and score similar names. - - *op* is "read" or "search" — kept separate because the two callers return - different error JSON shapes ("File not found:" vs "Path not found:"). - - Eviction: TTL or write_file/patch on the path (see invalidate_for_path). - """ - import os as _os - import time - with _read_tracker_lock: - task_data = _read_tracker.get(task_id) - if not task_data: - return None - nf = task_data.get("not_found") - if not nf: - return None - entry = nf.get((op, resolved_str)) - if entry is None: - return None - ts, cached_json = entry - if time.monotonic() - ts > _NOT_FOUND_TTL_SECONDS: - nf.pop((op, resolved_str), None) - return None - # Existence guard: the path may have been created since we cached the - # miss — by a terminal command, another agent, or any external process - # (write_file/patch invalidate explicitly, but they're not the only - # writers). The agent pattern "check file → create it → read it" is - # common; serving a stale miss for up to the TTL breaks it. One stat is - # ~free next to the subprocess walk we're skipping. - # - # The stat runs OUTSIDE _read_tracker_lock (matching the dedup mtime - # check below in read_file_tool): the lock is global across all tasks, - # and a hung stat on a dead network mount must not stall every other - # task's read/search bookkeeping. - if _os.path.exists(resolved_str): - with _read_tracker_lock: - task_data = _read_tracker.get(task_id) - nf = task_data.get("not_found") if task_data else None - if nf: - nf.pop((op, resolved_str), None) - return None - return cached_json - - -def _record_not_found(op: str, resolved_str: str, task_id: str, error_json: str) -> None: - """Cache a not-found error so the next *op* call for *resolved_str* skips I/O.""" - import time - with _read_tracker_lock: - task_data = _read_tracker.setdefault(task_id, { - "last_key": None, "consecutive": 0, - "read_history": set(), "dedup": {}, - "dedup_hits": {}, "read_timestamps": {}, - }) - nf = task_data.setdefault("not_found", {}) - nf[(op, resolved_str)] = (time.monotonic(), error_json) - _cap_read_tracker_data(task_data) - - -def _is_internal_file_status_text(content: str) -> bool: - """Return True when content looks like an internal file-tool status, not real file bytes. - - The read_file dedup status message must never be persisted as file - content. The obvious shape is the model echoing the message verbatim, - but in practice it also wraps it with small framing text (a leading - "Note:", a trailing newline + short comment, etc.) before calling - write_file. We treat any short-ish write whose body is dominated by - the status message as the same class of corruption. - - Heuristic: - * Strict equality (after strip) — the verbatim shape. - * OR the stripped content contains the full status message AND is - short enough that the status dominates it (<=2x the message length). - Short, status-dominated writes can't plausibly be real files — - legitimate docs/notes that happen to quote this internal message - are always dramatically longer. - """ - if not isinstance(content, str): - return False - stripped = content.strip() - if not stripped: - return False - if stripped == _READ_DEDUP_STATUS_MESSAGE: - return True - if _READ_DEDUP_STATUS_MESSAGE in stripped and \ - len(stripped) <= 2 * len(_READ_DEDUP_STATUS_MESSAGE): - return True - return False - - -def _looks_like_read_file_line_numbered_content(content: str) -> bool: - """Return True for content dominated by read_file's ``LINE_NUM|CONTENT`` display. - - ``read_file`` intentionally returns line-numbered text to the model. If - that display format is echoed into ``write_file``, config/source files are - silently corrupted with prefixes like `` 1|``. We reject writes where the - non-empty lines are mostly consecutive read_file-style numbered lines, while - allowing sparse literal pipe content such as a single ``1|value`` line. - """ - if not isinstance(content, str): - return False - - lines = [line for line in content.splitlines() if line.strip()] - if len(lines) < 2: - return False - - numbered: list[int] = [] - for line in lines: - stripped = line.lstrip() - prefix, sep, _rest = stripped.partition("|") - if sep and prefix.isdigit(): - numbered.append(int(prefix)) - - if len(numbered) < 2: - return False - if len(numbered) / len(lines) < 0.6: - return False - - consecutive_pairs = sum( - 1 for prev, current in zip(numbered, numbered[1:]) - if current == prev + 1 + from tools.terminal_tool import ( + _CONTAINER_BACKENDS, + _create_environment, + _get_env_config, + _is_container_backend, + _is_unusable_container_cwd, + _resolve_task_host_cwd, + get_session_cwd, + resolve_task_overrides, ) - return consecutive_pairs >= len(numbered) - 1 + config = _get_env_config() + env_type = config["env_type"] + overrides = resolve_task_overrides(raw_task_id) -def _is_internal_file_tool_content(content: str) -> bool: - """Return True when content is file-tool display text, not intended file bytes.""" - return ( - _is_internal_file_status_text(content) - or _looks_like_read_file_line_numbered_content(content) + image_key = _ENV_IMAGE_KEYS.get(env_type) + image = (overrides.get(image_key) or config[image_key]) if image_key else "" + + try: + recorded_cwd = get_session_cwd(raw_task_id) + except Exception: + recorded_cwd = None + cwd = overrides.get("cwd") or recorded_cwd or config["cwd"] + # Re-apply the container cwd guard _get_env_config() already ran on + # config["cwd"]: a gateway/TUI/ACP cwd override is a raw HOST path, and + # ``docker run -w `` starts the container in a directory that + # doesn't exist there, so search_files & co silently return nothing. + # Valid in-container overrides (/workspace, /root, ...) pass untouched. + if env_type in _CONTAINER_BACKENDS and _is_unusable_container_cwd(cwd): + if cwd != config["cwd"]: + logger.info( + "Ignoring host/relative cwd override %r for %s backend " + "(won't exist in sandbox). Using %r instead.", + cwd, env_type, config["cwd"], + ) + cwd = config["cwd"] + logger.info("Creating new %s environment for task %s...", env_type, task_id[:8]) + + container_config = None + if _is_container_backend(env_type): + container_config = {key: config.get(key, default) for key, default in _CONTAINER_CONFIG_DEFAULTS} + + ssh_config = None + if env_type == "ssh": + ssh_config = { + "host": config.get("ssh_host", ""), + "user": config.get("ssh_user", ""), + "port": config.get("ssh_port", 22), + "key": config.get("ssh_key", ""), + "persistent": config.get("ssh_persistent", False), + } + + local_config = None + if env_type == "local": + local_config = { + "persistent": config.get("local_persistent", False), + } + + terminal_env = _create_environment( + env_type=env_type, + image=image, + cwd=cwd, + timeout=config["timeout"], + ssh_config=ssh_config, + container_config=container_config, + local_config=local_config, + task_id=task_id, + host_cwd=_resolve_task_host_cwd(config, raw_task_id), ) + return env_type, terminal_env def _get_file_ops(task_id: str = "default") -> ShellFileOperations: - """Get or create ShellFileOperations for a terminal environment. + """Get or create ShellFileOperations for the task's terminal environment. - Respects the TERMINAL_ENV setting -- if the task_id doesn't have an - environment yet, creates one using the configured backend (local, docker, - modal, etc.) rather than always defaulting to local. - - Thread-safe: uses the same per-task creation locks as terminal_tool to - prevent duplicate sandbox creation from concurrent tool calls. - - Note: subagent task_ids are collapsed to "default" via - ``_resolve_container_task_id`` so delegate_task children share the - parent's container and its cached file_ops. RL/benchmark task_ids with - a registered env override keep their isolation. + Thread-safe via terminal_tool's per-task creation locks, so concurrent tool + calls never build duplicate sandboxes. Subagent task_ids collapse to + "default" (``_resolve_container_task_id``) so delegate_task children share + the parent's container and cached file_ops; RL/benchmark task_ids with a + registered env override keep their isolation. """ from tools.terminal_tool import ( - _active_environments, _env_lock, _create_environment, - _get_env_config, _last_activity, _start_cleanup_thread, + _active_environments, _env_lock, + _last_activity, _start_cleanup_thread, _creation_locks, _creation_locks_lock, _resolve_container_task_id, - _resolve_task_host_cwd, - _is_unusable_container_cwd, - _CONTAINER_BACKENDS, ) import time raw_task_id = task_id or "default" task_id = _resolve_container_task_id(raw_task_id) - # Fast path: check cache -- but also verify the underlying environment - # is still alive (it may have been killed by the cleanup thread). + # Fast path: cached AND the environment is still alive (the cleanup thread + # may have killed it). with _file_ops_lock: cached = _file_ops_cache.get(task_id) if cached is not None: @@ -1437,17 +464,11 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: _last_activity[task_id] = time.time() return cached else: - # Environment was cleaned up -- preserve the old cwd in the - # session record before invalidating the stale cache entry - # (fixes #26211: silent file-creation failures in long-running - # conversations). Usually a no-op: every completed command - # already recorded its cwd. - # - # Fill-only: ``cached.cwd`` is a snapshot of the SHARED env's - # cwd at cache-build time, so it is not attributable to this - # session (same class as the interrupted-command bug, #85658). - # Rescue a session that has no record, but never overwrite a - # record the session wrote for itself. + # Environment was cleaned up: rescue the old cwd into the + # session record before dropping the stale entry, FILL-ONLY — + # ``cached.cwd`` is a snapshot of the SHARED env's cwd, not + # attributable to this session, so never overwrite a record + # the session wrote for itself. old_cwd = getattr(cached, "cwd", None) if old_cwd: try: @@ -1462,15 +483,13 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: with _file_ops_lock: _file_ops_cache.pop(task_id, None) - # Need to ensure the environment exists before building file_ops. - # Acquire per-task lock so only one thread creates the sandbox. with _creation_locks_lock: if task_id not in _creation_locks: _creation_locks[task_id] = threading.Lock() task_lock = _creation_locks[task_id] with task_lock: - # Double-check: another thread may have created it while we waited + # Double-check: another thread may have created it while we waited. with _env_lock: if task_id in _active_environments: _last_activity[task_id] = time.time() @@ -1479,95 +498,7 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: terminal_env = None if terminal_env is None: - from tools.terminal_tool import resolve_task_overrides - - config = _get_env_config() - env_type = config["env_type"] - overrides = resolve_task_overrides(raw_task_id) - - if env_type == "docker": - image = overrides.get("docker_image") or config["docker_image"] - elif env_type == "singularity": - image = overrides.get("singularity_image") or config["singularity_image"] - elif env_type == "modal": - image = overrides.get("modal_image") or config["modal_image"] - elif env_type == "daytona": - image = overrides.get("daytona_image") or config["daytona_image"] - else: - image = "" - - try: - from tools.terminal_tool import get_session_cwd - recorded_cwd = get_session_cwd(raw_task_id) - except Exception: - recorded_cwd = None - cwd = overrides.get("cwd") or recorded_cwd or config["cwd"] - # Re-apply the container cwd guard that _get_env_config() already - # ran on config["cwd"] (see #50636). A per-task cwd override - # registered by the gateway/TUI/ACP for workspace tracking is a - # raw host path (e.g. a Desktop session's /Users//workspace or - # C:\\Users\\). On a container backend that reaches - # ``docker run -w `` and the container starts in a - # directory that doesn't exist inside the sandbox, so search_files - # and friends silently return empty results (#54447). Sanitize it - # back to the already-validated config["cwd"] so the override can't - # bypass the guard. Valid in-container override paths (RL/benchmark - # sandboxes that set cwd to /workspace, /root, etc.) are absolute - # non-host paths and pass through untouched. - if env_type in _CONTAINER_BACKENDS and _is_unusable_container_cwd(cwd): - if cwd != config["cwd"]: - logger.info( - "Ignoring host/relative cwd override %r for %s backend " - "(won't exist in sandbox). Using %r instead.", - cwd, env_type, config["cwd"], - ) - cwd = config["cwd"] - logger.info("Creating new %s environment for task %s...", env_type, task_id[:8]) - - container_config = None - from tools.terminal_tool import _is_container_backend as _is_container - - if _is_container(env_type): - container_config = { - "container_cpu": config.get("container_cpu", 1), - "container_memory": config.get("container_memory", 5120), - "container_disk": config.get("container_disk", 51200), - "container_persistent": config.get("container_persistent", True), - "vercel_runtime": config.get("vercel_runtime", ""), - "docker_volumes": config.get("docker_volumes", []), - "docker_mount_cwd_to_workspace": config.get("docker_mount_cwd_to_workspace", False), - "docker_forward_env": config.get("docker_forward_env", []), - "docker_run_as_host_user": config.get("docker_run_as_host_user", False), - "docker_network": config.get("docker_network", True), - } - - ssh_config = None - if env_type == "ssh": - ssh_config = { - "host": config.get("ssh_host", ""), - "user": config.get("ssh_user", ""), - "port": config.get("ssh_port", 22), - "key": config.get("ssh_key", ""), - "persistent": config.get("ssh_persistent", False), - } - - local_config = None - if env_type == "local": - local_config = { - "persistent": config.get("local_persistent", False), - } - - terminal_env = _create_environment( - env_type=env_type, - image=image, - cwd=cwd, - timeout=config["timeout"], - ssh_config=ssh_config, - container_config=container_config, - local_config=local_config, - task_id=task_id, - host_cwd=_resolve_task_host_cwd(config, raw_task_id), - ) + env_type, terminal_env = _create_terminal_env_for_file_ops(raw_task_id, task_id) with _env_lock: _active_environments[task_id] = terminal_env @@ -1576,7 +507,6 @@ def _get_file_ops(task_id: str = "default") -> ShellFileOperations: _start_cleanup_thread() logger.info("%s environment ready for task %s", env_type, task_id[:8]) - # Build file_ops from the (guaranteed live) environment and cache it file_ops = ShellFileOperations(terminal_env) with _file_ops_lock: _file_ops_cache[task_id] = file_ops @@ -1592,46 +522,154 @@ def clear_file_ops_cache(task_id: str = None): _file_ops_cache.clear() +_SPECIAL_FILE_KINDS = ( + (stat.S_ISFIFO, "a FIFO (named pipe)"), + (stat.S_ISSOCK, "a socket"), + (stat.S_ISCHR, "a character device"), + (stat.S_ISBLK, "a block device"), +) + + def _special_file_kind(path) -> str | None: - """Return a human name for non-regular file types that block reads. + """Human name for a non-regular file type that would hang a read, else None. - Stat-based sibling of the name-based ``_is_blocked_device`` guard: a - FIFO at ``logs/live.pipe`` or a socket in a workspace hangs ``read_file`` - just as hard as ``/dev/zero``, but carries no recognizable name. Only - called for host-visible filesystems (see ``_file_ops_uses_host_paths``); - remote backends cannot be statted from here. - - Returns None for regular files, missing paths, and anything unstattable - (those flow to the normal read path and its own error handling). + Stat-based sibling of the name-based ``_is_blocked_device`` guard: a FIFO + or socket inside a workspace hangs ``read_file`` as hard as ``/dev/zero`` + but carries no recognizable name. Host filesystems only (see + ``_file_ops_uses_host_paths``). Missing/unstattable paths return None and + flow to the normal read path's own error handling. """ - import stat as _stat - try: st = os.stat(os.fspath(path)) # follows symlinks, matching a real read except OSError: return None mode = st.st_mode - if _stat.S_ISREG(mode) or _stat.S_ISDIR(mode): + if stat.S_ISREG(mode) or stat.S_ISDIR(mode): return None - if _stat.S_ISFIFO(mode): - return "a FIFO (named pipe)" - if _stat.S_ISSOCK(mode): - return "a socket" - if _stat.S_ISCHR(mode): - return "a character device" - if _stat.S_ISBLK(mode): - return "a block device" + for predicate, label in _SPECIAL_FILE_KINDS: + if predicate(mode): + return label return "a special (non-regular) file" +def _read_extracted_document(path: str, _resolved, offset: int, limit: int, task_id: str) -> str | None: + """Render an extractable document (.docx/.xlsx/.pdf/...) as paginated text. + + Returns the JSON result, a tool_error for a binary document whose + extraction failed for an actionable reason (size cap, encrypted, + malformed), or ``None`` to fall through to the normal read path. Runs + BEFORE the binary-extension guard so Office files can render as text. + """ + from tools.read_extract import ( + ANYDOC_EXTENSIONS, + EXTRACTABLE_EXTENSIONS, + MAX_DOCUMENT_BYTES, + ExtractionError, + extract_document_bytes, + is_extractable_document, + ) + + if not is_extractable_document(str(_resolved)): + return None + file_ops = _get_file_ops(task_id) + try: + binary = file_ops.read_file_bytes( + str(_resolved), max_bytes=MAX_DOCUMENT_BYTES + ) + if binary.error or binary.base64_content is None: + raise ExtractionError(binary.error or "Document bytes unavailable") + document_bytes = base64.b64decode( + binary.base64_content, validate=True + ) + extracted_text = extract_document_bytes( + document_bytes, str(_resolved) + ) + except (ExtractionError, ValueError, base64.binascii.Error) as exc: + logger.debug("document extraction failed for %s", path, exc_info=True) + # Binary document formats surface the specific failure: the fallthrough + # can only yield a generic binary-file error or garbage bytes. .ipynb + # (plain JSON) and byte-transport errors (ValueError/binascii) fall + # through — only a specific ExtractionError carries an actionable reason. + _doc_ext = _resolved.suffix.lower() + _binary_doc = _doc_ext in ANYDOC_EXTENSIONS or ( + _doc_ext in EXTRACTABLE_EXTENSIONS and _doc_ext != ".ipynb" + ) + if ( + _binary_doc + and isinstance(exc, ExtractionError) + and not str(exc).startswith("Unsupported document type") + ): + return tool_error( + f"Cannot read '{path}' ({_doc_ext}): document " + f"extraction failed — {exc}. Use terminal utilities " + "to inspect or convert the file." + ) + return None + + lines = extracted_text.splitlines() + total_lines = len(lines) + end_line = offset + limit - 1 + page_text = "\n".join(lines[offset - 1:end_line]) + result_dict = { + "content": file_ops._add_line_numbers(page_text, offset) if page_text else "", + "total_lines": total_lines, + "file_size": binary.file_size, + "truncated": total_lines > end_line, + "extracted_document": True, + } + if result_dict["truncated"]: + result_dict["hint"] = ( + f"Use offset={end_line + 1} to continue reading " + f"(showing {offset}-{min(end_line, total_lines)} of {total_lines} lines)" + ) + max_chars = _get_max_read_chars() + if len(result_dict["content"]) > max_chars: + _apply_char_budget(result_dict, result_dict["content"], offset, total_lines, max_chars) + if result_dict["content"]: + result_dict["content"] = redact_sensitive_text(result_dict["content"], file_read=True) + return json.dumps(result_dict, ensure_ascii=False) + + +def _dedup_stub_or_block(task_data: dict, dedup_key: tuple, path: str) -> str: + """Return the "unchanged" stub for a repeated identical read, escalating to a + hard BLOCK after 2 stubs so weak tool-followers don't loop forever.""" + with _read_tracker_lock: + hits = task_data["dedup_hits"].get(dedup_key, 0) + 1 + task_data["dedup_hits"][dedup_key] = hits + _cap_read_tracker_data(task_data) + + if hits >= 2: + return tool_error( + f"BLOCKED: You have called read_file on this " + f"exact region {hits + 1} times and the file " + "has NOT changed. STOP calling read_file for " + "this path — the content from your earlier " + "read_file result in this conversation is " + "still current. Proceed with your task using " + "the information you already have.", + path=path, + already_read=hits + 1, + ) + + return json.dumps({ + "status": "unchanged", + "message": _READ_DEDUP_STATUS_MESSAGE, + "path": path, + "dedup": True, + "content_returned": False, + }, ensure_ascii=False) + + def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = "default") -> str: - """Read a file with pagination and line numbers.""" + """Read a file with pagination and line numbers. + + Guard order: device-path blocklist (no I/O) → stat-based special-file + guard (host only) → document extraction → binary-extension guard → Hermes + internal denylist → negative-result cache → dedup stub → real read. + """ try: offset, limit = normalize_read_pagination(offset, limit) - # ── Device path guard ───────────────────────────────────────── - # Block paths that would hang the process (infinite output, - # blocking on input). Pure path check — no I/O. device_base = None if Path(path).expanduser().is_absolute() else _resolve_base_dir(task_id) if _is_blocked_device(path, base_dir=device_base): return tool_error( @@ -1641,10 +679,7 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = _resolved = _resolve_path_for_task(path, task_id) - # ── Special-file type guard (stat-based) ────────────────────── - # The name blocklist above catches /dev/* and /proc/* aliases; this - # catches the class — any FIFO/socket/device wherever it lives. A - # read on a FIFO blocks until the exec timeout: a self-shipped DoS. + # A read on a FIFO/socket blocks until the exec timeout: a self-shipped DoS. if _file_ops_uses_host_paths(_get_file_ops(task_id)): kind = _special_file_kind(_resolved) if kind is not None: @@ -1658,110 +693,12 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = ), }) - # ── Structured-document extraction ──────────────────────────── - # Try before the binary-extension guard so .docx/.xlsx can render as text. - # Malformed documents fall through to the normal path/binary guard. - from tools.read_extract import ( - ANYDOC_EXTENSIONS, - EXTRACTABLE_EXTENSIONS, - MAX_DOCUMENT_BYTES, - ExtractionError, - extract_document_bytes, - is_extractable_document, - ) + extracted = _read_extracted_document(path, _resolved, offset, limit, task_id) + if extracted is not None: + return extracted - if is_extractable_document(str(_resolved)): - file_ops = _get_file_ops(task_id) - try: - binary = file_ops.read_file_bytes( - str(_resolved), max_bytes=MAX_DOCUMENT_BYTES - ) - if binary.error or binary.base64_content is None: - raise ExtractionError(binary.error or "Document bytes unavailable") - document_bytes = base64.b64decode( - binary.base64_content, validate=True - ) - extracted_text = extract_document_bytes( - document_bytes, str(_resolved) - ) - except (ExtractionError, ValueError, base64.binascii.Error) as exc: - logger.debug("document extraction failed for %s", path, exc_info=True) - # For binary document formats, surface the specific failure - # (size cap, encrypted, malformed…) instead of falling through - # — the fallthrough path can only produce a generic - # binary-file error or garbage raw bytes, hiding the - # actionable reason (e.g. "Document too large to convert"). - # .ipynb stays on the fallthrough path: it is plain JSON text - # and a raw read is genuinely useful. Byte-transport issues - # (ValueError / binascii) keep the fallthrough too — only a - # specific ExtractionError carries an actionable reason. - _doc_ext = _resolved.suffix.lower() - _binary_doc = _doc_ext in ANYDOC_EXTENSIONS or ( - _doc_ext in EXTRACTABLE_EXTENSIONS and _doc_ext != ".ipynb" - ) - if ( - _binary_doc - and isinstance(exc, ExtractionError) - and not str(exc).startswith("Unsupported document type") - ): - return tool_error( - f"Cannot read '{path}' ({_doc_ext}): document " - f"extraction failed — {exc}. Use terminal utilities " - "to inspect or convert the file." - ) - else: - lines = extracted_text.splitlines() - total_lines = len(lines) - end_line = offset + limit - 1 - page_text = "\n".join(lines[offset - 1:end_line]) - result_dict = { - "content": file_ops._add_line_numbers(page_text, offset) if page_text else "", - "total_lines": total_lines, - "file_size": binary.file_size, - "truncated": total_lines > end_line, - "extracted_document": True, - } - if result_dict["truncated"]: - result_dict["hint"] = ( - f"Use offset={end_line + 1} to continue reading " - f"(showing {offset}-{min(end_line, total_lines)} of {total_lines} lines)" - ) - content_len = len(result_dict["content"]) - max_chars = _get_max_read_chars() - if content_len > max_chars: - # Graceful char-budget truncation (nearai/ironclaw#5029): - # trim to the last complete line that fits and offer a - # next_offset rather than rejecting the whole extraction. - trimmed, lines_kept, _ = _truncate_to_char_budget( - result_dict["content"], max_chars - ) - next_offset = offset + lines_kept - shown_end = offset + lines_kept - 1 - result_dict["content"] = trimmed - result_dict["truncated"] = True - result_dict["truncated_by"] = "bytes" - result_dict["next_offset"] = next_offset - result_dict["hint"] = ( - f"Output truncated at the {max_chars:,}-char read budget " - f"after {lines_kept} line(s) (showing lines {offset}-" - f"{shown_end} of {total_lines}). Use offset={next_offset} " - "to continue." - ) - if len(trimmed.split("\n", 1)[0]) >= max_chars: - result_dict["hint"] += ( - " Note: the first line alone exceeded the budget and " - "was clamped mid-line; its remainder is not " - "retrievable via offset." - ) - if result_dict["content"]: - result_dict["content"] = redact_sensitive_text(result_dict["content"], file_read=True) - return json.dumps(result_dict, ensure_ascii=False) - - # ── Binary file guard ───────────────────────────────────────── - # Block binary files by extension (no I/O). Name what we know: - # the extension is a claim, so keep this branch's message to the - # extension itself — the content-sniffing path below names the - # actual magic-byte type for extension-less/lying files. + # The extension is a claim, so this message names only the extension; + # the content-sniffing path names the actual magic-byte type. if has_binary_extension(str(_resolved)): _ext = _resolved.suffix.lower() return tool_error( @@ -1769,152 +706,57 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = "Use vision_analyze for images, or terminal to inspect binary files." ) - # ── Hermes internal path guard ──────────────────────────────── - # Prevent prompt injection via catalog or hub metadata files, - # and block credential stores under HERMES_HOME. Pass the - # already-resolved path so a relative-path read against - # TERMINAL_CWD == HERMES_HOME (e.g. "auth.json") still hits the - # denylist — get_read_block_error's own resolve() runs against - # the Python process cwd, which can differ. + # Pass the already-resolved path: get_read_block_error's own resolve() + # runs against the process cwd, so a relative "auth.json" read with + # TERMINAL_CWD == HERMES_HOME would otherwise miss the denylist. block_error = get_read_block_error(str(_resolved)) if block_error: return tool_error(block_error) - # ── Negative-result cache ───────────────────────────────────── - # If we already discovered this path doesn't exist (within TTL), - # return the cached error without spawning the subprocess + - # similar-files walk. Cleared by write_file/patch on the same path. - resolved_str_for_neg = str(_resolved) - cached_not_found = _check_not_found_cache("read", resolved_str_for_neg, task_id) + resolved_str = str(_resolved) + cached_not_found = _check_not_found_cache("read", resolved_str, task_id) if cached_not_found is not None: return cached_not_found - # ── Dedup check ─────────────────────────────────────────────── - # If we already read this exact (path, offset, limit) and the - # file hasn't been modified since, return a lightweight stub - # instead of re-sending the same content. Saves context tokens. - resolved_str = str(_resolved) + # Dedup: identical (path, offset, limit) on an unchanged file returns a + # lightweight stub instead of re-sending the content. dedup_key = (resolved_str, offset, limit) with _read_tracker_lock: - task_data = _read_tracker.setdefault(task_id, { - "last_key": None, "consecutive": 0, - "read_history": set(), "dedup": {}, - "dedup_hits": {}, "read_timestamps": {}, - }) - # Backward-compat for pre-existing tracker entries that predate - # dedup_hits/read_timestamps (long-lived task or crossed an - # upgrade boundary). - if "dedup_hits" not in task_data: - task_data["dedup_hits"] = {} - if "read_timestamps" not in task_data: - task_data["read_timestamps"] = {} + task_data = _task_data(task_id) cached_mtime = task_data.get("dedup", {}).get(dedup_key) if cached_mtime is not None: try: - current_mtime = os.path.getmtime(resolved_str) - if current_mtime == cached_mtime: - # Count repeated stub returns so weak tool-followers that - # ignore the "refer to earlier result" hint don't burn - # their iteration budget in an infinite read loop. After - # 2 stubs for the same key we escalate to a hard block - # mirroring the count>=4 path on real reads. - with _read_tracker_lock: - hits = task_data["dedup_hits"].get(dedup_key, 0) + 1 - task_data["dedup_hits"][dedup_key] = hits - _cap_read_tracker_data(task_data) - - if hits >= 2: - return tool_error( - f"BLOCKED: You have called read_file on this " - f"exact region {hits + 1} times and the file " - "has NOT changed. STOP calling read_file for " - "this path — the content from your earlier " - "read_file result in this conversation is " - "still current. Proceed with your task using " - "the information you already have.", - path=path, - already_read=hits + 1, - ) - - return json.dumps({ - "status": "unchanged", - "message": _READ_DEDUP_STATUS_MESSAGE, - "path": path, - "dedup": True, - "content_returned": False, - }, ensure_ascii=False) + if os.path.getmtime(resolved_str) == cached_mtime: + return _dedup_stub_or_block(task_data, dedup_key, path) except OSError: pass # stat failed — fall through to full read - # ── Perform the read ────────────────────────────────────────── file_ops = _get_file_ops(task_id) result = file_ops.read_file(path, offset, limit) result_dict = result.to_dict() - # ── Populate negative-result cache on not-found ─────────────── - # _suggest_similar_files returns ReadResult(error="File not found: .."). - # Cache the JSON we'd return so a retry skips the parent-dir walk. - # Deliberately NO early return: on upstream, error results flow - # through the tracking block below (consecutive-loop detection, - # dedup bookkeeping via the resolved path) and the normal exit — - # short-circuiting here changes that behavior (and broke a real - # test interaction). Serving from the cache (above) is the - # optimization; recording must stay side-effect-identical. + # Cache a not-found result for retries. Deliberately NO early return: + # error results still flow through the tracking block below, exactly + # as before the cache existed. _err = result_dict.get("error") or "" if isinstance(_err, str) and _err.startswith("File not found:"): - _not_found_json = json.dumps(result_dict, ensure_ascii=False) - _record_not_found("read", resolved_str_for_neg, task_id, _not_found_json) + _record_not_found("read", resolved_str, task_id, json.dumps(result_dict, ensure_ascii=False)) - # ── Character-count guard ───────────────────────────────────── - # We're model-agnostic so we can't count tokens; characters are - # the best proxy we have. If the read produced an unreasonable - # amount of content, reject it and tell the model to narrow down. - # Note: we check the formatted content (with line-number prefixes), - # not the raw file size, because that's what actually enters context. - # Check BEFORE redaction to avoid expensive regex on huge content. - content_len = len(result.content or "") + # Char budget is checked on the FORMATTED content (that is what enters + # context) and BEFORE redaction, to skip the regex pass on huge content. file_size = result_dict.get("file_size", 0) max_chars = _get_max_read_chars() - if content_len > max_chars: - # Graceful char-budget truncation (ported from nearai/ironclaw#5029). - # Instead of rejecting the whole read — which forces the model to - # guess a smaller `limit` and wastes a round-trip returning nothing - # — trim to the last complete line that fits and offer a - # `next_offset` so the model can paginate forward. This rescues the - # "few but very long lines" case (logs, wide CSVs, minified data) - # that sails past the line-count `limit` but blows the char budget. - total_lines = result_dict.get("total_lines", "unknown") - trimmed, lines_kept, _ = _truncate_to_char_budget( - result.content or "", max_chars + if len(result.content or "") > max_chars: + result.content = _apply_char_budget( + result_dict, result.content or "", offset, + result_dict.get("total_lines", "unknown"), max_chars, ) - next_offset = offset + lines_kept - shown_end = offset + lines_kept - 1 - result.content = trimmed - result_dict["content"] = trimmed - result_dict["truncated"] = True - result_dict["truncated_by"] = "bytes" - result_dict["next_offset"] = next_offset - result_dict["hint"] = ( - f"Output truncated at the {max_chars:,}-char read budget after " - f"{lines_kept} line(s) (showing lines {offset}-{shown_end} of " - f"{total_lines}). Use offset={next_offset} to continue." - ) - if len(trimmed.split("\n", 1)[0]) >= max_chars: - result_dict["hint"] += ( - " Note: the first line alone exceeded the budget and was " - "clamped mid-line; its remainder is not retrievable via " - "offset." - ) - content_len = len(trimmed) - # ── Redact secrets (after guard check to skip oversized content) ── if result.content: result.content = redact_sensitive_text(result.content, file_read=True) result_dict["content"] = result.content - # Large-file hint: if the file is big and the caller didn't ask - # for a narrow window, nudge toward targeted reads. if (file_size and file_size > _LARGE_FILE_HINT_BYTES and limit > 200 and result_dict.get("truncated")): @@ -1924,63 +766,34 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = "to keep context usage efficient." )) - # ── Track for consecutive-loop detection ────────────────────── read_key = ("read", path, offset, limit) with _read_tracker_lock: - # Ensure "dedup" / "dedup_hits" keys exist (backward compat with - # old tracker state from pre-dedup-guard sessions). - if "dedup" not in task_data: - task_data["dedup"] = {} - if "dedup_hits" not in task_data: - task_data["dedup_hits"] = {} - # Real read succeeded — this key is no longer in a stub-loop, so - # reset its hit counter. (File either changed or stat failed - # earlier and we fell through.) + # A real read succeeded, so this key is no longer in a stub-loop. task_data["dedup_hits"].pop(dedup_key, None) task_data["read_history"].add((path, offset, limit)) - if task_data["last_key"] == read_key: - task_data["consecutive"] += 1 - else: - task_data["last_key"] = read_key - task_data["consecutive"] = 1 - count = task_data["consecutive"] - - # Store mtime at read time for two purposes: - # 1. Dedup: skip identical re-reads of unchanged files. - # 2. Staleness: warn on write/patch if the file changed since - # the agent last read it (external edit, concurrent agent, etc.). + count = _bump_consecutive(task_data, read_key) + # mtime at read time feeds dedup AND the write/patch staleness warning. try: _mtime_now = os.path.getmtime(resolved_str) task_data["dedup"][dedup_key] = _mtime_now task_data.setdefault("read_timestamps", {})[resolved_str] = _mtime_now except OSError: - pass # Can't stat — skip tracking for this entry - - # Bound the per-task containers so a long CLI session doesn't - # accumulate megabytes of dict/set state. See _cap_read_tracker_data. + pass _cap_read_tracker_data(task_data) - # Cross-agent file-state registry (separate from per-task read - # tracker above): records that THIS agent has read this path so - # write/patch can detect sibling-subagent writes that happened - # after our read. Partial read when offset>1 or the read was - # truncated (large file with more content than limit covered). - # Outside the _read_tracker_lock so the registry's own locking - # isn't nested under ours. + # Cross-agent registry (separate from the per-task tracker): lets + # write/patch detect sibling-subagent writes after our read. Partial + # when offset>1 or truncated. Outside our lock so the registry's own + # locking isn't nested under it. _partial = (offset > 1) or bool(result_dict.get("truncated")) try: file_state.record_read(task_id, resolved_str, partial=_partial) except Exception: logger.debug("file_state.record_read failed", exc_info=True) - # Background-review read-before-write guard integration (#61521): - # when the self-improvement review fork reads a skill file with - # read_file (now whitelisted dispatch-side), register the read the - # same way skill_view does, so a follow-up - # skill_manage(action='patch') on the loaded file is accepted. - # A partial read doesn't count — the guard requires the CURRENT - # full content to have been seen. No-op outside review forks - # (mark_background_review_skill_read gates on is_background_review). + # Background-review read-before-write guard: a FULL read of a skill + # file counts like skill_view so a follow-up skill_manage(patch) is + # accepted. No-op outside review forks. if not _partial: try: from tools.skill_manager_tool import mark_background_review_skill_read @@ -1992,7 +805,6 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = ) if count >= 4: - # Hard block: stop returning content to break the loop return tool_error( f"BLOCKED: You have read this exact file region {count} times in a row. " "The content has NOT changed. You already have this information. " @@ -2012,232 +824,28 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str = return tool_error(str(e)) +def _write_precheck_error(paths: list[str], content_paths: list[str], task_id: str, + cross_profile: bool) -> str | None: + """Run the shared write/patch guards in order; return the first error string. - -def reset_file_dedup(task_id: str = None): - """Clear the deduplication cache for file reads. - - Called after context compression — the original read content has been - summarised away, so the model needs the full content if it reads the - same file again. Without this, reads after compression would return - a "file unchanged" stub pointing at content that no longer exists in - context. - - Call with a task_id to clear just that task, or without to clear all. + Order matters: hard denies (sensitive path, mirror) and the corruption + guard run before anything that could prompt the user, and ONE approval + prompt covers every path of a multi-file patch. """ - with _read_tracker_lock: - if task_id: - task_data = _read_tracker.get(task_id) - if task_data: - if "dedup" in task_data: - task_data["dedup"].clear() - if "dedup_hits" in task_data: - task_data["dedup_hits"].clear() - else: - for task_data in _read_tracker.values(): - if "dedup" in task_data: - task_data["dedup"].clear() - if "dedup_hits" in task_data: - task_data["dedup_hits"].clear() - - -def notify_other_tool_call(task_id: str = "default"): - """Reset consecutive read/search counter for a task. - - Called by the tool dispatcher (model_tools.py) whenever a tool OTHER - than read_file / search_files is executed. This ensures we only warn - or block on *truly consecutive* repeated reads — if the agent does - anything else in between (write, patch, terminal, etc.) the counter - resets and the next read is treated as fresh. - """ - with _read_tracker_lock: - task_data = _read_tracker.get(task_id) - if task_data: - task_data["last_key"] = None - task_data["consecutive"] = 0 - # An intervening non-read tool call breaks any stub-loop in - # progress, so clear per-key dedup hit counters too. - if "dedup_hits" in task_data: - task_data["dedup_hits"].clear() - # Any other tool (terminal, delegate, ...) may have created a - # previously-missing path — a cached miss is no longer - # trustworthy. The serve-side existence guard in - # _check_not_found_cache already covers this, but clearing - # here keeps the cache honest and covers exotic cases the - # stat can't (e.g. permission flips). - nf = task_data.get("not_found") - if nf: - nf.clear() - - -def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None: - """Remove all dedup cache entries whose resolved path matches *filepath*. - - Called after write_file and patch so that a subsequent read_file on - the same path always returns fresh content instead of a stale - "File unchanged" stub. The dedup cache keys are tuples of - ``(resolved_path, offset, limit)``; we must evict **all** offset/limit - combinations for the written path because any cached range could now - be stale. - - Must be called with ``_read_tracker_lock`` **not** held — acquires it - internally. - """ - try: - resolved = str(_resolve_path(filepath, task_id)) - except (OSError, ValueError): - return - with _read_tracker_lock: - task_data = _read_tracker.get(task_id) - if task_data is None: - return - dedup = task_data.get("dedup") - if dedup: - # Collect keys to remove (can't mutate dict during iteration). - stale_keys = [k for k in dedup if k[0] == resolved] - for k in stale_keys: - del dedup[k] - # Also evict from the negative-result cache: a write_file that - # creates the path means subsequent reads (or searches under it) - # must hit disk. - nf = task_data.get("not_found") - if nf: - nf.pop(("read", resolved), None) - nf.pop(("search", resolved), None) - - -def _update_read_timestamp(filepath: str, task_id: str) -> None: - """Record the file's current modification time after a successful write. - - Called after write_file and patch so that consecutive edits by the - same task don't trigger false staleness warnings — each write - refreshes the stored timestamp to match the file's new state. - - Also invalidates the dedup cache for the written path so that - subsequent reads return fresh content (fixes #13144). - """ - # Invalidate dedup first (before acquiring lock for timestamp update). - _invalidate_dedup_for_path(filepath, task_id) - try: - resolved = str(_resolve_path_for_task(filepath, task_id)) - current_mtime = os.path.getmtime(resolved) - except (OSError, ValueError): - return - with _read_tracker_lock: - task_data = _read_tracker.get(task_id) - if task_data is not None: - task_data.setdefault("read_timestamps", {})[resolved] = current_mtime - _cap_read_tracker_data(task_data) - - -def _check_file_staleness(filepath: str, task_id: str) -> str | None: - """Check whether a file was modified since the agent last read it. - - Returns a warning string if the file is stale (mtime changed since - the last read_file call for this task), or None if the file is fresh - or was never read. Does not block — the write still proceeds. - """ - try: - resolved = str(_resolve_path_for_task(filepath, task_id)) - except (OSError, ValueError): - return None - with _read_tracker_lock: - task_data = _read_tracker.get(task_id) - if not task_data: - return None - read_mtime = task_data.get("read_timestamps", {}).get(resolved) - if read_mtime is None: - return None # File was never read — nothing to compare against - try: - current_mtime = os.path.getmtime(resolved) - except OSError: - return None # Can't stat — file may have been deleted, let write handle it - if current_mtime != read_mtime: - return ( - f"Warning: {filepath} was modified since you last read it " - "(external edit or concurrent agent). The content you read may be " - "stale. Consider re-reading the file to verify before writing." - ) - return None - - -def _mark_verification_stale( - task_id: str, - resolved_paths: list[str], - session_id: str | None = None, -) -> None: - """Best-effort note that successful edits made prior verification stale.""" - paths = [p for p in resolved_paths if p] - if not paths: - return - try: - from agent.coding_context import project_facts_for - from agent.verification_evidence import mark_workspace_edited - - cwd = None - for path in paths: - try: - candidate = str(Path(path).parent) - except Exception: - continue - if project_facts_for(candidate): - cwd = candidate - break - if cwd is None: - cwd = _authoritative_workspace_root(task_id) - if cwd is None: - try: - cwd = str(Path(paths[0]).parent) - except Exception: - cwd = None - mark_workspace_edited(session_id=session_id or task_id, cwd=cwd, paths=paths) - except Exception: - logger.debug("verification stale marker failed", exc_info=True) - - -def _check_binary_document_write(filepath: str, task_id: str = "default") -> str | None: - """Reject text-tool writes that would corrupt a binary document. - - ``read_file`` auto-extracts .docx/.xlsx/.pptx (and PDF, via anydoc) to - readable text, so the model plausibly believes it holds the file's - contents and tries to write the edited text back with write_file/patch. - A plain-text write can never produce a valid OOXML/OLE/ODF container, so - that write silently destroys the document (port of nearai/ironclaw#7109). - - Rules: - - Opaque container formats (.doc/.docx/.xls/.xlsx/.ppt/.pptx/.odt/.ods/ - .odp): always rejected — text bytes are never a valid document, whether - creating or overwriting. - - .pdf: rejected only when OVERWRITING an existing regular file. Raw PDF - syntax is text-authorable, so new-file creation stays allowed. - """ - if has_opaque_document_extension(filepath): - ext = filepath[filepath.rfind("."):].lower() - return ( - f"Refusing to write plain text to binary document '{filepath}' ({ext}). " - "A text write cannot produce a valid document container and would " - "corrupt the file (read_file showed you EXTRACTED text, not the real " - "bytes). Use the docx/xlsx/powerpoint skills or a library like " - "python-docx/openpyxl/python-pptx via the terminal to create or edit " - "this document." - ) - if is_pdf_path(filepath): - try: - resolved = Path(_resolve_path_for_task(filepath, task_id)) - except Exception: - resolved = Path(_expand_tilde(filepath)) - try: - if resolved.is_file(): - return ( - f"Refusing to overwrite existing PDF '{filepath}' with plain text. " - "read_file showed you EXTRACTED text, not the real bytes — writing " - "text back would destroy the document. Use the pdf skill or a PDF " - "library via the terminal to modify it. (Creating a NEW .pdf file " - "is allowed.)" - ) - except OSError: - pass - return None + for p in paths: + err = _check_sensitive_path(p, task_id) + if err: + return err + if not cross_profile: + err = _check_cross_profile_path(p, task_id) + if err: + return err + for p in content_paths: + err = _check_binary_document_write(p, task_id) + if err: + return err + return (_check_protected_instruction_write(paths, task_id) + or _check_approval_required_write(paths, task_id)) def write_file_tool(path: str, content: str, task_id: str = "default", @@ -2245,11 +853,11 @@ def write_file_tool(path: str, content: str, task_id: str = "default", session_id: str | None = None) -> str: """Write content to a file. - ``cross_profile`` bypasses the #32049 sandbox-mirror lost-write - guards (writes the host process would never read). Unadvertised in - the schema — the mirror rejection error teaches it. The cross-PROFILE - guard this flag was named for is removed (profiles are not isolated). + ``cross_profile`` bypasses the sandbox-mirror lost-write guards only + (unadvertised in the schema; the mirror rejection error teaches it — the + cross-PROFILE guard it was named for no longer exists). """ + # write_file checks the binary-document guard before the mirror guard. sensitive_err = _check_sensitive_path(path, task_id) if sensitive_err: return tool_error(sensitive_err) @@ -2273,9 +881,8 @@ def write_file_tool(path: str, content: str, task_id: str = "default", "file contents before writing." ) try: - # Resolve once for the registry lock + stale check. Failures here - # fall back to the legacy path — write proceeds, per-task staleness - # check below still runs. + # Resolution failure falls back to the legacy unlocked path (the write + # still proceeds; the per-task staleness check still runs). try: _resolved = str(_resolve_path_for_task(path, task_id)) except Exception: @@ -2293,16 +900,12 @@ def write_file_tool(path: str, content: str, task_id: str = "default", _update_read_timestamp(path, task_id) return json.dumps(result_dict, ensure_ascii=False) - # Serialize the read→modify→write region per-path so concurrent - # subagents can't interleave on the same file. Different paths - # remain fully parallel. + # Per-path lock serializes read→modify→write across concurrent subagents. with file_state.lock_path(_resolved): - # Cross-agent staleness wins over per-task warning when both - # fire — its message names the sibling subagent. + # Warning priority: cross-agent (names the sibling subagent) > + # per-task staleness > workspace divergence. cross_warning = file_state.check_stale(task_id, _resolved) stale_warning = _check_file_staleness(path, task_id) - # Workspace-divergence warning: relative path resolving outside the - # terminal's cwd (the worktree-cwd bug). Lowest priority of the three. cwd_warning = _path_resolution_warning(path, Path(_resolved), task_id) file_ops = _get_file_ops(task_id) result = file_ops.write_file(_resolved, content) @@ -2310,15 +913,11 @@ def write_file_tool(path: str, content: str, task_id: str = "default", effective_warning = cross_warning or stale_warning or cwd_warning if effective_warning: result_dict["_warning"] = effective_warning - # Always report the ABSOLUTE path actually written, so a wrong-cwd - # mismatch is visible in the response instead of silently routing - # the edit to the wrong checkout. + # Always report the ABSOLUTE path written so a wrong-cwd mismatch is visible. result_dict["resolved_path"] = _resolved if not result_dict.get("error"): result_dict["files_modified"] = [_resolved] _mark_verification_stale(task_id, [_resolved], session_id=session_id) - # Refresh stamps after the successful write so consecutive - # writes by this task don't trigger false staleness warnings. _update_read_timestamp(path, task_id) if not result_dict.get("error"): file_state.note_write(task_id, _resolved) @@ -2331,6 +930,55 @@ def write_file_tool(path: str, content: str, task_id: str = "default", return tool_error(str(e)) +def _collect_v4a_header_paths(patch: str) -> tuple[list[str], list[str]] | str: + """Extract every path named in V4A headers, rejecting ``..`` traversal. + + Returns ``(all_paths, content_write_paths)`` or a tool_error string. + Header paths come from patch CONTENT (skill text, web extract, prompt + injection) so they are more attacker-influenceable than the explicit + ``path=`` arg, which keeps its legitimate ``..`` use. Move headers check + BOTH endpoints (a Move onto /etc/crontab must hit the sensitive-path check). + Delete/Move don't write text, so only Update/Add feed the binary-document guard. + """ + from tools.path_security import has_traversal_component + + def _reject_v4a_traversal(v4a_path: str) -> str | None: + if has_traversal_component(v4a_path): + return tool_error( + f"V4A patch header contains '..' traversal: {v4a_path!r}. " + "Use the agent's cwd-relative path (no '..') or an absolute " + "path in '*** Update File:' / '*** Add File:' / " + "'*** Delete File:' / '*** Move File:' headers." + ) + return None + + paths: list[str] = [] + content_paths: list[str] = [] + for _m in _V4A_SINGLE_HEADER_OP_RE.finditer(patch): + _op = _m.group(1) + v4a_path = _m.group(2).strip() + _err = _reject_v4a_traversal(v4a_path) + if _err: + return _err + paths.append(v4a_path) + if _op in ("Update", "Add"): + content_paths.append(v4a_path) + for _m in _V4A_MOVE_HEADER_RE.finditer(patch): + for v4a_path in (_m.group(2).strip(), _m.group(3).strip()): + _err = _reject_v4a_traversal(v4a_path) + if _err: + return _err + paths.append(v4a_path) + return paths, content_paths + + +def _resolve_or_none(filepath: str, task_id: str) -> str | None: + try: + return str(_resolve_path_for_task(filepath, task_id)) + except Exception: + return None + + def patch_tool(mode: str = "replace", path: str = None, old_string: str = None, new_string: str = None, replace_all: bool = False, patch: str = None, task_id: str = "default", cross_profile: bool = False, @@ -2340,143 +988,57 @@ def patch_tool(mode: str = "replace", path: str = None, old_string: str = None, ``cross_profile``: same semantics as ``write_file``'s flag (mirror-guard bypass only; unadvertised). """ - # Check sensitive paths for both replace (explicit path) and V4A patch (extract paths) - _paths_to_check = [] - # Paths whose CONTENT will be text-written (Update/Add + explicit path). - # V4A Delete/Move don't write text, so they skip the binary-document guard. - _content_write_paths = [] - if path: - _paths_to_check.append(path) - _content_write_paths.append(path) + _paths_to_check = [path] if path else [] + _content_write_paths = list(_paths_to_check) if mode == "patch" and patch: - import re as _re - from tools.path_security import has_traversal_component - def _reject_v4a_traversal(v4a_path: str) -> str | None: - # V4A path headers come from patch CONTENT, not the explicit - # ``path=`` arg — so they're more attacker-influenceable (skill - # content, web extract, prompt injection). Reject ``..`` traversal - # in V4A headers: a legitimate multi-file patch from a single cwd - # can always emit absolute paths or paths relative to the agent's - # cwd without ``..``. The explicit ``path=`` arg is unchanged - # because the agent uses relative ``..`` paths legitimately - # (e.g. ``patch path="../other_module/x.py"`` from a worktree). - if has_traversal_component(v4a_path): - return tool_error( - f"V4A patch header contains '..' traversal: {v4a_path!r}. " - "Use the agent's cwd-relative path (no '..') or an absolute " - "path in '*** Update File:' / '*** Add File:' / " - "'*** Delete File:' / '*** Move File:' headers." - ) - return None - - # ``\s*`` (not ``\s+``) after ``***`` matches patch_parser leniency: - # it accepts ``***Update File:`` with no space after the asterisks - # (patch_parser.py uses ``\*\*\*\s*Update\s+File:``). Requiring a space - # here let a no-space header parse + apply while skipping this check. - for _m in _re.finditer(r'^\*\*\*\s*(Update|Add|Delete)\s+File:\s*(.+)$', patch, _re.MULTILINE): - _op = _m.group(1) - v4a_path = _m.group(2).strip() - _err = _reject_v4a_traversal(v4a_path) - if _err: - return _err - _paths_to_check.append(v4a_path) - if _op in ("Update", "Add"): - _content_write_paths.append(v4a_path) - # ``*** Move File: src -> dst`` is a valid V4A op (patch_parser.py:114) - # but was never extracted, so a Move targeting /etc/crontab skipped the - # sensitive-path pre-check. Check BOTH endpoints, and run them through - # the same ``..`` traversal rejection as the other headers. - for _m in _re.finditer(r'^\*\*\*\s*Move\s+File:\s*(.+?)\s*->\s*(.+)$', patch, _re.MULTILINE): - for v4a_path in (_m.group(1).strip(), _m.group(2).strip()): - _err = _reject_v4a_traversal(v4a_path) - if _err: - return _err - _paths_to_check.append(v4a_path) - for _p in _paths_to_check: - sensitive_err = _check_sensitive_path(_p, task_id) - if sensitive_err: - return tool_error(sensitive_err) - if not cross_profile: - cross_warning = _check_cross_profile_path(_p, task_id) - if cross_warning: - return tool_error(cross_warning) - for _p in _content_write_paths: - binary_doc_err = _check_binary_document_write(_p, task_id) - if binary_doc_err: - return tool_error(binary_doc_err) - # One approval prompt for the whole patch: a single protected file gates - # the ENTIRE patch (deny applies nothing — see the helper's docstring). - protected_err = _check_protected_instruction_write(_paths_to_check, task_id) - if protected_err: - return tool_error(protected_err) - approval_err = _check_approval_required_write(_paths_to_check, task_id) - if approval_err: - return tool_error(approval_err) + collected = _collect_v4a_header_paths(patch) + if isinstance(collected, str): + return collected + _paths_to_check += collected[0] + _content_write_paths += collected[1] + precheck_err = _write_precheck_error(_paths_to_check, _content_write_paths, task_id, cross_profile) + if precheck_err: + return tool_error(precheck_err) try: - # Resolve paths for locking. Ordered + deduplicated so concurrent - # callers lock in the same order — prevents deadlock on overlapping - # multi-file V4A patches. - _resolved_paths: list[str] = [] - _seen: set[str] = set() - for _p in _paths_to_check: - try: - _r = str(_resolve_path_for_task(_p, task_id)) - except Exception: - _r = None - if _r and _r not in _seen: - _resolved_paths.append(_r) - _seen.add(_r) - _resolved_paths.sort() + # Lock paths in sorted, deduplicated order so concurrent callers with + # overlapping multi-file patches can't deadlock. An unresolvable path + # is simply not locked. + _path_to_resolved: dict[str, str] = { + _p: _resolve_or_none(_p, task_id) for _p in _paths_to_check + } + _resolved_paths = sorted({_r for _r in _path_to_resolved.values() if _r}) - # Acquire per-path locks in sorted order via ExitStack. On single - # path this degenerates to one lock; on empty list (unresolvable) - # it's a no-op and execution falls through unchanged. from contextlib import ExitStack with ExitStack() as _locks: for _r in _resolved_paths: _locks.enter_context(file_state.lock_path(_r)) - # Collect warnings — cross-agent registry first (names sibling), - # then per-task tracker as a fallback. + # Warning priority per path: cross-agent registry (names the + # sibling) > per-task staleness > workspace divergence. stale_warnings: list[str] = [] - _path_to_resolved: dict[str, str] = {} for _p in _paths_to_check: - try: - _r = str(_resolve_path_for_task(_p, task_id)) - except Exception: - _r = None - _path_to_resolved[_p] = _r + _r = _path_to_resolved[_p] _cross = file_state.check_stale(task_id, _r) if _r else None _sw = _cross or _check_file_staleness(_p, task_id) if not _sw and _r: - # Workspace-divergence warning (worktree-cwd bug): relative - # path resolving outside the terminal's cwd. _sw = _path_resolution_warning(_p, Path(_r), task_id) if _sw: stale_warnings.append(_sw) file_ops = _get_file_ops(task_id) + # Hand the shell layer the RESOLVED targets so both layers agree on + # which file is edited even when the shell's cwd differs. if mode == "replace": if not path: return tool_error("path required") if old_string is None or new_string is None: return tool_error("old_string and new_string required") - # Pass the resolved ABSOLUTE path to the shell layer so it - # operates on the exact file the tool layer resolved — the - # shell's own cwd may differ (worktree-cwd bug), and a relative - # path would let the two layers disagree about which file is - # being edited. _replace_target = _path_to_resolved.get(path) or path result = file_ops.patch_replace(_replace_target, old_string, new_string, replace_all) elif mode == "patch": if not patch: return tool_error("patch content required") - # Rewrite V4A headers to the resolved absolute paths so the - # shell layer patches the exact files the tool layer resolved - # (locked/reported). Without this a relative header re-resolves - # against the shell's cwd, which can differ from the workspace - # (git-worktree cwd bug) — landing the edit elsewhere. patch_for_ops = _rewrite_v4a_patch_paths_for_host( patch, _path_to_resolved, file_ops ) @@ -2487,14 +1049,9 @@ def patch_tool(mode: str = "replace", path: str = None, old_string: str = None, result_dict = result.to_dict() if stale_warnings: result_dict["_warning"] = stale_warnings[0] if len(stale_warnings) == 1 else " | ".join(stale_warnings) - # Report the ABSOLUTE path(s) actually patched so a wrong-cwd - # mismatch (e.g. a worktree session editing the main checkout) is - # visible in the response instead of silently landing elsewhere. _resolved_modified = [ _path_to_resolved.get(_p) or _p for _p in _paths_to_check ] - # Refresh stored timestamps for all successfully-patched paths so - # consecutive edits by this task don't trigger false warnings. if not result_dict.get("error"): result_dict["files_modified"] = _resolved_modified if len(_resolved_modified) == 1: @@ -2505,33 +1062,20 @@ def patch_tool(mode: str = "replace", path: str = None, old_string: str = None, _r = _path_to_resolved.get(_p) if _r: file_state.note_write(task_id, _r) - # Successful patch: clear any prior consecutive-failure - # counters for the touched paths so a future failure on - # the same path starts the escalation cycle fresh. _reset_patch_failures(task_id, [ _r for _r in (_path_to_resolved.get(_p) for _p in _paths_to_check) if _r ]) - # Hint when old_string not found — saves iterations where the agent - # retries with stale content instead of re-reading the file. - # Suppressed when patch_replace already attached a rich "Did you mean?" - # snippet (which is strictly more useful than the generic hint). + # old_string-not-found hint. Per-file failure escalation is tracked for + # replace mode only (V4A failures are rare; the generic hint suffices). + # The generic hint is suppressed when patch_replace already attached a + # richer "Did you mean?" snippet. if result_dict.get("error") and "Could not find" in str(result_dict["error"]): - # Track per-file consecutive failures for replace mode. The - # ``path`` arg only exists for replace mode; for V4A patches - # we'd need to walk the headers, but in practice V4A failures - # are far rarer and the existing _hint covers them adequately. failure_count = 0 if mode == "replace" and path: resolved = _path_to_resolved.get(path) or path failure_count = _record_patch_failure(task_id, resolved) if failure_count >= 3: - # Escalating hint after multiple consecutive failures on the - # same path. Most common cause is a stale view of the file — - # the model is retrying with the same old_string against - # content that has since changed. Surface the failure count - # so the model recognises it's in a loop and breaks out by - # re-reading or falling back to write_file. result_dict["_hint"] = ( f"This is failure #{failure_count} patching {path!r}. " "Stop retrying with variations of the same old_string. " @@ -2559,9 +1103,8 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", try: offset, limit = normalize_search_pagination(offset, limit) - # Track searches to detect *consecutive* repeated search loops. - # Include pagination args so users can page through truncated - # results without tripping the repeated-search guard. + # Pagination args are part of the key so paging through truncated + # results doesn't trip the repeated-search guard. search_key = ( "search", pattern, @@ -2575,12 +1118,7 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", task_data = _read_tracker.setdefault(task_id, { "last_key": None, "consecutive": 0, "read_history": set(), }) - if task_data["last_key"] == search_key: - task_data["consecutive"] += 1 - else: - task_data["last_key"] = search_key - task_data["consecutive"] = 1 - count = task_data["consecutive"] + count = _bump_consecutive(task_data, search_key) if count >= 4: return tool_error( @@ -2599,11 +1137,8 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", if block_error: return tool_error(block_error) - # ── Negative-result cache ───────────────────────────────────── - # Search returns "Path not found: " when the search root - # doesn't exist. The error path also lists the parent directory - # (file_operations.py:1402) — expensive to repeat. Cache so the - # next call to a known-missing root skips both shells. + # A missing search root costs two shells (search + parent listing); + # cache the miss. try: resolved_search_path = str(_resolve_path_for_task(path, task_id)) except (OSError, ValueError): @@ -2630,13 +1165,10 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", "token, cache, or secret-bearing environment files." ) - # Populate negative cache when search root was missing. No early - # return — same rationale as the read path: error results keep - # flowing through the consecutive-search bookkeeping below. + # No early return on a cached miss — same rationale as the read path. _search_err = result_dict.get("error") or "" if isinstance(_search_err, str) and _search_err.startswith("Path not found:"): - _search_nf_json = json.dumps(result_dict, ensure_ascii=False) - _record_not_found("search", resolved_search_path, task_id, _search_nf_json) + _record_not_found("search", resolved_search_path, task_id, json.dumps(result_dict, ensure_ascii=False)) if count >= 3: result_dict["_warning"] = ( @@ -2645,8 +1177,6 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", ) result_json = json.dumps(result_dict, ensure_ascii=False) - # Hint when results were truncated — explicit next offset is clearer - # than relying on the model to infer it from total_count vs match count. if result_dict.get("truncated"): next_offset = offset + limit result_json += f"\n\n[Hint: Results truncated. Use offset={next_offset} to see more, or narrow with a more specific pattern or file_glob.]" @@ -2655,8 +1185,6 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", return tool_error(str(e)) - - # --------------------------------------------------------------------------- # Schemas + Registry # --------------------------------------------------------------------------- diff --git a/tools/file_tools_paths.py b/tools/file_tools_paths.py new file mode 100644 index 0000000000..bb79f5c8bd --- /dev/null +++ b/tools/file_tools_paths.py @@ -0,0 +1,270 @@ +"""Path resolution for the file tools: task-aware base dir, ``~`` expansion, workspace-divergence warning. + +Companion to ``tools.file_tools`` (which re-imports every name here). The core +invariant: the base directory used to anchor relative paths is ALWAYS absolute +and derived from the task's terminal cwd, never from the process cwd unless no +other anchor exists. A relative or sentinel ``TERMINAL_CWD`` would otherwise +silently anchor edits to the agent process cwd (e.g. the main repo while a +worktree session is active). +""" + +import os +import posixpath +import sys +from pathlib import Path, PurePosixPath + +# ``TERMINAL_CWD`` values that mean "not configured", not a directory to resolve +# against ("." from a stale config; "auto"/"cwd" are setup-wizard placeholders). +# The gateway sanitizes the same set at import time (gateway/run.py). +_TERMINAL_CWD_SENTINELS = frozenset({"", ".", "./", "auto", "cwd"}) +_CONTAINER_PATH_BACKENDS_FALLBACK = frozenset({"docker", "singularity", "modal", "daytona", "vercel_sandbox"}) +# Backend name inferred from the live environment's class name (first match wins). +_ENV_CLASS_NAME_HINTS = ("local", "ssh", "docker", "singularity", "modal", "daytona") + + +def _expand_tilde(path: str) -> str: + """Expand ``~`` using the effective profile home when available. + + In-process file tools share the gateway process's HOME, which may differ + from the profile-specific HOME interactive CLI sessions use; mirroring + ``hermes_constants.get_subprocess_home()`` keeps ``~`` consistent across + interactive and gateway-driven (cron) runs. + """ + if not path or "~" not in path: + return path + try: + from hermes_constants import get_subprocess_home + + home = get_subprocess_home() + except Exception: + home = None + if home and (path == "~" or path.startswith("~/")): + return home if path == "~" else os.path.join(home, path[2:]) + return os.path.expanduser(path) + + +def _terminal_env_type_for_task(task_id: str = "default") -> str: + """Best-effort terminal backend type for path-resolution decisions.""" + try: + from tools.terminal_tool import ( + _active_environments, + _env_lock, + _get_env_config, + _resolve_container_task_id, + ) + + try: + container_key = _resolve_container_task_id(task_id) + except Exception: + container_key = task_id + with _env_lock: + env = _active_environments.get(container_key) or _active_environments.get(task_id) + if env is not None: + name = env.__class__.__name__.lower() + for hint in _ENV_CLASS_NAME_HINTS: + if hint in name: + return hint + stamped = getattr(env, "_hermes_backend_name", None) + if isinstance(stamped, str) and stamped: + return stamped + cfg = _get_env_config() + return str(cfg.get("env_type") or os.getenv("TERMINAL_ENV") or "local").lower() + except Exception: + return str(os.getenv("TERMINAL_ENV") or "local").lower() + + +def _uses_container_paths(task_id: str = "default") -> bool: + env_type = _terminal_env_type_for_task(task_id) + try: + from tools.terminal_tool import _is_container_backend + + return _is_container_backend(env_type) + except Exception: + return env_type in _CONTAINER_PATH_BACKENDS_FALLBACK + + +def _normalize_without_host_deref(path: str | Path | PurePosixPath) -> PurePosixPath: + """Normalize path syntax without following host symlinks. + + Container paths are meaningful inside the sandbox; ``Path.resolve()`` on the + host could dereference a host-side symlink (e.g. ``/workspace``) and rewrite + the path before Docker sees it. + """ + return PurePosixPath(posixpath.normpath(str(path))) + + +def _sentinel_free_abs_cwd(raw: str | None) -> str | None: + """Return *raw* expanded when it is a non-sentinel ABSOLUTE anchor, else ``None``. + + A relative anchor is meaningless without knowing which cwd it is relative + to — exactly the ambiguity that misroutes worktree edits. + """ + raw = str(raw or "").strip() + if raw.lower() in _TERMINAL_CWD_SENTINELS: + return None + expanded = _expand_tilde(raw) + if not os.path.isabs(expanded): + return None + return expanded + + +def _configured_terminal_cwd() -> str | None: + """Return ``$TERMINAL_CWD`` only when it names a real (absolute, non-sentinel) anchor. + + Scope-aware: under gateway multiplexing the routed profile's cwd lives in + the per-turn terminal scope, not the process env. + """ + from agent.runtime_cwd import scope_terminal_cwd + + return _sentinel_free_abs_cwd(scope_terminal_cwd() or None) + + +def _registered_task_cwd_override(task_id: str = "default") -> str | None: + """Return a registered cwd override keyed by the RAW task id, when available. + + ``terminal_tool`` collapses CWD-only task overrides to the shared + ``"default"`` environment (TUI/dashboard/ACP sessions share one sandbox), + but the cwd value itself stays keyed by the raw session id — so read the + raw override before falling back to the collapsed container key. + """ + try: + from tools.terminal_tool import resolve_task_overrides + + overrides = resolve_task_overrides(task_id) + except Exception: + return None + + return _sentinel_free_abs_cwd(overrides.get("cwd")) + + +def _authoritative_workspace_root(task_id: str = "default") -> str | None: + """Best-effort absolute workspace root, or ``None`` when no reliable anchor exists. + + Order: (1) the session's own cwd record (written on every completed terminal + command; per-session, so one session's ``cd`` never leaks into another); + (2) a registered raw-keyed task/session cwd override (TUI/Desktop/ACP); + (3) a sentinel-free absolute ``$TERMINAL_CWD`` (``-w`` sessions). + """ + try: + from tools.terminal_tool import get_session_cwd + + recorded = get_session_cwd(task_id) + except Exception: + recorded = None + if recorded: + return recorded + registered = _registered_task_cwd_override(task_id) + if registered: + return registered + return _configured_terminal_cwd() + + +def _resolve_base_dir( + task_id: str = "default", + *, + container_paths: bool | None = None, +) -> Path | PurePosixPath: + """Return the ABSOLUTE base directory for resolving relative paths. + + Uses ``_authoritative_workspace_root`` (live cwd → registered override → + ``$TERMINAL_CWD``), falling back to the process cwd only as a last resort. + Sentinel/relative ``TERMINAL_CWD`` values are rejected outright rather than + anchored to the process cwd, so the result never depends on where the + agent process happens to run. + """ + root = _authoritative_workspace_root(task_id) + if container_paths is None: + container_paths = _uses_container_paths(task_id) + base_text = _expand_tilde(root) if root else os.getcwd() + if container_paths: + if not posixpath.isabs(base_text): + base_text = posixpath.join(os.getcwd(), base_text) + return _normalize_without_host_deref(base_text) + # Git Bash ``pwd -P`` reports ``/c/Users/...``; translate before Path so + # relative file-tool paths don't anchor under a nonexistent ``\\c\\Users``. + from tools.environments.local import _msys_to_windows_path + + base_text = _msys_to_windows_path(base_text) + if sys.platform == "win32": + import ntpath + + if not ntpath.isabs(base_text): + base_text = ntpath.join(os.getcwd(), base_text) + return Path(ntpath.normpath(base_text)) + base = Path(base_text) + if not base.is_absolute(): + # A backend reporting a relative cwd is anchored to the process cwd + # once, here, so the result no longer depends on cwd at resolve(). + base = Path(os.getcwd()) / base + return base.resolve() + + +def _resolve_path_for_task(filepath: str, task_id: str = "default") -> Path | PurePosixPath: + """Resolve *filepath* against the task's absolute base directory. + + Absolute inputs are returned resolved-but-unanchored. On native Windows, + Git Bash / MSYS drive paths (``/c/Users/...``) are translated first so + they aren't treated as relative ``\\c\\Users\\...`` under the process cwd; + container/WSL Linux paths are never rewritten. + """ + container_paths = _uses_container_paths(task_id) + if container_paths: + expanded = _expand_tilde(filepath) + if posixpath.isabs(expanded): + return _normalize_without_host_deref(expanded) + resolved = _resolve_base_dir(task_id, container_paths=True) / expanded + return _normalize_without_host_deref(resolved) + + from tools.environments.local import _msys_to_windows_path + + expanded = _expand_tilde(_msys_to_windows_path(filepath)) + if sys.platform == "win32": + import ntpath + + if ntpath.isabs(expanded): + return Path(ntpath.normpath(expanded)) + joined = ntpath.join(str(_resolve_base_dir(task_id, container_paths=False)), expanded) + return Path(ntpath.normpath(joined)) + + p = Path(expanded) + if p.is_absolute(): + return p.resolve() + resolved = _resolve_base_dir(task_id, container_paths=False) / p + return resolved.resolve() + + +# Back-compat alias (imported by agent.context_references and tests). +_resolve_path = _resolve_path_for_task + + +def _path_resolution_warning(filepath: str, resolved: Path, task_id: str = "default") -> str | None: + """Warn when a RELATIVE path resolved OUTSIDE the task's workspace root. + + Surfaces the worktree-cwd divergence the moment it matters — the edit is + about to land in a different checkout than the terminal's cwd. ``None`` for + absolute paths, an unknown root, or a path correctly under the root. Fires + on the very first write even before any ``cd`` populated the cwd registry. + """ + try: + if Path(_expand_tilde(filepath)).is_absolute(): + return None + workspace_root = _authoritative_workspace_root(task_id) + if not workspace_root: + return None + if _uses_container_paths(task_id): + root = _normalize_without_host_deref(Path(_expand_tilde(workspace_root))) + else: + root = Path(_expand_tilde(workspace_root)).resolve() + try: + resolved.relative_to(root) + return None + except ValueError: + return ( + f"Relative path {filepath!r} resolved to {str(resolved)!r}, which is " + f"OUTSIDE the active workspace ({str(root)!r}). The edit will land in " + f"a different directory than the terminal's cwd. If this is not " + f"intended (e.g. a git-worktree session writing into the main " + f"checkout), pass an absolute path under the workspace instead." + ) + except Exception: + return None diff --git a/tools/file_tools_read_tracking.py b/tools/file_tools_read_tracking.py new file mode 100644 index 0000000000..1789c76bdc --- /dev/null +++ b/tools/file_tools_read_tracking.py @@ -0,0 +1,329 @@ +"""Per-task read/search bookkeeping for the file tools. + +Process-lifetime state behind ``read_file`` / ``search_files`` / ``write_file`` / +``patch``; ``tools.file_tools`` re-imports every name here. Per task_id +``_read_tracker`` stores: + + last_key / consecutive most recent read-or-search key and its repeat count + (loop detection; reset by any OTHER tool call). + read_history set of (path, offset, limit) — diagnostic summaries only. + dedup (resolved_path, offset, limit) -> mtime; skip identical + re-reads of unchanged files. Cleared on context + compression (the original content was summarised away). + dedup_hits per-key count of stub returns, to break stub loops. + read_timestamps resolved_path -> mtime at last read/write by this task; + write/patch warn when the file changed underneath. + not_found (op, resolved_path) -> (monotonic, cached error JSON); + short-TTL negative cache for retried missing paths. + +Every container is hard-capped (``_cap_read_tracker_data``) so a long CLI +session accretes a few hundred KB at most instead of ~1.5MB per 10k reads. +""" + +import logging +import os +import threading +import time + +from tools.file_tools_paths import _authoritative_workspace_root, _resolve_path_for_task + +logger = logging.getLogger("tools.file_tools") + +_read_tracker_lock = threading.Lock() +_read_tracker: dict = {} + +# Consecutive patch failures per (task_id, resolved_path); escalates the hint +# when the model keeps failing the same file (stale view, ambiguous old_string). +# Reset on a successful patch to that path. +_patch_failure_lock = threading.Lock() +_patch_failure_tracker: dict = {} # {task_id: {resolved_path: count}} +_PATCH_FAILURE_PATHS_CAP = 64 + +# Only the most recent reads matter for dedup, loop detection and external-edit +# warnings; caps bound accretion regardless of session length. +_READ_HISTORY_CAP = 500 +_DEDUP_CAP = 1000 +_READ_TIMESTAMPS_CAP = 1000 +_NOT_FOUND_CAP = 500 +_NOT_FOUND_TTL_SECONDS = 60.0 # a path that didn't exist may be created soon + + +def _new_task_data() -> dict: + return { + "last_key": None, "consecutive": 0, + "read_history": set(), "dedup": {}, + "dedup_hits": {}, "read_timestamps": {}, + } + + +def _task_data(task_id: str) -> dict: + """Get-or-create the tracker entry for *task_id*, back-filling any missing keys. + + Must be called with ``_read_tracker_lock`` held. Entries created by older + code paths (or injected by tests) may lack the newer containers. + """ + task_data = _read_tracker.setdefault(task_id, _new_task_data()) + for key, factory in (("dedup", dict), ("dedup_hits", dict), ("read_timestamps", dict)): + if key not in task_data: + task_data[key] = factory() + return task_data + + +def _record_patch_failure(task_id: str, resolved_path: str) -> int: + """Increment and return the consecutive-failure count for this path.""" + with _patch_failure_lock: + task_failures = _patch_failure_tracker.setdefault(task_id, {}) + # Evict the oldest entry once a task has failed on many distinct files. + if len(task_failures) >= _PATCH_FAILURE_PATHS_CAP and resolved_path not in task_failures: + try: + del task_failures[next(iter(task_failures))] + except StopIteration: + pass + task_failures[resolved_path] = task_failures.get(resolved_path, 0) + 1 + return task_failures[resolved_path] + + +def _reset_patch_failures(task_id: str, resolved_paths: list) -> None: + """Clear consecutive-failure counts for the given paths.""" + if not resolved_paths: + return + with _patch_failure_lock: + task_failures = _patch_failure_tracker.get(task_id) + if not task_failures: + return + for rp in resolved_paths: + task_failures.pop(rp, None) + + +def _evict_oldest(container, cap: int) -> None: + """Pop entries until *container* is within *cap*. + + Sets pop arbitrary entries (they only feed diagnostic summaries); dicts pop + oldest by insertion order. An evicted entry costs one redundant re-send + (dedup) or one non-mtime staleness check — graceful degradation, not a bug. + """ + for _ in range(len(container) - cap): + try: + if isinstance(container, set): + container.pop() + else: + container.pop(next(iter(container))) + except (StopIteration, KeyError): + break + + +def _cap_read_tracker_data(task_data: dict) -> None: + """Enforce size caps on the per-task sub-containers. Call with ``_read_tracker_lock`` held.""" + # Caps are read at call time so tests can monkeypatch the module constants. + for key, cap in ( + ("read_history", _READ_HISTORY_CAP), + ("dedup", _DEDUP_CAP), + ("dedup_hits", _DEDUP_CAP), + ("read_timestamps", _READ_TIMESTAMPS_CAP), + ("not_found", _NOT_FOUND_CAP), + ): + container = task_data.get(key) + if container is not None and len(container) > cap: + _evict_oldest(container, cap) + + +def _check_not_found_cache(op: str, resolved_str: str, task_id: str) -> str | None: + """Return cached not-found JSON for *(op, resolved_str)* if still fresh. + + Skips the subprocess + similar-name walk when the model retries the same + missing path. *op* is "read" or "search" (different error JSON shapes). + Evicted by TTL, by write_file/patch on the path, or by any other tool call. + """ + with _read_tracker_lock: + task_data = _read_tracker.get(task_id) + if not task_data: + return None + nf = task_data.get("not_found") + if not nf: + return None + entry = nf.get((op, resolved_str)) + if entry is None: + return None + ts, cached_json = entry + if time.monotonic() - ts > _NOT_FOUND_TTL_SECONDS: + nf.pop((op, resolved_str), None) + return None + # The path may have been created since the miss was cached (terminal, + # another agent, ...) — the "check → create → read" pattern is common, so + # serving a stale miss breaks it. The stat runs OUTSIDE the global tracker + # lock: a hung stat on a dead network mount must not stall every task. + if os.path.exists(resolved_str): + with _read_tracker_lock: + task_data = _read_tracker.get(task_id) + nf = task_data.get("not_found") if task_data else None + if nf: + nf.pop((op, resolved_str), None) + return None + return cached_json + + +def _record_not_found(op: str, resolved_str: str, task_id: str, error_json: str) -> None: + """Cache a not-found error so the next *op* call for *resolved_str* skips I/O.""" + with _read_tracker_lock: + task_data = _task_data(task_id) + nf = task_data.setdefault("not_found", {}) + nf[(op, resolved_str)] = (time.monotonic(), error_json) + _cap_read_tracker_data(task_data) + + +def _bump_consecutive(task_data: dict, key: tuple) -> int: + """Update last_key/consecutive for *key* and return the new count. Lock must be held.""" + if task_data["last_key"] == key: + task_data["consecutive"] += 1 + else: + task_data["last_key"] = key + task_data["consecutive"] = 1 + return task_data["consecutive"] + + +def reset_file_dedup(task_id: str = None): + """Clear the read-dedup cache (one task, or all when ``task_id`` is None). + + Called after context compression: the original read content was summarised + away, so a "file unchanged" stub would point at content no longer in context. + """ + with _read_tracker_lock: + if task_id: + targets = [_read_tracker[task_id]] if _read_tracker.get(task_id) else [] + else: + targets = list(_read_tracker.values()) + for task_data in targets: + for key in ("dedup", "dedup_hits"): + if key in task_data: + task_data[key].clear() + + +def notify_other_tool_call(task_id: str = "default"): + """Reset the consecutive read/search counter for a task. + + Called by the dispatcher for every tool OTHER than read_file/search_files, + so loop detection only fires on truly consecutive repeats. Also clears the + stub-hit counters and the not-found cache: any other tool may have created + a previously-missing path (the serve-side stat covers most cases; clearing + covers the rest, e.g. permission flips). + """ + with _read_tracker_lock: + task_data = _read_tracker.get(task_id) + if task_data: + task_data["last_key"] = None + task_data["consecutive"] = 0 + if "dedup_hits" in task_data: + task_data["dedup_hits"].clear() + nf = task_data.get("not_found") + if nf: + nf.clear() + + +def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None: + """Evict every dedup entry (all offset/limit ranges) and not-found entry for *filepath*. + + Called after write_file/patch so the next read returns fresh content + instead of a stale "unchanged" stub. Acquires ``_read_tracker_lock`` itself. + """ + try: + resolved = str(_resolve_path_for_task(filepath, task_id)) + except (OSError, ValueError): + return + with _read_tracker_lock: + task_data = _read_tracker.get(task_id) + if task_data is None: + return + dedup = task_data.get("dedup") + if dedup: + for k in [k for k in dedup if k[0] == resolved]: + del dedup[k] + nf = task_data.get("not_found") + if nf: + nf.pop(("read", resolved), None) + nf.pop(("search", resolved), None) + + +def _update_read_timestamp(filepath: str, task_id: str) -> None: + """After a successful write: invalidate dedup and refresh the stored mtime so + consecutive edits by the same task don't trigger false staleness warnings.""" + _invalidate_dedup_for_path(filepath, task_id) + try: + resolved = str(_resolve_path_for_task(filepath, task_id)) + current_mtime = os.path.getmtime(resolved) + except (OSError, ValueError): + return + with _read_tracker_lock: + task_data = _read_tracker.get(task_id) + if task_data is not None: + task_data.setdefault("read_timestamps", {})[resolved] = current_mtime + _cap_read_tracker_data(task_data) + + +def _check_file_staleness(filepath: str, task_id: str) -> str | None: + """Warn (don't block) when the file's mtime changed since this task last read it. + + ``None`` when never read, fresh, or unstattable (a deleted file is the + write's problem to report). + """ + try: + resolved = str(_resolve_path_for_task(filepath, task_id)) + except (OSError, ValueError): + return None + with _read_tracker_lock: + task_data = _read_tracker.get(task_id) + if not task_data: + return None + read_mtime = task_data.get("read_timestamps", {}).get(resolved) + if read_mtime is None: + return None + try: + current_mtime = os.path.getmtime(resolved) + except OSError: + return None + if current_mtime != read_mtime: + return ( + f"Warning: {filepath} was modified since you last read it " + "(external edit or concurrent agent). The content you read may be " + "stale. Consider re-reading the file to verify before writing." + ) + return None + + +def _mark_verification_stale( + task_id: str, + resolved_paths: list[str], + session_id: str | None = None, +) -> None: + """Best-effort note that successful edits made prior verification stale. + + The workspace cwd is the first edited path's project root when one is + recognised, else the task's workspace root, else the first path's parent. + """ + from pathlib import Path + + paths = [p for p in resolved_paths if p] + if not paths: + return + try: + from agent.coding_context import project_facts_for + from agent.verification_evidence import mark_workspace_edited + + cwd = None + for path in paths: + try: + candidate = str(Path(path).parent) + except Exception: + continue + if project_facts_for(candidate): + cwd = candidate + break + if cwd is None: + cwd = _authoritative_workspace_root(task_id) + if cwd is None: + try: + cwd = str(Path(paths[0]).parent) + except Exception: + cwd = None + mark_workspace_edited(session_id=session_id or task_id, cwd=cwd, paths=paths) + except Exception: + logger.debug("verification stale marker failed", exc_info=True) diff --git a/tools/file_tools_write_guards.py b/tools/file_tools_write_guards.py new file mode 100644 index 0000000000..294ce52715 --- /dev/null +++ b/tools/file_tools_write_guards.py @@ -0,0 +1,530 @@ +"""Write-side safety guards for write_file / patch. + +Every guard returns ``None`` when the write may proceed, else an error string +the tool returns verbatim. ``tools.file_tools`` re-imports every name here. + +Guards (in the order the tools apply them): + * ``_check_sensitive_path`` — system paths + the Hermes config.yaml (hard deny). + * ``_check_binary_document_write`` — text write would corrupt an Office/PDF container. + * ``_check_protected_instruction_write`` — AGENTS.md-style files: ALWAYS ask, no yolo bypass. + * ``_check_approval_required_write`` — ~/.ssh/config-style files: normal approval gate. + * ``_check_cross_profile_path`` — sandbox-mirror writes the host never reads (lost-work guard). + * ``_is_internal_file_tool_content`` — refuse to persist read_file display text as a file. +""" + +import fnmatch +import os +from pathlib import Path + +from tools.binary_extensions import has_opaque_document_extension, is_pdf_path +from tools.file_tools_paths import _expand_tilde, _resolve_path_for_task + +# Prefixes matched after realpath. macOS: /private/var mirrors /var — block the +# sensitive subtrees only; a blanket "/private/var/" refuses every temp-file +# write because $TMPDIR, /tmp and /var/folders all realpath there. +_SENSITIVE_PATH_PREFIXES = ( + "/etc/", "/boot/", "/usr/lib/systemd/", + "/private/etc/", + "/private/var/db/", "/private/var/root/", +) +_SENSITIVE_EXACT_PATHS = {"/var/run/docker.sock", "/run/docker.sock"} + +_hermes_config_resolved: str | None = None +_hermes_config_resolved_loaded = False + + +def _get_hermes_config_resolved() -> str | None: + """Return the resolved absolute path of the Hermes config file (cached).""" + global _hermes_config_resolved, _hermes_config_resolved_loaded + if _hermes_config_resolved_loaded: + return _hermes_config_resolved + _hermes_config_resolved_loaded = True + try: + from hermes_cli.config import get_config_path + _hermes_config_resolved = str(get_config_path().resolve()) + except Exception: + try: + _hermes_config_resolved = str(Path(_expand_tilde("~/.hermes/config.yaml")).resolve()) + except Exception: + _hermes_config_resolved = None + return _hermes_config_resolved + + +def _resolved_or_raw(filepath: str, task_id: str) -> str: + """Task-resolved path string, falling back to the raw input on resolution failure.""" + try: + return str(_resolve_path_for_task(filepath, task_id)) + except (OSError, ValueError): + return filepath + + +def _check_sensitive_path(filepath: str, task_id: str = "default") -> str | None: + """Return an error message if the path targets a sensitive system location.""" + resolved = _resolved_or_raw(filepath, task_id) + normalized = os.path.normpath(_expand_tilde(filepath)) + _err = ( + f"Refusing to write to sensitive system path: {filepath}\n" + "Use the terminal tool with sudo if you need to modify system files." + ) + for prefix in _SENSITIVE_PATH_PREFIXES: + if resolved.startswith(prefix) or normalized.startswith(prefix): + return _err + if resolved in _SENSITIVE_EXACT_PATHS or normalized in _SENSITIVE_EXACT_PATHS: + return _err + # approvals.mode and other security settings live in config.yaml; a + # prompt-injected agent could silently disable exec approval by editing it. + hermes_config = _get_hermes_config_resolved() + if hermes_config and (resolved == hermes_config or normalized == hermes_config): + return ( + f"Refusing to write to Hermes config file: {filepath}\n" + "Agent cannot modify security-sensitive configuration. " + "Edit ~/.hermes/config.yaml directly or use 'hermes config' instead." + ) + return None + + +# --------------------------------------------------------------------------- +# Protected agent-instruction files (always-ask approval gate) +# --------------------------------------------------------------------------- +# Files that steer FUTURE agent behavior are a prompt-injection persistence +# vector: an injected edit to AGENTS.md / CLAUDE.md / SOUL.md / .cursorrules +# (or a project-local .hermes tree) outlives the turn and poisons every later +# session. Writes ALWAYS require human approval — even under --yolo — and fail +# closed when no human channel exists. (Ported from Roo-Code's +# RooProtectedController; the terminal-tool vector is gated separately.) +# +# Basenames match in ANY directory (instruction files load from cwd trees) and +# case-insensitively (case-insensitive filesystems; loaders probe variants). +_PROTECTED_INSTRUCTION_BASENAMES = frozenset({ + "agents.md", "claude.md", "soul.md", ".cursorrules", +}) + +_real_hermes_home_cached: str | None = None +_real_hermes_home_loaded = False + + +def _get_real_hermes_home() -> str | None: + """Return the realpath of the authoritative Hermes home (cached).""" + global _real_hermes_home_cached, _real_hermes_home_loaded + if _real_hermes_home_loaded: + return _real_hermes_home_cached + _real_hermes_home_loaded = True + try: + from hermes_constants import get_hermes_home + _real_hermes_home_cached = os.path.realpath(str(get_hermes_home())) + except Exception: + try: + _real_hermes_home_cached = os.path.realpath(_expand_tilde("~/.hermes")) + except Exception: + _real_hermes_home_cached = None + return _real_hermes_home_cached + + +def _protected_instruction_config() -> tuple[bool, list[str]]: + """Return ``(enabled, extra_patterns)`` from ``security.protected_instruction_files`` / + ``security.protected_instruction_extra_patterns`` (fnmatch on basename). + + Config read failures keep the gate ON — fail-safe for a security boundary. + """ + try: + from hermes_cli.config import load_config, cfg_get + cfg = load_config() + enabled = cfg_get(cfg, "security", "protected_instruction_files", + default=True) + extra = cfg_get(cfg, "security", "protected_instruction_extra_patterns", + default=[]) + except Exception: + return True, [] + if not isinstance(enabled, bool): + enabled = True + if not isinstance(extra, list): + extra = [] + return enabled, [str(p) for p in extra if p] + + +def _protected_instruction_reason(filepath: str, task_id: str = "default", + *, enabled: bool | None = None, + extra_patterns: list[str] | None = None) -> str | None: + """Return a short label when ``filepath`` targets a protected instruction file, else ``None``. + + Matches BOTH the normalized input and its realpath so neither a symlink + pointing AT a protected file nor a protected name that is itself a symlink + escapes; ``..`` traversal is neutralized by normpath/realpath first. + """ + if enabled is None or extra_patterns is None: + enabled, extra_patterns = _protected_instruction_config() + if not enabled: + return None + + normalized = os.path.normpath(_expand_tilde(filepath)) + try: + resolved = os.path.realpath(str(_resolve_path_for_task(filepath, task_id))) + except (OSError, ValueError, RuntimeError): + resolved = os.path.realpath(normalized) + + # ~/.hermes itself is governed by its own guards (config.yaml hard-block, + # mirror guard, write_approval); this gate targets PROJECT-LOCAL files only. + # Must run before the ``.hermes`` component rule, which would match the home. + real_home = _get_real_hermes_home() + if real_home and (resolved == real_home + or resolved.startswith(real_home + os.sep)): + return None + + for candidate in (normalized, resolved): + base = os.path.basename(candidate) + base_lower = base.lower() + if base_lower in _PROTECTED_INSTRUCTION_BASENAMES: + return base + for pattern in extra_patterns: + if fnmatch.fnmatch(base_lower, pattern.lower()): + return base + # Project-local .hermes config dirs (/.hermes/config.yaml) steer + # behavior too. Only the IMMEDIATE parent counts — matching any ancestor + # would gate every write inside a checkout living under ~/.hermes. + parts = candidate.replace("\\", "/").rstrip("/").split("/") + if len(parts) >= 2 and parts[-2] == ".hermes": + return candidate + return None + + +def _request_protected_instruction_approval( + reasons: list[str], task_id: str = "default") -> str | None: + """Ask the human to approve a write to protected instruction file(s). + + Returns ``None`` when approved, else a BLOCKED error string. Deliberately + NOT routed through ``_run_approval_gate``: that honors --yolo and + session/permanent allowlists, and this gate is one-operation approval EVERY + time with no persisted scope. Fail-closed when no human channel exists. + """ + targets = ", ".join(dict.fromkeys(reasons)) + description = ( + f"Write to protected agent-instruction file(s): {targets}. " + "These files steer future agent behavior; approval is always " + "required (not bypassed by auto-approve)." + ) + display = f"" + blocked = ( + f"BLOCKED: write to protected agent-instruction file(s) ({targets}) " + "{why} The user has NOT consented to this write. Do NOT retry it or " + "attempt the same edit via another path (terminal, execute_code, " + "etc.)." + ) + timed_out = blocked.format( + why="approval prompt timed out without a user response. " + "Silence is not consent.") + denied = blocked.format(why="was denied by the user.") + + try: + import tools.approval as _approval + except Exception: + return blocked.format(why="requires approval but the approval " + "subsystem is unavailable.") + + # Gateway surface: block on the button round-trip when a notify callback + # is registered for this session. One-operation only — no scope buttons. + session_key = _approval.get_current_session_key() + notify_cb = None + try: + with _approval._lock: + notify_cb = _approval._gateway_notify_cbs.get(session_key) + except Exception: + notify_cb = None + + if notify_cb is not None: + approval_data = { + "command": display, + "pattern_key": "protected_instruction_file", + "pattern_keys": ["protected_instruction_file"], + "description": description, + "allow_permanent": False, + "allow_session": False, + } + decision = _approval._await_gateway_decision( + session_key, notify_cb, approval_data, surface="gateway", + ) + if decision.get("notify_failed"): + return blocked.format( + why="requires approval but the approval request could not " + "be delivered.") + choice = decision.get("choice") + # Any tapped scope is a one-operation grant; nothing is persisted. + if decision.get("resolved") and choice in {"once", "session", "always"}: + return None + if not decision.get("resolved"): + return timed_out + return denied + + # CLI surface: per-thread approval callback (prompt_toolkit panel). + callback = None + try: + from tools.terminal_tool import _get_approval_callback + callback = _get_approval_callback() + except Exception: + callback = None + + if callback is not None: + choice = _approval.prompt_dangerous_approval( + display, description, + allow_permanent=False, + allow_session=False, + approval_callback=callback, + ) + if choice in {"once", "session", "always"}: + return None + if choice == "timeout": + return timed_out + return denied + + # No human channel (script, cron, background thread): fail closed — + # auto-approving here would recreate the persistence vector. + return blocked.format( + why="requires approval but no interactive user or gateway is " + "present to approve it.") + + +def _check_protected_instruction_write(paths: list[str], + task_id: str = "default") -> str | None: + """Gate a write/patch touching protected instruction files. + + ONE protected file gates the ENTIRE multi-file patch: a single prompt lists + every protected target and a deny applies nothing (atomic all-or-nothing + beats a partially-applied patch). + """ + enabled, extra = _protected_instruction_config() + if not enabled: + return None + reasons = [ + r for r in ( + _protected_instruction_reason(p, task_id, enabled=enabled, extra_patterns=extra) + for p in paths + ) if r + ] + if not reasons: + return None + return _request_protected_instruction_approval(reasons, task_id) + + +def _check_approval_required_write(paths: list[str], + task_id: str = "default") -> str | None: + """Gate a write/patch touching an approval-required path (``~/.ssh/config``). + + Not credentials and not hard-denied, but they can steer process execution + (SSH ``ProxyCommand`` / ``Match exec``). Unlike the protected-instruction + gate this is a routine user edit: the prompt offers once/session/always and + honors --yolo. Fail-closed when no interactive/gateway channel exists. + """ + try: + from agent.file_safety import is_write_approval_required + except Exception: + return None + + targets = [p for p in paths if is_write_approval_required(p)] + if not targets: + return None + + display_targets = ", ".join(dict.fromkeys(targets)) + description = ( + f"Write to SSH client config file(s): {display_targets}. " + "The SSH config can carry ProxyCommand / Match exec directives that " + "run commands, so writes require your approval." + ) + blocked = ( + f"BLOCKED: write to SSH config file(s) ({display_targets}) " + "{why} Do NOT retry it via another path (terminal, execute_code) " + "without the user's explicit consent." + ) + + try: + import tools.approval as _approval + except Exception: + return blocked.format(why="requires approval but the approval " + "subsystem is unavailable.") + + result = _approval._run_approval_gate( + pattern_key="ssh_config_write", + description=description, + display_target=f"", + cron_deny_message=blocked.format( + why="requires approval but this cron session denies it."), + single_query_deny_message=blocked.format( + why="requires approval but single-query (-q) sessions run " + "without a user present to approve it. To allow flagged " + "actions in single-query mode, set approvals.single_query_mode: " + "approve in config.yaml."), + autoapprove_log_prefix="ssh_config_write", + fail_closed_when_no_human=True, + no_human_block_message=blocked.format( + why="requires approval but no interactive user or gateway is " + "present to approve it."), + ) + if result.get("approved"): + return None + return result.get("message") or blocked.format(why="was denied.") + + +def _get_container_mirror_prefix_for_task(task_id: str = "default") -> str | None: + """Return the container-side Hermes mirror prefix for persistent Docker file tools.""" + try: + from tools.terminal_tool import ( + _active_environments, + _env_lock, + _get_env_config, + _resolve_container_task_id, + ) + + container_key = _resolve_container_task_id(task_id) + except Exception: + return None + + try: + with _env_lock: + env = _active_environments.get(container_key) or _active_environments.get(task_id) + + if env is not None: + if env.__class__.__name__ == "DockerEnvironment" and bool( + getattr(env, "_persistent", False) + ): + return "/root/.hermes" + return None + + config = _get_env_config() + except Exception: + return None + + if config.get("env_type") == "docker" and config.get("container_persistent", True): + return "/root/.hermes" + return None + + +def _check_cross_profile_path(filepath: str, task_id: str = "default") -> str | None: + """Soft-guard: warn when ``filepath`` lands on a host-side or Docker sandbox + MIRROR of Hermes state — a write the host process never reads (lost work). + + Not profile isolation: the former cross-PROFILE guard was removed by + maintainer decision (profiles were never isolated). ``cross_profile=True`` + on the tools still bypasses these mirror guards (name kept for replay compat). + Fails open on import error — the sensitive-path guard and denylist still apply. + """ + try: + from agent.file_safety import ( + get_container_mirror_warning, + get_sandbox_mirror_warning, + ) + except Exception: + return None + + resolved = _resolved_or_raw(filepath, task_id) + + warning = get_sandbox_mirror_warning(resolved) + if warning is not None: + return warning + + return get_container_mirror_warning( + resolved, + mirror_prefix=_get_container_mirror_prefix_for_task(task_id), + ) + + +def _check_binary_document_write(filepath: str, task_id: str = "default") -> str | None: + """Reject text-tool writes that would corrupt a binary document. + + ``read_file`` auto-extracts Office/PDF to text, so the model plausibly + believes it holds the file's bytes and writes edited text back — which can + never form a valid container. Opaque formats (.docx/.xlsx/.pptx/.odt/...) + are always rejected; .pdf only when OVERWRITING an existing regular file + (raw PDF syntax is text-authorable, so new-file creation stays allowed). + """ + if has_opaque_document_extension(filepath): + ext = filepath[filepath.rfind("."):].lower() + return ( + f"Refusing to write plain text to binary document '{filepath}' ({ext}). " + "A text write cannot produce a valid document container and would " + "corrupt the file (read_file showed you EXTRACTED text, not the real " + "bytes). Use the docx/xlsx/powerpoint skills or a library like " + "python-docx/openpyxl/python-pptx via the terminal to create or edit " + "this document." + ) + if is_pdf_path(filepath): + try: + resolved = Path(_resolve_path_for_task(filepath, task_id)) + except Exception: + resolved = Path(_expand_tilde(filepath)) + try: + if resolved.is_file(): + return ( + f"Refusing to overwrite existing PDF '{filepath}' with plain text. " + "read_file showed you EXTRACTED text, not the real bytes — writing " + "text back would destroy the document. Use the pdf skill or a PDF " + "library via the terminal to modify it. (Creating a NEW .pdf file " + "is allowed.)" + ) + except OSError: + pass + return None + + +# --------------------------------------------------------------------------- +# Internal display text must never be persisted as file content +# --------------------------------------------------------------------------- +_READ_DEDUP_STATUS_MESSAGE = ( + "File unchanged since last read. The content from " + "the earlier read_file result in this conversation is " + "still current — refer to that instead of re-reading." +) + + +def _is_internal_file_status_text(content: str) -> bool: + """True when content is the read_file dedup status message (verbatim or lightly framed). + + Models echo the message verbatim OR wrap it with short framing ("Note:", + a trailing comment). Any write whose stripped body contains the full + message and is <=2x its length is status-dominated — a real file quoting + this message would be dramatically longer. + """ + if not isinstance(content, str): + return False + stripped = content.strip() + if not stripped: + return False + if stripped == _READ_DEDUP_STATUS_MESSAGE: + return True + return (_READ_DEDUP_STATUS_MESSAGE in stripped + and len(stripped) <= 2 * len(_READ_DEDUP_STATUS_MESSAGE)) + + +def _looks_like_read_file_line_numbered_content(content: str) -> bool: + """True for content dominated by read_file's ``LINE_NUM|CONTENT`` display. + + Rejects writes whose non-empty lines are mostly (>=60%) consecutive + numbered lines, while allowing sparse literal pipe content such as a + single ``1|value`` line. + """ + if not isinstance(content, str): + return False + + lines = [line for line in content.splitlines() if line.strip()] + if len(lines) < 2: + return False + + numbered: list[int] = [] + for line in lines: + prefix, sep, _rest = line.lstrip().partition("|") + if sep and prefix.isdigit(): + numbered.append(int(prefix)) + + if len(numbered) < 2 or len(numbered) / len(lines) < 0.6: + return False + + consecutive_pairs = sum( + 1 for prev, current in zip(numbered, numbered[1:]) + if current == prev + 1 + ) + return consecutive_pairs >= len(numbered) - 1 + + +def _is_internal_file_tool_content(content: str) -> bool: + """Return True when content is file-tool display text, not intended file bytes.""" + return ( + _is_internal_file_status_text(content) + or _looks_like_read_file_line_numbered_content(content) + ) diff --git a/tools/focus_pane_tool.py b/tools/focus_pane_tool.py index b729e93b5e..5e18decc62 100644 --- a/tools/focus_pane_tool.py +++ b/tools/focus_pane_tool.py @@ -1,16 +1,10 @@ #!/usr/bin/env python3 -"""Reveal/focus a pane in the Hermes desktop GUI. +"""Reveal/focus a pane in the Hermes desktop GUI (``pane.reveal`` via ``desktop_ui``). -Lives in the ``desktop_ui`` toolset (like the other GUI affordances), which the -GUI gateway enables only for desktop-sourced sessions. Emits ``pane.reveal`` -through the shared ``desktop_ui`` bridge; the renderer runs each pane's own -reveal path and only acts on the active window (a background turn never moves -the user's focus). To show a URL/file, use ``open_preview``; to close it, use -``close_preview``. +The renderer runs each pane's own reveal path and only acts on the active window, so +a background turn never moves the user's focus. URLs/files go through `desktop_preview`. """ -import json - from tools import desktop_ui from tools.registry import registry, tool_error @@ -22,15 +16,13 @@ def focus_pane_tool(pane: str) -> str: name = (pane or "").strip().lower() if name not in PANES: return tool_error(f"pane must be one of: {', '.join(PANES)}.") - - try: - ok = desktop_ui.emit("pane.reveal", {"pane": name}) - except Exception as exc: - return tool_error(f"Failed to focus the {name} pane: {exc}") - if not ok: - return tool_error("Pane focus is only available in the Hermes desktop app.") - - return json.dumps({"success": True, "pane": name}, ensure_ascii=False) + return desktop_ui.emit_or_error( + "pane.reveal", + {"pane": name}, + f"Failed to focus the {name} pane: ", + "Pane focus is only available in the Hermes desktop app.", + {"success": True, "pane": name}, + ) FOCUS_PANE_SCHEMA = { diff --git a/tools/fuzzy_match.py b/tools/fuzzy_match.py index 360645b4d4..a991c00f6b 100644 --- a/tools/fuzzy_match.py +++ b/tools/fuzzy_match.py @@ -1,61 +1,38 @@ #!/usr/bin/env python3 -""" -Fuzzy Matching Module for File Operations +"""Fuzzy find-and-replace for LLM-generated edits. -Implements a multi-strategy matching chain to robustly find and replace text, -accommodating variations in whitespace, indentation, and escaping common -in LLM-generated code. +Tries an ordered chain of increasingly permissive matching strategies (see +:mod:`tools.fuzzy_match_strategies`) so whitespace, indentation, escaping and +Unicode drift in tool-call arguments still land on the intended region. -The 9-strategy chain (inspired by OpenCode), tried in order: -1. Exact match - Direct string comparison -2. Line-trimmed - Strip leading/trailing whitespace per line -3. Whitespace normalized - Collapse multiple spaces/tabs to single space -4. Indentation flexible - Ignore indentation differences entirely -5. Escape normalized - Convert \\n literals to actual newlines -6. Trimmed boundary - Trim first/last line whitespace only -7. Block anchor - Match first+last lines, use similarity for middle -8. Context-aware - 50% line similarity threshold - -Multi-occurrence matching is handled via the replace_all flag. - -Usage: - from tools.fuzzy_match import fuzzy_find_and_replace - new_content, match_count, strategy, error = fuzzy_find_and_replace( - content="def foo():\\n pass", - old_string="def foo():", - new_string="def bar():", - replace_all=False - ) + content, old_string, new_string, replace_all=False) """ -import re -from typing import Tuple, Optional, List, Callable from difflib import SequenceMatcher +from typing import List, Optional, Tuple -UNICODE_MAP = { - "\u201c": '"', "\u201d": '"', # smart double quotes - "\u2018": "'", "\u2019": "'", # smart single quotes - "\u2014": "--", "\u2013": "-", # em/en dashes - "\u2026": "...", "\u00a0": " ", # ellipsis and non-breaking space - # Unicode minus sign — models type ASCII '-' for file content that uses - # the typographic minus (math/scientific docs). - "\u2212": "-", - # Space-separator family (Zs) beyond NBSP. Files with typographic - # spacing (en/em/thin spaces, narrow NBSP in French text, ideographic - # space in CJK text) never match a model's ASCII-space old_string via - # the precise strategies, falling through to the similarity-based - # context_aware fallback — which can pick the wrong region and flattens - # the file's Unicode on replacement. (anomalyco/opencode#38133 corpus) - "\u2000": " ", "\u2001": " ", # en/em quad - "\u2002": " ", "\u2003": " ", # en/em space - "\u2004": " ", "\u2005": " ", "\u2006": " ", # three/four/six-per-em - "\u2007": " ", "\u2008": " ", # figure/punctuation space - "\u2009": " ", "\u200a": " ", # thin/hair space - "\u202f": " ", # narrow no-break space - "\u205f": " ", # medium mathematical space - "\u3000": " ", # ideographic (CJK full-width) space -} +from tools.fuzzy_match_strategies import ( # noqa: F401 — re-exported names + SIMILARITY_STRATEGIES, + STRATEGIES, + UNICODE_MAP, + _build_orig_to_norm_map, + _calculate_line_positions, + _invert_norm_map, + _map_normalized_positions, + _map_positions_norm_to_orig, + _norm_end_to_orig, + _strategy_block_anchor, + _strategy_context_aware, + _strategy_escape_normalized, + _strategy_exact, + _strategy_indentation_flexible, + _strategy_line_trimmed, + _strategy_trimmed_boundary, + _strategy_unicode_normalized, + _strategy_whitespace_normalized, + _unicode_normalize, +) IDENTICAL_STRINGS_ERROR = ( "No edit was applied because old_string and new_string are identical. " @@ -64,30 +41,11 @@ IDENTICAL_STRINGS_ERROR = ( ) -def _unicode_normalize(text: str) -> str: - """Normalizes Unicode characters to their standard ASCII equivalents.""" - for char, repl in UNICODE_MAP.items(): - text = text.replace(char, repl) - return text - - def is_already_applied(content: str, old_string: str, new_string: str) -> bool: - """Return True when the requested edit is already present in the file. + """True when the requested edit is already present (re-sent edit -> success-shaped no-op). - Production trajectory mining shows the most common patch failure is a - re-send of an edit that already landed (old_string == new_string, or - old_string gone while new_string is present) — the model's intent is - "make the file contain this text", and it already does. Callers use - this to convert those errors into an explicit success-shaped no-op so - the model moves on instead of re-reading and re-patching. - - Deliberately conservative: - - new_string must be non-trivial (>= 8 chars stripped) — a tiny target - matching by coincidence must not mask a genuine typo'd edit; - - new_string must appear EXACTLY in the content (no fuzzy matching — - approximate presence is not proof the edit landed); - - when old_string differs from new_string, old_string must be GONE - (still-present old text means the edit is at best half-applied). + Conservative: new_string must be non-trivial (>= 8 chars stripped) and + appear EXACTLY; when it differs from old_string, old_string must be gone. """ if not new_string or len(new_string.strip()) < 8: return False @@ -98,14 +56,13 @@ def is_already_applied(content: str, old_string: str, new_string: str) -> bool: return old_string not in content +def _matched_regions(content: str, matches: List[Tuple[int, int]]) -> str: + return "".join(content[start:end] for start, end in matches) + + def _format_match_locations(content: str, matches: List[Tuple[int, int]], cap: int = 5) -> str: - """Render up to ``cap`` match positions as 'L: ' rows. - - Gives the model the information it needs to disambiguate an ambiguous - old_string in ONE follow-up (add neighboring context, or choose - replace_all) instead of re-reading the file to find the occurrences. - """ + """Render up to ``cap`` match positions as 'L: ' rows.""" rows = [] for start, _end in matches[:cap]: line_no = content.count("\n", 0, start) + 1 @@ -124,173 +81,81 @@ def _format_match_locations(content: str, matches: List[Tuple[int, int]], def fuzzy_find_and_replace(content: str, old_string: str, new_string: str, - replace_all: bool = False) -> Tuple[str, int, Optional[str], Optional[str]]: - """ - Find and replace text using a chain of increasingly fuzzy matching strategies. + replace_all: bool = False) -> Tuple[str, int, Optional[str], Optional[str]]: + """Find and replace via the strategy chain. - Args: - content: The file content to search in - old_string: The text to find - new_string: The replacement text - replace_all: If True, replace all occurrences; if False, require uniqueness - - Returns: - Tuple of (new_content, match_count, strategy_name, error_message) - - If successful: (modified_content, number_of_replacements, strategy_used, None) - - If failed: (original_content, 0, None, error_description) + Returns ``(new_content, match_count, strategy_name, error)``; on failure + ``(content, 0, None, error)``. """ if not old_string: return content, 0, None, "old_string cannot be empty" - if not old_string.strip(): - # A whitespace-only old_string matches trivially (a blank line, run of - # spaces, etc.) and, when it recurs, either mass-replaces under - # replace_all or raises a hard-to-diagnose ambiguity error. It's never - # a meaningful anchor — reject it so the caller provides real context. + # Whitespace-only anchors match trivially and mass-replace or + # ambiguity-error; never meaningful. return content, 0, None, "old_string is only whitespace — provide non-blank text to match" - if old_string == new_string: return content, 0, None, IDENTICAL_STRINGS_ERROR - # Try each matching strategy in order - strategies: List[Tuple[str, Callable]] = [ - ("exact", _strategy_exact), - ("line_trimmed", _strategy_line_trimmed), - ("whitespace_normalized", _strategy_whitespace_normalized), - ("indentation_flexible", _strategy_indentation_flexible), - ("escape_normalized", _strategy_escape_normalized), - ("trimmed_boundary", _strategy_trimmed_boundary), - ("unicode_normalized", _strategy_unicode_normalized), - ("block_anchor", _strategy_block_anchor), - ("context_aware", _strategy_context_aware), - ] - - # Strategies whose matches are similarity-based rather than exact-content: - # they can accept a region that only *approximately* resembles old_string. - # Safe for a single unique replacement (the caller asked to change that one - # spot), but NEVER safe under replace_all — "replace every approximate - # match" silently rewrites regions that don't actually contain old_string. - _SIMILARITY_STRATEGIES = {"block_anchor", "context_aware"} - - for strategy_name, strategy_fn in strategies: + for strategy_name, strategy_fn in STRATEGIES: matches = strategy_fn(content, old_string) + if not matches: + continue - if matches: - # Found matches with this strategy - if len(matches) > 1 and not replace_all: - locations = _format_match_locations(content, matches) - return content, 0, None, ( - f"Found {len(matches)} matches for old_string. " - f"Provide more context to make it unique, or use replace_all=True. " - f"Matches:\n{locations}" - ) - - # replace_all with a similarity-based strategy would overwrite - # every approximately-matching block, not just exact ones — refuse - # and make the caller narrow old_string to something a precise - # strategy can match exactly. - if replace_all and len(matches) > 1 and strategy_name in _SIMILARITY_STRATEGIES: - return content, 0, None, ( - f"Found {len(matches)} approximate matches via the " - f"'{strategy_name}' strategy; replace_all only applies to exact " - f"matches. Provide the precise text (whitespace included) so an " - f"exact/line-trimmed match can be made." - ) - - # Escape-drift guard: when the matched strategy is NOT `exact`, - # we matched via some form of normalization. If new_string - # contains shell/JSON-style escape sequences (\' or \") that - # would be written literally into the file but the matched - # region of the file has no such sequences, this is almost - # certainly tool-call serialization drift — the model typed - # an apostrophe/quote and the transport added a stray - # backslash. Writing new_string as-is would corrupt the file. - # Block with a helpful error so the model re-reads and retries - # instead of the caller silently persisting garbage (or not). - if strategy_name != "exact": - drift_err = _detect_escape_drift(content, matches, old_string, new_string) - if drift_err: - return content, 0, None, drift_err - - # Perform replacement. When the matched strategy is NOT `exact`, - # the file's indentation may differ from what the LLM sent in - # old_string/new_string — e.g. LLM used 2-space indent but the - # file is 4-space. Shift new_string by the indentation delta so - # the replacement matches the file's actual indent pattern. - # LLMs frequently serialize tabs / carriage returns in JSON - # tool-call arguments as the two-character sequences ``\t`` and - # ``\r`` (backslash + letter) instead of the real control bytes. - # If we write new_string verbatim, the file ends up with literal - # backslash sequences where the surrounding code uses real tabs. - # - # Strategy: only unescape when the matched region of the file - # *actually contains* the corresponding real control character. - # That mirrors the region-based heuristic in - # ``_detect_escape_drift`` and keeps legitimate writes of the - # literal two-character string ``"\t"`` (e.g. patching Python - # source that contains a tab string literal in source text) - # untouched — those files have a backslash+t in the matched - # region, not a real tab, so we leave new_string alone. - # - # ``\n`` is intentionally excluded: newlines serialize correctly - # through JSON, and rewriting backslash-n would mangle escape - # sequences in source code constants far more often than help. - effective_new = _maybe_unescape_new_string( - new_string, content, matches, + if len(matches) > 1 and not replace_all: + locations = _format_match_locations(content, matches) + return content, 0, None, ( + f"Found {len(matches)} matches for old_string. " + f"Provide more context to make it unique, or use replace_all=True. " + f"Matches:\n{locations}" ) - # Unicode-preservation guard: when strategy 7 (unicode_normalized) - # matched, the file has Unicode characters (em-dashes, smart quotes, - # ellipsis) but old_string/new_string from the LLM are ASCII - # equivalents. Writing new_string verbatim would silently corrupt - # the file's Unicode — em-dashes become two hyphens, smart quotes - # become straight quotes. Align the replacement with the file's - # actual Unicode so only the LLM's intended changes are applied - # and unchanged portions keep their original characters. - if strategy_name == "unicode_normalized": - effective_new = _preserve_unicode_in_replacement( - content, matches, old_string, effective_new, - ) - new_content = _apply_replacements( - content, matches, effective_new, - old_string=old_string if strategy_name != "exact" else None, + if replace_all and len(matches) > 1 and strategy_name in SIMILARITY_STRATEGIES: + return content, 0, None, ( + f"Found {len(matches)} approximate matches via the " + f"'{strategy_name}' strategy; replace_all only applies to exact " + f"matches. Provide the precise text (whitespace included) so an " + f"exact/line-trimmed match can be made." ) - return new_content, len(matches), strategy_name, None - # No strategy found a match + # Non-exact matches came through some normalization, so new_string may + # carry serialization drift the file doesn't have. + if strategy_name != "exact": + drift_err = _detect_escape_drift(content, matches, old_string, new_string) + if drift_err: + return content, 0, None, drift_err + + effective_new = _maybe_unescape_new_string(new_string, content, matches) + if strategy_name == "unicode_normalized": + effective_new = _preserve_unicode_in_replacement( + content, matches, old_string, effective_new, + ) + new_content = _apply_replacements( + content, matches, effective_new, + old_string=old_string if strategy_name != "exact" else None, + ) + return new_content, len(matches), strategy_name, None + return content, 0, None, "Could not find a match for old_string in the file" +# ── Escape-drift guards ────────────────────────────────────────────────── + def _detect_escape_drift(content: str, matches: List[Tuple[int, int]], old_string: str, new_string: str) -> Optional[str]: - """Detect tool-call escape-drift artifacts in new_string. + """Error string when new_string carries tool-call escape artifacts, else None. - Looks for ``\\'`` or ``\\"`` sequences that are present in both - old_string and new_string (i.e. the model copy-pasted them as "context" - it intended to preserve) but don't exist in the matched region of the - file. That pattern indicates the transport layer inserted spurious - shell-style escapes around apostrophes or quotes — writing new_string - verbatim would literally insert ``\\'`` into source code. - - Returns an error string if drift is detected, None otherwise. + Fires on ``\\'``/``\\"`` present in both old_string and new_string but + absent from the matched region (spurious shell-style escaping), and on + JSON double-escaped backslash runs (see ``_detect_backslash_doubling``). """ - # Cheap pre-check: bail out unless new_string actually contains a - # suspect escape sequence or any backslash at all (the doubling guard - # below only matters when backslashes are present). This keeps the - # guard free for all the common, correct cases. has_quote_suspects = "\\'" in new_string or '\\"' in new_string if not has_quote_suspects and "\\" not in old_string: return None - # Aggregate matched regions of the file — that's what new_string will - # replace. If the suspect escapes are present there already, the - # model is genuinely preserving them (valid for some languages / - # escaped strings); accept the patch. - matched_regions = "".join(content[start:end] for start, end in matches) - + matched_regions = _matched_regions(content, matches) if has_quote_suspects: for suspect in ("\\'", '\\"'): if suspect in new_string and suspect in old_string and suspect not in matched_regions: - plain = suspect[1] # "'" or '"' + plain = suspect[1] return ( f"Escape-drift detected: old_string and new_string contain " f"the literal sequence {suspect!r} but the matched region of " @@ -300,23 +165,11 @@ def _detect_escape_drift(content: str, matches: List[Tuple[int, int]], f"read_file and pass old_string/new_string without " f"backslash-escaping {plain!r} characters." ) - - # Backslash-run doubling: the model sent old_string with 2x the - # backslashes the file actually has (JSON string double-escaping — - # source text `\\` arrives as `\\\\`). A similarity strategy can match - # the region anyway, and writing new_string verbatim then doubles every - # backslash run in the file (`C:\\Users` becomes `C:\\\\Users`). - # Detect the halving relationship between the shared (unchanged) text - # of old_string and the matched region and block with guidance rather - # than silently corrupting escape sequences. - drift = _detect_backslash_doubling(matched_regions, old_string, new_string) - if drift: - return drift - return None + return _detect_backslash_doubling(matched_regions, old_string, new_string) def _backslash_runs(s: str) -> List[int]: - """Return the lengths of maximal backslash runs in ``s``, in order.""" + """Lengths of maximal backslash runs in ``s``, in order.""" runs: List[int] = [] n = 0 for ch in s: @@ -332,14 +185,13 @@ def _backslash_runs(s: str) -> List[int]: def _detect_backslash_doubling(matched_regions: str, old_string: str, new_string: str) -> Optional[str]: - """Detect JSON double-escaped backslashes in old_string/new_string. + """Detect old_string whose every backslash run is exactly 2x the file's. - Fires when every backslash run in old_string is exactly twice the - length of the corresponding run in the matched file region (with the - same number of runs, and at least one run of length >= 2 so a single - doubled backslash in prose can't trigger it). That pattern means the - tool-call arguments went through an extra JSON-escaping pass; writing - new_string verbatim would double every backslash in the file. + That pattern means the arguments were JSON-escaped one extra time; a + similarity strategy still matches, and writing new_string verbatim would + double every backslash in the file. Requires the same run count, a + non-trivial signal (a run >= 2 or 2+ runs), and new_string not already + matching the file's counts. """ old_runs = _backslash_runs(old_string) file_runs = _backslash_runs(matched_regions) @@ -347,19 +199,11 @@ def _detect_backslash_doubling(matched_regions: str, old_string: str, return None if old_runs == file_runs: return None - # Every old run must be exactly double its file counterpart, and the - # doubling must be non-trivial (>= 2 backslashes in the file) for at - # least one run — a lone `\` vs `\\` is too weak a signal on its own - # unless it is consistent across 2+ runs. if any(o != f * 2 for o, f in zip(old_runs, file_runs)): return None if not (any(f >= 2 for f in file_runs) or len(file_runs) >= 2): return None - # new_string must exhibit the same doubling (the model copy-pasted the - # doubled form); if it already matches the file's counts, writing it is - # harmless and we let the edit through. - new_runs = _backslash_runs(new_string) - if new_runs == file_runs: + if _backslash_runs(new_string) == file_runs: return None return ( "Escape-drift detected: every backslash run in old_string is exactly " @@ -372,118 +216,19 @@ def _detect_backslash_doubling(matched_regions: str, old_string: str, ) -def _leading_whitespace(line: str) -> str: - """Return the leading whitespace prefix of a line (spaces/tabs).""" - i = 0 - while i < len(line) and line[i] in (" ", "\t"): - i += 1 - return line[:i] - - -def _first_meaningful_line(text: str) -> Optional[str]: - """Return the first line of ``text`` that has any non-whitespace content. - - Returns ``None`` if no such line exists (text is empty or all whitespace). - """ - for line in text.split("\n"): - if line.strip(): - return line - return None - - -def _reindent_replacement(file_region: str, old_string: str, new_string: str) -> str: - """Adjust ``new_string`` so its indentation matches ``file_region``. - - Used after a non-exact fuzzy match: the LLM may have sent old_string and - new_string with a different indent than the file actually has (e.g. - 2-space indent in tool args vs 4-space indent on disk). The fuzzy - strategy successfully matched anyway, but writing ``new_string`` verbatim - would corrupt the file's indentation. - - Approach: - - 1. For each non-blank line in ``new_string``, compute its indent - *relative* to the shallowest non-blank line of ``old_string`` (the - LLM's base indent). - 2. Anchor that relative indent onto the file's actual base indent (the - leading whitespace of the file_region's first non-blank line). - 3. Re-emit each non-blank line as ``file_base + (line_indent - llm_base)``. - - Blank lines and lines less-indented than the LLM's base are anchored - directly to the file's base indent. - - No-op cases (returns ``new_string`` unchanged): - - file_region or old_string has no meaningful line - - LLM base indent equals file base indent - - new_string is empty - """ - if not new_string: - return new_string - - old_first = _first_meaningful_line(old_string) - file_first = _first_meaningful_line(file_region) - if old_first is None or file_first is None: - return new_string - - old_indent = _leading_whitespace(old_first) - file_indent = _leading_whitespace(file_first) - - if old_indent == file_indent: - return new_string - - # Re-indent each line of new_string. Strategy: replace the LLM's base - # indent prefix with the file's base indent prefix, preserving any - # additional indent the LLM added on top. This is the same approach - # Roo Code uses (multi-search-replace.ts:466-500). It preserves the - # LLM's intended *relative* nesting between lines while anchoring to - # the file's actual indent style. - out_lines: List[str] = [] - for line in new_string.split("\n"): - if not line.strip(): - # Blank lines: leave whitespace untouched. - out_lines.append(line) - continue - line_indent = _leading_whitespace(line) - if line_indent.startswith(old_indent): - # Common case: line has the LLM's base indent (possibly plus - # extra). Swap base prefix for the file's base prefix. - remainder = line[len(old_indent):] - out_lines.append(file_indent + remainder) - else: - # Line is less-indented than the LLM's base — e.g. a dedent at - # the start of new_string. Anchor to the file's base. - out_lines.append(file_indent + line.lstrip(" \t")) - return "\n".join(out_lines) - - -def _maybe_unescape_new_string(new_string: str, - content: str, +def _maybe_unescape_new_string(new_string: str, content: str, matches: List[Tuple[int, int]]) -> str: - """Conditionally unescape ``\\t``/``\\r`` in new_string. + """Convert literal ``\\t``/``\\r`` in new_string to control chars, per sequence, + only when the matched file region already contains the real control char. - LLMs frequently send the two-character sequences ``\\t`` (backslash + t) - and ``\\r`` (backslash + r) inside JSON tool-call arguments where they - meant a real tab or carriage-return byte. Writing the string verbatim - corrupts tab-indented files with literal backslash-letter pairs. - - The unescape is only applied per-sequence when the *matched region of - the file* actually contains the corresponding control character — that - is, we only convert ``\\t`` -> tab when the file region we're replacing - contains a real tab byte. Files that legitimately contain the literal - two-character string ``"\\t"`` (e.g. a Python source line that defines - ``sep = "\\t"``) get a backslash+t in the matched region instead of a - tab, so we leave new_string alone. - - ``\\n`` is intentionally excluded: newlines serialize correctly through - JSON and rewriting backslash-n would corrupt escape sequences in - string literals far more often than it would help. + Files that legitimately contain the two-char string (e.g. ``sep = "\\t"``) + have a backslash+t in the region, not a tab, so they're left alone. + ``\\n`` is deliberately excluded: newlines serialize correctly through + JSON and rewriting them would mangle escape sequences in source literals. """ - # Cheap pre-check — bail out unless new_string actually contains one of - # the suspect sequences. Keeps the common case free. if "\\t" not in new_string and "\\r" not in new_string: return new_string - - matched_regions = "".join(content[start:end] for start, end in matches) + matched_regions = _matched_regions(content, matches) out = new_string if "\\t" in out and "\t" in matched_regions: out = out.replace("\\t", "\t") @@ -492,594 +237,99 @@ def _maybe_unescape_new_string(new_string: str, return out +# ── Replacement shaping ────────────────────────────────────────────────── + +def _leading_whitespace(line: str) -> str: + return line[:len(line) - len(line.lstrip(" \t"))] + + +def _first_meaningful_line(text: str) -> Optional[str]: + for line in text.split("\n"): + if line.strip(): + return line + return None + + +def _reindent_replacement(file_region: str, old_string: str, new_string: str) -> str: + """Re-anchor ``new_string``'s indentation onto the file's actual base indent. + + After a non-exact match the LLM's base indent (first non-blank line of + old_string) may differ from the file's. Each non-blank new_string line + swaps the LLM base prefix for the file's, preserving relative nesting; + lines shallower than the LLM base are anchored to the file base. + """ + if not new_string: + return new_string + old_first = _first_meaningful_line(old_string) + file_first = _first_meaningful_line(file_region) + if old_first is None or file_first is None: + return new_string + old_indent = _leading_whitespace(old_first) + file_indent = _leading_whitespace(file_first) + if old_indent == file_indent: + return new_string + + out_lines: List[str] = [] + for line in new_string.split("\n"): + if not line.strip(): + out_lines.append(line) + elif _leading_whitespace(line).startswith(old_indent): + out_lines.append(file_indent + line[len(old_indent):]) + else: + out_lines.append(file_indent + line.lstrip(" \t")) + return "\n".join(out_lines) + + def _preserve_unicode_in_replacement( content: str, matches: List[Tuple[int, int]], old_string: str, new_string: str, ) -> str: - """Preserve Unicode characters from the file in the replacement string. + """Apply only the old->new edits onto the file's original (Unicode) text. - When strategy 7 (unicode_normalized) matched, the file has Unicode - characters (em-dashes, smart quotes, ellipsis, non-breaking spaces) - but old_string/new_string from the LLM are ASCII equivalents. - Writing new_string verbatim would silently corrupt the file's - Unicode — em-dashes become two hyphens, smart quotes become - straight quotes. - - This function aligns the replacement with the file's actual Unicode - by diffing old_string→new_string and applying only the actual edits - to the file's original text, preserving Unicode for unchanged portions. + After a unicode_normalized match, writing the LLM's ASCII new_string + verbatim would flatten the file's em-dashes/smart quotes. Diff the + normalized old_string against new_string and keep the file's original + characters for every ``equal`` span. """ - # Aggregate the matched file regions - file_region = "".join(content[start:end] for start, end in matches) - - # Normalize both for comparison + file_region = _matched_regions(content, matches) norm_old = _unicode_normalize(old_string) - norm_file = _unicode_normalize(file_region) + if norm_old != _unicode_normalize(file_region): + return new_string # strategy shouldn't have fired; fall back - # If the normalized forms don't match, the strategy shouldn't have - # fired — fall back to direct replacement. - if norm_old != norm_file: - return new_string - - # Build position maps from normalized space back to original space - # for both old_string and file_region. UNICODE_MAP replacements can - # expand characters (em-dash → '--'), so normalized positions don't - # map 1:1 to original positions. Reuse the module-level - # _build_orig_to_norm_map, then invert it (same inversion as - # _map_positions_norm_to_orig) to get norm→orig lookups. file_orig_to_norm = _build_orig_to_norm_map(file_region) - file_norm_to_orig: dict[int, int] = {} - for orig_pos, np in enumerate(file_orig_to_norm[:-1]): - if np not in file_norm_to_orig: - file_norm_to_orig[np] = orig_pos + file_norm_to_orig = _invert_norm_map(file_orig_to_norm) - # Diff norm_old → new_string to find the actual edits - sm = SequenceMatcher(None, norm_old, new_string) - opcodes = sm.get_opcodes() - - # Apply edits to file_region, preserving Unicode for unchanged spans result_parts: List[str] = [] - for tag, i1, i2, j1, j2 in opcodes: + for tag, i1, i2, j1, j2 in SequenceMatcher(None, norm_old, new_string).get_opcodes(): if tag == "equal": - # Keep the original file_region text for this span orig_start = file_norm_to_orig.get(i1, 0) - orig_end = orig_start - while ( - orig_end < len(file_region) - and file_orig_to_norm[orig_end] < i2 - ): - orig_end += 1 + orig_end = _norm_end_to_orig(file_orig_to_norm, orig_start, i2) result_parts.append(file_region[orig_start:orig_end]) - elif tag == "replace": + elif tag != "delete": result_parts.append(new_string[j1:j2]) - elif tag == "delete": - pass # skip deleted portion - elif tag == "insert": - result_parts.append(new_string[j1:j2]) - return "".join(result_parts) def _apply_replacements(content: str, matches: List[Tuple[int, int]], new_string: str, old_string: Optional[str] = None) -> str: + """Splice ``new_string`` over each span (end-to-start so offsets stay valid). + + ``old_string`` non-None signals a non-exact match: new_string is + re-indented per region to the file's actual indentation. """ - Apply replacements at the given positions. - - Args: - content: Original content - matches: List of (start, end) positions to replace - new_string: Replacement text - old_string: When non-None, signals that the match came from a - non-exact fuzzy strategy; ``new_string`` is re-indented to - match the file's actual indentation before substitution. - - Returns: - Content with replacements applied - """ - # Sort matches by position (descending) to replace from end to start - # This preserves positions of earlier matches - sorted_matches = sorted(matches, key=lambda x: x[0], reverse=True) - result = content - for start, end in sorted_matches: + for start, end in sorted(matches, key=lambda x: x[0], reverse=True): + adjusted = new_string if old_string is not None: - file_region = content[start:end] - adjusted = _reindent_replacement(file_region, old_string, new_string) - else: - adjusted = new_string + adjusted = _reindent_replacement(content[start:end], old_string, new_string) result = result[:start] + adjusted + result[end:] - return result -# ============================================================================= -# Matching Strategies -# ============================================================================= - -def _strategy_exact(content: str, pattern: str) -> List[Tuple[int, int]]: - """Strategy 1: Exact string match.""" - matches = [] - start = 0 - while True: - pos = content.find(pattern, start) - if pos == -1: - break - matches.append((pos, pos + len(pattern))) - # Advance past the whole match, not just one char, so self-overlapping - # patterns (e.g. "aa" in "aaaa") produce non-overlapping spans matching - # str.replace() semantics. Advancing by 1 yielded overlapping matches - # that corrupt the file under replace_all=True (reverse-order apply on - # stale offsets). - start = pos + len(pattern) - return matches - - -def _strategy_line_trimmed(content: str, pattern: str) -> List[Tuple[int, int]]: - """ - Strategy 2: Match with line-by-line whitespace trimming. - - Strips leading/trailing whitespace from each line before matching. - """ - # Normalize pattern and content by trimming each line - pattern_lines = [line.strip() for line in pattern.split('\n')] - pattern_normalized = '\n'.join(pattern_lines) - - content_lines = content.split('\n') - content_normalized_lines = [line.strip() for line in content_lines] - - # Build mapping from normalized positions back to original positions - return _find_normalized_matches( - content, content_lines, content_normalized_lines, - pattern, pattern_normalized - ) - - -def _strategy_whitespace_normalized(content: str, pattern: str) -> List[Tuple[int, int]]: - """ - Strategy 3: Collapse multiple whitespace to single space. - """ - def normalize(s): - # Collapse multiple spaces/tabs to single space, preserve newlines - return re.sub(r'[ \t]+', ' ', s) - - pattern_normalized = normalize(pattern) - content_normalized = normalize(content) - - # Find in normalized, map back to original - matches_in_normalized = _strategy_exact(content_normalized, pattern_normalized) - - if not matches_in_normalized: - return [] - - # Map positions back to original content - return _map_normalized_positions(content, content_normalized, matches_in_normalized) - - -def _strategy_indentation_flexible(content: str, pattern: str) -> List[Tuple[int, int]]: - """ - Strategy 4: Ignore indentation differences entirely. - - Strips all leading whitespace from lines before matching. - """ - content_lines = content.split('\n') - content_stripped_lines = [line.lstrip() for line in content_lines] - pattern_lines = [line.lstrip() for line in pattern.split('\n')] - - return _find_normalized_matches( - content, content_lines, content_stripped_lines, - pattern, '\n'.join(pattern_lines) - ) - - -def _strategy_escape_normalized(content: str, pattern: str) -> List[Tuple[int, int]]: - """ - Strategy 5: Convert escape sequences to actual characters. - - Handles \\n -> newline, \\t -> tab, etc. - """ - def unescape(s): - # Convert common escape sequences - return s.replace('\\n', '\n').replace('\\t', '\t').replace('\\r', '\r') - - pattern_unescaped = unescape(pattern) - - if pattern_unescaped == pattern: - # No escapes to convert, skip this strategy - return [] - - return _strategy_exact(content, pattern_unescaped) - - -def _strategy_trimmed_boundary(content: str, pattern: str) -> List[Tuple[int, int]]: - """ - Strategy 6: Trim whitespace from first and last lines only. - - Useful when the pattern boundaries have whitespace differences. - """ - pattern_lines = pattern.split('\n') - if not pattern_lines: - return [] - - # Trim only first and last lines - pattern_lines[0] = pattern_lines[0].strip() - if len(pattern_lines) > 1: - pattern_lines[-1] = pattern_lines[-1].strip() - - modified_pattern = '\n'.join(pattern_lines) - - content_lines = content.split('\n') - - # Search through content for matching block - matches = [] - pattern_line_count = len(pattern_lines) - - for i in range(len(content_lines) - pattern_line_count + 1): - block_lines = content_lines[i:i + pattern_line_count] - - # Trim first and last of this block - check_lines = block_lines.copy() - check_lines[0] = check_lines[0].strip() - if len(check_lines) > 1: - check_lines[-1] = check_lines[-1].strip() - - if '\n'.join(check_lines) == modified_pattern: - # Found match - calculate original positions - start_pos, end_pos = _calculate_line_positions( - content_lines, i, i + pattern_line_count, len(content) - ) - matches.append((start_pos, end_pos)) - - return matches - - -def _build_orig_to_norm_map(original: str) -> List[int]: - """Build a list mapping each original character index to its normalized index. - - Because UNICODE_MAP replacements may expand characters (e.g. em-dash → '--', - ellipsis → '...'), the normalised string can be longer than the original. - This map lets us convert positions in the normalised string back to the - corresponding positions in the original string. - - Returns a list of length ``len(original) + 1``; entry ``i`` is the - normalised index that character ``i`` maps to. - """ - result: List[int] = [] - norm_pos = 0 - for char in original: - result.append(norm_pos) - repl = UNICODE_MAP.get(char) - norm_pos += len(repl) if repl is not None else 1 - result.append(norm_pos) # sentinel: one past the last character - return result - - -def _map_positions_norm_to_orig( - orig_to_norm: List[int], - norm_matches: List[Tuple[int, int]], -) -> List[Tuple[int, int]]: - """Convert (start, end) positions in the normalised string to original positions.""" - # Invert the map: norm_pos -> first original position with that norm_pos - norm_to_orig_start: dict[int, int] = {} - for orig_pos, norm_pos in enumerate(orig_to_norm[:-1]): - if norm_pos not in norm_to_orig_start: - norm_to_orig_start[norm_pos] = orig_pos - - results: List[Tuple[int, int]] = [] - orig_len = len(orig_to_norm) - 1 # number of original characters - - for norm_start, norm_end in norm_matches: - if norm_start not in norm_to_orig_start: - continue - orig_start = norm_to_orig_start[norm_start] - - # Walk forward until orig_to_norm[orig_end] >= norm_end - orig_end = orig_start - while orig_end < orig_len and orig_to_norm[orig_end] < norm_end: - orig_end += 1 - - results.append((orig_start, orig_end)) - - return results - - -def _strategy_unicode_normalized(content: str, pattern: str) -> List[Tuple[int, int]]: - """Strategy 7: Unicode normalisation. - - Normalises smart quotes, em/en-dashes, ellipsis, and non-breaking spaces - to their ASCII equivalents in both *content* and *pattern*, then runs - exact and line_trimmed matching on the normalised copies. - - Positions are mapped back to the *original* string via - ``_build_orig_to_norm_map`` — necessary because some UNICODE_MAP - replacements expand a single character into multiple ASCII characters, - making a naïve position copy incorrect. - """ - # Normalize both sides. Either the content or the pattern (or both) may - # carry unicode variants — e.g. content has an em-dash that should match - # the LLM's ASCII '--', or vice-versa. Skip only when neither changes. - norm_pattern = _unicode_normalize(pattern) - norm_content = _unicode_normalize(content) - if norm_content == content and norm_pattern == pattern: - return [] - - norm_matches = _strategy_exact(norm_content, norm_pattern) - if not norm_matches: - norm_matches = _strategy_line_trimmed(norm_content, norm_pattern) - - if not norm_matches: - return [] - - orig_to_norm = _build_orig_to_norm_map(content) - return _map_positions_norm_to_orig(orig_to_norm, norm_matches) - - -def _strategy_block_anchor(content: str, pattern: str) -> List[Tuple[int, int]]: - """ - Strategy 8: Match by anchoring on first and last lines. - Adjusted with permissive thresholds and unicode normalization. - """ - # Normalize both strings for comparison while keeping original content for offset calculation - norm_pattern = _unicode_normalize(pattern) - norm_content = _unicode_normalize(content) - - pattern_lines = norm_pattern.split('\n') - if len(pattern_lines) < 2: - return [] - - first_line = pattern_lines[0].strip() - last_line = pattern_lines[-1].strip() - - # Use normalized lines for matching logic - norm_content_lines = norm_content.split('\n') - # BUT use original lines for calculating start/end positions to prevent index shift - orig_content_lines = content.split('\n') - - pattern_line_count = len(pattern_lines) - - potential_matches = [] - for i in range(len(norm_content_lines) - pattern_line_count + 1): - if (norm_content_lines[i].strip() == first_line and - norm_content_lines[i + pattern_line_count - 1].strip() == last_line): - potential_matches.append(i) - - matches = [] - candidate_count = len(potential_matches) - - # Thresholding logic: 0.50 for unique matches, 0.70 for multiple candidates. - # Previous values (0.10 / 0.30) were dangerously loose — a 10% middle-section - # similarity could match completely unrelated blocks. - threshold = 0.50 if candidate_count == 1 else 0.70 - - for i in potential_matches: - if pattern_line_count <= 2: - similarity = 1.0 - else: - # Compare normalized middle sections - content_middle = '\n'.join(norm_content_lines[i+1:i+pattern_line_count-1]) - pattern_middle = '\n'.join(pattern_lines[1:-1]) - similarity = SequenceMatcher(None, content_middle, pattern_middle).ratio() - - if similarity >= threshold: - # Calculate positions using ORIGINAL lines to ensure correct character offsets in the file - start_pos, end_pos = _calculate_line_positions( - orig_content_lines, i, i + pattern_line_count, len(content) - ) - matches.append((start_pos, end_pos)) - - return matches - - -def _strategy_context_aware(content: str, pattern: str) -> List[Tuple[int, int]]: - """ - Strategy 9 (last resort): anchored line-by-line similarity. - - Only considers blocks whose first AND last lines closely match the - pattern's first/last lines (an anchor pre-filter), then requires EVERY - non-blank pattern line to be highly similar (>=0.80) to the aligned - content line. The anchor filter keeps this from being an O(file x pattern) - scan on every miss, and the all-lines requirement stops a single - coincidental line-match from silently replacing an unrelated block - (the old 50%-of-lines threshold accepted half-garbage patterns and - destroyed the non-matching lines). - """ - pattern_lines = pattern.split('\n') - content_lines = content.split('\n') - - if not pattern_lines: - return [] - - pattern_line_count = len(pattern_lines) - if pattern_line_count > len(content_lines): - return [] - - # Anchor pre-filter: a block is only a candidate when its first and last - # lines are strong matches for the pattern's first/last lines. This is the - # cheap gate that avoids scoring every window in the file. - first_pat = pattern_lines[0].strip() - last_pat = pattern_lines[-1].strip() - ANCHOR_THRESHOLD = 0.80 - - def _sim(a: str, b: str) -> float: - if a == b: - return 1.0 - return SequenceMatcher(None, a, b).ratio() - - matches = [] - for i in range(len(content_lines) - pattern_line_count + 1): - block_lines = content_lines[i:i + pattern_line_count] - - # Cheap anchor check first — skip non-candidate windows without - # scoring their interior. - if _sim(first_pat, block_lines[0].strip()) < ANCHOR_THRESHOLD: - continue - if _sim(last_pat, block_lines[-1].strip()) < ANCHOR_THRESHOLD: - continue - - # Candidate: require EVERY non-blank pattern line to match its aligned - # content line closely. One garbage line disqualifies the block. - all_match = True - for p_line, c_line in zip(pattern_lines, block_lines): - p_stripped = p_line.strip() - if not p_stripped: - continue # blank pattern lines don't constrain the match - if _sim(p_stripped, c_line.strip()) < 0.80: - all_match = False - break - - if all_match: - start_pos, end_pos = _calculate_line_positions( - content_lines, i, i + pattern_line_count, len(content) - ) - matches.append((start_pos, end_pos)) - - return matches - - -# ============================================================================= -# Helper Functions -# ============================================================================= - -def _calculate_line_positions(content_lines: List[str], start_line: int, - end_line: int, content_length: int) -> Tuple[int, int]: - """Calculate start and end character positions from line indices. - - Args: - content_lines: List of lines (without newlines) - start_line: Starting line index (0-based) - end_line: Ending line index (exclusive, 0-based) - content_length: Total length of the original content string - - Returns: - Tuple of (start_pos, end_pos) in the original content - """ - start_pos = sum(len(line) + 1 for line in content_lines[:start_line]) - end_pos = sum(len(line) + 1 for line in content_lines[:end_line]) - 1 - end_pos = min(content_length, end_pos) - return start_pos, end_pos - - -def _find_normalized_matches(content: str, content_lines: List[str], - content_normalized_lines: List[str], - pattern: str, pattern_normalized: str) -> List[Tuple[int, int]]: - """ - Find matches in normalized content and map back to original positions. - - Args: - content: Original content string - content_lines: Original content split by lines - content_normalized_lines: Normalized content lines - pattern: Original pattern - pattern_normalized: Normalized pattern - - Returns: - List of (start, end) positions in the original content - """ - pattern_norm_lines = pattern_normalized.split('\n') - num_pattern_lines = len(pattern_norm_lines) - - matches = [] - - for i in range(len(content_normalized_lines) - num_pattern_lines + 1): - # Check if this block matches - block = '\n'.join(content_normalized_lines[i:i + num_pattern_lines]) - - if block == pattern_normalized: - # Found a match - calculate original positions - start_pos, end_pos = _calculate_line_positions( - content_lines, i, i + num_pattern_lines, len(content) - ) - matches.append((start_pos, end_pos)) - - return matches - - -def _map_normalized_positions(original: str, normalized: str, - normalized_matches: List[Tuple[int, int]]) -> List[Tuple[int, int]]: - """ - Map positions from normalized string back to original. - - This is a best-effort mapping that works for whitespace normalization. - """ - if not normalized_matches: - return [] - - # Build character mapping from normalized to original - orig_to_norm = [] # orig_to_norm[i] = position in normalized - - orig_idx = 0 - norm_idx = 0 - - while orig_idx < len(original) and norm_idx < len(normalized): - if original[orig_idx] == normalized[norm_idx]: - orig_to_norm.append(norm_idx) - orig_idx += 1 - norm_idx += 1 - elif original[orig_idx] in ' \t' and normalized[norm_idx] == ' ': - # Original has space/tab, normalized collapsed to space - orig_to_norm.append(norm_idx) - orig_idx += 1 - # Don't advance norm_idx yet - wait until all whitespace consumed - if orig_idx < len(original) and original[orig_idx] not in ' \t': - norm_idx += 1 - elif original[orig_idx] in ' \t': - # Extra whitespace in original - orig_to_norm.append(norm_idx) - orig_idx += 1 - else: - # Mismatch - shouldn't happen with our normalization - orig_to_norm.append(norm_idx) - orig_idx += 1 - - # Fill remaining - while orig_idx < len(original): - orig_to_norm.append(len(normalized)) - orig_idx += 1 - - # Reverse mapping: for each normalized position, find original range - norm_to_orig_start = {} - norm_to_orig_end = {} - - for orig_pos, norm_pos in enumerate(orig_to_norm): - if norm_pos not in norm_to_orig_start: - norm_to_orig_start[norm_pos] = orig_pos - norm_to_orig_end[norm_pos] = orig_pos - - # Map matches - original_matches = [] - for norm_start, norm_end in normalized_matches: - # Find original start - if norm_start in norm_to_orig_start: - orig_start = norm_to_orig_start[norm_start] - else: - # Find nearest - orig_start = min(i for i, n in enumerate(orig_to_norm) if n >= norm_start) - - # Find original end - if norm_end - 1 in norm_to_orig_end: - orig_end = norm_to_orig_end[norm_end - 1] + 1 - else: - orig_end = orig_start + (norm_end - norm_start) - - # Expand to include trailing whitespace that was normalized, - # but only when the normalized match itself ended with whitespace. - # When the match ends with a non-space character, the first - # whitespace in the original is a word boundary and must not be - # consumed. See https://github.com/NousResearch/hermes-agent/issues/52491 - if norm_end < len(normalized) and normalized[norm_end - 1] == ' ': - while orig_end < len(original) and original[orig_end] in ' \t': - orig_end += 1 - - original_matches.append((orig_start, min(orig_end, len(original)))) - - return original_matches - +# ── "Did you mean?" diagnostics ────────────────────────────────────────── def _visualize_whitespace(line: str) -> str: - """Render leading whitespace visibly (→ = tab, · = space). - - Only the leading run is visualized — interior spacing is rarely the - culprit and full visualization makes lines unreadable. - """ + """Render the leading whitespace run visibly (→ = tab, · = space).""" i = 0 prefix = [] while i < len(line) and line[i] in (" ", "\t"): @@ -1089,30 +339,21 @@ def _visualize_whitespace(line: str) -> str: def find_closest_lines(old_string: str, content: str, context_lines: int = 2, max_results: int = 3) -> str: - """Find lines in content most similar to old_string for "did you mean?" feedback. - - Returns a formatted string showing the closest matching lines with context, - or empty string if no useful match is found. - """ + """Numbered snippets of the lines most similar to old_string's anchor line, or ''.""" if not old_string or not content: return "" - old_lines = old_string.splitlines() content_lines = content.splitlines() - if not old_lines or not content_lines: return "" - # Use first line of old_string as anchor for search anchor = old_lines[0].strip() if not anchor: - # Try second line if first is blank candidates = [l.strip() for l in old_lines if l.strip()] if not candidates: return "" anchor = candidates[0] - # Score each line in content by similarity to anchor scored = [] for i, line in enumerate(content_lines): stripped = line.strip() @@ -1121,11 +362,8 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma ratio = SequenceMatcher(None, anchor, stripped).ratio() if ratio > 0.3: scored.append((ratio, i)) - if not scored: return "" - - # Take top matches scored.sort(key=lambda x: -x[0]) top = scored[:max_results] @@ -1134,26 +372,19 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma for _, line_idx in top: start = max(0, line_idx - context_lines) end = min(len(content_lines), line_idx + len(old_lines) + context_lines) - key = (start, end) - if key in seen_ranges: + if (start, end) in seen_ranges: continue - seen_ranges.add(key) - snippet = "\n".join( + seen_ranges.add((start, end)) + parts.append("\n".join( f"{start + j + 1:4d}| {content_lines[start + j]}" for j in range(end - start) - ) - parts.append(snippet) - + )) if not parts: return "" - result = "\n---\n".join(parts) - # Whitespace diagnosis (pattern from crush's diagnoseMismatch): when the - # best candidate line matches the anchor after stripping but differs in - # raw text, the failure is whitespace-shaped. Show BOTH lines with - # leading whitespace made visible so the model can copy the file's - # exact indentation instead of guessing again. + # Whitespace-shaped miss: best line equals the anchor once stripped. Show + # both with visible leading whitespace so the model copies the file's. best_line = content_lines[top[0][1]] if best_line.strip() == anchor and best_line != old_lines[0]: result += ( @@ -1162,20 +393,15 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma f" you sent: {_visualize_whitespace(old_lines[0])}\n" "Use the exact whitespace shown in 'file has'." ) - return result def format_no_match_hint(error: Optional[str], match_count: int, old_string: str, content: str) -> str: - """Return a '\\n\\nDid you mean...' snippet for plain no-match errors. + """'\\n\\nDid you mean...' snippet for plain no-match errors only, else ''. - Gated so the hint only fires for actual "old_string not found" failures. - Ambiguous-match ("Found N matches"), escape-drift, and identical-strings - errors all have ``match_count == 0`` but a "did you mean?" snippet would - be misleading — those failed for unrelated reasons. - - Returns an empty string when there's nothing useful to append. + Ambiguous-match, escape-drift and identical-strings errors also have + ``match_count == 0`` but a hint would mislead there. """ if match_count != 0: return "" diff --git a/tools/fuzzy_match_strategies.py b/tools/fuzzy_match_strategies.py new file mode 100644 index 0000000000..eb3e3464bd --- /dev/null +++ b/tools/fuzzy_match_strategies.py @@ -0,0 +1,334 @@ +"""Matching strategies for :mod:`tools.fuzzy_match`. + +Each strategy takes ``(content, pattern)`` and returns a list of +``(start, end)`` spans in the ORIGINAL content. ``STRATEGIES`` is the ordered +chain the orchestrator tries; ``SIMILARITY_STRATEGIES`` names the ones whose +matches are approximate and therefore unsafe under ``replace_all``. +""" + +import re +from difflib import SequenceMatcher +from typing import Callable, Dict, List, Tuple + +UNICODE_MAP = { + "\u201c": '"', "\u201d": '"', # smart double quotes + "\u2018": "'", "\u2019": "'", # smart single quotes + "\u2014": "--", "\u2013": "-", # em/en dashes + "\u2026": "...", "\u00a0": " ", # ellipsis and non-breaking space + "\u2212": "-", # typographic minus (math/scientific docs) + # Space-separator family (Zs) beyond NBSP: files with typographic spacing + # otherwise miss every precise strategy and fall through to the + # similarity fallback, which can pick the wrong region. + "\u2000": " ", "\u2001": " ", "\u2002": " ", "\u2003": " ", + "\u2004": " ", "\u2005": " ", "\u2006": " ", "\u2007": " ", + "\u2008": " ", "\u2009": " ", "\u200a": " ", "\u202f": " ", + "\u205f": " ", "\u3000": " ", +} + +Span = Tuple[int, int] + + +def _unicode_normalize(text: str) -> str: + """Map typographic Unicode variants to ASCII equivalents.""" + for char, repl in UNICODE_MAP.items(): + text = text.replace(char, repl) + return text + + +# ── Position helpers ───────────────────────────────────────────────────── + +def _calculate_line_positions(content_lines: List[str], start_line: int, + end_line: int, content_length: int) -> Span: + """Character span covering ``content_lines[start_line:end_line]`` (end exclusive).""" + start_pos = sum(len(line) + 1 for line in content_lines[:start_line]) + end_pos = sum(len(line) + 1 for line in content_lines[:end_line]) - 1 + return start_pos, min(content_length, end_pos) + + +def _match_transformed_lines(content: str, pattern: str, + transform: Callable[[str], str]) -> List[Span]: + """Match ``pattern`` against ``content`` after applying ``transform`` per line.""" + content_lines = content.split('\n') + norm_lines = [transform(line) for line in content_lines] + pattern_norm = '\n'.join(transform(line) for line in pattern.split('\n')) + n = pattern_norm.count('\n') + 1 + matches = [] + for i in range(len(norm_lines) - n + 1): + if '\n'.join(norm_lines[i:i + n]) == pattern_norm: + matches.append(_calculate_line_positions(content_lines, i, i + n, len(content))) + return matches + + +def _build_orig_to_norm_map(original: str) -> List[int]: + """Map each original index to its index in ``_unicode_normalize(original)``. + + UNICODE_MAP replacements can expand one char into several, so the map is + needed to translate normalised spans back. Length is ``len(original)+1``; + the last entry is a sentinel one past the final character. + """ + result: List[int] = [] + norm_pos = 0 + for char in original: + result.append(norm_pos) + repl = UNICODE_MAP.get(char) + norm_pos += len(repl) if repl is not None else 1 + result.append(norm_pos) + return result + + +def _invert_norm_map(orig_to_norm: List[int]) -> Dict[int, int]: + """norm_pos -> first original position mapping to it.""" + inverted: Dict[int, int] = {} + for orig_pos, norm_pos in enumerate(orig_to_norm[:-1]): + if norm_pos not in inverted: + inverted[norm_pos] = orig_pos + return inverted + + +def _norm_end_to_orig(orig_to_norm: List[int], orig_start: int, norm_end: int) -> int: + """Walk from ``orig_start`` until the mapped position reaches ``norm_end``.""" + orig_len = len(orig_to_norm) - 1 + orig_end = orig_start + while orig_end < orig_len and orig_to_norm[orig_end] < norm_end: + orig_end += 1 + return orig_end + + +def _map_positions_norm_to_orig(orig_to_norm: List[int], + norm_matches: List[Span]) -> List[Span]: + """Convert spans in the normalised string to original-string spans.""" + norm_to_orig_start = _invert_norm_map(orig_to_norm) + results: List[Span] = [] + for norm_start, norm_end in norm_matches: + if norm_start not in norm_to_orig_start: + continue + orig_start = norm_to_orig_start[norm_start] + results.append((orig_start, _norm_end_to_orig(orig_to_norm, orig_start, norm_end))) + return results + + +def _map_normalized_positions(original: str, normalized: str, + normalized_matches: List[Span]) -> List[Span]: + """Best-effort span mapping for ``[ \\t]+`` -> ``' '`` whitespace collapsing.""" + orig_to_norm = [] # orig_to_norm[i] = position in normalized + orig_idx = norm_idx = 0 + while orig_idx < len(original) and norm_idx < len(normalized): + if original[orig_idx] == normalized[norm_idx]: + orig_to_norm.append(norm_idx) + orig_idx += 1 + norm_idx += 1 + elif original[orig_idx] in ' \t' and normalized[norm_idx] == ' ': + # Collapsed run: advance norm_idx only once the run is consumed. + orig_to_norm.append(norm_idx) + orig_idx += 1 + if orig_idx < len(original) and original[orig_idx] not in ' \t': + norm_idx += 1 + else: + # Extra whitespace in original, or a mismatch that normalization + # should never produce — either way, pin to the current norm_idx. + orig_to_norm.append(norm_idx) + orig_idx += 1 + while orig_idx < len(original): + orig_to_norm.append(len(normalized)) + orig_idx += 1 + + norm_to_orig_start = {} + norm_to_orig_end = {} + for orig_pos, norm_pos in enumerate(orig_to_norm): + if norm_pos not in norm_to_orig_start: + norm_to_orig_start[norm_pos] = orig_pos + norm_to_orig_end[norm_pos] = orig_pos + + original_matches = [] + for norm_start, norm_end in normalized_matches: + if norm_start in norm_to_orig_start: + orig_start = norm_to_orig_start[norm_start] + else: + orig_start = min(i for i, n in enumerate(orig_to_norm) if n >= norm_start) + if norm_end - 1 in norm_to_orig_end: + orig_end = norm_to_orig_end[norm_end - 1] + 1 + else: + orig_end = orig_start + (norm_end - norm_start) + # Absorb trailing collapsed whitespace only when the normalized match + # itself ended in a space; otherwise the first whitespace after the + # match is a word boundary that must survive (#52491). + if norm_end < len(normalized) and normalized[norm_end - 1] == ' ': + while orig_end < len(original) and original[orig_end] in ' \t': + orig_end += 1 + original_matches.append((orig_start, min(orig_end, len(original)))) + return original_matches + + +# ── Strategies ─────────────────────────────────────────────────────────── + +def _strategy_exact(content: str, pattern: str) -> List[Span]: + """Strategy 1: exact, non-overlapping occurrences (str.replace semantics).""" + matches = [] + start = 0 + while True: + pos = content.find(pattern, start) + if pos == -1: + break + matches.append((pos, pos + len(pattern))) + # Advance past the whole match: overlapping spans would corrupt the + # file under replace_all (reverse-order apply on stale offsets). + start = pos + len(pattern) + return matches + + +def _strategy_line_trimmed(content: str, pattern: str) -> List[Span]: + """Strategy 2: strip each line before comparing.""" + return _match_transformed_lines(content, pattern, str.strip) + + +def _strategy_whitespace_normalized(content: str, pattern: str) -> List[Span]: + """Strategy 3: collapse runs of spaces/tabs to a single space.""" + def normalize(s): + return re.sub(r'[ \t]+', ' ', s) + + content_normalized = normalize(content) + matches_in_normalized = _strategy_exact(content_normalized, normalize(pattern)) + if not matches_in_normalized: + return [] + return _map_normalized_positions(content, content_normalized, matches_in_normalized) + + +def _strategy_indentation_flexible(content: str, pattern: str) -> List[Span]: + """Strategy 4: ignore leading indentation entirely.""" + return _match_transformed_lines(content, pattern, str.lstrip) + + +def _strategy_escape_normalized(content: str, pattern: str) -> List[Span]: + """Strategy 5: treat literal ``\\n``/``\\t``/``\\r`` in the pattern as control chars.""" + pattern_unescaped = pattern.replace('\\n', '\n').replace('\\t', '\t').replace('\\r', '\r') + if pattern_unescaped == pattern: + return [] + return _strategy_exact(content, pattern_unescaped) + + +def _strategy_trimmed_boundary(content: str, pattern: str) -> List[Span]: + """Strategy 6: strip whitespace on the first and last lines only.""" + pattern_lines = pattern.split('\n') + pattern_lines[0] = pattern_lines[0].strip() + if len(pattern_lines) > 1: + pattern_lines[-1] = pattern_lines[-1].strip() + modified_pattern = '\n'.join(pattern_lines) + n = len(pattern_lines) + + content_lines = content.split('\n') + matches = [] + for i in range(len(content_lines) - n + 1): + check_lines = content_lines[i:i + n] + check_lines[0] = check_lines[0].strip() + if n > 1: + check_lines[-1] = check_lines[-1].strip() + if '\n'.join(check_lines) == modified_pattern: + matches.append(_calculate_line_positions(content_lines, i, i + n, len(content))) + return matches + + +def _strategy_unicode_normalized(content: str, pattern: str) -> List[Span]: + """Strategy 7: exact/line-trimmed match after Unicode->ASCII normalisation of both sides.""" + norm_pattern = _unicode_normalize(pattern) + norm_content = _unicode_normalize(content) + if norm_content == content and norm_pattern == pattern: + return [] + norm_matches = _strategy_exact(norm_content, norm_pattern) + if not norm_matches: + norm_matches = _strategy_line_trimmed(norm_content, norm_pattern) + if not norm_matches: + return [] + return _map_positions_norm_to_orig(_build_orig_to_norm_map(content), norm_matches) + + +def _strategy_block_anchor(content: str, pattern: str) -> List[Span]: + """Strategy 8: anchor on first+last lines, similarity-score the middle.""" + pattern_lines = _unicode_normalize(pattern).split('\n') + if len(pattern_lines) < 2: + return [] + first_line = pattern_lines[0].strip() + last_line = pattern_lines[-1].strip() + n = len(pattern_lines) + + # Match on normalized lines; compute offsets from the ORIGINAL lines so + # multi-char expansions (em-dash -> '--') don't shift positions. + norm_content_lines = _unicode_normalize(content).split('\n') + orig_content_lines = content.split('\n') + + potential_matches = [ + i for i in range(len(norm_content_lines) - n + 1) + if norm_content_lines[i].strip() == first_line + and norm_content_lines[i + n - 1].strip() == last_line + ] + # Looser thresholds (0.10/0.30) matched unrelated blocks; these are the safe floor. + threshold = 0.50 if len(potential_matches) == 1 else 0.70 + + matches = [] + for i in potential_matches: + if n <= 2: + similarity = 1.0 + else: + content_middle = '\n'.join(norm_content_lines[i + 1:i + n - 1]) + pattern_middle = '\n'.join(pattern_lines[1:-1]) + similarity = SequenceMatcher(None, content_middle, pattern_middle).ratio() + if similarity >= threshold: + matches.append(_calculate_line_positions(orig_content_lines, i, i + n, len(content))) + return matches + + +def _strategy_context_aware(content: str, pattern: str) -> List[Span]: + """Strategy 9 (last resort): anchored per-line similarity, every non-blank line >= 0.80. + + The first/last-line anchor pre-filter keeps a miss from being an + O(file x pattern) scan; the all-lines requirement stops one coincidental + line match from replacing an unrelated block. + """ + pattern_lines = pattern.split('\n') + content_lines = content.split('\n') + n = len(pattern_lines) + if n > len(content_lines): + return [] + + first_pat = pattern_lines[0].strip() + last_pat = pattern_lines[-1].strip() + ANCHOR_THRESHOLD = 0.80 + + def _sim(a: str, b: str) -> float: + if a == b: + return 1.0 + return SequenceMatcher(None, a, b).ratio() + + matches = [] + for i in range(len(content_lines) - n + 1): + block_lines = content_lines[i:i + n] + if _sim(first_pat, block_lines[0].strip()) < ANCHOR_THRESHOLD: + continue + if _sim(last_pat, block_lines[-1].strip()) < ANCHOR_THRESHOLD: + continue + all_match = True + for p_line, c_line in zip(pattern_lines, block_lines): + p_stripped = p_line.strip() + if p_stripped and _sim(p_stripped, c_line.strip()) < 0.80: + all_match = False + break + if all_match: + matches.append(_calculate_line_positions(content_lines, i, i + n, len(content))) + return matches + + +# Ordered chain: precise strategies first, similarity-based last. +STRATEGIES: List[Tuple[str, Callable[[str, str], List[Span]]]] = [ + ("exact", _strategy_exact), + ("line_trimmed", _strategy_line_trimmed), + ("whitespace_normalized", _strategy_whitespace_normalized), + ("indentation_flexible", _strategy_indentation_flexible), + ("escape_normalized", _strategy_escape_normalized), + ("trimmed_boundary", _strategy_trimmed_boundary), + ("unicode_normalized", _strategy_unicode_normalized), + ("block_anchor", _strategy_block_anchor), + ("context_aware", _strategy_context_aware), +] + +# Matches from these only *approximately* resemble old_string — fine for one +# unique replacement, never safe under replace_all. +SIMILARITY_STRATEGIES = frozenset({"block_anchor", "context_aware"}) diff --git a/tools/homeassistant_tool.py b/tools/homeassistant_tool.py index 771387b4d2..4de74292ab 100644 --- a/tools/homeassistant_tool.py +++ b/tools/homeassistant_tool.py @@ -1,13 +1,8 @@ """Home Assistant tool for controlling smart home devices via REST API. -Registers four LLM-callable tools: -- ``ha_list_entities`` -- list/filter entities by domain or area -- ``ha_get_state`` -- get detailed state of a single entity -- ``ha_list_services`` -- list available services (actions) per domain -- ``ha_call_service`` -- call a HA service (turn_on, turn_off, set_temperature, etc.) - -Authentication uses a Long-Lived Access Token via ``HASS_TOKEN`` env var. -The HA instance URL is read from ``HASS_URL`` (default: http://homeassistant.local:8123). +Registers ``ha_list_entities``, ``ha_get_state``, ``ha_list_services``, ``ha_call_service``. +Auth is a Long-Lived Access Token (``HASS_TOKEN``); the instance URL comes from +``HASS_URL`` (default http://homeassistant.local:8123). """ import asyncio @@ -17,13 +12,10 @@ import re from typing import Any, Dict, Optional from agent.secret_scope import get_secret +from tools.registry import registry, tool_error logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Configuration -# --------------------------------------------------------------------------- - # Kept for backward compatibility (e.g. test monkeypatching); prefer _get_config(). _HASS_URL: str = "" _HASS_TOKEN: str = "" @@ -36,21 +28,16 @@ def _get_config(): _HASS_TOKEN or get_secret("HASS_TOKEN", "") or "", ) -# Regex for valid HA entity_id format (e.g. "light.living_room", "sensor.temperature_1") +# Valid HA entity_id (e.g. "light.living_room", "sensor.temperature_1"). _ENTITY_ID_RE = re.compile(r"^[a-z_][a-z0-9_]*\.[a-z0-9_]+$") -# Regex for valid HA service/domain names (e.g. "light", "turn_on", "shell_command"). -# Only lowercase ASCII letters, digits, and underscores — no slashes, dots, or -# other characters that could allow path traversal in URL construction. -# The domain and service are interpolated into /api/services/{domain}/{service}, -# so allowing arbitrary strings would enable SSRF via path traversal -# (e.g. domain="../../api/config") or blocked-domain bypass -# (e.g. domain="shell_command/../light"). +# Domain/service names are interpolated into /api/services/{domain}/{service}, so only +# [a-z0-9_] is allowed: anything else enables SSRF via path traversal +# (domain="../../api/config") or blocklist bypass (domain="shell_command/../light"). _SERVICE_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*$") -# Service domains blocked for security -- these allow arbitrary code/command -# execution on the HA host or enable SSRF attacks on the local network. -# HA provides zero service-level access control; all safety must be in our layer. +# Domains that allow arbitrary code/command execution on the HA host or SSRF on the +# local network. HA has zero service-level access control; all safety lives here. _BLOCKED_DOMAINS = frozenset({ "shell_command", # arbitrary shell commands as root in HA container "command_line", # sensors/switches that execute shell commands @@ -71,8 +58,22 @@ def _get_headers(token: str = "") -> Dict[str, str]: } +async def _api_json(method: str, path: str, timeout: float, payload: Any = None) -> Any: + """One HA REST call (GET or POST JSON) that raises on HTTP errors and returns the JSON body.""" + import aiohttp + + hass_url, hass_token = _get_config() + kwargs: Dict[str, Any] = {"headers": _get_headers(hass_token), "timeout": aiohttp.ClientTimeout(total=timeout)} + if method == "POST": + kwargs["json"] = payload + async with aiohttp.ClientSession() as session: + async with session.request(method, f"{hass_url}{path}", **kwargs) as resp: + resp.raise_for_status() + return await resp.json() + + # --------------------------------------------------------------------------- -# Async helpers (called from sync handlers via run_until_complete) +# Async helpers (called from sync handlers via _run_async) # --------------------------------------------------------------------------- def _filter_and_summarize( @@ -92,14 +93,14 @@ def _filter_and_summarize( or area_lower in (s.get("attributes", {}).get("area", "") or "").lower() ] - entities = [] - for s in states: - entities.append({ + entities = [ + { "entity_id": s["entity_id"], "state": s["state"], "friendly_name": s.get("attributes", {}).get("friendly_name", ""), - }) - + } + for s in states + ] return {"count": len(entities), "entities": entities} @@ -108,29 +109,13 @@ async def _async_list_entities( area: Optional[str] = None, ) -> Dict[str, Any]: """Fetch entity states from HA and optionally filter by domain/area.""" - import aiohttp - - hass_url, hass_token = _get_config() - url = f"{hass_url}/api/states" - async with aiohttp.ClientSession() as session: - async with session.get(url, headers=_get_headers(hass_token), timeout=aiohttp.ClientTimeout(total=15)) as resp: - resp.raise_for_status() - states = await resp.json() - + states = await _api_json("GET", "/api/states", 15) return _filter_and_summarize(states, domain, area) async def _async_get_state(entity_id: str) -> Dict[str, Any]: """Fetch detailed state of a single entity.""" - import aiohttp - - hass_url, hass_token = _get_config() - url = f"{hass_url}/api/states/{entity_id}" - async with aiohttp.ClientSession() as session: - async with session.get(url, headers=_get_headers(hass_token), timeout=aiohttp.ClientTimeout(total=10)) as resp: - resp.raise_for_status() - data = await resp.json() - + data = await _api_json("GET", f"/api/states/{entity_id}", 10) return { "entity_id": data["entity_id"], "state": data["state"], @@ -144,11 +129,8 @@ def _build_service_payload( entity_id: Optional[str] = None, data: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: - """Build the JSON payload for a HA service call.""" - payload: Dict[str, Any] = {} - if data: - payload.update(data) - # entity_id parameter takes precedence over data["entity_id"] + """Build the JSON payload for a HA service call; ``entity_id`` overrides data["entity_id"].""" + payload: Dict[str, Any] = dict(data or {}) if entity_id: payload["entity_id"] = entity_id return payload @@ -162,12 +144,7 @@ def _parse_service_response( """Parse HA service call response into a structured result.""" affected = [] if isinstance(result, list): - for s in result: - affected.append({ - "entity_id": s.get("entity_id", ""), - "state": s.get("state", ""), - }) - + affected = [{"entity_id": s.get("entity_id", ""), "state": s.get("state", "")} for s in result] return { "success": True, "service": f"{domain}.{service}", @@ -182,83 +159,85 @@ async def _async_call_service( data: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Call a Home Assistant service.""" - import aiohttp - - hass_url, hass_token = _get_config() - url = f"{hass_url}/api/services/{domain}/{service}" - payload = _build_service_payload(entity_id, data) - - async with aiohttp.ClientSession() as session: - async with session.post( - url, - headers=_get_headers(hass_token), - json=payload, - timeout=aiohttp.ClientTimeout(total=15), - ) as resp: - resp.raise_for_status() - result = await resp.json() - + result = await _api_json( + "POST", f"/api/services/{domain}/{service}", 15, _build_service_payload(entity_id, data) + ) return _parse_service_response(domain, service, result) +async def _async_list_services(domain: Optional[str] = None) -> Dict[str, Any]: + """Fetch available services from HA, optionally filtered by domain, compacted for context.""" + services = await _api_json("GET", "/api/services", 15) + if domain: + services = [s for s in services if s.get("domain") == domain] + + result = [] + for svc_domain in services: + domain_services = {} + for svc_name, svc_info in svc_domain.get("services", {}).items(): + svc_entry: Dict[str, Any] = {"description": svc_info.get("description", "")} + fields = svc_info.get("fields", {}) + if fields: + svc_entry["fields"] = { + k: v.get("description", "") for k, v in fields.items() + if isinstance(v, dict) + } + domain_services[svc_name] = svc_entry + result.append({"domain": svc_domain.get("domain", ""), "services": domain_services}) + + return {"count": len(result), "domains": result} + + # --------------------------------------------------------------------------- # Sync wrappers (handler signature: (args, **kw) -> str) # --------------------------------------------------------------------------- def _run_async(coro): - """Run an async coroutine from a sync handler.""" + """Run a coroutine from a sync handler; hops to a thread if a loop is already running.""" try: loop = asyncio.get_running_loop() except RuntimeError: loop = None if loop and loop.is_running(): - # Already inside an event loop -- create a new thread import concurrent.futures with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - future = pool.submit(asyncio.run, coro) - return future.result(timeout=30) - else: - return asyncio.run(coro) + return pool.submit(asyncio.run, coro).result(timeout=30) + return asyncio.run(coro) + + +def _dispatch(coro, log_name: str, fail_msg: str) -> str: + """Run ``coro`` and wrap as ``{"result": ...}``; on error log and return tool_error.""" + try: + return json.dumps({"result": _run_async(coro)}) + except Exception as e: + logger.error("%s error: %s", log_name, e) + return tool_error(f"{fail_msg}: {e}") def _handle_list_entities(args: dict, **kw) -> str: - """Handler for ha_list_entities tool.""" - domain = args.get("domain") - area = args.get("area") - try: - result = _run_async(_async_list_entities(domain=domain, area=area)) - return json.dumps({"result": result}) - except Exception as e: - logger.error("ha_list_entities error: %s", e) - return tool_error(f"Failed to list entities: {e}") + return _dispatch( + _async_list_entities(domain=args.get("domain"), area=args.get("area")), + "ha_list_entities", "Failed to list entities", + ) def _handle_get_state(args: dict, **kw) -> str: - """Handler for ha_get_state tool.""" entity_id = args.get("entity_id", "") if not entity_id: return tool_error("Missing required parameter: entity_id") if not _ENTITY_ID_RE.match(entity_id): return tool_error(f"Invalid entity_id format: {entity_id}") - try: - result = _run_async(_async_get_state(entity_id)) - return json.dumps({"result": result}) - except Exception as e: - logger.error("ha_get_state error: %s", e) - return tool_error(f"Failed to get state for {entity_id}: {e}") + return _dispatch(_async_get_state(entity_id), "ha_get_state", f"Failed to get state for {entity_id}") def _handle_call_service(args: dict, **kw) -> str: - """Handler for ha_call_service tool.""" domain = args.get("domain", "") service = args.get("service", "") if not domain or not service: return tool_error("Missing required parameters: domain and service") - # Validate domain/service format BEFORE the blocklist check — prevents - # path traversal in /api/services/{domain}/{service} and blocklist bypass - # via payloads like "shell_command/../light". + # Format check BEFORE the blocklist: rejects "shell_command/../light" style bypasses. if not _SERVICE_NAME_RE.match(domain): return tool_error(f"Invalid domain format: {domain!r}") if not _SERVICE_NAME_RE.match(service): @@ -275,73 +254,24 @@ def _handle_call_service(args: dict, **kw) -> str: return tool_error(f"Invalid entity_id format: {entity_id}") data = args.get("data") - if isinstance(data, str): + if isinstance(data, str): # XML tool-calling mode delivers data as a JSON string try: data = json.loads(data) if data.strip() else None except json.JSONDecodeError as e: return tool_error(f"Invalid JSON string in 'data' parameter: {e}") - try: - result = _run_async(_async_call_service(domain, service, entity_id, data)) - return json.dumps({"result": result}) - except Exception as e: - logger.error("ha_call_service error: %s", e) - return tool_error(f"Failed to call {domain}.{service}: {e}") - - -# --------------------------------------------------------------------------- -# List services -# --------------------------------------------------------------------------- - -async def _async_list_services(domain: Optional[str] = None) -> Dict[str, Any]: - """Fetch available services from HA and optionally filter by domain.""" - import aiohttp - - hass_url, hass_token = _get_config() - url = f"{hass_url}/api/services" - headers = {"Authorization": f"Bearer {hass_token}", "Content-Type": "application/json"} - async with aiohttp.ClientSession() as session: - async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=15)) as resp: - resp.raise_for_status() - services = await resp.json() - - if domain: - services = [s for s in services if s.get("domain") == domain] - - # Compact the output for context efficiency - result = [] - for svc_domain in services: - d = svc_domain.get("domain", "") - domain_services = {} - for svc_name, svc_info in svc_domain.get("services", {}).items(): - svc_entry: Dict[str, Any] = {"description": svc_info.get("description", "")} - fields = svc_info.get("fields", {}) - if fields: - svc_entry["fields"] = { - k: v.get("description", "") for k, v in fields.items() - if isinstance(v, dict) - } - domain_services[svc_name] = svc_entry - result.append({"domain": d, "services": domain_services}) - - return {"count": len(result), "domains": result} + return _dispatch( + _async_call_service(domain, service, entity_id, data), + "ha_call_service", f"Failed to call {domain}.{service}", + ) def _handle_list_services(args: dict, **kw) -> str: - """Handler for ha_list_services tool.""" - domain = args.get("domain") - try: - result = _run_async(_async_list_services(domain=domain)) - return json.dumps({"result": result}) - except Exception as e: - logger.error("ha_list_services error: %s", e) - return tool_error(f"Failed to list services: {e}") + return _dispatch( + _async_list_services(domain=args.get("domain")), "ha_list_services", "Failed to list services" + ) -# --------------------------------------------------------------------------- -# Availability check -# --------------------------------------------------------------------------- - def _check_ha_available() -> bool: """Tool is only available when HASS_TOKEN is set.""" return bool(get_secret("HASS_TOKEN")) @@ -475,8 +405,6 @@ HA_CALL_SERVICE_SCHEMA = { # Registration # --------------------------------------------------------------------------- -from tools.registry import registry, tool_error - registry.register( name="ha_list_entities", toolset="homeassistant", diff --git a/tools/hook_output_spill.py b/tools/hook_output_spill.py index f7d011bc3b..c518281f03 100644 --- a/tools/hook_output_spill.py +++ b/tools/hook_output_spill.py @@ -1,22 +1,10 @@ """Spill oversized hook-injected context to disk with a preview placeholder. -Ported from openai/codex PR #21069 (``Spill large hook outputs from context``). - -Background ----------- -Both shell hooks (``agent/shell_hooks.py``) and Python plugins -(``pre_llm_call`` hook in ``run_agent.py``) can return ``{"context": "..."}`` -which gets concatenated into the current turn's user message on EVERY -subsequent API call. If a hook emits a large blob (e.g. a debug dump, a -full file, or a runaway prompt-engineering script), that blob inflates -every turn of the session and blows out the prompt cache prefix the -moment it's appended. - -This mirrors what Codex does for its ``PreToolUse``/``Stop``/feedback -hooks: once the injected text exceeds a configured budget, write the -full content to a per-session directory on disk and replace the in-prompt -payload with a head/tail preview plus the saved path. The model can still -inspect the full content via ``read_file`` or ``terminal`` if it needs to. +Shell hooks and plugin ``pre_llm_call`` hooks can return ``{"context": ...}`` +that is concatenated into the user message on EVERY subsequent API call, so a +large blob inflates every turn and breaks the prompt-cache prefix. Above a +configured budget the full text is written to a per-session directory and the +in-prompt payload becomes a head/tail preview plus the saved path. Config (``config.yaml``):: @@ -28,16 +16,8 @@ Config (``config.yaml``):: preview_tail: 500 # chars shown at the end of the preview directory: null # default: /hook_outputs -Design invariants ------------------ -* Behaviour-preserving when ``enabled: false`` or when content is under - the cap — return the input string unchanged. -* Never raises. Any I/O error (disk full, permission denied, missing - HERMES_HOME, etc.) falls back to a byte-length truncation with an - in-prompt notice — the hook context still reaches the model, just - bounded in size. -* Spill files are grouped by session so a ``/new`` session doesn't grow - them forever in one directory. +Invariants: unchanged input when disabled or under the cap; never raises — +an I/O failure still returns a bounded preview with an in-prompt notice. """ from __future__ import annotations @@ -48,6 +28,8 @@ import uuid from pathlib import Path from typing import Any, Dict, Optional +from tools.tool_output_limits import _coerce_int, _coerce_positive_int + logger = logging.getLogger(__name__) @@ -57,25 +39,9 @@ DEFAULT_PREVIEW_TAIL = 500 DEFAULT_ENABLED = True -def _coerce_positive_int(value: Any, default: int) -> int: - try: - iv = int(value) - except (TypeError, ValueError): - return default - if iv <= 0: - return default - return iv - - def _coerce_non_negative_int(value: Any, default: int) -> int: """Like ``_coerce_positive_int`` but allows zero (e.g. empty tail).""" - try: - iv = int(value) - except (TypeError, ValueError): - return default - if iv < 0: - return default - return iv + return _coerce_int(value, default, 0) def get_spill_config() -> Dict[str, Any]: @@ -113,7 +79,7 @@ def get_spill_config() -> Dict[str, Any]: def _resolve_spill_dir(directory_override: Optional[str], session_id: Optional[str]) -> Path: - """Return the directory where spill files for this session live.""" + """Per-session spill directory; session id is sanitised so it can't escape ``base``.""" if directory_override: base = Path(os.path.expanduser(directory_override)) else: @@ -121,10 +87,7 @@ def _resolve_spill_dir(directory_override: Optional[str], session_id: Optional[s base = Path(get_hermes_home()) / "hook_outputs" - # Group by session so spills are contained per conversation. session_segment = session_id or "no-session" - # Defensive: strip path separators so a weird session id can't - # escape the directory. session_segment = session_segment.replace("/", "_").replace("\\", "_").replace("..", "_") return base / session_segment @@ -164,24 +127,9 @@ def spill_if_oversized( ) -> str: """Spill ``text`` to disk if it exceeds the configured cap. - Returns either ``text`` unchanged (when under the cap, disabled, or - empty) or a preview string with a filesystem path pointing at the - full content. - - Parameters - ---------- - text: - The raw injected-context string from a hook. Non-string inputs - are coerced with ``str()``. - session_id: - Used to group spill files by conversation. Falls back to - ``"no-session"`` if missing. - source: - Human-readable label used in the preview header (``"hook"``, - ``"plugin hook"``, ``"shell hook"``, etc.). Free-form. - config: - Optional override for tests; normally resolved from - ``config.yaml``. + Returns ``text`` unchanged (under cap, disabled, or empty) or a preview + string pointing at the full content. Non-string input is ``str()``-coerced; + ``source`` labels the preview header; ``config`` overrides config.yaml. """ if text is None: return "" @@ -203,21 +151,18 @@ def spill_if_oversized( tail = int(cfg.get("preview_tail") or 0) directory_override = cfg.get("directory") - # Try to write the spill file. If that fails we still need to return - # something bounded — never let a disk failure blow up the turn. + # A disk failure must never blow up the turn — fall through to a preview + # without a saved path. saved_path: Optional[str] = None try: spill_dir = _resolve_spill_dir(directory_override, session_id) from tools.spill_safety import ensure_spill_dir, write_text_exclusive - # Hook context may embed raw secrets: private dir/file perms, and an - # exclusive symlink-refusing create so a planted link can't redirect - # the write (predictable per-session directory). + # Hook context may embed raw secrets: private perms + exclusive, + # symlink-refusing create (the per-session dir is predictable). ensure_spill_dir(spill_dir, private=True) - filename = f"{uuid.uuid4().hex}.txt" - spill_path = spill_dir / filename - # Write the raw text plus a trailing newline so tail readers - # (``tail -f``, editors) don't report "missing newline". + spill_path = spill_dir / f"{uuid.uuid4().hex}.txt" + # Trailing newline so tail readers don't report "missing newline". write_text_exclusive( spill_path, text if text.endswith("\n") else text + "\n", diff --git a/tools/image_generation_catalog.py b/tools/image_generation_catalog.py new file mode 100644 index 0000000000..8cd93af163 --- /dev/null +++ b/tools/image_generation_catalog.py @@ -0,0 +1,567 @@ +"""FAL image model catalog + upscaler constants for ``tools.image_generation_tool``. + +Each entry declares how to translate the unified inputs (prompt + aspect_ratio) +into the model's native payload. ``size_style`` picks the family: +``"image_size_preset"`` (FAL preset enum), ``"aspect_ratio"`` (ratio enum), +``"gpt_literal"`` (literal "WxH" strings). ``supports`` / ``edit_supports`` are +whitelists — keys outside them are stripped so models never receive rejected +parameters. ``upscale`` (Clarity Upscaler chained after generation) is False +everywhere: Clarity redraws content (creativity 0.35) and degraded text/CJK/ +faces when default-on, so upscaling is strictly per-call opt-in. +Pricing strings are as-of-commit and allowed to drift. +""" + +from typing import Any, Dict + +_PRESET_SIZES = { + "landscape": "landscape_16_9", + "square": "square_hd", + "portrait": "portrait_16_9", +} +_ASPECT_SIZES = {"landscape": "16:9", "square": "1:1", "portrait": "9:16"} + +FAL_MODELS: Dict[str, Dict[str, Any]] = { + "fal-ai/flux-2/klein/9b": { + "display": "FLUX 2 Klein 9B", + "speed": "<1s", + "strengths": "Fast, crisp text", + "price": "$0.006/MP", + "size_style": "image_size_preset", + "sizes": _PRESET_SIZES, + "defaults": { + "num_inference_steps": 4, + "output_format": "png", + "enable_safety_checker": False, + }, + "supports": { + "prompt", "image_size", "num_inference_steps", "seed", + "output_format", "enable_safety_checker", + }, + "upscale": False, + # Image-to-image / editing: FLUX.2 [klein] 9B edit endpoint takes + # `image_urls` (list). Natural-language edits, multi-ref. + "edit_endpoint": "fal-ai/flux-2/klein/9b/edit", + "edit_supports": { + "prompt", "image_urls", "num_inference_steps", "seed", + "output_format", "enable_safety_checker", + }, + "max_reference_images": 9, + }, + "fal-ai/flux-2-pro": { + "display": "FLUX 2 Pro", + "speed": "~6s", + "strengths": "Studio photorealism", + "price": "$0.03/MP", + "size_style": "image_size_preset", + "sizes": _PRESET_SIZES, + "defaults": { + "num_inference_steps": 50, + "guidance_scale": 4.5, + "num_images": 1, + "output_format": "png", + "enable_safety_checker": False, + "safety_tolerance": "5", + "sync_mode": True, + }, + "supports": { + "prompt", "image_size", "num_inference_steps", "guidance_scale", + "num_images", "output_format", "enable_safety_checker", + "safety_tolerance", "sync_mode", "seed", + }, + "upscale": False, + # Edit endpoint accepts up to 9 reference images. + "edit_endpoint": "fal-ai/flux-2-pro/edit", + "edit_supports": { + "prompt", "image_urls", "num_inference_steps", "guidance_scale", + "num_images", "output_format", "enable_safety_checker", + "safety_tolerance", "sync_mode", "seed", + }, + "max_reference_images": 9, + }, + "fal-ai/z-image/turbo": { + "display": "Z-Image Turbo", + "speed": "~2s", + "strengths": "Bilingual EN/CN, 6B", + "price": "$0.005/MP", + "size_style": "image_size_preset", + "sizes": _PRESET_SIZES, + "defaults": { + "num_inference_steps": 8, + "num_images": 1, + "output_format": "png", + "enable_safety_checker": False, + "enable_prompt_expansion": False, # avoid the extra per-request charge + }, + "supports": { + "prompt", "image_size", "num_inference_steps", "num_images", + "seed", "output_format", "enable_safety_checker", + "enable_prompt_expansion", + }, + "upscale": False, + }, + "fal-ai/nano-banana-pro": { + "display": "Nano Banana Pro (Gemini 3 Pro Image)", + "speed": "~8s", + "strengths": "Gemini 3 Pro, reasoning depth, text rendering", + "price": "$0.15/image (1K)", + "size_style": "aspect_ratio", + "sizes": _ASPECT_SIZES, + "defaults": { + "num_images": 1, + "output_format": "png", + "safety_tolerance": "5", + # "1K" is the cheapest tier; 4K doubles the per-image cost. + # Users on Nous Subscription should stay at 1K for predictable billing. + "resolution": "1K", + }, + "supports": { + "prompt", "aspect_ratio", "num_images", "output_format", + "safety_tolerance", "seed", "sync_mode", "resolution", + "enable_web_search", "limit_generations", + }, + "upscale": False, + # Nano Banana Pro edit (Gemini 3 Pro Image): natural-language edits + # with up to 2 reference images via `image_urls`. + "edit_endpoint": "fal-ai/nano-banana-pro/edit", + "edit_supports": { + "prompt", "image_urls", "aspect_ratio", "num_images", + "output_format", "safety_tolerance", "seed", "sync_mode", + "resolution", "enable_web_search", "limit_generations", + }, + "max_reference_images": 2, + }, + "fal-ai/nano-banana-2": { + "display": "Nano Banana 2 (Gemini 3.1 Flash Image)", + "speed": "~3s", + "strengths": "Fast reasoning, multilingual text, infographics", + "price": "Lower-cost Flash tier", + "size_style": "aspect_ratio", + "sizes": _ASPECT_SIZES, + "defaults": { + "num_images": 1, + "output_format": "png", + "safety_tolerance": "4", + "resolution": "1K", + "limit_generations": True, + }, + "supports": { + "prompt", "aspect_ratio", "num_images", "output_format", + "safety_tolerance", "seed", "sync_mode", "system_prompt", + "resolution", "enable_web_search", "limit_generations", + "thinking_level", + }, + "upscale": False, + "edit_endpoint": "fal-ai/nano-banana-2/edit", + "edit_supports": { + "prompt", "image_urls", "aspect_ratio", "num_images", + "output_format", "safety_tolerance", "seed", "sync_mode", + "system_prompt", "resolution", "enable_web_search", + "limit_generations", "thinking_level", + }, + "max_reference_images": 14, + }, + "fal-ai/gpt-image-1.5": { + "display": "GPT Image 1.5", + "speed": "~15s", + "strengths": "Prompt adherence", + "price": "$0.034/image", + "size_style": "gpt_literal", + "sizes": { + "landscape": "1536x1024", + "square": "1024x1024", + "portrait": "1024x1536", + }, + "defaults": { + # Quality is pinned to medium to keep portal billing predictable + # across all users (low is too rough, high is 4-6x more expensive). + "quality": "medium", + "num_images": 1, + "output_format": "png", + }, + "supports": { + "prompt", "image_size", "quality", "num_images", "output_format", + "background", "sync_mode", + }, + "upscale": False, + # Edit endpoint: high-fidelity edits preserving composition/lighting. + "edit_endpoint": "fal-ai/gpt-image-1.5/edit", + "edit_supports": { + "prompt", "image_urls", "image_size", "quality", "num_images", + "output_format", "sync_mode", + }, + "max_reference_images": 16, + }, + "fal-ai/gpt-image-2": { + "display": "GPT Image 2", + "speed": "~20s", + "strengths": "SOTA text rendering + CJK, world-aware photorealism", + "price": "$0.04–0.06/image", + # GPT Image 2 uses FAL's standard preset enum (unlike 1.5's literal + # dimensions). We map to the 4:3 variants — the 16:9 presets + # (1024x576) fall below GPT-Image-2's 655,360 min-pixel requirement + # and would be rejected. 4:3 keeps us above the minimum on all + # three aspect ratios. + "size_style": "image_size_preset", + "sizes": { + "landscape": "landscape_4_3", # 1024x768 + "square": "square_hd", # 1024x1024 + "portrait": "portrait_4_3", # 768x1024 + }, + "defaults": { + # Same quality pinning as gpt-image-1.5: medium keeps Nous + # Portal billing predictable. "high" is 3-4x the per-image + # cost at the same size; "low" is too rough for production use. + "quality": "medium", + "num_images": 1, + "output_format": "png", + }, + "supports": { + "prompt", "image_size", "quality", "num_images", "output_format", + "sync_mode", + # openai_api_key (BYOK) intentionally omitted — all users go + # through the shared FAL billing path. + }, + "upscale": False, + # GPT Image 2 edit endpoint lives under the OpenAI namespace on FAL + # (NOT fal-ai/). Takes `image_urls` (list) + optional mask. We don't + # send `image_size` on edit so the model auto-infers from input. + "edit_endpoint": "openai/gpt-image-2/edit", + "edit_supports": { + "prompt", "image_urls", "quality", "num_images", "output_format", + "sync_mode", "mask_image_url", + }, + "max_reference_images": 16, + }, + "fal-ai/ideogram/v3": { + "display": "Ideogram V3", + "speed": "~5s", + "strengths": "Best typography", + "price": "$0.03-0.09/image", + "size_style": "image_size_preset", + "sizes": _PRESET_SIZES, + "defaults": { + "rendering_speed": "BALANCED", + "expand_prompt": True, + "style": "AUTO", + }, + "supports": { + "prompt", "image_size", "rendering_speed", "expand_prompt", + "style", "seed", + }, + "upscale": False, + # Ideogram V3 edit endpoint takes `image_urls` (list). + "edit_endpoint": "fal-ai/ideogram/v3/edit", + "edit_supports": { + "prompt", "image_urls", "rendering_speed", "expand_prompt", + "style", "seed", + }, + "max_reference_images": 1, + }, + "fal-ai/recraft/v4/pro/text-to-image": { + "display": "Recraft V4 Pro", + "speed": "~8s", + "strengths": "Design, brand systems, production-ready", + "price": "$0.25/image", + "size_style": "image_size_preset", + "sizes": _PRESET_SIZES, + "defaults": { + # V4 Pro dropped V3's required `style` enum — defaults handle taste now. + "enable_safety_checker": False, + }, + "supports": { + "prompt", "image_size", "enable_safety_checker", + "colors", "background_color", + }, + "upscale": False, + }, + "fal-ai/qwen-image": { + "display": "Qwen Image", + "speed": "~12s", + "strengths": "LLM-based, complex text", + "price": "$0.02/MP", + "size_style": "image_size_preset", + "sizes": _PRESET_SIZES, + "defaults": { + "num_inference_steps": 30, + "guidance_scale": 2.5, + "num_images": 1, + "output_format": "png", + "acceleration": "regular", + }, + "supports": { + "prompt", "image_size", "num_inference_steps", "guidance_scale", + "num_images", "output_format", "acceleration", "seed", "sync_mode", + }, + "upscale": False, + # Qwen edit uses the Qwen Image 2.0 Pro editing endpoint, which takes + # `image_urls` (list) + natural-language edit instructions. + "edit_endpoint": "fal-ai/qwen-image-2/pro/edit", + "edit_supports": { + "prompt", "image_urls", "num_inference_steps", "guidance_scale", + "num_images", "output_format", "acceleration", "seed", "sync_mode", + }, + "max_reference_images": 3, + }, + # Krea 2 on FAL — same model family as ``plugins/image_gen/krea``, but billed + # through FAL / the FAL managed gateway. Native ``krea-2-*`` ids route to the + # dedicated Krea plugin instead. + "fal-ai/krea/v2/medium/text-to-image": { + "display": "Krea 2 Medium", + "speed": "~15-25s", + "strengths": "Illustration, anime, painting, expressive/artistic styles", + "price": "$0.030 (text) / $0.035 (style refs)", + "size_style": "aspect_ratio", + "sizes": _ASPECT_SIZES, + "defaults": { + "creativity": "medium", + }, + "supports": { + "prompt", "aspect_ratio", "creativity", "seed", + "image_style_references", + }, + "upscale": False, + }, + "fal-ai/krea/v2/large/text-to-image": { + "display": "Krea 2 Large", + "speed": "~25-60s", + "strengths": "Photorealism, raw textured looks (motion blur, grain, film)", + "price": "$0.060 (text) / $0.065 (style refs)", + "size_style": "aspect_ratio", + "sizes": _ASPECT_SIZES, + "defaults": { + "creativity": "medium", + }, + "supports": { + "prompt", "aspect_ratio", "creativity", "seed", + "image_style_references", + }, + "upscale": False, + }, + # ─── Aug 2026 catalog expansion ──────────────────────────────────────── + # Endpoint ids, `supports` whitelists and enum defaults below are taken + # from each model's FAL OpenAPI schema, so a key we send is a key the + # vendor declares. Paired `/edit` apps hang off their text-to-image entry + # rather than appearing as separate picker rows. + "bytedance/seedream/v5/pro/text-to-image": { + "display": "Seedream 5.0 Pro", + "speed": "~10s", + "strengths": "ByteDance flagship, dense layouts, native text in 14 languages", + "price": "$0.0675/image (≤1536²)", + "size_style": "image_size_preset", + # Pro requires total pixels between 1024x1024 and 2048x2048 — + # explicit ImageSize dicts keep every aspect inside that window. + "sizes": { + "landscape": {"width": 2048, "height": 1152}, + "square": {"width": 1536, "height": 1536}, + "portrait": {"width": 1152, "height": 2048}, + }, + "defaults": { + "num_images": 1, + "output_format": "png", + "enable_safety_checker": False, + }, + "supports": { + "prompt", "image_size", "num_images", "output_format", + "sync_mode", "enable_safety_checker", + }, + "upscale": False, + # Region-precise editing with up to 10 reference images. + "edit_endpoint": "bytedance/seedream/v5/pro/edit", + "edit_supports": { + "prompt", "image_urls", "image_size", "num_images", + "output_format", "sync_mode", "enable_safety_checker", + }, + "max_reference_images": 10, + }, + "bytedance/seedream/v5/lite/text-to-image": { + "display": "Seedream 5.0 Lite", + "speed": "~5s", + "strengths": "Fast/cheap Seedream tier, high-res output", + "price": "$0.035/image", + "size_style": "image_size_preset", + # Lite wants total pixels between 2560x1440 and 4096x4096. Use the + # documented presets (FAL auto-scales if a preset is under the floor) + # instead of hand-rolled ImageSize dicts that drift from the schema. + "sizes": _PRESET_SIZES, + "defaults": { + "num_images": 1, + "enable_safety_checker": False, + }, + "supports": { + "prompt", "image_size", "num_images", "max_images", + "sync_mode", "enable_safety_checker", + }, + "upscale": False, + }, + "ideogram/v4/instant": { + "display": "Ideogram V4 (Instant)", + "speed": "<1s", + "strengths": "Latest Ideogram typography, posters/logos, instant", + "price": "$0.0075/MP", + "size_style": "image_size_preset", + "sizes": _PRESET_SIZES, + "defaults": { + "expansion_model": "Medium", + "output_format": "png", + "enable_safety_checker": False, + }, + "supports": { + "prompt", "image_size", "expansion_model", "num_images", + "seed", "sync_mode", "enable_safety_checker", "output_format", + }, + "upscale": False, + }, + "ideogram/v4/fast": { + "display": "Ideogram V4 (Fast)", + "speed": "~1s", + "strengths": "Ideogram V4 quality tiers via rendering_speed", + "price": "$0.005-0.018/MP", + "size_style": "image_size_preset", + "sizes": _PRESET_SIZES, + "defaults": { + "expansion_model": "Medium", + "rendering_speed": "BALANCED", + }, + "supports": { + "prompt", "image_size", "expansion_model", "rendering_speed", + "num_images", "seed", "sync_mode", + }, + "upscale": False, + }, + "alibaba/qwen-image-3/text-to-image": { + "display": "Qwen Image 3", + "speed": "~8s", + "strengths": "Complex CN/EN text rendering, prompt-guided resolution", + "price": "$0.04 (1K) / $0.075 (2K) per image", + "size_style": "image_size_preset", + "sizes": _PRESET_SIZES, + "defaults": { + "num_images": 1, + "output_format": "png", + "enable_prompt_expansion": False, # avoid the LLM rewrite surprise + "enable_safety_checker": False, + }, + "supports": { + "prompt", "negative_prompt", "image_size", "num_images", + "seed", "sync_mode", "output_format", + "enable_prompt_expansion", "enable_safety_checker", + }, + "upscale": False, + # Qwen Image 3 edit: 1-3 reference images, identity-preserving edits. + "edit_endpoint": "alibaba/qwen-image-3/edit", + "edit_supports": { + "prompt", "image_urls", "negative_prompt", "num_images", + "seed", "sync_mode", "output_format", + "enable_prompt_expansion", "enable_safety_checker", + }, + "max_reference_images": 3, + }, + "microsoft/mai-image-2.5-pro": { + "display": "MAI Image 2.5 Pro", + "speed": "~10s", + "strengths": "Microsoft flagship, hero imagery, precise typography", + "price": "~$0.17/image", + "size_style": "aspect_ratio", + "sizes": _ASPECT_SIZES, + "defaults": { + "num_images": 1, + "output_format": "png", + }, + "supports": { + "prompt", "aspect_ratio", "num_images", "output_format", + "sync_mode", + }, + "upscale": False, + }, + "google/nano-banana-2-lite": { + "display": "Nano Banana 2 Lite", + "speed": "<2s", + "strengths": "Gemini image family, sub-2s, 14 aspect ratios incl. extreme", + "price": "~$0.04/image (1K fixed)", + "size_style": "aspect_ratio", + "sizes": _ASPECT_SIZES, + "defaults": { + "num_images": 1, + "output_format": "png", + "safety_tolerance": "5", + }, + "supports": { + "prompt", "aspect_ratio", "num_images", "seed", + "output_format", "safety_tolerance", "sync_mode", + "system_prompt", "limit_generations", "thinking_level", + }, + "upscale": False, + # Fast multi-turn local edits with reference images via `image_urls`. + "edit_endpoint": "google/nano-banana-2-lite/edit", + "edit_supports": { + "prompt", "image_urls", "aspect_ratio", "num_images", + "seed", "output_format", "safety_tolerance", "sync_mode", + "system_prompt", + }, + "max_reference_images": 4, + }, + "fal-ai/recraft/v4.1/text-to-image": { + "display": "Recraft V4.1", + "speed": "~8s", + "strengths": "Design-first raster, brand systems, editorial", + "price": "$0.035/image", + "size_style": "image_size_preset", + "sizes": _PRESET_SIZES, + "defaults": { + "enable_safety_checker": False, + }, + "supports": { + "prompt", "image_size", "enable_safety_checker", + "colors", "background_color", + }, + "upscale": False, + }, + "xai/grok-imagine-image/v2.0/text-to-image": { + "display": "Grok Imagine Image 2.0", + "speed": "~5s", + "strengths": "xAI. Design-grade typography/layout, instruction following", + "price": "$0.06/image (1K medium)", + "size_style": "aspect_ratio", + "sizes": _ASPECT_SIZES, + "defaults": { + "num_images": 1, + "output_format": "png", + # 1k + medium is the cheapest sensible tier ($0.06/image); + # 2k roughly +33% per image. + "resolution": "1k", + "quality": "medium", + }, + "supports": { + "prompt", "aspect_ratio", "num_images", "output_format", + "resolution", "quality", "sync_mode", + }, + "upscale": False, + # Edit endpoint takes `image_urls` (max 3) + the same knobs; + # aspect_ratio defaults to "auto" (follows the first input image), + # so we don't send it on edits. + "edit_endpoint": "xai/grok-imagine-image/v2.0/edit", + "edit_supports": { + "prompt", "image_urls", "num_images", "output_format", + "resolution", "quality", "sync_mode", + }, + "max_reference_images": 3, + }, +} + + +# Fastest reasonable option; cheap and sub-1s. +DEFAULT_MODEL = "fal-ai/flux-2/klein/9b" + +DEFAULT_ASPECT_RATIO = "landscape" +VALID_ASPECT_RATIOS = ("landscape", "square", "portrait") + +# Clarity Upscaler settings. +UPSCALER_MODEL = "fal-ai/clarity-upscaler" +UPSCALER_FACTOR = 2 +UPSCALER_SAFETY_CHECKER = False +UPSCALER_DEFAULT_PROMPT = "masterpiece, best quality, highres" +UPSCALER_NEGATIVE_PROMPT = "(worst quality, low quality, normal quality:2)" +UPSCALER_CREATIVITY = 0.35 +UPSCALER_RESEMBLANCE = 0.6 +UPSCALER_GUIDANCE_SCALE = 4 +UPSCALER_NUM_INFERENCE_STEPS = 18 diff --git a/tools/image_generation_tool.py b/tools/image_generation_tool.py index c914418970..b98f3a4361 100644 --- a/tools/image_generation_tool.py +++ b/tools/image_generation_tool.py @@ -7,19 +7,13 @@ selectable via ``hermes tools`` → Image Generation; the active model is persisted to ``image_gen.model`` in ``config.yaml``. Architecture: -- ``FAL_MODELS`` is a catalog of supported models with per-model metadata +- ``FAL_MODELS`` (``tools.image_generation_catalog``) holds per-model metadata (size-style family, defaults, ``supports`` whitelist, upscaler flag). -- ``_build_fal_payload()`` translates the agent's unified inputs (prompt + - aspect_ratio) into the model-specific payload and filters to the - ``supports`` whitelist so models never receive rejected keys. -- Upscaling via FAL's Clarity Upscaler is gated per-model via the ``upscale`` - flag — OFF by default for every model. Clarity is an SD1.5 creative - tile-diffusion enhancer (creativity 0.35 redraws content); chained by - default it mangled GPT Image 2 / Ideogram text rendering, CJK, and faces - (Aug 2026 quality regression). Upscaling is strictly per-call opt-in. - -Pricing shown in UI strings is as-of the initial commit; we accept drift and -update when it's noticed. +- ``_build_fal_payload()`` / ``_build_fal_edit_payload()`` translate the + unified inputs into the model-specific payload, filtered to the whitelist so + models never receive rejected keys. +- Upscaling (Clarity Upscaler) is strictly per-call opt-in: chained by default + it degraded text rendering, CJK and faces, so ``upscale`` is False everywhere. """ import json @@ -30,26 +24,15 @@ import threading import uuid from typing import Any, Dict, Optional -# fal_client is imported lazily — see _load_fal_client(). Pulling it -# eagerly added ~64 ms to every CLI cold start because -# discover_builtin_tools() imports this module unconditionally during -# the registry walk, even when image generation is never used. -# -# Tests that monkeypatch this attribute (e.g. -# ``monkeypatch.setattr(image_tool, "fal_client", fake_fal_client)``) -# still work: _load_fal_client() short-circuits when the attribute is -# anything truthy, so a test-installed mock is not overwritten by a -# subsequent real import. +# fal_client is imported lazily (see _load_fal_client): an eager import cost +# ~64 ms on every CLI cold start because discover_builtin_tools() imports this +# module unconditionally. Tests that monkeypatch this attribute keep working: +# _load_fal_client() short-circuits when it is already truthy. fal_client: Any = None def _load_fal_client() -> Any: - """Lazily import fal_client and rebind the module global on first use. - - Idempotent. Returns the (now-loaded) ``fal_client`` module reference. - Skips the import if the global is already truthy — this preserves the - test pattern of monkeypatching the module global to install a mock. - """ + """Lazily import fal_client into the module global (idempotent; keeps a test-installed mock).""" global fal_client if fal_client is not None: return fal_client @@ -64,6 +47,21 @@ from tools.fal_common import ( _extract_http_status, _normalize_fal_queue_url_format, # noqa: F401 — re-exported for tests ) +from tools.image_generation_catalog import ( # noqa: F401 — re-exported (plugins/tests/tools_config) + DEFAULT_ASPECT_RATIO, + DEFAULT_MODEL, + FAL_MODELS, + UPSCALER_CREATIVITY, + UPSCALER_DEFAULT_PROMPT, + UPSCALER_FACTOR, + UPSCALER_GUIDANCE_SCALE, + UPSCALER_MODEL, + UPSCALER_NEGATIVE_PROMPT, + UPSCALER_NUM_INFERENCE_STEPS, + UPSCALER_RESEMBLANCE, + UPSCALER_SAFETY_CHECKER, + VALID_ASPECT_RATIOS, +) from tools.managed_tool_gateway import resolve_managed_tool_gateway from tools.tool_backend_helpers import ( NOUS_MANAGED_PROVIDER, @@ -76,654 +74,6 @@ from tools.tool_backend_helpers import ( logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# FAL model catalog -# --------------------------------------------------------------------------- -# -# Each entry declares how to translate our unified inputs into the model's -# native payload shape. Size specification falls into three families: -# -# "image_size_preset" — preset enum ("square_hd", "landscape_16_9", ...) -# used by the flux family, z-image, qwen, recraft, -# ideogram. -# "aspect_ratio" — aspect ratio enum ("16:9", "1:1", ...) used by -# nano-banana (Gemini). -# "gpt_literal" — literal dimension strings ("1024x1024", etc.) -# used by gpt-image-1.5. -# -# ``supports`` is a whitelist of keys allowed in the outgoing payload — any -# key outside this set is stripped before submission so models never receive -# rejected parameters (each FAL model rejects unknown keys differently). -# -# ``upscale`` controls whether to chain Clarity Upscaler after generation. -# Policy (Aug 2026): False everywhere — the default-on experiment degraded -# output quality (Clarity redraws content). Opt-in per call only. - -FAL_MODELS: Dict[str, Dict[str, Any]] = { - "fal-ai/flux-2/klein/9b": { - "display": "FLUX 2 Klein 9B", - "speed": "<1s", - "strengths": "Fast, crisp text", - "price": "$0.006/MP", - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - "num_inference_steps": 4, - "output_format": "png", - "enable_safety_checker": False, - }, - "supports": { - "prompt", "image_size", "num_inference_steps", "seed", - "output_format", "enable_safety_checker", - }, - "upscale": False, - # Image-to-image / editing: FLUX.2 [klein] 9B edit endpoint takes - # `image_urls` (list). Natural-language edits, multi-ref. - "edit_endpoint": "fal-ai/flux-2/klein/9b/edit", - "edit_supports": { - "prompt", "image_urls", "num_inference_steps", "seed", - "output_format", "enable_safety_checker", - }, - "max_reference_images": 9, - }, - "fal-ai/flux-2-pro": { - "display": "FLUX 2 Pro", - "speed": "~6s", - "strengths": "Studio photorealism", - "price": "$0.03/MP", - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - "num_inference_steps": 50, - "guidance_scale": 4.5, - "num_images": 1, - "output_format": "png", - "enable_safety_checker": False, - "safety_tolerance": "5", - "sync_mode": True, - }, - "supports": { - "prompt", "image_size", "num_inference_steps", "guidance_scale", - "num_images", "output_format", "enable_safety_checker", - "safety_tolerance", "sync_mode", "seed", - }, - "upscale": False, # opt-in only (was default-on pre-Aug 2026) - # Edit endpoint accepts up to 9 reference images. - "edit_endpoint": "fal-ai/flux-2-pro/edit", - "edit_supports": { - "prompt", "image_urls", "num_inference_steps", "guidance_scale", - "num_images", "output_format", "enable_safety_checker", - "safety_tolerance", "sync_mode", "seed", - }, - "max_reference_images": 9, - }, - "fal-ai/z-image/turbo": { - "display": "Z-Image Turbo", - "speed": "~2s", - "strengths": "Bilingual EN/CN, 6B", - "price": "$0.005/MP", - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - "num_inference_steps": 8, - "num_images": 1, - "output_format": "png", - "enable_safety_checker": False, - "enable_prompt_expansion": False, # avoid the extra per-request charge - }, - "supports": { - "prompt", "image_size", "num_inference_steps", "num_images", - "seed", "output_format", "enable_safety_checker", - "enable_prompt_expansion", - }, - "upscale": False, - }, - "fal-ai/nano-banana-pro": { - "display": "Nano Banana Pro (Gemini 3 Pro Image)", - "speed": "~8s", - "strengths": "Gemini 3 Pro, reasoning depth, text rendering", - "price": "$0.15/image (1K)", - "size_style": "aspect_ratio", - "sizes": { - "landscape": "16:9", - "square": "1:1", - "portrait": "9:16", - }, - "defaults": { - "num_images": 1, - "output_format": "png", - "safety_tolerance": "5", - # "1K" is the cheapest tier; 4K doubles the per-image cost. - # Users on Nous Subscription should stay at 1K for predictable billing. - "resolution": "1K", - }, - "supports": { - "prompt", "aspect_ratio", "num_images", "output_format", - "safety_tolerance", "seed", "sync_mode", "resolution", - "enable_web_search", "limit_generations", - }, - "upscale": False, - # Nano Banana Pro edit (Gemini 3 Pro Image): natural-language edits - # with up to 2 reference images via `image_urls`. - "edit_endpoint": "fal-ai/nano-banana-pro/edit", - "edit_supports": { - "prompt", "image_urls", "aspect_ratio", "num_images", - "output_format", "safety_tolerance", "seed", "sync_mode", - "resolution", "enable_web_search", "limit_generations", - }, - "max_reference_images": 2, - }, - "fal-ai/nano-banana-2": { - "display": "Nano Banana 2 (Gemini 3.1 Flash Image)", - "speed": "~3s", - "strengths": "Fast reasoning, multilingual text, infographics", - "price": "Lower-cost Flash tier", - "size_style": "aspect_ratio", - "sizes": { - "landscape": "16:9", - "square": "1:1", - "portrait": "9:16", - }, - "defaults": { - "num_images": 1, - "output_format": "png", - "safety_tolerance": "4", - "resolution": "1K", - "limit_generations": True, - }, - "supports": { - "prompt", "aspect_ratio", "num_images", "output_format", - "safety_tolerance", "seed", "sync_mode", "system_prompt", - "resolution", "enable_web_search", "limit_generations", - "thinking_level", - }, - "upscale": False, - "edit_endpoint": "fal-ai/nano-banana-2/edit", - "edit_supports": { - "prompt", "image_urls", "aspect_ratio", "num_images", - "output_format", "safety_tolerance", "seed", "sync_mode", - "system_prompt", "resolution", "enable_web_search", - "limit_generations", "thinking_level", - }, - "max_reference_images": 14, - }, - "fal-ai/gpt-image-1.5": { - "display": "GPT Image 1.5", - "speed": "~15s", - "strengths": "Prompt adherence", - "price": "$0.034/image", - "size_style": "gpt_literal", - "sizes": { - "landscape": "1536x1024", - "square": "1024x1024", - "portrait": "1024x1536", - }, - "defaults": { - # Quality is pinned to medium to keep portal billing predictable - # across all users (low is too rough, high is 4-6x more expensive). - "quality": "medium", - "num_images": 1, - "output_format": "png", - }, - "supports": { - "prompt", "image_size", "quality", "num_images", "output_format", - "background", "sync_mode", - }, - "upscale": False, - # Edit endpoint: high-fidelity edits preserving composition/lighting. - "edit_endpoint": "fal-ai/gpt-image-1.5/edit", - "edit_supports": { - "prompt", "image_urls", "image_size", "quality", "num_images", - "output_format", "sync_mode", - }, - "max_reference_images": 16, - }, - "fal-ai/gpt-image-2": { - "display": "GPT Image 2", - "speed": "~20s", - "strengths": "SOTA text rendering + CJK, world-aware photorealism", - "price": "$0.04–0.06/image", - # GPT Image 2 uses FAL's standard preset enum (unlike 1.5's literal - # dimensions). We map to the 4:3 variants — the 16:9 presets - # (1024x576) fall below GPT-Image-2's 655,360 min-pixel requirement - # and would be rejected. 4:3 keeps us above the minimum on all - # three aspect ratios. - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_4_3", # 1024x768 - "square": "square_hd", # 1024x1024 - "portrait": "portrait_4_3", # 768x1024 - }, - "defaults": { - # Same quality pinning as gpt-image-1.5: medium keeps Nous - # Portal billing predictable. "high" is 3-4x the per-image - # cost at the same size; "low" is too rough for production use. - "quality": "medium", - "num_images": 1, - "output_format": "png", - }, - "supports": { - "prompt", "image_size", "quality", "num_images", "output_format", - "sync_mode", - # openai_api_key (BYOK) intentionally omitted — all users go - # through the shared FAL billing path. - }, - "upscale": False, - # GPT Image 2 edit endpoint lives under the OpenAI namespace on FAL - # (NOT fal-ai/). Takes `image_urls` (list) + optional mask. We don't - # send `image_size` on edit so the model auto-infers from input. - "edit_endpoint": "openai/gpt-image-2/edit", - "edit_supports": { - "prompt", "image_urls", "quality", "num_images", "output_format", - "sync_mode", "mask_image_url", - }, - "max_reference_images": 16, - }, - "fal-ai/ideogram/v3": { - "display": "Ideogram V3", - "speed": "~5s", - "strengths": "Best typography", - "price": "$0.03-0.09/image", - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - "rendering_speed": "BALANCED", - "expand_prompt": True, - "style": "AUTO", - }, - "supports": { - "prompt", "image_size", "rendering_speed", "expand_prompt", - "style", "seed", - }, - "upscale": False, - # Ideogram V3 edit endpoint takes `image_urls` (list). - "edit_endpoint": "fal-ai/ideogram/v3/edit", - "edit_supports": { - "prompt", "image_urls", "rendering_speed", "expand_prompt", - "style", "seed", - }, - "max_reference_images": 1, - }, - "fal-ai/recraft/v4/pro/text-to-image": { - "display": "Recraft V4 Pro", - "speed": "~8s", - "strengths": "Design, brand systems, production-ready", - "price": "$0.25/image", - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - # V4 Pro dropped V3's required `style` enum — defaults handle taste now. - "enable_safety_checker": False, - }, - "supports": { - "prompt", "image_size", "enable_safety_checker", - "colors", "background_color", - }, - "upscale": False, - }, - "fal-ai/qwen-image": { - "display": "Qwen Image", - "speed": "~12s", - "strengths": "LLM-based, complex text", - "price": "$0.02/MP", - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - "num_inference_steps": 30, - "guidance_scale": 2.5, - "num_images": 1, - "output_format": "png", - "acceleration": "regular", - }, - "supports": { - "prompt", "image_size", "num_inference_steps", "guidance_scale", - "num_images", "output_format", "acceleration", "seed", "sync_mode", - }, - "upscale": False, - # Qwen edit uses the Qwen Image 2.0 Pro editing endpoint, which takes - # `image_urls` (list) + natural-language edit instructions. - "edit_endpoint": "fal-ai/qwen-image-2/pro/edit", - "edit_supports": { - "prompt", "image_urls", "num_inference_steps", "guidance_scale", - "num_images", "output_format", "acceleration", "seed", "sync_mode", - }, - "max_reference_images": 3, - }, - # Krea 2 on FAL — same model family as ``plugins/image_gen/krea``, but billed - # through FAL / the FAL managed gateway. Native ``krea-2-*`` ids route to the - # dedicated Krea plugin instead. - "fal-ai/krea/v2/medium/text-to-image": { - "display": "Krea 2 Medium", - "speed": "~15-25s", - "strengths": "Illustration, anime, painting, expressive/artistic styles", - "price": "$0.030 (text) / $0.035 (style refs)", - "size_style": "aspect_ratio", - "sizes": { - "landscape": "16:9", - "square": "1:1", - "portrait": "9:16", - }, - "defaults": { - "creativity": "medium", - }, - "supports": { - "prompt", "aspect_ratio", "creativity", "seed", - "image_style_references", - }, - "upscale": False, - }, - "fal-ai/krea/v2/large/text-to-image": { - "display": "Krea 2 Large", - "speed": "~25-60s", - "strengths": "Photorealism, raw textured looks (motion blur, grain, film)", - "price": "$0.060 (text) / $0.065 (style refs)", - "size_style": "aspect_ratio", - "sizes": { - "landscape": "16:9", - "square": "1:1", - "portrait": "9:16", - }, - "defaults": { - "creativity": "medium", - }, - "supports": { - "prompt", "aspect_ratio", "creativity", "seed", - "image_style_references", - }, - "upscale": False, - }, - # ─── Aug 2026 catalog expansion ──────────────────────────────────────── - # Endpoint ids, `supports` whitelists and enum defaults below are taken - # from each model's FAL OpenAPI schema, so a key we send is a key the - # vendor declares. Paired `/edit` apps hang off their text-to-image entry - # rather than appearing as separate picker rows. - "bytedance/seedream/v5/pro/text-to-image": { - "display": "Seedream 5.0 Pro", - "speed": "~10s", - "strengths": "ByteDance flagship, dense layouts, native text in 14 languages", - "price": "$0.0675/image (≤1536²)", - "size_style": "image_size_preset", - # Pro requires total pixels between 1024x1024 and 2048x2048 — - # explicit ImageSize dicts keep every aspect inside that window. - "sizes": { - "landscape": {"width": 2048, "height": 1152}, - "square": {"width": 1536, "height": 1536}, - "portrait": {"width": 1152, "height": 2048}, - }, - "defaults": { - "num_images": 1, - "output_format": "png", - "enable_safety_checker": False, - }, - "supports": { - "prompt", "image_size", "num_images", "output_format", - "sync_mode", "enable_safety_checker", - }, - "upscale": False, - # Region-precise editing with up to 10 reference images. - "edit_endpoint": "bytedance/seedream/v5/pro/edit", - "edit_supports": { - "prompt", "image_urls", "image_size", "num_images", - "output_format", "sync_mode", "enable_safety_checker", - }, - "max_reference_images": 10, - }, - "bytedance/seedream/v5/lite/text-to-image": { - "display": "Seedream 5.0 Lite", - "speed": "~5s", - "strengths": "Fast/cheap Seedream tier, high-res output", - "price": "$0.035/image", - "size_style": "image_size_preset", - # Lite wants total pixels between 2560x1440 and 4096x4096. Use the - # documented presets (FAL auto-scales if a preset is under the floor) - # instead of hand-rolled ImageSize dicts that drift from the schema. - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - "num_images": 1, - "enable_safety_checker": False, - }, - "supports": { - "prompt", "image_size", "num_images", "max_images", - "sync_mode", "enable_safety_checker", - }, - "upscale": False, - }, - "ideogram/v4/instant": { - "display": "Ideogram V4 (Instant)", - "speed": "<1s", - "strengths": "Latest Ideogram typography, posters/logos, instant", - "price": "$0.0075/MP", - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - "expansion_model": "Medium", - "output_format": "png", - "enable_safety_checker": False, - }, - "supports": { - "prompt", "image_size", "expansion_model", "num_images", - "seed", "sync_mode", "enable_safety_checker", "output_format", - }, - "upscale": False, - }, - "ideogram/v4/fast": { - "display": "Ideogram V4 (Fast)", - "speed": "~1s", - "strengths": "Ideogram V4 quality tiers via rendering_speed", - "price": "$0.005-0.018/MP", - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - "expansion_model": "Medium", - "rendering_speed": "BALANCED", - }, - "supports": { - "prompt", "image_size", "expansion_model", "rendering_speed", - "num_images", "seed", "sync_mode", - }, - "upscale": False, - }, - "alibaba/qwen-image-3/text-to-image": { - "display": "Qwen Image 3", - "speed": "~8s", - "strengths": "Complex CN/EN text rendering, prompt-guided resolution", - "price": "$0.04 (1K) / $0.075 (2K) per image", - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - "num_images": 1, - "output_format": "png", - "enable_prompt_expansion": False, # avoid the LLM rewrite surprise - "enable_safety_checker": False, - }, - "supports": { - "prompt", "negative_prompt", "image_size", "num_images", - "seed", "sync_mode", "output_format", - "enable_prompt_expansion", "enable_safety_checker", - }, - "upscale": False, - # Qwen Image 3 edit: 1-3 reference images, identity-preserving edits. - "edit_endpoint": "alibaba/qwen-image-3/edit", - "edit_supports": { - "prompt", "image_urls", "negative_prompt", "num_images", - "seed", "sync_mode", "output_format", - "enable_prompt_expansion", "enable_safety_checker", - }, - "max_reference_images": 3, - }, - "microsoft/mai-image-2.5-pro": { - "display": "MAI Image 2.5 Pro", - "speed": "~10s", - "strengths": "Microsoft flagship, hero imagery, precise typography", - "price": "~$0.17/image", - "size_style": "aspect_ratio", - "sizes": { - "landscape": "16:9", - "square": "1:1", - "portrait": "9:16", - }, - "defaults": { - "num_images": 1, - "output_format": "png", - }, - "supports": { - "prompt", "aspect_ratio", "num_images", "output_format", - "sync_mode", - }, - "upscale": False, - }, - "google/nano-banana-2-lite": { - "display": "Nano Banana 2 Lite", - "speed": "<2s", - "strengths": "Gemini image family, sub-2s, 14 aspect ratios incl. extreme", - "price": "~$0.04/image (1K fixed)", - "size_style": "aspect_ratio", - "sizes": { - "landscape": "16:9", - "square": "1:1", - "portrait": "9:16", - }, - "defaults": { - "num_images": 1, - "output_format": "png", - "safety_tolerance": "5", - }, - "supports": { - "prompt", "aspect_ratio", "num_images", "seed", - "output_format", "safety_tolerance", "sync_mode", - "system_prompt", "limit_generations", "thinking_level", - }, - "upscale": False, - # Fast multi-turn local edits with reference images via `image_urls`. - "edit_endpoint": "google/nano-banana-2-lite/edit", - "edit_supports": { - "prompt", "image_urls", "aspect_ratio", "num_images", - "seed", "output_format", "safety_tolerance", "sync_mode", - "system_prompt", - }, - "max_reference_images": 4, - }, - "fal-ai/recraft/v4.1/text-to-image": { - "display": "Recraft V4.1", - "speed": "~8s", - "strengths": "Design-first raster, brand systems, editorial", - "price": "$0.035/image", - "size_style": "image_size_preset", - "sizes": { - "landscape": "landscape_16_9", - "square": "square_hd", - "portrait": "portrait_16_9", - }, - "defaults": { - "enable_safety_checker": False, - }, - "supports": { - "prompt", "image_size", "enable_safety_checker", - "colors", "background_color", - }, - "upscale": False, - }, - "xai/grok-imagine-image/v2.0/text-to-image": { - "display": "Grok Imagine Image 2.0", - "speed": "~5s", - "strengths": "xAI. Design-grade typography/layout, instruction following", - "price": "$0.06/image (1K medium)", - "size_style": "aspect_ratio", - "sizes": { - "landscape": "16:9", - "square": "1:1", - "portrait": "9:16", - }, - "defaults": { - "num_images": 1, - "output_format": "png", - # 1k + medium is the cheapest sensible tier ($0.06/image); - # 2k roughly +33% per image. - "resolution": "1k", - "quality": "medium", - }, - "supports": { - "prompt", "aspect_ratio", "num_images", "output_format", - "resolution", "quality", "sync_mode", - }, - # Opt-in only (policy: default-on upscaling was disabled everywhere - # Aug 2026; the upscaler is a creative enhancer that can alter fine - # detail). 1k native is sub-2MP — pass upscale=true when needed. - "upscale": False, - # Edit endpoint takes `image_urls` (max 3) + the same knobs; - # aspect_ratio defaults to "auto" (follows the first input image), - # so we don't send it on edits. - "edit_endpoint": "xai/grok-imagine-image/v2.0/edit", - "edit_supports": { - "prompt", "image_urls", "num_images", "output_format", - "resolution", "quality", "sync_mode", - }, - "max_reference_images": 3, - }, -} - -# Default model is the fastest reasonable option. Kept cheap and sub-1s. -DEFAULT_MODEL = "fal-ai/flux-2/klein/9b" - -DEFAULT_ASPECT_RATIO = "landscape" -VALID_ASPECT_RATIOS = ("landscape", "square", "portrait") - - -# --------------------------------------------------------------------------- -# Upscaler (Clarity Upscaler — unchanged from previous implementation) -# --------------------------------------------------------------------------- -UPSCALER_MODEL = "fal-ai/clarity-upscaler" -UPSCALER_FACTOR = 2 -UPSCALER_SAFETY_CHECKER = False -UPSCALER_DEFAULT_PROMPT = "masterpiece, best quality, highres" -UPSCALER_NEGATIVE_PROMPT = "(worst quality, low quality, normal quality:2)" -UPSCALER_CREATIVITY = 0.35 -UPSCALER_RESEMBLANCE = 0.6 -UPSCALER_GUIDANCE_SCALE = 4 -UPSCALER_NUM_INFERENCE_STEPS = 18 - - _debug = DebugSession("image_tools", env_var="IMAGE_TOOLS_DEBUG") _managed_fal_client = None _managed_fal_client_config = None @@ -736,19 +86,14 @@ _managed_fal_client_lock = threading.Lock() def _resolve_managed_fal_gateway(): """Resolve the FAL route from the stored `hermes tools` selection. - Dispatch is a plain switch on the stored ``image_gen`` provider string: - - ``"nous"`` (or legacy ``use_gateway: true``) → managed fal-queue - gateway ONLY; unentitled/unreachable is a selection-naming error - (never a silent fall back to FAL_KEY). - - any other stored provider (``"fal"``, ...) → direct FAL ONLY; a - missing FAL_KEY is an error naming FAL_KEY and the selection (never a - silent managed reroute). - - no selection ever written → legacy credential autodetect: direct when - FAL_KEY is set, else the managed gateway when resolvable, else None. + - ``"nous"`` (or legacy ``use_gateway: true``) → managed gateway ONLY; not + entitled/unreachable is a selection-naming error, never a silent FAL_KEY fallback. + - any other stored provider → direct FAL ONLY; missing FAL_KEY is an error + naming FAL_KEY and the selection, never a silent managed reroute. + - never configured → legacy autodetect: direct when FAL_KEY is set, else + the managed gateway when resolvable, else None. Returns the managed gateway config, or ``None`` for the direct route. - Raises ``ValueError`` with the honest error contract when the stored - selection cannot run. """ selected = read_selection("image_gen") if selected == NOUS_MANAGED_PROVIDER: @@ -787,8 +132,8 @@ def _get_managed_fal_client(managed_gateway): if _managed_fal_client is not None and _managed_fal_client_config == client_config: return _managed_fal_client - # Resolve fal_client on the legacy module — preserves the test - # pattern of monkey-patching ``image_generation_tool.fal_client``. + # Resolve fal_client on this module so monkeypatching + # ``image_generation_tool.fal_client`` still takes effect. _load_fal_client() _managed_fal_client = _ManagedFalSyncClient( fal_client, @@ -806,13 +151,10 @@ class ImageGenerationInterrupted(Exception): def _wait_fal_result(handler, *, poll_seconds: float = 0.5): """Interrupt-aware replacement for a blind ``handler.get()``. - ``handler.get()`` blocks inside the FAL SDK until the remote job - finishes — a 30-60s window where a user interrupt was previously - invisible (the reported symptom: redirects queued behind a running - generation). Run the blocking get on a daemon worker and poll the - per-thread interrupt bit between join slices; on interrupt, abandon - the worker (daemon thread, remote job keeps running server-side but - we stop waiting) and raise ``ImageGenerationInterrupted``. + ``handler.get()`` blocks inside the FAL SDK for 30-60s, during which a user + interrupt was invisible. Run it on a daemon worker and poll the per-thread + interrupt bit between join slices; on interrupt, abandon the worker (the + remote job keeps running) and raise ``ImageGenerationInterrupted``. """ from tools.interrupt import is_interrupted @@ -841,7 +183,6 @@ def _wait_fal_result(handler, *, poll_seconds: float = 0.5): def _submit_fal_request(model: str, arguments: Dict[str, Any]): """Submit a FAL request using direct credentials or the managed queue gateway.""" - # Trigger the lazy import on first call. Idempotent. _load_fal_client() request_headers = {"x-idempotency-key": str(uuid.uuid4())} managed_gateway = _resolve_managed_fal_gateway() @@ -856,10 +197,9 @@ def _submit_fal_request(model: str, arguments: Dict[str, Any]): headers=request_headers, ) except Exception as exc: - # 4xx from the managed gateway typically means the portal doesn't - # currently proxy this model (allowlist miss, billing gate, etc.) - # — surface a clearer message with actionable remediation instead - # of a raw HTTP error from httpx. + # A 4xx from the managed gateway usually means the portal doesn't proxy + # this model (allowlist miss, billing gate) — give actionable remediation + # instead of a raw httpx error. status = _extract_http_status(exc) if status is not None and 400 <= status < 500: gateway_message = "" @@ -883,29 +223,48 @@ def _submit_fal_request(model: str, arguments: Dict[str, Any]): # --------------------------------------------------------------------------- -# Model resolution + payload construction +# Config readers, model resolution + payload construction # --------------------------------------------------------------------------- +def _read_image_gen_key(key: str) -> Optional[str]: + """Return the stripped ``image_gen.`` string from config.yaml, or None.""" + try: + from hermes_cli.config import load_config + cfg = load_config() + section = cfg.get("image_gen") if isinstance(cfg, dict) else None + if isinstance(section, dict): + value = section.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + except Exception as exc: + logger.debug("Could not read image_gen.%s: %s", key, exc) + return None + + +def _read_configured_image_model(): + """Return the value of ``image_gen.model`` from config.yaml, or None.""" + return _read_image_gen_key("model") + + +def _read_configured_image_provider(): + """Return ``image_gen.provider`` from config.yaml, or None. + + The plugin registry is consulted only when this is explicitly set — an + unset value keeps users on the in-tree FAL fallback even when other + providers happen to be registered (e.g. OPENAI_API_KEY present for other + features). ``"fal"`` explicitly routes through ``plugins/image_gen/fal/``, + which delegates back into this module via call-time indirection. + """ + return _read_image_gen_key("provider") + + def _resolve_fal_model() -> tuple: """Resolve the active FAL model from config.yaml (primary) or default. Returns (model_id, metadata_dict). Falls back to DEFAULT_MODEL if the configured model is unknown (logged as a warning). """ - model_id = "" - try: - from hermes_cli.config import load_config - cfg = load_config() - img_cfg = cfg.get("image_gen") if isinstance(cfg, dict) else None - if isinstance(img_cfg, dict): - raw = img_cfg.get("model") - if isinstance(raw, str): - model_id = raw.strip() - except Exception as exc: - logger.debug("Could not load image_gen.model from config: %s", exc) - - # Env var escape hatch (undocumented; backward-compat for tests/scripts). - if not model_id: - model_id = os.getenv("FAL_IMAGE_MODEL", "").strip() + # FAL_IMAGE_MODEL is an undocumented escape hatch (backward-compat for tests/scripts). + model_id = _read_image_gen_key("model") or os.getenv("FAL_IMAGE_MODEL", "").strip() if not model_id: return DEFAULT_MODEL, FAL_MODELS[DEFAULT_MODEL] @@ -920,20 +279,26 @@ def _resolve_fal_model() -> tuple: return model_id, FAL_MODELS[model_id] -def _build_fal_payload( +def _build_payload( model_id: str, prompt: str, - aspect_ratio: str = DEFAULT_ASPECT_RATIO, - seed: Optional[int] = None, - overrides: Optional[Dict[str, Any]] = None, + aspect_ratio: str, + seed: Optional[int], + overrides: Optional[Dict[str, Any]], + image_urls: Optional[list] = None, ) -> Dict[str, Any]: - """Build a FAL request payload for `model_id` from unified inputs. + """Shared text-to-image / edit payload builder (``image_urls`` selects edit mode). - Translates aspect_ratio into the model's native size spec (preset enum, - aspect-ratio enum, or GPT literal string), merges model defaults, applies - caller overrides, then filters to the model's ``supports`` whitelist. + Translates aspect_ratio into the model's native size spec, merges model + defaults, applies caller overrides, then filters to the model's whitelist. + Edit endpoints mostly auto-infer output size from the input image, so the + size key is only sent when ``edit_supports`` advertises it. ``prompt`` (and + ``image_urls`` on edits) are required by every FAL endpoint and are kept + even if a whitelist omits them, so a catalog gap can't send a broken request. """ meta = FAL_MODELS[model_id] + edit = image_urls is not None + supports = (meta.get("edit_supports") or set()) if edit else meta["supports"] size_style = meta["size_style"] sizes = meta["sizes"] @@ -943,13 +308,21 @@ def _build_fal_payload( payload: Dict[str, Any] = dict(meta.get("defaults", {})) payload["prompt"] = (prompt or "").strip() + required = {"prompt"} + if edit: + payload["image_urls"] = list(image_urls) + required.add("image_urls") if size_style in {"image_size_preset", "gpt_literal"}: - payload["image_size"] = sizes[aspect] + size_key = "image_size" elif size_style == "aspect_ratio": - payload["aspect_ratio"] = sizes[aspect] + size_key = "aspect_ratio" + elif edit: + size_key = None else: raise ValueError(f"Unknown size_style: {size_style!r}") + if size_key is not None and (not edit or size_key in supports): + payload[size_key] = sizes[aspect] if seed is not None and isinstance(seed, int): payload["seed"] = seed @@ -959,14 +332,18 @@ def _build_fal_payload( if v is not None: payload[k] = v - supports = meta["supports"] - # ``prompt`` is required by every FAL text-to-image endpoint; keep it even - # if a model's ``supports`` whitelist omits it, so a missing whitelist entry - # can't silently strip the prompt and send an empty request. - return { - k: v for k, v in payload.items() - if k in supports or k == "prompt" - } + return {k: v for k, v in payload.items() if k in supports or k in required} + + +def _build_fal_payload( + model_id: str, + prompt: str, + aspect_ratio: str = DEFAULT_ASPECT_RATIO, + seed: Optional[int] = None, + overrides: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Build a FAL text-to-image payload for `model_id` from unified inputs.""" + return _build_payload(model_id, prompt, aspect_ratio, seed, overrides) def _build_fal_edit_payload( @@ -977,64 +354,15 @@ def _build_fal_edit_payload( seed: Optional[int] = None, overrides: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: - """Build a FAL *edit* request payload (image-to-image) from unified inputs. - - Every FAL edit endpoint takes ``image_urls`` (a list of source/reference - image URLs) plus the prompt. Size handling differs from text-to-image: - most edit endpoints auto-infer output dimensions from the input image, so - we only send ``image_size`` / ``aspect_ratio`` when the edit endpoint's - ``edit_supports`` whitelist accepts it. Keys outside ``edit_supports`` are - stripped before submission. - """ - meta = FAL_MODELS[model_id] - edit_supports = meta.get("edit_supports") or set() - size_style = meta["size_style"] - sizes = meta["sizes"] - - aspect = (aspect_ratio or DEFAULT_ASPECT_RATIO).lower().strip() - if aspect not in sizes: - aspect = DEFAULT_ASPECT_RATIO - - payload: Dict[str, Any] = dict(meta.get("defaults", {})) - payload["prompt"] = (prompt or "").strip() - payload["image_urls"] = list(image_urls) - - # Only express output size when the edit endpoint advertises the key. - # gpt-image-2 edit auto-infers size from the input, so `image_size` is - # intentionally absent from its edit_supports whitelist. - if size_style in {"image_size_preset", "gpt_literal"} and "image_size" in edit_supports: - payload["image_size"] = sizes[aspect] - elif size_style == "aspect_ratio" and "aspect_ratio" in edit_supports: - payload["aspect_ratio"] = sizes[aspect] - - if seed is not None and isinstance(seed, int): - payload["seed"] = seed - - if overrides: - for k, v in overrides.items(): - if v is not None: - payload[k] = v - - # ``prompt`` and ``image_urls`` are required by every FAL edit endpoint; - # keep them even if a model's ``edit_supports`` whitelist omits them, so a - # missing whitelist entry can't silently drop the prompt or the source - # images and send a broken edit request. - _required = {"prompt", "image_urls"} - return { - k: v for k, v in payload.items() - if k in edit_supports or k in _required - } + """Build a FAL *edit* (image-to-image) payload: ``image_urls`` + prompt, filtered to ``edit_supports``.""" + return _build_payload(model_id, prompt, aspect_ratio, seed, overrides, image_urls=image_urls) # --------------------------------------------------------------------------- # Upscaler # --------------------------------------------------------------------------- def _upscale_image(image_url: str, original_prompt: str) -> Optional[Dict[str, Any]]: - """Upscale an image using FAL.ai's Clarity Upscaler. - - Returns upscaled image dict, or None on failure (caller falls back to - the original image). - """ + """Upscale via FAL's Clarity Upscaler; None on failure (caller keeps the original).""" try: logger.info("Upscaling image with Clarity Upscaler...") @@ -1071,8 +399,7 @@ def _upscale_image(image_url: str, original_prompt: str) -> Optional[Dict[str, A return None except ImageGenerationInterrupted: - # Propagate: the user interrupt must not degrade into a silent - # "upscale failed, use original" fallback that keeps the turn alive. + # A user interrupt must not degrade into a silent "use original" fallback. raise except Exception as e: logger.error("Error upscaling image: %s", e, exc_info=True) @@ -1080,7 +407,7 @@ def _upscale_image(image_url: str, original_prompt: str) -> Optional[Dict[str, A # --------------------------------------------------------------------------- -# Tool entry point +# Artifact path hinting for non-local terminal backends # --------------------------------------------------------------------------- def _looks_like_absolute_file_path(value: str) -> bool: if not value or not isinstance(value, str): @@ -1105,10 +432,8 @@ def _active_terminal_env(task_id: str | None): def _agent_cache_base_for_env(env: Any) -> str | None: if env is not None: - # Forward-looking optional override: an environment may expose its own - # agent-visible cache root via this callable. No backend defines it yet - # — it's an extension hook, not a typo. The getattr/callable guards make - # it a safe no-op until a producer exists. + # Optional extension hook: an environment may expose its own agent-visible + # cache root. No backend defines it yet; the guards make it a safe no-op. explicit = getattr(env, "agent_visible_cache_base", None) if callable(explicit): try: @@ -1126,10 +451,9 @@ def _agent_cache_base_for_env(env: Any) -> str | None: if env_name in {"DockerEnvironment", "SingularityEnvironment", "ModalEnvironment"}: return "/root/.hermes" - # If no environment has been created yet, only backends with deterministic - # Hermes cache roots can be translated without side effects. SSH can still - # use a shell-visible tilde path; its first environment sync will upload - # the cache file before the first command runs. + # No environment yet: only backends with deterministic cache roots can be + # translated without side effects. SSH can use a shell-visible tilde path; + # its first environment sync uploads the cache file before the first command. backend = (os.getenv("TERMINAL_ENV") or "local").strip().lower() if backend in {"docker", "singularity", "modal"}: return "/root/.hermes" @@ -1168,9 +492,9 @@ def _force_artifact_sync(env: Any) -> None: def _postprocess_image_generate_result(raw: str, task_id: str | None = None) -> str: """Annotate successful local image results with backend-visible paths. - ``image`` remains the host/gateway-deliverable path. When the active - terminal backend has a different filesystem, ``agent_visible_image`` gives - the path the agent can use with terminal/file tools. + ``image`` stays the host/gateway-deliverable path; when the active terminal + backend has a different filesystem, ``agent_visible_image`` is the path the + agent can use with terminal/file tools. """ try: payload = json.loads(raw) if isinstance(raw, str) else raw @@ -1197,6 +521,9 @@ def _postprocess_image_generate_result(raw: str, task_id: str | None = None) -> return json.dumps(payload, ensure_ascii=False) +# --------------------------------------------------------------------------- +# Tool entry point +# --------------------------------------------------------------------------- def image_generate_tool( prompt: str, aspect_ratio: str = DEFAULT_ASPECT_RATIO, @@ -1211,15 +538,11 @@ def image_generate_tool( ) -> str: """Generate an image from a text prompt, or edit a source image, via FAL. - Routing: when ``image_url`` (or ``reference_image_urls``) is provided AND - the configured model declares an ``edit_endpoint``, the call routes to that - image-to-image / edit endpoint; otherwise it's plain text-to-image. - - The agent-facing schema exposes ``prompt``, ``aspect_ratio``, ``image_url`` - and ``reference_image_urls``; the remaining kwargs are overrides for direct - Python callers and are filtered per-model via the ``supports`` / - ``edit_supports`` whitelist (unsupported overrides are silently dropped so - legacy callers don't break when switching models). + Routing: ``image_url`` / ``reference_image_urls`` plus a model with an + ``edit_endpoint`` → image-to-image; otherwise text-to-image. The extra + kwargs are overrides for direct Python callers, filtered per-model via the + ``supports`` / ``edit_supports`` whitelist (unsupported ones are dropped + silently so legacy callers survive model switches). Returns a JSON string with ``{"success": bool, "image": url | None, "modality": "text" | "image", "error": str, "error_type": str}``. @@ -1264,15 +587,14 @@ def image_generate_tool( if not prompt or not isinstance(prompt, str) or len(prompt.strip()) == 0: raise ValueError("Prompt is required and must be a non-empty string") - # Strict selection check: a stored-but-broken selection raises the - # honest selection-naming error from _resolve_managed_fal_gateway(); - # only the never-configured path can report "no backend at all". + # A stored-but-broken selection raises the selection-naming error from + # _resolve_managed_fal_gateway(); only the never-configured path can + # report "no backend at all". if not (fal_key_is_configured() or _resolve_managed_fal_gateway()): raise ValueError(_build_no_backend_setup_message()) - # If the caller supplied source images but the active model has no - # edit endpoint, fail with a clear, actionable message instead of - # silently dropping the images and producing an unrelated picture. + # Source images on a model without an edit endpoint: fail clearly rather + # than silently dropping them and producing an unrelated picture. if source_images and not edit_endpoint: raise ValueError( f"Model '{meta.get('display', model_id)}' ({model_id}) is not " @@ -1289,15 +611,14 @@ def image_generate_tool( ) aspect_lc = DEFAULT_ASPECT_RATIO - overrides: Dict[str, Any] = {} - if num_inference_steps is not None: - overrides["num_inference_steps"] = num_inference_steps - if guidance_scale is not None: - overrides["guidance_scale"] = guidance_scale - if num_images is not None: - overrides["num_images"] = num_images - if output_format is not None: - overrides["output_format"] = output_format + overrides: Dict[str, Any] = { + k: v for k, v in ( + ("num_inference_steps", num_inference_steps), + ("guidance_scale", guidance_scale), + ("num_images", num_images), + ("output_format", output_format), + ) if v is not None + } if use_edit: # Clamp reference count to the model's declared cap. @@ -1335,11 +656,10 @@ def image_generate_tool( if not images: raise ValueError("No images were generated") - # Explicit ``upscale`` (agent/user opt-in via the tool schema) wins - # over the per-model catalog default — including for edits, where an - # explicit request is intentional. The catalog default keeps skipping - # edits (Clarity is a text-to-image quality pass and must not alter - # edit compositions silently). + # An explicit ``upscale`` wins over the catalog default, including for + # edits (an explicit request is intentional). The catalog default never + # upscales edits: Clarity is a text-to-image quality pass and must not + # silently alter edit compositions. if upscale is not None: should_upscale = bool(upscale) else: @@ -1411,12 +731,10 @@ def image_generate_tool( def check_fal_api_key() -> bool: - """True if the FAL backend selected via `hermes tools` (or, on a - never-configured install, any FAL backend) is available. + """True if the FAL backend selected via `hermes tools` (or, never configured, any FAL backend) is available. - A stored-but-broken selection reports False here (registry gating); - the honest selection-naming error surfaces at call time from - ``_resolve_managed_fal_gateway``. + A stored-but-broken selection reports False here (registry gating); the + selection-naming error surfaces at call time from ``_resolve_managed_fal_gateway``. """ selected = read_selection("image_gen") if selected == NOUS_MANAGED_PROVIDER: @@ -1427,14 +745,8 @@ def check_fal_api_key() -> bool: def _build_no_backend_setup_message() -> str: - """Build an actionable error string when no FAL backend is reachable. - - Used by the in-tree FAL path. Mentions: - - FAL_KEY signup link - - managed-gateway status (if Nous tools are enabled) - - plugin alternative pointer (so users on a stale ``image_gen.provider`` - know the registry exists and how to inspect it) - """ + """Actionable error when no FAL backend is reachable: FAL_KEY signup, + managed-gateway status (if Nous tools enabled), and the plugin alternative.""" lines = ["Image generation is unavailable in this environment.", ""] lines.append("Missing requirements:") if managed_nous_tools_enabled(): @@ -1467,14 +779,21 @@ def _build_no_backend_setup_message() -> str: return "\n".join(lines) +def _get_plugin_provider(name: str): + """Discover plugins (import is local so importing this module never triggers discovery) and return the named provider.""" + from agent.image_gen_registry import get_provider + from hermes_cli.plugins import _ensure_plugins_discovered + + _ensure_plugins_discovered() + return get_provider(name) + + def check_image_generation_requirements() -> bool: """True if FAL or the explicitly configured image backend is available.""" try: if check_fal_api_key(): - # Trigger the lazy fal_client import here as the SDK presence - # check. Raises ImportError if the optional ``fal-client`` - # package isn't installed; the caller's except ImportError - # below catches that and continues to plugin probing. + # The lazy import doubles as the SDK presence check: ImportError + # when ``fal-client`` isn't installed falls through to plugin probing. _load_fal_client() return True except ImportError: @@ -1487,51 +806,12 @@ def check_image_generation_requirements() -> bool: # Probe only the explicitly selected plugin. Merely possessing a cloud # provider key must not opt a user into a paid image-generation backend. try: - from agent.image_gen_registry import get_provider - from hermes_cli.plugins import _ensure_plugins_discovered - - _ensure_plugins_discovered() - provider = get_provider(configured) + provider = _get_plugin_provider(configured) return bool(provider and provider.is_available()) except Exception: return False -# --------------------------------------------------------------------------- -# Demo / CLI entry point -# --------------------------------------------------------------------------- -if __name__ == "__main__": - print("🎨 Image Generation Tools — FAL.ai multi-model support") - print("=" * 60) - - if not check_fal_api_key(): - print("❌ FAL_KEY environment variable not set") - print(" Set it via: export FAL_KEY='your-key-here'") - print(" Get a key: https://fal.ai/") - raise SystemExit(1) - print("✅ FAL.ai API key found") - - try: - import fal_client # noqa: F401 - print("✅ fal_client library available") - except ImportError: - print("❌ fal_client library not found — pip install fal-client") - raise SystemExit(1) - - model_id, meta = _resolve_fal_model() - print(f"🤖 Active model: {meta.get('display', model_id)} ({model_id})") - print(f" Speed: {meta.get('speed', '?')} · Price: {meta.get('price', '?')}") - print(f" Upscaler: {'on' if meta.get('upscale') else 'off'}") - - print("\nAvailable models:") - for mid, m in FAL_MODELS.items(): - marker = " ← active" if mid == model_id else "" - print(f" {mid:<32} {m.get('speed', '?'):<6} {m.get('price', '?')}{marker}") - - if _debug.active: - print(f"\n🐛 Debug mode enabled — session {_debug.session_id}") - - # --------------------------------------------------------------------------- # Registry # --------------------------------------------------------------------------- @@ -1567,52 +847,49 @@ IMAGE_GENERATE_SCHEMA = { "description": "The aspect ratio of the generated image. 'landscape' is 16:9 wide, 'portrait' is 16:9 tall, 'square' is 1:1.", "default": DEFAULT_ASPECT_RATIO, }, - # NOTE (schema diet, #95681): image_url / reference_image_urls / - # upscale are added per-capability by _build_dynamic_image_schema. - # Do not re-add them statically. + # image_url / reference_image_urls / upscale are added per-capability + # by _build_dynamic_image_schema. Do not re-add them statically. }, "required": ["prompt"], }, } -def _read_configured_image_model(): - """Return the value of ``image_gen.model`` from config.yaml, or None.""" - try: - from hermes_cli.config import load_config - cfg = load_config() - section = cfg.get("image_gen") if isinstance(cfg, dict) else None - if isinstance(section, dict): - value = section.get("model") - if isinstance(value, str) and value.strip(): - return value.strip() - except Exception as exc: - logger.debug("Could not read image_gen.model: %s", exc) - return None +# --------------------------------------------------------------------------- +# Plugin provider dispatch + managed-mode Krea routing +# --------------------------------------------------------------------------- +def _provider_error(error: str, error_type: str) -> str: + """JSON error envelope shared by every provider-dispatch failure path.""" + return json.dumps({ + "success": False, + "image": None, + "error": error, + "error_type": error_type, + }) -def _read_configured_image_provider(): - """Return ``image_gen.provider`` from config.yaml, or None. +def _add_provider_kwargs( + kwargs: Dict[str, Any], + image_url: Optional[str], + reference_image_urls: Optional[list], + upscale: Optional[bool], + model: Optional[str] = None, +) -> Dict[str, Any]: + """Add the optional ``provider.generate(**kwargs)`` args in place (edit args only when supplied).""" + if model: + kwargs["model"] = model + if isinstance(image_url, str) and image_url.strip(): + kwargs["image_url"] = image_url.strip() + norm_refs = None + if reference_image_urls is not None: + from agent.image_gen_provider import normalize_reference_images - We only consult the plugin registry when this is explicitly set — an - unset value keeps users on the in-tree FAL fallback even when other - providers happen to be registered (e.g. a user has OPENAI_API_KEY set - for other features but never asked for OpenAI image gen). ``"fal"`` - explicitly routes through ``plugins/image_gen/fal/`` (which delegates - back into this module's pipeline via call-time indirection — see - issue #26241). - """ - try: - from hermes_cli.config import load_config - cfg = load_config() - section = cfg.get("image_gen") if isinstance(cfg, dict) else None - if isinstance(section, dict): - value = section.get("provider") - if isinstance(value, str) and value.strip(): - return value.strip() - except Exception as exc: - logger.debug("Could not read image_gen.provider: %s", exc) - return None + norm_refs = normalize_reference_images(reference_image_urls) + if norm_refs: + kwargs["reference_image_urls"] = norm_refs + if upscale is not None: + kwargs["upscale"] = bool(upscale) + return kwargs def _dispatch_to_plugin_provider( @@ -1625,38 +902,24 @@ def _dispatch_to_plugin_provider( """Route the call to a plugin-registered provider when one is selected. Returns a JSON string on dispatch, or ``None`` to fall through to the - in-tree FAL fallback in ``image_generate_tool``. + in-tree FAL pipeline. Fires when ``image_gen.provider`` is set to anything + other than unset / ``"fal"`` / ``"nous"`` — those run the legacy pipeline + (``"nous"`` routes it through the managed fal-queue gateway). - Dispatch fires when ``image_gen.provider`` is explicitly set — including - ``"fal"`` itself, which now resolves to the - ``plugins/image_gen/fal/`` plugin (the plugin re-enters this module's - pipeline via ``_it`` indirection so behavior is identical to the - direct call, just routed through the registry). - - ``image_url`` / ``reference_image_urls`` enable image-to-image / editing: - they are forwarded to the provider's ``generate()`` so the backend can - route to its edit endpoint. ``upscale`` (when explicitly set) requests a - post-generation high-resolution pass; providers without upscale support - ignore it via their ``**kwargs`` (the ABC contract). + ``image_url`` / ``reference_image_urls`` are forwarded so the backend can + route to its edit endpoint; ``upscale`` requests a post-generation + high-res pass (providers without it ignore it via ``**kwargs``). """ configured = _read_configured_image_provider() if not configured or configured in ("fal", NOUS_MANAGED_PROVIDER): - # Unset/explicit FAL keeps the legacy FAL path; "nous" (managed - # Nous Subscription selection) also runs the legacy pipeline, which - # routes through the managed fal-queue gateway. return None - # Also read configured model so we can pass it to the plugin configured_model = _read_configured_image_model() try: - # Import locally so plugin discovery isn't triggered just by - # importing this module (tests rely on that). - from agent.image_gen_registry import get_provider from hermes_cli.plugins import _ensure_plugins_discovered - _ensure_plugins_discovered() - provider = get_provider(configured) + provider = _get_plugin_provider(configured) except Exception as exc: logger.debug("image_gen plugin dispatch skipped: %s", exc) return None @@ -1664,104 +927,61 @@ def _dispatch_to_plugin_provider( if provider is None: try: # Long-lived sessions may have discovered plugins before a bundled - # backend was patched in or before config changed. Retry once with - # a forced refresh before surfacing a missing-provider error. + # backend was patched in or config changed: retry once with a forced + # refresh before surfacing a missing-provider error. + from agent.image_gen_registry import get_provider + _ensure_plugins_discovered(force=True) provider = get_provider(configured) except Exception as exc: logger.debug("image_gen plugin force-refresh skipped: %s", exc) if provider is None: - return json.dumps({ - "success": False, - "image": None, - "error": ( - f"image_gen.provider='{configured}' is set but no plugin " - f"registered that name. Run `hermes plugins list` to see " - f"available image gen backends." - ), - "error_type": "provider_not_registered", - }) + return _provider_error( + f"image_gen.provider='{configured}' is set but no plugin " + f"registered that name. Run `hermes plugins list` to see " + f"available image gen backends.", + "provider_not_registered", + ) + pname = getattr(provider, "name", "?") kwargs: Dict[str, Any] = {"prompt": prompt, "aspect_ratio": aspect_ratio} try: - if configured_model: - kwargs["model"] = configured_model - if isinstance(image_url, str) and image_url.strip(): - kwargs["image_url"] = image_url.strip() - norm_refs = None - if reference_image_urls is not None: - from agent.image_gen_provider import normalize_reference_images - - norm_refs = normalize_reference_images(reference_image_urls) - if norm_refs: - kwargs["reference_image_urls"] = norm_refs - if upscale is not None: - kwargs["upscale"] = bool(upscale) + _add_provider_kwargs( + kwargs, image_url, reference_image_urls, upscale, model=configured_model, + ) result = provider.generate(**kwargs) except TypeError as exc: - # A provider whose generate() signature predates image_url support - # (third-party plugin not yet updated) — retry without the new kwargs - # so text-to-image keeps working, but surface a clear note when the - # user actually asked for an edit. + # A provider whose generate() predates image_url support (third-party + # plugin not yet updated): text-to-image keeps working, but surface a + # clear note when the user actually asked for an edit. if "image_url" in kwargs or "reference_image_urls" in kwargs: logger.warning( "image_gen provider '%s' rejected image-to-image kwargs " "(signature too narrow): %s", - getattr(provider, "name", "?"), exc, + pname, exc, ) - return json.dumps({ - "success": False, - "image": None, - "error": ( - f"Provider '{getattr(provider, 'name', '?')}' does not " - f"support image-to-image / editing (its generate() " - f"signature is out of date with the image_generate schema). " - f"Omit image_url for text-to-image, or pick a backend that " - f"supports editing via `hermes tools` → Image Generation." - ), - "error_type": "modality_unsupported", - }) - logger.warning( - "Image gen provider '%s' raised TypeError: %s", - getattr(provider, "name", "?"), exc, - ) - return json.dumps({ - "success": False, - "image": None, - "error": f"Provider '{getattr(provider, 'name', '?')}' error: {exc}", - "error_type": "provider_exception", - }) + return _provider_error( + f"Provider '{pname}' does not " + f"support image-to-image / editing (its generate() " + f"signature is out of date with the image_generate schema). " + f"Omit image_url for text-to-image, or pick a backend that " + f"supports editing via `hermes tools` → Image Generation.", + "modality_unsupported", + ) + logger.warning("Image gen provider '%s' raised TypeError: %s", pname, exc) + return _provider_error(f"Provider '{pname}' error: {exc}", "provider_exception") except Exception as exc: - logger.warning( - "Image gen provider '%s' raised: %s", - getattr(provider, "name", "?"), exc, - ) - return json.dumps({ - "success": False, - "image": None, - "error": f"Provider '{getattr(provider, 'name', '?')}' error: {exc}", - "error_type": "provider_exception", - }) + logger.warning("Image gen provider '%s' raised: %s", pname, exc) + return _provider_error(f"Provider '{pname}' error: {exc}", "provider_exception") if not isinstance(result, dict): - return json.dumps({ - "success": False, - "image": None, - "error": "Provider returned a non-dict result", - "error_type": "provider_contract", - }) + return _provider_error("Provider returned a non-dict result", "provider_contract") return json.dumps(result) -# --------------------------------------------------------------------------- -# Managed-mode Krea routing -# --------------------------------------------------------------------------- -# # Native ``krea-2-*`` plugin model ids are served by the dedicated Krea managed -# gateway. ``fal-ai/krea/v2/*`` FAL catalog ids stay on the FAL path (BYO key -# or FAL managed gateway). Routing only fires in managed mode; direct/BYO users -# keep their unchanged pipeline. - +# gateway; ``fal-ai/krea/v2/*`` catalog ids stay on the FAL path. Routing only +# fires in managed mode — direct/BYO users keep their unchanged pipeline. _KREA_NATIVE_MODELS = {"krea-2-medium", "krea-2-large", "krea-2-medium-turbo"} @@ -1775,11 +995,6 @@ def _normalize_krea_model(model_id: Optional[str]) -> Optional[str]: return None -def is_krea_model(model_id: Optional[str]) -> bool: - """True when ``model_id`` is a native Krea plugin id (``krea-2-*``).""" - return _normalize_krea_model(model_id) is not None - - def _maybe_route_managed_krea( prompt: str, aspect_ratio: str, @@ -1789,21 +1004,12 @@ def _maybe_route_managed_krea( ) -> Optional[str]: """Route a native ``krea-2-*`` model to the managed Krea gateway, in managed mode. - Returns a JSON result string when handled by the Krea managed gateway, or - ``None`` to fall through to the normal plugin/FAL pipeline. Fires only when - all hold: - - the configured image model is a native ``krea-2-*`` id, AND - - the user isn't already routed to the Krea plugin via - ``image_gen.provider`` (that path dispatches normally), AND - - the managed Krea gateway is resolvable (portal/managed mode). - - Direct/BYO users (no managed gateway) fall through untouched. + Returns a JSON result string when handled, or ``None`` to fall through to + the normal plugin/FAL pipeline. Fires only when the configured model is a + native ``krea-2-*`` id AND no explicit ``image_gen.provider`` other than the + managed ``"nous"`` selection is stored (a picker choice dispatches normally) + AND the managed Krea gateway is resolvable. """ - # Strict selection rule: an explicitly stored ``image_gen.provider`` - # (other than the managed "nous" selection, which IS a managed-mode - # opt-in) disables the model-driven managed interception — the user's - # picker choice dispatches normally. Interception is permitted only on - # never-configured installs or under the managed selection. configured_provider = _read_configured_image_provider() if configured_provider is not None and configured_provider != NOUS_MANAGED_PROVIDER: return None @@ -1812,7 +1018,6 @@ def _maybe_route_managed_krea( if normalized is None: return None - # Only intercept on the managed path; BYO/direct users keep their pipeline. try: from plugins.image_gen.krea import _resolve_managed_krea_gateway @@ -1823,50 +1028,22 @@ def _maybe_route_managed_krea( return None try: - from agent.image_gen_registry import get_provider - from hermes_cli.plugins import _ensure_plugins_discovered - - _ensure_plugins_discovered() - provider = get_provider("krea") + provider = _get_plugin_provider("krea") except Exception as exc: # noqa: BLE001 logger.debug("Managed Krea routing: provider unavailable: %s", exc) return None if provider is None: return None - kwargs: Dict[str, Any] = { - "prompt": prompt, - "aspect_ratio": aspect_ratio, - "model": normalized, - } + kwargs: Dict[str, Any] = {"prompt": prompt, "aspect_ratio": aspect_ratio, "model": normalized} try: - if isinstance(image_url, str) and image_url.strip(): - kwargs["image_url"] = image_url.strip() - norm_refs = None - if reference_image_urls is not None: - from agent.image_gen_provider import normalize_reference_images - - norm_refs = normalize_reference_images(reference_image_urls) - if norm_refs: - kwargs["reference_image_urls"] = norm_refs - if upscale is not None: - kwargs["upscale"] = bool(upscale) + _add_provider_kwargs(kwargs, image_url, reference_image_urls, upscale) result = provider.generate(**kwargs) except Exception as exc: # noqa: BLE001 logger.warning("Managed Krea routing failed: %s", exc) - return json.dumps({ - "success": False, - "image": None, - "error": f"Managed Krea generation error: {exc}", - "error_type": "provider_exception", - }) + return _provider_error(f"Managed Krea generation error: {exc}", "provider_exception") if not isinstance(result, dict): - return json.dumps({ - "success": False, - "image": None, - "error": "Krea provider returned a non-dict result", - "error_type": "provider_contract", - }) + return _provider_error("Krea provider returned a non-dict result", "provider_contract") return json.dumps(result) @@ -1876,12 +1053,11 @@ def _confine_source_images( """Route path-like source images through the sandbox-aware resolver. Under a non-local terminal backend (ssh/docker/…), model-supplied local - paths are resolved via ``tools.image_source`` (in-sandbox exec-read, - media-cache host reads, credential guard, lazy env bring-up) and converted - to ``data:`` URLs before any provider sees them — so generation tools obey - the same confinement boundary as vision/video analysis, and sandbox-only - files actually work as edit sources. URLs and data: URLs pass through - untouched; the local backend is a no-op (providers keep their host reads). + paths resolve via ``tools.image_source`` (in-sandbox exec-read, media-cache + host reads, credential guard) into ``data:`` URLs before any provider sees + them, so generation obeys the same confinement boundary as vision/video + analysis and sandbox-only files work as edit sources. URLs and data: URLs + pass through; the local backend is a no-op (providers keep host reads). Returns ``(image_url, reference_image_urls, error_json_or_None)``. """ @@ -1903,12 +1079,9 @@ def _confine_source_images( for ref in list(reference_image_urls) ] except ImageResolutionError as exc: - return image_url, reference_image_urls, json.dumps({ - "success": False, - "image": None, - "error": f"Could not read source image: {exc}", - "error_type": type(exc).__name__, - }) + return image_url, reference_image_urls, _provider_error( + f"Could not read source image: {exc}", type(exc).__name__, + ) return image_url, reference_image_urls, None @@ -1924,79 +1097,55 @@ def _handle_image_generate(args, **kw): upscale = None task_id = kw.get("task_id") - # Terminal-backend confinement chokepoint: convert path-like sources to - # data: URLs via the shared resolver BEFORE any provider dispatch, so - # every backend (plugin, managed Krea, in-tree FAL) gets the same - # sandbox-confined bytes. + # Confinement chokepoint: path-like sources become data: URLs BEFORE any + # dispatch, so plugin, managed Krea and in-tree FAL all get sandbox-confined bytes. image_url, reference_image_urls, confine_error = _confine_source_images( image_url, reference_image_urls, task_id) if confine_error is not None: return confine_error - # Route to a plugin-registered provider if one is active (and it's - # not the in-tree FAL path). When ``image_gen.provider == "krea"`` this - # already reaches the Krea plugin's managed gateway path. - dispatched = _dispatch_to_plugin_provider( + # Order matters: explicit plugin provider (incl. provider == "krea"), then + # model-driven managed Krea interception (only when no provider is set, so + # the BYO/direct FAL path stays untouched), then the in-tree FAL pipeline. + raw = _dispatch_to_plugin_provider( prompt, aspect_ratio, image_url=image_url, reference_image_urls=reference_image_urls, upscale=upscale, ) - if dispatched is not None: - return _postprocess_image_generate_result(dispatched, task_id=task_id) - - # Managed-mode Krea routing: when no explicit plugin provider is configured - # but the selected model is a native ``krea-2-*`` id, a portal user routes to - # the dedicated Krea managed gateway. ``fal-ai/krea/v2/*`` models stay on the - # FAL path below. Runs after plugin dispatch (which returns None when no - # provider is set) so the BYO/direct FAL path stays untouched. - krea_routed = _maybe_route_managed_krea( - prompt, aspect_ratio, - image_url=image_url, - reference_image_urls=reference_image_urls, - upscale=upscale, - ) - if krea_routed is not None: - return _postprocess_image_generate_result(krea_routed, task_id=task_id) - - raw = image_generate_tool( - prompt=prompt, - aspect_ratio=aspect_ratio, - image_url=image_url, - reference_image_urls=reference_image_urls, - upscale=upscale, - ) + if raw is None: + raw = _maybe_route_managed_krea( + prompt, aspect_ratio, + image_url=image_url, + reference_image_urls=reference_image_urls, + upscale=upscale, + ) + if raw is None: + raw = image_generate_tool( + prompt=prompt, + aspect_ratio=aspect_ratio, + image_url=image_url, + reference_image_urls=reference_image_urls, + upscale=upscale, + ) return _postprocess_image_generate_result(raw, task_id=task_id) # --------------------------------------------------------------------------- # Dynamic schema — reflect the active backend's image-to-image capability # --------------------------------------------------------------------------- -# -# Why dynamic: whether the active model supports image-to-image / editing -# depends entirely on the user's configured backend + model. Telling the -# model up front ("the active model is text-to-image only — image_url will be -# rejected") saves a wasted turn. Memoized by config.yaml mtime in -# model_tools.get_tool_definitions(), so it rebuilds when the user switches -# model/provider via `hermes tools` or `/skills`. - - -_GENERIC_IMAGE_DESCRIPTION = IMAGE_GENERATE_SCHEMA["description"] +# Whether the active model can edit depends on the configured backend + model; +# telling the model up front saves a wasted turn. Memoized by config.yaml mtime +# in model_tools.get_tool_definitions(), so it rebuilds on provider/model switch. def _active_image_capabilities() -> Dict[str, Any]: - """Best-effort: return the active backend/model's image capabilities. + """Best-effort capabilities of the active backend/model; never raises. - Resolution order mirrors the runtime dispatch: - 1. If ``image_gen.provider`` is set, ask that plugin provider. - 2. Otherwise inspect the in-tree FAL model catalog for the active model. - - Returns ``{"modalities": [...], "max_reference_images": N, - "supports_upscale": bool, "model": "...", "provider": "..."}``. - Fail-closed on every axis: an unknown/undeclared capability is - advertised as absent (a provider that can edit but didn't declare it - under-advertises — that is the provider's bug to fix in - ``capabilities()``, not a safety problem). Never raises. + Resolution mirrors runtime dispatch: a set ``image_gen.provider`` asks that + plugin, otherwise the in-tree FAL catalog. Fail-closed on every axis: an + undeclared capability is advertised as absent (an under-declaring provider + is that provider's bug, not a safety problem). """ info: Dict[str, Any] = { "modalities": ["text"], @@ -2007,11 +1156,7 @@ def _active_image_capabilities() -> Dict[str, Any]: configured_provider = _read_configured_image_provider() if configured_provider and configured_provider != "fal": try: - from agent.image_gen_registry import get_provider - from hermes_cli.plugins import _ensure_plugins_discovered - - _ensure_plugins_discovered() - provider = get_provider(configured_provider) + provider = _get_plugin_provider(configured_provider) if provider is not None: caps = {} try: @@ -2041,10 +1186,8 @@ def _active_image_capabilities() -> Dict[str, Any]: else: info["modalities"] = ["text"] info["max_reference_images"] = 0 - # FAL: the Clarity Upscaler is a separate endpoint chained on - # explicit request for ANY catalog model (the per-model ``upscale`` - # key is only the default-on flag, retired Aug 2026 — not a - # capability). Plugin providers must declare supports_upscale. + # FAL: Clarity is a separate endpoint chained on explicit request for ANY + # catalog model (the per-model ``upscale`` key is only the default flag). info["supports_upscale"] = True except Exception: # noqa: BLE001 pass @@ -2076,11 +1219,8 @@ _UPSCALE_PARAM = { def _build_dynamic_image_schema() -> Dict[str, Any]: """Render description AND params from the active model's capabilities. - Capability coverage is guaranteed: all in-tree FAL catalog entries - carry edit/refs/upscale metadata (contract-tested), and the plugin - provider ABC's capabilities() fail-closed default is text-only. Args a - model cannot honor are NOT advertised — the handler still accepts them - (replay compat) and answers with a capability error. + Args a model cannot honor are NOT advertised — the handler still accepts + them (replay compat) and answers with a capability error. """ base_desc = ( "Generate high-quality images from text prompts{edit_clause}. " diff --git a/tools/kanban_tools.py b/tools/kanban_tools.py index dd1db3ed3d..a4cb9a7704 100644 --- a/tools/kanban_tools.py +++ b/tools/kanban_tools.py @@ -1,42 +1,48 @@ """Kanban tools — structured tool-call surface for worker + orchestrator agents. -These tools are registered into the model's schema when the agent is -running under the dispatcher (env var ``HERMES_KANBAN_TASK`` set) or when -the active profile explicitly enables the ``kanban`` toolset for -orchestrator work. A normal ``hermes chat`` session still sees **zero** -kanban tools in its schema unless configured. +Registered into the model's schema only when running under the dispatcher +(``HERMES_KANBAN_TASK`` set) or when the active profile enables the ``kanban`` +toolset; a plain ``hermes chat`` session sees zero kanban tools. -Why tools instead of just shelling out to ``hermes kanban``? - -1. **Backend portability.** A worker whose terminal tool points at Docker - / Modal / Singularity / SSH would run ``hermes kanban complete …`` - inside the container, where ``hermes`` isn't installed and the DB - isn't mounted. Tools run in the agent's Python process, so they - always reach ``~/.hermes/kanban.db`` regardless of terminal backend. - -2. **No shell-quoting footguns.** Passing ``--metadata '{"x": [...]}'`` - through shlex+argparse is fragile. Structured tool args skip it. - -3. **Better errors.** Tool-call failures return structured JSON the - model can reason about, not stderr strings it has to parse. - -Humans continue to use the CLI (``hermes kanban …``), the dashboard -(``hermes dashboard``), and the slash command (``/kanban …``) — all -three bypass the agent entirely. The tools are for dispatcher-spawned -worker handoffs and for configured orchestrator profiles that route work -through the board. +Why tools rather than shelling out to ``hermes kanban``: tools run in the +agent's Python process, so they reach ``~/.hermes/kanban.db`` even when the +terminal backend is a container/SSH host without ``hermes`` installed; they +avoid shlex/argparse quoting of JSON metadata; and failures come back as +structured JSON the model can reason about. Humans keep using the CLI, +dashboard, and ``/kanban`` slash command, which bypass the agent entirely. """ from __future__ import annotations +import functools import json import logging import os -from typing import Any, Optional +from contextlib import contextmanager +from typing import Any, Callable, Optional from agent.redact import redact_sensitive_text from hermes_cli.goals import judge_goal from tools.registry import registry, tool_error from hermes_cli.config import cfg_get, load_config +from tools.kanban_tools_schemas import ( # noqa: F401 - re-exported for callers/tests + _DESC_BOARD, + _DESC_TASK_ID_DEFAULT, + _board_schema_prop, + KANBAN_ATTACH_SCHEMA, + KANBAN_ATTACH_URL_SCHEMA, + KANBAN_ATTACHMENTS_SCHEMA, + KANBAN_BLOCK_SCHEMA, + KANBAN_COMMENT_SCHEMA, + KANBAN_COMPLETE_SCHEMA, + KANBAN_CREATE_SCHEMA, + KANBAN_HEARTBEAT_SCHEMA, + KANBAN_LINK_SCHEMA, + KANBAN_LIST_SCHEMA, + KANBAN_REQUEST_CHANGES_SCHEMA, + KANBAN_REQUEST_REVIEW_SCHEMA, + KANBAN_SHOW_SCHEMA, + KANBAN_UNBLOCK_SCHEMA, +) logger = logging.getLogger(__name__) @@ -50,14 +56,12 @@ KANBAN_LIST_MAX_LIMIT = 200 def _profile_has_kanban_toolset() -> bool: - # Uses load_config() which has mtime-based caching, so this adds - # negligible overhead. The check_fn results are further TTL-cached - # (~30s) by the tool registry. + # load_config() is mtime-cached and check_fn results are TTL-cached (~30s) + # by the registry, so this is cheap. try: from hermes_cli.config import load_config cfg = load_config() - toolsets = cfg.get("toolsets", []) - return "kanban" in toolsets + return "kanban" in cfg.get("toolsets", []) except Exception: return False @@ -82,13 +86,17 @@ def _is_dispatcher_owned_worker() -> bool: return True +def _is_env_worker() -> bool: + """True only for a dispatcher-spawned worker scoped to HERMES_KANBAN_TASK.""" + return bool(os.environ.get("HERMES_KANBAN_TASK")) and _is_dispatcher_owned_worker() + + def _reject_delegated_child_mutation(tool_name: str) -> Optional[str]: """Deny Kanban mutations from delegate_task children. - A delegate_task child runs in the same process as its parent, so stale or - inherited HERMES_KANBAN_* env vars are not proof of dispatcher ownership. - The child may summarize findings to its parent, but it must not complete, - block, heartbeat, comment, create, link, or unblock board tasks directly. + A child runs in the same process as its parent, so inherited HERMES_KANBAN_* + env vars are not proof of dispatcher ownership. It may report findings to + the parent but must not mutate board state directly. """ if not _is_delegated_child_context(): return None @@ -101,36 +109,24 @@ def _reject_delegated_child_mutation(tool_name: str) -> Optional[str]: def _check_kanban_mode() -> bool: - """Task-lifecycle tools are available when: - - 1. ``HERMES_KANBAN_TASK`` is set (dispatcher-spawned worker), OR - 2. The current profile has ``kanban`` in its toolsets config - (orchestrator profiles like techlead that route work via Kanban). - - Humans running ``hermes chat`` without the kanban toolset see zero - kanban tools. Workers spawned by the kanban dispatcher (gateway- - embedded by default) and orchestrator profiles with the kanban - toolset enabled see the Kanban lifecycle tool surface. - """ + """Lifecycle tools: visible to dispatcher-spawned workers and to profiles + that enable the ``kanban`` toolset (orchestrators); never to delegate children.""" if _is_delegated_child_context(): return False - if os.environ.get("HERMES_KANBAN_TASK") and _is_dispatcher_owned_worker(): + if _is_env_worker(): return True return _profile_has_kanban_toolset() def _check_kanban_orchestrator_mode() -> bool: - """Board-routing tools (kanban_list, kanban_unblock) are intentionally - hidden from task workers. + """Board-routing tools (kanban_list, kanban_unblock): hidden from task workers. - Dispatcher-spawned workers should close their own task via the - lifecycle tools (complete/block/heartbeat), not enumerate or unblock - board state. Profiles that explicitly opt into the kanban toolset - and are NOT scoped to a single task are the orchestrator surface. + Workers close their own task via complete/block/heartbeat; only profiles + that opt into the toolset and are NOT scoped to a single task route work. """ if _is_delegated_child_context(): return False - if os.environ.get("HERMES_KANBAN_TASK") and _is_dispatcher_owned_worker(): + if _is_env_worker(): return False return _profile_has_kanban_toolset() @@ -139,18 +135,20 @@ def _check_kanban_orchestrator_mode() -> bool: # Shared helpers # --------------------------------------------------------------------------- +_TASK_ID_REQUIRED = "task_id is required (or set HERMES_KANBAN_TASK in the env)" + + def _default_task_id(arg: Optional[str]) -> Optional[str]: - """Resolve ``task_id`` arg or fall back to the env var the dispatcher set.""" + """Resolve ``task_id`` arg or fall back to the env var the dispatcher set. + + A delegate child or a cron job fired in-process from a worker must never + inherit the worker's task id as an implicit default. + """ if arg: return arg - if _is_delegated_child_context(): + if _is_delegated_child_context() or not _is_dispatcher_owned_worker(): return None - if not _is_dispatcher_owned_worker(): - # A cron job fired in-process from a worker must never inherit the - # worker's task id as an implicit default. - return None - env_tid = os.environ.get("HERMES_KANBAN_TASK") - return env_tid or None + return os.environ.get("HERMES_KANBAN_TASK") or None def _worker_run_id(task_id: str) -> Optional[int]: @@ -158,52 +156,30 @@ def _worker_run_id(task_id: str) -> Optional[int]: if os.environ.get("HERMES_KANBAN_TASK") != task_id: return None raw = os.environ.get("HERMES_KANBAN_RUN_ID") - if not raw: - return None try: - return int(raw) + return int(raw) if raw else None except ValueError: return None -def _stamp_worker_session_metadata( - task_id: str, metadata: Optional[dict] -) -> Optional[dict]: +def _stamp_worker_session_metadata(task_id: str, metadata: Optional[dict]) -> Optional[dict]: """Add trusted worker session id metadata for this worker's own task.""" - if os.environ.get("HERMES_KANBAN_TASK") != task_id: - return metadata session_id = os.environ.get("HERMES_SESSION_ID") - if not session_id: + if os.environ.get("HERMES_KANBAN_TASK") != task_id or not session_id: return metadata - stamped = dict(metadata or {}) - stamped["worker_session_id"] = session_id - return stamped + return {**(metadata or {}), "worker_session_id": session_id} def _enforce_worker_task_ownership(tid: str) -> Optional[str]: """Reject worker-driven destructive calls on foreign task IDs. - A process spawned by the dispatcher has ``HERMES_KANBAN_TASK`` set - to its own task id. Tools like ``kanban_complete`` / ``kanban_block`` - / ``kanban_heartbeat`` mutate run-lifecycle state, so a buggy or - prompt-injected worker that passed an explicit ``task_id`` for some - other task could corrupt sibling or cross-tenant runs (see #19534). - - Orchestrator profiles (kanban toolset enabled but **no** - ``HERMES_KANBAN_TASK`` in env) aren't subject to this check — their - job is routing, and they sometimes legitimately close out child - tasks or reopen blocked ones. Workers are narrowly scoped to their - one task. - - Returns ``None`` when the call is allowed, or a tool-error string - when it must be rejected. Callers should ``return`` the error - verbatim. + A dispatcher-spawned worker has ``HERMES_KANBAN_TASK`` set to its own task; + a buggy or prompt-injected explicit ``task_id`` must not corrupt sibling or + cross-tenant runs. Orchestrators (toolset enabled, no env task) are exempt: + routing legitimately closes or reopens child tasks. """ env_tid = os.environ.get("HERMES_KANBAN_TASK") - if not env_tid: - # Orchestrator or CLI context — no task-scope restriction. - return None - if tid != env_tid: + if env_tid and tid != env_tid: return tool_error( f"worker is scoped to task {env_tid}; refusing to mutate " f"{tid}. Use kanban_comment to hand off information to other " @@ -212,36 +188,191 @@ def _enforce_worker_task_ownership(tid: str) -> Optional[str]: return None -def _connect(board: Optional[str] = None): - """Import + connect lazily so the module imports cleanly in non-kanban - contexts (e.g. test rigs that import every tool module). +def _worker_guard(tool_name: str, args: dict) -> tuple[str, Optional[str]]: + """Common preamble for worker mutation tools: ``(task_id, error)``. - When ``board`` is provided it's forwarded to :func:`kb.connect`, which - routes the connection to that board's sqlite file. ``None`` (the - default) preserves the legacy resolution chain - (``HERMES_KANBAN_DB`` → ``HERMES_KANBAN_BOARD`` env → current symlink - → ``default``). Per-tool ``board`` lets a Telegram-side agent override - the env-pinned active board without restarting Hermes. + Order matters: delegate-child rejection, then task id resolution, then + task-scope ownership. ``task_id`` is only meaningful when ``error`` is None. + """ + err = _reject_delegated_child_mutation(tool_name) + if err: + return "", err + tid = _default_task_id(args.get("task_id")) + if not tid: + return "", tool_error(_TASK_ID_REQUIRED) + return tid, _enforce_worker_task_ownership(tid) + + +def _connect(board: Optional[str] = None): + """Import + connect lazily so the module imports cleanly in non-kanban contexts. + + ``board=None`` keeps the legacy resolution chain (``HERMES_KANBAN_DB`` → + ``HERMES_KANBAN_BOARD`` → current symlink → ``default``); an explicit slug + lets e.g. a Telegram-side agent override the env-pinned board per call. """ from hermes_cli import kanban_db as kb return kb, kb.connect(board=board) +@contextmanager +def _board(board: Optional[str]): + """``with _board(slug) as (kb, conn)`` — connection closed on exit.""" + kb, conn = _connect(board=board) + try: + yield kb, conn + finally: + conn.close() + + +def _close_quietly(conn) -> None: + try: + conn.close() + except Exception: + pass + + +def _kanban_handler(tool_name: str) -> Callable: + """Wrap a handler so every failure is a structured tool error. + + ``ValueError`` (invalid board slug, DB validation such as cycle/self-link, + ``AttachmentTooLarge``) is reported without a traceback; anything else is + logged with ``logger.exception``. + """ + def deco(fn): + @functools.wraps(fn) + def wrapper(args: dict, **kw) -> str: + try: + return fn(args, **kw) + except ValueError as e: + return tool_error(f"{tool_name}: {e}") + except Exception as e: + logger.exception(f"{tool_name} failed") + return tool_error(f"{tool_name}: {e}") + return wrapper + return deco + + +def _ok(**fields: Any) -> str: + return json.dumps({"ok": True, **fields}) + + +def _redact(value: Any) -> str: + return redact_sensitive_text(str(value), force=True) + + +def _redact_metadata(metadata: dict) -> Optional[dict]: + """Redact a metadata dict via a JSON round-trip; None if it can't be re-parsed.""" + try: + return json.loads(redact_sensitive_text(json.dumps(metadata), force=True)) + except json.JSONDecodeError: + return None + + +def _coerce_str_list( + value: Any, name: str, what: str, *, strip: bool = False +) -> tuple[Any, Optional[str]]: + """Accept a single string (convenience) or a list/tuple; ``(value, error)``. + + With ``strip`` the items are stringified, stripped, and empties dropped. + """ + if value is None: + return None, None + if isinstance(value, str): + value = [value] + if not isinstance(value, (list, tuple)): + return None, tool_error( + f"{name} must be a list of {what}, got {type(value).__name__}" + ) + if strip: + value = [str(x).strip() for x in value if str(x).strip()] + return value, None + + +def _parse_bool_arg(args: dict, name: str, *, default: bool = False): + value = args.get(name) + if value is None: + return default, None + if isinstance(value, bool): + return value, None + text = str(value).strip().lower() + if text in {"true", "1", "yes"}: + return True, None + if text in {"false", "0", "no"}: + return False, None + return default, f"{name} must be a boolean or 'true'/'false'" + + +def _require_orchestrator_tool(tool_name: str) -> Optional[str]: + """Runtime guard for orchestrator-only handlers. + + The check_fn already hides these from the worker schema; this catches a + stale registration or test harness routing a worker here anyway. + """ + if os.environ.get("HERMES_KANBAN_TASK"): + return tool_error( + f"{tool_name} is orchestrator-only; dispatcher-spawned workers " + "must use kanban_complete, kanban_block, kanban_heartbeat, or " + "kanban_comment for their assigned task." + ) + return None + + +_TASK_FIELDS = ( + "id", "title", "body", "assignee", "status", "tenant", "priority", + "workspace_kind", "workspace_path", "created_by", "created_at", + "started_at", "completed_at", "result", "current_run_id", + "model_override", "provider_override", +) +_TASK_SUMMARY_FIELDS = ( + "id", "title", "assignee", "status", "priority", "tenant", + "workspace_kind", "workspace_path", "project_id", "created_by", + "created_at", "started_at", "completed_at", "current_run_id", + "model_override", "provider_override", +) +_RUN_FIELDS = ( + "id", "profile", "status", "outcome", "summary", "error", "metadata", + "started_at", "ended_at", +) +_COMMENT_FIELDS = ("author", "body", "created_at") +_EVENT_FIELDS = ("kind", "payload", "created_at", "run_id") +_ATTACHMENT_FIELDS = ( + "id", "filename", "content_type", "size", "uploaded_by", "stored_path", + "created_at", +) + + +def _fields(obj: Any, names: tuple[str, ...]) -> dict[str, Any]: + return {n: getattr(obj, n) for n in names} + + +def _task_summary_dict(kb, conn, task) -> dict[str, Any]: + """Compact task shape for board-listing tools.""" + parents = kb.parent_ids(conn, task.id) + children = kb.child_ids(conn, task.id) + return { + **_fields(task, _TASK_SUMMARY_FIELDS), + "parents": parents, + "children": children, + "parent_count": len(parents), + "child_count": len(children), + } + + +# --------------------------------------------------------------------------- +# Goal-mode judge gate +# --------------------------------------------------------------------------- + _GOAL_MODE_BLOCK_ALLOWED_KINDS = frozenset({"dependency", "needs_input"}) def _goal_judge_available() -> bool: """True when an auxiliary client is configured for the goal judge. - ``judge_goal`` is fail-open at the source: when no auxiliary model can - be reached it returns a ``"continue"`` verdict that is indistinguishable - from a real "not done yet" judgment. The completion gate must not treat - that as a rejection, or an unconfigured/degraded auxiliary model would - wedge every ``goal_mode`` worker (it could never close its own task). - - So we probe availability first and only enforce the gate when a judge is - actually reachable. This mirrors the same client lookup ``judge_goal`` - performs internally. + ``judge_goal`` fails open: with no reachable auxiliary model it returns + ``"continue"``, indistinguishable from a real "not done yet". Treating that + as a rejection would wedge every goal_mode worker, so the completion gate + is enforced only when a judge is actually reachable (same client lookup + ``judge_goal`` performs internally). """ try: from agent.auxiliary_client import get_text_auxiliary_client @@ -254,10 +385,9 @@ def _goal_judge_available() -> bool: def _goal_mode_handoff_rejection(task, evidence: str): """Return ``(verdict, reason_or_None)`` for a goal-mode terminal handoff. - ``{"done", None}`` means the judge allows the handoff; anything else is - a rejection whose verdict disambiguates the guidance the caller gives - the worker (``continue`` = not done yet, ``blocked`` = judged - unachievable — see #100954). + ``("done", None)`` allows the handoff. Otherwise the verdict picks the + guidance: ``continue`` = not done yet, ``blocked`` = judged unachievable. + A broken judge fails open (logged) so it cannot permanently wedge work. """ if not task or not task.goal_mode or not _goal_judge_available(): return ("done", None) @@ -269,8 +399,6 @@ def _goal_mode_handoff_rejection(task, evidence: str): last_response=evidence.strip(), ) except Exception as judge_exc: - # Keep the existing fail-open semantics: an unavailable/broken - # auxiliary judge must not permanently wedge goal-mode work. logger.warning( "goal judge check failed, allowing lifecycle handoff: %s", judge_exc, @@ -280,50 +408,30 @@ def _goal_mode_handoff_rejection(task, evidence: str): # --------------------------------------------------------------------------- -# Runtime-activity → board-heartbeat bridge (#31752) +# Runtime-activity → board bridges (auto-heartbeat, live comment injection) # --------------------------------------------------------------------------- -# When the agent ticks ``_touch_activity`` during normal work (between -# tool calls, mid-stream chunks, etc.), we want the kanban board's -# ``last_heartbeat_at`` columns to reflect that liveness so the dispatcher -# watchdog (which reads ``tasks.last_heartbeat_at``, not the agent's -# in-process timestamp) doesn't reclaim an actively-running worker as -# stale. The model is not required to call the explicit ``kanban_heartbeat`` -# tool for this to work — that tool stays available for workers that want -# to attach a note or pre-emptively extend a claim across a known-long op. -# -# Constraints: -# - Best-effort: never raise. The agent loop must not care if the bridge -# fails (board missing, DB locked, etc.). -# - Rate-limited to one DB write per 60s per-process; runtime activity -# can tick on every chunk/tool result and we don't need that resolution. -# - No-op outside dispatcher-spawned worker context (no ``HERMES_KANBAN_TASK``). -# - No durable note on these auto-heartbeats; that's reserved for the -# explicit tool which carries a model-supplied note. +# The dispatcher watchdog reads ``tasks.last_heartbeat_at``, not the agent's +# in-process activity timestamp, so normal work (tool calls, stream chunks) is +# mirrored onto the board here; the explicit ``kanban_heartbeat`` tool stays +# for attaching a note or pre-extending a claim across a known-long op. +# Constraints: best-effort (never raise into the agent loop), rate-limited +# per process, no-op outside dispatcher-spawned worker context, no durable +# note on auto-heartbeats. _AUTO_HEARTBEAT_MIN_INTERVAL_SECONDS = 60.0 _auto_heartbeat_last_attempt: float = 0.0 def heartbeat_current_worker_from_env() -> bool: - """Best-effort: extend the kanban claim + bump board heartbeat for the - current dispatcher-spawned worker, using identity from env vars. + """Best-effort: extend the claim + bump board heartbeat for the current worker. - Returns True if a write was attempted (whether or not it succeeded); - False if the call was skipped (not a kanban worker, rate-limited, or - swallowed exception). The boolean is informational — callers should - not branch on it. - - Identity comes from: - * ``HERMES_KANBAN_TASK`` — task id (required; absence means no-op) - * ``HERMES_KANBAN_RUN_ID`` — pins the run row so we don't heartbeat - a stale run that may have already been reclaimed - * ``HERMES_KANBAN_CLAIM_LOCK`` — claim lock for ``heartbeat_claim``; - falls back to the default ``_claimer_id()`` for locally-driven - workers that never went through the dispatcher path - - Rate-limited via the module-level ``_auto_heartbeat_last_attempt`` - timestamp (monotonic clock); not thread-safe in the strict sense, but - the worst case is one extra DB write per race, which is harmless. + Returns True if a write was attempted, False if skipped (not a worker, + rate-limited, or failed) — informational only. Identity from env: + ``HERMES_KANBAN_TASK`` (required), ``HERMES_KANBAN_RUN_ID`` (pins the run + row so a reclaimed stale run is not heartbeated), ``HERMES_KANBAN_CLAIM_LOCK`` + (falls back to the default claimer for locally-driven workers). The + monotonic rate limit is not strictly thread-safe; a race costs one extra + harmless DB write. """ global _auto_heartbeat_last_attempt tid = os.environ.get("HERMES_KANBAN_TASK") @@ -337,26 +445,16 @@ def heartbeat_current_worker_from_env() -> bool: try: kb, conn = _connect() try: - claim_lock = os.environ.get("HERMES_KANBAN_CLAIM_LOCK") try: - kb.heartbeat_claim(conn, tid, claimer=claim_lock) + kb.heartbeat_claim(conn, tid, claimer=os.environ.get("HERMES_KANBAN_CLAIM_LOCK")) except Exception: logger.debug("auto-heartbeat: heartbeat_claim failed", exc_info=True) - run_id_raw = os.environ.get("HERMES_KANBAN_RUN_ID") - run_id: Optional[int] try: - run_id = int(run_id_raw) if run_id_raw else None - except (TypeError, ValueError): - run_id = None - try: - kb.heartbeat_worker(conn, tid, note=None, expected_run_id=run_id) + kb.heartbeat_worker(conn, tid, note=None, expected_run_id=_worker_run_id(tid)) except Exception: logger.debug("auto-heartbeat: heartbeat_worker failed", exc_info=True) finally: - try: - conn.close() - except Exception: - pass + _close_quietly(conn) return True except Exception: logger.debug("auto-heartbeat: bridge failed", exc_info=True) @@ -364,10 +462,9 @@ def heartbeat_current_worker_from_env() -> bool: # Live operator-note injection: poll the worker's task for new comments and -# fold them into the running agent via the OUT-OF-BAND steer channel, so a user -# can "talk to" a running kanban task without the block → comment → unblock -# dance (or a restart). Rate-limited on its own (tighter than the 60s heartbeat -# so notes land within a few seconds), watermarked per task id. +# fold them in via the OUT-OF-BAND steer channel, so a user can talk to a +# running task without block → comment → unblock (or a restart). Polled +# tighter than the heartbeat so notes land within seconds; watermarked per task. _COMMENT_POLL_MIN_INTERVAL_SECONDS = 6.0 _comment_poll_last_attempt: float = 0.0 # task_id -> highest comment id already seen (seeded on first poll so history @@ -378,14 +475,10 @@ _comment_watermark: dict[str, int] = {} def inject_new_comments_from_env(agent: Any) -> bool: """Fold new operator comments on the current worker's task into ``agent``. - Best-effort and self-gating: no-op unless this process is a kanban worker - (``HERMES_KANBAN_TASK`` set) and ``agent`` exposes ``steer``. Returns True - if a steer was injected, else False. Never raises into the agent loop. - - The first poll only *seeds* the watermark to the newest existing comment — - those are already in the worker's context — so only comments added after - the run started are injected. The worker's own authored comments (matched - by ``HERMES_PROFILE``) are skipped to avoid echoing itself. + Self-gating no-op unless ``HERMES_KANBAN_TASK`` is set and ``agent`` exposes + ``steer``; returns True iff a steer was injected; never raises. The first + poll only seeds the watermark (those comments are already in context), and + the worker's own comments (matched by ``HERMES_PROFILE``) are skipped. """ tid = os.environ.get("HERMES_KANBAN_TASK") if not tid or agent is None or not hasattr(agent, "steer"): @@ -403,23 +496,17 @@ def inject_new_comments_from_env(agent: Any) -> bool: try: rows = kb.list_comments_after(conn, tid, after_id=seen or 0) finally: - try: - conn.close() - except Exception: - pass + _close_quietly(conn) except Exception: logger.debug("comment-inject: bridge failed", exc_info=True) return False if seen is None: - # First poll for this task: seed past the existing thread, inject nothing. _comment_watermark[tid] = max((c.id for c in rows), default=0) return False if not rows: return False - - # Advance the watermark past everything we just read (including our own - # notes) so nothing is re-injected next poll. + # Advance past everything read (including our own notes) so nothing is re-injected. _comment_watermark[tid] = max(c.id for c in rows) own = (os.environ.get("HERMES_PROFILE") or "").strip() @@ -442,169 +529,39 @@ def inject_new_comments_from_env(agent: Any) -> bool: return False -def _ok(**fields: Any) -> str: - return json.dumps({"ok": True, **fields}) - - -def _normalize_profile(value: Any) -> Optional[str]: - """Normalize CLI-compatible assignee sentinels for the tool surface.""" - if value is None: - return None - text = str(value).strip() - if not text or text.lower() in {"none", "-", "null"}: - return None - return text - - -def _parse_bool_arg(args: dict, name: str, *, default: bool = False): - value = args.get(name) - if value is None: - return default, None - if isinstance(value, bool): - return value, None - text = str(value).strip().lower() - if text in {"true", "1", "yes"}: - return True, None - if text in {"false", "0", "no"}: - return False, None - return default, f"{name} must be a boolean or 'true'/'false'" - - -def _require_orchestrator_tool(tool_name: str) -> Optional[str]: - """Belt-and-suspenders runtime guard for orchestrator-only handlers. - - The check_fn (`_check_kanban_orchestrator_mode`) keeps these tools - out of the worker schema entirely, but in case a stale registration - or test harness routes a worker to one of them anyway, return a - structured tool_error so the model gets a clear refusal instead of - silently mutating board state from a worker context. - """ - if os.environ.get("HERMES_KANBAN_TASK"): - return tool_error( - f"{tool_name} is orchestrator-only; dispatcher-spawned workers " - "must use kanban_complete, kanban_block, kanban_heartbeat, or " - "kanban_comment for their assigned task." - ) - return None - - -def _task_summary_dict(kb, conn, task) -> dict[str, Any]: - """Compact task shape for board-listing tools.""" - parents = kb.parent_ids(conn, task.id) - children = kb.child_ids(conn, task.id) - return { - "id": task.id, - "title": task.title, - "assignee": task.assignee, - "status": task.status, - "priority": task.priority, - "tenant": task.tenant, - "workspace_kind": task.workspace_kind, - "workspace_path": task.workspace_path, - "project_id": task.project_id, - "created_by": task.created_by, - "created_at": task.created_at, - "started_at": task.started_at, - "completed_at": task.completed_at, - "current_run_id": task.current_run_id, - "model_override": task.model_override, - "provider_override": task.provider_override, - "parents": parents, - "children": children, - "parent_count": len(parents), - "child_count": len(children), - } - - # --------------------------------------------------------------------------- # Handlers # --------------------------------------------------------------------------- +@_kanban_handler("kanban_show") def _handle_show(args: dict, **kw) -> str: - """Read a task's full state: task row, parents, children, comments, - runs (attempt history), and the last N events.""" + """Read a task's full state: row, parents, children, comments, runs, last 50 events.""" tid = _default_task_id(args.get("task_id")) if not tid: - return tool_error( - "task_id is required (or set HERMES_KANBAN_TASK in the env)" - ) - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - task = kb.get_task(conn, tid) - if task is None: - return tool_error(f"task {tid} not found") - comments = kb.list_comments(conn, tid) - events = kb.list_events(conn, tid) - runs = kb.list_runs(conn, tid) - parents = kb.parent_ids(conn, tid) - children = kb.child_ids(conn, tid) - - def _task_dict(t): - return { - "id": t.id, "title": t.title, "body": t.body, - "assignee": t.assignee, "status": t.status, - "tenant": t.tenant, "priority": t.priority, - "workspace_kind": t.workspace_kind, - "workspace_path": t.workspace_path, - "created_by": t.created_by, "created_at": t.created_at, - "started_at": t.started_at, - "completed_at": t.completed_at, - "result": t.result, - "current_run_id": t.current_run_id, - "model_override": t.model_override, - "provider_override": t.provider_override, - } - - def _run_dict(r): - return { - "id": r.id, "profile": r.profile, - "status": r.status, "outcome": r.outcome, - "summary": r.summary, "error": r.error, - "metadata": r.metadata, - "started_at": r.started_at, "ended_at": r.ended_at, - } - - return json.dumps({ - "task": _task_dict(task), - "parents": parents, - "children": children, - "comments": [ - {"author": c.author, "body": c.body, - "created_at": c.created_at} - for c in comments - ], - "events": [ - {"kind": e.kind, "payload": e.payload, - "created_at": e.created_at, "run_id": e.run_id} - for e in events[-50:] # cap; full log via CLI - ], - "runs": [_run_dict(r) for r in runs], - # Also surface the worker's own context block so the - # agent can include it directly if it wants. This is - # the same string build_worker_context returns to the - # dispatcher at spawn time. - "worker_context": kb.build_worker_context(conn, tid), - }) - finally: - conn.close() - except ValueError as e: - # Invalid board slug surfaces as ValueError from _normalize_board_slug. - return tool_error(f"kanban_show: {e}") - except Exception as e: - logger.exception("kanban_show failed") - return tool_error(f"kanban_show: {e}") + return tool_error(_TASK_ID_REQUIRED) + with _board(args.get("board")) as (kb, conn): + task = kb.get_task(conn, tid) + if task is None: + return tool_error(f"task {tid} not found") + return json.dumps({ + "task": _fields(task, _TASK_FIELDS), + "parents": kb.parent_ids(conn, tid), + "children": kb.child_ids(conn, tid), + "comments": [_fields(c, _COMMENT_FIELDS) for c in kb.list_comments(conn, tid)], + # Capped; full log via CLI. + "events": [_fields(e, _EVENT_FIELDS) for e in kb.list_events(conn, tid)[-50:]], + "runs": [_fields(r, _RUN_FIELDS) for r in kb.list_runs(conn, tid)], + # Same string build_worker_context hands the dispatcher at spawn time. + "worker_context": kb.build_worker_context(conn, tid), + }) +@_kanban_handler("kanban_list") def _handle_list(args: dict, **kw) -> str: """List task summaries with the same core filters as the CLI.""" guard = _require_orchestrator_tool("kanban_list") if guard: return guard - assignee = args.get("assignee") - status = args.get("status") - tenant = args.get("tenant") include_archived, bool_error = _parse_bool_arg(args, "include_archived") if bool_error: return tool_error(bool_error) @@ -619,258 +576,166 @@ def _handle_list(args: dict, **kw) -> str: return tool_error("limit must be >= 1") if limit > KANBAN_LIST_MAX_LIMIT: return tool_error(f"limit must be <= {KANBAN_LIST_MAX_LIMIT}") - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - # Match CLI list: dependencies that cleared since the last - # dispatcher tick should be visible to orchestrators immediately. - promoted = kb.recompute_ready(conn) - # Fetch one extra row so model-facing output can report that - # a bounded listing was truncated without dumping the board. - rows = kb.list_tasks( - conn, - assignee=assignee, - status=status, - tenant=tenant, - include_archived=include_archived, - limit=limit + 1, - ) - truncated = len(rows) > limit - tasks = rows[:limit] - return json.dumps({ - "tasks": [_task_summary_dict(kb, conn, t) for t in tasks], - "count": len(tasks), - "limit": limit, - "truncated": truncated, - "next_limit": ( - min(limit * 2, KANBAN_LIST_MAX_LIMIT) - if truncated and limit < KANBAN_LIST_MAX_LIMIT else None - ), - "promoted": promoted, - }) - finally: - conn.close() - except ValueError as e: - return tool_error(f"kanban_list: {e}") - except Exception as e: - logger.exception("kanban_list failed") - return tool_error(f"kanban_list: {e}") + with _board(args.get("board")) as (kb, conn): + # Match CLI list: dependencies cleared since the last dispatcher tick + # should be visible to orchestrators immediately. + promoted = kb.recompute_ready(conn) + # One extra row lets the output report truncation without dumping the board. + rows = kb.list_tasks( + conn, + assignee=args.get("assignee"), + status=args.get("status"), + tenant=args.get("tenant"), + include_archived=include_archived, + limit=limit + 1, + ) + truncated = len(rows) > limit + tasks = rows[:limit] + return json.dumps({ + "tasks": [_task_summary_dict(kb, conn, t) for t in tasks], + "count": len(tasks), + "limit": limit, + "truncated": truncated, + "next_limit": ( + min(limit * 2, KANBAN_LIST_MAX_LIMIT) + if truncated and limit < KANBAN_LIST_MAX_LIMIT else None + ), + "promoted": promoted, + }) +@_kanban_handler("kanban_complete") def _handle_complete(args: dict, **kw) -> str: """Mark the current task done with a structured handoff.""" - delegated_err = _reject_delegated_child_mutation("kanban_complete") - if delegated_err: - return delegated_err - tid = _default_task_id(args.get("task_id")) - if not tid: - return tool_error( - "task_id is required (or set HERMES_KANBAN_TASK in the env)" - ) - ownership_err = _enforce_worker_task_ownership(tid) - if ownership_err: - return ownership_err + tid, err = _worker_guard("kanban_complete", args) + if err: + return err summary = args.get("summary") metadata = args.get("metadata") result = args.get("result") if summary: - summary = redact_sensitive_text(str(summary), force=True) + summary = _redact(summary) if result: - result = redact_sensitive_text(str(result), force=True) - if metadata is not None and isinstance(metadata, dict): - meta_json = json.dumps(metadata) - meta_json = redact_sensitive_text(meta_json, force=True) - try: - metadata = json.loads(meta_json) - except json.JSONDecodeError: - pass - created_cards = args.get("created_cards") - artifacts = args.get("artifacts") - if created_cards is not None: - if isinstance(created_cards, str): - # Accept a single id as a string for convenience. - created_cards = [created_cards] - if not isinstance(created_cards, (list, tuple)): + result = _redact(result) + if isinstance(metadata, dict): + # Keep the unredacted dict if the redacted JSON cannot be re-parsed. + redacted = _redact_metadata(metadata) + if redacted is not None: + metadata = redacted + created_cards, err = _coerce_str_list( + args.get("created_cards"), "created_cards", "task ids", strip=True + ) + if err: + return err + artifacts, err = _coerce_str_list( + args.get("artifacts"), "artifacts", "file paths", strip=True + ) + if err: + return err + if artifacts: + # Artifacts ride inside metadata so the completed-event payload needs + # no DB schema change; the gateway notifier reads payload['artifacts'] + # and uploads each path as a native attachment. Merge with (never + # overwrite) a metadata.artifacts the worker passed manually. + if metadata is None: + metadata = {} + elif not isinstance(metadata, dict): return tool_error( - f"created_cards must be a list of task ids, got " - f"{type(created_cards).__name__}" + f"metadata must be an object/dict, got {type(metadata).__name__}" ) - # Normalise: strings only, stripped, non-empty. - created_cards = [ - str(c).strip() for c in created_cards if str(c).strip() - ] - if artifacts is not None: - if isinstance(artifacts, str): - # Accept a single path as a string for convenience. - artifacts = [artifacts] - if not isinstance(artifacts, (list, tuple)): - return tool_error( - f"artifacts must be a list of file paths, got " - f"{type(artifacts).__name__}" - ) - artifacts = [ - str(p).strip() for p in artifacts if str(p).strip() - ] - # Carry the artifact list inside metadata so it rides the - # existing completed-event payload without a schema change at - # the DB layer. The gateway notifier reads payload['artifacts'] - # off the completion event and uploads each path as a native - # attachment. - if artifacts: - if metadata is None: - metadata = {} - elif not isinstance(metadata, dict): - return tool_error( - f"metadata must be an object/dict, got " - f"{type(metadata).__name__}" - ) - # Don't overwrite an existing metadata.artifacts the worker - # passed manually — merge instead. - existing = metadata.get("artifacts") - if isinstance(existing, (list, tuple)): - merged: list[str] = [] - seen: set[str] = set() - for item in list(existing) + artifacts: - s = str(item).strip() - if s and s not in seen: - seen.add(s) - merged.append(s) - metadata["artifacts"] = merged - else: - metadata["artifacts"] = artifacts + existing = metadata.get("artifacts") + if isinstance(existing, (list, tuple)): + merged = (str(item).strip() for item in [*existing, *artifacts]) + metadata["artifacts"] = list(dict.fromkeys(s for s in merged if s)) + else: + metadata["artifacts"] = artifacts if not (summary or result): - return tool_error( - "provide at least one of: summary (preferred), result" - ) + return tool_error("provide at least one of: summary (preferred), result") if metadata is not None and not isinstance(metadata, dict): return tool_error( f"metadata must be an object/dict, got {type(metadata).__name__}" ) metadata = _stamp_worker_session_metadata(tid, metadata) - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - # Goal-mode pre-completion judge gate (Issue #38367). - # Prevent workers from bypassing the auxiliary judge by - # calling kanban_complete before acceptance criteria are met. - # Only enforce when a judge is actually reachable — see - # _goal_judge_available for why an unavailable judge fails open. - task = kb.get_task(conn, tid) - gate_verdict, rejection = _goal_mode_handoff_rejection( - task, - (summary or result or "").strip(), + with _board(args.get("board")) as (kb, conn): + # Goal-mode pre-completion judge gate: a worker must not bypass the + # auxiliary judge by completing before acceptance criteria are met. + task = kb.get_task(conn, tid) + gate_verdict, rejection = _goal_mode_handoff_rejection( + task, (summary or result or "").strip() + ) + if gate_verdict == "blocked": + return tool_error( + f"Goal completion rejected: judge ruled the goal " + f"unachievable — {rejection}. The task will NOT complete " + f"silently. Either re-scope the task with kanban_edit, " + f"or record the block with kanban_block and hand the " + f"decision to a human / reviewer." ) - if gate_verdict == "blocked": - return tool_error( - f"Goal completion rejected: judge ruled the goal " - f"unachievable — {rejection}. The task will NOT complete " - f"silently. Either re-scope the task with kanban_edit, " - f"or record the block with kanban_block and hand the " - f"decision to a human / reviewer." - ) - if rejection is not None: - return tool_error( - f"Goal completion rejected by judge: {rejection}. " - f"To proceed, either: (1) provide explicit acceptance " - f"evidence in your summary matching the task's criteria, " - f"or (2) create continuation tasks with parents=[{tid}] " - f"and keep this task alive." - ) - - try: - ok = kb.complete_task( - conn, tid, - result=result, summary=summary, metadata=metadata, - created_cards=created_cards, - expected_run_id=_worker_run_id(tid), - ) - except kb.ArtifactPreservationError as artifact_err: - return tool_error( - f"kanban_complete could not preserve the declared artifacts: " - f"{artifact_err}. Your task is still in-flight and its " - f"scratch workspace was kept. Fix the artifact path or " - f"storage error, then retry kanban_complete with the same handoff." - ) - except kb.HallucinatedCardsError as hall_err: - # Structured rejection — surface the phantom ids so the - # worker can retry with a corrected list or drop the - # field. Audit event already landed in the DB. - # - # The task itself was NOT mutated (the gate runs before - # the write txn), so the worker can simply call - # kanban_complete again. Spell that out — without it the - # model often interprets a tool_error as a terminal - # failure and either blocks or crashes the run instead - # of retrying. See #22923. - return tool_error( - f"kanban_complete blocked: the following created_cards " - f"do not exist or were not created by this worker: " - f"{', '.join(hall_err.phantom)}. " - f"Your task is still in-flight (no state change). " - f"Retry kanban_complete with the same summary/metadata " - f"and either drop these ids from created_cards, or pass " - f"created_cards=[] to skip the card-claim check entirely." - ) - if not ok: - return tool_error( - f"could not complete {tid} (unknown id or already terminal)" - ) - run = kb.latest_run(conn, tid) - return _ok(task_id=tid, run_id=run.id if run else None) - finally: - conn.close() - except ValueError as e: - return tool_error(f"kanban_complete: {e}") - except Exception as e: - logger.exception("kanban_complete failed") - return tool_error(f"kanban_complete: {e}") + if rejection is not None: + return tool_error( + f"Goal completion rejected by judge: {rejection}. " + f"To proceed, either: (1) provide explicit acceptance " + f"evidence in your summary matching the task's criteria, " + f"or (2) create continuation tasks with parents=[{tid}] " + f"and keep this task alive." + ) + try: + ok = kb.complete_task( + conn, tid, + result=result, summary=summary, metadata=metadata, + created_cards=created_cards, + expected_run_id=_worker_run_id(tid), + ) + except kb.ArtifactPreservationError as artifact_err: + return tool_error( + f"kanban_complete could not preserve the declared artifacts: " + f"{artifact_err}. Your task is still in-flight and its " + f"scratch workspace was kept. Fix the artifact path or " + f"storage error, then retry kanban_complete with the same handoff." + ) + except kb.HallucinatedCardsError as hall_err: + # The gate runs before the write txn, so the task was NOT mutated; + # say so explicitly or the model treats the error as terminal and + # blocks/crashes instead of retrying. Audit event already landed. + return tool_error( + f"kanban_complete blocked: the following created_cards " + f"do not exist or were not created by this worker: " + f"{', '.join(hall_err.phantom)}. " + f"Your task is still in-flight (no state change). " + f"Retry kanban_complete with the same summary/metadata " + f"and either drop these ids from created_cards, or pass " + f"created_cards=[] to skip the card-claim check entirely." + ) + if not ok: + return tool_error( + f"could not complete {tid} (unknown id or already terminal)" + ) + run = kb.latest_run(conn, tid) + return _ok(task_id=tid, run_id=run.id if run else None) +@_kanban_handler("kanban_block") def _handle_block(args: dict, **kw) -> str: """Transition the task to blocked with a reason a human will read.""" - delegated_err = _reject_delegated_child_mutation("kanban_block") - if delegated_err: - return delegated_err - tid = _default_task_id(args.get("task_id")) - if not tid: - return tool_error( - "task_id is required (or set HERMES_KANBAN_TASK in the env)" - ) - ownership_err = _enforce_worker_task_ownership(tid) - if ownership_err: - return ownership_err + tid, err = _worker_guard("kanban_block", args) + if err: + return err reason = args.get("reason") if not reason or not str(reason).strip(): return tool_error("reason is required — explain what input you need") - reason = redact_sensitive_text(str(reason), force=True) + reason = _redact(reason) kind = args.get("kind") - board = args.get("board") - try: - kb, conn = _connect(board=board) + with _board(args.get("board")) as (kb, conn): if kind is not None and kind not in kb.VALID_BLOCK_KINDS: - conn.close() return tool_error( f"kind must be one of {sorted(kb.VALID_BLOCK_KINDS)} (or omit it)" ) - # Goal-mode block gate (Issue #38696, sibling of the kanban_complete - # judge gate in #38367). kanban_block is a second exit path out of - # the goal loop — run_kanban_goal_loop() treats ANY `blocked` status - # as terminal, identically to `done`, regardless of kind. Without - # this, a worker that learns kanban_complete is gated can just call - # kanban_block(reason="anything") to escape the loop instead. - # Restrict goal_mode tasks to the kinds that represent a genuine - # external blocker the worker cannot resolve itself; `capability` - # and `transient` (or an unset kind) route back through - # kanban_complete, which the judge now gates. + # Goal-mode block gate: the goal loop treats ANY blocked status as + # terminal, so kanban_block would be an escape hatch around the + # completion judge. Restrict goal_mode tasks to kinds that are genuine + # external blockers; everything else routes back through kanban_complete. task = kb.get_task(conn, tid) - if ( - task - and task.goal_mode - and kind not in _GOAL_MODE_BLOCK_ALLOWED_KINDS - ): - conn.close() + if task and task.goal_mode and kind not in _GOAL_MODE_BLOCK_ALLOWED_KINDS: return tool_error( f"goal_mode tasks can only block with kind in " f"{sorted(_GOAL_MODE_BLOCK_ALLOWED_KINDS)} (got {kind!r}). " @@ -878,223 +743,139 @@ def _handle_block(args: dict, **kw) -> str: f"another reason, call kanban_complete instead — the " f"completion judge will evaluate it." ) - try: - ok = kb.block_task( - conn, tid, - reason=reason, - kind=kind, - expected_run_id=_worker_run_id(tid), + ok = kb.block_task( + conn, tid, reason=reason, kind=kind, expected_run_id=_worker_run_id(tid), + ) + if not ok: + return tool_error( + f"could not block {tid} (unknown id or not in running/ready)" ) - if not ok: - return tool_error( - f"could not block {tid} (unknown id or not in " - f"running/ready)" - ) - run = kb.latest_run(conn, tid) - # Tell the worker where the task actually landed so it doesn't - # assume it's sitting in 'blocked' when routing sent it elsewhere. - landed = kb.get_task(conn, tid) - return _ok( - task_id=tid, - run_id=run.id if run else None, - status=landed.status if landed else "blocked", - block_kind=kind, - ) - finally: - conn.close() - except ValueError as e: - return tool_error(f"kanban_block: {e}") - except Exception as e: - logger.exception("kanban_block failed") - return tool_error(f"kanban_block: {e}") + run = kb.latest_run(conn, tid) + # Report where the task actually landed; routing may not leave it in 'blocked'. + landed = kb.get_task(conn, tid) + return _ok( + task_id=tid, + run_id=run.id if run else None, + status=landed.status if landed else "blocked", + block_kind=kind, + ) +@_kanban_handler("kanban_request_review") def _handle_request_review(args: dict, **kw) -> str: """Move implementation into the first-class review phase.""" - delegated_err = _reject_delegated_child_mutation("kanban_request_review") - if delegated_err: - return delegated_err - tid = _default_task_id(args.get("task_id")) - if not tid: - return tool_error( - "task_id is required (or set HERMES_KANBAN_TASK in the env)" - ) - ownership_err = _enforce_worker_task_ownership(tid) - if ownership_err: - return ownership_err + tid, err = _worker_guard("kanban_request_review", args) + if err: + return err summary = args.get("summary") if not summary or not str(summary).strip(): return tool_error( "summary is required — describe what was implemented and how it " "was verified so the reviewer has context" ) - summary = redact_sensitive_text(str(summary), force=True) + summary = _redact(summary) metadata = args.get("metadata") if metadata is not None and not isinstance(metadata, dict): return tool_error( f"metadata must be an object/dict, got {type(metadata).__name__}" ) if metadata is not None: - metadata_json = redact_sensitive_text(json.dumps(metadata), force=True) - try: - metadata = json.loads(metadata_json) - except json.JSONDecodeError: + metadata = _redact_metadata(metadata) + if metadata is None: return tool_error("metadata could not be safely serialized") metadata = _stamp_worker_session_metadata(tid, metadata) reviewer = args.get("reviewer") or None if reviewer: - # Model-supplied free text stored durably on the event payload — - # redact like summary / kanban_block's reason. - reviewer = redact_sensitive_text(str(reviewer), force=True) - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - task = kb.get_task(conn, tid) - gate_verdict, rejection = _goal_mode_handoff_rejection(task, summary) - if gate_verdict == "blocked": - return tool_error( - f"Goal review handoff rejected: judge ruled the goal " - f"unachievable — {rejection}. Record the block with " - f"kanban_block instead of requesting review." - ) - if rejection is not None: - return tool_error( - f"Goal review handoff rejected by judge: {rejection}. " - "Provide acceptance evidence matching the card before " - "requesting review." - ) - ok, fail_reason = kb.request_review( - conn, tid, - summary=summary, - metadata=metadata, - reviewer=reviewer, - expected_run_id=_worker_run_id(tid), - with_reason=True, + # Model-supplied free text stored durably on the event payload. + reviewer = _redact(reviewer) + with _board(args.get("board")) as (kb, conn): + task = kb.get_task(conn, tid) + gate_verdict, rejection = _goal_mode_handoff_rejection(task, summary) + if gate_verdict == "blocked": + return tool_error( + f"Goal review handoff rejected: judge ruled the goal " + f"unachievable — {rejection}. Record the block with " + f"kanban_block instead of requesting review." ) - if not ok: - detail = fail_reason or "unknown id or not in running/ready" - return tool_error( - f"could not request review for {tid}: {detail}" - ) - run = kb.latest_run(conn, tid) - landed = kb.get_task(conn, tid) - return _ok( - task_id=tid, - run_id=run.id if run else None, - status=landed.status if landed else "review", + if rejection is not None: + return tool_error( + f"Goal review handoff rejected by judge: {rejection}. " + "Provide acceptance evidence matching the card before " + "requesting review." ) - finally: - conn.close() - except ValueError as e: - return tool_error(f"kanban_request_review: {e}") - except Exception as e: - logger.exception("kanban_request_review failed") - return tool_error(f"kanban_request_review: {e}") + ok, fail_reason = kb.request_review( + conn, tid, + summary=summary, + metadata=metadata, + reviewer=reviewer, + expected_run_id=_worker_run_id(tid), + with_reason=True, + ) + if not ok: + detail = fail_reason or "unknown id or not in running/ready" + return tool_error(f"could not request review for {tid}: {detail}") + run = kb.latest_run(conn, tid) + landed = kb.get_task(conn, tid) + return _ok( + task_id=tid, + run_id=run.id if run else None, + status=landed.status if landed else "review", + ) +@_kanban_handler("kanban_request_changes") def _handle_request_changes(args: dict, **kw) -> str: """Return a reviewer-owned running task to its implementer.""" - delegated_err = _reject_delegated_child_mutation("kanban_request_changes") - if delegated_err: - return delegated_err - tid = _default_task_id(args.get("task_id")) - if not tid: - return tool_error( - "task_id is required (or set HERMES_KANBAN_TASK in the env)" - ) - ownership_err = _enforce_worker_task_ownership(tid) - if ownership_err: - return ownership_err + tid, err = _worker_guard("kanban_request_changes", args) + if err: + return err reason = args.get("reason") if not reason or not str(reason).strip(): return tool_error("reason is required — describe the changes needed") - reason = redact_sensitive_text(str(reason), force=True) - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - ok, detail = kb.request_changes( - conn, - tid, - reason=reason, - expected_run_id=_worker_run_id(tid), - ) - if not ok: - return tool_error( - f"could not request changes for {tid}: {detail or 'invalid review state'}" - ) - landed = kb.get_task(conn, tid) - run = kb.latest_run(conn, tid) - return _ok( - task_id=tid, - run_id=run.id if run else None, - status=landed.status if landed else "ready", - implementer=detail, - ) - finally: - conn.close() - except ValueError as e: - return tool_error(f"kanban_request_changes: {e}") - except Exception as e: - logger.exception("kanban_request_changes failed") - return tool_error(f"kanban_request_changes: {e}") - - -def _handle_heartbeat(args: dict, **kw) -> str: - """Signal that the worker is still alive during a long operation. - - Extends the claim TTL via ``heartbeat_claim`` AND records a heartbeat - event via ``heartbeat_worker``. Without the ``heartbeat_claim`` half, - a diligent worker that loops this tool while a single tool call - blocks the agent for >DEFAULT_CLAIM_TTL_SECONDS still gets reclaimed - by ``release_stale_claims`` — which is exactly the trap that - ``heartbeat_claim``'s docstring warns against. - """ - delegated_err = _reject_delegated_child_mutation("kanban_heartbeat") - if delegated_err: - return delegated_err - tid = _default_task_id(args.get("task_id")) - if not tid: - return tool_error( - "task_id is required (or set HERMES_KANBAN_TASK in the env)" + reason = _redact(reason) + with _board(args.get("board")) as (kb, conn): + ok, detail = kb.request_changes( + conn, tid, reason=reason, expected_run_id=_worker_run_id(tid), ) - ownership_err = _enforce_worker_task_ownership(tid) - if ownership_err: - return ownership_err - note = args.get("note") - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - # Extend the claim TTL first. The dispatcher pins - # HERMES_KANBAN_CLAIM_LOCK in the worker env at spawn time - # (see _default_spawn in kanban_db.py); falling back to the - # default _claimer_id() covers locally-driven workers that - # never went through the dispatcher path. - claim_lock = os.environ.get("HERMES_KANBAN_CLAIM_LOCK") - kb.heartbeat_claim(conn, tid, claimer=claim_lock) - - ok = kb.heartbeat_worker( - conn, - tid, - note=note, - expected_run_id=_worker_run_id(tid), + if not ok: + return tool_error( + f"could not request changes for {tid}: {detail or 'invalid review state'}" ) - if not ok: - return tool_error( - f"could not heartbeat {tid} (unknown id or not running)" - ) - return _ok(task_id=tid) - finally: - conn.close() - except ValueError as e: - return tool_error(f"kanban_heartbeat: {e}") - except Exception as e: - logger.exception("kanban_heartbeat failed") - return tool_error(f"kanban_heartbeat: {e}") + landed = kb.get_task(conn, tid) + run = kb.latest_run(conn, tid) + return _ok( + task_id=tid, + run_id=run.id if run else None, + status=landed.status if landed else "ready", + implementer=detail, + ) +@_kanban_handler("kanban_heartbeat") +def _handle_heartbeat(args: dict, **kw) -> str: + """Signal liveness during a long operation. + + Extends the claim TTL (``heartbeat_claim``) AND records a heartbeat event + (``heartbeat_worker``). Without the claim half, a worker looping this tool + while one tool call blocks longer than the claim TTL still gets reclaimed + by ``release_stale_claims``. + """ + tid, err = _worker_guard("kanban_heartbeat", args) + if err: + return err + with _board(args.get("board")) as (kb, conn): + # The dispatcher pins HERMES_KANBAN_CLAIM_LOCK at spawn; the default + # claimer covers locally-driven workers that bypassed the dispatcher. + kb.heartbeat_claim(conn, tid, claimer=os.environ.get("HERMES_KANBAN_CLAIM_LOCK")) + ok = kb.heartbeat_worker( + conn, tid, note=args.get("note"), expected_run_id=_worker_run_id(tid), + ) + if not ok: + return tool_error(f"could not heartbeat {tid} (unknown id or not running)") + return _ok(task_id=tid) + + +@_kanban_handler("kanban_comment") def _handle_comment(args: dict, **kw) -> str: """Append a comment to a task's thread.""" delegated_err = _reject_delegated_child_mutation("kanban_comment") @@ -1109,53 +890,41 @@ def _handle_comment(args: dict, **kw) -> str: body = args.get("body") if not body or not str(body).strip(): return tool_error("body is required") - body = redact_sensitive_text(str(body), force=True) - # Author is intentionally derived from the worker's own runtime - # identity, NOT from caller-supplied args. Comments are injected - # into the next worker's system prompt by ``build_worker_context`` - # as ``**{author}** (timestamp): {body}`` — accepting an - # ``args["author"]`` override let a worker forge a comment from - # an authoritative-looking name like ``hermes-system`` and poison - # the future-worker context with what reads as a system directive. - # Cross-task commenting itself remains unrestricted (see #19713) — - # comments are the deliberate handoff channel between tasks. + body = _redact(body) + # Author comes from the worker's runtime identity, never caller args: + # comments are injected into future workers' system prompts as + # ``**{author}** (timestamp): {body}``, so an args["author"] override + # could forge a directive from an authoritative-looking name like + # ``hermes-system``. Cross-task commenting stays unrestricted — it is the + # deliberate handoff channel between tasks. author = os.environ.get("HERMES_PROFILE") or "worker" - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - cid = kb.add_comment(conn, tid, author=author, body=str(body)) - return _ok(task_id=tid, comment_id=cid) - finally: - conn.close() - except ValueError as e: - return tool_error(f"kanban_comment: {e}") - except Exception as e: - logger.exception("kanban_comment failed") - return tool_error(f"kanban_comment: {e}") + with _board(args.get("board")) as (kb, conn): + cid = kb.add_comment(conn, tid, author=author, body=str(body)) + return _ok(task_id=tid, comment_id=cid) +def _store_attachment(kb, board, tid, filename, data, content_type) -> str: + with _board(board) as (_, conn): + att_id = kb.store_attachment_bytes( + conn, tid, str(filename), data, + content_type=content_type, uploaded_by="agent", board=board, + ) + return _ok(task_id=tid, attachment_id=att_id, size=len(data)) + + +@_kanban_handler("kanban_attach") def _handle_attach(args: dict, **kw) -> str: """Attach an inline (base64) file to a task. - Mirrors the dashboard's upload endpoint for the agent surface: decode - the payload, enforce the shared size cap, write it under the per-task - attachments dir, and record the metadata row — all via - ``kanban_db.store_attachment_bytes`` so the three surfaces stay in lockstep. + Goes through ``kanban_db.store_attachment_bytes`` (decode, shared size cap, + per-task attachments dir, metadata row) so agent, dashboard, and CLI + surfaces stay in lockstep. """ from hermes_cli import kanban_db as kb - delegated_err = _reject_delegated_child_mutation("kanban_attach") - if delegated_err: - return delegated_err - tid = _default_task_id(args.get("task_id")) - if not tid: - return tool_error( - "task_id is required (or set HERMES_KANBAN_TASK in the env)" - ) - ownership_err = _enforce_worker_task_ownership(tid) - if ownership_err: - return ownership_err + tid, err = _worker_guard("kanban_attach", args) + if err: + return err filename = args.get("filename") if not filename or not str(filename).strip(): return tool_error("filename is required") @@ -1168,30 +937,7 @@ def _handle_attach(args: dict, **kw) -> str: data = base64.b64decode(str(content_b64), validate=True) except (binascii.Error, ValueError) as e: return tool_error(f"content_base64 is not valid base64: {e}") - content_type = args.get("content_type") - board = args.get("board") - try: - _, conn = _connect(board=board) - try: - att_id = kb.store_attachment_bytes( - conn, - tid, - str(filename), - data, - content_type=content_type, - uploaded_by="agent", - board=board, - ) - return _ok(task_id=tid, attachment_id=att_id, size=len(data)) - finally: - conn.close() - except kb.AttachmentTooLarge as e: - return tool_error(f"kanban_attach: {e}") - except ValueError as e: - return tool_error(f"kanban_attach: {e}") - except Exception as e: - logger.exception("kanban_attach failed") - return tool_error(f"kanban_attach: {e}") + return _store_attachment(kb, args.get("board"), tid, filename, data, args.get("content_type")) _MAX_ATTACH_URL_REDIRECTS = 5 @@ -1200,18 +946,13 @@ _MAX_ATTACH_URL_REDIRECTS = 5 def _download_url_with_cap(url: str, max_bytes: int) -> tuple[bytes, Optional[str]]: """Fetch ``url`` over http(s) with SSRF guarding, capped at ``max_bytes``. - Every hop — the initial URL and each redirect target — is validated with - ``tools.url_safety.is_safe_url`` before it is fetched, so a - model-controlled URL (or a public host 302ing to one) cannot reach - loopback, private/CGNAT ranges, or cloud metadata endpoints. Redirects - are followed manually (``follow_redirects=False``) so each Location is - re-checked, mirroring ``tools.skills_hub._guarded_http_get``. - - Returns ``(data, content_type)``. Raises ``ValueError`` for a non-http(s) - scheme, an SSRF-blocked target, too many redirects, or a body that - overruns the cap (the caller maps it to a clean tool error). Reads in - chunks so an oversize response is rejected without buffering the whole - thing. + Every hop (initial URL and each redirect target) is validated with + ``tools.url_safety.is_safe_url`` before fetching, so a model-controlled URL + (or a public host 302ing to one) cannot reach loopback, private/CGNAT + ranges, or cloud metadata. Redirects are followed manually so each + Location is re-checked. Returns ``(data, content_type)``; raises + ``ValueError`` for a bad scheme, blocked target, too many redirects, or a + body over the cap (checked while streaming, so nothing oversize is buffered). """ from urllib.parse import urljoin, urlparse @@ -1258,26 +999,14 @@ def _download_url_with_cap(url: str, max_bytes: int) -> tuple[bytes, Optional[st raise ValueError(f"too many redirects fetching {url}") +@_kanban_handler("kanban_attach_url") def _handle_attach_url(args: dict, **kw) -> str: - """Attach a file fetched server-side from a URL. - - The agent passes a URL; Hermes downloads it (with the shared size cap) - and stores it as a real attachment. Useful when the agent has a link - rather than the bytes. Only http/https URLs are accepted. - """ + """Attach a file fetched server-side from an http(s) URL (shared size cap).""" from hermes_cli import kanban_db as kb - delegated_err = _reject_delegated_child_mutation("kanban_attach_url") - if delegated_err: - return delegated_err - tid = _default_task_id(args.get("task_id")) - if not tid: - return tool_error( - "task_id is required (or set HERMES_KANBAN_TASK in the env)" - ) - ownership_err = _enforce_worker_task_ownership(tid) - if ownership_err: - return ownership_err + tid, err = _worker_guard("kanban_attach_url", args) + if err: + return err url = args.get("url") if not url or not str(url).strip(): return tool_error("url is required") @@ -1288,8 +1017,6 @@ def _handle_attach_url(args: dict, **kw) -> str: from urllib.parse import unquote, urlparse leaf = unquote(urlparse(url).path.rsplit("/", 1)[-1]).strip() filename = leaf or "download" - content_type = args.get("content_type") - board = args.get("board") try: data, fetched_ct = _download_url_with_cap(url, kb.KANBAN_ATTACHMENT_MAX_BYTES) except ValueError as e: @@ -1297,75 +1024,30 @@ def _handle_attach_url(args: dict, **kw) -> str: except Exception as e: logger.exception("kanban_attach_url download failed") return tool_error(f"kanban_attach_url: failed to fetch {url}: {e}") - try: - _, conn = _connect(board=board) - try: - att_id = kb.store_attachment_bytes( - conn, - tid, - str(filename), - data, - content_type=content_type or fetched_ct, - uploaded_by="agent", - board=board, - ) - return _ok(task_id=tid, attachment_id=att_id, size=len(data)) - finally: - conn.close() - except kb.AttachmentTooLarge as e: - return tool_error(f"kanban_attach_url: {e}") - except ValueError as e: - return tool_error(f"kanban_attach_url: {e}") - except Exception as e: - logger.exception("kanban_attach_url failed") - return tool_error(f"kanban_attach_url: {e}") + return _store_attachment( + kb, args.get("board"), tid, filename, data, args.get("content_type") or fetched_ct + ) +@_kanban_handler("kanban_attachments") def _handle_attachments(args: dict, **kw) -> str: """List a task's attachments (read-only; no ownership restriction).""" tid = _default_task_id(args.get("task_id")) if not tid: - return tool_error( - "task_id is required (or set HERMES_KANBAN_TASK in the env)" - ) - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - if kb.get_task(conn, tid) is None: - return tool_error(f"task {tid} not found") - atts = kb.list_attachments(conn, tid) - return json.dumps({ - "ok": True, - "task_id": tid, - "attachments": [ - { - "id": a.id, - "filename": a.filename, - "content_type": a.content_type, - "size": a.size, - "uploaded_by": a.uploaded_by, - "stored_path": a.stored_path, - "created_at": a.created_at, - } - for a in atts - ], - }) - finally: - conn.close() - except ValueError as e: - return tool_error(f"kanban_attachments: {e}") - except Exception as e: - logger.exception("kanban_attachments failed") - return tool_error(f"kanban_attachments: {e}") + return tool_error(_TASK_ID_REQUIRED) + with _board(args.get("board")) as (kb, conn): + if kb.get_task(conn, tid) is None: + return tool_error(f"task {tid} not found") + return json.dumps({ + "ok": True, + "task_id": tid, + "attachments": [_fields(a, _ATTACHMENT_FIELDS) for a in kb.list_attachments(conn, tid)], + }) +@_kanban_handler("kanban_create") def _handle_create(args: dict, **kw) -> str: - """Create a child task. Orchestrator workers use this to fan out. - - ``parents`` can be a list of task ids; dependency-gated promotion - works as usual. - """ + """Create a (child) task; orchestrator workers use this to fan out.""" delegated_err = _reject_delegated_child_mutation("kanban_create") if delegated_err: return delegated_err @@ -1381,13 +1063,10 @@ def _handle_create(args: dict, **kw) -> str: body = args.get("body") parents = args.get("parents") or [] tenant = args.get("tenant") or os.environ.get("HERMES_TENANT") - # Stamp the originating session id when the agent loop runs under - # ACP (which sets HERMES_SESSION_ID before invoking tools). NULL on - # CLI / dashboard paths and on legacy hosts that don't set the env. - # Prefer the request-scoped api_server origin binding: HERMES_SESSION_ID - # is clobbered with a subagent's internal id whenever a child agent is - # constructed in-process (agent_init calls set_current_session_id), which - # would stamp — and later wake — the wrong session. + # Prefer the request-scoped api_server origin binding over HERMES_SESSION_ID: + # the env var is clobbered with a subagent's internal id whenever a child + # agent is constructed in-process, which would stamp — and later wake — + # the wrong session. NULL on CLI/dashboard paths that set neither. from tools.async_delegation import _current_origin_session_id session_id = ( @@ -1396,14 +1075,11 @@ def _handle_create(args: dict, **kw) -> str: or os.environ.get("HERMES_SESSION_ID") ) priority = args.get("priority") - # Resolve workspace. Workspace sharing is always explicit: omitted fields - # mean a fresh scratch workspace, even when a dispatcher-spawned worker - # creates the task. Reusing a parent's literal path would let a child - # mutate review evidence or race the parent's checkout (#67567). - # - # Project identity is the one safe context to inherit implicitly. The DB - # resolves a project-linked scratch request into a fresh per-task worktree, - # preserving the repository/branch convention without sharing a checkout. + # Workspace sharing is always explicit: omitted fields mean a fresh scratch + # workspace even for a dispatcher-spawned creator — reusing the parent's + # literal path would let a child mutate review evidence or race its + # checkout. Project identity is the one safe thing to inherit implicitly + # (the DB turns it into a fresh per-task worktree). workspace_kind = args.get("workspace_kind") workspace_path = args.get("workspace_path") project_id = args.get("project") or args.get("project_id") @@ -1417,14 +1093,9 @@ def _handle_create(args: dict, **kw) -> str: idempotency_key = args.get("idempotency_key") max_runtime_seconds = args.get("max_runtime_seconds") initial_status = args.get("initial_status") or "running" - skills = args.get("skills") - if isinstance(skills, str): - # Accept a single skill name as a string for convenience. - skills = [skills] - if skills is not None and not isinstance(skills, (list, tuple)): - return tool_error( - f"skills must be a list of skill names, got {type(skills).__name__}" - ) + skills, err = _coerce_str_list(args.get("skills"), "skills", "skill names") + if err: + return err goal_mode, goal_bool_error = _parse_bool_arg(args, "goal_mode") if goal_bool_error: return tool_error(goal_bool_error) @@ -1433,119 +1104,82 @@ def _handle_create(args: dict, **kw) -> str: provider_override = args.get("provider") if provider_override and not model_override: return tool_error("'provider' requires 'model' to be set as well") - if isinstance(parents, str): - parents = [parents] - if not isinstance(parents, (list, tuple)): - return tool_error( - f"parents must be a list of task ids, got {type(parents).__name__}" + parents, err = _coerce_str_list(parents, "parents", "task ids") + if err: + return err + with _board(args.get("board")) as (kb, conn): + if _inherit_project and project_id is None: + _self_tid = os.environ.get("HERMES_KANBAN_TASK") + if _self_tid: + _self_task = kb.get_task(conn, _self_tid) + if _self_task is not None and _self_task.project_id: + project_id = _self_task.project_id + project_source_task_id = _self_task.id + new_tid = kb.create_task( + conn, + title=str(title).strip(), + body=body, + assignee=str(assignee), + parents=tuple(parents), + tenant=tenant, + priority=int(priority) if priority is not None else 0, + workspace_kind=str(workspace_kind), + workspace_path=workspace_path, + project_id=project_id, + project_source_task_id=project_source_task_id, + triage=triage, + idempotency_key=idempotency_key, + max_runtime_seconds=( + int(max_runtime_seconds) if max_runtime_seconds is not None else None + ), + skills=skills, + model_override=model_override, + provider_override=provider_override, + goal_mode=goal_mode, + goal_max_turns=int(goal_max_turns) if goal_max_turns is not None else None, + initial_status=str(initial_status), + created_by=os.environ.get("HERMES_PROFILE") or "worker", + session_id=session_id, + ) + new_task = kb.get_task(conn, new_tid) + subscribed = _maybe_auto_subscribe(conn, new_tid) + return _ok( + task_id=new_tid, + status=new_task.status if new_task else None, + workspace_kind=new_task.workspace_kind if new_task else None, + workspace_path=new_task.workspace_path if new_task else None, + project_id=new_task.project_id if new_task else None, + subscribed=subscribed, ) - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - # A project link is safe to inherit because ``create_task`` turns - # it into a fresh per-task worktree. Never inherit the parent's - # literal workspace kind/path; directory sharing must be explicit. - if _inherit_project and project_id is None: - _self_tid = os.environ.get("HERMES_KANBAN_TASK") - if _self_tid: - _self_task = kb.get_task(conn, _self_tid) - if _self_task is not None and _self_task.project_id: - project_id = _self_task.project_id - project_source_task_id = _self_task.id - new_tid = kb.create_task( - conn, - title=str(title).strip(), - body=body, - assignee=str(assignee), - parents=tuple(parents), - tenant=tenant, - priority=int(priority) if priority is not None else 0, - workspace_kind=str(workspace_kind), - workspace_path=workspace_path, - project_id=project_id, - project_source_task_id=project_source_task_id, - triage=triage, - idempotency_key=idempotency_key, - max_runtime_seconds=( - int(max_runtime_seconds) - if max_runtime_seconds is not None else None - ), - skills=skills, - model_override=model_override, - provider_override=provider_override, - goal_mode=goal_mode, - goal_max_turns=( - int(goal_max_turns) if goal_max_turns is not None else None - ), - initial_status=str(initial_status), - created_by=os.environ.get("HERMES_PROFILE") or "worker", - session_id=session_id, - ) - new_task = kb.get_task(conn, new_tid) - subscribed = _maybe_auto_subscribe(conn, new_tid) - return _ok( - task_id=new_tid, - status=new_task.status if new_task else None, - workspace_kind=new_task.workspace_kind if new_task else None, - workspace_path=new_task.workspace_path if new_task else None, - project_id=new_task.project_id if new_task else None, - subscribed=subscribed, - ) - finally: - conn.close() - except ValueError as e: - return tool_error(f"kanban_create: {e}") - except Exception as e: - logger.exception("kanban_create failed") - return tool_error(f"kanban_create: {e}") def _maybe_auto_subscribe(conn: Any, task_id: str) -> bool: """Auto-subscribe the calling session to task completion / block events. - Returns True if a subscription row was written, False otherwise (no - session context, config gate disabled, or best-effort failure). The - caller surfaces this in the ``subscribed`` field of the kanban_create - response so an orchestrator can decide whether to fall back to an - explicit ``kanban_notify-subscribe`` or to polling. + Returns True iff a subscription row was written; surfaced as ``subscribed`` + on kanban_create so an orchestrator can fall back to an explicit + ``kanban_notify-subscribe`` or polling. Gated by + ``kanban.auto_subscribe_on_create`` (default True; unreadable config also + means True). - Gated by ``kanban.auto_subscribe_on_create`` in config.yaml (default - True). Disable to mirror pre-feature behaviour, e.g. when the - originating user/chat opted out via the per-platform notification - toggle (see ``hermes dashboard``). + Delivery targets: + - Gateway (telegram/discord/...): ``HERMES_SESSION_PLATFORM`` / + ``HERMES_SESSION_CHAT_ID`` ContextVars set before dispatch. + - TUI/desktop: those ContextVars are cleared, but the subprocess inherits + ``HERMES_SESSION_KEY``; subscribe as ``platform="tui"``, ``chat_id=`` + for the TUI notification poller. ``HERMES_SESSION_ID`` is deliberately + NOT a fallback — it is set for every CLI/ACP invocation for telemetry and + would auto-subscribe every CLI run. + - CLI / cron / tests: no persistent channel, no-op. - Subscription paths: - - - **Gateway** (telegram/discord/slack/etc): ``HERMES_SESSION_PLATFORM``, - ``HERMES_SESSION_CHAT_ID``, and ``HERMES_SESSION_CHAT_TYPE`` are set in - ContextVars by the messaging gateway before agent dispatch. The - notification poller already keys off these, so we just register a row. - - - **TUI** (herm desktop / herm TUI): the platform/chat_id ContextVars - are intentionally cleared (TUI is a single-channel local UI, not - a multi-tenant chat surface), but the agent subprocess inherits - ``HERMES_SESSION_KEY`` from the parent session. We subscribe with - ``platform="tui"`` and ``chat_id=``; the TUI notification - poller (``tui_gateway/server.py``) reads ``kanban_notify_subs`` - for these rows and posts the completion message into the running - session. - - - **CLI / cron / test / unattached**: no persistent delivery channel, - no-op. - - Failure mode: any exception inside the function is logged at WARNING - with the offending exception + diagnostic env vars and swallowed. - We never want a notification bookkeeping failure to fail the - kanban_create that the agent is mid-conversation about. + Any failure is logged at WARNING and swallowed: notification bookkeeping + must never fail the kanban_create the agent is mid-conversation about. """ try: cfg = load_config() if not cfg_get(cfg, "kanban", "auto_subscribe_on_create", default=True): return False except Exception: - # If config can't load we still default to True — this is the - # user-friendly behaviour that mirrors the pre-gate implementation. pass platform = "" @@ -1555,19 +1189,6 @@ def _maybe_auto_subscribe(conn: Any, task_id: str) -> bool: platform = get_session_env("HERMES_SESSION_PLATFORM", "") chat_id = get_session_env("HERMES_SESSION_CHAT_ID", "") if not platform or not chat_id: - # TUI / desktop fallback: platform/chat_id ContextVars are - # cleared for TUI sessions, but the parent process exports - # HERMES_SESSION_KEY into the subprocess env. Treat that - # as a "tui" subscription so the TUI notification poller - # (tui_gateway/server.py) can pick it up. - # - # HERMES_SESSION_ID is intentionally NOT a fallback here: - # it is set by ACP / the agent subprocess for telemetry - # regardless of whether the parent is a TUI or a CLI, so - # treating it as a notification target would auto-subscribe - # every CLI invocation, which is exactly the over-eager - # behaviour that got #19718 reverted upstream. The TUI - # poller keys on HERMES_SESSION_KEY. session_key = ( get_session_env("HERMES_SESSION_KEY", "") or os.environ.get("HERMES_SESSION_KEY", "") @@ -1609,7 +1230,6 @@ def _maybe_auto_subscribe(conn: Any, task_id: str) -> bool: if message_id: delivery_metadata["telegram_reply_to_message_id"] = str(message_id) - # Lazy-import to keep the module-level dependency light from hermes_cli import kanban_db as _kb _kb.add_notify_sub( conn, task_id=task_id, @@ -1629,6 +1249,7 @@ def _maybe_auto_subscribe(conn: Any, task_id: str) -> bool: return False +@_kanban_handler("kanban_unblock") def _handle_unblock(args: dict, **kw) -> str: """Transition a blocked task to ready, or todo while parents remain open.""" delegated_err = _reject_delegated_child_mutation("kanban_unblock") @@ -1643,26 +1264,17 @@ def _handle_unblock(args: dict, **kw) -> str: ownership_err = _enforce_worker_task_ownership(str(tid)) if ownership_err: return ownership_err - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - ok = kb.unblock_task(conn, str(tid)) - if not ok: - return tool_error(f"could not unblock {tid} (not blocked or unknown)") - task = kb.get_task(conn, str(tid)) - return _ok(task_id=str(tid), status=task.status if task else None) - finally: - conn.close() - except ValueError as e: - return tool_error(f"kanban_unblock: {e}") - except Exception as e: - logger.exception("kanban_unblock failed") - return tool_error(f"kanban_unblock: {e}") + with _board(args.get("board")) as (kb, conn): + ok = kb.unblock_task(conn, str(tid)) + if not ok: + return tool_error(f"could not unblock {tid} (not blocked or unknown)") + task = kb.get_task(conn, str(tid)) + return _ok(task_id=str(tid), status=task.status if task else None) +@_kanban_handler("kanban_link") def _handle_link(args: dict, **kw) -> str: - """Add a parent→child dependency edge after the fact.""" + """Add a parent→child dependency edge after the fact (cycles/self-links → ValueError).""" delegated_err = _reject_delegated_child_mutation("kanban_link") if delegated_err: return delegated_err @@ -1670,831 +1282,38 @@ def _handle_link(args: dict, **kw) -> str: child_id = args.get("child_id") if not parent_id or not child_id: return tool_error("both parent_id and child_id are required") - board = args.get("board") - try: - kb, conn = _connect(board=board) - try: - kb.link_tasks(conn, parent_id=parent_id, child_id=child_id) - return _ok(parent_id=parent_id, child_id=child_id) - finally: - conn.close() - except ValueError as e: - # Covers cycle + self-parent rejections - return tool_error(f"kanban_link: {e}") - except Exception as e: - logger.exception("kanban_link failed") - return tool_error(f"kanban_link: {e}") + with _board(args.get("board")) as (kb, conn): + kb.link_tasks(conn, parent_id=parent_id, child_id=child_id) + return _ok(parent_id=parent_id, child_id=child_id) # --------------------------------------------------------------------------- -# Schemas +# Registration (order preserved: it is the order tools appear in the schema) # --------------------------------------------------------------------------- -_DESC_TASK_ID_DEFAULT = ( - "Task id. If omitted, defaults to HERMES_KANBAN_TASK from the env " - "(the task the dispatcher spawned you to work on)." +_TOOLS = ( + ("kanban_show", KANBAN_SHOW_SCHEMA, _handle_show, _check_kanban_mode, "📋"), + ("kanban_list", KANBAN_LIST_SCHEMA, _handle_list, _check_kanban_orchestrator_mode, "📋"), + ("kanban_complete", KANBAN_COMPLETE_SCHEMA, _handle_complete, _check_kanban_mode, "✔"), + ("kanban_block", KANBAN_BLOCK_SCHEMA, _handle_block, _check_kanban_mode, "⏸"), + ("kanban_request_review", KANBAN_REQUEST_REVIEW_SCHEMA, _handle_request_review, _check_kanban_mode, "👀"), + ("kanban_request_changes", KANBAN_REQUEST_CHANGES_SCHEMA, _handle_request_changes, _check_kanban_mode, "↩"), + ("kanban_heartbeat", KANBAN_HEARTBEAT_SCHEMA, _handle_heartbeat, _check_kanban_mode, "💓"), + ("kanban_comment", KANBAN_COMMENT_SCHEMA, _handle_comment, _check_kanban_mode, "💬"), + ("kanban_attach", KANBAN_ATTACH_SCHEMA, _handle_attach, _check_kanban_mode, "📎"), + ("kanban_attach_url", KANBAN_ATTACH_URL_SCHEMA, _handle_attach_url, _check_kanban_mode, "📎"), + ("kanban_attachments", KANBAN_ATTACHMENTS_SCHEMA, _handle_attachments, _check_kanban_mode, "📎"), + ("kanban_create", KANBAN_CREATE_SCHEMA, _handle_create, _check_kanban_mode, "➕"), + ("kanban_unblock", KANBAN_UNBLOCK_SCHEMA, _handle_unblock, _check_kanban_orchestrator_mode, "▶"), + ("kanban_link", KANBAN_LINK_SCHEMA, _handle_link, _check_kanban_mode, "🔗"), ) -_DESC_BOARD = ( - "Kanban board slug to target. When omitted, the call resolves the " - "active board the usual way: HERMES_KANBAN_DB env → " - "HERMES_KANBAN_BOARD env → the 'current' symlink under the kanban " - "home → 'default'. Pass an explicit slug only when the caller (e.g. " - "a Telegram routing layer) needs to override the env-pinned active " - "board for this one call." -) - - -def _board_schema_prop() -> dict[str, str]: - """Schema fragment for the optional ``board`` parameter. - - Centralised so a future tweak to the description / validation hint - only has to land in one place. - """ - return {"type": "string", "description": _DESC_BOARD} - -KANBAN_SHOW_SCHEMA = { - "name": "kanban_show", - "description": ( - "Read a task's full state — title, body, assignee, parent task " - "handoffs, your prior attempts on this task if any, comments, " - "and recent events. Use this to (re)orient yourself before " - "starting work, especially on retries. The response includes a " - "pre-formatted ``worker_context`` string suitable for inclusion " - "verbatim in your reasoning." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": _DESC_TASK_ID_DEFAULT, - }, - "board": _board_schema_prop(), - }, - "required": [], - }, -} - -KANBAN_LIST_SCHEMA = { - "name": "kanban_list", - "description": ( - "List Kanban task summaries so an orchestrator profile can discover " - "work to route. Supports the same core filters as the CLI: assignee, " - "status, tenant, include_archived, and limit. Returns compact rows " - "with ids, title, status, assignee, priority, parent/child ids, and " - "counts. Bounded to 50 rows by default, 200 max, with truncation " - "metadata. Also recomputes ready tasks before listing, matching the " - "CLI. Orchestrator-only — dispatcher-spawned task workers never see " - "this tool." - ), - "parameters": { - "type": "object", - "properties": { - "assignee": { - "type": "string", - "description": "Optional assignee/profile filter.", - }, - "status": { - "type": "string", - "enum": [ - "triage", "todo", "ready", "running", - "blocked", "done", "archived", - ], - "description": "Optional task status filter.", - }, - "tenant": { - "type": "string", - "description": "Optional tenant/project namespace filter.", - }, - "include_archived": { - "type": "boolean", - "description": "Include archived tasks. Defaults to false.", - }, - "limit": { - "type": "integer", - "description": "Optional maximum rows to return (default 50, max 200).", - }, - "board": _board_schema_prop(), - }, - "required": [], - }, -} - -KANBAN_COMPLETE_SCHEMA = { - "name": "kanban_complete", - "description": ( - "Mark your current task done with a structured handoff for " - "downstream workers and humans. Prefer ``summary`` for a " - "human-readable 1-3 sentence description of what you did; put " - "machine-readable facts in ``metadata`` (changed_files, " - "tests_run, decisions, findings, etc). At least one of " - "``summary`` or ``result`` is required. If you created new " - "tasks via ``kanban_create`` during this run, list their ids " - "in ``created_cards`` — the kernel verifies them so phantom " - "references are caught before they leak into downstream " - "automation. If you produced deliverable files (charts, PDFs, " - "spreadsheets, generated images), list their absolute paths " - "in ``artifacts`` — the gateway notifier will upload them as " - "native attachments to the human who subscribed to the task, " - "so the deliverable lands in their chat alongside the summary " - "instead of being a path they have to fetch by hand." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": _DESC_TASK_ID_DEFAULT, - }, - "summary": { - "type": "string", - "description": ( - "Human-readable handoff, 1-3 sentences. Appears in " - "Run History on the dashboard and in downstream " - "workers' context." - ), - }, - "metadata": { - "type": "object", - "description": ( - "Free-form dict of structured facts about this " - "attempt — {\"changed_files\": [...], \"tests_run\": 12, " - "\"findings\": [...]}. Surfaced to downstream " - "workers alongside ``summary``." - ), - }, - "result": { - "type": "string", - "description": ( - "Short result log line (legacy field, maps to " - "task.result). Use ``summary`` instead when " - "possible; this exists for compatibility with " - "callers that still set --result on the CLI." - ), - }, - "created_cards": { - "type": "array", - "items": {"type": "string"}, - "description": ( - "Optional structured manifest of task ids you " - "created via ``kanban_create`` during this run. " - "The kernel verifies each id exists and was " - "created by this worker's profile; any phantom " - "id blocks the completion with an error listing " - "what went wrong (auditable in the task's events). " - "Only list ids you got back from a successful " - "``kanban_create`` call — do not invent or " - "remember ids from prose. Omit the field if you " - "did not create any cards." - ), - }, - "artifacts": { - "type": "array", - "items": {"type": "string"}, - "description": ( - "Optional list of absolute paths to deliverable " - "files you produced during this run — generated " - "charts, PDFs, spreadsheets, images, archives. " - "Examples: [\"/tmp/q3-revenue.png\", " - "\"/tmp/report.pdf\"]. The gateway notifier " - "uploads each path as a native attachment to the " - "subscribed chat (images embed inline, everything " - "else uploads as a file) so the deliverable " - "lands with the completion notification. Skip " - "intermediate scratch files and references that " - "are not the deliverable. The path must exist " - "on disk at completion. Files inside a managed scratch " - "workspace are copied to durable task attachments before " - "cleanup; a missing declared scratch artifact keeps the " - "task in-flight so you can fix the path and retry." - ), - }, - "board": _board_schema_prop(), - }, - "required": [], - }, -} - -KANBAN_BLOCK_SCHEMA = { - "name": "kanban_block", - "description": ( - "Stop work on this task and route it according to WHY you're stuck. " - "Set ``kind`` to say which: 'dependency' (waiting on another task — " - "goes to todo and auto-resumes when that task finishes, no human " - "needed), 'needs_input' (you need a human decision/answer), " - "'capability' (a hard wall: no access, missing credentials, an action " - "no agent can do), or 'transient' (a flaky failure that may clear). " - "``reason`` is shown to the human on the board. If a task keeps " - "getting unblocked and re-blocked for the same reason, it is " - "auto-escalated to triage. Use for genuine blockers only — don't " - "block on things you can resolve yourself." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": _DESC_TASK_ID_DEFAULT, - }, - "reason": { - "type": "string", - "description": ( - "What you need answered or what stopped you, in one or " - "two sentences. Don't paste the whole conversation; the " - "human has the board and can ask follow-ups via comments." - ), - }, - "kind": { - "type": "string", - "enum": ["dependency", "needs_input", "capability", "transient"], - "description": ( - "Why you're blocked. 'dependency' waits in todo and " - "resumes automatically; the others surface to a human. " - "Omit only if none apply." - ), - }, - "board": _board_schema_prop(), - }, - "required": ["reason"], - }, -} - -KANBAN_REQUEST_REVIEW_SCHEMA = { - "name": "kanban_request_review", - "description": ( - "Hand the task off for review: implementation, self-review, and " - "verification are complete and you want a human (or reviewer) to " - "look before it is marked done. Moves the task to the 'review' " - "column and notifies the subscriber. Unlike ``kanban_block`` this is " - "NOT a blocker — it never counts toward unblock-loop detection, so a " - "task can cycle through review across follow-ups without ever being " - "falsely escalated to triage. Use this instead of blocking with a " - "free-form 'review-required:' reason." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": _DESC_TASK_ID_DEFAULT, - }, - "summary": { - "type": "string", - "description": ( - "What was implemented and how it was verified, in one or " - "two sentences — shown to the reviewer. Don't paste " - "the whole diff; the reviewer has the board and the PR." - ), - }, - "reviewer": { - "type": "string", - "description": ( - "Optional reviewer profile. When provided, the task is " - "reassigned to that profile before review dispatch." - ), - }, - "metadata": { - "type": "object", - "description": ( - "Optional structured handoff facts for the reviewer, such " - "as changed_files, tests_run, commit, or decisions." - ), - "additionalProperties": True, - }, - "board": _board_schema_prop(), - }, - "required": ["summary"], - }, -} - -KANBAN_REQUEST_CHANGES_SCHEMA = { - "name": "kanban_request_changes", - "description": ( - "Reviewer verdict: return the current review run to the original " - "implementer with concrete required changes. This closes the review " - "run, reapplies parent dependency gating, and requeues the task without " - "using block-loop accounting. Only use from a task claimed from the " - "review column; use kanban_block only for a genuine external blocker." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": _DESC_TASK_ID_DEFAULT, - }, - "reason": { - "type": "string", - "description": ( - "Specific, actionable changes the implementer must make " - "before requesting another review." - ), - }, - "board": _board_schema_prop(), - }, - "required": ["reason"], - }, -} - -KANBAN_HEARTBEAT_SCHEMA = { - "name": "kanban_heartbeat", - "description": ( - "Signal that you're still alive during a long operation " - "(training, encoding, large crawls). Call every few minutes so " - "humans see liveness separately from PID checks. Pure side " - "effect — no work changes." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": _DESC_TASK_ID_DEFAULT, - }, - "note": { - "type": "string", - "description": ( - "Optional short note describing current progress. " - "Shown in the event log." - ), - }, - "board": _board_schema_prop(), - }, - "required": [], - }, -} - -KANBAN_COMMENT_SCHEMA = { - "name": "kanban_comment", - "description": ( - "Append a comment to a task's thread. Use for durable notes " - "that should outlive this run (questions for the next worker, " - "partial findings, rationale). Ephemeral reasoning doesn't " - "belong here — use your normal response instead." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": ( - "Task id. Required (may be your own task or " - "another's — comment threads are per-task)." - ), - }, - "body": { - "type": "string", - "description": "Markdown-supported comment body.", - }, - "board": _board_schema_prop(), - }, - "required": ["task_id", "body"], - }, -} - -KANBAN_ATTACH_SCHEMA = { - "name": "kanban_attach", - "description": ( - "Attach a file to a task by passing its bytes inline (base64). " - "Use for genuine file artifacts the next worker or a human should " - "be able to download — generated reports, images, exports. The " - "file is stored as a real attachment (not a comment link) under " - "the task's attachments dir, capped at 25 MB. Prefer " - "kanban_attach_url when you only have a URL." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": _DESC_TASK_ID_DEFAULT, - }, - "filename": { - "type": "string", - "description": ( - "File name to store it under (e.g. 'report.pdf'). " - "Directory components are stripped; only the leaf is kept." - ), - }, - "content_base64": { - "type": "string", - "description": "The file contents, base64-encoded. Max 25 MB decoded.", - }, - "content_type": { - "type": "string", - "description": "Optional MIME type (e.g. 'application/pdf').", - }, - "board": _board_schema_prop(), - }, - "required": ["filename", "content_base64"], - }, -} - -KANBAN_ATTACH_URL_SCHEMA = { - "name": "kanban_attach_url", - "description": ( - "Attach a file to a task by URL — Hermes downloads it server-side " - "and stores it as a real attachment (capped at 25 MB). Use when " - "you have a link rather than the bytes. Only http/https URLs are " - "accepted." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": _DESC_TASK_ID_DEFAULT, - }, - "url": { - "type": "string", - "description": "http(s) URL to fetch and store.", - }, - "filename": { - "type": "string", - "description": ( - "Optional name to store it under. Defaults to the URL " - "path's leaf component." - ), - }, - "content_type": { - "type": "string", - "description": ( - "Optional MIME type override. Defaults to the " - "Content-Type the server returns." - ), - }, - "board": _board_schema_prop(), - }, - "required": ["url"], - }, -} - -KANBAN_ATTACHMENTS_SCHEMA = { - "name": "kanban_attachments", - "description": ( - "List the files attached to a task: id, filename, content_type, " - "size, who uploaded it, and the absolute on-disk path you can read." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": _DESC_TASK_ID_DEFAULT, - }, - "board": _board_schema_prop(), - }, - "required": [], - }, -} - -KANBAN_CREATE_SCHEMA = { - "name": "kanban_create", - "description": ( - "Create a new kanban task, optionally as a child of the current " - "one (pass the current task id in ``parents``). Used by " - "orchestrator workers to fan out — decompose work into child " - "tasks with specific assignees, link them into a pipeline, " - "then complete your own task. The dispatcher picks up the new " - "tasks on its next tick and spawns the assigned profiles." - ), - "parameters": { - "type": "object", - "properties": { - "title": { - "type": "string", - "description": "Short task title (required).", - }, - "assignee": { - "type": "string", - "description": ( - "Profile name that should execute this task " - "(e.g. 'researcher-a', 'reviewer', 'writer'). " - "Required — tasks without an assignee are never " - "dispatched." - ), - }, - "body": { - "type": "string", - "description": ( - "Opening post: full spec, acceptance criteria, " - "links. The assigned worker reads this as part of " - "its context." - ), - }, - "parents": { - "type": "array", - "items": {"type": "string"}, - "description": ( - "Parent task ids. The new task stays in 'todo' " - "until every parent reaches 'done'; then it " - "auto-promotes to 'ready'. Typical fan-in: list " - "all the researcher task ids when creating a " - "synthesizer task." - ), - }, - "tenant": { - "type": "string", - "description": ( - "Optional namespace for multi-project isolation. " - "Defaults to HERMES_TENANT env if set." - ), - }, - "priority": { - "type": "integer", - "description": ( - "Dispatcher tiebreaker. Higher = picked sooner " - "when multiple ready tasks share an assignee." - ), - }, - "workspace_kind": { - "type": "string", - "enum": ["scratch", "dir", "worktree"], - "description": ( - "Workspace flavor: 'scratch' (fresh tmp dir, " - "default), 'dir' (shared directory, requires " - "absolute workspace_path), 'worktree' (git worktree)." - ), - }, - "workspace_path": { - "type": "string", - "description": ( - "Absolute path for 'dir' or 'worktree' workspace. " - "Relative paths are rejected at dispatch." - ), - }, - "project": { - "type": "string", - "description": ( - "Optional project id or slug to link the task to. When " - "set, the task becomes a git worktree under the project's " - "primary repo with a deterministic branch (project slug + " - "task id), instead of a random branch." - ), - }, - "triage": { - "type": "boolean", - "description": ( - "If true, task lands in 'triage' instead of 'todo' " - "— a specifier profile is expected to flesh out " - "the body before work starts." - ), - }, - "idempotency_key": { - "type": "string", - "description": ( - "If a non-archived task with this key already " - "exists, return that task's id instead of creating " - "a duplicate. Useful for retry-safe automation." - ), - }, - "max_runtime_seconds": { - "type": "integer", - "description": ( - "Per-task runtime cap. When exceeded, the " - "dispatcher SIGTERMs the worker and re-queues the " - "task with outcome='timed_out'." - ), - }, - "initial_status": { - "type": "string", - "enum": ["running", "blocked"], - "description": ( - "Initial card status. Use 'blocked' for tasks that " - "require immediate human ops (R3 gate) to skip the " - "brief running-to-blocked transition. Defaults to " - "'running', which preserves the usual dispatch path." - ), - }, - "skills": { - "type": "array", - "items": {"type": "string"}, - "description": ( - "Skill names to force-load into the dispatched " - "worker. The kanban lifecycle is already injected " - "automatically; use this to pin a task to a specialist " - "context — e.g. ['translation'] for a translation " - "task, ['github-code-review'] for a reviewer task. " - "The names must match skills installed on the " - "assignee's profile." - ), - }, - "goal_mode": { - "type": "boolean", - "description": ( - "Run the dispatched worker in a goal loop. When true, " - "after each turn an auxiliary judge checks the worker's " - "response against this card's title/body; if the work " - "isn't done and budget remains, the worker keeps going " - "in the same session until the judge agrees it's " - "complete (or the goal-turn budget is exhausted, which " - "blocks the task for human review). Use this for " - "open-ended cards where one shot rarely finishes the " - "work. Defaults to false (classic single-shot worker)." - ), - }, - "goal_max_turns": { - "type": "integer", - "description": ( - "Turn budget for goal_mode workers. Caps how many " - "continuation turns the worker may take before the task " - "is blocked for review. Ignored unless goal_mode is " - "true. Defaults to the goal-engine default (20)." - ), - }, - "model": { - "type": "string", - "description": ( - "Pin the dispatched worker to this model instead of " - "the assignee profile's configured model. Use the " - "exact model name the target provider expects. Omit " - "to use the profile default." - ), - }, - "provider": { - "type": "string", - "description": ( - "Provider the 'model' belongs to (e.g. 'openrouter', " - "'anthropic', 'nous'). Set this whenever the model " - "is not from the assignee profile's configured " - "provider — a model name alone is resolved against " - "the profile's provider and will fail if it belongs " - "to a different one. Requires 'model'." - ), - }, - "board": _board_schema_prop(), - }, - "required": ["title", "assignee"], - }, -} - -KANBAN_UNBLOCK_SCHEMA = { - "name": "kanban_unblock", - "description": ( - "Unblock a Kanban task. It moves to ready when all parents are done, " - "or todo while any parent remains open. Orchestrator-only — only " - "profiles with the kanban toolset can unblock routed work; " - "dispatcher-spawned task workers never see this tool." - ), - "parameters": { - "type": "object", - "properties": { - "task_id": { - "type": "string", - "description": "Blocked task id to move to ready or parent-gated todo.", - }, - "board": _board_schema_prop(), - }, - "required": ["task_id"], - }, -} - -KANBAN_LINK_SCHEMA = { - "name": "kanban_link", - "description": ( - "Add a parent→child dependency edge after both tasks already " - "exist. The child won't promote to 'ready' until all parents " - "are 'done'. Cycles and self-links are rejected." - ), - "parameters": { - "type": "object", - "properties": { - "parent_id": {"type": "string", "description": "Parent task id."}, - "child_id": {"type": "string", "description": "Child task id."}, - "board": _board_schema_prop(), - }, - "required": ["parent_id", "child_id"], - }, -} - - -# --------------------------------------------------------------------------- -# Registration -# --------------------------------------------------------------------------- - -registry.register( - name="kanban_show", - toolset="kanban", - schema=KANBAN_SHOW_SCHEMA, - handler=_handle_show, - check_fn=_check_kanban_mode, - emoji="📋", -) - -registry.register( - name="kanban_list", - toolset="kanban", - schema=KANBAN_LIST_SCHEMA, - handler=_handle_list, - check_fn=_check_kanban_orchestrator_mode, - emoji="📋", -) - -registry.register( - name="kanban_complete", - toolset="kanban", - schema=KANBAN_COMPLETE_SCHEMA, - handler=_handle_complete, - check_fn=_check_kanban_mode, - emoji="✔", -) - -registry.register( - name="kanban_block", - toolset="kanban", - schema=KANBAN_BLOCK_SCHEMA, - handler=_handle_block, - check_fn=_check_kanban_mode, - emoji="⏸", -) - -registry.register( - name="kanban_request_review", - toolset="kanban", - schema=KANBAN_REQUEST_REVIEW_SCHEMA, - handler=_handle_request_review, - check_fn=_check_kanban_mode, - emoji="👀", -) - -registry.register( - name="kanban_request_changes", - toolset="kanban", - schema=KANBAN_REQUEST_CHANGES_SCHEMA, - handler=_handle_request_changes, - check_fn=_check_kanban_mode, - emoji="↩", -) - -registry.register( - name="kanban_heartbeat", - toolset="kanban", - schema=KANBAN_HEARTBEAT_SCHEMA, - handler=_handle_heartbeat, - check_fn=_check_kanban_mode, - emoji="💓", -) - -registry.register( - name="kanban_comment", - toolset="kanban", - schema=KANBAN_COMMENT_SCHEMA, - handler=_handle_comment, - check_fn=_check_kanban_mode, - emoji="💬", -) - -registry.register( - name="kanban_attach", - toolset="kanban", - schema=KANBAN_ATTACH_SCHEMA, - handler=_handle_attach, - check_fn=_check_kanban_mode, - emoji="📎", -) - -registry.register( - name="kanban_attach_url", - toolset="kanban", - schema=KANBAN_ATTACH_URL_SCHEMA, - handler=_handle_attach_url, - check_fn=_check_kanban_mode, - emoji="📎", -) - -registry.register( - name="kanban_attachments", - toolset="kanban", - schema=KANBAN_ATTACHMENTS_SCHEMA, - handler=_handle_attachments, - check_fn=_check_kanban_mode, - emoji="📎", -) - -registry.register( - name="kanban_create", - toolset="kanban", - schema=KANBAN_CREATE_SCHEMA, - handler=_handle_create, - check_fn=_check_kanban_mode, - emoji="➕", -) - -registry.register( - name="kanban_unblock", - toolset="kanban", - schema=KANBAN_UNBLOCK_SCHEMA, - handler=_handle_unblock, - check_fn=_check_kanban_orchestrator_mode, - emoji="▶", -) - -registry.register( - name="kanban_link", - toolset="kanban", - schema=KANBAN_LINK_SCHEMA, - handler=_handle_link, - check_fn=_check_kanban_mode, - emoji="🔗", -) +for _name, _sch, _handler, _check_fn, _emoji in _TOOLS: + registry.register( + name=_name, + toolset="kanban", + schema=_sch, + handler=_handler, + check_fn=_check_fn, + emoji=_emoji, + ) diff --git a/tools/kanban_tools_schemas.py b/tools/kanban_tools_schemas.py new file mode 100644 index 0000000000..c09a9afe6d --- /dev/null +++ b/tools/kanban_tools_schemas.py @@ -0,0 +1,511 @@ +"""Tool schemas for tools.kanban_tools (model-facing; strings are byte-frozen).""" +from __future__ import annotations + +from typing import Any + +_DESC_TASK_ID_DEFAULT = ( + "Task id. If omitted, defaults to HERMES_KANBAN_TASK from the env " + "(the task the dispatcher spawned you to work on)." +) + +_DESC_BOARD = ( + "Kanban board slug to target. When omitted, the call resolves the " + "active board the usual way: HERMES_KANBAN_DB env → " + "HERMES_KANBAN_BOARD env → the 'current' symlink under the kanban " + "home → 'default'. Pass an explicit slug only when the caller (e.g. " + "a Telegram routing layer) needs to override the env-pinned active " + "board for this one call." +) + + +def _prop(type_: str, description: str) -> dict[str, str]: + return {"type": type_, "description": description} + + +def _board_schema_prop() -> dict[str, str]: + """Schema fragment for the optional ``board`` parameter (one place to tweak).""" + return _prop("string", _DESC_BOARD) + + +def _schema(name: str, description: str, properties: dict[str, Any], required: list[str]) -> dict[str, Any]: + """Build a tool schema; every kanban tool takes an optional trailing ``board``.""" + return { + "name": name, + "description": description, + "parameters": { + "type": "object", + "properties": {**properties, "board": _board_schema_prop()}, + "required": required, + }, + } + + +KANBAN_SHOW_SCHEMA = _schema( + "kanban_show", + ( + "Read a task's full state — title, body, assignee, parent task " + "handoffs, your prior attempts on this task if any, comments, " + "and recent events. Use this to (re)orient yourself before " + "starting work, especially on retries. The response includes a " + "pre-formatted ``worker_context`` string suitable for inclusion " + "verbatim in your reasoning." + ), + { + "task_id": _prop("string", _DESC_TASK_ID_DEFAULT), + }, + [], +) + +KANBAN_LIST_SCHEMA = _schema( + "kanban_list", + ( + "List Kanban task summaries so an orchestrator profile can discover " + "work to route. Supports the same core filters as the CLI: assignee, " + "status, tenant, include_archived, and limit. Returns compact rows " + "with ids, title, status, assignee, priority, parent/child ids, and " + "counts. Bounded to 50 rows by default, 200 max, with truncation " + "metadata. Also recomputes ready tasks before listing, matching the " + "CLI. Orchestrator-only — dispatcher-spawned task workers never see " + "this tool." + ), + { + "assignee": _prop("string", "Optional assignee/profile filter."), + "status": { + "type": "string", + "enum": [ + "triage", "todo", "ready", "running", + "blocked", "done", "archived", + ], + "description": "Optional task status filter.", + }, + "tenant": _prop("string", "Optional tenant/project namespace filter."), + "include_archived": _prop("boolean", "Include archived tasks. Defaults to false."), + "limit": _prop("integer", "Optional maximum rows to return (default 50, max 200)."), + }, + [], +) + +KANBAN_COMPLETE_SCHEMA = _schema( + "kanban_complete", + ( + "Mark your current task done with a structured handoff for " + "downstream workers and humans. Prefer ``summary`` for a " + "human-readable 1-3 sentence description of what you did; put " + "machine-readable facts in ``metadata`` (changed_files, " + "tests_run, decisions, findings, etc). At least one of " + "``summary`` or ``result`` is required. If you created new " + "tasks via ``kanban_create`` during this run, list their ids " + "in ``created_cards`` — the kernel verifies them so phantom " + "references are caught before they leak into downstream " + "automation. If you produced deliverable files (charts, PDFs, " + "spreadsheets, generated images), list their absolute paths " + "in ``artifacts`` — the gateway notifier will upload them as " + "native attachments to the human who subscribed to the task, " + "so the deliverable lands in their chat alongside the summary " + "instead of being a path they have to fetch by hand." + ), + { + "task_id": _prop("string", _DESC_TASK_ID_DEFAULT), + "summary": _prop("string", ( + "Human-readable handoff, 1-3 sentences. Appears in " + "Run History on the dashboard and in downstream " + "workers' context." + )), + "metadata": _prop("object", ( + "Free-form dict of structured facts about this " + "attempt — {\"changed_files\": [...], \"tests_run\": 12, " + "\"findings\": [...]}. Surfaced to downstream " + "workers alongside ``summary``." + )), + "result": _prop("string", ( + "Short result log line (legacy field, maps to " + "task.result). Use ``summary`` instead when " + "possible; this exists for compatibility with " + "callers that still set --result on the CLI." + )), + "created_cards": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Optional structured manifest of task ids you " + "created via ``kanban_create`` during this run. " + "The kernel verifies each id exists and was " + "created by this worker's profile; any phantom " + "id blocks the completion with an error listing " + "what went wrong (auditable in the task's events). " + "Only list ids you got back from a successful " + "``kanban_create`` call — do not invent or " + "remember ids from prose. Omit the field if you " + "did not create any cards." + ), + }, + "artifacts": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Optional list of absolute paths to deliverable " + "files you produced during this run — generated " + "charts, PDFs, spreadsheets, images, archives. " + "Examples: [\"/tmp/q3-revenue.png\", " + "\"/tmp/report.pdf\"]. The gateway notifier " + "uploads each path as a native attachment to the " + "subscribed chat (images embed inline, everything " + "else uploads as a file) so the deliverable " + "lands with the completion notification. Skip " + "intermediate scratch files and references that " + "are not the deliverable. The path must exist " + "on disk at completion. Files inside a managed scratch " + "workspace are copied to durable task attachments before " + "cleanup; a missing declared scratch artifact keeps the " + "task in-flight so you can fix the path and retry." + ), + }, + }, + [], +) + +KANBAN_BLOCK_SCHEMA = _schema( + "kanban_block", + ( + "Stop work on this task and route it according to WHY you're stuck. " + "Set ``kind`` to say which: 'dependency' (waiting on another task — " + "goes to todo and auto-resumes when that task finishes, no human " + "needed), 'needs_input' (you need a human decision/answer), " + "'capability' (a hard wall: no access, missing credentials, an action " + "no agent can do), or 'transient' (a flaky failure that may clear). " + "``reason`` is shown to the human on the board. If a task keeps " + "getting unblocked and re-blocked for the same reason, it is " + "auto-escalated to triage. Use for genuine blockers only — don't " + "block on things you can resolve yourself." + ), + { + "task_id": _prop("string", _DESC_TASK_ID_DEFAULT), + "reason": _prop("string", ( + "What you need answered or what stopped you, in one or " + "two sentences. Don't paste the whole conversation; the " + "human has the board and can ask follow-ups via comments." + )), + "kind": { + "type": "string", + "enum": ["dependency", "needs_input", "capability", "transient"], + "description": ( + "Why you're blocked. 'dependency' waits in todo and " + "resumes automatically; the others surface to a human. " + "Omit only if none apply." + ), + }, + }, + ["reason"], +) + +KANBAN_REQUEST_REVIEW_SCHEMA = _schema( + "kanban_request_review", + ( + "Hand the task off for review: implementation, self-review, and " + "verification are complete and you want a human (or reviewer) to " + "look before it is marked done. Moves the task to the 'review' " + "column and notifies the subscriber. Unlike ``kanban_block`` this is " + "NOT a blocker — it never counts toward unblock-loop detection, so a " + "task can cycle through review across follow-ups without ever being " + "falsely escalated to triage. Use this instead of blocking with a " + "free-form 'review-required:' reason." + ), + { + "task_id": _prop("string", _DESC_TASK_ID_DEFAULT), + "summary": _prop("string", ( + "What was implemented and how it was verified, in one or " + "two sentences — shown to the reviewer. Don't paste " + "the whole diff; the reviewer has the board and the PR." + )), + "reviewer": _prop("string", ( + "Optional reviewer profile. When provided, the task is " + "reassigned to that profile before review dispatch." + )), + "metadata": { + "type": "object", + "description": ( + "Optional structured handoff facts for the reviewer, such " + "as changed_files, tests_run, commit, or decisions." + ), + "additionalProperties": True, + }, + }, + ["summary"], +) + +KANBAN_REQUEST_CHANGES_SCHEMA = _schema( + "kanban_request_changes", + ( + "Reviewer verdict: return the current review run to the original " + "implementer with concrete required changes. This closes the review " + "run, reapplies parent dependency gating, and requeues the task without " + "using block-loop accounting. Only use from a task claimed from the " + "review column; use kanban_block only for a genuine external blocker." + ), + { + "task_id": _prop("string", _DESC_TASK_ID_DEFAULT), + "reason": _prop("string", ( + "Specific, actionable changes the implementer must make " + "before requesting another review." + )), + }, + ["reason"], +) + +KANBAN_HEARTBEAT_SCHEMA = _schema( + "kanban_heartbeat", + ( + "Signal that you're still alive during a long operation " + "(training, encoding, large crawls). Call every few minutes so " + "humans see liveness separately from PID checks. Pure side " + "effect — no work changes." + ), + { + "task_id": _prop("string", _DESC_TASK_ID_DEFAULT), + "note": _prop("string", ( + "Optional short note describing current progress. " + "Shown in the event log." + )), + }, + [], +) + +KANBAN_COMMENT_SCHEMA = _schema( + "kanban_comment", + ( + "Append a comment to a task's thread. Use for durable notes " + "that should outlive this run (questions for the next worker, " + "partial findings, rationale). Ephemeral reasoning doesn't " + "belong here — use your normal response instead." + ), + { + "task_id": _prop("string", ( + "Task id. Required (may be your own task or " + "another's — comment threads are per-task)." + )), + "body": _prop("string", "Markdown-supported comment body."), + }, + ["task_id", "body"], +) + +KANBAN_ATTACH_SCHEMA = _schema( + "kanban_attach", + ( + "Attach a file to a task by passing its bytes inline (base64). " + "Use for genuine file artifacts the next worker or a human should " + "be able to download — generated reports, images, exports. The " + "file is stored as a real attachment (not a comment link) under " + "the task's attachments dir, capped at 25 MB. Prefer " + "kanban_attach_url when you only have a URL." + ), + { + "task_id": _prop("string", _DESC_TASK_ID_DEFAULT), + "filename": _prop("string", ( + "File name to store it under (e.g. 'report.pdf'). " + "Directory components are stripped; only the leaf is kept." + )), + "content_base64": { + "type": "string", + "description": "The file contents, base64-encoded. Max 25 MB decoded.", + }, + "content_type": _prop("string", "Optional MIME type (e.g. 'application/pdf')."), + }, + ["filename", "content_base64"], +) + +KANBAN_ATTACH_URL_SCHEMA = _schema( + "kanban_attach_url", + ( + "Attach a file to a task by URL — Hermes downloads it server-side " + "and stores it as a real attachment (capped at 25 MB). Use when " + "you have a link rather than the bytes. Only http/https URLs are " + "accepted." + ), + { + "task_id": _prop("string", _DESC_TASK_ID_DEFAULT), + "url": _prop("string", "http(s) URL to fetch and store."), + "filename": _prop("string", ( + "Optional name to store it under. Defaults to the URL " + "path's leaf component." + )), + "content_type": _prop("string", ( + "Optional MIME type override. Defaults to the " + "Content-Type the server returns." + )), + }, + ["url"], +) + +KANBAN_ATTACHMENTS_SCHEMA = _schema( + "kanban_attachments", + ( + "List the files attached to a task: id, filename, content_type, " + "size, who uploaded it, and the absolute on-disk path you can read." + ), + { + "task_id": _prop("string", _DESC_TASK_ID_DEFAULT), + }, + [], +) + +KANBAN_CREATE_SCHEMA = _schema( + "kanban_create", + ( + "Create a new kanban task, optionally as a child of the current " + "one (pass the current task id in ``parents``). Used by " + "orchestrator workers to fan out — decompose work into child " + "tasks with specific assignees, link them into a pipeline, " + "then complete your own task. The dispatcher picks up the new " + "tasks on its next tick and spawns the assigned profiles." + ), + { + "title": _prop("string", "Short task title (required)."), + "assignee": _prop("string", ( + "Profile name that should execute this task " + "(e.g. 'researcher-a', 'reviewer', 'writer'). " + "Required — tasks without an assignee are never " + "dispatched." + )), + "body": _prop("string", ( + "Opening post: full spec, acceptance criteria, " + "links. The assigned worker reads this as part of " + "its context." + )), + "parents": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Parent task ids. The new task stays in 'todo' " + "until every parent reaches 'done'; then it " + "auto-promotes to 'ready'. Typical fan-in: list " + "all the researcher task ids when creating a " + "synthesizer task." + ), + }, + "tenant": _prop("string", ( + "Optional namespace for multi-project isolation. " + "Defaults to HERMES_TENANT env if set." + )), + "priority": _prop("integer", ( + "Dispatcher tiebreaker. Higher = picked sooner " + "when multiple ready tasks share an assignee." + )), + "workspace_kind": { + "type": "string", + "enum": ["scratch", "dir", "worktree"], + "description": ( + "Workspace flavor: 'scratch' (fresh tmp dir, " + "default), 'dir' (shared directory, requires " + "absolute workspace_path), 'worktree' (git worktree)." + ), + }, + "workspace_path": _prop("string", ( + "Absolute path for 'dir' or 'worktree' workspace. " + "Relative paths are rejected at dispatch." + )), + "project": _prop("string", ( + "Optional project id or slug to link the task to. When " + "set, the task becomes a git worktree under the project's " + "primary repo with a deterministic branch (project slug + " + "task id), instead of a random branch." + )), + "triage": _prop("boolean", ( + "If true, task lands in 'triage' instead of 'todo' " + "— a specifier profile is expected to flesh out " + "the body before work starts." + )), + "idempotency_key": _prop("string", ( + "If a non-archived task with this key already " + "exists, return that task's id instead of creating " + "a duplicate. Useful for retry-safe automation." + )), + "max_runtime_seconds": _prop("integer", ( + "Per-task runtime cap. When exceeded, the " + "dispatcher SIGTERMs the worker and re-queues the " + "task with outcome='timed_out'." + )), + "initial_status": { + "type": "string", + "enum": ["running", "blocked"], + "description": ( + "Initial card status. Use 'blocked' for tasks that " + "require immediate human ops (R3 gate) to skip the " + "brief running-to-blocked transition. Defaults to " + "'running', which preserves the usual dispatch path." + ), + }, + "skills": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Skill names to force-load into the dispatched " + "worker. The kanban lifecycle is already injected " + "automatically; use this to pin a task to a specialist " + "context — e.g. ['translation'] for a translation " + "task, ['github-code-review'] for a reviewer task. " + "The names must match skills installed on the " + "assignee's profile." + ), + }, + "goal_mode": _prop("boolean", ( + "Run the dispatched worker in a goal loop. When true, " + "after each turn an auxiliary judge checks the worker's " + "response against this card's title/body; if the work " + "isn't done and budget remains, the worker keeps going " + "in the same session until the judge agrees it's " + "complete (or the goal-turn budget is exhausted, which " + "blocks the task for human review). Use this for " + "open-ended cards where one shot rarely finishes the " + "work. Defaults to false (classic single-shot worker)." + )), + "goal_max_turns": _prop("integer", ( + "Turn budget for goal_mode workers. Caps how many " + "continuation turns the worker may take before the task " + "is blocked for review. Ignored unless goal_mode is " + "true. Defaults to the goal-engine default (20)." + )), + "model": _prop("string", ( + "Pin the dispatched worker to this model instead of " + "the assignee profile's configured model. Use the " + "exact model name the target provider expects. Omit " + "to use the profile default." + )), + "provider": _prop("string", ( + "Provider the 'model' belongs to (e.g. 'openrouter', " + "'anthropic', 'nous'). Set this whenever the model " + "is not from the assignee profile's configured " + "provider — a model name alone is resolved against " + "the profile's provider and will fail if it belongs " + "to a different one. Requires 'model'." + )), + }, + ["title", "assignee"], +) + +KANBAN_UNBLOCK_SCHEMA = _schema( + "kanban_unblock", + ( + "Unblock a Kanban task. It moves to ready when all parents are done, " + "or todo while any parent remains open. Orchestrator-only — only " + "profiles with the kanban toolset can unblock routed work; " + "dispatcher-spawned task workers never see this tool." + ), + { + "task_id": _prop("string", "Blocked task id to move to ready or parent-gated todo."), + }, + ["task_id"], +) + +KANBAN_LINK_SCHEMA = _schema( + "kanban_link", + ( + "Add a parent→child dependency edge after both tasks already " + "exist. The child won't promote to 'ready' until all parents " + "are 'done'. Cycles and self-links are rejected." + ), + { + "parent_id": {"type": "string", "description": "Parent task id."}, + "child_id": {"type": "string", "description": "Child task id."}, + }, + ["parent_id", "child_id"], +) diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index f990f340eb..f2ef410e29 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -1,68 +1,27 @@ -""" -Lazy dependency installer for opt-in Hermes Agent backends. +"""Lazy dependency installer for opt-in Hermes backends. -Many Hermes features (Mistral TTS, ElevenLabs TTS, Honcho memory, Bedrock, -Slack, Matrix, etc.) require Python packages that not every user needs. The -historical approach was to bundle them all under ``pyproject.toml`` extras -(``hermes-agent[all]``) and install them eagerly at setup time. That has -two problems: - -1. **Fragility.** When one extra's transitive dependency becomes - unavailable on PyPI (quarantined for malware, yanked, broken upload), - the *entire* ``[all]`` resolve fails and fresh installs silently fall - back to a stripped tier — losing 10+ unrelated extras at once. - -2. **Bloat.** A user who only ever talks to one provider pulls hundreds - of packages they will never import. - -The lazy-install pattern fixes both. Backends call :func:`ensure` at the -top of their first-import path. If the deps are missing, ``ensure`` checks -the ``security.allow_lazy_installs`` config flag (default true) and runs -a venv-scoped pip install. If the user has explicitly disabled lazy -installs, ``ensure`` raises :class:`FeatureUnavailable` with a clear -remediation hint pointing at ``hermes tools`` or the manual pip command. +Backends call :func:`ensure(feature)` on first import; missing packages are +pip-installed into the active venv (or the durable target) unless the user set +``security.allow_lazy_installs: false``, in which case :class:`FeatureUnavailable` +carries a remediation hint. Eager ``[all]`` extras were both fragile (one yanked +transitive broke every extra) and bloated; lazy installs fix both. Security model: +* Venv-scoped: installs target ``sys.executable``'s venv, never system Python. +* Durable-target mode (sealed images): ``HERMES_LAZY_INSTALL_TARGET`` redirects + installs to a writable volume that is APPENDED to ``sys.path`` — never + prepended, never via PYTHONPATH — so core site-packages wins every collision. + A lazily installed package can only add modules, never shadow or break core; + that guarantee is what made sealing the venv safe. An ABI stamp on the target + wipes stale compiled wheels across interpreter rebuilds. +* PyPI by name only: no ``--index-url``, ``git+``, or file specs (``_spec_is_safe``). +* Allowlist: only specs in :data:`LAZY_DEPS` flow into pip via ``ensure``. +* Opt-out ``security.allow_lazy_installs: false`` disables installs in both modes. +* Install failures surface pip's stderr as FeatureUnavailable — no retries, no cache. -* **Venv-scoped by default.** Installs target ``sys.executable`` in the - active venv. We never touch the system Python. -* **Durable-target mode (immutable images).** When the deployment seals the - agent's own venv (the Docker image sets ``HERMES_DISABLE_LAZY_INSTALLS=1`` - and makes ``/opt/hermes`` read-only), setting - ``HERMES_LAZY_INSTALL_TARGET`` redirects lazy installs to a writable - directory on the durable data volume (e.g. ``/opt/data/lazy-packages``). - That directory is **appended to the end of ``sys.path``** — never - prepended, never exported via ``PYTHONPATH`` — so the agent's own - site-packages wins every name collision. A package installed this way can - only ADD new importable modules; it can never shadow, downgrade, or break - a module the core already ships. The worst a bad/incompatible backend - package can do is fail to import and report itself unavailable — the agent - core stays healthy. This is the structural guarantee that a lazily - installed package cannot brick Hermes, which is what made it safe to seal - the venv in the first place. Compiled-wheel safety across image rebuilds - is handled by an ABI/Python-version stamp on the target subdir (see - :func:`_ensure_target_ready`). -* **PyPI by package name only.** Specs may be ``"package>=1.0,<2"`` etc. - We do NOT support ``--index-url`` overrides, ``git+https://``, file: - paths, or any other input that could be hijacked by a malicious config. -* **Allowlist.** Only specs that appear in :data:`LAZY_DEPS` can be - installed via this path. A typo in feature name doesn't get the user - install-anything semantics. -* **Opt-out.** Setting ``security.allow_lazy_installs: false`` in - ``config.yaml`` disables runtime installs in BOTH modes. Users in - restricted networks or strict security postures can pin themselves to - whatever was installed at setup time. -* **Offline detection.** If the install fails (offline, mirror down, - PyPI 404 / quarantine), we surface the failure as - :class:`FeatureUnavailable` with the actual pip stderr — no silent - retries, no caching of bad state. - -Adding a new backend: - -1. Add an entry to :data:`LAZY_DEPS` with the package specs. -2. At the top of the backend module's import path, call - ``ensure("feature.name")`` inside a try/except that converts - :class:`FeatureUnavailable` to a useful runtime error. +Adding a backend: add a :data:`LAZY_DEPS` entry, then call ``ensure("ns.name")`` +at the top of the backend's import path, converting FeatureUnavailable to a +useful runtime error. """ from __future__ import annotations @@ -84,34 +43,24 @@ from hermes_cli._subprocess_compat import windows_hide_flags logger = logging.getLogger(__name__) -# ============================================================================= -# Allowlist of lazy-installable backends. -# -# Keys are dot-separated feature names ("namespace.backend"). Values are -# tuples of pip-installable specs that match the corresponding extra in -# pyproject.toml. The framework enforces that only specs from this map -# can flow into the pip install command. -# ============================================================================= - +# Allowlist: "namespace.backend" -> pip specs matching the pyproject extra. +# Pins are exact (no ranges, security posture); bump here AND in pyproject. +# Shared patched floors (prior CVEs + GHSA-cq5v-8q36-5273/GHSA-mfx4-hv73-q22v/ +# GHSA-mq44-7p77-q5h7; CVE-2026-48710 BadHost) — keep in sync with pyproject. +_AIOHTTP_PIN = "aiohttp==3.14.3" +_STARLETTE_PIN = "starlette==1.3.1" LAZY_DEPS: dict[str, tuple[str, ...]] = { # ─── Inference providers ─────────────────────────────────────────────── - # Native Anthropic SDK — needed when provider=anthropic (not via - # OpenRouter / aggregators which use the openai SDK). + # Native Anthropic SDK (provider=anthropic; aggregators use the openai SDK). "provider.anthropic": ("anthropic==0.87.0",), # CVE-2026-34450, CVE-2026-34452 - # AWS Bedrock provider "provider.bedrock": ("boto3==1.42.89",), - # Google Vertex AI provider — OAuth2 token minting for the Gemini - # OpenAI-compatible endpoint. Only loaded when provider=vertex is selected; - # google-auth is NOT in [all] so plain installs don't carry it. + # Vertex OAuth2 token minting; google-auth is NOT in [all] on purpose. "provider.vertex": ( "google-auth==2.55.1", "pyasn1==0.6.4", ), - # Microsoft Foundry — Entra ID auth (managed identity, workload identity, - # service principal, az login, VS Code, azd, PowerShell). Only loaded - # when model.auth_mode=entra_id is selected; key-based azure-foundry - # users never pay this import. + # Foundry Entra ID auth; only when model.auth_mode=entra_id. "provider.azure_identity": ("azure-identity==1.25.3",), # ─── Web search backends ─────────────────────────────────────────────── @@ -120,23 +69,15 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { "search.parallel": ("parallel-web==0.4.2",), # ─── Monitoring ───────────────────────────────────────────────────────── - # OTLP gateway monitoring export. Lazily installed on first use of - # monitoring.gateway_health_export / monitoring.export.otlp. Tracks the - # `otlp` extra in pyproject.toml — bump both together. + # OTLP export; tracks the `otlp` extra. "export.otlp": ( "opentelemetry-sdk==1.39.1", "opentelemetry-exporter-otlp-proto-http==1.39.1", ), # ─── TTS providers ───────────────────────────────────────────────────── - # Pinned to exact versions to match pyproject.toml's no-ranges policy - # (see comment at top of [project.dependencies]). When bumping, update - # both this map AND the corresponding extra in pyproject.toml. - # - # mistralai pin tracks the `mistral` extra in pyproject.toml. PyPI - # quarantined the project 2026-05-12 (malicious 2.4.6, Mini Shai-Hulud); - # 2.4.6 was removed and clean releases resumed (2.4.7, 2.4.8). Voxtral - # STT + TTS share the same SDK. + # mistralai: 2.4.6 was a malicious quarantined release — never pin below 2.4.7. + # Voxtral STT + TTS share the SDK. "tts.mistral": ("mistralai==2.4.8",), "tts.edge": ("edge-tts==7.2.7",), "tts.elevenlabs": ("elevenlabs==1.59.0",), @@ -148,20 +89,14 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { "sounddevice==0.5.5", "numpy==2.4.3", ), - # SILK voice-note decoding (WeChat/QQ .silk voice messages). pilk is a - # small silk-v3 codec binding; installed on first .silk transcription. + # SILK voice-note decoding (WeChat/QQ); silk-v3 codec binding. "stt.silk": ("pilk==0.2.4",), - # ─── Wake word ("Hey Hermes") engines ────────────────────────────────── - # Keep in sync with the `wake` extra in pyproject.toml. openWakeWord is the - # free, local default (ONNX runtime); Porcupine is the premium engine. - # openWakeWord's ONNX embedding model returns near-zero scores on macOS - # ARM64 (dscripka/openWakeWord#336), so the wake word runs on the tflite - # backend there. Upstream declares tflite-runtime for Linux only; - # ai-edge-litert is the macOS equivalent, bridged in tools/wake_word.py. - # It lives in its own feature because lazy-dep specs cannot carry PEP 508 - # environment markers (_spec_is_safe rejects ";"), so the platform gate is - # applied by the caller instead. + # ─── Wake word ("Hey Hermes") engines (sync with the `wake` extra) ────── + # openWakeWord's ONNX model scores ~0 on macOS ARM64, so macOS uses the tflite + # backend (ai-edge-litert, bridged in tools/wake_word.py). Separate feature + # because specs cannot carry PEP 508 markers (";" is rejected) — the caller + # applies the platform gate. "wake.openwakeword.tflite": ( "ai-edge-litert==2.1.6", ), @@ -171,9 +106,8 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { "sounddevice==0.5.5", "numpy==2.4.3", ), - # Open-vocabulary keyword spotting: any typed phrase, zero training. - # sentencepiece is required by sherpa_onnx.text2token (runtime phrase - # tokenization) even though sherpa-onnx doesn't declare it. + # Open-vocabulary keyword spotting. sentencepiece is needed by + # sherpa_onnx.text2token but undeclared by sherpa-onnx. "wake.sherpa": ( "sherpa-onnx==1.13.4", "sentencepiece==0.2.2", @@ -192,46 +126,33 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { # ─── Memory providers ────────────────────────────────────────────────── "memory.honcho": ("honcho-ai==2.2.0",), "memory.hindsight": ("hindsight-client==0.6.1",), - # supermemory + mem0 are opt-in cloud memory providers with their own - # SDKs. On the published Docker image the agent venv is sealed - # (HERMES_DISABLE_LAZY_INSTALLS=1) and lazy installs are redirected to the - # durable target — so, like honcho/hindsight, these MUST go through - # ensure() to be installable there. Without an allowlist entry + an - # ensure() call at the import site, the SDK never installs on a hosted - # instance and the provider silently reports itself unavailable. + # Cloud memory SDKs MUST be allowlisted + ensure()'d at the import site, or + # they never install on the sealed Docker image (durable-target only). "memory.supermemory": ("supermemory==3.50.0",), "memory.mem0": ("mem0ai==2.0.10",), # ─── Messaging platforms (lazy-installable on demand) ────────────────── "platform.telegram": ("python-telegram-bot[webhooks]==22.8",), - # brotlicffi gives aiohttp a working 2-arg Decompressor.process() for - # Discord CDN's Brotli-encoded attachments. Without it, aiohttp falls - # back to google's `Brotli` package (1-arg API), and any .txt/.md/.doc - # uploaded to the Discord gateway fails to decode at att.read() with - # "Can not decode content-encoding: br" — see #12511 / #15744. + # brotlicffi: aiohttp needs its 2-arg Decompressor for Discord CDN's + # Brotli attachments; google's `Brotli` (1-arg) fails "Can not decode br". + # aiohttp is only capped transitively by these adapters, so a vulnerable + # already-installed copy would satisfy them — pin the patched floor explicitly. "platform.discord": ( "discord.py[voice]==2.7.1", "brotlicffi==1.2.0.1", - # discord.py pulls aiohttp transitively (>=3.7.4,<4) as its HTTP - # backbone. Pin the patched floor here too so the lazy Discord path - # can't keep an already-installed vulnerable aiohttp satisfying that - # range — mirrors the messaging extra and platform.slack. - "aiohttp==3.14.3", # prior CVEs + GHSA-cq5v-8q36-5273/GHSA-mfx4-hv73-q22v/GHSA-mq44-7p77-q5h7 + _AIOHTTP_PIN, ), "platform.slack": ( "slack-bolt==1.30.0", "slack-sdk==3.43.0", - "aiohttp==3.14.3", # prior CVEs + GHSA-cq5v-8q36-5273/GHSA-mfx4-hv73-q22v/GHSA-mq44-7p77-q5h7 + _AIOHTTP_PIN, ), "platform.matrix": ( "mautrix[encryption]==0.21.1", "aiosqlite==0.22.1", "asyncpg==0.31.0", "aiohttp-socks==0.11.0", - # mautrix (aiohttp>=3,<4) and aiohttp-socks (aiohttp>=3.10.0) only cap - # aiohttp transitively, so a vulnerable already-installed aiohttp still - # satisfies both — pin the patched floor here too, like platform.discord. - "aiohttp==3.14.3", # prior CVEs + GHSA-cq5v-8q36-5273/GHSA-mfx4-hv73-q22v/GHSA-mq44-7p77-q5h7 + _AIOHTTP_PIN, ), "platform.dingtalk": ( "dingtalk-stream==0.24.3", @@ -242,15 +163,10 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { "lark-oapi==1.6.8", "qrcode==7.4.2", ), - # WeCom callback-mode adapter — parses untrusted XML POST bodies. Pulls - # defusedxml only; aiohttp/httpx are core dependencies of every messaging - # adapter and ship via `platform.discord` / `platform.slack` / etc. + # WeCom callback adapter parses untrusted XML POST bodies -> defusedxml. "platform.wecom_callback": ("defusedxml==0.7.1",), - # Microsoft Teams adapter — microsoft-teams-apps pulls a heavy tree - # (microsoft-teams-api/cards/common, dependency-injector, msal). Lazy- - # installed on demand like every other messaging platform; also exposed - # as the `teams` extra in pyproject for packagers / explicit installs. - "platform.teams": ("microsoft-teams-apps==2.0.13.4", "aiohttp==3.14.3"), # aiohttp 3.14.3: prior CVEs + GHSA-cq5v-8q36-5273/GHSA-mfx4-hv73-q22v/GHSA-mq44-7p77-q5h7 + # Teams pulls a heavy tree (msal, dependency-injector); also the `teams` extra. + "platform.teams": ("microsoft-teams-apps==2.0.13.4", _AIOHTTP_PIN), # ─── Terminal backends ───────────────────────────────────────────────── "terminal.modal": ("modal==1.3.4",), @@ -263,84 +179,51 @@ LAZY_DEPS: dict[str, tuple[str, ...]] = { "google-auth==2.55.1", "google-auth-oauthlib==1.3.1", "google-auth-httplib2==0.3.1", - # Transitive via google-api-python-client/google-auth-httplib2; keep explicit - # so lazy installs do not resolve vulnerable transitives: httplib2 0.31.2 - # (GHSA-j5g9-f88f-gfj3 decompression bomb DoS), stale pyasn1/google-auth. + # Explicit transitive pins: httplib2 <0.32 has a decompression-bomb DoS. "httplib2==0.32.0", "pyasn1==0.6.4", ), "skill.youtube": ("youtube-transcript-api==1.2.4",), # ─── Tools ───────────────────────────────────────────────────────────── - # ACP adapter (VS Code / Zed / JetBrains integration) + # ACP adapter (VS Code / Zed / JetBrains) "tool.acp": ("agent-client-protocol==0.9.0",), - # Dashboard (`hermes dashboard`) "tool.dashboard": ( "fastapi==0.133.1", "uvicorn[standard]==0.41.0", - "starlette==1.3.1", # CVE-2026-48710 (BadHost) — keep lazy-install in sync with pyproject [web] - "python-multipart==0.0.32", # FastAPI UploadFile/Form for streaming uploads (NS-501) + _STARLETTE_PIN, + "python-multipart==0.0.32", # FastAPI UploadFile/Form streaming uploads ), - # Vision image-resize recovery (Pillow). Pillow is now a CORE dependency - # (pyproject `dependencies`), so this entry is a belt-and-suspenders fallback - # for stripped/source-build installs that somehow dropped it. The vision - # call site uses prompt=False so it can never raise a blocking input() - # prompt mid-session (#40490). + # Pillow and firecrawl-anydoc are CORE deps; these entries are the self-heal + # path for lean/partial installs. Call sites use prompt=False so read_file / + # vision can never block on an input() prompt mid-session. "tool.vision": ("Pillow==12.3.0",), - # Document-to-Markdown extraction for read_file (firecrawl-anydoc, Rust - # core, imports as `anydoc`). Widens read_file's auto-extraction beyond - # the stdlib .ipynb/.docx/.xlsx to PDF, legacy Office (.doc/.ppt/.xls), - # OpenDocument, RTF, and EPUB. Installed on first read of such a file; - # the call site uses prompt=False so read_file never blocks on a prompt. - # NOTE: bundled in core pyproject dependencies since the hosted-OCR - # wiring (keep this lazy pin in lockstep with pyproject) — this entry - # survives as the self-heal path for lean/partial installs. "tool.doc_extract": ("firecrawl-anydoc==0.2.4",), # lockstep with pyproject - # Computer Use (cua-driver) — the MCP client SDK used to spawn and talk - # to the cua-driver process over stdio. Matches the `mcp` / `computer-use` - # extras in pyproject.toml. The one-liner installer pulls this in via - # `[all]`; lazy-installing here covers lean / partial / broken-extra - # installs so computer_use never dead-ends on `No module named 'mcp'`. + # MCP client SDK for the cua-driver; covers lean/broken-extra installs so + # computer_use never dead-ends on `No module named 'mcp'`. "tool.computer_use": ( "mcp==2.0.0", - "httpx2==2.7.0", # mcp 2.x HTTP stack — keep in sync with pyproject [computer-use] - "starlette==1.3.1", # CVE-2026-48710 — keep in sync with pyproject [computer-use] + "httpx2==2.7.0", # mcp 2.x HTTP stack — sync with pyproject [computer-use] + _STARLETTE_PIN, ), - # HF Agent Trace Viewer upload (hermes trace upload / /upload-trace). - # - # huggingface-hub is a SHARED dependency: transformers (pulled by - # sentence-transformers for local Hindsight embeddings) requires - # >=1.5.0,<2, and faster-whisper/tokenizers depend on it transitively. - # Because active_features() marks a feature active from mere package - # presence, the `hermes update` lazy-refresh pass re-asserts THIS pin on - # every install where hub is present — so an exact pin below 1.5.0 - # force-downgrades the shared package and breaks Hindsight startup - # (#60783). Policy: keep the exact pin (no ranges — security posture), - # but it MUST stay inside transformers' accepted window and MUST match - # uv.lock so the whole tree converges on ONE hub version - # (tests/test_project_metadata.py enforces both). When bumping: update - # here AND `uv lock --upgrade-package huggingface-hub` in lockstep. + # huggingface-hub is SHARED with transformers (>=1.5.0,<2 via Hindsight) and + # active_features() marks it active on mere presence, so `hermes update` + # re-asserts this pin everywhere hub exists. It MUST stay inside transformers' + # window and match uv.lock (tests/test_project_metadata.py enforces both); + # bump with `uv lock --upgrade-package huggingface-hub` in lockstep. "tool.trace_upload": ("huggingface-hub==1.24.0",), } -# Conservative regex for spec validation — package name plus optional -# version range. Reject anything that looks like a URL, file path, or shell -# metacharacter. -_SAFE_SPEC = re.compile( - r"^[A-Za-z0-9_][A-Za-z0-9_.\-]*" # package name - r"(?:\[[A-Za-z0-9_,\-]+\])?" # optional [extras] - r"(?:[<>=!~]=?[A-Za-z0-9_.\-+,*<>=!~]+)?" # optional version specifier - r"$" -) +# Spec validation: name[extras]specifier only — no URLs, paths, or shell metachars. +_NAME_RE = r"[A-Za-z0-9_][A-Za-z0-9_.\-]*" +_NAME_EXTRAS_RE = re.compile(rf"^{_NAME_RE}(?:\[[A-Za-z0-9_,\-]+\])?") +_SAFE_SPEC = re.compile(rf"^{_NAME_RE}(?:\[[A-Za-z0-9_,\-]+\])?(?:[<>=!~]=?[A-Za-z0-9_.\-+,*<>=!~]+)?$") class FeatureUnavailable(RuntimeError): - """A lazily-installable feature is missing and cannot be made available. - - Either the deps were never installed and the user has disabled lazy - installs, or the install attempt failed. - """ + """A lazily-installable feature is missing and cannot be made available + (lazy installs disabled, or the install attempt failed).""" def __init__(self, feature: str, missing: tuple[str, ...], reason: str): self.feature = feature @@ -364,71 +247,45 @@ class _InstallResult: stderr: str -# ============================================================================= -# Internals -# ============================================================================= +# ---- Internals --------------------------------------------------------------- - -# Environment variable that redirects lazy installs away from the (sealed) -# agent venv and into a writable directory on a durable volume. Set by the -# Docker image to /opt/data/lazy-packages. This is an internal bridge var, -# not user-facing config: the user-facing knob remains -# security.allow_lazy_installs in config.yaml. When unset, lazy installs go -# into the active venv as before. +# Internal bridge var (set by the Docker image, not user config) redirecting +# lazy installs from the sealed venv to a writable durable volume. _LAZY_TARGET_ENV = "HERMES_LAZY_INSTALL_TARGET" - -# Name of the stamp file written into the target dir recording the Python -# X.Y + ABI it was populated for. If a container rebuild bumps the -# interpreter, compiled wheels (.so) in the durable store would be ABI- -# incompatible; we detect the mismatch and wipe the store so packages get -# re-resolved against the new interpreter rather than importing a stale .so. +# Stamp recording the Python X.Y + ABI the target was populated for; a mismatch +# after an image rebuild wipes the store so stale .so files are never imported. _TARGET_STAMP_NAME = ".python-abi" +_SUBPROCESS_KW = dict(capture_output=True, text=True, encoding="utf-8", errors="replace", + stdin=subprocess.DEVNULL) + def _python_abi_tag() -> str: - """A stable token identifying the running interpreter's ABI. - - Combines the X.Y version with the EXT_SUFFIX (which encodes the ABI - tag and platform, e.g. ``cpython-313-x86_64-linux-gnu``). Two - interpreters that can share compiled wheels produce the same token. - """ + """X.Y version + EXT_SUFFIX (ABI tag + platform); interpreters that can + share compiled wheels produce the same token.""" ver = f"{sys.version_info.major}.{sys.version_info.minor}" ext = sysconfig.get_config_var("EXT_SUFFIX") or "" return f"{ver}:{ext}" def _lazy_install_target() -> Optional[Path]: - """Return the durable install-target dir, or None for venv-scoped mode. - - Returns a path only when :data:`_LAZY_TARGET_ENV` is set to a non-empty - value. The directory is created on demand by :func:`_ensure_target_ready`. - """ + """Durable install-target dir (from :data:`_LAZY_TARGET_ENV`), or None for + venv-scoped mode. Created on demand by :func:`_ensure_target_ready`.""" raw = os.environ.get(_LAZY_TARGET_ENV, "").strip() - if not raw: - return None - return Path(raw) + return Path(raw) if raw else None def _ensure_target_ready(target: Path) -> Optional[str]: - """Create the target dir and validate its ABI stamp. - - If the stamp is missing it is written. If it is present but records a - different interpreter ABI than the one now running (e.g. the container - image was rebuilt onto a newer Python), the directory's contents are - wiped and the stamp rewritten, so stale compiled wheels can't be - imported against an incompatible interpreter. - - Returns ``None`` on success, or an error string if the directory can't - be created / written (e.g. read-only mount, permission error). - """ + """Create the target dir and validate its ABI stamp; a stamp for a different + interpreter ABI wipes the contents first (stale .so must never import). + Returns None on success or an error string if the dir is not writable.""" want = _python_abi_tag() stamp = target / _TARGET_STAMP_NAME try: if target.exists(): - have = "" try: have = stamp.read_text(encoding="utf-8").strip() - except (OSError, FileNotFoundError): + except OSError: have = "" if have and have != want: logger.info( @@ -452,42 +309,36 @@ def _ensure_target_ready(target: Path) -> Optional[str]: def _activate_target_on_syspath(target: Path) -> None: - """Append the durable target to ``sys.path`` so its packages import. - - Appended to the END (never prepended) so the agent's own venv - site-packages takes precedence on every name collision. Idempotent. - Uses :func:`site.addsitedir` so ``.pth`` files (namespace packages, - editable installs) inside the target are honoured, then enforces the - append ordering — ``addsitedir`` would otherwise insert near the front. - """ + """Append the durable target to ``sys.path`` (idempotent). ``site.addsitedir`` + honours ``.pth`` files but inserts near the front, so every newly added + entry is moved to the END — core venv site-packages must win collisions.""" target_str = str(target) - # Snapshot existing entries so we can restore precedence afterwards. before = list(sys.path) if target_str not in before: site.addsitedir(target_str) - # site.addsitedir may have inserted target (and any .pth-added dirs) at - # the front. Move every newly-added entry to the end, preserving the - # core venv's precedence. New entries are those not present `before`. new_entries = [p for p in sys.path if p not in before] if new_entries: sys.path[:] = [p for p in sys.path if p not in new_entries] + new_entries - # importlib.metadata caches the path-based distribution finder; clear it - # so a just-activated dir is visible to version() checks this process. + _invalidate_import_caches() + + +def _invalidate_import_caches() -> None: + """Make just-installed/activated dists visible to importers and + importlib.metadata version() checks in this process.""" try: import importlib importlib.invalidate_caches() + import importlib.metadata as _md + if hasattr(_md, "_cache_clear"): + _md._cache_clear() # type: ignore[attr-defined] except Exception: pass def activate_durable_lazy_target() -> None: - """Public: wire the durable lazy-install target onto ``sys.path``. - - Safe no-op when :data:`_LAZY_TARGET_ENV` is unset or the directory does - not yet exist. Called once early in process startup (before backends - import) so packages installed into the durable store on a previous run - are importable on this run. Never raises. - """ + """Wire the durable target onto ``sys.path`` early in startup so packages + installed on a previous run import on this one. No-op when unset or the + dir does not exist yet. Never raises.""" target = _lazy_install_target() if target is None: return @@ -499,24 +350,11 @@ def activate_durable_lazy_target() -> None: def _allow_lazy_installs() -> bool: - """Return whether lazy installs are permitted in this environment. - - Resolution order: - - 1. ``security.allow_lazy_installs: false`` in config.yaml is an absolute - opt-out — it disables installs in BOTH venv-scoped and durable-target - modes. This is the user-facing kill switch. - 2. ``HERMES_DISABLE_LAZY_INSTALLS=1`` seals the *agent venv* (set by the - immutable Docker image). It blocks venv-scoped installs — UNLESS a - durable install target is configured, in which case installs are - redirected there (a path that structurally cannot break the sealed - venv) and are therefore allowed. - - Defaults to True. If config is unreadable we fail open (allow), because - refusing to install would lock people out of their own backends; the - decision to block is an explicit user opt-in. - """ - # (1) Config kill switch wins in every mode. + """Whether lazy installs are permitted. Order: (1) the config kill switch + ``security.allow_lazy_installs: false`` blocks in BOTH modes; (2) the sealed + venv (``HERMES_DISABLE_LAZY_INSTALLS=1``) blocks only when no durable target + exists to redirect into. Unreadable config fails OPEN — blocking is an + explicit user opt-in, not a default.""" try: from hermes_cli.config import load_config cfg = load_config() @@ -527,22 +365,14 @@ def _allow_lazy_installs() -> bool: if not bool(sec.get("allow_lazy_installs", True)): return False - # (2) Sealed-venv env var: blocks ONLY when there is no safe durable - # target to redirect into. With a target set, the install goes to the - # data volume (append-only on sys.path), so the seal is preserved. if os.environ.get("HERMES_DISABLE_LAZY_INSTALLS") == "1": return _lazy_install_target() is not None - return True def _unsupported_feature_reason(feature: str) -> Optional[str]: - """Return why a lazy feature cannot work on this host, or ``None``. - - This is a platform capability gate, not a security policy gate. It keeps - known-impossible installs out of both first-use lazy installation and the - ``hermes update`` lazy-refresh pass. - """ + """Platform capability gate (not policy): why a feature cannot work on + this host, or None. Keeps impossible installs out of ensure() and refresh.""" if sys.platform == "win32" and feature == "platform.matrix": return ( "unsupported on Windows: Matrix E2EE depends on python-olm, " @@ -564,129 +394,71 @@ def _spec_is_safe(spec: str) -> bool: def _pkg_name_from_spec(spec: str) -> str: - """Extract the bare package name from a pip spec. - - ``"slack-bolt>=1.18.0,<2"`` → ``"slack-bolt"`` - ``"mautrix[encryption]>=0.20"`` → ``"mautrix"`` - """ - m = re.match(r"^([A-Za-z0-9_][A-Za-z0-9_.\-]*)", spec) + """``"mautrix[encryption]>=0.20"`` -> ``"mautrix"``.""" + m = re.match(rf"^({_NAME_RE})", spec) return m.group(1) if m else spec def _specifier_from_spec(spec: str) -> str: - """Extract just the version-specifier portion of a pip spec. + """``"mautrix[encryption]>=0.20,<1"`` -> ``">=0.20,<1"``; ``""`` if unconstrained.""" + m = _NAME_EXTRAS_RE.match(spec) + return spec[m.end():] if m else "" - ``"honcho-ai==2.2.0"`` → ``"==2.2.0"`` - ``"mautrix[encryption]>=0.20,<1"`` → ``">=0.20,<1"`` - ``"package"`` → ``""`` (no version constraint) - """ - # Strip the package name + optional [extras] block. - m = re.match(r"^[A-Za-z0-9_][A-Za-z0-9_.\-]*(?:\[[A-Za-z0-9_,\-]+\])?", spec) - if not m: - return "" - return spec[m.end():] + +def _installed_version(spec: str) -> Optional[str]: + """Installed version of the spec's package, or None when absent.""" + try: + from importlib.metadata import version + + return version(_pkg_name_from_spec(spec)) + except Exception: + return None def _is_satisfied(spec: str) -> bool: - """Is ``spec`` already satisfied in the current env? - - Checks both presence AND version. If the package is installed at a - version outside the spec's range, returns False so the caller will - upgrade/downgrade to the pinned version. This is what makes - ``hermes update`` propagate pin bumps in :data:`LAZY_DEPS` to already- - installed backends instead of silently leaving stale versions in place. - - If ``packaging`` is unavailable for any reason (it's a transitive of - pip so this should never happen), we fall back to a presence-only check - so we err on the side of "don't churn". - """ - pkg = _pkg_name_from_spec(spec) - try: - from importlib.metadata import PackageNotFoundError, version - except ImportError: + """Present AND inside the spec's version range. A version outside the + range returns False so ``hermes update`` propagates pin bumps to installed + backends. Unparseable specs/versions or a missing ``packaging`` count as + satisfied — err toward "don't churn".""" + installed = _installed_version(spec) + if installed is None: return False - try: - installed = version(pkg) - except PackageNotFoundError: - return False - except Exception: - return False - spec_tail = _specifier_from_spec(spec) if not spec_tail: - # Bare ``"package"`` — no version constraint, presence is enough. return True - try: - from packaging.specifiers import InvalidSpecifier, SpecifierSet - from packaging.version import InvalidVersion, Version - except ImportError: - # packaging unavailable — fall back to "installed counts as satisfied". - return True + from packaging.specifiers import SpecifierSet + from packaging.version import Version - try: return Version(installed) in SpecifierSet(spec_tail) - except (InvalidSpecifier, InvalidVersion, Exception): - # Malformed spec or installed version we can't parse — don't churn. + except Exception: return True def _is_present(spec: str) -> bool: - """Cheap presence-only check (package name installed at any version). - - Used by :func:`active_features` to detect backends the user has - previously activated, regardless of whether the version pin moved. - """ - pkg = _pkg_name_from_spec(spec) - try: - from importlib.metadata import PackageNotFoundError, version - except ImportError: - return False - try: - version(pkg) - return True - except PackageNotFoundError: - return False - except Exception: - return False + """Presence-only check (any version); how :func:`active_features` detects + backends the user activated even if the pin has since moved.""" + return _installed_version(spec) is not None def _core_constraints_file() -> Optional[Path]: - """Write a pip constraints file pinning every package already importable - in the core environment to its installed version. - - Passed as ``--constraint`` for durable-target installs so the resolver - pins shared transitive deps (httpx, pydantic, aiohttp, …) to the exact - versions the core venv already ships, instead of pulling newer copies - into the durable store. Two payoffs: - - * The durable store stays minimal — only genuinely-new packages land - there; shared deps resolve to "already satisfied" against core. - * A backend that *requires* a version conflicting with core fails loudly - at install time (resolver conflict) rather than silently installing a - shadowed copy that can never win on sys.path anyway. - - Returns the path to a temp constraints file, or None if enumeration - failed (in which case the caller installs without constraints — still - safe, just less tidy). - """ - try: - from importlib.metadata import distributions - except ImportError: - return None + """Temp pip constraints file pinning every core-venv package to its installed + version, passed as ``--constraint`` for durable-target installs: shared deps + resolve as already-satisfied (store stays minimal) and a backend needing a + conflicting version fails loudly instead of installing a shadowed copy that + can never win on sys.path. None if enumeration failed (install unconstrained).""" try: import tempfile + from importlib.metadata import distributions + lines = [] seen = set() for dist in distributions(): name = dist.metadata["Name"] if dist.metadata else None ver = dist.version - if not name or not ver: + if not name or not ver or name.lower() in seen: continue - key = name.lower() - if key in seen: - continue - seen.add(key) + seen.add(name.lower()) lines.append(f"{name}=={ver}") if not lines: return None @@ -700,13 +472,8 @@ def _core_constraints_file() -> Optional[Path]: def _installed_dist_roots(spec: str, target: Optional[Path]) -> set[Path]: - """Return the package directories a freshly installed *spec* owns. - - Resolved from the distribution's own file list rather than guessing the - import name from the spec — ``python-telegram-bot`` ships ``telegram``, - ``firecrawl-anydoc`` ships ``anydoc``, and several specs ship more than - one top-level package. - """ + """Package directories a freshly installed *spec* owns, from the dist's own + file list (``python-telegram-bot`` ships ``telegram``; some ship several).""" name = _pkg_name_from_spec(spec) try: import importlib.metadata as _md @@ -727,9 +494,7 @@ def _installed_dist_roots(spec: str, target: Optional[Path]) -> set[Path]: parts = entry.parts if not parts or parts[0].startswith(".") or parts[0] == "__pycache__": continue - # Metadata dirs (``foo-1.0.dist-info``, legacy ``.egg-info``) own - # no importable code; compiling them is wasted work. - if parts[0].endswith((".dist-info", ".egg-info")): + if parts[0].endswith((".dist-info", ".egg-info")): # no importable code continue root = Path(dist.locate_file(parts[0])) if root.is_dir(): @@ -740,29 +505,14 @@ def _installed_dist_roots(spec: str, target: Optional[Path]) -> set[Path]: def _warm_installed_bytecode(specs: tuple[str, ...], target: Optional[Path]) -> None: - """Byte-compile what we just installed, so no user request has to. - - A pip/uv install writes ``.py`` sources and no ``__pycache__`` — and an - install of the *same* version still deletes the cache the old copy had. - Whoever imports the package next pays the whole compile: for - ``anthropic==0.87.0`` (541 modules) on cpython-3.12.13 that measured - 2.2-2.7s cold against 0.7-1.0s warm, and 10.5s cold under concurrent - load. That bill lands wherever the first import happens, and - for a lazily-installed backend that is the foreground of a user request - (#100461) — with nothing printed while it runs, so it reads as a hang. - Worse, N per-profile daemons cold-starting together each pay it in full - before any of them has written the cache. - - Paying it here instead is strictly better: the caller is already waiting - on an installer and can see why. Best-effort — a compile failure never - invalidates an install that succeeded. - """ + """Byte-compile what was just installed. A fresh install writes no + ``__pycache__`` (and drops the old one), so the next import — often the + foreground of a user request, silent, reading as a hang (~2-10s for a big + SDK) — would pay the compile. Pay it here while the caller already waits + on an installer. Best-effort; never invalidates a successful install.""" if sys.dont_write_bytecode: return - try: - import compileall - except Exception: # pragma: no cover — stdlib, but never break an install - return + import compileall for spec in specs: try: @@ -778,54 +528,46 @@ def _warm_installed_bytecode(specs: tuple[str, ...], target: Optional[Path]) -> def _venv_pip_install(specs: tuple[str, ...], *, timeout: int = 300) -> _InstallResult: - """Install ``specs`` using the uv → pip → ensurepip ladder. - - Two modes: - - * **Venv-scoped (default).** Installs into the active venv - (``sys.executable``). Used on normal installs. - * **Durable-target.** When :data:`_LAZY_TARGET_ENV` is set, installs into - that directory via ``--target`` and constrains shared deps to the - core venv's versions (see :func:`_core_constraints_file`). The target - is append-only on ``sys.path`` so it can never shadow core. Used by - the immutable Docker image to keep lazy installs off the sealed venv. - - Mirrors the strategy in ``hermes_cli.tools_config._pip_install`` but - kept independent here so this module has no CLI dependency. - """ + """Install ``specs`` via the uv -> pip -> ensurepip ladder, venv-scoped by + default or into the durable ``--target`` (constrained to core versions, see + :func:`_core_constraints_file`) when :data:`_LAZY_TARGET_ENV` is set. + Independent of ``hermes_cli.tools_config._pip_install`` so this module has + no CLI dependency.""" if not specs: return _InstallResult(True, "", "") target = _lazy_install_target() constraints: Optional[Path] = None - if target is not None: err = _ensure_target_ready(target) if err: return _InstallResult(False, "", err) constraints = _core_constraints_file() - target_args: list[str] = [] + extra_args: list[str] = [] if target is not None: - # --target tells both uv and pip to install into an arbitrary dir. - target_args = ["--target", str(target)] - constraint_args: list[str] = [] + extra_args += ["--target", str(target)] if constraints is not None: - constraint_args = ["--constraint", str(constraints)] + extra_args += ["--constraint", str(constraints)] + + def _run(cmd: list[str], **kw) -> subprocess.CompletedProcess: + return subprocess.run(cmd, **_SUBPROCESS_KW, creationflags=windows_hide_flags(), **kw) + + def _finish(r: subprocess.CompletedProcess) -> _InstallResult: + if r.returncode == 0: + if target is not None: + _activate_target_on_syspath(target) + _warm_installed_bytecode(specs, target) + return _InstallResult(r.returncode == 0, r.stdout or "", r.stderr or "") try: - venv_root = Path(sys.executable).parent.parent from tools.environments.local import hermes_subprocess_env uv_env = hermes_subprocess_env(inherit_credentials=False) - uv_env["VIRTUAL_ENV"] = str(venv_root) + uv_env["VIRTUAL_ENV"] = str(Path(sys.executable).parent.parent) - # Tier 1: uv (preferred — fast, doesn't need pip in the venv) - # Managed uv first: $HERMES_HOME/bin is never on PATH, so a bare - # which() misses the uv Hermes installed and falls through to the - # slower pip tier. Deliberately a lookup and not ensure_uv(): this runs - # mid-turn to install an optional dependency, and downloading uv + - # migrating the Python runtime as a side effect of that is a far bigger - # action than the caller asked for. Tier 2 pip covers the no-uv case. + # Tier 1: uv. Managed uv first ($HERMES_HOME/bin is never on PATH). A + # lookup, not ensure_uv(): downloading uv mid-turn is far more than the + # caller asked for; the pip tier covers the no-uv case. try: from hermes_cli.managed_uv import resolve_uv @@ -834,73 +576,39 @@ def _venv_pip_install(specs: tuple[str, ...], *, timeout: int = 300) -> _Install uv_bin = shutil.which("uv") if uv_bin: try: - # --compile-bytecode: uv does NOT write __pycache__ by default - # (pip does), so without it the first `import ` in - # the foreground of a user request recompiles every module of - # the backend *and* its transitive deps (#100461). This covers - # the whole install; _warm_installed_bytecode below is the - # belt-and-braces pass for the spec's own roots on any tier. - r = subprocess.run( - [uv_bin, "pip", "install", "--compile-bytecode", - *target_args, *constraint_args, *specs], - capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout, env=uv_env, - stdin=subprocess.DEVNULL, - creationflags=windows_hide_flags(), - ) - if r.returncode == 0: - if target is not None: - _activate_target_on_syspath(target) - _warm_installed_bytecode(specs, target) - return _InstallResult(True, r.stdout or "", r.stderr or "") - logger.debug("uv pip install failed: %s", r.stderr) - # A resolver failure is authoritative. Falling through to pip - # here would silently discard uv policy such as exclude-newer - # and could install a release that the project quarantined. - return _InstallResult(False, r.stdout or "", r.stderr or "") + # --compile-bytecode: uv writes no __pycache__ by default, so the + # first import would recompile the backend AND its transitives. + r = _run([uv_bin, "pip", "install", "--compile-bytecode", *extra_args, *specs], + timeout=timeout, env=uv_env) + if r.returncode != 0: + logger.debug("uv pip install failed: %s", r.stderr) + # A uv resolver failure is authoritative: falling through to pip + # would discard uv policy (exclude-newer) and could install a + # quarantined release. + return _finish(r) except subprocess.TimeoutExpired as e: logger.debug("uv invocation failed: %s", e) return _InstallResult(False, "", f"uv pip install timed out: {e}") except FileNotFoundError as e: - # The resolved uv path disappeared between lookup and spawn. - # In that narrow availability failure, the pip tier remains a - # valid fallback because uv never evaluated the requirements. + # uv vanished between lookup and spawn; it never evaluated the + # requirements, so pip remains a valid fallback. logger.debug("uv invocation failed: %s", e) - # Tier 2: python -m pip (with ensurepip bootstrap if needed) + # Tier 2: python -m pip (ensurepip bootstrap if needed) pip_cmd = [sys.executable, "-m", "pip"] try: - probe = subprocess.run( - pip_cmd + ["--version"], - capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=15, - stdin=subprocess.DEVNULL, - creationflags=windows_hide_flags(), - ) - if probe.returncode != 0: + if _run(pip_cmd + ["--version"], timeout=15).returncode != 0: raise FileNotFoundError("pip not in venv") except (subprocess.TimeoutExpired, FileNotFoundError): try: - subprocess.run( - [sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"], - capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=120, check=True, - stdin=subprocess.DEVNULL, - creationflags=windows_hide_flags(), - ) + _run([sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"], + timeout=120, check=True) except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: return _InstallResult(False, "", f"pip not available and ensurepip failed: {e}") try: - r = subprocess.run( - pip_cmd + ["install", *target_args, *constraint_args, *specs], - capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout, - stdin=subprocess.DEVNULL, - creationflags=windows_hide_flags(), - ) - if r.returncode == 0: - if target is not None: - _activate_target_on_syspath(target) - _warm_installed_bytecode(specs, target) - return _InstallResult(r.returncode == 0, r.stdout or "", r.stderr or "") + return _finish(_run(pip_cmd + ["install", *extra_args, *specs], timeout=timeout)) except subprocess.TimeoutExpired as e: return _InstallResult(False, "", f"pip install timed out: {e}") except Exception as e: @@ -913,9 +621,7 @@ def _venv_pip_install(specs: tuple[str, ...], *, timeout: int = 300) -> _Install pass -# ============================================================================= -# Public API -# ============================================================================= +# ---- Public API --------------------------------------------------------------- def feature_specs(feature: str) -> tuple[str, ...]: @@ -930,18 +636,25 @@ def feature_missing(feature: str) -> tuple[str, ...]: return tuple(s for s in feature_specs(feature) if not _is_satisfied(s)) +def _prompt_toolkit_active() -> bool: + """A bare input() deadlocks while a prompt_toolkit app owns the terminal + (keystrokes go to its loop, not stdin), so ensure() skips the confirmation + under the TUI — reaching it is already gated by security.allow_lazy_installs.""" + if "prompt_toolkit.application.current" not in sys.modules: + return False + try: + from prompt_toolkit.application.current import get_app_or_none + app = get_app_or_none() + return app is not None and bool(getattr(app, "is_running", False)) + except Exception: + return False + + def ensure(feature: str, *, prompt: bool = True) -> None: - """Make sure all packages for ``feature`` are importable. - - If they're missing, attempts to install them in the active venv. Raises - :class:`FeatureUnavailable` if the user has disabled lazy installs or - if the install attempt fails. - - ``prompt``: when True (default) and stdin is a TTY, asks the user to - confirm before installing. Non-interactive callers (gateway, cron, - batch) get prompt=False and skip the confirmation — config flag is - the gate in that case. - """ + """Make every package for ``feature`` importable, installing if needed. + Raises :class:`FeatureUnavailable` when installs are disabled or fail. + ``prompt``: confirm on a TTY first; non-interactive callers pass False and + rely on the config flag as the gate.""" if feature not in LAZY_DEPS: raise FeatureUnavailable( feature, (), f"feature {feature!r} not in LAZY_DEPS allowlist" @@ -955,19 +668,10 @@ def ensure(feature: str, *, prompt: bool = True) -> None: if unsupported: raise FeatureUnavailable(feature, missing, unsupported) - # Package-manager installs (NixOS, and any other distro that ships Hermes - # from a read-only store) cannot receive lazy pip installs: the venv's - # site-packages lives in the store, so the uv -> pip -> ensurepip ladder - # below burns ~15s bootstrapping ensurepip only to fail on a read-only - # target. Fail fast with an actionable message instead. - # - # Skipped when a durable install target is configured: the container - # deployment sets HERMES_MANAGED=true *and* HERMES_LAZY_INSTALL_TARGET - # (a writable volume), where lazy installs legitimately work. - # - # The reason string starts with "unsupported " on purpose: - # refresh_active_features classifies FeatureUnavailable by that prefix and - # reports anything else as a hard failure rather than a skip. + # Package-manager installs (NixOS etc.) have a read-only site-packages: the + # ladder would burn ~15s on ensurepip then fail. Fail fast — unless a durable + # target is configured, where installs legitimately work. The reason MUST + # start with "unsupported ": _refresh_features classifies skips by that prefix. if _lazy_install_target() is None: try: from hermes_cli.config import get_managed_system @@ -984,9 +688,7 @@ def ensure(feature: str, *, prompt: bool = True) -> None: f"{managed_by} (or run a pip/uv install of Hermes instead)." ) - # Validate every spec against the allowlist + safety regex. Belt and - # braces — the keys-in-LAZY_DEPS check above already constrains this. - for spec in missing: + for spec in missing: # belt and braces on top of the allowlist if not _spec_is_safe(spec): raise FeatureUnavailable( feature, missing, @@ -999,23 +701,7 @@ def ensure(feature: str, *, prompt: bool = True) -> None: "lazy installs disabled (security.allow_lazy_installs=false)" ) - # Only show the interactive confirmation when we own a TTY and - # prompt_toolkit isn't running. A bare input() deadlocks when a - # prompt_toolkit app owns the terminal because keystrokes route to - # its event loop rather than stdin, so the prompt blocks forever. - # Under the TUI we skip the prompt and proceed — lazy installs are - # gated by security.allow_lazy_installs, so reaching here is - # already user opt-in. - _pt_active = False - if "prompt_toolkit.application.current" in sys.modules: - try: - from prompt_toolkit.application.current import get_app_or_none - _app = get_app_or_none() - _pt_active = _app is not None and getattr(_app, "is_running", False) - except Exception: - _pt_active = False - - if prompt and not _pt_active and sys.stdin.isatty() and sys.stdout.isatty(): + if prompt and not _prompt_toolkit_active() and sys.stdin.isatty() and sys.stdout.isatty(): spec_list = ", ".join(missing) try: answer = input( @@ -1032,26 +718,15 @@ def ensure(feature: str, *, prompt: bool = True) -> None: logger.info("Lazy-installing %s for feature %r", " ".join(missing), feature) result = _venv_pip_install(missing) if not result.success: - # Surface the actual pip error so the user can debug PyPI-side - # issues (404 quarantine, network down, etc.). - snippet = (result.stderr or result.stdout or "").strip() - if snippet: - # Clip to a readable size — pip can dump pages of resolution traces. - snippet = snippet[-2000:] + # Surface pip's own error (quarantine 404, network) — tail-clipped, + # since pip can dump pages of resolution traces. + snippet = (result.stderr or result.stdout or "").strip()[-2000:] raise FeatureUnavailable( feature, missing, f"pip install failed: {snippet or 'no error output'}" ) - # Verify post-install. importlib.metadata caches per-process, so if we - # just installed something the cache may not see it without a refresh. - try: - import importlib.metadata as _md - if hasattr(_md, "_cache_clear"): - _md._cache_clear() # type: ignore[attr-defined] - except Exception: - pass - + _invalidate_import_caches() still_missing = feature_missing(feature) if still_missing: raise FeatureUnavailable( @@ -1071,19 +746,12 @@ def is_available(feature: str) -> bool: def feature_install_command(feature: str, *, venv_pip: bool = False) -> Optional[str]: - """Return the ``pip install`` command a user could run manually, or None. - - ``venv_pip=True`` targets the running interpreter's pip - (``{sys.executable} -m pip install …``) — correct in every layout - (default install, ``HERMES_HOME`` overrides, profile installs) and - immune to Ubuntu 24.04's PEP 668 ``externally-managed-environment`` - failure that a bare/system ``pip install`` hint invites. The default - ``uv pip install`` form is kept for contexts that document uv usage. - """ + """Manual install command for a feature, or None. ``venv_pip=True`` uses + ``{sys.executable} -m pip`` — correct in every layout and immune to PEP 668 + ``externally-managed-environment`` failures a bare ``pip install`` invites.""" if feature not in LAZY_DEPS: return None - specs = LAZY_DEPS[feature] - joined = " ".join(repr(s) for s in specs) + joined = " ".join(repr(s) for s in LAZY_DEPS[feature]) if venv_pip: return f"{sys.executable} -m pip install {joined}" return "uv pip install " + joined @@ -1108,30 +776,11 @@ class InstallSpecsResult: def install_specs(specs: list[str] | tuple[str, ...], *, timeout: int = 300) -> InstallSpecsResult: - """Install arbitrary (validated) pip specs through the lazy-install pipeline. - - This is the environment-aware install path for callers whose package - lists come from data (e.g. memory-provider plugin manifests declaring - ``pip_dependencies``) rather than the static :data:`LAZY_DEPS` allowlist. - It applies the exact same environment routing as :func:`ensure`: - - * **Venv-scoped by default** — installs into ``sys.executable``'s venv. - * **Durable-target on immutable images** — when the deployment seals the - agent venv (``HERMES_DISABLE_LAZY_INSTALLS=1``) and sets - ``HERMES_LAZY_INSTALL_TARGET``, installs are redirected to the writable - data-volume dir (``--target`` + core-venv constraints), then activated - on ``sys.path`` so the packages import in this process immediately. - * **Gated** — honors ``security.allow_lazy_installs`` and refuses to run - when the venv is sealed with no durable target (never attempts a write - to a read-only tree; reports *why* instead of surfacing EROFS/EACCES). - - Every spec must pass :func:`_spec_is_safe` (no URLs, paths, or shell - metacharacters). Unlike :func:`ensure`, unknown packages are permitted — - the caller owns manifest trust; this function owns spec hygiene and - environment routing. - - Never raises; inspect the returned :class:`InstallSpecsResult`. - """ + """Install data-driven pip specs (e.g. plugin manifest ``pip_dependencies``) + with the same environment routing and gating as :func:`ensure`. Unlike + ``ensure``, unknown packages are allowed — the caller owns manifest trust, + this function owns spec hygiene (:func:`_spec_is_safe`) and routing. Never + raises; inspect the :class:`InstallSpecsResult`.""" cleaned = tuple(str(s).strip() for s in specs if str(s).strip()) if not cleaned: return InstallSpecsResult(ok=True, command="") @@ -1169,17 +818,7 @@ def install_specs(specs: list[str] | tuple[str, ...], *, timeout: int = 300) -> ok=False, command=display, stderr=f"install failed: {exc}" ) - # Freshly-installed dists must be visible to importers and metadata - # checks in this same process (dashboard rechecks availability inline). - try: - import importlib - importlib.invalidate_caches() - import importlib.metadata as _md - if hasattr(_md, "_cache_clear"): - _md._cache_clear() # type: ignore[attr-defined] - except Exception: - pass - + _invalidate_import_caches() # dashboard rechecks availability inline return InstallSpecsResult( ok=result.success, command=display, @@ -1189,49 +828,22 @@ def install_specs(specs: list[str] | tuple[str, ...], *, timeout: int = 300) -> def active_features() -> list[str]: - """Return the list of features the user has ever lazy-installed. - - A feature counts as "active" if its anchor package (the first declared - spec) is currently installed in the venv (presence check, ignoring - version). We intentionally do NOT treat shared helper packages as proof - that a backend was enabled: for example ``platform.matrix`` depends on - generic packages like ``asyncpg``/``aiosqlite`` that can be installed for - unrelated reasons, while the actual Matrix adapter anchor is ``mautrix``. - Features the user has never enabled stay quiet. - - Used by ``hermes update`` to figure out which lazy backends need a - refresh pass when pins move in :data:`LAZY_DEPS`. - """ - active = [] - for feature, specs in LAZY_DEPS.items(): - if specs and _is_present(specs[0]): - active.append(feature) - return active + """Features whose ANCHOR package (first spec) is present at any version — + shared helpers like asyncpg are deliberately not proof a backend was + enabled. Drives the ``hermes update`` refresh pass.""" + return [f for f, specs in LAZY_DEPS.items() if specs and _is_present(specs[0])] def refresh_active_features(*, prompt: bool = False) -> dict[str, str]: - """Re-run ``ensure`` for every feature the user has previously activated. - - Returns a ``{feature: status}`` map where status is one of: - ``"current"`` — pins already satisfied, no install run - ``"refreshed"`` — pins were stale, reinstall succeeded - ``"failed: "`` — install attempt failed; caller decides - whether to surface it (we don't raise) - ``"skipped: "`` — gated off (config flag, user decline) - - Intended for ``hermes update``. Never raises; lazy-install failures - here must not block the rest of the update flow. - """ + """Re-run ``ensure`` for every active feature (``hermes update``). Returns + ``{feature: "current" | "refreshed" | "failed: " | "skipped: "}``. + Never raises — lazy failures must not block the update flow.""" return _refresh_features(active_features(), prompt=prompt, restoring=False) def restore_features(features: list[str]) -> dict[str, str]: - """Restore features captured before an explicit managed-runtime rebuild. - - Feature names are checked against :data:`LAZY_DEPS`, and installs remain - subject to ``security.allow_lazy_installs``. An explicit opt-out therefore - leaves the captured feature absent and reports it as skipped. - """ + """Restore features captured before a managed-runtime rebuild; still + subject to ``security.allow_lazy_installs`` (opt-out -> "skipped").""" return _refresh_features(features, prompt=False, restoring=True) @@ -1254,16 +866,10 @@ def _refresh_features( continue try: - if restoring: - ensure(feature, prompt=False) - results[feature] = "restored" - else: - ensure(feature, prompt=prompt) - results[feature] = "refreshed" + ensure(feature, prompt=False if restoring else prompt) + results[feature] = "restored" if restoring else "refreshed" except FeatureUnavailable as e: - # Distinguish "user opted out" or platform-incompatible features - # from install failures so the update command can render the - # right non-error message. + # Opt-outs and platform-incompatible features are skips, not failures. if ( "lazy installs disabled" in str(e) or "declined" in str(e) @@ -1284,37 +890,17 @@ def ensure_and_bind( *, prompt: bool = False, ) -> bool: - """Ensure a feature is installed, then rebind names into the caller's globals. + """:func:`ensure` the feature, then ``target_globals.update(importer())`` so + module-level names are rebound after a lazy install without hand-listing + them. ``importer`` runs only after ensure succeeds. Returns False (and + logs) if deps could not be installed or imported. - Combines :func:`ensure` with a post-install import step that rebinds - module-level names. This eliminates the error-prone pattern of manually - listing every global that needs updating after lazy-install. + Example:: - ``importer`` is a zero-arg callable that returns a dict of - ``{name: value}`` for all symbols the caller needs rebound. It is called - only after :func:`ensure` succeeds (or if the packages are already - installed). - - Returns True on success, False if deps couldn't be installed or imported. - - Example usage in a platform adapter:: - - def check_slack_requirements() -> bool: - if SLACK_AVAILABLE: - return True - def _import(): - from slack_bolt.async_app import AsyncApp - from slack_bolt.adapter.socket_mode.async_handler import AsyncSocketModeHandler - from slack_sdk.web.async_client import AsyncWebClient - import aiohttp - return { - "AsyncApp": AsyncApp, - "AsyncSocketModeHandler": AsyncSocketModeHandler, - "AsyncWebClient": AsyncWebClient, - "aiohttp": aiohttp, - "SLACK_AVAILABLE": True, - } - return ensure_and_bind("platform.slack", _import, globals(), prompt=False) + def _import(): + from slack_bolt.async_app import AsyncApp + return {"AsyncApp": AsyncApp, "SLACK_AVAILABLE": True} + return ensure_and_bind("platform.slack", _import, globals(), prompt=False) """ try: ensure(feature, prompt=prompt) @@ -1328,9 +914,7 @@ def ensure_and_bind( try: bindings = importer() except ImportError as exc: - logger.warning( - "Failed to import feature %r after install: %s", feature, exc - ) + logger.warning("Failed to import feature %r after install: %s", feature, exc) return False target_globals.update(bindings) diff --git a/tools/managed_tool_gateway.py b/tools/managed_tool_gateway.py index 0ec3b52460..1a6bce3b8a 100644 --- a/tools/managed_tool_gateway.py +++ b/tools/managed_tool_gateway.py @@ -8,13 +8,12 @@ import os from datetime import datetime, timezone from dataclasses import dataclass from typing import Callable, Optional -from urllib.parse import urlsplit - -logger = logging.getLogger(__name__) from hermes_constants import get_hermes_home from tools.tool_backend_helpers import managed_nous_tools_enabled +logger = logging.getLogger(__name__) + _DEFAULT_TOOL_GATEWAY_DOMAIN = "nousresearch.com" _DEFAULT_TOOL_GATEWAY_SCHEME = "https" _NOUS_ACCESS_TOKEN_REFRESH_SKEW_SECONDS = 120 @@ -74,12 +73,10 @@ def _access_token_is_expiring(expires_at: object, skew_seconds: int) -> bool: def _read_user_token_override() -> Optional[str]: - """Read the TOOL_GATEWAY_USER_TOKEN env override through the secret scope. + """Read the TOOL_GATEWAY_USER_TOKEN override through the secret scope. - Availability scans run both inside agent turns (scope installed) and in - unscoped CLI paths, so this uses the Slack pattern: honor the scope's - verdict when installed (a scoped miss does NOT borrow the process env - under multiplex), fall back to ``os.environ`` only when unscoped. + Scope verdict is authoritative when installed (a scoped miss must NOT + borrow the process env under multiplex); ``os.environ`` only when unscoped. """ try: from agent.secret_scope import UnscopedSecretError, get_secret @@ -96,15 +93,9 @@ def _read_user_token_override() -> Optional[str]: def peek_nous_access_token() -> Optional[str]: - """Cheap probe for a Nous gateway token without triggering refresh. - - Availability scans (`hermes tools`, banner/status paint, provider - `is_available()` checks) must stay off the synchronous OAuth refresh path. - This helper therefore only inspects the explicit env override and the - cached auth-store token, without checking expiry and without making any - network calls. Truthful refresh handling stays in request/session paths - that call :func:`read_nous_access_token`. - """ + """Cheap token probe: env override or cached auth-store token, no expiry + check and no network — availability scans must stay off the synchronous + OAuth refresh path (that lives in :func:`read_nous_access_token`).""" explicit = _read_user_token_override() if explicit: return explicit @@ -149,10 +140,8 @@ def get_tool_gateway_scheme() -> str: scheme = os.getenv("TOOL_GATEWAY_SCHEME", "").strip().lower() if not scheme: return _DEFAULT_TOOL_GATEWAY_SCHEME - if scheme in {"http", "https"}: return scheme - raise ValueError("TOOL_GATEWAY_SCHEME must be 'http' or 'https'") @@ -164,11 +153,8 @@ def build_vendor_gateway_url(vendor: str) -> str: return explicit_vendor_url shared_scheme = get_tool_gateway_scheme() - shared_domain = os.getenv("TOOL_GATEWAY_DOMAIN", "").strip().strip("/") - if shared_domain: - return f"{shared_scheme}://{vendor}-gateway.{shared_domain}" - - return f"{shared_scheme}://{vendor}-gateway.{_DEFAULT_TOOL_GATEWAY_DOMAIN}" + shared_domain = os.getenv("TOOL_GATEWAY_DOMAIN", "").strip().strip("/") or _DEFAULT_TOOL_GATEWAY_DOMAIN + return f"{shared_scheme}://{vendor}-gateway.{shared_domain}" def resolve_managed_tool_gateway( @@ -201,252 +187,13 @@ def is_managed_tool_gateway_ready( gateway_builder: Optional[Callable[[str], str]] = None, token_reader: Optional[Callable[[], Optional[str]]] = None, ) -> bool: - """Return True when gateway URL and a likely-usable Nous token are present. + """True when a gateway URL and a likely-usable Nous token are present. - Defaults to :func:`peek_nous_access_token` so read-only availability scans - avoid synchronous OAuth refresh. Callers that are about to make a real - gateway request should use :func:`resolve_managed_tool_gateway` (which - still defaults to the refresh-aware :func:`read_nous_access_token`). + Defaults to :func:`peek_nous_access_token` (no OAuth refresh); callers about + to make a real request use :func:`resolve_managed_tool_gateway` instead. """ return resolve_managed_tool_gateway( vendor, gateway_builder=gateway_builder, token_reader=token_reader or peek_nous_access_token, ) is not None - - -# --------------------------------------------------------------------------- -# Managed vendor endpoints -# --------------------------------------------------------------------------- -# -# Vendors the gateway serves on its own origin (rather than on a -# `{vendor}-gateway` host) are pinned HERE, in code, the same way every other -# managed vendor's gateway URL is pinned: adding one is a Hermes release, and -# the exact URL a user's agent may connect to is reviewable in this file. A -# runtime discovery catalog was tried and deliberately removed — a remote -# endpoint that can add tools to every entitled install is a bigger trust -# surface than a code diff. -# -# The gateway exposes a Nous-owned REST contract per vendor; it names the -# vendor but not the vendor's own API, so nothing here needs to know the -# upstream's endpoint or field names. - -# Pseudo-vendor used only to resolve the shared tool-gateway origin via -# build_vendor_gateway_url (honors TOOL_GATEWAY_URL / TOOL_GATEWAY_DOMAIN). -_MANAGED_GATEWAY_VENDOR = "tool" - -def managed_vendor_base_path(vendor: str) -> str: - """Base path for a managed vendor's REST routes on the gateway host.""" - return f"/api/{vendor}" - - -def managed_vendor_upload_path(vendor: str) -> str: - """Media upload endpoint for a managed vendor, on the same host.""" - return f"/api/uploads/{vendor}" - - -def managed_vendor_endpoints( - vendor: str, - gateway_builder: Optional[Callable[[str], str]] = None, -) -> Optional[dict]: - """Absolute URLs for a managed vendor, or ``None`` when none resolves. - - Address resolution only: entitlement is deliberately not consulted here. - What an account may spend on a managed vendor is the gateway's own - decision, stated in its refusals, and re-deciding it on the client can only - ever disagree with the server. A caller that wants to hide its tools from - users who could not call them at all does that in its ``check_fn``. - - ``None`` means no origin could be resolved — a misconfigured - ``TOOL_GATEWAY_SCHEME`` — so there is nothing to call. - """ - builder = gateway_builder or build_vendor_gateway_url - try: - origin = builder(_MANAGED_GATEWAY_VENDOR).rstrip("/") - except ValueError: - return None - if not origin: - return None - - return { - "origin": origin, - "base_url": f"{origin}{managed_vendor_base_path(vendor)}", - "upload_path": managed_vendor_upload_path(vendor), - } - - -def is_managed_nous_gateway_url( - url: object, - gateway_builder: Optional[Callable[[str], str]] = None, -) -> bool: - """True when ``url`` is on the Nous tool-gateway origin this client builds. - - Anything granting a URL extra trust — our bearer, reading files off disk to - upload — must gate on this rather than on a name, so an arbitrary URL can - never inherit that trust. - """ - if not isinstance(url, str) or not url.strip(): - return False - - builder = gateway_builder or build_vendor_gateway_url - try: - expected = urlsplit(builder(_MANAGED_GATEWAY_VENDOR)) - actual = urlsplit(url.strip()) - except ValueError: - return False - - return bool(actual.scheme) and (actual.scheme, actual.netloc) == (expected.scheme, expected.netloc) - - -def managed_gateway_auth_headers( - url: object, - gateway_builder: Optional[Callable[[str], str]] = None, - token_reader: Optional[Callable[[], Optional[str]]] = None, -) -> dict: - """Live auth headers for a managed gateway URL, or ``{}`` when not managed. - - Read fresh on every call rather than cached: a Nous access token expires - within the hour, and a long session would otherwise keep presenting a dead - bearer. Returns ``{}`` rather than raising when no token is available, so a - caller can report "sign in" instead of sending an unauthenticated request. - """ - if not is_managed_nous_gateway_url(url, gateway_builder): - return {} - - resolved_token_reader = token_reader or read_nous_access_token - try: - token = resolved_token_reader() - except Exception as exc: # pragma: no cover — defensive - logger.debug("Managed gateway token read failed for %s: %s", url, exc) - return {} - if not isinstance(token, str) or not token.strip(): - return {} - - return {"Authorization": f"Bearer {token.strip()}"} - - -# --------------------------------------------------------------------------- -# Managed media uploads -# --------------------------------------------------------------------------- -# -# Media arguments used to be inlined as base64, which capped a whole tool call -# at ~2MB of real bytes under the gateway's request ceiling and ruled out video -# entirely. Each pinned managed server carries an upload endpoint -# (`upload_path`); the bytes go straight to storage via a presigned URL, and -# the tool argument carries an opaque `nous-upload:` reference instead. -# -# The protocol lives HERE rather than in a vendor tool module: the presign -# request shape, the response contract, and the `nous-upload:` scheme are Nous -# gateway specifics shared by every managed vendor that takes media. - -_MEDIA_UPLOAD_PRESIGN_TIMEOUT_SECONDS = 15.0 -# The PUT carries up to 50MB of video; a flat 60s would fail a legitimate -# clip on an ordinary residential uplink, so only the write phase is long. -_MEDIA_UPLOAD_PUT_READ_TIMEOUT_SECONDS = 60.0 -_MEDIA_UPLOAD_PUT_WRITE_TIMEOUT_SECONDS = 300.0 - - -def _describe_media_upload_refusal(response) -> str: - """A model-actionable reason from a gateway refusal, or a generic one. - - The gateway's 4xx bodies carry deliberate guidance (rate-limit waits, size - caps, "you could not submit anyway"), so surface `error.message` verbatim - rather than a bare status code. - """ - try: - payload = response.json() - message = payload.get("error", {}).get("message") - if isinstance(message, str) and message.strip(): - return message.strip() - except Exception: - pass - return f"the gateway refused the upload (HTTP {response.status_code})" - - -def build_managed_media_uploader( - server_url: object, - upload_path: object, - gateway_builder: Optional[Callable[[str], str]] = None, - token_reader: Optional[Callable[[], Optional[str]]] = None, -) -> Optional[Callable]: - """Async ``(data, mime) -> argument value`` uploader for one managed vendor. - - Returns ``None`` when there is no usable upload endpoint (not a managed - Nous URL, or no ``upload_path``); callers then refuse local paths with a - clear message instead of silently forwarding them. - - The three steps of the protocol: - - 1. POST ``origin + upload_path`` with the declared content type and exact - byte length, using the same live auth headers as the vendor calls. - The gateway answers with a presigned single-object PUT URL (short - expiry; type and length are signed into it) and an upload token. - 2. PUT the bytes to that URL. This goes directly to storage — never - through the gateway — which is what removes the request-size ceiling. - 3. Return ``nous-upload:`` for the tool argument. The token is - bound to this Nous principal and is redeemable only through the - gateway, so it is inert anywhere else it might end up. - """ - if not is_managed_nous_gateway_url(server_url, gateway_builder): - return None - if not isinstance(upload_path, str) or not upload_path.startswith("/"): - return None - - parts = urlsplit(str(server_url).strip()) - origin = f"{parts.scheme}://{parts.netloc}" - presign_url = f"{origin}{upload_path}" - - async def upload(data: bytes, mime: str) -> str: - import httpx - - from tools.url_safety import create_ssrf_safe_async_client - - headers = managed_gateway_auth_headers(server_url, gateway_builder, token_reader) - if not headers: - raise RuntimeError("no Nous credential is available for the upload") - - # Two clients on purpose, split by whose address we are trusting. - # - # The presign POST goes to `presign_url`, which is entirely determined - # by the managed gateway origin (already validated by - # is_managed_nous_gateway_url) plus the pinned upload_path — the same - # first-party host the vendor calls go to freely. SSRF-guarding it - # protects against nothing and would reject a local gateway on - # 127.0.0.1, so it uses a plain client. The PUT target, by contrast, is - # a URL the gateway *returned*, so it keeps the SSRF-safe client as - # defense in depth (real presigned URLs are public R2, which it allows). - presign_timeout = httpx.Timeout(_MEDIA_UPLOAD_PRESIGN_TIMEOUT_SECONDS) - async with httpx.AsyncClient(timeout=presign_timeout) as client: - presign = await client.post( - presign_url, - headers=headers, - json={"contentType": mime, "contentLength": len(data)}, - ) - if presign.status_code != 200: - raise RuntimeError(_describe_media_upload_refusal(presign)) - - try: - payload = presign.json() - except Exception: - payload = None - upload_url = payload.get("uploadUrl") if isinstance(payload, dict) else None - token = payload.get("token") if isinstance(payload, dict) else None - if not (isinstance(upload_url, str) and upload_url and isinstance(token, str) and token): - raise RuntimeError("the gateway's upload response was malformed") - - put_timeout = httpx.Timeout( - _MEDIA_UPLOAD_PRESIGN_TIMEOUT_SECONDS, - read=_MEDIA_UPLOAD_PUT_READ_TIMEOUT_SECONDS, - write=_MEDIA_UPLOAD_PUT_WRITE_TIMEOUT_SECONDS, - ) - async with create_ssrf_safe_async_client(timeout=put_timeout) as client: - # The presigned URL signs the exact Content-Type and Content-Length, - # so this PUT must send precisely what was declared above. - put = await client.put(upload_url, content=data, headers={"Content-Type": mime}) - if put.status_code != 200: - raise RuntimeError(f"storage refused the upload (HTTP {put.status_code})") - - return f"nous-upload:{token}" - - return upload - diff --git a/tools/open_preview_tool.py b/tools/open_preview_tool.py index a2c44b14eb..03f3a44d5a 100644 --- a/tools/open_preview_tool.py +++ b/tools/open_preview_tool.py @@ -1,27 +1,23 @@ #!/usr/bin/env python3 """Open a URL, dev server, or file in the Hermes desktop GUI's preview pane. -Lives in the ``desktop_ui`` toolset, which the GUI gateway enables only for a -session whose source is the desktop app — so the schema never reaches a CLI, -messaging, or cron agent, and it DOES reach a desktop client on a remote/cloud -backend. Emits ``preview.open`` through the shared ``desktop_ui`` bridge; the -renderer opens the pane beside the chat for the window that asked and never -steals focus for a background session. +Registration moved into the `desktop_preview` tool; this module keeps the normalizer + +open action for ``tools.preview_tool``. Emits ``preview.open`` via ``desktop_ui``: the +renderer opens the pane for the window that asked and never steals focus for a +background session. The desktop_ui toolset reaches desktop clients on any backend. """ -import json import re from tools import desktop_ui -from tools.registry import registry, tool_error +from tools.registry import tool_error def _normalize_target(raw: str) -> str: """Coax a bare host/domain into a fetchable URL; leave paths + schemes alone. - ``www.cnn.com`` → ``https://www.cnn.com``; ``localhost:3000`` → - ``http://localhost:3000``. File paths and explicit schemes pass through for - the renderer's preview normalizer to classify. + ``www.cnn.com`` -> ``https://www.cnn.com``; ``localhost:3000`` -> ``http://localhost:3000``. + File paths and explicit schemes pass through for the renderer's preview normalizer. """ v = raw.strip().strip("`").strip() if not v or "://" in v or v.startswith(("/", "./", "../", "~", "file:")): @@ -43,46 +39,10 @@ def open_preview_tool(url: str, label: str = "") -> str: ) label = (label or "").strip() - try: - ok = desktop_ui.emit("preview.open", {"url": target, "label": label}) - except Exception as exc: - return tool_error(f"Failed to open the preview pane: {exc}") - if not ok: - return tool_error("The preview pane is only available in the Hermes desktop app.") - - return json.dumps({"success": True, "url": target, "label": label}, ensure_ascii=False) - - -OPEN_PREVIEW_SCHEMA = { - "name": "open_preview", - "description": ( - "Open something in the preview pane beside the chat in the Hermes desktop " - "app. Use this when the user asks to see a page, dev server, or file in the " - "preview pane — e.g. \"open cnn.com in the preview pane\" or \"preview " - "localhost:3000\". Accepts a web URL (a bare domain like www.cnn.com is fine), " - "a localhost dev-server URL, or a file path (HTML renders live; other files " - "show their contents). The pane opens for the current window only. To close " - "the pane or a tab, use close_preview." - ), - "parameters": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": ( - "What to preview: a web URL (https://… or a bare domain), a " - "localhost URL (localhost:3000), or a file path." - ), - }, - "label": { - "type": "string", - "description": "Optional tab label; defaults to the target's name.", - }, - }, - "required": ["url"], - }, -} - - -# Registration removed: consolidated into the `preview` tool (#95681); -# this module keeps its functions for the preview_tool. + return desktop_ui.emit_or_error( + "preview.open", + {"url": target, "label": label}, + "Failed to open the preview pane: ", + "The preview pane is only available in the Hermes desktop app.", + {"success": True, "url": target, "label": label}, + ) diff --git a/tools/openrouter_client.py b/tools/openrouter_client.py index 9c857076c8..83d8cbbc55 100644 --- a/tools/openrouter_client.py +++ b/tools/openrouter_client.py @@ -1,39 +1,13 @@ -"""Shared OpenRouter API client for Hermes tools. - -Provides a single lazy-initialized AsyncOpenAI client that all tool modules -can share. Routes through the centralized provider router in -agent/auxiliary_client.py so auth, headers, and API format are handled -consistently. -""" +"""OpenRouter API key probe shared by Hermes tools.""" import os -_client = None - - -def get_async_client(): - """Return a shared async OpenAI-compatible client for OpenRouter. - - The client is created lazily on first call and reused thereafter. - Uses the centralized provider router for auth and client construction. - Raises ValueError if OPENROUTER_API_KEY is not set. - """ - global _client - if _client is None: - from agent.auxiliary_client import resolve_provider_client - client, _model = resolve_provider_client("openrouter", async_mode=True) - if client is None: - raise ValueError("OPENROUTER_API_KEY environment variable not set") - _client = client - return _client - def check_api_key() -> bool: - """Check whether the OpenRouter API key is present. + """Return True if OPENROUTER_API_KEY is present. - Scope-aware (Slack pattern): tool paths run inside an installed profile - secret scope, whose verdict is authoritative under multiplex; unscoped - CLI probes keep the legacy env read. + Scope-aware: an installed profile secret scope is authoritative under + multiplex; unscoped CLI probes fall back to the plain env read. """ try: from agent.secret_scope import UnscopedSecretError, get_secret diff --git a/tools/osv_check.py b/tools/osv_check.py index b543dd7f33..69036d1a45 100644 --- a/tools/osv_check.py +++ b/tools/osv_check.py @@ -25,14 +25,11 @@ _OSV_ENDPOINT = os.getenv("OSV_ENDPOINT", "https://api.osv.dev/v1/query") _TIMEOUT = 10 # seconds # Result cache: (ecosystem, package, version) -> (expiry_monotonic, result). -# MCP reconnect ladders, stdio recycles, and parked-server self-probes re-run -# the preflight for the SAME package on every spawn attempt. Without a cache, -# a flapping server turns into a sustained OSV query/DNS stream — the #75485 -# incident logged 779K api.osv.dev DNS queries in 16h from revival loops. -# Malware advisories don't appear or vanish on second-to-second timescales, -# so a successful verdict (clean OR blocked) is reusable. Network failures -# are NOT cached: fail-open already covers them, and caching a failure could -# mask a real advisory once connectivity returns. +# MCP reconnect ladders and parked-server self-probes re-run the preflight for +# the SAME package on every spawn; uncached, a flapping server becomes a +# sustained OSV/DNS query stream. Advisories don't flip on second timescales, +# so clean AND blocked verdicts are reusable. Network failures are NOT cached: +# fail-open covers them and caching one could mask a real advisory later. _CACHE_TTL_S = float(os.getenv("OSV_CHECK_CACHE_TTL", "3600")) _CACHE_MAX_ENTRIES = 256 _cache: dict = {} @@ -111,14 +108,15 @@ def check_package_for_malware( return result +_ECOSYSTEM_BY_COMMAND = { + "npx": "npm", "npx.cmd": "npm", + "uvx": "PyPI", "uvx.cmd": "PyPI", "pipx": "PyPI", +} + + def _infer_ecosystem(command: str) -> Optional[str]: """Infer package ecosystem from the command name.""" - base = os.path.basename(command).lower() - if base in {"npx", "npx.cmd"}: - return "npm" - if base in {"uvx", "uvx.cmd", "pipx"}: - return "PyPI" - return None + return _ECOSYSTEM_BY_COMMAND.get(os.path.basename(command).lower()) def _parse_package_from_args( @@ -131,11 +129,9 @@ def _parse_package_from_args( if not args: return None, None - # Skip flags to find the package token. - # Honor npx's explicit install target: --package=NAME / --package NAME and - # the -p NAME short form, which name a package distinct from the executed - # binary. Without this the first bare positional (often the command name) - # is mistaken for the package. + # Skip flags to find the package token. Honor npx's explicit install target + # (--package=NAME / --package NAME / -p NAME), which names a package distinct + # from the executed binary; otherwise the first bare positional is used. package_token = None take_next = False for arg in args: @@ -157,12 +153,8 @@ def _parse_package_from_args( if not package_token: return None, None - - if ecosystem == "npm": - return _parse_npm_package(package_token) - elif ecosystem == "PyPI": - return _parse_pypi_package(package_token) - return package_token, None + parser = _PACKAGE_PARSERS.get(ecosystem) + return parser(package_token) if parser else (package_token, None) def _parse_npm_package(token: str) -> Tuple[Optional[str], Optional[str]]: @@ -191,6 +183,9 @@ def _parse_pypi_package(token: str) -> Tuple[Optional[str], Optional[str]]: return token, None +_PACKAGE_PARSERS = {"npm": _parse_npm_package, "PyPI": _parse_pypi_package} + + def _query_osv( package: str, ecosystem: str, version: Optional[str] = None ) -> list: diff --git a/tools/patch_parser.py b/tools/patch_parser.py index a95859d5e4..e48e1b84e5 100644 --- a/tools/patch_parser.py +++ b/tools/patch_parser.py @@ -1,39 +1,28 @@ #!/usr/bin/env python3 -""" -V4A Patch Format Parser +"""V4A patch format parser and applier (format used by codex, cline, etc.). -Parses the V4A patch format used by codex, cline, and other coding agents. - -V4A Format: *** Begin Patch *** Update File: path/to/file.py @@ optional context hint @@ context line (space prefix) - -removed line (minus prefix) - +added line (plus prefix) + -removed line + +added line *** Add File: path/to/new.py +new file content - +line 2 *** Delete File: path/to/old.py *** Move File: old/path.py -> new/path.py *** End Patch -Usage: - from tools.patch_parser import parse_v4a_patch, apply_v4a_operations - operations, error = parse_v4a_patch(patch_content) - if error: - print(f"Parse error: {error}") - else: - result = apply_v4a_operations(operations, file_ops) + result = apply_v4a_operations(operations, file_ops) """ import difflib import inspect import re from dataclasses import dataclass, field -from typing import List, Optional, Tuple, Any from enum import Enum +from typing import Any, Callable, Dict, List, Optional, Tuple class OperationType(Enum): @@ -67,157 +56,80 @@ class PatchOperation: content: Optional[str] = None # For add file operations -def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[str]]: - """ - Parse a V4A format patch. - - Args: - patch_content: The patch text in V4A format - - Returns: - Tuple of (operations, error_message) - - If successful: (list_of_operations, None) - - If failed: ([], error_description) - """ - # Split into lines, tolerating a CRLF patch body: strip the trailing - # ``\r`` from each line. Without this, a CRLF-encoded patch keeps ``\r`` - # inside every HunkLine.content and injects stray carriage returns into an - # LF target file (and the anchored ``...\s*$`` Begin/End markers would fail - # to match because of the trailing ``\r``). - lines = [ln[:-1] if ln.endswith('\r') else ln for ln in patch_content.split('\n')] - operations: List[PatchOperation] = [] +# Markers must occupy the whole line at column 0 so content lines that merely +# mention the format ("+*** End Patch") can't truncate or reset the patch. +_BEGIN_MARKER = re.compile(r'^\*\*\*\s*Begin\s+Patch\s*$') +_END_MARKER = re.compile(r'^\*\*\*\s*End\s+Patch\s*$') +_OP_MARKERS: List[Tuple[OperationType, re.Pattern]] = [ + (OperationType.UPDATE, re.compile(r'\*\*\*\s*Update\s+File:\s*(.+)')), + (OperationType.ADD, re.compile(r'\*\*\*\s*Add\s+File:\s*(.+)')), + (OperationType.DELETE, re.compile(r'\*\*\*\s*Delete\s+File:\s*(.+)')), + (OperationType.MOVE, re.compile(r'\*\*\*\s*Move\s+File:\s*(.+?)\s*->\s*(.+)')), +] +_HINT_RE = re.compile(r'@@\s*(.+?)\s*@@') - # Find patch boundaries. Markers must occupy the whole line at column 0: - # content lines like "+*** End Patch" or " *** End Patch" (e.g. docs - # about the patch format) must not truncate the patch or reset the - # start boundary. - start_idx = None - end_idx = None - begin_marker = re.compile(r'^\*\*\*\s*Begin\s+Patch\s*$') - end_marker = re.compile(r'^\*\*\*\s*End\s+Patch\s*$') + +def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[str]]: + """Parse a V4A patch into operations. + + Returns ``(operations, None)`` — ``[]`` for an empty patch is not an + error — or ``([], "Parse error: ...")`` for malformed operations. + """ + # Tolerate CRLF bodies: a stray ``\r`` would otherwise end up in every + # HunkLine.content and defeat the anchored Begin/End markers. + lines = [ln[:-1] if ln.endswith('\r') else ln for ln in patch_content.split('\n')] + + start_idx = -1 # parse from the top when no Begin marker is present + end_idx = len(lines) for i, line in enumerate(lines): - if begin_marker.match(line): + if _BEGIN_MARKER.match(line): start_idx = i - elif end_marker.match(line): + elif _END_MARKER.match(line): end_idx = i break - - if start_idx is None: - # Try to parse without explicit begin marker - start_idx = -1 - - if end_idx is None: - end_idx = len(lines) - - # Parse operations between boundaries - i = start_idx + 1 + + operations: List[PatchOperation] = [] current_op: Optional[PatchOperation] = None current_hunk: Optional[Hunk] = None - - while i < end_idx: - line = lines[i] - - # Check for file operation markers - update_match = re.match(r'\*\*\*\s*Update\s+File:\s*(.+)', line) - add_match = re.match(r'\*\*\*\s*Add\s+File:\s*(.+)', line) - delete_match = re.match(r'\*\*\*\s*Delete\s+File:\s*(.+)', line) - move_match = re.match(r'\*\*\*\s*Move\s+File:\s*(.+?)\s*->\s*(.+)', line) - - if update_match: - # Save previous operation - if current_op: - if current_hunk and current_hunk.lines: - current_op.hunks.append(current_hunk) - operations.append(current_op) - - current_op = PatchOperation( - operation=OperationType.UPDATE, - file_path=update_match.group(1).strip() - ) - current_hunk = None - - elif add_match: - if current_op: - if current_hunk and current_hunk.lines: - current_op.hunks.append(current_hunk) - operations.append(current_op) - - current_op = PatchOperation( - operation=OperationType.ADD, - file_path=add_match.group(1).strip() - ) - current_hunk = Hunk() - - elif delete_match: - if current_op: - if current_hunk and current_hunk.lines: - current_op.hunks.append(current_hunk) - operations.append(current_op) - - current_op = PatchOperation( - operation=OperationType.DELETE, - file_path=delete_match.group(1).strip() - ) + + def _flush() -> None: + if current_op: + if current_hunk and current_hunk.lines: + current_op.hunks.append(current_hunk) operations.append(current_op) - current_op = None - current_hunk = None - - elif move_match: - if current_op: - if current_hunk and current_hunk.lines: - current_op.hunks.append(current_hunk) - operations.append(current_op) - + + for line in lines[start_idx + 1:end_idx]: + op_match = next( + ((kind, m) for kind, rx in _OP_MARKERS if (m := rx.match(line))), None, + ) + if op_match: + kind, m = op_match + _flush() current_op = PatchOperation( - operation=OperationType.MOVE, - file_path=move_match.group(1).strip(), - new_path=move_match.group(2).strip() + operation=kind, + file_path=m.group(1).strip(), + new_path=m.group(2).strip() if kind is OperationType.MOVE else None, ) - operations.append(current_op) - current_op = None - current_hunk = None - + # UPDATE hunks start lazily (at '@@' or the first hunk line); ADD + # collects all '+' lines into one hunk; DELETE/MOVE are complete. + current_hunk = Hunk() if kind is OperationType.ADD else None + if kind in (OperationType.DELETE, OperationType.MOVE): + operations.append(current_op) + current_op = None elif line.startswith('@@'): - # Context hint / hunk marker if current_op: if current_hunk and current_hunk.lines: current_op.hunks.append(current_hunk) - - # Extract context hint - hint_match = re.match(r'@@\s*(.+?)\s*@@', line) - hint = hint_match.group(1) if hint_match else None - current_hunk = Hunk(context_hint=hint) - + hint_match = _HINT_RE.match(line) + current_hunk = Hunk(context_hint=hint_match.group(1) if hint_match else None) elif current_op and line: - # Parse hunk line if current_hunk is None: current_hunk = Hunk() - - if line.startswith('+'): - current_hunk.lines.append(HunkLine('+', line[1:])) - elif line.startswith('-'): - current_hunk.lines.append(HunkLine('-', line[1:])) - elif line.startswith(' '): - current_hunk.lines.append(HunkLine(' ', line[1:])) - elif line.startswith('\\'): - # "\ No newline at end of file" marker - skip - pass - else: - # Treat as context line (implicit space prefix) - current_hunk.lines.append(HunkLine(' ', line)) - - i += 1 - - # Don't forget the last operation - if current_op: - if current_hunk and current_hunk.lines: - current_op.hunks.append(current_hunk) - operations.append(current_op) - - # Validate the parsed result - if not operations: - # Empty patch is not an error — callers get [] and can decide - return operations, None + if line[0] in '+- ': + current_hunk.lines.append(HunkLine(line[0], line[1:])) + elif line[0] != '\\': # "\ No newline at end of file" marker is skipped + current_hunk.lines.append(HunkLine(' ', line)) # implicit context line + _flush() parse_errors: List[str] = [] for op in operations: @@ -227,15 +139,13 @@ def parse_v4a_patch(patch_content: str) -> Tuple[List[PatchOperation], Optional[ parse_errors.append(f"UPDATE {op.file_path!r}: no hunks found") if op.operation == OperationType.MOVE and not op.new_path: parse_errors.append(f"MOVE {op.file_path!r}: missing destination path (expected 'src -> dst')") - if parse_errors: return [], "Parse error: " + "; ".join(parse_errors) - return operations, None def _count_occurrences(text: str, pattern: str) -> int: - """Count non-overlapping occurrences of *pattern* in *text*.""" + """Count occurrences of *pattern* in *text*, advancing one char per hit (overlaps count).""" count = 0 start = 0 while True: @@ -247,33 +157,42 @@ def _count_occurrences(text: str, pattern: str) -> int: return count +def _split_hunk(hunk: Hunk) -> Tuple[List[str], List[str]]: + """``(search_lines, replace_lines)``: context+removed vs context+added.""" + search = [l.content for l in hunk.lines if l.prefix in {' ', '-'}] + replace = [l.content for l in hunk.lines if l.prefix in {' ', '+'}] + return search, replace + + +def _no_match_hint(error: Optional[str], search_pattern: str, content: str) -> str: + """Best-effort 'Did you mean...' suffix; never lets a hint failure mask the real error.""" + try: + from tools.fuzzy_match import format_no_match_hint + return format_no_match_hint(error, 0, search_pattern, content) + except Exception: + return "" + + def _validate_operations( operations: List[PatchOperation], file_ops: Any, ) -> List[str]: - """Validate all operations without writing any files. + """Dry-run every operation; return error strings (empty list = safe to apply). - Returns a list of error strings; an empty list means all operations - are valid and the apply phase can proceed safely. - - For UPDATE operations, hunks are simulated in order so that later - hunks validate against post-earlier-hunk content (matching apply order). + UPDATE hunks are simulated in order so later hunks validate against + post-earlier-hunk content, exactly as the apply phase will see it. """ - # Deferred import: breaks the patch_parser ↔ fuzzy_match circular dependency - from tools.fuzzy_match import fuzzy_find_and_replace + from tools.fuzzy_match import fuzzy_find_and_replace, is_already_applied errors: List[str] = [] real_change_count = 0 - # Virtual filesystem overlay so inter-op state (notably a MOVE creating the - # destination a later UPDATE targets) validates correctly. Maps a path to - # its pending content; ``None`` marks a path moved/deleted away. UPDATE and - # MOVE reads consult this overlay before hitting disk. + # Virtual overlay so inter-op state validates (e.g. a MOVE creating the + # destination a later UPDATE targets). UPDATE/MOVE reads consult it first. pending_content: dict = {} # path -> content produced by an earlier op removed_paths: set = set() # paths a MOVE/DELETE has taken away def _read(path: str): - """Read a path honoring the pending-move overlay.""" if path in removed_paths and path not in pending_content: return None, "file not found" if path in pending_content: @@ -294,16 +213,14 @@ def _validate_operations( simulated = content for hunk_index, hunk in enumerate(op.hunks, start=1): - search_lines = [l.content for l in hunk.lines if l.prefix in {' ', '-'}] - removed_lines = [l.content for l in hunk.lines if l.prefix == '-'] - added_lines = [l.content for l in hunk.lines if l.prefix == '+'] - if not removed_lines and not added_lines: - # Models occasionally emit inert anchor hunks between real - # changes. Ignore them without poisoning the atomic patch. + search_lines, replace_lines = _split_hunk(hunk) + if not any(l.prefix in '-+' for l in hunk.lines): + # Inert anchor hunk (context only) — models emit these + # between real changes; ignore without failing the patch. continue real_change_count += 1 if not search_lines: - # Addition-only hunk: validate context hint uniqueness + # Addition-only hunk: the context hint must be unique. if hunk.context_hint: occurrences = _count_occurrences(simulated, hunk.context_hint) if occurrences == 0: @@ -320,47 +237,29 @@ def _validate_operations( continue search_pattern = '\n'.join(search_lines) - replace_lines = [l.content for l in hunk.lines if l.prefix in {' ', '+'}] replacement = '\n'.join(replace_lines) - if search_lines == replace_lines: - # Degenerate hunk whose -/+ lines are identical: the apply - # phase skips it as a no-op, so validation must not fail it - # — fuzzy_find_and_replace would reject the identical - # search/replacement with old_string/new_string guidance - # that has no meaning in V4A patch mode. + # Identical -/+ lines: apply skips it as a no-op, so + # validation must not reject it with the identical-strings error. continue new_simulated, count, _strategy, match_error = fuzzy_find_and_replace( simulated, search_pattern, replacement, replace_all=False ) if count == 0: - # Already-applied hunk: validate as a no-op when the - # replacement text is already present (and the search - # text gone) — the edit landed earlier. Keeps multi-hunk - # patches from failing wholesale because one hunk was - # already applied in a prior call. The apply phase - # performs the same skip. - from tools.fuzzy_match import is_already_applied + # Already-applied hunk (edit landed in a prior call): treat + # as a no-op so multi-hunk patches don't fail wholesale. + # The apply phase performs the same skip. if is_already_applied(simulated or "", search_pattern, replacement): continue label = f"'{hunk.context_hint}'" if hunk.context_hint else "(no hint)" - msg = ( + errors.append( f"{op.file_path}: hunk {hunk_index} {label} not found" + (f" — {match_error}" if match_error else "") + + _no_match_hint(match_error, search_pattern, simulated) ) - try: - from tools.fuzzy_match import format_no_match_hint - msg += format_no_match_hint(match_error, count, search_pattern, simulated) - except Exception: - pass - errors.append(msg) else: - # Advance simulation so subsequent hunks validate correctly. - # Reuse the result from the call above — no second fuzzy run. simulated = new_simulated - # Record the post-update content so a later op (e.g. a MOVE of this - # file) sees the edited version in the overlay. pending_content[op.file_path] = simulated elif op.operation == OperationType.DELETE: @@ -378,14 +277,12 @@ def _validate_operations( src_content, src_err = _read(op.file_path) if src_err: errors.append(f"{op.file_path}: source file not found for move") - dst_content, dst_err = _read(op.new_path) + _dst, dst_err = _read(op.new_path) if not dst_err: errors.append( f"{op.new_path}: destination already exists — move would overwrite" ) - # Reflect the move in the overlay so a subsequent UPDATE of the - # destination validates against the moved content, and the source - # reads as gone. Only when the move itself validated cleanly. + # Only a cleanly-validated move updates the overlay. if not src_err and dst_err: pending_content[op.new_path] = src_content if src_content is not None else "" pending_content.pop(op.file_path, None) @@ -395,32 +292,23 @@ def _validate_operations( if not errors and real_change_count == 0: errors.append("Patch contains no changes (only context lines were provided)") - return errors +# Every _apply_* returns (success, diff_or_error, lsp_diagnostics, lint_result). +ApplyResult = Tuple[bool, str, Optional[str], Optional[dict]] + + def apply_v4a_operations(operations: List[PatchOperation], - file_ops: Any) -> 'PatchResult': - """Apply V4A patch operations using a file operations interface. + file_ops: Any) -> 'PatchResult': + """Validate all operations, then apply them (two-phase, atomic on validation failure). - Uses a two-phase validate-then-apply approach: - - Phase 1: validate all operations against current file contents without - writing anything. If any validation error is found, return immediately - with no filesystem changes. - - Phase 2: apply all operations. A failure here (e.g. a race between - validation and apply) is reported with a note to run ``git diff``. - - Args: - operations: List of PatchOperation from parse_v4a_patch - file_ops: Object with read_file_raw, write_file methods - - Returns: - PatchResult with results of all operations + A phase-2 failure (e.g. a race between validation and apply) is reported + with a note to run ``git diff`` since state may be inconsistent. + ``file_ops`` needs ``read_file_raw``, ``write_file``, ``delete_file``, ``move_file``. """ - # Import here to avoid circular imports - from tools.file_operations import PatchResult + from tools.file_operations import PatchResult # avoid circular import - # ---- Phase 1: validate ---- validation_errors = _validate_operations(operations, file_ops) if validation_errors: return PatchResult( @@ -429,111 +317,69 @@ def apply_v4a_operations(operations: List[PatchOperation], + "\n".join(f" • {e}" for e in validation_errors), ) - # ---- Phase 2: apply ---- - files_modified = [] - files_created = [] - files_deleted = [] - all_diffs = [] - # Per-file LSP diagnostics blocks captured from underlying write_file - # calls. V4A bypasses the WriteResult / PatchResult plumbing that - # write_file and patch_replace use, so without explicit propagation - # the LSP tier's output gets silently dropped — see - # ``PatchResult.lsp_diagnostics`` aggregation below. + files_modified: List[str] = [] + files_created: List[str] = [] + files_deleted: List[str] = [] + all_diffs: List[str] = [] + # V4A bypasses the WriteResult/PatchResult plumbing that write_file uses, + # so LSP diagnostics and lint must be propagated explicitly per file. lsp_blocks: List[str] = [] - errors = [] - lint_results = {} + errors: List[str] = [] + lint_results: Dict[str, dict] = {} + + dispatch: Dict[OperationType, Tuple[Callable[[PatchOperation, Any], ApplyResult], List[str], str]] = { + OperationType.ADD: (_apply_add, files_created, "add"), + OperationType.DELETE: (_apply_delete, files_deleted, "delete"), + OperationType.MOVE: (_apply_move, files_modified, "move"), + OperationType.UPDATE: (_apply_update, files_modified, "update"), + } for op in operations: try: - if op.operation == OperationType.ADD: - result = _apply_add(op, file_ops) - if result[0]: - files_created.append(op.file_path) - all_diffs.append(result[1]) - if result[2]: - lsp_blocks.append(result[2]) - if result[3]: - lint_results[op.file_path] = result[3] - else: - errors.append(f"Failed to add {op.file_path}: {result[1]}") - - elif op.operation == OperationType.DELETE: - result = _apply_delete(op, file_ops) - if result[0]: - files_deleted.append(op.file_path) - all_diffs.append(result[1]) - else: - errors.append(f"Failed to delete {op.file_path}: {result[1]}") - - elif op.operation == OperationType.MOVE: - result = _apply_move(op, file_ops) - if result[0]: - files_modified.append(f"{op.file_path} -> {op.new_path}") - all_diffs.append(result[1]) - else: - errors.append(f"Failed to move {op.file_path}: {result[1]}") - - elif op.operation == OperationType.UPDATE: - result = _apply_update(op, file_ops) - if result[0]: - files_modified.append(op.file_path) - all_diffs.append(result[1]) - if result[2]: - lsp_blocks.append(result[2]) - if result[3]: - lint_results[op.file_path] = result[3] - else: - errors.append(f"Failed to update {op.file_path}: {result[1]}") - + handler, bucket, verb = dispatch[op.operation] + ok, payload, lsp, lint = handler(op, file_ops) + if not ok: + errors.append(f"Failed to {verb} {op.file_path}: {payload}") + continue + label = op.file_path + if op.operation is OperationType.MOVE: + label = f"{op.file_path} -> {op.new_path}" + bucket.append(label) + all_diffs.append(payload) + if lsp: + lsp_blocks.append(lsp) + if lint: + lint_results[op.file_path] = lint except Exception as e: errors.append(f"Error processing {op.file_path}: {str(e)}") - # Lint results were collected from write_file's internal _check_lint_delta - # via the four-tuple return of _apply_add / _apply_update — zero extra - # subprocess calls vs. the old approach of re-reading each file with a - # bare _check_lint(f) that lacked post_content context. - - combined_diff = '\n'.join(all_diffs) - - # Combine per-file LSP diagnostics blocks. Each block already has - # the ```` header from - # ``LSPService.report_for_file`` so concatenation is safe — the - # agent (and any downstream parsers) can still attribute each - # diagnostic to its file. - combined_lsp = "\n\n".join(lsp_blocks) if lsp_blocks else None - - if errors: - return PatchResult( - success=False, - diff=combined_diff, - files_modified=files_modified, - files_created=files_created, - files_deleted=files_deleted, - lint=lint_results if lint_results else None, - lsp_diagnostics=combined_lsp, - error="Apply phase failed (state may be inconsistent — run `git diff` to assess):\n" - + "\n".join(f" • {e}" for e in errors), - ) - - return PatchResult( - success=True, - diff=combined_diff, + # Each LSP block carries its own header, so plain + # concatenation keeps per-file attribution. + result_kwargs = dict( + diff='\n'.join(all_diffs), files_modified=files_modified, files_created=files_created, files_deleted=files_deleted, lint=lint_results if lint_results else None, - lsp_diagnostics=combined_lsp, + lsp_diagnostics="\n\n".join(lsp_blocks) if lsp_blocks else None, ) + if errors: + return PatchResult( + success=False, + error="Apply phase failed (state may be inconsistent — run `git diff` to assess):\n" + + "\n".join(f" • {e}" for e in errors), + **result_kwargs, + ) + return PatchResult(success=True, **result_kwargs) def _write_file_accepts_pre_content(file_ops: Any) -> bool: """True when ``file_ops.write_file`` accepts a ``pre_content`` kwarg. - Decided from the signature (not by catching TypeError around the call) - so a TypeError raised *inside* a capable ``write_file`` propagates - instead of triggering a second, duplicate write. Unintrospectable - callables (some C-implemented ones) conservatively get the basic - two-argument form. + Decided from the signature rather than catching TypeError around the call, + so a TypeError raised *inside* a capable write_file propagates instead of + triggering a second, duplicate write. Unintrospectable callables get the + basic two-argument form. """ try: params = inspect.signature(file_ops.write_file).parameters @@ -544,194 +390,118 @@ def _write_file_accepts_pre_content(file_ops: Any) -> bool: ) -def _apply_add(op: PatchOperation, file_ops: Any) -> Tuple[bool, str, Optional[str], Optional[dict]]: - """Apply an add file operation. - - Returns ``(success, diff_or_error, lsp_diagnostics, lint_result)``. - The third element carries the formatted ```` block from - :class:`WriteResult.lsp_diagnostics` so V4A patches can surface - semantic diagnostics from the LSP layer. The fourth element carries - the ``WriteResult.lint`` dict (syntax check result) so V4A patches - can propagate lint to ``PatchResult.lint`` without a redundant - ``_check_lint`` re-read — write_file already ran the check internally. - """ - # Extract content from hunks (all + lines) - content_lines = [] - for hunk in op.hunks: - for line in hunk.lines: - if line.prefix == '+': - content_lines.append(line.content) - - content = '\n'.join(content_lines) - - # _apply_add creates a new file, no pre_content to pass - result = file_ops.write_file(op.file_path, content) +def _apply_add(op: PatchOperation, file_ops: Any) -> ApplyResult: + """Create a file from the hunks' '+' lines.""" + content_lines = [line.content for hunk in op.hunks for line in hunk.lines if line.prefix == '+'] + result = file_ops.write_file(op.file_path, '\n'.join(content_lines)) if result.error: return False, result.error, None, None - - diff = f"--- /dev/null\n+++ b/{op.file_path}\n" - diff += '\n'.join(f"+{line}" for line in content_lines) - + diff = f"--- /dev/null\n+++ b/{op.file_path}\n" + '\n'.join(f"+{line}" for line in content_lines) return True, diff, getattr(result, "lsp_diagnostics", None), getattr(result, "lint", None) -def _apply_delete(op: PatchOperation, file_ops: Any) -> Tuple[bool, str]: - """Apply a delete file operation.""" - # Read before deleting so we can produce a real unified diff. - # Validation already confirmed existence; this guards against races. +def _apply_delete(op: PatchOperation, file_ops: Any) -> ApplyResult: + """Delete a file, producing a real unified diff of the removed content.""" + # Validation already confirmed existence; the re-read guards against races. read_result = file_ops.read_file_raw(op.file_path) if read_result.error: - return False, f"Cannot delete {op.file_path}: file not found" - + return False, f"Cannot delete {op.file_path}: file not found", None, None result = file_ops.delete_file(op.file_path) if result.error: - return False, result.error - - removed_lines = read_result.content.splitlines(keepends=True) + return False, result.error, None, None diff = ''.join(difflib.unified_diff( - removed_lines, [], - fromfile=f"a/{op.file_path}", - tofile="/dev/null", + read_result.content.splitlines(keepends=True), [], + fromfile=f"a/{op.file_path}", tofile="/dev/null", )) - return True, diff or f"# Deleted: {op.file_path}" + return True, diff or f"# Deleted: {op.file_path}", None, None -def _apply_move(op: PatchOperation, file_ops: Any) -> Tuple[bool, str]: - """Apply a move file operation.""" +def _apply_move(op: PatchOperation, file_ops: Any) -> ApplyResult: result = file_ops.move_file(op.file_path, op.new_path) if result.error: - return False, result.error - - diff = f"# Moved: {op.file_path} -> {op.new_path}" - return True, diff + return False, result.error, None, None + return True, f"# Moved: {op.file_path} -> {op.new_path}", None, None -def _apply_update(op: PatchOperation, file_ops: Any) -> Tuple[bool, str, Optional[str], Optional[dict]]: - """Apply an update file operation. +def _insert_addition_only(new_content: str, hunk: Hunk, insert_text: str) -> Tuple[Optional[str], Optional[str]]: + """Place an addition-only hunk after its context hint (or at EOF). Returns (content, error).""" + if hunk.context_hint: + occurrences = _count_occurrences(new_content, hunk.context_hint) + if occurrences > 1: + return None, ( + f"Addition-only hunk: context hint '{hunk.context_hint}' is ambiguous " + f"({occurrences} occurrences) — provide a more unique hint" + ) + if occurrences == 1: + hint_pos = new_content.find(hunk.context_hint) + eol = new_content.find('\n', hint_pos) + if eol != -1: + return new_content[:eol + 1] + insert_text + '\n' + new_content[eol + 1:], None + return new_content + '\n' + insert_text, None + # Hint not found — append at end as a safe fallback. + return new_content.rstrip('\n') + '\n' + insert_text + '\n', None - Returns ``(success, diff_or_error, lsp_diagnostics, lint_result)`` — see - :func:`_apply_add` for the rationale on the third and fourth elements. - """ - # Deferred import: breaks the patch_parser ↔ fuzzy_match circular dependency - from tools.fuzzy_match import fuzzy_find_and_replace - # Read current content — raw so no line-number prefixes or per-line truncation +def _apply_update(op: PatchOperation, file_ops: Any) -> ApplyResult: + """Apply each hunk via fuzzy replace, then write once.""" + from tools.fuzzy_match import fuzzy_find_and_replace, is_already_applied + + # Raw read: no line-number prefixes or per-line truncation. read_result = file_ops.read_file_raw(op.file_path) - if read_result.error: return False, f"Cannot read file: {read_result.error}", None, None - current_content = read_result.content - - # Apply each hunk new_content = current_content for hunk in op.hunks: - # Build search pattern from context and removed lines - search_lines = [] - replace_lines = [] - - for line in hunk.lines: - if line.prefix == ' ': - search_lines.append(line.content) - replace_lines.append(line.content) - elif line.prefix == '-': - search_lines.append(line.content) - elif line.prefix == '+': - replace_lines.append(line.content) - + search_lines, replace_lines = _split_hunk(hunk) if search_lines and search_lines == replace_lines: continue - if search_lines: - search_pattern = '\n'.join(search_lines) - replacement = '\n'.join(replace_lines) + if not search_lines: + new_content, err = _insert_addition_only(new_content, hunk, '\n'.join(replace_lines)) + if err: + return False, err, None, None + continue - new_content, count, _strategy, error = fuzzy_find_and_replace( - new_content, search_pattern, replacement, replace_all=False - ) + search_pattern = '\n'.join(search_lines) + replacement = '\n'.join(replace_lines) + new_content, count, _strategy, error = fuzzy_find_and_replace( + new_content, search_pattern, replacement, replace_all=False + ) + if not (error and count == 0): + continue - if error and count == 0: - # Try with context hint if available - if hunk.context_hint: - # Find the context hint location and search nearby - hint_pos = new_content.find(hunk.context_hint) - if hint_pos != -1: - # Search in a window around the hint - window_start = max(0, hint_pos - 500) - window_end = min(len(new_content), hint_pos + 2000) - window = new_content[window_start:window_end] + # Retry inside a window around the context hint, if any. + if hunk.context_hint: + hint_pos = new_content.find(hunk.context_hint) + if hint_pos != -1: + window_start = max(0, hint_pos - 500) + window_end = min(len(new_content), hint_pos + 2000) + window_new, count, _strategy, error = fuzzy_find_and_replace( + new_content[window_start:window_end], search_pattern, replacement, replace_all=False + ) + if count > 0: + new_content = new_content[:window_start] + window_new + new_content[window_end:] + error = None + if error: + # Mirror the validation-phase already-applied skip, or the two + # phases disagree and the whole patch fails here. + if is_already_applied(new_content, search_pattern, replacement): + continue + return False, f"Could not apply hunk: {error}" + _no_match_hint(error, search_pattern, new_content), None, None - window_new, count, _strategy, error = fuzzy_find_and_replace( - window, search_pattern, replacement, replace_all=False - ) - - if count > 0: - new_content = new_content[:window_start] + window_new + new_content[window_end:] - error = None - - if error: - # Already-applied hunk: skip it, mirroring the - # validation-phase check (validation may also have - # passed via this path, so apply MUST skip too or the - # two phases disagree and the whole patch fails here). - from tools.fuzzy_match import is_already_applied - if is_already_applied(new_content, search_pattern, replacement): - continue - err_msg = f"Could not apply hunk: {error}" - try: - from tools.fuzzy_match import format_no_match_hint - err_msg += format_no_match_hint(error, 0, search_pattern, new_content) - except Exception: - pass - return False, err_msg, None, None - else: - # Addition-only hunk (no context or removed lines). - # Insert at the location indicated by the context hint, or at end of file. - insert_text = '\n'.join(replace_lines) - if hunk.context_hint: - occurrences = _count_occurrences(new_content, hunk.context_hint) - if occurrences == 0: - # Hint not found — append at end as a safe fallback - new_content = new_content.rstrip('\n') + '\n' + insert_text + '\n' - elif occurrences > 1: - return False, ( - f"Addition-only hunk: context hint '{hunk.context_hint}' is ambiguous " - f"({occurrences} occurrences) — provide a more unique hint" - ), None, None - else: - hint_pos = new_content.find(hunk.context_hint) - # Insert after the line containing the context hint - eol = new_content.find('\n', hint_pos) - if eol != -1: - new_content = new_content[:eol + 1] + insert_text + '\n' + new_content[eol + 1:] - else: - new_content = new_content + '\n' + insert_text - else: - new_content = new_content.rstrip('\n') + '\n' + insert_text + '\n' - - # Write new content — pass current_content (already read above) to avoid - # a redundant cat subprocess inside write_file. Fall back to the - # two-argument form when the file_ops implementation doesn't accept - # ``pre_content`` (duck-typed callers that only implement the basic - # ``write_file(path, content)`` contract). Feature-detect via the - # signature instead of catching TypeError around the call: a TypeError - # raised *inside* a pre_content-capable write_file must propagate, not - # trigger a second (double) write. + # Pass pre_content to skip a redundant re-read inside write_file when supported. if _write_file_accepts_pre_content(file_ops): - write_result = file_ops.write_file(op.file_path, new_content, - pre_content=current_content) + write_result = file_ops.write_file(op.file_path, new_content, pre_content=current_content) else: write_result = file_ops.write_file(op.file_path, new_content) if write_result.error: return False, write_result.error, None, None - - # Generate diff - diff_lines = difflib.unified_diff( + + diff = ''.join(difflib.unified_diff( current_content.splitlines(keepends=True), new_content.splitlines(keepends=True), fromfile=f"a/{op.file_path}", - tofile=f"b/{op.file_path}" - ) - diff = ''.join(diff_lines) - + tofile=f"b/{op.file_path}", + )) return True, diff, getattr(write_result, "lsp_diagnostics", None), getattr(write_result, "lint", None) diff --git a/tools/path_security.py b/tools/path_security.py index 9d73a613c5..79051e53db 100644 --- a/tools/path_security.py +++ b/tools/path_security.py @@ -1,9 +1,4 @@ -"""Shared path validation helpers for tool implementations. - -Extracts the ``resolve() + relative_to()`` and ``..`` traversal check -patterns previously duplicated across skill_manager_tool, skills_tool, -skills_hub, cronjob_tools, and credential_files. -""" +"""Shared path validation helpers for tool implementations (skills, cron, credential files).""" import logging from pathlib import Path @@ -13,31 +8,17 @@ logger = logging.getLogger(__name__) def validate_within_dir(path: Path, root: Path) -> Optional[str]: - """Ensure *path* resolves to a location within *root*. + """Return an error message if *path* does not resolve inside *root*, else None. - Returns an error message string if validation fails, or ``None`` if the - path is safe. Uses ``Path.resolve()`` to follow symlinks and normalize - ``..`` components. - - Usage:: - - error = validate_within_dir(user_path, allowed_root) - if error: - return tool_error(error) + ``Path.resolve()`` follows symlinks and normalises ``..`` before the check. """ try: - resolved = path.resolve() - root_resolved = root.resolve() - resolved.relative_to(root_resolved) + path.resolve().relative_to(root.resolve()) except (ValueError, OSError) as exc: return f"Path escapes allowed directory: {exc}" return None def has_traversal_component(path_str: str) -> bool: - """Return True if *path_str* contains ``..`` traversal components. - - Quick check for obvious traversal attempts before doing full resolution. - """ - parts = Path(path_str).parts - return ".." in parts + """Cheap pre-check for a literal ``..`` component before full resolution.""" + return ".." in Path(path_str).parts diff --git a/tools/plugin_guard.py b/tools/plugin_guard.py index 8f7de1f04a..250d10808a 100644 --- a/tools/plugin_guard.py +++ b/tools/plugin_guard.py @@ -1,39 +1,26 @@ #!/usr/bin/env python3 -""" -Plugin Guard — Security scanner for externally-installed plugins. +"""Plugin Guard — security scanner for externally-installed plugins. -Inspired by Claude Cowork's skill & plugin security scanning (announced -2026-08-06: third-party skills and plugins are automatically checked for -malicious content when someone uploads or edits them, returning pass / -warn / fail). Hermes already scans hub-installed *skills* via -``tools/skills_guard.py``; this module extends the same static-analysis -engine to ``hermes plugins install`` and ``hermes plugins update``, which -previously cloned and executed arbitrary Git repositories unscanned. +Extends the ``tools/skills_guard.py`` static-analysis engine to +``hermes plugins install`` / ``update``, which otherwise clone and execute +arbitrary Git repositories unscanned. -Plugins are strictly more dangerous than skills — they run Python -in-process with the agent — but they are also *expected* to do things a -skill never should: read their own API keys from environment variables -(the documented ``requires_env`` pattern), call provider HTTP APIs with -those keys, and spawn subprocesses. A naive reuse of the skill threat -patterns would flag every legitimate provider plugin. So this scanner: +Plugins run Python in-process, so they are more dangerous than skills — but +they are also *expected* to read their own API keys from env vars, call +provider HTTP APIs with them, and spawn subprocesses. Reusing the skill +patterns naively would flag every legitimate provider plugin, so this scanner: -- Runs the full skills_guard pattern set on documentation/config files - (README, after-install.md, plugin.yaml, ...), where prompt-injection - and social-engineering content lives. -- Exempts the "reads own env secret" / "HTTP call with key" pattern - family on *code* files, while keeping genuinely malicious signals: - foreign credential-store access (~/.ssh, ~/.aws, ~/.hermes/.env), - reverse shells, destructive commands, persistence mechanisms, - obfuscated execution, and known exfiltration services. +- Runs the full skills_guard pattern set on documentation/config files, where + prompt-injection and social-engineering content lives. +- Exempts the "reads own env secret" / "HTTP call with key" pattern family on + *code* files while keeping genuinely malicious signals (foreign credential + stores, reverse shells, destructive commands, persistence, obfuscation, + known exfiltration services). - Applies plugin-sized structural limits and skips VCS/venv noise. -Verdict → install policy (Cowork's pass/warn/fail, adapted): - -- ``safe`` → install normally. -- ``caution`` → warn; requires explicit confirmation (interactive - prompt, ``--force``, or a caller-supplied decision - callback). -- ``dangerous`` → blocked. ``--force`` does NOT override. +Verdict → install policy: ``safe`` installs; ``caution`` requires explicit +confirmation (prompt, ``--force``, or caller callback); ``dangerous`` is +blocked and ``--force`` does NOT override. Usage: from tools.plugin_guard import scan_plugin, should_allow_plugin_install @@ -44,8 +31,9 @@ Usage: from __future__ import annotations +from datetime import datetime, timezone from pathlib import Path -from typing import List, Optional, Tuple +from typing import Iterator, List, Optional, Tuple from tools.skills_guard import ( Finding, @@ -65,16 +53,14 @@ EXCLUDED_DIRS = { } # Code file extensions where "reads an env secret" / "HTTP call with a key -# variable" is the NORMAL, documented plugin pattern (provider plugins read -# their own API keys via requires_env and call their backend with them). +# variable" is the NORMAL, documented plugin pattern (requires_env). CODE_FILE_EXTENSIONS = { ".py", ".js", ".ts", ".sh", ".bash", ".rb", ".pl", ".php", } -# Pattern ids from skills_guard.THREAT_PATTERNS that are exempt on code -# files. Each of these describes behavior every legitimate provider plugin -# exhibits. They still apply in full to documentation and config files, -# where such content is a strong injection/social-engineering signal. +# skills_guard pattern ids exempt on code files (every legitimate provider +# plugin exhibits them). They still apply in full to docs/config files, where +# such content is a strong injection/social-engineering signal. CODE_EXEMPT_PATTERN_IDS = { "python_environ_get_secret", "python_getenv_secret", @@ -98,17 +84,12 @@ CODE_EXEMPT_PATTERN_IDS = { "encoded_exfil", } -# Findings whose severity is remapped for plugins. Skills treat any bundled -# binary as critical (a skill is documentation and should never ship one); -# plugin repos occasionally vendor a compiled artifact legitimately, so a -# binary is a warn-tier signal instead of an instant block. -# -# ``hermes_env_access`` (a reference to ``~/.hermes/.env``) is the DOCUMENTED -# way plugins tell users where to put their API keys — nearly every legit -# plugin README mentions it. A mere reference is informational for plugins; -# actually READING the file still trips ``read_secrets_file`` (critical). -# ``curl | sh`` install instructions are common in plugin READMEs; keep them -# at warn tier (caution) rather than an unoverridable block. +# Severity remaps for plugins. A bundled binary is warn-tier (plugin repos +# occasionally vendor one legitimately; skills never should). A mere +# ``~/.hermes/.env`` reference is the DOCUMENTED way plugin READMEs tell users +# where keys go — informational; actually READING it still trips +# ``read_secrets_file`` (critical). ``curl | sh`` install instructions are +# common in READMEs: caution, not an unoverridable block. SEVERITY_REMAP = { "binary_file": "high", "hermes_env_access": "medium", @@ -125,17 +106,30 @@ def _is_excluded(rel_parts: Tuple[str, ...]) -> bool: return any(part in EXCLUDED_DIRS for part in rel_parts) +def _walk(plugin_dir: Path) -> Iterator[Tuple[Path, str]]: + """Yield (path, "a/b/c" relative path) for every non-excluded entry under plugin_dir.""" + for f in plugin_dir.rglob("*"): + try: + rel_parts = f.relative_to(plugin_dir).parts + except ValueError: + continue + if not _is_excluded(rel_parts): + yield f, "/".join(rel_parts) + + +def _finding(pattern_id: str, severity: str, category: str, file: str, match: str, description: str) -> Finding: + return Finding(pattern_id=pattern_id, severity=severity, category=category, + file=file, line=0, match=match, description=description) + + def _filter_findings(findings: List[Finding], rel_path: str) -> List[Finding]: """Apply plugin-specific exemptions and severity remaps to raw findings.""" - ext = Path(rel_path).suffix.lower() - is_code = ext in CODE_FILE_EXTENSIONS + is_code = Path(rel_path).suffix.lower() in CODE_FILE_EXTENSIONS out: List[Finding] = [] for f in findings: if is_code and f.pattern_id in CODE_EXEMPT_PATTERN_IDS: continue - remapped = SEVERITY_REMAP.get(f.pattern_id) - if remapped: - f.severity = remapped + f.severity = SEVERITY_REMAP.get(f.pattern_id) or f.severity out.append(f) return out @@ -146,38 +140,20 @@ def _check_plugin_structure(plugin_dir: Path) -> List[Finding]: file_count = 0 total_size = 0 - for f in plugin_dir.rglob("*"): - try: - rel_parts = f.relative_to(plugin_dir).parts - except ValueError: - continue - if _is_excluded(rel_parts): - continue - rel = "/".join(rel_parts) - + for f, rel in _walk(plugin_dir): if f.is_symlink(): file_count += 1 try: resolved = f.resolve() if not resolved.is_relative_to(plugin_dir.resolve()): - findings.append(Finding( - pattern_id="symlink_escape", - severity="critical", - category="traversal", - file=rel, - line=0, - match=f"symlink -> {resolved}", - description="symlink points outside the plugin directory", + findings.append(_finding( + "symlink_escape", "critical", "traversal", rel, + f"symlink -> {resolved}", "symlink points outside the plugin directory", )) except OSError: - findings.append(Finding( - pattern_id="broken_symlink", - severity="medium", - category="traversal", - file=rel, - line=0, - match="broken symlink", - description="broken or circular symlink", + findings.append(_finding( + "broken_symlink", "medium", "traversal", rel, + "broken symlink", "broken or circular symlink", )) continue @@ -192,96 +168,47 @@ def _check_plugin_structure(plugin_dir: Path) -> List[Finding]: total_size += size if size > MAX_PLUGIN_SINGLE_FILE_KB * 1024: - findings.append(Finding( - pattern_id="oversized_file", - severity="medium", - category="structural", - file=rel, - line=0, - match=f"{size // 1024}KB", - description=( - f"file is {size // 1024}KB " - f"(limit: {MAX_PLUGIN_SINGLE_FILE_KB}KB)" - ), + findings.append(_finding( + "oversized_file", "medium", "structural", rel, f"{size // 1024}KB", + f"file is {size // 1024}KB (limit: {MAX_PLUGIN_SINGLE_FILE_KB}KB)", )) ext = f.suffix.lower() if ext in SUSPICIOUS_BINARY_EXTENSIONS: - findings.append(Finding( - pattern_id="binary_file", - severity=SEVERITY_REMAP.get("binary_file", "high"), - category="structural", - file=rel, - line=0, - match=f"binary: {ext}", - description=( - f"binary/executable file ({ext}) bundled in plugin " - f"(cannot be scanned)" - ), + findings.append(_finding( + "binary_file", SEVERITY_REMAP.get("binary_file", "high"), "structural", rel, + f"binary: {ext}", f"binary/executable file ({ext}) bundled in plugin (cannot be scanned)", )) if file_count > MAX_PLUGIN_FILE_COUNT: - findings.append(Finding( - pattern_id="too_many_files", - severity="medium", - category="structural", - file="(directory)", - line=0, - match=f"{file_count} files", - description=( - f"plugin has {file_count} files " - f"(limit: {MAX_PLUGIN_FILE_COUNT})" - ), + findings.append(_finding( + "too_many_files", "medium", "structural", "(directory)", f"{file_count} files", + f"plugin has {file_count} files (limit: {MAX_PLUGIN_FILE_COUNT})", )) if total_size > MAX_PLUGIN_TOTAL_SIZE_KB * 1024: - findings.append(Finding( - pattern_id="oversized_bundle", - severity="medium", - category="structural", - file="(directory)", - line=0, - match=f"{total_size // 1024}KB", - description=( - f"plugin is {total_size // 1024}KB total " - f"(limit: {MAX_PLUGIN_TOTAL_SIZE_KB}KB)" - ), + findings.append(_finding( + "oversized_bundle", "medium", "structural", "(directory)", f"{total_size // 1024}KB", + f"plugin is {total_size // 1024}KB total (limit: {MAX_PLUGIN_TOTAL_SIZE_KB}KB)", )) return findings def scan_plugin(plugin_dir: Path, source: str = "") -> ScanResult: - """Scan a plugin directory for security threats. + """Scan a plugin directory (typically the temp clone) for security threats. - Args: - plugin_dir: Path to the plugin directory (typically the temp clone, - before it is moved into ``~/.hermes/plugins/``). - source: Identifier for display (git URL or owner/repo shorthand). - - Returns: - ScanResult with verdict ``safe`` | ``caution`` | ``dangerous``. - Every externally installed plugin is ``community`` trust. + Returns a ScanResult with verdict ``safe`` | ``caution`` | ``dangerous``; + every externally installed plugin is ``community`` trust. """ all_findings: List[Finding] = [] if plugin_dir.is_dir(): all_findings.extend(_check_plugin_structure(plugin_dir)) - for f in sorted(plugin_dir.rglob("*")): - if not f.is_file() or f.is_symlink(): - continue - try: - rel_parts = f.relative_to(plugin_dir).parts - except ValueError: - continue - if _is_excluded(rel_parts): - continue - rel = "/".join(rel_parts) - raw = scan_file(f, rel_path=rel) - all_findings.extend(_filter_findings(raw, rel)) + for f, rel in sorted(_walk(plugin_dir)): + if f.is_file() and not f.is_symlink(): + all_findings.extend(_filter_findings(scan_file(f, rel_path=rel), rel)) verdict = _determine_verdict(all_findings) - from datetime import datetime, timezone - result = ScanResult( skill_name=plugin_dir.name, source=source or plugin_dir.name, @@ -310,27 +237,20 @@ def should_allow_plugin_install( result: ScanResult, force: bool = False, ) -> Tuple[Optional[bool], str]: - """Map a plugin scan verdict to an install decision. + """Map a plugin scan verdict to ``(allowed, reason)``. - Returns ``(allowed, reason)``: - - ``(True, ...)`` install proceeds. - - ``(None, ...)`` needs explicit confirmation (caution verdict). - - ``(False, ...)`` blocked; ``force`` never overrides ``dangerous``. + ``True`` installs, ``None`` needs explicit confirmation (caution), ``False`` + is blocked — ``force`` never overrides ``dangerous``. """ + n = len(result.findings) if result.verdict == "safe": return True, "Allowed (clean scan)" if result.verdict == "caution": if force: - return True, ( - f"Force-installed despite caution verdict " - f"({len(result.findings)} findings)" - ) - return None, ( - f"Requires confirmation (caution verdict, " - f"{len(result.findings)} findings)" - ) + return True, f"Force-installed despite caution verdict ({n} findings)" + return None, f"Requires confirmation (caution verdict, {n} findings)" return False, ( - f"Blocked (dangerous verdict, {len(result.findings)} findings). " + f"Blocked (dangerous verdict, {n} findings). " f"--force does not override a dangerous verdict." ) diff --git a/tools/preview_tool.py b/tools/preview_tool.py index e3890c02d3..8bc7f3804d 100644 --- a/tools/preview_tool.py +++ b/tools/preview_tool.py @@ -1,17 +1,12 @@ #!/usr/bin/env python3 """The `desktop_preview` tool — the preview pane beside the chat, as ONE tool. -Consolidation (#95681, maintainer-directed): open_preview, close_preview, -and read_preview each re-taught "the preview pane beside the chat" world; -one action enum states it once (576 -> ~210 tok). The read action keeps -its agent-callback dispatch (agent_runtime_helpers routes action=read -through agent.read_preview_callback, same as read_preview did). - +open/close/read used to be three tools that each re-taught "the preview pane" world; +one action enum states it once (~576 -> ~210 schema tokens). action=read keeps its +agent-level callback dispatch (agent_runtime_helpers -> agent.read_preview_callback). Lives in the ``desktop_ui`` toolset — desktop-app sessions only. """ -import json - from tools import desktop_ui from tools.open_preview_tool import _normalize_target, open_preview_tool from tools.registry import registry, tool_error @@ -23,28 +18,32 @@ def preview_open(url: str, label: str = "") -> str: def preview_close(url: str = "") -> str: target = _normalize_target(url or "") - try: - ok = desktop_ui.emit("preview.close", {"url": target}) - except Exception as exc: # noqa: BLE001 - return tool_error(f"Failed to close the preview: {exc}") - if not ok: - return tool_error("The preview pane is only available in the Hermes desktop app.") - return json.dumps({"success": True, "closed": target or "all"}, ensure_ascii=False) + return desktop_ui.emit_or_error( + "preview.close", + {"url": target}, + "Failed to close the preview: ", + "The preview pane is only available in the Hermes desktop app.", + {"success": True, "closed": target or "all"}, + ) + + +_ACTIONS = { + "open": lambda args: preview_open(url=args.get("url", ""), label=args.get("label", "")), + "close": lambda args: preview_close(url=args.get("url", "")), + # read needs the GUI callback and is dispatched at the agent level. + "read": lambda args: tool_error( + "preview read must run inside a desktop session (no GUI callback here)." + ), +} def _handle_preview(args, **kw): - """Non-read actions only: action=read is dispatched at the agent level - (needs the GUI callback), mirroring the old read_preview special path.""" + """Non-read actions only: action=read is dispatched at the agent level.""" action = (args.get("action") or "").strip() - if action == "open": - return preview_open(url=args.get("url", ""), label=args.get("label", "")) - if action == "close": - return preview_close(url=args.get("url", "")) - if action == "read": - return tool_error( - "preview read must run inside a desktop session (no GUI callback here)." - ) - return tool_error("action must be one of: open, close, read.") + fn = _ACTIONS.get(action) + if fn is None: + return tool_error("action must be one of: open, close, read.") + return fn(args) PREVIEW_SCHEMA = { diff --git a/tools/process_registry_notifications.py b/tools/process_registry_notifications.py new file mode 100644 index 0000000000..ccd8d2fd60 --- /dev/null +++ b/tools/process_registry_notifications.py @@ -0,0 +1,369 @@ +"""Human-readable rendering of background-process notification events. + +Events come off ``ProcessRegistry.completion_queue`` (completion, watch_match, +watch_disabled, watch_overflow_*, async_delegation) and are turned into the +``[IMPORTANT: ...]`` / ``[ASYNC DELEGATION ...]`` text injected into the agent +conversation by the CLI drain loop, the gateway, and the TUI. +""" + +import time + + +def _format_age(seconds: float) -> str: + """Human-friendly elapsed string ('18m', '2h3m', '45s').""" + try: + s = int(max(0, seconds)) + except (TypeError, ValueError): + return "?" + if s < 60: + return f"{s}s" + m, s = divmod(s, 60) + if m < 60: + return f"{m}m" if s == 0 else f"{m}m{s}s" + h, m = divmod(m, 60) + return f"{h}h" if m == 0 else f"{h}h{m}m" + + +def _model_not_found_patterns() -> "list[str]": + """Model-not-found phrases shared with the failover classifier. + + Imported from ``agent.error_classifier`` so the batch renderer applies the + same classification the failover path uses (no hand-copied list to drift). + Fails open to a minimal built-in set so an import problem never hides the + per-task blocks. + """ + try: + from agent.error_classifier import _MODEL_NOT_FOUND_PATTERNS + + return list(_MODEL_NOT_FOUND_PATTERNS) + except Exception: + return ["is not a valid model", "model not found", "model_not_found"] + + +def _delegation_config() -> dict: + """Active delegation config (model/provider/fallbacks); ``{}`` on any error. + + Mirrors ``tools.delegate_tool._load_config`` lazily so the renderer sees the + same model/provider the dispatcher used without importing the heavy + delegation module at import time. + """ + try: + from tools.delegate_tool import _load_config as _cfg + + return _cfg() or {} + except Exception: + return {} + + +def _delegation_model_not_found(results, config) -> bool: + """True when a result reflects a config-level model_not_found rejection. + + Requires both a model-not-found phrase AND the currently-configured model + name in the same error/summary text, so a stale task failing on a + different (removed) model is not mis-attributed to the config. + """ + model = (config or {}).get("model") + if not model: + return False + model = str(model).lower() + for r in results or []: + text = " ".join( + str(part) for part in (r.get("error"), r.get("summary")) if part + ).lower() + if not text or model not in text: + continue + if any(p in text for p in _model_not_found_patterns()): + return True + return False + + +def _delegation_model_not_found_notice(results) -> "list[str] | None": + """Config-level model_not_found notice lines, or None (fail-open) — once per batch.""" + config = _delegation_config() + if not _delegation_model_not_found(results, config): + return None + model = config.get("model") or "?" + provider = config.get("provider") or "configured provider" + lines = [ + "⚠ SUBAGENT MODEL REJECTED: the configured Subagent Model " + f'"{model}" was rejected by provider "{provider}" ' + "(HTTP 400: not a valid model ID).", + "Every task in this batch failed for this reason before doing any work.", + "Check Settings → Advanced → Subagent Model (or: " + "hermes config get delegation.model).", + ] + try: + from hermes_cli.fallback_config import get_fallback_chain + + if not get_fallback_chain(config): + lines.append( + "No fallback chain is configured, so no failover was attempted." + ) + except Exception: + pass + return lines + + +_TRUNCATED_SUMMARY_NOTE = ( + "[TRUNCATED — subagent hit its iteration cap; the summary below " + "may be incomplete. Verify before relying on it, or re-dispatch " + "the unfinished part.]" +) + + +def _is_truncated(entry: dict) -> bool: + return bool(entry.get("truncated") or entry.get("exit_reason") == "max_iterations") + + +def _dispatched_line(dispatched_at, completed_at) -> "str | None": + if not isinstance(dispatched_at, (int, float)): + return None + ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(dispatched_at)) + return f"Dispatched: {ts} ({_format_age(completed_at - dispatched_at)} ago)" + + +def _task_source_lines(evt: dict) -> "list[str]": + lines = [] + if evt.get("context"): + lines.append(f"Context you provided: {evt['context']}") + if evt.get("toolsets"): + lines.append(f"Toolsets: {', '.join(evt['toolsets'])}") + return lines + + +def _format_batch_delegation(evt: dict, deleg_id: str, completed_at: float) -> str: + """Consolidated block for a delegate_task fan-out that finished as one unit.""" + results = evt.get("results") or [] + goals = evt.get("goals") or [] + n = len(results) if results else len(goals) + total_dur = evt.get("total_duration_seconds", evt.get("duration_seconds", "?")) + error = evt.get("error") + lines = [ + f"[ASYNC DELEGATION BATCH COMPLETE — {deleg_id}]", + f"A background fan-out of {n} subagent(s) you dispatched earlier " + "has finished. All ran in parallel and waited on each other; their " + "consolidated results are below. You may have moved on since " + "dispatching — act on these or re-dispatch if things have changed.", + "", + ] + dispatched = _dispatched_line(evt.get("dispatched_at"), completed_at) + if dispatched: + lines.append(dispatched) + lines.extend(_task_source_lines(evt)) + lines.append( + f"Role: {evt.get('role') or 'leaf'} Model: {evt.get('model') or '?'}" + f" Total duration: {total_dur}s" + ) + if error and not results: + lines.append("--- ERROR ---") + lines.append(f"The batch did not complete successfully: {error}") + return "\n".join(lines) + # Config-level rejection notice BEFORE the per-task wall — a rejected + # delegation model fails every task identically and must not stay buried. + _notice = _delegation_model_not_found_notice(results) + if _notice: + lines.append("") + lines.extend(_notice) + for r in sorted(results, key=lambda x: x.get("task_index", 0)): + idx = r.get("task_index", 0) + r_status = r.get("status", "?") + r_summary = r.get("summary") + r_error = r.get("error") + r_goal = goals[idx] if idx < len(goals) else r.get("goal", "") + r_truncated = _is_truncated(r) + icon = "⚠" if r_truncated else ("✓" if r_status in ("completed", "success") else "✗") + lines.append("") + header = f"--- {icon} TASK {idx + 1}/{n}" + if r_goal: + header += f": {r_goal}" + header += f" (status={r_status}" + if r.get("api_calls"): + header += f", api_calls={r['api_calls']}" + if r.get("duration_seconds") is not None: + header += f", {r['duration_seconds']}s" + if r_truncated: + header += ", TRUNCATED: hit max_iterations — work may be incomplete" + header += ") ---" + lines.append(header) + if r_status in ("completed", "success") and r_summary: + if r_truncated: + lines.append(_TRUNCATED_SUMMARY_NOTE) + lines.append(r_summary) + elif r_summary: + if r_error: + lines.append(f"({r_status}: {r_error})") + lines.append("Partial output:") + lines.append(r_summary) + else: + lines.append( + f"(no summary — status={r_status}" + + (f": {r_error}" if r_error else "") + + ")" + ) + r_live = r.get("live_transcript") + if r_live: + lines.append( + f"Full live transcript (complete tool/assistant trace): {r_live}" + ) + return "\n".join(lines) + + +def _format_async_delegation(evt: dict) -> str: + """Format an async-delegation completion into a self-contained re-injection. + + Carries the FULL original task source (goal, context, toolsets, role, + model) plus dispatch time, status, and the complete result summary: when + this re-enters the conversation the agent may be deep in unrelated context + and must be able to use the result OR re-dispatch without remembering why + the subagent existed. + """ + deleg_id = evt.get("delegation_id", "unknown") + completed_at = evt.get("completed_at") or time.time() + if evt.get("is_batch") or isinstance(evt.get("results"), list): + return _format_batch_delegation(evt, deleg_id, completed_at) + + status = evt.get("status") or "completed" + summary = evt.get("summary") + error = evt.get("error") + truncated = _is_truncated(evt) + lines = [ + f"[ASYNC DELEGATION COMPLETE — {deleg_id}]", + "A background subagent you dispatched earlier has finished. You may " + "have moved on since dispatching it; the full task source is below so " + "you can act on the result or re-dispatch if things have changed.", + "", + ] + dispatched = _dispatched_line(evt.get("dispatched_at"), completed_at) + if dispatched: + lines.append(dispatched) + lines.append(f"Original goal: {evt.get('goal', '') or ''}") + lines.extend(_task_source_lines(evt)) + lines.append(f"Role: {evt.get('role') or 'leaf'} Model: {evt.get('model') or '?'}") + _notice = _delegation_model_not_found_notice([evt]) + if _notice: + lines.append("") + lines.extend(_notice) + _trunc = " [TRUNCATED: hit max_iterations — work may be incomplete]" if truncated else "" + lines.append( + f"Status: {status} API calls: {evt.get('api_calls', 0)} " + f"Duration: {evt.get('duration_seconds', '?')}s{_trunc}" + ) + lines.append("--- RESULT ---") + if status in ("completed", "success") and summary: + if truncated: + lines.append(_TRUNCATED_SUMMARY_NOTE) + lines.append(summary) + else: + if status == "interrupted": + lines.append( + "The subagent was interrupted before completing" + + (f": {error}" if error else ".") + ) + else: # error / timeout / failed + lines.append( + f"The subagent did not complete successfully (status={status})." + + (f"\n{error}" if error else "") + ) + if summary: + lines.append("Partial output:") + lines.append(summary) + return "\n".join(lines) + + +def _delegation_attribution_line(evt: dict) -> "str | None": + """One-line provenance for a subagent-owned process event, else None. + + Subagents run terminal sessions under ``task_id == subagent_id``; a + background process they started outlives the child and is routed to the + PARENT conversation, which otherwise sees an anonymous raw output wall. + Judged on ``owner_task_id`` (the raw spawning id) — ``task_id`` is the + container key and may be collapsed to the session key. + """ + task_id = str(evt.get("owner_task_id") or evt.get("task_id") or "") + if not task_id.startswith("sa-"): + return None + try: + from tools.delegate_tool import get_subagent_attribution + + info = get_subagent_attribution(task_id) + except Exception: + info = None + if not info: + # Registry entry aged out — still attribute generically, not anonymously. + return f"Started by subagent {task_id} (delegate_task)." + goal = str(info.get("goal") or "").strip() + if len(goal) > 120: + goal = goal[:117] + "..." + deleg = info.get("delegation_id") + parts = [f"Started by subagent {task_id}"] + if deleg: + parts.append(f"of delegation {deleg}") + line = " ".join(parts) + "." + if goal: + line += f' Task: "{goal}"' + return line + + +def _completion_status(evt: dict) -> str: + _exit = evt.get("exit_code", "?") + _reason = evt.get("completion_reason") or "exited" + if _reason == "killed": + return f"terminated by {evt.get('termination_source') or 'Hermes'}" + if _reason == "lost": + return "marked lost because the process backend disappeared" + if _reason == "failed_start": + return "failed to start" + if _exit == 0: + return "completed normally" + return "exited" + + +def format_process_notification(evt: dict) -> "str | None": + """Format a completion_queue event into an ``[IMPORTANT: ...]`` message.""" + evt_type = evt.get("type", "completion") + _sid = evt.get("session_id", "unknown") + _cmd = evt.get("command", "unknown") + _attribution = _delegation_attribution_line(evt) + + # watch_disabled and overflow events carry their own human-readable + # `message`; without this branch overflow events would fall through to the + # completion formatter as a phantom "process exited (exit code ?)". + if evt_type in ("watch_disabled", "watch_overflow_tripped", "watch_overflow_released"): + return f"[IMPORTANT: {evt.get('message', '')}]" + + if evt_type == "watch_match": + _sup = evt.get("suppressed", 0) + text = ( + f"[IMPORTANT: Background process {_sid} matched " + f"watch pattern \"{evt.get('pattern', '?')}\".\n" + ) + if _attribution: + text += f"{_attribution}\n" + text += f"Command: {_cmd}\nMatched output:\n{evt.get('output', '')}" + if _sup: + text += f"\n({_sup} earlier matches were suppressed by rate limit)" + return text + "]" + + if evt_type == "async_delegation": + return _format_async_delegation(evt) + + _exit = evt.get("exit_code", "?") + _out = evt.get("output", "") + _signal = ", SIGTERM" if _exit in {-15, 143, "-15", "143"} else "" + text = ( + f"[IMPORTANT: Background process {_sid} {_completion_status(evt)} " + f"(exit code {_exit}{_signal}).\n" + ) + if _attribution: + text += f"{_attribution}\n" + # A subagent-owned process's full output belongs in the child's + # transcript, not as a raw wall in the parent — trim hard but keep + # enough tail to recognise failures. + if isinstance(_out, str) and len(_out) > 600: + _out = ( + "...(output trimmed — subagent-owned process; see the " + "delegation's live transcript for full output)\n" + + _out[-600:] + ) + text += f"Command: {_cmd}\nOutput:\n{_out}]" + return text diff --git a/tools/read_extract.py b/tools/read_extract.py index 260aa045f4..0deeebb4eb 100644 --- a/tools/read_extract.py +++ b/tools/read_extract.py @@ -1,17 +1,17 @@ """Stdlib document-to-text extraction for ``read_file``. -Supports Jupyter notebooks, DOCX, and XLSX without adding hard dependencies. -When the optional ``firecrawl-anydoc`` package is installed (``pip install -firecrawl-anydoc``, imports as ``anydoc``), coverage widens to legacy Office -(.doc/.ppt/.xls), OpenDocument, RTF, EPUB, and PDF — converted to Markdown by -its Rust core. The stdlib extractors remain authoritative for their three -formats so behavior is identical whether or not anydoc is present. -Malformed documents raise :class:`ExtractionError`; callers can then fall back to -normal text/binary handling. +Supports Jupyter notebooks, DOCX, and XLSX without hard dependencies. When the +optional ``firecrawl-anydoc`` package is installed (imports as ``anydoc``), +coverage widens to legacy Office (.doc/.ppt/.xls), OpenDocument, RTF, EPUB, and +PDF via its Rust core. The stdlib extractors stay authoritative for their three +formats so behavior is identical whether or not anydoc is present. Malformed +documents raise :class:`ExtractionError`; callers then fall back to normal +text/binary handling. """ from __future__ import annotations +import contextlib import importlib import json import os @@ -24,7 +24,7 @@ import threading import time import zipfile from pathlib import Path -from typing import Any, Optional +from typing import Any, Callable, Iterator, Optional from xml.etree import ElementTree as ET __all__ = [ @@ -44,10 +44,8 @@ ANYDOC_EXTENSIONS = frozenset({ ".odt", ".ods", ".odp", ".rtf", ".epub", ".pdf", }) -MAX_XLSX_BYTES = 50 * 1024 * 1024 -# Refuse to convert huge documents. anydoc loads the whole file through its -# Rust core with no streaming, and the read_file char budget only applies -# after conversion, so an unbounded input can pin a tool turn and spike RAM. +# anydoc loads the whole file through its Rust core with no streaming, and the +# read_file char budget only applies after conversion — cap the input size. MAX_ANYDOC_BYTES = 50 * 1024 * 1024 MAX_DOCUMENT_BYTES = 50 * 1024 * 1024 _MAX_XLSX_ROWS_PER_SHEET = 5000 @@ -65,9 +63,7 @@ class ExtractionError(Exception): def _extension(path: str) -> str: ext = Path(path).suffix.lower() - if ext in EXTRACTABLE_EXTENSIONS: - return ext - if ext in ANYDOC_EXTENSIONS and _anydoc() is not None: + if ext in EXTRACTABLE_EXTENSIONS or (ext in ANYDOC_EXTENSIONS and _anydoc() is not None): return ext return "" @@ -75,9 +71,8 @@ def _extension(path: str) -> str: _ANYDOC_UNSET = object() _anydoc_module: Any = _ANYDOC_UNSET _anydoc_lock = threading.Lock() -# After a failed first load, wait this long before trying again. The attempt -# can shell out to pip, so retrying on every call would hammer the network -# in environments where the install can never succeed. +# After a failed load, wait this long before retrying: the attempt can shell out +# to pip, so retrying every call would hammer the network where install can't succeed. ANYDOC_RETRY_SECONDS = 300.0 _anydoc_failed_at: Optional[float] = None @@ -86,8 +81,7 @@ def _anydoc() -> Optional[Any]: """Lazily import the optional anydoc converter; None when unavailable. A failed load is retried after :data:`ANYDOC_RETRY_SECONDS` rather than - disabling extraction for the rest of the process, so one transient - failure (network blip, pip race) does not stick in long-lived workers. + disabling extraction for the rest of a long-lived process. """ global _anydoc_module, _anydoc_failed_at if _anydoc_module is not _ANYDOC_UNSET: @@ -121,39 +115,24 @@ def is_extractable_document(path: str) -> bool: return bool(_extension(path)) -def extract_document_text(path: str) -> str: - ext = _extension(path) - if ext == ".ipynb": - return _extract_notebook(path) - if ext == ".docx": - return _extract_docx(path) - if ext == ".xlsx": - return _extract_xlsx(path) - if ext in ANYDOC_EXTENSIONS: - return _extract_anydoc(path) - raise ExtractionError(f"Unsupported document type: {path!r}") +def _unsupported(path: str) -> ExtractionError: + return ExtractionError(f"Unsupported document type: {path!r}") -def extract_document_bytes(data: bytes, path: str) -> str: - """Extract a document already fetched across a file backend boundary.""" - if len(data) > MAX_DOCUMENT_BYTES: - raise ExtractionError( - f"Document too large to convert ({len(data):,} bytes, limit is {MAX_DOCUMENT_BYTES:,})" - ) - ext = _extension(path) - if ext in ANYDOC_EXTENSIONS: - return _extract_anydoc_bytes(data, path) - if ext not in EXTRACTABLE_EXTENSIONS: - raise ExtractionError(f"Unsupported document type: {path!r}") +def _check_size(size: int, limit: int) -> None: + if size > limit: + raise ExtractionError(f"Document too large to convert ({size:,} bytes, limit is {limit:,})") - # The stdlib extractors are path-oriented. Materialize backend bytes in a - # private host temp file, then remove it even when parsing fails. + +@contextlib.contextmanager +def _temp_copy(data: bytes, suffix: str) -> Iterator[str]: + """Materialize backend bytes in a private host temp file; removed even when parsing fails.""" temp_path = "" try: - with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as fh: + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as fh: fh.write(data) temp_path = fh.name - return extract_document_text(temp_path) + yield temp_path finally: if temp_path: try: @@ -162,13 +141,34 @@ def extract_document_bytes(data: bytes, path: str) -> str: pass +def extract_document_text(path: str) -> str: + ext = _extension(path) + extractor = _STDLIB_EXTRACTORS.get(ext) + if extractor is not None: + return extractor(path) + if ext in ANYDOC_EXTENSIONS: + return _extract_anydoc(path) + raise _unsupported(path) + + +def extract_document_bytes(data: bytes, path: str) -> str: + """Extract a document already fetched across a file backend boundary.""" + _check_size(len(data), MAX_DOCUMENT_BYTES) + ext = _extension(path) + if ext in ANYDOC_EXTENSIONS: + return _extract_anydoc_bytes(data, path) + if ext not in EXTRACTABLE_EXTENSIONS: + raise _unsupported(path) + # The stdlib extractors are path-oriented. + with _temp_copy(data, ext) as temp_path: + return extract_document_text(temp_path) + + def _anydoc_missing_error(path: str) -> str: """Teaching error for anydoc-gated formats when the converter is absent. - Response-time hint (#95681 pattern): the schema no longer lists the - anydoc-gated formats or the availability caveat — a session that never - touches a .doc/.odt/.epub never pays for the explanation, and one that - does gets the full story here, with the fix. + The schema deliberately omits these formats and this caveat; the explanation + (and the fix) is paid for only by sessions that actually hit one. """ return ( f"Cannot convert {path!r}: this format needs the optional anydoc " @@ -181,15 +181,13 @@ def _anydoc_missing_error(path: str) -> str: def _hosted_ocr_config() -> tuple: - """Resolve hosted-OCR settings: (enabled, api_key, api_url). + """Resolve hosted-OCR settings: (enabled, api_key, api_url). Never raises. Maintainer decision: the ONLY route is a direct ``FIRECRAWL_API_KEY`` - (anydoc defaults api_url to https://api.firecrawl.dev). The Nous - managed gateway is NOT used — its Parse proxy was live-probed broken - (uniform HTTP 500, 2026-08-28) while scrape/search worked; revisit - when the gateway grows Parse support. ``file_tools.hosted_ocr``: - false disables even with a key; true/unset → enabled iff key - present. Never raises. + (anydoc defaults api_url to https://api.firecrawl.dev); the Nous managed + gateway is NOT used — its Parse proxy live-probed broken while scrape/search + worked. ``file_tools.hosted_ocr: false`` disables even with a key; + true/unset → enabled iff the key is present. Env probe only, no network. """ api_key = os.environ.get("FIRECRAWL_API_KEY") or None enabled = api_key is not None @@ -208,38 +206,17 @@ def _hosted_ocr_config() -> tuple: def hosted_ocr_available() -> bool: """Public probe for read_file's schema line: is hosted OCR unlocked? - Maintainer decision: ONE gate — a direct ``FIRECRAWL_API_KEY`` in the - environment. Nothing else unlocks the "PDF (scanned or text)" wording - (not the Nous gateway — Parse proxy live-probed broken 2026-08-28 — - and not config assertions). ``file_tools.hosted_ocr: false`` still - disables. Env probe only — no network at schema-build time; a key - that fails at conversion time lands in the NEEDS-OCR warning. + Same single gate as :func:`_hosted_ocr_config`. A key that fails at + conversion time lands in the NEEDS-OCR warning instead. """ - try: - if not os.environ.get("FIRECRAWL_API_KEY"): - return False - try: - from hermes_cli.config import load_config_readonly - - cfg = load_config_readonly() - section = cfg.get("file_tools") if isinstance(cfg, dict) else None - if isinstance(section, dict) and section.get("hosted_ocr") is False: - return False - except Exception: # noqa: BLE001 - pass - return True - except Exception: # noqa: BLE001 - return False + return _hosted_ocr_config()[0] def _needs_ocr_warning(path: str, pages, hosted_error: str = "") -> str: - """Typed replacement for the heuristic coverage note on full-OCR PDFs. + """Result text when anydoc raises NeedsOcrError and hosted OCR is off/failed. - Fired when anydoc raises NeedsOcrError and hosted OCR is disabled, - unavailable, or failed. Maintainer-directed shape: hint at CHECKING - for an OCR skill (never name one — none is guaranteed to exist), and - never advertise the hosted_ocr config knob — when hosted fails or is - absent, a skill or ignoring the gap are the paths that exist. + Hints at CHECKING for an OCR skill (never names one — none is guaranteed to + exist) and never advertises the hosted_ocr config knob. """ page_list = ", ".join(str(p) for p in pages) if pages else "unknown" msg = ( @@ -257,81 +234,108 @@ def _needs_ocr_warning(path: str, pages, hosted_error: str = "") -> str: return msg + "]\n" -def _extract_anydoc(path: str) -> str: +def _convert_anydoc(path: str, size: int, convert: Callable[[Any], str], pdf_note: Callable[[], str]) -> str: + """Shared anydoc pipeline: availability + size gate, convert, normalize, PDF note. + + ``convert(mod)`` runs the converter; its exceptions become ExtractionError + (anydoc raises one ConvertError subclass per failure mode — Unsupported, + Malformed, Encrypted, ResourceLimit, MissingPart — all meaning "no meaningful + text", so read_file falls back to normal path/binary handling). The PDF + coverage note is PREPENDED: read_file paginates the extraction, so a footer + on a long document would sit on a page the model may never fetch. It covers + PARTIAL gaps (text layer plus some scanned pages) that convert without + raising NeedsOcrError. + """ mod = _anydoc() if mod is None: raise ExtractionError(_anydoc_missing_error(path)) + _check_size(size, MAX_ANYDOC_BYTES) try: - size = os.path.getsize(path) - except OSError as exc: - raise ExtractionError(str(exc)) from exc - if size > MAX_ANYDOC_BYTES: - raise ExtractionError( - f"Document too large to convert ({size:,} bytes, limit is {MAX_ANYDOC_BYTES:,})" - ) - needs_ocr = getattr(mod, "NeedsOcrError", None) - try: - text = mod.to_markdown(path) - except OSError as exc: - raise ExtractionError(str(exc)) from exc + text = convert(mod) + except ExtractionError: + raise except Exception as exc: - if needs_ocr is not None and isinstance(exc, needs_ocr): - # Typed scanned-pages signal (anydoc >= 0.2). Try hosted OCR - # when a Firecrawl route exists; otherwise teach recovery. - pages = list(getattr(exc, "pages", []) or []) - enabled, api_key, api_url = _hosted_ocr_config() - hosted_error = "" - if enabled: - try: - kwargs = {"ocr": "hosted"} - if api_key: - kwargs["api_key"] = api_key - if api_url: - kwargs["api_url"] = api_url - text = mod.to_markdown(path, **kwargs) - return text.rstrip("\n") + "\n" - except Exception as hosted_exc: # noqa: BLE001 - hosted_error = f"{type(hosted_exc).__name__}: {hosted_exc}" - # No route / disabled / hosted failed: whole doc is scans — - # nothing to extract, so the warning IS the result. - return _needs_ocr_warning(path, pages, hosted_error) - # anydoc raises one ConvertError subclass per failure mode - # (Unsupported, Malformed, Encrypted, ResourceLimit, MissingPart). - # Any of them means "no meaningful text": fall back to the normal - # path/binary handling rather than crash read_file. raise ExtractionError(f"{type(exc).__name__}: {exc}") from exc if not isinstance(text, str) or not text.strip(): raise ExtractionError("Document contains no extractable text") text = text.rstrip("\n") + "\n" if Path(path).suffix.lower() == ".pdf": - note = _pdf_coverage_note(path) + note = pdf_note() if note: - # Prepend: read_file paginates the extraction, so a footer on a - # long document would sit on a page the model may never fetch. - # This heuristic note survives for PARTIAL coverage gaps — - # documents with a text layer plus some scanned pages, which - # convert without raising NeedsOcrError. text = note + text return text +def _ocr_scanned_pdf(mod: Any, path: str, exc: BaseException) -> str: + """Typed scanned-pages signal (anydoc >= 0.2): try hosted OCR, else teach recovery.""" + pages = list(getattr(exc, "pages", []) or []) + enabled, api_key, api_url = _hosted_ocr_config() + hosted_error = "" + if enabled: + try: + kwargs = {"ocr": "hosted"} + if api_key: + kwargs["api_key"] = api_key + if api_url: + kwargs["api_url"] = api_url + return mod.to_markdown(path, **kwargs).rstrip("\n") + "\n" + except Exception as hosted_exc: # noqa: BLE001 + hosted_error = f"{type(hosted_exc).__name__}: {hosted_exc}" + # Whole doc is scans — nothing to extract, so the warning IS the result. + return _needs_ocr_warning(path, pages, hosted_error) + + +class _ScannedPdfResult(ExtractionError): + """Internal: carries the hosted-OCR/NEEDS-OCR text out of the converter callback.""" + + +def _extract_anydoc(path: str) -> str: + def convert(mod: Any) -> str: + try: + return mod.to_markdown(path) + except OSError as exc: + raise ExtractionError(str(exc)) from exc + except Exception as exc: + needs_ocr = getattr(mod, "NeedsOcrError", None) + if needs_ocr is not None and isinstance(exc, needs_ocr): + raise _ScannedPdfResult(_ocr_scanned_pdf(mod, path, exc)) from exc + raise + + if _anydoc() is None: + raise ExtractionError(_anydoc_missing_error(path)) + try: + size = os.path.getsize(path) + except OSError as exc: + raise ExtractionError(str(exc)) from exc + try: + return _convert_anydoc(path, size, convert, lambda: _pdf_coverage_note(path)) + except _ScannedPdfResult as result: + return str(result) + + +def _extract_anydoc_bytes(data: bytes, path: str) -> str: + return _convert_anydoc( + path, len(data), lambda mod: mod.to_markdown_bytes(data), + lambda: _pdf_coverage_note_from_bytes(data, path), + ) + + # ── Scanned-PDF coverage detection ────────────────────────────────── -# -# anydoc (like every text-layer extractor) returns nothing for scanned -# image pages and emits no image placeholders or page markers, so a -# mostly-scanned PDF converts "successfully" into a few headers with -# empty bodies — silent data loss the model cannot detect. Count per-page -# text via poppler's pdftotext (form-feed page separators) and append a -# loud footer when a meaningful share of pages yielded no text. +# Text-layer extractors return nothing for scanned pages and emit no +# placeholders, so a mostly-scanned PDF converts "successfully" into headers +# with empty bodies — silent data loss the model cannot detect. Count per-page +# text via pdftotext (form-feed separators) and warn when many pages are empty. # A page with fewer extracted characters than this is considered empty. PDF_EMPTY_PAGE_CHARS = 20 -# Warn when at least this many pages are empty AND they exceed the ratio, -# or when the absolute count alone is overwhelming. +# Warn when empty pages reach both MIN_EMPTY and MIN_RATIO, or ABSOLUTE_EMPTY alone. PDF_COVERAGE_MIN_EMPTY = 2 PDF_COVERAGE_MIN_RATIO = 0.2 PDF_COVERAGE_ABSOLUTE_EMPTY = 10 PDF_PAGE_SCAN_TIMEOUT = 20.0 +# Cap the per-gap breakdown so alternating text/scan pages can't balloon the warning. +PDF_GAP_MAP_MAX_ENTRIES = 20 +_GAP_CONTEXT_CHARS = 60 def _pdf_page_texts(path: str) -> Optional[list[str]]: @@ -354,22 +358,6 @@ def _pdf_page_texts(path: str) -> Optional[list[str]]: return pages or None -def _pdf_page_char_counts(path: str) -> Optional[list[int]]: - """Per-page extracted-text char counts, or None when undeterminable.""" - pages = _pdf_page_texts(path) - if pages is None: - return None - return [len(page.strip()) for page in pages] - - -def _page_ranges(pages: list[int]) -> str: - """Compact 1-based range list, e.g. '2-29, 33-35, 42'.""" - parts = [f"{a}-{b}" if a != b else str(a) for a, b in _group_ranges(pages)] - if len(parts) > 12: - parts = parts[:12] + ["…"] - return ", ".join(parts) - - def _group_ranges(pages: list[int]) -> list[list[int]]: """Group sorted 1-based page numbers into [start, end] runs.""" ranges: list[list[int]] = [] @@ -381,24 +369,14 @@ def _group_ranges(pages: list[int]) -> list[list[int]]: return ranges -# Cap the per-gap breakdown so a pathological PDF (hundreds of alternating -# text/scan pages) cannot balloon the warning. Ranges beyond the cap are -# summarized in one line. -PDF_GAP_MAP_MAX_ENTRIES = 20 -_GAP_CONTEXT_CHARS = 60 - - def _gap_map(counts: list[int], texts: list[str], empty: list[int]) -> str: - """Per-gap breakdown: each empty range labeled with the last text seen - before it (usually a section divider/header page), so the agent can - decide WHICH gaps it actually needs to read instead of OCRing all of - them.""" + """Per-gap breakdown, each empty range labeled with the last text seen before + it (usually a section header), so the agent can pick WHICH gaps to OCR.""" ranges = _group_ranges(empty) lines: list[str] = [] for a, b in ranges[:PDF_GAP_MAP_MAX_ENTRIES]: label = "" - # Walk back to the nearest preceding page with text. - for prev in range(a - 2, -1, -1): + for prev in range(a - 2, -1, -1): # nearest preceding page with text if counts[prev] >= PDF_EMPTY_PAGE_CHARS: snippet = " ".join(texts[prev].split())[:_GAP_CONTEXT_CHARS] label = f' — after "{snippet}" (p{prev + 1})' @@ -414,11 +392,10 @@ def _gap_map(counts: list[int], texts: list[str], empty: list[int]) -> str: def _pdf_coverage_note(path: str, display_path: Optional[str] = None) -> str: - """A warning header when many PDF pages produced no text, else ''. + """Warning header when many PDF pages produced no text, else ''. - ``path`` is the file scanned with pdftotext (may be a host temp file - for backend-transferred bytes); ``display_path`` is the path shown in - the recovery command — the one the agent's terminal can actually see. + ``path`` is scanned with pdftotext (may be a host temp file); ``display_path`` + is what the recovery command shows — the path the agent's terminal can see. """ texts = _pdf_page_texts(path) if not texts or len(texts) < 2: @@ -451,51 +428,17 @@ def _pdf_coverage_note(path: str, display_path: Optional[str] = None) -> str: ) -def _extract_anydoc_bytes(data: bytes, path: str) -> str: - mod = _anydoc() - if mod is None: - raise ExtractionError(_anydoc_missing_error(path)) - if len(data) > MAX_ANYDOC_BYTES: - raise ExtractionError( - f"Document too large to convert ({len(data):,} bytes, limit is {MAX_ANYDOC_BYTES:,})" - ) - try: - text = mod.to_markdown_bytes(data) - except Exception as exc: - raise ExtractionError(f"{type(exc).__name__}: {exc}") from exc - if not isinstance(text, str) or not text.strip(): - raise ExtractionError("Document contains no extractable text") - text = text.rstrip("\n") + "\n" - if Path(path).suffix.lower() == ".pdf": - note = _pdf_coverage_note_from_bytes(data, path) - if note: - # Prepend: read_file paginates the extraction, so a footer on a - # long document would sit on a page the model may never fetch. - text = note + text - return text - - def _pdf_coverage_note_from_bytes(data: bytes, display_path: str) -> str: """Coverage note for backend-transferred PDF bytes. - pdftotext is path-oriented, so materialize the bytes in a private host - temp file for the scan; the recovery command still names - ``display_path`` — the path the agent's terminal backend can see. + pdftotext is path-oriented, so scan a host temp copy; the recovery command + still names ``display_path`` — the path the agent's terminal backend can see. """ - temp_path = "" try: - with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as fh: - fh.write(data) - temp_path = fh.name - return _pdf_coverage_note(temp_path, display_path=display_path) + with _temp_copy(data, ".pdf") as temp_path: + return _pdf_coverage_note(temp_path, display_path=display_path) except OSError: return "" - finally: - if temp_path: - try: - os.unlink(temp_path) - except OSError: - pass def _source_text(source) -> str: @@ -520,10 +463,8 @@ def _base64_bytes(payload: str) -> int: def _clean_stream_text(text: str) -> str: """Strip ANSI escapes and collapse ``\\r`` progress-bar rewrites. - tqdm and friends redraw the same line via carriage returns; Jupyter - renders only the final frame, so keeping the text after the last ``\\r`` - of each line reproduces what the notebook displays without the invisible - intermediate frames. + Jupyter renders only the final frame of a ``\\r``-redrawn line (tqdm), so + keep the text after the last ``\\r`` of each line. """ from tools.ansi_strip import strip_ansi @@ -539,14 +480,16 @@ def _clean_stream_text(text: str) -> str: # single runaway training log cannot flood the extracted text. _MAX_OUTPUT_CHARS = 20_000 +# nbformat v3 stores mime data flat on the output dict under these keys. +_V3_MIME_KEYS = (("png", "image/png"), ("jpeg", "image/jpeg"), ("svg", "image/svg+xml"), ("html", "text/html")) + def _notebook_output_text(output: Any) -> str: """Render one notebook output as compact text. - Keeps stream text, error tracebacks, and textual results; replaces - token-heavy payloads (base64 images, HTML, widget state) with short - sized placeholders. Handles both nbformat v4 output shapes and the - legacy v3 ones (``pyout``/``pyerr``; data flat on the output dict). + Keeps stream text, tracebacks, and textual results; replaces token-heavy + payloads (base64 images, HTML, widget state) with short sized placeholders. + Handles nbformat v4 and legacy v3 (``pyout``/``pyerr``) shapes. """ if not isinstance(output, dict): return "" @@ -569,20 +512,17 @@ def _notebook_output_text(output: Any) -> str: if otype in {"execute_result", "display_data", "pyout"}: data = output.get("data") if not isinstance(data, dict): - # nbformat v3 stores mime data flat on the output dict. data = {} if isinstance(output.get("text"), (str, list)): data["text/plain"] = output["text"] - for v3_key, mime in (("png", "image/png"), ("jpeg", "image/jpeg"), - ("svg", "image/svg+xml"), ("html", "text/html")): + for v3_key, mime in _V3_MIME_KEYS: if v3_key in output: data[mime] = output[v3_key] if "application/vnd.jupyter.widget-view+json" in data: return "[interactive widget — omitted]" - # Prefer readable text: models consume text/plain (e.g. the pandas - # twin of an HTML table) far better than markup. + # Prefer readable text: models consume text/plain far better than markup. for mime in ("text/plain", "text/markdown"): if mime in data: body = _clean_stream_text(_source_text(data[mime])) @@ -614,13 +554,14 @@ def _notebook_outputs(cell: dict, jq_pointer: str = "", filename: str = "") -> s joined = "\n".join(blocks) if len(joined) > _MAX_OUTPUT_CHARS: omitted = len(joined) - _MAX_OUTPUT_CHARS - hint = "" - if jq_pointer and filename: - hint = f" — full output: jq -r '{jq_pointer}' {filename}" + hint = f" — full output: jq -r '{jq_pointer}' {filename}" if jq_pointer and filename else "" joined = joined[:_MAX_OUTPUT_CHARS] + f"\n… [{omitted:,} output chars truncated{hint}]" return joined +_CELL_LABELS = {"markdown": "Markdown", "code": "Code", "raw": "Raw"} + + def _extract_notebook(path: str) -> str: try: with open(path, encoding="utf-8", errors="replace") as fh: @@ -644,18 +585,17 @@ def _extract_notebook(path: str) -> str: raise ExtractionError("Notebook contains no cells") nb_name = os.path.basename(path) - counts = {"markdown": 0, "code": 0, "raw": 0} - labels = {"markdown": "Markdown", "code": "Code", "raw": "Raw"} + counts = dict.fromkeys(_CELL_LABELS, 0) out: list[str] = [] for jq_pointer, cell in cells: if not isinstance(cell, dict): continue typ = cell.get("cell_type") - if typ not in labels: + if typ not in _CELL_LABELS: continue counts[typ] += 1 suffix = f" {counts[typ]}" if typ != "raw" else "" - out.extend((f"# ── {labels[typ]} cell{suffix} ──", _source_text(cell.get("source", "")).rstrip("\n"), "")) + out.extend((f"# ── {_CELL_LABELS[typ]} cell{suffix} ──", _source_text(cell.get("source", "")).rstrip("\n"), "")) if typ == "code": rendered = _notebook_outputs(cell, jq_pointer, nb_name) if rendered: @@ -674,6 +614,16 @@ def _zip_xml(zf: zipfile.ZipFile, name: str) -> ET.Element: raise ExtractionError(f"Malformed XML in {name}: {exc}") from exc +def _optional_zip_xml(zf: zipfile.ZipFile, names: set[str], name: str) -> Optional[ET.Element]: + """Parse an optional package part; None when absent or malformed.""" + if name not in names: + return None + try: + return ET.fromstring(zf.read(name)) + except ET.ParseError: + return None + + def _extract_docx(path: str) -> str: try: with zipfile.ZipFile(path) as zf: @@ -734,11 +684,8 @@ def _extract_xlsx(path: str) -> str: def _shared_strings(zf: zipfile.ZipFile, names: set[str]) -> list[str]: - if "xl/sharedStrings.xml" not in names: - return [] - try: - root = ET.fromstring(zf.read("xl/sharedStrings.xml")) - except ET.ParseError: + root = _optional_zip_xml(zf, names, "xl/sharedStrings.xml") + if root is None: return [] s = f"{{{_NS_S}}}" return ["".join(t.text or "" for t in item.iter(f"{s}t")) for item in root.iter(f"{s}si")] @@ -754,12 +701,8 @@ def _workbook_sheets(zf: zipfile.ZipFile) -> list[tuple[str, str, str]]: def _workbook_rels(zf: zipfile.ZipFile, names: set[str]) -> dict[str, str]: - rels_path = "xl/_rels/workbook.xml.rels" - if rels_path not in names: - return {} - try: - root = ET.fromstring(zf.read(rels_path)) - except ET.ParseError: + root = _optional_zip_xml(zf, names, "xl/_rels/workbook.xml.rels") + if root is None: return {} rel_tag = f"{{{_NS_PKG_REL}}}Relationship" return {rel.get("Id", ""): rel.get("Target", "") for rel in root.iter(rel_tag) if rel.get("Id")} @@ -816,3 +759,11 @@ def _cell_value(cell: ET.Element, shared: list[str], s: str) -> str: if typ == "e": return value or "#ERROR" return value + + +# Extension -> stdlib extractor; anydoc formats fall through in extract_document_text. +_STDLIB_EXTRACTORS: dict[str, Callable[[str], str]] = { + ".ipynb": _extract_notebook, + ".docx": _extract_docx, + ".xlsx": _extract_xlsx, +} diff --git a/tools/read_preview_tool.py b/tools/read_preview_tool.py index 53111ab148..66f4cf490b 100644 --- a/tools/read_preview_tool.py +++ b/tools/read_preview_tool.py @@ -1,21 +1,16 @@ #!/usr/bin/env python3 """Read the in-app browser / preview pane in the Hermes desktop GUI. -The preview's content lives in the desktop renderer (a sandboxed ```` -for URL tabs), so this tool round-trips through the gateway's blocking-prompt -bridge — the same one ``read_terminal`` uses: tui_gateway emits -``preview.read.request``, the renderer serializes the active preview tab and -answers with ``preview.read.respond``. This module is just schema + a thin -dispatcher over the platform-injected callback. - -Lives in the ``desktop_ui`` toolset, which the GUI gateway enables only for -desktop-sourced sessions. +The preview's content lives in the renderer (a sandboxed ````), so this +round-trips through the gateway's blocking-prompt bridge like ``read_terminal`` +(``preview.read.request`` -> ``preview.read.respond``). Registration moved into +`desktop_preview`; the agent dispatches action=read here with the injected callback. """ -import json from typing import Callable, Optional -from tools.registry import registry, tool_error +from tools.desktop_ui import passthrough_json +from tools.registry import tool_error def read_preview_tool( @@ -43,43 +38,4 @@ def read_preview_tool( if not raw: return tool_error("No preview tab is open, or the read timed out.") - - # Desktop answers with a JSON object; pass it through, else wrap the raw text. - try: - return json.dumps(json.loads(raw), ensure_ascii=False) - except (TypeError, ValueError): - return json.dumps({"text": str(raw)}, ensure_ascii=False) - - -READ_PREVIEW_SCHEMA = { - "name": "read_preview", - "description": ( - "Read what's currently shown in the in-app browser / preview pane of the " - "Hermes desktop GUI (the pane open_preview opens beside this chat). Call " - "with no arguments for the first window of the active tab's content. " - "Returns JSON {kind, url, title, text, start, end, total_chars, note?}: " - "a URL (Browser) tab's text is the rendered page's visible text — page " - "through longer pages with `start`/`count` (character offsets, capped " - "per read); a file tab answers identity only (read the file with " - "read_file); an artifact tab points back at the conversation. Use after " - "open_preview, or whenever the user refers to what's on screen in the " - "browser ('what does this page say?'). To close the pane, use close_preview." - ), - "parameters": { - "type": "object", - "properties": { - "start": { - "type": "integer", - "description": "0-indexed character offset into the page text. Omit for the start.", - }, - "count": { - "type": "integer", - "description": "Characters to return from start. Defaults to (and is capped at) the per-read maximum.", - }, - }, - }, -} - - -# Registration removed: consolidated into the `preview` tool (#95681); -# this module keeps its functions for the agent-level preview action=read dispatch. + return passthrough_json(raw) diff --git a/tools/read_window_tool.py b/tools/read_window_tool.py index 171476edf9..c772110eb8 100644 --- a/tools/read_window_tool.py +++ b/tools/read_window_tool.py @@ -1,17 +1,14 @@ #!/usr/bin/env python3 """Read which OS window sits directly underneath the Hermes desktop window. -The window list lives with the OS, so this tool round-trips through the -gateway's blocking-prompt bridge — the same one `read_terminal` uses: -tui_gateway emits ``window.read.request``, the desktop renderer asks its main -process (which owns native window enumeration) and answers with -``window.read.respond``. This module is just schema + a thin dispatcher over -the platform-injected callback. +The window list lives with the OS, so this round-trips through the gateway's +blocking-prompt bridge like `read_terminal`: ``window.read.request`` -> the renderer's +main process (native window enumeration) -> ``window.read.respond``. """ -import json from typing import Callable, Optional +from tools.desktop_ui import passthrough_json from tools.registry import registry, tool_error @@ -32,12 +29,7 @@ def read_window_below_tool(callback: Optional[Callable] = None) -> str: "Could not determine the window underneath (the desktop app did " "not answer, or window enumeration is unavailable on this system)." ) - - # Desktop answers with a JSON object; pass it through, else wrap the raw text. - try: - return json.dumps(json.loads(raw), ensure_ascii=False) - except (TypeError, ValueError): - return json.dumps({"text": str(raw)}, ensure_ascii=False) + return passthrough_json(raw) READ_WINDOW_BELOW_SCHEMA = { diff --git a/tools/self_repo_guard.py b/tools/self_repo_guard.py index 53d3a9675c..dba15d6f8d 100644 --- a/tools/self_repo_guard.py +++ b/tools/self_repo_guard.py @@ -8,6 +8,7 @@ import shlex import subprocess from dataclasses import dataclass, field from pathlib import Path +from typing import Callable from tools.approval import ( _bash_exec_payload, @@ -17,111 +18,49 @@ from tools.approval import ( ) +# bisect is included: it drives repeated checkouts of the running root — the +# exact module-version-skew hazard this guard exists for. _WORKTREE_MUTATIONS = frozenset({ - "checkout", - "switch", - "rebase", - "merge", - "pull", - "restore", - "clean", - "cherry-pick", - "revert", - # bisect drives repeated checkouts of the running root — the exact - # module-version-skew hazard this guard exists for. - "bisect", + "checkout", "switch", "rebase", "merge", "pull", "restore", "clean", + "cherry-pick", "revert", "bisect", }) _WORKTREE_TARGET_ACTIONS = frozenset({"move", "remove"}) _STASH_SAFE_ACTIONS = frozenset({"list", "show", "create", "store", "drop", "clear"}) _RESET_WORKTREE_MODES = frozenset({"--hard", "--merge", "--keep"}) +# `reset`/`stash`/`clean`/`restore` reach this set only in their SAFE forms +# (_mutates_worktree classifies the dangerous forms first in _inspect_git); +# listing them only avoids a pointless `git config --get alias.` +# subprocess for `stash list`, `reset --soft`, `clean -n`, `restore --staged`. _KNOWN_GIT_BUILTINS = frozenset({ - "add", - "am", - "apply", - "blame", - "branch", - "bundle", - "cat-file", - # `reset`/`stash`/`clean`/`restore` reach this set only in their SAFE - # forms — _mutates_worktree classifies the dangerous forms first (see - # _inspect_git) — so listing them here only prevents a pointless - # `git config --get alias.` subprocess for `stash list`, - # `reset --soft`, `clean -n`, `restore --staged`, which agent dev - # sessions run constantly inside the source repo. - "clean", - "clone", - "commit", - "config", - "describe", - "diff", - "fetch", - "format-patch", - "grep", - "help", - "init", - "log", - "ls-files", - "ls-remote", - "ls-tree", - "maintenance", - "merge-base", - "mv", - "notes", - "push", - "range-diff", - "reflog", - "remote", - "repack", - "replace", - "reset", - "restore", - "rev-list", - "rev-parse", - "rm", - "shortlog", - "show", - "show-ref", - "stash", - "status", - "submodule", - "tag", + "add", "am", "apply", "blame", "branch", "bundle", "cat-file", "clean", + "clone", "commit", "config", "describe", "diff", "fetch", "format-patch", + "grep", "help", "init", "log", "ls-files", "ls-remote", "ls-tree", + "maintenance", "merge-base", "mv", "notes", "push", "range-diff", "reflog", + "remote", "repack", "replace", "reset", "restore", "rev-list", "rev-parse", + "rm", "shortlog", "show", "show-ref", "stash", "status", "submodule", "tag", "worktree", }) _SHELL_EXECUTABLES = frozenset({"bash", "dash", "ksh", "sh", "zsh"}) _ASSIGNMENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*=(.*)", re.DOTALL) -_SUDO_OPTIONS_WITH_ARG = frozenset({ - "-C", - "--chdir", - "-c", - "--close-from", - "-g", - "--group", - "-h", - "--host", - "-p", - "--prompt", - "-R", - "--chroot", - "-T", - "--command-timeout", - "-u", - "--user", -}) -_ENV_OPTIONS_WITH_ARG = frozenset({ - "-a", - "--argv0", - "-C", - "--chdir", - "-S", - "--split-string", - "-u", - "--unset", -}) -_WRAPPER_OPTIONS_WITH_ARG = { +_RESET_HARD_RE = re.compile(r"--h(?:a(?:r(?:d)?)?)?\Z") +_NO_OPTIONS: frozenset[str] = frozenset() +# Wrapper executables that are skipped to reach the real command, mapped to +# the options that consume a following argument. +_WRAPPER_OPTIONS_WITH_ARG: dict[str, frozenset[str]] = { + "sudo": frozenset({ + "-C", "--chdir", "-c", "--close-from", "-g", "--group", "-h", "--host", + "-p", "--prompt", "-R", "--chroot", "-T", "--command-timeout", "-u", "--user", + }), + "env": frozenset({ + "-a", "--argv0", "-C", "--chdir", "-S", "--split-string", "-u", "--unset", + }), + "command": _NO_OPTIONS, + "builtin": _NO_OPTIONS, "exec": frozenset({"-a"}), + "nohup": _NO_OPTIONS, + "setsid": _NO_OPTIONS, "time": frozenset({"-f", "--format", "-o", "--output"}), } -_SIMPLE_WRAPPERS = frozenset({"builtin", "exec", "nohup", "setsid", "time"}) _MAX_RECURSION = 4 @@ -187,8 +126,9 @@ def _shell_words_at(command: str, start: int) -> list[str]: def _consume_options( words: list[str], start: int, - options_with_arg: frozenset[str], + options_with_arg: frozenset[str] = _NO_OPTIONS, ) -> int: + """Index of the first positional at/after ``start`` (``--`` ends options).""" index = start while index < len(words): option = words[index] @@ -205,6 +145,7 @@ def _consume_options( def _command_parts(words: list[str]) -> tuple[dict[str, str], str | None, list[str]]: + """Split leading VAR=value assignments and wrappers off -> (env, executable, args).""" env: dict[str, str] = {} index = 0 @@ -216,30 +157,19 @@ def _command_parts(words: list[str]) -> tuple[dict[str, str], str | None, list[s continue executable = _executable_name(words[index]) - if executable == "sudo": - index = _consume_options(words, index + 1, _SUDO_OPTIONS_WITH_ARG) - continue - if executable == "env": - index = _consume_options(words, index + 1, _ENV_OPTIONS_WITH_ARG) - continue - if executable == "command": - if index + 1 < len(words) and words[index + 1] in {"-v", "-V"}: - return env, None, [] - index = _consume_options(words, index + 1, frozenset()) - continue - if executable in _SIMPLE_WRAPPERS: - index = _consume_options( - words, - index + 1, - _WRAPPER_OPTIONS_WITH_ARG.get(executable, frozenset()), - ) - continue - return env, words[index], words[index + 1 :] + wrapper_options = _WRAPPER_OPTIONS_WITH_ARG.get(executable) + if wrapper_options is None: + return env, words[index], words[index + 1 :] + # `command -v/-V` only reports; nothing runs. + if executable == "command" and words[index + 1 : index + 2] in (["-v"], ["-V"]): + return env, None, [] + index = _consume_options(words, index + 1, wrapper_options) return env, None, [] def _scope_keys(command: str, starts: list[int]) -> dict[int, tuple[int, ...]]: + """Map each command start to the tuple of enclosing ``(``/``$(``/backtick openers.""" contexts = [_ShellContext("root", -1)] scopes: dict[int, tuple[int, ...]] = {} cursor = 0 @@ -255,44 +185,34 @@ def _scope_keys(command: str, starts: list[int]) -> dict[int, tuple[int, ...]]: context.quote = None cursor += 1 continue + # Unquoted or inside double quotes: substitutions still open scopes. + if char == "\\" and cursor + 1 < start: + cursor += 2 + continue if quote == '"': - if char == "\\" and cursor + 1 < start: - cursor += 2 - continue if char == '"': context.quote = None cursor += 1 continue - if command.startswith("$(", cursor): - contexts.append(_ShellContext("$(", cursor)) - cursor += 2 - continue - if char == "`": - contexts.append(_ShellContext("`", cursor)) - cursor += 1 - continue - - if char in {"'", '"'}: + elif char in {"'", '"'}: context.quote = char cursor += 1 continue - if char == "\\" and cursor + 1 < start: - cursor += 2 - continue if command.startswith("$(", cursor): contexts.append(_ShellContext("$(", cursor)) cursor += 2 continue - if char == "(": - contexts.append(_ShellContext("(", cursor)) - cursor += 1 - continue - if char == ")" and len(contexts) > 1 and contexts[-1].kind in {"(", "$("}: - contexts.pop() - cursor += 1 - continue + if quote is None: + if char == "(": + contexts.append(_ShellContext("(", cursor)) + cursor += 1 + continue + if char == ")" and len(contexts) > 1 and contexts[-1].kind in {"(", "$("}: + contexts.pop() + cursor += 1 + continue if char == "`": - if len(contexts) > 1 and contexts[-1].kind == "`": + if quote is None and len(contexts) > 1 and contexts[-1].kind == "`": contexts.pop() else: contexts.append(_ShellContext("`", cursor)) @@ -321,7 +241,7 @@ def _operator_before(command: str, start: int) -> str | None: def _cd_target(executable: str, args: list[str], cwd: Path) -> Path | None: if _executable_name(executable) not in {"cd", "pushd"}: return None - index = _consume_options(args, 0, frozenset()) + index = _consume_options(args, 0) if index >= len(args) or args[index] == "-": return None target = _resolve(args[index], cwd) @@ -331,13 +251,10 @@ def _cd_target(executable: str, args: list[str], cwd: Path) -> Path | None: def _shell_script_arg(args: list[str]) -> str | None: """Return the script string owned by a shell's ``-c``, if present. - Tries approval.py's ``_bash_exec_payload`` first: it parses bash's real - option grammar (``-O/-o`` consume the next argument, short-option - bundles, ``--init-file``/``--rcfile``), catching payloads a naive scan - misses — ``bash -o pipefail -c '