From 65f033a1a20e847b7a150fe6168cd345385c7d07 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 7 Sep 2026 03:33:05 -0700 Subject: [PATCH] fix(execute-code): teach the working helper import contract Slim adaptation of #83772 to the current schema and failure-hint table. Generated helpers are module exports on every execution path, not globals. Correct schema, recovery hints and CLI tip rather than injecting names or changing the execution boundary. Two registry-driven invariants reproduce both misleading instructions on main and execute the corrected guidance. Additional tool fix discovered during campaign #104904. Original diagnosis and correction: @yuzilongleif-collab (#83772). Co-authored-by: yuzilongleif-collab <235949691+yuzilongleif-collab@users.noreply.github.com> --- hermes_cli/tips.py | 2 +- tests/tools/test_execute_helper_contract.py | 58 +++++++++++++++++++++ tests/tools/test_sandbox_failure_hints.py | 6 +-- tools/code_execution_tool.py | 17 +++--- 4 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 tests/tools/test_execute_helper_contract.py diff --git a/hermes_cli/tips.py b/hermes_cli/tips.py index d9e83c5766..68397a957f 100644 --- a/hermes_cli/tips.py +++ b/hermes_cli/tips.py @@ -254,7 +254,7 @@ TIPS = [ "The terminal tool annotates common exit codes: grep returning 1 = 'No matches found (not an error)'.", "Failed foreground terminal commands auto-retry up to 3 times with exponential backoff (2s, 4s, 8s).", "Bare sudo commands are auto-rewritten to pipe SUDO_PASSWORD from .env — no interactive prompt needed.", - "execute_code has built-in helpers: json_parse() for tolerant parsing, shell_quote(), and retry() with backoff.", + "execute_code helpers require explicit imports: from hermes_tools import json_parse, shell_quote, retry.", "execute_code's 7 sandbox tools (web_search, terminal, read/write/search/patch) use RPC — never enter context.", "Reading the same file region 3+ times triggers a warning. At 4+, it's hard-blocked to prevent loops.", "write_file and patch detect if a file was externally modified since the last read and warn about staleness.", diff --git a/tests/tools/test_execute_helper_contract.py b/tests/tools/test_execute_helper_contract.py new file mode 100644 index 0000000000..bd4c621c67 --- /dev/null +++ b/tests/tools/test_execute_helper_contract.py @@ -0,0 +1,58 @@ +"""Execute the helper usage taught by the public schema and recovery hints.""" + +import json +import re + +import pytest + +from tools import code_execution_tool +from tools.code_kernel import shutdown_all_kernels +from tools.registry import registry + + +PROBE = ''' +from __future__ import annotations +{imports} +import json +print(json.dumps([json_parse('{{"value": 7}}')["value"], + shell_quote("two words"), retry(lambda: "ok", delay=0)])) +''' + + +@pytest.fixture(autouse=True) +def local_kernel(tmp_path, monkeypatch): + monkeypatch.setenv("TERMINAL_ENV", "local") + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + (tmp_path / "config.yaml").write_text("code_execution:\n mode: strict\n", encoding="utf-8") + shutdown_all_kernels() + yield + shutdown_all_kernels() + + +def run(code, reset=False): + return json.loads(registry.dispatch("execute_code", {"code": code, "reset": reset}, + task_id="helper-contract", enabled_tools=[])) + + +def test_schema_helper_instructions_work_across_kernel_lifetimes(): + description = registry.get_schema("execute_code")["description"] + imports = "\n".join(re.findall(r"`(from hermes_tools import [\w, ]+)`", description)) + for reset, reused in ((False, False), (False, True), (True, False)): + result = run(PROBE.format(imports=imports), reset=reset) + assert result["status"] == "success", result + assert json.loads(result["output"]) == [7, "'two words'", "ok"] + assert result["kernel"]["reused"] is reused + + +def test_missing_helper_recovery_instructions_execute(): + for helper in ("json_parse", "shell_quote", "retry"): + failed = run(f"print({helper})", reset=True) + assert failed["status"] == "error", failed + instruction = re.search(r"from hermes_tools import \w+", failed["hint"]) + assert instruction, failed + recovered = run(instruction.group() + f"\nprint(callable({helper}))") + assert recovered["status"] == "success", recovered + assert recovered["output"].strip() == "True" + skew = code_execution_tool._sandbox_failure_hint( + f"ImportError: cannot import name '{helper}' from 'hermes_tools'") + assert instruction.group() in skew diff --git a/tests/tools/test_sandbox_failure_hints.py b/tests/tools/test_sandbox_failure_hints.py index 1c52100e27..53c8a49486 100644 --- a/tests/tools/test_sandbox_failure_hints.py +++ b/tests/tools/test_sandbox_failure_hints.py @@ -16,11 +16,11 @@ class TestSandboxFailureHint: assert "read_file" in h and "terminal" in h assert "normal tool call" in h - def test_builtin_helper_import_redirects(self): + def test_helper_import_failure_reports_module_skew(self): err = "ImportError: cannot import name 'json_parse' from 'hermes_tools'" h = _sandbox_failure_hint(err) - assert "BUILT-IN" in h - assert "no import" in h.lower() + assert "from hermes_tools import json_parse" in h + assert "sys.path" in h def test_missing_third_party_module(self): err = "ModuleNotFoundError: No module named 'matplotlib'" diff --git a/tools/code_execution_tool.py b/tools/code_execution_tool.py index a06bc98679..c7d77d23d5 100644 --- a/tools/code_execution_tool.py +++ b/tools/code_execution_tool.py @@ -144,8 +144,10 @@ _TOOL_STUBS = { def _missing_hermes_tools_import_hint(m, enabled_tools) -> str: missing = m.group(1) if missing in {"json_parse", "shell_quote", "retry"}: - return (f"{missing} is a BUILT-IN helper in the sandbox — no import " - f"needed. Remove it from the import line and call {missing}(...) directly.") + return (f"Import helpers with `from hermes_tools import {missing}`. " + "If that import failed, the generated module may be stale or another " + "hermes_tools may be first on sys.path. Check hermes_tools.__file__ " + "and retry with reset=true.") available = sorted(SANDBOX_ALLOWED_TOOLS & set(enabled_tools or SANDBOX_ALLOWED_TOOLS)) return (f"'{missing}' is not available inside the execute_code sandbox. " f"Importable tools here: {', '.join(available)}. For anything " @@ -153,13 +155,13 @@ def _missing_hermes_tools_import_hint(m, enabled_tools) -> str: # (regex, formatter(match, enabled_tools)) — first match wins. Production mining (state.db) ranked -# these as the top execute_code failure classes: hermes_tools import misuse, importing the built-in -# helpers, treating tool results as strings, importing third-party packages absent from the sandbox. +# these as the top execute_code failure classes: hermes_tools import misuse, missing helper +# imports, treating tool results as strings, importing third-party packages absent from the sandbox. _FAILURE_HINT_RULES = ( (r"cannot import name '(\w+)' from 'hermes_tools'", _missing_hermes_tools_import_hint), (r"NameError: name '(json_parse|shell_quote|retry)' is not defined", - lambda m, _: f"{m.group(1)} is built into the generated sandbox module — " - "call it directly at module scope without importing it."), + lambda m, _: f"Import {m.group(1)} before calling it: " + f"from hermes_tools import {m.group(1)}"), (r"ModuleNotFoundError: No module named '([\w.]+)'", lambda m, _: f"'{m.group(1)}' is not installed in the sandbox interpreter. " "Use Python stdlib inside execute_code, or run the code via " @@ -850,7 +852,8 @@ def build_execute_code_schema(enabled_sandbox_tools: set = None, "Limits: 5-minute timeout, max 50 tool calls per call. Stdout over " "50KB shows head/tail inline; the FULL text is auto-saved to a file whose path rides in the result.\n\n" f"{cwd_note}\n\n" - "Built-in helpers (no import): json_parse(text) — tolerant " + "Helpers require imports: `from hermes_tools import json_parse, shell_quote, retry`. " + "json_parse(text) — tolerant " "json.loads for terminal() output; shell_quote(s) — shlex.quote for " "dynamic shell args; retry(fn, max_attempts=3, delay=2) — exponential backoff." )