diff --git a/hermes_cli/tips.py b/hermes_cli/tips.py index d9e83c5766..68397a957f 100644 --- a/hermes_cli/tips.py +++ b/hermes_cli/tips.py @@ -254,7 +254,7 @@ TIPS = [ "The terminal tool annotates common exit codes: grep returning 1 = 'No matches found (not an error)'.", "Failed foreground terminal commands auto-retry up to 3 times with exponential backoff (2s, 4s, 8s).", "Bare sudo commands are auto-rewritten to pipe SUDO_PASSWORD from .env — no interactive prompt needed.", - "execute_code has built-in helpers: json_parse() for tolerant parsing, shell_quote(), and retry() with backoff.", + "execute_code helpers require explicit imports: from hermes_tools import json_parse, shell_quote, retry.", "execute_code's 7 sandbox tools (web_search, terminal, read/write/search/patch) use RPC — never enter context.", "Reading the same file region 3+ times triggers a warning. At 4+, it's hard-blocked to prevent loops.", "write_file and patch detect if a file was externally modified since the last read and warn about staleness.", diff --git a/tests/tools/test_execute_helper_contract.py b/tests/tools/test_execute_helper_contract.py new file mode 100644 index 0000000000..bd4c621c67 --- /dev/null +++ b/tests/tools/test_execute_helper_contract.py @@ -0,0 +1,58 @@ +"""Execute the helper usage taught by the public schema and recovery hints.""" + +import json +import re + +import pytest + +from tools import code_execution_tool +from tools.code_kernel import shutdown_all_kernels +from tools.registry import registry + + +PROBE = ''' +from __future__ import annotations +{imports} +import json +print(json.dumps([json_parse('{{"value": 7}}')["value"], + shell_quote("two words"), retry(lambda: "ok", delay=0)])) +''' + + +@pytest.fixture(autouse=True) +def local_kernel(tmp_path, monkeypatch): + monkeypatch.setenv("TERMINAL_ENV", "local") + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + (tmp_path / "config.yaml").write_text("code_execution:\n mode: strict\n", encoding="utf-8") + shutdown_all_kernels() + yield + shutdown_all_kernels() + + +def run(code, reset=False): + return json.loads(registry.dispatch("execute_code", {"code": code, "reset": reset}, + task_id="helper-contract", enabled_tools=[])) + + +def test_schema_helper_instructions_work_across_kernel_lifetimes(): + description = registry.get_schema("execute_code")["description"] + imports = "\n".join(re.findall(r"`(from hermes_tools import [\w, ]+)`", description)) + for reset, reused in ((False, False), (False, True), (True, False)): + result = run(PROBE.format(imports=imports), reset=reset) + assert result["status"] == "success", result + assert json.loads(result["output"]) == [7, "'two words'", "ok"] + assert result["kernel"]["reused"] is reused + + +def test_missing_helper_recovery_instructions_execute(): + for helper in ("json_parse", "shell_quote", "retry"): + failed = run(f"print({helper})", reset=True) + assert failed["status"] == "error", failed + instruction = re.search(r"from hermes_tools import \w+", failed["hint"]) + assert instruction, failed + recovered = run(instruction.group() + f"\nprint(callable({helper}))") + assert recovered["status"] == "success", recovered + assert recovered["output"].strip() == "True" + skew = code_execution_tool._sandbox_failure_hint( + f"ImportError: cannot import name '{helper}' from 'hermes_tools'") + assert instruction.group() in skew diff --git a/tests/tools/test_sandbox_failure_hints.py b/tests/tools/test_sandbox_failure_hints.py index 1c52100e27..53c8a49486 100644 --- a/tests/tools/test_sandbox_failure_hints.py +++ b/tests/tools/test_sandbox_failure_hints.py @@ -16,11 +16,11 @@ class TestSandboxFailureHint: assert "read_file" in h and "terminal" in h assert "normal tool call" in h - def test_builtin_helper_import_redirects(self): + def test_helper_import_failure_reports_module_skew(self): err = "ImportError: cannot import name 'json_parse' from 'hermes_tools'" h = _sandbox_failure_hint(err) - assert "BUILT-IN" in h - assert "no import" in h.lower() + assert "from hermes_tools import json_parse" in h + assert "sys.path" in h def test_missing_third_party_module(self): err = "ModuleNotFoundError: No module named 'matplotlib'" diff --git a/tools/code_execution_tool.py b/tools/code_execution_tool.py index a06bc98679..c7d77d23d5 100644 --- a/tools/code_execution_tool.py +++ b/tools/code_execution_tool.py @@ -144,8 +144,10 @@ _TOOL_STUBS = { def _missing_hermes_tools_import_hint(m, enabled_tools) -> str: missing = m.group(1) if missing in {"json_parse", "shell_quote", "retry"}: - return (f"{missing} is a BUILT-IN helper in the sandbox — no import " - f"needed. Remove it from the import line and call {missing}(...) directly.") + return (f"Import helpers with `from hermes_tools import {missing}`. " + "If that import failed, the generated module may be stale or another " + "hermes_tools may be first on sys.path. Check hermes_tools.__file__ " + "and retry with reset=true.") available = sorted(SANDBOX_ALLOWED_TOOLS & set(enabled_tools or SANDBOX_ALLOWED_TOOLS)) return (f"'{missing}' is not available inside the execute_code sandbox. " f"Importable tools here: {', '.join(available)}. For anything " @@ -153,13 +155,13 @@ def _missing_hermes_tools_import_hint(m, enabled_tools) -> str: # (regex, formatter(match, enabled_tools)) — first match wins. Production mining (state.db) ranked -# these as the top execute_code failure classes: hermes_tools import misuse, importing the built-in -# helpers, treating tool results as strings, importing third-party packages absent from the sandbox. +# these as the top execute_code failure classes: hermes_tools import misuse, missing helper +# imports, treating tool results as strings, importing third-party packages absent from the sandbox. _FAILURE_HINT_RULES = ( (r"cannot import name '(\w+)' from 'hermes_tools'", _missing_hermes_tools_import_hint), (r"NameError: name '(json_parse|shell_quote|retry)' is not defined", - lambda m, _: f"{m.group(1)} is built into the generated sandbox module — " - "call it directly at module scope without importing it."), + lambda m, _: f"Import {m.group(1)} before calling it: " + f"from hermes_tools import {m.group(1)}"), (r"ModuleNotFoundError: No module named '([\w.]+)'", lambda m, _: f"'{m.group(1)}' is not installed in the sandbox interpreter. " "Use Python stdlib inside execute_code, or run the code via " @@ -850,7 +852,8 @@ def build_execute_code_schema(enabled_sandbox_tools: set = None, "Limits: 5-minute timeout, max 50 tool calls per call. Stdout over " "50KB shows head/tail inline; the FULL text is auto-saved to a file whose path rides in the result.\n\n" f"{cwd_note}\n\n" - "Built-in helpers (no import): json_parse(text) — tolerant " + "Helpers require imports: `from hermes_tools import json_parse, shell_quote, retry`. " + "json_parse(text) — tolerant " "json.loads for terminal() output; shell_quote(s) — shlex.quote for " "dynamic shell args; retry(fn, max_attempts=3, delay=2) — exponential backoff." )