Teknium
2a95791992
run_agent.py: delete the `# noqa: F401` re-export block (agent.process_bootstrap
OpenAI/_SafeWriter/_get_proxy_*, model_tools get_tool_definitions/
handle_function_call/check_toolset_requirements, FailoverReason,
_qwen_portal_headers/_routermint_headers, session_persistence names,
estimate_request_tokens_rough, ContextCompressor + friends, jittered_backoff,
prompt_builder names, message_sanitization names, tool_dispatch_helpers
names) — 41 names run_agent never used itself — and the `_STREAM_DIAG_HEADERS`
back-compat class alias (no in-tree reader). run_agent now imports only what
it uses (get_toolset_for_tool, is_local_endpoint, coalesce/uniquify tool-call
ids, cleanup_vm/get_active_env from terminal_tool_lifecycle).
agent/*: `_ra().X` late-binds that only reached a re-export now import the
defining module directly (agent_runtime_helpers -> process_bootstrap.OpenAI,
model_tools.handle_function_call, session_persistence._safe_session_filename_component;
agent_init -> model_tools.get_tool_definitions/check_toolset_requirements,
_lazy_headers("agent.client_lifecycle", ...) for qwen/routermint;
system_prompt -> agent.prompt_builder / model_tools directly, dropping its
own _ra() shim and the `_r` parameter threading). `_ra()` stays for
run_agent-resident names (logger, AIAgent, _hermes_home, _set_interrupt, ...).
toolsets.py: remove resolve_multiple_toolsets (shim-only, restored by
34abf954bd); tests/test_toolsets.py pins the same union behavior via
resolve_toolset over each name.
providers/__init__.py: drop the OMIT_TEMPERATURE re-export (no callers via the
package); ProviderProfile stays because __init__ uses it for annotations —
2 tests repointed to providers.base.
agent/iteration_budget.py: drop the "run_agent re-exports the class"
docstring pointer; 4 tests import IterationBudget from its home.
model_tools.py (arg_coercion names), agent/tool_executor.py, and
hermes_cli/cli_session_mixin.py repoints landed via a sibling commit on this
shared worktree.
Callers repointed: gateway/run.py, hermes_cli/cli_chat_turn_mixin.py,
hermes_cli/cli_tui_mixin.py, tui_gateway/session_workdir.py,
agent/transports/codex.py (one-line imports) + comment pointers in
tools/file_state.py, tools/schema_sanitizer.py, scripts/tool_search_livetest.py.
Tests: patch("run_agent.X") / monkeypatch.setattr(run_agent, "X") /
`from run_agent import X` -> defining module across 99 test files.
358 lines
13 KiB
Python
358 lines
13 KiB
Python
"""Tests for toolsets.py — toolset resolution, validation, and composition."""
|
|
|
|
import toolsets as toolsets_mod
|
|
from tools.registry import ToolRegistry
|
|
from toolsets import (
|
|
TOOLSETS,
|
|
get_toolset,
|
|
resolve_toolset,
|
|
get_all_toolsets,
|
|
validate_toolset,
|
|
create_custom_toolset,
|
|
get_toolset_info,
|
|
)
|
|
|
|
|
|
def _dummy_handler(args, **kwargs):
|
|
return "{}"
|
|
|
|
|
|
def _make_schema(name: str, description: str = "test tool"):
|
|
return {
|
|
"name": name,
|
|
"description": description,
|
|
"parameters": {"type": "object", "properties": {}},
|
|
}
|
|
|
|
|
|
class TestGetToolset:
|
|
def test_known_toolset(self):
|
|
ts = get_toolset("web")
|
|
assert ts is not None
|
|
assert "web_search" in ts["tools"]
|
|
|
|
def test_x_search_toolset_marks_read_only_and_points_to_xurl(self):
|
|
ts = get_toolset("x_search")
|
|
assert ts is not None
|
|
assert ts["tools"] == ["x_search"]
|
|
description = ts["description"].lower()
|
|
assert "read-only" in description
|
|
assert "xurl" in description
|
|
assert "authenticated" in description
|
|
|
|
def test_merges_registry_tools_into_builtin_toolset(self, monkeypatch):
|
|
reg = ToolRegistry()
|
|
reg.register(
|
|
name="web_search_plus",
|
|
toolset="web",
|
|
schema=_make_schema("web_search_plus", "Plugin web search"),
|
|
handler=_dummy_handler,
|
|
)
|
|
|
|
monkeypatch.setattr("tools.registry.registry", reg)
|
|
|
|
ts = get_toolset("web")
|
|
assert ts is not None
|
|
assert set(ts["tools"]) == {"web_search", "web_extract", "web_search_plus"}
|
|
|
|
|
|
|
|
class TestResolveToolset:
|
|
def test_leaf_toolset(self):
|
|
tools = resolve_toolset("web")
|
|
assert set(tools) == {"web_search", "web_extract"}
|
|
|
|
def test_composite_toolset(self):
|
|
tools = resolve_toolset("debugging")
|
|
assert "terminal" in tools
|
|
assert "web_search" in tools
|
|
assert "web_extract" in tools
|
|
|
|
def test_cycle_detection(self):
|
|
# Create a cycle: A includes B, B includes A
|
|
TOOLSETS["_cycle_a"] = {"description": "test", "tools": ["t1"], "includes": ["_cycle_b"]}
|
|
TOOLSETS["_cycle_b"] = {"description": "test", "tools": ["t2"], "includes": ["_cycle_a"]}
|
|
try:
|
|
tools = resolve_toolset("_cycle_a")
|
|
# Should not infinite loop — cycle is detected
|
|
assert "t1" in tools
|
|
assert "t2" in tools
|
|
finally:
|
|
del TOOLSETS["_cycle_a"]
|
|
del TOOLSETS["_cycle_b"]
|
|
|
|
|
|
def test_plugin_toolset_uses_registry_snapshot(self, monkeypatch):
|
|
reg = ToolRegistry()
|
|
reg.register(
|
|
name="plugin_b",
|
|
toolset="plugin_example",
|
|
schema=_make_schema("plugin_b", "B"),
|
|
handler=_dummy_handler,
|
|
)
|
|
reg.register(
|
|
name="plugin_a",
|
|
toolset="plugin_example",
|
|
schema=_make_schema("plugin_a", "A"),
|
|
handler=_dummy_handler,
|
|
)
|
|
|
|
monkeypatch.setattr("tools.registry.registry", reg)
|
|
|
|
assert resolve_toolset("plugin_example") == ["plugin_a", "plugin_b"]
|
|
|
|
|
|
|
|
|
|
|
|
class TestResolveToolsetComposition:
|
|
def test_union_over_names_combines_and_deduplicates(self):
|
|
tools = sorted({t for name in ("web", "terminal") for t in resolve_toolset(name)})
|
|
assert "web_search" in tools
|
|
assert "web_extract" in tools
|
|
assert "terminal" in tools
|
|
# No duplicates
|
|
assert len(tools) == len(set(tools))
|
|
|
|
|
|
class TestValidateToolset:
|
|
def test_valid(self):
|
|
assert validate_toolset("web") is True
|
|
assert validate_toolset("terminal") is True
|
|
|
|
|
|
def test_invalid(self):
|
|
assert validate_toolset("nonexistent") is False
|
|
|
|
def test_mcp_alias_uses_live_registry(self, monkeypatch):
|
|
reg = ToolRegistry()
|
|
reg.register(
|
|
name="mcp__dynserver__ping",
|
|
toolset="mcp-dynserver",
|
|
schema=_make_schema("mcp__dynserver__ping", "Ping"),
|
|
handler=_dummy_handler,
|
|
)
|
|
reg.register_toolset_alias("dynserver", "mcp-dynserver")
|
|
|
|
monkeypatch.setattr("tools.registry.registry", reg)
|
|
|
|
assert validate_toolset("dynserver") is True
|
|
assert validate_toolset("mcp-dynserver") is True
|
|
assert "mcp__dynserver__ping" in resolve_toolset("dynserver")
|
|
|
|
|
|
class TestGetToolsetInfo:
|
|
def test_leaf(self):
|
|
info = get_toolset_info("web")
|
|
assert info["name"] == "web"
|
|
assert info["is_composite"] is False
|
|
assert info["tool_count"] == 2
|
|
|
|
def test_composite(self):
|
|
info = get_toolset_info("debugging")
|
|
assert info["is_composite"] is True
|
|
assert info["tool_count"] > len(info["direct_tools"])
|
|
|
|
|
|
|
|
class TestCreateCustomToolset:
|
|
def test_runtime_creation(self):
|
|
create_custom_toolset(
|
|
name="_test_custom",
|
|
description="Test toolset",
|
|
tools=["web_search"],
|
|
includes=["terminal"],
|
|
)
|
|
try:
|
|
tools = resolve_toolset("_test_custom")
|
|
assert "web_search" in tools
|
|
assert "terminal" in tools
|
|
assert validate_toolset("_test_custom") is True
|
|
finally:
|
|
del TOOLSETS["_test_custom"]
|
|
|
|
|
|
class TestRegistryOwnedToolsets:
|
|
def test_registry_membership_is_live(self, monkeypatch):
|
|
reg = ToolRegistry()
|
|
reg.register(
|
|
name="test_live_toolset_tool",
|
|
toolset="test-live-toolset",
|
|
schema=_make_schema("test_live_toolset_tool", "Live"),
|
|
handler=_dummy_handler,
|
|
)
|
|
|
|
monkeypatch.setattr("tools.registry.registry", reg)
|
|
|
|
assert validate_toolset("test-live-toolset") is True
|
|
assert get_toolset("test-live-toolset")["tools"] == ["test_live_toolset_tool"]
|
|
assert resolve_toolset("test-live-toolset") == ["test_live_toolset_tool"]
|
|
|
|
|
|
class TestToolsetConsistency:
|
|
"""Verify structural integrity of the built-in TOOLSETS dict."""
|
|
|
|
def test_all_toolsets_have_required_keys(self):
|
|
for name, ts in TOOLSETS.items():
|
|
assert "description" in ts, f"{name} missing description"
|
|
assert "tools" in ts, f"{name} missing tools"
|
|
assert "includes" in ts, f"{name} missing includes"
|
|
|
|
|
|
def test_hermes_platforms_share_core_tools(self):
|
|
"""All hermes-* platform toolsets share the same core tools.
|
|
|
|
Platform-specific additions (e.g. ``discord`` / ``discord_admin``
|
|
on hermes-discord, gated on DISCORD_BOT_TOKEN) are allowed on top —
|
|
the invariant is that the core set is identical across platforms.
|
|
"""
|
|
platforms = ["hermes-cli", "hermes-telegram", "hermes-discord", "hermes-whatsapp", "hermes-slack", "hermes-signal", "hermes-homeassistant"]
|
|
tool_sets = [set(TOOLSETS[p]["tools"]) for p in platforms]
|
|
# All platforms must contain the shared core; platform-specific
|
|
# extras are OK (subset check, not equality).
|
|
core = set.intersection(*tool_sets)
|
|
for name, ts in zip(platforms, tool_sets):
|
|
assert core.issubset(ts), f"{name} is missing core tools: {core - ts}"
|
|
# Sanity: the shared core must be non-trivial (i.e. we didn't
|
|
# silently let a platform diverge so far that nothing is shared).
|
|
assert len(core) > 20, f"Suspiciously small shared core: {len(core)} tools"
|
|
|
|
|
|
class TestPluginToolsets:
|
|
def test_get_all_toolsets_includes_plugin_toolset(self, monkeypatch):
|
|
reg = ToolRegistry()
|
|
reg.register(
|
|
name="plugin_tool",
|
|
toolset="plugin_bundle",
|
|
schema=_make_schema("plugin_tool", "Plugin tool"),
|
|
handler=_dummy_handler,
|
|
)
|
|
|
|
monkeypatch.setattr("tools.registry.registry", reg)
|
|
|
|
all_toolsets = get_all_toolsets()
|
|
assert "plugin_bundle" in all_toolsets
|
|
assert all_toolsets["plugin_bundle"]["tools"] == ["plugin_tool"]
|
|
|
|
|
|
class TestDefaultPlatformWebSearchCoverage:
|
|
def test_hermes_whatsapp_toolset_includes_web_search(self):
|
|
assert "web_search" in resolve_toolset("hermes-whatsapp")
|
|
|
|
|
|
|
|
class TestResolveToolsetIncludeRegistry:
|
|
"""include_registry flag exposes the static (pre-registry-merge) view used
|
|
by platform reverse-mapping. Regression harness for issue #49622."""
|
|
|
|
def test_include_registry_false_excludes_registry_tools(self):
|
|
from tools.registry import discover_builtin_tools, registry
|
|
discover_builtin_tools()
|
|
|
|
# Register a tool into `terminal` at runtime, the way plugins and MCP
|
|
# servers do, so the split is exercised on the mechanism rather than on
|
|
# whichever built-in currently happens to live where.
|
|
registry.register(
|
|
name="__probe_registry_only_tool__",
|
|
toolset="terminal",
|
|
schema={"name": "__probe_registry_only_tool__", "parameters": {"type": "object", "properties": {}}},
|
|
handler=lambda args, **kw: "",
|
|
)
|
|
try:
|
|
merged = set(resolve_toolset("terminal"))
|
|
static = set(resolve_toolset("terminal", include_registry=False))
|
|
finally:
|
|
registry.deregister("__probe_registry_only_tool__")
|
|
|
|
assert static == {"terminal", "process_manage"}, static
|
|
# Registered into 'terminal' but not part of the static definition — it
|
|
# must only appear in the merged view.
|
|
assert "__probe_registry_only_tool__" in merged
|
|
assert "__probe_registry_only_tool__" not in static
|
|
|
|
|
|
def test_static_view_threads_through_includes(self):
|
|
# 'debugging' has direct tools [terminal, process] and includes [web, file]
|
|
static = set(resolve_toolset("debugging", include_registry=False))
|
|
assert {"terminal", "process_manage"} <= static
|
|
assert "web_search" in static
|
|
assert "read_file" in static
|
|
|
|
|
|
def test_registry_only_toolset_static_view_is_empty(self):
|
|
assert resolve_toolset("__definitely_not_a_real_toolset__", include_registry=False) == []
|
|
|
|
|
|
class TestResolveToolsetMemo:
|
|
"""Measured-work pins for the generation-keyed resolution memo."""
|
|
|
|
def test_second_resolution_is_cached(self, monkeypatch):
|
|
"""Repeated resolves of the same toolset must not re-walk the registry.
|
|
|
|
resolve_toolset is called dozens of times per _get_platform_tools()
|
|
(every /tools completion keystroke). The memo keyed on the registry
|
|
generation makes repeat calls a dict lookup instead of a full
|
|
includes-walk + registry snapshot.
|
|
"""
|
|
from tools.registry import registry
|
|
|
|
toolsets_mod._resolve_toolset_memo.clear()
|
|
get_toolset_calls = {"n": 0}
|
|
|
|
orig_get_toolset = toolsets_mod.get_toolset
|
|
|
|
def counting_get_toolset(name, *, include_registry=True):
|
|
get_toolset_calls["n"] += 1
|
|
return orig_get_toolset(name, include_registry=include_registry)
|
|
|
|
monkeypatch.setattr(toolsets_mod, "get_toolset", counting_get_toolset)
|
|
|
|
registry_id = id(registry)
|
|
generation = registry._generation
|
|
|
|
first = resolve_toolset("hermes-cli")
|
|
second = resolve_toolset("hermes-cli")
|
|
|
|
assert first == second
|
|
assert get_toolset_calls["n"] == 1, (
|
|
"second resolution must be a memo hit (no get_toolset re-walk), "
|
|
f"got {get_toolset_calls['n']} calls"
|
|
)
|
|
assert (
|
|
"hermes-cli", True, registry_id, generation
|
|
) in toolsets_mod._resolve_toolset_memo
|
|
|
|
def test_generation_bump_invalidates_memo(self, monkeypatch):
|
|
"""A registry mutation (generation bump) must force a fresh resolve."""
|
|
from tools.registry import registry
|
|
|
|
toolsets_mod._resolve_toolset_memo.clear()
|
|
get_toolset_calls = {"n": 0}
|
|
|
|
orig_get_toolset = toolsets_mod.get_toolset
|
|
|
|
def counting_get_toolset(name, *, include_registry=True):
|
|
get_toolset_calls["n"] += 1
|
|
return orig_get_toolset(name, include_registry=include_registry)
|
|
|
|
monkeypatch.setattr(toolsets_mod, "get_toolset", counting_get_toolset)
|
|
|
|
resolve_toolset("hermes-cli")
|
|
assert get_toolset_calls["n"] == 1
|
|
|
|
# Simulate a registry mutation bumping the generation.
|
|
registry._generation += 1
|
|
resolve_toolset("hermes-cli")
|
|
assert get_toolset_calls["n"] == 2, (
|
|
"generation bump must invalidate the memo and re-resolve"
|
|
)
|
|
|
|
def test_memo_result_matches_fresh_resolution(self):
|
|
"""The memo must never change the resolved result."""
|
|
toolsets_mod._resolve_toolset_memo.clear()
|
|
first = resolve_toolset("hermes-cli", include_registry=False)
|
|
second = resolve_toolset("hermes-cli", include_registry=False)
|
|
assert first == second
|
|
assert first # non-empty sanity
|
|
|