ouyangbo/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix(evals): align Kanban probe with managed MCP overrides

Browse Source 
Use hermes-tools for the probe's managed server configuration and tool-call target so it exercises the endpoint receiving worker overrides. Leave intentional user-defined hermes-mcp fixtures unchanged.

Scope-risk: narrow
Tested: Three complete isolated probe runs on macOS with Codex CLI 0.137.0; related regression selection 202 passed, 2 existing skips; Ruff and repository static checks
Not-tested: Full repository suite, native Linux/Windows, hosted CI, cloud-model turns
This commit is contained in:
Shenrui Ma
2026-09-11 23:51:43 +08:00
committed by Teknium
parent 6973f2622d
commit 4521e04add
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evals/kanban_scope_probe.py
+3 -3
View File
@@ -34,8 +34,8 @@ ch.mkdir()
(ch / "config.toml").write_text(
'model="fixture"\nmodel_provider="fixture"\n'
'[model_providers.fixture]\nname="fixture"\nbase_url="http://127.0.0.1:9/v1"\nwire_api="responses"\n'
'[mcp_servers.hermes-mcp]\ncommand=' + json.dumps(sys.executable) + '\nargs=["-m","agent.transports.hermes_tools_mcp_server"]\nstartup_timeout_sec=40\n'
'[mcp_servers.hermes-mcp.env]\nPYTHONPATH=' + json.dumps(str(repo)) + '\nHERMES_HOME=' + json.dumps(str(hh)) + '\n'
'[mcp_servers.hermes-tools]\ncommand=' + json.dumps(sys.executable) + '\nargs=["-m","agent.transports.hermes_tools_mcp_server"]\nstartup_timeout_sec=40\n'
'[mcp_servers.hermes-tools.env]\nPYTHONPATH=' + json.dumps(str(repo)) + '\nHERMES_HOME=' + json.dumps(str(hh)) + '\n'
)
report = {}
with CodexAppServerClient(codex_bin=shutil.which("codex") or "codex", codex_home=str(ch)) as c:
@@ -51,7 +51,7 @@ with CodexAppServerClient(codex_bin=shutil.which("codex") or "codex", codex_home
status = c.request("mcpServerStatus/list", {"threadId": thread_id}, timeout=50)
report["mcp_servers"] = [{"name": x.get("name"), "tools": list(x.get("tools", {}))} for x in status.get("data", [])]
for name, tid in (("foreign", foreign), ("own", own)):
report[name] = c.request("mcpServer/tool/call", {"threadId": thread_id, "server": "hermes-mcp", "tool": "kanban_complete", "arguments": {"task_id": tid, "summary": "supervised parent handoff"}}, timeout=50)
report[name] = c.request("mcpServer/tool/call", {"threadId": thread_id, "server": "hermes-tools", "tool": "kanban_complete", "arguments": {"task_id": tid, "summary": "supervised parent handoff"}}, timeout=50)
report["stderr"] = c._stderr_lines[-8:]
report["readback"] = {"own": kb.get_task(conn, own).status, "foreign": kb.get_task(conn, foreign).status, "integrity": conn.execute("PRAGMA integrity_check").fetchone()[0]}
print(json.dumps(report, indent=2))