From 4521e04add661eb4f3628be2ebbaa7b4ad5abae7 Mon Sep 17 00:00:00 2001 From: Shenrui Ma <115921185+Shenrui-Ma@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:51:43 +0800 Subject: [PATCH] fix(evals): align Kanban probe with managed MCP overrides Use hermes-tools for the probe's managed server configuration and tool-call target so it exercises the endpoint receiving worker overrides. Leave intentional user-defined hermes-mcp fixtures unchanged. Scope-risk: narrow Tested: Three complete isolated probe runs on macOS with Codex CLI 0.137.0; related regression selection 202 passed, 2 existing skips; Ruff and repository static checks Not-tested: Full repository suite, native Linux/Windows, hosted CI, cloud-model turns --- evals/kanban_scope_probe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/evals/kanban_scope_probe.py b/evals/kanban_scope_probe.py index 46f6764f95..b36c48a457 100644 --- a/evals/kanban_scope_probe.py +++ b/evals/kanban_scope_probe.py @@ -34,8 +34,8 @@ ch.mkdir() (ch / "config.toml").write_text( 'model="fixture"\nmodel_provider="fixture"\n' '[model_providers.fixture]\nname="fixture"\nbase_url="http://127.0.0.1:9/v1"\nwire_api="responses"\n' - '[mcp_servers.hermes-mcp]\ncommand=' + json.dumps(sys.executable) + '\nargs=["-m","agent.transports.hermes_tools_mcp_server"]\nstartup_timeout_sec=40\n' - '[mcp_servers.hermes-mcp.env]\nPYTHONPATH=' + json.dumps(str(repo)) + '\nHERMES_HOME=' + json.dumps(str(hh)) + '\n' + '[mcp_servers.hermes-tools]\ncommand=' + json.dumps(sys.executable) + '\nargs=["-m","agent.transports.hermes_tools_mcp_server"]\nstartup_timeout_sec=40\n' + '[mcp_servers.hermes-tools.env]\nPYTHONPATH=' + json.dumps(str(repo)) + '\nHERMES_HOME=' + json.dumps(str(hh)) + '\n' ) report = {} with CodexAppServerClient(codex_bin=shutil.which("codex") or "codex", codex_home=str(ch)) as c: @@ -51,7 +51,7 @@ with CodexAppServerClient(codex_bin=shutil.which("codex") or "codex", codex_home status = c.request("mcpServerStatus/list", {"threadId": thread_id}, timeout=50) report["mcp_servers"] = [{"name": x.get("name"), "tools": list(x.get("tools", {}))} for x in status.get("data", [])] for name, tid in (("foreign", foreign), ("own", own)): - report[name] = c.request("mcpServer/tool/call", {"threadId": thread_id, "server": "hermes-mcp", "tool": "kanban_complete", "arguments": {"task_id": tid, "summary": "supervised parent handoff"}}, timeout=50) + report[name] = c.request("mcpServer/tool/call", {"threadId": thread_id, "server": "hermes-tools", "tool": "kanban_complete", "arguments": {"task_id": tid, "summary": "supervised parent handoff"}}, timeout=50) report["stderr"] = c._stderr_lines[-8:] report["readback"] = {"own": kb.get_task(conn, own).status, "foreign": kb.get_task(conn, foreign).status, "integrity": conn.execute("PRAGMA integrity_check").fetchone()[0]} print(json.dumps(report, indent=2))