diff --git a/tests/tools/test_mcp_tool_session_expired.py b/tests/tools/test_mcp_tool_session_expired.py index 5c824d8f53..a4c068cb5c 100644 --- a/tests/tools/test_mcp_tool_session_expired.py +++ b/tests/tools/test_mcp_tool_session_expired.py @@ -186,6 +186,10 @@ def test_call_tool_handler_rebuilds_configured_server_transport( mcp_tool._servers["resumed"] = server mcp_tool._server_error_counts.pop("resumed", None) mcp_tool._server_breaker_opened_at.pop("resumed", None) + # Auto-retry after session expiry is only safe (and only performed) for + # tools positively annotated read-only; a write may already have + # executed server-side (#88821). + mcp_tool._tool_read_only_hints.setdefault("resumed", {})["health"] = True loop = mcp_tool._mcp_loop assert loop is not None run_future = asyncio.run_coroutine_threadsafe( @@ -218,6 +222,7 @@ def test_call_tool_handler_rebuilds_configured_server_transport( mcp_tool._servers.pop("resumed", None) mcp_tool._server_error_counts.pop("resumed", None) mcp_tool._server_breaker_opened_at.pop("resumed", None) + mcp_tool._tool_read_only_hints.pop("resumed", None) def test_session_expired_retry_waits_for_new_session(monkeypatch, tmp_path): @@ -278,6 +283,10 @@ def test_session_expired_retry_waits_for_new_session(monkeypatch, tmp_path): server._reconnect_event = _ReconnectAdapter() mcp_tool._servers["hindsight"] = server mcp_tool._server_error_counts["hindsight"] = 7 + # Read-only annotation: the session-expired auto-retry now only fires + # for tools positively marked readOnlyHint=True (write-capable calls + # get the outcome-unknown path instead). + mcp_tool._tool_read_only_hints.setdefault("hindsight", {})["get_bank"] = True # Stamp the breaker "open" far enough in the past that the cooldown has # provably elapsed, so this call is a half-open probe. The breaker compares # against time.monotonic() (tools/mcp_tool.py), whose origin is arbitrary and @@ -297,6 +306,7 @@ def test_session_expired_retry_waits_for_new_session(monkeypatch, tmp_path): mcp_tool._servers.pop("hindsight", None) mcp_tool._server_error_counts.pop("hindsight", None) mcp_tool._server_breaker_opened_at.pop("hindsight", None) + mcp_tool._tool_read_only_hints.pop("hindsight", None) def test_session_expired_handler_returns_none_without_loop(monkeypatch): @@ -417,3 +427,73 @@ def test_non_tool_handlers_also_reconnect_on_session_expired( finally: mcp_tool._servers.pop(f"srv-{op_label}", None) mcp_tool._server_error_counts.pop(f"srv-{op_label}", None) + + +# --------------------------------------------------------------------------- +# At-most-once guard for write-capable tools: a session-expired/transport +# failure on a call that may already have executed server-side must NOT be +# auto-retried. The transport is healed and an outcome_uncertain error tells +# the model to verify first. Only readOnlyHint=True tools keep the retry. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("read_only", [False, True], ids=["write-capable", "read-only"]) +def test_session_expired_retry_only_for_read_only_tools(monkeypatch, tmp_path, read_only): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + from tools import mcp_tool + from tools.mcp_tool_handlers import _make_tool_handler + + _mcp_loop._ensure_mcp_loop() + server, reconnect_flag = _install_stub_server("srv") + call_count = {"n": 0} + + async def _sequence(*a, **kw): + call_count["n"] += 1 + if call_count["n"] == 1: + raise RuntimeError("Session terminated") + result = MagicMock() + result.is_error = False + result.content = [MagicMock(type="text", text="fresh data")] + result.structured_content = None + return result + + server.session.call_tool = _sequence + mcp_tool._servers["srv"] = server + mcp_tool._server_error_counts.pop("srv", None) + if read_only: + mcp_tool._tool_read_only_hints.setdefault("srv", {})["tool"] = True + # else: no readOnlyHint entry -> fails safe to write-capable. + + try: + parsed = json.loads(_make_tool_handler("srv", "tool", 10.0)({})) + assert reconnect_flag.is_set() # transport healed either way + if read_only: + assert parsed.get("result") == "fresh data", parsed + assert call_count["n"] == 2 # original + one retry + else: + assert parsed.get("outcome_uncertain") is True, parsed + assert "may or may not have taken effect" in parsed["error"] + assert call_count["n"] == 1 # exactly one dispatch + # Successful reconnect clears breaker state (session-state failure, not server health). + assert mcp_tool._server_error_counts.get("srv", 0) == 0 + finally: + mcp_tool._servers.pop("srv", None) + mcp_tool._server_error_counts.pop("srv", None) + mcp_tool._server_breaker_opened_at.pop("srv", None) + mcp_tool._tool_read_only_hints.pop("srv", None) + + +def test_tool_is_read_only_fails_safe(): + """Unknown server/tool metadata classifies as write-capable (False).""" + from tools import mcp_tool + from tools.mcp_tool_handlers import _tool_is_read_only + + assert _tool_is_read_only("no-such-server", "tool") is False + mcp_tool._tool_read_only_hints["known"] = {"reader": True, "writer": False} + try: + assert _tool_is_read_only("known", "reader") is True + assert _tool_is_read_only("known", "writer") is False + assert _tool_is_read_only("known", "unlisted") is False + finally: + mcp_tool._tool_read_only_hints.pop("known", None) diff --git a/tools/mcp_tool_handlers.py b/tools/mcp_tool_handlers.py index e8df28d9a9..b9e11079e4 100644 --- a/tools/mcp_tool_handlers.py +++ b/tools/mcp_tool_handlers.py @@ -8,6 +8,7 @@ import inspect import json import time from contextlib import asynccontextmanager +from functools import partial from types import SimpleNamespace from typing import Any, Callable, Dict, List, Optional, Tuple from tools.registry import tool_error @@ -36,17 +37,28 @@ _STDIO_DIED_AGAIN_MSG = ( _STDIO_OUTCOME_UNCERTAIN_MSG = ( "MCP server '{s}' lost its stdio subprocess after the tool call began. The operation may have completed, so " "Hermes did not replay it. Do NOT retry automatically; inspect the external state first.") +_SESSION_OUTCOME_UNCERTAIN_MSG = ( + "The MCP transport session to '{s}' expired while this write-capable call was in flight, so the outcome is " + "UNKNOWN — the operation may or may not have taken effect server-side. It was NOT automatically retried to " + "avoid a duplicate side effect. The connection has {state}. Verify whether the operation took effect (e.g. " + "with a read-only tool) before re-invoking it.") + + +def _tool_is_read_only(server_name: str, tool_name: str) -> bool: + """True only when discovery captured ``readOnlyHint=True`` for the tool. Missing or malformed + metadata fails safe to False (treated as write-capable). readOnlyHint is a property of the + connection's tools, so it lives under the connection key.""" + from tools.mcp_tool_scope import _resolve_server_key + return _core._tool_read_only_hints.get(_resolve_server_key(server_name), {}).get(tool_name) is True def _trust_gate_check(server_name: str, tool_name: str) -> Optional[str]: """Approval gate for write-capable tools on ``trust: untrusted`` servers. None to proceed, else a ``tool_error``. Fail-closed: approval-system errors block.""" - from tools.mcp_tool_scope import _resolve_server_key, _server_key - # Trust is the calling profile's own policy (an adopter of a shared connection keeps its own tier); - # readOnlyHint is a property of the connection's tools, so it lives under the connection key. + from tools.mcp_tool_scope import _server_key + # Trust is the calling profile's own policy (an adopter of a shared connection keeps its own tier). trust = _core._server_trust_levels.get(_server_key(server_name), _core._TRUST_FULL) - if (trust != _core._TRUST_UNTRUSTED - or _core._tool_read_only_hints.get(_resolve_server_key(server_name), {}).get(tool_name) is True): + if trust != _core._TRUST_UNTRUSTED or _tool_is_read_only(server_name, tool_name): return None try: # lazy: tools.approval routes the prompt to whichever surface owns the session from tools.approval_prompt import request_elicitation_consent @@ -181,7 +193,8 @@ def _handle_auth_error_and_retry(server_name: str, exc: BaseException, retry_cal return _strike(server_name, _NEEDS_REAUTH_MSG.format(s=server_name), needs_reauth=True, server=server_name) -def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retry_call, op_description: str): +def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retry_call, op_description: str, + *, call_may_have_side_effects: bool = False): """Transport reconnect + one retry on session expiry; None to fall through. Skips ``handle_401``: the token is valid, only the server-side session is stale. @@ -190,10 +203,29 @@ def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retr ``_reconnect_event`` causes the server task's lifecycle loop to tear down the current ``streamablehttp_client`` + ``ClientSession`` and rebuild them, reusing the existing OAuth provider instance. See #13383. + + At-most-once for writes: a session-expired shape can be synthesized by a proxy AFTER the upstream + executed the request, and the transport errors this classifier also matches (``ClosedResourceError``, + broken pipe) routinely fire mid-response. With ``call_may_have_side_effects`` the transport is still + healed but the call is never re-run; the model gets an ``outcome_uncertain`` error instead (same + contract as the mid-call stdio death path). Callers pass True unless the tool is positively read-only. """ srv = _lookup_reconnectable_server(server_name, require_loop=True) if _is_session_expired_error(exc) else None if srv is None: return None + if call_may_have_side_effects: + reconnected = _loop._signal_reconnect_and_wait( + server_name, srv, op_description=f"{op_description} (write, no auto-retry)", timeout=15) + if reconnected: # session state failed, not server health: no breaker strike + _core._reset_server_error(server_name) + else: + _core._bump_server_error(server_name) + logger.warning("MCP server '%s': %s failed with a session-expired/transport error after the request may " + "have been dispatched; NOT auto-retrying a write-capable tool (reconnect %s).", + server_name, op_description, "succeeded" if reconnected else "failed") + return tool_error(_SESSION_OUTCOME_UNCERTAIN_MSG.format( + s=server_name, state="been re-established" if reconnected else "not recovered yet"), + outcome_uncertain=True, server=server_name) logger.info("MCP server '%s': %s failed with session-expired error (%s); signalling transport reconnect " "and retrying once.", server_name, op_description, exc) if not _loop._signal_reconnect_and_wait(server_name, srv, op_description=op_description, timeout=15): @@ -502,9 +534,13 @@ def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float): def _on_failure(exc): _core._bump_server_error(server_name) logger.error("MCP tool %s/%s call failed: %s", server_name, tool_name, exc) + # Only a tool annotated readOnlyHint=True is replayed after session expiry; a 401 is always + # pre-dispatch so the auth recoverer keeps its retry for every tool. + session_expired = partial(_handle_session_expired_and_retry, + call_may_have_side_effects=not _tool_is_read_only(server_name, tool_name)) return _dispatch( server_name, server, op, _call, tool_timeout, - (_handle_stdio_child_exited_and_retry, _handle_auth_error_and_retry, _handle_session_expired_and_retry), + (_handle_stdio_child_exited_and_retry, _handle_auth_error_and_retry, session_expired), _on_failure, record_outcome=True) return _handler