Port from cloudflare/cloudflare-os#168: never auto-retry write-capable MCP tools after mid-flight session expiry
A 'session expired' / transport-closed failure can arrive AFTER the server already accepted and executed the request (proxy-synthesized 404s, pod rotation, ClosedResourceError firing mid-response). Auto-retrying a write-capable tool in that window risks a duplicate side effect that MCP offers no way to undo. The session-expired recovery path now consults the discovery-time readOnlyHint capture (same data the trust gate uses): only tools whose annotation is exactly True keep the reconnect+retry-once behavior. Write- capable calls still get the transport healed (reconnect, breaker reset on success) but return a structured outcome_unknown error telling the model to verify with a read before re-invoking. The OAuth 401 path keeps its retry for all tools: a 401 means the server demanded authorization before dispatch, so the call never executed — matching the upstream classifier's McpAuthRequiredError => safe rule. Fails safe: missing/unknown annotations classify as write-capable.
This commit is contained in:
@@ -186,6 +186,10 @@ def test_call_tool_handler_rebuilds_configured_server_transport(
|
||||
mcp_tool._servers["resumed"] = server
|
||||
mcp_tool._server_error_counts.pop("resumed", None)
|
||||
mcp_tool._server_breaker_opened_at.pop("resumed", None)
|
||||
# Auto-retry after session expiry is only safe (and only performed) for
|
||||
# tools positively annotated read-only; a write may already have
|
||||
# executed server-side (#88821).
|
||||
mcp_tool._tool_read_only_hints.setdefault("resumed", {})["health"] = True
|
||||
loop = mcp_tool._mcp_loop
|
||||
assert loop is not None
|
||||
run_future = asyncio.run_coroutine_threadsafe(
|
||||
@@ -218,6 +222,7 @@ def test_call_tool_handler_rebuilds_configured_server_transport(
|
||||
mcp_tool._servers.pop("resumed", None)
|
||||
mcp_tool._server_error_counts.pop("resumed", None)
|
||||
mcp_tool._server_breaker_opened_at.pop("resumed", None)
|
||||
mcp_tool._tool_read_only_hints.pop("resumed", None)
|
||||
|
||||
|
||||
def test_session_expired_retry_waits_for_new_session(monkeypatch, tmp_path):
|
||||
@@ -278,6 +283,10 @@ def test_session_expired_retry_waits_for_new_session(monkeypatch, tmp_path):
|
||||
server._reconnect_event = _ReconnectAdapter()
|
||||
mcp_tool._servers["hindsight"] = server
|
||||
mcp_tool._server_error_counts["hindsight"] = 7
|
||||
# Read-only annotation: the session-expired auto-retry now only fires
|
||||
# for tools positively marked readOnlyHint=True (write-capable calls
|
||||
# get the outcome-unknown path instead).
|
||||
mcp_tool._tool_read_only_hints.setdefault("hindsight", {})["get_bank"] = True
|
||||
# Stamp the breaker "open" far enough in the past that the cooldown has
|
||||
# provably elapsed, so this call is a half-open probe. The breaker compares
|
||||
# against time.monotonic() (tools/mcp_tool.py), whose origin is arbitrary and
|
||||
@@ -297,6 +306,7 @@ def test_session_expired_retry_waits_for_new_session(monkeypatch, tmp_path):
|
||||
mcp_tool._servers.pop("hindsight", None)
|
||||
mcp_tool._server_error_counts.pop("hindsight", None)
|
||||
mcp_tool._server_breaker_opened_at.pop("hindsight", None)
|
||||
mcp_tool._tool_read_only_hints.pop("hindsight", None)
|
||||
|
||||
|
||||
def test_session_expired_handler_returns_none_without_loop(monkeypatch):
|
||||
@@ -417,3 +427,73 @@ def test_non_tool_handlers_also_reconnect_on_session_expired(
|
||||
finally:
|
||||
mcp_tool._servers.pop(f"srv-{op_label}", None)
|
||||
mcp_tool._server_error_counts.pop(f"srv-{op_label}", None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# At-most-once guard for write-capable tools: a session-expired/transport
|
||||
# failure on a call that may already have executed server-side must NOT be
|
||||
# auto-retried. The transport is healed and an outcome_uncertain error tells
|
||||
# the model to verify first. Only readOnlyHint=True tools keep the retry.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("read_only", [False, True], ids=["write-capable", "read-only"])
|
||||
def test_session_expired_retry_only_for_read_only_tools(monkeypatch, tmp_path, read_only):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
||||
from tools import mcp_tool
|
||||
from tools.mcp_tool_handlers import _make_tool_handler
|
||||
|
||||
_mcp_loop._ensure_mcp_loop()
|
||||
server, reconnect_flag = _install_stub_server("srv")
|
||||
call_count = {"n": 0}
|
||||
|
||||
async def _sequence(*a, **kw):
|
||||
call_count["n"] += 1
|
||||
if call_count["n"] == 1:
|
||||
raise RuntimeError("Session terminated")
|
||||
result = MagicMock()
|
||||
result.is_error = False
|
||||
result.content = [MagicMock(type="text", text="fresh data")]
|
||||
result.structured_content = None
|
||||
return result
|
||||
|
||||
server.session.call_tool = _sequence
|
||||
mcp_tool._servers["srv"] = server
|
||||
mcp_tool._server_error_counts.pop("srv", None)
|
||||
if read_only:
|
||||
mcp_tool._tool_read_only_hints.setdefault("srv", {})["tool"] = True
|
||||
# else: no readOnlyHint entry -> fails safe to write-capable.
|
||||
|
||||
try:
|
||||
parsed = json.loads(_make_tool_handler("srv", "tool", 10.0)({}))
|
||||
assert reconnect_flag.is_set() # transport healed either way
|
||||
if read_only:
|
||||
assert parsed.get("result") == "fresh data", parsed
|
||||
assert call_count["n"] == 2 # original + one retry
|
||||
else:
|
||||
assert parsed.get("outcome_uncertain") is True, parsed
|
||||
assert "may or may not have taken effect" in parsed["error"]
|
||||
assert call_count["n"] == 1 # exactly one dispatch
|
||||
# Successful reconnect clears breaker state (session-state failure, not server health).
|
||||
assert mcp_tool._server_error_counts.get("srv", 0) == 0
|
||||
finally:
|
||||
mcp_tool._servers.pop("srv", None)
|
||||
mcp_tool._server_error_counts.pop("srv", None)
|
||||
mcp_tool._server_breaker_opened_at.pop("srv", None)
|
||||
mcp_tool._tool_read_only_hints.pop("srv", None)
|
||||
|
||||
|
||||
def test_tool_is_read_only_fails_safe():
|
||||
"""Unknown server/tool metadata classifies as write-capable (False)."""
|
||||
from tools import mcp_tool
|
||||
from tools.mcp_tool_handlers import _tool_is_read_only
|
||||
|
||||
assert _tool_is_read_only("no-such-server", "tool") is False
|
||||
mcp_tool._tool_read_only_hints["known"] = {"reader": True, "writer": False}
|
||||
try:
|
||||
assert _tool_is_read_only("known", "reader") is True
|
||||
assert _tool_is_read_only("known", "writer") is False
|
||||
assert _tool_is_read_only("known", "unlisted") is False
|
||||
finally:
|
||||
mcp_tool._tool_read_only_hints.pop("known", None)
|
||||
|
||||
@@ -8,6 +8,7 @@ import inspect
|
||||
import json
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
from tools.registry import tool_error
|
||||
@@ -36,17 +37,28 @@ _STDIO_DIED_AGAIN_MSG = (
|
||||
_STDIO_OUTCOME_UNCERTAIN_MSG = (
|
||||
"MCP server '{s}' lost its stdio subprocess after the tool call began. The operation may have completed, so "
|
||||
"Hermes did not replay it. Do NOT retry automatically; inspect the external state first.")
|
||||
_SESSION_OUTCOME_UNCERTAIN_MSG = (
|
||||
"The MCP transport session to '{s}' expired while this write-capable call was in flight, so the outcome is "
|
||||
"UNKNOWN — the operation may or may not have taken effect server-side. It was NOT automatically retried to "
|
||||
"avoid a duplicate side effect. The connection has {state}. Verify whether the operation took effect (e.g. "
|
||||
"with a read-only tool) before re-invoking it.")
|
||||
|
||||
|
||||
def _tool_is_read_only(server_name: str, tool_name: str) -> bool:
|
||||
"""True only when discovery captured ``readOnlyHint=True`` for the tool. Missing or malformed
|
||||
metadata fails safe to False (treated as write-capable). readOnlyHint is a property of the
|
||||
connection's tools, so it lives under the connection key."""
|
||||
from tools.mcp_tool_scope import _resolve_server_key
|
||||
return _core._tool_read_only_hints.get(_resolve_server_key(server_name), {}).get(tool_name) is True
|
||||
|
||||
|
||||
def _trust_gate_check(server_name: str, tool_name: str) -> Optional[str]:
|
||||
"""Approval gate for write-capable tools on ``trust: untrusted`` servers. None to proceed,
|
||||
else a ``tool_error``. Fail-closed: approval-system errors block."""
|
||||
from tools.mcp_tool_scope import _resolve_server_key, _server_key
|
||||
# Trust is the calling profile's own policy (an adopter of a shared connection keeps its own tier);
|
||||
# readOnlyHint is a property of the connection's tools, so it lives under the connection key.
|
||||
from tools.mcp_tool_scope import _server_key
|
||||
# Trust is the calling profile's own policy (an adopter of a shared connection keeps its own tier).
|
||||
trust = _core._server_trust_levels.get(_server_key(server_name), _core._TRUST_FULL)
|
||||
if (trust != _core._TRUST_UNTRUSTED
|
||||
or _core._tool_read_only_hints.get(_resolve_server_key(server_name), {}).get(tool_name) is True):
|
||||
if trust != _core._TRUST_UNTRUSTED or _tool_is_read_only(server_name, tool_name):
|
||||
return None
|
||||
try: # lazy: tools.approval routes the prompt to whichever surface owns the session
|
||||
from tools.approval_prompt import request_elicitation_consent
|
||||
@@ -181,7 +193,8 @@ def _handle_auth_error_and_retry(server_name: str, exc: BaseException, retry_cal
|
||||
return _strike(server_name, _NEEDS_REAUTH_MSG.format(s=server_name), needs_reauth=True, server=server_name)
|
||||
|
||||
|
||||
def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retry_call, op_description: str):
|
||||
def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retry_call, op_description: str,
|
||||
*, call_may_have_side_effects: bool = False):
|
||||
"""Transport reconnect + one retry on session expiry; None to fall through. Skips
|
||||
``handle_401``: the token is valid, only the server-side session is stale.
|
||||
|
||||
@@ -190,10 +203,29 @@ def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retr
|
||||
``_reconnect_event`` causes the server task's lifecycle loop to tear down the current
|
||||
``streamablehttp_client`` + ``ClientSession`` and rebuild them, reusing the existing OAuth provider
|
||||
instance. See #13383.
|
||||
|
||||
At-most-once for writes: a session-expired shape can be synthesized by a proxy AFTER the upstream
|
||||
executed the request, and the transport errors this classifier also matches (``ClosedResourceError``,
|
||||
broken pipe) routinely fire mid-response. With ``call_may_have_side_effects`` the transport is still
|
||||
healed but the call is never re-run; the model gets an ``outcome_uncertain`` error instead (same
|
||||
contract as the mid-call stdio death path). Callers pass True unless the tool is positively read-only.
|
||||
"""
|
||||
srv = _lookup_reconnectable_server(server_name, require_loop=True) if _is_session_expired_error(exc) else None
|
||||
if srv is None:
|
||||
return None
|
||||
if call_may_have_side_effects:
|
||||
reconnected = _loop._signal_reconnect_and_wait(
|
||||
server_name, srv, op_description=f"{op_description} (write, no auto-retry)", timeout=15)
|
||||
if reconnected: # session state failed, not server health: no breaker strike
|
||||
_core._reset_server_error(server_name)
|
||||
else:
|
||||
_core._bump_server_error(server_name)
|
||||
logger.warning("MCP server '%s': %s failed with a session-expired/transport error after the request may "
|
||||
"have been dispatched; NOT auto-retrying a write-capable tool (reconnect %s).",
|
||||
server_name, op_description, "succeeded" if reconnected else "failed")
|
||||
return tool_error(_SESSION_OUTCOME_UNCERTAIN_MSG.format(
|
||||
s=server_name, state="been re-established" if reconnected else "not recovered yet"),
|
||||
outcome_uncertain=True, server=server_name)
|
||||
logger.info("MCP server '%s': %s failed with session-expired error (%s); signalling transport reconnect "
|
||||
"and retrying once.", server_name, op_description, exc)
|
||||
if not _loop._signal_reconnect_and_wait(server_name, srv, op_description=op_description, timeout=15):
|
||||
@@ -502,9 +534,13 @@ def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float):
|
||||
def _on_failure(exc):
|
||||
_core._bump_server_error(server_name)
|
||||
logger.error("MCP tool %s/%s call failed: %s", server_name, tool_name, exc)
|
||||
# Only a tool annotated readOnlyHint=True is replayed after session expiry; a 401 is always
|
||||
# pre-dispatch so the auth recoverer keeps its retry for every tool.
|
||||
session_expired = partial(_handle_session_expired_and_retry,
|
||||
call_may_have_side_effects=not _tool_is_read_only(server_name, tool_name))
|
||||
return _dispatch(
|
||||
server_name, server, op, _call, tool_timeout,
|
||||
(_handle_stdio_child_exited_and_retry, _handle_auth_error_and_retry, _handle_session_expired_and_retry),
|
||||
(_handle_stdio_child_exited_and_retry, _handle_auth_error_and_retry, session_expired),
|
||||
_on_failure, record_outcome=True)
|
||||
return _handler
|
||||
|
||||
|
||||
Reference in New Issue
Block a user