Port from cloudflare/cloudflare-os#168: never auto-retry write-capable MCP tools after mid-flight session expiry

A 'session expired' / transport-closed failure can arrive AFTER the server
already accepted and executed the request (proxy-synthesized 404s, pod
rotation, ClosedResourceError firing mid-response). Auto-retrying a
write-capable tool in that window risks a duplicate side effect that MCP
offers no way to undo.

The session-expired recovery path now consults the discovery-time
readOnlyHint capture (same data the trust gate uses): only tools whose
annotation is exactly True keep the reconnect+retry-once behavior. Write-
capable calls still get the transport healed (reconnect, breaker reset on
success) but return a structured outcome_unknown error telling the model
to verify with a read before re-invoking.

The OAuth 401 path keeps its retry for all tools: a 401 means the server
demanded authorization before dispatch, so the call never executed —
matching the upstream classifier's McpAuthRequiredError => safe rule.

Fails safe: missing/unknown annotations classify as write-capable.
This commit is contained in:
Teknium
2026-08-17 18:19:36 -07:00
parent 087c4f4f4e
commit 26f5643dc9
2 changed files with 123 additions and 7 deletions
@@ -186,6 +186,10 @@ def test_call_tool_handler_rebuilds_configured_server_transport(
mcp_tool._servers["resumed"] = server
mcp_tool._server_error_counts.pop("resumed", None)
mcp_tool._server_breaker_opened_at.pop("resumed", None)
# Auto-retry after session expiry is only safe (and only performed) for
# tools positively annotated read-only; a write may already have
# executed server-side (#88821).
mcp_tool._tool_read_only_hints.setdefault("resumed", {})["health"] = True
loop = mcp_tool._mcp_loop
assert loop is not None
run_future = asyncio.run_coroutine_threadsafe(
@@ -218,6 +222,7 @@ def test_call_tool_handler_rebuilds_configured_server_transport(
mcp_tool._servers.pop("resumed", None)
mcp_tool._server_error_counts.pop("resumed", None)
mcp_tool._server_breaker_opened_at.pop("resumed", None)
mcp_tool._tool_read_only_hints.pop("resumed", None)
def test_session_expired_retry_waits_for_new_session(monkeypatch, tmp_path):
@@ -278,6 +283,10 @@ def test_session_expired_retry_waits_for_new_session(monkeypatch, tmp_path):
server._reconnect_event = _ReconnectAdapter()
mcp_tool._servers["hindsight"] = server
mcp_tool._server_error_counts["hindsight"] = 7
# Read-only annotation: the session-expired auto-retry now only fires
# for tools positively marked readOnlyHint=True (write-capable calls
# get the outcome-unknown path instead).
mcp_tool._tool_read_only_hints.setdefault("hindsight", {})["get_bank"] = True
# Stamp the breaker "open" far enough in the past that the cooldown has
# provably elapsed, so this call is a half-open probe. The breaker compares
# against time.monotonic() (tools/mcp_tool.py), whose origin is arbitrary and
@@ -297,6 +306,7 @@ def test_session_expired_retry_waits_for_new_session(monkeypatch, tmp_path):
mcp_tool._servers.pop("hindsight", None)
mcp_tool._server_error_counts.pop("hindsight", None)
mcp_tool._server_breaker_opened_at.pop("hindsight", None)
mcp_tool._tool_read_only_hints.pop("hindsight", None)
def test_session_expired_handler_returns_none_without_loop(monkeypatch):
@@ -417,3 +427,73 @@ def test_non_tool_handlers_also_reconnect_on_session_expired(
finally:
mcp_tool._servers.pop(f"srv-{op_label}", None)
mcp_tool._server_error_counts.pop(f"srv-{op_label}", None)
# ---------------------------------------------------------------------------
# At-most-once guard for write-capable tools: a session-expired/transport
# failure on a call that may already have executed server-side must NOT be
# auto-retried. The transport is healed and an outcome_uncertain error tells
# the model to verify first. Only readOnlyHint=True tools keep the retry.
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("read_only", [False, True], ids=["write-capable", "read-only"])
def test_session_expired_retry_only_for_read_only_tools(monkeypatch, tmp_path, read_only):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
from tools import mcp_tool
from tools.mcp_tool_handlers import _make_tool_handler
_mcp_loop._ensure_mcp_loop()
server, reconnect_flag = _install_stub_server("srv")
call_count = {"n": 0}
async def _sequence(*a, **kw):
call_count["n"] += 1
if call_count["n"] == 1:
raise RuntimeError("Session terminated")
result = MagicMock()
result.is_error = False
result.content = [MagicMock(type="text", text="fresh data")]
result.structured_content = None
return result
server.session.call_tool = _sequence
mcp_tool._servers["srv"] = server
mcp_tool._server_error_counts.pop("srv", None)
if read_only:
mcp_tool._tool_read_only_hints.setdefault("srv", {})["tool"] = True
# else: no readOnlyHint entry -> fails safe to write-capable.
try:
parsed = json.loads(_make_tool_handler("srv", "tool", 10.0)({}))
assert reconnect_flag.is_set() # transport healed either way
if read_only:
assert parsed.get("result") == "fresh data", parsed
assert call_count["n"] == 2 # original + one retry
else:
assert parsed.get("outcome_uncertain") is True, parsed
assert "may or may not have taken effect" in parsed["error"]
assert call_count["n"] == 1 # exactly one dispatch
# Successful reconnect clears breaker state (session-state failure, not server health).
assert mcp_tool._server_error_counts.get("srv", 0) == 0
finally:
mcp_tool._servers.pop("srv", None)
mcp_tool._server_error_counts.pop("srv", None)
mcp_tool._server_breaker_opened_at.pop("srv", None)
mcp_tool._tool_read_only_hints.pop("srv", None)
def test_tool_is_read_only_fails_safe():
"""Unknown server/tool metadata classifies as write-capable (False)."""
from tools import mcp_tool
from tools.mcp_tool_handlers import _tool_is_read_only
assert _tool_is_read_only("no-such-server", "tool") is False
mcp_tool._tool_read_only_hints["known"] = {"reader": True, "writer": False}
try:
assert _tool_is_read_only("known", "reader") is True
assert _tool_is_read_only("known", "writer") is False
assert _tool_is_read_only("known", "unlisted") is False
finally:
mcp_tool._tool_read_only_hints.pop("known", None)
+43 -7
View File
@@ -8,6 +8,7 @@ import inspect
import json
import time
from contextlib import asynccontextmanager
from functools import partial
from types import SimpleNamespace
from typing import Any, Callable, Dict, List, Optional, Tuple
from tools.registry import tool_error
@@ -36,17 +37,28 @@ _STDIO_DIED_AGAIN_MSG = (
_STDIO_OUTCOME_UNCERTAIN_MSG = (
"MCP server '{s}' lost its stdio subprocess after the tool call began. The operation may have completed, so "
"Hermes did not replay it. Do NOT retry automatically; inspect the external state first.")
_SESSION_OUTCOME_UNCERTAIN_MSG = (
"The MCP transport session to '{s}' expired while this write-capable call was in flight, so the outcome is "
"UNKNOWN — the operation may or may not have taken effect server-side. It was NOT automatically retried to "
"avoid a duplicate side effect. The connection has {state}. Verify whether the operation took effect (e.g. "
"with a read-only tool) before re-invoking it.")
def _tool_is_read_only(server_name: str, tool_name: str) -> bool:
"""True only when discovery captured ``readOnlyHint=True`` for the tool. Missing or malformed
metadata fails safe to False (treated as write-capable). readOnlyHint is a property of the
connection's tools, so it lives under the connection key."""
from tools.mcp_tool_scope import _resolve_server_key
return _core._tool_read_only_hints.get(_resolve_server_key(server_name), {}).get(tool_name) is True
def _trust_gate_check(server_name: str, tool_name: str) -> Optional[str]:
"""Approval gate for write-capable tools on ``trust: untrusted`` servers. None to proceed,
else a ``tool_error``. Fail-closed: approval-system errors block."""
from tools.mcp_tool_scope import _resolve_server_key, _server_key
# Trust is the calling profile's own policy (an adopter of a shared connection keeps its own tier);
# readOnlyHint is a property of the connection's tools, so it lives under the connection key.
from tools.mcp_tool_scope import _server_key
# Trust is the calling profile's own policy (an adopter of a shared connection keeps its own tier).
trust = _core._server_trust_levels.get(_server_key(server_name), _core._TRUST_FULL)
if (trust != _core._TRUST_UNTRUSTED
or _core._tool_read_only_hints.get(_resolve_server_key(server_name), {}).get(tool_name) is True):
if trust != _core._TRUST_UNTRUSTED or _tool_is_read_only(server_name, tool_name):
return None
try: # lazy: tools.approval routes the prompt to whichever surface owns the session
from tools.approval_prompt import request_elicitation_consent
@@ -181,7 +193,8 @@ def _handle_auth_error_and_retry(server_name: str, exc: BaseException, retry_cal
return _strike(server_name, _NEEDS_REAUTH_MSG.format(s=server_name), needs_reauth=True, server=server_name)
def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retry_call, op_description: str):
def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retry_call, op_description: str,
*, call_may_have_side_effects: bool = False):
"""Transport reconnect + one retry on session expiry; None to fall through. Skips
``handle_401``: the token is valid, only the server-side session is stale.
@@ -190,10 +203,29 @@ def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retr
``_reconnect_event`` causes the server task's lifecycle loop to tear down the current
``streamablehttp_client`` + ``ClientSession`` and rebuild them, reusing the existing OAuth provider
instance. See #13383.
At-most-once for writes: a session-expired shape can be synthesized by a proxy AFTER the upstream
executed the request, and the transport errors this classifier also matches (``ClosedResourceError``,
broken pipe) routinely fire mid-response. With ``call_may_have_side_effects`` the transport is still
healed but the call is never re-run; the model gets an ``outcome_uncertain`` error instead (same
contract as the mid-call stdio death path). Callers pass True unless the tool is positively read-only.
"""
srv = _lookup_reconnectable_server(server_name, require_loop=True) if _is_session_expired_error(exc) else None
if srv is None:
return None
if call_may_have_side_effects:
reconnected = _loop._signal_reconnect_and_wait(
server_name, srv, op_description=f"{op_description} (write, no auto-retry)", timeout=15)
if reconnected: # session state failed, not server health: no breaker strike
_core._reset_server_error(server_name)
else:
_core._bump_server_error(server_name)
logger.warning("MCP server '%s': %s failed with a session-expired/transport error after the request may "
"have been dispatched; NOT auto-retrying a write-capable tool (reconnect %s).",
server_name, op_description, "succeeded" if reconnected else "failed")
return tool_error(_SESSION_OUTCOME_UNCERTAIN_MSG.format(
s=server_name, state="been re-established" if reconnected else "not recovered yet"),
outcome_uncertain=True, server=server_name)
logger.info("MCP server '%s': %s failed with session-expired error (%s); signalling transport reconnect "
"and retrying once.", server_name, op_description, exc)
if not _loop._signal_reconnect_and_wait(server_name, srv, op_description=op_description, timeout=15):
@@ -502,9 +534,13 @@ def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float):
def _on_failure(exc):
_core._bump_server_error(server_name)
logger.error("MCP tool %s/%s call failed: %s", server_name, tool_name, exc)
# Only a tool annotated readOnlyHint=True is replayed after session expiry; a 401 is always
# pre-dispatch so the auth recoverer keeps its retry for every tool.
session_expired = partial(_handle_session_expired_and_retry,
call_may_have_side_effects=not _tool_is_read_only(server_name, tool_name))
return _dispatch(
server_name, server, op, _call, tool_timeout,
(_handle_stdio_child_exited_and_retry, _handle_auth_error_and_retry, _handle_session_expired_and_retry),
(_handle_stdio_child_exited_and_retry, _handle_auth_error_and_retry, session_expired),
_on_failure, record_outcome=True)
return _handler