diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 72a6be2aed..70e9a12762 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -1304,6 +1304,11 @@ class _LoopState: failed: bool = False codex_ack_continuations: int = 0 length_continue_retries: int = 0 + # Per-turn backstop for the refunding restarts (redirect / rebuilt-for-fallback). + # Unlike ``retry_count`` (rebound to 0 each iteration) this accumulates for the whole + # turn so a runaway interrupt/redirect that keeps re-arming a restart flag cannot + # refund the iteration budget forever and hold the turn lease indefinitely. + restart_count: int = 0 _outer_error_count: int = 0 # outer-loop exceptions this turn (#92450), see _MAX_OUTER_LOOP_ERRORS truncated_tool_call_retries: int = 0 truncated_response_parts: List[str] = field(default_factory=list) diff --git a/agent/turn_iteration_prep.py b/agent/turn_iteration_prep.py index 2c219d5dec..e30d756f85 100644 --- a/agent/turn_iteration_prep.py +++ b/agent/turn_iteration_prep.py @@ -375,6 +375,7 @@ class RetryRestartVerdict: current_turn_user_idx: Any final_response: Any retry_count: Any + restart_count: Any api_call_count: Any _preflight_compression_blocked: Any _turn_exit_reason: Any @@ -383,13 +384,20 @@ class RetryRestartVerdict: def apply_retry_restarts( agent: Any, *, _retry: Any, response: Any, interrupted: Any, messages: Any, conversation_history: Any, user_message: Any, api_kwargs: Any, current_turn_user_idx: Any, - final_response: Any, retry_count: Any, api_call_count: Any, length_continue_retries: Any, + final_response: Any, retry_count: Any, max_retries: Any, api_call_count: Any, + restart_count: Any, length_continue_retries: Any, _preflight_compression_blocked: Any, _turn_exit_reason: Any, ) -> RetryRestartVerdict: """Consume the ``TurnRetryState`` restart flags after the retry loop, in the original priority order. Refunds the iteration budget/count for restarts that produced no valid assistant item; ``restart_with_rebuilt_messages`` is the single consumer that clears - ``_preflight_compression_blocked`` so the fallback gets a fresh preflight (#84733).""" + ``_preflight_compression_blocked`` so the fallback gets a fresh preflight (#84733). + + The two refunding restart paths (redirect and rebuilt-for-fallback) are bounded by + ``max_retries`` via ``restart_count`` (a per-turn accumulator) so a runaway + interrupt/redirect that keeps re-arming a restart flag cannot refund the budget + forever and hold the turn lease indefinitely.""" + from agent.conversation_loop import ( _HANDOFF_SKIP_FINAL_RESPONSE, _should_skip_model_call_for_reference_handoff ) @@ -397,12 +405,25 @@ def apply_retry_restarts( def _verdict(action: str) -> RetryRestartVerdict: return RetryRestartVerdict( action=action, current_turn_user_idx=current_turn_user_idx, - final_response=final_response, retry_count=retry_count, api_call_count=api_call_count, + final_response=final_response, retry_count=retry_count, restart_count=restart_count, + api_call_count=api_call_count, _preflight_compression_blocked=_preflight_compression_blocked, _turn_exit_reason=_turn_exit_reason, ) if _retry.restart_with_redirected_messages: + restart_count += 1 + if restart_count > max_retries: + # A redirect/interrupt keeps re-arming this flag: stop refunding the iteration + # budget and re-issuing the same logical iteration, or a runaway turn holds the + # turn lease indefinitely (redirect restarts previously had no bound). + _turn_exit_reason = "redirect_restart_limit_exceeded" + logger.warning( + "Redirected-message restart limit (%s) exceeded; ending turn instead of " + "refunding the iteration budget indefinitely.", + max_retries, + ) + return _verdict("break") # Cancelled request produced no valid assistant item: reuse the same logical # iteration after the outer loop appends partial context + correction. api_call_count -= 1 @@ -444,6 +465,18 @@ def apply_retry_restarts( return _verdict("continue") if _retry.restart_with_rebuilt_messages: + restart_count += 1 + if restart_count > max_retries: + # A stall/failure keeps re-escalating to the fallback chain: stop refunding the + # iteration budget and re-issuing, or a runaway turn holds the turn lease + # indefinitely (rebuilt restarts previously had no bound). + _turn_exit_reason = "rebuilt_restart_limit_exceeded" + logger.warning( + "Rebuilt-message restart limit (%s) exceeded; ending turn instead of " + "refunding the iteration budget indefinitely.", + max_retries, + ) + return _verdict("break") # A stall/failure escalated to the fallback chain: re-issue against the # active fallback provider, refunding budget/count for the stalled attempt. api_call_count -= 1 diff --git a/tests/run_agent/test_turn_iteration_prep.py b/tests/run_agent/test_turn_iteration_prep.py new file mode 100644 index 0000000000..d0252c4c28 --- /dev/null +++ b/tests/run_agent/test_turn_iteration_prep.py @@ -0,0 +1,130 @@ +"""Bound the refunding restart paths so a runaway turn cannot hold the turn lease. + +``apply_retry_restarts`` has three restart paths that refund the iteration +budget and re-issue the iteration. The redirect and rebuilt-for-fallback paths +were unbounded: a runaway interrupt/redirect that keeps re-arming the restart +flag refunded the budget forever, so the turn loop never exited, the durable +turn lease (``turn_facade_lease.py``) never released in the ``finally`` block, +and concurrent processes blocked on the lease for up to ``LEASE_WAIT_SECONDS``. + +The compression path was already capped (via the shared per-turn +``compression_attempts`` backstop); this change gives redirect and rebuilt the +same treatment with a per-turn ``restart_count`` accumulator capped at +``max_retries`` (the retry-loop bound). + +These tests assert the *behavior contract* — a repeated restart eventually +``break`` s instead of refunding forever — by driving ``apply_retry_restarts`` +directly. No source inspection. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +from agent.turn_iteration_prep import apply_retry_restarts +from agent.turn_retry_state import TurnRetryState + + +class _RefundBudget: + """Minimal iteration budget that counts refunds (the runaway signal).""" + + def __init__(self) -> None: + self.refund_count = 0 + + def refund(self) -> None: + self.refund_count += 1 + + +def _apply(agent, _retry, restart_count: int): + """Invoke ``apply_retry_restarts`` with a minimal loop-locals payload.""" + return apply_retry_restarts( + agent, + _retry=_retry, + response=None, + interrupted=False, + messages=[], + conversation_history=[], + user_message="hi", + api_kwargs={}, + current_turn_user_idx=0, + final_response=None, + retry_count=0, + max_retries=agent._api_max_retries, + api_call_count=1, + restart_count=restart_count, + length_continue_retries=0, + _preflight_compression_blocked=False, + _turn_exit_reason="unknown", + ) + + +def _make_agent(max_retries: int) -> SimpleNamespace: + return SimpleNamespace(iteration_budget=_RefundBudget(), _api_max_retries=max_retries) + + +def _drive_repeated_restart(agent, arm): + """Re-arm one restart flag every iteration, threading ``restart_count`` back in.""" + restart_count = 0 + actions = [] + reasons = [] + for _ in range(agent._api_max_retries + 5): + _retry = TurnRetryState() + arm(_retry) + verdict = _apply(agent, _retry, restart_count=restart_count) + restart_count = verdict.restart_count + actions.append(verdict.action) + reasons.append(verdict._turn_exit_reason) + if verdict.action == "break": + break + return actions, reasons + + +class TestRedirectRestartBound: + def test_single_redirect_restart_still_continues(self): + """A lone user correction must still re-issue the iteration (one refund).""" + agent = _make_agent(max_retries=3) + _retry = TurnRetryState() + _retry.restart_with_redirected_messages = True + verdict = _apply(agent, _retry, restart_count=0) + + assert verdict.action == "continue" + assert agent.iteration_budget.refund_count == 1 + + def test_repeated_redirect_restarts_break_after_bounded_count(self): + """A runaway redirect must break instead of refunding forever.""" + agent = _make_agent(max_retries=3) + actions, reasons = _drive_repeated_restart( + agent, lambda r: setattr(r, "restart_with_redirected_messages", True) + ) + + assert "break" in actions, "redirect restarts must eventually break, not refund forever" + # Exactly ``max_retries`` refunds are allowed before the cap trips. + assert actions.count("continue") == agent._api_max_retries + assert agent.iteration_budget.refund_count == agent._api_max_retries + assert reasons[-1] == "redirect_restart_limit_exceeded" + + +class TestRebuiltRestartBound: + def test_single_rebuilt_restart_still_continues(self): + """A lone fallback activation must still re-issue the iteration (one refund).""" + agent = _make_agent(max_retries=3) + _retry = TurnRetryState() + _retry.restart_with_rebuilt_messages = True + verdict = _apply(agent, _retry, restart_count=0) + + assert verdict.action == "continue" + assert agent.iteration_budget.refund_count == 1 + # The single consumer still clears the preflight block for the fallback. + assert verdict._preflight_compression_blocked is False + + def test_repeated_rebuilt_restarts_break_after_bounded_count(self): + """A stall that keeps re-escalating to fallback must break, not refund forever.""" + agent = _make_agent(max_retries=3) + actions, reasons = _drive_repeated_restart( + agent, lambda r: setattr(r, "restart_with_rebuilt_messages", True) + ) + + assert "break" in actions, "rebuilt restarts must eventually break, not refund forever" + assert actions.count("continue") == agent._api_max_retries + assert agent.iteration_budget.refund_count == agent._api_max_retries + assert reasons[-1] == "rebuilt_restart_limit_exceeded"