fix(agent): bound redirect/rebuilt restart refunds so a runaway turn can't hold the session lease

The redirect and rebuilt-for-fallback restart paths in apply_retry_restarts
refund the iteration budget and re-issue the iteration with no per-turn
bound. A redirect/interrupt that keeps re-arming the flag refunds forever,
so the turn loop never exits and the durable session turn lease is held
indefinitely (concurrent processes block up to LEASE_WAIT_SECONDS).

Add a per-turn restart_count accumulator (threaded through _run_phase like
the other loop locals) and break out once it exceeds max_retries, matching
the bound the compression path already has.
This commit is contained in:
yoyodine-industries
2026-09-08 18:38:26 -04:00
committed by Teknium
parent 91adf584a4
commit e9312da68b
3 changed files with 171 additions and 3 deletions
+5
View File
@@ -1304,6 +1304,11 @@ class _LoopState:
failed: bool = False
codex_ack_continuations: int = 0
length_continue_retries: int = 0
# Per-turn backstop for the refunding restarts (redirect / rebuilt-for-fallback).
# Unlike ``retry_count`` (rebound to 0 each iteration) this accumulates for the whole
# turn so a runaway interrupt/redirect that keeps re-arming a restart flag cannot
# refund the iteration budget forever and hold the turn lease indefinitely.
restart_count: int = 0
_outer_error_count: int = 0 # outer-loop exceptions this turn (#92450), see _MAX_OUTER_LOOP_ERRORS
truncated_tool_call_retries: int = 0
truncated_response_parts: List[str] = field(default_factory=list)
+36 -3
View File
@@ -375,6 +375,7 @@ class RetryRestartVerdict:
current_turn_user_idx: Any
final_response: Any
retry_count: Any
restart_count: Any
api_call_count: Any
_preflight_compression_blocked: Any
_turn_exit_reason: Any
@@ -383,13 +384,20 @@ class RetryRestartVerdict:
def apply_retry_restarts(
agent: Any, *, _retry: Any, response: Any, interrupted: Any, messages: Any,
conversation_history: Any, user_message: Any, api_kwargs: Any, current_turn_user_idx: Any,
final_response: Any, retry_count: Any, api_call_count: Any, length_continue_retries: Any,
final_response: Any, retry_count: Any, max_retries: Any, api_call_count: Any,
restart_count: Any, length_continue_retries: Any,
_preflight_compression_blocked: Any, _turn_exit_reason: Any,
) -> RetryRestartVerdict:
"""Consume the ``TurnRetryState`` restart flags after the retry loop, in the original
priority order. Refunds the iteration budget/count for restarts that produced no valid
assistant item; ``restart_with_rebuilt_messages`` is the single consumer that clears
``_preflight_compression_blocked`` so the fallback gets a fresh preflight (#84733)."""
``_preflight_compression_blocked`` so the fallback gets a fresh preflight (#84733).
The two refunding restart paths (redirect and rebuilt-for-fallback) are bounded by
``max_retries`` via ``restart_count`` (a per-turn accumulator) so a runaway
interrupt/redirect that keeps re-arming a restart flag cannot refund the budget
forever and hold the turn lease indefinitely."""
from agent.conversation_loop import (
_HANDOFF_SKIP_FINAL_RESPONSE, _should_skip_model_call_for_reference_handoff
)
@@ -397,12 +405,25 @@ def apply_retry_restarts(
def _verdict(action: str) -> RetryRestartVerdict:
return RetryRestartVerdict(
action=action, current_turn_user_idx=current_turn_user_idx,
final_response=final_response, retry_count=retry_count, api_call_count=api_call_count,
final_response=final_response, retry_count=retry_count, restart_count=restart_count,
api_call_count=api_call_count,
_preflight_compression_blocked=_preflight_compression_blocked,
_turn_exit_reason=_turn_exit_reason,
)
if _retry.restart_with_redirected_messages:
restart_count += 1
if restart_count > max_retries:
# A redirect/interrupt keeps re-arming this flag: stop refunding the iteration
# budget and re-issuing the same logical iteration, or a runaway turn holds the
# turn lease indefinitely (redirect restarts previously had no bound).
_turn_exit_reason = "redirect_restart_limit_exceeded"
logger.warning(
"Redirected-message restart limit (%s) exceeded; ending turn instead of "
"refunding the iteration budget indefinitely.",
max_retries,
)
return _verdict("break")
# Cancelled request produced no valid assistant item: reuse the same logical
# iteration after the outer loop appends partial context + correction.
api_call_count -= 1
@@ -444,6 +465,18 @@ def apply_retry_restarts(
return _verdict("continue")
if _retry.restart_with_rebuilt_messages:
restart_count += 1
if restart_count > max_retries:
# A stall/failure keeps re-escalating to the fallback chain: stop refunding the
# iteration budget and re-issuing, or a runaway turn holds the turn lease
# indefinitely (rebuilt restarts previously had no bound).
_turn_exit_reason = "rebuilt_restart_limit_exceeded"
logger.warning(
"Rebuilt-message restart limit (%s) exceeded; ending turn instead of "
"refunding the iteration budget indefinitely.",
max_retries,
)
return _verdict("break")
# A stall/failure escalated to the fallback chain: re-issue against the
# active fallback provider, refunding budget/count for the stalled attempt.
api_call_count -= 1
+130
View File
@@ -0,0 +1,130 @@
"""Bound the refunding restart paths so a runaway turn cannot hold the turn lease.
``apply_retry_restarts`` has three restart paths that refund the iteration
budget and re-issue the iteration. The redirect and rebuilt-for-fallback paths
were unbounded: a runaway interrupt/redirect that keeps re-arming the restart
flag refunded the budget forever, so the turn loop never exited, the durable
turn lease (``turn_facade_lease.py``) never released in the ``finally`` block,
and concurrent processes blocked on the lease for up to ``LEASE_WAIT_SECONDS``.
The compression path was already capped (via the shared per-turn
``compression_attempts`` backstop); this change gives redirect and rebuilt the
same treatment with a per-turn ``restart_count`` accumulator capped at
``max_retries`` (the retry-loop bound).
These tests assert the *behavior contract* — a repeated restart eventually
``break`` s instead of refunding forever — by driving ``apply_retry_restarts``
directly. No source inspection.
"""
from __future__ import annotations
from types import SimpleNamespace
from agent.turn_iteration_prep import apply_retry_restarts
from agent.turn_retry_state import TurnRetryState
class _RefundBudget:
"""Minimal iteration budget that counts refunds (the runaway signal)."""
def __init__(self) -> None:
self.refund_count = 0
def refund(self) -> None:
self.refund_count += 1
def _apply(agent, _retry, restart_count: int):
"""Invoke ``apply_retry_restarts`` with a minimal loop-locals payload."""
return apply_retry_restarts(
agent,
_retry=_retry,
response=None,
interrupted=False,
messages=[],
conversation_history=[],
user_message="hi",
api_kwargs={},
current_turn_user_idx=0,
final_response=None,
retry_count=0,
max_retries=agent._api_max_retries,
api_call_count=1,
restart_count=restart_count,
length_continue_retries=0,
_preflight_compression_blocked=False,
_turn_exit_reason="unknown",
)
def _make_agent(max_retries: int) -> SimpleNamespace:
return SimpleNamespace(iteration_budget=_RefundBudget(), _api_max_retries=max_retries)
def _drive_repeated_restart(agent, arm):
"""Re-arm one restart flag every iteration, threading ``restart_count`` back in."""
restart_count = 0
actions = []
reasons = []
for _ in range(agent._api_max_retries + 5):
_retry = TurnRetryState()
arm(_retry)
verdict = _apply(agent, _retry, restart_count=restart_count)
restart_count = verdict.restart_count
actions.append(verdict.action)
reasons.append(verdict._turn_exit_reason)
if verdict.action == "break":
break
return actions, reasons
class TestRedirectRestartBound:
def test_single_redirect_restart_still_continues(self):
"""A lone user correction must still re-issue the iteration (one refund)."""
agent = _make_agent(max_retries=3)
_retry = TurnRetryState()
_retry.restart_with_redirected_messages = True
verdict = _apply(agent, _retry, restart_count=0)
assert verdict.action == "continue"
assert agent.iteration_budget.refund_count == 1
def test_repeated_redirect_restarts_break_after_bounded_count(self):
"""A runaway redirect must break instead of refunding forever."""
agent = _make_agent(max_retries=3)
actions, reasons = _drive_repeated_restart(
agent, lambda r: setattr(r, "restart_with_redirected_messages", True)
)
assert "break" in actions, "redirect restarts must eventually break, not refund forever"
# Exactly ``max_retries`` refunds are allowed before the cap trips.
assert actions.count("continue") == agent._api_max_retries
assert agent.iteration_budget.refund_count == agent._api_max_retries
assert reasons[-1] == "redirect_restart_limit_exceeded"
class TestRebuiltRestartBound:
def test_single_rebuilt_restart_still_continues(self):
"""A lone fallback activation must still re-issue the iteration (one refund)."""
agent = _make_agent(max_retries=3)
_retry = TurnRetryState()
_retry.restart_with_rebuilt_messages = True
verdict = _apply(agent, _retry, restart_count=0)
assert verdict.action == "continue"
assert agent.iteration_budget.refund_count == 1
# The single consumer still clears the preflight block for the fallback.
assert verdict._preflight_compression_blocked is False
def test_repeated_rebuilt_restarts_break_after_bounded_count(self):
"""A stall that keeps re-escalating to fallback must break, not refund forever."""
agent = _make_agent(max_retries=3)
actions, reasons = _drive_repeated_restart(
agent, lambda r: setattr(r, "restart_with_rebuilt_messages", True)
)
assert "break" in actions, "rebuilt restarts must eventually break, not refund forever"
assert actions.count("continue") == agent._api_max_retries
assert agent.iteration_budget.refund_count == agent._api_max_retries
assert reasons[-1] == "rebuilt_restart_limit_exceeded"