fix(agent): bound redirect/rebuilt restart refunds so a runaway turn can't hold the session lease
The redirect and rebuilt-for-fallback restart paths in apply_retry_restarts refund the iteration budget and re-issue the iteration with no per-turn bound. A redirect/interrupt that keeps re-arming the flag refunds forever, so the turn loop never exits and the durable session turn lease is held indefinitely (concurrent processes block up to LEASE_WAIT_SECONDS). Add a per-turn restart_count accumulator (threaded through _run_phase like the other loop locals) and break out once it exceeds max_retries, matching the bound the compression path already has.
This commit is contained in:
committed by
Teknium
parent
91adf584a4
commit
e9312da68b
@@ -1304,6 +1304,11 @@ class _LoopState:
|
||||
failed: bool = False
|
||||
codex_ack_continuations: int = 0
|
||||
length_continue_retries: int = 0
|
||||
# Per-turn backstop for the refunding restarts (redirect / rebuilt-for-fallback).
|
||||
# Unlike ``retry_count`` (rebound to 0 each iteration) this accumulates for the whole
|
||||
# turn so a runaway interrupt/redirect that keeps re-arming a restart flag cannot
|
||||
# refund the iteration budget forever and hold the turn lease indefinitely.
|
||||
restart_count: int = 0
|
||||
_outer_error_count: int = 0 # outer-loop exceptions this turn (#92450), see _MAX_OUTER_LOOP_ERRORS
|
||||
truncated_tool_call_retries: int = 0
|
||||
truncated_response_parts: List[str] = field(default_factory=list)
|
||||
|
||||
@@ -375,6 +375,7 @@ class RetryRestartVerdict:
|
||||
current_turn_user_idx: Any
|
||||
final_response: Any
|
||||
retry_count: Any
|
||||
restart_count: Any
|
||||
api_call_count: Any
|
||||
_preflight_compression_blocked: Any
|
||||
_turn_exit_reason: Any
|
||||
@@ -383,13 +384,20 @@ class RetryRestartVerdict:
|
||||
def apply_retry_restarts(
|
||||
agent: Any, *, _retry: Any, response: Any, interrupted: Any, messages: Any,
|
||||
conversation_history: Any, user_message: Any, api_kwargs: Any, current_turn_user_idx: Any,
|
||||
final_response: Any, retry_count: Any, api_call_count: Any, length_continue_retries: Any,
|
||||
final_response: Any, retry_count: Any, max_retries: Any, api_call_count: Any,
|
||||
restart_count: Any, length_continue_retries: Any,
|
||||
_preflight_compression_blocked: Any, _turn_exit_reason: Any,
|
||||
) -> RetryRestartVerdict:
|
||||
"""Consume the ``TurnRetryState`` restart flags after the retry loop, in the original
|
||||
priority order. Refunds the iteration budget/count for restarts that produced no valid
|
||||
assistant item; ``restart_with_rebuilt_messages`` is the single consumer that clears
|
||||
``_preflight_compression_blocked`` so the fallback gets a fresh preflight (#84733)."""
|
||||
``_preflight_compression_blocked`` so the fallback gets a fresh preflight (#84733).
|
||||
|
||||
The two refunding restart paths (redirect and rebuilt-for-fallback) are bounded by
|
||||
``max_retries`` via ``restart_count`` (a per-turn accumulator) so a runaway
|
||||
interrupt/redirect that keeps re-arming a restart flag cannot refund the budget
|
||||
forever and hold the turn lease indefinitely."""
|
||||
|
||||
from agent.conversation_loop import (
|
||||
_HANDOFF_SKIP_FINAL_RESPONSE, _should_skip_model_call_for_reference_handoff
|
||||
)
|
||||
@@ -397,12 +405,25 @@ def apply_retry_restarts(
|
||||
def _verdict(action: str) -> RetryRestartVerdict:
|
||||
return RetryRestartVerdict(
|
||||
action=action, current_turn_user_idx=current_turn_user_idx,
|
||||
final_response=final_response, retry_count=retry_count, api_call_count=api_call_count,
|
||||
final_response=final_response, retry_count=retry_count, restart_count=restart_count,
|
||||
api_call_count=api_call_count,
|
||||
_preflight_compression_blocked=_preflight_compression_blocked,
|
||||
_turn_exit_reason=_turn_exit_reason,
|
||||
)
|
||||
|
||||
if _retry.restart_with_redirected_messages:
|
||||
restart_count += 1
|
||||
if restart_count > max_retries:
|
||||
# A redirect/interrupt keeps re-arming this flag: stop refunding the iteration
|
||||
# budget and re-issuing the same logical iteration, or a runaway turn holds the
|
||||
# turn lease indefinitely (redirect restarts previously had no bound).
|
||||
_turn_exit_reason = "redirect_restart_limit_exceeded"
|
||||
logger.warning(
|
||||
"Redirected-message restart limit (%s) exceeded; ending turn instead of "
|
||||
"refunding the iteration budget indefinitely.",
|
||||
max_retries,
|
||||
)
|
||||
return _verdict("break")
|
||||
# Cancelled request produced no valid assistant item: reuse the same logical
|
||||
# iteration after the outer loop appends partial context + correction.
|
||||
api_call_count -= 1
|
||||
@@ -444,6 +465,18 @@ def apply_retry_restarts(
|
||||
return _verdict("continue")
|
||||
|
||||
if _retry.restart_with_rebuilt_messages:
|
||||
restart_count += 1
|
||||
if restart_count > max_retries:
|
||||
# A stall/failure keeps re-escalating to the fallback chain: stop refunding the
|
||||
# iteration budget and re-issuing, or a runaway turn holds the turn lease
|
||||
# indefinitely (rebuilt restarts previously had no bound).
|
||||
_turn_exit_reason = "rebuilt_restart_limit_exceeded"
|
||||
logger.warning(
|
||||
"Rebuilt-message restart limit (%s) exceeded; ending turn instead of "
|
||||
"refunding the iteration budget indefinitely.",
|
||||
max_retries,
|
||||
)
|
||||
return _verdict("break")
|
||||
# A stall/failure escalated to the fallback chain: re-issue against the
|
||||
# active fallback provider, refunding budget/count for the stalled attempt.
|
||||
api_call_count -= 1
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Bound the refunding restart paths so a runaway turn cannot hold the turn lease.
|
||||
|
||||
``apply_retry_restarts`` has three restart paths that refund the iteration
|
||||
budget and re-issue the iteration. The redirect and rebuilt-for-fallback paths
|
||||
were unbounded: a runaway interrupt/redirect that keeps re-arming the restart
|
||||
flag refunded the budget forever, so the turn loop never exited, the durable
|
||||
turn lease (``turn_facade_lease.py``) never released in the ``finally`` block,
|
||||
and concurrent processes blocked on the lease for up to ``LEASE_WAIT_SECONDS``.
|
||||
|
||||
The compression path was already capped (via the shared per-turn
|
||||
``compression_attempts`` backstop); this change gives redirect and rebuilt the
|
||||
same treatment with a per-turn ``restart_count`` accumulator capped at
|
||||
``max_retries`` (the retry-loop bound).
|
||||
|
||||
These tests assert the *behavior contract* — a repeated restart eventually
|
||||
``break`` s instead of refunding forever — by driving ``apply_retry_restarts``
|
||||
directly. No source inspection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from agent.turn_iteration_prep import apply_retry_restarts
|
||||
from agent.turn_retry_state import TurnRetryState
|
||||
|
||||
|
||||
class _RefundBudget:
|
||||
"""Minimal iteration budget that counts refunds (the runaway signal)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.refund_count = 0
|
||||
|
||||
def refund(self) -> None:
|
||||
self.refund_count += 1
|
||||
|
||||
|
||||
def _apply(agent, _retry, restart_count: int):
|
||||
"""Invoke ``apply_retry_restarts`` with a minimal loop-locals payload."""
|
||||
return apply_retry_restarts(
|
||||
agent,
|
||||
_retry=_retry,
|
||||
response=None,
|
||||
interrupted=False,
|
||||
messages=[],
|
||||
conversation_history=[],
|
||||
user_message="hi",
|
||||
api_kwargs={},
|
||||
current_turn_user_idx=0,
|
||||
final_response=None,
|
||||
retry_count=0,
|
||||
max_retries=agent._api_max_retries,
|
||||
api_call_count=1,
|
||||
restart_count=restart_count,
|
||||
length_continue_retries=0,
|
||||
_preflight_compression_blocked=False,
|
||||
_turn_exit_reason="unknown",
|
||||
)
|
||||
|
||||
|
||||
def _make_agent(max_retries: int) -> SimpleNamespace:
|
||||
return SimpleNamespace(iteration_budget=_RefundBudget(), _api_max_retries=max_retries)
|
||||
|
||||
|
||||
def _drive_repeated_restart(agent, arm):
|
||||
"""Re-arm one restart flag every iteration, threading ``restart_count`` back in."""
|
||||
restart_count = 0
|
||||
actions = []
|
||||
reasons = []
|
||||
for _ in range(agent._api_max_retries + 5):
|
||||
_retry = TurnRetryState()
|
||||
arm(_retry)
|
||||
verdict = _apply(agent, _retry, restart_count=restart_count)
|
||||
restart_count = verdict.restart_count
|
||||
actions.append(verdict.action)
|
||||
reasons.append(verdict._turn_exit_reason)
|
||||
if verdict.action == "break":
|
||||
break
|
||||
return actions, reasons
|
||||
|
||||
|
||||
class TestRedirectRestartBound:
|
||||
def test_single_redirect_restart_still_continues(self):
|
||||
"""A lone user correction must still re-issue the iteration (one refund)."""
|
||||
agent = _make_agent(max_retries=3)
|
||||
_retry = TurnRetryState()
|
||||
_retry.restart_with_redirected_messages = True
|
||||
verdict = _apply(agent, _retry, restart_count=0)
|
||||
|
||||
assert verdict.action == "continue"
|
||||
assert agent.iteration_budget.refund_count == 1
|
||||
|
||||
def test_repeated_redirect_restarts_break_after_bounded_count(self):
|
||||
"""A runaway redirect must break instead of refunding forever."""
|
||||
agent = _make_agent(max_retries=3)
|
||||
actions, reasons = _drive_repeated_restart(
|
||||
agent, lambda r: setattr(r, "restart_with_redirected_messages", True)
|
||||
)
|
||||
|
||||
assert "break" in actions, "redirect restarts must eventually break, not refund forever"
|
||||
# Exactly ``max_retries`` refunds are allowed before the cap trips.
|
||||
assert actions.count("continue") == agent._api_max_retries
|
||||
assert agent.iteration_budget.refund_count == agent._api_max_retries
|
||||
assert reasons[-1] == "redirect_restart_limit_exceeded"
|
||||
|
||||
|
||||
class TestRebuiltRestartBound:
|
||||
def test_single_rebuilt_restart_still_continues(self):
|
||||
"""A lone fallback activation must still re-issue the iteration (one refund)."""
|
||||
agent = _make_agent(max_retries=3)
|
||||
_retry = TurnRetryState()
|
||||
_retry.restart_with_rebuilt_messages = True
|
||||
verdict = _apply(agent, _retry, restart_count=0)
|
||||
|
||||
assert verdict.action == "continue"
|
||||
assert agent.iteration_budget.refund_count == 1
|
||||
# The single consumer still clears the preflight block for the fallback.
|
||||
assert verdict._preflight_compression_blocked is False
|
||||
|
||||
def test_repeated_rebuilt_restarts_break_after_bounded_count(self):
|
||||
"""A stall that keeps re-escalating to fallback must break, not refund forever."""
|
||||
agent = _make_agent(max_retries=3)
|
||||
actions, reasons = _drive_repeated_restart(
|
||||
agent, lambda r: setattr(r, "restart_with_rebuilt_messages", True)
|
||||
)
|
||||
|
||||
assert "break" in actions, "rebuilt restarts must eventually break, not refund forever"
|
||||
assert actions.count("continue") == agent._api_max_retries
|
||||
assert agent.iteration_budget.refund_count == agent._api_max_retries
|
||||
assert reasons[-1] == "rebuilt_restart_limit_exceeded"
|
||||
Reference in New Issue
Block a user