diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index deab15b077..3b7d8b0361 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1712,11 +1712,15 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool current_provider = (getattr(agent, "provider", "") or "").strip().lower() primary_provider = ((agent._primary_runtime or {}).get("provider") or "").strip().lower() if (not fallback_already_active) or (primary_provider and current_provider == primary_provider): - # Exponential backoff: 30min → 1h → 2h → 4h cap + # Exponential backoff: keep upstream's 60s first-hit cooldown and + # escalate on CONSECUTIVE rate-limits: 60s → 2m → 4m → 8m → ... → + # 4h cap. The first 429 must NOT bench the primary for half an + # hour — fast primary restore is the common case; escalation only + # punishes providers that keep 429ing. # Counter is reset by restore_primary_runtime on successful restore. backoff_count = getattr(agent, "_rate_limit_backoff_count", 0) agent._rate_limit_backoff_count = backoff_count + 1 - backoff_seconds = min(1800 * (2 ** backoff_count), 14400) + backoff_seconds = min(60 * (2 ** backoff_count), 14400) agent._rate_limited_until = time.monotonic() + backoff_seconds logging.info( "Rate-limit backoff level %d: cooldown %d s (%.1f min, backoff#%d)", diff --git a/tests/run_agent/test_24996_fallback_exhaustion_cooldown.py b/tests/run_agent/test_24996_fallback_exhaustion_cooldown.py index 83991c2471..2e86c795c5 100644 --- a/tests/run_agent/test_24996_fallback_exhaustion_cooldown.py +++ b/tests/run_agent/test_24996_fallback_exhaustion_cooldown.py @@ -130,3 +130,103 @@ class TestExhaustionArmsCooldown: assert agent._try_activate_fallback() is False cooldown = getattr(agent, "_rate_limited_until", 0) assert cooldown == far_future + + +class TestRateLimitBackoffEscalation: + """Exponential backoff for consecutive rate-limit failures (#29702). + + The first rate-limit keeps the historical 60s cooldown; each consecutive + rate-limit within one degradation window doubles it (60 → 120 → 240 → ...) + capped at 4h (14400s). A successful primary restore resets the counter. + """ + + @staticmethod + def _back_on_primary(agent, snapshot): + """Simulate the primary provider rate-limiting again on a later turn + (without a successful restore, which would reset the counter): put + the agent's identity back on the primary and reset the turn-scoped + fallback chain state.""" + agent.provider, agent.model, agent.base_url = snapshot + agent._fallback_activated = False + agent._fallback_index = 0 + + def test_backoff_doubles_per_consecutive_rate_limit(self): + """Each consecutive primary rate-limit doubles the cooldown: + 60s, then 120s, then 240s.""" + fbs = [{"provider": "openai", "model": "gpt-4o"}] + agent = _make_agent(fallback_model=fbs) + agent._rate_limited_until = 0 + snapshot = (agent.provider, agent.model, agent.base_url) + frozen = 1_000.0 + with ( + patch("agent.chat_completion_helpers.time.monotonic", return_value=frozen), + patch( + "agent.auxiliary_client.resolve_provider_client", + return_value=(_mock_client(), "resolved"), + ), + ): + expected = [60, 120, 240] + for n, want in enumerate(expected): + self._back_on_primary(agent, snapshot) + agent._try_activate_fallback(reason=FailoverReason.rate_limit) + assert agent._rate_limited_until == frozen + want, ( + f"backoff #{n + 1}: expected {want}s cooldown" + ) + assert agent._rate_limit_backoff_count == n + 1 + + def test_backoff_caps_at_four_hours(self): + """Escalation is capped at 14400s (4h) no matter how many + consecutive rate-limits occurred.""" + fbs = [{"provider": "openai", "model": "gpt-4o"}] + agent = _make_agent(fallback_model=fbs) + agent._rate_limited_until = 0 + # 60 * 2**10 = 61440s, far past the cap. + agent._rate_limit_backoff_count = 10 + frozen = 1_000.0 + with ( + patch("agent.chat_completion_helpers.time.monotonic", return_value=frozen), + patch( + "agent.auxiliary_client.resolve_provider_client", + return_value=(_mock_client(), "resolved"), + ), + ): + agent._try_activate_fallback(reason=FailoverReason.rate_limit) + assert agent._rate_limited_until == frozen + 14400 + + def test_backoff_counter_resets_on_successful_primary_restore(self): + """A successful restore_primary_runtime resets the backoff counter, + so the next rate-limit starts back at the 60s base.""" + fbs = [{"provider": "openai", "model": "gpt-4o"}] + agent = _make_agent(fallback_model=fbs) + snapshot = (agent.provider, agent.model, agent.base_url) + frozen = 1_000.0 + with ( + patch("agent.chat_completion_helpers.time.monotonic", return_value=frozen), + patch( + "agent.auxiliary_client.resolve_provider_client", + return_value=(_mock_client(), "resolved"), + ), + ): + # Two consecutive rate-limits escalate the counter to 2. + agent._rate_limited_until = 0 + agent._try_activate_fallback(reason=FailoverReason.rate_limit) + self._back_on_primary(agent, snapshot) + agent._try_activate_fallback(reason=FailoverReason.rate_limit) + assert agent._rate_limit_backoff_count == 2 + + # Cooldown expired; the primary restores successfully. + agent._fallback_activated = True + agent._rate_limited_until = 0 + assert agent._restore_primary_runtime() is True + assert agent._rate_limit_backoff_count == 0 + + # The next rate-limit is treated as a fresh first failure: 60s. + with ( + patch("agent.chat_completion_helpers.time.monotonic", return_value=frozen), + patch( + "agent.auxiliary_client.resolve_provider_client", + return_value=(_mock_client(), "resolved"), + ), + ): + agent._try_activate_fallback(reason=FailoverReason.rate_limit) + assert agent._rate_limited_until == frozen + 60