Teknium
6b81590c55
Systematic prune per AGENTS.md test policy, one pass over every major test tree (gateway, hermes_cli, tools, agent, run_agent, plugins, cli, cron, tui_gateway, honcho/openviking, root-level): - DELETE: source-reading tests (read_text/getsource on prod files), change-detector tests (exact catalog counts, model-name snapshots, config version literals), mock-echo tests (assert a mock returns what it was told), assertion-free/trivial tests, near-duplicate parametrizations (boundaries + one representative kept), async/sync twin duplicates, cosmetic within-file variations. - KEEP (mandatory): security/redaction/approval guards, message-role alternation invariants, prompt-caching/deterministic-call-id invariants, issue-number regression tests (deduped), E2E tests. - 6 test files deleted outright (script-style/no-assert or fully redundant); conftest.py, fakes/, fixtures/ untouched. - tests/acp/conftest.py added: autouse fixture stubs the live models.dev/GitHub/Copilot/Anthropic inventory fetches that ACP server tests performed on every session create — test_server.py 147s → 3.4s, and the tests are now genuinely hermetic. - Sleep-based slowness shrunk where safe (codex_ttfb_watchdog, compression_concurrent_fork, etc.); no wall-clock assertion tightened. Verification: full hermetic suite via scripts/run_tests.sh — 2439 files, 31,130 tests passed, 0 failed, 0 flaky retries, 315s wall (baseline: 583s wall, 13,564s subprocess CPU).
94 lines
3.8 KiB
Python
94 lines
3.8 KiB
Python
"""Regression guard for #31273: HTTP 402 (billing exhaustion) must abort
|
|
after credential-pool rotation and provider fallback have failed.
|
|
|
|
Before the fix, ``FailoverReason.billing`` was in the exclusion set that
|
|
prevents the loop's ``is_client_error`` branch from firing. When a user
|
|
ran a pay-per-token provider (OpenRouter, etc.) with no credential pool
|
|
and no fallback configured, a single 402 cascaded into
|
|
``agent.api_max_retries`` paid requests against an exhausted balance.
|
|
Real-world impact: ~$40 burned in 48h on a 24/7 gateway routing Telegram
|
|
+ Discord traffic.
|
|
|
|
The fix removes ``FailoverReason.billing`` from the exclusion set. By
|
|
the time control reaches the ``is_client_error`` check:
|
|
* credential-pool rotation has already run (and either ``continue``d
|
|
on rotation, or returned False because the pool is exhausted/absent).
|
|
* the eager-fallback branch for billing has also run (and either
|
|
``continue``d on fallback activation, or fell through because no
|
|
fallback is configured).
|
|
Falling through to the retry-backoff path from here just burns paid
|
|
requests with no recovery mechanism left. Aborting mirrors how 401/403
|
|
(also ``should_fallback=True``) already behave once their recovery paths
|
|
have failed.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
|
|
class TestBillingTriggersClientErrorAbort:
|
|
"""Mirror the ``is_client_error`` predicate shape used in
|
|
``agent/conversation_loop.py`` and verify ``FailoverReason.billing``
|
|
now resolves to True (i.e. aborts the loop).
|
|
"""
|
|
|
|
def _mirror_is_client_error(
|
|
self,
|
|
*,
|
|
classified_retryable: bool,
|
|
classified_reason,
|
|
classified_should_compress: bool = False,
|
|
is_local_validation_error: bool = False,
|
|
is_context_length_error: bool = False,
|
|
) -> bool:
|
|
"""Exact shape of conversation_loop.py's is_client_error check.
|
|
|
|
Kept in lock-step with the source. If you change one, change
|
|
both — or, better, refactor the predicate into a shared helper
|
|
and have both sites import it.
|
|
"""
|
|
from agent.error_classifier import FailoverReason
|
|
|
|
return (
|
|
is_local_validation_error
|
|
or (
|
|
not classified_retryable
|
|
and not classified_should_compress
|
|
and classified_reason not in {
|
|
FailoverReason.rate_limit,
|
|
FailoverReason.overloaded,
|
|
FailoverReason.context_overflow,
|
|
FailoverReason.payload_too_large,
|
|
FailoverReason.long_context_tier,
|
|
FailoverReason.thinking_signature,
|
|
}
|
|
)
|
|
) and not is_context_length_error
|
|
|
|
def test_billing_now_aborts_the_loop(self):
|
|
"""402 with no fallback / no pool entry → ``is_client_error`` True."""
|
|
from agent.error_classifier import FailoverReason
|
|
|
|
# This is what classify_api_error() returns for a plain 402:
|
|
# reason=billing, retryable=False, should_compress=False
|
|
assert self._mirror_is_client_error(
|
|
classified_retryable=False,
|
|
classified_reason=FailoverReason.billing,
|
|
), (
|
|
"FailoverReason.billing must trigger is_client_error abort after "
|
|
"credential-pool rotation and provider fallback have failed — see #31273."
|
|
)
|
|
|
|
|
|
|
|
def test_context_overflow_still_falls_through_to_compression(self):
|
|
"""Sanity check: context-overflow must NOT be classified as
|
|
client error — compression is the recovery path."""
|
|
from agent.error_classifier import FailoverReason
|
|
|
|
assert not self._mirror_is_client_error(
|
|
classified_retryable=True,
|
|
classified_reason=FailoverReason.context_overflow,
|
|
classified_should_compress=True,
|
|
)
|
|
|
|
|