Teknium
562e6e4824
Codex Responses reasoning and compaction items carry ciphertext the provider prices by its own token count, never by bytes; a single native compaction checkpoint is ~5M chars, which the bytes/4 estimator turned into ~1.29M "tokens" against a 204K threshold (#100611). #104192 deferred that decision for one request; this removes the mis-pricing at the source so the preflight estimator and the tail-budget walk agree (a mismatched size class protects blob-heavy rows as "small" and compaction re-fires). Only real usage prices these items, and the usage anchor carries that price forward. evals/native_compaction/ab_checkpoint_preflight.py: preflight estimate after checkpoint 1,292,413 -> 58 rough tokens; the over-threshold negative arm still compresses.
126 lines
5.3 KiB
Python
126 lines
5.3 KiB
Python
"""Regression tests for #73298 (second site): the tail-budget walk must not
|
|
charge the signed/base64 ``reasoning_details`` envelope at chars/4.
|
|
|
|
The preflight estimator (``model_metadata._estimate_message_tokens_without_images``)
|
|
excludes ``reasoning_details`` entirely; ``_estimate_msg_budget_tokens`` in the
|
|
compressor mirrors that for the envelope while — per the #51800 counter-argument —
|
|
still counting actual thinking TEXT when it is not already carried by
|
|
``reasoning``/``reasoning_content``.
|
|
"""
|
|
|
|
from agent.context_compressor import (
|
|
_estimate_msg_budget_tokens,
|
|
_reasoning_details_text_chars,
|
|
)
|
|
|
|
|
|
def _signed_thinking_block(thinking_chars: int, sig_chars: int) -> dict:
|
|
return {
|
|
"type": "thinking",
|
|
"thinking": "t" * thinking_chars,
|
|
"signature": "s" * sig_chars,
|
|
}
|
|
|
|
|
|
class TestBudgetExcludesReasoningDetailsEnvelope:
|
|
def test_signature_blob_not_charged(self):
|
|
base = {"role": "assistant", "content": "hello there"}
|
|
with_envelope = dict(
|
|
base,
|
|
reasoning_details=[_signed_thinking_block(0, 40_000)],
|
|
)
|
|
plain = _estimate_msg_budget_tokens(base)
|
|
loaded = _estimate_msg_budget_tokens(with_envelope)
|
|
# 40K chars of base64 signature would add ~10K phantom tokens at
|
|
# chars/4. The envelope must be invisible to the budget.
|
|
assert loaded - plain < 50, (
|
|
f"signed envelope inflated the budget by {loaded - plain} tokens"
|
|
)
|
|
|
|
def test_thinking_text_still_counted(self):
|
|
"""#51800: real reasoning text must stay visible to the budget when
|
|
it is not duplicated into reasoning/reasoning_content."""
|
|
base = {"role": "assistant", "content": "hello there"}
|
|
with_text = dict(
|
|
base,
|
|
reasoning_details=[_signed_thinking_block(8_000, 0)],
|
|
)
|
|
plain = _estimate_msg_budget_tokens(base)
|
|
loaded = _estimate_msg_budget_tokens(with_text)
|
|
# 8K chars of thinking ≈ 2K tokens at chars/4.
|
|
assert loaded - plain > 1_000, (
|
|
"thinking TEXT inside reasoning_details was dropped from the "
|
|
f"budget (delta {loaded - plain}); only the envelope may be excluded"
|
|
)
|
|
|
|
def test_duplicated_thinking_text_not_double_charged(self):
|
|
"""Anthropic-wire sessions carry the prose byte-identical in both
|
|
reasoning_content and reasoning_details; charge it once."""
|
|
prose = "deep thought " * 1_000
|
|
msg = {
|
|
"role": "assistant",
|
|
"content": "hi",
|
|
"reasoning_content": prose,
|
|
"reasoning_details": [
|
|
{"type": "thinking", "thinking": prose, "signature": "s" * 5_000},
|
|
],
|
|
}
|
|
once = _estimate_msg_budget_tokens(
|
|
{"role": "assistant", "content": "hi", "reasoning_content": prose}
|
|
)
|
|
both = _estimate_msg_budget_tokens(msg)
|
|
assert both - once < 100, (
|
|
f"thinking prose charged twice (delta {both - once} tokens)"
|
|
)
|
|
|
|
def test_codex_encrypted_content_is_opaque_to_local_estimates(self):
|
|
"""Ciphertext is priced by the provider's own count, never by its bytes (a native
|
|
compaction checkpoint alone is ~5M chars, #100611): the budget walk and the trigger
|
|
estimator both charge it ~0, so only real usage ever prices it (#104462)."""
|
|
from agent.model_metadata import estimate_messages_tokens_rough
|
|
|
|
base = {"role": "assistant", "content": "hi"}
|
|
loaded = dict(base, codex_reasoning_items=[{"type": "reasoning", "encrypted_content": "e" * 20_000}])
|
|
assert _estimate_msg_budget_tokens(loaded) - _estimate_msg_budget_tokens(base) < 50
|
|
assert estimate_messages_tokens_rough([loaded]) - estimate_messages_tokens_rough([base]) < 50
|
|
|
|
|
|
class TestReasoningDetailsTextChars:
|
|
def test_none_and_empty(self):
|
|
assert _reasoning_details_text_chars(None) == 0
|
|
assert _reasoning_details_text_chars([]) == 0
|
|
assert _reasoning_details_text_chars("") == 0
|
|
|
|
def test_string_counts_fully(self):
|
|
assert _reasoning_details_text_chars("abcd" * 10) == 40
|
|
|
|
def test_envelope_keys_skipped(self):
|
|
blocks = [
|
|
{"type": "thinking", "thinking": "abc", "signature": "S" * 999},
|
|
{"type": "redacted_thinking", "data": "D" * 999},
|
|
{"type": "text", "text": "xy"},
|
|
]
|
|
assert _reasoning_details_text_chars(blocks) == 5
|
|
|
|
|
|
class TestPreflightExcludesReasoningDetails:
|
|
"""Regression coverage for the preflight half (#73306 shipped without a
|
|
test): _estimate_message_tokens_without_images must not count the
|
|
reasoning_details envelope."""
|
|
|
|
def test_preflight_estimate_ignores_reasoning_details(self):
|
|
from agent.model_metadata import _estimate_message_tokens_without_images
|
|
|
|
base = {"role": "assistant", "content": "hello there"}
|
|
loaded = dict(
|
|
base,
|
|
reasoning_details=[_signed_thinking_block(10_000, 40_000)],
|
|
)
|
|
plain = _estimate_message_tokens_without_images(base)
|
|
with_details = _estimate_message_tokens_without_images(loaded)
|
|
assert with_details - plain < 50, (
|
|
"preflight estimator counted the reasoning_details payload "
|
|
f"(delta {with_details - plain} tokens) — compression would fire "
|
|
"at a fraction of real usage on thinking models (#73298)"
|
|
)
|