fix(agent): count non-CJK sparse text by UTF-8 bytes in the rough estimator

The ~4 chars/token rule is calibrated for ASCII; Cyrillic, Greek, Arabic and
similar 2-byte scripts tokenize at ~2-3 chars/token, so chars/4 under-counts
them ~2x and the pre-flight pressure figure trails real usage by tens of
percent on non-English sessions. Counting UTF-8 BYTES at ~4/token uses the
encoding width itself as the corrective: ASCII is unchanged (1 byte/char),
2-byte scripts count at chars/2, and the CJK dense path keeps its explicit
~1 token/char rule with the sparse remainder byte-counted. The ASCII
isascii() O(1) fast path is preserved; the non-ASCII paths add a single
C-level encode over text that was already being regex-scanned.

Complements the last-real-prompt floor: the floor catches sessions that are
already at the ceiling, this keeps the estimate from lagging in the first
place.
This commit is contained in:
Darafei Praliaskouski
2026-08-16 23:16:58 +04:00
committed by kshitij
parent 73b8ec3cea
commit f07b6ff426
2 changed files with 26 additions and 10 deletions
+13 -6
View File
@@ -3652,13 +3652,20 @@ def estimate_tokens_rough(text: str) -> int:
if text.isascii():
# O(1) fast path — ASCII text cannot contain token-dense CJK chars.
return (len(text) + 3) // 4
dense = len(text) - len(_CJK_DENSE_RE.sub("", text))
stripped = _CJK_DENSE_RE.sub("", text)
dense = len(text) - len(stripped)
if not dense:
# Non-ASCII but no CJK (accents, Cyrillic, emoji, ...): keep the
# classic ~4 chars/token rule.
return (len(text) + 3) // 4
sparse = len(text) - dense
return dense + ((sparse + 3) // 4)
# Non-ASCII but no CJK (accents, Cyrillic, emoji, ...): count UTF-8
# BYTES at ~4/token instead of characters. The byte width is the
# corrective: Cyrillic/Greek/Arabic are 2 bytes per char, so they
# count as ~chars/2 — matching their real BPE cost (~2-3 chars per
# token) where chars/4 under-counted them ~2x and let sessions ride
# the provider's context ceiling below the compaction threshold.
# ASCII spans inside mixed text still count at 1 byte each.
return (len(text.encode("utf-8")) + 3) // 4
# Mixed CJK + other: dense chars stay ~1 token each; the sparse
# remainder is byte-counted for the same corrective.
return dense + ((len(stripped.encode("utf-8")) + 3) // 4)
def estimate_messages_tokens_rough(
+13 -4
View File
@@ -49,15 +49,16 @@ def test_cjk_tail_does_not_expand_to_english_char_budget():
def _reference_per_char_estimate(text: str) -> int:
"""The pre-perf-gate per-character reference implementation."""
"""Per-character reference: CJK ~1 token/char, everything else UTF-8
bytes/4 (the byte width corrects Cyrillic/Greek/Arabic under-counting)."""
dense = 0
sparse = 0
sparse_bytes = 0
for ch in text:
if _is_cjk_token_dense_char(ch):
dense += 1
else:
sparse += 1
return dense + ((sparse + 3) // 4)
sparse_bytes += len(ch.encode("utf-8"))
return dense + ((sparse_bytes + 3) // 4)
def test_perf_gated_estimator_matches_per_char_reference():
@@ -79,3 +80,11 @@ def test_perf_gated_estimator_matches_per_char_reference():
def test_cyrillic_counts_by_utf8_bytes():
# «русский текст» = 12 Cyrillic chars (2 bytes each) + 1 ASCII space:
# 25 bytes -> ceil(25/4) = 7 tokens; the old chars/4 rule said 4 —
# the ~2x under-count that let real prompts ride the context ceiling.
from agent.model_metadata import estimate_tokens_rough
assert estimate_tokens_rough("русский текст") == 7
# Pure ASCII unchanged.
assert estimate_tokens_rough("a" * 400) == 100