fix(agent): count non-CJK sparse text by UTF-8 bytes in the rough estimator
The ~4 chars/token rule is calibrated for ASCII; Cyrillic, Greek, Arabic and similar 2-byte scripts tokenize at ~2-3 chars/token, so chars/4 under-counts them ~2x and the pre-flight pressure figure trails real usage by tens of percent on non-English sessions. Counting UTF-8 BYTES at ~4/token uses the encoding width itself as the corrective: ASCII is unchanged (1 byte/char), 2-byte scripts count at chars/2, and the CJK dense path keeps its explicit ~1 token/char rule with the sparse remainder byte-counted. The ASCII isascii() O(1) fast path is preserved; the non-ASCII paths add a single C-level encode over text that was already being regex-scanned. Complements the last-real-prompt floor: the floor catches sessions that are already at the ceiling, this keeps the estimate from lagging in the first place.
This commit is contained in:
committed by
kshitij
parent
73b8ec3cea
commit
f07b6ff426
+13
-6
@@ -3652,13 +3652,20 @@ def estimate_tokens_rough(text: str) -> int:
|
||||
if text.isascii():
|
||||
# O(1) fast path — ASCII text cannot contain token-dense CJK chars.
|
||||
return (len(text) + 3) // 4
|
||||
dense = len(text) - len(_CJK_DENSE_RE.sub("", text))
|
||||
stripped = _CJK_DENSE_RE.sub("", text)
|
||||
dense = len(text) - len(stripped)
|
||||
if not dense:
|
||||
# Non-ASCII but no CJK (accents, Cyrillic, emoji, ...): keep the
|
||||
# classic ~4 chars/token rule.
|
||||
return (len(text) + 3) // 4
|
||||
sparse = len(text) - dense
|
||||
return dense + ((sparse + 3) // 4)
|
||||
# Non-ASCII but no CJK (accents, Cyrillic, emoji, ...): count UTF-8
|
||||
# BYTES at ~4/token instead of characters. The byte width is the
|
||||
# corrective: Cyrillic/Greek/Arabic are 2 bytes per char, so they
|
||||
# count as ~chars/2 — matching their real BPE cost (~2-3 chars per
|
||||
# token) where chars/4 under-counted them ~2x and let sessions ride
|
||||
# the provider's context ceiling below the compaction threshold.
|
||||
# ASCII spans inside mixed text still count at 1 byte each.
|
||||
return (len(text.encode("utf-8")) + 3) // 4
|
||||
# Mixed CJK + other: dense chars stay ~1 token each; the sparse
|
||||
# remainder is byte-counted for the same corrective.
|
||||
return dense + ((len(stripped.encode("utf-8")) + 3) // 4)
|
||||
|
||||
|
||||
def estimate_messages_tokens_rough(
|
||||
|
||||
@@ -49,15 +49,16 @@ def test_cjk_tail_does_not_expand_to_english_char_budget():
|
||||
|
||||
|
||||
def _reference_per_char_estimate(text: str) -> int:
|
||||
"""The pre-perf-gate per-character reference implementation."""
|
||||
"""Per-character reference: CJK ~1 token/char, everything else UTF-8
|
||||
bytes/4 (the byte width corrects Cyrillic/Greek/Arabic under-counting)."""
|
||||
dense = 0
|
||||
sparse = 0
|
||||
sparse_bytes = 0
|
||||
for ch in text:
|
||||
if _is_cjk_token_dense_char(ch):
|
||||
dense += 1
|
||||
else:
|
||||
sparse += 1
|
||||
return dense + ((sparse + 3) // 4)
|
||||
sparse_bytes += len(ch.encode("utf-8"))
|
||||
return dense + ((sparse_bytes + 3) // 4)
|
||||
|
||||
|
||||
def test_perf_gated_estimator_matches_per_char_reference():
|
||||
@@ -79,3 +80,11 @@ def test_perf_gated_estimator_matches_per_char_reference():
|
||||
|
||||
|
||||
|
||||
def test_cyrillic_counts_by_utf8_bytes():
|
||||
# «русский текст» = 12 Cyrillic chars (2 bytes each) + 1 ASCII space:
|
||||
# 25 bytes -> ceil(25/4) = 7 tokens; the old chars/4 rule said 4 —
|
||||
# the ~2x under-count that let real prompts ride the context ceiling.
|
||||
from agent.model_metadata import estimate_tokens_rough
|
||||
assert estimate_tokens_rough("русский текст") == 7
|
||||
# Pure ASCII unchanged.
|
||||
assert estimate_tokens_rough("a" * 400) == 100
|
||||
|
||||
Reference in New Issue
Block a user