From f07b6ff426a5dd7f72fb9e40fe6f9dc3a451d2a9 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 16 Aug 2026 23:16:58 +0400 Subject: [PATCH] fix(agent): count non-CJK sparse text by UTF-8 bytes in the rough estimator The ~4 chars/token rule is calibrated for ASCII; Cyrillic, Greek, Arabic and similar 2-byte scripts tokenize at ~2-3 chars/token, so chars/4 under-counts them ~2x and the pre-flight pressure figure trails real usage by tens of percent on non-English sessions. Counting UTF-8 BYTES at ~4/token uses the encoding width itself as the corrective: ASCII is unchanged (1 byte/char), 2-byte scripts count at chars/2, and the CJK dense path keeps its explicit ~1 token/char rule with the sparse remainder byte-counted. The ASCII isascii() O(1) fast path is preserved; the non-ASCII paths add a single C-level encode over text that was already being regex-scanned. Complements the last-real-prompt floor: the floor catches sessions that are already at the ceiling, this keeps the estimate from lagging in the first place. --- agent/model_metadata.py | 19 +++++++++++++------ tests/agent/test_cjk_token_estimation.py | 17 +++++++++++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index fb78714afd..d93055252a 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -3652,13 +3652,20 @@ def estimate_tokens_rough(text: str) -> int: if text.isascii(): # O(1) fast path — ASCII text cannot contain token-dense CJK chars. return (len(text) + 3) // 4 - dense = len(text) - len(_CJK_DENSE_RE.sub("", text)) + stripped = _CJK_DENSE_RE.sub("", text) + dense = len(text) - len(stripped) if not dense: - # Non-ASCII but no CJK (accents, Cyrillic, emoji, ...): keep the - # classic ~4 chars/token rule. - return (len(text) + 3) // 4 - sparse = len(text) - dense - return dense + ((sparse + 3) // 4) + # Non-ASCII but no CJK (accents, Cyrillic, emoji, ...): count UTF-8 + # BYTES at ~4/token instead of characters. The byte width is the + # corrective: Cyrillic/Greek/Arabic are 2 bytes per char, so they + # count as ~chars/2 — matching their real BPE cost (~2-3 chars per + # token) where chars/4 under-counted them ~2x and let sessions ride + # the provider's context ceiling below the compaction threshold. + # ASCII spans inside mixed text still count at 1 byte each. + return (len(text.encode("utf-8")) + 3) // 4 + # Mixed CJK + other: dense chars stay ~1 token each; the sparse + # remainder is byte-counted for the same corrective. + return dense + ((len(stripped.encode("utf-8")) + 3) // 4) def estimate_messages_tokens_rough( diff --git a/tests/agent/test_cjk_token_estimation.py b/tests/agent/test_cjk_token_estimation.py index 003d39d1eb..c584597ad6 100644 --- a/tests/agent/test_cjk_token_estimation.py +++ b/tests/agent/test_cjk_token_estimation.py @@ -49,15 +49,16 @@ def test_cjk_tail_does_not_expand_to_english_char_budget(): def _reference_per_char_estimate(text: str) -> int: - """The pre-perf-gate per-character reference implementation.""" + """Per-character reference: CJK ~1 token/char, everything else UTF-8 + bytes/4 (the byte width corrects Cyrillic/Greek/Arabic under-counting).""" dense = 0 - sparse = 0 + sparse_bytes = 0 for ch in text: if _is_cjk_token_dense_char(ch): dense += 1 else: - sparse += 1 - return dense + ((sparse + 3) // 4) + sparse_bytes += len(ch.encode("utf-8")) + return dense + ((sparse_bytes + 3) // 4) def test_perf_gated_estimator_matches_per_char_reference(): @@ -79,3 +80,11 @@ def test_perf_gated_estimator_matches_per_char_reference(): +def test_cyrillic_counts_by_utf8_bytes(): + # «русский текст» = 12 Cyrillic chars (2 bytes each) + 1 ASCII space: + # 25 bytes -> ceil(25/4) = 7 tokens; the old chars/4 rule said 4 — + # the ~2x under-count that let real prompts ride the context ceiling. + from agent.model_metadata import estimate_tokens_rough + assert estimate_tokens_rough("русский текст") == 7 + # Pure ASCII unchanged. + assert estimate_tokens_rough("a" * 400) == 100