refactor(tui_gateway): shared token estimator in compress feedback, iterator subsequence rank

This commit is contained in:
Teknium
2026-09-03 00:47:01 -07:00
parent af0eaf691c
commit 47c781c2a8
2 changed files with 14 additions and 22 deletions
+8 -14
View File
@@ -85,15 +85,13 @@ def _fuzzy_basename_rank(name: str, query: str) -> tuple[int, int] | None:
· 3 substring · 4 subsequence (query chars appear in order)."""
if not query:
return (3, len(name))
nl = name.lower()
ql = query.lower()
nl, ql = name.lower(), query.lower()
if nl == ql:
return (0, len(name))
if nl.startswith(ql):
return (1, len(name))
# Split on -_. and camelCase (`appChrome` → ["app","Chrome"]); cheap approximation,
# falls through to substring/subsequence if it misses.
# Word boundaries: split on -_. and camelCase (`appChrome` → ["app","Chrome"]); cheap
# approximation, falls through to substring/subsequence if it misses.
parts: list[str] = []
buf = ""
for ch in name:
@@ -105,17 +103,13 @@ def _fuzzy_basename_rank(name: str, query: str) -> tuple[int, int] | None:
buf += ch
if buf:
parts.append(buf)
for p in parts:
if p.lower().startswith(ql):
return (2, len(name))
if any(p.lower().startswith(ql) for p in parts):
return (2, len(name))
if ql in nl:
return (3, len(name))
i = 0
for ch in nl:
if ch == ql[i]:
i += 1
if i == len(ql):
return (4, len(name))
it = iter(nl)
if all(any(c == q for c in it) for q in ql):
return (4, len(name))
return None
+6 -8
View File
@@ -265,9 +265,10 @@ def _compress_live_with_feedback(sid: str, session: dict, agent, arg: str, *, sn
history_version = int(session.get("history_version", 0))
sys_prompt = getattr(agent, "_cached_system_prompt", "") or ""
tools = getattr(agent, "tools", None) or None
before_tokens = (
estimate_request_tokens_rough(before_messages, system_prompt=sys_prompt, tools=tools) if before_messages else 0
)
def estimate(messages, prompt, tool_defs) -> int:
return estimate_request_tokens_rough(messages, system_prompt=prompt, tools=tool_defs) if messages else 0
before_tokens = estimate(before_messages, sys_prompt, tools)
try:
if snapshot_kwargs:
_compress_session_history(
@@ -280,11 +281,8 @@ def _compress_live_with_feedback(sid: str, session: dict, agent, arg: str, *, sn
_sync_session_key_after_compress(sid, session)
with session["history_lock"]:
after_messages = list(session.get("history", []))
after_tokens = (
estimate_request_tokens_rough(
after_messages, system_prompt=getattr(agent, "_cached_system_prompt", "") or sys_prompt,
tools=getattr(agent, "tools", None) or tools)
if after_messages else 0)
after_tokens = estimate(
after_messages, getattr(agent, "_cached_system_prompt", "") or sys_prompt, getattr(agent, "tools", None) or tools)
_emit("session.info", sid, _session_info(agent, session))
fb = summarize_manual_compression(
before_messages, after_messages, before_tokens, after_tokens,