refactor(adapters/base): base.py 7719->5348, helpers.py 942->640; extract _process_message_background helpers, unify media send/fallback/cache/notice paths, drop dead _resolve_extensionless_candidate + TextBatchAggregator, compact docs

This commit is contained in:
Teknium
2026-09-02 14:06:31 -07:00
parent 661fc669a0
commit d3db321828
5 changed files with 1997 additions and 4716 deletions
+12 -28
View File
@@ -1,27 +1,10 @@
"""
Platform adapters for messaging integrations.
Each adapter handles:
- Receiving messages from a platform
- Sending messages/responses back
- Platform-specific authentication
- Message formatting and media handling
"""
"""Platform adapters for messaging integrations (receive, send, auth, media)."""
from .base import BasePlatformAdapter, MessageEvent, SendResult
# QQAdapter and YuanbaoAdapter were previously imported eagerly here, but
# nothing in the codebase consumes ``from gateway.platforms import
# QQAdapter`` (every real call site uses the long-form path
# ``from gateway.platforms.qqbot import QQAdapter``). The eager imports
# pulled in qqbot's chunked-upload + keyboards + onboard machinery and
# yuanbao's websocket stack — about 48 ms wall and ~8 MB RSS on every
# CLI invocation, even ones that never touch a gateway adapter.
#
# Use PEP 562 module ``__getattr__`` to keep the public re-export working
# while deferring the actual import to first attribute access. This is
# 100% backward-compatible for any external code that still imports the
# adapters from the package root.
# QQAdapter / YuanbaoAdapter are exposed lazily (PEP 562 ``__getattr__``): eager
# imports cost ~48 ms / ~8 MB RSS on every CLI invocation and nothing in-tree
# imports them from the package root.
__all__ = [
"BasePlatformAdapter",
"MessageEvent",
@@ -30,15 +13,16 @@ __all__ = [
"YuanbaoAdapter",
]
_LAZY_ADAPTERS = {"QQAdapter": ".qqbot", "YuanbaoAdapter": ".yuanbao"}
def __getattr__(name):
if name == "QQAdapter":
from .qqbot import QQAdapter # noqa: F401
return QQAdapter
if name == "YuanbaoAdapter":
from .yuanbao import YuanbaoAdapter # noqa: F401
return YuanbaoAdapter
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
module = _LAZY_ADAPTERS.get(name)
if module is None:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
from importlib import import_module
return getattr(import_module(module, __name__), name)
def __dir__():
+30 -57
View File
@@ -1,26 +1,12 @@
"""Shared HTTP client factory for long-lived platform adapters.
Gateway messaging platforms (QQ Bot, Feishu, WeCom, DingTalk, Signal,
BlueBubbles, WeCom-callback) keep a persistent ``httpx.AsyncClient``
alive for the adapter's lifetime. That amortises TLS/connection setup
across many API calls, but it also means the process's file-descriptor
pressure is sensitive to how aggressively the pool recycles idle keep-
alive connections.
httpx's default ``keepalive_expiry`` is 5 seconds. On macOS behind
Cloudflare Warp (and other transparent proxies), peer-initiated FIN can
sit in ``CLOSE_WAIT`` longer than that before the local socket actually
drains — which, multiplied across 7 long-lived adapters plus the LLM
client and MCP clients, walks straight into the default 256 fd limit.
See #18451.
``platform_httpx_limits()`` returns a tighter ``httpx.Limits`` the
adapter factories use instead of the httpx default. The values chosen:
* ``max_keepalive_connections=10`` — plenty for any single adapter;
platform APIs rarely parallelise beyond this.
* ``keepalive_expiry=2.0`` — close idle sockets aggressively so a
proxy's lingering CLOSE_WAIT window can't starve the process.
Persistent ``httpx.AsyncClient`` pools amortise TLS setup, but httpx's default
``keepalive_expiry`` (5s) lets peer-initiated FIN sit in ``CLOSE_WAIT`` behind
transparent proxies (macOS + Cloudflare Warp) — multiplied across 7 adapters
plus LLM/MCP clients that walks into the default 256 fd limit (#18451).
``platform_httpx_limits()`` returns a tighter ``httpx.Limits``:
``max_keepalive_connections=10`` (platform APIs rarely parallelise beyond
this), ``keepalive_expiry=2.0`` (close idle sockets aggressively).
Override via ``HERMES_GATEWAY_HTTPX_KEEPALIVE_EXPIRY`` /
``HERMES_GATEWAY_HTTPX_MAX_KEEPALIVE`` env vars when tuning under load.
@@ -40,45 +26,32 @@ _DEFAULT_KEEPALIVE_EXPIRY_S = 2.0
_DEFAULT_MAX_KEEPALIVE = 10
def platform_httpx_limits() -> "httpx.Limits | None":
"""Return ``httpx.Limits`` tuned for persistent platform-adapter clients.
def _positive_env(name: str, default, cast):
"""``cast(env)`` when set, parseable and > 0; else *default*."""
raw = os.environ.get(name, "").strip()
if not raw:
return default
try:
val = cast(raw)
except (TypeError, ValueError):
return default
return val if val > 0 else default
Returns ``None`` when httpx isn't importable, so callers can fall
back to httpx's built-in default without a hard dependency on this
helper being reachable.
def platform_httpx_limits() -> "httpx.Limits | None":
"""``httpx.Limits`` tuned for persistent platform-adapter clients.
Returns ``None`` when httpx isn't importable so callers can fall back to
httpx's built-in default without a hard dependency on this helper.
"""
if httpx is None:
return None
def _env_float(name: str, default: float) -> float:
raw = os.environ.get(name, "").strip()
if not raw:
return default
try:
val = float(raw)
except (TypeError, ValueError):
return default
return val if val > 0 else default
def _env_int(name: str, default: int) -> int:
raw = os.environ.get(name, "").strip()
if not raw:
return default
try:
val = int(raw)
except (TypeError, ValueError):
return default
return val if val > 0 else default
keepalive_expiry = _env_float(
"HERMES_GATEWAY_HTTPX_KEEPALIVE_EXPIRY", _DEFAULT_KEEPALIVE_EXPIRY_S
)
max_keepalive = _env_int(
"HERMES_GATEWAY_HTTPX_MAX_KEEPALIVE", _DEFAULT_MAX_KEEPALIVE
)
return httpx.Limits(
max_keepalive_connections=max_keepalive,
# Leave max_connections at httpx default (100) — plenty of headroom.
keepalive_expiry=keepalive_expiry,
max_keepalive_connections=_positive_env(
"HERMES_GATEWAY_HTTPX_MAX_KEEPALIVE", _DEFAULT_MAX_KEEPALIVE, int
),
# max_connections stays at the httpx default (100) — plenty of headroom.
keepalive_expiry=_positive_env(
"HERMES_GATEWAY_HTTPX_KEEPALIVE_EXPIRY", _DEFAULT_KEEPALIVE_EXPIRY_S, float
),
)
+1851 -4222
View File
File diff suppressed because it is too large Load Diff
+104 -406
View File
@@ -1,23 +1,16 @@
"""Shared helper classes for gateway platform adapters.
"""Shared helpers for gateway platform adapters: message dedup, markdown
stripping, thread participation tracking, GFM table → bullets, mention-pattern
compilation, and fence-aware markdown chunking."""
Extracts common patterns that were duplicated across 5-7 adapters:
message deduplication, text batch aggregation, markdown stripping,
and thread participation tracking.
"""
import asyncio
import json
import logging
import re
import time
from pathlib import Path
from typing import TYPE_CHECKING, Dict
from typing import Dict
from utils import atomic_json_write
if TYPE_CHECKING:
from gateway.platforms.base import MessageEvent
logger = logging.getLogger(__name__)
@@ -25,20 +18,7 @@ logger = logging.getLogger(__name__)
class MessageDeduplicator:
"""TTL-based message deduplication cache.
Replaces the identical ``_seen_messages`` / ``_is_duplicate()`` pattern
previously duplicated in discord, slack, dingtalk, wecom, weixin,
mattermost, and feishu adapters.
Usage::
self._dedup = MessageDeduplicator()
# In message handler:
if self._dedup.is_duplicate(msg_id):
return
"""
"""TTL-based message deduplication cache (``if dedup.is_duplicate(msg_id): return``)."""
def __init__(self, max_size: int = 2000, ttl_seconds: float = 300):
self._seen: Dict[str, float] = {}
@@ -60,14 +40,8 @@ class MessageDeduplicator:
cutoff = now - self._ttl
self._seen = {k: v for k, v in self._seen.items() if v > cutoff}
if len(self._seen) > self._max_size:
# TTL pruning alone does not cap the cache when every entry is
# still fresh. Keep the newest entries so the helper's
# max_size bound is enforced under sustained traffic.
newest = sorted(
self._seen.items(),
key=lambda item: item[1],
)[-self._max_size:]
self._seen = dict(newest)
# All entries still fresh: keep the newest so max_size holds under load.
self._seen = dict(sorted(self._seen.items(), key=lambda item: item[1])[-self._max_size:])
return False
def contains(self, msg_id: str) -> bool:
@@ -91,94 +65,6 @@ class MessageDeduplicator:
self._seen.clear()
# ─── Text Batch Aggregation ──────────────────────────────────────────────────
class TextBatchAggregator:
"""Aggregates rapid-fire text events into single messages.
Replaces the ``_enqueue_text_event`` / ``_flush_text_batch`` pattern
previously duplicated in telegram, discord, matrix, wecom, and feishu.
Usage::
self._text_batcher = TextBatchAggregator(
handler=self._message_handler,
batch_delay=0.6,
split_threshold=1900,
)
# In message dispatch:
if msg_type == MessageType.TEXT and self._text_batcher.is_enabled():
self._text_batcher.enqueue(event, session_key)
return
"""
def __init__(
self,
handler,
*,
batch_delay: float = 0.6,
split_delay: float = 2.0,
split_threshold: int = 4000,
):
self._handler = handler
self._batch_delay = batch_delay
self._split_delay = split_delay
self._split_threshold = split_threshold
self._pending: Dict[str, "MessageEvent"] = {}
self._pending_tasks: Dict[str, asyncio.Task] = {}
def is_enabled(self) -> bool:
"""Return True if batching is active (delay > 0)."""
return self._batch_delay > 0
def enqueue(self, event: "MessageEvent", key: str) -> None:
"""Add *event* to the pending batch for *key*."""
chunk_len = len(event.text or "")
existing = self._pending.get(key)
if not existing:
event._last_chunk_len = chunk_len # type: ignore[attr-defined]
self._pending[key] = event
else:
existing.text = f"{existing.text}\n{event.text}"
existing._last_chunk_len = chunk_len # type: ignore[attr-defined]
# Cancel prior flush timer, start a new one
prior = self._pending_tasks.get(key)
if prior and not prior.done():
prior.cancel()
self._pending_tasks[key] = asyncio.create_task(self._flush(key))
async def _flush(self, key: str) -> None:
"""Wait then dispatch the batched event for *key*."""
current_task = self._pending_tasks.get(key)
pending = self._pending.get(key)
last_len = getattr(pending, "_last_chunk_len", 0) if pending else 0
# Use longer delay when the last chunk looks like a split message
delay = self._split_delay if last_len >= self._split_threshold else self._batch_delay
await asyncio.sleep(delay)
event = self._pending.pop(key, None)
if event:
try:
await self._handler(event)
except Exception:
logger.exception("[TextBatchAggregator] Error dispatching batched event for %s", key)
if self._pending_tasks.get(key) is current_task:
self._pending_tasks.pop(key, None)
def cancel_all(self) -> None:
"""Cancel all pending flush tasks."""
for task in self._pending_tasks.values():
if not task.done():
task.cancel()
self._pending_tasks.clear()
self._pending.clear()
# ─── Markdown Stripping ──────────────────────────────────────────────────────
# Pre-compiled regexes for performance
@@ -194,11 +80,7 @@ _RE_MULTI_NEWLINE = re.compile(r"\n{3,}")
def strip_markdown(text: str) -> str:
"""Strip markdown formatting for plain-text platforms (SMS, iMessage, etc.).
Replaces the identical ``_strip_markdown()`` functions previously
duplicated in sms.py, bluebubbles.py, and feishu.py.
"""
"""Strip markdown formatting for plain-text platforms (SMS, iMessage, etc.)."""
text = _RE_BOLD.sub(r"\1", text)
text = _RE_ITALIC_STAR.sub(r"\1", text)
text = _RE_BOLD_UNDER.sub(r"\1", text)
@@ -215,22 +97,9 @@ def strip_markdown(text: str) -> str:
class ThreadParticipationTracker:
"""Persistent tracking of threads the bot has participated in.
"""Persistent set of threads the bot has participated in (``<platform>_threads.json``).
Replaces the identical ``_load/_save_participated_threads`` +
``_mark_thread_participated`` pattern previously duplicated in
discord.py and matrix.py.
Usage::
self._threads = ThreadParticipationTracker("discord")
# Check membership:
if thread_id in self._threads:
...
# Mark participation:
self._threads.mark(thread_id)
``thread_id in tracker`` checks membership; ``tracker.mark(thread_id)`` persists.
"""
_MAX_TRACKED = 500
@@ -238,9 +107,7 @@ class ThreadParticipationTracker:
def __init__(self, platform_name: str, max_tracked: int = 500):
self._platform = platform_name
self._max_tracked = max_tracked
self._threads: dict[str, None] = {
str(thread_id): None for thread_id in self._load()
}
self._threads: dict[str, None] = dict.fromkeys(str(t) for t in self._load())
def _state_path(self) -> Path:
from hermes_constants import get_hermes_home
@@ -282,11 +149,7 @@ class ThreadParticipationTracker:
def redact_phone(phone: str) -> str:
"""Redact a phone number for logging, preserving country code and last 4.
Replaces the identical ``_redact_phone()`` functions in signal.py,
sms.py, and bluebubbles.py.
"""
"""Redact a phone number for logging, preserving country code and last 4."""
if not phone:
return "<none>"
if len(phone) <= 8:
@@ -295,17 +158,12 @@ def redact_phone(phone: str) -> str:
# ─── GFM Markdown Table → Bullet Conversion ─────────────────────────────────
# Shared by Discord and Telegram adapters. Discord calls
# convert_table_to_bullets() directly; Telegram imports the primitives
# but keeps its own MarkdownV2-aware renderer.
# Discord calls convert_table_to_bullets(); Telegram imports the primitives but
# keeps its own MarkdownV2-aware renderer.
# Matches a GFM table delimiter row: optional outer pipes, cells of dashes
# (with optional alignment colons) separated by '|'.
# Requires at least one internal '|' so lone '---' rules are NOT matched.
TABLE_SEPARATOR_RE = re.compile(
r'^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*){1,}\|?\s*$'
)
# GFM delimiter row: optional outer pipes, dash cells (optional alignment colons).
# Requires at least one internal '|' so a lone '---' rule is NOT matched.
TABLE_SEPARATOR_RE = re.compile(r'^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*){1,}\|?\s*$')
def is_table_row(line: str) -> bool:
@@ -315,40 +173,24 @@ def is_table_row(line: str) -> bool:
def split_markdown_table_row(line: str) -> list[str]:
"""Split a GFM table row into stripped cell values.
Thin delegate to the canonical implementation in
:mod:`agent.markdown_tables` (``split_table_row``) so the three
formerly byte-identical copies (here, ``agent/markdown_tables.py``,
``weixin._split_table_row``) share one body.
"""
"""Split a GFM table row into stripped cells (delegates to agent.markdown_tables)."""
from agent.markdown_tables import split_table_row
return split_table_row(line)
def _render_table_block(table_block: list[str]) -> str:
"""Render a detected GFM table as bold-heading + bullet groups.
Uses the same alignment logic as Telegram's renderer: for non-row-label
tables, ``data_cells = cells`` (the full row) and the bullet whose value
duplicates the heading is skipped. This keeps header→value alignment
correct.
Same alignment logic as Telegram's renderer: without a row-label column the
full row is the data and the bullet duplicating the heading is skipped.
"""
if len(table_block) < 3:
return "\n".join(table_block)
headers = split_markdown_table_row(table_block[0])
if len(headers) < 2:
return "\n".join(table_block)
first_data_row = (
split_markdown_table_row(table_block[2])
if len(table_block) > 2
else []
)
first_data_row = split_markdown_table_row(table_block[2])
has_row_label_col = len(first_data_row) == len(headers) + 1
rendered_groups: list[str] = []
for index, row in enumerate(table_block[2:], start=1):
cells = split_markdown_table_row(row)
@@ -358,21 +200,16 @@ def _render_table_block(table_block: list[str]) -> str:
else:
heading = next((cell for cell in cells if cell), f"Row {index}")
data_cells = cells
if len(data_cells) < len(headers):
data_cells.extend([""] * (len(headers) - len(data_cells)))
elif len(data_cells) > len(headers):
data_cells = data_cells[: len(headers)]
bullets: list[str] = []
for header, value in zip(headers, data_cells):
if not has_row_label_col and value == heading:
continue
bullets.append(f"• {header}: {value}")
group_lines = [f"**{heading}**", *bullets]
rendered_groups.append("\n".join(group_lines))
bullets = [
f"• {header}: {value}"
for header, value in zip(headers, data_cells)
if has_row_label_col or value != heading
]
rendered_groups.append("\n".join([f"**{heading}**", *bullets]))
return "\n\n".join(rendered_groups)
@@ -383,7 +220,6 @@ def convert_table_to_bullets(text: str) -> str:
"""
if '|' not in text or '-' not in text:
return text
lines = text.split('\n')
out: list[str] = []
in_fence = False
@@ -391,22 +227,13 @@ def convert_table_to_bullets(text: str) -> str:
while i < len(lines):
line = lines[i]
stripped = line.lstrip()
if stripped.startswith('```'):
in_fence = not in_fence
if in_fence or stripped.startswith('```'):
out.append(line)
i += 1
continue
if in_fence:
out.append(line)
i += 1
continue
if (
'|' in line
and i + 1 < len(lines)
and TABLE_SEPARATOR_RE.match(lines[i + 1])
):
if '|' in line and i + 1 < len(lines) and TABLE_SEPARATOR_RE.match(lines[i + 1]):
table_block = [line, lines[i + 1]]
j = i + 2
while j < len(lines) and is_table_row(lines[j]):
@@ -415,10 +242,8 @@ def convert_table_to_bullets(text: str) -> str:
out.append(_render_table_block(table_block))
i = j
continue
out.append(line)
i += 1
return '\n'.join(out)
@@ -436,25 +261,18 @@ def compile_mention_patterns(
) -> 'list[re.Pattern]':
"""Compile regex wake-word/mention patterns from config or env values.
Two adapter families share this logic:
* **Config-style** (dingtalk, telegram): pass ``platform_label``. ``raw``
must be a list or string (anything else warns and yields ``[]``);
non-string entries are skipped; a summary info log is emitted on load.
* **Wakeword-style** (photon, bluebubbles): pass ``defaults``. ``raw`` may be
None (defaults), a string (JSON list or comma/newline separated), a list,
or a scalar; entries are coerced via ``str()``.
* **Config-style** (dingtalk, telegram): pass ``platform_label`` (e.g.
``"dingtalk"``). ``raw`` is the value from ``config.extra`` after env
fallback parsing; must be a list or string, anything else logs a warning
and yields ``[]``. Non-string entries are skipped. A summary info log is
emitted when patterns load.
* **Wakeword-style** (photon, bluebubbles): pass ``defaults``. ``raw`` may
be None (use defaults), a string (JSON list or comma/newline separated),
a list, or a scalar (wrapped in a list). Entries are coerced via
``str()``.
``log_prefix`` is interpolated into every log message so per-adapter log
output stays byte-identical to the historical inline implementations.
``log_prefix`` is interpolated into every log line so per-adapter output
stays byte-identical to the historical inline implementations.
"""
log = logger_ or logger
if platform_label is not None:
# Config-style (dingtalk/telegram) semantics.
display = display_label or platform_label
patterns = raw
if patterns is None:
@@ -464,12 +282,9 @@ def compile_mention_patterns(
if not isinstance(patterns, list):
log.warning(
"[%s] %s mention_patterns must be a list or string; got %s",
log_prefix,
platform_label,
type(patterns).__name__,
log_prefix, platform_label, type(patterns).__name__,
)
return []
compiled: list[re.Pattern] = []
for pattern in patterns:
if not isinstance(pattern, str) or not pattern.strip():
@@ -477,23 +292,10 @@ def compile_mention_patterns(
try:
compiled.append(re.compile(pattern, re.IGNORECASE))
except re.error as exc:
log.warning(
"[%s] Invalid %s mention pattern %r: %s",
log_prefix,
display,
pattern,
exc,
)
log.warning("[%s] Invalid %s mention pattern %r: %s", log_prefix, display, pattern, exc)
if compiled:
log.info(
"[%s] Loaded %d %s mention pattern(s)",
log_prefix,
len(compiled),
display,
)
log.info("[%s] Loaded %d %s mention pattern(s)", log_prefix, len(compiled), display)
return compiled
# Wakeword-style (photon/bluebubbles) semantics.
if raw is None:
patterns = list(defaults or [])
elif isinstance(raw, str):
@@ -503,15 +305,12 @@ def compile_mention_patterns(
except Exception:
loaded = None
patterns = loaded if isinstance(loaded, list) else [
part.strip()
for line in text.splitlines()
for part in line.split(",")
part.strip() for line in text.splitlines() for part in line.split(",")
]
elif isinstance(raw, list):
patterns = raw
else:
patterns = [raw]
compiled = []
for pattern in patterns:
text = str(pattern).strip()
@@ -525,28 +324,15 @@ def compile_mention_patterns(
# ─── Fence-Aware Markdown Chunking ───────────────────────────────────────────
# Shared core for the fence-aware markdown chunkers that previously lived as
# near-duplicates in gateway/stream_consumer.py, gateway/platforms/yuanbao.py
# (MarkdownProcessor — the richest version, which this core is derived from),
# and gateway/platforms/weixin.py. Each caller keeps its own knobs:
#
# * stream_consumer: newline-preferred splitting + close/reopen fence
# balancing (``prefer_paragraphs=False, balance_fences=True``)
# * yuanbao: atomic-block extraction + paragraph-boundary splitting, fences
# kept intact as atoms (``prefer_paragraphs=True, balance_fences=False``)
# * weixin: keeps its own block splitter (anchored ``_FENCE_RE``, per-line
# rstrip semantics) but reuses ``greedy_pack_blocks`` for packing.
#
# The typing helpers below use ``Optional``/``Callable`` from ``typing`` to
# match the module's existing import style.
# Shared core for the chunkers in gateway/stream_consumer.py (newline-preferred
# + fence balancing: ``prefer_paragraphs=False, balance_fences=True``), yuanbao
# (atomic blocks + paragraph splitting, fences kept as atoms:
# ``prefer_paragraphs=True, balance_fences=False``) and weixin (own block
# splitter, reuses ``greedy_pack_blocks``).
def text_has_unclosed_fence(text: str) -> bool:
"""Return True when *text* ends inside an unclosed ``` code fence.
Scans line by line, toggling in/out state on lines starting with ```.
An odd number of toggles means the trailing fence is unclosed.
"""
"""Return True when *text* ends inside an unclosed ``` code fence."""
in_fence = False
for line in text.split('\n'):
if line.startswith('```'):
@@ -557,10 +343,7 @@ def text_has_unclosed_fence(text: str) -> bool:
def text_ends_with_table_row(text: str) -> bool:
"""True when the last non-empty line starts and ends with ``|``."""
trimmed = text.rstrip()
if not trimmed:
return False
last_line = trimmed.split('\n')[-1].strip()
return last_line.startswith('|') and last_line.endswith('|')
return bool(trimmed) and _is_pipe_row(trimmed.split('\n')[-1])
def is_fence_atom(text: str) -> bool:
@@ -568,82 +351,57 @@ def is_fence_atom(text: str) -> bool:
return text.lstrip().startswith('```')
def _is_pipe_row(line: str) -> bool:
stripped = line.strip()
return stripped.startswith('|') and stripped.endswith('|')
def is_table_atom(text: str) -> bool:
"""True when an atomic block is a table (first line is ``|...|``)."""
first_line = text.split('\n')[0].strip()
return first_line.startswith('|') and first_line.endswith('|')
return _is_pipe_row(text.split('\n')[0])
_SENTENCE_END_NEWLINE_RE = re.compile(r'[。!?.!?]\n')
def split_at_paragraph_boundary(text, max_chars, len_fn=None):
"""Find the nearest paragraph boundary within *max_chars*; return (head, tail).
"""Split at the nearest paragraph boundary within *max_chars*; return (head, tail).
Split priority:
1. Blank line (paragraph boundary)
2. Newline after sentence-ending punctuation (CJK and ASCII)
3. Last newline
4. Force split at the *max_chars* window boundary
``head + tail == text`` always holds. *len_fn* allows measuring in
custom units (e.g. UTF-16 code units); a binary search finds the largest
prefix that fits when it is provided.
Priority: blank line → newline after sentence-ending punctuation (CJK/ASCII)
→ last newline → forced split at the window boundary. ``head + tail == text``
always holds. *len_fn* measures in custom units (e.g. UTF-16 code units).
"""
_len = len_fn or len
if _len(text) <= max_chars:
return text, ''
if _len is len:
window = text[:max_chars]
else:
lo, hi = 0, len(text)
while lo < hi:
mid = (lo + hi + 1) // 2
if _len(text[:mid]) <= max_chars:
lo = mid
else:
hi = mid - 1
window = text[:lo]
# 1. Prefer the last blank line (\n\n) as paragraph boundary
from gateway.platforms.base import _custom_unit_to_cp # heavyweight; lazy
window = text[:_custom_unit_to_cp(text, max_chars, _len)]
pos = window.rfind('\n\n')
if pos > 0:
return text[:pos + 2], text[pos + 2:]
# 2. Then the last newline following sentence-ending punctuation
best_pos = -1
for m in _SENTENCE_END_NEWLINE_RE.finditer(window):
best_pos = m.end()
if best_pos > 0:
return text[:best_pos], text[best_pos:]
# 3. Fallback: last newline
pos = window.rfind('\n')
if pos > 0:
return text[:pos + 1], text[pos + 1:]
# 4. No valid split point: force split at the window boundary
cut = len(window)
return text[:cut], text[cut:]
def split_markdown_atoms(text: str) -> "list[str]":
"""Split markdown into indivisible "atomic blocks".
Atoms are: fenced code blocks (``` ... ``` inclusive), tables
(consecutive ``|...|`` lines), and plain paragraphs separated by blank
lines. Blank lines are separators and belong to no atom.
"""
"""Split markdown into indivisible atoms: fenced code blocks, tables
(consecutive ``|...|`` lines) and paragraphs. Blank lines belong to no atom."""
lines = text.split('\n')
atoms: "list[str]" = []
current_lines: "list[str]" = []
in_fence = False
def _is_table_line(line: str) -> bool:
stripped = line.strip()
return stripped.startswith('|') and stripped.endswith('|')
_is_table_line = _is_pipe_row
def _flush_current() -> None:
if current_lines:
@@ -651,7 +409,6 @@ def split_markdown_atoms(text: str) -> "list[str]":
if atom.strip():
atoms.append(atom)
current_lines.clear()
for line in lines:
if in_fence:
current_lines.append(line)
@@ -672,41 +429,26 @@ def split_markdown_atoms(text: str) -> "list[str]":
if current_lines and _is_table_line(current_lines[-1]):
_flush_current()
current_lines.append(line)
_flush_current()
return atoms
def infer_block_separator(prev_chunk: str, next_chunk: str) -> str:
"""Infer the separator (``'\\n'`` or ``'\\n\\n'``) between two chunks.
Single newline when the boundary sits at a code fence or a continued
table; paragraph separator otherwise.
"""
"""``'\\n'`` when the boundary sits at a code fence or a continued table, else ``'\\n\\n'``."""
prev_trimmed = prev_chunk.rstrip()
next_trimmed = next_chunk.lstrip()
if prev_trimmed.endswith('```') or next_trimmed.startswith('```'):
return '\n'
if text_ends_with_table_row(prev_chunk):
first_line = next_trimmed.split('\n')[0].strip() if next_trimmed else ''
if first_line.startswith('|') and first_line.endswith('|'):
return '\n'
if text_ends_with_table_row(prev_chunk) and next_trimmed and _is_pipe_row(next_trimmed.split('\n')[0]):
return '\n'
return '\n\n'
def merge_streaming_fences(chunks: "list[str]") -> "list[str]":
"""Stream-aware fence merge: rejoin chunks truncated mid-fence.
While chunk *i* has an unclosed fence and a successor exists, merge the
successor into it using :func:`infer_block_separator`.
"""
"""Rejoin chunks truncated mid-fence: while chunk *i* has an unclosed fence
and a successor exists, merge the successor in via :func:`infer_block_separator`."""
if not chunks:
return []
result: "list[str]" = []
i = 0
while i < len(chunks):
@@ -717,36 +459,19 @@ def merge_streaming_fences(chunks: "list[str]") -> "list[str]":
i += 1
result.append(current)
i += 1
return result
def balance_fences_across_chunks(chunks: "list[str]") -> "list[str]":
"""Close orphaned ``` fences at each chunk boundary and reopen on the next.
When a split lands inside a triple-backtick code block, close the fence
at the end of the head chunk and reopen it (with the original language
tag) at the start of the next, so every delivered chunk is
fence-balanced on its own.
"""
"""Close orphaned ``` fences at each chunk boundary and reopen (with the
original language tag) on the next, so every chunk is fence-balanced alone."""
if len(chunks) <= 1:
return chunks
out: "list[str]" = []
carry_lang = None
for chunk in chunks:
prefix = f"```{carry_lang}\n" if carry_lang is not None else ""
in_code = carry_lang is not None
lang = carry_lang or ""
for line in chunk.split("\n"):
stripped = line.strip()
if stripped.startswith("```"):
if in_code:
in_code = False
lang = ""
else:
in_code = True
tag = stripped[3:].strip()
lang = tag.split()[0] if tag else ""
in_code, lang = fence_state_after(chunk, carry_lang is not None, carry_lang or "")
body = prefix + chunk
if in_code:
body += "\n```"
@@ -757,12 +482,24 @@ def balance_fences_across_chunks(chunks: "list[str]") -> "list[str]":
return out
def greedy_pack_blocks(blocks, max_length, len_fn=None, sep="\n\n", overflow=None):
"""Greedily pack pre-split *blocks* into chunks of at most *max_length*.
def fence_state_after(text: str, in_code: bool = False, lang: str = "") -> "tuple[bool, str]":
"""Walk ``text`` line by line toggling on ``` lines; return the final (in_code, lang)."""
for line in text.split("\n"):
stripped = line.strip()
if stripped.startswith("```"):
if in_code:
in_code, lang = False, ""
else:
tag = stripped[3:].strip()
in_code, lang = True, (tag.split()[0] if tag else "")
return in_code, lang
Blocks are joined with *sep* while they fit. A block that alone exceeds
the limit is passed to *overflow(block)* (which must return a list of
chunks) when provided, else emitted as-is.
def greedy_pack_blocks(blocks, max_length, len_fn=None, sep="\n\n", overflow=None):
"""Greedily pack *blocks* (joined with *sep*) into chunks of at most *max_length*.
A block that alone exceeds the limit goes through *overflow(block)* (must
return a list of chunks) when provided, else is emitted as-is.
"""
_len = len_fn or len
packed: "list[str]" = []
@@ -795,36 +532,24 @@ def split_text_fence_aware(
prefer_paragraphs=True,
balance_fences=False,
):
"""Split markdown text into chunks of at most *limit*, respecting fences.
"""Split markdown into chunks of at most *limit*, respecting fences.
Two strategies, selected by ``prefer_paragraphs``:
``prefer_paragraphs=True`` (yuanbao-derived, the richest):
Extract atomic blocks (code fences, tables, paragraphs), greedily merge
them up to *limit*, split still-oversized non-atomic chunks at
paragraph boundaries, then re-merge small neighbours. Code blocks and
tables are never split in the middle; a single atom larger than
*limit* is emitted oversize rather than broken.
``prefer_paragraphs=False`` (stream_consumer-derived):
Newline-preferred hard splitting with headroom reserved for fence
markers when the text contains ```.
``balance_fences=True`` post-processes the chunks so a split inside a
code block closes the fence on the head chunk and reopens it (with the
language tag) on the tail — required by callers whose chunks are
delivered as independent messages that each must render standalone.
``prefer_paragraphs=True`` (yuanbao-derived): atoms (fences, tables,
paragraphs) are greedily merged, oversized non-atomic chunks split at
paragraph boundaries, small neighbours re-merged; a single atom larger than
*limit* is emitted oversize rather than broken.
``prefer_paragraphs=False`` (stream_consumer-derived): newline-preferred
hard splitting with headroom reserved for fence markers.
``balance_fences=True`` closes/reopens fences at chunk boundaries so each
chunk renders standalone.
"""
_len = len_fn or len
if not text:
return []
if prefer_paragraphs:
chunks = _chunk_markdown_paragraphs(text, limit, len_fn)
else:
chunks = _chunk_newline_preferred(text, limit, _len)
if balance_fences:
chunks = balance_fences_across_chunks(chunks)
return chunks
@@ -833,14 +558,10 @@ def split_text_fence_aware(
def _chunk_markdown_paragraphs(text, max_chars, len_fn=None):
"""Yuanbao-derived paragraph/atom chunking pipeline (see module docs)."""
_len = len_fn or len
if _len(text) <= max_chars:
return [text]
# Phase 1: Extract atomic blocks
atoms = split_markdown_atoms(text)
# Phase 2: Greedy merge
# Phase 2: greedy merge; oversized fence/table atoms stay indivisible.
chunks: "list[str]" = []
indivisible_set: "set[int]" = set()
current_parts: "list[str]" = []
@@ -849,58 +570,40 @@ def _chunk_markdown_paragraphs(text, max_chars, len_fn=None):
def _flush_parts() -> None:
if current_parts:
chunks.append('\n\n'.join(current_parts))
for atom in atoms:
atom_len = _len(atom)
sep_len = 2 if current_parts else 0
projected_len = current_len + sep_len + atom_len
if projected_len > max_chars and current_parts:
_flush_parts()
current_parts = []
current_len = 0
sep_len = 0
if (not current_parts
and atom_len > max_chars
and (is_fence_atom(atom) or is_table_atom(atom))):
indivisible_set.add(len(chunks))
chunks.append(atom)
continue
current_parts.append(atom)
current_len += sep_len + atom_len
_flush_parts()
# Phase 3: Split still-oversized chunks at paragraph boundaries
# Phase 3: split still-oversized divisible chunks at paragraph boundaries.
result: "list[str]" = []
for idx, chunk in enumerate(chunks):
if _len(chunk) <= max_chars:
if _len(chunk) <= max_chars or idx in indivisible_set or text_has_unclosed_fence(chunk):
result.append(chunk)
continue
if idx in indivisible_set:
result.append(chunk)
continue
if text_has_unclosed_fence(chunk):
result.append(chunk)
continue
remaining = chunk
while _len(remaining) > max_chars:
head, remaining = split_at_paragraph_boundary(
remaining, max_chars, len_fn=len_fn,
)
head, remaining = split_at_paragraph_boundary(remaining, max_chars, len_fn=len_fn)
if not head:
head, remaining = remaining[:max_chars], remaining[max_chars:]
if head:
result.append(head)
if remaining:
result.append(remaining)
# Phase 4: Merge small trailing/leading chunks with neighbours
# Phase 4: merge small chunks with neighbours.
if len(result) > 1:
merged: "list[str]" = [result[0]]
for chunk in result[1:]:
@@ -911,7 +614,6 @@ def _chunk_markdown_paragraphs(text, max_chars, len_fn=None):
else:
merged.append(chunk)
result = merged
return [c for c in result if c]
@@ -919,15 +621,11 @@ def _chunk_newline_preferred(text, limit, len_fn):
"""Stream-consumer-derived newline-preferred splitting (no balancing)."""
if len_fn(text) <= limit:
return [text]
# Reserve headroom for the close/reopen fence markers a balancing pass
# may add, so balanced chunks stay within the platform limit.
# Reserve headroom for fence markers a balancing pass may add.
split_limit = limit
if "```" in text:
split_limit = max(limit - 16, limit // 2, 1)
# Local import: gateway.platforms.base is heavyweight and pulls config;
# helpers must stay import-light for adapters that import it first.
from gateway.platforms.base import _custom_unit_to_cp
from gateway.platforms.base import _custom_unit_to_cp # heavyweight; lazy
chunks: "list[str]" = []
remaining = text
while len_fn(remaining) > split_limit:
-3
View File
@@ -177,7 +177,6 @@ def cache_media_bytes(
cache_document_from_bytes,
cache_image_from_bytes,
)
primary = _normalize_mime(mime)
kind = kind_hint
if kind is None:
@@ -187,14 +186,12 @@ def cache_media_bytes(
kind = "audio"
else:
kind = "document"
if kind == "image":
ext = ext_for_mime(primary, overrides=ext_overrides, fallback=".jpg") or ".jpg"
return cache_image_from_bytes(data, ext)
if kind == "audio":
ext = ext_for_mime(primary, overrides=ext_overrides, fallback=".ogg") or ".ogg"
return cache_audio_from_bytes(data, ext)
filename = filename_hint
if not filename:
ext = ext_for_mime(primary, overrides=ext_overrides, fallback=".bin")