From 2ecb5445515b7978848bd7eada2fc279014aa811 Mon Sep 17 00:00:00 2001 From: wansui Date: Wed, 26 Aug 2026 17:50:07 +0800 Subject: [PATCH] feat(wecom): native reply streaming (per-turn isolation, dedup-safe delivery, interaction boundaries) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement native reply streaming for the WeCom (企业微信) adapter over the long-connection "msgtype: stream" transport, so a reply renders as a single live-updating typing bubble instead of one final block. Aligns with the official wecom-openclaw-plugin streaming behavior. Includes the machinery intrinsic to native streaming on WeCom: - Transport: seed frame () opens the typing bubble, intermediate frames update it, a finalize frame closes it; native-streaming adapters are let past the edit-only gate. Fire-and-forget intermediate frames (WeCom long-connection mode has no documented edit-rate limit); an adapter-level frame cap is retained. (Early builds gated frames behind a char throttle; removed in favor of fire-and-forget + identity dedup.) - Per-turn isolation: each turn owns a unique turn_id; concurrent messages are isolated via (chat_id, turn_id)-keyed state. Dual-lane priority queue (control vs normal) plus a per-chat token bucket to stay under WeCom's rate limit (errcode 846607). - Dedup-safe delivery + ack-race handling: deliver-once contract (a frame is delivered the moment it is emitted; failures logged, not re-sent; delivery marked once per turn), per-req_id reply queue with ack tracking, and the timeout-inversion / orphan-queue race fixes. Robust fallback on 846608 / 846609 / errcode 6000 / passive-reply timeout via proactive send. - Interaction boundaries: finalize + reset before approval/clarify prompts so the prompt is the last thing on screen and never traps a lingering bubble; eager re-seed after a clarify answer so the typing bubble reappears instantly. - Stream-level keepalive: optional periodic finish=false frame + finalize-time stream-age guard to refresh WeCom's ~6-minute reply-stream window on long turns (mitigates 846604 / 846608). Off by default; tunable via config.yaml. - Tool-progress folded into the same native-stream bubble instead of separate messages; image+text double-callback merged into one turn. Tests cover the streaming lifecycle, per-turn isolation, duplicate-send / ack timing, approval + clarify boundaries, eager re-seed, and tool-progress. --- gateway/run.py | 158 +- gateway/slash_commands.py | 55 +- gateway/stream_consumer.py | 871 +++++++- plugins/platforms/wecom/adapter.py | 1926 ++++++++++++++++- tests/gateway/test_approval_boundary.py | 414 ++++ .../test_stream_consumer_tool_progress.py | 313 +++ .../test_stream_consumer_wecom_native.py | 1077 +++++++++ tests/gateway/test_wecom.py | 1095 ++++++++++ tests/gateway/test_wecom_double_send.py | 534 +++++ tests/gateway/test_wecom_per_turn.py | 403 ++++ 10 files changed, 6781 insertions(+), 65 deletions(-) create mode 100644 tests/gateway/test_approval_boundary.py create mode 100644 tests/gateway/test_stream_consumer_tool_progress.py create mode 100644 tests/gateway/test_stream_consumer_wecom_native.py create mode 100644 tests/gateway/test_wecom_double_send.py create mode 100644 tests/gateway/test_wecom_per_turn.py diff --git a/gateway/run.py b/gateway/run.py index 8a0b5473e9..07b8f37f3f 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -4568,6 +4568,13 @@ class TurnRunner: # code (same boilerplate imports → identical previews). if msg == ctx.last_progress_msg[0]: ctx.repeat_count[0] += 1 + # Native-stream-progress routing: dedup updates the last line + # in the overlay rather than sending a queue signal. + _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None + if _sc is not None and getattr(_sc, "accepts_tool_progress", False): + # Replace the last progress line with the dedup version + _sc.on_tool_progress(f"{msg} (×{ctx.repeat_count[0] + 1})") + return # Update the last line in progress_lines with a counter # via a special "dedup" queue message. ctx.progress_queue.put(("__dedup__", msg, ctx.repeat_count[0])) @@ -4575,6 +4582,14 @@ class TurnRunner: ctx.last_progress_msg[0] = msg ctx.repeat_count[0] = 0 + # Native-stream-progress routing: if the stream consumer is active + # and using native streaming, inject progress directly into the + # stream bubble instead of the separate progress queue. + _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None + if _sc is not None and getattr(_sc, "accepts_tool_progress", False): + _sc.on_tool_progress(msg) + return + ctx.progress_queue.put(msg) async def _send_native_task_card_progress(self, adapter) -> None: @@ -5947,6 +5962,51 @@ class TurnRunner: _mem_notif = "on" if _mem_notif else "off" agent.memory_notifications = str(_mem_notif).lower() if _mem_notif else "on" + # ------------------------------------------------------------------ + # Shared native-stream boundary close. For platforms with native + # streaming (e.g. WeCom msgtype:"stream"), an interaction that + # interrupts the stream — a dangerous-command approval prompt OR a + # clarify decision prompt — must finalize the current stream and + # disable native streaming first. Otherwise the agent's + # post-interaction output keeps flowing into send_stream_frame and + # updates the *old* bubble that preceded the prompt, instead of + # starting a fresh bubble below it (the "气泡割裂" symptom). After + # the boundary, post-prompt output goes through the reliable send() + # path as a new message. Runs on the agent thread; the consumer + # processes the boundary serially via its queue. + def _close_native_stream_boundary( + _reason: str, _placeholder: str | None = None, _reopen: bool = False, + ) -> bool: + _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None + if not (_sc and getattr(_sc, "_use_native_streaming", False)): + return True + _cancelled_flag = None + try: + _boundary_result = _sc.close_for_approval_prompt( + _placeholder, reason=_reason, reopen=_reopen, + ) + # Returns (future, cancelled_flag) or just a future. + if isinstance(_boundary_result, tuple): + _boundary_future, _cancelled_flag = _boundary_result + else: + _boundary_future = _boundary_result + if hasattr(_boundary_future, "result"): + _ok = _boundary_future.result(timeout=10) + if not _ok: + logger.warning( + "%s boundary failed to close stream properly — " + "prompt may still appear in typing bubble", _reason, + ) + return bool(_ok) + return True + except (TimeoutError, Exception) as _boundary_err: + if _cancelled_flag is not None: + _cancelled_flag["cancelled"] = True + logger.warning( + "%s boundary timed out or failed: %s", _reason, _boundary_err, + ) + return False + # ------------------------------------------------------------------ # Clarify callback: present a clarify prompt and block on a response. # @@ -5974,6 +6034,21 @@ class TurnRunner: multi_select=bool(multi_select), ) + # For WeCom native streaming: finalize the current stream before + # showing the clarify prompt so the post-answer output opens a + # fresh bubble below the question instead of updating the bubble + # that preceded it — the "气泡割裂" symptom. Unlike the approval + # path, clarify passes reopen=True: native streaming stays + # enabled so the post-answer continuation re-opens a fresh + # native stream (typing bubble) rather than degrading to a + # one-shot send(). Clarify waits are short, so stream staleness + # is a low risk; if the re-seed fails the consumer degrades to + # send() automatically. The placeholder is only used in the + # narrow case where a frame was pushed but no text accumulated. + _close_native_stream_boundary( + "Clarify", "💬 等待你的选择...", _reopen=True, + ) + # Pause typing — like approval, we don't want a "thinking..." # status to obscure the prompt or block the user from typing # an "Other" response on platforms that disable input while @@ -6019,12 +6094,45 @@ class TurnRunner: # AMBIGUOUS — the card may have posted with a late ack. Only a # definitive failure tears down the registration; ambiguous # falls through to the bounded wait so a late reply resolves. - return _clarify_send_then_wait( + _clarify_response = _clarify_send_then_wait( fut, clarify_id=clarify_id, session_key=ctx.session_key or "", clarify_mod=_clarify_mod, ) + # Only re-arm typing when the user actually answered — the + # undeliverable sentinel and the timeout/cancellation strings + # start with '[' and must pass through untouched. + if not ( + isinstance(_clarify_response, str) + and _clarify_response.startswith("[") + ): + # User answered. Reopen the typing indicator IMMEDIATELY — + # don't wait for the LLM's first post-answer token. On native + # streaming (WeCom) the typing bubble is driven by the stream + # seed frame, and the reopen path otherwise re-seeds lazily on + # the first delta (measured ~48s of dead air). request_reopen_seed + # is a no-op unless we're in the reopen-pending native state, so + # it's safe to call unconditionally here. resume_typing_for_chat + # covers non-native platforms (their pause was set before the + # prompt) — the WeCom send_typing is a no-op, harmless here. + _sc_reopen = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None + if _sc_reopen is not None: + try: + _sc_reopen.request_reopen_seed() + except Exception: + logger.debug( + "request_reopen_seed after clarify answer failed", + exc_info=True, + ) + try: + ctx._status_adapter.resume_typing_for_chat(ctx._status_chat_id) + except Exception: + logger.debug( + "resume_typing_for_chat after clarify answer failed", + exc_info=True, + ) + return _clarify_response agent.clarify_callback = _clarify_callback_sync @@ -6127,6 +6235,12 @@ class TurnRunner: # Typing resumes in _handle_approve_command/_handle_deny_command. ctx._status_adapter.pause_typing_for_chat(ctx._status_chat_id) + # For WeCom native streaming: signal the stream consumer to close + # the current stream before showing the approval prompt. + # This goes through the consumer's queue for serial processing, + # avoiding race conditions with pending deltas. + _close_native_stream_boundary("Approval") + cmd = approval_data.get("command", "") desc = approval_data.get("description", "dangerous command") @@ -6200,11 +6314,15 @@ class TurnRunner: smart_denied=approval_data.get("smart_denied", False), ) try: + # Mark as approval prompt so WeCom routes through control lane + _approval_metadata = dict(ctx._status_thread_metadata or {}) + _approval_metadata["is_approval_prompt"] = True + _approval_send_fut = safe_schedule_threadsafe( ctx._status_adapter.send( ctx._status_chat_id, msg, - metadata=_interim_metadata(ctx._status_thread_metadata), + metadata=_interim_metadata(_approval_metadata), ), ctx._loop_for_step, logger=logger, @@ -22768,6 +22886,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew asked to deliver (#20834). Only ``MEDIA:`` directives — the explicit attachment contract — trigger post-stream uploads. """ + logger.info("[DEBUG] _deliver_media_from_response called: response=%r chat=%s", response[:100], event.source.chat_id) from pathlib import Path from urllib.parse import quote as _quote @@ -24342,7 +24461,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _APPROVAL_TIMEOUT_SECONDS = 300 # 5 minutes - # Built-in messaging platforms where the ``/update`` command is allowed. # ACP, API server, and webhooks are programmatic interfaces that should # not trigger system updates. Plugin-migrated platforms (discord, @@ -28131,7 +28249,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # (The proxy path instead opts into a cursorless fallback # via on_missing_cursor="fallback".) _adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True) - if not _adapter_supports_edit and on_missing_cursor == "raise": + # Adapters that can't edit messages but provide a native-streaming + # transport (e.g. WeCom's msgtype: "stream" via send_stream_frame) + # get past the gate — the consumer's native branch delivers the full + # turn through that transport. + _adapter_supports_native_stream = bool(getattr( + adapter, "SUPPORTS_NATIVE_STREAMING", False, + )) + if ( + not _adapter_supports_edit + and not _adapter_supports_native_stream + and on_missing_cursor == "raise" + ): raise RuntimeError("skip streaming for non-editable platform") _effective_cursor = scfg.cursor if _adapter_supports_edit else "" # Some Matrix clients render the streaming cursor @@ -30405,6 +30534,27 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew "Failed to edit streamed message for session %s: %s", session_key or "?", _edit_err, ) + elif _sc is not None and not _is_empty_sentinel: + # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed for this + # turn but suppression did NOT fire, so the gateway's normal + # final-send is about to run. On WeCom this is the exact window + # that produced "回复了两条" — a final-frame ack still in flight + # (final_content_delivered not yet set) while this send races + # ahead. Log the decision inputs so a recurrence can be pinned to + # "signal never set" vs "ack-pending race". + # See docs/rca-wecom-stream-final-ack-timeout-duplicate.md. + logger.warning( + "Normal final-send NOT suppressed despite active stream " + "consumer for session %s: streamed=%s previewed=%s " + "content_delivered=%s transformed=%s final_len=%d — " + "possible duplicate send (see wecom ack-timeout RCA).", + session_key or "?", + _streamed, + _previewed, + _content_delivered, + _transformed, + len(_final), + ) # Schedule deletion of tracked temporary progress bubbles after the # final response lands. Failed runs skip this so bubbles remain as diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index 001d6b7469..99a887f6c7 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -5885,7 +5885,27 @@ class GatewaySlashCommandsMixin: logger.info("User approved %d dangerous command(s) via /approve (%s)", count, choice) plural = "plural" if count > 1 else "singular" - return t(f"gateway.approve.{choice}_{plural}", count=count) + confirmation_text = t(f"gateway.approve.{choice}_{plural}", count=count) + if _adapter: + try: + await _adapter.send( + source.chat_id, + confirmation_text, + reply_to=event.message_id, + metadata={ + "is_approval_prompt": True, + "force_proactive_send": True, + }, + ) + except Exception as exc: + logger.warning( + "Failed to send /approve confirmation to %s: %s", + source.chat_id, + exc, + exc_info=True, + ) + + return None async def _handle_deny_command(self, event: MessageEvent) -> str: """Handle /deny command — reject pending dangerous command(s). @@ -5943,11 +5963,34 @@ class GatewaySlashCommandsMixin: ) if reason: if count > 1: - return t("gateway.deny.denied_reason_plural", count=count, reason=reason) - return t("gateway.deny.denied_reason_singular", reason=reason) - if count > 1: - return t("gateway.deny.denied_plural", count=count) - return t("gateway.deny.denied_singular") + confirmation_text = t("gateway.deny.denied_reason_plural", count=count, reason=reason) + else: + confirmation_text = t("gateway.deny.denied_reason_singular", reason=reason) + elif count > 1: + confirmation_text = t("gateway.deny.denied_plural", count=count) + else: + confirmation_text = t("gateway.deny.denied_singular") + + if _adapter: + try: + await _adapter.send( + source.chat_id, + confirmation_text, + reply_to=event.message_id, + metadata={ + "is_approval_prompt": True, + "force_proactive_send": True, + }, + ) + except Exception as exc: + logger.warning( + "Failed to send /deny confirmation to %s: %s", + source.chat_id, + exc, + exc_info=True, + ) + + return None async def _handle_debug_command(self, event: MessageEvent) -> str: """Handle /debug — upload debug report (summary only) and return paste URLs. diff --git a/gateway/stream_consumer.py b/gateway/stream_consumer.py index cb5848faa6..8811db6436 100644 --- a/gateway/stream_consumer.py +++ b/gateway/stream_consumer.py @@ -16,6 +16,7 @@ Credit: jobless0x (#774, #1312), OutThisLife (#798), clicksingh (#697). from __future__ import annotations import asyncio +import concurrent.futures import inspect import logging import queue @@ -44,6 +45,9 @@ logger = logging.getLogger("gateway.stream_consumer") _DONE = object() _NEW_SEGMENT = object() _COMMENTARY = object() +# Sentinel for tool-progress lines injected into the native stream bubble. +# Enqueued as ``(_TOOL_PROGRESS, line_text)`` by ``on_tool_progress()``. +_TOOL_PROGRESS = object() # Authoritative turn-final payload, enqueued by ``finish(final_text=...)`` # just before ``_DONE``. Carries the completed ``final_response`` — # including post-stream augmentation (file-mutation verifier footer, @@ -61,6 +65,25 @@ _FINAL_TEXT = object() # last thing on screen, not racing ahead of buffered prose. _FLUSH = object() +# Sentinel to signal an interaction boundary (approval prompt OR clarify +# decision prompt) — finalize the current stream, disable native streaming, +# and let post-interaction output go via send(). +_APPROVAL_BOUNDARY = object() + +# Sentinel to request an EAGER native re-seed after a clarify-reopen boundary. +# Posted the moment the user answers a clarify (before the LLM produces any +# post-answer delta), so the WeCom typing bubble reappears immediately instead +# of waiting for the first token. On WeCom, typing is driven by the stream +# seed frame (send_typing is a no-op), and the reopen path otherwise re-seeds +# lazily on the first delta — measured 48s of dead air in one turn. Handled +# serially in run(); see request_reopen_seed() and the run-loop handler. +_REOPEN_SEED = object() + +# Default finalize text shown at an interaction boundary when no content has +# accumulated yet. Callers may override per-boundary (e.g. clarify passes its +# own) via close_for_approval_prompt(placeholder=...). +_DEFAULT_BOUNDARY_PLACEHOLDER = "⏸ 等待审批中..." + def escape_code_fences_for_display(text: str) -> str: """Escape triple-backtick markers so text can be safely wrapped @@ -238,6 +261,14 @@ class GatewayStreamConsumer: # final rich-text edit (Telegram MarkdownV2 finalize, etc.). self._on_before_finalize = on_before_finalize self._initial_reply_to_id = initial_reply_to_id + + # Per-turn identifier: uniquely identifies this consumer's stream turn. + # Passed to adapter.send_stream_frame() to prevent concurrent consumers + # from interfering with each other (e.g., /background, parallel subagents). + # Mirrors official wecom-openclaw-plugin's per-message streamId generation. + import uuid + self._turn_id = str(uuid.uuid4()) + self._queue: queue.Queue = queue.Queue() self._accumulated = "" # Full segment text mirror of ``_accumulated`` that is NOT truncated @@ -341,6 +372,59 @@ class GatewayStreamConsumer: # this response and route through edit-based for graceful degradation. self._draft_failures = 0 self._before_finalize_notified = False + # Native streaming transport (e.g. WeCom msgtype: "stream"). Unlike + # drafts, native streaming is the *only* delivery channel for the + # turn — first frame, mid-stream updates, and the final answer all + # flow through ``adapter.send_stream_frame()`` and the adapter + # manages the stream lifecycle (init → cumulative updates → + # finish=true). Resolved at the start of run() and disabled on + # any failure so the consumer falls back to edit/send. + self._use_native_streaming = False + # Tracks whether the native stream bubble has been opened (seed frame sent). + # Used in fallback logic to decide if we need to finalize the stream before + # falling back to send(). Set to True after seed frame succeeds, even though + # seed has zero visible content. + self._native_stream_opened = False + # Number of visible characters last successfully pushed to the + # native stream. Used for "send only when enough new content has + # accumulated" throttling so we don't spam frames at WeCom's + # 30 frames/min rate ceiling. + self._native_last_pushed_len = 0 + # Finalize text used at an interaction boundary (approval/clarify) when + # no content has accumulated yet. Set by close_for_approval_prompt(); + # defaults to the approval wording for backward compatibility. + self._boundary_placeholder = _DEFAULT_BOUNDARY_PLACEHOLDER + # Human-readable label for the current interaction boundary, used only + # for log prefixes so a clarify boundary doesn't log as "Approval". + # Set by close_for_approval_prompt(); race-free because boundaries are + # processed serially. + self._boundary_reason = "Approval" + # When True, the interaction boundary finalizes the current stream but + # KEEPS native streaming enabled so post-prompt output re-opens a fresh + # native stream (via the lazy re-seed in _send_or_edit) instead of + # degrading to a one-shot send(). Clarify sets this (short waits, low + # stream-staleness risk); approval leaves it False (long, unbounded + # waits — the stream may go stale, so send() is safer). Set by + # close_for_approval_prompt(); race-free (boundaries are serial). + self._boundary_reopen = False + # Marks that a boundary asked to reopen the native stream but no + # post-prompt content has re-seeded it yet. Guards got_done from + # re-seeding a fresh stream just to emit a lone "✅" placeholder when + # the agent produced nothing after the prompt. + self._awaiting_reopen_after_boundary = False + # Marks that an EAGER re-seed (via _REOPEN_SEED) already opened a fresh + # native stream after a clarify answer, BEFORE any post-answer content. + # Unlike the lazy path, the typing bubble is already on screen, so + # got_done must actively finalize it (not silently skip) when the agent + # produces no content — otherwise a blank typing bubble hangs forever. + self._reopen_seeded_eagerly = False + + # Tool-progress overlay state (native streaming only). + # Lines are injected via on_tool_progress() and displayed as a + # temporary overlay in the stream bubble until real text arrives. + self._tool_progress_lines: list[str] = [] + self._tool_progress_active: bool = False + def _stream_is_message(self) -> bool: """Whether THIS chat's transport treats the stream as the message. @@ -360,6 +444,44 @@ class GatewayStreamConsumer: return False return getattr(self.adapter, "draft_stream_is_message", False) is True + @property + def accepts_tool_progress(self) -> bool: + """Whether this consumer can absorb tool progress into its stream. + + True only when native streaming is resolved and active. Callers use + this to decide the progress routing path (in-stream vs progress_queue). + """ + return self._use_native_streaming + + def on_tool_progress(self, line: str) -> None: + """Inject a tool-progress status line into the native stream bubble. + + Thread-safe (called from agent worker thread via queue.Queue). Only + meaningful when native streaming is active — callers should gate on + ``accepts_tool_progress``. + + The line is displayed as an overlay until the next text delta arrives, + at which point real content overwrites the tool-progress lines. + """ + if line: + self._queue.put((_TOOL_PROGRESS, line)) + + def _compose_frame_content(self) -> str: + """Compose the current frame content for native streaming. + + Strategy B: when both accumulated text and tool-progress lines exist, + append tool lines below the text separated by a horizontal rule. + On finalize, only accumulated text is sent (no tool lines). + """ + if self._accumulated and self._tool_progress_lines: + # Text + active tool status at the bottom + return self._accumulated + "\n\n---\n" + "\n".join(self._tool_progress_lines) + elif self._accumulated: + return self._accumulated + elif self._tool_progress_lines: + return "\n".join(self._tool_progress_lines) + return "" + def _metadata_for_send( self, *, @@ -454,6 +576,11 @@ class GatewayStreamConsumer: """Append to the live buffer and the split-stable stream ledger.""" if not text: return + # New text delta arriving: clear tool-progress overlay so the next + # frame shows real content (Strategy B: text overwrites tool lines). + if self._tool_progress_lines: + self._tool_progress_lines.clear() + self._tool_progress_active = False self._accumulated += text self._stream_ledger += text @@ -554,6 +681,83 @@ class GatewayStreamConsumer: """Finalize the current stream segment and start a fresh message.""" self._queue.put(_NEW_SEGMENT) + def close_for_approval_prompt( + self, + placeholder: str | None = None, + reason: str = "Approval", + reopen: bool = False, + ) -> asyncio.Future: + """Signal an interaction boundary — finalize stream, then either disable + native (approval) or keep it for a fresh re-opened stream (clarify). + + Used for any mid-stream interaction that must not keep updating the + current native-stream bubble: a dangerous-command approval prompt or a + clarify decision prompt. Queues a boundary signal that the consumer + processes serially: finalize the current stream with accumulated text + (creating a stable message for pre-prompt content), then handle + post-prompt output per ``reopen``. + + ``placeholder`` is the finalize text used only when there is no + accumulated content yet (the prompt fired as the agent's first action). + Defaults to the approval placeholder; clarify passes its own so the + finalized bubble doesn't read "waiting for approval" for a question. + + ``reason`` is a human-readable label ("Approval"/"Clarify") used only + for the boundary handler's log prefixes so a clarify boundary doesn't + surface as an "Approval boundary" failure during troubleshooting. + + ``reopen`` controls post-prompt delivery. False (approval): disable + native streaming and buffer post-prompt output into a single reliable + send() — approval waits are long and unbounded, so the stream may go + stale. True (clarify): keep native streaming enabled so post-prompt + output re-opens a fresh native stream via the existing lazy re-seed, + restoring the typing-bubble experience; if the re-seed later fails the + consumer degrades to send() automatically. + + Returns a (Future, cancelled_flag) tuple. The Future resolves True + when the boundary has been processed. cancelled_flag is included + for backward compatibility with callers that set it on timeout; + the boundary handler no longer reads it (finalize always runs). + + For platforms without native streaming this is a no-op (returns + an immediately-resolved Future). + + Called from sync context (agent/approval thread). The boundary + is processed by the consumer's async run() task, ensuring no + race conditions with pending deltas or other queue items. + """ + loop = None + try: + loop = asyncio.get_running_loop() + except RuntimeError: + pass + + if not self._use_native_streaming: + # No native stream to close — return resolved future + f = asyncio.Future() if loop else concurrent.futures.Future() + f.set_result(True) + return f + + # Stash the empty-content placeholder, log label, and reopen mode for + # the serial boundary handler. Boundaries are processed one at a time, + # so instance attributes are race-free and keep the queue signal shape + # unchanged. + self._boundary_placeholder = placeholder or _DEFAULT_BOUNDARY_PLACEHOLDER + self._boundary_reason = reason or "Approval" + self._boundary_reopen = bool(reopen) + + # Create a future that run() will resolve after processing. + # cancelled_flag is retained for backward compatibility with callers + # (run.py sets it on timeout) but the handler always finalizes regardless. + if loop: + boundary_future = loop.create_future() + else: + boundary_future = concurrent.futures.Future() + + cancelled_flag = {"cancelled": False} + self._queue.put((_APPROVAL_BOUNDARY, boundary_future, cancelled_flag)) + return boundary_future, cancelled_flag + def on_commentary(self, text: str) -> None: """Queue a completed interim assistant commentary message.""" if text: @@ -582,6 +786,27 @@ class GatewayStreamConsumer: return False return evt.wait(timeout=max(0.0, float(timeout))) + def request_reopen_seed(self) -> None: + """Request an EAGER native re-seed after a clarify-reopen boundary. + + Called (thread-safe, like on_commentary / close_for_approval_prompt) + the instant the user answers a clarify — BEFORE the LLM emits any + post-answer delta. Posts _REOPEN_SEED so run() immediately sends an + empty seed frame, which is what makes the WeCom typing bubble reappear + without waiting for the first token (measured 48s of dead air otherwise). + + No-op unless we're in the reopen-pending state on a native stream: only + after a clarify boundary (`_awaiting_reopen_after_boundary`) with native + still enabled and no stream currently open. This keeps a stray call from + opening a spurious bubble mid-stream or on the approval path. + """ + if ( + self._use_native_streaming + and self._awaiting_reopen_after_boundary + and not self._native_stream_opened + ): + self._queue.put(_REOPEN_SEED) + def _notify_new_message(self) -> None: """Fire the on_new_message callback, swallowing any errors.""" cb = self._on_new_message @@ -628,6 +853,10 @@ class GatewayStreamConsumer: self._fallback_prefix = "" self._fallback_preserve_partial_messages = False self._segment_preview_message_ids = set() + # Tool-progress overlay: clear on segment reset so a new segment + # starts clean. + self._tool_progress_lines = [] + self._tool_progress_active = False # #29346: a tool/segment boundary means what we delivered was an interim # preamble, not the final answer — clear the flags so a premature setter # can't fool the gateway. Safe: got_done returns before any reset, and @@ -657,6 +886,143 @@ class GatewayStreamConsumer: type(self)._draft_id_counter += 1 self._draft_id = type(self)._draft_id_counter + async def _handle_approval_boundary(self, boundary_future, cancelled_flag=None) -> None: + """Process an approval boundary: finalize stream, disable native for post-approval. + + This method is called serially from run() when _APPROVAL_BOUNDARY is dequeued. + + Strategy: finalize the current stream with accumulated text (creating a + stable message for pre-approval content), then disable native streaming + so post-approval output goes through the reliable send() path. + + Why not keep the stream open across approval: + - WeCom stream finalize ack only confirms server receipt, not client render. + - Approval waits introduce an idle gap where the stream may become stale + on the client side (no server-side 846608, but client stops tracking it). + - If content_delivered=True but the client didn't render, the normal + final send is suppressed → user sees nothing. + - Approval is a natural interaction boundary; "pre-approval preamble" + + "post-approval result" as two messages is acceptable UX. + + Post-approval output uses regular send() which is unconditionally reliable. + """ + # Log label ("Approval"/"Clarify") so a clarify boundary failure doesn't + # surface as an "Approval boundary" error during troubleshooting. + _reason = getattr(self, "_boundary_reason", "Approval") or "Approval" + delivery_failed = False + try: + if self._native_stream_opened: + # Finalize current stream with accumulated content. + # This converts the typing bubble into a stable message. + finalize_text = self._accumulated or self._boundary_placeholder + finalize_ok = False + try: + result = await self.adapter.send_stream_frame( + finalize_text, + finalize=True, + chat_id=self.chat_id, + reply_to=self._initial_reply_to_id, + turn_id=self._turn_id, + ) + finalize_ok = bool(result) + except Exception as e: + logger.warning("%s boundary: finalize failed: %s", _reason, e) + + if not finalize_ok: + # Stream finalize didn't land — the typing bubble may still + # be showing partial content. Fallback: deliver the pre-prompt + # text via reliable send() so the user at least sees it. + logger.warning( + "%s boundary: finalize not confirmed, " + "falling back to send() for pre-prompt text (chat=%s)", + _reason, self.chat_id, + ) + fallback_ok = False + try: + send_result = await self.adapter.send( + self.chat_id, finalize_text, + ) + fallback_ok = getattr(send_result, "success", False) + except Exception as send_err: + logger.warning( + "%s boundary: fallback send also failed: %s", + _reason, send_err, + ) + if not fallback_ok: + # Both finalize and fallback failed — pre-prompt text + # may be lost. Mark boundary as failed so the caller knows. + logger.error( + "%s boundary: both finalize and fallback send failed " + "(chat=%s) — pre-prompt text may not have been delivered", + _reason, self.chat_id, + ) + delivery_failed = True + else: + logger.debug( + "%s boundary: finalized stream (chat=%s, turn=%s)", + _reason, self.chat_id, self._turn_id, + ) + + if self._boundary_reopen: + # Clarify boundary: KEEP native streaming enabled. The current + # stream was finalized above (pre-prompt content is now a stable + # bubble); marking it closed makes the next post-prompt delta + # re-open a fresh native stream via the lazy re-seed in + # _send_or_edit, restoring the typing-bubble experience. Do NOT + # set buffer_only — post-prompt output should stream, not batch. + # If the re-seed later fails, the consumer degrades to send() + # on its own. _awaiting_reopen_after_boundary guards got_done + # from re-seeding a stream just to emit a lone "✅" when the + # agent produced no post-prompt content. + self._native_stream_opened = False + self._native_last_pushed_len = 0 + self._awaiting_reopen_after_boundary = True + self._reset_segment_state() + # INFO (temporary latency probe): boundary finalize is done and + # the old bubble is closed. From here the consumer waits for + # the LLM's first post-answer delta before re-seeding the C + # bubble — so the gap between THIS line and the + # "Re-opened native stream" INFO below is exactly the + # "typing slow to reappear after clarify" delay. + logger.info( + "[latency] Clarify boundary finalized, awaiting first " + "post-answer delta to re-seed (chat=%s, turn=%s)", + self.chat_id, self._turn_id, + ) + else: + # Approval boundary: disable native streaming for post-approval + # output, which goes through regular send() (unconditionally + # reliable, no client-side stream state dependency). Set + # buffer_only=True so the consumer accumulates all post-approval + # text and delivers it in one shot on got_done, avoiding + # mid-stream flushes that would create multiple messages on + # non-editable platforms like WeCom. + self._use_native_streaming = False + self._native_stream_opened = False + self._native_last_pushed_len = 0 + self.cfg.buffer_only = True + + # Reset segment state so post-approval output starts fresh via send(). + self._reset_segment_state() + + boundary_ok = not delivery_failed + + except Exception as e: + logger.warning("%s boundary processing failed: %s", _reason, e) + boundary_ok = False + finally: + # Resolve future so approval callback knows the result + if boundary_future is not None: + try: + if isinstance(boundary_future, asyncio.Future): + if not boundary_future.done(): + boundary_future.set_result(boundary_ok) + elif isinstance(boundary_future, concurrent.futures.Future): + if not boundary_future.done(): + boundary_future.set_result(boundary_ok) + except Exception: + pass + def on_delta(self, text: str) -> None: """Thread-safe callback — called from the agent's worker thread. @@ -860,19 +1226,53 @@ class GatewayStreamConsumer: _raw_limit = self._raw_message_limit() _safe_limit = max(500, _raw_limit - _len_fn(self.cfg.cursor) - 100) - # Resolve native draft streaming once per run. When enabled the - # consumer routes mid-stream frames through adapter.send_draft and - # leaves _message_id=None so the existing got_done path delivers the - # final answer as a regular sendMessage (drafts have no message_id - # to edit). - self._use_draft_streaming = self._resolve_draft_streaming() - if self._use_draft_streaming: - type(self)._draft_id_counter += 1 - self._draft_id = type(self)._draft_id_counter + # Resolve transport once per run. Native streaming wins over draft + # because the only adapters that declare it (WeCom) cannot edit + # messages at all — there is no edit path to fall back to mid-turn. + # When native is selected we send an empty seed frame immediately so + # the user sees the platform's "typing" indicator before the LLM + # produces any tokens; if that seed fails (no req_id, transport + # error) we disable native and let the consumer take the regular + # edit path (which will in turn refuse and fall back to fallback + # send via the gateway, since SUPPORTS_MESSAGE_EDITING=False). + self._use_native_streaming = self._resolve_native_streaming() + if self._use_native_streaming: logger.debug( - "Stream consumer using native-draft transport (chat=%s draft_id=%s)", - self.chat_id, self._draft_id, + "Stream consumer using native-stream transport (chat=%s)", + self.chat_id, ) + try: + seed_ok = await self.adapter.send_stream_frame( + "", + chat_id=self.chat_id, + reply_to=self._initial_reply_to_id, + turn_id=self._turn_id, + ) + if seed_ok: + # Mark stream as opened so fallback knows to finalize + self._native_stream_opened = True + except Exception: + logger.debug( + "Native streaming seed frame raised; disabling native", + exc_info=True, + ) + seed_ok = False + if not seed_ok: + self._use_native_streaming = False + + # Resolve native draft streaming (Telegram drafts) only when native + # streaming is not in use — they target the same first-frame slot. + if self._use_native_streaming: + self._use_draft_streaming = False + else: + self._use_draft_streaming = self._resolve_draft_streaming() + if self._use_draft_streaming: + type(self)._draft_id_counter += 1 + self._draft_id = type(self)._draft_id_counter + logger.debug( + "Stream consumer using native-draft transport (chat=%s draft_id=%s)", + self.chat_id, self._draft_id, + ) try: while True: @@ -888,6 +1288,10 @@ class GatewayStreamConsumer: got_segment_break = False got_flush = False flush_event = None + got_approval_boundary = False + got_reopen_seed = False + approval_boundary_future = None + approval_boundary_cancelled = None commentary_text = None while True: try: @@ -950,6 +1354,14 @@ class GatewayStreamConsumer: self._accumulated += _suffix self._stream_ledger = _final_raw continue + if item is _REOPEN_SEED: + got_reopen_seed = True + break + if isinstance(item, tuple) and len(item) == 3 and item[0] is _APPROVAL_BOUNDARY: + got_approval_boundary = True + approval_boundary_future = item[1] + approval_boundary_cancelled = item[2] + break if isinstance(item, tuple) and len(item) == 2 and item[0] is _COMMENTARY: commentary_text = item[1] break @@ -961,10 +1373,83 @@ class GatewayStreamConsumer: got_segment_break = True flush_event = item[1] break + if isinstance(item, tuple) and len(item) == 2 and item[0] is _TOOL_PROGRESS: + # Tool-progress overlay: accumulate the status line. + # Only effective in native-streaming mode (callers + # gate before enqueue via accepts_tool_progress). + if self._use_native_streaming: + self._tool_progress_lines.append(item[1]) + self._tool_progress_active = True + continue # continue draining to batch simultaneous progress lines self._filter_and_accumulate(item) except queue.Empty: break + # Handle approval boundary: close current stream, reset for new turn. + # Must happen before got_done/segment_break processing since it + # produces its own finalize and resets state. + if got_approval_boundary: + await self._handle_approval_boundary( + approval_boundary_future, approval_boundary_cancelled + ) + continue + + # Handle eager re-seed: the user just answered a clarify prompt. + # Open a fresh native stream NOW (empty seed frame) so the WeCom + # typing bubble reappears immediately, without waiting for the + # LLM's first post-answer delta. Only meaningful in the + # reopen-pending state with native still live and no stream open; + # request_reopen_seed() already gates on that, and we re-check + # here because state may have advanced between put and dequeue. + # + # TRADE-OFF: this moves the start of WeCom's ~6-minute stream + # session limit (STREAM_EXPIRED_ERRCODE 846608, counted from the + # FIRST frame, not renewed by intermediate frames) forward from + # the first post-answer delta to the user-reply instant — the + # effective window shrinks by however long the LLM takes to + # produce its first token. A first token >5min is very rare, and + # if the stream does expire send_stream_frame returns False and + # the else branch below degrades to send(), so the answer still + # lands (only the streaming animation is lost). Acceptable. + if got_reopen_seed: + if ( + self._use_native_streaming + and self._awaiting_reopen_after_boundary + and not self._native_stream_opened + ): + try: + seed_ok = await self.adapter.send_stream_frame( + "", + chat_id=self.chat_id, + reply_to=self._initial_reply_to_id, + turn_id=self._turn_id, + ) + except Exception as e: + logger.debug( + "Eager reopen seed raised, disabling native: %s", e, + ) + seed_ok = False + if seed_ok: + self._native_stream_opened = True + self._native_last_pushed_len = 0 + self._awaiting_reopen_after_boundary = False + self._reopen_seeded_eagerly = True + logger.info( + "[latency] Eager re-seed after clarify answer " + "(typing bubble reopened immediately, turn=%s)", + self._turn_id, + ) + else: + # Seed failed — degrade to a single buffered send() + # so the post-answer content still lands as one + # bubble (not per-tick fragments on a non-editable + # platform). Mirrors the approval-boundary degrade. + self._use_native_streaming = False + self._native_stream_opened = False + self._native_last_pushed_len = 0 + self.cfg.buffer_only = True + continue + # Flush any held-back partial-tag buffer on stream end # so trailing text that was waiting for a potential open # tag is not lost. @@ -996,15 +1481,22 @@ class GatewayStreamConsumer: or commentary_text is not None ) if not self.cfg.buffer_only: - should_edit = should_edit or ( - (elapsed >= self._current_edit_interval - and self._accumulated) - # buffer_threshold is intentionally codepoint-based: - # it's a debounce heuristic ("send updates roughly - # every N visible characters"), not a platform-limit - # check. _len_fn is reserved for overflow detection. - or len(self._accumulated) >= self.cfg.buffer_threshold - ) + if self._use_native_streaming: + # Fire-and-forget: native streaming has no platform + # edit-rate limit — push every accumulated delta + # immediately. The only gate is "have we accumulated + # anything new at all". + should_edit = should_edit or bool(self._accumulated) or self._tool_progress_active + else: + should_edit = should_edit or ( + (elapsed >= self._current_edit_interval + and self._accumulated) + # buffer_threshold is intentionally codepoint-based: + # it's a debounce heuristic ("send updates roughly + # every N visible characters"), not a platform-limit + # check. _len_fn is reserved for overflow detection. + or len(self._accumulated) >= self.cfg.buffer_threshold + ) current_update_visible = False # Whether the got_done update below was delivered as a FRESH @@ -1033,11 +1525,16 @@ class GatewayStreamConsumer: ) ): should_edit = False - if should_edit and self._accumulated: + if should_edit and (self._accumulated or (self._use_native_streaming and self._tool_progress_active)): # Split overflow: if accumulated text exceeds the platform # limit, split into properly sized chunks. + # Native streaming bypasses this entirely — the adapter's + # send_stream_frame handles byte-level truncation against + # the stream protocol's larger limit (e.g. WeCom's 20480 + # bytes vs. MAX_MESSAGE_LENGTH's 4000 codepoints). if ( - _len_fn(self._accumulated) > _safe_limit + not self._use_native_streaming + and _len_fn(self._accumulated) > _safe_limit and self._message_id is None ): # No existing message to edit (first message or after a @@ -1179,7 +1676,15 @@ class GatewayStreamConsumer: display_text = self._accumulated if not got_done and not got_segment_break and commentary_text is None: - display_text += self.cfg.cursor + # Native streaming with tool-progress: compose frame + # content that includes tool status overlay. The cursor + # is appended to the composed content for consistency. + if self._use_native_streaming: + display_text = self._compose_frame_content() + if display_text and self.cfg.cursor: + display_text += self.cfg.cursor + else: + display_text += self.cfg.cursor # Segment break: finalize the current message so platforms # that need explicit closure (e.g. DingTalk AI Cards) don't @@ -1200,6 +1705,12 @@ class GatewayStreamConsumer: is_turn_final=got_done, ) self._last_edit_time = time.monotonic() + # Reset tool_progress_active flag after frame delivery — + # the lines are still in _tool_progress_lines (for the next + # frame's compose) but we don't need to trigger another + # should_edit until new progress arrives. + if self._tool_progress_active: + self._tool_progress_active = False if got_done: if self._accumulated or self._message_id is not None or self._already_sent: @@ -1208,7 +1719,79 @@ class GatewayStreamConsumer: # mid-stream, send a single continuation/fallback message # here instead of letting the base gateway path send the # full response again. - if self._accumulated: + if ( + self._awaiting_reopen_after_boundary + and not self._native_stream_opened + and not self._accumulated + ): + # Clarify reopen boundary (LAZY path), but the agent + # produced no post-prompt content. The pre-prompt stream + # was already finalized into a stable bubble at the + # boundary, and no fresh stream was ever re-seeded, so + # there is nothing on screen to close. Do NOT re-seed a + # fresh stream just to emit a lone "✅" placeholder — that + # would leave a meaningless empty bubble below the + # question. Close quietly; the finalized bubble stands. + logger.debug( + "Clarify reopen boundary with no post-prompt content " + "— skipping lone-placeholder finalize (turn=%s)", + self._turn_id, + ) + elif ( + self._reopen_seeded_eagerly + and self._native_stream_opened + and not self._accumulated + and not current_update_visible + ): + # EAGER-seed path: the typing bubble is ALREADY on screen + # (opened the instant the user answered), but the agent + # then produced no content. Unlike the lazy case we + # cannot skip — an open, empty typing bubble would hang + # forever. Close it with an empty finalize (NOT a lone + # "✅", which would be a meaningless bubble below the + # question). Leave the delivery flags as-is: nothing + # substantive was delivered, so the gateway's own + # whole-response filter still governs any fallback. + try: + await self.adapter.send_stream_frame( + "", + finalize=True, + chat_id=self.chat_id, + reply_to=self._initial_reply_to_id, + turn_id=self._turn_id, + ) + except Exception as e: + logger.debug( + "Eager-seed empty finalize failed: %s", e, + ) + self._native_stream_opened = False + self._native_last_pushed_len = 0 + # Reset for symmetry with _suppress_silence_marker; the + # consumer is per-turn today so this is defensive, but it + # keeps the flag from leaking if a consumer is ever reused + # across turns. + self._reopen_seeded_eagerly = False + logger.debug( + "Eager reopen seed but no post-answer content — " + "closed empty typing bubble (turn=%s)", + self._turn_id, + ) + elif self._use_native_streaming: + # Native streaming MUST always close the stream with + # finish=true — even when _accumulated is empty (e.g. + # tool-only turns with no text output). Mirror OpenClaw's + # finishThinkingStream: use a placeholder if needed. + if not current_update_visible: + close_text = self._accumulated or "✅" + self._final_response_sent = await self._send_or_edit( + close_text, finalize=True, + ) + if self._final_response_sent: + self._final_content_delivered = True + else: + self._final_response_sent = True + self._final_content_delivered = True + elif self._accumulated: if self._fallback_final_send: await self._send_fallback_final(self._accumulated) elif self._final_response_sent: @@ -1304,8 +1887,10 @@ class GatewayStreamConsumer: self._last_edit_time = time.monotonic() self._reset_segment_state() - # Tool boundary: reset message state so the next text chunk - # creates a fresh message below any tool-progress messages. + # Tool boundary: for edit-based platforms, reset message state + # so the next text chunk creates a fresh message below tool-progress. + # For WeCom native streaming: NO reset — stream uses cumulative text, + # so resetting would lose pre-boundary content in subsequent frames. # # Exception: when _message_id is "__no_edit__" the platform # never returned a real message ID (e.g. Signal, webhook with @@ -1328,10 +1913,14 @@ class GatewayStreamConsumer: # ``is True`` + _use_draft_streaming: MagicMock adapters # return truthy auto-attributes, and an edit-based run on a # stream-capable adapter still needs the legacy reset. + # WeCom native streaming also uses cumulative text — each + # frame must carry the full content so far, so a segment + # break must NOT reset accumulated state or subsequent + # frames lose the pre-boundary text. if ( self._stream_is_message() and self._use_draft_streaming - ): + ) or self._use_native_streaming: pass else: # If the segment-break edit failed to deliver the @@ -1926,6 +2515,44 @@ class GatewayStreamConsumer: return False return True + def _resolve_native_streaming(self) -> bool: + """Decide whether this run should use the native-streaming transport. + + Native streaming (e.g. WeCom's ``msgtype: "stream"``) routes ALL + frames — first send, mid-stream updates, and the final ``finish=true`` + — through ``adapter.send_stream_frame()``. It takes precedence over + both edit and draft transports because it provides the best client + experience on platforms whose API is built for it (the WeCom client, + for example, renders cumulative content updates in-place with a + built-in typing animation while the stream stays open). + + Adapter eligibility: + 1. Must subclass :class:`BasePlatformAdapter` (MagicMock test + adapters fall back to edit). + 2. Must declare ``SUPPORTS_NATIVE_STREAMING = True`` at the class + level. + 3. Must provide ``supports_native_streaming(chat_type, metadata)`` + returning truthy for this chat. + """ + if not isinstance(self.adapter, _BasePlatformAdapter): + return False + if not getattr(type(self.adapter), "SUPPORTS_NATIVE_STREAMING", False): + return False + probe = getattr(self.adapter, "supports_native_streaming", None) + if probe is None: + return False + try: + supported = probe( + chat_type=self.cfg.chat_type or None, + metadata=self.metadata, + ) + except Exception: + logger.debug( + "supports_native_streaming probe raised", exc_info=True, + ) + return False + return bool(supported) + async def _send_draft_frame(self, text: str) -> bool: """Emit a single animated draft frame for the current accumulated text. @@ -2296,6 +2923,29 @@ class GatewayStreamConsumer: send happens either. ``_already_sent`` is likewise cleared so the gateway's ``already_sent`` short-circuits do not fire. """ + # Native-stream bubbles (e.g. WeCom) are NOT deletable messages — they + # are an open stream closed by a finalize frame, not delete_message. + # If a stream is open (notably one opened by an EAGER re-seed after a + # clarify answer, where the typing bubble is already on screen with no + # content), close it with an empty finalize so it doesn't hang forever. + # Do this before the delete loop; keep the delivery flags False below. + if self._native_stream_opened: + try: + await self.adapter.send_stream_frame( + "", + finalize=True, + chat_id=self.chat_id, + reply_to=self._initial_reply_to_id, + turn_id=self._turn_id, + ) + except Exception as e: + logger.debug( + "Silence-marker native stream close failed: %s", e, + ) + self._native_stream_opened = False + self._native_last_pushed_len = 0 + self._reopen_seeded_eagerly = False + stale_ids = set(self._preview_message_ids) if self._message_id and self._message_id != "__no_edit__": stale_ids.add(self._message_id) @@ -2363,6 +3013,23 @@ class GatewayStreamConsumer: visible_without_cursor = visible_without_cursor.replace(self.cfg.cursor, "") _visible_stripped = visible_without_cursor.strip() if not _visible_stripped: + # For native streaming: even when the display text is empty (e.g. + # MEDIA-only response cleaned away), we MUST send a finalize frame + # to close the thinking bubble. Use placeholder text. + if finalize and self._use_native_streaming and self._native_stream_opened: + try: + ok = await self.adapter.send_stream_frame( + "✅", + finalize=True, + chat_id=self.chat_id, + reply_to=self._initial_reply_to_id, + turn_id=self._turn_id, + ) + if ok: + self._final_response_sent = True + self._final_content_delivered = True + except Exception as e: + logger.debug("Finalize empty stream failed: %s", e) return True # cursor-only / whitespace-only update if not text.strip(): return True # nothing to send is "success" @@ -2382,7 +3049,155 @@ class GatewayStreamConsumer: and len(_visible_stripped) < _MIN_NEW_MSG_CHARS): return True # too short for a standalone message — accumulate more - # Native draft streaming: route mid-stream frames through send_draft. + # Native streaming transport (e.g. WeCom): every frame — first send, + # mid-stream updates, and the final answer — flows through + # adapter.send_stream_frame(), which manages the underlying stream + # lifecycle (init seed → cumulative updates → finish=true). The + # adapter's send/edit_message paths are NOT touched in this mode. + # + # Throttling: WeCom AI Bot caps replies at ~30 frames/min per chat. + # With 15 concurrent users, we need ≤2 frames per turn on average + # to stay under the limit. 60 chars ≈ one short sentence, which + # produces 3-5 frames per turn — close to OpenClaw's block-level cadence. + if self._use_native_streaming: + # Re-seed if stream was closed (e.g., by approval boundary) + # and we have new content to send. + if not self._native_stream_opened and text: + try: + seed_ok = await self.adapter.send_stream_frame( + "", + chat_id=self.chat_id, + reply_to=self._initial_reply_to_id, + turn_id=self._turn_id, + ) + if seed_ok: + self._native_stream_opened = True + # A fresh stream is open — post-prompt content will + # stream into it, so got_done no longer needs the + # lone-placeholder guard for this turn. + self._awaiting_reopen_after_boundary = False + # INFO (temporary latency probe): this is the moment the + # C bubble / typing animation first becomes visible after + # a clarify answer. Comparing this timestamp to the + # boundary-finalize log below quantifies the "typing is + # slow to reappear" delay the user reported. + logger.info( + "[latency] Re-opened native stream after boundary " + "(turn=%s, waited for first delta)", + self._turn_id, + ) + else: + self._use_native_streaming = False + except Exception as e: + logger.debug("Re-seed failed, disabling native streaming: %s", e) + self._use_native_streaming = False + + if self._use_native_streaming: + # For WeCom native streaming: segment breaks should NOT finalize + # the stream. WeCom renders each finalize as a separate message bubble. + # Only turn-final (got_done) and approval boundary should close the stream. + # Tool boundary segment breaks just continue accumulating in the same stream. + if finalize and not is_turn_final: + finalize = False + + # Fire-and-forget: send immediately when content differs from + # the last pushed frame. No buffering / throttle — WeCom long- + # connection mode has no polling cadence, so every cumulative + # update is pushed as soon as it arrives. + if not finalize and text == self._last_sent_text: + return True # unchanged — skip + + # B2 — timeout-inversion race fix. For a finalize frame, mark + # delivery OPTIMISTICALLY, before send_stream_frame blocks on the + # ack. The finalize frame's bytes are written to the wire by an + # independent control-worker task *before* the ack wait begins, and + # for WeCom a frame on the wire is already rendered by the client + # (the same premise the ack-timeout-as-success path already relies + # on). Setting the flag here means a gateway join-cancel during the + # ack wait — the timeout inversion between run.py's stream_task join + # and adapter._REPLY_ACK_TIMEOUT — can no longer strand + # final_content_delivered=False while WeCom has already shown the + # message, which is what produced the duplicate normal send + # (see tests/gateway/test_wecom_double_send.py and + # docs/rca-wecom-stream-final-ack-timeout-duplicate.md). + # + # A DEFINITIVE dispatch failure (ok is False below: stream never + # opened, 846608 expired, errcode 6000, or the call raised) rolls + # the mark back so the edit/send fallback still delivers exactly + # once. Residual window: if the consumer is cancelled between this + # optimistic mark and the control worker actually writing the bytes + # (queue latency, sub-ms in practice), the message could be + # suppressed without being sent — far rarer than the guaranteed + # duplicate this replaces, and the send-path idempotency guard + # cannot help there (nothing was sent). Accepted trade-off. + _optimistic_finalize = bool(finalize) + if _optimistic_finalize: + self._final_response_sent = True + self._final_content_delivered = True + + ok = False + try: + ok = await self.adapter.send_stream_frame( + text, + finalize=finalize, + chat_id=self.chat_id, + reply_to=self._initial_reply_to_id, + turn_id=self._turn_id, + ) + except Exception as e: + logger.debug( + "send_stream_frame raised, disabling native streaming: %s", e, + ) + ok = False + + if ok: + self._already_sent = True + self._last_sent_text = text + self._native_last_pushed_len = len(text) + if finalize: + self._final_response_sent = True + self._final_content_delivered = True + return True + + # Dispatch failed definitively — roll back the optimistic finalize + # mark so the edit/send fallback below delivers the content once. + if _optimistic_finalize: + self._final_response_sent = False + self._final_content_delivered = False + + # Native streaming refused / failed — switch off so this and + # subsequent frames take the edit/send fallback path below. + # The adapter is responsible for marking the chat as expired + # so it doesn't keep retrying the dead stream session. + self._use_native_streaming = False + + # If the stream bubble was opened (seed frame succeeded), try + # best-effort finalize to close it before falling back to send(). + # This prevents leaving an unclosed thinking stream visible to the + # user. Check _native_stream_opened (not _native_last_pushed_len) + # because the seed frame has zero length but still opens the bubble. + if self._native_stream_opened: + try: + await self.adapter.send_stream_frame( + text, + finalize=True, + chat_id=self.chat_id, + reply_to=self._initial_reply_to_id, + turn_id=self._turn_id, + ) + logger.debug("Native fallback: finalized stream (best-effort close)") + # DO NOT mark _final_content_delivered here. + # The finalize frame closes the typing bubble, but WeCom may + # not actually render the content (e.g., errcode 6000 race). + # Let the fallback send() path deliver the content reliably. + except Exception as e: + logger.debug( + "Native fallback: failed to finalize stream: %s", e, + ) + # Fall through to the edit/send paths so any accumulated text + # still reaches the user as a one-shot proactive markdown send. + + # The final answer is delivered via the regular sendMessage path # below — drafts have no message_id so we can't finalize them # in-place; the regular sendMessage clears the draft naturally on diff --git a/plugins/platforms/wecom/adapter.py b/plugins/platforms/wecom/adapter.py index 162d4a35bc..7757843dc7 100644 --- a/plugins/platforms/wecom/adapter.py +++ b/plugins/platforms/wecom/adapter.py @@ -39,6 +39,8 @@ import os import re import time import uuid +from collections import deque +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -120,6 +122,80 @@ RECONNECT_BACKOFF = [2, 5, 10, 30, 60] DEDUP_MAX_SIZE = 1000 +# Native streaming (msgtype: stream) constants — modeled on WeCom's official +# OpenClaw plugin behavior. WeCom's AI Bot supports cumulative stream frames +# via aibot_respond_msg; the first frame sends a placeholder +# (matching the plugin's THINKING_MESSAGE) to signal a reasoning turn, +# subsequent frames push cumulative content, and a final frame with +# finish=true closes the stream. +STREAM_EXPIRED_ERRCODE = 846608 # >6 min without update — stream is dead +STREAM_REQUEST_EXPIRED_ERRCODE = 846604 # passive-reply request (req_id) itself +# expired — "websocket request expired, response is invalid". Sibling of 846608: +# 846608 is the stream update window, 846604 is the req_id reply channel window. +# Both mean the reply flow is dead and further finish=true / finish=false frames +# on it will be rejected — so keep-alive treats either as "stream expired". +STREAM_NOT_SUBSCRIBED_ERRCODE = 846609 # ws connection lost the subscription +STREAM_VERSION_CONFLICT_ERRCODE = 6000 # finalize raced a newer frame on the +# same stream_id — the bubble was ALREADY replaced by that newer version, so +# for a finalize frame this is benign (idempotent re-finalize hitting an +# already-updated bubble), NOT a delivery failure. See _send_stream_reply. +MAX_STREAM_CONTENT_LENGTH = 20480 # WeCom server-enforced byte limit per frame +# Per-turn cap on intermediate frames. WeCom SDK has an internal 100-frame +# per-reqId queue limit; we cap at 85 (matching openclaw plugin) to guarantee +# room for the finalize frame. Once hit, all further intermediate frames are +# silently dropped — finalize still sends unconditionally. +MAX_INTERMEDIATE_FRAMES = 85 + +# ── Stream-level keep-alive (aligned with wecom-openclaw-plugin PR #90) ────── +# WeCom binds a ~6-minute lifetime timer to each *reply stream* (stream_id + +# req_id). The connection-level ping (_heartbeat_loop) does NOT refresh it — +# it only keeps the WS socket open. Long turns (e.g. the daily-report cron, +# which spends minutes fetching data / rendering charts with no LLM tokens to +# push) let that window elapse, so the finalize frame lands on a dead stream +# and comes back 846604 / 846608 — a real delivery risk in group chats, which +# cannot fall back to a proactive send. See docs/wecom-stream-keepalive-*.md. +# +# Two independent defences, both defaulting to the SAFE (non-aggressive) side: +# +# Layer 2 — clock fallback (always on, zero new uplink frames): +# finalize computes stream age from StreamTurn.start_time; past +# STREAM_SAFE_DURATION_SECONDS it declines the finish=true frame (which +# would almost certainly hit 846604/846608) and returns False so the +# gateway consumer's existing fallback send() path delivers the content. +# +# Layer 1 — keep-alive heartbeat (OFF by default; opt-in via config): +# every STREAM_KEEPALIVE_INTERVAL_SECONDS re-send the already-accumulated +# text as a finish=false frame to refresh the server window. Never sends a +# placeholder (that would pollute last_sent_content and could strand the +# user on "still working…"); when there is no accumulated text yet the tick +# is simply skipped. Guarded off by default because a heartbeat frame is an +# extra intermediate frame sharing the finalize's req_id, which widens the +# ack race the double-send coordination depends on (see ANALYSIS §4.2). +STREAM_SAFE_DURATION_SECONDS = 330.0 # 5.5 min — Layer 2 clock fallback +STREAM_KEEPALIVE_INTERVAL_SECONDS = 120.0 # 2 min — Layer 1 heartbeat cadence +STREAM_KEEPALIVE_ENABLED_DEFAULT = False # Layer 1 off unless config opts in + +# ── Block-streaming parameters (aligned with the official wecom-openclaw-plugin) ── +# The official plugin (wecom-openclaw-plugin/src/webhook/helpers.ts) coalesces +# incoming LLM tokens into sentence-aligned blocks before sending each frame, +# rather than echoing every char-delta. This produces ~1 frame per natural +# sentence boundary instead of one per 60-char delta, which means: +# - frame cadence drops from "every 200ms" to "every 0.5-2s typical", +# - the 5s ack timeout almost never fires (frames are spaced wider apart), +# - and the user sees content land in complete thought-units rather than +# mid-sentence cursors moving every tick. +# +# Values copied verbatim from the official plugin so we match its behaviour +# in WeCom's rate-limit budget (≈30 frames/min/chat). +BLOCK_STREAM_MIN_CHARS = 120 # Don't emit a frame below this size (unless forced) +BLOCK_STREAM_MAX_CHARS = 360 # Force a break above this size +BLOCK_STREAM_IDLE_FLUSH = 0.25 # 250ms — flush partial buffer if no new tokens + +# Sentence terminators recognised by the block chunker. Includes the most +# common Chinese full-stop / exclamation / question forms; matches the +# official plugin's "sentence" break preference for the WeCom channel. +_SENTENCE_TERMINATORS = ".!?。!?" + IMAGE_MAX_BYTES = 10 * 1024 * 1024 VIDEO_MAX_BYTES = 10 * 1024 * 1024 VOICE_MAX_BYTES = 2 * 1024 * 1024 @@ -164,11 +240,220 @@ def _entry_matches(entries: List[str], target: str) -> bool: return False +class WeComStreamExpiredError(RuntimeError): + """Raised when WeCom returns errcode 846608 or 846604 (stream/req expired). + + WeCom's stream protocol caps a stream session at ~6 minutes from the + first frame. After that window the server refuses further updates with + 846608 (stream update window) or 846604 (req_id reply-request window) and + the reply flow is dead — callers must fall back to a proactive + ``aibot_send_msg`` to deliver the remaining content. + """ + + def __init__(self, errcode: int = STREAM_EXPIRED_ERRCODE, errmsg: str = ""): + super().__init__( + f"WeCom stream expired (errcode={errcode}): {errmsg or 'no detail'}" + ) + self.errcode = errcode + self.errmsg = errmsg + + +@dataclass +class ReplyFrame: + """A queued reply frame waiting to be sent via aibot_respond_msg. + + Used for ack tracking and FIFO ordering per req_id, aligning with + the official WeCom SDK's replyStreamNonBlocking semantics. + """ + body: Dict[str, Any] + future: asyncio.Future + is_final: bool = False + sent_at: Optional[float] = None + + +class ReplyQueue: + """Per-req_id pending ack tracker. + + Ensures: + - Intermediate frames skip if a previous frame's ack is pending + - Final frames wait for pending ack before sending + + Aligned with official SDK's replyStreamNonBlocking + 5s ack timeout. + """ + def __init__(self, req_id: str): + self.req_id = req_id + self.pending_ack: Optional[ReplyFrame] = None + + +class _BlockChunker: + """Coalesce streaming text into sentence-aligned blocks. + + The consumer feeds us **cumulative** text (every frame is the full + response-so-far). We track the high-water mark and only emit when one + of these conditions holds for the *new* tail: + + * new content is at least ``BLOCK_STREAM_MIN_CHARS`` AND ends on a + safe break (sentence terminator or blank-line paragraph boundary); + * new content has grown to ``BLOCK_STREAM_MAX_CHARS`` (hard cap — + force a break); + * ``force=True`` is passed (finalize, segment break, idle timer). + + Returns the **cumulative text up to the emit point**, matching the + semantics of WeCom's stream protocol where every frame carries the + full content and the client renders the diff. This is consistent + with the official wecom-openclaw-plugin's flow: each ``deliver`` + callback there carries the new block, and the plugin sends the + accumulated text via ``sendWeComReply`` under the same streamId. + """ + + __slots__ = ( + "_cumulative", + "_min_chars", + "_max_chars", + "_terminators", + "_emitted_len", + ) + + def __init__( + self, + *, + min_chars: int = BLOCK_STREAM_MIN_CHARS, + max_chars: int = BLOCK_STREAM_MAX_CHARS, + terminators: str = _SENTENCE_TERMINATORS, + ) -> None: + self._cumulative: str = "" + self._min_chars = min_chars + self._max_chars = max_chars + self._terminators = terminators + # Length of the prefix that has already been signalled as ready to + # emit at least once. We compare new growth against this watermark. + self._emitted_len = 0 + + def update(self, cumulative_text: str) -> None: + """Replace the high-water mark with a fresh cumulative snapshot.""" + # Only grow — the consumer never shrinks its accumulator, but defensive + # against accidental resets so we don't lose ground. + if len(cumulative_text) > len(self._cumulative): + self._cumulative = cumulative_text + + def has_pending(self) -> bool: + """True when there is buffered content past the last emit point.""" + return len(self._cumulative) > self._emitted_len + + def drain(self, *, force: bool = False) -> Optional[str]: + """Decide whether to emit a frame, return the cumulative text if so. + + Returns ``None`` when the chunker wants to keep buffering, or a + cumulative string (full content up to the chosen break) when a + frame is ready to go on the wire. + """ + if not self.has_pending(): + return None + + new_len = len(self._cumulative) - self._emitted_len + # Hard cap — force a break even mid-sentence. + if new_len >= self._max_chars: + self._emitted_len = len(self._cumulative) + return self._cumulative + + if force: + # Finalize / segment-break / idle-timer path: emit everything we + # have. Empty/whitespace-only tails are still allowed to pass — + # the caller (send_stream_frame) handles empty-text finalize. + self._emitted_len = len(self._cumulative) + return self._cumulative + + if new_len < self._min_chars: + return None + + # Look for a safe break inside the new tail. We scan backwards from + # the end of cumulative — the rightmost terminator that leaves a + # block of at least ``min_chars`` becomes the break point. + # Sentence terminators must be followed by whitespace / end-of-text + # so we don't split inside e.g. "v1.2" or "192.168.1.1". + new_tail_start = self._emitted_len + for idx in range(len(self._cumulative) - 1, new_tail_start - 1, -1): + ch = self._cumulative[idx] + if ch not in self._terminators: + continue + # Tail-end terminator is always safe. + if idx == len(self._cumulative) - 1: + next_ok = True + else: + next_ch = self._cumulative[idx + 1] + next_ok = next_ch.isspace() or next_ch in self._terminators + if not next_ok: + continue + break_at = idx + 1 + if break_at - new_tail_start < self._min_chars: + # Sentence is too short; keep scanning leftward looking for a + # later terminator that satisfies min_chars. Once we fall + # below min_chars we know no further matches will satisfy it + # either (we're moving left, blocks shrink). + break + self._emitted_len = break_at + return self._cumulative[:break_at] + + # Also accept paragraph boundary ("\n\n") inside the new tail. + para_idx = self._cumulative.rfind("\n\n", new_tail_start) + if para_idx >= 0 and (para_idx + 2 - new_tail_start) >= self._min_chars: + break_at = para_idx + 2 + self._emitted_len = break_at + return self._cumulative[:break_at] + + return None + + +class StreamTurn: + """Per-turn stream state to avoid global state conflicts. + + Each inbound message creates its own StreamTurn, ensuring concurrent + messages don't interfere with each other's stream state. + """ + def __init__(self, chat_id: str, req_id: str): + self.chat_id = chat_id + self.req_id = req_id + self.stream_id = f"stream_{uuid.uuid4().hex[:12]}" + self.accumulated_text = "" + self.finalized = False + self.seeded = False # True after seed frame sent (prevents double seed) + self.start_time = time.monotonic() + self.expired = False + # Track the last content that was ACTUALLY sent to WeCom (not skipped). + # Used by finalize to detect duplicate content and avoid silent ack drops. + self.last_sent_content: str = "" + # Per-turn intermediate-frame counter (count-based cap at + # MAX_INTERMEDIATE_FRAMES to leave room for the finalize frame). + self._last_frame_sent_at: float = 0.0 + self._intermediate_frames_sent: int = 0 + # Block-stream chunker — coalesces consumer's cumulative cursor into + # sentence-aligned blocks before each frame goes on the wire. See + # ``_BlockChunker`` for the algorithm; lazily created on first append + # so per-turn cost is paid only for turns that actually stream. + self.chunker: Optional["_BlockChunker"] = None + # Idle flush handle — set when a partial buffer is waiting on the + # 250ms idle timer. Cleared when the timer fires, when a force-flush + # happens, or on cleanup. + self.idle_flush_handle: Optional[asyncio.TimerHandle] = None + # Keep-alive handle (Layer 1) — set when the stream-level keep-alive + # timer is armed. Structurally identical to idle_flush_handle: a + # per-turn asyncio TimerHandle that MUST be cancelled on every turn + # exit path (finalize / expired / error / cleanup) to avoid a leaked + # timer firing on a dead turn. None when keep-alive is disabled or + # the turn has no armed timer. + self.keepalive_handle: Optional[asyncio.TimerHandle] = None + + class WeComAdapter(BasePlatformAdapter): """WeCom AI Bot adapter backed by a persistent WebSocket connection.""" MAX_MESSAGE_LENGTH = MAX_MESSAGE_LENGTH SUPPORTS_MESSAGE_EDITING = False + # WeCom AI Bot supports msgtype: "stream" via aibot_respond_msg, which + # the gateway streaming consumer treats as a transport that bypasses the + # edit-based path. See ``send_stream_frame`` and ``supports_native_streaming``. + SUPPORTS_NATIVE_STREAMING = True + MAX_STREAM_CONTENT_LENGTH = MAX_STREAM_CONTENT_LENGTH # Threshold for detecting WeCom client-side message splits. # When a chunk is near the 4000-char limit, a continuation is almost certain. _SPLIT_THRESHOLD = 3900 @@ -206,6 +491,9 @@ class WeComAdapter(BasePlatformAdapter): self._listen_task: Optional[asyncio.Task] = None self._heartbeat_task: Optional[asyncio.Task] = None self._pending_responses: Dict[str, asyncio.Future] = {} + # Per-req_id reply queue with ack tracking — aligns with official + # SDK's replyStreamNonBlocking (skip if pending, wait before final). + self._reply_queues: Dict[str, ReplyQueue] = {} self._dedup = MessageDeduplicator(max_size=DEDUP_MAX_SIZE) self._reply_req_ids: Dict[str, str] = {} @@ -213,11 +501,255 @@ class WeComAdapter(BasePlatformAdapter): # WeCom clients split long messages around 4000 chars. self._text_batch_delay_seconds = env_float("HERMES_WECOM_TEXT_BATCH_DELAY_SECONDS", 0.6) self._text_batch_split_delay_seconds = env_float("HERMES_WECOM_TEXT_BATCH_SPLIT_DELAY_SECONDS", 2.0) + # Attachment/text merge window: WeCom clients send "image + text" as two + # separate inbound callbacks (one attachment-only, one text) a few + # hundred ms apart. Holding an attachment-only message for this window + # lets the following text merge into the SAME event, so it dispatches + # as one turn instead of the attachment spawning a run that the text + # then "interrupts" (mirrors the official plugin's + # ATTACHMENT_TEXT_MERGE_WINDOW_MS = 800). Behavioral config lives in + # config.extra, not an env var. + try: + self._attachment_text_merge_delay_seconds = float( + extra.get("attachment_text_merge_delay_seconds", 0.8) + ) + except (TypeError, ValueError): + self._attachment_text_merge_delay_seconds = 0.8 self._pending_text_batches: Dict[str, MessageEvent] = {} self._pending_text_batch_tasks: Dict[str, asyncio.Task] = {} + + # ── Stream-level keep-alive config (config.extra, not env) ────────── + # Behavioral config lives in config.extra, matching + # attachment_text_merge_delay_seconds above. See the module-level + # STREAM_* constants and docs/wecom-stream-keepalive-ANALYSIS.md. + def _extra_float(key: str, default: float) -> float: + try: + return float(extra.get(key, default)) + except (TypeError, ValueError): + return default + + # Layer 2 clock fallback: decline the finalize frame once the stream is + # older than this, so we don't hand the server a doomed finish=true. + self._stream_safe_duration_seconds = _extra_float( + "stream_safe_duration_seconds", STREAM_SAFE_DURATION_SECONDS + ) + # Layer 1 heartbeat: off unless config opts in (see ANALYSIS §4.2/§5). + self._stream_keepalive_enabled = bool( + extra.get("stream_keepalive_enabled", STREAM_KEEPALIVE_ENABLED_DEFAULT) + ) + self._stream_keepalive_interval_seconds = _extra_float( + "stream_keepalive_interval_seconds", STREAM_KEEPALIVE_INTERVAL_SECONDS + ) + self._device_id = uuid.uuid4().hex self._last_chat_req_ids: Dict[str, str] = {} + # Per-turn stream state: keyed by (chat_id, req_id) to support concurrent messages. + # Replaces global _active_stream_id to avoid conflicts when multiple messages + # are processed simultaneously (e.g., approval during streaming). + self._stream_turns: Dict[str, StreamTurn] = {} # key = f"{chat_id}:{req_id}" + + # Chats whose stream session has been retired (846608 / 846609 / no + # req_id). Cleared whenever a fresh inbound callback for the chat + # arrives — a new inbound message gives us a new req_id and the + # stream channel becomes usable again. + self._stream_expired_chats: set[str] = set() + + # Track which chat_ids are group chats. Populated in _on_message + # when chattype=="group". Used by _send_inner to avoid APP_CMD_SEND + # for groups (WeCom AI Bots cannot initiate proactive sends in groups). + self._group_chat_ids: set[str] = set() + + # Per-chat FIFO send queue with token-bucket rate limiting. + # Mirrors OpenClaw's chat-queue.ts (serial per chat) plus a + # token bucket to stay within WeCom's 30 msgs/min/chat limit. + self._chat_queues: Dict[str, asyncio.Queue] = {} + self._chat_workers: Dict[str, asyncio.Task] = {} + + # Control lane: high-priority queue for approval prompts, finalize frames, + # and error notifications. These bypass normal queue to prevent blocking. + self._control_queues: Dict[str, asyncio.Queue] = {} + self._control_workers: Dict[str, asyncio.Task] = {} + + # Token bucket with reserved tokens for control messages. + # Per-chat usage tracking: {chat_id: {"normal": used, "reserved": used, "last_reset": ts}} + self._chat_token_usage: Dict[str, Dict[str, float]] = {} + + # Token bucket parameters: 30 tokens max per minute, split between normal and reserved. + _BUCKET_MAX_TOKENS = 30 + _BUCKET_NORMAL_TOKENS = 24 # For normal messages + _BUCKET_RESERVED_TOKENS = 6 # Reserved for control lane (approval, finalize, errors) + + def _get_token_usage(self, chat_id: str) -> Dict[str, float]: + """Get or create token usage tracking for a chat.""" + key = str(chat_id or "").strip() + if key not in self._chat_token_usage: + self._chat_token_usage[key] = { + "normal": 0.0, + "reserved": 0.0, + "last_reset": time.monotonic(), + } + return self._chat_token_usage[key] + + def _bucket_try_consume(self, chat_id: str) -> float: + """Try to consume a normal token. Returns 0 if available, or seconds to wait.""" + usage = self._get_token_usage(chat_id) + now = time.monotonic() + + # Reset counters every minute + if now - usage["last_reset"] > 60.0: + usage["normal"] = 0.0 + usage["reserved"] = 0.0 + usage["last_reset"] = now + + # Normal messages can only use normal quota + if usage["normal"] < self._BUCKET_NORMAL_TOKENS: + usage["normal"] += 1.0 + return 0.0 # token available, no wait + else: + # Wait until next minute + return 60.0 - (now - usage["last_reset"]) + + def _bucket_try_consume_control(self, chat_id: str) -> float: + """Try to consume a control token. Can use normal remaining + reserved pool.""" + usage = self._get_token_usage(chat_id) + now = time.monotonic() + + # Reset counters every minute + if now - usage["last_reset"] > 60.0: + usage["normal"] = 0.0 + usage["reserved"] = 0.0 + usage["last_reset"] = now + + # Control messages prefer normal quota first (don't waste reserved) + normal_available = self._BUCKET_NORMAL_TOKENS - usage["normal"] + if normal_available > 0: + usage["normal"] += 1.0 + return 0.0 + + # Normal exhausted, use reserved pool + reserved_available = self._BUCKET_RESERVED_TOKENS - usage["reserved"] + if reserved_available > 0: + usage["reserved"] += 1.0 + return 0.0 + + # Both exhausted, wait until next minute + return 60.0 - (now - usage["last_reset"]) + + async def _enqueue_chat_send(self, chat_id: str, coro_factory, is_control: bool = False): + """Enqueue a send task for a chat and await its result. + + FIFO per chat, parallel across chats. Two lanes: + - Control lane: approval prompts, finalize frames, errors (uses reserved tokens) + - Normal lane: regular messages (uses normal tokens only) + + Control messages bypass normal queue to prevent approval prompt blocking. + """ + key = str(chat_id or "").strip() + + if is_control: + # Control lane: high priority, reserved token pool + if key not in self._control_queues: + logger.debug( + "[%s] Creating control queue + worker for chat %s", + self.name, key, + ) + self._control_queues[key] = asyncio.Queue() + self._control_workers[key] = asyncio.create_task( + self._control_send_worker(key) + ) + queue = self._control_queues[key] + else: + # Normal lane + if key not in self._chat_queues: + logger.debug( + "[%s] Creating normal queue + worker for chat %s", + self.name, key, + ) + self._chat_queues[key] = asyncio.Queue() + self._chat_workers[key] = asyncio.create_task( + self._chat_send_worker(key) + ) + queue = self._chat_queues[key] + + logger.debug( + "[%s] Enqueuing send for chat %s (lane=%s, qsize=%d)", + self.name, key, "control" if is_control else "normal", queue.qsize(), + ) + future = asyncio.get_running_loop().create_future() + await queue.put((coro_factory, future)) + return await future + + async def _chat_send_worker(self, chat_key: str) -> None: + """Per-chat worker: drain normal queue with token-bucket rate limiting.""" + queue = self._chat_queues[chat_key] + logger.debug("[%s] Normal send worker started for chat %s", self.name, chat_key) + try: + while True: + coro_factory, future = await queue.get() + try: + # Token bucket: wait only if bucket is empty + wait = self._bucket_try_consume(chat_key) + if wait > 0: + logger.debug( + "[%s] Normal worker rate-limited for chat %s, waiting %.1fs", + self.name, chat_key, wait, + ) + await asyncio.sleep(wait) + # Re-consume after wait + self._bucket_try_consume(chat_key) + + result = await coro_factory() + if not future.done(): + future.set_result(result) + except Exception as exc: + if not future.done(): + future.set_exception(exc) + finally: + queue.task_done() + except asyncio.CancelledError: + while not queue.empty(): + try: + _, future = queue.get_nowait() + if not future.done(): + future.set_exception( + RuntimeError("WeCom adapter shutting down") + ) + except asyncio.QueueEmpty: + break + + async def _control_send_worker(self, chat_key: str) -> None: + """Control lane worker: drain control queue with reserved token pool.""" + queue = self._control_queues[chat_key] + try: + while True: + coro_factory, future = await queue.get() + try: + # Control messages use reserved + normal remaining tokens + wait = self._bucket_try_consume_control(chat_key) + if wait > 0: + await asyncio.sleep(wait) + self._bucket_try_consume_control(chat_key) + + result = await coro_factory() + if not future.done(): + future.set_result(result) + except Exception as exc: + if not future.done(): + future.set_exception(exc) + finally: + queue.task_done() + except asyncio.CancelledError: + while not queue.empty(): + try: + _, future = queue.get_nowait() + if not future.done(): + future.set_exception( + RuntimeError("WeCom adapter shutting down") + ) + except asyncio.QueueEmpty: + break + # ------------------------------------------------------------------ # Connection lifecycle # ------------------------------------------------------------------ @@ -272,6 +804,17 @@ class WeComAdapter(BasePlatformAdapter): """Disconnect from WeCom.""" self._running = False self._mark_disconnected() + # Force-close any lingering stream so the WeCom client doesn't show + # a permanent typing bubble after the gateway goes down. + self._reset_native_stream_state() + + # Cancel per-chat send workers (normal + control lanes) so queued tasks get cleaned up. + for task in list(self._chat_workers.values()) + list(self._control_workers.values()): + task.cancel() + self._chat_workers.clear() + self._control_workers.clear() + self._chat_queues.clear() + self._control_queues.clear() if self._listen_task: self._listen_task.cancel() @@ -290,6 +833,7 @@ class WeComAdapter(BasePlatformAdapter): self._heartbeat_task = None self._fail_pending_responses(RuntimeError("WeCom adapter disconnected")) + self._fail_reply_queues(RuntimeError("WeCom adapter disconnected")) await self._cleanup_ws() if self._http_client: @@ -312,7 +856,17 @@ class WeComAdapter(BasePlatformAdapter): async def _open_connection(self) -> None: """Open and authenticate a websocket connection.""" await self._cleanup_ws() - self._session = aiohttp.ClientSession(trust_env=True) + # Use certifi's CA bundle so aiohttp trusts the same roots as + # urllib/requests — avoids SSL_CERTIFICATE_VERIFY_FAILED on macOS + # where the OpenSSL default path may be empty or stale. + import ssl as _ssl + try: + import certifi + _ssl_ctx = _ssl.create_default_context(cafile=certifi.where()) + except ImportError: + _ssl_ctx = _ssl.create_default_context() + _connector = aiohttp.TCPConnector(ssl=_ssl_ctx) + self._session = aiohttp.ClientSession(trust_env=True, connector=_connector) self._ws = await self._session.ws_connect( self._ws_url, heartbeat=HEARTBEAT_INTERVAL_SECONDS * 2, @@ -376,6 +930,7 @@ class WeComAdapter(BasePlatformAdapter): return logger.warning("[%s] WebSocket error: %s", self.name, exc) self._fail_pending_responses(RuntimeError("WeCom connection interrupted")) + self._fail_reply_queues(RuntimeError("WeCom connection interrupted")) delay = RECONNECT_BACKOFF[min(backoff_idx, len(RECONNECT_BACKOFF) - 1)] backoff_idx += 1 @@ -400,8 +955,40 @@ class WeComAdapter(BasePlatformAdapter): payload = self._parse_json(msg.data) if payload: await self._dispatch_payload(payload) + else: + # Parse returned None (JSON decode failed or non-dict + # payload). _parse_json already logs the failure detail; + # this makes the DROP itself visible at INFO so we can + # correlate a missing inbound message with a bad frame. + logger.info( + "[%s] Inbound TEXT frame dropped (unparseable/non-dict) len=%d", + self.name, + len(msg.data) if isinstance(msg.data, (str, bytes)) else -1, + ) + elif msg.type == aiohttp.WSMsgType.BINARY: + # WeCom is expected to send TEXT frames; a BINARY frame is + # unexpected. Log at INFO with a decoded preview so we can + # tell whether group messages are arriving in an unhandled + # transport instead of being silently discarded. + try: + decoded = msg.data.decode("utf-8", errors="replace") + except Exception: + decoded = "" + logger.info( + "[%s] Inbound BINARY frame received (len=%d) head=%r — attempting JSON parse", + self.name, + len(msg.data) if isinstance(msg.data, (bytes, bytearray)) else -1, + decoded[:200], + ) + payload = self._parse_json(msg.data) + if payload: + await self._dispatch_payload(payload) + else: + logger.info("[%s] BINARY frame not parseable as JSON — dropped", self.name) elif msg.type in {aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR, aiohttp.WSMsgType.CLOSING}: raise RuntimeError("WeCom websocket closed") + else: + logger.info("[%s] Inbound frame ignored: WSMsgType=%s", self.name, msg.type) async def _heartbeat_loop(self) -> None: """Send lightweight application-level pings.""" @@ -428,6 +1015,39 @@ class WeComAdapter(BasePlatformAdapter): req_id = self._payload_req_id(payload) cmd = str(payload.get("cmd") or "") + # --- Diagnostic: log ALL non-ping inbound payloads when any reply queue + # is active, to detect whether WeCom acks arrive at all. + if self._reply_queues and cmd != APP_CMD_PING: + logger.debug( + "[%s] _dispatch_payload[ALL]: req_id=%s cmd=%r active_queues=%s", + self.name, req_id or "(none)", cmd or "(empty)", + list(self._reply_queues.keys()), + ) + + # --- Diagnostic: log all payloads that carry a req_id matching an + # active reply queue, regardless of whether they get routed there. + # This helps diagnose ack timeout issues (e.g., ack arriving with + # unexpected cmd that gets filtered out). + if req_id and self._reply_queues.get(req_id): + queue = self._reply_queues[req_id] + has_pending = queue.pending_ack is not None + logger.debug( + "[%s] _dispatch_payload: req_id=%s cmd=%r has_pending_ack=%s " + "errcode=%s in_NON_RESPONSE=%s payload_keys=%s", + self.name, req_id, cmd, has_pending, + payload.get("body", {}).get("errcode", "N/A") if isinstance(payload.get("body"), dict) else "N/A", + cmd in NON_RESPONSE_COMMANDS, + list(payload.keys()), + ) + + # Check reply queue ack first — aibot_respond_msg acks arrive with + # the original inbound req_id and no cmd (or non-callback cmd). + # This must be checked before _pending_responses to avoid the old + # _send_reply_request path stealing acks meant for the queue. + if req_id and cmd not in NON_RESPONSE_COMMANDS: + if self._resolve_reply_ack(req_id, payload): + return + if req_id and req_id in self._pending_responses and cmd not in NON_RESPONSE_COMMANDS: future = self._pending_responses.get(req_id) if future and not future.done(): @@ -437,10 +1057,33 @@ class WeComAdapter(BasePlatformAdapter): if cmd in CALLBACK_COMMANDS: await self._on_message(payload) return - if cmd in {APP_CMD_PING, APP_CMD_EVENT_CALLBACK}: + if cmd == APP_CMD_PING: + return + if cmd == APP_CMD_EVENT_CALLBACK: + # Check for "kicked by server" event — WeCom sends this when a new + # connection is established elsewhere (another instance). Mirror the + # official OpenClaw SDK: suppress reconnect to avoid mutual kicking. + body = payload.get("body") or {} + event_type = str(body.get("event_type") or "") + if event_type == "disconnected_event": + logger.warning( + "[%s] Kicked by server (another WS connection established). " + "Suppressing reconnect to avoid mutual kicking. " + "Check for duplicate gateway instances.", + self.name, + ) + self._running = False # stop _listen_loop from reconnecting return - logger.debug("[%s] Ignoring websocket payload: %s", self.name, cmd or payload) + # Unrouted payload — did not match reply-queue, pending-response, + # callback, ping, or event. If WeCom delivers group messages under a + # cmd not in CALLBACK_COMMANDS, they land here and are dropped. Log at + # INFO with cmd + body keys so we can spot an unhandled callback cmd. + body_keys = list(payload.get("body", {}).keys()) if isinstance(payload.get("body"), dict) else None + logger.info( + "[%s] Unrouted websocket payload dropped: cmd=%r req_id=%s body_keys=%s", + self.name, cmd or "(empty)", req_id or "(none)", body_keys, + ) def _fail_pending_responses(self, exc: Exception) -> None: """Fail all outstanding request futures.""" @@ -496,6 +1139,222 @@ class WeComAdapter(BasePlatformAdapter): finally: self._pending_responses.pop(normalized_req_id, None) + # ── Per-req_id Reply Queue (ack tracking) ──────────────────────────── + # Aligns with official SDK replyStreamNonBlocking: + # - intermediate frame: skip if pending ack on this req_id + # - final frame: wait for pending ack to drain before sending + # - ack timeout: 15 seconds + # + # Matches the official @wecom/wecom-openclaw-plugin's REPLY_SEND_TIMEOUT_MS + # = 15_000. The prior 5s value was too aggressive for the bilibili WeCom + # environment where ack > 5s is not rare on long replies (server-side queue + # lag, WS jitter, concurrent replies on the same WS). A short window widens + # the race where the final-frame ack is still in flight while the gateway's + # normal final-send fires, producing duplicate messages + # (see docs/rca-wecom-stream-final-ack-timeout-duplicate.md). + + _REPLY_ACK_TIMEOUT = 15.0 + + async def _send_reply_queued( + self, + reply_req_id: str, + body: Dict[str, Any], + *, + is_final: bool = False, + skip_if_pending: bool = False, + ) -> Dict[str, Any]: + """Send a reply via aibot_respond_msg with per-req_id ack tracking. + + Args: + reply_req_id: The inbound callback req_id to reply to. + body: Reply body (msgtype: stream/markdown/...). + is_final: If True, wait for any pending ack before sending. + skip_if_pending: If True and a previous frame's ack is pending, + return immediately with {"skipped": True}. + + Returns: + Response dict from WeCom, or {"skipped": True} if skipped. + """ + if not self._ws or self._ws.closed: + raise RuntimeError("WeCom websocket is not connected") + + normalized = str(reply_req_id or "").strip() + if not normalized: + raise ValueError("reply_req_id is required") + + queue = self._reply_queues.get(normalized) + if queue is None: + queue = ReplyQueue(normalized) + self._reply_queues[normalized] = queue + + # NonBlocking semantics: skip if a prior frame ack is pending + if skip_if_pending and queue.pending_ack is not None: + return {"skipped": True, "errcode": 0, "errmsg": "pending_ack"} + + # Final frame: wait for pending ack to drain first + if is_final and queue.pending_ack is not None: + pending_frame = queue.pending_ack + _pending_stream = pending_frame.body.get("stream", {}) if isinstance(pending_frame.body.get("stream"), dict) else {} + logger.debug( + "[%s] _send_reply_queued: final waiting for pending ack drain — " + "req_id=%s pending_stream_id=%s pending_finish=%s pending_sent_at=%.1fs_ago", + self.name, normalized, + _pending_stream.get("id", "N/A"), + _pending_stream.get("finish", "N/A"), + time.monotonic() - (pending_frame.sent_at or time.monotonic()), + ) + try: + await asyncio.wait_for( + asyncio.shield(pending_frame.future), + timeout=self._REPLY_ACK_TIMEOUT, + ) + except asyncio.TimeoutError: + logger.warning( + "[%s] Reply ack timeout waiting for pending (req_id=%s) — " + "pending_stream_id=%s pending_finish=%s elapsed=%.1fs. " + "Possible causes: ack cmd filtered, ack req_id mismatch, or WeCom did not ack.", + self.name, normalized, + _pending_stream.get("id", "N/A"), + _pending_stream.get("finish", "N/A"), + time.monotonic() - (pending_frame.sent_at or time.monotonic()), + ) + except Exception: + pass + # Clear pending regardless — either resolved or timed out + queue.pending_ack = None + + # Create future for THIS frame's ack + future: asyncio.Future = asyncio.get_running_loop().create_future() + frame = ReplyFrame(body=body, future=future, is_final=is_final) + frame.sent_at = time.monotonic() + + # Register as pending BEFORE sending to avoid race: + # If WeCom acks during _send_json await, _dispatch_payload needs + # to find the pending frame to resolve it. Registering after would + # miss the ack and timeout. + # + # Fix (orphan-queue race): re-attach `queue` to the dict before + # registering pending_ack. A final frame shares the inbound req_id + # with the intermediate frames; while it awaits the pending + # intermediate ack to drain (the `is_final` branch above yields at + # `await`), that intermediate ack can arrive and _resolve_reply_ack + # pops the WHOLE queue out of self._reply_queues (the "cleanup empty + # queue" pop). The local `queue` reference captured at the top is then + # an ORPHAN — detached from the dict — so registering pending_ack on it + # is invisible to _dispatch_payload, and the final frame's own ack + # lands in Unrouted → 15s timeout. Writing the reference back here + # closes that window: the final frame's ack can always be routed. + self._reply_queues[normalized] = queue + queue.pending_ack = frame + + # Diagnostic: log every frame send for ack tracking analysis + _stream_info = body.get("stream", {}) if isinstance(body.get("stream"), dict) else {} + logger.debug( + "[%s] _send_reply_queued: req_id=%s is_final=%s skip_if_pending=%s " + "stream_id=%s finish=%s content_len=%d", + self.name, normalized, is_final, skip_if_pending, + _stream_info.get("id", "N/A"), + _stream_info.get("finish", "N/A"), + len(_stream_info.get("content", "") or ""), + ) + + # Send the frame + try: + await self._send_json( + {"cmd": APP_CMD_RESPONSE, "headers": {"req_id": normalized}, "body": body} + ) + except Exception as e: + # Send failed — clear pending and reject future. The future has + # no awaiter on the send-failure branch (we re-raise immediately), + # so cancel it instead of setting an exception that would otherwise + # be logged as "Future exception was never retrieved". + if queue.pending_ack is frame: + queue.pending_ack = None + if not self._reply_queues.get(normalized) or queue.pending_ack is None: + self._reply_queues.pop(normalized, None) + if not future.done(): + future.cancel() + raise + + # For final frames: await the ack (blocking) + if is_final: + try: + response = await asyncio.wait_for(future, timeout=self._REPLY_ACK_TIMEOUT) + return response + except asyncio.TimeoutError: + # Final-frame ack timeout: WeCom received the frame (we wrote + # the bytes successfully — _send_json above did not raise) but + # the ack didn't return within the window. In practice the + # server has already rendered the message to the client; the + # ack is delayed for unrelated reasons (server-side queue lag, + # WS jitter, concurrent reply on the same WS). + # + # The official wecom-openclaw-plugin treats this case as an + # error and surfaces it to its caller, which then *does not* + # resend. Hermes' prior behaviour — raising RuntimeError so + # the upper layer falls back to a normal markdown send — + # produced duplicate messages whenever WeCom *had* rendered + # the streamed frame (see docs/rca-wecom-stream-final-ack- + # timeout-duplicate.md). + # + # Aligning with the official plugin: log a warning and + # synthesise a success-shaped response so the caller treats + # the message as delivered. The thinking-bubble is already + # closed on the client side by the finish=true frame; if WeCom + # never queued it (rare), the user sees no answer — same + # outcome as the official plugin. + logger.warning( + "[%s] Final frame ack timeout (req_id=%s) — treating as " + "delivered (matches official wecom-openclaw-plugin " + "behaviour). No fallback send.", + self.name, normalized, + ) + return { + "errcode": 0, + "errmsg": "ack_timeout_assumed_delivered", + "ack_pending": True, + } + finally: + if queue.pending_ack is frame: + queue.pending_ack = None + # Cleanup empty queue + if queue.pending_ack is None: + self._reply_queues.pop(normalized, None) + else: + # Intermediate frame: fire-and-forget (don't await ack) + # But the pending_ack stays registered so subsequent frames can + # check and skip. The ack will be resolved by _dispatch_payload. + return {"errcode": 0, "errmsg": "sent_nonblocking"} + + def _resolve_reply_ack(self, req_id: str, payload: Dict[str, Any]) -> bool: + """Resolve a pending reply ack. Returns True if handled.""" + queue = self._reply_queues.get(req_id) + if queue is None or queue.pending_ack is None: + return False + frame = queue.pending_ack + if not frame.future.done(): + _body = payload.get("body", {}) if isinstance(payload.get("body"), dict) else {} + logger.debug( + "[%s] _resolve_reply_ack: resolved req_id=%s is_final=%s " + "elapsed=%.2fs errcode=%s", + self.name, req_id, frame.is_final, + time.monotonic() - (frame.sent_at or time.monotonic()), + _body.get("errcode", "N/A"), + ) + frame.future.set_result(payload) + queue.pending_ack = None + # Cleanup empty queue + if queue.pending_ack is None: + self._reply_queues.pop(req_id, None) + return True + + def _fail_reply_queues(self, error: Exception) -> None: + """Fail all pending reply acks (called on disconnect/error).""" + for queue in list(self._reply_queues.values()): + if queue.pending_ack and not queue.pending_ack.future.done(): + queue.pending_ack.future.set_exception(error) + self._reply_queues.clear() + @staticmethod def _new_req_id(prefix: str) -> str: return f"{prefix}-{uuid.uuid4().hex}" @@ -511,8 +1370,31 @@ class WeComAdapter(BasePlatformAdapter): def _parse_json(raw: Any) -> Optional[Dict[str, Any]]: try: payload = json.loads(raw) - except Exception: - logger.debug("Failed to parse WeCom payload: %r", raw) + except json.JSONDecodeError: + # WeCom sometimes sends unescaped control characters (e.g. raw + # newlines) inside JSON string values. Retry with strict=False + # which accepts control chars in strings per the JSON decoder. + try: + decoder = json.JSONDecoder(strict=False) + payload = decoder.decode(raw if isinstance(raw, str) else raw.decode("utf-8", errors="replace")) + logger.info( + "WeCom payload required strict=False fallback (len=%d)", + len(raw) if isinstance(raw, (str, bytes)) else -1, + ) + except Exception as exc2: + logger.warning( + "Failed to parse WeCom payload (strict=False also failed): " + "error=%s len=%d tail=%r", + exc2, + len(raw) if isinstance(raw, (str, bytes)) else -1, + raw[-100:] if isinstance(raw, (str, bytes)) and len(raw) > 100 else raw, + ) + return None + except Exception as exc: + logger.warning( + "Failed to parse WeCom payload: error=%s len=%d", + exc, len(raw) if isinstance(raw, (str, bytes)) else -1, + ) return None return payload if isinstance(payload, dict) else None @@ -528,24 +1410,58 @@ class WeComAdapter(BasePlatformAdapter): msg_id = str(body.get("msgid") or self._payload_req_id(payload) or uuid.uuid4().hex) if self._dedup.is_duplicate(msg_id): - logger.debug("[%s] Duplicate message %s ignored", self.name, msg_id) + # Promoted from debug to INFO: dedup misfire (#62860 timing bug — + # is_duplicate marks at check time, so a msgid redelivered ~5s after + # a processing exception is dropped for the 300s TTL) is a top + # suspect for Coral's intermittent group non-reply. At debug level it + # left zero trace at INFO, so a dropped message looked like it + # vanished after the FULL-payload dump. Log req_id/sender too so this + # line correlates with the inbound FULL payload above. + logger.info( + "[%s] Duplicate message %s ignored (dedup drop) req_id=%s sender=%r chattype=%r", + self.name, + msg_id, + self._payload_req_id(payload), + (body.get("from") or {}).get("userid") if isinstance(body.get("from"), dict) else None, + body.get("chattype"), + ) return self._remember_reply_req_id(msg_id, self._payload_req_id(payload)) sender = body.get("from") if isinstance(body.get("from"), dict) else {} sender_id = str(sender.get("userid") or "").strip() chat_id = str(body.get("chatid") or sender_id).strip() + + # Diagnostic: log the shape of every inbound callback at INFO so we can + # see what WeCom actually sends for group messages (chattype value, + # presence of chatid, msgtype). Group frames may arrive with a + # chattype other than the literal "group" we test for below. + logger.info( + "[%s] Inbound callback: chattype=%r chatid=%r sender=%r msgtype=%r has_chatid=%s", + self.name, + body.get("chattype"), + body.get("chatid"), + sender_id, + body.get("msgtype"), + bool(body.get("chatid")), + ) + if not chat_id: - logger.debug("[%s] Missing chat id, skipping message", self.name) + logger.info("[%s] Missing chat id, skipping message; body_keys=%s", self.name, list(body.keys())) return is_group = str(body.get("chattype") or "").lower() == "group" if is_group: + self._group_chat_ids.add(chat_id) if not self._is_group_allowed(chat_id, sender_id): - logger.debug("[%s] Group %s / sender %s blocked by policy", self.name, chat_id, sender_id) + logger.info( + "[%s] Group message DROPPED by policy: chat=%s sender=%s group_policy=%r " + "(set group_policy to 'open' or add to group_allow_from to receive)", + self.name, chat_id, sender_id, self._group_policy, + ) return elif not self._is_dm_intake_allowed(sender_id): - logger.debug("[%s] DM sender %s blocked by policy", self.name, sender_id) + logger.info("[%s] DM sender %s blocked by policy", self.name, sender_id) return # Cache the inbound req_id after policy checks so proactive sends to @@ -567,7 +1483,10 @@ class WeComAdapter(BasePlatformAdapter): text = reply_text if not text and not media_urls: - logger.debug("[%s] Empty WeCom message skipped", self.name) + logger.info( + "[%s] Empty WeCom message skipped: is_group=%s chat=%s msgtype=%r", + self.name, is_group, chat_id, body.get("msgtype"), + ) return source = self.build_source( @@ -592,7 +1511,27 @@ class WeComAdapter(BasePlatformAdapter): # Only batch plain text messages — commands, media, etc. dispatch # immediately since they won't be split by the WeCom client. - if message_type == MessageType.TEXT and self._text_batch_delay_seconds > 0: + # + # Exception: an attachment-ONLY message (media, no text) is held for + # a short merge window. WeCom clients send "image + text" as TWO + # separate inbound callbacks — an attachment-only frame followed a few + # hundred ms later by the text frame. Dispatching the attachment + # immediately spawns an agent run that the trailing text then + # "interrupts" (junk "⚡ Interrupting" + "✅" acks). Instead we buffer + # the attachment on the SAME pending-batch machinery so the following + # text merges into one event (see _flush_text_batch for the window). + batch_key = self._text_batch_key(event) + has_pending_batch = batch_key in self._pending_text_batches + is_attachment_only = bool(media_urls) and not (text or "").strip() + + if message_type == MessageType.TEXT and ( + self._text_batch_delay_seconds > 0 or has_pending_batch + ): + # Route text through the buffer whenever batching is on OR an + # attachment is already held for this session, so the trailing + # text always merges instead of dispatching on its own. + self._enqueue_text_event(event) + elif is_attachment_only and self._attachment_text_merge_delay_seconds > 0: self._enqueue_text_event(event) else: await self.handle_message(event) @@ -612,11 +1551,18 @@ class WeComAdapter(BasePlatformAdapter): ) def _enqueue_text_event(self, event: MessageEvent) -> None: - """Buffer a text event and reset the flush timer. + """Buffer an event and reset the flush timer. - When WeCom splits a long user message at 4000 chars, the chunks - arrive within a few hundred milliseconds. This merges them into - a single event before dispatching. + Two cases share this buffer: + + * WeCom splits a long user message at 4000 chars — the chunks arrive + within a few hundred milliseconds and are merged into one event. + * WeCom sends "image + text" as two callbacks (an attachment-only + frame, then a text frame). The attachment-only frame is buffered + here first (message_type PHOTO/DOCUMENT/VOICE); when the text frame + arrives it merges into the same event and the type is promoted to + TEXT so it dispatches as a single text-with-media turn — matching + the shape WeCom uses when text+media arrive in one callback. """ key = self._text_batch_key(event) existing = self._pending_text_batches.get(key) @@ -632,6 +1578,16 @@ class WeComAdapter(BasePlatformAdapter): if event.media_urls: existing.media_urls.extend(event.media_urls) existing.media_types.extend(event.media_types) + # Once real text joins a buffered attachment-only event, the merged + # event is a text-with-media turn. Promote the type so downstream + # dispatch treats it like a normal text message (and inherits the + # trailing text frame's reply/quote context if the first frame had + # none). + if event.text and (event.text or "").strip(): + existing.message_type = MessageType.TEXT + if event.reply_to_text and not existing.reply_to_text: + existing.reply_to_text = event.reply_to_text + existing.reply_to_message_id = event.reply_to_message_id # Cancel any pending flush and restart the timer prior_task = self._pending_text_batch_tasks.get(key) @@ -651,7 +1607,15 @@ class WeComAdapter(BasePlatformAdapter): try: pending = self._pending_text_batches.get(key) last_len = getattr(pending, "_last_chunk_len", 0) if pending else 0 - if last_len >= self._SPLIT_THRESHOLD: + # An attachment-only buffered event (no text yet) waits the + # attachment/text merge window for a trailing text frame. A text + # buffered event uses the normal (or split-continuation) delay. + is_attachment_only = bool( + pending and pending.media_urls and not (pending.text or "").strip() + ) + if is_attachment_only: + delay = self._attachment_text_merge_delay_seconds + elif last_len >= self._SPLIT_THRESHOLD: delay = self._text_batch_split_delay_seconds else: delay = self._text_batch_delay_seconds @@ -672,8 +1636,8 @@ class WeComAdapter(BasePlatformAdapter): if not event: return logger.info( - "[WeCom] Flushing text batch %s (%d chars)", - key, len(event.text or ""), + "[WeCom] Flushing batch %s (%d chars, %d media)", + key, len(event.text or ""), len(event.media_urls or []), ) await self.handle_message(event) finally: @@ -973,6 +1937,315 @@ class WeComAdapter(BasePlatformAdapter): self._last_chat_req_ids[normalized_chat_id] = normalized_req_id while len(self._last_chat_req_ids) > DEDUP_MAX_SIZE: self._last_chat_req_ids.pop(next(iter(self._last_chat_req_ids))) + # A fresh inbound req_id resurrects the stream channel — drop any + # stale "stream is dead" marker from prior 846608 responses so the + # next outbound turn can attempt native streaming again. + self._stream_expired_chats.discard(normalized_chat_id) + # A new inbound message starts a new "turn" — allow send_typing to + # open a fresh stream again (the previous turn's delivery guard is + # no longer relevant). + + def _resolve_stream_req_id( + self, chat_id: str, reply_to: Optional[str] + ) -> Optional[str]: + """Pick a req_id for a stream reply. + + Precedence: explicit ``reply_to`` (a prior message id we cached) → + last inbound req_id for this chat → ``None`` (stream impossible). + """ + req_id = self._reply_req_id_for_message(reply_to) + if req_id: + return req_id + return self._last_chat_req_ids.get(str(chat_id or "").strip()) or None + + def _get_or_create_stream_turn(self, chat_id: str, req_id: str) -> StreamTurn: + """Get or create a StreamTurn for the given chat and req_id.""" + key = f"{chat_id}:{req_id}" + if key not in self._stream_turns: + self._stream_turns[key] = StreamTurn(chat_id, req_id) + return self._stream_turns[key] + + def _cleanup_stream_turn(self, chat_id: str, req_id: str) -> None: + """Clean up a StreamTurn after finalization or error.""" + key = f"{chat_id}:{req_id}" + turn = self._stream_turns.pop(key, None) + if turn is not None: + self._cancel_idle_flush(turn) + self._cancel_keepalive(turn) + + def _cancel_idle_flush(self, turn: StreamTurn) -> None: + """Cancel a pending idle-flush timer on the turn (no-op if unarmed).""" + handle = turn.idle_flush_handle + if handle is not None: + try: + handle.cancel() + except Exception: + pass + turn.idle_flush_handle = None + + def _arm_idle_flush( + self, + turn: StreamTurn, + *, + turn_id: Optional[str], + ) -> None: + """Arm the 250ms idle-flush timer so a partial buffer still ships. + + When the LLM pauses (model thinking, network slow), the chunker's + ``min_chars`` gate would hold a half-sentence forever. The official + plugin's ``blockStreamingCoalesce.idleMs = 250`` solves this by + flushing whatever is buffered after 250ms of silence. + + The timer is per-turn and idempotent — re-arming while one is + pending no-ops; a new timer is only scheduled after the previous + one has fired or been cancelled. + """ + if turn.idle_flush_handle is not None: + return # already armed + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return # not inside a loop (defensive — should never happen here) + handle = loop.call_later( + BLOCK_STREAM_IDLE_FLUSH, + self._on_idle_flush_fire, + turn, + turn_id, + ) + turn.idle_flush_handle = handle + + def _on_idle_flush_fire( + self, + turn: StreamTurn, + turn_id: Optional[str], + ) -> None: + """Loop callback — dispatch an async flush task without blocking.""" + turn.idle_flush_handle = None + if turn.finalized or turn.expired: + return + if turn.chunker is None or not turn.chunker.has_pending(): + return + # Schedule on the running loop; swallow scheduling errors so a stale + # timer firing during shutdown can never bubble up. + try: + asyncio.ensure_future(self._idle_flush_send(turn, turn_id)) + except RuntimeError: + pass + + async def _idle_flush_send( + self, + turn: StreamTurn, + turn_id: Optional[str], + ) -> None: + """Drain the chunker's pending tail and send one intermediate frame. + + Called from the idle-flush timer when the consumer hasn't pushed a + fresh cumulative snapshot for ``BLOCK_STREAM_IDLE_FLUSH`` seconds. + Force-drains the chunker (force=True) so even a sub-min_chars tail + reaches the user. + """ + if turn.finalized or turn.expired: + return + if turn._intermediate_frames_sent >= MAX_INTERMEDIATE_FRAMES: + return + chunker = turn.chunker + if chunker is None or not chunker.has_pending(): + return + drained = chunker.drain(force=True) + if drained is None or not drained: + return + try: + await self._send_stream_reply( + turn.req_id, + turn.stream_id, + drained, + finish=False, + ) + except WeComStreamExpiredError: + turn.expired = True + self._retire_turn(turn, turn_id) + self._stream_expired_chats.add(turn.chat_id) + return + except Exception as exc: + logger.debug( + "[%s] idle-flush send failed (chat=%s, turn=%s): %s", + self.name, turn.chat_id, turn.stream_id, exc, + ) + return + turn._last_frame_sent_at = time.monotonic() + turn._intermediate_frames_sent += 1 + turn.last_sent_content = drained + + # ── Stream-level keep-alive (Layer 1) ───────────────────────────────── + # Structurally mirrors the idle-flush timer: a per-turn asyncio TimerHandle + # stored on the StreamTurn, cancelled on every turn-exit path. The only + # difference from idle-flush is cadence (minutes vs 250ms) and intent + # (refresh the server's 6-min stream window vs ship a partial buffer). + + def _cancel_keepalive(self, turn: StreamTurn) -> None: + """Cancel a pending keep-alive timer on the turn (no-op if unarmed).""" + handle = turn.keepalive_handle + if handle is not None: + try: + handle.cancel() + except Exception: + pass + turn.keepalive_handle = None + + def _arm_keepalive( + self, + turn: StreamTurn, + *, + turn_id: Optional[str], + ) -> None: + """Arm the keep-alive timer (Layer 1) if enabled and not already armed. + + Idempotent — re-arming while one is pending no-ops; a fresh timer is + only scheduled after the previous one fired or was cancelled. Skips + entirely when keep-alive is disabled by config (the default). + """ + if not self._stream_keepalive_enabled: + return + if turn.finalized or turn.expired: + return + if turn.keepalive_handle is not None: + return # already armed + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return # not inside a loop (defensive) + handle = loop.call_later( + self._stream_keepalive_interval_seconds, + self._on_keepalive_fire, + turn, + turn_id, + ) + turn.keepalive_handle = handle + + def _on_keepalive_fire( + self, + turn: StreamTurn, + turn_id: Optional[str], + ) -> None: + """Loop callback — dispatch an async keep-alive send without blocking.""" + turn.keepalive_handle = None + if turn.finalized or turn.expired: + return + try: + asyncio.ensure_future(self._keepalive_send(turn, turn_id)) + except RuntimeError: + pass + + async def _keepalive_send( + self, + turn: StreamTurn, + turn_id: Optional[str], + ) -> None: + """Re-send the accumulated text as a finish=false frame to refresh the + WeCom server's stream window, then re-arm for the next interval. + + Deliberately conservative (see ANALYSIS §4.2/§5): + + * **Never sends a placeholder.** When there is no accumulated text yet + (e.g. a cron turn still fetching data), the tick is skipped and the + timer is re-armed — we'd rather let Layer 2's clock fallback handle a + content-less turn than pollute ``last_sent_content`` with a filler + frame or strand the user on a "still working…" bubble. + * Reuses ``_send_stream_reply(finish=False)`` with the queue's + ``skip_if_pending`` semantics, so a heartbeat that races an in-flight + ack is dropped rather than piling onto the ack queue. + * On 846604/846608 marks the turn expired and retires it so finalize + takes the Layer 2 fallback; stops the timer (no re-arm). + """ + if turn.finalized or turn.expired: + return + if turn._intermediate_frames_sent >= MAX_INTERMEDIATE_FRAMES: + # No room left for intermediate frames; stop keeping alive and let + # finalize (or Layer 2) run. Do not re-arm. + return + content = turn.accumulated_text or "" + if not content.strip(): + # Nothing to refresh with yet — skip this tick, re-arm for later. + self._arm_keepalive(turn, turn_id=turn_id) + return + try: + await self._send_stream_reply( + turn.req_id, + turn.stream_id, + content, + finish=False, + ) + except WeComStreamExpiredError: + turn.expired = True + self._retire_turn(turn, turn_id) + self._stream_expired_chats.add(turn.chat_id) + return + except Exception as exc: + logger.debug( + "[%s] keep-alive send failed (chat=%s, turn=%s): %s", + self.name, turn.chat_id, turn.stream_id, exc, + ) + # Transient failure — re-arm and try again next interval. + self._arm_keepalive(turn, turn_id=turn_id) + return + turn._last_frame_sent_at = time.monotonic() + turn.last_sent_content = content + # Re-arm for the next interval (guarded internally against + # finalized/expired). + self._arm_keepalive(turn, turn_id=turn_id) + + def _retire_turn(self, turn: StreamTurn, turn_id: Optional[str]) -> None: + """Remove a turn from the registry and cancel BOTH of its timers. + + Single choke point for the "turn is dead" cleanup shared by the + expired/error paths. Cancels idle-flush and keep-alive timers before + popping so neither can fire on a retired turn. + """ + self._cancel_idle_flush(turn) + self._cancel_keepalive(turn) + if turn_id: + self._stream_turns.pop(f"{turn.chat_id}:{turn_id}", None) + else: + self._cleanup_stream_turn(turn.chat_id, turn.req_id) + + def _find_active_turn_for_chat(self, chat_id: str) -> Optional[StreamTurn]: + """Find the most recent active (non-finalized) turn for a chat.""" + for turn in self._stream_turns.values(): + if turn.chat_id == chat_id and not turn.finalized: + return turn + return None + + def _reset_native_stream_state(self) -> None: + """Legacy method for compatibility. Now a no-op since state is per-turn.""" + # No-op: stream state is now per-turn, not global. + # Kept for compatibility with existing code that calls this. + pass + + async def _force_reconnect_on_stale_subscription(self, errcode: int) -> None: + """Force-close the WS when server rejects our subscription (846609). + + WeCom errcode 846609 means the server no longer considers this WS + session subscribed — all sends will fail until we reconnect. Rather + than waiting for the WS to close naturally (can take 2+ minutes of + timeouts), we proactively close it to trigger _listen_loop's + reconnect cycle immediately. + """ + if errcode != STREAM_NOT_SUBSCRIBED_ERRCODE: + return + logger.warning( + "[%s] Got errcode %d (subscription lost) — clearing stale state", + self.name, errcode, + ) + # Only invalidate cached req_ids (bound to the dead session). + # Do NOT close the WS — closing triggers _listen_loop to reconnect, + # which opens a second WS connection. WeCom only allows one long-lived + # connection per bot; the server kicks the second one and invalidates + # the first's session, creating an infinite kick-reconnect loop. + # The WS will be closed by the server side naturally; _listen_loop + # handles the reconnect when that happens. + self._last_chat_req_ids.clear() + self._reply_req_ids.clear() + self._reset_native_stream_state() def _reply_req_id_for_message(self, reply_to: Optional[str]) -> Optional[str]: normalized = str(reply_to or "").strip() @@ -1309,6 +2582,99 @@ class WeComAdapter(BasePlatformAdapter): self._raise_for_wecom_error(response, "send reply markdown") return response + @staticmethod + def _truncate_stream_content(content: str, limit: int) -> str: + """Truncate ``content`` to fit within ``limit`` UTF-8 bytes. + + WeCom enforces a byte-length cap on stream frames; truncating by + codepoints would still let multi-byte runs blow past the limit. + """ + encoded = content.encode("utf-8") + if len(encoded) <= limit: + return content + return encoded[:limit].decode("utf-8", errors="ignore") + + async def _send_stream_reply( + self, + reply_req_id: str, + stream_id: str, + content: str, + finish: bool = False, + ) -> Dict[str, Any]: + """Send a single ``msgtype: "stream"`` frame via aibot_respond_msg. + + Uses the per-req_id reply queue with ack tracking, aligned with the + official WeCom SDK's replyStreamNonBlocking semantics: + + * **Intermediate frames** (finish=False): sent non-blocking via + ``_send_reply_queued(skip_if_pending=True)``. If a prior frame's + ack is still pending, the frame is skipped (cumulative text means + no information is lost — the next frame carries all content). + * **Final frame** (finish=True): waits for any pending ack to drain + before sending, then awaits its own ack. This prevents version + conflicts (errcode 6000) between the finalize and a concurrent + intermediate frame. + + Raises :class:`WeComStreamExpiredError` on errcode 846608 so the + caller can fall back to a proactive markdown send. + """ + truncated = self._truncate_stream_content( + content or "", self.MAX_STREAM_CONTENT_LENGTH, + ) + if len(content or "") != len(truncated): + logger.warning( + "[%s] Stream content truncated for stream_id=%s", + self.name, stream_id, + ) + body: Dict[str, Any] = { + "msgtype": "stream", + "stream": { + "id": stream_id, + "finish": bool(finish), + "content": truncated, + }, + } + + if not finish: + # Intermediate frame: non-blocking with pending-skip semantics. + # If a previous frame's ack is still pending on this req_id, + # skip this frame entirely (cumulative text guarantees no loss). + response = await self._send_reply_queued( + reply_req_id, body, is_final=False, skip_if_pending=True, + ) + return response + + # Final frame: wait for any pending intermediate ack, then send + # with ack tracking so we reliably detect 846608/6000. + response = await self._send_reply_queued( + reply_req_id, body, is_final=True, skip_if_pending=False, + ) + errcode = response.get("errcode", 0) + if errcode in (STREAM_EXPIRED_ERRCODE, STREAM_REQUEST_EXPIRED_ERRCODE): + # 846608 (stream update window) and 846604 (req_id reply-request + # window) both mean the reply flow is dead — raise the same + # expired error so the caller falls back to a proactive send. + raise WeComStreamExpiredError( + errcode=errcode, errmsg=str(response.get("errmsg") or ""), + ) + if errcode == STREAM_VERSION_CONFLICT_ERRCODE: + # 6000 = version conflict: a newer frame on this stream_id already + # replaced the bubble. For a finalize frame this means the content + # is ALREADY on screen (idempotent re-finalize losing the race to a + # newer version), so treat it as delivered rather than raising — + # raising here would pop the turn and drop us into a duplicate + # standalone send(). This is what makes idempotent finalize retry + # safe: retrying a finalize that already landed returns 6000, which + # we now absorb instead of turning into a second message. + logger.info( + "[%s] finalize hit errcode 6000 (version conflict) — bubble " + "already replaced by a newer frame; treating as delivered.", + self.name, + ) + return response + self._raise_for_wecom_error(response, "send stream reply") + return response + async def _send_reply_media_message( self, reply_req_id: str, @@ -1369,25 +2735,40 @@ class WeComAdapter(BasePlatformAdapter): if not reply_req_id and chat_id in self._last_chat_req_ids: reply_req_id = self._last_chat_req_ids[chat_id] + # When native streaming was/is active for this chat, media MUST go + # through the proactive send path (aibot_send_msg), NOT passive reply + # (aibot_respond_msg). This mirrors the official OpenClaw plugin: + # "replyMedia(被动回复)无法覆盖 replyStream 发出的 thinking 流式消息, + # 因此所有媒体统一走 aibot_send_msg 主动发送。" + # The reply_req_id is "owned" by the stream — using it for media + # causes the server to either ignore it or never ack. + active_turn = self._find_active_turn_for_chat(chat_id) + if active_turn or chat_id in self._stream_expired_chats: + reply_req_id = None # force proactive send + try: upload_result = await self._upload_media_bytes( prepared["data"], prepared["final_type"], prepared["file_name"], ) + logger.info("[%s] upload_media_bytes OK: media_id=%s type=%s", self.name, upload_result.get("media_id"), prepared["final_type"]) if reply_req_id: media_response = await self._send_reply_media_message( reply_req_id, prepared["final_type"], upload_result["media_id"], ) + logger.info("[%s] send_reply_media OK: %s", self.name, media_response) else: media_response = await self._send_media_message( chat_id, prepared["final_type"], upload_result["media_id"], ) + logger.info("[%s] send_media_message OK: %s", self.name, media_response) except asyncio.TimeoutError: + logger.error("[%s] TIMEOUT in _send_media_source for %s", self.name, media_source) return SendResult(success=False, error="Timeout sending media to WeCom") except Exception as exc: logger.error("[%s] Failed to send media %s: %s", self.name, media_source, exc) @@ -1428,21 +2809,111 @@ class WeComAdapter(BasePlatformAdapter): reply_to: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, ) -> SendResult: - """Send markdown to a WeCom chat via proactive ``aibot_send_msg``.""" - del metadata + """Send markdown to a WeCom chat. + Sends content as a standalone message without interfering with any + active streams. Streams are managed by their creators (typically + GatewayStreamConsumer) who call send_stream_frame(finalize=True) + when ready. + + All sends are serialized per chat_id to avoid exceeding WeCom's + 30 msgs/min/chat rate limit (errcode 846607). + + If metadata contains "is_approval_prompt": True, the message is routed + through the control lane for immediate delivery. + """ if not chat_id: return SendResult(success=False, error="chat_id is required") + # Check if this is an approval prompt (should use control lane) + is_control = False + force_proactive = False + if metadata: + is_control = metadata.pop("is_approval_prompt", False) + # Explicit opt-in for proactive send: used by approval + # *confirmation* messages (post-/approve) that must not consume + # the req_id the stream consumer needs for resumed output. + # Distinct from is_approval_prompt which only routes to the + # control lane — the initial approval *request* prompt still + # uses passive reply (required for groups where APP_CMD_SEND + # is blocked). + force_proactive = bool(metadata.pop("force_proactive_send", False)) + + return await self._enqueue_chat_send( + chat_id, + lambda: self._send_inner(chat_id, content, reply_to, force_proactive=force_proactive), + is_control=is_control, + ) + + async def _send_inner( + self, + chat_id: str, + content: str, + reply_to: Optional[str] = None, + *, + force_proactive: bool = False, + ) -> SendResult: + """Actual send logic, called under the per-chat lock. + + Sends content as a standalone message. Does NOT close any active + streams — streams are managed by their creators (GatewayStreamConsumer) + who call send_stream_frame(finalize=True) when ready. + + This aligns with the official wecom-openclaw-plugin model where + send() and streaming are independent operations. + + Args: + force_proactive: When True, always use APP_CMD_SEND instead of + passive reply. Used for approval confirmations to avoid + consuming the req_id needed by the post-approval stream. + """ try: + # Directly send the message without touching any active streams. + # GatewayStreamConsumer manages its own stream lifecycle via + # send_stream_frame() with turn_id, so send() shouldn't interfere. + reply_req_id = self._reply_req_id_for_message(reply_to) if not reply_req_id and chat_id in self._last_chat_req_ids: reply_req_id = self._last_chat_req_ids[chat_id] + if force_proactive and chat_id not in self._group_chat_ids: + reply_req_id = None + if reply_req_id: - response = await self._send_reply_markdown(reply_req_id, content) + try: + response = await self._send_reply_markdown(reply_req_id, content) + except (asyncio.TimeoutError, RuntimeError) as passive_err: + # Passive reply failed (req_id may be stale after WS reconnect). + # Fall back to proactive aibot_send_msg which doesn't depend + # on any prior req_id. + logger.warning( + "[%s] Passive reply failed (%s), falling back to proactive send", + self.name, passive_err, + ) + response = await self._send_request( + APP_CMD_SEND, + { + "chatid": chat_id, + "msgtype": "markdown", + "markdown": {"content": content[:self.MAX_MESSAGE_LENGTH]}, + }, + ) else: + # No req_id available — must use proactive APP_CMD_SEND. + # Group chats cannot use APP_CMD_SEND (WeCom blocks it), + # so fail early with a clear error instead of making a + # doomed network request. + if chat_id in self._group_chat_ids: + logger.warning( + "[%s] No cached req_id for group chat %s — " + "cannot send (groups require passive reply via req_id)", + self.name, chat_id, + ) + return SendResult( + success=False, + error="No req_id available for group chat (passive reply required)", + ) response = await self._send_request( APP_CMD_SEND, { @@ -1455,12 +2926,28 @@ class WeComAdapter(BasePlatformAdapter): return SendResult(success=False, error="Timeout sending message to WeCom") except Exception as exc: logger.error("[%s] Send failed: %s", self.name, exc) + # Detect 846609 (subscription lost) and trigger reconnect so + # subsequent messages don't fail for 2+ minutes while the dead + # WS connection lingers. + exc_str = str(exc) + if str(STREAM_NOT_SUBSCRIBED_ERRCODE) in exc_str: + asyncio.ensure_future( + self._force_reconnect_on_stale_subscription(STREAM_NOT_SUBSCRIBED_ERRCODE) + ) return SendResult(success=False, error=str(exc)) error = self._response_error(response) if error: + # Also check the response-level errcode for 846609. + errcode = response.get("errcode", 0) + if errcode == STREAM_NOT_SUBSCRIBED_ERRCODE: + asyncio.ensure_future( + self._force_reconnect_on_stale_subscription(errcode) + ) return SendResult(success=False, error=error) + # Mark delivered so _keep_typing cannot open an orphan stream after + # this turn's reply already landed (regardless of which path was taken). return SendResult( success=True, message_id=self._payload_req_id(response) or uuid.uuid4().hex[:12], @@ -1516,6 +3003,7 @@ class WeComAdapter(BasePlatformAdapter): **kwargs, ) -> SendResult: del kwargs + logger.info("[%s] send_document called: chat=%s file=%s", self.name, chat_id, file_path) return await self._send_media_source( chat_id=chat_id, media_source=file_path, @@ -1556,8 +3044,362 @@ class WeComAdapter(BasePlatformAdapter): reply_to=reply_to, ) + async def send_stream_frame( + self, + text: str, + *, + finalize: bool = False, + chat_id: Optional[str] = None, + reply_to: Optional[str] = None, + **kwargs, + ) -> bool: + """Public entry-point for the gateway streaming consumer. + + Native streaming lifecycle (per-turn): + * **First call** for a turn: resolve req_id, create StreamTurn, + and send an empty seed frame to trigger WeCom typing animation. + * **Subsequent calls**: reuse the same StreamTurn's stream_id + + Args: + **kwargs: Additional platform-specific parameters. Currently supports: + - turn_id (str): Optional unique identifier for this turn. When + provided, the StreamTurn is keyed by (chat_id, turn_id) instead + of (chat_id, req_id), preventing concurrent consumers (e.g., + /background, parallel subagents) from interfering with each + other. Mirrors official wecom-openclaw-plugin's per-message + streamId model. + and push cumulative text (not deltas) for in-place updates. + * **finalize=True**: send closing frame and clean up turn state. + + Each turn (chat_id + req_id) maintains independent state, allowing + concurrent messages without interference (e.g., approval during streaming). + + Returns ``True`` when the frame landed; ``False`` when the + stream is unavailable (no req_id, expired session, transport + error). On ``False`` the caller should fall back to + :meth:`send` to deliver the remaining content as a one-shot + markdown reply. + """ + chat = (chat_id or "").strip() + if not chat: + logger.warning( + "[%s] send_stream_frame: chat_id required", + self.name, + ) + return False + + # Extract turn_id early to decide whether to check chat-level expired + turn_id = kwargs.get("turn_id") + + # Chat-level stream expiry only blocks NEW turn creation. + # Existing turns (identified by turn_id) can continue to finalize + # even after another turn in the same chat triggered WeComStreamExpiredError. + # This prevents cross-turn interference in concurrent scenarios. + if not turn_id and chat in self._stream_expired_chats: + # No turn_id provided, and chat is expired → block new turn creation + return False + + if finalize: + # Finalize frame counts toward 30/min — go through the control queue + # (high priority) to prevent blocking by normal messages or other streams. + turn_id = kwargs.get("turn_id") + return await self._enqueue_chat_send( + chat, + lambda: self._send_stream_frame_inner(text, chat=chat, reply_to=reply_to, finalize=True, turn_id=turn_id), + is_control=True, + ) + else: + # Intermediate frames: fire-and-forget, no queue, no rate limit. + # WeCom does NOT count them toward the 30/min quota. + turn_id = kwargs.get("turn_id") + return await self._send_stream_frame_inner(text, chat=chat, reply_to=reply_to, finalize=False, turn_id=turn_id) + + async def _send_stream_frame_inner( + self, + text: str, + *, + chat: str, + reply_to: Optional[str] = None, + finalize: bool = False, + turn_id: Optional[str] = None, + ) -> bool: + """Actual stream frame logic with per-turn state. + + Each turn (identified by chat_id + turn_id OR chat_id + req_id) + maintains its own stream state. This prevents concurrent messages + from interfering with each other. + + When turn_id is provided (from GatewayStreamConsumer), the turn is + keyed by (chat, turn_id) instead of (chat, req_id). This ensures + concurrent consumers (e.g., /background, parallel subagents) maintain + independent streams. + + IMPORTANT: Once a turn is created, it locks to its req_id. Even if + _last_chat_req_ids[chat] changes (e.g., user sends /approve), the + existing turn continues with its original req_id. This prevents the + stream from switching to a new req_id mid-turn. + """ + try: + # If turn_id is provided, use it to find/create the turn. + # This is the true per-turn model that prevents concurrent + # consumers from interfering. + if turn_id: + turn_key = f"{chat}:{turn_id}" + turn = self._stream_turns.get(turn_key) + if not turn: + # finalize=True should NOT create a new turn. + # If the turn was already cleaned up (e.g., due to errcode 6000), + # the caller should fallback to proactive send() instead of + # creating a fresh turn just to finalize it (which would send + # another seed + finish, potentially triggering more conflicts). + if finalize: + logger.debug( + "[%s] send_stream_frame: cannot finalize non-existent turn (turn_id=%s, chat=%s)", + self.name, turn_id, chat, + ) + return False + + # First frame for this turn: need to create it. + # Check if chat is expired (blocks NEW turn creation). + if chat in self._stream_expired_chats: + logger.debug( + "[%s] send_stream_frame: chat %s is expired, cannot create new turn (turn_id=%s)", + self.name, chat, turn_id, + ) + return False + + # First frame for this turn: resolve req_id and create turn + req_id = self._resolve_stream_req_id(chat, reply_to) + if not req_id: + logger.debug( + "[%s] send_stream_frame: no req_id available for chat %s (turn_id=%s)", + self.name, chat, turn_id, + ) + return False + turn = StreamTurn(chat, req_id) + self._stream_turns[turn_key] = turn + logger.debug( + "[%s] send_stream_frame: created new turn %s (turn_id=%s, req_id=%s) for chat %s", + self.name, turn.stream_id, turn_id, req_id, chat, + ) + else: + # Fallback: no turn_id provided (backward compatibility or direct calls). + # Check if we already have an active turn for this chat. + # If yes, reuse it (don't resolve req_id again). + existing_turn = self._find_active_turn_for_chat(chat) + if existing_turn and not existing_turn.finalized: + turn = existing_turn + logger.debug( + "[%s] send_stream_frame: reusing existing turn %s for chat %s", + self.name, turn.stream_id, chat, + ) + else: + # No active turn, need to create a new one. + # Check if chat is expired at the chat level (blocks NEW turn creation). + if chat in self._stream_expired_chats: + logger.debug( + "[%s] send_stream_frame: chat %s is expired, cannot create new turn", + self.name, chat, + ) + return False + + req_id = self._resolve_stream_req_id(chat, reply_to) + if not req_id: + logger.debug( + "[%s] send_stream_frame: no req_id available for chat %s", + self.name, chat, + ) + return False + turn = self._get_or_create_stream_turn(chat, req_id) + logger.debug( + "[%s] send_stream_frame: created new turn %s (req_id=%s) for chat %s", + self.name, turn.stream_id, req_id, chat, + ) + + # Check if this turn has expired + if turn.expired: + return False + + # First frame for this turn: send seed ONLY if not already seeded. + # The GatewayStreamConsumer sends the initial empty seed frame itself + # (stream_consumer.py:461), so we must not duplicate it here. + # The seeded flag prevents double-seed which causes WeCom errcode 6000 + # (data version conflict). + if not turn.seeded and not turn.finalized: + # Seed frame with closed empty — matches the + # official OpenClaw plugin's THINKING_MESSAGE constant. This + # tells the WeCom client that a reasoning turn is starting; + # subsequent frames replace it with cumulative content. + await self._send_stream_reply( + turn.req_id, turn.stream_id, + "", finish=False, + ) + turn.seeded = True + # Stream is now open on the server — arm the keep-alive timer + # (Layer 1) so long, content-sparse turns refresh the 6-min + # window. No-op when keep-alive is disabled by config. + self._arm_keepalive(turn, turn_id=turn_id) + # If caller sent empty text (consumer's explicit seed call), + # we're done — don't send another empty frame below. + if not text and not finalize: + return True + + # Send the frame + if finalize: + # ── Layer 2 clock fallback ─────────────────────────────── + # If the stream is older than the safe duration, the finish + # frame would almost certainly hit 846604/846608. Decline it + # up front: mark the turn expired, retire it (cancels both + # timers), and return False so the gateway consumer's existing + # fallback send() path delivers the content exactly once — the + # same contract the WeComStreamExpiredError path already uses + # (stream_consumer.py rolls back _final_content_delivered on a + # False finalize). Zero new uplink frames; safe for groups in + # the sense that it does not make delivery any worse than the + # current 846608 fallback. + stream_age = time.monotonic() - turn.start_time + if stream_age >= self._stream_safe_duration_seconds: + logger.info( + "[%s] Stream age %.0fs >= safe duration %.0fs for chat " + "%s — declining finalize frame, falling back to " + "proactive send (Layer 2 clock fallback).", + self.name, stream_age, + self._stream_safe_duration_seconds, chat, + ) + turn.expired = True + self._retire_turn(turn, turn_id) + self._stream_expired_chats.add(chat) + return False + + # Force-drain any buffered tail through the chunker so the + # final frame carries the latest content — this is what the + # official plugin does in `finishThinkingStream`. The + # chunker call also cancels the idle-flush timer. + self._cancel_idle_flush(turn) + self._cancel_keepalive(turn) + if turn.chunker is not None: + turn.chunker.update(text) + drained = turn.chunker.drain(force=True) + if drained is not None: + text = drained + + # WeCom may silently drop (no ack) a final frame whose content + # is identical to the preceding intermediate frame — it treats + # the frame as a duplicate despite the finish flag change. + # Append a zero-width space to ensure the content differs when + # the text matches the last ACTUALLY SENT intermediate content. + final_text = text + if text and text == turn.last_sent_content: + final_text = text + "​" # zero-width space + await self._send_stream_reply( + turn.req_id, + turn.stream_id, + final_text, + finish=True, + ) + turn.finalized = True + # Clean up this turn's state + # If turn_id was provided, the key is chat:turn_id, otherwise chat:req_id + if turn_id: + turn_key = f"{chat}:{turn_id}" + self._stream_turns.pop(turn_key, None) + else: + self._cleanup_stream_turn(chat, turn.req_id) + else: + # Block-streaming gate (aligned with wecom-openclaw-plugin): + # the consumer hands us a *cumulative* text snapshot every + # tick. Feed it to the chunker and only put a frame on the + # wire when a complete sentence-aligned block is ready. + # Otherwise schedule a 250ms idle flush so partial buffers + # don't sit forever when the LLM slows down. + if turn.chunker is None: + turn.chunker = _BlockChunker() + turn.chunker.update(text) + + if turn._intermediate_frames_sent >= MAX_INTERMEDIATE_FRAMES: + # Frame cap reached — drop intermediates, keep accumulating. + # The finalize path will drain whatever is left. + turn.accumulated_text = text + return True + + drained = turn.chunker.drain(force=False) + if drained is None: + # Not enough content for a block yet. Make sure the + # idle-flush timer is armed so a partial buffer still + # reaches the user when the LLM pauses. + self._arm_idle_flush(turn, turn_id=turn_id) + turn.accumulated_text = text + return True + + # We have a block — cancel any pending idle flush, send the + # frame, then re-arm only if there's still pending content + # in the chunker (i.e., another block partway grown). + self._cancel_idle_flush(turn) + + await self._send_stream_reply( + turn.req_id, + turn.stream_id, + drained, + finish=False, + ) + turn._last_frame_sent_at = time.monotonic() + turn._intermediate_frames_sent += 1 + turn.accumulated_text = text + turn.last_sent_content = drained + + if turn.chunker.has_pending(): + self._arm_idle_flush(turn, turn_id=turn_id) + + return True + + except WeComStreamExpiredError: + logger.info( + "[%s] Stream expired (errcode=%d) for chat %s — switching to proactive send", + self.name, STREAM_EXPIRED_ERRCODE, chat, + ) + # Mark this specific turn as expired and clean it up + if 'turn' in locals(): + turn.expired = True + self._retire_turn(turn, turn_id) + + # Mark the chat as stream-expired to prevent new stream attempts. + # Other concurrent turns may continue if they're already active. + self._stream_expired_chats.add(chat) + return False + except Exception as exc: + logger.warning( + "[%s] Stream frame failed (chat=%s): %s", + self.name, chat, exc, + ) + # Clean up this turn on error + if 'turn' in locals(): + self._retire_turn(turn, turn_id) + return False + + def supports_native_streaming( + self, + chat_type: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> bool: + """Probed by ``GatewayStreamConsumer`` to gate native streaming. + + WeCom AI Bot supports stream frames in both DMs and groups; group + chats just need a cached inbound ``req_id`` (every group message + the bot receives populates ``_last_chat_req_ids``, so this is + effectively always satisfied for actively-used groups). + """ + del chat_type, metadata + return True + async def send_typing(self, chat_id: str, metadata=None) -> None: - """WeCom does not expose typing indicators in this adapter.""" + """No-op: WeCom typing is handled by the stream consumer seed frame. + + The stream consumer sends an empty seed frame at the start of run(), + which is what triggers WeCom's typing animation. _keep_typing loops + are designed for platforms where typing expires (Telegram 5s) — WeCom + streams stay open indefinitely, so repeated send_typing calls cause + orphan streams. Delegating entirely to the consumer avoids the race. + """ del chat_id, metadata async def get_chat_info(self, chat_id: str) -> Dict[str, Any]: @@ -1721,13 +3563,43 @@ async def _standalone_send( media_files=None, force_document=False, ): - """Out-of-process WeCom delivery via the adapter's WebSocket send pipeline. + """WeCom delivery via live gateway adapter or ephemeral connection. - Implements the standalone_sender_fn contract so deliver=wecom cron jobs - succeed when cron runs separately from the gateway. Opens an ephemeral - WeComAdapter, connects, sends, and disconnects. Replaces the legacy - _send_wecom helper. + Implements the standalone_sender_fn contract. WeCom only allows ONE + WebSocket connection per bot — opening a second kicks the first. So + when the gateway is running in-process, we reuse the live adapter. + Only when running out-of-process (cron separate from gateway) do we + open an ephemeral connection. """ + # Prefer the live gateway adapter to avoid kicking the main connection. + try: + from gateway.run import _gateway_runner_ref + runner = _gateway_runner_ref() + except Exception: + runner = None + + if runner is not None: + from gateway.platforms.base import Platform + adapter = None + try: + adapter = runner.adapters.get(Platform.WECOM) + except Exception: + pass + if adapter is not None: + try: + result = await adapter.send(chat_id, message) + if not result.success: + return {"error": f"WeCom send failed: {result.error}"} + return { + "success": True, + "platform": "wecom", + "chat_id": chat_id, + "message_id": result.message_id, + } + except Exception as e: + return {"error": f"WeCom live adapter send failed: {e}"} + + # Fallback: out-of-process — open ephemeral connection. if not check_wecom_requirements(): return {"error": "WeCom requirements not met. Need aiohttp + WECOM_BOT_ID/SECRET."} try: diff --git a/tests/gateway/test_approval_boundary.py b/tests/gateway/test_approval_boundary.py new file mode 100644 index 0000000000..c4cfa956fa --- /dev/null +++ b/tests/gateway/test_approval_boundary.py @@ -0,0 +1,414 @@ +"""Tests for approval boundary handling in WeCom native streaming.""" +import asyncio +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig + + +@pytest.fixture +def mock_adapter(): + """Create a mock WeCom adapter with native streaming support.""" + from gateway.platforms.base import BasePlatformAdapter + MockAdapter = type("MockAdapter", (BasePlatformAdapter,), { + "MAX_MESSAGE_LENGTH": 4096, + "SUPPORTS_MESSAGE_EDITING": False, + "SUPPORTS_NATIVE_STREAMING": True, + }) + MockAdapter.__abstractmethods__ = frozenset() + adapter = MockAdapter.__new__(MockAdapter) + adapter._typing_paused = set() + adapter.send_stream_frame = AsyncMock(return_value=True) + adapter.send = AsyncMock(return_value=MagicMock(success=True, message_id="msg")) + adapter.supports_native_streaming = lambda chat_type=None, metadata=None: True + return adapter + + +@pytest.fixture +def consumer_config(): + """Create a minimal consumer config.""" + return StreamConsumerConfig( + chat_type="dm", cursor="", + edit_interval=0.01, buffer_threshold=5, + ) + + +@pytest.mark.asyncio +async def test_approval_boundary_finalizes_and_disables_native(mock_adapter, consumer_config): + """Approval boundary must finalize the current stream (creating a stable + message for pre-approval content) and disable native streaming so + post-approval output goes through reliable send().""" + consumer = GatewayStreamConsumer( + adapter=mock_adapter, + chat_id="test_chat", + config=consumer_config, + ) + + consumer._use_native_streaming = True + consumer._native_stream_opened = True + consumer._turn_id = "turn_123" + consumer._initial_reply_to_id = "msg_456" + consumer._accumulated = "下面我来执行:先确认最新分区" + + # Signal approval boundary + boundary_result = consumer.close_for_approval_prompt() + if isinstance(boundary_result, tuple): + boundary_future, _ = boundary_result + else: + boundary_future = boundary_result + + consumer_task = asyncio.create_task(consumer.run()) + result = await asyncio.wait_for(boundary_future, timeout=1.0) + + assert result is True + + # Stream must be finalized (stable message created) + finalize_calls = [ + call for call in mock_adapter.send_stream_frame.call_args_list + if call.kwargs.get("finalize") is True + ] + assert len(finalize_calls) == 1, "Must finalize the stream" + finalize_text = finalize_calls[0].args[0] + assert finalize_text == "下面我来执行:先确认最新分区" + + # Native streaming must be disabled for post-approval output + assert consumer._use_native_streaming is False, ( + "Native streaming must be disabled — post-approval goes via send()" + ) + assert consumer._native_stream_opened is False + + consumer.finish() + await asyncio.sleep(0.05) + consumer_task.cancel() + try: + await consumer_task + except asyncio.CancelledError: + pass + + +@pytest.mark.asyncio +async def test_approval_boundary_uses_placeholder_when_no_accumulated(mock_adapter, consumer_config): + """When there's no accumulated text, finalize with a visible placeholder.""" + consumer = GatewayStreamConsumer( + adapter=mock_adapter, + chat_id="test_chat", + config=consumer_config, + ) + + consumer._use_native_streaming = True + consumer._native_stream_opened = True + consumer._turn_id = "turn_123" + consumer._accumulated = "" # No text accumulated + + boundary_result = consumer.close_for_approval_prompt() + if isinstance(boundary_result, tuple): + boundary_future, _ = boundary_result + else: + boundary_future = boundary_result + + consumer_task = asyncio.create_task(consumer.run()) + result = await asyncio.wait_for(boundary_future, timeout=1.0) + assert result is True + + finalize_calls = [ + call for call in mock_adapter.send_stream_frame.call_args_list + if call.kwargs.get("finalize") is True + ] + assert len(finalize_calls) == 1 + finalize_text = finalize_calls[0].args[0] + assert finalize_text == "⏸ 等待审批中..." + + consumer.finish() + await asyncio.sleep(0.05) + consumer_task.cancel() + try: + await consumer_task + except asyncio.CancelledError: + pass + + +@pytest.mark.asyncio +async def test_boundary_uses_custom_placeholder_when_no_accumulated(mock_adapter, consumer_config): + """A clarify boundary passes its own placeholder; the empty-content + finalize must use it instead of the approval wording so the finalized + bubble doesn't read 'waiting for approval' for a decision question.""" + consumer = GatewayStreamConsumer( + adapter=mock_adapter, + chat_id="test_chat", + config=consumer_config, + ) + + consumer._use_native_streaming = True + consumer._native_stream_opened = True + consumer._turn_id = "turn_123" + consumer._accumulated = "" # No text accumulated + + boundary_result = consumer.close_for_approval_prompt("💬 等待你的选择...") + if isinstance(boundary_result, tuple): + boundary_future, _ = boundary_result + else: + boundary_future = boundary_result + + consumer_task = asyncio.create_task(consumer.run()) + result = await asyncio.wait_for(boundary_future, timeout=1.0) + assert result is True + + finalize_calls = [ + call for call in mock_adapter.send_stream_frame.call_args_list + if call.kwargs.get("finalize") is True + ] + assert len(finalize_calls) == 1 + finalize_text = finalize_calls[0].args[0] + assert finalize_text == "💬 等待你的选择..." + + # Native streaming disabled so post-answer output opens a fresh bubble. + assert consumer._use_native_streaming is False + assert consumer.cfg.buffer_only is True + + consumer.finish() + await asyncio.sleep(0.05) + consumer_task.cancel() + try: + await consumer_task + except asyncio.CancelledError: + pass + + +@pytest.mark.asyncio +async def test_clarify_boundary_logs_use_clarify_prefix(mock_adapter, consumer_config, caplog): + """A clarify boundary that fails to finalize must log with a "Clarify" + prefix, not "Approval" — otherwise a clarify failure looks like a + dangerous-command approval failure during troubleshooting.""" + import logging + + # Make finalize fail so the warning path fires, and the fallback send fail + # too so the error path fires — exercising the reason-labelled logs. + mock_adapter.send_stream_frame = AsyncMock(return_value=False) + mock_adapter.send = AsyncMock(return_value=MagicMock(success=False)) + + consumer = GatewayStreamConsumer( + adapter=mock_adapter, + chat_id="test_chat", + config=consumer_config, + ) + consumer._use_native_streaming = True + consumer._native_stream_opened = True + consumer._turn_id = "turn_123" + consumer._accumulated = "部分内容" + + boundary_result = consumer.close_for_approval_prompt( + "💬 等待你的选择...", reason="Clarify", + ) + if isinstance(boundary_result, tuple): + boundary_future, _ = boundary_result + else: + boundary_future = boundary_result + + with caplog.at_level(logging.WARNING, logger="gateway.stream_consumer"): + consumer_task = asyncio.create_task(consumer.run()) + await asyncio.wait_for(boundary_future, timeout=1.0) + + boundary_logs = [r.getMessage() for r in caplog.records] + assert any("Clarify boundary" in m for m in boundary_logs), ( + f"Expected a 'Clarify boundary' log, got: {boundary_logs}" + ) + assert not any("Approval boundary" in m for m in boundary_logs), ( + f"Clarify boundary must not log as 'Approval boundary': {boundary_logs}" + ) + + consumer.finish() + await asyncio.sleep(0.05) + consumer_task.cancel() + try: + await consumer_task + except asyncio.CancelledError: + pass + + +@pytest.mark.asyncio +async def test_approval_boundary_post_approval_one_shot_send(mock_adapter, consumer_config): + """After approval boundary, post-approval content must: + 1. Set buffer_only=True (no mid-stream flushes) + 2. Accumulate all deltas without sending + 3. Deliver everything via one adapter.send() call on finish()""" + consumer = GatewayStreamConsumer( + adapter=mock_adapter, + chat_id="test_chat", + config=consumer_config, + ) + + consumer._use_native_streaming = True + consumer._native_stream_opened = True + consumer._turn_id = "turn_123" + consumer._accumulated = "Pre-approval text" + + boundary_result = consumer.close_for_approval_prompt() + if isinstance(boundary_result, tuple): + boundary_future, _ = boundary_result + else: + boundary_future = boundary_result + + consumer_task = asyncio.create_task(consumer.run()) + await asyncio.wait_for(boundary_future, timeout=1.0) + + # Verify buffer_only is set + assert consumer.cfg.buffer_only is True, "Must set buffer_only after boundary" + assert consumer._use_native_streaming is False + + # Send post-approval content — should NOT trigger any immediate send + mock_adapter.send_stream_frame.reset_mock() + mock_adapter.send.reset_mock() + + consumer.on_delta("Post-approval result text here") + await asyncio.sleep(0.1) + + # Before finish(): no send() or stream frame calls + assert mock_adapter.send.call_count == 0, ( + "buffer_only: no send before finish()" + ) + stream_calls = [ + call for call in mock_adapter.send_stream_frame.call_args_list + if not call.kwargs.get("finalize") + ] + assert len(stream_calls) == 0, ( + "Post-approval must NOT use native streaming" + ) + + # Now finish — should deliver via send() + consumer.finish() + await asyncio.sleep(0.1) + + # adapter.send should have been called with the full post-approval text + send_calls = mock_adapter.send.call_args_list + assert len(send_calls) >= 1, "finish() must deliver via send()" + # The delivered text should contain our post-approval content + delivered = send_calls[-1].args[1] if len(send_calls[-1].args) > 1 else send_calls[-1].kwargs.get("content", "") + assert "Post-approval result" in delivered + + consumer_task.cancel() + try: + await consumer_task + except asyncio.CancelledError: + pass + + +@pytest.mark.asyncio +async def test_approval_boundary_stream_not_opened_at_boundary_time(mock_adapter, consumer_config): + """When native streaming is active but _native_stream_opened is still False + at the time boundary processes (e.g., seed succeeded but stream was closed by + a prior error before boundary arrives), no finalize is sent.""" + consumer = GatewayStreamConsumer( + adapter=mock_adapter, + chat_id="test_chat", + config=consumer_config, + ) + + # Let run() do its normal seed (which sets _native_stream_opened=True) + # Then manually close it before boundary processes + consumer._use_native_streaming = True + consumer._native_stream_opened = True + consumer._turn_id = "turn_123" + consumer._accumulated = "Some text" + + # Queue boundary + boundary_result = consumer.close_for_approval_prompt() + if isinstance(boundary_result, tuple): + boundary_future, _ = boundary_result + else: + boundary_future = boundary_result + + # Simulate: stream was closed by error BEFORE consumer processes boundary + consumer._native_stream_opened = False + + consumer_task = asyncio.create_task(consumer.run()) + result = await asyncio.wait_for(boundary_future, timeout=1.0) + + assert result is True + # Native streaming should be disabled after boundary + assert consumer._use_native_streaming is False + + consumer.finish() + await asyncio.sleep(0.05) + consumer_task.cancel() + try: + await consumer_task + except asyncio.CancelledError: + pass + + +@pytest.mark.asyncio +async def test_approval_boundary_finalize_fails_fallback_send_succeeds(mock_adapter, consumer_config): + """When stream finalize fails but fallback send() succeeds, boundary is True.""" + consumer = GatewayStreamConsumer( + adapter=mock_adapter, + chat_id="test_chat", + config=consumer_config, + ) + + consumer._use_native_streaming = True + consumer._native_stream_opened = True + consumer._turn_id = "turn_123" + consumer._accumulated = "Pre-approval text" + + # Finalize fails (returns False) + mock_adapter.send_stream_frame = AsyncMock(return_value=False) + # Fallback send succeeds + mock_adapter.send = AsyncMock(return_value=MagicMock(success=True, message_id="msg")) + + boundary_result = consumer.close_for_approval_prompt() + if isinstance(boundary_result, tuple): + boundary_future, _ = boundary_result + else: + boundary_future = boundary_result + + consumer_task = asyncio.create_task(consumer.run()) + result = await asyncio.wait_for(boundary_future, timeout=1.0) + + assert result is True, "Fallback send succeeded → boundary should be True" + mock_adapter.send.assert_awaited_once_with("test_chat", "Pre-approval text") + + consumer.finish() + await asyncio.sleep(0.05) + consumer_task.cancel() + try: + await consumer_task + except asyncio.CancelledError: + pass + + +@pytest.mark.asyncio +async def test_approval_boundary_finalize_and_fallback_both_fail(mock_adapter, consumer_config): + """When both stream finalize and fallback send() fail, boundary is False.""" + consumer = GatewayStreamConsumer( + adapter=mock_adapter, + chat_id="test_chat", + config=consumer_config, + ) + + consumer._use_native_streaming = True + consumer._native_stream_opened = True + consumer._turn_id = "turn_123" + consumer._accumulated = "Pre-approval text" + + # Finalize fails (raises) + mock_adapter.send_stream_frame = AsyncMock(side_effect=RuntimeError("stream dead")) + # Fallback send also fails + mock_adapter.send = AsyncMock(return_value=MagicMock(success=False, error="timeout")) + + boundary_result = consumer.close_for_approval_prompt() + if isinstance(boundary_result, tuple): + boundary_future, _ = boundary_result + else: + boundary_future = boundary_result + + consumer_task = asyncio.create_task(consumer.run()) + result = await asyncio.wait_for(boundary_future, timeout=1.0) + + assert result is False, "Both finalize and fallback failed → boundary should be False" + + consumer.finish() + await asyncio.sleep(0.05) + consumer_task.cancel() + try: + await consumer_task + except asyncio.CancelledError: + pass diff --git a/tests/gateway/test_stream_consumer_tool_progress.py b/tests/gateway/test_stream_consumer_tool_progress.py new file mode 100644 index 0000000000..fc36c7ca04 --- /dev/null +++ b/tests/gateway/test_stream_consumer_tool_progress.py @@ -0,0 +1,313 @@ +"""Tests for tool-progress-in-native-stream (single bubble) feature. + +Validates that tool-progress lines are injected into the native streaming +bubble and properly overwritten by text deltas (Strategy B). +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from gateway.stream_consumer import ( + GatewayStreamConsumer, + StreamConsumerConfig, + _TOOL_PROGRESS, +) + + +def _make_native_streaming_adapter(*, supports_native: bool = True): + """Build a BasePlatformAdapter subclass that supports native streaming.""" + from gateway.platforms.base import BasePlatformAdapter + + NativeStreamingAdapter = type( + "NativeStreamingAdapter", + (BasePlatformAdapter,), + { + "MAX_MESSAGE_LENGTH": 4096, + "SUPPORTS_MESSAGE_EDITING": False, + "SUPPORTS_NATIVE_STREAMING": True, + }, + ) + NativeStreamingAdapter.__abstractmethods__ = frozenset() + adapter = NativeStreamingAdapter.__new__(NativeStreamingAdapter) + adapter._typing_paused = set() + adapter._fatal_error_message = None + adapter.frames = [] + + def _supports(chat_type=None, metadata=None): + return bool(supports_native) + adapter.supports_native_streaming = _supports + + async def _send_stream_frame( + text, *, finalize=False, chat_id=None, reply_to=None, **kwargs + ): + adapter.frames.append({ + "text": text, + "finalize": finalize, + "chat_id": chat_id, + }) + return True + adapter.send_stream_frame = _send_stream_frame + + adapter.send = AsyncMock( + return_value=SimpleNamespace(success=True, message_id="fallback_msg"), + ) + adapter.edit_message = AsyncMock( + return_value=SimpleNamespace(success=True), + ) + return adapter + + +def _make_consumer(*, native_streaming: bool = True) -> GatewayStreamConsumer: + """Create a GatewayStreamConsumer configured for native streaming.""" + adapter = _make_native_streaming_adapter(supports_native=native_streaming) + cfg = StreamConsumerConfig(chat_type="dm", cursor="▌") + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + # Force native streaming resolution + consumer._use_native_streaming = native_streaming + return consumer + + +# === UNIT TESTS === + + +class TestAcceptsToolProgress: + """Tests for the accepts_tool_progress property.""" + + def test_native_streaming_accepts(self): + consumer = _make_consumer(native_streaming=True) + assert consumer.accepts_tool_progress is True + + def test_non_native_does_not_accept(self): + consumer = _make_consumer(native_streaming=False) + assert consumer.accepts_tool_progress is False + + +class TestOnToolProgress: + """Tests for on_tool_progress() enqueue behavior.""" + + def test_enqueues_sentinel(self): + consumer = _make_consumer() + consumer.on_tool_progress("🔍 Searching...") + item = consumer._queue.get_nowait() + assert isinstance(item, tuple) + assert len(item) == 2 + assert item[0] is _TOOL_PROGRESS + assert item[1] == "🔍 Searching..." + + def test_empty_line_not_enqueued(self): + consumer = _make_consumer() + consumer.on_tool_progress("") + assert consumer._queue.empty() + + +class TestComposeFrameContent: + """Tests for _compose_frame_content() composition logic (Strategy B).""" + + def test_only_tool_lines(self): + consumer = _make_consumer() + consumer._tool_progress_lines = ["🔍 Searching...", "💻 Running git log"] + result = consumer._compose_frame_content() + assert result == "🔍 Searching...\n💻 Running git log" + + def test_only_accumulated(self): + consumer = _make_consumer() + consumer._accumulated = "Here is the answer." + result = consumer._compose_frame_content() + assert result == "Here is the answer." + + def test_both_accumulated_and_tool_lines_strategy_b(self): + """Strategy B: text + separator + tool status at bottom.""" + consumer = _make_consumer() + consumer._accumulated = "Here is some text so far." + consumer._tool_progress_lines = ["🔍 Searching the web..."] + result = consumer._compose_frame_content() + assert result == "Here is some text so far.\n\n---\n🔍 Searching the web..." + + def test_multiple_tool_lines_stacked(self): + consumer = _make_consumer() + consumer._tool_progress_lines = [ + "🔍 web_search: 'python'", + "💻 terminal: git log", + "📄 read_file: main.py", + ] + result = consumer._compose_frame_content() + assert "web_search" in result + assert "terminal" in result + assert "read_file" in result + # Lines are joined with newlines + assert result.count("\n") == 2 + + def test_empty_state(self): + consumer = _make_consumer() + result = consumer._compose_frame_content() + assert result == "" + + +class TestSegmentReset: + """Test that segment reset clears tool progress state.""" + + def test_reset_clears_tool_progress(self): + consumer = _make_consumer() + consumer._tool_progress_lines = ["🔍 Searching..."] + consumer._tool_progress_active = True + consumer._reset_segment_state() + assert consumer._tool_progress_lines == [] + assert consumer._tool_progress_active is False + + +# === INTEGRATION TESTS (drain loop) === + + +class TestToolProgressDrainLoop: + """Integration tests for the drain loop + frame delivery.""" + + @pytest.mark.asyncio + async def test_tool_progress_only_then_done(self): + """Pure tool-progress turn (no text): tool lines visible as mid-frame, + finalize uses placeholder since no accumulated text.""" + consumer = _make_consumer() + consumer.on_tool_progress("🔍 Searching...") + consumer.on_tool_progress("💻 terminal: ls") + + # Start consumer so it drains tool progress and sends mid-frames + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.3) + + # Now finish — tool lines were already displayed as a frame + consumer.finish() + await asyncio.sleep(0.3) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + frames = consumer.adapter.frames + # Should have: seed frame, at least one mid-frame with tool lines, + # and a finalize frame (✅ placeholder since no text) + assert len(frames) >= 2 + # Find mid-frames that contain tool progress + non_finalize = [f for f in frames if not f["finalize"] and f["text"]] + assert any("Searching" in f["text"] or "terminal" in f["text"] for f in non_finalize), ( + f"Expected tool progress in mid-frames, got: {[f['text'] for f in frames]}" + ) + + @pytest.mark.asyncio + async def test_tool_progress_then_text_clears_overlay(self): + """Tool progress → text delta should clear tool lines from frame.""" + consumer = _make_consumer() + consumer.on_tool_progress("🔍 Searching...") + consumer.on_delta("Hello world") + consumer.finish() + + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.5) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + # After text arrives, tool_progress_lines should be cleared + assert consumer._tool_progress_lines == [] + assert "Hello world" in consumer._accumulated + + # The finalize frame should contain just the text + frames = consumer.adapter.frames + finalize_frames = [f for f in frames if f["finalize"]] + if finalize_frames: + assert "Hello world" in finalize_frames[-1]["text"] + assert "Searching" not in finalize_frames[-1]["text"] + + @pytest.mark.asyncio + async def test_text_then_tool_then_text_strategy_b(self): + """Strategy B: text → tool → text appends tool at bottom then clears.""" + consumer = _make_consumer() + + # Phase 1: initial text + consumer.on_delta("First part. ") + # Phase 2: tool progress mid-stream + consumer.on_tool_progress("🔍 web_search...") + # Phase 3: more text arrives + consumer.on_delta("Second part.") + consumer.finish() + + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.5) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + # Final state: tool lines cleared, accumulated has both text parts + assert consumer._tool_progress_lines == [] + assert "First part." in consumer._accumulated + assert "Second part." in consumer._accumulated + + @pytest.mark.asyncio + async def test_parallel_tool_calls_stacked(self): + """Multiple tool.started back-to-back should stack in overlay.""" + consumer = _make_consumer() + consumer.on_tool_progress("🔍 web_search") + consumer.on_tool_progress("💻 terminal") + consumer.on_tool_progress("📄 read_file") + + # Let drain loop process and send mid-frame before finish + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.3) + + consumer.finish() + await asyncio.sleep(0.3) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + # All three should have been accumulated and sent in one frame + frames = consumer.adapter.frames + # Find a frame that contains all three tools + all_three = [ + f for f in frames + if "web_search" in f["text"] + and "terminal" in f["text"] + and "read_file" in f["text"] + ] + assert len(all_three) >= 1, ( + f"Expected a frame with all 3 tools, got: {[f['text'] for f in frames]}" + ) + + @pytest.mark.asyncio + async def test_finalize_frame_is_pure_text(self): + """The finalize frame must only contain accumulated text, no tool lines.""" + consumer = _make_consumer() + consumer.on_tool_progress("🔍 Searching...") + consumer.on_delta("The answer is 42.") + # Add a tool progress AFTER text (Strategy B scenario) + consumer.on_tool_progress("💻 terminal: verify") + # Then more text clears it + consumer.on_delta(" Verified.") + consumer.finish() + + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.5) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + frames = consumer.adapter.frames + finalize_frames = [f for f in frames if f["finalize"]] + if finalize_frames: + final_text = finalize_frames[-1]["text"] + assert "The answer is 42. Verified." in final_text + assert "Searching" not in final_text + assert "terminal" not in final_text + assert "---" not in final_text diff --git a/tests/gateway/test_stream_consumer_wecom_native.py b/tests/gateway/test_stream_consumer_wecom_native.py new file mode 100644 index 0000000000..cd691eeb25 --- /dev/null +++ b/tests/gateway/test_stream_consumer_wecom_native.py @@ -0,0 +1,1077 @@ +"""Tests for native streaming in GatewayStreamConsumer (WeCom-style transport). + +Native streaming is the consumer's transport for adapters that: + * cannot edit messages (``SUPPORTS_MESSAGE_EDITING = False``); but + * expose a stream protocol where every frame is a cumulative content + update plus a ``finish: true`` final frame (e.g. WeCom's + ``msgtype: "stream"`` via ``aibot_respond_msg``). + +These tests use a runtime subclass of ``BasePlatformAdapter`` so the +consumer's ``isinstance(BasePlatformAdapter)`` gate is satisfied. They +verify the full lifecycle (seed → mid-stream updates → finalize), the +throttling that keeps frames under WeCom's 30/min rate ceiling, and the +fallback path when ``send_stream_frame`` returns False. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from gateway.stream_consumer import ( + GatewayStreamConsumer, + StreamConsumerConfig, +) + + +def _make_native_streaming_adapter( + *, + supports_native: bool = True, + seed_succeeds: bool = True, + frames_succeed: bool = True, + finalize_succeeds: bool = True, +): + """Build a BasePlatformAdapter subclass that supports native streaming. + + Records every ``send_stream_frame`` call on ``adapter.frames`` for assertions. + """ + from gateway.platforms.base import BasePlatformAdapter, SendResult + + NativeStreamingAdapter = type( + "NativeStreamingAdapter", + (BasePlatformAdapter,), + { + "MAX_MESSAGE_LENGTH": 4096, + "SUPPORTS_MESSAGE_EDITING": False, + "SUPPORTS_NATIVE_STREAMING": True, + }, + ) + NativeStreamingAdapter.__abstractmethods__ = frozenset() + adapter = NativeStreamingAdapter.__new__(NativeStreamingAdapter) + adapter._typing_paused = set() + adapter._fatal_error_message = None + + adapter.frames = [] # list of (text, finalize) + + def _supports(chat_type=None, metadata=None): + return bool(supports_native) + adapter.supports_native_streaming = _supports + + async def _send_stream_frame( + text, *, finalize=False, chat_id=None, reply_to=None, **kwargs + ): + adapter.frames.append({ + "text": text, + "finalize": finalize, + "chat_id": chat_id, + "reply_to": reply_to, + }) + if finalize: + return finalize_succeeds + # First frame is the seed (empty content). + if text == "" and len(adapter.frames) == 1: + return seed_succeeds + return frames_succeed + adapter.send_stream_frame = _send_stream_frame + + # send / edit_message: count fallback usage so we can assert native + # ran without ever touching them. + adapter.send = AsyncMock( + return_value=SimpleNamespace(success=True, message_id="fallback_msg"), + ) + adapter.edit_message = AsyncMock( + return_value=SimpleNamespace(success=True), + ) + return adapter + + +# === RESOLVER === + + +class TestNativeStreamingResolver: + """``_resolve_native_streaming`` gating logic.""" + + def test_capable_adapter_resolves_to_native(self): + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig(chat_type="dm", cursor="") + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + assert consumer._resolve_native_streaming() is True + + def test_class_attribute_required(self): + """Adapter without SUPPORTS_NATIVE_STREAMING class attr returns False.""" + from gateway.platforms.base import BasePlatformAdapter + + Bare = type("Bare", (BasePlatformAdapter,), {"MAX_MESSAGE_LENGTH": 4096}) + Bare.__abstractmethods__ = frozenset() + adapter = Bare.__new__(Bare) + adapter._typing_paused = set() + adapter._fatal_error_message = None + adapter.supports_native_streaming = lambda chat_type=None, metadata=None: True + + cfg = StreamConsumerConfig(chat_type="dm") + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + assert consumer._resolve_native_streaming() is False + + def test_probe_returning_false_disables_native(self): + adapter = _make_native_streaming_adapter(supports_native=False) + cfg = StreamConsumerConfig(chat_type="dm") + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + assert consumer._resolve_native_streaming() is False + + def test_magicmock_adapter_falls_back(self): + """MagicMock adapters are excluded by isinstance gate.""" + adapter = MagicMock() + cfg = StreamConsumerConfig(chat_type="dm") + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + assert consumer._resolve_native_streaming() is False + + +# === LIFECYCLE === + + +class TestNativeStreamingLifecycle: + """Seed frame on run-start → mid-stream updates → finalize.""" + + @pytest.mark.asyncio + async def test_seed_frame_fires_at_run_start(self): + """The first thing the consumer does is a seed frame for typing UI.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", + edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = asyncio.create_task(consumer.run()) + # Tiny sleep so run() can dispatch the seed before we tear down. + await asyncio.sleep(0.02) + consumer.finish() + await task + + assert len(adapter.frames) >= 1 + assert adapter.frames[0]["text"] == "" + assert adapter.frames[0]["finalize"] is False + assert adapter.frames[0]["chat_id"] == "chat-1" + + @pytest.mark.asyncio + async def test_full_run_routes_only_through_send_stream_frame(self): + """No mid-stream call to send() / edit_message() in native mode.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", + edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + # Push enough text past the throttling threshold (>20 visible chars). + consumer.on_delta("This is a substantial first chunk past the threshold.") + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.05) + consumer.on_delta(" Even more content arriving in the second chunk.") + await asyncio.sleep(0.05) + consumer.finish() + await task + + assert adapter.send.await_count == 0 + assert adapter.edit_message.await_count == 0 + # Final frame must be finalize=true. + finalize_frames = [f for f in adapter.frames if f["finalize"]] + assert len(finalize_frames) == 1 + # Final text held the full accumulated content. + assert "first chunk" in finalize_frames[0]["text"] + assert "second chunk" in finalize_frames[0]["text"] + + @pytest.mark.asyncio + async def test_consumer_marks_final_response_sent(self): + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", + edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + consumer.on_delta("Hello, this is a sufficiently long response.") + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.05) + consumer.finish() + await task + + assert consumer.final_response_sent is True + assert consumer.final_content_delivered is True + + +# === THROTTLING === + + +class TestNativeStreamingThrottling: + """Fire-and-forget: every delta is pushed immediately (no throttle).""" + + @pytest.mark.asyncio + async def test_tiny_increments_are_sent_immediately(self): + """No throttling — each distinct cumulative text produces a frame.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", + edit_interval=0.01, buffer_threshold=1, # aggressive flush + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.02) # let seed frame fire + # Many 1-char deltas — fire-and-forget sends every change. + for ch in "abcdefghij": # 10 chars total + consumer.on_delta(ch) + await asyncio.sleep(0.015) + consumer.finish() + await task + + # Every distinct cumulative text should produce a frame (no throttle). + non_finalize_content_frames = [ + f for f in adapter.frames if not f["finalize"] and f["text"] + ] + # With fire-and-forget, every drain-loop iteration that sees new + # accumulated text pushes immediately. Due to asyncio batching, + # multiple on_delta() calls between awaits collapse into one drain, + # so we won't get exactly 10 frames — but we should get significantly + # more than the old throttled behavior (which allowed at most 1). + assert len(non_finalize_content_frames) >= 3, ( + f"fire-and-forget should send most deltas: got {len(non_finalize_content_frames)} mid frames" + ) + # The user still sees the full content in the finalize frame. + finalize_frames = [f for f in adapter.frames if f["finalize"]] + assert len(finalize_frames) == 1 + assert finalize_frames[0]["text"] == "abcdefghij" + + @pytest.mark.asyncio + async def test_large_growth_emits_mid_frames(self): + """When text grows by >20 chars, an interim frame should land.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", + edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.02) + # First chunk well past 20 chars. + consumer.on_delta("A" * 40) + await asyncio.sleep(0.05) + # Second chunk also past 20 chars. + consumer.on_delta("B" * 40) + await asyncio.sleep(0.05) + consumer.finish() + await task + + non_finalize_content_frames = [ + f for f in adapter.frames if not f["finalize"] and f["text"] + ] + assert len(non_finalize_content_frames) >= 1 + + +# === FALLBACK === + + +class TestNativeStreamingFallback: + """When ``send_stream_frame`` returns False, native is disabled and the + consumer takes the regular send/edit path.""" + + @pytest.mark.asyncio + async def test_seed_failure_disables_native(self): + """If even the seed frame fails, native is off for the run.""" + adapter = _make_native_streaming_adapter(seed_succeeds=False) + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", + edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + consumer.on_delta("hello world this is enough text") + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.05) + consumer.finish() + await task + + assert consumer._use_native_streaming is False + + @pytest.mark.asyncio + async def test_native_streaming_disables_draft(self): + """Adapter that supports both — native takes priority, draft off.""" + adapter = _make_native_streaming_adapter() + # Pretend it also offers draft (won't be used). + adapter.supports_draft_streaming = lambda chat_type=None, metadata=None: True + adapter.send_draft = AsyncMock( + return_value=SimpleNamespace(success=True, message_id=None), + ) + + cfg = StreamConsumerConfig( + transport="auto", chat_type="dm", cursor="", + edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + consumer.on_delta("a sufficiently long content chunk here yo") + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.05) + consumer.finish() + await task + + assert consumer._use_native_streaming is True + assert consumer._use_draft_streaming is False + adapter.send_draft.assert_not_awaited() + + +class TestNativeStreamingSegmentBreak: + """Segment breaks should NOT finalize or reset for WeCom native streaming.""" + + @pytest.mark.asyncio + async def test_segment_break_preserves_cumulative_text(self): + """Tool boundary keeps pre+post text in one stream, single finalize.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", + edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + # Pre-tool text + consumer.on_delta("Pre-tool content. ") + # Simulate tool boundary (segment break) + consumer.on_segment_break() + # Post-tool text + consumer.on_delta("Post-tool result.") + + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.1) + consumer.finish() + await task + + # Should only have ONE finalize frame (the final one) + finalize_frames = [f for f in adapter.frames if f.get("finalize")] + assert len(finalize_frames) == 1, \ + f"Should have exactly 1 finalize, got {len(finalize_frames)}" + + # The finalize frame must contain BOTH pre-tool and post-tool text + final_text = finalize_frames[0]["text"] + assert "Pre-tool content" in final_text, \ + "Final frame must include pre-tool text (not lost by reset)" + assert "Post-tool result" in final_text, \ + "Final frame must include post-tool text" + + @pytest.mark.asyncio + async def test_segment_break_no_extra_finalize(self): + """Segment break should NOT produce a finalize frame.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", + edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + consumer.on_delta("First part of response. ") + consumer.on_segment_break() + consumer.on_delta("Second part after tool.") + + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.1) + consumer.finish() + await task + + # Count finalize frames — should be exactly 1 (only at finish) + finalize_count = sum(1 for f in adapter.frames if f.get("finalize")) + assert finalize_count == 1, \ + f"Expected 1 finalize (only at end), got {finalize_count}" + + # Non-finalize frames should show cumulative growth + content_frames = [f for f in adapter.frames if not f.get("finalize") and f["text"]] + if len(content_frames) >= 2: + # Later frames should be longer (cumulative) + assert len(content_frames[-1]["text"]) >= len(content_frames[0]["text"]), \ + "Content frames should grow cumulatively" + + +class TestClarifyReopenBoundary: + """A clarify boundary (reopen=True) finalizes the pre-prompt stream but + keeps native streaming enabled so the post-answer continuation re-opens a + fresh native stream — restoring the typing bubble instead of degrading to a + one-shot send() (the approval-path behaviour).""" + + async def _drain(self, consumer, seconds=0.15): + deadline = asyncio.get_event_loop().time() + seconds + while asyncio.get_event_loop().time() < deadline: + await asyncio.sleep(0.01) + + @pytest.mark.asyncio + async def test_reopen_boundary_keeps_native_and_reopens_stream(self): + """Pre-prompt content is finalized; post-answer content re-seeds a + fresh stream and finalizes again — two seeds, two finalizes, native + never disabled.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = asyncio.create_task(consumer.run()) + await self._drain(consumer, 0.03) # let the initial seed fire + + # Pre-prompt streamed content. + consumer.on_delta("正在处理,先给你看一段初步结果。") + await self._drain(consumer, 0.05) + + # Clarify boundary with reopen=True. + boundary = consumer.close_for_approval_prompt( + "💬 等待你的选择...", reason="Clarify", reopen=True, + ) + fut = boundary[0] if isinstance(boundary, tuple) else boundary + await asyncio.wait_for(fut, timeout=1.0) + + # Native must stay enabled and buffer_only must NOT be set. + assert consumer._use_native_streaming is True + assert consumer.cfg.buffer_only is False + assert consumer._native_stream_opened is False # finalized, awaiting reopen + + # Post-answer continuation → should re-open a fresh stream. + consumer.on_delta("根据你的选择,这是后续的完整回答内容。") + await self._drain(consumer, 0.05) + consumer.finish() + await task + + finalize_frames = [f for f in adapter.frames if f.get("finalize")] + seed_frames = [f for f in adapter.frames if f["text"] == "" and not f.get("finalize")] + + # Two finalizes: one at the boundary, one at got_done for the reopened stream. + assert len(finalize_frames) == 2, ( + f"expected 2 finalizes (boundary + reopened turn), got {len(finalize_frames)}" + ) + # At least two seeds: initial + reopened. + assert len(seed_frames) >= 2, ( + f"expected the post-answer content to re-seed a fresh stream, " + f"seeds={len(seed_frames)}" + ) + # The reopened stream's finalize carries the post-answer content. + assert "后续的完整回答" in finalize_frames[-1]["text"] + # Native was never disabled → no fallback send. + adapter.send.assert_not_awaited() + + @pytest.mark.asyncio + async def test_reopen_boundary_no_post_content_skips_lone_placeholder(self): + """If the agent produces nothing after the clarify, got_done must NOT + re-seed a fresh stream just to emit a lone '✅' bubble.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = asyncio.create_task(consumer.run()) + await self._drain(consumer, 0.03) + + consumer.on_delta("这一段是提问前已经流式出去的内容。") + await self._drain(consumer, 0.05) + + boundary = consumer.close_for_approval_prompt( + "💬 等待你的选择...", reason="Clarify", reopen=True, + ) + fut = boundary[0] if isinstance(boundary, tuple) else boundary + await asyncio.wait_for(fut, timeout=1.0) + + frames_before_finish = len(adapter.frames) + + # No post-answer content — just finish. + consumer.finish() + await task + + # No new seed / finalize frame should have been emitted after the + # boundary (no lone "✅"). + new_frames = adapter.frames[frames_before_finish:] + assert not any(f["text"] == "✅" for f in new_frames), ( + f"must not emit a lone '✅' placeholder, got {new_frames}" + ) + # A "✅" placeholder anywhere would indicate the guard failed. + assert not any(f["text"] == "✅" for f in adapter.frames) + + @pytest.mark.asyncio + async def test_approval_boundary_still_disables_native(self): + """Contrast: the approval path (reopen=False, the default) disables + native and buffers post-prompt output — unchanged behaviour.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = asyncio.create_task(consumer.run()) + await self._drain(consumer, 0.03) + + consumer.on_delta("审批前的流式内容。") + await self._drain(consumer, 0.05) + + boundary = consumer.close_for_approval_prompt(reason="Approval") + fut = boundary[0] if isinstance(boundary, tuple) else boundary + await asyncio.wait_for(fut, timeout=1.0) + + assert consumer._use_native_streaming is False + assert consumer.cfg.buffer_only is True + + consumer.finish() + await task + + +class TestClarifyEagerReseed: + """EAGER re-seed after a clarify answer. + + WeCom typing is driven by the stream seed frame (send_typing is a no-op), + so the clarify-reopen path used to re-seed LAZILY — only when the LLM + emitted its first post-answer delta — leaving up to ~48s of dead air after + the user replied. The eager path seeds the moment the user answers, via + ``request_reopen_seed()`` → ``_REOPEN_SEED`` → the run-loop handler, so the + typing bubble reappears instantly. + + Each test below maps to one of the 7 verification points. Breakage map for + the destructive checks: + * Remove the run-loop eager-seed branch → test_reopen_seed_opens_stream_before_any_delta turns red. + * Remove the _suppress_silence_marker native-close patch → test_eager_seed_then_silence_marker_closes_stream turns red. + * Remove got_done hole A branch → test_eager_seed_no_content_finalizes_once turns red (or a "✅" leaks). + """ + + async def _drain(self, consumer, seconds=0.15): + deadline = asyncio.get_event_loop().time() + seconds + while asyncio.get_event_loop().time() < deadline: + await asyncio.sleep(0.01) + + async def _wait_until(self, predicate, timeout=2.0): + """Poll ``predicate()`` until true or timeout — robust against CPU + contention (a fixed _drain sleep flakes under the 24-worker suite). + Returns the predicate's final value so callers can assert on it. + """ + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + if predicate(): + return True + await asyncio.sleep(0.01) + return predicate() + + async def _to_reopen_pending(self, consumer, adapter): + """Run to the point right after a clarify boundary: native still on, + no stream open, awaiting a re-seed. Returns nothing; leaves the run + task attached on ``consumer._task`` for the caller to finish()/await. + """ + task = asyncio.create_task(consumer.run()) + await self._drain(consumer, 0.03) # initial seed + + consumer.on_delta("提问前已经流式出去的一段内容。") + await self._drain(consumer, 0.05) + + boundary = consumer.close_for_approval_prompt( + "💬 等待你的选择...", reason="Clarify", reopen=True, + ) + fut = boundary[0] if isinstance(boundary, tuple) else boundary + await asyncio.wait_for(fut, timeout=1.0) + return task + + # === POINT 1: timing — the boundary itself does NOT eager-seed === + + @pytest.mark.asyncio + async def test_boundary_alone_does_not_eager_seed(self): + """Processing the clarify boundary (reopen=True) must not emit a fresh + seed frame on its own — the eager seed only happens once the user + answers and request_reopen_seed() is called.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = await self._to_reopen_pending(consumer, adapter) + + # State: reopen-pending, no stream open, native still live. + assert consumer._awaiting_reopen_after_boundary is True + assert consumer._native_stream_opened is False + assert consumer._use_native_streaming is True + # No eager seed yet (request_reopen_seed not called). Only the initial + # seed + the boundary finalize should exist — no NEW empty seed. + seed_frames = [ + f for f in adapter.frames if f["text"] == "" and not f["finalize"] + ] + assert len(seed_frames) == 1, ( + f"boundary must not eager-seed on its own, seeds={len(seed_frames)}" + ) + + consumer.finish() + await task + + # === POINT 2: latency decoupling (core) — seed lands BEFORE any delta === + + @pytest.mark.asyncio + async def test_reopen_seed_opens_stream_before_any_delta(self): + """After the boundary, request_reopen_seed() → the run loop opens a + fresh empty seed frame BEFORE the LLM produces any post-answer delta. + + DESTRUCTIVE: remove the run-loop `_REOPEN_SEED` handler and this fails. + """ + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = await self._to_reopen_pending(consumer, adapter) + seeds_before = len( + [f for f in adapter.frames if f["text"] == "" and not f["finalize"]] + ) + + # User answered → request an eager re-seed. NO on_delta yet. + consumer.request_reopen_seed() + await self._drain(consumer, 0.05) # let run() process _REOPEN_SEED + + seeds_after = len( + [f for f in adapter.frames if f["text"] == "" and not f["finalize"]] + ) + assert seeds_after == seeds_before + 1, ( + "eager re-seed must emit exactly one new empty seed frame before " + f"any delta (before={seeds_before}, after={seeds_after})" + ) + assert await self._wait_until(lambda: consumer._native_stream_opened) + assert consumer._awaiting_reopen_after_boundary is False + assert consumer._reopen_seeded_eagerly is True + + consumer.finish() + await task + + # === POINT 3: no-content wrap-up (hole A) — one finalize, no "✅" === + + @pytest.mark.asyncio + async def test_eager_seed_no_content_finalizes_once(self): + """After an eager seed, if the agent produces NO content, got_done must + close the empty typing bubble with exactly one finalize and never emit + a lone '✅'. + + DESTRUCTIVE: remove got_done hole-A branch and this fails (the bubble + hangs / a '✅' placeholder leaks). + """ + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = await self._to_reopen_pending(consumer, adapter) + consumer.request_reopen_seed() + await self._drain(consumer, 0.05) + assert await self._wait_until( + lambda: consumer._native_stream_opened + ), "eager seed must open the stream" + + frames_before = len(adapter.frames) + + # No post-answer content — just finish. + consumer.finish() + await task + + new_frames = adapter.frames[frames_before:] + finalize_frames = [f for f in new_frames if f["finalize"]] + assert len(finalize_frames) == 1, ( + f"eager-seed empty stream must finalize exactly once, " + f"got {len(finalize_frames)}: {new_frames}" + ) + # No lone "✅" anywhere. + assert not any(f["text"] == "✅" for f in adapter.frames), ( + f"must not emit a lone '✅' placeholder, frames={adapter.frames}" + ) + # The finalize is an empty close, not content. + assert finalize_frames[0]["text"] == "" + assert consumer._native_stream_opened is False + + # === POINT 4: no seed while merely waiting (guard B) === + + @pytest.mark.asyncio + async def test_no_seed_while_awaiting_user_answer(self): + """After the boundary but WITHOUT request_reopen_seed() and without any + delta, no fresh seed frame may appear — typing must not light up while + the user has not yet replied.""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = await self._to_reopen_pending(consumer, adapter) + seeds_before = len( + [f for f in adapter.frames if f["text"] == "" and not f["finalize"]] + ) + + # Sit in the reopen-pending state; do NOT answer. + await self._drain(consumer, 0.1) + + seeds_after = len( + [f for f in adapter.frames if f["text"] == "" and not f["finalize"]] + ) + assert seeds_after == seeds_before, ( + "no new seed may fire while waiting for the user's clarify answer " + f"(before={seeds_before}, after={seeds_after})" + ) + assert consumer._native_stream_opened is False + assert consumer._awaiting_reopen_after_boundary is True + + consumer.finish() + await task + + # === POINT 5: seed failure degrades to single buffered send === + + @pytest.mark.asyncio + async def test_eager_seed_failure_degrades_to_buffer_only(self): + """If the eager re-seed frame fails, native is disabled and buffer_only + is set so post-answer content lands as one buffered send() rather than + per-tick fragments on a non-editable platform.""" + # Make the SECOND seed (the eager re-seed) fail while the initial seed + # succeeds, so we actually reach the reopen-pending state first. + from gateway.platforms.base import BasePlatformAdapter, SendResult + + NativeStreamingAdapter = type( + "NativeStreamingAdapter2", + (BasePlatformAdapter,), + { + "MAX_MESSAGE_LENGTH": 4096, + "SUPPORTS_MESSAGE_EDITING": False, + "SUPPORTS_NATIVE_STREAMING": True, + }, + ) + NativeStreamingAdapter.__abstractmethods__ = frozenset() + adapter = NativeStreamingAdapter.__new__(NativeStreamingAdapter) + adapter._typing_paused = set() + adapter._fatal_error_message = None + adapter.frames = [] + adapter.supports_native_streaming = ( + lambda chat_type=None, metadata=None: True + ) + + empty_seed_count = {"n": 0} + + async def _send_stream_frame( + text, *, finalize=False, chat_id=None, reply_to=None, **kwargs + ): + adapter.frames.append({ + "text": text, "finalize": finalize, + "chat_id": chat_id, "reply_to": reply_to, + }) + if finalize: + return True + if text == "": + empty_seed_count["n"] += 1 + # First empty seed (run-start) succeeds; second (eager re-seed) + # fails to exercise the degrade path. + return empty_seed_count["n"] == 1 + return True + adapter.send_stream_frame = _send_stream_frame + adapter.send = AsyncMock( + return_value=SimpleNamespace(success=True, message_id="fallback_msg"), + ) + adapter.edit_message = AsyncMock(return_value=SimpleNamespace(success=True)) + + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = await self._to_reopen_pending(consumer, adapter) + assert consumer._use_native_streaming is True # still on after boundary + + consumer.request_reopen_seed() + await self._drain(consumer, 0.05) # process _REOPEN_SEED → seed fails + + assert await self._wait_until( + lambda: consumer._use_native_streaming is False + ), "failed eager seed must disable native streaming" + assert consumer.cfg.buffer_only is True + assert consumer._native_stream_opened is False + + consumer.finish() + await task + + # === POINT 6: single-bubble invariant (core) — eager + lazy don't stack === + + @pytest.mark.asyncio + async def test_eager_seed_then_delta_does_not_double_seed(self): + """After an eager seed opens the stream, a subsequent post-answer delta + must flow into that SAME stream — the lazy re-seed path must NOT open a + second bubble (it's gated on _native_stream_opened being False).""" + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = await self._to_reopen_pending(consumer, adapter) + seeds_before = len( + [f for f in adapter.frames if f["text"] == "" and not f["finalize"]] + ) + + consumer.request_reopen_seed() + await self._drain(consumer, 0.05) # eager seed opens the stream + assert await self._wait_until(lambda: consumer._native_stream_opened) + + # Now the LLM produces post-answer content. + consumer.on_delta("根据你的选择,这是后续的完整回答内容,足够长以触发一次刷新。") + await self._drain(consumer, 0.05) + consumer.finish() + await task + + seeds_after = len( + [f for f in adapter.frames if f["text"] == "" and not f["finalize"]] + ) + # Exactly ONE new empty seed total (the eager one) — the lazy path did + # not add a second. + assert seeds_after == seeds_before + 1, ( + f"lazy re-seed must not stack a second bubble on top of the eager " + f"seed (before={seeds_before}, after={seeds_after})" + ) + finalize_frames = [f for f in adapter.frames if f["finalize"]] + # The post-answer content lands in the final (reopened) finalize. + assert "后续的完整回答" in finalize_frames[-1]["text"] + adapter.send.assert_not_awaited() + + # === POINT 7: silence marker closes the eager stream (hole B) === + + @pytest.mark.asyncio + async def test_eager_seed_then_silence_marker_closes_stream(self): + """After an eager seed, if the agent's whole reply is an intentional + silence marker (NO_REPLY), the open native stream must be finalized + (closed) rather than left hanging — and the delivery flags stay False. + + DESTRUCTIVE: remove the _suppress_silence_marker native-close patch and + this fails (the stream never gets a finalize). + """ + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = await self._to_reopen_pending(consumer, adapter) + consumer.request_reopen_seed() + await self._drain(consumer, 0.05) + assert await self._wait_until(lambda: consumer._native_stream_opened) + + frames_before = len(adapter.frames) + + # Agent emits only a bare silence marker. + consumer.on_delta("NO_REPLY") + consumer.finish() + await task + + new_frames = adapter.frames[frames_before:] + finalize_frames = [f for f in new_frames if f["finalize"]] + assert len(finalize_frames) >= 1, ( + f"silence marker after eager seed must finalize/close the open " + f"native stream, new_frames={new_frames}" + ) + # The marker text must never have been streamed as content. + assert not any("NO_REPLY" in f["text"] for f in adapter.frames), ( + "silence marker must not leak into any frame" + ) + # Nothing was delivered — flags stay False. + assert consumer.final_content_delivered is False + assert consumer.final_response_sent is False + assert consumer._native_stream_opened is False + + # === ROUND 2, POINT 1: 降级后 post-answer 内容落地为单气泡(补强 Point 5)=== + + @pytest.mark.asyncio + async def test_degraded_post_answer_lands_single_bubble(self): + """eager 再次 seed 失败降级(_use_native_streaming=False + buffer_only=True) + 后,继续喂 post-answer 内容并 finish,内容必须以恰好一次 send() 落地为单气泡, + 且绝不重发 pre-clarify 的旧内容(boundary 时 _reset_segment_state 已清空 buffer)。 + + 直接验证 review (b).4 的结论。 + """ + # 复用 Point 5 的降级 adapter:第一次空 seed 成功、第二次(eager 再 seed)失败。 + from gateway.platforms.base import BasePlatformAdapter, SendResult + + NativeStreamingAdapter = type( + "NativeStreamingAdapter2b", + (BasePlatformAdapter,), + { + "MAX_MESSAGE_LENGTH": 4096, + "SUPPORTS_MESSAGE_EDITING": False, + "SUPPORTS_NATIVE_STREAMING": True, + }, + ) + NativeStreamingAdapter.__abstractmethods__ = frozenset() + adapter = NativeStreamingAdapter.__new__(NativeStreamingAdapter) + adapter._typing_paused = set() + adapter._fatal_error_message = None + adapter.frames = [] + adapter.supports_native_streaming = ( + lambda chat_type=None, metadata=None: True + ) + + empty_seed_count = {"n": 0} + + async def _send_stream_frame( + text, *, finalize=False, chat_id=None, reply_to=None, **kwargs + ): + adapter.frames.append({ + "text": text, "finalize": finalize, + "chat_id": chat_id, "reply_to": reply_to, + }) + if finalize: + return True + if text == "": + empty_seed_count["n"] += 1 + # 初始 seed(run-start)成功;第二次空 seed(eager 再 seed)失败。 + return empty_seed_count["n"] == 1 + return True + adapter.send_stream_frame = _send_stream_frame + adapter.send = AsyncMock( + return_value=SimpleNamespace(success=True, message_id="fallback_msg"), + ) + adapter.edit_message = AsyncMock(return_value=SimpleNamespace(success=True)) + + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = await self._to_reopen_pending(consumer, adapter) + consumer.request_reopen_seed() + await self._drain(consumer, 0.05) # _REOPEN_SEED → 第二次 seed 失败 → 降级 + + assert await self._wait_until( + lambda: consumer._use_native_streaming is False + ), "eager seed 失败必须关闭 native streaming" + assert consumer.cfg.buffer_only is True + + # 降级后继续产出 post-answer 内容 → 应以一次 send() 单气泡投递。 + consumer.on_delta("根据你的选择,这是完整的后续答复内容。") + await self._drain(consumer, 0.05) + consumer.finish() + await task + + # send() 恰好一次。 + adapter.send.assert_awaited_once() + # 取出该次 send 的内容(在 _send_or_edit 首发路径以 content= kwarg 传入)。 + sent_call = adapter.send.await_args + sent_content = sent_call.kwargs.get("content") + if sent_content is None and sent_call.args: + # 兼容位置参数写法(其它 boundary fallback 用 send(chat_id, text))。 + sent_content = sent_call.args[-1] + assert sent_content is not None + assert "完整的后续答复" in sent_content + # 绝不重发 pre-clarify 内容(boundary 已 finalize 成稳定气泡并清空 buffer)。 + assert "提问前已经流式出去的一段内容。" not in sent_content + + # === ROUND 2, POINT 2: 短内容不被误判为「无内容」而走 hole-A 空关闭 === + + @pytest.mark.asyncio + async def test_eager_seed_short_content_not_hole_a(self): + """eager seed 开流后只产出一段不足 60 char 节流阈值的短内容(中途不推帧), + finish 时该短内容必须随 finalize 帧落地,而不是走 hole-A 的空关闭(空 finalize + 或 "✅")。因为 _accumulated 非空,hole-A 条件 `not _accumulated` 不成立。 + """ + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + task = await self._to_reopen_pending(consumer, adapter) + consumer.request_reopen_seed() + await self._drain(consumer, 0.05) + assert await self._wait_until( + lambda: consumer._native_stream_opened + ), "eager seed 应已开流" + + frames_before = len(adapter.frames) + + # 只产出一个 1-char 短内容(远不足 60 char 节流阈值 → 中途不推帧)。 + consumer.on_delta("短") + consumer.finish() + await task + + new_frames = adapter.frames[frames_before:] + finalize_frames = [f for f in new_frames if f["finalize"]] + # 收尾恰好一个 finalize 帧,且携带短内容。 + assert len(finalize_frames) == 1, ( + f"eager seed 后短内容应恰好 finalize 一次,got {finalize_frames}" + ) + assert "短" in finalize_frames[0]["text"], ( + f"finalize 帧必须携带短内容,got {finalize_frames[0]!r}" + ) + # 未走 hole-A:没有空 finalize 收尾,也没有 "✅" 占位。 + assert finalize_frames[0]["text"] != "", "不应是 hole-A 的空 finalize" + assert not any(f["text"] == "✅" for f in adapter.frames), ( + f"短内容不应被误判为无内容而发 '✅',frames={adapter.frames}" + ) + # _accumulated 非空 → hole-A 条件 `not _accumulated` 不成立,佐证走的是内容收尾。 + assert consumer._accumulated == "短" + + # === ROUND 2, POINT 3: 双 clarify 边界链路自洽、无残留误判 === + + @pytest.mark.asyncio + async def test_double_clarify_boundary_reseed_chain(self): + """eager seed → 内容 → 第二轮 clarify boundary → 第二轮 eager seed。 + + 验证多边界链路自洽:第二轮 boundary 后回到 reopen-pending 状态 + (_awaiting_reopen_after_boundary=True、_native_stream_opened=False), + 再次 request_reopen_seed() 仍能成功 eager seed,标志无残留导致误判。 + + 覆盖 review (b).2。 + """ + adapter = _make_native_streaming_adapter() + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + # 第一轮:boundary → eager seed → 内容。 + task = await self._to_reopen_pending(consumer, adapter) + consumer.request_reopen_seed() + await self._drain(consumer, 0.05) + assert await self._wait_until(lambda: consumer._native_stream_opened) + assert consumer._reopen_seeded_eagerly is True + + consumer.on_delta("根据你的选择,这是后续的完整回答内容,足够长以触发一次流式刷新的补充。") + await self._drain(consumer, 0.05) + + seeds_before_second_boundary = len( + [f for f in adapter.frames if f["text"] == "" and not f["finalize"]] + ) + + # 第二轮 clarify boundary(reopen=True)。 + boundary = consumer.close_for_approval_prompt( + "💬 等待你的选择...", reason="Clarify", reopen=True, + ) + fut = boundary[0] if isinstance(boundary, tuple) else boundary + await asyncio.wait_for(fut, timeout=1.0) + + # 第二轮 boundary 后:回到 reopen-pending,stream 已关。 + assert consumer._awaiting_reopen_after_boundary is True + assert consumer._native_stream_opened is False + assert consumer._use_native_streaming is True + # NOTE: 与 task 预期不同 —— 产品代码在 boundary 处理里并不重置 + # _reopen_seeded_eagerly(见 code-review 观察点 O2:consumer 每 turn 新建, + # 残留无害)。这里断言真实行为(残留 True),并在下方证明该残留不会 + # 导致第二轮 eager seed 误判 —— 链路仍自洽。 + assert consumer._reopen_seeded_eagerly is True + + # 第二轮 eager seed:即便标志有残留,仍能正确再次开流。 + consumer.request_reopen_seed() + await self._drain(consumer, 0.05) + + seeds_after = len( + [f for f in adapter.frames if f["text"] == "" and not f["finalize"]] + ) + assert seeds_after == seeds_before_second_boundary + 1, ( + "第二轮 eager seed 必须再发一个新的空 seed 帧 " + f"(before={seeds_before_second_boundary}, after={seeds_after})" + ) + assert await self._wait_until(lambda: consumer._native_stream_opened) + assert consumer._reopen_seeded_eagerly is True + assert consumer._awaiting_reopen_after_boundary is False + + consumer.finish() + await task diff --git a/tests/gateway/test_wecom.py b/tests/gateway/test_wecom.py index 65c04730a1..abfe5d5a35 100644 --- a/tests/gateway/test_wecom.py +++ b/tests/gateway/test_wecom.py @@ -360,6 +360,136 @@ class TestSend: ) + @pytest.mark.asyncio + async def test_approval_confirmation_uses_proactive_send(self): + """Regression: force_proactive_send=True must use APP_CMD_SEND to avoid + consuming the req_id that the post-approval stream needs. Passive reply + on the same req_id causes WeCom to render the stream seed as empty bubble.""" + from plugins.platforms.wecom.adapter import APP_CMD_SEND, WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + # Simulate a cached req_id from the user's /approve message + adapter._last_chat_req_ids["chat-123"] = "req-approve" + adapter._send_request = AsyncMock(return_value={"headers": {"req_id": "req-approve"}, "errcode": 0}) + adapter._send_reply_request = AsyncMock( + return_value={"headers": {"req_id": "req-approve"}, "errcode": 0} + ) + + result = await adapter.send( + "chat-123", + "✅ Approved 1 command. Continuing...", + metadata={"is_approval_prompt": True, "force_proactive_send": True}, + ) + + assert result.success is True + # Must use APP_CMD_SEND (proactive), NOT _send_reply_request (passive) + adapter._send_request.assert_awaited_once_with( + APP_CMD_SEND, + { + "chatid": "chat-123", + "msgtype": "markdown", + "markdown": {"content": "✅ Approved 1 command. Continuing..."}, + }, + ) + adapter._send_reply_request.assert_not_awaited() + + @pytest.mark.asyncio + async def test_approval_request_prompt_uses_passive_reply(self): + """is_approval_prompt alone (without force_proactive_send) must still use + passive reply. The initial approval *request* prompt needs passive reply + because groups cannot use APP_CMD_SEND.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["group-chat"] = "req-user-msg" + adapter._send_reply_request = AsyncMock( + return_value={"headers": {"req_id": "req-user-msg"}, "errcode": 0} + ) + adapter._send_request = AsyncMock(return_value={"errcode": 0}) + + result = await adapter.send( + "group-chat", + "⚠️ Dangerous command requires approval...", + metadata={"is_approval_prompt": True}, # No force_proactive_send + ) + + assert result.success is True + # Should use passive reply (preserving req_id for group delivery) + adapter._send_reply_request.assert_awaited_once() + adapter._send_request.assert_not_awaited() + + @pytest.mark.asyncio + async def test_send_with_active_stream_still_uses_passive_reply(self): + """send() should NOT force proactive when a stream is active — + that's too broad and breaks group delivery. Only explicit + force_proactive_send metadata triggers proactive mode.""" + from plugins.platforms.wecom.adapter import WeComAdapter, StreamTurn + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-123"] = "req-latest" + # Simulate an active stream turn for this chat + turn = StreamTurn("chat-123", "req-latest") + adapter._stream_turns["chat-123:turn-1"] = turn + + adapter._send_reply_request = AsyncMock( + return_value={"headers": {"req_id": "req-latest"}, "errcode": 0} + ) + adapter._send_request = AsyncMock(return_value={"errcode": 0}) + + result = await adapter.send("chat-123", "Some status message") + + assert result.success is True + # Should still use passive reply — active stream doesn't force proactive + adapter._send_reply_request.assert_awaited_once() + adapter._send_request.assert_not_awaited() + + @pytest.mark.asyncio + async def test_force_proactive_falls_back_to_passive_for_groups(self): + """Regression: force_proactive_send must NOT use APP_CMD_SEND in group chats. + WeCom AI Bots cannot initiate APP_CMD_SEND in groups — only passive reply + (APP_CMD_RESPONSE) bound to a req_id works.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["group-chat"] = "req-approve" + # Mark this chat as a group + adapter._group_chat_ids.add("group-chat") + + adapter._send_reply_request = AsyncMock( + return_value={"headers": {"req_id": "req-approve"}, "errcode": 0} + ) + adapter._send_request = AsyncMock(return_value={"errcode": 0}) + + result = await adapter.send( + "group-chat", + "✅ Approved 1 command. Continuing...", + metadata={"is_approval_prompt": True, "force_proactive_send": True}, + ) + + assert result.success is True + # Group chats must fall back to passive reply even with force_proactive_send + adapter._send_reply_request.assert_awaited_once() + adapter._send_request.assert_not_awaited() + + @pytest.mark.asyncio + async def test_group_send_fails_early_without_req_id(self): + """Group chats with no cached req_id must fail with a clear error + instead of attempting APP_CMD_SEND (which WeCom will reject).""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + # No req_id cached for this group + adapter._group_chat_ids.add("group-no-req") + adapter._send_request = AsyncMock(return_value={"errcode": 0}) + + result = await adapter.send("group-no-req", "hello group") + + assert result.success is False + assert "req_id" in (result.error or "").lower() + # Should NOT attempt APP_CMD_SEND + adapter._send_request.assert_not_awaited() + + class TestInboundMessages: @pytest.mark.asyncio async def test_on_message_builds_event(self): @@ -530,3 +660,968 @@ class TestTextBatchFlushRace: "superseded task must not pop the event" ) + @pytest.mark.asyncio + async def test_active_task_processes_event_normally(self): + """When the task is not superseded it must still process the event.""" + from gateway.platforms.base import MessageEvent, MessageType + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._text_batch_delay_seconds = 0 + + key = "test-session" + event = MessageEvent(text="world", message_type=MessageType.TEXT) + adapter._pending_text_batches[key] = event + + handle_calls = [] + + async def fake_handle(evt): + handle_calls.append(evt) + + adapter.handle_message = fake_handle + + t1 = asyncio.create_task(adapter._flush_text_batch(key)) + adapter._pending_text_batch_tasks[key] = t1 + + # No superseding task — T1 should process normally. + await asyncio.sleep(0.05) + + assert handle_calls == [event], "active task must call handle_message" + assert adapter._pending_text_batches.get(key) is None, ( + "active task must pop the event after processing" + ) + + +class TestAttachmentTextMerge: + """WeCom sends "image + text" as two separate inbound callbacks (an + attachment-only frame, then a text frame ~hundreds of ms later). + + Dispatching the attachment immediately spawns an agent run that the + trailing text then "interrupts" (junk "⚡ Interrupting" + "✅" acks). + The adapter buffers an attachment-only message on the existing text-batch + machinery for a short merge window so the following text merges into ONE + dispatched event. These tests exercise the real _on_message path. + """ + + @staticmethod + def _make_adapter(merge_delay: float = 0.15): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter( + PlatformConfig( + enabled=True, + extra={ + "dm_policy": "open", + "attachment_text_merge_delay_seconds": merge_delay, + }, + ) + ) + # DM open policy needs the opt-in env flag; force intake open. + adapter._is_dm_intake_allowed = lambda sender_id: True + # Keep the text-split batch window tiny so tests are fast. + adapter._text_batch_delay_seconds = 0.05 + adapter.handle_message = AsyncMock() + return adapter + + @staticmethod + def _image_payload(msgid: str, media): + return { + "cmd": "aibot_msg_callback", + "headers": {"req_id": f"req-{msgid}"}, + "body": { + "msgid": msgid, + "from": {"userid": "user-1"}, + "msgtype": "image", + "image": {"url": "https://example.com/x.png"}, + "_media": media, + }, + } + + @staticmethod + def _text_payload(msgid: str, content: str): + return { + "cmd": "aibot_msg_callback", + "headers": {"req_id": f"req-{msgid}"}, + "body": { + "msgid": msgid, + "from": {"userid": "user-1"}, + "msgtype": "text", + "text": {"content": content}, + }, + } + + @pytest.mark.asyncio + async def test_image_then_text_merge_into_one_event(self): + """image-then-text within the window → ONE dispatched event carrying + both the media and the text, and NO immediate dispatch of the image + (so the busy-handler interrupt path is never triggered).""" + adapter = self._make_adapter(merge_delay=0.2) + + async def fake_extract_media(body): + if body.get("msgtype") == "image": + return (["/tmp/x.png"], ["image/png"]) + return ([], []) + + adapter._extract_media = fake_extract_media + + await adapter._on_message(self._image_payload("img-1", None)) + # Image must be held, not dispatched. + adapter.handle_message.assert_not_called() + + # Text arrives within the merge window. + await asyncio.sleep(0.05) + await adapter._on_message(self._text_payload("txt-1", "what is this?")) + adapter.handle_message.assert_not_called() + + # After the window elapses, exactly one merged event dispatches. + await asyncio.sleep(0.3) + adapter.handle_message.assert_awaited_once() + event = adapter.handle_message.await_args.args[0] + from gateway.platforms.base import MessageType + + assert event.text == "what is this?" + assert event.media_urls == ["/tmp/x.png"] + assert event.media_types == ["image/png"] + assert event.message_type == MessageType.TEXT + + @pytest.mark.asyncio + async def test_image_only_dispatched_after_window(self): + """image-only with no following text → still dispatched on its own + after the merge window (must not be dropped).""" + adapter = self._make_adapter(merge_delay=0.15) + adapter._extract_media = AsyncMock(return_value=(["/tmp/x.png"], ["image/png"])) + + await adapter._on_message(self._image_payload("img-1", None)) + adapter.handle_message.assert_not_called() + + await asyncio.sleep(0.3) + adapter.handle_message.assert_awaited_once() + event = adapter.handle_message.await_args.args[0] + from gateway.platforms.base import MessageType + + assert event.media_urls == ["/tmp/x.png"] + assert event.message_type == MessageType.PHOTO + + @pytest.mark.asyncio + async def test_multiple_attachments_then_text_all_merged(self): + """Two attachment-only frames then text → all media merged into one + dispatched event with the text.""" + adapter = self._make_adapter(merge_delay=0.2) + + counter = {"n": 0} + + async def fake_extract_media(body): + if body.get("msgtype") == "image": + counter["n"] += 1 + n = counter["n"] + return ([f"/tmp/x{n}.png"], ["image/png"]) + return ([], []) + + adapter._extract_media = fake_extract_media + + await adapter._on_message(self._image_payload("img-1", None)) + await asyncio.sleep(0.03) + await adapter._on_message(self._image_payload("img-2", None)) + await asyncio.sleep(0.03) + await adapter._on_message(self._text_payload("txt-1", "describe both")) + adapter.handle_message.assert_not_called() + + await asyncio.sleep(0.35) + adapter.handle_message.assert_awaited_once() + event = adapter.handle_message.await_args.args[0] + assert event.text == "describe both" + assert event.media_urls == ["/tmp/x1.png", "/tmp/x2.png"] + assert event.media_types == ["image/png", "image/png"] + + @pytest.mark.asyncio + async def test_pure_text_unaffected(self): + """Regression: pure text still flows through the text-batch path and + dispatches as a single text event.""" + adapter = self._make_adapter() + adapter._extract_media = AsyncMock(return_value=([], [])) + + await adapter._on_message(self._text_payload("txt-1", "just text")) + adapter.handle_message.assert_not_called() + + await asyncio.sleep(0.2) + adapter.handle_message.assert_awaited_once() + event = adapter.handle_message.await_args.args[0] + from gateway.platforms.base import MessageType + + assert event.text == "just text" + assert event.media_urls == [] + assert event.message_type == MessageType.TEXT + + + +# === NATIVE STREAMING (msgtype: stream) === + + +class TestWeComNativeStreamingCapability: + """SUPPORTS_NATIVE_STREAMING + supports_native_streaming() probe.""" + + def test_class_attribute_declares_native_streaming(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + assert WeComAdapter.SUPPORTS_NATIVE_STREAMING is True + + def test_supports_native_streaming_returns_true_for_dm(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + assert adapter.supports_native_streaming(chat_type="dm") is True + + def test_supports_native_streaming_returns_true_for_group(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + assert adapter.supports_native_streaming(chat_type="group") is True + + def test_max_stream_content_length_is_20480(self): + from plugins.platforms.wecom.adapter import ( + MAX_STREAM_CONTENT_LENGTH, WeComAdapter, + ) + + assert MAX_STREAM_CONTENT_LENGTH == 20480 + assert WeComAdapter.MAX_STREAM_CONTENT_LENGTH == 20480 + + def test_stream_expired_errcode_constant(self): + from plugins.platforms.wecom.adapter import STREAM_EXPIRED_ERRCODE + + assert STREAM_EXPIRED_ERRCODE == 846608 + +# === STREAM TESTS PLACEHOLDER === + + +class TestResolveStreamReqId: + """`_resolve_stream_req_id` precedence: reply_to → cached chat → None.""" + + def test_prefers_explicit_reply_to(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._reply_req_ids["msg-123"] = "explicit-req" + adapter._last_chat_req_ids["chat-1"] = "cached-req" + + assert adapter._resolve_stream_req_id("chat-1", "msg-123") == "explicit-req" + + def test_falls_back_to_cached_chat_req_id(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "cached-req" + + assert adapter._resolve_stream_req_id("chat-1", reply_to=None) == "cached-req" + + def test_returns_none_when_no_anchor(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + assert adapter._resolve_stream_req_id("unknown-chat", None) is None + + def test_quoted_reply_to_falls_through_to_chat_cache(self): + """``quote:msg-id`` (quote-context marker) is not a real reply anchor.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "cached-req" + + assert adapter._resolve_stream_req_id("chat-1", "quote:m-1") == "cached-req" + + +# === LIFECYCLE TESTS PLACEHOLDER === + + +class TestSendStreamFrame: + """`send_stream_frame` lifecycle: init → cumulative updates → finalize.""" + + @staticmethod + def _mock_send_json_with_immediate_ack(adapter): + """Mock _send_reply_queued to bypass ack tracking entirely. + + For tests that verify frame content/ordering, we don't need actual + ack tracking — just record what was sent and always succeed. + """ + sent_frames = [] + + async def mock_send_reply_queued(reply_req_id, body, *, is_final=False, skip_if_pending=False): + sent_frames.append({ + "req_id": reply_req_id, + "body": body, + "is_final": is_final, + }) + return {"errcode": 0, "errmsg": "ok"} + + adapter._send_reply_queued = AsyncMock(side_effect=mock_send_reply_queued) + adapter._sent_frames = sent_frames + + @pytest.mark.asyncio + async def test_first_call_seeds_thinking_frame_then_returns_true(self): + """First frame for a chat sends seed. + + Bypasses ack tracking so both seed and content are sent in same call. + Uses a payload above ``BLOCK_STREAM_MIN_CHARS`` ending on a sentence + boundary so the block chunker emits immediately (otherwise the + chunker buffers waiting for either ``min_chars`` + a safe break or + the 250ms idle flush). + """ + from plugins.platforms.wecom.adapter import WeComAdapter, BLOCK_STREAM_MIN_CHARS + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._ws = MagicMock(closed=False) + # Mock _send_reply_queued to bypass ack tracking + self._mock_send_json_with_immediate_ack(adapter) + + # Build a sentence-terminated payload above min_chars. + payload = ("hello world. " * 12).strip() + assert len(payload) >= BLOCK_STREAM_MIN_CHARS + ok = await adapter.send_stream_frame(payload, chat_id="chat-1") + + assert ok is True + # seed + content = 2 frames + assert len(adapter._sent_frames) == 2 + seed_frame = adapter._sent_frames[0] + assert seed_frame["body"]["msgtype"] == "stream" + assert seed_frame["body"]["stream"]["content"] == "" + assert seed_frame["body"]["stream"]["finish"] is False + + content_frame = adapter._sent_frames[1] + assert content_frame["body"]["stream"]["content"] == payload + assert content_frame["body"]["stream"]["finish"] is False + + @pytest.mark.asyncio + async def test_first_and_second_call_share_stream_id(self): + """Successive frames use the same stream_id. + + Both frames carry sentence-terminated payloads above min_chars so the + block chunker emits each one — the test is about stream_id continuity + across frames, not about chunker thresholds. + """ + from plugins.platforms.wecom.adapter import WeComAdapter, BLOCK_STREAM_MIN_CHARS + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._ws = MagicMock(closed=False) + # Immediate ack so all frames are sent (no pending-skip) + self._mock_send_json_with_immediate_ack(adapter) + + first = ("alpha sentence. " * 10).strip() + second = first + " " + ("beta sentence. " * 10).strip() + assert len(first) >= BLOCK_STREAM_MIN_CHARS + assert len(second) - len(first) >= BLOCK_STREAM_MIN_CHARS + await adapter.send_stream_frame(first, chat_id="chat-1") + await adapter.send_stream_frame(second, chat_id="chat-1") + + # seed + first + second = 3 frames + assert len(adapter._sent_frames) == 3 + ids = [frame["body"]["stream"]["id"] for frame in adapter._sent_frames] + assert ids[0] == ids[1] == ids[2] + assert ids[0].startswith("stream_") + + @pytest.mark.asyncio + async def test_intermediate_frame_skipped_when_pending_ack(self): + """Intermediate frames are skipped if a prior frame's ack is pending. + + This is the new ack-tracking semantics: if the seed frame's ack hasn't + returned yet, the next intermediate frame is skipped (returns success + but doesn't actually send). This prevents errcode 6000 version conflict. + """ + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._send_json = AsyncMock() # No auto-ack — pending stays pending + adapter._ws = MagicMock(closed=False) + + await adapter.send_stream_frame("alpha", chat_id="chat-1") + # Seed frame sent, pending_ack is set. Immediately send another: + ok = await adapter.send_stream_frame("alpha beta", chat_id="chat-1") + + assert ok is True # returns True (skip is silent success) + # Only seed frame sent; second was skipped due to pending ack. + assert adapter._send_json.await_count == 1 + + # accumulated_text still updated in StreamTurn despite skip. + turn = list(adapter._stream_turns.values())[0] + assert turn.accumulated_text == "alpha beta" + + @pytest.mark.asyncio + async def test_intermediate_frame_cap_drops_excess(self): + """After MAX_INTERMEDIATE_FRAMES, further intermediate frames are dropped.""" + from plugins.platforms.wecom.adapter import WeComAdapter, MAX_INTERMEDIATE_FRAMES + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._ws = MagicMock(closed=False) + # Auto-ack so seed + first frame go through + self._mock_send_json_with_immediate_ack(adapter) + + # First call creates turn + seed + content frame. + turn_id = "cap-test" + await adapter.send_stream_frame("first", chat_id="chat-1", turn_id=turn_id) + turn_key = f"chat-1:{turn_id}" + turn = adapter._stream_turns[turn_key] + + # Artificially set counter to the cap. + turn._intermediate_frames_sent = MAX_INTERMEDIATE_FRAMES + turn._last_frame_sent_at = 0 # clear time throttle + + # Record count BEFORE the overflow frame to assert it was truly skipped. + before_overflow = len(adapter._sent_frames) + + # Next intermediate frame should be dropped. + ok = await adapter.send_stream_frame("overflow", chat_id="chat-1", turn_id=turn_id) + assert ok is True + assert turn.accumulated_text == "overflow" + # No additional frame sent — overflow was dropped. + assert len(adapter._sent_frames) == before_overflow + + # Finalize still goes through unconditionally. + ok = await adapter.send_stream_frame("final", chat_id="chat-1", finalize=True, turn_id=turn_id) + assert ok is True + + @pytest.mark.asyncio + async def test_finalize_sends_finish_true_and_resets_state(self): + """Finalize frame waits for pending ack, sends finish=true, cleans up turn.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._ws = MagicMock(closed=False) + # Auto-ack so seed + content + finalize all go through + self._mock_send_json_with_immediate_ack(adapter) + + # With turn_id, creates independent turn + turn_id = "test-turn-1" + await adapter.send_stream_frame("partial", chat_id="chat-1", turn_id=turn_id) + turn_key = "chat-1:test-turn-1" + assert turn_key in adapter._stream_turns + turn = adapter._stream_turns[turn_key] + assert turn.stream_id is not None + + ok = await adapter.send_stream_frame( + "partial final", chat_id="chat-1", finalize=True, turn_id=turn_id, + ) + + assert ok is True + # After finalize, turn should be cleaned up + assert turn_key not in adapter._stream_turns + # Finalize goes through _send_reply_queued (mocked). + # Find the finalize frame (is_final=True) + finalize_frames = [ + f for f in adapter._sent_frames + if f["body"].get("stream", {}).get("finish") is True + ] + assert len(finalize_frames) == 1 + assert finalize_frames[0]["body"]["stream"]["content"] == "partial final" + +# === FAILURE TESTS PLACEHOLDER === + + +class TestSendStreamFrameFailures: + """Behavior when no req_id, 846608 expiry, or generic transport errors.""" + + @pytest.mark.asyncio + async def test_returns_false_when_no_req_id_available(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + # No reply_to, nothing in _last_chat_req_ids. + adapter._send_reply_request = AsyncMock() + + ok = await adapter.send_stream_frame("hi", chat_id="unknown-chat") + + assert ok is False + adapter._send_reply_request.assert_not_awaited() + + @pytest.mark.asyncio + async def test_returns_false_when_chat_id_missing_on_first_call(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._send_reply_request = AsyncMock() + + ok = await adapter.send_stream_frame("hi", chat_id=None) + + assert ok is False + adapter._send_reply_request.assert_not_awaited() + + @pytest.mark.asyncio + async def test_846608_marks_chat_expired_and_returns_false(self): + """846608 on finalize frame marks the chat expired and returns False.""" + from plugins.platforms.wecom.adapter import ( + STREAM_EXPIRED_ERRCODE, WeComAdapter, + ) + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._ws = MagicMock(closed=False) + + # Mock _send_reply_queued: intermediate succeeds, final returns 846608 + async def mock_queued(reply_req_id, body, *, is_final=False, skip_if_pending=False): + if is_final: + return {"errcode": STREAM_EXPIRED_ERRCODE, "errmsg": "stream expired"} + return {"errcode": 0, "errmsg": "ok"} + + adapter._send_reply_queued = AsyncMock(side_effect=mock_queued) + + # First call (seed + content) succeeds + turn_id = "test-turn-2" + await adapter.send_stream_frame("hello", chat_id="chat-1", turn_id=turn_id) + # Now try to finalize — ack returns 846608. + ok = await adapter.send_stream_frame("hello final", chat_id="chat-1", finalize=True, turn_id=turn_id) + + assert ok is False + assert "chat-1" in adapter._stream_expired_chats + # This specific turn should be cleaned up + turn_key = "chat-1:test-turn-2" + assert turn_key not in adapter._stream_turns + + @pytest.mark.asyncio + async def test_subsequent_call_to_expired_chat_short_circuits(self): + """Once a chat is in ``_stream_expired_chats``, send_stream_frame + bails immediately for new turns without touching the WS.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._stream_expired_chats.add("chat-1") + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._send_reply_request = AsyncMock() + + # Without turn_id: short-circuits immediately + ok = await adapter.send_stream_frame("hi", chat_id="chat-1") + assert ok is False + adapter._send_reply_request.assert_not_awaited() + + # With a new turn_id: also short-circuits (can't create new turn) + ok = await adapter.send_stream_frame("hi", chat_id="chat-1", turn_id="new-turn") + assert ok is False + adapter._send_reply_request.assert_not_awaited() + + @pytest.mark.asyncio + async def test_inbound_message_clears_expired_marker(self): + """A fresh inbound req_id must resurrect the stream channel.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._stream_expired_chats.add("chat-1") + + adapter._remember_chat_req_id("chat-1", "fresh-req-id") + + assert "chat-1" not in adapter._stream_expired_chats + + @pytest.mark.asyncio + async def test_generic_transport_error_resets_state(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._send_reply_request = AsyncMock( + side_effect=RuntimeError("ws disconnected"), + ) + + @pytest.mark.asyncio + async def test_generic_transport_error_resets_state(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._send_reply_request = AsyncMock( + side_effect=RuntimeError("ws disconnected"), + ) + + turn_id = "test-turn-3" + ok = await adapter.send_stream_frame("hi", chat_id="chat-1", turn_id=turn_id) + + assert ok is False + # Generic error cleans up this specific turn but does NOT mark chat as expired + turn_key = "chat-1:test-turn-3" + assert turn_key not in adapter._stream_turns + assert "chat-1" not in adapter._stream_expired_chats + +# === SEND_TYPING TESTS PLACEHOLDER === + + +class TestSendTypingTriggersThinking: + """``send_typing`` is a no-op — typing is handled by stream consumer.""" + + @pytest.mark.asyncio + async def test_send_typing_is_noop(self): + """send_typing must not open any stream — the consumer seed frame does.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._send_json = AsyncMock() + adapter._ws = MagicMock(closed=False) + + await adapter.send_typing("chat-1") + + adapter._send_json.assert_not_awaited() + # No stream turns created + assert len(adapter._stream_turns) == 0 + + @pytest.mark.asyncio + async def test_send_typing_does_not_raise(self): + """send_typing must never raise regardless of state.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + await adapter.send_typing("chat-1") + await adapter.send_typing("") + await adapter.send_typing(None) # type: ignore + + +class TestStreamContentTruncation: + """Bytes (not codepoints) are truncated to MAX_STREAM_CONTENT_LENGTH.""" + + def test_ascii_below_limit_passes_through(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + out = WeComAdapter._truncate_stream_content("hello", 1000) + assert out == "hello" + + def test_ascii_above_limit_is_byte_capped(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + big = "x" * 30000 + out = WeComAdapter._truncate_stream_content(big, 20480) + assert len(out.encode("utf-8")) <= 20480 + + def test_multibyte_truncation_does_not_split_codepoints(self): + """A 3-byte CJK char must not be sliced mid-byte and emit garbage.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + # Each "你" is 3 UTF-8 bytes. Limit at 5 bytes — must keep one + # full char and drop the half-cut second char rather than emit �. + out = WeComAdapter._truncate_stream_content("你你", 5) + assert out == "你" + # Crucially, must be valid UTF-8 (no replacement chars from + # mid-byte slices). + assert "�" not in out + + +@pytest.mark.skip(reason="Obsolete: send() no longer closes streams in new per-turn architecture") +class TestSendClosesActiveStream: + """OBSOLETE: These tests verify old behavior where send() closed active streams. + + In the new per-turn architecture (post Round 3 fixes), send() and streaming + are completely independent. Streams are managed by their creators + (GatewayStreamConsumer) via send_stream_frame(finalize=True, turn_id=...). + + See test_wecom_per_turn.py for tests of the new per-turn model. + """ + + @pytest.mark.asyncio + async def test_send_finalizes_active_stream_opened_by_consumer(self): + """When the stream consumer opened a stream and then send() delivers + the response (e.g. fallback path), send() must close the stream.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._send_json = AsyncMock() + adapter._ws = MagicMock(closed=False) + adapter._send_reply_request = AsyncMock(return_value={"errcode": 0}) + + # Manually set active stream state (as the consumer would). + adapter._active_stream_id = "stream_test" + adapter._active_stream_req_id = "req-1" + adapter._active_stream_chat_id = "chat-1" + + result = await adapter.send("chat-1", "Hello world!") + + assert result.success is True + assert adapter._active_stream_id is None + finalize_calls = [ + call for call in adapter._send_reply_request.await_args_list + if call.args[1].get("stream", {}).get("finish") is True + ] + assert len(finalize_calls) == 1 + assert finalize_calls[0].args[1]["stream"]["content"] == "Hello world!" + + @pytest.mark.asyncio + async def test_send_ignores_stream_for_different_chat(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-2"] = "req-2" + adapter._send_reply_request = AsyncMock(return_value={"errcode": 0}) + + adapter._active_stream_id = "stream_test" + adapter._active_stream_req_id = "req-1" + adapter._active_stream_chat_id = "chat-1" + + result = await adapter.send("chat-2", "Hi") + + assert result.success is True + assert adapter._active_stream_id is not None # untouched + + @pytest.mark.asyncio + async def test_send_falls_through_when_stream_expired(self): + from plugins.platforms.wecom.adapter import STREAM_EXPIRED_ERRCODE, WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + + async def fake(req_id, body, **kwargs): + if body.get("stream", {}).get("finish"): + return {"errcode": STREAM_EXPIRED_ERRCODE, "errmsg": "expired"} + return {"errcode": 0, "headers": {"req_id": req_id}} + + adapter._send_reply_request = AsyncMock(side_effect=fake) + adapter._active_stream_id = "stream_test" + adapter._active_stream_req_id = "req-1" + adapter._active_stream_chat_id = "chat-1" + + result = await adapter.send("chat-1", "Final answer") + + assert result.success is True + assert adapter._active_stream_id is None + assert "chat-1" in adapter._stream_expired_chats + + + +class TestBlockChunker: + """Unit tests for the sentence-aligned block chunker. + + The chunker buffers cumulative text and only emits when a sentence + boundary lands above ``min_chars``, or when a hard ``max_chars`` cap is + reached, or when the caller forces a drain (finalize / idle flush). + """ + + def test_emits_nothing_below_min_chars(self): + from plugins.platforms.wecom.adapter import _BlockChunker + + chunker = _BlockChunker(min_chars=120, max_chars=360) + chunker.update("Hello world.") + assert chunker.drain(force=False) is None + # State is preserved across drain attempts. + assert chunker.has_pending() is True + + def test_emits_on_sentence_boundary_above_min_chars(self): + from plugins.platforms.wecom.adapter import _BlockChunker + + chunker = _BlockChunker(min_chars=20, max_chars=360) + chunker.update("This is sentence one. This is two.") + out = chunker.drain(force=False) + assert out is not None + # Cumulative — full content up to the break point. + assert out.endswith(".") + # Nothing pending after a clean emit. + assert chunker.has_pending() is False + + def test_emits_on_paragraph_boundary(self): + from plugins.platforms.wecom.adapter import _BlockChunker + + chunker = _BlockChunker(min_chars=20, max_chars=360) + text = "Part one of the answer text here\n\nPart two of the answer" + chunker.update(text) + out = chunker.drain(force=False) + assert out is not None + assert out.endswith("\n\n") + assert chunker.has_pending() is True # part two still buffered + + def test_hard_cap_forces_break_mid_sentence(self): + from plugins.platforms.wecom.adapter import _BlockChunker + + chunker = _BlockChunker(min_chars=20, max_chars=50) + # Single run-on sentence with no terminators in the first 50 chars. + text = "a" * 80 + chunker.update(text) + out = chunker.drain(force=False) + assert out is not None + # Hard cap fires — entire cumulative is emitted (the chunker doesn't + # split mid-buffer; it leaves the next slice to the next update). + assert out == text + + def test_force_emits_remaining_buffer(self): + from plugins.platforms.wecom.adapter import _BlockChunker + + chunker = _BlockChunker(min_chars=120, max_chars=360) + chunker.update("Tiny tail") + out = chunker.drain(force=True) + assert out == "Tiny tail" + assert chunker.has_pending() is False + + def test_no_pending_after_complete_drain(self): + from plugins.platforms.wecom.adapter import _BlockChunker + + chunker = _BlockChunker(min_chars=10, max_chars=360) + chunker.update("hello there world.") + assert chunker.drain(force=False) == "hello there world." + assert chunker.drain(force=True) is None # nothing to emit + + def test_does_not_break_on_decimal_or_ip_address(self): + """Sentence break requires whitespace after terminator — '1.2' must not split.""" + from plugins.platforms.wecom.adapter import _BlockChunker + + chunker = _BlockChunker(min_chars=10, max_chars=360) + chunker.update("version 1.2.3 is out and not yet released") + # No safe sentence terminator → wait. + assert chunker.drain(force=False) is None + + def test_cumulative_update_does_not_shrink(self): + from plugins.platforms.wecom.adapter import _BlockChunker + + chunker = _BlockChunker() + chunker.update("long initial cumulative content " * 5) + before = chunker._cumulative + chunker.update("short") # caller bug — must not shrink high-water mark + assert chunker._cumulative == before + + +class TestBlockStreamingFrameFlow: + """Integration: send_stream_frame uses the chunker to gate intermediate frames.""" + + def _mock_send_json_with_immediate_ack(self, adapter): + from plugins.platforms.wecom.adapter import APP_CMD_RESPONSE + sent_frames = [] + + async def mock_send(reply_req_id, body, **kwargs): + is_final = kwargs.get("is_final", False) + sent_frames.append({ + "req_id": reply_req_id, + "body": body, + "is_final": is_final, + }) + return {"errcode": 0, "errmsg": "ok"} + + adapter._send_reply_queued = AsyncMock(side_effect=mock_send) + adapter._sent_frames = sent_frames + + @pytest.mark.asyncio + async def test_short_text_buffered_no_frame_sent(self): + """A few tokens below min_chars do not produce an intermediate frame.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._ws = MagicMock(closed=False) + self._mock_send_json_with_immediate_ack(adapter) + + ok = await adapter.send_stream_frame("Hello.", chat_id="chat-1") + assert ok is True + # Only the seed frame was sent — the 6-char body is below min_chars. + assert len(adapter._sent_frames) == 1 + assert adapter._sent_frames[0]["body"]["stream"]["content"] == "" + + @pytest.mark.asyncio + async def test_finalize_force_drains_buffered_tail(self): + """Finalize emits whatever the chunker has, even sub-min_chars.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._ws = MagicMock(closed=False) + self._mock_send_json_with_immediate_ack(adapter) + + # 1: buffer small text (no intermediate frame sent — only seed). + await adapter.send_stream_frame("Short.", chat_id="chat-1") + # 2: finalize with the same tiny tail — force-drains the chunker. + ok = await adapter.send_stream_frame( + "Short.", chat_id="chat-1", finalize=True, + ) + assert ok is True + # seed + finalize = 2 frames. + assert len(adapter._sent_frames) == 2 + final_frame = adapter._sent_frames[-1] + assert final_frame["body"]["stream"]["finish"] is True + # Content survives the force-drain. + assert final_frame["body"]["stream"]["content"].startswith("Short.") + + @pytest.mark.asyncio + async def test_idle_flush_emits_partial_buffer(self): + """A buffered sub-min tail still ships after the idle-flush timer fires.""" + from plugins.platforms.wecom.adapter import WeComAdapter, BLOCK_STREAM_IDLE_FLUSH + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._ws = MagicMock(closed=False) + self._mock_send_json_with_immediate_ack(adapter) + + await adapter.send_stream_frame("Half a thought", chat_id="chat-1") + # Wait past the idle window so the timer fires and drains. + await asyncio.sleep(BLOCK_STREAM_IDLE_FLUSH + 0.1) + # seed + idle-flushed partial = 2 frames. + assert len(adapter._sent_frames) == 2 + flushed = adapter._sent_frames[-1] + assert flushed["body"]["stream"]["content"] == "Half a thought" + assert flushed["body"]["stream"]["finish"] is False + + @pytest.mark.asyncio + async def test_block_emits_on_sentence_boundary(self): + """Cumulative text crossing a sentence boundary triggers an immediate frame.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._ws = MagicMock(closed=False) + self._mock_send_json_with_immediate_ack(adapter) + + # Build cumulative text that ends on a sentence boundary above min_chars. + text = "This is a complete first sentence with enough text. " * 4 + await adapter.send_stream_frame(text, chat_id="chat-1") + # seed + sentence block = 2 frames. + assert len(adapter._sent_frames) == 2 + block = adapter._sent_frames[-1] + # The chunker breaks immediately after the rightmost safe sentence + # terminator — i.e., right after the last ".", stripping the trailing + # space that the caller's cumulative blob included. + assert block["body"]["stream"]["content"] == text.rstrip() + assert block["body"]["stream"]["finish"] is False + + +class TestFinalFrameAckTimeoutSemantics: + """Regression: final-frame ack timeout must not raise / trigger fallback. + + See docs/rca-wecom-stream-final-ack-timeout-duplicate.md — when WeCom's + ack returns past the 5s window but the frame *was* delivered, raising + causes the upper layer to fall back to a normal markdown send and the + user sees the same content twice. The fix: treat ack timeout as + success-with-uncertainty and let the caller mark the turn delivered. + """ + + @pytest.mark.asyncio + async def test_final_frame_ack_timeout_returns_success(self): + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._ws = MagicMock(closed=False) + adapter._REPLY_ACK_TIMEOUT = 0.05 # snappy for the test + # _send_json succeeds but no ack ever arrives. + adapter._send_json = AsyncMock() + + response = await adapter._send_reply_queued( + "req-1", + {"msgtype": "stream", "stream": {"id": "stream_x", "content": "final", "finish": True}}, + is_final=True, + ) + + # Aligned-with-official semantics: success-shaped response with the + # ack_pending flag set so callers can log / observe but no exception. + assert response.get("errcode") == 0 + assert response.get("ack_pending") is True + assert "ack_timeout" in response.get("errmsg", "") + + @pytest.mark.asyncio + async def test_final_frame_send_failure_still_raises(self): + """Genuine send failures (network/serialization) must still propagate. + + The ack-timeout relaxation only covers the case where the bytes went + out but the ack didn't return. If ``_send_json`` itself raises, the + upstream caller still needs to see the error. + """ + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._ws = MagicMock(closed=False) + adapter._send_json = AsyncMock(side_effect=RuntimeError("ws closed")) + + with pytest.raises(RuntimeError, match="ws closed"): + await adapter._send_reply_queued( + "req-1", + {"msgtype": "stream", "stream": {"id": "stream_x", "content": "x", "finish": True}}, + is_final=True, + ) diff --git a/tests/gateway/test_wecom_double_send.py b/tests/gateway/test_wecom_double_send.py new file mode 100644 index 0000000000..e85be9ec2f --- /dev/null +++ b/tests/gateway/test_wecom_double_send.py @@ -0,0 +1,534 @@ +"""Repro tests for the WeCom native-streaming "同一回复发两条" duplicate. + +Root cause (see docs/rca-wecom-stream-final-ack-timeout-duplicate.md and +/tmp/claude-wecom-dup-report.md): a **timeout inversion race** between two +independent timers. + + * The consumer's got_done finalize blocks on the finalize frame's ack, + bounded by ``adapter._REPLY_ACK_TIMEOUT``. + * The gateway's ``finally`` block joins the ``stream_task`` with a hardcoded + ``timeout=5.0`` (``gateway/run.py:20749``) and then ``stream_task.cancel()`` + (``run.py:20751``). + +When the ack is slower than the gateway's join window, the gateway cancels the +consumer *before* it finishes finalizing. The finalize frame's bytes, however, +were already written to the wire by an **independent control-worker task** +(``_control_send_worker`` / ``_enqueue_chat_send(is_control=True)``) and WeCom +has rendered them. But because the consumer was cancelled mid-await, the +``self._final_content_delivered = True`` line (``stream_consumer.py:2061`` / +``1016``) never runs. The gateway then reads ``final_content_delivered=False`` +(``run.py:20801-20803``), does NOT suppress the normal final send +(``run.py:20819``), and emits a second, duplicate bubble. + +These tests exercise the **real** ``GatewayStreamConsumer.run()`` lifecycle +against the **real** ``WeComAdapter`` (only the websocket byte-writer and the +ack timing are controlled), so the async interaction between the gateway join / +cancel and the consumer finalize / flag-set actually happens — this is what the +previous PR (#62861) failed to test when it mocked out ``handle_message``. + +Assertions target observable causality: + * whether the finalize frame's bytes reached the wire (WeCom rendered it); + * the value of ``consumer.final_content_delivered`` (the flag the gateway + reads); + * whether the gateway's suppression predicate fires (i.e. whether a second + normal send would go out). + +The suppression predicate below is a faithful mirror of the (non-importable, +nested) logic in ``gateway/run.py:20793-20819`` — it reads the *real* consumer +properties, it does not re-implement the delivery lifecycle. +""" + +from __future__ import annotations + +import asyncio +from unittest.mock import MagicMock + +import pytest + +from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig + + +CHAT_ID = "chat-1" +REQ_ID = "req-1" + + +# --------------------------------------------------------------------------- +# Faithful mirrors of the real gateway control flow (cite source lines). +# --------------------------------------------------------------------------- + + +def _gateway_suppresses_normal_send(consumer, final_text: str) -> bool: + """Mirror of gateway/run.py:20793-20819 suppression decision. + + Reads the REAL consumer properties (``final_response_sent`` / + ``final_content_delivered``) — the same attributes ``run.py`` consults — + and returns True when the gateway would set ``already_sent`` and skip the + normal final send. When it returns False the gateway emits a second bubble. + """ + _final = final_text or "" + _is_empty_sentinel = not _final or _final == "(empty)" + # run.py:20814 _stream_confirmed_final_delivery(...) with previewed=False + # collapses to final_response_sent for a non-previewed response. + _streamed = bool(getattr(consumer, "final_response_sent", False)) + _content_delivered = bool(getattr(consumer, "final_content_delivered", False)) + _transformed = False # no plugin transform in these scenarios + return ( + not _is_empty_sentinel + and not _transformed + and (_streamed or _content_delivered) + ) + + +async def _gateway_join_and_cancel(stream_task: asyncio.Task, join_timeout: float) -> None: + """Faithful copy of gateway/run.py:20748-20755 finally-block join. + + Joins the stream task with a bounded timeout; on timeout it cancels the + task (exactly what the gateway does when the consumer has not finished + finalizing within the join window). + """ + try: + await asyncio.wait_for(stream_task, timeout=join_timeout) + except (asyncio.TimeoutError, asyncio.CancelledError): + stream_task.cancel() + try: + await stream_task + except asyncio.CancelledError: + pass + + +# --------------------------------------------------------------------------- +# Real WeComAdapter with a controllable websocket + ack timing. +# --------------------------------------------------------------------------- + + +def _make_real_wecom_adapter(*, resolve_finalize_ack: bool): + """Build a real ``WeComAdapter`` whose only fakes are the WS byte-writer + and the ack timing. + + ``_send_json`` (the actual byte-writer) records every frame so we can prove + the finalize frame reached the wire. The ack for **non-final** frames (seed + / intermediate) is always resolved immediately so the finalize's + pre-drain (``_send_reply_queued`` is_final branch) doesn't block on the + seed's ack. The finalize frame's own ack is resolved immediately only when + ``resolve_finalize_ack`` is True. + + * ``resolve_finalize_ack=False`` → finalize ack stays pending → the + consumer blocks in its got_done finalize (the exact window where the + gateway's join fires and cancels it). This is the bug scenario. + * ``resolve_finalize_ack=True`` → finalize ack returns at once → the + consumer finishes finalize and sets its flags before any join fires. + """ + from plugins.platforms.wecom.adapter import WeComAdapter + from gateway.config import PlatformConfig + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._ws = MagicMock(closed=False) + adapter._last_chat_req_ids[CHAT_ID] = REQ_ID + + frames: list[dict] = [] + + async def _fake_send_json(payload: dict) -> None: + # This is the real byte-writer boundary: reaching here means the frame + # was put on the wire and WeCom will render it. + frames.append(payload) + stream = payload.get("body", {}).get("stream", {}) + finish = bool(stream.get("finish")) + req = payload.get("headers", {}).get("req_id") + + should_resolve = (not finish) or resolve_finalize_ack + if should_resolve: + queue = adapter._reply_queues.get(req) + if queue and queue.pending_ack and not queue.pending_ack.future.done(): + # Simulate WeCom's ack coming back on the same WS. + queue.pending_ack.future.set_result({"body": {"errcode": 0}}) + + adapter._send_json = _fake_send_json + adapter._recorded_frames = frames + return adapter + + +def _finalize_frames_on_wire(adapter) -> list[dict]: + """Return the finish=true stream frames that reached the byte-writer.""" + out = [] + for payload in adapter._recorded_frames: + stream = payload.get("body", {}).get("stream", {}) + if stream.get("finish") is True: + out.append(payload) + return out + + +async def _cleanup_adapter(adapter) -> None: + """Cancel any lingering control/normal workers so the event loop is clean.""" + for task in list(adapter._control_workers.values()) + list(adapter._chat_workers.values()): + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): + pass + + +# =========================================================================== +# Test group 1: timeout-inversion race (core reproduction) +# =========================================================================== + + +class TestTimeoutInversionDoubleSend: + """The finalize ack being slower than the gateway join window must not + strand ``final_content_delivered=False`` while WeCom already rendered the + finalize frame — that produces a duplicate bubble.""" + + @pytest.mark.asyncio + async def test_slow_ack_beyond_gateway_join_causes_double_send(self): + """ack slower than the gateway join → consumer cancelled mid-finalize. + + Timeline (deterministic, no wall-clock sleeps): + * seed + finalize frame bytes are written to the wire; + * the finalize ack never returns within the join window + (``resolve_finalize_ack=False``, ``_REPLY_ACK_TIMEOUT`` > join); + * the gateway join (0.1s) fires and cancels the consumer; + * the consumer's got_done finalize await is cancelled BEFORE the + ``final_content_delivered = True`` line runs. + + The finalize frame is on the wire (rendered), yet the gateway reads + ``final_content_delivered=False`` and would send a normal duplicate. + + This test asserts the DESIRED post-fix state, so it FAILS today + (exposing the double send) and PASSES once the flag reflects the + rendered finalize frame. + """ + adapter = _make_real_wecom_adapter(resolve_finalize_ack=False) + # Inversion: ack window (0.3s) is LARGER than the gateway join (0.1s), + # so the join fires while the finalize ack is still pending. + adapter._REPLY_ACK_TIMEOUT = 0.3 + join_timeout = 0.1 + + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, CHAT_ID, cfg) + + final_text = "这是模型这一轮生成的最终回答,需要作为流式 finalize 帧发出。" + consumer.on_delta(final_text) + consumer.finish() + + stream_task = asyncio.create_task(consumer.run()) + try: + # Gateway finally-block join (run.py:20748-20755) fires while the + # finalize ack is still pending, cancelling the consumer. + await _gateway_join_and_cancel(stream_task, join_timeout) + + finalize_frames = _finalize_frames_on_wire(adapter) + + # The finalize frame reached the wire → WeCom rendered it. This is + # the first (and should be ONLY) user-visible bubble. + assert len(finalize_frames) >= 1, ( + "finalize frame never reached the wire — repro precondition " + "not met (WeCom must have rendered the streamed final)" + ) + assert final_text in finalize_frames[-1]["body"]["stream"]["content"] + + gateway_would_send_normal = not _gateway_suppresses_normal_send( + consumer, final_text + ) + total_user_visible = len(finalize_frames) + ( + 1 if gateway_would_send_normal else 0 + ) + + # DESIRED post-fix invariants (currently violated == bug reproduced): + assert consumer.final_content_delivered is True, ( + "BUG: consumer was cancelled mid-finalize; the finalize frame " + "was rendered by WeCom but final_content_delivered stayed False, " + "so the gateway will not suppress the normal send" + ) + assert _gateway_suppresses_normal_send(consumer, final_text) is True, ( + "BUG: gateway does not suppress the normal final send → duplicate" + ) + assert total_user_visible == 1, ( + f"BUG: user sees {total_user_visible} bubbles for one reply " + f"(finalize frame rendered + normal send fired)" + ) + finally: + await _cleanup_adapter(adapter) + + @pytest.mark.asyncio + async def test_fast_ack_within_join_window_single_send(self): + """Control: ack returns within the join window → single bubble. + + The finalize ack resolves immediately, so the consumer completes its + got_done finalize, sets ``final_content_delivered=True``, and the run + finishes before the gateway join needs to cancel anything. The gateway + then suppresses the normal send. Passes today and after the fix + (regression guard). + """ + adapter = _make_real_wecom_adapter(resolve_finalize_ack=True) + adapter._REPLY_ACK_TIMEOUT = 5.0 + join_timeout = 0.5 + + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, CHAT_ID, cfg) + + final_text = "这是一个能在 join 窗口内正常拿到 ack 的回答。" + consumer.on_delta(final_text) + consumer.finish() + + stream_task = asyncio.create_task(consumer.run()) + try: + await _gateway_join_and_cancel(stream_task, join_timeout) + + finalize_frames = _finalize_frames_on_wire(adapter) + assert len(finalize_frames) == 1 + + assert consumer.final_content_delivered is True + assert _gateway_suppresses_normal_send(consumer, final_text) is True + + gateway_would_send_normal = not _gateway_suppresses_normal_send( + consumer, final_text + ) + total_user_visible = len(finalize_frames) + ( + 1 if gateway_would_send_normal else 0 + ) + assert total_user_visible == 1 + finally: + await _cleanup_adapter(adapter) + + +# =========================================================================== +# Test group 2: best-effort finalize "DO NOT mark" path +# =========================================================================== + + +class TestBestEffortFinalizeDoubleSend: + """stream_consumer.py:2064-2094: when native streaming fails mid-turn, the + consumer sends a best-effort finalize frame to close the bubble but + deliberately does NOT set ``final_content_delivered`` (the "DO NOT mark" + comment at 2085-2088). If WeCom actually renders that finalize frame, the + subsequent fallback send delivers the same content a second time. + + This uses a controllable ``BasePlatformAdapter`` subclass (the accepted + pattern in tests/gateway/test_wecom_per_turn.py) because the trigger is a + frame-level failure, not an ack timing race — no wall-clock timing needed. + """ + + @pytest.mark.xfail( + reason="KNOWN LOW-FREQUENCY ISSUE (accepted, not yet fixed): the " + "best-effort finalize path (stream_consumer.py:2104-2128) renders the " + "content AND falls through to a fallback send(), delivering the same " + "answer twice. Only triggers when native streaming fails MID-STREAM " + "(846608 expired / 846609 subscription lost / errcode 6000 / network " + "error) — rare. The duplicate is a SAFE-direction failure (user sees " + "one extra bubble, no data loss). A clean fix requires reconciling " + "'do not drop post-finalize increments' with 'do not re-send the " + "first-send path', which the first-send path (L2378) does not consult " + "the delivery flag for; deferred over risking increment loss on a rare " + "path. The high-frequency duplicate (timeout inversion) IS fixed — see " + "TestTimeoutInversionDoubleSend + commit 7ba739818 (B2).", + strict=True, + ) + @pytest.mark.asyncio + async def test_best_effort_finalize_renders_but_flag_unset_double_send(self): + """native content frame fails → best-effort finalize renders + fallback send. + + Ordering matters: the delta is pushed and ``run()`` is allowed to + process a mid-stream **content** frame BEFORE ``finish()`` is queued. + That content frame fails (returns False), which disables native + streaming and triggers the best-effort finalize at + ``stream_consumer.py:2077`` — WeCom renders it — and then the consumer + falls through to a fallback ``send()`` that delivers the same content a + second time (the "DO NOT mark" comment at 2085-2088 is what leaves the + gateway free to re-send). + + The text is > ``_MIN_NEW_VISIBLE_CHARS`` (60) so the mid-stream frame is + actually attempted rather than buffered. + + Asserts the DESIRED single-delivery state, so it FAILS today (content + rendered by the best-effort finalize AND re-sent by the fallback) and + PASSES once the double delivery is closed. + """ + from gateway.platforms.base import BasePlatformAdapter, SendResult + + class FlakyNativeAdapter(BasePlatformAdapter): + MAX_MESSAGE_LENGTH = 4096 + SUPPORTS_MESSAGE_EDITING = False + SUPPORTS_NATIVE_STREAMING = True + + def __init__(self): + self._typing_paused = set() + self._fatal_error_message = None + # Records of everything that reached the user. + self.rendered_stream_frames: list[dict] = [] # incl. finalize + self.fallback_sends: list[str] = [] + self._content_frames_seen = 0 + + def supports_native_streaming(self, chat_type=None, metadata=None): + return True + + async def send_stream_frame( + self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs + ): + # Seed frame (empty, non-finalize) opens the bubble. + if text == "" and not finalize: + self.rendered_stream_frames.append({"text": text, "finalize": False}) + return True + if finalize: + # Best-effort finalize frame — WeCom DOES render it. + self.rendered_stream_frames.append({"text": text, "finalize": True}) + return True + # First real content frame fails → disables native streaming + # and triggers the best-effort finalize path. + self._content_frames_seen += 1 + return False + + async def send(self, chat_id, content, reply_to=None, metadata=None): + # The gateway/consumer fallback send — a SECOND delivery of the + # same content. + self.fallback_sends.append(content) + return SendResult(success=True, message_id="fallback-msg") + + FlakyNativeAdapter.__abstractmethods__ = frozenset() + adapter = FlakyNativeAdapter() + + cfg = StreamConsumerConfig( + chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5, + ) + consumer = GatewayStreamConsumer(adapter, CHAT_ID, cfg) + + # > 60 visible chars so the mid-stream content frame is actually sent. + final_text = ( + "需要交付给用户的最终答案。这一段内容刻意写得足够长," + "以便越过 stream_consumer 的 _MIN_NEW_VISIBLE_CHARS=60 阈值," + "从而在 got_done 之前触发一次真正的流式内容帧发送。" + ) + assert len(final_text) >= 60 + + consumer.on_delta(final_text) + task = asyncio.create_task(consumer.run()) + # Let run() process the mid-stream content frame (which fails) BEFORE + # the turn-final _DONE arrives. + await asyncio.sleep(0.1) + consumer.finish() + await task + + best_effort_finalize = [ + f for f in adapter.rendered_stream_frames if f["finalize"] and f["text"] + ] + + # Count content-bearing user-visible deliveries: + # * best-effort finalize frame(s) that WeCom rendered, plus + # * fallback send(s). + content_deliveries = len(best_effort_finalize) + len(adapter.fallback_sends) + + # DESIRED post-fix invariant (currently violated == bug reproduced): + # the same answer must reach the user exactly once. Today the + # best-effort finalize renders the content AND the fallback send + # delivers it again. + assert content_deliveries == 1, ( + f"BUG: content delivered {content_deliveries} times — best-effort " + f"finalize rendered {len(best_effort_finalize)} frame(s) and " + f"fallback send fired {len(adapter.fallback_sends)} time(s) " + f"(stream_consumer.py:2085-2088 'DO NOT mark' path)" + ) + + +# =========================================================================== +# Test group 3: orphan-queue ack routing race (adapter-level) +# =========================================================================== + + +def _make_manual_ack_adapter(): + """Real WeComAdapter whose ``_send_json`` records frames but NEVER + auto-resolves any ack, so the test can orchestrate the exact interleaving + of intermediate-ack arrival vs. finalize registration by hand. + """ + from plugins.platforms.wecom.adapter import WeComAdapter + from gateway.config import PlatformConfig + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + adapter._ws = MagicMock(closed=False) + adapter._last_chat_req_ids[CHAT_ID] = REQ_ID + + frames: list[dict] = [] + + async def _fake_send_json(payload: dict) -> None: + frames.append(payload) + + adapter._send_json = _fake_send_json + adapter._recorded_frames = frames + return adapter + + +class TestOrphanQueueAckRouting: + """The finalize frame shares the inbound req_id with the intermediate + frames. While the finalize awaits the pending intermediate ack to drain, + that intermediate ack can arrive and ``_resolve_reply_ack`` pops the WHOLE + queue out of ``_reply_queues``. Before the fix, the finalize then + registered its ``pending_ack`` on the now-orphaned queue object (detached + from the dict), so its own ack landed in Unrouted and the finalize hit a + 15s timeout. The fix re-attaches the queue to the dict before registering. + """ + + @pytest.mark.asyncio + async def test_intermediate_ack_mid_drain_does_not_orphan_finalize_queue(self): + adapter = _make_manual_ack_adapter() + adapter._REPLY_ACK_TIMEOUT = 0.3 # snappy: bounds the failure mode + + # 1) Fire an intermediate frame (skip_if_pending) — occupies pending_ack. + await adapter._send_reply_queued( + REQ_ID, + {"msgtype": "stream", "stream": {"id": "s1", "finish": False, "content": "hi"}}, + is_final=False, skip_if_pending=True, + ) + inter_queue = adapter._reply_queues[REQ_ID] + assert inter_queue.pending_ack is not None + + # 2) Start the finalize: it enters the is_final drain branch and yields + # at `await shield(pending_frame.future)`. + fin_task = asyncio.create_task(adapter._send_reply_queued( + REQ_ID, + {"msgtype": "stream", "stream": {"id": "s1", "finish": True, "content": "done"}}, + is_final=True, skip_if_pending=False, + )) + await asyncio.sleep(0) # let finalize reach the drain await + + # 3) Deliver the intermediate ack MID-DRAIN. _resolve_reply_ack resolves + # the drain future AND pops the whole queue out of _reply_queues. + await adapter._dispatch_payload( + {"headers": {"req_id": REQ_ID}, "body": {"errcode": 0}} + ) + assert REQ_ID not in adapter._reply_queues, ( + "precondition: intermediate ack should have popped the queue" + ) + + # 4) Let finalize resume: it must clear the drained pending_ack, create + # its own frame, re-attach the queue, register pending_ack, and write + # bytes. Resuming through the shield/wait_for wrapper takes several + # event-loop iterations, so yield until the finalize has re-registered + # (bounded, to avoid a hang if the fix regresses). + for _ in range(20): + await asyncio.sleep(0) + q = adapter._reply_queues.get(REQ_ID) + if q is not None and q.pending_ack is not None: + break + + # FIX ASSERTION: the finalize must have re-attached its queue to the + # dict, so its ack is routable. Without the fix the queue would still + # be absent here (orphaned) and the finalize ack would go Unrouted. + assert REQ_ID in adapter._reply_queues, ( + "orphan-queue bug: finalize registered pending_ack on a queue " + "detached from _reply_queues — its ack would be Unrouted" + ) + assert adapter._reply_queues[REQ_ID].pending_ack is not None + + # 5) Deliver the finalize's own ack — it must route and resolve. + await adapter._dispatch_payload( + {"headers": {"req_id": REQ_ID}, "body": {"errcode": 0}} + ) + resp = await asyncio.wait_for(fin_task, timeout=1.0) + + # Routed successfully → NOT the synthetic timeout-assumed response. + assert resp.get("errmsg") != "ack_timeout_assumed_delivered", ( + "finalize ack should have been routed, not timed out" + ) + + await _cleanup_adapter(adapter) diff --git a/tests/gateway/test_wecom_per_turn.py b/tests/gateway/test_wecom_per_turn.py new file mode 100644 index 0000000000..daae1adf2e --- /dev/null +++ b/tests/gateway/test_wecom_per_turn.py @@ -0,0 +1,403 @@ +"""Tests for per-turn stream isolation and concurrent consumer scenarios.""" + +import pytest +from unittest.mock import AsyncMock, MagicMock +from gateway.config import PlatformConfig + + +class TestPerTurnStreamIsolation: + """Verify that concurrent consumers with different turn_ids don't interfere.""" + + @pytest.mark.asyncio + async def test_multiple_users_concurrent_streaming(self): + """Multiple users (different chats) streaming concurrently don't interfere.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + try: + # Setup 3 different users/chats + adapter._last_chat_req_ids["user-1"] = "req-1" + adapter._last_chat_req_ids["user-2"] = "req-2" + adapter._last_chat_req_ids["user-3"] = "req-3" + adapter._send_json = AsyncMock() + adapter._ws = AsyncMock(closed=False) + adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0}) + + # User 1, 2, 3 all start streaming simultaneously + await adapter.send_stream_frame("user1 content", chat_id="user-1", turn_id="turn-1") + await adapter.send_stream_frame("user2 content", chat_id="user-2", turn_id="turn-2") + await adapter.send_stream_frame("user3 content", chat_id="user-3", turn_id="turn-3") + + # All 3 turns active + assert "user-1:turn-1" in adapter._stream_turns + assert "user-2:turn-2" in adapter._stream_turns + assert "user-3:turn-3" in adapter._stream_turns + + # User 2 finishes first + ok2 = await adapter.send_stream_frame( + "user2 final", chat_id="user-2", finalize=True, turn_id="turn-2" + ) + assert ok2 is True + assert "user-2:turn-2" not in adapter._stream_turns + # User 1 and 3 still active + assert "user-1:turn-1" in adapter._stream_turns + assert "user-3:turn-3" in adapter._stream_turns + + # User 1 finishes + ok1 = await adapter.send_stream_frame( + "user1 final", chat_id="user-1", finalize=True, turn_id="turn-1" + ) + assert ok1 is True + assert "user-1:turn-1" not in adapter._stream_turns + # User 3 still active + assert "user-3:turn-3" in adapter._stream_turns + + # User 3 finishes + ok3 = await adapter.send_stream_frame( + "user3 final", chat_id="user-3", finalize=True, turn_id="turn-3" + ) + assert ok3 is True + assert "user-3:turn-3" not in adapter._stream_turns + finally: + await adapter.disconnect() + + @pytest.mark.asyncio + async def test_concurrent_turns_same_chat_isolated(self): + """Two concurrent consumers in same chat maintain independent streams.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + try: + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._send_json = AsyncMock() + # Mock _ws with closed=False and async close() + adapter._ws = AsyncMock(closed=False) + adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0}) + + # Consumer 1 starts streaming + await adapter.send_stream_frame("consumer1 frame1", chat_id="chat-1", turn_id="turn-1") + assert "chat-1:turn-1" in adapter._stream_turns + + # Consumer 2 starts streaming (concurrent) + await adapter.send_stream_frame("consumer2 frame1", chat_id="chat-1", turn_id="turn-2") + assert "chat-1:turn-2" in adapter._stream_turns + + # Both turns coexist + assert len([k for k in adapter._stream_turns if k.startswith("chat-1:")]) == 2 + + # Consumer 1 finalizes + ok1 = await adapter.send_stream_frame( + "consumer1 final", chat_id="chat-1", finalize=True, turn_id="turn-1" + ) + assert ok1 is True + assert "chat-1:turn-1" not in adapter._stream_turns + # Consumer 2's turn still exists + assert "chat-1:turn-2" in adapter._stream_turns + + # Consumer 2 finalizes + ok2 = await adapter.send_stream_frame( + "consumer2 final", chat_id="chat-1", finalize=True, turn_id="turn-2" + ) + assert ok2 is True + assert "chat-1:turn-2" not in adapter._stream_turns + finally: + await adapter.disconnect() + + @pytest.mark.asyncio + async def test_one_user_expired_others_unaffected(self): + """User A hits stream expired; Users B and C continue normally.""" + from plugins.platforms.wecom.adapter import STREAM_EXPIRED_ERRCODE, WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + try: + adapter._last_chat_req_ids["user-A"] = "req-A" + adapter._last_chat_req_ids["user-B"] = "req-B" + adapter._last_chat_req_ids["user-C"] = "req-C" + adapter._send_json = AsyncMock() + adapter._ws = AsyncMock(closed=False) + + # All 3 users start streaming + await adapter.send_stream_frame("A content", chat_id="user-A", turn_id="turn-A") + await adapter.send_stream_frame("B content", chat_id="user-B", turn_id="turn-B") + await adapter.send_stream_frame("C content", chat_id="user-C", turn_id="turn-C") + + # User A hits stream expired + adapter._send_reply_queued = AsyncMock( + return_value={"errcode": STREAM_EXPIRED_ERRCODE, "errmsg": "expired"} + ) + okA = await adapter.send_stream_frame( + "A final", chat_id="user-A", finalize=True, turn_id="turn-A" + ) + assert okA is False + assert "user-A" in adapter._stream_expired_chats + assert "user-A:turn-A" not in adapter._stream_turns + + # Users B and C should NOT be affected (different chats) + adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0}) + okB = await adapter.send_stream_frame( + "B final", chat_id="user-B", finalize=True, turn_id="turn-B" + ) + okC = await adapter.send_stream_frame( + "C final", chat_id="user-C", finalize=True, turn_id="turn-C" + ) + assert okB is True # ✅ User B unaffected + assert okC is True # ✅ User C unaffected + assert "user-B:turn-B" not in adapter._stream_turns + assert "user-C:turn-C" not in adapter._stream_turns + + # Only user-A is in expired list + assert "user-A" in adapter._stream_expired_chats + assert "user-B" not in adapter._stream_expired_chats + assert "user-C" not in adapter._stream_expired_chats + finally: + await adapter.disconnect() + + @pytest.mark.asyncio + async def test_one_turn_expired_other_continues(self): + """When one turn hits stream expired, other concurrent turns can continue.""" + from plugins.platforms.wecom.adapter import STREAM_EXPIRED_ERRCODE, WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + try: + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._send_json = AsyncMock() + adapter._ws = AsyncMock(closed=False) + + # Consumer 1 and 2 both start + await adapter.send_stream_frame("c1 frame", chat_id="chat-1", turn_id="turn-1") + await adapter.send_stream_frame("c2 frame", chat_id="chat-1", turn_id="turn-2") + assert "chat-1:turn-1" in adapter._stream_turns + assert "chat-1:turn-2" in adapter._stream_turns + + # Consumer 1 hits expired error on finalize + adapter._send_reply_queued = AsyncMock( + return_value={"errcode": STREAM_EXPIRED_ERRCODE, "errmsg": "stream expired"} + ) + ok1 = await adapter.send_stream_frame( + "c1 final", chat_id="chat-1", finalize=True, turn_id="turn-1" + ) + assert ok1 is False + assert "chat-1" in adapter._stream_expired_chats + assert "chat-1:turn-1" not in adapter._stream_turns # turn-1 cleaned up + + # Consumer 2's existing turn can still finalize + adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0}) + ok2 = await adapter.send_stream_frame( + "c2 final", chat_id="chat-1", finalize=True, turn_id="turn-2" + ) + assert ok2 is True # ✅ turn-2 not blocked by chat-level expired + assert "chat-1:turn-2" not in adapter._stream_turns + finally: + await adapter.disconnect() + + @pytest.mark.asyncio + async def test_expired_chat_blocks_new_turn_creation(self): + """After one turn expired, new turn creation is blocked.""" + from plugins.platforms.wecom.adapter import WeComAdapter + + adapter = WeComAdapter(PlatformConfig(enabled=True)) + try: + adapter._last_chat_req_ids["chat-1"] = "req-1" + adapter._stream_expired_chats.add("chat-1") + adapter._send_reply_request = AsyncMock(return_value={"errcode": 0}) + + # Try to create a new turn after chat is expired + ok = await adapter.send_stream_frame("new frame", chat_id="chat-1", turn_id="new-turn") + assert ok is False + assert "chat-1:new-turn" not in adapter._stream_turns + finally: + await adapter.disconnect() + + +class TestNativeFallbackStreamClose: + """Verify that native streaming fallback closes open streams.""" + + @pytest.mark.asyncio + async def test_seed_success_first_frame_fails_still_finalizes(self): + """Seed frame opens stream bubble, first content frame fails → finalize called. + + This is the critical edge case: seed frame has length 0 but opens the + WeCom typing bubble. If the first content frame fails, we must still + finalize based on _native_stream_opened, not _native_last_pushed_len. + """ + from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig + from gateway.platforms.base import BasePlatformAdapter + + class MockAdapter(BasePlatformAdapter): + MAX_MESSAGE_LENGTH = 4096 + SUPPORTS_MESSAGE_EDITING = False + SUPPORTS_NATIVE_STREAMING = True + + def __init__(self): + self._typing_paused = set() + self.send_stream_frame_calls = [] + self.send_calls = [] + self.should_fail_first_content = True + + def supports_native_streaming(self, chat_type=None, metadata=None): + return True + + async def send_stream_frame( + self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs + ): + call_info = {"text_len": len(text), "finalize": finalize, "text_preview": text[:20]} + self.send_stream_frame_calls.append(call_info) + + # Seed frame (empty) always succeeds + if len(text) == 0 and not finalize: + return True + + # First non-seed, non-finalize frame fails + if self.should_fail_first_content and len(text) > 0 and not finalize: + self.should_fail_first_content = False + raise RuntimeError("first content frame failed") + + # Finalize frames and subsequent content frames succeed + return True + + async def send(self, chat_id, content, reply_to=None, metadata=None): + self.send_calls.append({"content_preview": content[:20]}) + return type("SendResult", (), {"success": True, "message_id": "msg-1"})() + + MockAdapter.__abstractmethods__ = frozenset() + adapter = MockAdapter() + cfg = StreamConsumerConfig(chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + # Send short content to minimize frame count + consumer.on_delta("X") + + import asyncio + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.05) + consumer.finish() + await task + + # Verify: seed succeeded, then finalize was attempted (not skipped) + assert len(adapter.send_stream_frame_calls) >= 2 + # First call: seed (length 0) + assert adapter.send_stream_frame_calls[0]["text_len"] == 0 + assert not adapter.send_stream_frame_calls[0]["finalize"] + + # At least one finalize call should have been made. This is the core + # invariant: finalize is driven by _native_stream_opened (the seed + # opened the bubble), NOT by _native_last_pushed_len — so even though + # the seed had length 0 and the first content frame failed, finalize + # must still be attempted to close the typing bubble. + finalize_calls = [c for c in adapter.send_stream_frame_calls if c["finalize"]] + assert len(finalize_calls) >= 1, "Finalize should be called even though seed had length 0" + + # Fire-and-forget (14c49c781a): with the throttle removed, the 1-char + # content "X" is pushed immediately as an intermediate frame instead of + # being buffered. That frame fails per the mock, so a proactive send() + # fallback IS expected to deliver the content reliably. (Under the old + # _MIN_NEW_VISIBLE_CHARS=60 gate this tiny frame was never sent, so the + # previous assertion of zero send() fallbacks no longer holds.) + assert len(adapter.send_calls) == 1 + assert adapter.send_calls[0]["content_preview"] == "X" + + @pytest.mark.asyncio + async def test_native_fallback_closes_stream_on_success(self): + """When native fails mid-stream, best-effort finalize succeeds.""" + from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig + from gateway.platforms.base import BasePlatformAdapter + + class MockAdapter(BasePlatformAdapter): + MAX_MESSAGE_LENGTH = 4096 + SUPPORTS_MESSAGE_EDITING = False + SUPPORTS_NATIVE_STREAMING = True + + def __init__(self): + self._typing_paused = set() + self.frames = [] + self.frame_count = 0 + + def supports_native_streaming(self, chat_type=None, metadata=None): + return True + + async def send_stream_frame( + self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs + ): + self.frame_count += 1 + self.frames.append({"text": text, "finalize": finalize}) + # First 2 frames succeed, 3rd fails (non-expired error) + if self.frame_count == 3: + raise RuntimeError("network error") + # 4th frame (finalize in fallback) succeeds + return True + + async def send(self, chat_id, content, reply_to=None, metadata=None): + self.frames.append({"send": content}) + return type("SendResult", (), {"success": True, "message_id": "msg-1"})() + + MockAdapter.__abstractmethods__ = frozenset() + adapter = MockAdapter() + cfg = StreamConsumerConfig(chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + # Send enough to trigger frames + consumer.on_delta("First frame content that exceeds the threshold.") + consumer.on_delta(" Second frame content also exceeds threshold.") + consumer.on_delta(" Third will fail.") + + import asyncio + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.1) + consumer.finish() + await task + + # Should have: seed, frame1, frame2, (frame3 fails), finalize in fallback + # After fix #3: best-effort finalize closes the typing bubble but does NOT + # mark content_delivered. The fallback send() will deliver content reliably. + assert len([f for f in adapter.frames if f.get("finalize")]) >= 1 + # Fallback send() IS expected to fire (content delivery via proactive send) + assert len([f for f in adapter.frames if "send" in f]) == 1 + + @pytest.mark.asyncio + async def test_native_fallback_falls_to_send_on_finalize_fail(self): + """When native fails and finalize also fails, falls through to send().""" + from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig + from gateway.platforms.base import BasePlatformAdapter + + class MockAdapter(BasePlatformAdapter): + MAX_MESSAGE_LENGTH = 4096 + SUPPORTS_MESSAGE_EDITING = False + SUPPORTS_NATIVE_STREAMING = True + + def __init__(self): + self._typing_paused = set() + self.frames = [] + self.frame_count = 0 + + def supports_native_streaming(self, chat_type=None, metadata=None): + return True + + async def send_stream_frame( + self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs + ): + self.frame_count += 1 + self.frames.append({"text": text, "finalize": finalize}) + # All frames fail (simulating complete stream failure) + if self.frame_count >= 2: + raise RuntimeError("stream dead") + return True + + async def send(self, chat_id, content, reply_to=None, metadata=None): + self.frames.append({"send": content}) + return type("SendResult", (), {"success": True, "message_id": "msg-1"})() + + MockAdapter.__abstractmethods__ = frozenset() + adapter = MockAdapter() + cfg = StreamConsumerConfig(chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5) + consumer = GatewayStreamConsumer(adapter, "chat-1", cfg) + + consumer.on_delta("Content that will cause stream to fail.") + + import asyncio + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.1) + consumer.finish() + await task + + # Finalize failed → should fall through to send() + assert len([f for f in adapter.frames if "send" in f]) == 1