feat(wecom): native reply streaming (per-turn isolation, dedup-safe delivery, interaction boundaries)
Implement native reply streaming for the WeCom (企业微信) adapter over the long-connection "msgtype: stream" transport, so a reply renders as a single live-updating typing bubble instead of one final block. Aligns with the official wecom-openclaw-plugin streaming behavior. Includes the machinery intrinsic to native streaming on WeCom: - Transport: seed frame (<think></think>) opens the typing bubble, intermediate frames update it, a finalize frame closes it; native-streaming adapters are let past the edit-only gate. Fire-and-forget intermediate frames (WeCom long-connection mode has no documented edit-rate limit); an adapter-level frame cap is retained. (Early builds gated frames behind a char throttle; removed in favor of fire-and-forget + identity dedup.) - Per-turn isolation: each turn owns a unique turn_id; concurrent messages are isolated via (chat_id, turn_id)-keyed state. Dual-lane priority queue (control vs normal) plus a per-chat token bucket to stay under WeCom's rate limit (errcode 846607). - Dedup-safe delivery + ack-race handling: deliver-once contract (a frame is delivered the moment it is emitted; failures logged, not re-sent; delivery marked once per turn), per-req_id reply queue with ack tracking, and the timeout-inversion / orphan-queue race fixes. Robust fallback on 846608 / 846609 / errcode 6000 / passive-reply timeout via proactive send. - Interaction boundaries: finalize + reset before approval/clarify prompts so the prompt is the last thing on screen and never traps a lingering bubble; eager re-seed after a clarify answer so the typing bubble reappears instantly. - Stream-level keepalive: optional periodic finish=false frame + finalize-time stream-age guard to refresh WeCom's ~6-minute reply-stream window on long turns (mitigates 846604 / 846608). Off by default; tunable via config.yaml. - Tool-progress folded into the same native-stream bubble instead of separate messages; image+text double-callback merged into one turn. Tests cover the streaming lifecycle, per-turn isolation, duplicate-send / ack timing, approval + clarify boundaries, eager re-seed, and tool-progress.
This commit is contained in:
+154
-4
@@ -4568,6 +4568,13 @@ class TurnRunner:
|
||||
# code (same boilerplate imports → identical previews).
|
||||
if msg == ctx.last_progress_msg[0]:
|
||||
ctx.repeat_count[0] += 1
|
||||
# Native-stream-progress routing: dedup updates the last line
|
||||
# in the overlay rather than sending a queue signal.
|
||||
_sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None
|
||||
if _sc is not None and getattr(_sc, "accepts_tool_progress", False):
|
||||
# Replace the last progress line with the dedup version
|
||||
_sc.on_tool_progress(f"{msg} (×{ctx.repeat_count[0] + 1})")
|
||||
return
|
||||
# Update the last line in progress_lines with a counter
|
||||
# via a special "dedup" queue message.
|
||||
ctx.progress_queue.put(("__dedup__", msg, ctx.repeat_count[0]))
|
||||
@@ -4575,6 +4582,14 @@ class TurnRunner:
|
||||
ctx.last_progress_msg[0] = msg
|
||||
ctx.repeat_count[0] = 0
|
||||
|
||||
# Native-stream-progress routing: if the stream consumer is active
|
||||
# and using native streaming, inject progress directly into the
|
||||
# stream bubble instead of the separate progress queue.
|
||||
_sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None
|
||||
if _sc is not None and getattr(_sc, "accepts_tool_progress", False):
|
||||
_sc.on_tool_progress(msg)
|
||||
return
|
||||
|
||||
ctx.progress_queue.put(msg)
|
||||
|
||||
async def _send_native_task_card_progress(self, adapter) -> None:
|
||||
@@ -5947,6 +5962,51 @@ class TurnRunner:
|
||||
_mem_notif = "on" if _mem_notif else "off"
|
||||
agent.memory_notifications = str(_mem_notif).lower() if _mem_notif else "on"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Shared native-stream boundary close. For platforms with native
|
||||
# streaming (e.g. WeCom msgtype:"stream"), an interaction that
|
||||
# interrupts the stream — a dangerous-command approval prompt OR a
|
||||
# clarify decision prompt — must finalize the current stream and
|
||||
# disable native streaming first. Otherwise the agent's
|
||||
# post-interaction output keeps flowing into send_stream_frame and
|
||||
# updates the *old* bubble that preceded the prompt, instead of
|
||||
# starting a fresh bubble below it (the "气泡割裂" symptom). After
|
||||
# the boundary, post-prompt output goes through the reliable send()
|
||||
# path as a new message. Runs on the agent thread; the consumer
|
||||
# processes the boundary serially via its queue.
|
||||
def _close_native_stream_boundary(
|
||||
_reason: str, _placeholder: str | None = None, _reopen: bool = False,
|
||||
) -> bool:
|
||||
_sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None
|
||||
if not (_sc and getattr(_sc, "_use_native_streaming", False)):
|
||||
return True
|
||||
_cancelled_flag = None
|
||||
try:
|
||||
_boundary_result = _sc.close_for_approval_prompt(
|
||||
_placeholder, reason=_reason, reopen=_reopen,
|
||||
)
|
||||
# Returns (future, cancelled_flag) or just a future.
|
||||
if isinstance(_boundary_result, tuple):
|
||||
_boundary_future, _cancelled_flag = _boundary_result
|
||||
else:
|
||||
_boundary_future = _boundary_result
|
||||
if hasattr(_boundary_future, "result"):
|
||||
_ok = _boundary_future.result(timeout=10)
|
||||
if not _ok:
|
||||
logger.warning(
|
||||
"%s boundary failed to close stream properly — "
|
||||
"prompt may still appear in typing bubble", _reason,
|
||||
)
|
||||
return bool(_ok)
|
||||
return True
|
||||
except (TimeoutError, Exception) as _boundary_err:
|
||||
if _cancelled_flag is not None:
|
||||
_cancelled_flag["cancelled"] = True
|
||||
logger.warning(
|
||||
"%s boundary timed out or failed: %s", _reason, _boundary_err,
|
||||
)
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Clarify callback: present a clarify prompt and block on a response.
|
||||
#
|
||||
@@ -5974,6 +6034,21 @@ class TurnRunner:
|
||||
multi_select=bool(multi_select),
|
||||
)
|
||||
|
||||
# For WeCom native streaming: finalize the current stream before
|
||||
# showing the clarify prompt so the post-answer output opens a
|
||||
# fresh bubble below the question instead of updating the bubble
|
||||
# that preceded it — the "气泡割裂" symptom. Unlike the approval
|
||||
# path, clarify passes reopen=True: native streaming stays
|
||||
# enabled so the post-answer continuation re-opens a fresh
|
||||
# native stream (typing bubble) rather than degrading to a
|
||||
# one-shot send(). Clarify waits are short, so stream staleness
|
||||
# is a low risk; if the re-seed fails the consumer degrades to
|
||||
# send() automatically. The placeholder is only used in the
|
||||
# narrow case where a frame was pushed but no text accumulated.
|
||||
_close_native_stream_boundary(
|
||||
"Clarify", "💬 等待你的选择...", _reopen=True,
|
||||
)
|
||||
|
||||
# Pause typing — like approval, we don't want a "thinking..."
|
||||
# status to obscure the prompt or block the user from typing
|
||||
# an "Other" response on platforms that disable input while
|
||||
@@ -6019,12 +6094,45 @@ class TurnRunner:
|
||||
# AMBIGUOUS — the card may have posted with a late ack. Only a
|
||||
# definitive failure tears down the registration; ambiguous
|
||||
# falls through to the bounded wait so a late reply resolves.
|
||||
return _clarify_send_then_wait(
|
||||
_clarify_response = _clarify_send_then_wait(
|
||||
fut,
|
||||
clarify_id=clarify_id,
|
||||
session_key=ctx.session_key or "",
|
||||
clarify_mod=_clarify_mod,
|
||||
)
|
||||
# Only re-arm typing when the user actually answered — the
|
||||
# undeliverable sentinel and the timeout/cancellation strings
|
||||
# start with '[' and must pass through untouched.
|
||||
if not (
|
||||
isinstance(_clarify_response, str)
|
||||
and _clarify_response.startswith("[")
|
||||
):
|
||||
# User answered. Reopen the typing indicator IMMEDIATELY —
|
||||
# don't wait for the LLM's first post-answer token. On native
|
||||
# streaming (WeCom) the typing bubble is driven by the stream
|
||||
# seed frame, and the reopen path otherwise re-seeds lazily on
|
||||
# the first delta (measured ~48s of dead air). request_reopen_seed
|
||||
# is a no-op unless we're in the reopen-pending native state, so
|
||||
# it's safe to call unconditionally here. resume_typing_for_chat
|
||||
# covers non-native platforms (their pause was set before the
|
||||
# prompt) — the WeCom send_typing is a no-op, harmless here.
|
||||
_sc_reopen = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None
|
||||
if _sc_reopen is not None:
|
||||
try:
|
||||
_sc_reopen.request_reopen_seed()
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"request_reopen_seed after clarify answer failed",
|
||||
exc_info=True,
|
||||
)
|
||||
try:
|
||||
ctx._status_adapter.resume_typing_for_chat(ctx._status_chat_id)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"resume_typing_for_chat after clarify answer failed",
|
||||
exc_info=True,
|
||||
)
|
||||
return _clarify_response
|
||||
|
||||
agent.clarify_callback = _clarify_callback_sync
|
||||
|
||||
@@ -6127,6 +6235,12 @@ class TurnRunner:
|
||||
# Typing resumes in _handle_approve_command/_handle_deny_command.
|
||||
ctx._status_adapter.pause_typing_for_chat(ctx._status_chat_id)
|
||||
|
||||
# For WeCom native streaming: signal the stream consumer to close
|
||||
# the current stream before showing the approval prompt.
|
||||
# This goes through the consumer's queue for serial processing,
|
||||
# avoiding race conditions with pending deltas.
|
||||
_close_native_stream_boundary("Approval")
|
||||
|
||||
cmd = approval_data.get("command", "")
|
||||
desc = approval_data.get("description", "dangerous command")
|
||||
|
||||
@@ -6200,11 +6314,15 @@ class TurnRunner:
|
||||
smart_denied=approval_data.get("smart_denied", False),
|
||||
)
|
||||
try:
|
||||
# Mark as approval prompt so WeCom routes through control lane
|
||||
_approval_metadata = dict(ctx._status_thread_metadata or {})
|
||||
_approval_metadata["is_approval_prompt"] = True
|
||||
|
||||
_approval_send_fut = safe_schedule_threadsafe(
|
||||
ctx._status_adapter.send(
|
||||
ctx._status_chat_id,
|
||||
msg,
|
||||
metadata=_interim_metadata(ctx._status_thread_metadata),
|
||||
metadata=_interim_metadata(_approval_metadata),
|
||||
),
|
||||
ctx._loop_for_step,
|
||||
logger=logger,
|
||||
@@ -22768,6 +22886,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
asked to deliver (#20834). Only ``MEDIA:`` directives — the explicit
|
||||
attachment contract — trigger post-stream uploads.
|
||||
"""
|
||||
logger.info("[DEBUG] _deliver_media_from_response called: response=%r chat=%s", response[:100], event.source.chat_id)
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote as _quote
|
||||
|
||||
@@ -24342,7 +24461,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
_APPROVAL_TIMEOUT_SECONDS = 300 # 5 minutes
|
||||
|
||||
|
||||
|
||||
# Built-in messaging platforms where the ``/update`` command is allowed.
|
||||
# ACP, API server, and webhooks are programmatic interfaces that should
|
||||
# not trigger system updates. Plugin-migrated platforms (discord,
|
||||
@@ -28131,7 +28249,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
# (The proxy path instead opts into a cursorless fallback
|
||||
# via on_missing_cursor="fallback".)
|
||||
_adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True)
|
||||
if not _adapter_supports_edit and on_missing_cursor == "raise":
|
||||
# Adapters that can't edit messages but provide a native-streaming
|
||||
# transport (e.g. WeCom's msgtype: "stream" via send_stream_frame)
|
||||
# get past the gate — the consumer's native branch delivers the full
|
||||
# turn through that transport.
|
||||
_adapter_supports_native_stream = bool(getattr(
|
||||
adapter, "SUPPORTS_NATIVE_STREAMING", False,
|
||||
))
|
||||
if (
|
||||
not _adapter_supports_edit
|
||||
and not _adapter_supports_native_stream
|
||||
and on_missing_cursor == "raise"
|
||||
):
|
||||
raise RuntimeError("skip streaming for non-editable platform")
|
||||
_effective_cursor = scfg.cursor if _adapter_supports_edit else ""
|
||||
# Some Matrix clients render the streaming cursor
|
||||
@@ -30405,6 +30534,27 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
"Failed to edit streamed message for session %s: %s",
|
||||
session_key or "?", _edit_err,
|
||||
)
|
||||
elif _sc is not None and not _is_empty_sentinel:
|
||||
# DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed for this
|
||||
# turn but suppression did NOT fire, so the gateway's normal
|
||||
# final-send is about to run. On WeCom this is the exact window
|
||||
# that produced "回复了两条" — a final-frame ack still in flight
|
||||
# (final_content_delivered not yet set) while this send races
|
||||
# ahead. Log the decision inputs so a recurrence can be pinned to
|
||||
# "signal never set" vs "ack-pending race".
|
||||
# See docs/rca-wecom-stream-final-ack-timeout-duplicate.md.
|
||||
logger.warning(
|
||||
"Normal final-send NOT suppressed despite active stream "
|
||||
"consumer for session %s: streamed=%s previewed=%s "
|
||||
"content_delivered=%s transformed=%s final_len=%d — "
|
||||
"possible duplicate send (see wecom ack-timeout RCA).",
|
||||
session_key or "?",
|
||||
_streamed,
|
||||
_previewed,
|
||||
_content_delivered,
|
||||
_transformed,
|
||||
len(_final),
|
||||
)
|
||||
|
||||
# Schedule deletion of tracked temporary progress bubbles after the
|
||||
# final response lands. Failed runs skip this so bubbles remain as
|
||||
|
||||
@@ -5885,7 +5885,27 @@ class GatewaySlashCommandsMixin:
|
||||
|
||||
logger.info("User approved %d dangerous command(s) via /approve (%s)", count, choice)
|
||||
plural = "plural" if count > 1 else "singular"
|
||||
return t(f"gateway.approve.{choice}_{plural}", count=count)
|
||||
confirmation_text = t(f"gateway.approve.{choice}_{plural}", count=count)
|
||||
if _adapter:
|
||||
try:
|
||||
await _adapter.send(
|
||||
source.chat_id,
|
||||
confirmation_text,
|
||||
reply_to=event.message_id,
|
||||
metadata={
|
||||
"is_approval_prompt": True,
|
||||
"force_proactive_send": True,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to send /approve confirmation to %s: %s",
|
||||
source.chat_id,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
async def _handle_deny_command(self, event: MessageEvent) -> str:
|
||||
"""Handle /deny command — reject pending dangerous command(s).
|
||||
@@ -5943,11 +5963,34 @@ class GatewaySlashCommandsMixin:
|
||||
)
|
||||
if reason:
|
||||
if count > 1:
|
||||
return t("gateway.deny.denied_reason_plural", count=count, reason=reason)
|
||||
return t("gateway.deny.denied_reason_singular", reason=reason)
|
||||
if count > 1:
|
||||
return t("gateway.deny.denied_plural", count=count)
|
||||
return t("gateway.deny.denied_singular")
|
||||
confirmation_text = t("gateway.deny.denied_reason_plural", count=count, reason=reason)
|
||||
else:
|
||||
confirmation_text = t("gateway.deny.denied_reason_singular", reason=reason)
|
||||
elif count > 1:
|
||||
confirmation_text = t("gateway.deny.denied_plural", count=count)
|
||||
else:
|
||||
confirmation_text = t("gateway.deny.denied_singular")
|
||||
|
||||
if _adapter:
|
||||
try:
|
||||
await _adapter.send(
|
||||
source.chat_id,
|
||||
confirmation_text,
|
||||
reply_to=event.message_id,
|
||||
metadata={
|
||||
"is_approval_prompt": True,
|
||||
"force_proactive_send": True,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to send /deny confirmation to %s: %s",
|
||||
source.chat_id,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
async def _handle_debug_command(self, event: MessageEvent) -> str:
|
||||
"""Handle /debug — upload debug report (summary only) and return paste URLs.
|
||||
|
||||
+843
-28
File diff suppressed because it is too large
Load Diff
+1899
-27
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,414 @@
|
||||
"""Tests for approval boundary handling in WeCom native streaming."""
|
||||
import asyncio
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_adapter():
|
||||
"""Create a mock WeCom adapter with native streaming support."""
|
||||
from gateway.platforms.base import BasePlatformAdapter
|
||||
MockAdapter = type("MockAdapter", (BasePlatformAdapter,), {
|
||||
"MAX_MESSAGE_LENGTH": 4096,
|
||||
"SUPPORTS_MESSAGE_EDITING": False,
|
||||
"SUPPORTS_NATIVE_STREAMING": True,
|
||||
})
|
||||
MockAdapter.__abstractmethods__ = frozenset()
|
||||
adapter = MockAdapter.__new__(MockAdapter)
|
||||
adapter._typing_paused = set()
|
||||
adapter.send_stream_frame = AsyncMock(return_value=True)
|
||||
adapter.send = AsyncMock(return_value=MagicMock(success=True, message_id="msg"))
|
||||
adapter.supports_native_streaming = lambda chat_type=None, metadata=None: True
|
||||
return adapter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def consumer_config():
|
||||
"""Create a minimal consumer config."""
|
||||
return StreamConsumerConfig(
|
||||
chat_type="dm", cursor="",
|
||||
edit_interval=0.01, buffer_threshold=5,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_approval_boundary_finalizes_and_disables_native(mock_adapter, consumer_config):
|
||||
"""Approval boundary must finalize the current stream (creating a stable
|
||||
message for pre-approval content) and disable native streaming so
|
||||
post-approval output goes through reliable send()."""
|
||||
consumer = GatewayStreamConsumer(
|
||||
adapter=mock_adapter,
|
||||
chat_id="test_chat",
|
||||
config=consumer_config,
|
||||
)
|
||||
|
||||
consumer._use_native_streaming = True
|
||||
consumer._native_stream_opened = True
|
||||
consumer._turn_id = "turn_123"
|
||||
consumer._initial_reply_to_id = "msg_456"
|
||||
consumer._accumulated = "下面我来执行:先确认最新分区"
|
||||
|
||||
# Signal approval boundary
|
||||
boundary_result = consumer.close_for_approval_prompt()
|
||||
if isinstance(boundary_result, tuple):
|
||||
boundary_future, _ = boundary_result
|
||||
else:
|
||||
boundary_future = boundary_result
|
||||
|
||||
consumer_task = asyncio.create_task(consumer.run())
|
||||
result = await asyncio.wait_for(boundary_future, timeout=1.0)
|
||||
|
||||
assert result is True
|
||||
|
||||
# Stream must be finalized (stable message created)
|
||||
finalize_calls = [
|
||||
call for call in mock_adapter.send_stream_frame.call_args_list
|
||||
if call.kwargs.get("finalize") is True
|
||||
]
|
||||
assert len(finalize_calls) == 1, "Must finalize the stream"
|
||||
finalize_text = finalize_calls[0].args[0]
|
||||
assert finalize_text == "下面我来执行:先确认最新分区"
|
||||
|
||||
# Native streaming must be disabled for post-approval output
|
||||
assert consumer._use_native_streaming is False, (
|
||||
"Native streaming must be disabled — post-approval goes via send()"
|
||||
)
|
||||
assert consumer._native_stream_opened is False
|
||||
|
||||
consumer.finish()
|
||||
await asyncio.sleep(0.05)
|
||||
consumer_task.cancel()
|
||||
try:
|
||||
await consumer_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_approval_boundary_uses_placeholder_when_no_accumulated(mock_adapter, consumer_config):
|
||||
"""When there's no accumulated text, finalize with a visible placeholder."""
|
||||
consumer = GatewayStreamConsumer(
|
||||
adapter=mock_adapter,
|
||||
chat_id="test_chat",
|
||||
config=consumer_config,
|
||||
)
|
||||
|
||||
consumer._use_native_streaming = True
|
||||
consumer._native_stream_opened = True
|
||||
consumer._turn_id = "turn_123"
|
||||
consumer._accumulated = "" # No text accumulated
|
||||
|
||||
boundary_result = consumer.close_for_approval_prompt()
|
||||
if isinstance(boundary_result, tuple):
|
||||
boundary_future, _ = boundary_result
|
||||
else:
|
||||
boundary_future = boundary_result
|
||||
|
||||
consumer_task = asyncio.create_task(consumer.run())
|
||||
result = await asyncio.wait_for(boundary_future, timeout=1.0)
|
||||
assert result is True
|
||||
|
||||
finalize_calls = [
|
||||
call for call in mock_adapter.send_stream_frame.call_args_list
|
||||
if call.kwargs.get("finalize") is True
|
||||
]
|
||||
assert len(finalize_calls) == 1
|
||||
finalize_text = finalize_calls[0].args[0]
|
||||
assert finalize_text == "⏸ 等待审批中..."
|
||||
|
||||
consumer.finish()
|
||||
await asyncio.sleep(0.05)
|
||||
consumer_task.cancel()
|
||||
try:
|
||||
await consumer_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_boundary_uses_custom_placeholder_when_no_accumulated(mock_adapter, consumer_config):
|
||||
"""A clarify boundary passes its own placeholder; the empty-content
|
||||
finalize must use it instead of the approval wording so the finalized
|
||||
bubble doesn't read 'waiting for approval' for a decision question."""
|
||||
consumer = GatewayStreamConsumer(
|
||||
adapter=mock_adapter,
|
||||
chat_id="test_chat",
|
||||
config=consumer_config,
|
||||
)
|
||||
|
||||
consumer._use_native_streaming = True
|
||||
consumer._native_stream_opened = True
|
||||
consumer._turn_id = "turn_123"
|
||||
consumer._accumulated = "" # No text accumulated
|
||||
|
||||
boundary_result = consumer.close_for_approval_prompt("💬 等待你的选择...")
|
||||
if isinstance(boundary_result, tuple):
|
||||
boundary_future, _ = boundary_result
|
||||
else:
|
||||
boundary_future = boundary_result
|
||||
|
||||
consumer_task = asyncio.create_task(consumer.run())
|
||||
result = await asyncio.wait_for(boundary_future, timeout=1.0)
|
||||
assert result is True
|
||||
|
||||
finalize_calls = [
|
||||
call for call in mock_adapter.send_stream_frame.call_args_list
|
||||
if call.kwargs.get("finalize") is True
|
||||
]
|
||||
assert len(finalize_calls) == 1
|
||||
finalize_text = finalize_calls[0].args[0]
|
||||
assert finalize_text == "💬 等待你的选择..."
|
||||
|
||||
# Native streaming disabled so post-answer output opens a fresh bubble.
|
||||
assert consumer._use_native_streaming is False
|
||||
assert consumer.cfg.buffer_only is True
|
||||
|
||||
consumer.finish()
|
||||
await asyncio.sleep(0.05)
|
||||
consumer_task.cancel()
|
||||
try:
|
||||
await consumer_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clarify_boundary_logs_use_clarify_prefix(mock_adapter, consumer_config, caplog):
|
||||
"""A clarify boundary that fails to finalize must log with a "Clarify"
|
||||
prefix, not "Approval" — otherwise a clarify failure looks like a
|
||||
dangerous-command approval failure during troubleshooting."""
|
||||
import logging
|
||||
|
||||
# Make finalize fail so the warning path fires, and the fallback send fail
|
||||
# too so the error path fires — exercising the reason-labelled logs.
|
||||
mock_adapter.send_stream_frame = AsyncMock(return_value=False)
|
||||
mock_adapter.send = AsyncMock(return_value=MagicMock(success=False))
|
||||
|
||||
consumer = GatewayStreamConsumer(
|
||||
adapter=mock_adapter,
|
||||
chat_id="test_chat",
|
||||
config=consumer_config,
|
||||
)
|
||||
consumer._use_native_streaming = True
|
||||
consumer._native_stream_opened = True
|
||||
consumer._turn_id = "turn_123"
|
||||
consumer._accumulated = "部分内容"
|
||||
|
||||
boundary_result = consumer.close_for_approval_prompt(
|
||||
"💬 等待你的选择...", reason="Clarify",
|
||||
)
|
||||
if isinstance(boundary_result, tuple):
|
||||
boundary_future, _ = boundary_result
|
||||
else:
|
||||
boundary_future = boundary_result
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="gateway.stream_consumer"):
|
||||
consumer_task = asyncio.create_task(consumer.run())
|
||||
await asyncio.wait_for(boundary_future, timeout=1.0)
|
||||
|
||||
boundary_logs = [r.getMessage() for r in caplog.records]
|
||||
assert any("Clarify boundary" in m for m in boundary_logs), (
|
||||
f"Expected a 'Clarify boundary' log, got: {boundary_logs}"
|
||||
)
|
||||
assert not any("Approval boundary" in m for m in boundary_logs), (
|
||||
f"Clarify boundary must not log as 'Approval boundary': {boundary_logs}"
|
||||
)
|
||||
|
||||
consumer.finish()
|
||||
await asyncio.sleep(0.05)
|
||||
consumer_task.cancel()
|
||||
try:
|
||||
await consumer_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_approval_boundary_post_approval_one_shot_send(mock_adapter, consumer_config):
|
||||
"""After approval boundary, post-approval content must:
|
||||
1. Set buffer_only=True (no mid-stream flushes)
|
||||
2. Accumulate all deltas without sending
|
||||
3. Deliver everything via one adapter.send() call on finish()"""
|
||||
consumer = GatewayStreamConsumer(
|
||||
adapter=mock_adapter,
|
||||
chat_id="test_chat",
|
||||
config=consumer_config,
|
||||
)
|
||||
|
||||
consumer._use_native_streaming = True
|
||||
consumer._native_stream_opened = True
|
||||
consumer._turn_id = "turn_123"
|
||||
consumer._accumulated = "Pre-approval text"
|
||||
|
||||
boundary_result = consumer.close_for_approval_prompt()
|
||||
if isinstance(boundary_result, tuple):
|
||||
boundary_future, _ = boundary_result
|
||||
else:
|
||||
boundary_future = boundary_result
|
||||
|
||||
consumer_task = asyncio.create_task(consumer.run())
|
||||
await asyncio.wait_for(boundary_future, timeout=1.0)
|
||||
|
||||
# Verify buffer_only is set
|
||||
assert consumer.cfg.buffer_only is True, "Must set buffer_only after boundary"
|
||||
assert consumer._use_native_streaming is False
|
||||
|
||||
# Send post-approval content — should NOT trigger any immediate send
|
||||
mock_adapter.send_stream_frame.reset_mock()
|
||||
mock_adapter.send.reset_mock()
|
||||
|
||||
consumer.on_delta("Post-approval result text here")
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Before finish(): no send() or stream frame calls
|
||||
assert mock_adapter.send.call_count == 0, (
|
||||
"buffer_only: no send before finish()"
|
||||
)
|
||||
stream_calls = [
|
||||
call for call in mock_adapter.send_stream_frame.call_args_list
|
||||
if not call.kwargs.get("finalize")
|
||||
]
|
||||
assert len(stream_calls) == 0, (
|
||||
"Post-approval must NOT use native streaming"
|
||||
)
|
||||
|
||||
# Now finish — should deliver via send()
|
||||
consumer.finish()
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# adapter.send should have been called with the full post-approval text
|
||||
send_calls = mock_adapter.send.call_args_list
|
||||
assert len(send_calls) >= 1, "finish() must deliver via send()"
|
||||
# The delivered text should contain our post-approval content
|
||||
delivered = send_calls[-1].args[1] if len(send_calls[-1].args) > 1 else send_calls[-1].kwargs.get("content", "")
|
||||
assert "Post-approval result" in delivered
|
||||
|
||||
consumer_task.cancel()
|
||||
try:
|
||||
await consumer_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_approval_boundary_stream_not_opened_at_boundary_time(mock_adapter, consumer_config):
|
||||
"""When native streaming is active but _native_stream_opened is still False
|
||||
at the time boundary processes (e.g., seed succeeded but stream was closed by
|
||||
a prior error before boundary arrives), no finalize is sent."""
|
||||
consumer = GatewayStreamConsumer(
|
||||
adapter=mock_adapter,
|
||||
chat_id="test_chat",
|
||||
config=consumer_config,
|
||||
)
|
||||
|
||||
# Let run() do its normal seed (which sets _native_stream_opened=True)
|
||||
# Then manually close it before boundary processes
|
||||
consumer._use_native_streaming = True
|
||||
consumer._native_stream_opened = True
|
||||
consumer._turn_id = "turn_123"
|
||||
consumer._accumulated = "Some text"
|
||||
|
||||
# Queue boundary
|
||||
boundary_result = consumer.close_for_approval_prompt()
|
||||
if isinstance(boundary_result, tuple):
|
||||
boundary_future, _ = boundary_result
|
||||
else:
|
||||
boundary_future = boundary_result
|
||||
|
||||
# Simulate: stream was closed by error BEFORE consumer processes boundary
|
||||
consumer._native_stream_opened = False
|
||||
|
||||
consumer_task = asyncio.create_task(consumer.run())
|
||||
result = await asyncio.wait_for(boundary_future, timeout=1.0)
|
||||
|
||||
assert result is True
|
||||
# Native streaming should be disabled after boundary
|
||||
assert consumer._use_native_streaming is False
|
||||
|
||||
consumer.finish()
|
||||
await asyncio.sleep(0.05)
|
||||
consumer_task.cancel()
|
||||
try:
|
||||
await consumer_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_approval_boundary_finalize_fails_fallback_send_succeeds(mock_adapter, consumer_config):
|
||||
"""When stream finalize fails but fallback send() succeeds, boundary is True."""
|
||||
consumer = GatewayStreamConsumer(
|
||||
adapter=mock_adapter,
|
||||
chat_id="test_chat",
|
||||
config=consumer_config,
|
||||
)
|
||||
|
||||
consumer._use_native_streaming = True
|
||||
consumer._native_stream_opened = True
|
||||
consumer._turn_id = "turn_123"
|
||||
consumer._accumulated = "Pre-approval text"
|
||||
|
||||
# Finalize fails (returns False)
|
||||
mock_adapter.send_stream_frame = AsyncMock(return_value=False)
|
||||
# Fallback send succeeds
|
||||
mock_adapter.send = AsyncMock(return_value=MagicMock(success=True, message_id="msg"))
|
||||
|
||||
boundary_result = consumer.close_for_approval_prompt()
|
||||
if isinstance(boundary_result, tuple):
|
||||
boundary_future, _ = boundary_result
|
||||
else:
|
||||
boundary_future = boundary_result
|
||||
|
||||
consumer_task = asyncio.create_task(consumer.run())
|
||||
result = await asyncio.wait_for(boundary_future, timeout=1.0)
|
||||
|
||||
assert result is True, "Fallback send succeeded → boundary should be True"
|
||||
mock_adapter.send.assert_awaited_once_with("test_chat", "Pre-approval text")
|
||||
|
||||
consumer.finish()
|
||||
await asyncio.sleep(0.05)
|
||||
consumer_task.cancel()
|
||||
try:
|
||||
await consumer_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_approval_boundary_finalize_and_fallback_both_fail(mock_adapter, consumer_config):
|
||||
"""When both stream finalize and fallback send() fail, boundary is False."""
|
||||
consumer = GatewayStreamConsumer(
|
||||
adapter=mock_adapter,
|
||||
chat_id="test_chat",
|
||||
config=consumer_config,
|
||||
)
|
||||
|
||||
consumer._use_native_streaming = True
|
||||
consumer._native_stream_opened = True
|
||||
consumer._turn_id = "turn_123"
|
||||
consumer._accumulated = "Pre-approval text"
|
||||
|
||||
# Finalize fails (raises)
|
||||
mock_adapter.send_stream_frame = AsyncMock(side_effect=RuntimeError("stream dead"))
|
||||
# Fallback send also fails
|
||||
mock_adapter.send = AsyncMock(return_value=MagicMock(success=False, error="timeout"))
|
||||
|
||||
boundary_result = consumer.close_for_approval_prompt()
|
||||
if isinstance(boundary_result, tuple):
|
||||
boundary_future, _ = boundary_result
|
||||
else:
|
||||
boundary_future = boundary_result
|
||||
|
||||
consumer_task = asyncio.create_task(consumer.run())
|
||||
result = await asyncio.wait_for(boundary_future, timeout=1.0)
|
||||
|
||||
assert result is False, "Both finalize and fallback failed → boundary should be False"
|
||||
|
||||
consumer.finish()
|
||||
await asyncio.sleep(0.05)
|
||||
consumer_task.cancel()
|
||||
try:
|
||||
await consumer_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Tests for tool-progress-in-native-stream (single bubble) feature.
|
||||
|
||||
Validates that tool-progress lines are injected into the native streaming
|
||||
bubble and properly overwritten by text deltas (Strategy B).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from gateway.stream_consumer import (
|
||||
GatewayStreamConsumer,
|
||||
StreamConsumerConfig,
|
||||
_TOOL_PROGRESS,
|
||||
)
|
||||
|
||||
|
||||
def _make_native_streaming_adapter(*, supports_native: bool = True):
|
||||
"""Build a BasePlatformAdapter subclass that supports native streaming."""
|
||||
from gateway.platforms.base import BasePlatformAdapter
|
||||
|
||||
NativeStreamingAdapter = type(
|
||||
"NativeStreamingAdapter",
|
||||
(BasePlatformAdapter,),
|
||||
{
|
||||
"MAX_MESSAGE_LENGTH": 4096,
|
||||
"SUPPORTS_MESSAGE_EDITING": False,
|
||||
"SUPPORTS_NATIVE_STREAMING": True,
|
||||
},
|
||||
)
|
||||
NativeStreamingAdapter.__abstractmethods__ = frozenset()
|
||||
adapter = NativeStreamingAdapter.__new__(NativeStreamingAdapter)
|
||||
adapter._typing_paused = set()
|
||||
adapter._fatal_error_message = None
|
||||
adapter.frames = []
|
||||
|
||||
def _supports(chat_type=None, metadata=None):
|
||||
return bool(supports_native)
|
||||
adapter.supports_native_streaming = _supports
|
||||
|
||||
async def _send_stream_frame(
|
||||
text, *, finalize=False, chat_id=None, reply_to=None, **kwargs
|
||||
):
|
||||
adapter.frames.append({
|
||||
"text": text,
|
||||
"finalize": finalize,
|
||||
"chat_id": chat_id,
|
||||
})
|
||||
return True
|
||||
adapter.send_stream_frame = _send_stream_frame
|
||||
|
||||
adapter.send = AsyncMock(
|
||||
return_value=SimpleNamespace(success=True, message_id="fallback_msg"),
|
||||
)
|
||||
adapter.edit_message = AsyncMock(
|
||||
return_value=SimpleNamespace(success=True),
|
||||
)
|
||||
return adapter
|
||||
|
||||
|
||||
def _make_consumer(*, native_streaming: bool = True) -> GatewayStreamConsumer:
|
||||
"""Create a GatewayStreamConsumer configured for native streaming."""
|
||||
adapter = _make_native_streaming_adapter(supports_native=native_streaming)
|
||||
cfg = StreamConsumerConfig(chat_type="dm", cursor="▌")
|
||||
consumer = GatewayStreamConsumer(adapter, "chat-1", cfg)
|
||||
# Force native streaming resolution
|
||||
consumer._use_native_streaming = native_streaming
|
||||
return consumer
|
||||
|
||||
|
||||
# === UNIT TESTS ===
|
||||
|
||||
|
||||
class TestAcceptsToolProgress:
|
||||
"""Tests for the accepts_tool_progress property."""
|
||||
|
||||
def test_native_streaming_accepts(self):
|
||||
consumer = _make_consumer(native_streaming=True)
|
||||
assert consumer.accepts_tool_progress is True
|
||||
|
||||
def test_non_native_does_not_accept(self):
|
||||
consumer = _make_consumer(native_streaming=False)
|
||||
assert consumer.accepts_tool_progress is False
|
||||
|
||||
|
||||
class TestOnToolProgress:
|
||||
"""Tests for on_tool_progress() enqueue behavior."""
|
||||
|
||||
def test_enqueues_sentinel(self):
|
||||
consumer = _make_consumer()
|
||||
consumer.on_tool_progress("🔍 Searching...")
|
||||
item = consumer._queue.get_nowait()
|
||||
assert isinstance(item, tuple)
|
||||
assert len(item) == 2
|
||||
assert item[0] is _TOOL_PROGRESS
|
||||
assert item[1] == "🔍 Searching..."
|
||||
|
||||
def test_empty_line_not_enqueued(self):
|
||||
consumer = _make_consumer()
|
||||
consumer.on_tool_progress("")
|
||||
assert consumer._queue.empty()
|
||||
|
||||
|
||||
class TestComposeFrameContent:
|
||||
"""Tests for _compose_frame_content() composition logic (Strategy B)."""
|
||||
|
||||
def test_only_tool_lines(self):
|
||||
consumer = _make_consumer()
|
||||
consumer._tool_progress_lines = ["🔍 Searching...", "💻 Running git log"]
|
||||
result = consumer._compose_frame_content()
|
||||
assert result == "🔍 Searching...\n💻 Running git log"
|
||||
|
||||
def test_only_accumulated(self):
|
||||
consumer = _make_consumer()
|
||||
consumer._accumulated = "Here is the answer."
|
||||
result = consumer._compose_frame_content()
|
||||
assert result == "Here is the answer."
|
||||
|
||||
def test_both_accumulated_and_tool_lines_strategy_b(self):
|
||||
"""Strategy B: text + separator + tool status at bottom."""
|
||||
consumer = _make_consumer()
|
||||
consumer._accumulated = "Here is some text so far."
|
||||
consumer._tool_progress_lines = ["🔍 Searching the web..."]
|
||||
result = consumer._compose_frame_content()
|
||||
assert result == "Here is some text so far.\n\n---\n🔍 Searching the web..."
|
||||
|
||||
def test_multiple_tool_lines_stacked(self):
|
||||
consumer = _make_consumer()
|
||||
consumer._tool_progress_lines = [
|
||||
"🔍 web_search: 'python'",
|
||||
"💻 terminal: git log",
|
||||
"📄 read_file: main.py",
|
||||
]
|
||||
result = consumer._compose_frame_content()
|
||||
assert "web_search" in result
|
||||
assert "terminal" in result
|
||||
assert "read_file" in result
|
||||
# Lines are joined with newlines
|
||||
assert result.count("\n") == 2
|
||||
|
||||
def test_empty_state(self):
|
||||
consumer = _make_consumer()
|
||||
result = consumer._compose_frame_content()
|
||||
assert result == ""
|
||||
|
||||
|
||||
class TestSegmentReset:
|
||||
"""Test that segment reset clears tool progress state."""
|
||||
|
||||
def test_reset_clears_tool_progress(self):
|
||||
consumer = _make_consumer()
|
||||
consumer._tool_progress_lines = ["🔍 Searching..."]
|
||||
consumer._tool_progress_active = True
|
||||
consumer._reset_segment_state()
|
||||
assert consumer._tool_progress_lines == []
|
||||
assert consumer._tool_progress_active is False
|
||||
|
||||
|
||||
# === INTEGRATION TESTS (drain loop) ===
|
||||
|
||||
|
||||
class TestToolProgressDrainLoop:
|
||||
"""Integration tests for the drain loop + frame delivery."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_progress_only_then_done(self):
|
||||
"""Pure tool-progress turn (no text): tool lines visible as mid-frame,
|
||||
finalize uses placeholder since no accumulated text."""
|
||||
consumer = _make_consumer()
|
||||
consumer.on_tool_progress("🔍 Searching...")
|
||||
consumer.on_tool_progress("💻 terminal: ls")
|
||||
|
||||
# Start consumer so it drains tool progress and sends mid-frames
|
||||
task = asyncio.create_task(consumer.run())
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
# Now finish — tool lines were already displayed as a frame
|
||||
consumer.finish()
|
||||
await asyncio.sleep(0.3)
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
frames = consumer.adapter.frames
|
||||
# Should have: seed frame, at least one mid-frame with tool lines,
|
||||
# and a finalize frame (✅ placeholder since no text)
|
||||
assert len(frames) >= 2
|
||||
# Find mid-frames that contain tool progress
|
||||
non_finalize = [f for f in frames if not f["finalize"] and f["text"]]
|
||||
assert any("Searching" in f["text"] or "terminal" in f["text"] for f in non_finalize), (
|
||||
f"Expected tool progress in mid-frames, got: {[f['text'] for f in frames]}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_progress_then_text_clears_overlay(self):
|
||||
"""Tool progress → text delta should clear tool lines from frame."""
|
||||
consumer = _make_consumer()
|
||||
consumer.on_tool_progress("🔍 Searching...")
|
||||
consumer.on_delta("Hello world")
|
||||
consumer.finish()
|
||||
|
||||
task = asyncio.create_task(consumer.run())
|
||||
await asyncio.sleep(0.5)
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
# After text arrives, tool_progress_lines should be cleared
|
||||
assert consumer._tool_progress_lines == []
|
||||
assert "Hello world" in consumer._accumulated
|
||||
|
||||
# The finalize frame should contain just the text
|
||||
frames = consumer.adapter.frames
|
||||
finalize_frames = [f for f in frames if f["finalize"]]
|
||||
if finalize_frames:
|
||||
assert "Hello world" in finalize_frames[-1]["text"]
|
||||
assert "Searching" not in finalize_frames[-1]["text"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_then_tool_then_text_strategy_b(self):
|
||||
"""Strategy B: text → tool → text appends tool at bottom then clears."""
|
||||
consumer = _make_consumer()
|
||||
|
||||
# Phase 1: initial text
|
||||
consumer.on_delta("First part. ")
|
||||
# Phase 2: tool progress mid-stream
|
||||
consumer.on_tool_progress("🔍 web_search...")
|
||||
# Phase 3: more text arrives
|
||||
consumer.on_delta("Second part.")
|
||||
consumer.finish()
|
||||
|
||||
task = asyncio.create_task(consumer.run())
|
||||
await asyncio.sleep(0.5)
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
# Final state: tool lines cleared, accumulated has both text parts
|
||||
assert consumer._tool_progress_lines == []
|
||||
assert "First part." in consumer._accumulated
|
||||
assert "Second part." in consumer._accumulated
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parallel_tool_calls_stacked(self):
|
||||
"""Multiple tool.started back-to-back should stack in overlay."""
|
||||
consumer = _make_consumer()
|
||||
consumer.on_tool_progress("🔍 web_search")
|
||||
consumer.on_tool_progress("💻 terminal")
|
||||
consumer.on_tool_progress("📄 read_file")
|
||||
|
||||
# Let drain loop process and send mid-frame before finish
|
||||
task = asyncio.create_task(consumer.run())
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
consumer.finish()
|
||||
await asyncio.sleep(0.3)
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
# All three should have been accumulated and sent in one frame
|
||||
frames = consumer.adapter.frames
|
||||
# Find a frame that contains all three tools
|
||||
all_three = [
|
||||
f for f in frames
|
||||
if "web_search" in f["text"]
|
||||
and "terminal" in f["text"]
|
||||
and "read_file" in f["text"]
|
||||
]
|
||||
assert len(all_three) >= 1, (
|
||||
f"Expected a frame with all 3 tools, got: {[f['text'] for f in frames]}"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_frame_is_pure_text(self):
|
||||
"""The finalize frame must only contain accumulated text, no tool lines."""
|
||||
consumer = _make_consumer()
|
||||
consumer.on_tool_progress("🔍 Searching...")
|
||||
consumer.on_delta("The answer is 42.")
|
||||
# Add a tool progress AFTER text (Strategy B scenario)
|
||||
consumer.on_tool_progress("💻 terminal: verify")
|
||||
# Then more text clears it
|
||||
consumer.on_delta(" Verified.")
|
||||
consumer.finish()
|
||||
|
||||
task = asyncio.create_task(consumer.run())
|
||||
await asyncio.sleep(0.5)
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
frames = consumer.adapter.frames
|
||||
finalize_frames = [f for f in frames if f["finalize"]]
|
||||
if finalize_frames:
|
||||
final_text = finalize_frames[-1]["text"]
|
||||
assert "The answer is 42. Verified." in final_text
|
||||
assert "Searching" not in final_text
|
||||
assert "terminal" not in final_text
|
||||
assert "---" not in final_text
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,534 @@
|
||||
"""Repro tests for the WeCom native-streaming "同一回复发两条" duplicate.
|
||||
|
||||
Root cause (see docs/rca-wecom-stream-final-ack-timeout-duplicate.md and
|
||||
/tmp/claude-wecom-dup-report.md): a **timeout inversion race** between two
|
||||
independent timers.
|
||||
|
||||
* The consumer's got_done finalize blocks on the finalize frame's ack,
|
||||
bounded by ``adapter._REPLY_ACK_TIMEOUT``.
|
||||
* The gateway's ``finally`` block joins the ``stream_task`` with a hardcoded
|
||||
``timeout=5.0`` (``gateway/run.py:20749``) and then ``stream_task.cancel()``
|
||||
(``run.py:20751``).
|
||||
|
||||
When the ack is slower than the gateway's join window, the gateway cancels the
|
||||
consumer *before* it finishes finalizing. The finalize frame's bytes, however,
|
||||
were already written to the wire by an **independent control-worker task**
|
||||
(``_control_send_worker`` / ``_enqueue_chat_send(is_control=True)``) and WeCom
|
||||
has rendered them. But because the consumer was cancelled mid-await, the
|
||||
``self._final_content_delivered = True`` line (``stream_consumer.py:2061`` /
|
||||
``1016``) never runs. The gateway then reads ``final_content_delivered=False``
|
||||
(``run.py:20801-20803``), does NOT suppress the normal final send
|
||||
(``run.py:20819``), and emits a second, duplicate bubble.
|
||||
|
||||
These tests exercise the **real** ``GatewayStreamConsumer.run()`` lifecycle
|
||||
against the **real** ``WeComAdapter`` (only the websocket byte-writer and the
|
||||
ack timing are controlled), so the async interaction between the gateway join /
|
||||
cancel and the consumer finalize / flag-set actually happens — this is what the
|
||||
previous PR (#62861) failed to test when it mocked out ``handle_message``.
|
||||
|
||||
Assertions target observable causality:
|
||||
* whether the finalize frame's bytes reached the wire (WeCom rendered it);
|
||||
* the value of ``consumer.final_content_delivered`` (the flag the gateway
|
||||
reads);
|
||||
* whether the gateway's suppression predicate fires (i.e. whether a second
|
||||
normal send would go out).
|
||||
|
||||
The suppression predicate below is a faithful mirror of the (non-importable,
|
||||
nested) logic in ``gateway/run.py:20793-20819`` — it reads the *real* consumer
|
||||
properties, it does not re-implement the delivery lifecycle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig
|
||||
|
||||
|
||||
CHAT_ID = "chat-1"
|
||||
REQ_ID = "req-1"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Faithful mirrors of the real gateway control flow (cite source lines).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _gateway_suppresses_normal_send(consumer, final_text: str) -> bool:
|
||||
"""Mirror of gateway/run.py:20793-20819 suppression decision.
|
||||
|
||||
Reads the REAL consumer properties (``final_response_sent`` /
|
||||
``final_content_delivered``) — the same attributes ``run.py`` consults —
|
||||
and returns True when the gateway would set ``already_sent`` and skip the
|
||||
normal final send. When it returns False the gateway emits a second bubble.
|
||||
"""
|
||||
_final = final_text or ""
|
||||
_is_empty_sentinel = not _final or _final == "(empty)"
|
||||
# run.py:20814 _stream_confirmed_final_delivery(...) with previewed=False
|
||||
# collapses to final_response_sent for a non-previewed response.
|
||||
_streamed = bool(getattr(consumer, "final_response_sent", False))
|
||||
_content_delivered = bool(getattr(consumer, "final_content_delivered", False))
|
||||
_transformed = False # no plugin transform in these scenarios
|
||||
return (
|
||||
not _is_empty_sentinel
|
||||
and not _transformed
|
||||
and (_streamed or _content_delivered)
|
||||
)
|
||||
|
||||
|
||||
async def _gateway_join_and_cancel(stream_task: asyncio.Task, join_timeout: float) -> None:
|
||||
"""Faithful copy of gateway/run.py:20748-20755 finally-block join.
|
||||
|
||||
Joins the stream task with a bounded timeout; on timeout it cancels the
|
||||
task (exactly what the gateway does when the consumer has not finished
|
||||
finalizing within the join window).
|
||||
"""
|
||||
try:
|
||||
await asyncio.wait_for(stream_task, timeout=join_timeout)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
stream_task.cancel()
|
||||
try:
|
||||
await stream_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Real WeComAdapter with a controllable websocket + ack timing.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_real_wecom_adapter(*, resolve_finalize_ack: bool):
|
||||
"""Build a real ``WeComAdapter`` whose only fakes are the WS byte-writer
|
||||
and the ack timing.
|
||||
|
||||
``_send_json`` (the actual byte-writer) records every frame so we can prove
|
||||
the finalize frame reached the wire. The ack for **non-final** frames (seed
|
||||
/ intermediate) is always resolved immediately so the finalize's
|
||||
pre-drain (``_send_reply_queued`` is_final branch) doesn't block on the
|
||||
seed's ack. The finalize frame's own ack is resolved immediately only when
|
||||
``resolve_finalize_ack`` is True.
|
||||
|
||||
* ``resolve_finalize_ack=False`` → finalize ack stays pending → the
|
||||
consumer blocks in its got_done finalize (the exact window where the
|
||||
gateway's join fires and cancels it). This is the bug scenario.
|
||||
* ``resolve_finalize_ack=True`` → finalize ack returns at once → the
|
||||
consumer finishes finalize and sets its flags before any join fires.
|
||||
"""
|
||||
from plugins.platforms.wecom.adapter import WeComAdapter
|
||||
from gateway.config import PlatformConfig
|
||||
|
||||
adapter = WeComAdapter(PlatformConfig(enabled=True))
|
||||
adapter._ws = MagicMock(closed=False)
|
||||
adapter._last_chat_req_ids[CHAT_ID] = REQ_ID
|
||||
|
||||
frames: list[dict] = []
|
||||
|
||||
async def _fake_send_json(payload: dict) -> None:
|
||||
# This is the real byte-writer boundary: reaching here means the frame
|
||||
# was put on the wire and WeCom will render it.
|
||||
frames.append(payload)
|
||||
stream = payload.get("body", {}).get("stream", {})
|
||||
finish = bool(stream.get("finish"))
|
||||
req = payload.get("headers", {}).get("req_id")
|
||||
|
||||
should_resolve = (not finish) or resolve_finalize_ack
|
||||
if should_resolve:
|
||||
queue = adapter._reply_queues.get(req)
|
||||
if queue and queue.pending_ack and not queue.pending_ack.future.done():
|
||||
# Simulate WeCom's ack coming back on the same WS.
|
||||
queue.pending_ack.future.set_result({"body": {"errcode": 0}})
|
||||
|
||||
adapter._send_json = _fake_send_json
|
||||
adapter._recorded_frames = frames
|
||||
return adapter
|
||||
|
||||
|
||||
def _finalize_frames_on_wire(adapter) -> list[dict]:
|
||||
"""Return the finish=true stream frames that reached the byte-writer."""
|
||||
out = []
|
||||
for payload in adapter._recorded_frames:
|
||||
stream = payload.get("body", {}).get("stream", {})
|
||||
if stream.get("finish") is True:
|
||||
out.append(payload)
|
||||
return out
|
||||
|
||||
|
||||
async def _cleanup_adapter(adapter) -> None:
|
||||
"""Cancel any lingering control/normal workers so the event loop is clean."""
|
||||
for task in list(adapter._control_workers.values()) + list(adapter._chat_workers.values()):
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Test group 1: timeout-inversion race (core reproduction)
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestTimeoutInversionDoubleSend:
|
||||
"""The finalize ack being slower than the gateway join window must not
|
||||
strand ``final_content_delivered=False`` while WeCom already rendered the
|
||||
finalize frame — that produces a duplicate bubble."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_slow_ack_beyond_gateway_join_causes_double_send(self):
|
||||
"""ack slower than the gateway join → consumer cancelled mid-finalize.
|
||||
|
||||
Timeline (deterministic, no wall-clock sleeps):
|
||||
* seed + finalize frame bytes are written to the wire;
|
||||
* the finalize ack never returns within the join window
|
||||
(``resolve_finalize_ack=False``, ``_REPLY_ACK_TIMEOUT`` > join);
|
||||
* the gateway join (0.1s) fires and cancels the consumer;
|
||||
* the consumer's got_done finalize await is cancelled BEFORE the
|
||||
``final_content_delivered = True`` line runs.
|
||||
|
||||
The finalize frame is on the wire (rendered), yet the gateway reads
|
||||
``final_content_delivered=False`` and would send a normal duplicate.
|
||||
|
||||
This test asserts the DESIRED post-fix state, so it FAILS today
|
||||
(exposing the double send) and PASSES once the flag reflects the
|
||||
rendered finalize frame.
|
||||
"""
|
||||
adapter = _make_real_wecom_adapter(resolve_finalize_ack=False)
|
||||
# Inversion: ack window (0.3s) is LARGER than the gateway join (0.1s),
|
||||
# so the join fires while the finalize ack is still pending.
|
||||
adapter._REPLY_ACK_TIMEOUT = 0.3
|
||||
join_timeout = 0.1
|
||||
|
||||
cfg = StreamConsumerConfig(
|
||||
chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5,
|
||||
)
|
||||
consumer = GatewayStreamConsumer(adapter, CHAT_ID, cfg)
|
||||
|
||||
final_text = "这是模型这一轮生成的最终回答,需要作为流式 finalize 帧发出。"
|
||||
consumer.on_delta(final_text)
|
||||
consumer.finish()
|
||||
|
||||
stream_task = asyncio.create_task(consumer.run())
|
||||
try:
|
||||
# Gateway finally-block join (run.py:20748-20755) fires while the
|
||||
# finalize ack is still pending, cancelling the consumer.
|
||||
await _gateway_join_and_cancel(stream_task, join_timeout)
|
||||
|
||||
finalize_frames = _finalize_frames_on_wire(adapter)
|
||||
|
||||
# The finalize frame reached the wire → WeCom rendered it. This is
|
||||
# the first (and should be ONLY) user-visible bubble.
|
||||
assert len(finalize_frames) >= 1, (
|
||||
"finalize frame never reached the wire — repro precondition "
|
||||
"not met (WeCom must have rendered the streamed final)"
|
||||
)
|
||||
assert final_text in finalize_frames[-1]["body"]["stream"]["content"]
|
||||
|
||||
gateway_would_send_normal = not _gateway_suppresses_normal_send(
|
||||
consumer, final_text
|
||||
)
|
||||
total_user_visible = len(finalize_frames) + (
|
||||
1 if gateway_would_send_normal else 0
|
||||
)
|
||||
|
||||
# DESIRED post-fix invariants (currently violated == bug reproduced):
|
||||
assert consumer.final_content_delivered is True, (
|
||||
"BUG: consumer was cancelled mid-finalize; the finalize frame "
|
||||
"was rendered by WeCom but final_content_delivered stayed False, "
|
||||
"so the gateway will not suppress the normal send"
|
||||
)
|
||||
assert _gateway_suppresses_normal_send(consumer, final_text) is True, (
|
||||
"BUG: gateway does not suppress the normal final send → duplicate"
|
||||
)
|
||||
assert total_user_visible == 1, (
|
||||
f"BUG: user sees {total_user_visible} bubbles for one reply "
|
||||
f"(finalize frame rendered + normal send fired)"
|
||||
)
|
||||
finally:
|
||||
await _cleanup_adapter(adapter)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fast_ack_within_join_window_single_send(self):
|
||||
"""Control: ack returns within the join window → single bubble.
|
||||
|
||||
The finalize ack resolves immediately, so the consumer completes its
|
||||
got_done finalize, sets ``final_content_delivered=True``, and the run
|
||||
finishes before the gateway join needs to cancel anything. The gateway
|
||||
then suppresses the normal send. Passes today and after the fix
|
||||
(regression guard).
|
||||
"""
|
||||
adapter = _make_real_wecom_adapter(resolve_finalize_ack=True)
|
||||
adapter._REPLY_ACK_TIMEOUT = 5.0
|
||||
join_timeout = 0.5
|
||||
|
||||
cfg = StreamConsumerConfig(
|
||||
chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5,
|
||||
)
|
||||
consumer = GatewayStreamConsumer(adapter, CHAT_ID, cfg)
|
||||
|
||||
final_text = "这是一个能在 join 窗口内正常拿到 ack 的回答。"
|
||||
consumer.on_delta(final_text)
|
||||
consumer.finish()
|
||||
|
||||
stream_task = asyncio.create_task(consumer.run())
|
||||
try:
|
||||
await _gateway_join_and_cancel(stream_task, join_timeout)
|
||||
|
||||
finalize_frames = _finalize_frames_on_wire(adapter)
|
||||
assert len(finalize_frames) == 1
|
||||
|
||||
assert consumer.final_content_delivered is True
|
||||
assert _gateway_suppresses_normal_send(consumer, final_text) is True
|
||||
|
||||
gateway_would_send_normal = not _gateway_suppresses_normal_send(
|
||||
consumer, final_text
|
||||
)
|
||||
total_user_visible = len(finalize_frames) + (
|
||||
1 if gateway_would_send_normal else 0
|
||||
)
|
||||
assert total_user_visible == 1
|
||||
finally:
|
||||
await _cleanup_adapter(adapter)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Test group 2: best-effort finalize "DO NOT mark" path
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestBestEffortFinalizeDoubleSend:
|
||||
"""stream_consumer.py:2064-2094: when native streaming fails mid-turn, the
|
||||
consumer sends a best-effort finalize frame to close the bubble but
|
||||
deliberately does NOT set ``final_content_delivered`` (the "DO NOT mark"
|
||||
comment at 2085-2088). If WeCom actually renders that finalize frame, the
|
||||
subsequent fallback send delivers the same content a second time.
|
||||
|
||||
This uses a controllable ``BasePlatformAdapter`` subclass (the accepted
|
||||
pattern in tests/gateway/test_wecom_per_turn.py) because the trigger is a
|
||||
frame-level failure, not an ack timing race — no wall-clock timing needed.
|
||||
"""
|
||||
|
||||
@pytest.mark.xfail(
|
||||
reason="KNOWN LOW-FREQUENCY ISSUE (accepted, not yet fixed): the "
|
||||
"best-effort finalize path (stream_consumer.py:2104-2128) renders the "
|
||||
"content AND falls through to a fallback send(), delivering the same "
|
||||
"answer twice. Only triggers when native streaming fails MID-STREAM "
|
||||
"(846608 expired / 846609 subscription lost / errcode 6000 / network "
|
||||
"error) — rare. The duplicate is a SAFE-direction failure (user sees "
|
||||
"one extra bubble, no data loss). A clean fix requires reconciling "
|
||||
"'do not drop post-finalize increments' with 'do not re-send the "
|
||||
"first-send path', which the first-send path (L2378) does not consult "
|
||||
"the delivery flag for; deferred over risking increment loss on a rare "
|
||||
"path. The high-frequency duplicate (timeout inversion) IS fixed — see "
|
||||
"TestTimeoutInversionDoubleSend + commit 7ba739818 (B2).",
|
||||
strict=True,
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_best_effort_finalize_renders_but_flag_unset_double_send(self):
|
||||
"""native content frame fails → best-effort finalize renders + fallback send.
|
||||
|
||||
Ordering matters: the delta is pushed and ``run()`` is allowed to
|
||||
process a mid-stream **content** frame BEFORE ``finish()`` is queued.
|
||||
That content frame fails (returns False), which disables native
|
||||
streaming and triggers the best-effort finalize at
|
||||
``stream_consumer.py:2077`` — WeCom renders it — and then the consumer
|
||||
falls through to a fallback ``send()`` that delivers the same content a
|
||||
second time (the "DO NOT mark" comment at 2085-2088 is what leaves the
|
||||
gateway free to re-send).
|
||||
|
||||
The text is > ``_MIN_NEW_VISIBLE_CHARS`` (60) so the mid-stream frame is
|
||||
actually attempted rather than buffered.
|
||||
|
||||
Asserts the DESIRED single-delivery state, so it FAILS today (content
|
||||
rendered by the best-effort finalize AND re-sent by the fallback) and
|
||||
PASSES once the double delivery is closed.
|
||||
"""
|
||||
from gateway.platforms.base import BasePlatformAdapter, SendResult
|
||||
|
||||
class FlakyNativeAdapter(BasePlatformAdapter):
|
||||
MAX_MESSAGE_LENGTH = 4096
|
||||
SUPPORTS_MESSAGE_EDITING = False
|
||||
SUPPORTS_NATIVE_STREAMING = True
|
||||
|
||||
def __init__(self):
|
||||
self._typing_paused = set()
|
||||
self._fatal_error_message = None
|
||||
# Records of everything that reached the user.
|
||||
self.rendered_stream_frames: list[dict] = [] # incl. finalize
|
||||
self.fallback_sends: list[str] = []
|
||||
self._content_frames_seen = 0
|
||||
|
||||
def supports_native_streaming(self, chat_type=None, metadata=None):
|
||||
return True
|
||||
|
||||
async def send_stream_frame(
|
||||
self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs
|
||||
):
|
||||
# Seed frame (empty, non-finalize) opens the bubble.
|
||||
if text == "" and not finalize:
|
||||
self.rendered_stream_frames.append({"text": text, "finalize": False})
|
||||
return True
|
||||
if finalize:
|
||||
# Best-effort finalize frame — WeCom DOES render it.
|
||||
self.rendered_stream_frames.append({"text": text, "finalize": True})
|
||||
return True
|
||||
# First real content frame fails → disables native streaming
|
||||
# and triggers the best-effort finalize path.
|
||||
self._content_frames_seen += 1
|
||||
return False
|
||||
|
||||
async def send(self, chat_id, content, reply_to=None, metadata=None):
|
||||
# The gateway/consumer fallback send — a SECOND delivery of the
|
||||
# same content.
|
||||
self.fallback_sends.append(content)
|
||||
return SendResult(success=True, message_id="fallback-msg")
|
||||
|
||||
FlakyNativeAdapter.__abstractmethods__ = frozenset()
|
||||
adapter = FlakyNativeAdapter()
|
||||
|
||||
cfg = StreamConsumerConfig(
|
||||
chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5,
|
||||
)
|
||||
consumer = GatewayStreamConsumer(adapter, CHAT_ID, cfg)
|
||||
|
||||
# > 60 visible chars so the mid-stream content frame is actually sent.
|
||||
final_text = (
|
||||
"需要交付给用户的最终答案。这一段内容刻意写得足够长,"
|
||||
"以便越过 stream_consumer 的 _MIN_NEW_VISIBLE_CHARS=60 阈值,"
|
||||
"从而在 got_done 之前触发一次真正的流式内容帧发送。"
|
||||
)
|
||||
assert len(final_text) >= 60
|
||||
|
||||
consumer.on_delta(final_text)
|
||||
task = asyncio.create_task(consumer.run())
|
||||
# Let run() process the mid-stream content frame (which fails) BEFORE
|
||||
# the turn-final _DONE arrives.
|
||||
await asyncio.sleep(0.1)
|
||||
consumer.finish()
|
||||
await task
|
||||
|
||||
best_effort_finalize = [
|
||||
f for f in adapter.rendered_stream_frames if f["finalize"] and f["text"]
|
||||
]
|
||||
|
||||
# Count content-bearing user-visible deliveries:
|
||||
# * best-effort finalize frame(s) that WeCom rendered, plus
|
||||
# * fallback send(s).
|
||||
content_deliveries = len(best_effort_finalize) + len(adapter.fallback_sends)
|
||||
|
||||
# DESIRED post-fix invariant (currently violated == bug reproduced):
|
||||
# the same answer must reach the user exactly once. Today the
|
||||
# best-effort finalize renders the content AND the fallback send
|
||||
# delivers it again.
|
||||
assert content_deliveries == 1, (
|
||||
f"BUG: content delivered {content_deliveries} times — best-effort "
|
||||
f"finalize rendered {len(best_effort_finalize)} frame(s) and "
|
||||
f"fallback send fired {len(adapter.fallback_sends)} time(s) "
|
||||
f"(stream_consumer.py:2085-2088 'DO NOT mark' path)"
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Test group 3: orphan-queue ack routing race (adapter-level)
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def _make_manual_ack_adapter():
|
||||
"""Real WeComAdapter whose ``_send_json`` records frames but NEVER
|
||||
auto-resolves any ack, so the test can orchestrate the exact interleaving
|
||||
of intermediate-ack arrival vs. finalize registration by hand.
|
||||
"""
|
||||
from plugins.platforms.wecom.adapter import WeComAdapter
|
||||
from gateway.config import PlatformConfig
|
||||
|
||||
adapter = WeComAdapter(PlatformConfig(enabled=True))
|
||||
adapter._ws = MagicMock(closed=False)
|
||||
adapter._last_chat_req_ids[CHAT_ID] = REQ_ID
|
||||
|
||||
frames: list[dict] = []
|
||||
|
||||
async def _fake_send_json(payload: dict) -> None:
|
||||
frames.append(payload)
|
||||
|
||||
adapter._send_json = _fake_send_json
|
||||
adapter._recorded_frames = frames
|
||||
return adapter
|
||||
|
||||
|
||||
class TestOrphanQueueAckRouting:
|
||||
"""The finalize frame shares the inbound req_id with the intermediate
|
||||
frames. While the finalize awaits the pending intermediate ack to drain,
|
||||
that intermediate ack can arrive and ``_resolve_reply_ack`` pops the WHOLE
|
||||
queue out of ``_reply_queues``. Before the fix, the finalize then
|
||||
registered its ``pending_ack`` on the now-orphaned queue object (detached
|
||||
from the dict), so its own ack landed in Unrouted and the finalize hit a
|
||||
15s timeout. The fix re-attaches the queue to the dict before registering.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_intermediate_ack_mid_drain_does_not_orphan_finalize_queue(self):
|
||||
adapter = _make_manual_ack_adapter()
|
||||
adapter._REPLY_ACK_TIMEOUT = 0.3 # snappy: bounds the failure mode
|
||||
|
||||
# 1) Fire an intermediate frame (skip_if_pending) — occupies pending_ack.
|
||||
await adapter._send_reply_queued(
|
||||
REQ_ID,
|
||||
{"msgtype": "stream", "stream": {"id": "s1", "finish": False, "content": "hi"}},
|
||||
is_final=False, skip_if_pending=True,
|
||||
)
|
||||
inter_queue = adapter._reply_queues[REQ_ID]
|
||||
assert inter_queue.pending_ack is not None
|
||||
|
||||
# 2) Start the finalize: it enters the is_final drain branch and yields
|
||||
# at `await shield(pending_frame.future)`.
|
||||
fin_task = asyncio.create_task(adapter._send_reply_queued(
|
||||
REQ_ID,
|
||||
{"msgtype": "stream", "stream": {"id": "s1", "finish": True, "content": "done"}},
|
||||
is_final=True, skip_if_pending=False,
|
||||
))
|
||||
await asyncio.sleep(0) # let finalize reach the drain await
|
||||
|
||||
# 3) Deliver the intermediate ack MID-DRAIN. _resolve_reply_ack resolves
|
||||
# the drain future AND pops the whole queue out of _reply_queues.
|
||||
await adapter._dispatch_payload(
|
||||
{"headers": {"req_id": REQ_ID}, "body": {"errcode": 0}}
|
||||
)
|
||||
assert REQ_ID not in adapter._reply_queues, (
|
||||
"precondition: intermediate ack should have popped the queue"
|
||||
)
|
||||
|
||||
# 4) Let finalize resume: it must clear the drained pending_ack, create
|
||||
# its own frame, re-attach the queue, register pending_ack, and write
|
||||
# bytes. Resuming through the shield/wait_for wrapper takes several
|
||||
# event-loop iterations, so yield until the finalize has re-registered
|
||||
# (bounded, to avoid a hang if the fix regresses).
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0)
|
||||
q = adapter._reply_queues.get(REQ_ID)
|
||||
if q is not None and q.pending_ack is not None:
|
||||
break
|
||||
|
||||
# FIX ASSERTION: the finalize must have re-attached its queue to the
|
||||
# dict, so its ack is routable. Without the fix the queue would still
|
||||
# be absent here (orphaned) and the finalize ack would go Unrouted.
|
||||
assert REQ_ID in adapter._reply_queues, (
|
||||
"orphan-queue bug: finalize registered pending_ack on a queue "
|
||||
"detached from _reply_queues — its ack would be Unrouted"
|
||||
)
|
||||
assert adapter._reply_queues[REQ_ID].pending_ack is not None
|
||||
|
||||
# 5) Deliver the finalize's own ack — it must route and resolve.
|
||||
await adapter._dispatch_payload(
|
||||
{"headers": {"req_id": REQ_ID}, "body": {"errcode": 0}}
|
||||
)
|
||||
resp = await asyncio.wait_for(fin_task, timeout=1.0)
|
||||
|
||||
# Routed successfully → NOT the synthetic timeout-assumed response.
|
||||
assert resp.get("errmsg") != "ack_timeout_assumed_delivered", (
|
||||
"finalize ack should have been routed, not timed out"
|
||||
)
|
||||
|
||||
await _cleanup_adapter(adapter)
|
||||
@@ -0,0 +1,403 @@
|
||||
"""Tests for per-turn stream isolation and concurrent consumer scenarios."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from gateway.config import PlatformConfig
|
||||
|
||||
|
||||
class TestPerTurnStreamIsolation:
|
||||
"""Verify that concurrent consumers with different turn_ids don't interfere."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_users_concurrent_streaming(self):
|
||||
"""Multiple users (different chats) streaming concurrently don't interfere."""
|
||||
from plugins.platforms.wecom.adapter import WeComAdapter
|
||||
|
||||
adapter = WeComAdapter(PlatformConfig(enabled=True))
|
||||
try:
|
||||
# Setup 3 different users/chats
|
||||
adapter._last_chat_req_ids["user-1"] = "req-1"
|
||||
adapter._last_chat_req_ids["user-2"] = "req-2"
|
||||
adapter._last_chat_req_ids["user-3"] = "req-3"
|
||||
adapter._send_json = AsyncMock()
|
||||
adapter._ws = AsyncMock(closed=False)
|
||||
adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0})
|
||||
|
||||
# User 1, 2, 3 all start streaming simultaneously
|
||||
await adapter.send_stream_frame("user1 content", chat_id="user-1", turn_id="turn-1")
|
||||
await adapter.send_stream_frame("user2 content", chat_id="user-2", turn_id="turn-2")
|
||||
await adapter.send_stream_frame("user3 content", chat_id="user-3", turn_id="turn-3")
|
||||
|
||||
# All 3 turns active
|
||||
assert "user-1:turn-1" in adapter._stream_turns
|
||||
assert "user-2:turn-2" in adapter._stream_turns
|
||||
assert "user-3:turn-3" in adapter._stream_turns
|
||||
|
||||
# User 2 finishes first
|
||||
ok2 = await adapter.send_stream_frame(
|
||||
"user2 final", chat_id="user-2", finalize=True, turn_id="turn-2"
|
||||
)
|
||||
assert ok2 is True
|
||||
assert "user-2:turn-2" not in adapter._stream_turns
|
||||
# User 1 and 3 still active
|
||||
assert "user-1:turn-1" in adapter._stream_turns
|
||||
assert "user-3:turn-3" in adapter._stream_turns
|
||||
|
||||
# User 1 finishes
|
||||
ok1 = await adapter.send_stream_frame(
|
||||
"user1 final", chat_id="user-1", finalize=True, turn_id="turn-1"
|
||||
)
|
||||
assert ok1 is True
|
||||
assert "user-1:turn-1" not in adapter._stream_turns
|
||||
# User 3 still active
|
||||
assert "user-3:turn-3" in adapter._stream_turns
|
||||
|
||||
# User 3 finishes
|
||||
ok3 = await adapter.send_stream_frame(
|
||||
"user3 final", chat_id="user-3", finalize=True, turn_id="turn-3"
|
||||
)
|
||||
assert ok3 is True
|
||||
assert "user-3:turn-3" not in adapter._stream_turns
|
||||
finally:
|
||||
await adapter.disconnect()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_turns_same_chat_isolated(self):
|
||||
"""Two concurrent consumers in same chat maintain independent streams."""
|
||||
from plugins.platforms.wecom.adapter import WeComAdapter
|
||||
|
||||
adapter = WeComAdapter(PlatformConfig(enabled=True))
|
||||
try:
|
||||
adapter._last_chat_req_ids["chat-1"] = "req-1"
|
||||
adapter._send_json = AsyncMock()
|
||||
# Mock _ws with closed=False and async close()
|
||||
adapter._ws = AsyncMock(closed=False)
|
||||
adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0})
|
||||
|
||||
# Consumer 1 starts streaming
|
||||
await adapter.send_stream_frame("consumer1 frame1", chat_id="chat-1", turn_id="turn-1")
|
||||
assert "chat-1:turn-1" in adapter._stream_turns
|
||||
|
||||
# Consumer 2 starts streaming (concurrent)
|
||||
await adapter.send_stream_frame("consumer2 frame1", chat_id="chat-1", turn_id="turn-2")
|
||||
assert "chat-1:turn-2" in adapter._stream_turns
|
||||
|
||||
# Both turns coexist
|
||||
assert len([k for k in adapter._stream_turns if k.startswith("chat-1:")]) == 2
|
||||
|
||||
# Consumer 1 finalizes
|
||||
ok1 = await adapter.send_stream_frame(
|
||||
"consumer1 final", chat_id="chat-1", finalize=True, turn_id="turn-1"
|
||||
)
|
||||
assert ok1 is True
|
||||
assert "chat-1:turn-1" not in adapter._stream_turns
|
||||
# Consumer 2's turn still exists
|
||||
assert "chat-1:turn-2" in adapter._stream_turns
|
||||
|
||||
# Consumer 2 finalizes
|
||||
ok2 = await adapter.send_stream_frame(
|
||||
"consumer2 final", chat_id="chat-1", finalize=True, turn_id="turn-2"
|
||||
)
|
||||
assert ok2 is True
|
||||
assert "chat-1:turn-2" not in adapter._stream_turns
|
||||
finally:
|
||||
await adapter.disconnect()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_user_expired_others_unaffected(self):
|
||||
"""User A hits stream expired; Users B and C continue normally."""
|
||||
from plugins.platforms.wecom.adapter import STREAM_EXPIRED_ERRCODE, WeComAdapter
|
||||
|
||||
adapter = WeComAdapter(PlatformConfig(enabled=True))
|
||||
try:
|
||||
adapter._last_chat_req_ids["user-A"] = "req-A"
|
||||
adapter._last_chat_req_ids["user-B"] = "req-B"
|
||||
adapter._last_chat_req_ids["user-C"] = "req-C"
|
||||
adapter._send_json = AsyncMock()
|
||||
adapter._ws = AsyncMock(closed=False)
|
||||
|
||||
# All 3 users start streaming
|
||||
await adapter.send_stream_frame("A content", chat_id="user-A", turn_id="turn-A")
|
||||
await adapter.send_stream_frame("B content", chat_id="user-B", turn_id="turn-B")
|
||||
await adapter.send_stream_frame("C content", chat_id="user-C", turn_id="turn-C")
|
||||
|
||||
# User A hits stream expired
|
||||
adapter._send_reply_queued = AsyncMock(
|
||||
return_value={"errcode": STREAM_EXPIRED_ERRCODE, "errmsg": "expired"}
|
||||
)
|
||||
okA = await adapter.send_stream_frame(
|
||||
"A final", chat_id="user-A", finalize=True, turn_id="turn-A"
|
||||
)
|
||||
assert okA is False
|
||||
assert "user-A" in adapter._stream_expired_chats
|
||||
assert "user-A:turn-A" not in adapter._stream_turns
|
||||
|
||||
# Users B and C should NOT be affected (different chats)
|
||||
adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0})
|
||||
okB = await adapter.send_stream_frame(
|
||||
"B final", chat_id="user-B", finalize=True, turn_id="turn-B"
|
||||
)
|
||||
okC = await adapter.send_stream_frame(
|
||||
"C final", chat_id="user-C", finalize=True, turn_id="turn-C"
|
||||
)
|
||||
assert okB is True # ✅ User B unaffected
|
||||
assert okC is True # ✅ User C unaffected
|
||||
assert "user-B:turn-B" not in adapter._stream_turns
|
||||
assert "user-C:turn-C" not in adapter._stream_turns
|
||||
|
||||
# Only user-A is in expired list
|
||||
assert "user-A" in adapter._stream_expired_chats
|
||||
assert "user-B" not in adapter._stream_expired_chats
|
||||
assert "user-C" not in adapter._stream_expired_chats
|
||||
finally:
|
||||
await adapter.disconnect()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_turn_expired_other_continues(self):
|
||||
"""When one turn hits stream expired, other concurrent turns can continue."""
|
||||
from plugins.platforms.wecom.adapter import STREAM_EXPIRED_ERRCODE, WeComAdapter
|
||||
|
||||
adapter = WeComAdapter(PlatformConfig(enabled=True))
|
||||
try:
|
||||
adapter._last_chat_req_ids["chat-1"] = "req-1"
|
||||
adapter._send_json = AsyncMock()
|
||||
adapter._ws = AsyncMock(closed=False)
|
||||
|
||||
# Consumer 1 and 2 both start
|
||||
await adapter.send_stream_frame("c1 frame", chat_id="chat-1", turn_id="turn-1")
|
||||
await adapter.send_stream_frame("c2 frame", chat_id="chat-1", turn_id="turn-2")
|
||||
assert "chat-1:turn-1" in adapter._stream_turns
|
||||
assert "chat-1:turn-2" in adapter._stream_turns
|
||||
|
||||
# Consumer 1 hits expired error on finalize
|
||||
adapter._send_reply_queued = AsyncMock(
|
||||
return_value={"errcode": STREAM_EXPIRED_ERRCODE, "errmsg": "stream expired"}
|
||||
)
|
||||
ok1 = await adapter.send_stream_frame(
|
||||
"c1 final", chat_id="chat-1", finalize=True, turn_id="turn-1"
|
||||
)
|
||||
assert ok1 is False
|
||||
assert "chat-1" in adapter._stream_expired_chats
|
||||
assert "chat-1:turn-1" not in adapter._stream_turns # turn-1 cleaned up
|
||||
|
||||
# Consumer 2's existing turn can still finalize
|
||||
adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0})
|
||||
ok2 = await adapter.send_stream_frame(
|
||||
"c2 final", chat_id="chat-1", finalize=True, turn_id="turn-2"
|
||||
)
|
||||
assert ok2 is True # ✅ turn-2 not blocked by chat-level expired
|
||||
assert "chat-1:turn-2" not in adapter._stream_turns
|
||||
finally:
|
||||
await adapter.disconnect()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expired_chat_blocks_new_turn_creation(self):
|
||||
"""After one turn expired, new turn creation is blocked."""
|
||||
from plugins.platforms.wecom.adapter import WeComAdapter
|
||||
|
||||
adapter = WeComAdapter(PlatformConfig(enabled=True))
|
||||
try:
|
||||
adapter._last_chat_req_ids["chat-1"] = "req-1"
|
||||
adapter._stream_expired_chats.add("chat-1")
|
||||
adapter._send_reply_request = AsyncMock(return_value={"errcode": 0})
|
||||
|
||||
# Try to create a new turn after chat is expired
|
||||
ok = await adapter.send_stream_frame("new frame", chat_id="chat-1", turn_id="new-turn")
|
||||
assert ok is False
|
||||
assert "chat-1:new-turn" not in adapter._stream_turns
|
||||
finally:
|
||||
await adapter.disconnect()
|
||||
|
||||
|
||||
class TestNativeFallbackStreamClose:
|
||||
"""Verify that native streaming fallback closes open streams."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seed_success_first_frame_fails_still_finalizes(self):
|
||||
"""Seed frame opens stream bubble, first content frame fails → finalize called.
|
||||
|
||||
This is the critical edge case: seed frame has length 0 but opens the
|
||||
WeCom typing bubble. If the first content frame fails, we must still
|
||||
finalize based on _native_stream_opened, not _native_last_pushed_len.
|
||||
"""
|
||||
from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig
|
||||
from gateway.platforms.base import BasePlatformAdapter
|
||||
|
||||
class MockAdapter(BasePlatformAdapter):
|
||||
MAX_MESSAGE_LENGTH = 4096
|
||||
SUPPORTS_MESSAGE_EDITING = False
|
||||
SUPPORTS_NATIVE_STREAMING = True
|
||||
|
||||
def __init__(self):
|
||||
self._typing_paused = set()
|
||||
self.send_stream_frame_calls = []
|
||||
self.send_calls = []
|
||||
self.should_fail_first_content = True
|
||||
|
||||
def supports_native_streaming(self, chat_type=None, metadata=None):
|
||||
return True
|
||||
|
||||
async def send_stream_frame(
|
||||
self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs
|
||||
):
|
||||
call_info = {"text_len": len(text), "finalize": finalize, "text_preview": text[:20]}
|
||||
self.send_stream_frame_calls.append(call_info)
|
||||
|
||||
# Seed frame (empty) always succeeds
|
||||
if len(text) == 0 and not finalize:
|
||||
return True
|
||||
|
||||
# First non-seed, non-finalize frame fails
|
||||
if self.should_fail_first_content and len(text) > 0 and not finalize:
|
||||
self.should_fail_first_content = False
|
||||
raise RuntimeError("first content frame failed")
|
||||
|
||||
# Finalize frames and subsequent content frames succeed
|
||||
return True
|
||||
|
||||
async def send(self, chat_id, content, reply_to=None, metadata=None):
|
||||
self.send_calls.append({"content_preview": content[:20]})
|
||||
return type("SendResult", (), {"success": True, "message_id": "msg-1"})()
|
||||
|
||||
MockAdapter.__abstractmethods__ = frozenset()
|
||||
adapter = MockAdapter()
|
||||
cfg = StreamConsumerConfig(chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5)
|
||||
consumer = GatewayStreamConsumer(adapter, "chat-1", cfg)
|
||||
|
||||
# Send short content to minimize frame count
|
||||
consumer.on_delta("X")
|
||||
|
||||
import asyncio
|
||||
task = asyncio.create_task(consumer.run())
|
||||
await asyncio.sleep(0.05)
|
||||
consumer.finish()
|
||||
await task
|
||||
|
||||
# Verify: seed succeeded, then finalize was attempted (not skipped)
|
||||
assert len(adapter.send_stream_frame_calls) >= 2
|
||||
# First call: seed (length 0)
|
||||
assert adapter.send_stream_frame_calls[0]["text_len"] == 0
|
||||
assert not adapter.send_stream_frame_calls[0]["finalize"]
|
||||
|
||||
# At least one finalize call should have been made. This is the core
|
||||
# invariant: finalize is driven by _native_stream_opened (the seed
|
||||
# opened the bubble), NOT by _native_last_pushed_len — so even though
|
||||
# the seed had length 0 and the first content frame failed, finalize
|
||||
# must still be attempted to close the typing bubble.
|
||||
finalize_calls = [c for c in adapter.send_stream_frame_calls if c["finalize"]]
|
||||
assert len(finalize_calls) >= 1, "Finalize should be called even though seed had length 0"
|
||||
|
||||
# Fire-and-forget (14c49c781a): with the throttle removed, the 1-char
|
||||
# content "X" is pushed immediately as an intermediate frame instead of
|
||||
# being buffered. That frame fails per the mock, so a proactive send()
|
||||
# fallback IS expected to deliver the content reliably. (Under the old
|
||||
# _MIN_NEW_VISIBLE_CHARS=60 gate this tiny frame was never sent, so the
|
||||
# previous assertion of zero send() fallbacks no longer holds.)
|
||||
assert len(adapter.send_calls) == 1
|
||||
assert adapter.send_calls[0]["content_preview"] == "X"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_fallback_closes_stream_on_success(self):
|
||||
"""When native fails mid-stream, best-effort finalize succeeds."""
|
||||
from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig
|
||||
from gateway.platforms.base import BasePlatformAdapter
|
||||
|
||||
class MockAdapter(BasePlatformAdapter):
|
||||
MAX_MESSAGE_LENGTH = 4096
|
||||
SUPPORTS_MESSAGE_EDITING = False
|
||||
SUPPORTS_NATIVE_STREAMING = True
|
||||
|
||||
def __init__(self):
|
||||
self._typing_paused = set()
|
||||
self.frames = []
|
||||
self.frame_count = 0
|
||||
|
||||
def supports_native_streaming(self, chat_type=None, metadata=None):
|
||||
return True
|
||||
|
||||
async def send_stream_frame(
|
||||
self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs
|
||||
):
|
||||
self.frame_count += 1
|
||||
self.frames.append({"text": text, "finalize": finalize})
|
||||
# First 2 frames succeed, 3rd fails (non-expired error)
|
||||
if self.frame_count == 3:
|
||||
raise RuntimeError("network error")
|
||||
# 4th frame (finalize in fallback) succeeds
|
||||
return True
|
||||
|
||||
async def send(self, chat_id, content, reply_to=None, metadata=None):
|
||||
self.frames.append({"send": content})
|
||||
return type("SendResult", (), {"success": True, "message_id": "msg-1"})()
|
||||
|
||||
MockAdapter.__abstractmethods__ = frozenset()
|
||||
adapter = MockAdapter()
|
||||
cfg = StreamConsumerConfig(chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5)
|
||||
consumer = GatewayStreamConsumer(adapter, "chat-1", cfg)
|
||||
|
||||
# Send enough to trigger frames
|
||||
consumer.on_delta("First frame content that exceeds the threshold.")
|
||||
consumer.on_delta(" Second frame content also exceeds threshold.")
|
||||
consumer.on_delta(" Third will fail.")
|
||||
|
||||
import asyncio
|
||||
task = asyncio.create_task(consumer.run())
|
||||
await asyncio.sleep(0.1)
|
||||
consumer.finish()
|
||||
await task
|
||||
|
||||
# Should have: seed, frame1, frame2, (frame3 fails), finalize in fallback
|
||||
# After fix #3: best-effort finalize closes the typing bubble but does NOT
|
||||
# mark content_delivered. The fallback send() will deliver content reliably.
|
||||
assert len([f for f in adapter.frames if f.get("finalize")]) >= 1
|
||||
# Fallback send() IS expected to fire (content delivery via proactive send)
|
||||
assert len([f for f in adapter.frames if "send" in f]) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_fallback_falls_to_send_on_finalize_fail(self):
|
||||
"""When native fails and finalize also fails, falls through to send()."""
|
||||
from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig
|
||||
from gateway.platforms.base import BasePlatformAdapter
|
||||
|
||||
class MockAdapter(BasePlatformAdapter):
|
||||
MAX_MESSAGE_LENGTH = 4096
|
||||
SUPPORTS_MESSAGE_EDITING = False
|
||||
SUPPORTS_NATIVE_STREAMING = True
|
||||
|
||||
def __init__(self):
|
||||
self._typing_paused = set()
|
||||
self.frames = []
|
||||
self.frame_count = 0
|
||||
|
||||
def supports_native_streaming(self, chat_type=None, metadata=None):
|
||||
return True
|
||||
|
||||
async def send_stream_frame(
|
||||
self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs
|
||||
):
|
||||
self.frame_count += 1
|
||||
self.frames.append({"text": text, "finalize": finalize})
|
||||
# All frames fail (simulating complete stream failure)
|
||||
if self.frame_count >= 2:
|
||||
raise RuntimeError("stream dead")
|
||||
return True
|
||||
|
||||
async def send(self, chat_id, content, reply_to=None, metadata=None):
|
||||
self.frames.append({"send": content})
|
||||
return type("SendResult", (), {"success": True, "message_id": "msg-1"})()
|
||||
|
||||
MockAdapter.__abstractmethods__ = frozenset()
|
||||
adapter = MockAdapter()
|
||||
cfg = StreamConsumerConfig(chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5)
|
||||
consumer = GatewayStreamConsumer(adapter, "chat-1", cfg)
|
||||
|
||||
consumer.on_delta("Content that will cause stream to fail.")
|
||||
|
||||
import asyncio
|
||||
task = asyncio.create_task(consumer.run())
|
||||
await asyncio.sleep(0.1)
|
||||
consumer.finish()
|
||||
await task
|
||||
|
||||
# Finalize failed → should fall through to send()
|
||||
assert len([f for f in adapter.frames if "send" in f]) == 1
|
||||
Reference in New Issue
Block a user