feat(wecom): native reply streaming (per-turn isolation, dedup-safe delivery, interaction boundaries)

Implement native reply streaming for the WeCom (企业微信) adapter over the
long-connection "msgtype: stream" transport, so a reply renders as a single
live-updating typing bubble instead of one final block. Aligns with the
official wecom-openclaw-plugin streaming behavior.

Includes the machinery intrinsic to native streaming on WeCom:

- Transport: seed frame (<think></think>) opens the typing bubble, intermediate
  frames update it, a finalize frame closes it; native-streaming adapters are
  let past the edit-only gate. Fire-and-forget intermediate frames (WeCom
  long-connection mode has no documented edit-rate limit); an adapter-level
  frame cap is retained. (Early builds gated frames behind a char throttle;
  removed in favor of fire-and-forget + identity dedup.)
- Per-turn isolation: each turn owns a unique turn_id; concurrent messages are
  isolated via (chat_id, turn_id)-keyed state. Dual-lane priority queue
  (control vs normal) plus a per-chat token bucket to stay under WeCom's rate
  limit (errcode 846607).
- Dedup-safe delivery + ack-race handling: deliver-once contract (a frame is
  delivered the moment it is emitted; failures logged, not re-sent; delivery
  marked once per turn), per-req_id reply queue with ack tracking, and the
  timeout-inversion / orphan-queue race fixes. Robust fallback on 846608 /
  846609 / errcode 6000 / passive-reply timeout via proactive send.
- Interaction boundaries: finalize + reset before approval/clarify prompts so
  the prompt is the last thing on screen and never traps a lingering bubble;
  eager re-seed after a clarify answer so the typing bubble reappears instantly.
- Stream-level keepalive: optional periodic finish=false frame + finalize-time
  stream-age guard to refresh WeCom's ~6-minute reply-stream window on long
  turns (mitigates 846604 / 846608). Off by default; tunable via config.yaml.
- Tool-progress folded into the same native-stream bubble instead of separate
  messages; image+text double-callback merged into one turn.

Tests cover the streaming lifecycle, per-turn isolation, duplicate-send / ack
timing, approval + clarify boundaries, eager re-seed, and tool-progress.
This commit is contained in:
wansui
2026-08-26 17:50:07 +08:00
committed by Teknium
parent 42dc0dea70
commit 2ecb544551
10 changed files with 6781 additions and 65 deletions
+154 -4
View File
@@ -4568,6 +4568,13 @@ class TurnRunner:
# code (same boilerplate imports → identical previews).
if msg == ctx.last_progress_msg[0]:
ctx.repeat_count[0] += 1
# Native-stream-progress routing: dedup updates the last line
# in the overlay rather than sending a queue signal.
_sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None
if _sc is not None and getattr(_sc, "accepts_tool_progress", False):
# Replace the last progress line with the dedup version
_sc.on_tool_progress(f"{msg} (×{ctx.repeat_count[0] + 1})")
return
# Update the last line in progress_lines with a counter
# via a special "dedup" queue message.
ctx.progress_queue.put(("__dedup__", msg, ctx.repeat_count[0]))
@@ -4575,6 +4582,14 @@ class TurnRunner:
ctx.last_progress_msg[0] = msg
ctx.repeat_count[0] = 0
# Native-stream-progress routing: if the stream consumer is active
# and using native streaming, inject progress directly into the
# stream bubble instead of the separate progress queue.
_sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None
if _sc is not None and getattr(_sc, "accepts_tool_progress", False):
_sc.on_tool_progress(msg)
return
ctx.progress_queue.put(msg)
async def _send_native_task_card_progress(self, adapter) -> None:
@@ -5947,6 +5962,51 @@ class TurnRunner:
_mem_notif = "on" if _mem_notif else "off"
agent.memory_notifications = str(_mem_notif).lower() if _mem_notif else "on"
# ------------------------------------------------------------------
# Shared native-stream boundary close. For platforms with native
# streaming (e.g. WeCom msgtype:"stream"), an interaction that
# interrupts the stream — a dangerous-command approval prompt OR a
# clarify decision prompt — must finalize the current stream and
# disable native streaming first. Otherwise the agent's
# post-interaction output keeps flowing into send_stream_frame and
# updates the *old* bubble that preceded the prompt, instead of
# starting a fresh bubble below it (the "气泡割裂" symptom). After
# the boundary, post-prompt output goes through the reliable send()
# path as a new message. Runs on the agent thread; the consumer
# processes the boundary serially via its queue.
def _close_native_stream_boundary(
_reason: str, _placeholder: str | None = None, _reopen: bool = False,
) -> bool:
_sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None
if not (_sc and getattr(_sc, "_use_native_streaming", False)):
return True
_cancelled_flag = None
try:
_boundary_result = _sc.close_for_approval_prompt(
_placeholder, reason=_reason, reopen=_reopen,
)
# Returns (future, cancelled_flag) or just a future.
if isinstance(_boundary_result, tuple):
_boundary_future, _cancelled_flag = _boundary_result
else:
_boundary_future = _boundary_result
if hasattr(_boundary_future, "result"):
_ok = _boundary_future.result(timeout=10)
if not _ok:
logger.warning(
"%s boundary failed to close stream properly — "
"prompt may still appear in typing bubble", _reason,
)
return bool(_ok)
return True
except (TimeoutError, Exception) as _boundary_err:
if _cancelled_flag is not None:
_cancelled_flag["cancelled"] = True
logger.warning(
"%s boundary timed out or failed: %s", _reason, _boundary_err,
)
return False
# ------------------------------------------------------------------
# Clarify callback: present a clarify prompt and block on a response.
#
@@ -5974,6 +6034,21 @@ class TurnRunner:
multi_select=bool(multi_select),
)
# For WeCom native streaming: finalize the current stream before
# showing the clarify prompt so the post-answer output opens a
# fresh bubble below the question instead of updating the bubble
# that preceded it — the "气泡割裂" symptom. Unlike the approval
# path, clarify passes reopen=True: native streaming stays
# enabled so the post-answer continuation re-opens a fresh
# native stream (typing bubble) rather than degrading to a
# one-shot send(). Clarify waits are short, so stream staleness
# is a low risk; if the re-seed fails the consumer degrades to
# send() automatically. The placeholder is only used in the
# narrow case where a frame was pushed but no text accumulated.
_close_native_stream_boundary(
"Clarify", "💬 等待你的选择...", _reopen=True,
)
# Pause typing — like approval, we don't want a "thinking..."
# status to obscure the prompt or block the user from typing
# an "Other" response on platforms that disable input while
@@ -6019,12 +6094,45 @@ class TurnRunner:
# AMBIGUOUS — the card may have posted with a late ack. Only a
# definitive failure tears down the registration; ambiguous
# falls through to the bounded wait so a late reply resolves.
return _clarify_send_then_wait(
_clarify_response = _clarify_send_then_wait(
fut,
clarify_id=clarify_id,
session_key=ctx.session_key or "",
clarify_mod=_clarify_mod,
)
# Only re-arm typing when the user actually answered — the
# undeliverable sentinel and the timeout/cancellation strings
# start with '[' and must pass through untouched.
if not (
isinstance(_clarify_response, str)
and _clarify_response.startswith("[")
):
# User answered. Reopen the typing indicator IMMEDIATELY —
# don't wait for the LLM's first post-answer token. On native
# streaming (WeCom) the typing bubble is driven by the stream
# seed frame, and the reopen path otherwise re-seeds lazily on
# the first delta (measured ~48s of dead air). request_reopen_seed
# is a no-op unless we're in the reopen-pending native state, so
# it's safe to call unconditionally here. resume_typing_for_chat
# covers non-native platforms (their pause was set before the
# prompt) — the WeCom send_typing is a no-op, harmless here.
_sc_reopen = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None
if _sc_reopen is not None:
try:
_sc_reopen.request_reopen_seed()
except Exception:
logger.debug(
"request_reopen_seed after clarify answer failed",
exc_info=True,
)
try:
ctx._status_adapter.resume_typing_for_chat(ctx._status_chat_id)
except Exception:
logger.debug(
"resume_typing_for_chat after clarify answer failed",
exc_info=True,
)
return _clarify_response
agent.clarify_callback = _clarify_callback_sync
@@ -6127,6 +6235,12 @@ class TurnRunner:
# Typing resumes in _handle_approve_command/_handle_deny_command.
ctx._status_adapter.pause_typing_for_chat(ctx._status_chat_id)
# For WeCom native streaming: signal the stream consumer to close
# the current stream before showing the approval prompt.
# This goes through the consumer's queue for serial processing,
# avoiding race conditions with pending deltas.
_close_native_stream_boundary("Approval")
cmd = approval_data.get("command", "")
desc = approval_data.get("description", "dangerous command")
@@ -6200,11 +6314,15 @@ class TurnRunner:
smart_denied=approval_data.get("smart_denied", False),
)
try:
# Mark as approval prompt so WeCom routes through control lane
_approval_metadata = dict(ctx._status_thread_metadata or {})
_approval_metadata["is_approval_prompt"] = True
_approval_send_fut = safe_schedule_threadsafe(
ctx._status_adapter.send(
ctx._status_chat_id,
msg,
metadata=_interim_metadata(ctx._status_thread_metadata),
metadata=_interim_metadata(_approval_metadata),
),
ctx._loop_for_step,
logger=logger,
@@ -22768,6 +22886,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
asked to deliver (#20834). Only ``MEDIA:`` directives — the explicit
attachment contract — trigger post-stream uploads.
"""
logger.info("[DEBUG] _deliver_media_from_response called: response=%r chat=%s", response[:100], event.source.chat_id)
from pathlib import Path
from urllib.parse import quote as _quote
@@ -24342,7 +24461,6 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
_APPROVAL_TIMEOUT_SECONDS = 300 # 5 minutes
# Built-in messaging platforms where the ``/update`` command is allowed.
# ACP, API server, and webhooks are programmatic interfaces that should
# not trigger system updates. Plugin-migrated platforms (discord,
@@ -28131,7 +28249,18 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
# (The proxy path instead opts into a cursorless fallback
# via on_missing_cursor="fallback".)
_adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True)
if not _adapter_supports_edit and on_missing_cursor == "raise":
# Adapters that can't edit messages but provide a native-streaming
# transport (e.g. WeCom's msgtype: "stream" via send_stream_frame)
# get past the gate — the consumer's native branch delivers the full
# turn through that transport.
_adapter_supports_native_stream = bool(getattr(
adapter, "SUPPORTS_NATIVE_STREAMING", False,
))
if (
not _adapter_supports_edit
and not _adapter_supports_native_stream
and on_missing_cursor == "raise"
):
raise RuntimeError("skip streaming for non-editable platform")
_effective_cursor = scfg.cursor if _adapter_supports_edit else ""
# Some Matrix clients render the streaming cursor
@@ -30405,6 +30534,27 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
"Failed to edit streamed message for session %s: %s",
session_key or "?", _edit_err,
)
elif _sc is not None and not _is_empty_sentinel:
# DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed for this
# turn but suppression did NOT fire, so the gateway's normal
# final-send is about to run. On WeCom this is the exact window
# that produced "回复了两条" — a final-frame ack still in flight
# (final_content_delivered not yet set) while this send races
# ahead. Log the decision inputs so a recurrence can be pinned to
# "signal never set" vs "ack-pending race".
# See docs/rca-wecom-stream-final-ack-timeout-duplicate.md.
logger.warning(
"Normal final-send NOT suppressed despite active stream "
"consumer for session %s: streamed=%s previewed=%s "
"content_delivered=%s transformed=%s final_len=%d — "
"possible duplicate send (see wecom ack-timeout RCA).",
session_key or "?",
_streamed,
_previewed,
_content_delivered,
_transformed,
len(_final),
)
# Schedule deletion of tracked temporary progress bubbles after the
# final response lands. Failed runs skip this so bubbles remain as
+49 -6
View File
@@ -5885,7 +5885,27 @@ class GatewaySlashCommandsMixin:
logger.info("User approved %d dangerous command(s) via /approve (%s)", count, choice)
plural = "plural" if count > 1 else "singular"
return t(f"gateway.approve.{choice}_{plural}", count=count)
confirmation_text = t(f"gateway.approve.{choice}_{plural}", count=count)
if _adapter:
try:
await _adapter.send(
source.chat_id,
confirmation_text,
reply_to=event.message_id,
metadata={
"is_approval_prompt": True,
"force_proactive_send": True,
},
)
except Exception as exc:
logger.warning(
"Failed to send /approve confirmation to %s: %s",
source.chat_id,
exc,
exc_info=True,
)
return None
async def _handle_deny_command(self, event: MessageEvent) -> str:
"""Handle /deny command — reject pending dangerous command(s).
@@ -5943,11 +5963,34 @@ class GatewaySlashCommandsMixin:
)
if reason:
if count > 1:
return t("gateway.deny.denied_reason_plural", count=count, reason=reason)
return t("gateway.deny.denied_reason_singular", reason=reason)
if count > 1:
return t("gateway.deny.denied_plural", count=count)
return t("gateway.deny.denied_singular")
confirmation_text = t("gateway.deny.denied_reason_plural", count=count, reason=reason)
else:
confirmation_text = t("gateway.deny.denied_reason_singular", reason=reason)
elif count > 1:
confirmation_text = t("gateway.deny.denied_plural", count=count)
else:
confirmation_text = t("gateway.deny.denied_singular")
if _adapter:
try:
await _adapter.send(
source.chat_id,
confirmation_text,
reply_to=event.message_id,
metadata={
"is_approval_prompt": True,
"force_proactive_send": True,
},
)
except Exception as exc:
logger.warning(
"Failed to send /deny confirmation to %s: %s",
source.chat_id,
exc,
exc_info=True,
)
return None
async def _handle_debug_command(self, event: MessageEvent) -> str:
"""Handle /debug — upload debug report (summary only) and return paste URLs.
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+414
View File
@@ -0,0 +1,414 @@
"""Tests for approval boundary handling in WeCom native streaming."""
import asyncio
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig
@pytest.fixture
def mock_adapter():
"""Create a mock WeCom adapter with native streaming support."""
from gateway.platforms.base import BasePlatformAdapter
MockAdapter = type("MockAdapter", (BasePlatformAdapter,), {
"MAX_MESSAGE_LENGTH": 4096,
"SUPPORTS_MESSAGE_EDITING": False,
"SUPPORTS_NATIVE_STREAMING": True,
})
MockAdapter.__abstractmethods__ = frozenset()
adapter = MockAdapter.__new__(MockAdapter)
adapter._typing_paused = set()
adapter.send_stream_frame = AsyncMock(return_value=True)
adapter.send = AsyncMock(return_value=MagicMock(success=True, message_id="msg"))
adapter.supports_native_streaming = lambda chat_type=None, metadata=None: True
return adapter
@pytest.fixture
def consumer_config():
"""Create a minimal consumer config."""
return StreamConsumerConfig(
chat_type="dm", cursor="",
edit_interval=0.01, buffer_threshold=5,
)
@pytest.mark.asyncio
async def test_approval_boundary_finalizes_and_disables_native(mock_adapter, consumer_config):
"""Approval boundary must finalize the current stream (creating a stable
message for pre-approval content) and disable native streaming so
post-approval output goes through reliable send()."""
consumer = GatewayStreamConsumer(
adapter=mock_adapter,
chat_id="test_chat",
config=consumer_config,
)
consumer._use_native_streaming = True
consumer._native_stream_opened = True
consumer._turn_id = "turn_123"
consumer._initial_reply_to_id = "msg_456"
consumer._accumulated = "下面我来执行:先确认最新分区"
# Signal approval boundary
boundary_result = consumer.close_for_approval_prompt()
if isinstance(boundary_result, tuple):
boundary_future, _ = boundary_result
else:
boundary_future = boundary_result
consumer_task = asyncio.create_task(consumer.run())
result = await asyncio.wait_for(boundary_future, timeout=1.0)
assert result is True
# Stream must be finalized (stable message created)
finalize_calls = [
call for call in mock_adapter.send_stream_frame.call_args_list
if call.kwargs.get("finalize") is True
]
assert len(finalize_calls) == 1, "Must finalize the stream"
finalize_text = finalize_calls[0].args[0]
assert finalize_text == "下面我来执行:先确认最新分区"
# Native streaming must be disabled for post-approval output
assert consumer._use_native_streaming is False, (
"Native streaming must be disabled — post-approval goes via send()"
)
assert consumer._native_stream_opened is False
consumer.finish()
await asyncio.sleep(0.05)
consumer_task.cancel()
try:
await consumer_task
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_approval_boundary_uses_placeholder_when_no_accumulated(mock_adapter, consumer_config):
"""When there's no accumulated text, finalize with a visible placeholder."""
consumer = GatewayStreamConsumer(
adapter=mock_adapter,
chat_id="test_chat",
config=consumer_config,
)
consumer._use_native_streaming = True
consumer._native_stream_opened = True
consumer._turn_id = "turn_123"
consumer._accumulated = "" # No text accumulated
boundary_result = consumer.close_for_approval_prompt()
if isinstance(boundary_result, tuple):
boundary_future, _ = boundary_result
else:
boundary_future = boundary_result
consumer_task = asyncio.create_task(consumer.run())
result = await asyncio.wait_for(boundary_future, timeout=1.0)
assert result is True
finalize_calls = [
call for call in mock_adapter.send_stream_frame.call_args_list
if call.kwargs.get("finalize") is True
]
assert len(finalize_calls) == 1
finalize_text = finalize_calls[0].args[0]
assert finalize_text == "⏸ 等待审批中..."
consumer.finish()
await asyncio.sleep(0.05)
consumer_task.cancel()
try:
await consumer_task
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_boundary_uses_custom_placeholder_when_no_accumulated(mock_adapter, consumer_config):
"""A clarify boundary passes its own placeholder; the empty-content
finalize must use it instead of the approval wording so the finalized
bubble doesn't read 'waiting for approval' for a decision question."""
consumer = GatewayStreamConsumer(
adapter=mock_adapter,
chat_id="test_chat",
config=consumer_config,
)
consumer._use_native_streaming = True
consumer._native_stream_opened = True
consumer._turn_id = "turn_123"
consumer._accumulated = "" # No text accumulated
boundary_result = consumer.close_for_approval_prompt("💬 等待你的选择...")
if isinstance(boundary_result, tuple):
boundary_future, _ = boundary_result
else:
boundary_future = boundary_result
consumer_task = asyncio.create_task(consumer.run())
result = await asyncio.wait_for(boundary_future, timeout=1.0)
assert result is True
finalize_calls = [
call for call in mock_adapter.send_stream_frame.call_args_list
if call.kwargs.get("finalize") is True
]
assert len(finalize_calls) == 1
finalize_text = finalize_calls[0].args[0]
assert finalize_text == "💬 等待你的选择..."
# Native streaming disabled so post-answer output opens a fresh bubble.
assert consumer._use_native_streaming is False
assert consumer.cfg.buffer_only is True
consumer.finish()
await asyncio.sleep(0.05)
consumer_task.cancel()
try:
await consumer_task
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_clarify_boundary_logs_use_clarify_prefix(mock_adapter, consumer_config, caplog):
"""A clarify boundary that fails to finalize must log with a "Clarify"
prefix, not "Approval" — otherwise a clarify failure looks like a
dangerous-command approval failure during troubleshooting."""
import logging
# Make finalize fail so the warning path fires, and the fallback send fail
# too so the error path fires — exercising the reason-labelled logs.
mock_adapter.send_stream_frame = AsyncMock(return_value=False)
mock_adapter.send = AsyncMock(return_value=MagicMock(success=False))
consumer = GatewayStreamConsumer(
adapter=mock_adapter,
chat_id="test_chat",
config=consumer_config,
)
consumer._use_native_streaming = True
consumer._native_stream_opened = True
consumer._turn_id = "turn_123"
consumer._accumulated = "部分内容"
boundary_result = consumer.close_for_approval_prompt(
"💬 等待你的选择...", reason="Clarify",
)
if isinstance(boundary_result, tuple):
boundary_future, _ = boundary_result
else:
boundary_future = boundary_result
with caplog.at_level(logging.WARNING, logger="gateway.stream_consumer"):
consumer_task = asyncio.create_task(consumer.run())
await asyncio.wait_for(boundary_future, timeout=1.0)
boundary_logs = [r.getMessage() for r in caplog.records]
assert any("Clarify boundary" in m for m in boundary_logs), (
f"Expected a 'Clarify boundary' log, got: {boundary_logs}"
)
assert not any("Approval boundary" in m for m in boundary_logs), (
f"Clarify boundary must not log as 'Approval boundary': {boundary_logs}"
)
consumer.finish()
await asyncio.sleep(0.05)
consumer_task.cancel()
try:
await consumer_task
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_approval_boundary_post_approval_one_shot_send(mock_adapter, consumer_config):
"""After approval boundary, post-approval content must:
1. Set buffer_only=True (no mid-stream flushes)
2. Accumulate all deltas without sending
3. Deliver everything via one adapter.send() call on finish()"""
consumer = GatewayStreamConsumer(
adapter=mock_adapter,
chat_id="test_chat",
config=consumer_config,
)
consumer._use_native_streaming = True
consumer._native_stream_opened = True
consumer._turn_id = "turn_123"
consumer._accumulated = "Pre-approval text"
boundary_result = consumer.close_for_approval_prompt()
if isinstance(boundary_result, tuple):
boundary_future, _ = boundary_result
else:
boundary_future = boundary_result
consumer_task = asyncio.create_task(consumer.run())
await asyncio.wait_for(boundary_future, timeout=1.0)
# Verify buffer_only is set
assert consumer.cfg.buffer_only is True, "Must set buffer_only after boundary"
assert consumer._use_native_streaming is False
# Send post-approval content — should NOT trigger any immediate send
mock_adapter.send_stream_frame.reset_mock()
mock_adapter.send.reset_mock()
consumer.on_delta("Post-approval result text here")
await asyncio.sleep(0.1)
# Before finish(): no send() or stream frame calls
assert mock_adapter.send.call_count == 0, (
"buffer_only: no send before finish()"
)
stream_calls = [
call for call in mock_adapter.send_stream_frame.call_args_list
if not call.kwargs.get("finalize")
]
assert len(stream_calls) == 0, (
"Post-approval must NOT use native streaming"
)
# Now finish — should deliver via send()
consumer.finish()
await asyncio.sleep(0.1)
# adapter.send should have been called with the full post-approval text
send_calls = mock_adapter.send.call_args_list
assert len(send_calls) >= 1, "finish() must deliver via send()"
# The delivered text should contain our post-approval content
delivered = send_calls[-1].args[1] if len(send_calls[-1].args) > 1 else send_calls[-1].kwargs.get("content", "")
assert "Post-approval result" in delivered
consumer_task.cancel()
try:
await consumer_task
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_approval_boundary_stream_not_opened_at_boundary_time(mock_adapter, consumer_config):
"""When native streaming is active but _native_stream_opened is still False
at the time boundary processes (e.g., seed succeeded but stream was closed by
a prior error before boundary arrives), no finalize is sent."""
consumer = GatewayStreamConsumer(
adapter=mock_adapter,
chat_id="test_chat",
config=consumer_config,
)
# Let run() do its normal seed (which sets _native_stream_opened=True)
# Then manually close it before boundary processes
consumer._use_native_streaming = True
consumer._native_stream_opened = True
consumer._turn_id = "turn_123"
consumer._accumulated = "Some text"
# Queue boundary
boundary_result = consumer.close_for_approval_prompt()
if isinstance(boundary_result, tuple):
boundary_future, _ = boundary_result
else:
boundary_future = boundary_result
# Simulate: stream was closed by error BEFORE consumer processes boundary
consumer._native_stream_opened = False
consumer_task = asyncio.create_task(consumer.run())
result = await asyncio.wait_for(boundary_future, timeout=1.0)
assert result is True
# Native streaming should be disabled after boundary
assert consumer._use_native_streaming is False
consumer.finish()
await asyncio.sleep(0.05)
consumer_task.cancel()
try:
await consumer_task
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_approval_boundary_finalize_fails_fallback_send_succeeds(mock_adapter, consumer_config):
"""When stream finalize fails but fallback send() succeeds, boundary is True."""
consumer = GatewayStreamConsumer(
adapter=mock_adapter,
chat_id="test_chat",
config=consumer_config,
)
consumer._use_native_streaming = True
consumer._native_stream_opened = True
consumer._turn_id = "turn_123"
consumer._accumulated = "Pre-approval text"
# Finalize fails (returns False)
mock_adapter.send_stream_frame = AsyncMock(return_value=False)
# Fallback send succeeds
mock_adapter.send = AsyncMock(return_value=MagicMock(success=True, message_id="msg"))
boundary_result = consumer.close_for_approval_prompt()
if isinstance(boundary_result, tuple):
boundary_future, _ = boundary_result
else:
boundary_future = boundary_result
consumer_task = asyncio.create_task(consumer.run())
result = await asyncio.wait_for(boundary_future, timeout=1.0)
assert result is True, "Fallback send succeeded → boundary should be True"
mock_adapter.send.assert_awaited_once_with("test_chat", "Pre-approval text")
consumer.finish()
await asyncio.sleep(0.05)
consumer_task.cancel()
try:
await consumer_task
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_approval_boundary_finalize_and_fallback_both_fail(mock_adapter, consumer_config):
"""When both stream finalize and fallback send() fail, boundary is False."""
consumer = GatewayStreamConsumer(
adapter=mock_adapter,
chat_id="test_chat",
config=consumer_config,
)
consumer._use_native_streaming = True
consumer._native_stream_opened = True
consumer._turn_id = "turn_123"
consumer._accumulated = "Pre-approval text"
# Finalize fails (raises)
mock_adapter.send_stream_frame = AsyncMock(side_effect=RuntimeError("stream dead"))
# Fallback send also fails
mock_adapter.send = AsyncMock(return_value=MagicMock(success=False, error="timeout"))
boundary_result = consumer.close_for_approval_prompt()
if isinstance(boundary_result, tuple):
boundary_future, _ = boundary_result
else:
boundary_future = boundary_result
consumer_task = asyncio.create_task(consumer.run())
result = await asyncio.wait_for(boundary_future, timeout=1.0)
assert result is False, "Both finalize and fallback failed → boundary should be False"
consumer.finish()
await asyncio.sleep(0.05)
consumer_task.cancel()
try:
await consumer_task
except asyncio.CancelledError:
pass
@@ -0,0 +1,313 @@
"""Tests for tool-progress-in-native-stream (single bubble) feature.
Validates that tool-progress lines are injected into the native streaming
bubble and properly overwritten by text deltas (Strategy B).
"""
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from gateway.stream_consumer import (
GatewayStreamConsumer,
StreamConsumerConfig,
_TOOL_PROGRESS,
)
def _make_native_streaming_adapter(*, supports_native: bool = True):
"""Build a BasePlatformAdapter subclass that supports native streaming."""
from gateway.platforms.base import BasePlatformAdapter
NativeStreamingAdapter = type(
"NativeStreamingAdapter",
(BasePlatformAdapter,),
{
"MAX_MESSAGE_LENGTH": 4096,
"SUPPORTS_MESSAGE_EDITING": False,
"SUPPORTS_NATIVE_STREAMING": True,
},
)
NativeStreamingAdapter.__abstractmethods__ = frozenset()
adapter = NativeStreamingAdapter.__new__(NativeStreamingAdapter)
adapter._typing_paused = set()
adapter._fatal_error_message = None
adapter.frames = []
def _supports(chat_type=None, metadata=None):
return bool(supports_native)
adapter.supports_native_streaming = _supports
async def _send_stream_frame(
text, *, finalize=False, chat_id=None, reply_to=None, **kwargs
):
adapter.frames.append({
"text": text,
"finalize": finalize,
"chat_id": chat_id,
})
return True
adapter.send_stream_frame = _send_stream_frame
adapter.send = AsyncMock(
return_value=SimpleNamespace(success=True, message_id="fallback_msg"),
)
adapter.edit_message = AsyncMock(
return_value=SimpleNamespace(success=True),
)
return adapter
def _make_consumer(*, native_streaming: bool = True) -> GatewayStreamConsumer:
"""Create a GatewayStreamConsumer configured for native streaming."""
adapter = _make_native_streaming_adapter(supports_native=native_streaming)
cfg = StreamConsumerConfig(chat_type="dm", cursor="▌")
consumer = GatewayStreamConsumer(adapter, "chat-1", cfg)
# Force native streaming resolution
consumer._use_native_streaming = native_streaming
return consumer
# === UNIT TESTS ===
class TestAcceptsToolProgress:
"""Tests for the accepts_tool_progress property."""
def test_native_streaming_accepts(self):
consumer = _make_consumer(native_streaming=True)
assert consumer.accepts_tool_progress is True
def test_non_native_does_not_accept(self):
consumer = _make_consumer(native_streaming=False)
assert consumer.accepts_tool_progress is False
class TestOnToolProgress:
"""Tests for on_tool_progress() enqueue behavior."""
def test_enqueues_sentinel(self):
consumer = _make_consumer()
consumer.on_tool_progress("🔍 Searching...")
item = consumer._queue.get_nowait()
assert isinstance(item, tuple)
assert len(item) == 2
assert item[0] is _TOOL_PROGRESS
assert item[1] == "🔍 Searching..."
def test_empty_line_not_enqueued(self):
consumer = _make_consumer()
consumer.on_tool_progress("")
assert consumer._queue.empty()
class TestComposeFrameContent:
"""Tests for _compose_frame_content() composition logic (Strategy B)."""
def test_only_tool_lines(self):
consumer = _make_consumer()
consumer._tool_progress_lines = ["🔍 Searching...", "💻 Running git log"]
result = consumer._compose_frame_content()
assert result == "🔍 Searching...\n💻 Running git log"
def test_only_accumulated(self):
consumer = _make_consumer()
consumer._accumulated = "Here is the answer."
result = consumer._compose_frame_content()
assert result == "Here is the answer."
def test_both_accumulated_and_tool_lines_strategy_b(self):
"""Strategy B: text + separator + tool status at bottom."""
consumer = _make_consumer()
consumer._accumulated = "Here is some text so far."
consumer._tool_progress_lines = ["🔍 Searching the web..."]
result = consumer._compose_frame_content()
assert result == "Here is some text so far.\n\n---\n🔍 Searching the web..."
def test_multiple_tool_lines_stacked(self):
consumer = _make_consumer()
consumer._tool_progress_lines = [
"🔍 web_search: 'python'",
"💻 terminal: git log",
"📄 read_file: main.py",
]
result = consumer._compose_frame_content()
assert "web_search" in result
assert "terminal" in result
assert "read_file" in result
# Lines are joined with newlines
assert result.count("\n") == 2
def test_empty_state(self):
consumer = _make_consumer()
result = consumer._compose_frame_content()
assert result == ""
class TestSegmentReset:
"""Test that segment reset clears tool progress state."""
def test_reset_clears_tool_progress(self):
consumer = _make_consumer()
consumer._tool_progress_lines = ["🔍 Searching..."]
consumer._tool_progress_active = True
consumer._reset_segment_state()
assert consumer._tool_progress_lines == []
assert consumer._tool_progress_active is False
# === INTEGRATION TESTS (drain loop) ===
class TestToolProgressDrainLoop:
"""Integration tests for the drain loop + frame delivery."""
@pytest.mark.asyncio
async def test_tool_progress_only_then_done(self):
"""Pure tool-progress turn (no text): tool lines visible as mid-frame,
finalize uses placeholder since no accumulated text."""
consumer = _make_consumer()
consumer.on_tool_progress("🔍 Searching...")
consumer.on_tool_progress("💻 terminal: ls")
# Start consumer so it drains tool progress and sends mid-frames
task = asyncio.create_task(consumer.run())
await asyncio.sleep(0.3)
# Now finish — tool lines were already displayed as a frame
consumer.finish()
await asyncio.sleep(0.3)
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
frames = consumer.adapter.frames
# Should have: seed frame, at least one mid-frame with tool lines,
# and a finalize frame (✅ placeholder since no text)
assert len(frames) >= 2
# Find mid-frames that contain tool progress
non_finalize = [f for f in frames if not f["finalize"] and f["text"]]
assert any("Searching" in f["text"] or "terminal" in f["text"] for f in non_finalize), (
f"Expected tool progress in mid-frames, got: {[f['text'] for f in frames]}"
)
@pytest.mark.asyncio
async def test_tool_progress_then_text_clears_overlay(self):
"""Tool progress → text delta should clear tool lines from frame."""
consumer = _make_consumer()
consumer.on_tool_progress("🔍 Searching...")
consumer.on_delta("Hello world")
consumer.finish()
task = asyncio.create_task(consumer.run())
await asyncio.sleep(0.5)
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
# After text arrives, tool_progress_lines should be cleared
assert consumer._tool_progress_lines == []
assert "Hello world" in consumer._accumulated
# The finalize frame should contain just the text
frames = consumer.adapter.frames
finalize_frames = [f for f in frames if f["finalize"]]
if finalize_frames:
assert "Hello world" in finalize_frames[-1]["text"]
assert "Searching" not in finalize_frames[-1]["text"]
@pytest.mark.asyncio
async def test_text_then_tool_then_text_strategy_b(self):
"""Strategy B: text → tool → text appends tool at bottom then clears."""
consumer = _make_consumer()
# Phase 1: initial text
consumer.on_delta("First part. ")
# Phase 2: tool progress mid-stream
consumer.on_tool_progress("🔍 web_search...")
# Phase 3: more text arrives
consumer.on_delta("Second part.")
consumer.finish()
task = asyncio.create_task(consumer.run())
await asyncio.sleep(0.5)
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
# Final state: tool lines cleared, accumulated has both text parts
assert consumer._tool_progress_lines == []
assert "First part." in consumer._accumulated
assert "Second part." in consumer._accumulated
@pytest.mark.asyncio
async def test_parallel_tool_calls_stacked(self):
"""Multiple tool.started back-to-back should stack in overlay."""
consumer = _make_consumer()
consumer.on_tool_progress("🔍 web_search")
consumer.on_tool_progress("💻 terminal")
consumer.on_tool_progress("📄 read_file")
# Let drain loop process and send mid-frame before finish
task = asyncio.create_task(consumer.run())
await asyncio.sleep(0.3)
consumer.finish()
await asyncio.sleep(0.3)
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
# All three should have been accumulated and sent in one frame
frames = consumer.adapter.frames
# Find a frame that contains all three tools
all_three = [
f for f in frames
if "web_search" in f["text"]
and "terminal" in f["text"]
and "read_file" in f["text"]
]
assert len(all_three) >= 1, (
f"Expected a frame with all 3 tools, got: {[f['text'] for f in frames]}"
)
@pytest.mark.asyncio
async def test_finalize_frame_is_pure_text(self):
"""The finalize frame must only contain accumulated text, no tool lines."""
consumer = _make_consumer()
consumer.on_tool_progress("🔍 Searching...")
consumer.on_delta("The answer is 42.")
# Add a tool progress AFTER text (Strategy B scenario)
consumer.on_tool_progress("💻 terminal: verify")
# Then more text clears it
consumer.on_delta(" Verified.")
consumer.finish()
task = asyncio.create_task(consumer.run())
await asyncio.sleep(0.5)
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
frames = consumer.adapter.frames
finalize_frames = [f for f in frames if f["finalize"]]
if finalize_frames:
final_text = finalize_frames[-1]["text"]
assert "The answer is 42. Verified." in final_text
assert "Searching" not in final_text
assert "terminal" not in final_text
assert "---" not in final_text
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+534
View File
@@ -0,0 +1,534 @@
"""Repro tests for the WeCom native-streaming "同一回复发两条" duplicate.
Root cause (see docs/rca-wecom-stream-final-ack-timeout-duplicate.md and
/tmp/claude-wecom-dup-report.md): a **timeout inversion race** between two
independent timers.
* The consumer's got_done finalize blocks on the finalize frame's ack,
bounded by ``adapter._REPLY_ACK_TIMEOUT``.
* The gateway's ``finally`` block joins the ``stream_task`` with a hardcoded
``timeout=5.0`` (``gateway/run.py:20749``) and then ``stream_task.cancel()``
(``run.py:20751``).
When the ack is slower than the gateway's join window, the gateway cancels the
consumer *before* it finishes finalizing. The finalize frame's bytes, however,
were already written to the wire by an **independent control-worker task**
(``_control_send_worker`` / ``_enqueue_chat_send(is_control=True)``) and WeCom
has rendered them. But because the consumer was cancelled mid-await, the
``self._final_content_delivered = True`` line (``stream_consumer.py:2061`` /
``1016``) never runs. The gateway then reads ``final_content_delivered=False``
(``run.py:20801-20803``), does NOT suppress the normal final send
(``run.py:20819``), and emits a second, duplicate bubble.
These tests exercise the **real** ``GatewayStreamConsumer.run()`` lifecycle
against the **real** ``WeComAdapter`` (only the websocket byte-writer and the
ack timing are controlled), so the async interaction between the gateway join /
cancel and the consumer finalize / flag-set actually happens — this is what the
previous PR (#62861) failed to test when it mocked out ``handle_message``.
Assertions target observable causality:
* whether the finalize frame's bytes reached the wire (WeCom rendered it);
* the value of ``consumer.final_content_delivered`` (the flag the gateway
reads);
* whether the gateway's suppression predicate fires (i.e. whether a second
normal send would go out).
The suppression predicate below is a faithful mirror of the (non-importable,
nested) logic in ``gateway/run.py:20793-20819`` — it reads the *real* consumer
properties, it does not re-implement the delivery lifecycle.
"""
from __future__ import annotations
import asyncio
from unittest.mock import MagicMock
import pytest
from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig
CHAT_ID = "chat-1"
REQ_ID = "req-1"
# ---------------------------------------------------------------------------
# Faithful mirrors of the real gateway control flow (cite source lines).
# ---------------------------------------------------------------------------
def _gateway_suppresses_normal_send(consumer, final_text: str) -> bool:
"""Mirror of gateway/run.py:20793-20819 suppression decision.
Reads the REAL consumer properties (``final_response_sent`` /
``final_content_delivered``) — the same attributes ``run.py`` consults —
and returns True when the gateway would set ``already_sent`` and skip the
normal final send. When it returns False the gateway emits a second bubble.
"""
_final = final_text or ""
_is_empty_sentinel = not _final or _final == "(empty)"
# run.py:20814 _stream_confirmed_final_delivery(...) with previewed=False
# collapses to final_response_sent for a non-previewed response.
_streamed = bool(getattr(consumer, "final_response_sent", False))
_content_delivered = bool(getattr(consumer, "final_content_delivered", False))
_transformed = False # no plugin transform in these scenarios
return (
not _is_empty_sentinel
and not _transformed
and (_streamed or _content_delivered)
)
async def _gateway_join_and_cancel(stream_task: asyncio.Task, join_timeout: float) -> None:
"""Faithful copy of gateway/run.py:20748-20755 finally-block join.
Joins the stream task with a bounded timeout; on timeout it cancels the
task (exactly what the gateway does when the consumer has not finished
finalizing within the join window).
"""
try:
await asyncio.wait_for(stream_task, timeout=join_timeout)
except (asyncio.TimeoutError, asyncio.CancelledError):
stream_task.cancel()
try:
await stream_task
except asyncio.CancelledError:
pass
# ---------------------------------------------------------------------------
# Real WeComAdapter with a controllable websocket + ack timing.
# ---------------------------------------------------------------------------
def _make_real_wecom_adapter(*, resolve_finalize_ack: bool):
"""Build a real ``WeComAdapter`` whose only fakes are the WS byte-writer
and the ack timing.
``_send_json`` (the actual byte-writer) records every frame so we can prove
the finalize frame reached the wire. The ack for **non-final** frames (seed
/ intermediate) is always resolved immediately so the finalize's
pre-drain (``_send_reply_queued`` is_final branch) doesn't block on the
seed's ack. The finalize frame's own ack is resolved immediately only when
``resolve_finalize_ack`` is True.
* ``resolve_finalize_ack=False`` → finalize ack stays pending → the
consumer blocks in its got_done finalize (the exact window where the
gateway's join fires and cancels it). This is the bug scenario.
* ``resolve_finalize_ack=True`` → finalize ack returns at once → the
consumer finishes finalize and sets its flags before any join fires.
"""
from plugins.platforms.wecom.adapter import WeComAdapter
from gateway.config import PlatformConfig
adapter = WeComAdapter(PlatformConfig(enabled=True))
adapter._ws = MagicMock(closed=False)
adapter._last_chat_req_ids[CHAT_ID] = REQ_ID
frames: list[dict] = []
async def _fake_send_json(payload: dict) -> None:
# This is the real byte-writer boundary: reaching here means the frame
# was put on the wire and WeCom will render it.
frames.append(payload)
stream = payload.get("body", {}).get("stream", {})
finish = bool(stream.get("finish"))
req = payload.get("headers", {}).get("req_id")
should_resolve = (not finish) or resolve_finalize_ack
if should_resolve:
queue = adapter._reply_queues.get(req)
if queue and queue.pending_ack and not queue.pending_ack.future.done():
# Simulate WeCom's ack coming back on the same WS.
queue.pending_ack.future.set_result({"body": {"errcode": 0}})
adapter._send_json = _fake_send_json
adapter._recorded_frames = frames
return adapter
def _finalize_frames_on_wire(adapter) -> list[dict]:
"""Return the finish=true stream frames that reached the byte-writer."""
out = []
for payload in adapter._recorded_frames:
stream = payload.get("body", {}).get("stream", {})
if stream.get("finish") is True:
out.append(payload)
return out
async def _cleanup_adapter(adapter) -> None:
"""Cancel any lingering control/normal workers so the event loop is clean."""
for task in list(adapter._control_workers.values()) + list(adapter._chat_workers.values()):
task.cancel()
try:
await task
except (asyncio.CancelledError, Exception):
pass
# ===========================================================================
# Test group 1: timeout-inversion race (core reproduction)
# ===========================================================================
class TestTimeoutInversionDoubleSend:
"""The finalize ack being slower than the gateway join window must not
strand ``final_content_delivered=False`` while WeCom already rendered the
finalize frame — that produces a duplicate bubble."""
@pytest.mark.asyncio
async def test_slow_ack_beyond_gateway_join_causes_double_send(self):
"""ack slower than the gateway join → consumer cancelled mid-finalize.
Timeline (deterministic, no wall-clock sleeps):
* seed + finalize frame bytes are written to the wire;
* the finalize ack never returns within the join window
(``resolve_finalize_ack=False``, ``_REPLY_ACK_TIMEOUT`` > join);
* the gateway join (0.1s) fires and cancels the consumer;
* the consumer's got_done finalize await is cancelled BEFORE the
``final_content_delivered = True`` line runs.
The finalize frame is on the wire (rendered), yet the gateway reads
``final_content_delivered=False`` and would send a normal duplicate.
This test asserts the DESIRED post-fix state, so it FAILS today
(exposing the double send) and PASSES once the flag reflects the
rendered finalize frame.
"""
adapter = _make_real_wecom_adapter(resolve_finalize_ack=False)
# Inversion: ack window (0.3s) is LARGER than the gateway join (0.1s),
# so the join fires while the finalize ack is still pending.
adapter._REPLY_ACK_TIMEOUT = 0.3
join_timeout = 0.1
cfg = StreamConsumerConfig(
chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5,
)
consumer = GatewayStreamConsumer(adapter, CHAT_ID, cfg)
final_text = "这是模型这一轮生成的最终回答,需要作为流式 finalize 帧发出。"
consumer.on_delta(final_text)
consumer.finish()
stream_task = asyncio.create_task(consumer.run())
try:
# Gateway finally-block join (run.py:20748-20755) fires while the
# finalize ack is still pending, cancelling the consumer.
await _gateway_join_and_cancel(stream_task, join_timeout)
finalize_frames = _finalize_frames_on_wire(adapter)
# The finalize frame reached the wire → WeCom rendered it. This is
# the first (and should be ONLY) user-visible bubble.
assert len(finalize_frames) >= 1, (
"finalize frame never reached the wire — repro precondition "
"not met (WeCom must have rendered the streamed final)"
)
assert final_text in finalize_frames[-1]["body"]["stream"]["content"]
gateway_would_send_normal = not _gateway_suppresses_normal_send(
consumer, final_text
)
total_user_visible = len(finalize_frames) + (
1 if gateway_would_send_normal else 0
)
# DESIRED post-fix invariants (currently violated == bug reproduced):
assert consumer.final_content_delivered is True, (
"BUG: consumer was cancelled mid-finalize; the finalize frame "
"was rendered by WeCom but final_content_delivered stayed False, "
"so the gateway will not suppress the normal send"
)
assert _gateway_suppresses_normal_send(consumer, final_text) is True, (
"BUG: gateway does not suppress the normal final send → duplicate"
)
assert total_user_visible == 1, (
f"BUG: user sees {total_user_visible} bubbles for one reply "
f"(finalize frame rendered + normal send fired)"
)
finally:
await _cleanup_adapter(adapter)
@pytest.mark.asyncio
async def test_fast_ack_within_join_window_single_send(self):
"""Control: ack returns within the join window → single bubble.
The finalize ack resolves immediately, so the consumer completes its
got_done finalize, sets ``final_content_delivered=True``, and the run
finishes before the gateway join needs to cancel anything. The gateway
then suppresses the normal send. Passes today and after the fix
(regression guard).
"""
adapter = _make_real_wecom_adapter(resolve_finalize_ack=True)
adapter._REPLY_ACK_TIMEOUT = 5.0
join_timeout = 0.5
cfg = StreamConsumerConfig(
chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5,
)
consumer = GatewayStreamConsumer(adapter, CHAT_ID, cfg)
final_text = "这是一个能在 join 窗口内正常拿到 ack 的回答。"
consumer.on_delta(final_text)
consumer.finish()
stream_task = asyncio.create_task(consumer.run())
try:
await _gateway_join_and_cancel(stream_task, join_timeout)
finalize_frames = _finalize_frames_on_wire(adapter)
assert len(finalize_frames) == 1
assert consumer.final_content_delivered is True
assert _gateway_suppresses_normal_send(consumer, final_text) is True
gateway_would_send_normal = not _gateway_suppresses_normal_send(
consumer, final_text
)
total_user_visible = len(finalize_frames) + (
1 if gateway_would_send_normal else 0
)
assert total_user_visible == 1
finally:
await _cleanup_adapter(adapter)
# ===========================================================================
# Test group 2: best-effort finalize "DO NOT mark" path
# ===========================================================================
class TestBestEffortFinalizeDoubleSend:
"""stream_consumer.py:2064-2094: when native streaming fails mid-turn, the
consumer sends a best-effort finalize frame to close the bubble but
deliberately does NOT set ``final_content_delivered`` (the "DO NOT mark"
comment at 2085-2088). If WeCom actually renders that finalize frame, the
subsequent fallback send delivers the same content a second time.
This uses a controllable ``BasePlatformAdapter`` subclass (the accepted
pattern in tests/gateway/test_wecom_per_turn.py) because the trigger is a
frame-level failure, not an ack timing race — no wall-clock timing needed.
"""
@pytest.mark.xfail(
reason="KNOWN LOW-FREQUENCY ISSUE (accepted, not yet fixed): the "
"best-effort finalize path (stream_consumer.py:2104-2128) renders the "
"content AND falls through to a fallback send(), delivering the same "
"answer twice. Only triggers when native streaming fails MID-STREAM "
"(846608 expired / 846609 subscription lost / errcode 6000 / network "
"error) — rare. The duplicate is a SAFE-direction failure (user sees "
"one extra bubble, no data loss). A clean fix requires reconciling "
"'do not drop post-finalize increments' with 'do not re-send the "
"first-send path', which the first-send path (L2378) does not consult "
"the delivery flag for; deferred over risking increment loss on a rare "
"path. The high-frequency duplicate (timeout inversion) IS fixed — see "
"TestTimeoutInversionDoubleSend + commit 7ba739818 (B2).",
strict=True,
)
@pytest.mark.asyncio
async def test_best_effort_finalize_renders_but_flag_unset_double_send(self):
"""native content frame fails → best-effort finalize renders + fallback send.
Ordering matters: the delta is pushed and ``run()`` is allowed to
process a mid-stream **content** frame BEFORE ``finish()`` is queued.
That content frame fails (returns False), which disables native
streaming and triggers the best-effort finalize at
``stream_consumer.py:2077`` — WeCom renders it — and then the consumer
falls through to a fallback ``send()`` that delivers the same content a
second time (the "DO NOT mark" comment at 2085-2088 is what leaves the
gateway free to re-send).
The text is > ``_MIN_NEW_VISIBLE_CHARS`` (60) so the mid-stream frame is
actually attempted rather than buffered.
Asserts the DESIRED single-delivery state, so it FAILS today (content
rendered by the best-effort finalize AND re-sent by the fallback) and
PASSES once the double delivery is closed.
"""
from gateway.platforms.base import BasePlatformAdapter, SendResult
class FlakyNativeAdapter(BasePlatformAdapter):
MAX_MESSAGE_LENGTH = 4096
SUPPORTS_MESSAGE_EDITING = False
SUPPORTS_NATIVE_STREAMING = True
def __init__(self):
self._typing_paused = set()
self._fatal_error_message = None
# Records of everything that reached the user.
self.rendered_stream_frames: list[dict] = [] # incl. finalize
self.fallback_sends: list[str] = []
self._content_frames_seen = 0
def supports_native_streaming(self, chat_type=None, metadata=None):
return True
async def send_stream_frame(
self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs
):
# Seed frame (empty, non-finalize) opens the bubble.
if text == "" and not finalize:
self.rendered_stream_frames.append({"text": text, "finalize": False})
return True
if finalize:
# Best-effort finalize frame — WeCom DOES render it.
self.rendered_stream_frames.append({"text": text, "finalize": True})
return True
# First real content frame fails → disables native streaming
# and triggers the best-effort finalize path.
self._content_frames_seen += 1
return False
async def send(self, chat_id, content, reply_to=None, metadata=None):
# The gateway/consumer fallback send — a SECOND delivery of the
# same content.
self.fallback_sends.append(content)
return SendResult(success=True, message_id="fallback-msg")
FlakyNativeAdapter.__abstractmethods__ = frozenset()
adapter = FlakyNativeAdapter()
cfg = StreamConsumerConfig(
chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5,
)
consumer = GatewayStreamConsumer(adapter, CHAT_ID, cfg)
# > 60 visible chars so the mid-stream content frame is actually sent.
final_text = (
"需要交付给用户的最终答案。这一段内容刻意写得足够长,"
"以便越过 stream_consumer 的 _MIN_NEW_VISIBLE_CHARS=60 阈值,"
"从而在 got_done 之前触发一次真正的流式内容帧发送。"
)
assert len(final_text) >= 60
consumer.on_delta(final_text)
task = asyncio.create_task(consumer.run())
# Let run() process the mid-stream content frame (which fails) BEFORE
# the turn-final _DONE arrives.
await asyncio.sleep(0.1)
consumer.finish()
await task
best_effort_finalize = [
f for f in adapter.rendered_stream_frames if f["finalize"] and f["text"]
]
# Count content-bearing user-visible deliveries:
# * best-effort finalize frame(s) that WeCom rendered, plus
# * fallback send(s).
content_deliveries = len(best_effort_finalize) + len(adapter.fallback_sends)
# DESIRED post-fix invariant (currently violated == bug reproduced):
# the same answer must reach the user exactly once. Today the
# best-effort finalize renders the content AND the fallback send
# delivers it again.
assert content_deliveries == 1, (
f"BUG: content delivered {content_deliveries} times — best-effort "
f"finalize rendered {len(best_effort_finalize)} frame(s) and "
f"fallback send fired {len(adapter.fallback_sends)} time(s) "
f"(stream_consumer.py:2085-2088 'DO NOT mark' path)"
)
# ===========================================================================
# Test group 3: orphan-queue ack routing race (adapter-level)
# ===========================================================================
def _make_manual_ack_adapter():
"""Real WeComAdapter whose ``_send_json`` records frames but NEVER
auto-resolves any ack, so the test can orchestrate the exact interleaving
of intermediate-ack arrival vs. finalize registration by hand.
"""
from plugins.platforms.wecom.adapter import WeComAdapter
from gateway.config import PlatformConfig
adapter = WeComAdapter(PlatformConfig(enabled=True))
adapter._ws = MagicMock(closed=False)
adapter._last_chat_req_ids[CHAT_ID] = REQ_ID
frames: list[dict] = []
async def _fake_send_json(payload: dict) -> None:
frames.append(payload)
adapter._send_json = _fake_send_json
adapter._recorded_frames = frames
return adapter
class TestOrphanQueueAckRouting:
"""The finalize frame shares the inbound req_id with the intermediate
frames. While the finalize awaits the pending intermediate ack to drain,
that intermediate ack can arrive and ``_resolve_reply_ack`` pops the WHOLE
queue out of ``_reply_queues``. Before the fix, the finalize then
registered its ``pending_ack`` on the now-orphaned queue object (detached
from the dict), so its own ack landed in Unrouted and the finalize hit a
15s timeout. The fix re-attaches the queue to the dict before registering.
"""
@pytest.mark.asyncio
async def test_intermediate_ack_mid_drain_does_not_orphan_finalize_queue(self):
adapter = _make_manual_ack_adapter()
adapter._REPLY_ACK_TIMEOUT = 0.3 # snappy: bounds the failure mode
# 1) Fire an intermediate frame (skip_if_pending) — occupies pending_ack.
await adapter._send_reply_queued(
REQ_ID,
{"msgtype": "stream", "stream": {"id": "s1", "finish": False, "content": "hi"}},
is_final=False, skip_if_pending=True,
)
inter_queue = adapter._reply_queues[REQ_ID]
assert inter_queue.pending_ack is not None
# 2) Start the finalize: it enters the is_final drain branch and yields
# at `await shield(pending_frame.future)`.
fin_task = asyncio.create_task(adapter._send_reply_queued(
REQ_ID,
{"msgtype": "stream", "stream": {"id": "s1", "finish": True, "content": "done"}},
is_final=True, skip_if_pending=False,
))
await asyncio.sleep(0) # let finalize reach the drain await
# 3) Deliver the intermediate ack MID-DRAIN. _resolve_reply_ack resolves
# the drain future AND pops the whole queue out of _reply_queues.
await adapter._dispatch_payload(
{"headers": {"req_id": REQ_ID}, "body": {"errcode": 0}}
)
assert REQ_ID not in adapter._reply_queues, (
"precondition: intermediate ack should have popped the queue"
)
# 4) Let finalize resume: it must clear the drained pending_ack, create
# its own frame, re-attach the queue, register pending_ack, and write
# bytes. Resuming through the shield/wait_for wrapper takes several
# event-loop iterations, so yield until the finalize has re-registered
# (bounded, to avoid a hang if the fix regresses).
for _ in range(20):
await asyncio.sleep(0)
q = adapter._reply_queues.get(REQ_ID)
if q is not None and q.pending_ack is not None:
break
# FIX ASSERTION: the finalize must have re-attached its queue to the
# dict, so its ack is routable. Without the fix the queue would still
# be absent here (orphaned) and the finalize ack would go Unrouted.
assert REQ_ID in adapter._reply_queues, (
"orphan-queue bug: finalize registered pending_ack on a queue "
"detached from _reply_queues — its ack would be Unrouted"
)
assert adapter._reply_queues[REQ_ID].pending_ack is not None
# 5) Deliver the finalize's own ack — it must route and resolve.
await adapter._dispatch_payload(
{"headers": {"req_id": REQ_ID}, "body": {"errcode": 0}}
)
resp = await asyncio.wait_for(fin_task, timeout=1.0)
# Routed successfully → NOT the synthetic timeout-assumed response.
assert resp.get("errmsg") != "ack_timeout_assumed_delivered", (
"finalize ack should have been routed, not timed out"
)
await _cleanup_adapter(adapter)
+403
View File
@@ -0,0 +1,403 @@
"""Tests for per-turn stream isolation and concurrent consumer scenarios."""
import pytest
from unittest.mock import AsyncMock, MagicMock
from gateway.config import PlatformConfig
class TestPerTurnStreamIsolation:
"""Verify that concurrent consumers with different turn_ids don't interfere."""
@pytest.mark.asyncio
async def test_multiple_users_concurrent_streaming(self):
"""Multiple users (different chats) streaming concurrently don't interfere."""
from plugins.platforms.wecom.adapter import WeComAdapter
adapter = WeComAdapter(PlatformConfig(enabled=True))
try:
# Setup 3 different users/chats
adapter._last_chat_req_ids["user-1"] = "req-1"
adapter._last_chat_req_ids["user-2"] = "req-2"
adapter._last_chat_req_ids["user-3"] = "req-3"
adapter._send_json = AsyncMock()
adapter._ws = AsyncMock(closed=False)
adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0})
# User 1, 2, 3 all start streaming simultaneously
await adapter.send_stream_frame("user1 content", chat_id="user-1", turn_id="turn-1")
await adapter.send_stream_frame("user2 content", chat_id="user-2", turn_id="turn-2")
await adapter.send_stream_frame("user3 content", chat_id="user-3", turn_id="turn-3")
# All 3 turns active
assert "user-1:turn-1" in adapter._stream_turns
assert "user-2:turn-2" in adapter._stream_turns
assert "user-3:turn-3" in adapter._stream_turns
# User 2 finishes first
ok2 = await adapter.send_stream_frame(
"user2 final", chat_id="user-2", finalize=True, turn_id="turn-2"
)
assert ok2 is True
assert "user-2:turn-2" not in adapter._stream_turns
# User 1 and 3 still active
assert "user-1:turn-1" in adapter._stream_turns
assert "user-3:turn-3" in adapter._stream_turns
# User 1 finishes
ok1 = await adapter.send_stream_frame(
"user1 final", chat_id="user-1", finalize=True, turn_id="turn-1"
)
assert ok1 is True
assert "user-1:turn-1" not in adapter._stream_turns
# User 3 still active
assert "user-3:turn-3" in adapter._stream_turns
# User 3 finishes
ok3 = await adapter.send_stream_frame(
"user3 final", chat_id="user-3", finalize=True, turn_id="turn-3"
)
assert ok3 is True
assert "user-3:turn-3" not in adapter._stream_turns
finally:
await adapter.disconnect()
@pytest.mark.asyncio
async def test_concurrent_turns_same_chat_isolated(self):
"""Two concurrent consumers in same chat maintain independent streams."""
from plugins.platforms.wecom.adapter import WeComAdapter
adapter = WeComAdapter(PlatformConfig(enabled=True))
try:
adapter._last_chat_req_ids["chat-1"] = "req-1"
adapter._send_json = AsyncMock()
# Mock _ws with closed=False and async close()
adapter._ws = AsyncMock(closed=False)
adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0})
# Consumer 1 starts streaming
await adapter.send_stream_frame("consumer1 frame1", chat_id="chat-1", turn_id="turn-1")
assert "chat-1:turn-1" in adapter._stream_turns
# Consumer 2 starts streaming (concurrent)
await adapter.send_stream_frame("consumer2 frame1", chat_id="chat-1", turn_id="turn-2")
assert "chat-1:turn-2" in adapter._stream_turns
# Both turns coexist
assert len([k for k in adapter._stream_turns if k.startswith("chat-1:")]) == 2
# Consumer 1 finalizes
ok1 = await adapter.send_stream_frame(
"consumer1 final", chat_id="chat-1", finalize=True, turn_id="turn-1"
)
assert ok1 is True
assert "chat-1:turn-1" not in adapter._stream_turns
# Consumer 2's turn still exists
assert "chat-1:turn-2" in adapter._stream_turns
# Consumer 2 finalizes
ok2 = await adapter.send_stream_frame(
"consumer2 final", chat_id="chat-1", finalize=True, turn_id="turn-2"
)
assert ok2 is True
assert "chat-1:turn-2" not in adapter._stream_turns
finally:
await adapter.disconnect()
@pytest.mark.asyncio
async def test_one_user_expired_others_unaffected(self):
"""User A hits stream expired; Users B and C continue normally."""
from plugins.platforms.wecom.adapter import STREAM_EXPIRED_ERRCODE, WeComAdapter
adapter = WeComAdapter(PlatformConfig(enabled=True))
try:
adapter._last_chat_req_ids["user-A"] = "req-A"
adapter._last_chat_req_ids["user-B"] = "req-B"
adapter._last_chat_req_ids["user-C"] = "req-C"
adapter._send_json = AsyncMock()
adapter._ws = AsyncMock(closed=False)
# All 3 users start streaming
await adapter.send_stream_frame("A content", chat_id="user-A", turn_id="turn-A")
await adapter.send_stream_frame("B content", chat_id="user-B", turn_id="turn-B")
await adapter.send_stream_frame("C content", chat_id="user-C", turn_id="turn-C")
# User A hits stream expired
adapter._send_reply_queued = AsyncMock(
return_value={"errcode": STREAM_EXPIRED_ERRCODE, "errmsg": "expired"}
)
okA = await adapter.send_stream_frame(
"A final", chat_id="user-A", finalize=True, turn_id="turn-A"
)
assert okA is False
assert "user-A" in adapter._stream_expired_chats
assert "user-A:turn-A" not in adapter._stream_turns
# Users B and C should NOT be affected (different chats)
adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0})
okB = await adapter.send_stream_frame(
"B final", chat_id="user-B", finalize=True, turn_id="turn-B"
)
okC = await adapter.send_stream_frame(
"C final", chat_id="user-C", finalize=True, turn_id="turn-C"
)
assert okB is True # ✅ User B unaffected
assert okC is True # ✅ User C unaffected
assert "user-B:turn-B" not in adapter._stream_turns
assert "user-C:turn-C" not in adapter._stream_turns
# Only user-A is in expired list
assert "user-A" in adapter._stream_expired_chats
assert "user-B" not in adapter._stream_expired_chats
assert "user-C" not in adapter._stream_expired_chats
finally:
await adapter.disconnect()
@pytest.mark.asyncio
async def test_one_turn_expired_other_continues(self):
"""When one turn hits stream expired, other concurrent turns can continue."""
from plugins.platforms.wecom.adapter import STREAM_EXPIRED_ERRCODE, WeComAdapter
adapter = WeComAdapter(PlatformConfig(enabled=True))
try:
adapter._last_chat_req_ids["chat-1"] = "req-1"
adapter._send_json = AsyncMock()
adapter._ws = AsyncMock(closed=False)
# Consumer 1 and 2 both start
await adapter.send_stream_frame("c1 frame", chat_id="chat-1", turn_id="turn-1")
await adapter.send_stream_frame("c2 frame", chat_id="chat-1", turn_id="turn-2")
assert "chat-1:turn-1" in adapter._stream_turns
assert "chat-1:turn-2" in adapter._stream_turns
# Consumer 1 hits expired error on finalize
adapter._send_reply_queued = AsyncMock(
return_value={"errcode": STREAM_EXPIRED_ERRCODE, "errmsg": "stream expired"}
)
ok1 = await adapter.send_stream_frame(
"c1 final", chat_id="chat-1", finalize=True, turn_id="turn-1"
)
assert ok1 is False
assert "chat-1" in adapter._stream_expired_chats
assert "chat-1:turn-1" not in adapter._stream_turns # turn-1 cleaned up
# Consumer 2's existing turn can still finalize
adapter._send_reply_queued = AsyncMock(return_value={"errcode": 0})
ok2 = await adapter.send_stream_frame(
"c2 final", chat_id="chat-1", finalize=True, turn_id="turn-2"
)
assert ok2 is True # ✅ turn-2 not blocked by chat-level expired
assert "chat-1:turn-2" not in adapter._stream_turns
finally:
await adapter.disconnect()
@pytest.mark.asyncio
async def test_expired_chat_blocks_new_turn_creation(self):
"""After one turn expired, new turn creation is blocked."""
from plugins.platforms.wecom.adapter import WeComAdapter
adapter = WeComAdapter(PlatformConfig(enabled=True))
try:
adapter._last_chat_req_ids["chat-1"] = "req-1"
adapter._stream_expired_chats.add("chat-1")
adapter._send_reply_request = AsyncMock(return_value={"errcode": 0})
# Try to create a new turn after chat is expired
ok = await adapter.send_stream_frame("new frame", chat_id="chat-1", turn_id="new-turn")
assert ok is False
assert "chat-1:new-turn" not in adapter._stream_turns
finally:
await adapter.disconnect()
class TestNativeFallbackStreamClose:
"""Verify that native streaming fallback closes open streams."""
@pytest.mark.asyncio
async def test_seed_success_first_frame_fails_still_finalizes(self):
"""Seed frame opens stream bubble, first content frame fails → finalize called.
This is the critical edge case: seed frame has length 0 but opens the
WeCom typing bubble. If the first content frame fails, we must still
finalize based on _native_stream_opened, not _native_last_pushed_len.
"""
from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig
from gateway.platforms.base import BasePlatformAdapter
class MockAdapter(BasePlatformAdapter):
MAX_MESSAGE_LENGTH = 4096
SUPPORTS_MESSAGE_EDITING = False
SUPPORTS_NATIVE_STREAMING = True
def __init__(self):
self._typing_paused = set()
self.send_stream_frame_calls = []
self.send_calls = []
self.should_fail_first_content = True
def supports_native_streaming(self, chat_type=None, metadata=None):
return True
async def send_stream_frame(
self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs
):
call_info = {"text_len": len(text), "finalize": finalize, "text_preview": text[:20]}
self.send_stream_frame_calls.append(call_info)
# Seed frame (empty) always succeeds
if len(text) == 0 and not finalize:
return True
# First non-seed, non-finalize frame fails
if self.should_fail_first_content and len(text) > 0 and not finalize:
self.should_fail_first_content = False
raise RuntimeError("first content frame failed")
# Finalize frames and subsequent content frames succeed
return True
async def send(self, chat_id, content, reply_to=None, metadata=None):
self.send_calls.append({"content_preview": content[:20]})
return type("SendResult", (), {"success": True, "message_id": "msg-1"})()
MockAdapter.__abstractmethods__ = frozenset()
adapter = MockAdapter()
cfg = StreamConsumerConfig(chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5)
consumer = GatewayStreamConsumer(adapter, "chat-1", cfg)
# Send short content to minimize frame count
consumer.on_delta("X")
import asyncio
task = asyncio.create_task(consumer.run())
await asyncio.sleep(0.05)
consumer.finish()
await task
# Verify: seed succeeded, then finalize was attempted (not skipped)
assert len(adapter.send_stream_frame_calls) >= 2
# First call: seed (length 0)
assert adapter.send_stream_frame_calls[0]["text_len"] == 0
assert not adapter.send_stream_frame_calls[0]["finalize"]
# At least one finalize call should have been made. This is the core
# invariant: finalize is driven by _native_stream_opened (the seed
# opened the bubble), NOT by _native_last_pushed_len — so even though
# the seed had length 0 and the first content frame failed, finalize
# must still be attempted to close the typing bubble.
finalize_calls = [c for c in adapter.send_stream_frame_calls if c["finalize"]]
assert len(finalize_calls) >= 1, "Finalize should be called even though seed had length 0"
# Fire-and-forget (14c49c781a): with the throttle removed, the 1-char
# content "X" is pushed immediately as an intermediate frame instead of
# being buffered. That frame fails per the mock, so a proactive send()
# fallback IS expected to deliver the content reliably. (Under the old
# _MIN_NEW_VISIBLE_CHARS=60 gate this tiny frame was never sent, so the
# previous assertion of zero send() fallbacks no longer holds.)
assert len(adapter.send_calls) == 1
assert adapter.send_calls[0]["content_preview"] == "X"
@pytest.mark.asyncio
async def test_native_fallback_closes_stream_on_success(self):
"""When native fails mid-stream, best-effort finalize succeeds."""
from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig
from gateway.platforms.base import BasePlatformAdapter
class MockAdapter(BasePlatformAdapter):
MAX_MESSAGE_LENGTH = 4096
SUPPORTS_MESSAGE_EDITING = False
SUPPORTS_NATIVE_STREAMING = True
def __init__(self):
self._typing_paused = set()
self.frames = []
self.frame_count = 0
def supports_native_streaming(self, chat_type=None, metadata=None):
return True
async def send_stream_frame(
self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs
):
self.frame_count += 1
self.frames.append({"text": text, "finalize": finalize})
# First 2 frames succeed, 3rd fails (non-expired error)
if self.frame_count == 3:
raise RuntimeError("network error")
# 4th frame (finalize in fallback) succeeds
return True
async def send(self, chat_id, content, reply_to=None, metadata=None):
self.frames.append({"send": content})
return type("SendResult", (), {"success": True, "message_id": "msg-1"})()
MockAdapter.__abstractmethods__ = frozenset()
adapter = MockAdapter()
cfg = StreamConsumerConfig(chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5)
consumer = GatewayStreamConsumer(adapter, "chat-1", cfg)
# Send enough to trigger frames
consumer.on_delta("First frame content that exceeds the threshold.")
consumer.on_delta(" Second frame content also exceeds threshold.")
consumer.on_delta(" Third will fail.")
import asyncio
task = asyncio.create_task(consumer.run())
await asyncio.sleep(0.1)
consumer.finish()
await task
# Should have: seed, frame1, frame2, (frame3 fails), finalize in fallback
# After fix #3: best-effort finalize closes the typing bubble but does NOT
# mark content_delivered. The fallback send() will deliver content reliably.
assert len([f for f in adapter.frames if f.get("finalize")]) >= 1
# Fallback send() IS expected to fire (content delivery via proactive send)
assert len([f for f in adapter.frames if "send" in f]) == 1
@pytest.mark.asyncio
async def test_native_fallback_falls_to_send_on_finalize_fail(self):
"""When native fails and finalize also fails, falls through to send()."""
from gateway.stream_consumer import GatewayStreamConsumer, StreamConsumerConfig
from gateway.platforms.base import BasePlatformAdapter
class MockAdapter(BasePlatformAdapter):
MAX_MESSAGE_LENGTH = 4096
SUPPORTS_MESSAGE_EDITING = False
SUPPORTS_NATIVE_STREAMING = True
def __init__(self):
self._typing_paused = set()
self.frames = []
self.frame_count = 0
def supports_native_streaming(self, chat_type=None, metadata=None):
return True
async def send_stream_frame(
self, text, *, finalize=False, chat_id=None, reply_to=None, **kwargs
):
self.frame_count += 1
self.frames.append({"text": text, "finalize": finalize})
# All frames fail (simulating complete stream failure)
if self.frame_count >= 2:
raise RuntimeError("stream dead")
return True
async def send(self, chat_id, content, reply_to=None, metadata=None):
self.frames.append({"send": content})
return type("SendResult", (), {"success": True, "message_id": "msg-1"})()
MockAdapter.__abstractmethods__ = frozenset()
adapter = MockAdapter()
cfg = StreamConsumerConfig(chat_type="dm", cursor="", edit_interval=0.01, buffer_threshold=5)
consumer = GatewayStreamConsumer(adapter, "chat-1", cfg)
consumer.on_delta("Content that will cause stream to fail.")
import asyncio
task = asyncio.create_task(consumer.run())
await asyncio.sleep(0.1)
consumer.finish()
await task
# Finalize failed → should fall through to send()
assert len([f for f in adapter.frames if "send" in f]) == 1