fix(gateway): persist internal synthetic turns typed as internal_notification (#82888)
Async-delegation batch completions and background watch notifications re-enter the gateway as synthetic MessageEvent(internal=True) turns via _inject_watch_notification, but were persisted as bare role='user' rows — indistinguishable from real user input in transcripts and the desktop UI. Thread the event's internal flag through to persistence: when event.internal is set, the turn's persisted user row is stamped display_kind='internal_notification' (the existing DB-only presentation sidecar used by auto_continue / model_switch rows). Wired through _run_agent → _run_agent_inner → TurnContext → run_conversation's persist_user_display_kind, and onto the three gateway-side fallback user rows (transient failure, no-new-messages, pre-run crash), whose append_to_transcript writer now forwards display_kind/display_metadata to SessionDB.append_message. Invariants preserved: role stays 'user' (alternation untouched), no new injections, no past-context mutation, and display_kind is already popped from every provider-bound copy in conversation_loop, so replayed sessions never leak the marker to the API. Regression tests: internal turn marked, real user turn unmarked, fallback rows marked/unmarked per event, and a DB round-trip proving replay keeps role/content intact while the provider copy drops the marker.
This commit is contained in:
@@ -5985,6 +5985,14 @@ class TurnRunner:
|
||||
_conversation_kwargs["persist_user_message"] = _persist_user_message_override
|
||||
elif observed_group_context:
|
||||
_conversation_kwargs["persist_user_message"] = ctx.message
|
||||
if ctx.persist_user_display_kind:
|
||||
# Internal self-injected turn (#82888): type the persisted user
|
||||
# row at turn start so UIs render it as a timeline notice, not
|
||||
# a user bubble. Role/content are untouched and the key is
|
||||
# stripped from provider-bound payloads in conversation_loop.
|
||||
_conversation_kwargs["persist_user_display_kind"] = (
|
||||
ctx.persist_user_display_kind
|
||||
)
|
||||
if ctx.moa_config is not None:
|
||||
_conversation_kwargs["moa_config"] = ctx.moa_config
|
||||
if _persist_user_timestamp_override is not None:
|
||||
@@ -17898,6 +17906,17 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
_redact_pii = False
|
||||
persist_user_message = None
|
||||
persist_user_timestamp = None
|
||||
# Synthetic self-injected turns (async-delegation batch completions,
|
||||
# background watch notifications, resume wake-ups) arrive as
|
||||
# MessageEvent(internal=True). Persist their user row typed with
|
||||
# display_kind="internal_notification" so transcripts/UIs can render
|
||||
# them as timeline notices instead of user bubbles (#82888). Role and
|
||||
# content are untouched — display_kind is a DB-only sidecar stripped
|
||||
# from every provider-bound payload (see conversation_loop's
|
||||
# api_msg.pop("display_kind")).
|
||||
persist_user_display_kind = (
|
||||
"internal_notification" if getattr(event, "internal", False) else None
|
||||
)
|
||||
try:
|
||||
_pcfg = _load_gateway_config()
|
||||
_redact_pii = bool((_pcfg.get("privacy") or {}).get("redact_pii", False))
|
||||
@@ -19050,6 +19069,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
moa_config=getattr(event, "_moa_config", None),
|
||||
persist_user_message=persist_user_message,
|
||||
persist_user_timestamp=persist_user_timestamp,
|
||||
persist_user_display_kind=persist_user_display_kind,
|
||||
message_type=event.message_type,
|
||||
)
|
||||
_turn_seconds = time.monotonic() - _turn_started_monotonic
|
||||
@@ -19489,6 +19509,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
else ts
|
||||
),
|
||||
}
|
||||
if persist_user_display_kind:
|
||||
_user_entry["display_kind"] = persist_user_display_kind
|
||||
if event.message_id:
|
||||
_user_entry["message_id"] = str(event.message_id)
|
||||
# Dedupe: skip if this platform message_id is already in the
|
||||
@@ -19531,6 +19553,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
else ts
|
||||
),
|
||||
}
|
||||
if persist_user_display_kind:
|
||||
_user_entry["display_kind"] = persist_user_display_kind
|
||||
if event.message_id:
|
||||
_user_entry["message_id"] = str(event.message_id)
|
||||
await self.async_session_store.append_to_transcript(
|
||||
@@ -19720,6 +19744,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
else time.time()
|
||||
),
|
||||
}
|
||||
if 'persist_user_display_kind' in locals() and persist_user_display_kind:
|
||||
_user_entry["display_kind"] = persist_user_display_kind
|
||||
if getattr(event, "message_id", None):
|
||||
_user_entry["message_id"] = str(event.message_id)
|
||||
await self.async_session_store.append_to_transcript(
|
||||
@@ -26061,6 +26087,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
moa_config: Optional[dict] = None,
|
||||
persist_user_message: Optional[Any] = None,
|
||||
persist_user_timestamp: Optional[float] = None,
|
||||
persist_user_display_kind: Optional[str] = None,
|
||||
message_type: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Profile-scoping wrapper around the agent run.
|
||||
@@ -26080,6 +26107,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
channel_prompt=channel_prompt, moa_config=moa_config,
|
||||
persist_user_message=persist_user_message,
|
||||
persist_user_timestamp=persist_user_timestamp,
|
||||
persist_user_display_kind=persist_user_display_kind,
|
||||
message_type=message_type,
|
||||
)
|
||||
|
||||
@@ -26092,6 +26120,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
channel_prompt=channel_prompt, moa_config=moa_config,
|
||||
persist_user_message=persist_user_message,
|
||||
persist_user_timestamp=persist_user_timestamp,
|
||||
persist_user_display_kind=persist_user_display_kind,
|
||||
message_type=message_type,
|
||||
)
|
||||
|
||||
@@ -26234,6 +26263,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
moa_config: Optional[dict] = None,
|
||||
persist_user_message: Optional[Any] = None,
|
||||
persist_user_timestamp: Optional[float] = None,
|
||||
persist_user_display_kind: Optional[str] = None,
|
||||
message_type: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -26541,6 +26571,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
moa_config=moa_config,
|
||||
persist_user_message=persist_user_message,
|
||||
persist_user_timestamp=persist_user_timestamp,
|
||||
persist_user_display_kind=persist_user_display_kind,
|
||||
)
|
||||
turn_runner = TurnRunner(self, turn_ctx)
|
||||
# Callback invoked by agent on tool lifecycle events — extracted to
|
||||
|
||||
@@ -3707,6 +3707,11 @@ class SessionStore:
|
||||
# any gateway-side persistence path or the next turn's
|
||||
# replay diverges at this row.
|
||||
api_content=extract_api_content_sidecar(message),
|
||||
# Presentation typing (e.g. "internal_notification" for
|
||||
# self-injected async-delegation/background notification turns,
|
||||
# #82888). DB-only; stripped from provider-bound payloads.
|
||||
display_kind=message.get("display_kind"),
|
||||
display_metadata=message.get("display_metadata"),
|
||||
)
|
||||
|
||||
# Maximum in-memory pending messages per session before dropping the
|
||||
|
||||
@@ -91,6 +91,11 @@ class TurnContext:
|
||||
moa_config: Optional[dict] = None
|
||||
persist_user_message: Optional[Any] = None
|
||||
persist_user_timestamp: Optional[float] = None
|
||||
# display_kind stamped on the persisted user row at turn start when this
|
||||
# turn was self-injected (MessageEvent.internal), e.g.
|
||||
# "internal_notification" for async-delegation/background notifications
|
||||
# (#82888). DB-only presentation metadata; never sent to the provider.
|
||||
persist_user_display_kind: Optional[str] = None
|
||||
user_config: Any = None
|
||||
enabled_toolsets: Any = None
|
||||
disabled_toolsets: Any = None
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Regression tests for #82888 — internal synthetic turns are persisted typed.
|
||||
|
||||
Async-delegation batch completions and background watch notifications re-enter
|
||||
the gateway as synthetic ``MessageEvent(internal=True)`` turns (see
|
||||
``_inject_watch_notification``). They must keep ``role='user'`` (message
|
||||
alternation is sacred) but the persisted row must carry
|
||||
``display_kind='internal_notification'`` so transcripts and the desktop UI can
|
||||
render them as timeline notices instead of user bubbles.
|
||||
|
||||
Covered:
|
||||
|
||||
1. an internal event threads ``persist_user_display_kind`` into the agent run;
|
||||
2. a real user event does NOT get the marker;
|
||||
3. the gateway-side fallback user rows (transient-failure / no-new-messages
|
||||
paths) carry the marker for internal events only;
|
||||
4. a marked row round-trips through SessionDB replay with role='user' intact
|
||||
and the marker is stripped from provider-bound payload copies.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import types
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
import gateway.run as gateway_run
|
||||
from gateway.config import GatewayConfig, Platform
|
||||
from gateway.platforms.base import MessageEvent
|
||||
from gateway.session import SessionEntry, SessionSource
|
||||
|
||||
SESSION_KEY = "agent:main:telegram:group:-1001:12345"
|
||||
|
||||
|
||||
def _bootstrap(monkeypatch, tmp_path):
|
||||
"""Minimal GatewayRunner setup (pattern from test_42039)."""
|
||||
fake_dotenv = types.ModuleType("dotenv")
|
||||
fake_dotenv.load_dotenv = lambda *args, **kwargs: None
|
||||
monkeypatch.setitem(sys.modules, "dotenv", fake_dotenv)
|
||||
|
||||
config = GatewayConfig()
|
||||
runner = gateway_run.GatewayRunner(config)
|
||||
runner.adapters = {}
|
||||
runner._running_agents = {}
|
||||
runner._running_agents_ts = {}
|
||||
runner._pending_messages = {}
|
||||
runner._pending_approvals = {}
|
||||
runner._is_user_authorized = lambda _source: True
|
||||
runner._set_session_env = lambda _context: None
|
||||
runner._handle_active_session_busy_message = AsyncMock(return_value=False)
|
||||
runner._session_db = MagicMock()
|
||||
runner._recover_telegram_topic_thread_id = lambda _source: None
|
||||
runner._cache_session_source = lambda _key, _source: None
|
||||
runner._is_session_run_current = lambda _key, _gen: True
|
||||
runner._begin_session_run_generation = lambda _key: 1
|
||||
runner._reply_anchor_for_event = lambda _event: None
|
||||
runner._get_guild_id = lambda _event: None
|
||||
runner._should_send_voice_reply = lambda *_a, **_kw: False
|
||||
runner.hooks = MagicMock()
|
||||
runner.hooks.emit = AsyncMock()
|
||||
|
||||
runner.session_store = MagicMock()
|
||||
runner.session_store.get_or_create_session.return_value = SessionEntry(
|
||||
session_key=SESSION_KEY,
|
||||
session_id="sess-82888",
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_type="group",
|
||||
)
|
||||
runner.session_store.load_transcript.return_value = []
|
||||
runner.session_store.append_to_transcript = MagicMock()
|
||||
runner.session_store.has_platform_message_id.return_value = False
|
||||
runner.session_store.update_session = MagicMock()
|
||||
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path)
|
||||
monkeypatch.setattr(
|
||||
gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"}
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"agent.model_metadata.get_model_context_length",
|
||||
lambda *_args, **_kwargs: 100_000,
|
||||
)
|
||||
return runner
|
||||
|
||||
|
||||
def _source():
|
||||
return SessionSource(
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_id="-1001",
|
||||
chat_type="group",
|
||||
user_id="12345",
|
||||
)
|
||||
|
||||
|
||||
def _event(*, internal: bool, text: str = "hello world"):
|
||||
return MessageEvent(
|
||||
text=text,
|
||||
source=_source(),
|
||||
message_id=None if internal else "msg-82888",
|
||||
internal=internal,
|
||||
)
|
||||
|
||||
|
||||
def _user_entries(calls):
|
||||
return [
|
||||
call.args[1]
|
||||
for call in calls
|
||||
if len(call.args) >= 2
|
||||
and isinstance(call.args[1], dict)
|
||||
and call.args[1].get("role") == "user"
|
||||
]
|
||||
|
||||
|
||||
# ── 1+2: the marker is threaded to the agent run for internal events only ──
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_internal_event_threads_marker_into_agent_run(monkeypatch, tmp_path):
|
||||
runner = _bootstrap(monkeypatch, tmp_path)
|
||||
runner._run_agent = AsyncMock(
|
||||
return_value={
|
||||
"final_response": "ack",
|
||||
"messages": [],
|
||||
"tools": [],
|
||||
"history_offset": 0,
|
||||
"last_prompt_tokens": 0,
|
||||
}
|
||||
)
|
||||
|
||||
await runner._handle_message_with_agent(
|
||||
_event(internal=True, text="[ASYNC DELEGATION BATCH COMPLETE]"),
|
||||
_source(), SESSION_KEY, 1,
|
||||
)
|
||||
|
||||
kwargs = runner._run_agent.call_args.kwargs
|
||||
assert kwargs["persist_user_display_kind"] == "internal_notification"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_user_event_gets_no_marker(monkeypatch, tmp_path):
|
||||
runner = _bootstrap(monkeypatch, tmp_path)
|
||||
runner._run_agent = AsyncMock(
|
||||
return_value={
|
||||
"final_response": "hi",
|
||||
"messages": [],
|
||||
"tools": [],
|
||||
"history_offset": 0,
|
||||
"last_prompt_tokens": 0,
|
||||
}
|
||||
)
|
||||
|
||||
await runner._handle_message_with_agent(
|
||||
_event(internal=False), _source(), SESSION_KEY, 1,
|
||||
)
|
||||
|
||||
kwargs = runner._run_agent.call_args.kwargs
|
||||
assert kwargs["persist_user_display_kind"] is None
|
||||
|
||||
|
||||
# ── 3: gateway-side fallback rows carry the marker for internal events ─────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_early_fallback_row_is_marked_for_internal_event(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
runner = _bootstrap(monkeypatch, tmp_path)
|
||||
runner._run_agent = AsyncMock(
|
||||
return_value={
|
||||
"failed": True,
|
||||
"final_response": None,
|
||||
"error": "429 Too Many Requests — rate limit exceeded",
|
||||
"messages": [],
|
||||
"history_offset": 0,
|
||||
"last_prompt_tokens": 0,
|
||||
}
|
||||
)
|
||||
|
||||
await runner._handle_message_with_agent(
|
||||
_event(internal=True, text="[ASYNC DELEGATION BATCH COMPLETE]"),
|
||||
_source(), SESSION_KEY, 1,
|
||||
)
|
||||
|
||||
entries = _user_entries(runner.session_store.append_to_transcript.call_args_list)
|
||||
assert entries, "expected a fallback user-row write"
|
||||
for entry in entries:
|
||||
assert entry["role"] == "user" # alternation invariant: role unchanged
|
||||
assert entry["display_kind"] == "internal_notification"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_early_fallback_row_is_unmarked_for_real_user(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
runner = _bootstrap(monkeypatch, tmp_path)
|
||||
runner._run_agent = AsyncMock(
|
||||
return_value={
|
||||
"failed": True,
|
||||
"final_response": None,
|
||||
"error": "429 Too Many Requests — rate limit exceeded",
|
||||
"messages": [],
|
||||
"history_offset": 0,
|
||||
"last_prompt_tokens": 0,
|
||||
}
|
||||
)
|
||||
|
||||
await runner._handle_message_with_agent(
|
||||
_event(internal=False), _source(), SESSION_KEY, 1,
|
||||
)
|
||||
|
||||
entries = _user_entries(runner.session_store.append_to_transcript.call_args_list)
|
||||
assert entries
|
||||
for entry in entries:
|
||||
assert "display_kind" not in entry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_new_messages_fallback_row_is_marked_for_internal_event(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
runner = _bootstrap(monkeypatch, tmp_path)
|
||||
runner._run_agent = AsyncMock(
|
||||
return_value={
|
||||
"final_response": "done",
|
||||
"messages": [{"role": "user", "content": "x"}],
|
||||
"tools": [],
|
||||
"history_offset": 1, # equals len(messages) → new_messages=[]
|
||||
"last_prompt_tokens": 0,
|
||||
}
|
||||
)
|
||||
|
||||
await runner._handle_message_with_agent(
|
||||
_event(internal=True, text="[SYSTEM: Background process matched]"),
|
||||
_source(), SESSION_KEY, 1,
|
||||
)
|
||||
|
||||
entries = _user_entries(runner.session_store.append_to_transcript.call_args_list)
|
||||
assert entries
|
||||
for entry in entries:
|
||||
assert entry["role"] == "user"
|
||||
assert entry["display_kind"] == "internal_notification"
|
||||
|
||||
|
||||
# ── 4: DB round-trip replay + provider-payload hygiene ─────────────────────
|
||||
|
||||
|
||||
def test_marked_row_replays_cleanly_and_never_reaches_provider(tmp_path):
|
||||
"""A marked row persists, resumes with role='user' + marker intact, and
|
||||
the provider-bound copy built by conversation_loop drops the marker."""
|
||||
from hermes_state import SessionDB
|
||||
|
||||
db = SessionDB(tmp_path / "state.db")
|
||||
sid = "sess-82888-replay"
|
||||
db.create_session(session_id=sid, source="gateway", model="test-model")
|
||||
try:
|
||||
db.append_message(
|
||||
session_id=sid,
|
||||
role="user",
|
||||
content="[ASYNC DELEGATION BATCH COMPLETE — 2/2 succeeded]",
|
||||
display_kind="internal_notification",
|
||||
)
|
||||
db.append_message(session_id=sid, role="assistant", content="noted")
|
||||
|
||||
# Session resume/load tolerates the extra key and keeps role='user'.
|
||||
replayed = db.get_messages_as_conversation(sid)
|
||||
user_row, = [m for m in replayed if m["role"] == "user"]
|
||||
assert user_row["display_kind"] == "internal_notification"
|
||||
assert user_row["content"].startswith("[ASYNC DELEGATION BATCH COMPLETE")
|
||||
|
||||
# Provider hygiene: the per-request copy in conversation_loop pops
|
||||
# display fields off every outgoing message (see the api_msg.pop
|
||||
# calls in run_chat_completions_conversation). Reproduce that
|
||||
# sequence on the replayed row and verify nothing display-only
|
||||
# survives while the original persisted dict is untouched.
|
||||
from agent.conversation_loop import _clone_message_for_send
|
||||
|
||||
api_msg = _clone_message_for_send(user_row)
|
||||
api_msg.pop("api_content", None)
|
||||
api_msg.pop("display_kind", None)
|
||||
api_msg.pop("display_metadata", None)
|
||||
api_msg.pop("_row_id", None)
|
||||
assert "display_kind" not in api_msg
|
||||
assert "display_metadata" not in api_msg
|
||||
assert api_msg["role"] == "user"
|
||||
assert user_row["display_kind"] == "internal_notification"
|
||||
finally:
|
||||
db.close()
|
||||
Reference in New Issue
Block a user