diff --git a/gateway/run_turn.py b/gateway/run_turn.py index 325eac294a..d36a0a788c 100644 --- a/gateway/run_turn.py +++ b/gateway/run_turn.py @@ -1,8 +1,6 @@ -"""Agent-turn execution (_handle_message_with_agent, _run_agent*, proxy path, background tasks, MCP reload) for GatewayRunner. - -Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO. -``gateway.run`` internals are imported lazily inside method bodies (import cycle), -so ``patch("gateway.run.X")`` keeps intercepting them at call time. +"""Agent-turn execution for GatewayRunner: _handle_message_with_agent, _run_agent*, proxy path, +background tasks, MCP reload. Bound onto ``GatewayRunner`` via the MRO; ``gateway.run`` internals +are imported lazily inside method bodies (import cycle) so ``patch("gateway.run.X")`` still works. """ from __future__ import annotations @@ -24,10 +22,7 @@ from gateway.config import Platform from gateway.media_repair import repair_explicit_computer_use_media_paths from gateway.platforms.base import BasePlatformAdapter, MessageEvent from gateway.session import ( - SessionSource, - TranscriptReadError, - _session_key_namespace, - build_channel_continuity_note, + SessionSource, TranscriptReadError, _session_key_namespace, build_channel_continuity_note, build_session_context, ) from gateway.turn_context import TurnContext @@ -45,7 +40,7 @@ logger = logging.getLogger("gateway.run") class GatewayTurnMixin: - """Agent-turn execution (_handle_message_with_agent, _run_agent*, proxy path, background tasks, MCP reload) for GatewayRunner.""" + """Agent-turn execution for GatewayRunner (see module docstring).""" def _resolve_session_agent_runtime( self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, @@ -151,8 +146,7 @@ class GatewayTurnMixin: logger.warning( "Empty model resolved for session=%s — recovering " "last-known-good model %s (config read likely returned " - "empty; see #35314)", - skey or "", _recovered, + "empty; see #35314)", skey or "", _recovered, ) model = _recovered else: @@ -164,11 +158,8 @@ class GatewayTurnMixin: return model, runtime_kwargs def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwargs: dict) -> dict: - """Build the effective model/runtime config for a single turn. - - Always uses the session's primary model/provider. With `/fast` priority on and a model that - supports it, fast-mode ``request_overrides`` are deep-merged OVER the per-provider - ``request_overrides`` (e.g. ``custom_providers`` ``extra_body``) so both reach the model. + """Effective model/runtime config for one turn. With `/fast` priority on, fast-mode + ``request_overrides`` are deep-merged OVER the per-provider ones so both reach the model. """ from gateway.run import _deep_merge_request_overrides from hermes_cli.models import resolve_fast_mode_overrides @@ -408,10 +399,8 @@ class GatewayTurnMixin: platform=source.platform, session_type=getattr(source, 'chat_type', 'dm'), ) platform_name = source.platform.value if source.platform else "" - # Suspended / restart-recovery-expired sessions always notify regardless of - # policy.notify — an active session was silently replaced and the user must learn they - # can /resume it. Idle/daily resets respect the policy flag, the excluded platforms - # (api_server, webhook) and require prior activity. + # Suspended / restart-recovery-expired sessions always notify (the user must learn they + # can /resume); idle/daily resets respect policy.notify + excluded platforms + activity. should_notify = reset_reason in {"suspended", "resume_pending_expired"} or ( policy.notify and getattr(session_entry, 'reset_had_activity', False) @@ -465,12 +454,10 @@ class GatewayTurnMixin: logger.warning("[Gateway] Failed to auto-load skill(s) %s: %s", _skill_names, e) async def _hmwa_acquire_turn_lease(self, _quick_key, run_generation, session_entry, _session_env_tokens): - """Serialize [load history → run → flush] per resolved SESSION_ID (session resolution is - FINAL here): another routing key mapped to the same session_id waits for the prior flush - instead of loading a stale base. Fail-closed on timeout — never enter the transcript - region without a lease; outer dispatch returns a bounded resend notice. Released in - _handle_message's finally, granted per (routing key, run generation) so a stale unwind - can't release a newer turn's.""" + """Serialize [load history → run → flush] per resolved SESSION_ID so another routing key on + the same session waits for the prior flush. Fail-closed on timeout (outer dispatch returns + a resend notice). Released in _handle_message's finally, granted per (routing key, run + generation) so a stale unwind can't release a newer turn's.""" from gateway.run import _float_env _lease_registry = getattr(self, "_turn_leases", None) if _lease_registry is None: @@ -543,13 +530,11 @@ class GatewayTurnMixin: ) async def _hmwa_hygiene_settings(self, source, session_key): - """Resolve model/provider/context-length + hygiene knobs for the pre-agent compression - safety net (fail-soft: any config/runtime error keeps the defaults). + """Resolve model/provider/context-length + hygiene knobs (fail-soft: errors keep defaults). - The hygiene threshold (0.85) is deliberately HIGHER than the agent's own compressor - (0.50): it is a safety net for sessions that grew between turns. ``max_turn_hold_seconds`` - bounds how long the user's TURN waits on hygiene before proceeding uncompressed (the - compressor keeps running detached, commit fenced); kept below transport idle-timeouts.""" + The 0.85 threshold is deliberately HIGHER than the agent's compressor (0.50): a safety net + for sessions that grew between turns. ``max_turn_hold_seconds`` bounds the TURN wait + (compressor keeps running detached, commit fenced); kept below transport idle-timeouts.""" from gateway.run import _load_gateway_config hs = self._HygieneSettings( model="anthropic/claude-sonnet-4.6", threshold_pct=0.85, compression_enabled=True, @@ -622,10 +607,8 @@ class GatewayTurnMixin: _warn_token_threshold = int(_hyg_context_length * 0.95) _msg_count = len(history) - # Prefer the API-reported prompt tokens from the last turn over the rough char estimate. - # Rough estimates run 30-50% high on code/JSON-heavy sessions, which only makes hygiene - # fire early (safe). Do NOT compensate with a threshold multiplier: 85% * 1.4 = 119% of - # context kept hygiene from ever firing for ~200K models. + # Prefer the API-reported prompt tokens over the rough estimate (runs 30-50% high, which only + # fires hygiene early — safe). Do NOT compensate with a threshold multiplier. if session_entry.last_prompt_tokens > 0: _approx_tokens, _token_source = session_entry.last_prompt_tokens, "actual" else: @@ -637,9 +620,8 @@ class GatewayTurnMixin: _needs_compress = _approx_tokens >= _compress_token_threshold or _msg_count >= hs.hard_msg_limit if _needs_compress: - # Persistent DB-backed cooldown (shared with context_compressor.py) so it survives - # gateway restarts; an in-memory dict re-triggered the same failing compression on - # every restart and wedged session storage. + # DB-backed cooldown (shared with context_compressor.py): survives gateway restarts, so a + # failing compression is not re-triggered on every restart. _session_db = getattr(self, "_session_db", None) if _session_db is not None: _session_db = getattr(_session_db, "_db", _session_db) @@ -681,10 +663,7 @@ class GatewayTurnMixin: transcript; raises ``HygieneTurnHoldExceeded`` (turn-hold budget) or ``asyncio.TimeoutError`` (idle/ceiling/fence cancel) for the caller's handlers. - The timeout is an INACTIVITY budget — the worker ticks the fence per streamed token, so a - slow but still-generating model extends the deadline; a hard ceiling bounds the total so - a trickle stream can't hold the turn; the turn-hold budget caps how long the user's TURN - waits regardless.""" + Idle timeout (fence ticks per streamed token) + hard ceiling + turn-hold cap.""" from gateway.run import HygieneTurnHoldExceeded, hygiene_wait_should_extend fence = attempt.commit_fence while True: @@ -741,12 +720,9 @@ class GatewayTurnMixin: raise async def _hmwa_hygiene_cancel_before_commit(self, fence): - """Cancel the detached worker at the commit fence. Returns ``False`` when the worker had - already crossed into its commit (caller must consume the result — aborting mid-commit - would corrupt the message-store transaction), ``True`` when cancelled in time. - - A hung commit retains the fence lock; the lock-free ``commit_in_flight`` marker keeps - this loop from spinning forever. 25ms polls ride transient lock-setup windows.""" + """Cancel the detached worker at the commit fence: ``True`` when cancelled in time, ``False`` + when it already crossed into its commit (caller must consume the result). The lock-free + ``commit_in_flight`` marker keeps this loop from spinning on a hung commit.""" while True: if fence.commit_in_flight: return False @@ -802,15 +778,12 @@ class GatewayTurnMixin: notify the user, and re-raise; returns the compressed transcript only when the worker was already committing. - Turn-hold expiry is an availability boundary, not a failure: the compressor is healthy, - so the failure streak must NOT advance — only flat, non-escalating retry spacing is - recorded. When the worker's commit is watermark-fenced (rows appended after compression - start — this turn included — survive its late commit as cloned concurrent tail) the - attempt KEEPS commit admission: the turn proceeds uncompressed NOW and the summary is - adopted at the worker's own fenced commit; always cancelling burned every attempt for - thinking summary models whose reasoning prefix alone exceeds the hold. Without the - watermark fence (no session_db, capture failed, legacy lock API) a late commit could - clobber newer turns, so cancel.""" + Turn-hold expiry is an availability boundary, not a failure: the streak must NOT advance, + only flat retry spacing is recorded. A watermark-fenced commit (rows appended after + compression start survive as cloned tail) KEEPS admission: the turn proceeds uncompressed + now and the summary is adopted at the worker's fenced commit — always cancelling burned + every attempt for thinking summary models. Without the fence a late commit could clobber + newer turns, so cancel.""" from gateway.run import ( _HYGIENE_TURNHOLD_RETRY_SECONDS, _record_hygiene_cooldown, _reset_hygiene_failure_streak ) @@ -818,11 +791,9 @@ class GatewayTurnMixin: _hyg_keep_admission = bool(getattr(fence, "commit_watermark_fenced", False)) and not fence.is_cancelled if _hyg_keep_admission: self._hmwa_hygiene_defer_cleanup(attempt, "session hygiene turn-hold") - # NO retry-after here: the attempt is still running toward a real commit, and the flat - # 60s retry-after would also block the agent-side preflight compressor (same-session - # cooldown). Re-attempt spacing comes from the durable compression lock: the next - # turn's hygiene pre-check skips while this worker's lease is held. The done-callback - # records the flat retry-after ONLY if the worker ends without committing anything. + # NO retry-after here (it would also block the agent-side preflight compressor); spacing + # comes from the durable compression lock. The done-callback records the flat retry-after + # ONLY if the worker ends without committing anything. _sid, _skey, _agent = session_entry.session_id, session_key, attempt.agent def _hyg_adopt_or_space_retry(_fut, _gw=self, _sid=_sid, _skey=_skey, _agent=_agent): @@ -950,13 +921,9 @@ class GatewayTurnMixin: raise def _hmwa_hygiene_on_unwind(self, attempt, hs, session_entry, session_key): - """``except BaseException`` body (caller re-raises): revoke commit admission and record a - cooldown so the next turn does not immediately re-arm hygiene. - - Non-timeout unwind (KeyboardInterrupt, task cancel, unexpected error) while the detached - worker may still run: revoke admission (and release its durable lease) BEFORE the host - unwinds so it can never commit later. Restart drain / task cancel must record a cooldown, - or the next turn re-arms hygiene and waits up to 600s behind a fence that refuses again.""" + """``except BaseException`` body (caller re-raises): revoke commit admission BEFORE the host + unwinds so the detached worker can never commit later, and record a cooldown — otherwise + the next turn re-arms hygiene and waits up to 600s behind a fence that refuses again.""" from gateway.run import _hygiene_cooldown_for_failure, _record_hygiene_cooldown attempt.commit_fence.revoke_commit_admission() if not attempt.cleanup_deferred: @@ -983,17 +950,13 @@ class GatewayTurnMixin: async def _hmwa_hygiene_adopt_transcript( self, attempt, _compressed, history, plan, *, session_entry, source, _quick_key, run_generation, ): - """Adopt a finished compression (rotation / in-place / refused) and publish the transcript - to continue with on ``attempt.history``. + """Adopt a finished compression (rotation / in-place / refused); publishes the transcript to + continue with on ``attempt.history``. - Rewrite the transcript only when rotation produced a NEW session id. In-place compaction - needs none: archive_and_compact() already soft-archived the previous rows, and - rewrite_transcript() would replace_messages(active_only=False) and DELETE the archived - turns. A summary with neither rotation nor a completed archive_and_compact() signals - FAILURE; an unconditional rewrite would replace the originals with only the summary. - Write-before-repoint (mirrors manual /compress): if session_entry were repointed first and - rewrite_transcript then failed (lock/ENOSPC), the live entry would reference an empty - session and the conversation silently vanishes.""" + Rewrite only on rotation (NEW session id): in-place compaction already soft-archived the + old rows and rewrite_transcript() would DELETE them; neither rotation nor in-place signals + FAILURE and an unconditional rewrite would leave only the summary. Write-before-repoint: + a repoint-then-failed-rewrite would point the live entry at an empty session.""" from agent.model_metadata import estimate_messages_tokens_rough _hyg_agent = attempt.agent # _compress_context ends the old session and creates a new session_id; compressed messages @@ -1007,9 +970,8 @@ class GatewayTurnMixin: _hyg_out_toks = estimate_messages_tokens_rough(_compressed) if _hyg_rotated and _hyg_out_toks > _hyg_in_toks: logger.warning( - "Gateway hygiene compression for session %s " - "would grow transcript (~%s -> ~%s tokens); " - "keeping the original transcript unchanged", + "Gateway hygiene compression for session %s would grow transcript (~%s -> ~%s " + "tokens); keeping the original transcript unchanged", session_entry.session_id, f"{_hyg_in_toks:,}", f"{_hyg_out_toks:,}", ) _hyg_rotated = False @@ -1017,10 +979,8 @@ class GatewayTurnMixin: if _hyg_rotated: if not await self.async_session_store.rewrite_transcript(_hyg_new_sid, _compressed): logger.error( - "Session hygiene: failed to persist " - "compressed transcript for rotated " - "session %s → %s; keeping the live " - "entry on the original session so the " + "Session hygiene: failed to persist compressed transcript for rotated session " + "%s → %s; keeping the live entry on the original session so the " "conversation is not dropped", session_entry.session_id, _hyg_new_sid, ) # Fail closed: treat like no rotation. @@ -1048,16 +1008,13 @@ class GatewayTurnMixin: _new_count = plan.msg_count _new_tokens = plan.approx_tokens logger.warning( - "Gateway hygiene compression for session %s " - "did not rotate or compact in place " - "(no session_db on the hygiene agent) — " - "preserving the original transcript instead " + "Gateway hygiene compression for session %s did not rotate or compact in place (no " + "session_db on the hygiene agent) — preserving the original transcript instead " "of overwriting it with the summary (#21301).", session_entry.session_id, ) logger.info( - "Session hygiene: compressed %s → %s msgs, " - "~%s → ~%s tokens", + "Session hygiene: compressed %s → %s msgs, ~%s → ~%s tokens", plan.msg_count, _new_count, f"{plan.approx_tokens:,}", f"{_new_tokens:,}", ) if _new_tokens >= plan.warn_token_threshold: @@ -1115,10 +1072,8 @@ class GatewayTurnMixin: await self._hmwa_hygiene_notify( source, attempt.meta, "⚠️ Context compression aborted " f"({_err}). No messages were dropped — " - "conversation is unchanged. Run /compress " - "to retry, /reset for a clean session, or " - "check your auxiliary.compression model " - "configuration.", + "conversation is unchanged. Run /compress to retry, /reset for a clean " + "session, or check your auxiliary.compression model configuration.", "compression-failure warning", ) # If the CONFIGURED aux model failed and we recovered on the main model, tell the user — @@ -1150,8 +1105,7 @@ class GatewayTurnMixin: failure_cooldown_seconds=hs.failure_cooldown_seconds, ) logger.info( - "Session hygiene (codex app-server): %s " - "(session=%s, mode=%s, ~%s tokens)", + "Session hygiene (codex app-server): %s (session=%s, mode=%s, ~%s tokens)", _hyg_codex_outcome, session_entry.session_id, _hyg_codex_auto, f"{plan.approx_tokens:,}", ) @@ -1165,9 +1119,8 @@ class GatewayTurnMixin: except Exception as exc: _hyg_session_row = None logger.warning( - "Session hygiene could not restore the system " - "prompt for session %s: %s. Preserving an empty " - "prompt so the live turn rebuilds it with its " + "Session hygiene could not restore the system prompt for session %s: %s. " + "Preserving an empty prompt so the live turn rebuilds it with its " "configured providers.", session_entry.session_id, exc, exc_info=True, ) _hyg_session_db = getattr(self._session_db, "_db", self._session_db) @@ -1202,10 +1155,8 @@ class GatewayTurnMixin: _hyg_agent, _hyg_session_db = await self._hmwa_hygiene_build_agent(_hyg_model, _hyg_runtime, session_entry) attempt.agent = _hyg_agent try: - # Hygiene runs before the turn and owns the session binding, so prefer in-place - # compaction: archive old rows under the same session id rather than minting a - # continuation child that must be published back to SessionStore/topic bindings. - # Without a SessionDB this stays False and the apply guard preserves it. + # Hygiene owns the session binding, so prefer in-place compaction over minting a + # continuation child. Without a SessionDB this stays False. _hyg_agent.compression_in_place = True _bind_hyg_state = getattr(getattr(_hyg_agent, "context_compressor", None), "bind_session_state", None) if callable(_bind_hyg_state): @@ -1216,10 +1167,8 @@ class GatewayTurnMixin: loop = asyncio.get_running_loop() _hyg_commit_fence = CompressionCommitFence(total_ceiling_seconds=hs.total_ceiling_seconds) - # Default executor (NOT self._get_executor): a fence-cancelled hung summary must never - # occupy an agent-work slot. But it MUST run in the caller's contextvars: under - # multiplex_profiles the secret scope / HERMES_HOME live in ContextVars, and an empty - # Context makes get_secret() fail closed → lossy truncation. + # Default executor (NOT self._get_executor): a hung summary must never occupy an + # agent-work slot. MUST run in the caller's contextvars (multiplex secret scope). attempt.commit_fence = _hyg_commit_fence attempt.future = loop.run_in_executor( None, @@ -1360,10 +1309,9 @@ class GatewayTurnMixin: sethome_cmd = "/hermes sethome" if source.platform == Platform.SLACK else "/sethome" await self._deliver_platform_notice( source, f"📬 No home channel is set for {platform_name.title()}. " - f"A home channel is where Hermes delivers cron job results " - f"and cross-platform messages.\n\n" - f"Type {sethome_cmd} to make this chat your home channel, " - f"or ignore to skip.", + f"A home channel is where Hermes delivers cron job results and cross-platform " + f"messages.\n\nType {sethome_cmd} to make this chat your home channel, or ignore " + f"to skip.", ) def _hmwa_apply_message_timestamp(self, event, message_text): @@ -1435,9 +1383,8 @@ class GatewayTurnMixin: # "(empty)" = the model produced no visible content after exhausting all retries. if response == "(empty)" and not _intentional_silence: response = ( - "⚠️ The model returned no response after processing tool " - "results. This can happen with some models — try again or " - "rephrase your question." + "⚠️ The model returned no response after processing tool results. This can happen " + "with some models — try again or rephrase your question." ) agent_messages = agent_result.get("messages", []) logger.info( @@ -1479,17 +1426,15 @@ class GatewayTurnMixin: ) else: logger.info( - "Skipping agent-result session split sync for %s because " - "the session binding moved from %s to %s before " - "compression finished", + "Skipping agent-result session split sync for %s because the session binding " + "moved from %s to %s before compression finished", session_key or "?", _run_start_session_id, session_entry.session_id, ) return response, _intentional_silence, agent_messages # reasoning_style → (header line, per-line quote prefix for blank / non-blank lines) _REASONING_QUOTE_STYLES = { - "subtext": ("-# 💭 Reasoning", "-# ", "-#"), - "blockquote": ("> 💭 **Reasoning:**", "> ", ">"), + "subtext": ("-# 💭 Reasoning", "-# ", "-#"), "blockquote": ("> 💭 **Reasoning:**", "> ", ">") } def _hmwa_prepend_reasoning(self, agent_result, response, source, _intentional_silence): @@ -1593,10 +1538,8 @@ class GatewayTurnMixin: """Classify a finished turn for transcript persistence. Returns ``(agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure)``. - Context-overflow failures (compression exhausted, generic 400 on large sessions) must - NOT persist the user message — it would grow the session and reproduce the failure - forever. Transient failures (429, timeout, connection error, 5xx) DO persist it: the - session is not oversized and dropping the turn causes severe context loss on retry.""" + Context-overflow failures must NOT persist the user message (session would grow and + reproduce the failure forever); transient failures (429/timeout/5xx) DO.""" from gateway.run import _is_gateway_hidden_reasoning_incomplete_turn agent_failed_early = bool(agent_result.get("failed")) hidden_reasoning_incomplete = _is_gateway_hidden_reasoning_incomplete_turn(agent_result) @@ -1611,8 +1554,7 @@ class GatewayTurnMixin: if is_context_overflow_failure: logger.info( "Skipping transcript persistence for context-overflow " - "failure in session %s to prevent session growth loop.", - session_entry.session_id, + "failure in session %s to prevent session growth loop.", session_entry.session_id, ) elif agent_failed_early: logger.info( @@ -1621,8 +1563,7 @@ class GatewayTurnMixin: ) elif hidden_reasoning_incomplete: logger.warning( - "Suppressing hidden-reasoning-only incomplete gateway turn " - "for session %s: %s", + "Suppressing hidden-reasoning-only incomplete gateway turn for session %s: %s", session_entry.session_id, agent_result.get("error", "processing incomplete"), ) return agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure @@ -1649,19 +1590,16 @@ class GatewayTurnMixin: # dict (see _CONVERSATION_SCOPED_STATE). self._clear_conversation_scope(session_key, reason="compression_exhausted_reset") if new_entry is not None: - # Re-point the Telegram topic binding at the fresh session: compression rotated - # session_entry.session_id to the bloated child earlier this turn and that _sync - # also rewrote the (chat_id, thread_id) binding. Without a re-sync the binding-heal - # walk switches the next inbound message back onto the child and re-triggers - # exhaustion forever. No-op on non-topic lanes. + # Re-point the Telegram topic binding at the fresh session, or the binding-heal walk + # switches the next message back onto the bloated child and re-triggers exhaustion + # forever. No-op on non-topic lanes. session_entry = new_entry await asyncio.to_thread( self._sync_telegram_topic_binding, source, session_entry, reason="compression-exhausted-reset", ) response = (response or "") + ( - "\n\n🔄 Session auto-reset — the conversation exceeded the " - "maximum context size and could not be compressed further. " - "Your next message will start a fresh session." + "\n\n🔄 Session auto-reset — the conversation exceeded the maximum context size and " + "could not be compressed further. Your next message will start a fresh session." ) return response, session_entry @@ -1714,15 +1652,12 @@ class GatewayTurnMixin: "timestamp": ts, }) if agent_failed_early or hidden_reasoning_incomplete: - # Transient failure (429/timeout/5xx): persist only the user message so the next - # message loads a transcript that reflects what was said; the assistant error text is - # a gateway-generated hint, not model output. Hidden-reasoning incomplete turns follow - # the same rule so peer-agent channels don't ingest them. - # Dedupe on platform message_id (Telegram retries after transient failures). + # Transient failure / hidden-reasoning incomplete: persist only the user message (the + # assistant error text is a gateway hint, not model output). Dedupe on platform + # message_id (Telegram retries after transient failures). if event.message_id and await store.has_platform_message_id(sid, str(event.message_id)): logger.info( - "Skipping duplicate user turn " - "(message_id=%s) in session %s", + "Skipping duplicate user turn (message_id=%s) in session %s", event.message_id, sid, ) else: @@ -1730,8 +1665,8 @@ class GatewayTurnMixin: sid, self._hmwa_user_transcript_entry(event, prepared, ts), skip_db=agent_persisted, ) else: - # Only the NEW messages from this turn: use history_offset (what the agent saw), not - # len(history), which counts session_meta entries stripped before the agent saw them. + # Only the NEW messages: history_offset (what the agent saw), not len(history), which + # counts session_meta entries stripped before the agent saw them. history_len = agent_result.get("history_offset", len(history)) new_messages = agent_messages[history_len:] if len(agent_messages) > history_len else [] if not new_messages: @@ -1770,11 +1705,9 @@ class GatewayTurnMixin: touch_activity=not bool(getattr(event, "internal", False)), ) - # Re-baseline the cached agent's message_count snapshot now that ALL of this turn's writes - # are done (flushed rows AND the first-turn `session_meta` marker, which bumps the count - # too). The cross-process coherence guard snapshots at agent-BUILD time and never - # refreshes on reuse, so our own writes would trigger a rebuild next turn (destroying - # prompt caching). + # Re-baseline the cached agent's message_count now that ALL of this turn's writes are done: + # the coherence guard snapshots at agent-BUILD time, so our own writes would otherwise + # trigger a rebuild next turn (destroying prompt caching). await self._refresh_agent_cache_message_count(session_key, sid) async def _hmwa_deliver_turn_response( @@ -1883,9 +1816,8 @@ class GatewayTurnMixin: # payload is too large for the API — treat it the same way. if len(prepared.history) > 50: return ( - "⚠️ Session too large for the model's context window.\n" - "Use /compact to compress the conversation, or " - "/reset to start fresh." + "⚠️ Session too large for the model's context window.\nUse /compact to " + "compress the conversation, or /reset to start fresh." ) elif status_code == 400: status_hint = " The request was rejected by the API." @@ -1922,10 +1854,8 @@ class GatewayTurnMixin: a ``str`` is a reply to send instead of running (history unreadable); ``None`` drops the turn (inbound text rejected).""" from gateway.run import _load_gateway_config - # Synthetic self-injected turns (batch completions, watch notifications, resume wake-ups) - # arrive as MessageEvent(internal=True). Persist with display_kind="internal_notification" - # so UIs render timeline notices, not user bubbles. display_kind is a DB-only sidecar - # stripped from every provider-bound payload; role/content untouched. + # Self-injected turns (MessageEvent(internal=True)) persist with a DB-only display_kind so + # UIs render timeline notices, not user bubbles; role/content untouched. persist_user_display_kind = "internal_notification" if getattr(event, "internal", False) else None _redact_pii = False # privacy.redact_pii, re-read per message with suppress(Exception): @@ -1936,9 +1866,8 @@ class GatewayTurnMixin: # drift turn-over-turn; a miss (thread rename, /sethome, redact_pii flip) re-renders. context_prompt = self._pinned_session_context_prompt(context, _redact_pii, session_key) - # Per-turn must-deliver notes ride the user message via the api_content sidecar (staged - # below, consumed in run_sync → build_turn_context), NOT context_prompt: appending them to - # the ephemeral system prompt guaranteed a turn1→turn2 diff and a full agent rebuild. + # Per-turn notes ride the user message via the api_content sidecar, NOT context_prompt + # (appending to the ephemeral system prompt forced a full agent rebuild). turn_sidecar_notes: List[str] = [] if _was_auto_reset: await self._hmwa_deliver_auto_reset_notice(session_entry, source, turn_sidecar_notes) @@ -1956,19 +1885,16 @@ class GatewayTurnMixin: # routed message that never began processing if the gateway died while it was still waiting. await self._mark_durable_active_turn(event, session_entry.session_key) - # An unreadable canonical store is not an empty conversation: stop before the agent can - # invent continuity from a plausible-looking []. This return happens before the broad - # cleanup finally, so restore task-local context here; the outer dispatch still clears the - # durable marker and turn lease. + # An unreadable store is not an empty conversation: stop before the agent invents continuity + # from []. Restore task-local context here (before the broad cleanup finally). try: history = await self.async_session_store.load_transcript(session_entry.session_id) except TranscriptReadError: self._clear_session_env(_session_env_tokens) return ( - "⚠️ This session's history is temporarily unavailable, so " - "this message was not processed. Ask the operator to inspect " - "state.db, then resend after it is healthy. Use /reset only " - "if you intentionally want to start a new conversation." + "⚠️ This session's history is temporarily unavailable, so this message was not " + "processed. Ask the operator to inspect state.db, then resend after it is healthy. " + "Use /reset only if you intentionally want to start a new conversation." ) history = await self._hmwa_run_session_hygiene( @@ -1977,9 +1903,8 @@ class GatewayTurnMixin: await self._hmwa_first_contact_notes(source, history, turn_sidecar_notes) - # Voice channel state (who is present / speaking) rides the user message ONLY when changed - # since the previous turn; in the ephemeral system prompt it forced a full agent rebuild + - # prompt-cache re-key per message (the prompt carries a static pointer line instead). + # Voice channel state rides the user message ONLY when changed (in the system prompt it + # forced a rebuild + prompt-cache re-key per message). _vc_note = self._voice_channel_sidecar_note(event, source, session_key) if _vc_note: turn_sidecar_notes.append(_vc_note) @@ -2169,11 +2094,9 @@ class GatewayTurnMixin: def _resolve_enabled_toolsets_for_source( self, user_config: dict, source: "SessionSource", platform_key: str, ) -> list: - """Enabled toolsets for an agent run, honoring an adapter ``toolsets_for_source()`` override. - - The override (e.g. per-route webhook toolsets) is validated through the SAME - ``_get_platform_tools`` path as normal platform config, so unknown and platform-restricted - toolsets are dropped rather than trusted. Absent an override: ``platform_toolsets.``. + """Enabled toolsets for an agent run, honoring an adapter ``toolsets_for_source()`` override + validated through the SAME ``_get_platform_tools`` path (unknown / platform-restricted + toolsets dropped, not trusted). """ from hermes_cli.tools_config import _get_platform_tools try: @@ -2457,10 +2380,8 @@ class GatewayTurnMixin: self, source: "SessionSource", scfg: Any, adapter: Any, *, on_missing_cursor: str, ) -> "tuple[Any, Optional[Callable[[], None]]]": """Build the shared ``StreamConsumerConfig`` and optional Telegram pause-typing closure. - - ``on_missing_cursor`` handles adapters with ``SUPPORTS_MESSAGE_EDITING = False``: - ``"fallback"`` (proxy path) streams with an empty cursor; ``"raise"`` (in-process path) - raises ``RuntimeError`` so the caller's ``except`` skips streaming entirely. + For non-editing adapters ``on_missing_cursor="fallback"`` streams with an empty cursor; + ``"raise"`` raises ``RuntimeError`` so the caller skips streaming entirely. """ from gateway.stream_consumer import StreamConsumerConfig @@ -2468,10 +2389,8 @@ class GatewayTurnMixin: if source.platform == Platform.TELEGRAM and hasattr(adapter, "pause_typing_for_chat"): def _pause_typing_before_finalize(_adapter=adapter, _chat_id=source.chat_id) -> None: _adapter.pause_typing_for_chat(_chat_id) - # Platforms that can't edit sent messages (e.g. QQ, WeChat) skip streaming entirely: the - # partial first message could never be updated, yielding duplicates (partial + final) — - # unless they have a native-streaming transport (e.g. WeCom msgtype "stream"), which the - # consumer's native branch delivers whole. + # Non-editing platforms (QQ, WeChat) skip streaming — the partial first message could never + # be updated — unless they have a native-streaming transport (WeCom msgtype "stream"). _adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True) _adapter_supports_native_stream = bool(getattr(adapter, "SUPPORTS_NATIVE_STREAMING", False)) if not _adapter_supports_edit and not _adapter_supports_native_stream and on_missing_cursor == "raise": @@ -2820,19 +2739,14 @@ class GatewayTurnMixin: self, disp: "GatewayRunner._RunAgentDisplay", AIAgent: Any, *, message: str, source: SessionSource, session_key: Optional[str], run_generation: Optional[int], **turn_params, ) -> Tuple[TurnContext, TurnRunner, Any]: - """Build the progress holders, the ``TurnContext`` and its ``TurnRunner``. - - ``turn_params`` (history, context_prompt, channel_prompt, session_id, _interrupt_depth, - event_message_id, inbound_message_id, moa_config, persist_user_*) are stored verbatim. - Returns ``(turn_ctx, turn_runner, cleanup_adapter)``; the progress-bubble cleanup flags travel - on ``turn_ctx._cleanup_progress`` / ``turn_ctx._cleanup_msg_ids``. + """Build the ``TurnContext`` and its ``TurnRunner``; ``turn_params`` (history, context_prompt, + session_id, persist_user_*, …) are stored verbatim. Returns ``(turn_ctx, turn_runner, + cleanup_adapter)``. """ from gateway.run import TurnRunner - # Discord voice "verbal ack before tool calls": with the continuous mixer installed - # (discord.voice_fx.enabled), speak a short phrase on the FIRST tool call of the turn; once - # per turn. source.chat_id is the linked text channel; resolve the guild whose voice - # connection is bound to it (mirrors DiscordAdapter.play_tts). + # Discord voice "verbal ack" on the FIRST tool call (discord.voice_fx.enabled): resolve the + # guild whose voice connection is bound to this text channel (mirrors DiscordAdapter.play_tts). _voice_ack_guild: List[Optional[int]] = [None] if source.platform == Platform.DISCORD: _va = self.adapters.get(Platform.DISCORD) @@ -2857,30 +2771,14 @@ class GatewayTurnMixin: _cleanup_progress = False _cleanup_adapter = None + # The one-slot progress/holder containers shared with the callbacks are TurnContext defaults. turn_ctx = TurnContext( - source=source, + source=source, message=message, AIAgent=AIAgent, session_key=session_key, + run_generation=run_generation, _cleanup_progress=_cleanup_progress, _run_still_current=self._run_still_current_fn(session_key, run_generation), progress_queue=queue.Queue() if disp.needs_progress_queue else None, - # Mutable one-slot containers shared with the callbacks: last progress line (dedup), last - # tool, whether the previous line was a terminal fenced block (consecutive terminal calls - # drop the repeated header), repeat counter, first-touch long-tool onboarding latch. - last_progress_msg=[None], - last_tool=[None], - last_was_terminal_block=[False], - repeat_count=[0], - long_tool_hint_fired=[False], - _LONG_TOOL_THRESHOLD_S=30.0, - _cleanup_progress=_cleanup_progress, - _cleanup_msg_ids=[], - message=message, - AIAgent=AIAgent, - _voice_ack_fired=[False], - _voice_ack_guild=_voice_ack_guild, - _voice_ack_loop=asyncio.get_running_loop(), - session_key=session_key, - run_generation=run_generation, - **{name: getattr(disp, name) for name in self._DISPLAY_TO_TURN_CTX}, - **turn_params, + _voice_ack_guild=_voice_ack_guild, _voice_ack_loop=asyncio.get_running_loop(), + **{name: getattr(disp, name) for name in self._DISPLAY_TO_TURN_CTX}, **turn_params, ) turn_runner = TurnRunner(self, turn_ctx) # Agent tool-lifecycle callbacks live on the runner (bound methods, same signatures). @@ -2915,14 +2813,11 @@ class GatewayTurnMixin: def _run_agent_progress_threading( self, source: SessionSource, event_message_id: Optional[str], _native_slack_task_cards: bool ) -> Tuple[Optional[dict], Optional[str], Any, Optional[str]]: - """Resolve where progress bubbles are threaded. + """Resolve where progress bubbles are threaded (platform-specific). Returns ``(progress_metadata, progress_reply_to, progress_thread_id, relay_prospective_thread_id)``. - Threading is platform-specific: Slack DM threading needs the event_message_id fallback; - Telegram forum topics use message_thread_id and Hermes-created private DM topic lanes need - thread metadata plus a reply anchor; Feishu only honors reply_in_thread on a reply; others use - explicit source.thread_id only. Slack and Buzz honour the user's reply_in_thread opt-out: - never synthesise a thread for progress, or every later reply inherits it. + Slack and Buzz honour the user's reply_in_thread opt-out: never synthesise a thread for + progress, or every later reply inherits it. """ from gateway.run import _non_conversational_metadata, _resolve_progress_thread_id is_buzz = str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz" @@ -3025,11 +2920,8 @@ class GatewayTurnMixin: self, source: SessionSource, event_message_id: Optional[str], _progress_thread_id: Any, _relay_prospective_thread_id: Optional[str], ) -> Optional[Dict[str, Any]]: - """Thread metadata for status / approval / stream sends. - - Feishu topics only keep messages inside the topic when sent via the reply API with - reply_in_thread=true; those paths usually only get metadata, so carry the triggering - message id as a Feishu-specific fallback. + """Thread metadata for status / approval / stream sends; Feishu topics need the triggering + message id (reply API with reply_in_thread=true) so carry it as a fallback. """ if source.platform == Platform.FEISHU and source.thread_id and event_message_id: return {"thread_id": _progress_thread_id, "reply_to_message_id": event_message_id} @@ -3043,10 +2935,8 @@ class GatewayTurnMixin: ) -> None: """Start the streaming-TTS consumer for a voice-input turn on an auto-TTS chat. - Created on the gateway event-loop thread (here), NOT inside run_sync's executor worker: the - outer interrupt / finalisation paths reference it via ``streaming_tts_consumer_holder[0]``. - An inactive consumer (no streaming provider) leaves the holder None so the whole-file - fallback path runs. + Created on the gateway loop thread (not run_sync's executor); an inactive consumer leaves + the holder None so the whole-file fallback path runs. """ _stts_adapter = self._adapter_for_source(source) _is_voice_input = ( @@ -3088,11 +2978,8 @@ class GatewayTurnMixin: await stream_task async def _run_agent_track_agent(self, turn_ctx: TurnContext) -> None: - """Track this agent as running for the session (interrupt support) once it is created. - - Only promote the sentinel to the real agent if this run is still current: if /stop or /new - bumped the generation while we were spinning up, leave the newer run's slot alone (the - stale-result check in _handle_message_with_agent discards us). + """Track this agent as running for the session (interrupt support) once it is created — only + if this run is still current, else leave the newer run's slot alone. """ session_key, run_generation, agent_holder = turn_ctx.session_key, turn_ctx.run_generation, turn_ctx.agent_holder while agent_holder[0] is None: @@ -3116,10 +3003,9 @@ class GatewayTurnMixin: ) -> None: """Peek the adapter's pending event, transcribe voice, then signal the agent + abort streaming TTS. - Peek WITHOUT consuming: the message must stay in _pending_messages for the post-run - ``_dequeue_pending_event()`` (full MessageEvent + media). Popping here races: the agent may - finish before checking _interrupt_requested, losing it. Audio is transcribed BEFORE signaling - so voice messages interrupt with the real transcript, not an empty string / path placeholder. + Peek WITHOUT consuming: the event must stay for the post-run ``_dequeue_pending_event()`` + (popping races the agent finishing). Transcribe BEFORE signaling so voice interrupts carry + the real transcript. """ from gateway.run import _build_media_placeholder _peek_event = adapter._pending_messages.get(session_key) @@ -3144,10 +3030,8 @@ class GatewayTurnMixin: async def _run_agent_monitor_for_interrupt(self, turn_ctx: TurnContext, _interrupt_detected: "asyncio.Event") -> None: """Poll the adapter for interrupts (new messages) every 200ms and signal the agent. - PRIMARY interrupt path for regular text: Level 1 (base.py) catches them before - _handle_message(), so the Level 2 running_agent.interrupt() path never fires. The inactivity - poll loop has a BACKUP check in case this task dies silently. Interrupts are keyed by - session_key (build_session_key output), NOT source.chat_id. + Level 1 (base.py) catches regular text before _handle_message(); the inactivity poll loop + has a BACKUP check in case this task dies. Keyed by session_key, NOT source.chat_id. """ source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder streaming_tts_consumer_holder = turn_ctx.streaming_tts_consumer_holder @@ -3198,12 +3082,9 @@ class GatewayTurnMixin: @staticmethod def _run_agent_stream_confirmed_final_delivery(consumer, final_text: str, *, previewed: bool = False) -> bool: - """Return True only when the actual final reply reached the user. - - A successful finalize call is not proof the *content* was final: the edit may carry only the - last preview snapshot. Reconcile against the recorded turn-final payload: only a demonstrable - mismatch (False, incl. payload-less split delivery) overrides the flag; None keeps legacy - trust so timeout dedup isn't regressed. + """True only when the actual final reply reached the user: a finalize call may carry only the + last preview snapshot, so reconcile against the recorded payload — a demonstrable mismatch + (False) overrides the flag; None keeps legacy trust. """ if consumer is None: return False @@ -3226,11 +3107,9 @@ class GatewayTurnMixin: def _run_agent_start_turn_worker(self, turn_ctx: TurnContext, run_sync: Callable[[], Any]) -> "GatewayRunner._RunAgentWorker": """Schedule ``run_sync`` on the executor plus the inactivity watchdog thread. - *Inactivity* timeout, not wall-clock: the agent may run for hours while actively calling - tools / streaming, but a hung API call or stuck tool is killed. agent.gateway_timeout / - HERMES_AGENT_TIMEOUT (env wins); default 1800s; 0 = unlimited. The daemon watchdog is - independent of asyncio: cgroup memory reclaim can starve the loop that runs the normal - timeout poll, and cleanup must not wait for the loop to recover. + *Inactivity* timeout (agent.gateway_timeout / HERMES_AGENT_TIMEOUT, env wins; 0 = unlimited), + not wall-clock. The daemon watchdog is independent of asyncio: cgroup memory reclaim can + starve the loop that runs the normal timeout poll. """ from gateway.run import _float_env, _watch_gateway_turn_inactivity from tools.process_registry import process_registry @@ -3265,10 +3144,9 @@ class GatewayTurnMixin: return run_sync() finally: worker.worker_done.set() - # `.turn.agent` is only reset to _AGENT_PENDING_SENTINEL when the *next* turn is - # claimed, so this agent stays reachable from _interrupt_and_clear_session() until - # then. Clearing ownership markers the instant our worker finishes means a /stop on - # the finished turn no longer reaps background work it left running. + # `.turn.agent` stays reachable until the *next* turn is claimed; clearing the + # ownership markers now means a /stop on the finished turn no longer reaps background + # work it left running. _finished_agent = agent_holder[0] if agent_holder else None if _finished_agent is not None: _finished_agent._gateway_turn_process_task_id = "" @@ -3320,9 +3198,8 @@ class GatewayTurnMixin: try: await _warn_adapter.send( source.chat_id, f"⚠️ No activity for {_elapsed_warn} min. " - f"If the agent does not respond soon, it will " - f"be timed out in {_remaining_mins} min. " - f"You can continue waiting or use /reset.", + f"If the agent does not respond soon, it will be timed out in {_remaining_mins} " + f"min. You can continue waiting or use /reset.", metadata=_interim_metadata(_status_thread_metadata), ) except Exception as _warn_err: @@ -3354,9 +3231,8 @@ class GatewayTurnMixin: ] if _cur_tool: _diag_lines.append( - f"The agent appears stuck on tool `{_cur_tool}` " - f"({_secs_ago:.0f}s since last activity, " - f"iteration {_iter_n}/{_iter_max})." + f"The agent appears stuck on tool `{_cur_tool}` ({_secs_ago:.0f}s since last " + f"activity, iteration {_iter_n}/{_iter_max})." ) else: _diag_lines.append( @@ -3365,9 +3241,8 @@ class GatewayTurnMixin: "The agent may have been waiting on an API response." ) _diag_lines.append( - "To increase the limit, set agent.gateway_timeout in config.yaml " - "(value in seconds, 0 = no limit) and restart the gateway.\n" - "Try again, or use /reset to start fresh." + "To increase the limit, set agent.gateway_timeout in config.yaml (value in seconds, 0 " + "= no limit) and restart the gateway.\nTry again, or use /reset to start fresh." ) return { "final_response": "\n".join(_diag_lines), @@ -3382,12 +3257,9 @@ class GatewayTurnMixin: self, worker: "GatewayRunner._RunAgentWorker", turn_ctx: TurnContext, _interrupt_detected: "asyncio.Event", interrupt_monitor: "asyncio.Task", ) -> Any: - """Poll the executor future (inactivity timeout + backup interrupt checks); return its result. - - Polls the agent's activity tracker (updated by _touch_activity() on every tool call, API - call and stream delta) every few seconds; with an unlimited timeout it still polls for the - backup interrupt check in case monitor_for_interrupt() silently died. On inactivity timeout - the result is a synthetic failed run dict carrying the diagnostic. + """Poll the executor future (inactivity timeout + backup interrupt checks); return its result, + or a synthetic failed run dict on inactivity timeout. Polls even with an unlimited timeout + so the backup interrupt check runs if monitor_for_interrupt() silently died. """ from gateway.run import _abandon_timed_out_gateway_turn agent_holder = turn_ctx.agent_holder @@ -3410,19 +3282,16 @@ class GatewayTurnMixin: threading.Thread( target=_abandon_timed_out_gateway_turn, kwargs={"agent_holder": agent_holder, **self._reaper_kwargs(worker)}, - name=f"gateway-turn-reaper-{worker.task_id[:12]}", - daemon=True, + name=f"gateway-turn-reaper-{worker.task_id[:12]}", daemon=True, ).start() break await self._run_agent_backup_interrupt_check(turn_ctx, _interrupt_detected, interrupt_monitor) return self._run_agent_timeout_result(worker, turn_ctx) def _run_agent_evict_on_fallback(self, turn_ctx: TurnContext) -> None: - """Evict the cached agent when a fallback model activated on a successful run. - - So /model shows the actually-active model and the next message retries the primary. Skip - when the run failed — evicting forces MCP reinit on the next message for no benefit (bad - model → fallback → evict → recreate → same 400 loop burning CPU). + """Evict the cached agent when a fallback model activated on a SUCCESSFUL run (so /model shows + the active model and the next message retries the primary). Skip failed runs: evicting + would loop bad model → fallback → evict → recreate. """ from gateway.run import _resolve_gateway_model session_key = turn_ctx.session_key @@ -3444,12 +3313,9 @@ class GatewayTurnMixin: self._evict_cached_agent(session_key) async def _run_agent_finalize_streaming_tts(self, turn_ctx: TurnContext, adapter: Any) -> None: - """Finalize the streaming-TTS consumer on the outer event-loop thread. - - finish() here covers early returns from run_sync too. wait_complete() drains queued audio; - on timeout abort unconditionally to free the consumer task — audible streams retain - whole-file suppression (no replay from the start); silent streams stay eligible for the - whole-file fallback. + """Finalize the streaming-TTS consumer on the outer event-loop thread (covers early returns + from run_sync). On drain timeout abort to free the task — audible streams keep whole-file + suppression, silent streams stay eligible for the whole-file fallback. """ _stts = turn_ctx.streaming_tts_consumer_holder[0] if _stts is None: @@ -3481,10 +3347,8 @@ class GatewayTurnMixin: pending = None if result and adapter and session_key: pending_event = _dequeue_pending_event(adapter, session_key) - # /queue overflow: after consuming the adapter's "next-up" slot, promote the next queued - # event into it so the recursive run's drain will see it. Keeping the slot occupied for - # the whole FIFO chain preserves order and makes a mid-chain /queue route to overflow - # instead of jumping the queue. + # /queue overflow: promote the next queued event into the consumed "next-up" slot so the + # recursive drain sees it (keeps FIFO order; a mid-chain /queue can't jump the queue). pending_event = self._promote_queued_event(session_key, adapter, pending_event) if result.get("interrupted") and not pending_event and result.get("interrupt_message"): interrupt_message = result.get("interrupt_message") @@ -3749,11 +3613,10 @@ class GatewayTurnMixin: async def _run_agent_mark_streamed_delivery(self, response: Any, turn_ctx: TurnContext) -> None: """Set ``response["already_sent"]`` when streaming already delivered the final reply. - Never when the agent failed (the error is unseen content) or on "(empty)": interim text - ("Let me search…") set already_sent but is NOT the final answer; suppressing would leave the - user with silence. Both suppression flags reflect call success, not content, so reconcile - against the recorded turn-final payload: on mismatch (False, incl. payload-less split - delivery) neither flag may suppress the final send; None (no record) keeps legacy trust. + Never when the agent failed (the error is unseen content) or on "(empty)". Both suppression + flags reflect call success, not content, so reconcile against the recorded turn-final + payload: a mismatch (False, incl. payload-less split delivery) never suppresses; None (no + record) keeps legacy trust. """ _sc, source, session_key = turn_ctx.stream_consumer_holder[0], turn_ctx.source, turn_ctx.session_key if not isinstance(response, dict) or response.get("failed"): @@ -3790,10 +3653,8 @@ class GatewayTurnMixin: ) response["already_sent"] = True elif not _transformed and _stale_finalized and _sc is not None: - # Stale finalize: the streamed message holds only the last preview snapshot. Edit it up to - # the complete response; on edit failure leave already_sent unset so the normal send - # delivers. Not for split delivery: message_id is only the LAST chunk, so editing it would - # repeat every sealed head chunk — fall through to the normal send. + # Stale finalize: edit the streamed message up to the complete response (on failure the + # normal send delivers). Not for split delivery — message_id is only the LAST chunk. _sc_msg_id = _sc.message_id if getattr(_sc, "_turn_split_delivery", False): logger.info( @@ -3847,9 +3708,8 @@ class GatewayTurnMixin: # normal final-send is about to run. Log the decision inputs so a recurrence can be pinned # to "signal never set" vs "ack-pending race". logger.warning( - "Normal final-send NOT suppressed despite active stream " - "consumer for session %s: streamed=%s previewed=%s " - "content_delivered=%s transformed=%s final_len=%d — " + "Normal final-send NOT suppressed despite active stream consumer for session %s: " + "streamed=%s previewed=%s content_delivered=%s transformed=%s final_len=%d — " "possible duplicate send (see wecom ack-timeout RCA).", _sk, _streamed, _previewed, _content_delivered, _transformed, len(_final), ) @@ -3897,22 +3757,14 @@ class GatewayTurnMixin: self, turn_ctx: TurnContext, turn_runner: TurnRunner, source: SessionSource, event_message_id: Optional[str], _native_slack_task_cards: bool, ) -> Optional[Dict[str, Any]]: - """Resolve progress threading, then publish progress metadata, result holders and the - sync→async bridges onto ``turn_ctx``. - - Returns ``_status_thread_metadata``; the holders are read back via ``turn_ctx.*_holder``. - The one-slot holders share agent / result / tool-defs / stream consumer / streaming-TTS - consumer between run_sync's executor thread and the outer finalisation + interrupt paths. + """Resolve progress threading, then publish progress metadata and the sync→async bridges onto + ``turn_ctx`` (the one-slot holders shared with run_sync's executor thread are TurnContext + defaults). Returns ``_status_thread_metadata``. """ ( turn_ctx._progress_metadata, turn_ctx._progress_reply_to, _progress_thread_id, _relay_prospective_thread_id, ) = self._run_agent_progress_threading(source, event_message_id, _native_slack_task_cards) - turn_ctx.agent_holder = [None] - turn_ctx.result_holder = [None] - turn_ctx.tools_holder = [None] - turn_ctx.stream_consumer_holder = [None] - turn_ctx.streaming_tts_consumer_holder = [None] # Bridges: sync step_callback / event_callback / status_callback → async hooks.emit and # adapter.send (TurnRunner methods bound onto the shared ctx). turn_ctx._loop_for_step = asyncio.get_running_loop() @@ -3929,18 +3781,21 @@ class GatewayTurnMixin: return _status_thread_metadata async def _run_agent_notify_long_running( - self, disp: "GatewayRunner._RunAgentDisplay", turn_ctx: TurnContext, *, _executor_task_holder: list, - _NOTIFY_INTERVAL: Optional[float], _long_running_mode: str, _notify_start: float, + self, disp: "GatewayRunner._RunAgentDisplay", turn_ctx: TurnContext, _executor_task_holder: list, ) -> None: - """Periodic "still working" heartbeat, edited in place where the adapter supports it. + """Periodic "still working" heartbeat, edited in place where supported. Stops once this run + no longer owns the session slot or the executor finished. ``_executor_task_holder[0]`` is + bound just after this task is scheduled (reads as None until then). - Stops once this run no longer owns the session slot or the executor has finished, else a - stale "running: delegate_task" bubble outlives its run. ``_executor_task_holder[0]`` is bound - just after this task is scheduled; tolerate the brief window before then (reads as None). + Interval: agent.gateway_notify_interval / HERMES_AGENT_NOTIFY_INTERVAL (default 180s; 0 or + long_running_notifications=off disables). """ - from gateway.run import _interim_metadata, _non_conversational_metadata - if _NOTIFY_INTERVAL is None: - return # Notifications disabled (gateway_notify_interval: 0) + from gateway.run import _float_env, _interim_metadata, _non_conversational_metadata + _notify_start = time.time() + _NOTIFY_INTERVAL = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180) + _long_running_mode = disp._display_surface_mode("long_running_notifications", default=True, allow_generic=True) + if _NOTIFY_INTERVAL <= 0 or _long_running_mode == "off": + return source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder _status_thread_metadata = turn_ctx._status_thread_metadata _notify_adapter = self._adapter_for_source(source) @@ -4009,7 +3864,6 @@ class GatewayTurnMixin: Keys: "final_response", "messages", "api_calls", "completed". """ - from gateway.run import _float_env if self._get_proxy_url(): return await self._run_agent_via_proxy( message=message, context_prompt=context_prompt, history=history, source=source, @@ -4048,20 +3902,9 @@ class GatewayTurnMixin: _interrupt_detected = asyncio.Event() # shared with backup check interrupt_monitor = asyncio.create_task(self._run_agent_monitor_for_interrupt(turn_ctx, _interrupt_detected)) - # Periodic "still working" notifications so the user knows the agent hasn't died. Config: - # agent.gateway_notify_interval or HERMES_AGENT_NOTIFY_INTERVAL env; default 180s. - _NOTIFY_INTERVAL_RAW = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180) - _NOTIFY_INTERVAL = _NOTIFY_INTERVAL_RAW if _NOTIFY_INTERVAL_RAW > 0 else None - _long_running_mode = disp._display_surface_mode("long_running_notifications", default=True, allow_generic=True) - if _long_running_mode == "off": - _NOTIFY_INTERVAL = None + # Periodic "still working" notifications so the user knows the agent hasn't died. _executor_task_holder: list = [None] # bound once the executor future exists (see below) - _notify_task = asyncio.create_task( - self._run_agent_notify_long_running( - disp, turn_ctx, _executor_task_holder=_executor_task_holder, _NOTIFY_INTERVAL=_NOTIFY_INTERVAL, - _long_running_mode=_long_running_mode, _notify_start=time.time(), - ) - ) + _notify_task = asyncio.create_task(self._run_agent_notify_long_running(disp, turn_ctx, _executor_task_holder)) try: # run_sync is TurnRunner.run_sync (bound method; executor call unchanged).