refactor(gateway): run_turn — TurnContext defaults replace explicit holder inits, heartbeat task owns its interval resolution, docstring/comment compaction, AST-neutral layout

This commit is contained in:
Teknium
2026-09-02 23:34:47 -07:00
parent 01835246a6
commit a2007bc76f
+182 -339
View File
@@ -1,8 +1,6 @@
"""Agent-turn execution (_handle_message_with_agent, _run_agent*, proxy path, background tasks, MCP reload) for GatewayRunner.
Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO.
``gateway.run`` internals are imported lazily inside method bodies (import cycle),
so ``patch("gateway.run.X")`` keeps intercepting them at call time.
"""Agent-turn execution for GatewayRunner: _handle_message_with_agent, _run_agent*, proxy path,
background tasks, MCP reload. Bound onto ``GatewayRunner`` via the MRO; ``gateway.run`` internals
are imported lazily inside method bodies (import cycle) so ``patch("gateway.run.X")`` still works.
"""
from __future__ import annotations
@@ -24,10 +22,7 @@ from gateway.config import Platform
from gateway.media_repair import repair_explicit_computer_use_media_paths
from gateway.platforms.base import BasePlatformAdapter, MessageEvent
from gateway.session import (
SessionSource,
TranscriptReadError,
_session_key_namespace,
build_channel_continuity_note,
SessionSource, TranscriptReadError, _session_key_namespace, build_channel_continuity_note,
build_session_context,
)
from gateway.turn_context import TurnContext
@@ -45,7 +40,7 @@ logger = logging.getLogger("gateway.run")
class GatewayTurnMixin:
"""Agent-turn execution (_handle_message_with_agent, _run_agent*, proxy path, background tasks, MCP reload) for GatewayRunner."""
"""Agent-turn execution for GatewayRunner (see module docstring)."""
def _resolve_session_agent_runtime(
self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None,
@@ -151,8 +146,7 @@ class GatewayTurnMixin:
logger.warning(
"Empty model resolved for session=%s — recovering "
"last-known-good model %s (config read likely returned "
"empty; see #35314)",
skey or "", _recovered,
"empty; see #35314)", skey or "", _recovered,
)
model = _recovered
else:
@@ -164,11 +158,8 @@ class GatewayTurnMixin:
return model, runtime_kwargs
def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwargs: dict) -> dict:
"""Build the effective model/runtime config for a single turn.
Always uses the session's primary model/provider. With `/fast` priority on and a model that
supports it, fast-mode ``request_overrides`` are deep-merged OVER the per-provider
``request_overrides`` (e.g. ``custom_providers`` ``extra_body``) so both reach the model.
"""Effective model/runtime config for one turn. With `/fast` priority on, fast-mode
``request_overrides`` are deep-merged OVER the per-provider ones so both reach the model.
"""
from gateway.run import _deep_merge_request_overrides
from hermes_cli.models import resolve_fast_mode_overrides
@@ -408,10 +399,8 @@ class GatewayTurnMixin:
platform=source.platform, session_type=getattr(source, 'chat_type', 'dm'),
)
platform_name = source.platform.value if source.platform else ""
# Suspended / restart-recovery-expired sessions always notify regardless of
# policy.notify — an active session was silently replaced and the user must learn they
# can /resume it. Idle/daily resets respect the policy flag, the excluded platforms
# (api_server, webhook) and require prior activity.
# Suspended / restart-recovery-expired sessions always notify (the user must learn they
# can /resume); idle/daily resets respect policy.notify + excluded platforms + activity.
should_notify = reset_reason in {"suspended", "resume_pending_expired"} or (
policy.notify
and getattr(session_entry, 'reset_had_activity', False)
@@ -465,12 +454,10 @@ class GatewayTurnMixin:
logger.warning("[Gateway] Failed to auto-load skill(s) %s: %s", _skill_names, e)
async def _hmwa_acquire_turn_lease(self, _quick_key, run_generation, session_entry, _session_env_tokens):
"""Serialize [load history → run → flush] per resolved SESSION_ID (session resolution is
FINAL here): another routing key mapped to the same session_id waits for the prior flush
instead of loading a stale base. Fail-closed on timeout — never enter the transcript
region without a lease; outer dispatch returns a bounded resend notice. Released in
_handle_message's finally, granted per (routing key, run generation) so a stale unwind
can't release a newer turn's."""
"""Serialize [load history → run → flush] per resolved SESSION_ID so another routing key on
the same session waits for the prior flush. Fail-closed on timeout (outer dispatch returns
a resend notice). Released in _handle_message's finally, granted per (routing key, run
generation) so a stale unwind can't release a newer turn's."""
from gateway.run import _float_env
_lease_registry = getattr(self, "_turn_leases", None)
if _lease_registry is None:
@@ -543,13 +530,11 @@ class GatewayTurnMixin:
)
async def _hmwa_hygiene_settings(self, source, session_key):
"""Resolve model/provider/context-length + hygiene knobs for the pre-agent compression
safety net (fail-soft: any config/runtime error keeps the defaults).
"""Resolve model/provider/context-length + hygiene knobs (fail-soft: errors keep defaults).
The hygiene threshold (0.85) is deliberately HIGHER than the agent's own compressor
(0.50): it is a safety net for sessions that grew between turns. ``max_turn_hold_seconds``
bounds how long the user's TURN waits on hygiene before proceeding uncompressed (the
compressor keeps running detached, commit fenced); kept below transport idle-timeouts."""
The 0.85 threshold is deliberately HIGHER than the agent's compressor (0.50): a safety net
for sessions that grew between turns. ``max_turn_hold_seconds`` bounds the TURN wait
(compressor keeps running detached, commit fenced); kept below transport idle-timeouts."""
from gateway.run import _load_gateway_config
hs = self._HygieneSettings(
model="anthropic/claude-sonnet-4.6", threshold_pct=0.85, compression_enabled=True,
@@ -622,10 +607,8 @@ class GatewayTurnMixin:
_warn_token_threshold = int(_hyg_context_length * 0.95)
_msg_count = len(history)
# Prefer the API-reported prompt tokens from the last turn over the rough char estimate.
# Rough estimates run 30-50% high on code/JSON-heavy sessions, which only makes hygiene
# fire early (safe). Do NOT compensate with a threshold multiplier: 85% * 1.4 = 119% of
# context kept hygiene from ever firing for ~200K models.
# Prefer the API-reported prompt tokens over the rough estimate (runs 30-50% high, which only
# fires hygiene early — safe). Do NOT compensate with a threshold multiplier.
if session_entry.last_prompt_tokens > 0:
_approx_tokens, _token_source = session_entry.last_prompt_tokens, "actual"
else:
@@ -637,9 +620,8 @@ class GatewayTurnMixin:
_needs_compress = _approx_tokens >= _compress_token_threshold or _msg_count >= hs.hard_msg_limit
if _needs_compress:
# Persistent DB-backed cooldown (shared with context_compressor.py) so it survives
# gateway restarts; an in-memory dict re-triggered the same failing compression on
# every restart and wedged session storage.
# DB-backed cooldown (shared with context_compressor.py): survives gateway restarts, so a
# failing compression is not re-triggered on every restart.
_session_db = getattr(self, "_session_db", None)
if _session_db is not None:
_session_db = getattr(_session_db, "_db", _session_db)
@@ -681,10 +663,7 @@ class GatewayTurnMixin:
transcript; raises ``HygieneTurnHoldExceeded`` (turn-hold budget) or
``asyncio.TimeoutError`` (idle/ceiling/fence cancel) for the caller's handlers.
The timeout is an INACTIVITY budget — the worker ticks the fence per streamed token, so a
slow but still-generating model extends the deadline; a hard ceiling bounds the total so
a trickle stream can't hold the turn; the turn-hold budget caps how long the user's TURN
waits regardless."""
Idle timeout (fence ticks per streamed token) + hard ceiling + turn-hold cap."""
from gateway.run import HygieneTurnHoldExceeded, hygiene_wait_should_extend
fence = attempt.commit_fence
while True:
@@ -741,12 +720,9 @@ class GatewayTurnMixin:
raise
async def _hmwa_hygiene_cancel_before_commit(self, fence):
"""Cancel the detached worker at the commit fence. Returns ``False`` when the worker had
already crossed into its commit (caller must consume the result — aborting mid-commit
would corrupt the message-store transaction), ``True`` when cancelled in time.
A hung commit retains the fence lock; the lock-free ``commit_in_flight`` marker keeps
this loop from spinning forever. 25ms polls ride transient lock-setup windows."""
"""Cancel the detached worker at the commit fence: ``True`` when cancelled in time, ``False``
when it already crossed into its commit (caller must consume the result). The lock-free
``commit_in_flight`` marker keeps this loop from spinning on a hung commit."""
while True:
if fence.commit_in_flight:
return False
@@ -802,15 +778,12 @@ class GatewayTurnMixin:
notify the user, and re-raise; returns the compressed transcript only when the worker
was already committing.
Turn-hold expiry is an availability boundary, not a failure: the compressor is healthy,
so the failure streak must NOT advance — only flat, non-escalating retry spacing is
recorded. When the worker's commit is watermark-fenced (rows appended after compression
start — this turn included — survive its late commit as cloned concurrent tail) the
attempt KEEPS commit admission: the turn proceeds uncompressed NOW and the summary is
adopted at the worker's own fenced commit; always cancelling burned every attempt for
thinking summary models whose reasoning prefix alone exceeds the hold. Without the
watermark fence (no session_db, capture failed, legacy lock API) a late commit could
clobber newer turns, so cancel."""
Turn-hold expiry is an availability boundary, not a failure: the streak must NOT advance,
only flat retry spacing is recorded. A watermark-fenced commit (rows appended after
compression start survive as cloned tail) KEEPS admission: the turn proceeds uncompressed
now and the summary is adopted at the worker's fenced commit — always cancelling burned
every attempt for thinking summary models. Without the fence a late commit could clobber
newer turns, so cancel."""
from gateway.run import (
_HYGIENE_TURNHOLD_RETRY_SECONDS, _record_hygiene_cooldown, _reset_hygiene_failure_streak
)
@@ -818,11 +791,9 @@ class GatewayTurnMixin:
_hyg_keep_admission = bool(getattr(fence, "commit_watermark_fenced", False)) and not fence.is_cancelled
if _hyg_keep_admission:
self._hmwa_hygiene_defer_cleanup(attempt, "session hygiene turn-hold")
# NO retry-after here: the attempt is still running toward a real commit, and the flat
# 60s retry-after would also block the agent-side preflight compressor (same-session
# cooldown). Re-attempt spacing comes from the durable compression lock: the next
# turn's hygiene pre-check skips while this worker's lease is held. The done-callback
# records the flat retry-after ONLY if the worker ends without committing anything.
# NO retry-after here (it would also block the agent-side preflight compressor); spacing
# comes from the durable compression lock. The done-callback records the flat retry-after
# ONLY if the worker ends without committing anything.
_sid, _skey, _agent = session_entry.session_id, session_key, attempt.agent
def _hyg_adopt_or_space_retry(_fut, _gw=self, _sid=_sid, _skey=_skey, _agent=_agent):
@@ -950,13 +921,9 @@ class GatewayTurnMixin:
raise
def _hmwa_hygiene_on_unwind(self, attempt, hs, session_entry, session_key):
"""``except BaseException`` body (caller re-raises): revoke commit admission and record a
cooldown so the next turn does not immediately re-arm hygiene.
Non-timeout unwind (KeyboardInterrupt, task cancel, unexpected error) while the detached
worker may still run: revoke admission (and release its durable lease) BEFORE the host
unwinds so it can never commit later. Restart drain / task cancel must record a cooldown,
or the next turn re-arms hygiene and waits up to 600s behind a fence that refuses again."""
"""``except BaseException`` body (caller re-raises): revoke commit admission BEFORE the host
unwinds so the detached worker can never commit later, and record a cooldown — otherwise
the next turn re-arms hygiene and waits up to 600s behind a fence that refuses again."""
from gateway.run import _hygiene_cooldown_for_failure, _record_hygiene_cooldown
attempt.commit_fence.revoke_commit_admission()
if not attempt.cleanup_deferred:
@@ -983,17 +950,13 @@ class GatewayTurnMixin:
async def _hmwa_hygiene_adopt_transcript(
self, attempt, _compressed, history, plan, *, session_entry, source, _quick_key, run_generation,
):
"""Adopt a finished compression (rotation / in-place / refused) and publish the transcript
to continue with on ``attempt.history``.
"""Adopt a finished compression (rotation / in-place / refused); publishes the transcript to
continue with on ``attempt.history``.
Rewrite the transcript only when rotation produced a NEW session id. In-place compaction
needs none: archive_and_compact() already soft-archived the previous rows, and
rewrite_transcript() would replace_messages(active_only=False) and DELETE the archived
turns. A summary with neither rotation nor a completed archive_and_compact() signals
FAILURE; an unconditional rewrite would replace the originals with only the summary.
Write-before-repoint (mirrors manual /compress): if session_entry were repointed first and
rewrite_transcript then failed (lock/ENOSPC), the live entry would reference an empty
session and the conversation silently vanishes."""
Rewrite only on rotation (NEW session id): in-place compaction already soft-archived the
old rows and rewrite_transcript() would DELETE them; neither rotation nor in-place signals
FAILURE and an unconditional rewrite would leave only the summary. Write-before-repoint:
a repoint-then-failed-rewrite would point the live entry at an empty session."""
from agent.model_metadata import estimate_messages_tokens_rough
_hyg_agent = attempt.agent
# _compress_context ends the old session and creates a new session_id; compressed messages
@@ -1007,9 +970,8 @@ class GatewayTurnMixin:
_hyg_out_toks = estimate_messages_tokens_rough(_compressed)
if _hyg_rotated and _hyg_out_toks > _hyg_in_toks:
logger.warning(
"Gateway hygiene compression for session %s "
"would grow transcript (~%s -> ~%s tokens); "
"keeping the original transcript unchanged",
"Gateway hygiene compression for session %s would grow transcript (~%s -> ~%s "
"tokens); keeping the original transcript unchanged",
session_entry.session_id, f"{_hyg_in_toks:,}", f"{_hyg_out_toks:,}",
)
_hyg_rotated = False
@@ -1017,10 +979,8 @@ class GatewayTurnMixin:
if _hyg_rotated:
if not await self.async_session_store.rewrite_transcript(_hyg_new_sid, _compressed):
logger.error(
"Session hygiene: failed to persist "
"compressed transcript for rotated "
"session %s → %s; keeping the live "
"entry on the original session so the "
"Session hygiene: failed to persist compressed transcript for rotated session "
"%s → %s; keeping the live entry on the original session so the "
"conversation is not dropped", session_entry.session_id, _hyg_new_sid,
)
# Fail closed: treat like no rotation.
@@ -1048,16 +1008,13 @@ class GatewayTurnMixin:
_new_count = plan.msg_count
_new_tokens = plan.approx_tokens
logger.warning(
"Gateway hygiene compression for session %s "
"did not rotate or compact in place "
"(no session_db on the hygiene agent) — "
"preserving the original transcript instead "
"Gateway hygiene compression for session %s did not rotate or compact in place (no "
"session_db on the hygiene agent) — preserving the original transcript instead "
"of overwriting it with the summary (#21301).", session_entry.session_id,
)
logger.info(
"Session hygiene: compressed %s → %s msgs, "
"~%s → ~%s tokens",
"Session hygiene: compressed %s → %s msgs, ~%s → ~%s tokens",
plan.msg_count, _new_count, f"{plan.approx_tokens:,}", f"{_new_tokens:,}",
)
if _new_tokens >= plan.warn_token_threshold:
@@ -1115,10 +1072,8 @@ class GatewayTurnMixin:
await self._hmwa_hygiene_notify(
source, attempt.meta, "⚠️ Context compression aborted "
f"({_err}). No messages were dropped — "
"conversation is unchanged. Run /compress "
"to retry, /reset for a clean session, or "
"check your auxiliary.compression model "
"configuration.",
"conversation is unchanged. Run /compress to retry, /reset for a clean "
"session, or check your auxiliary.compression model configuration.",
"compression-failure warning",
)
# If the CONFIGURED aux model failed and we recovered on the main model, tell the user —
@@ -1150,8 +1105,7 @@ class GatewayTurnMixin:
failure_cooldown_seconds=hs.failure_cooldown_seconds,
)
logger.info(
"Session hygiene (codex app-server): %s "
"(session=%s, mode=%s, ~%s tokens)",
"Session hygiene (codex app-server): %s (session=%s, mode=%s, ~%s tokens)",
_hyg_codex_outcome, session_entry.session_id, _hyg_codex_auto, f"{plan.approx_tokens:,}",
)
@@ -1165,9 +1119,8 @@ class GatewayTurnMixin:
except Exception as exc:
_hyg_session_row = None
logger.warning(
"Session hygiene could not restore the system "
"prompt for session %s: %s. Preserving an empty "
"prompt so the live turn rebuilds it with its "
"Session hygiene could not restore the system prompt for session %s: %s. "
"Preserving an empty prompt so the live turn rebuilds it with its "
"configured providers.", session_entry.session_id, exc, exc_info=True,
)
_hyg_session_db = getattr(self._session_db, "_db", self._session_db)
@@ -1202,10 +1155,8 @@ class GatewayTurnMixin:
_hyg_agent, _hyg_session_db = await self._hmwa_hygiene_build_agent(_hyg_model, _hyg_runtime, session_entry)
attempt.agent = _hyg_agent
try:
# Hygiene runs before the turn and owns the session binding, so prefer in-place
# compaction: archive old rows under the same session id rather than minting a
# continuation child that must be published back to SessionStore/topic bindings.
# Without a SessionDB this stays False and the apply guard preserves it.
# Hygiene owns the session binding, so prefer in-place compaction over minting a
# continuation child. Without a SessionDB this stays False.
_hyg_agent.compression_in_place = True
_bind_hyg_state = getattr(getattr(_hyg_agent, "context_compressor", None), "bind_session_state", None)
if callable(_bind_hyg_state):
@@ -1216,10 +1167,8 @@ class GatewayTurnMixin:
loop = asyncio.get_running_loop()
_hyg_commit_fence = CompressionCommitFence(total_ceiling_seconds=hs.total_ceiling_seconds)
# Default executor (NOT self._get_executor): a fence-cancelled hung summary must never
# occupy an agent-work slot. But it MUST run in the caller's contextvars: under
# multiplex_profiles the secret scope / HERMES_HOME live in ContextVars, and an empty
# Context makes get_secret() fail closed → lossy truncation.
# Default executor (NOT self._get_executor): a hung summary must never occupy an
# agent-work slot. MUST run in the caller's contextvars (multiplex secret scope).
attempt.commit_fence = _hyg_commit_fence
attempt.future = loop.run_in_executor(
None,
@@ -1360,10 +1309,9 @@ class GatewayTurnMixin:
sethome_cmd = "/hermes sethome" if source.platform == Platform.SLACK else "/sethome"
await self._deliver_platform_notice(
source, f"📬 No home channel is set for {platform_name.title()}. "
f"A home channel is where Hermes delivers cron job results "
f"and cross-platform messages.\n\n"
f"Type {sethome_cmd} to make this chat your home channel, "
f"or ignore to skip.",
f"A home channel is where Hermes delivers cron job results and cross-platform "
f"messages.\n\nType {sethome_cmd} to make this chat your home channel, or ignore "
f"to skip.",
)
def _hmwa_apply_message_timestamp(self, event, message_text):
@@ -1435,9 +1383,8 @@ class GatewayTurnMixin:
# "(empty)" = the model produced no visible content after exhausting all retries.
if response == "(empty)" and not _intentional_silence:
response = (
"⚠️ The model returned no response after processing tool "
"results. This can happen with some models — try again or "
"rephrase your question."
"⚠️ The model returned no response after processing tool results. This can happen "
"with some models — try again or rephrase your question."
)
agent_messages = agent_result.get("messages", [])
logger.info(
@@ -1479,17 +1426,15 @@ class GatewayTurnMixin:
)
else:
logger.info(
"Skipping agent-result session split sync for %s because "
"the session binding moved from %s to %s before "
"compression finished",
"Skipping agent-result session split sync for %s because the session binding "
"moved from %s to %s before compression finished",
session_key or "?", _run_start_session_id, session_entry.session_id,
)
return response, _intentional_silence, agent_messages
# reasoning_style → (header line, per-line quote prefix for blank / non-blank lines)
_REASONING_QUOTE_STYLES = {
"subtext": ("-# 💭 Reasoning", "-# ", "-#"),
"blockquote": ("> 💭 **Reasoning:**", "> ", ">"),
"subtext": ("-# 💭 Reasoning", "-# ", "-#"), "blockquote": ("> 💭 **Reasoning:**", "> ", ">")
}
def _hmwa_prepend_reasoning(self, agent_result, response, source, _intentional_silence):
@@ -1593,10 +1538,8 @@ class GatewayTurnMixin:
"""Classify a finished turn for transcript persistence. Returns
``(agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure)``.
Context-overflow failures (compression exhausted, generic 400 on large sessions) must
NOT persist the user message — it would grow the session and reproduce the failure
forever. Transient failures (429, timeout, connection error, 5xx) DO persist it: the
session is not oversized and dropping the turn causes severe context loss on retry."""
Context-overflow failures must NOT persist the user message (session would grow and
reproduce the failure forever); transient failures (429/timeout/5xx) DO."""
from gateway.run import _is_gateway_hidden_reasoning_incomplete_turn
agent_failed_early = bool(agent_result.get("failed"))
hidden_reasoning_incomplete = _is_gateway_hidden_reasoning_incomplete_turn(agent_result)
@@ -1611,8 +1554,7 @@ class GatewayTurnMixin:
if is_context_overflow_failure:
logger.info(
"Skipping transcript persistence for context-overflow "
"failure in session %s to prevent session growth loop.",
session_entry.session_id,
"failure in session %s to prevent session growth loop.", session_entry.session_id,
)
elif agent_failed_early:
logger.info(
@@ -1621,8 +1563,7 @@ class GatewayTurnMixin:
)
elif hidden_reasoning_incomplete:
logger.warning(
"Suppressing hidden-reasoning-only incomplete gateway turn "
"for session %s: %s",
"Suppressing hidden-reasoning-only incomplete gateway turn for session %s: %s",
session_entry.session_id, agent_result.get("error", "processing incomplete"),
)
return agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure
@@ -1649,19 +1590,16 @@ class GatewayTurnMixin:
# dict (see _CONVERSATION_SCOPED_STATE).
self._clear_conversation_scope(session_key, reason="compression_exhausted_reset")
if new_entry is not None:
# Re-point the Telegram topic binding at the fresh session: compression rotated
# session_entry.session_id to the bloated child earlier this turn and that _sync
# also rewrote the (chat_id, thread_id) binding. Without a re-sync the binding-heal
# walk switches the next inbound message back onto the child and re-triggers
# exhaustion forever. No-op on non-topic lanes.
# Re-point the Telegram topic binding at the fresh session, or the binding-heal walk
# switches the next message back onto the bloated child and re-triggers exhaustion
# forever. No-op on non-topic lanes.
session_entry = new_entry
await asyncio.to_thread(
self._sync_telegram_topic_binding, source, session_entry, reason="compression-exhausted-reset",
)
response = (response or "") + (
"\n\n🔄 Session auto-reset — the conversation exceeded the "
"maximum context size and could not be compressed further. "
"Your next message will start a fresh session."
"\n\n🔄 Session auto-reset — the conversation exceeded the maximum context size and "
"could not be compressed further. Your next message will start a fresh session."
)
return response, session_entry
@@ -1714,15 +1652,12 @@ class GatewayTurnMixin:
"timestamp": ts,
})
if agent_failed_early or hidden_reasoning_incomplete:
# Transient failure (429/timeout/5xx): persist only the user message so the next
# message loads a transcript that reflects what was said; the assistant error text is
# a gateway-generated hint, not model output. Hidden-reasoning incomplete turns follow
# the same rule so peer-agent channels don't ingest them.
# Dedupe on platform message_id (Telegram retries after transient failures).
# Transient failure / hidden-reasoning incomplete: persist only the user message (the
# assistant error text is a gateway hint, not model output). Dedupe on platform
# message_id (Telegram retries after transient failures).
if event.message_id and await store.has_platform_message_id(sid, str(event.message_id)):
logger.info(
"Skipping duplicate user turn "
"(message_id=%s) in session %s",
"Skipping duplicate user turn (message_id=%s) in session %s",
event.message_id, sid,
)
else:
@@ -1730,8 +1665,8 @@ class GatewayTurnMixin:
sid, self._hmwa_user_transcript_entry(event, prepared, ts), skip_db=agent_persisted,
)
else:
# Only the NEW messages from this turn: use history_offset (what the agent saw), not
# len(history), which counts session_meta entries stripped before the agent saw them.
# Only the NEW messages: history_offset (what the agent saw), not len(history), which
# counts session_meta entries stripped before the agent saw them.
history_len = agent_result.get("history_offset", len(history))
new_messages = agent_messages[history_len:] if len(agent_messages) > history_len else []
if not new_messages:
@@ -1770,11 +1705,9 @@ class GatewayTurnMixin:
touch_activity=not bool(getattr(event, "internal", False)),
)
# Re-baseline the cached agent's message_count snapshot now that ALL of this turn's writes
# are done (flushed rows AND the first-turn `session_meta` marker, which bumps the count
# too). The cross-process coherence guard snapshots at agent-BUILD time and never
# refreshes on reuse, so our own writes would trigger a rebuild next turn (destroying
# prompt caching).
# Re-baseline the cached agent's message_count now that ALL of this turn's writes are done:
# the coherence guard snapshots at agent-BUILD time, so our own writes would otherwise
# trigger a rebuild next turn (destroying prompt caching).
await self._refresh_agent_cache_message_count(session_key, sid)
async def _hmwa_deliver_turn_response(
@@ -1883,9 +1816,8 @@ class GatewayTurnMixin:
# payload is too large for the API — treat it the same way.
if len(prepared.history) > 50:
return (
"⚠️ Session too large for the model's context window.\n"
"Use /compact to compress the conversation, or "
"/reset to start fresh."
"⚠️ Session too large for the model's context window.\nUse /compact to "
"compress the conversation, or /reset to start fresh."
)
elif status_code == 400:
status_hint = " The request was rejected by the API."
@@ -1922,10 +1854,8 @@ class GatewayTurnMixin:
a ``str`` is a reply to send instead of running (history unreadable); ``None`` drops the
turn (inbound text rejected)."""
from gateway.run import _load_gateway_config
# Synthetic self-injected turns (batch completions, watch notifications, resume wake-ups)
# arrive as MessageEvent(internal=True). Persist with display_kind="internal_notification"
# so UIs render timeline notices, not user bubbles. display_kind is a DB-only sidecar
# stripped from every provider-bound payload; role/content untouched.
# Self-injected turns (MessageEvent(internal=True)) persist with a DB-only display_kind so
# UIs render timeline notices, not user bubbles; role/content untouched.
persist_user_display_kind = "internal_notification" if getattr(event, "internal", False) else None
_redact_pii = False # privacy.redact_pii, re-read per message
with suppress(Exception):
@@ -1936,9 +1866,8 @@ class GatewayTurnMixin:
# drift turn-over-turn; a miss (thread rename, /sethome, redact_pii flip) re-renders.
context_prompt = self._pinned_session_context_prompt(context, _redact_pii, session_key)
# Per-turn must-deliver notes ride the user message via the api_content sidecar (staged
# below, consumed in run_sync → build_turn_context), NOT context_prompt: appending them to
# the ephemeral system prompt guaranteed a turn1→turn2 diff and a full agent rebuild.
# Per-turn notes ride the user message via the api_content sidecar, NOT context_prompt
# (appending to the ephemeral system prompt forced a full agent rebuild).
turn_sidecar_notes: List[str] = []
if _was_auto_reset:
await self._hmwa_deliver_auto_reset_notice(session_entry, source, turn_sidecar_notes)
@@ -1956,19 +1885,16 @@ class GatewayTurnMixin:
# routed message that never began processing if the gateway died while it was still waiting.
await self._mark_durable_active_turn(event, session_entry.session_key)
# An unreadable canonical store is not an empty conversation: stop before the agent can
# invent continuity from a plausible-looking []. This return happens before the broad
# cleanup finally, so restore task-local context here; the outer dispatch still clears the
# durable marker and turn lease.
# An unreadable store is not an empty conversation: stop before the agent invents continuity
# from []. Restore task-local context here (before the broad cleanup finally).
try:
history = await self.async_session_store.load_transcript(session_entry.session_id)
except TranscriptReadError:
self._clear_session_env(_session_env_tokens)
return (
"⚠️ This session's history is temporarily unavailable, so "
"this message was not processed. Ask the operator to inspect "
"state.db, then resend after it is healthy. Use /reset only "
"if you intentionally want to start a new conversation."
"⚠️ This session's history is temporarily unavailable, so this message was not "
"processed. Ask the operator to inspect state.db, then resend after it is healthy. "
"Use /reset only if you intentionally want to start a new conversation."
)
history = await self._hmwa_run_session_hygiene(
@@ -1977,9 +1903,8 @@ class GatewayTurnMixin:
await self._hmwa_first_contact_notes(source, history, turn_sidecar_notes)
# Voice channel state (who is present / speaking) rides the user message ONLY when changed
# since the previous turn; in the ephemeral system prompt it forced a full agent rebuild +
# prompt-cache re-key per message (the prompt carries a static pointer line instead).
# Voice channel state rides the user message ONLY when changed (in the system prompt it
# forced a rebuild + prompt-cache re-key per message).
_vc_note = self._voice_channel_sidecar_note(event, source, session_key)
if _vc_note:
turn_sidecar_notes.append(_vc_note)
@@ -2169,11 +2094,9 @@ class GatewayTurnMixin:
def _resolve_enabled_toolsets_for_source(
self, user_config: dict, source: "SessionSource", platform_key: str,
) -> list:
"""Enabled toolsets for an agent run, honoring an adapter ``toolsets_for_source()`` override.
The override (e.g. per-route webhook toolsets) is validated through the SAME
``_get_platform_tools`` path as normal platform config, so unknown and platform-restricted
toolsets are dropped rather than trusted. Absent an override: ``platform_toolsets.<platform>``.
"""Enabled toolsets for an agent run, honoring an adapter ``toolsets_for_source()`` override
validated through the SAME ``_get_platform_tools`` path (unknown / platform-restricted
toolsets dropped, not trusted).
"""
from hermes_cli.tools_config import _get_platform_tools
try:
@@ -2457,10 +2380,8 @@ class GatewayTurnMixin:
self, source: "SessionSource", scfg: Any, adapter: Any, *, on_missing_cursor: str,
) -> "tuple[Any, Optional[Callable[[], None]]]":
"""Build the shared ``StreamConsumerConfig`` and optional Telegram pause-typing closure.
``on_missing_cursor`` handles adapters with ``SUPPORTS_MESSAGE_EDITING = False``:
``"fallback"`` (proxy path) streams with an empty cursor; ``"raise"`` (in-process path)
raises ``RuntimeError`` so the caller's ``except`` skips streaming entirely.
For non-editing adapters ``on_missing_cursor="fallback"`` streams with an empty cursor;
``"raise"`` raises ``RuntimeError`` so the caller skips streaming entirely.
"""
from gateway.stream_consumer import StreamConsumerConfig
@@ -2468,10 +2389,8 @@ class GatewayTurnMixin:
if source.platform == Platform.TELEGRAM and hasattr(adapter, "pause_typing_for_chat"):
def _pause_typing_before_finalize(_adapter=adapter, _chat_id=source.chat_id) -> None:
_adapter.pause_typing_for_chat(_chat_id)
# Platforms that can't edit sent messages (e.g. QQ, WeChat) skip streaming entirely: the
# partial first message could never be updated, yielding duplicates (partial + final) —
# unless they have a native-streaming transport (e.g. WeCom msgtype "stream"), which the
# consumer's native branch delivers whole.
# Non-editing platforms (QQ, WeChat) skip streaming — the partial first message could never
# be updated — unless they have a native-streaming transport (WeCom msgtype "stream").
_adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True)
_adapter_supports_native_stream = bool(getattr(adapter, "SUPPORTS_NATIVE_STREAMING", False))
if not _adapter_supports_edit and not _adapter_supports_native_stream and on_missing_cursor == "raise":
@@ -2820,19 +2739,14 @@ class GatewayTurnMixin:
self, disp: "GatewayRunner._RunAgentDisplay", AIAgent: Any, *, message: str, source: SessionSource,
session_key: Optional[str], run_generation: Optional[int], **turn_params,
) -> Tuple[TurnContext, TurnRunner, Any]:
"""Build the progress holders, the ``TurnContext`` and its ``TurnRunner``.
``turn_params`` (history, context_prompt, channel_prompt, session_id, _interrupt_depth,
event_message_id, inbound_message_id, moa_config, persist_user_*) are stored verbatim.
Returns ``(turn_ctx, turn_runner, cleanup_adapter)``; the progress-bubble cleanup flags travel
on ``turn_ctx._cleanup_progress`` / ``turn_ctx._cleanup_msg_ids``.
"""Build the ``TurnContext`` and its ``TurnRunner``; ``turn_params`` (history, context_prompt,
session_id, persist_user_*, …) are stored verbatim. Returns ``(turn_ctx, turn_runner,
cleanup_adapter)``.
"""
from gateway.run import TurnRunner
# Discord voice "verbal ack before tool calls": with the continuous mixer installed
# (discord.voice_fx.enabled), speak a short phrase on the FIRST tool call of the turn; once
# per turn. source.chat_id is the linked text channel; resolve the guild whose voice
# connection is bound to it (mirrors DiscordAdapter.play_tts).
# Discord voice "verbal ack" on the FIRST tool call (discord.voice_fx.enabled): resolve the
# guild whose voice connection is bound to this text channel (mirrors DiscordAdapter.play_tts).
_voice_ack_guild: List[Optional[int]] = [None]
if source.platform == Platform.DISCORD:
_va = self.adapters.get(Platform.DISCORD)
@@ -2857,30 +2771,14 @@ class GatewayTurnMixin:
_cleanup_progress = False
_cleanup_adapter = None
# The one-slot progress/holder containers shared with the callbacks are TurnContext defaults.
turn_ctx = TurnContext(
source=source,
source=source, message=message, AIAgent=AIAgent, session_key=session_key,
run_generation=run_generation, _cleanup_progress=_cleanup_progress,
_run_still_current=self._run_still_current_fn(session_key, run_generation),
progress_queue=queue.Queue() if disp.needs_progress_queue else None,
# Mutable one-slot containers shared with the callbacks: last progress line (dedup), last
# tool, whether the previous line was a terminal fenced block (consecutive terminal calls
# drop the repeated header), repeat counter, first-touch long-tool onboarding latch.
last_progress_msg=[None],
last_tool=[None],
last_was_terminal_block=[False],
repeat_count=[0],
long_tool_hint_fired=[False],
_LONG_TOOL_THRESHOLD_S=30.0,
_cleanup_progress=_cleanup_progress,
_cleanup_msg_ids=[],
message=message,
AIAgent=AIAgent,
_voice_ack_fired=[False],
_voice_ack_guild=_voice_ack_guild,
_voice_ack_loop=asyncio.get_running_loop(),
session_key=session_key,
run_generation=run_generation,
**{name: getattr(disp, name) for name in self._DISPLAY_TO_TURN_CTX},
**turn_params,
_voice_ack_guild=_voice_ack_guild, _voice_ack_loop=asyncio.get_running_loop(),
**{name: getattr(disp, name) for name in self._DISPLAY_TO_TURN_CTX}, **turn_params,
)
turn_runner = TurnRunner(self, turn_ctx)
# Agent tool-lifecycle callbacks live on the runner (bound methods, same signatures).
@@ -2915,14 +2813,11 @@ class GatewayTurnMixin:
def _run_agent_progress_threading(
self, source: SessionSource, event_message_id: Optional[str], _native_slack_task_cards: bool
) -> Tuple[Optional[dict], Optional[str], Any, Optional[str]]:
"""Resolve where progress bubbles are threaded.
"""Resolve where progress bubbles are threaded (platform-specific).
Returns ``(progress_metadata, progress_reply_to, progress_thread_id, relay_prospective_thread_id)``.
Threading is platform-specific: Slack DM threading needs the event_message_id fallback;
Telegram forum topics use message_thread_id and Hermes-created private DM topic lanes need
thread metadata plus a reply anchor; Feishu only honors reply_in_thread on a reply; others use
explicit source.thread_id only. Slack and Buzz honour the user's reply_in_thread opt-out:
never synthesise a thread for progress, or every later reply inherits it.
Slack and Buzz honour the user's reply_in_thread opt-out: never synthesise a thread for
progress, or every later reply inherits it.
"""
from gateway.run import _non_conversational_metadata, _resolve_progress_thread_id
is_buzz = str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz"
@@ -3025,11 +2920,8 @@ class GatewayTurnMixin:
self, source: SessionSource, event_message_id: Optional[str], _progress_thread_id: Any,
_relay_prospective_thread_id: Optional[str],
) -> Optional[Dict[str, Any]]:
"""Thread metadata for status / approval / stream sends.
Feishu topics only keep messages inside the topic when sent via the reply API with
reply_in_thread=true; those paths usually only get metadata, so carry the triggering
message id as a Feishu-specific fallback.
"""Thread metadata for status / approval / stream sends; Feishu topics need the triggering
message id (reply API with reply_in_thread=true) so carry it as a fallback.
"""
if source.platform == Platform.FEISHU and source.thread_id and event_message_id:
return {"thread_id": _progress_thread_id, "reply_to_message_id": event_message_id}
@@ -3043,10 +2935,8 @@ class GatewayTurnMixin:
) -> None:
"""Start the streaming-TTS consumer for a voice-input turn on an auto-TTS chat.
Created on the gateway event-loop thread (here), NOT inside run_sync's executor worker: the
outer interrupt / finalisation paths reference it via ``streaming_tts_consumer_holder[0]``.
An inactive consumer (no streaming provider) leaves the holder None so the whole-file
fallback path runs.
Created on the gateway loop thread (not run_sync's executor); an inactive consumer leaves
the holder None so the whole-file fallback path runs.
"""
_stts_adapter = self._adapter_for_source(source)
_is_voice_input = (
@@ -3088,11 +2978,8 @@ class GatewayTurnMixin:
await stream_task
async def _run_agent_track_agent(self, turn_ctx: TurnContext) -> None:
"""Track this agent as running for the session (interrupt support) once it is created.
Only promote the sentinel to the real agent if this run is still current: if /stop or /new
bumped the generation while we were spinning up, leave the newer run's slot alone (the
stale-result check in _handle_message_with_agent discards us).
"""Track this agent as running for the session (interrupt support) once it is created — only
if this run is still current, else leave the newer run's slot alone.
"""
session_key, run_generation, agent_holder = turn_ctx.session_key, turn_ctx.run_generation, turn_ctx.agent_holder
while agent_holder[0] is None:
@@ -3116,10 +3003,9 @@ class GatewayTurnMixin:
) -> None:
"""Peek the adapter's pending event, transcribe voice, then signal the agent + abort streaming TTS.
Peek WITHOUT consuming: the message must stay in _pending_messages for the post-run
``_dequeue_pending_event()`` (full MessageEvent + media). Popping here races: the agent may
finish before checking _interrupt_requested, losing it. Audio is transcribed BEFORE signaling
so voice messages interrupt with the real transcript, not an empty string / path placeholder.
Peek WITHOUT consuming: the event must stay for the post-run ``_dequeue_pending_event()``
(popping races the agent finishing). Transcribe BEFORE signaling so voice interrupts carry
the real transcript.
"""
from gateway.run import _build_media_placeholder
_peek_event = adapter._pending_messages.get(session_key)
@@ -3144,10 +3030,8 @@ class GatewayTurnMixin:
async def _run_agent_monitor_for_interrupt(self, turn_ctx: TurnContext, _interrupt_detected: "asyncio.Event") -> None:
"""Poll the adapter for interrupts (new messages) every 200ms and signal the agent.
PRIMARY interrupt path for regular text: Level 1 (base.py) catches them before
_handle_message(), so the Level 2 running_agent.interrupt() path never fires. The inactivity
poll loop has a BACKUP check in case this task dies silently. Interrupts are keyed by
session_key (build_session_key output), NOT source.chat_id.
Level 1 (base.py) catches regular text before _handle_message(); the inactivity poll loop
has a BACKUP check in case this task dies. Keyed by session_key, NOT source.chat_id.
"""
source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder
streaming_tts_consumer_holder = turn_ctx.streaming_tts_consumer_holder
@@ -3198,12 +3082,9 @@ class GatewayTurnMixin:
@staticmethod
def _run_agent_stream_confirmed_final_delivery(consumer, final_text: str, *, previewed: bool = False) -> bool:
"""Return True only when the actual final reply reached the user.
A successful finalize call is not proof the *content* was final: the edit may carry only the
last preview snapshot. Reconcile against the recorded turn-final payload: only a demonstrable
mismatch (False, incl. payload-less split delivery) overrides the flag; None keeps legacy
trust so timeout dedup isn't regressed.
"""True only when the actual final reply reached the user: a finalize call may carry only the
last preview snapshot, so reconcile against the recorded payload — a demonstrable mismatch
(False) overrides the flag; None keeps legacy trust.
"""
if consumer is None:
return False
@@ -3226,11 +3107,9 @@ class GatewayTurnMixin:
def _run_agent_start_turn_worker(self, turn_ctx: TurnContext, run_sync: Callable[[], Any]) -> "GatewayRunner._RunAgentWorker":
"""Schedule ``run_sync`` on the executor plus the inactivity watchdog thread.
*Inactivity* timeout, not wall-clock: the agent may run for hours while actively calling
tools / streaming, but a hung API call or stuck tool is killed. agent.gateway_timeout /
HERMES_AGENT_TIMEOUT (env wins); default 1800s; 0 = unlimited. The daemon watchdog is
independent of asyncio: cgroup memory reclaim can starve the loop that runs the normal
timeout poll, and cleanup must not wait for the loop to recover.
*Inactivity* timeout (agent.gateway_timeout / HERMES_AGENT_TIMEOUT, env wins; 0 = unlimited),
not wall-clock. The daemon watchdog is independent of asyncio: cgroup memory reclaim can
starve the loop that runs the normal timeout poll.
"""
from gateway.run import _float_env, _watch_gateway_turn_inactivity
from tools.process_registry import process_registry
@@ -3265,10 +3144,9 @@ class GatewayTurnMixin:
return run_sync()
finally:
worker.worker_done.set()
# `.turn.agent` is only reset to _AGENT_PENDING_SENTINEL when the *next* turn is
# claimed, so this agent stays reachable from _interrupt_and_clear_session() until
# then. Clearing ownership markers the instant our worker finishes means a /stop on
# the finished turn no longer reaps background work it left running.
# `.turn.agent` stays reachable until the *next* turn is claimed; clearing the
# ownership markers now means a /stop on the finished turn no longer reaps background
# work it left running.
_finished_agent = agent_holder[0] if agent_holder else None
if _finished_agent is not None:
_finished_agent._gateway_turn_process_task_id = ""
@@ -3320,9 +3198,8 @@ class GatewayTurnMixin:
try:
await _warn_adapter.send(
source.chat_id, f"⚠️ No activity for {_elapsed_warn} min. "
f"If the agent does not respond soon, it will "
f"be timed out in {_remaining_mins} min. "
f"You can continue waiting or use /reset.",
f"If the agent does not respond soon, it will be timed out in {_remaining_mins} "
f"min. You can continue waiting or use /reset.",
metadata=_interim_metadata(_status_thread_metadata),
)
except Exception as _warn_err:
@@ -3354,9 +3231,8 @@ class GatewayTurnMixin:
]
if _cur_tool:
_diag_lines.append(
f"The agent appears stuck on tool `{_cur_tool}` "
f"({_secs_ago:.0f}s since last activity, "
f"iteration {_iter_n}/{_iter_max})."
f"The agent appears stuck on tool `{_cur_tool}` ({_secs_ago:.0f}s since last "
f"activity, iteration {_iter_n}/{_iter_max})."
)
else:
_diag_lines.append(
@@ -3365,9 +3241,8 @@ class GatewayTurnMixin:
"The agent may have been waiting on an API response."
)
_diag_lines.append(
"To increase the limit, set agent.gateway_timeout in config.yaml "
"(value in seconds, 0 = no limit) and restart the gateway.\n"
"Try again, or use /reset to start fresh."
"To increase the limit, set agent.gateway_timeout in config.yaml (value in seconds, 0 "
"= no limit) and restart the gateway.\nTry again, or use /reset to start fresh."
)
return {
"final_response": "\n".join(_diag_lines),
@@ -3382,12 +3257,9 @@ class GatewayTurnMixin:
self, worker: "GatewayRunner._RunAgentWorker", turn_ctx: TurnContext,
_interrupt_detected: "asyncio.Event", interrupt_monitor: "asyncio.Task",
) -> Any:
"""Poll the executor future (inactivity timeout + backup interrupt checks); return its result.
Polls the agent's activity tracker (updated by _touch_activity() on every tool call, API
call and stream delta) every few seconds; with an unlimited timeout it still polls for the
backup interrupt check in case monitor_for_interrupt() silently died. On inactivity timeout
the result is a synthetic failed run dict carrying the diagnostic.
"""Poll the executor future (inactivity timeout + backup interrupt checks); return its result,
or a synthetic failed run dict on inactivity timeout. Polls even with an unlimited timeout
so the backup interrupt check runs if monitor_for_interrupt() silently died.
"""
from gateway.run import _abandon_timed_out_gateway_turn
agent_holder = turn_ctx.agent_holder
@@ -3410,19 +3282,16 @@ class GatewayTurnMixin:
threading.Thread(
target=_abandon_timed_out_gateway_turn,
kwargs={"agent_holder": agent_holder, **self._reaper_kwargs(worker)},
name=f"gateway-turn-reaper-{worker.task_id[:12]}",
daemon=True,
name=f"gateway-turn-reaper-{worker.task_id[:12]}", daemon=True,
).start()
break
await self._run_agent_backup_interrupt_check(turn_ctx, _interrupt_detected, interrupt_monitor)
return self._run_agent_timeout_result(worker, turn_ctx)
def _run_agent_evict_on_fallback(self, turn_ctx: TurnContext) -> None:
"""Evict the cached agent when a fallback model activated on a successful run.
So /model shows the actually-active model and the next message retries the primary. Skip
when the run failed — evicting forces MCP reinit on the next message for no benefit (bad
model → fallback → evict → recreate → same 400 loop burning CPU).
"""Evict the cached agent when a fallback model activated on a SUCCESSFUL run (so /model shows
the active model and the next message retries the primary). Skip failed runs: evicting
would loop bad model → fallback → evict → recreate.
"""
from gateway.run import _resolve_gateway_model
session_key = turn_ctx.session_key
@@ -3444,12 +3313,9 @@ class GatewayTurnMixin:
self._evict_cached_agent(session_key)
async def _run_agent_finalize_streaming_tts(self, turn_ctx: TurnContext, adapter: Any) -> None:
"""Finalize the streaming-TTS consumer on the outer event-loop thread.
finish() here covers early returns from run_sync too. wait_complete() drains queued audio;
on timeout abort unconditionally to free the consumer task — audible streams retain
whole-file suppression (no replay from the start); silent streams stay eligible for the
whole-file fallback.
"""Finalize the streaming-TTS consumer on the outer event-loop thread (covers early returns
from run_sync). On drain timeout abort to free the task — audible streams keep whole-file
suppression, silent streams stay eligible for the whole-file fallback.
"""
_stts = turn_ctx.streaming_tts_consumer_holder[0]
if _stts is None:
@@ -3481,10 +3347,8 @@ class GatewayTurnMixin:
pending = None
if result and adapter and session_key:
pending_event = _dequeue_pending_event(adapter, session_key)
# /queue overflow: after consuming the adapter's "next-up" slot, promote the next queued
# event into it so the recursive run's drain will see it. Keeping the slot occupied for
# the whole FIFO chain preserves order and makes a mid-chain /queue route to overflow
# instead of jumping the queue.
# /queue overflow: promote the next queued event into the consumed "next-up" slot so the
# recursive drain sees it (keeps FIFO order; a mid-chain /queue can't jump the queue).
pending_event = self._promote_queued_event(session_key, adapter, pending_event)
if result.get("interrupted") and not pending_event and result.get("interrupt_message"):
interrupt_message = result.get("interrupt_message")
@@ -3749,11 +3613,10 @@ class GatewayTurnMixin:
async def _run_agent_mark_streamed_delivery(self, response: Any, turn_ctx: TurnContext) -> None:
"""Set ``response["already_sent"]`` when streaming already delivered the final reply.
Never when the agent failed (the error is unseen content) or on "(empty)": interim text
("Let me search…") set already_sent but is NOT the final answer; suppressing would leave the
user with silence. Both suppression flags reflect call success, not content, so reconcile
against the recorded turn-final payload: on mismatch (False, incl. payload-less split
delivery) neither flag may suppress the final send; None (no record) keeps legacy trust.
Never when the agent failed (the error is unseen content) or on "(empty)". Both suppression
flags reflect call success, not content, so reconcile against the recorded turn-final
payload: a mismatch (False, incl. payload-less split delivery) never suppresses; None (no
record) keeps legacy trust.
"""
_sc, source, session_key = turn_ctx.stream_consumer_holder[0], turn_ctx.source, turn_ctx.session_key
if not isinstance(response, dict) or response.get("failed"):
@@ -3790,10 +3653,8 @@ class GatewayTurnMixin:
)
response["already_sent"] = True
elif not _transformed and _stale_finalized and _sc is not None:
# Stale finalize: the streamed message holds only the last preview snapshot. Edit it up to
# the complete response; on edit failure leave already_sent unset so the normal send
# delivers. Not for split delivery: message_id is only the LAST chunk, so editing it would
# repeat every sealed head chunk — fall through to the normal send.
# Stale finalize: edit the streamed message up to the complete response (on failure the
# normal send delivers). Not for split delivery — message_id is only the LAST chunk.
_sc_msg_id = _sc.message_id
if getattr(_sc, "_turn_split_delivery", False):
logger.info(
@@ -3847,9 +3708,8 @@ class GatewayTurnMixin:
# normal final-send is about to run. Log the decision inputs so a recurrence can be pinned
# to "signal never set" vs "ack-pending race".
logger.warning(
"Normal final-send NOT suppressed despite active stream "
"consumer for session %s: streamed=%s previewed=%s "
"content_delivered=%s transformed=%s final_len=%d — "
"Normal final-send NOT suppressed despite active stream consumer for session %s: "
"streamed=%s previewed=%s content_delivered=%s transformed=%s final_len=%d — "
"possible duplicate send (see wecom ack-timeout RCA).",
_sk, _streamed, _previewed, _content_delivered, _transformed, len(_final),
)
@@ -3897,22 +3757,14 @@ class GatewayTurnMixin:
self, turn_ctx: TurnContext, turn_runner: TurnRunner, source: SessionSource,
event_message_id: Optional[str], _native_slack_task_cards: bool,
) -> Optional[Dict[str, Any]]:
"""Resolve progress threading, then publish progress metadata, result holders and the
sync→async bridges onto ``turn_ctx``.
Returns ``_status_thread_metadata``; the holders are read back via ``turn_ctx.*_holder``.
The one-slot holders share agent / result / tool-defs / stream consumer / streaming-TTS
consumer between run_sync's executor thread and the outer finalisation + interrupt paths.
"""Resolve progress threading, then publish progress metadata and the sync→async bridges onto
``turn_ctx`` (the one-slot holders shared with run_sync's executor thread are TurnContext
defaults). Returns ``_status_thread_metadata``.
"""
(
turn_ctx._progress_metadata, turn_ctx._progress_reply_to, _progress_thread_id,
_relay_prospective_thread_id,
) = self._run_agent_progress_threading(source, event_message_id, _native_slack_task_cards)
turn_ctx.agent_holder = [None]
turn_ctx.result_holder = [None]
turn_ctx.tools_holder = [None]
turn_ctx.stream_consumer_holder = [None]
turn_ctx.streaming_tts_consumer_holder = [None]
# Bridges: sync step_callback / event_callback / status_callback → async hooks.emit and
# adapter.send (TurnRunner methods bound onto the shared ctx).
turn_ctx._loop_for_step = asyncio.get_running_loop()
@@ -3929,18 +3781,21 @@ class GatewayTurnMixin:
return _status_thread_metadata
async def _run_agent_notify_long_running(
self, disp: "GatewayRunner._RunAgentDisplay", turn_ctx: TurnContext, *, _executor_task_holder: list,
_NOTIFY_INTERVAL: Optional[float], _long_running_mode: str, _notify_start: float,
self, disp: "GatewayRunner._RunAgentDisplay", turn_ctx: TurnContext, _executor_task_holder: list,
) -> None:
"""Periodic "still working" heartbeat, edited in place where the adapter supports it.
"""Periodic "still working" heartbeat, edited in place where supported. Stops once this run
no longer owns the session slot or the executor finished. ``_executor_task_holder[0]`` is
bound just after this task is scheduled (reads as None until then).
Stops once this run no longer owns the session slot or the executor has finished, else a
stale "running: delegate_task" bubble outlives its run. ``_executor_task_holder[0]`` is bound
just after this task is scheduled; tolerate the brief window before then (reads as None).
Interval: agent.gateway_notify_interval / HERMES_AGENT_NOTIFY_INTERVAL (default 180s; 0 or
long_running_notifications=off disables).
"""
from gateway.run import _interim_metadata, _non_conversational_metadata
if _NOTIFY_INTERVAL is None:
return # Notifications disabled (gateway_notify_interval: 0)
from gateway.run import _float_env, _interim_metadata, _non_conversational_metadata
_notify_start = time.time()
_NOTIFY_INTERVAL = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180)
_long_running_mode = disp._display_surface_mode("long_running_notifications", default=True, allow_generic=True)
if _NOTIFY_INTERVAL <= 0 or _long_running_mode == "off":
return
source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder
_status_thread_metadata = turn_ctx._status_thread_metadata
_notify_adapter = self._adapter_for_source(source)
@@ -4009,7 +3864,6 @@ class GatewayTurnMixin:
Keys: "final_response", "messages", "api_calls", "completed".
"""
from gateway.run import _float_env
if self._get_proxy_url():
return await self._run_agent_via_proxy(
message=message, context_prompt=context_prompt, history=history, source=source,
@@ -4048,20 +3902,9 @@ class GatewayTurnMixin:
_interrupt_detected = asyncio.Event() # shared with backup check
interrupt_monitor = asyncio.create_task(self._run_agent_monitor_for_interrupt(turn_ctx, _interrupt_detected))
# Periodic "still working" notifications so the user knows the agent hasn't died. Config:
# agent.gateway_notify_interval or HERMES_AGENT_NOTIFY_INTERVAL env; default 180s.
_NOTIFY_INTERVAL_RAW = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180)
_NOTIFY_INTERVAL = _NOTIFY_INTERVAL_RAW if _NOTIFY_INTERVAL_RAW > 0 else None
_long_running_mode = disp._display_surface_mode("long_running_notifications", default=True, allow_generic=True)
if _long_running_mode == "off":
_NOTIFY_INTERVAL = None
# Periodic "still working" notifications so the user knows the agent hasn't died.
_executor_task_holder: list = [None] # bound once the executor future exists (see below)
_notify_task = asyncio.create_task(
self._run_agent_notify_long_running(
disp, turn_ctx, _executor_task_holder=_executor_task_holder, _NOTIFY_INTERVAL=_NOTIFY_INTERVAL,
_long_running_mode=_long_running_mode, _notify_start=time.time(),
)
)
_notify_task = asyncio.create_task(self._run_agent_notify_long_running(disp, turn_ctx, _executor_task_holder))
try:
# run_sync is TurnRunner.run_sync (bound method; executor call unchanged).