From 01835246a624df8a02c223ad5d6fb81e06248a68 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:16:04 -0700 Subject: [PATCH 1/6] refactor(gateway): run_turn _run_agent_* helpers read turn identity from TurnContext instead of 5-12 forwarded params; _run_agent forwards **turn_kwargs --- gateway/run_turn.py | 231 +++++++++++++++----------------------------- 1 file changed, 78 insertions(+), 153 deletions(-) diff --git a/gateway/run_turn.py b/gateway/run_turn.py index 31cee08c46..325eac294a 100644 --- a/gateway/run_turn.py +++ b/gateway/run_turn.py @@ -2683,25 +2683,12 @@ class GatewayTurnMixin: async def _run_agent( self, message: str, context_prompt: str, history: List[Dict[str, Any]], - source: SessionSource, session_id: str, session_key: str = None, - run_generation: Optional[int] = None, _interrupt_depth: int = 0, - event_message_id: Optional[str] = None, inbound_message_id: Optional[str] = None, - channel_prompt: Optional[str] = None, moa_config: Optional[dict] = None, - persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None, - persist_user_display_kind: Optional[str] = None, message_type: Optional[str] = None, + source: SessionSource, session_id: str, **turn_kwargs, ) -> Dict[str, Any]: - """Profile-scoping wrapper around the agent run (pass-through when multiplexing is off).""" + """Profile-scoping wrapper around ``_run_agent_inner`` (same keyword parameters; pass-through + when multiplexing is off).""" with self._profile_scope_for_source(source): - return await self._run_agent_inner( - message, context_prompt, history, source, session_id, - session_key=session_key, run_generation=run_generation, - _interrupt_depth=_interrupt_depth, event_message_id=event_message_id, - inbound_message_id=inbound_message_id, - channel_prompt=channel_prompt, moa_config=moa_config, - persist_user_message=persist_user_message, - persist_user_timestamp=persist_user_timestamp, - persist_user_display_kind=persist_user_display_kind, message_type=message_type, - ) + return await self._run_agent_inner(message, context_prompt, history, source, session_id, **turn_kwargs) def _run_agent_display_settings(self, source: SessionSource) -> "GatewayRunner._RunAgentDisplay": """Resolve per-platform display, progress, status and streaming-surface settings for a turn.""" @@ -3100,15 +3087,14 @@ class GatewayTurnMixin: with suppress(asyncio.CancelledError): await stream_task - async def _run_agent_track_agent( - self, session_key: Optional[str], run_generation: Optional[int], agent_holder: list, - ) -> None: + async def _run_agent_track_agent(self, turn_ctx: TurnContext) -> None: """Track this agent as running for the session (interrupt support) once it is created. Only promote the sentinel to the real agent if this run is still current: if /stop or /new bumped the generation while we were spinning up, leave the newer run's slot alone (the stale-result check in _handle_message_with_agent discards us). """ + session_key, run_generation, agent_holder = turn_ctx.session_key, turn_ctx.run_generation, turn_ctx.agent_holder while agent_holder[0] is None: await asyncio.sleep(0.05) if not session_key: @@ -3155,10 +3141,7 @@ class GatewayTurnMixin: if _stts is not None: _stts.abort("barge-in") - async def _run_agent_monitor_for_interrupt( - self, source: SessionSource, session_key: Optional[str], agent_holder: list, - _interrupt_detected: "asyncio.Event", streaming_tts_consumer_holder: list, - ) -> None: + async def _run_agent_monitor_for_interrupt(self, turn_ctx: TurnContext, _interrupt_detected: "asyncio.Event") -> None: """Poll the adapter for interrupts (new messages) every 200ms and signal the agent. PRIMARY interrupt path for regular text: Level 1 (base.py) catches them before @@ -3166,6 +3149,8 @@ class GatewayTurnMixin: poll loop has a BACKUP check in case this task dies silently. Interrupts are keyed by session_key (build_session_key output), NOT source.chat_id. """ + source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder + streaming_tts_consumer_holder = turn_ctx.streaming_tts_consumer_holder if not session_key: return while True: @@ -3190,21 +3175,20 @@ class GatewayTurnMixin: logger.debug("monitor_for_interrupt error (will retry): %s", _mon_err) async def _run_agent_backup_interrupt_check( - self, source: SessionSource, session_key: Optional[str], agent_holder: list, - _interrupt_detected: "asyncio.Event", interrupt_monitor: "asyncio.Task", - streaming_tts_consumer_holder: list, + self, turn_ctx: TurnContext, _interrupt_detected: "asyncio.Event", interrupt_monitor: "asyncio.Task", ) -> None: """Backup interrupt check: if the monitor task died or missed the interrupt, catch it here.""" + source, session_key = turn_ctx.source, turn_ctx.session_key if _interrupt_detected.is_set() or not session_key: return _backup_adapter = self._adapter_for_source(source) - _backup_agent = agent_holder[0] + _backup_agent = turn_ctx.agent_holder[0] if (_backup_adapter and _backup_agent and hasattr(_backup_adapter, 'has_pending_interrupt') and _backup_adapter.has_pending_interrupt(session_key)): await self._run_agent_fire_pending_interrupt( _backup_adapter, _backup_agent, source, session_key, _interrupt_detected, - streaming_tts_consumer_holder, + turn_ctx.streaming_tts_consumer_holder, log_context="Voice-backup-interrupt", log=lambda: logger.info( "Backup interrupt detected for session %s (monitor task state: %s)", @@ -3239,10 +3223,7 @@ class GatewayTurnMixin: return False return False - def _run_agent_start_turn_worker( - self, turn_ctx: TurnContext, run_sync: Callable[[], Any], agent_holder: list, - session_id: str, session_key: Optional[str], run_generation: Optional[int], - ) -> "GatewayRunner._RunAgentWorker": + def _run_agent_start_turn_worker(self, turn_ctx: TurnContext, run_sync: Callable[[], Any]) -> "GatewayRunner._RunAgentWorker": """Schedule ``run_sync`` on the executor plus the inactivity watchdog thread. *Inactivity* timeout, not wall-clock: the agent may run for hours while actively calling @@ -3253,6 +3234,7 @@ class GatewayTurnMixin: """ from gateway.run import _float_env, _watch_gateway_turn_inactivity from tools.process_registry import process_registry + agent_holder, session_key, run_generation = turn_ctx.agent_holder, turn_ctx.session_key, turn_ctx.run_generation _agent_timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) _agent_warning = _float_env("HERMES_AGENT_TIMEOUT_WARNING", 900) _agent_timeout = _agent_timeout if _agent_timeout > 0 else None @@ -3260,7 +3242,7 @@ class GatewayTurnMixin: # A background=true process intentionally survives a successful turn, so capture existing IDs # and reap only children created by THIS turn if it times out. - _turn_task_id = session_id or "" + _turn_task_id = turn_ctx.session_id or "" _turn_process_baseline = process_registry.snapshot_running_ids(_turn_task_id) turn_ctx.process_task_id = _turn_task_id turn_ctx.process_baseline = _turn_process_baseline @@ -3346,11 +3328,12 @@ class GatewayTurnMixin: except Exception as _warn_err: logger.debug("Inactivity warning send error: %s", _warn_err) - def _run_agent_timeout_result(self, worker, session_key, agent_holder, result_holder, tools_holder) -> dict: + def _run_agent_timeout_result(self, worker, turn_ctx: TurnContext) -> dict: """Synthetic failed run dict for an inactivity timeout, with the activity-tracker diagnostic; interrupts the agent if it is still running so the thread pool worker is freed.""" from gateway.run import _INTERRUPT_REASON_TIMEOUT, request_hard_interrupt - _timed_out_agent = agent_holder[0] + session_key, result_holder, tools_holder = turn_ctx.session_key, turn_ctx.result_holder, turn_ctx.tools_holder + _timed_out_agent = turn_ctx.agent_holder[0] _activity = self._agent_activity_summary(_timed_out_agent) _last_desc = _activity.get("last_activity_desc", "unknown") _secs_ago = _activity.get("seconds_since_activity", 0) @@ -3396,10 +3379,8 @@ class GatewayTurnMixin: } async def _run_agent_await_turn_worker( - self, worker: "GatewayRunner._RunAgentWorker", *, source: SessionSource, - session_key: Optional[str], agent_holder: list, result_holder: list, tools_holder: list, + self, worker: "GatewayRunner._RunAgentWorker", turn_ctx: TurnContext, _interrupt_detected: "asyncio.Event", interrupt_monitor: "asyncio.Task", - streaming_tts_consumer_holder: list, _status_thread_metadata: Optional[Dict[str, Any]], ) -> Any: """Poll the executor future (inactivity timeout + backup interrupt checks); return its result. @@ -3409,6 +3390,7 @@ class GatewayTurnMixin: the result is a synthetic failed run dict carrying the diagnostic. """ from gateway.run import _abandon_timed_out_gateway_turn + agent_holder = turn_ctx.agent_holder _warning_fired = False while True: done, _ = await asyncio.wait({worker.executor_task}, timeout=5.0) @@ -3423,7 +3405,7 @@ class GatewayTurnMixin: _idle_secs = self._agent_activity_summary(agent_holder[0]).get("seconds_since_activity", 0.0) if not _warning_fired and worker.agent_warning is not None and _idle_secs >= worker.agent_warning: _warning_fired = True - await self._run_agent_inactivity_warning(worker, source, _status_thread_metadata) + await self._run_agent_inactivity_warning(worker, turn_ctx.source, turn_ctx._status_thread_metadata) if _idle_secs >= worker.agent_timeout: threading.Thread( target=_abandon_timed_out_gateway_turn, @@ -3432,15 +3414,10 @@ class GatewayTurnMixin: daemon=True, ).start() break - await self._run_agent_backup_interrupt_check( - source, session_key, agent_holder, _interrupt_detected, interrupt_monitor, - streaming_tts_consumer_holder, - ) - return self._run_agent_timeout_result(worker, session_key, agent_holder, result_holder, tools_holder) + await self._run_agent_backup_interrupt_check(turn_ctx, _interrupt_detected, interrupt_monitor) + return self._run_agent_timeout_result(worker, turn_ctx) - def _run_agent_evict_on_fallback( - self, session_key: Optional[str], agent_holder: list, result_holder: list, - ) -> None: + def _run_agent_evict_on_fallback(self, turn_ctx: TurnContext) -> None: """Evict the cached agent when a fallback model activated on a successful run. So /model shows the actually-active model and the next message retries the primary. Skip @@ -3448,8 +3425,9 @@ class GatewayTurnMixin: model → fallback → evict → recreate → same 400 loop burning CPU). """ from gateway.run import _resolve_gateway_model - _agent = agent_holder[0] - _result_for_fb = result_holder[0] + session_key = turn_ctx.session_key + _agent = turn_ctx.agent_holder[0] + _result_for_fb = turn_ctx.result_holder[0] _run_failed = _result_for_fb.get("failed") if _result_for_fb else False if _agent is None or not hasattr(_agent, 'model') or _run_failed: return @@ -3465,10 +3443,7 @@ class GatewayTurnMixin: if _agent.model != _cfg_model and not self._is_intentional_model_switch(session_key, _agent.model): self._evict_cached_agent(session_key) - async def _run_agent_finalize_streaming_tts( - self, streaming_tts_consumer_holder: list, adapter: Any, session_key: Optional[str], - run_generation: Optional[int], - ) -> None: + async def _run_agent_finalize_streaming_tts(self, turn_ctx: TurnContext, adapter: Any) -> None: """Finalize the streaming-TTS consumer on the outer event-loop thread. finish() here covers early returns from run_sync too. wait_complete() drains queued audio; @@ -3476,7 +3451,7 @@ class GatewayTurnMixin: whole-file suppression (no replay from the start); silent streams stay eligible for the whole-file fallback. """ - _stts = streaming_tts_consumer_holder[0] + _stts = turn_ctx.streaming_tts_consumer_holder[0] if _stts is None: return _stts.finish() @@ -3490,7 +3465,7 @@ class GatewayTurnMixin: if _stts.suppress_whole_file and adapter is not None: _mark_turn = getattr(adapter, "_mark_streaming_tts_completed_turn", None) if callable(_mark_turn): - _mark_turn(session_key, run_generation) + _mark_turn(turn_ctx.session_key, turn_ctx.run_generation) async def _run_agent_drain_pending( self, result: Any, adapter: Any, source: SessionSource, session_key: Optional[str] @@ -3567,13 +3542,11 @@ class GatewayTurnMixin: return pending_event, pending async def _run_agent_deliver_first_response( - self, *, source: SessionSource, adapter: Any, session_key: Optional[str], - run_generation: Optional[int], event_message_id: Optional[str], response: Any, result: Any, - stream_consumer_holder: list, stream_task: Any, - _status_thread_metadata: Optional[Dict[str, Any]], + self, turn_ctx: TurnContext, adapter: Any, response: Any, result: Any, stream_task: Any, ) -> None: """Deliver the first response before a queued follow-up runs, unless streaming already did.""" - _sc = stream_consumer_holder[0] + session_key = turn_ctx.session_key + _sc = turn_ctx.stream_consumer_holder[0] if _sc and stream_task: try: await self._await_stream_task(stream_task) @@ -3603,8 +3576,8 @@ class GatewayTurnMixin: session_key or "?", ) await self._deliver_queued_first_response( - first_response, source=source, adapter=adapter, - metadata=_status_thread_metadata, event_message_id=event_message_id, + first_response, source=turn_ctx.source, adapter=adapter, + metadata=turn_ctx._status_thread_metadata, event_message_id=turn_ctx.event_message_id, text_already_delivered=_already_streamed, deliver_media=not _delivery_result.get("failed"), stream_consumer=_sc, ) @@ -3612,7 +3585,7 @@ class GatewayTurnMixin: logger.warning("Failed to send first response before queued message: %s", e) # Release deferred bg-review notifications now that the first response is delivered: pop from # the adapter's callback dict (no double-fire in base.py's finally) and call. - _bg_cb = self._pop_post_delivery_callback(adapter, session_key, run_generation) + _bg_cb = self._pop_post_delivery_callback(adapter, session_key, turn_ctx.run_generation) if callable(_bg_cb): with suppress(Exception): _bg_result = _bg_cb() @@ -3620,15 +3593,17 @@ class GatewayTurnMixin: await _bg_result async def _run_agent_queued_followup( - self, *, source: SessionSource, adapter: Any, session_id: str, session_key: Optional[str], - run_generation: Optional[int], _interrupt_depth: int, event_message_id: Optional[str], - context_prompt: str, history: List[Dict[str, Any]], pending: Optional[str], - pending_event: Any, response: Any, result: Any, result_holder: list, - stream_consumer_holder: list, stream_task: Any, - _status_thread_metadata: Optional[Dict[str, Any]], + self, turn_ctx: TurnContext, adapter: Any, pending: Optional[str], pending_event: Any, + response: Any, result: Any, stream_task: Any, ) -> Any: """Run the queued / interrupting follow-up as the next turn (recursive ``_run_agent``).""" from gateway.run import _preserve_queued_followup_history_offset, merge_pending_message_event + source, session_id, session_key, run_generation = ( + turn_ctx.source, turn_ctx.session_id, turn_ctx.session_key, turn_ctx.run_generation, + ) + _interrupt_depth, history, _status_thread_metadata = ( + turn_ctx._interrupt_depth, turn_ctx.history, turn_ctx._status_thread_metadata, + ) logger.debug("Processing pending message: '%s...'", pending[:40]) # Clear the adapter's interrupt event so the next _run_agent call doesn't re-trigger the @@ -3648,17 +3623,12 @@ class GatewayTurnMixin: merge_pending_message_event(adapter._pending_messages, session_key, pending_event) elif adapter and hasattr(adapter, 'queue_message'): adapter.queue_message(session_key, pending) - return result_holder[0] or {"final_response": response, "messages": history} + return turn_ctx.result_holder[0] or {"final_response": response, "messages": history} # Interrupted: discard the response ("Operation interrupted." is noise; the user knows they # sent a new message). if not result.get("interrupted"): - await self._run_agent_deliver_first_response( - source=source, adapter=adapter, session_key=session_key, - run_generation=run_generation, event_message_id=event_message_id, response=response, - result=result, stream_consumer_holder=stream_consumer_holder, - stream_task=stream_task, _status_thread_metadata=_status_thread_metadata, - ) + await self._run_agent_deliver_first_response(turn_ctx, adapter, response, result, stream_task) updated_history = result.get("messages", history) next_source = source @@ -3719,7 +3689,7 @@ class GatewayTurnMixin: await self._refresh_agent_cache_message_count(session_key, session_id) followup_result = await self._run_agent( - message=next_message, context_prompt=context_prompt, history=updated_history, + message=next_message, context_prompt=turn_ctx.context_prompt, history=updated_history, source=next_source, session_id=session_id, session_key=next_session_key, run_generation=run_generation, _interrupt_depth=_interrupt_depth + 1, event_message_id=next_message_id, channel_prompt=next_channel_prompt, @@ -3728,12 +3698,11 @@ class GatewayTurnMixin: return _preserve_queued_followup_history_offset(result, followup_result) async def _run_agent_cleanup_turn_tasks( - self, *, progress_task: Any, log_task: Any, interrupt_monitor: "asyncio.Task", + self, turn_ctx: TurnContext, *, progress_task: Any, log_task: Any, interrupt_monitor: "asyncio.Task", _notify_task: "asyncio.Task", tracking_task: "asyncio.Task", stream_task: Any, - stream_consumer_holder: list, streaming_tts_consumer_holder: list, - session_key: Optional[str], run_generation: Optional[int], ) -> None: """``finally`` half of a turn: cancel background tasks, flush stream, release the session slot.""" + stream_consumer_holder, session_key = turn_ctx.stream_consumer_holder, turn_ctx.session_key for task in (progress_task, log_task, interrupt_monitor, _notify_task): if task: task.cancel() @@ -3751,7 +3720,7 @@ class GatewayTurnMixin: # Unconditional abort + bounded wait for the streaming-TTS consumer: covers cancellation / # exception paths where the normal finalisation block was skipped. - _stts_finally = streaming_tts_consumer_holder[0] + _stts_finally = turn_ctx.streaming_tts_consumer_holder[0] if _stts_finally is not None and not _stts_finally.done: _stts_finally.abort("cleanup") with suppress(Exception): @@ -3761,7 +3730,7 @@ class GatewayTurnMixin: if session_key: # Release the slot only if this run's generation still owns it: a /stop or /new that bumped # the generation while we unwound already installed its own state; keep it. - self._release_running_agent_state(session_key, run_generation=run_generation) + self._release_running_agent_state(session_key, run_generation=turn_ctx.run_generation) if self._draining: self._update_runtime_status("draining") @@ -3777,10 +3746,7 @@ class GatewayTurnMixin: # bookkeeping after this loop still runs. logger.debug("background turn task failed during cleanup", exc_info=True) - async def _run_agent_mark_streamed_delivery( - self, response: Any, stream_consumer_holder: list, source: SessionSource, - session_key: Optional[str], - ) -> None: + async def _run_agent_mark_streamed_delivery(self, response: Any, turn_ctx: TurnContext) -> None: """Set ``response["already_sent"]`` when streaming already delivered the final reply. Never when the agent failed (the error is unseen content) or on "(empty)": interim text @@ -3789,7 +3755,7 @@ class GatewayTurnMixin: against the recorded turn-final payload: on mismatch (False, incl. payload-less split delivery) neither flag may suppress the final send; None (no record) keeps legacy trust. """ - _sc = stream_consumer_holder[0] + _sc, source, session_key = turn_ctx.stream_consumer_holder[0], turn_ctx.source, turn_ctx.session_key if not isinstance(response, dict) or response.get("failed"): return _final = response.get("final_response") or "" @@ -3888,18 +3854,15 @@ class GatewayTurnMixin: _sk, _streamed, _previewed, _content_delivered, _transformed, len(_final), ) - def _run_agent_schedule_bubble_cleanup( - self, response: Any, _cleanup_progress: bool, _cleanup_adapter: Any, - _cleanup_msg_ids: List[str], source: SessionSource, session_key: Optional[str], - run_generation: Optional[int], - ) -> None: + def _run_agent_schedule_bubble_cleanup(self, response: Any, _cleanup_adapter: Any, turn_ctx: TurnContext) -> None: """Schedule deletion of tracked temporary progress bubbles after the final response lands. Failed runs keep them as breadcrumbs. Only on adapters with ``delete_message``; failures swallowed. """ from gateway.run import safe_schedule_threadsafe + _cleanup_msg_ids, session_key = turn_ctx._cleanup_msg_ids, turn_ctx.session_key if not ( - _cleanup_progress + turn_ctx._cleanup_progress and _cleanup_adapter is not None and _cleanup_msg_ids and session_key @@ -3909,7 +3872,7 @@ class GatewayTurnMixin: ): return _ids_snapshot = list(_cleanup_msg_ids) - _chat_id_snapshot = source.chat_id + _chat_id_snapshot = turn_ctx.source.chat_id _loop_snapshot = asyncio.get_running_loop() def _cleanup_temp_bubbles() -> None: @@ -3925,7 +3888,7 @@ class GatewayTurnMixin: try: _cleanup_adapter.register_post_delivery_callback( - session_key, _cleanup_temp_bubbles, generation=run_generation, + session_key, _cleanup_temp_bubbles, generation=turn_ctx.run_generation, ) except Exception as _rpe: logger.debug("Post-delivery cleanup registration failed: %s", _rpe) @@ -3966,11 +3929,8 @@ class GatewayTurnMixin: return _status_thread_metadata async def _run_agent_notify_long_running( - self, disp: "GatewayRunner._RunAgentDisplay", *, source: SessionSource, - session_key: Optional[str], agent_holder: list, _executor_task_holder: list, + self, disp: "GatewayRunner._RunAgentDisplay", turn_ctx: TurnContext, *, _executor_task_holder: list, _NOTIFY_INTERVAL: Optional[float], _long_running_mode: str, _notify_start: float, - _status_thread_metadata: Optional[Dict[str, Any]], _cleanup_progress: bool, - _cleanup_msg_ids: List[str], ) -> None: """Periodic "still working" heartbeat, edited in place where the adapter supports it. @@ -3981,6 +3941,8 @@ class GatewayTurnMixin: from gateway.run import _interim_metadata, _non_conversational_metadata if _NOTIFY_INTERVAL is None: return # Notifications disabled (gateway_notify_interval: 0) + source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder + _status_thread_metadata = turn_ctx._status_thread_metadata _notify_adapter = self._adapter_for_source(source) if not _notify_adapter: return @@ -4029,8 +3991,8 @@ class GatewayTurnMixin: ) if getattr(_notify_res, "success", False) and getattr(_notify_res, "message_id", None): _heartbeat_msg_id = str(_notify_res.message_id) - if _cleanup_progress: - _cleanup_msg_ids.append(_heartbeat_msg_id) + if turn_ctx._cleanup_progress: + turn_ctx._cleanup_msg_ids.append(_heartbeat_msg_id) except Exception as _ne: logger.debug("Long-running notification error: %s", _ne) @@ -4068,18 +4030,11 @@ class GatewayTurnMixin: persist_user_timestamp=persist_user_timestamp, persist_user_display_kind=persist_user_display_kind, ) - _cleanup_progress = turn_ctx._cleanup_progress - _cleanup_msg_ids = turn_ctx._cleanup_msg_ids _status_thread_metadata = self._run_agent_bind_turn_wiring( turn_ctx, turn_runner, source, event_message_id, disp._native_slack_task_cards, ) - agent_holder = turn_ctx.agent_holder - result_holder = turn_ctx.result_holder - stream_consumer_holder = turn_ctx.stream_consumer_holder - streaming_tts_consumer_holder = turn_ctx.streaming_tts_consumer_holder - self._run_agent_start_streaming_tts( - source, message_type, _status_thread_metadata, streaming_tts_consumer_holder, + source, message_type, _status_thread_metadata, turn_ctx.streaming_tts_consumer_holder, ) # Progress sender gates on needs_progress_queue (tool_progress OR thinking_progress), not @@ -4088,16 +4043,10 @@ class GatewayTurnMixin: log_task = asyncio.create_task(self._run_agent_write_tool_log(disp.log_queue)) if disp.log_mode_enabled else None # The stream consumer is created inside run_sync (thread pool) after the agent is constructed; # this task polls for it. - stream_task = asyncio.create_task(self._run_agent_stream_consumer_task(stream_consumer_holder)) - tracking_task = asyncio.create_task( - self._run_agent_track_agent(session_key, run_generation, agent_holder) - ) + stream_task = asyncio.create_task(self._run_agent_stream_consumer_task(turn_ctx.stream_consumer_holder)) + tracking_task = asyncio.create_task(self._run_agent_track_agent(turn_ctx)) _interrupt_detected = asyncio.Event() # shared with backup check - interrupt_monitor = asyncio.create_task( - self._run_agent_monitor_for_interrupt( - source, session_key, agent_holder, _interrupt_detected, streaming_tts_consumer_holder, - ) - ) + interrupt_monitor = asyncio.create_task(self._run_agent_monitor_for_interrupt(turn_ctx, _interrupt_detected)) # Periodic "still working" notifications so the user knows the agent hasn't died. Config: # agent.gateway_notify_interval or HERMES_AGENT_NOTIFY_INTERVAL env; default 180s. @@ -4109,58 +4058,34 @@ class GatewayTurnMixin: _executor_task_holder: list = [None] # bound once the executor future exists (see below) _notify_task = asyncio.create_task( self._run_agent_notify_long_running( - disp, source=source, session_key=session_key, agent_holder=agent_holder, - _executor_task_holder=_executor_task_holder, _NOTIFY_INTERVAL=_NOTIFY_INTERVAL, + disp, turn_ctx, _executor_task_holder=_executor_task_holder, _NOTIFY_INTERVAL=_NOTIFY_INTERVAL, _long_running_mode=_long_running_mode, _notify_start=time.time(), - _status_thread_metadata=_status_thread_metadata, - _cleanup_progress=_cleanup_progress, _cleanup_msg_ids=_cleanup_msg_ids, ) ) try: # run_sync is TurnRunner.run_sync (bound method; executor call unchanged). - worker = self._run_agent_start_turn_worker( - turn_ctx, turn_runner.run_sync, agent_holder, session_id, session_key, run_generation, - ) + worker = self._run_agent_start_turn_worker(turn_ctx, turn_runner.run_sync) _executor_task_holder[0] = worker.executor_task # read late by _notify_long_running - response = await self._run_agent_await_turn_worker( - worker, source=source, session_key=session_key, agent_holder=agent_holder, - result_holder=result_holder, tools_holder=turn_ctx.tools_holder, - _interrupt_detected=_interrupt_detected, interrupt_monitor=interrupt_monitor, - streaming_tts_consumer_holder=streaming_tts_consumer_holder, - _status_thread_metadata=_status_thread_metadata, - ) - self._run_agent_evict_on_fallback(session_key, agent_holder, result_holder) + response = await self._run_agent_await_turn_worker(worker, turn_ctx, _interrupt_detected, interrupt_monitor) + self._run_agent_evict_on_fallback(turn_ctx) # Interrupted OR queued message (/queue)? - result = result_holder[0] + result = turn_ctx.result_holder[0] adapter = self._adapter_for_source(source) - await self._run_agent_finalize_streaming_tts( - streaming_tts_consumer_holder, adapter, session_key, run_generation, - ) + await self._run_agent_finalize_streaming_tts(turn_ctx, adapter) pending_event, pending = await self._run_agent_drain_pending(result, adapter, source, session_key) if pending_event or pending: return await self._run_agent_queued_followup( - source=source, adapter=adapter, session_id=session_id, session_key=session_key, - run_generation=run_generation, _interrupt_depth=_interrupt_depth, - event_message_id=event_message_id, context_prompt=context_prompt, - history=history, pending=pending, pending_event=pending_event, - response=response, result=result, result_holder=result_holder, - stream_consumer_holder=stream_consumer_holder, stream_task=stream_task, - _status_thread_metadata=_status_thread_metadata, + turn_ctx, adapter, pending, pending_event, response, result, stream_task, ) finally: await self._run_agent_cleanup_turn_tasks( - progress_task=progress_task, log_task=log_task, interrupt_monitor=interrupt_monitor, + turn_ctx, progress_task=progress_task, log_task=log_task, interrupt_monitor=interrupt_monitor, _notify_task=_notify_task, tracking_task=tracking_task, stream_task=stream_task, - stream_consumer_holder=stream_consumer_holder, - streaming_tts_consumer_holder=streaming_tts_consumer_holder, - session_key=session_key, run_generation=run_generation, ) - await self._run_agent_mark_streamed_delivery(response, stream_consumer_holder, source, session_key) - self._run_agent_schedule_bubble_cleanup( - response, _cleanup_progress, _cleanup_adapter, _cleanup_msg_ids, source, session_key, run_generation, - ) + await self._run_agent_mark_streamed_delivery(response, turn_ctx) + self._run_agent_schedule_bubble_cleanup(response, _cleanup_adapter, turn_ctx) return response From a2007bc76f2b9de854ef9263dfbde5838e248f9c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:34:47 -0700 Subject: [PATCH 2/6] =?UTF-8?q?refactor(gateway):=20run=5Fturn=20=E2=80=94?= =?UTF-8?q?=20TurnContext=20defaults=20replace=20explicit=20holder=20inits?= =?UTF-8?q?,=20heartbeat=20task=20owns=20its=20interval=20resolution,=20do?= =?UTF-8?q?cstring/comment=20compaction,=20AST-neutral=20layout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gateway/run_turn.py | 521 ++++++++++++++++---------------------------- 1 file changed, 182 insertions(+), 339 deletions(-) diff --git a/gateway/run_turn.py b/gateway/run_turn.py index 325eac294a..d36a0a788c 100644 --- a/gateway/run_turn.py +++ b/gateway/run_turn.py @@ -1,8 +1,6 @@ -"""Agent-turn execution (_handle_message_with_agent, _run_agent*, proxy path, background tasks, MCP reload) for GatewayRunner. - -Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO. -``gateway.run`` internals are imported lazily inside method bodies (import cycle), -so ``patch("gateway.run.X")`` keeps intercepting them at call time. +"""Agent-turn execution for GatewayRunner: _handle_message_with_agent, _run_agent*, proxy path, +background tasks, MCP reload. Bound onto ``GatewayRunner`` via the MRO; ``gateway.run`` internals +are imported lazily inside method bodies (import cycle) so ``patch("gateway.run.X")`` still works. """ from __future__ import annotations @@ -24,10 +22,7 @@ from gateway.config import Platform from gateway.media_repair import repair_explicit_computer_use_media_paths from gateway.platforms.base import BasePlatformAdapter, MessageEvent from gateway.session import ( - SessionSource, - TranscriptReadError, - _session_key_namespace, - build_channel_continuity_note, + SessionSource, TranscriptReadError, _session_key_namespace, build_channel_continuity_note, build_session_context, ) from gateway.turn_context import TurnContext @@ -45,7 +40,7 @@ logger = logging.getLogger("gateway.run") class GatewayTurnMixin: - """Agent-turn execution (_handle_message_with_agent, _run_agent*, proxy path, background tasks, MCP reload) for GatewayRunner.""" + """Agent-turn execution for GatewayRunner (see module docstring).""" def _resolve_session_agent_runtime( self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, @@ -151,8 +146,7 @@ class GatewayTurnMixin: logger.warning( "Empty model resolved for session=%s — recovering " "last-known-good model %s (config read likely returned " - "empty; see #35314)", - skey or "", _recovered, + "empty; see #35314)", skey or "", _recovered, ) model = _recovered else: @@ -164,11 +158,8 @@ class GatewayTurnMixin: return model, runtime_kwargs def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwargs: dict) -> dict: - """Build the effective model/runtime config for a single turn. - - Always uses the session's primary model/provider. With `/fast` priority on and a model that - supports it, fast-mode ``request_overrides`` are deep-merged OVER the per-provider - ``request_overrides`` (e.g. ``custom_providers`` ``extra_body``) so both reach the model. + """Effective model/runtime config for one turn. With `/fast` priority on, fast-mode + ``request_overrides`` are deep-merged OVER the per-provider ones so both reach the model. """ from gateway.run import _deep_merge_request_overrides from hermes_cli.models import resolve_fast_mode_overrides @@ -408,10 +399,8 @@ class GatewayTurnMixin: platform=source.platform, session_type=getattr(source, 'chat_type', 'dm'), ) platform_name = source.platform.value if source.platform else "" - # Suspended / restart-recovery-expired sessions always notify regardless of - # policy.notify — an active session was silently replaced and the user must learn they - # can /resume it. Idle/daily resets respect the policy flag, the excluded platforms - # (api_server, webhook) and require prior activity. + # Suspended / restart-recovery-expired sessions always notify (the user must learn they + # can /resume); idle/daily resets respect policy.notify + excluded platforms + activity. should_notify = reset_reason in {"suspended", "resume_pending_expired"} or ( policy.notify and getattr(session_entry, 'reset_had_activity', False) @@ -465,12 +454,10 @@ class GatewayTurnMixin: logger.warning("[Gateway] Failed to auto-load skill(s) %s: %s", _skill_names, e) async def _hmwa_acquire_turn_lease(self, _quick_key, run_generation, session_entry, _session_env_tokens): - """Serialize [load history → run → flush] per resolved SESSION_ID (session resolution is - FINAL here): another routing key mapped to the same session_id waits for the prior flush - instead of loading a stale base. Fail-closed on timeout — never enter the transcript - region without a lease; outer dispatch returns a bounded resend notice. Released in - _handle_message's finally, granted per (routing key, run generation) so a stale unwind - can't release a newer turn's.""" + """Serialize [load history → run → flush] per resolved SESSION_ID so another routing key on + the same session waits for the prior flush. Fail-closed on timeout (outer dispatch returns + a resend notice). Released in _handle_message's finally, granted per (routing key, run + generation) so a stale unwind can't release a newer turn's.""" from gateway.run import _float_env _lease_registry = getattr(self, "_turn_leases", None) if _lease_registry is None: @@ -543,13 +530,11 @@ class GatewayTurnMixin: ) async def _hmwa_hygiene_settings(self, source, session_key): - """Resolve model/provider/context-length + hygiene knobs for the pre-agent compression - safety net (fail-soft: any config/runtime error keeps the defaults). + """Resolve model/provider/context-length + hygiene knobs (fail-soft: errors keep defaults). - The hygiene threshold (0.85) is deliberately HIGHER than the agent's own compressor - (0.50): it is a safety net for sessions that grew between turns. ``max_turn_hold_seconds`` - bounds how long the user's TURN waits on hygiene before proceeding uncompressed (the - compressor keeps running detached, commit fenced); kept below transport idle-timeouts.""" + The 0.85 threshold is deliberately HIGHER than the agent's compressor (0.50): a safety net + for sessions that grew between turns. ``max_turn_hold_seconds`` bounds the TURN wait + (compressor keeps running detached, commit fenced); kept below transport idle-timeouts.""" from gateway.run import _load_gateway_config hs = self._HygieneSettings( model="anthropic/claude-sonnet-4.6", threshold_pct=0.85, compression_enabled=True, @@ -622,10 +607,8 @@ class GatewayTurnMixin: _warn_token_threshold = int(_hyg_context_length * 0.95) _msg_count = len(history) - # Prefer the API-reported prompt tokens from the last turn over the rough char estimate. - # Rough estimates run 30-50% high on code/JSON-heavy sessions, which only makes hygiene - # fire early (safe). Do NOT compensate with a threshold multiplier: 85% * 1.4 = 119% of - # context kept hygiene from ever firing for ~200K models. + # Prefer the API-reported prompt tokens over the rough estimate (runs 30-50% high, which only + # fires hygiene early — safe). Do NOT compensate with a threshold multiplier. if session_entry.last_prompt_tokens > 0: _approx_tokens, _token_source = session_entry.last_prompt_tokens, "actual" else: @@ -637,9 +620,8 @@ class GatewayTurnMixin: _needs_compress = _approx_tokens >= _compress_token_threshold or _msg_count >= hs.hard_msg_limit if _needs_compress: - # Persistent DB-backed cooldown (shared with context_compressor.py) so it survives - # gateway restarts; an in-memory dict re-triggered the same failing compression on - # every restart and wedged session storage. + # DB-backed cooldown (shared with context_compressor.py): survives gateway restarts, so a + # failing compression is not re-triggered on every restart. _session_db = getattr(self, "_session_db", None) if _session_db is not None: _session_db = getattr(_session_db, "_db", _session_db) @@ -681,10 +663,7 @@ class GatewayTurnMixin: transcript; raises ``HygieneTurnHoldExceeded`` (turn-hold budget) or ``asyncio.TimeoutError`` (idle/ceiling/fence cancel) for the caller's handlers. - The timeout is an INACTIVITY budget — the worker ticks the fence per streamed token, so a - slow but still-generating model extends the deadline; a hard ceiling bounds the total so - a trickle stream can't hold the turn; the turn-hold budget caps how long the user's TURN - waits regardless.""" + Idle timeout (fence ticks per streamed token) + hard ceiling + turn-hold cap.""" from gateway.run import HygieneTurnHoldExceeded, hygiene_wait_should_extend fence = attempt.commit_fence while True: @@ -741,12 +720,9 @@ class GatewayTurnMixin: raise async def _hmwa_hygiene_cancel_before_commit(self, fence): - """Cancel the detached worker at the commit fence. Returns ``False`` when the worker had - already crossed into its commit (caller must consume the result — aborting mid-commit - would corrupt the message-store transaction), ``True`` when cancelled in time. - - A hung commit retains the fence lock; the lock-free ``commit_in_flight`` marker keeps - this loop from spinning forever. 25ms polls ride transient lock-setup windows.""" + """Cancel the detached worker at the commit fence: ``True`` when cancelled in time, ``False`` + when it already crossed into its commit (caller must consume the result). The lock-free + ``commit_in_flight`` marker keeps this loop from spinning on a hung commit.""" while True: if fence.commit_in_flight: return False @@ -802,15 +778,12 @@ class GatewayTurnMixin: notify the user, and re-raise; returns the compressed transcript only when the worker was already committing. - Turn-hold expiry is an availability boundary, not a failure: the compressor is healthy, - so the failure streak must NOT advance — only flat, non-escalating retry spacing is - recorded. When the worker's commit is watermark-fenced (rows appended after compression - start — this turn included — survive its late commit as cloned concurrent tail) the - attempt KEEPS commit admission: the turn proceeds uncompressed NOW and the summary is - adopted at the worker's own fenced commit; always cancelling burned every attempt for - thinking summary models whose reasoning prefix alone exceeds the hold. Without the - watermark fence (no session_db, capture failed, legacy lock API) a late commit could - clobber newer turns, so cancel.""" + Turn-hold expiry is an availability boundary, not a failure: the streak must NOT advance, + only flat retry spacing is recorded. A watermark-fenced commit (rows appended after + compression start survive as cloned tail) KEEPS admission: the turn proceeds uncompressed + now and the summary is adopted at the worker's fenced commit — always cancelling burned + every attempt for thinking summary models. Without the fence a late commit could clobber + newer turns, so cancel.""" from gateway.run import ( _HYGIENE_TURNHOLD_RETRY_SECONDS, _record_hygiene_cooldown, _reset_hygiene_failure_streak ) @@ -818,11 +791,9 @@ class GatewayTurnMixin: _hyg_keep_admission = bool(getattr(fence, "commit_watermark_fenced", False)) and not fence.is_cancelled if _hyg_keep_admission: self._hmwa_hygiene_defer_cleanup(attempt, "session hygiene turn-hold") - # NO retry-after here: the attempt is still running toward a real commit, and the flat - # 60s retry-after would also block the agent-side preflight compressor (same-session - # cooldown). Re-attempt spacing comes from the durable compression lock: the next - # turn's hygiene pre-check skips while this worker's lease is held. The done-callback - # records the flat retry-after ONLY if the worker ends without committing anything. + # NO retry-after here (it would also block the agent-side preflight compressor); spacing + # comes from the durable compression lock. The done-callback records the flat retry-after + # ONLY if the worker ends without committing anything. _sid, _skey, _agent = session_entry.session_id, session_key, attempt.agent def _hyg_adopt_or_space_retry(_fut, _gw=self, _sid=_sid, _skey=_skey, _agent=_agent): @@ -950,13 +921,9 @@ class GatewayTurnMixin: raise def _hmwa_hygiene_on_unwind(self, attempt, hs, session_entry, session_key): - """``except BaseException`` body (caller re-raises): revoke commit admission and record a - cooldown so the next turn does not immediately re-arm hygiene. - - Non-timeout unwind (KeyboardInterrupt, task cancel, unexpected error) while the detached - worker may still run: revoke admission (and release its durable lease) BEFORE the host - unwinds so it can never commit later. Restart drain / task cancel must record a cooldown, - or the next turn re-arms hygiene and waits up to 600s behind a fence that refuses again.""" + """``except BaseException`` body (caller re-raises): revoke commit admission BEFORE the host + unwinds so the detached worker can never commit later, and record a cooldown — otherwise + the next turn re-arms hygiene and waits up to 600s behind a fence that refuses again.""" from gateway.run import _hygiene_cooldown_for_failure, _record_hygiene_cooldown attempt.commit_fence.revoke_commit_admission() if not attempt.cleanup_deferred: @@ -983,17 +950,13 @@ class GatewayTurnMixin: async def _hmwa_hygiene_adopt_transcript( self, attempt, _compressed, history, plan, *, session_entry, source, _quick_key, run_generation, ): - """Adopt a finished compression (rotation / in-place / refused) and publish the transcript - to continue with on ``attempt.history``. + """Adopt a finished compression (rotation / in-place / refused); publishes the transcript to + continue with on ``attempt.history``. - Rewrite the transcript only when rotation produced a NEW session id. In-place compaction - needs none: archive_and_compact() already soft-archived the previous rows, and - rewrite_transcript() would replace_messages(active_only=False) and DELETE the archived - turns. A summary with neither rotation nor a completed archive_and_compact() signals - FAILURE; an unconditional rewrite would replace the originals with only the summary. - Write-before-repoint (mirrors manual /compress): if session_entry were repointed first and - rewrite_transcript then failed (lock/ENOSPC), the live entry would reference an empty - session and the conversation silently vanishes.""" + Rewrite only on rotation (NEW session id): in-place compaction already soft-archived the + old rows and rewrite_transcript() would DELETE them; neither rotation nor in-place signals + FAILURE and an unconditional rewrite would leave only the summary. Write-before-repoint: + a repoint-then-failed-rewrite would point the live entry at an empty session.""" from agent.model_metadata import estimate_messages_tokens_rough _hyg_agent = attempt.agent # _compress_context ends the old session and creates a new session_id; compressed messages @@ -1007,9 +970,8 @@ class GatewayTurnMixin: _hyg_out_toks = estimate_messages_tokens_rough(_compressed) if _hyg_rotated and _hyg_out_toks > _hyg_in_toks: logger.warning( - "Gateway hygiene compression for session %s " - "would grow transcript (~%s -> ~%s tokens); " - "keeping the original transcript unchanged", + "Gateway hygiene compression for session %s would grow transcript (~%s -> ~%s " + "tokens); keeping the original transcript unchanged", session_entry.session_id, f"{_hyg_in_toks:,}", f"{_hyg_out_toks:,}", ) _hyg_rotated = False @@ -1017,10 +979,8 @@ class GatewayTurnMixin: if _hyg_rotated: if not await self.async_session_store.rewrite_transcript(_hyg_new_sid, _compressed): logger.error( - "Session hygiene: failed to persist " - "compressed transcript for rotated " - "session %s → %s; keeping the live " - "entry on the original session so the " + "Session hygiene: failed to persist compressed transcript for rotated session " + "%s → %s; keeping the live entry on the original session so the " "conversation is not dropped", session_entry.session_id, _hyg_new_sid, ) # Fail closed: treat like no rotation. @@ -1048,16 +1008,13 @@ class GatewayTurnMixin: _new_count = plan.msg_count _new_tokens = plan.approx_tokens logger.warning( - "Gateway hygiene compression for session %s " - "did not rotate or compact in place " - "(no session_db on the hygiene agent) — " - "preserving the original transcript instead " + "Gateway hygiene compression for session %s did not rotate or compact in place (no " + "session_db on the hygiene agent) — preserving the original transcript instead " "of overwriting it with the summary (#21301).", session_entry.session_id, ) logger.info( - "Session hygiene: compressed %s → %s msgs, " - "~%s → ~%s tokens", + "Session hygiene: compressed %s → %s msgs, ~%s → ~%s tokens", plan.msg_count, _new_count, f"{plan.approx_tokens:,}", f"{_new_tokens:,}", ) if _new_tokens >= plan.warn_token_threshold: @@ -1115,10 +1072,8 @@ class GatewayTurnMixin: await self._hmwa_hygiene_notify( source, attempt.meta, "⚠️ Context compression aborted " f"({_err}). No messages were dropped — " - "conversation is unchanged. Run /compress " - "to retry, /reset for a clean session, or " - "check your auxiliary.compression model " - "configuration.", + "conversation is unchanged. Run /compress to retry, /reset for a clean " + "session, or check your auxiliary.compression model configuration.", "compression-failure warning", ) # If the CONFIGURED aux model failed and we recovered on the main model, tell the user — @@ -1150,8 +1105,7 @@ class GatewayTurnMixin: failure_cooldown_seconds=hs.failure_cooldown_seconds, ) logger.info( - "Session hygiene (codex app-server): %s " - "(session=%s, mode=%s, ~%s tokens)", + "Session hygiene (codex app-server): %s (session=%s, mode=%s, ~%s tokens)", _hyg_codex_outcome, session_entry.session_id, _hyg_codex_auto, f"{plan.approx_tokens:,}", ) @@ -1165,9 +1119,8 @@ class GatewayTurnMixin: except Exception as exc: _hyg_session_row = None logger.warning( - "Session hygiene could not restore the system " - "prompt for session %s: %s. Preserving an empty " - "prompt so the live turn rebuilds it with its " + "Session hygiene could not restore the system prompt for session %s: %s. " + "Preserving an empty prompt so the live turn rebuilds it with its " "configured providers.", session_entry.session_id, exc, exc_info=True, ) _hyg_session_db = getattr(self._session_db, "_db", self._session_db) @@ -1202,10 +1155,8 @@ class GatewayTurnMixin: _hyg_agent, _hyg_session_db = await self._hmwa_hygiene_build_agent(_hyg_model, _hyg_runtime, session_entry) attempt.agent = _hyg_agent try: - # Hygiene runs before the turn and owns the session binding, so prefer in-place - # compaction: archive old rows under the same session id rather than minting a - # continuation child that must be published back to SessionStore/topic bindings. - # Without a SessionDB this stays False and the apply guard preserves it. + # Hygiene owns the session binding, so prefer in-place compaction over minting a + # continuation child. Without a SessionDB this stays False. _hyg_agent.compression_in_place = True _bind_hyg_state = getattr(getattr(_hyg_agent, "context_compressor", None), "bind_session_state", None) if callable(_bind_hyg_state): @@ -1216,10 +1167,8 @@ class GatewayTurnMixin: loop = asyncio.get_running_loop() _hyg_commit_fence = CompressionCommitFence(total_ceiling_seconds=hs.total_ceiling_seconds) - # Default executor (NOT self._get_executor): a fence-cancelled hung summary must never - # occupy an agent-work slot. But it MUST run in the caller's contextvars: under - # multiplex_profiles the secret scope / HERMES_HOME live in ContextVars, and an empty - # Context makes get_secret() fail closed → lossy truncation. + # Default executor (NOT self._get_executor): a hung summary must never occupy an + # agent-work slot. MUST run in the caller's contextvars (multiplex secret scope). attempt.commit_fence = _hyg_commit_fence attempt.future = loop.run_in_executor( None, @@ -1360,10 +1309,9 @@ class GatewayTurnMixin: sethome_cmd = "/hermes sethome" if source.platform == Platform.SLACK else "/sethome" await self._deliver_platform_notice( source, f"📬 No home channel is set for {platform_name.title()}. " - f"A home channel is where Hermes delivers cron job results " - f"and cross-platform messages.\n\n" - f"Type {sethome_cmd} to make this chat your home channel, " - f"or ignore to skip.", + f"A home channel is where Hermes delivers cron job results and cross-platform " + f"messages.\n\nType {sethome_cmd} to make this chat your home channel, or ignore " + f"to skip.", ) def _hmwa_apply_message_timestamp(self, event, message_text): @@ -1435,9 +1383,8 @@ class GatewayTurnMixin: # "(empty)" = the model produced no visible content after exhausting all retries. if response == "(empty)" and not _intentional_silence: response = ( - "⚠️ The model returned no response after processing tool " - "results. This can happen with some models — try again or " - "rephrase your question." + "⚠️ The model returned no response after processing tool results. This can happen " + "with some models — try again or rephrase your question." ) agent_messages = agent_result.get("messages", []) logger.info( @@ -1479,17 +1426,15 @@ class GatewayTurnMixin: ) else: logger.info( - "Skipping agent-result session split sync for %s because " - "the session binding moved from %s to %s before " - "compression finished", + "Skipping agent-result session split sync for %s because the session binding " + "moved from %s to %s before compression finished", session_key or "?", _run_start_session_id, session_entry.session_id, ) return response, _intentional_silence, agent_messages # reasoning_style → (header line, per-line quote prefix for blank / non-blank lines) _REASONING_QUOTE_STYLES = { - "subtext": ("-# 💭 Reasoning", "-# ", "-#"), - "blockquote": ("> 💭 **Reasoning:**", "> ", ">"), + "subtext": ("-# 💭 Reasoning", "-# ", "-#"), "blockquote": ("> 💭 **Reasoning:**", "> ", ">") } def _hmwa_prepend_reasoning(self, agent_result, response, source, _intentional_silence): @@ -1593,10 +1538,8 @@ class GatewayTurnMixin: """Classify a finished turn for transcript persistence. Returns ``(agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure)``. - Context-overflow failures (compression exhausted, generic 400 on large sessions) must - NOT persist the user message — it would grow the session and reproduce the failure - forever. Transient failures (429, timeout, connection error, 5xx) DO persist it: the - session is not oversized and dropping the turn causes severe context loss on retry.""" + Context-overflow failures must NOT persist the user message (session would grow and + reproduce the failure forever); transient failures (429/timeout/5xx) DO.""" from gateway.run import _is_gateway_hidden_reasoning_incomplete_turn agent_failed_early = bool(agent_result.get("failed")) hidden_reasoning_incomplete = _is_gateway_hidden_reasoning_incomplete_turn(agent_result) @@ -1611,8 +1554,7 @@ class GatewayTurnMixin: if is_context_overflow_failure: logger.info( "Skipping transcript persistence for context-overflow " - "failure in session %s to prevent session growth loop.", - session_entry.session_id, + "failure in session %s to prevent session growth loop.", session_entry.session_id, ) elif agent_failed_early: logger.info( @@ -1621,8 +1563,7 @@ class GatewayTurnMixin: ) elif hidden_reasoning_incomplete: logger.warning( - "Suppressing hidden-reasoning-only incomplete gateway turn " - "for session %s: %s", + "Suppressing hidden-reasoning-only incomplete gateway turn for session %s: %s", session_entry.session_id, agent_result.get("error", "processing incomplete"), ) return agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure @@ -1649,19 +1590,16 @@ class GatewayTurnMixin: # dict (see _CONVERSATION_SCOPED_STATE). self._clear_conversation_scope(session_key, reason="compression_exhausted_reset") if new_entry is not None: - # Re-point the Telegram topic binding at the fresh session: compression rotated - # session_entry.session_id to the bloated child earlier this turn and that _sync - # also rewrote the (chat_id, thread_id) binding. Without a re-sync the binding-heal - # walk switches the next inbound message back onto the child and re-triggers - # exhaustion forever. No-op on non-topic lanes. + # Re-point the Telegram topic binding at the fresh session, or the binding-heal walk + # switches the next message back onto the bloated child and re-triggers exhaustion + # forever. No-op on non-topic lanes. session_entry = new_entry await asyncio.to_thread( self._sync_telegram_topic_binding, source, session_entry, reason="compression-exhausted-reset", ) response = (response or "") + ( - "\n\n🔄 Session auto-reset — the conversation exceeded the " - "maximum context size and could not be compressed further. " - "Your next message will start a fresh session." + "\n\n🔄 Session auto-reset — the conversation exceeded the maximum context size and " + "could not be compressed further. Your next message will start a fresh session." ) return response, session_entry @@ -1714,15 +1652,12 @@ class GatewayTurnMixin: "timestamp": ts, }) if agent_failed_early or hidden_reasoning_incomplete: - # Transient failure (429/timeout/5xx): persist only the user message so the next - # message loads a transcript that reflects what was said; the assistant error text is - # a gateway-generated hint, not model output. Hidden-reasoning incomplete turns follow - # the same rule so peer-agent channels don't ingest them. - # Dedupe on platform message_id (Telegram retries after transient failures). + # Transient failure / hidden-reasoning incomplete: persist only the user message (the + # assistant error text is a gateway hint, not model output). Dedupe on platform + # message_id (Telegram retries after transient failures). if event.message_id and await store.has_platform_message_id(sid, str(event.message_id)): logger.info( - "Skipping duplicate user turn " - "(message_id=%s) in session %s", + "Skipping duplicate user turn (message_id=%s) in session %s", event.message_id, sid, ) else: @@ -1730,8 +1665,8 @@ class GatewayTurnMixin: sid, self._hmwa_user_transcript_entry(event, prepared, ts), skip_db=agent_persisted, ) else: - # Only the NEW messages from this turn: use history_offset (what the agent saw), not - # len(history), which counts session_meta entries stripped before the agent saw them. + # Only the NEW messages: history_offset (what the agent saw), not len(history), which + # counts session_meta entries stripped before the agent saw them. history_len = agent_result.get("history_offset", len(history)) new_messages = agent_messages[history_len:] if len(agent_messages) > history_len else [] if not new_messages: @@ -1770,11 +1705,9 @@ class GatewayTurnMixin: touch_activity=not bool(getattr(event, "internal", False)), ) - # Re-baseline the cached agent's message_count snapshot now that ALL of this turn's writes - # are done (flushed rows AND the first-turn `session_meta` marker, which bumps the count - # too). The cross-process coherence guard snapshots at agent-BUILD time and never - # refreshes on reuse, so our own writes would trigger a rebuild next turn (destroying - # prompt caching). + # Re-baseline the cached agent's message_count now that ALL of this turn's writes are done: + # the coherence guard snapshots at agent-BUILD time, so our own writes would otherwise + # trigger a rebuild next turn (destroying prompt caching). await self._refresh_agent_cache_message_count(session_key, sid) async def _hmwa_deliver_turn_response( @@ -1883,9 +1816,8 @@ class GatewayTurnMixin: # payload is too large for the API — treat it the same way. if len(prepared.history) > 50: return ( - "⚠️ Session too large for the model's context window.\n" - "Use /compact to compress the conversation, or " - "/reset to start fresh." + "⚠️ Session too large for the model's context window.\nUse /compact to " + "compress the conversation, or /reset to start fresh." ) elif status_code == 400: status_hint = " The request was rejected by the API." @@ -1922,10 +1854,8 @@ class GatewayTurnMixin: a ``str`` is a reply to send instead of running (history unreadable); ``None`` drops the turn (inbound text rejected).""" from gateway.run import _load_gateway_config - # Synthetic self-injected turns (batch completions, watch notifications, resume wake-ups) - # arrive as MessageEvent(internal=True). Persist with display_kind="internal_notification" - # so UIs render timeline notices, not user bubbles. display_kind is a DB-only sidecar - # stripped from every provider-bound payload; role/content untouched. + # Self-injected turns (MessageEvent(internal=True)) persist with a DB-only display_kind so + # UIs render timeline notices, not user bubbles; role/content untouched. persist_user_display_kind = "internal_notification" if getattr(event, "internal", False) else None _redact_pii = False # privacy.redact_pii, re-read per message with suppress(Exception): @@ -1936,9 +1866,8 @@ class GatewayTurnMixin: # drift turn-over-turn; a miss (thread rename, /sethome, redact_pii flip) re-renders. context_prompt = self._pinned_session_context_prompt(context, _redact_pii, session_key) - # Per-turn must-deliver notes ride the user message via the api_content sidecar (staged - # below, consumed in run_sync → build_turn_context), NOT context_prompt: appending them to - # the ephemeral system prompt guaranteed a turn1→turn2 diff and a full agent rebuild. + # Per-turn notes ride the user message via the api_content sidecar, NOT context_prompt + # (appending to the ephemeral system prompt forced a full agent rebuild). turn_sidecar_notes: List[str] = [] if _was_auto_reset: await self._hmwa_deliver_auto_reset_notice(session_entry, source, turn_sidecar_notes) @@ -1956,19 +1885,16 @@ class GatewayTurnMixin: # routed message that never began processing if the gateway died while it was still waiting. await self._mark_durable_active_turn(event, session_entry.session_key) - # An unreadable canonical store is not an empty conversation: stop before the agent can - # invent continuity from a plausible-looking []. This return happens before the broad - # cleanup finally, so restore task-local context here; the outer dispatch still clears the - # durable marker and turn lease. + # An unreadable store is not an empty conversation: stop before the agent invents continuity + # from []. Restore task-local context here (before the broad cleanup finally). try: history = await self.async_session_store.load_transcript(session_entry.session_id) except TranscriptReadError: self._clear_session_env(_session_env_tokens) return ( - "⚠️ This session's history is temporarily unavailable, so " - "this message was not processed. Ask the operator to inspect " - "state.db, then resend after it is healthy. Use /reset only " - "if you intentionally want to start a new conversation." + "⚠️ This session's history is temporarily unavailable, so this message was not " + "processed. Ask the operator to inspect state.db, then resend after it is healthy. " + "Use /reset only if you intentionally want to start a new conversation." ) history = await self._hmwa_run_session_hygiene( @@ -1977,9 +1903,8 @@ class GatewayTurnMixin: await self._hmwa_first_contact_notes(source, history, turn_sidecar_notes) - # Voice channel state (who is present / speaking) rides the user message ONLY when changed - # since the previous turn; in the ephemeral system prompt it forced a full agent rebuild + - # prompt-cache re-key per message (the prompt carries a static pointer line instead). + # Voice channel state rides the user message ONLY when changed (in the system prompt it + # forced a rebuild + prompt-cache re-key per message). _vc_note = self._voice_channel_sidecar_note(event, source, session_key) if _vc_note: turn_sidecar_notes.append(_vc_note) @@ -2169,11 +2094,9 @@ class GatewayTurnMixin: def _resolve_enabled_toolsets_for_source( self, user_config: dict, source: "SessionSource", platform_key: str, ) -> list: - """Enabled toolsets for an agent run, honoring an adapter ``toolsets_for_source()`` override. - - The override (e.g. per-route webhook toolsets) is validated through the SAME - ``_get_platform_tools`` path as normal platform config, so unknown and platform-restricted - toolsets are dropped rather than trusted. Absent an override: ``platform_toolsets.``. + """Enabled toolsets for an agent run, honoring an adapter ``toolsets_for_source()`` override + validated through the SAME ``_get_platform_tools`` path (unknown / platform-restricted + toolsets dropped, not trusted). """ from hermes_cli.tools_config import _get_platform_tools try: @@ -2457,10 +2380,8 @@ class GatewayTurnMixin: self, source: "SessionSource", scfg: Any, adapter: Any, *, on_missing_cursor: str, ) -> "tuple[Any, Optional[Callable[[], None]]]": """Build the shared ``StreamConsumerConfig`` and optional Telegram pause-typing closure. - - ``on_missing_cursor`` handles adapters with ``SUPPORTS_MESSAGE_EDITING = False``: - ``"fallback"`` (proxy path) streams with an empty cursor; ``"raise"`` (in-process path) - raises ``RuntimeError`` so the caller's ``except`` skips streaming entirely. + For non-editing adapters ``on_missing_cursor="fallback"`` streams with an empty cursor; + ``"raise"`` raises ``RuntimeError`` so the caller skips streaming entirely. """ from gateway.stream_consumer import StreamConsumerConfig @@ -2468,10 +2389,8 @@ class GatewayTurnMixin: if source.platform == Platform.TELEGRAM and hasattr(adapter, "pause_typing_for_chat"): def _pause_typing_before_finalize(_adapter=adapter, _chat_id=source.chat_id) -> None: _adapter.pause_typing_for_chat(_chat_id) - # Platforms that can't edit sent messages (e.g. QQ, WeChat) skip streaming entirely: the - # partial first message could never be updated, yielding duplicates (partial + final) — - # unless they have a native-streaming transport (e.g. WeCom msgtype "stream"), which the - # consumer's native branch delivers whole. + # Non-editing platforms (QQ, WeChat) skip streaming — the partial first message could never + # be updated — unless they have a native-streaming transport (WeCom msgtype "stream"). _adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True) _adapter_supports_native_stream = bool(getattr(adapter, "SUPPORTS_NATIVE_STREAMING", False)) if not _adapter_supports_edit and not _adapter_supports_native_stream and on_missing_cursor == "raise": @@ -2820,19 +2739,14 @@ class GatewayTurnMixin: self, disp: "GatewayRunner._RunAgentDisplay", AIAgent: Any, *, message: str, source: SessionSource, session_key: Optional[str], run_generation: Optional[int], **turn_params, ) -> Tuple[TurnContext, TurnRunner, Any]: - """Build the progress holders, the ``TurnContext`` and its ``TurnRunner``. - - ``turn_params`` (history, context_prompt, channel_prompt, session_id, _interrupt_depth, - event_message_id, inbound_message_id, moa_config, persist_user_*) are stored verbatim. - Returns ``(turn_ctx, turn_runner, cleanup_adapter)``; the progress-bubble cleanup flags travel - on ``turn_ctx._cleanup_progress`` / ``turn_ctx._cleanup_msg_ids``. + """Build the ``TurnContext`` and its ``TurnRunner``; ``turn_params`` (history, context_prompt, + session_id, persist_user_*, …) are stored verbatim. Returns ``(turn_ctx, turn_runner, + cleanup_adapter)``. """ from gateway.run import TurnRunner - # Discord voice "verbal ack before tool calls": with the continuous mixer installed - # (discord.voice_fx.enabled), speak a short phrase on the FIRST tool call of the turn; once - # per turn. source.chat_id is the linked text channel; resolve the guild whose voice - # connection is bound to it (mirrors DiscordAdapter.play_tts). + # Discord voice "verbal ack" on the FIRST tool call (discord.voice_fx.enabled): resolve the + # guild whose voice connection is bound to this text channel (mirrors DiscordAdapter.play_tts). _voice_ack_guild: List[Optional[int]] = [None] if source.platform == Platform.DISCORD: _va = self.adapters.get(Platform.DISCORD) @@ -2857,30 +2771,14 @@ class GatewayTurnMixin: _cleanup_progress = False _cleanup_adapter = None + # The one-slot progress/holder containers shared with the callbacks are TurnContext defaults. turn_ctx = TurnContext( - source=source, + source=source, message=message, AIAgent=AIAgent, session_key=session_key, + run_generation=run_generation, _cleanup_progress=_cleanup_progress, _run_still_current=self._run_still_current_fn(session_key, run_generation), progress_queue=queue.Queue() if disp.needs_progress_queue else None, - # Mutable one-slot containers shared with the callbacks: last progress line (dedup), last - # tool, whether the previous line was a terminal fenced block (consecutive terminal calls - # drop the repeated header), repeat counter, first-touch long-tool onboarding latch. - last_progress_msg=[None], - last_tool=[None], - last_was_terminal_block=[False], - repeat_count=[0], - long_tool_hint_fired=[False], - _LONG_TOOL_THRESHOLD_S=30.0, - _cleanup_progress=_cleanup_progress, - _cleanup_msg_ids=[], - message=message, - AIAgent=AIAgent, - _voice_ack_fired=[False], - _voice_ack_guild=_voice_ack_guild, - _voice_ack_loop=asyncio.get_running_loop(), - session_key=session_key, - run_generation=run_generation, - **{name: getattr(disp, name) for name in self._DISPLAY_TO_TURN_CTX}, - **turn_params, + _voice_ack_guild=_voice_ack_guild, _voice_ack_loop=asyncio.get_running_loop(), + **{name: getattr(disp, name) for name in self._DISPLAY_TO_TURN_CTX}, **turn_params, ) turn_runner = TurnRunner(self, turn_ctx) # Agent tool-lifecycle callbacks live on the runner (bound methods, same signatures). @@ -2915,14 +2813,11 @@ class GatewayTurnMixin: def _run_agent_progress_threading( self, source: SessionSource, event_message_id: Optional[str], _native_slack_task_cards: bool ) -> Tuple[Optional[dict], Optional[str], Any, Optional[str]]: - """Resolve where progress bubbles are threaded. + """Resolve where progress bubbles are threaded (platform-specific). Returns ``(progress_metadata, progress_reply_to, progress_thread_id, relay_prospective_thread_id)``. - Threading is platform-specific: Slack DM threading needs the event_message_id fallback; - Telegram forum topics use message_thread_id and Hermes-created private DM topic lanes need - thread metadata plus a reply anchor; Feishu only honors reply_in_thread on a reply; others use - explicit source.thread_id only. Slack and Buzz honour the user's reply_in_thread opt-out: - never synthesise a thread for progress, or every later reply inherits it. + Slack and Buzz honour the user's reply_in_thread opt-out: never synthesise a thread for + progress, or every later reply inherits it. """ from gateway.run import _non_conversational_metadata, _resolve_progress_thread_id is_buzz = str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz" @@ -3025,11 +2920,8 @@ class GatewayTurnMixin: self, source: SessionSource, event_message_id: Optional[str], _progress_thread_id: Any, _relay_prospective_thread_id: Optional[str], ) -> Optional[Dict[str, Any]]: - """Thread metadata for status / approval / stream sends. - - Feishu topics only keep messages inside the topic when sent via the reply API with - reply_in_thread=true; those paths usually only get metadata, so carry the triggering - message id as a Feishu-specific fallback. + """Thread metadata for status / approval / stream sends; Feishu topics need the triggering + message id (reply API with reply_in_thread=true) so carry it as a fallback. """ if source.platform == Platform.FEISHU and source.thread_id and event_message_id: return {"thread_id": _progress_thread_id, "reply_to_message_id": event_message_id} @@ -3043,10 +2935,8 @@ class GatewayTurnMixin: ) -> None: """Start the streaming-TTS consumer for a voice-input turn on an auto-TTS chat. - Created on the gateway event-loop thread (here), NOT inside run_sync's executor worker: the - outer interrupt / finalisation paths reference it via ``streaming_tts_consumer_holder[0]``. - An inactive consumer (no streaming provider) leaves the holder None so the whole-file - fallback path runs. + Created on the gateway loop thread (not run_sync's executor); an inactive consumer leaves + the holder None so the whole-file fallback path runs. """ _stts_adapter = self._adapter_for_source(source) _is_voice_input = ( @@ -3088,11 +2978,8 @@ class GatewayTurnMixin: await stream_task async def _run_agent_track_agent(self, turn_ctx: TurnContext) -> None: - """Track this agent as running for the session (interrupt support) once it is created. - - Only promote the sentinel to the real agent if this run is still current: if /stop or /new - bumped the generation while we were spinning up, leave the newer run's slot alone (the - stale-result check in _handle_message_with_agent discards us). + """Track this agent as running for the session (interrupt support) once it is created — only + if this run is still current, else leave the newer run's slot alone. """ session_key, run_generation, agent_holder = turn_ctx.session_key, turn_ctx.run_generation, turn_ctx.agent_holder while agent_holder[0] is None: @@ -3116,10 +3003,9 @@ class GatewayTurnMixin: ) -> None: """Peek the adapter's pending event, transcribe voice, then signal the agent + abort streaming TTS. - Peek WITHOUT consuming: the message must stay in _pending_messages for the post-run - ``_dequeue_pending_event()`` (full MessageEvent + media). Popping here races: the agent may - finish before checking _interrupt_requested, losing it. Audio is transcribed BEFORE signaling - so voice messages interrupt with the real transcript, not an empty string / path placeholder. + Peek WITHOUT consuming: the event must stay for the post-run ``_dequeue_pending_event()`` + (popping races the agent finishing). Transcribe BEFORE signaling so voice interrupts carry + the real transcript. """ from gateway.run import _build_media_placeholder _peek_event = adapter._pending_messages.get(session_key) @@ -3144,10 +3030,8 @@ class GatewayTurnMixin: async def _run_agent_monitor_for_interrupt(self, turn_ctx: TurnContext, _interrupt_detected: "asyncio.Event") -> None: """Poll the adapter for interrupts (new messages) every 200ms and signal the agent. - PRIMARY interrupt path for regular text: Level 1 (base.py) catches them before - _handle_message(), so the Level 2 running_agent.interrupt() path never fires. The inactivity - poll loop has a BACKUP check in case this task dies silently. Interrupts are keyed by - session_key (build_session_key output), NOT source.chat_id. + Level 1 (base.py) catches regular text before _handle_message(); the inactivity poll loop + has a BACKUP check in case this task dies. Keyed by session_key, NOT source.chat_id. """ source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder streaming_tts_consumer_holder = turn_ctx.streaming_tts_consumer_holder @@ -3198,12 +3082,9 @@ class GatewayTurnMixin: @staticmethod def _run_agent_stream_confirmed_final_delivery(consumer, final_text: str, *, previewed: bool = False) -> bool: - """Return True only when the actual final reply reached the user. - - A successful finalize call is not proof the *content* was final: the edit may carry only the - last preview snapshot. Reconcile against the recorded turn-final payload: only a demonstrable - mismatch (False, incl. payload-less split delivery) overrides the flag; None keeps legacy - trust so timeout dedup isn't regressed. + """True only when the actual final reply reached the user: a finalize call may carry only the + last preview snapshot, so reconcile against the recorded payload — a demonstrable mismatch + (False) overrides the flag; None keeps legacy trust. """ if consumer is None: return False @@ -3226,11 +3107,9 @@ class GatewayTurnMixin: def _run_agent_start_turn_worker(self, turn_ctx: TurnContext, run_sync: Callable[[], Any]) -> "GatewayRunner._RunAgentWorker": """Schedule ``run_sync`` on the executor plus the inactivity watchdog thread. - *Inactivity* timeout, not wall-clock: the agent may run for hours while actively calling - tools / streaming, but a hung API call or stuck tool is killed. agent.gateway_timeout / - HERMES_AGENT_TIMEOUT (env wins); default 1800s; 0 = unlimited. The daemon watchdog is - independent of asyncio: cgroup memory reclaim can starve the loop that runs the normal - timeout poll, and cleanup must not wait for the loop to recover. + *Inactivity* timeout (agent.gateway_timeout / HERMES_AGENT_TIMEOUT, env wins; 0 = unlimited), + not wall-clock. The daemon watchdog is independent of asyncio: cgroup memory reclaim can + starve the loop that runs the normal timeout poll. """ from gateway.run import _float_env, _watch_gateway_turn_inactivity from tools.process_registry import process_registry @@ -3265,10 +3144,9 @@ class GatewayTurnMixin: return run_sync() finally: worker.worker_done.set() - # `.turn.agent` is only reset to _AGENT_PENDING_SENTINEL when the *next* turn is - # claimed, so this agent stays reachable from _interrupt_and_clear_session() until - # then. Clearing ownership markers the instant our worker finishes means a /stop on - # the finished turn no longer reaps background work it left running. + # `.turn.agent` stays reachable until the *next* turn is claimed; clearing the + # ownership markers now means a /stop on the finished turn no longer reaps background + # work it left running. _finished_agent = agent_holder[0] if agent_holder else None if _finished_agent is not None: _finished_agent._gateway_turn_process_task_id = "" @@ -3320,9 +3198,8 @@ class GatewayTurnMixin: try: await _warn_adapter.send( source.chat_id, f"⚠️ No activity for {_elapsed_warn} min. " - f"If the agent does not respond soon, it will " - f"be timed out in {_remaining_mins} min. " - f"You can continue waiting or use /reset.", + f"If the agent does not respond soon, it will be timed out in {_remaining_mins} " + f"min. You can continue waiting or use /reset.", metadata=_interim_metadata(_status_thread_metadata), ) except Exception as _warn_err: @@ -3354,9 +3231,8 @@ class GatewayTurnMixin: ] if _cur_tool: _diag_lines.append( - f"The agent appears stuck on tool `{_cur_tool}` " - f"({_secs_ago:.0f}s since last activity, " - f"iteration {_iter_n}/{_iter_max})." + f"The agent appears stuck on tool `{_cur_tool}` ({_secs_ago:.0f}s since last " + f"activity, iteration {_iter_n}/{_iter_max})." ) else: _diag_lines.append( @@ -3365,9 +3241,8 @@ class GatewayTurnMixin: "The agent may have been waiting on an API response." ) _diag_lines.append( - "To increase the limit, set agent.gateway_timeout in config.yaml " - "(value in seconds, 0 = no limit) and restart the gateway.\n" - "Try again, or use /reset to start fresh." + "To increase the limit, set agent.gateway_timeout in config.yaml (value in seconds, 0 " + "= no limit) and restart the gateway.\nTry again, or use /reset to start fresh." ) return { "final_response": "\n".join(_diag_lines), @@ -3382,12 +3257,9 @@ class GatewayTurnMixin: self, worker: "GatewayRunner._RunAgentWorker", turn_ctx: TurnContext, _interrupt_detected: "asyncio.Event", interrupt_monitor: "asyncio.Task", ) -> Any: - """Poll the executor future (inactivity timeout + backup interrupt checks); return its result. - - Polls the agent's activity tracker (updated by _touch_activity() on every tool call, API - call and stream delta) every few seconds; with an unlimited timeout it still polls for the - backup interrupt check in case monitor_for_interrupt() silently died. On inactivity timeout - the result is a synthetic failed run dict carrying the diagnostic. + """Poll the executor future (inactivity timeout + backup interrupt checks); return its result, + or a synthetic failed run dict on inactivity timeout. Polls even with an unlimited timeout + so the backup interrupt check runs if monitor_for_interrupt() silently died. """ from gateway.run import _abandon_timed_out_gateway_turn agent_holder = turn_ctx.agent_holder @@ -3410,19 +3282,16 @@ class GatewayTurnMixin: threading.Thread( target=_abandon_timed_out_gateway_turn, kwargs={"agent_holder": agent_holder, **self._reaper_kwargs(worker)}, - name=f"gateway-turn-reaper-{worker.task_id[:12]}", - daemon=True, + name=f"gateway-turn-reaper-{worker.task_id[:12]}", daemon=True, ).start() break await self._run_agent_backup_interrupt_check(turn_ctx, _interrupt_detected, interrupt_monitor) return self._run_agent_timeout_result(worker, turn_ctx) def _run_agent_evict_on_fallback(self, turn_ctx: TurnContext) -> None: - """Evict the cached agent when a fallback model activated on a successful run. - - So /model shows the actually-active model and the next message retries the primary. Skip - when the run failed — evicting forces MCP reinit on the next message for no benefit (bad - model → fallback → evict → recreate → same 400 loop burning CPU). + """Evict the cached agent when a fallback model activated on a SUCCESSFUL run (so /model shows + the active model and the next message retries the primary). Skip failed runs: evicting + would loop bad model → fallback → evict → recreate. """ from gateway.run import _resolve_gateway_model session_key = turn_ctx.session_key @@ -3444,12 +3313,9 @@ class GatewayTurnMixin: self._evict_cached_agent(session_key) async def _run_agent_finalize_streaming_tts(self, turn_ctx: TurnContext, adapter: Any) -> None: - """Finalize the streaming-TTS consumer on the outer event-loop thread. - - finish() here covers early returns from run_sync too. wait_complete() drains queued audio; - on timeout abort unconditionally to free the consumer task — audible streams retain - whole-file suppression (no replay from the start); silent streams stay eligible for the - whole-file fallback. + """Finalize the streaming-TTS consumer on the outer event-loop thread (covers early returns + from run_sync). On drain timeout abort to free the task — audible streams keep whole-file + suppression, silent streams stay eligible for the whole-file fallback. """ _stts = turn_ctx.streaming_tts_consumer_holder[0] if _stts is None: @@ -3481,10 +3347,8 @@ class GatewayTurnMixin: pending = None if result and adapter and session_key: pending_event = _dequeue_pending_event(adapter, session_key) - # /queue overflow: after consuming the adapter's "next-up" slot, promote the next queued - # event into it so the recursive run's drain will see it. Keeping the slot occupied for - # the whole FIFO chain preserves order and makes a mid-chain /queue route to overflow - # instead of jumping the queue. + # /queue overflow: promote the next queued event into the consumed "next-up" slot so the + # recursive drain sees it (keeps FIFO order; a mid-chain /queue can't jump the queue). pending_event = self._promote_queued_event(session_key, adapter, pending_event) if result.get("interrupted") and not pending_event and result.get("interrupt_message"): interrupt_message = result.get("interrupt_message") @@ -3749,11 +3613,10 @@ class GatewayTurnMixin: async def _run_agent_mark_streamed_delivery(self, response: Any, turn_ctx: TurnContext) -> None: """Set ``response["already_sent"]`` when streaming already delivered the final reply. - Never when the agent failed (the error is unseen content) or on "(empty)": interim text - ("Let me search…") set already_sent but is NOT the final answer; suppressing would leave the - user with silence. Both suppression flags reflect call success, not content, so reconcile - against the recorded turn-final payload: on mismatch (False, incl. payload-less split - delivery) neither flag may suppress the final send; None (no record) keeps legacy trust. + Never when the agent failed (the error is unseen content) or on "(empty)". Both suppression + flags reflect call success, not content, so reconcile against the recorded turn-final + payload: a mismatch (False, incl. payload-less split delivery) never suppresses; None (no + record) keeps legacy trust. """ _sc, source, session_key = turn_ctx.stream_consumer_holder[0], turn_ctx.source, turn_ctx.session_key if not isinstance(response, dict) or response.get("failed"): @@ -3790,10 +3653,8 @@ class GatewayTurnMixin: ) response["already_sent"] = True elif not _transformed and _stale_finalized and _sc is not None: - # Stale finalize: the streamed message holds only the last preview snapshot. Edit it up to - # the complete response; on edit failure leave already_sent unset so the normal send - # delivers. Not for split delivery: message_id is only the LAST chunk, so editing it would - # repeat every sealed head chunk — fall through to the normal send. + # Stale finalize: edit the streamed message up to the complete response (on failure the + # normal send delivers). Not for split delivery — message_id is only the LAST chunk. _sc_msg_id = _sc.message_id if getattr(_sc, "_turn_split_delivery", False): logger.info( @@ -3847,9 +3708,8 @@ class GatewayTurnMixin: # normal final-send is about to run. Log the decision inputs so a recurrence can be pinned # to "signal never set" vs "ack-pending race". logger.warning( - "Normal final-send NOT suppressed despite active stream " - "consumer for session %s: streamed=%s previewed=%s " - "content_delivered=%s transformed=%s final_len=%d — " + "Normal final-send NOT suppressed despite active stream consumer for session %s: " + "streamed=%s previewed=%s content_delivered=%s transformed=%s final_len=%d — " "possible duplicate send (see wecom ack-timeout RCA).", _sk, _streamed, _previewed, _content_delivered, _transformed, len(_final), ) @@ -3897,22 +3757,14 @@ class GatewayTurnMixin: self, turn_ctx: TurnContext, turn_runner: TurnRunner, source: SessionSource, event_message_id: Optional[str], _native_slack_task_cards: bool, ) -> Optional[Dict[str, Any]]: - """Resolve progress threading, then publish progress metadata, result holders and the - sync→async bridges onto ``turn_ctx``. - - Returns ``_status_thread_metadata``; the holders are read back via ``turn_ctx.*_holder``. - The one-slot holders share agent / result / tool-defs / stream consumer / streaming-TTS - consumer between run_sync's executor thread and the outer finalisation + interrupt paths. + """Resolve progress threading, then publish progress metadata and the sync→async bridges onto + ``turn_ctx`` (the one-slot holders shared with run_sync's executor thread are TurnContext + defaults). Returns ``_status_thread_metadata``. """ ( turn_ctx._progress_metadata, turn_ctx._progress_reply_to, _progress_thread_id, _relay_prospective_thread_id, ) = self._run_agent_progress_threading(source, event_message_id, _native_slack_task_cards) - turn_ctx.agent_holder = [None] - turn_ctx.result_holder = [None] - turn_ctx.tools_holder = [None] - turn_ctx.stream_consumer_holder = [None] - turn_ctx.streaming_tts_consumer_holder = [None] # Bridges: sync step_callback / event_callback / status_callback → async hooks.emit and # adapter.send (TurnRunner methods bound onto the shared ctx). turn_ctx._loop_for_step = asyncio.get_running_loop() @@ -3929,18 +3781,21 @@ class GatewayTurnMixin: return _status_thread_metadata async def _run_agent_notify_long_running( - self, disp: "GatewayRunner._RunAgentDisplay", turn_ctx: TurnContext, *, _executor_task_holder: list, - _NOTIFY_INTERVAL: Optional[float], _long_running_mode: str, _notify_start: float, + self, disp: "GatewayRunner._RunAgentDisplay", turn_ctx: TurnContext, _executor_task_holder: list, ) -> None: - """Periodic "still working" heartbeat, edited in place where the adapter supports it. + """Periodic "still working" heartbeat, edited in place where supported. Stops once this run + no longer owns the session slot or the executor finished. ``_executor_task_holder[0]`` is + bound just after this task is scheduled (reads as None until then). - Stops once this run no longer owns the session slot or the executor has finished, else a - stale "running: delegate_task" bubble outlives its run. ``_executor_task_holder[0]`` is bound - just after this task is scheduled; tolerate the brief window before then (reads as None). + Interval: agent.gateway_notify_interval / HERMES_AGENT_NOTIFY_INTERVAL (default 180s; 0 or + long_running_notifications=off disables). """ - from gateway.run import _interim_metadata, _non_conversational_metadata - if _NOTIFY_INTERVAL is None: - return # Notifications disabled (gateway_notify_interval: 0) + from gateway.run import _float_env, _interim_metadata, _non_conversational_metadata + _notify_start = time.time() + _NOTIFY_INTERVAL = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180) + _long_running_mode = disp._display_surface_mode("long_running_notifications", default=True, allow_generic=True) + if _NOTIFY_INTERVAL <= 0 or _long_running_mode == "off": + return source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder _status_thread_metadata = turn_ctx._status_thread_metadata _notify_adapter = self._adapter_for_source(source) @@ -4009,7 +3864,6 @@ class GatewayTurnMixin: Keys: "final_response", "messages", "api_calls", "completed". """ - from gateway.run import _float_env if self._get_proxy_url(): return await self._run_agent_via_proxy( message=message, context_prompt=context_prompt, history=history, source=source, @@ -4048,20 +3902,9 @@ class GatewayTurnMixin: _interrupt_detected = asyncio.Event() # shared with backup check interrupt_monitor = asyncio.create_task(self._run_agent_monitor_for_interrupt(turn_ctx, _interrupt_detected)) - # Periodic "still working" notifications so the user knows the agent hasn't died. Config: - # agent.gateway_notify_interval or HERMES_AGENT_NOTIFY_INTERVAL env; default 180s. - _NOTIFY_INTERVAL_RAW = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180) - _NOTIFY_INTERVAL = _NOTIFY_INTERVAL_RAW if _NOTIFY_INTERVAL_RAW > 0 else None - _long_running_mode = disp._display_surface_mode("long_running_notifications", default=True, allow_generic=True) - if _long_running_mode == "off": - _NOTIFY_INTERVAL = None + # Periodic "still working" notifications so the user knows the agent hasn't died. _executor_task_holder: list = [None] # bound once the executor future exists (see below) - _notify_task = asyncio.create_task( - self._run_agent_notify_long_running( - disp, turn_ctx, _executor_task_holder=_executor_task_holder, _NOTIFY_INTERVAL=_NOTIFY_INTERVAL, - _long_running_mode=_long_running_mode, _notify_start=time.time(), - ) - ) + _notify_task = asyncio.create_task(self._run_agent_notify_long_running(disp, turn_ctx, _executor_task_holder)) try: # run_sync is TurnRunner.run_sync (bound method; executor call unchanged). From 757d85945be70bd2fbe40fe4f3ad7973aeba8161 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:42:39 -0700 Subject: [PATCH 3/6] =?UTF-8?q?refactor(gateway):=20run=5Fturn=20=E2=80=94?= =?UTF-8?q?=20media-sender=20dispatch,=20flattened=20429/home-channel/prox?= =?UTF-8?q?y-key=20ladders,=20display=20setter=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gateway/run_turn.py | 128 +++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 79 deletions(-) diff --git a/gateway/run_turn.py b/gateway/run_turn.py index d36a0a788c..2b93610155 100644 --- a/gateway/run_turn.py +++ b/gateway/run_turn.py @@ -206,12 +206,8 @@ class GatewayTurnMixin: model = getattr(agent, "model", None) if not model: return - runtime = { - "provider": getattr(agent, "provider", None), - "base_url": getattr(agent, "base_url", None), - "api_mode": getattr(agent, "api_mode", None), - "fallback_active": bool(getattr(agent, "_fallback_activated", False)), - } + runtime = {k: getattr(agent, k, None) for k in ("provider", "base_url", "api_mode")} + runtime["fallback_active"] = bool(getattr(agent, "_fallback_activated", False)) runtime = {k: v for k, v in runtime.items() if v not in (None, "")} try: db = self._session_db._db @@ -1283,14 +1279,11 @@ class GatewayTurnMixin: # Multiplex: the home channel may live only in the profile secret scope / PlatformConfig, # not process os.environ. home_env = "" - try: - from agent.secret_scope import get_secret - - home_env = (get_secret(env_key) or "").strip() if env_key else "" - except Exception: - home_env = "" - if not home_env: - home_env = (os.getenv(env_key) or "").strip() if env_key else "" + if env_key: + with suppress(Exception): + from agent.secret_scope import get_secret + home_env = (get_secret(env_key) or "").strip() + home_env = home_env or (os.getenv(env_key) or "").strip() # Also honor in-memory / yaml home_channel on this platform. with suppress(Exception): if not home_env and self.config.get_home_channel(source.platform): @@ -1349,13 +1342,10 @@ class GatewayTurnMixin: workspace, so preserve the routing metadata used by the response delivery path.""" try: _typing_adapter = self._adapter_for_source(source) - _stop_with_metadata = getattr(type(_typing_adapter), "_stop_typing_with_metadata", None) - _stop_typing = getattr(type(_typing_adapter), "stop_typing", None) - if _typing_adapter and callable(_stop_with_metadata): - await _typing_adapter._stop_typing_with_metadata( - source.chat_id, self._event_thread_metadata(event, source), - ) - elif _typing_adapter and callable(_stop_typing): + _kind = type(_typing_adapter) + if _typing_adapter and callable(getattr(_kind, "_stop_typing_with_metadata", None)): + await _typing_adapter._stop_typing_with_metadata(source.chat_id, self._event_thread_metadata(event, source)) + elif _typing_adapter and callable(getattr(_kind, "stop_typing", None)): await _typing_adapter.stop_typing(source.chat_id) except Exception: pass @@ -1794,23 +1784,19 @@ class GatewayTurnMixin: status_hint = self._STATUS_HINTS.get(status_code, "") if status_code == 429: # Plan usage limit (resets on a schedule) vs a transient rate limit - _err_body = getattr(e, "response", None) _err_json = {} with suppress(Exception): - if _err_body is not None: - _err_json = _err_body.json().get("error", {}) - if not isinstance(_err_json, dict): - _err_json = {} - if _err_json.get("type") == "usage_limit_reached": - _resets_in = _err_json.get("resets_in_seconds") - if _resets_in and _resets_in > 0: - import math - _hours = math.ceil(_resets_in / 3600) - status_hint = f" Your plan's usage limit has been reached. It resets in ~{_hours}h." - else: - status_hint = " Your plan's usage limit has been reached. Please wait until it resets." - else: + _err_json = e.response.json().get("error", {}) + if not isinstance(_err_json, dict): + _err_json = {} + _resets_in = _err_json.get("resets_in_seconds") + if _err_json.get("type") != "usage_limit_reached": status_hint = " You are being rate-limited. Please wait a moment and try again." + elif _resets_in and _resets_in > 0: + import math + status_hint = f" Your plan's usage limit has been reached. It resets in ~{math.ceil(_resets_in / 3600)}h." + else: + status_hint = " Your plan's usage limit has been reached. Please wait until it resets." elif status_code in {400, 500}: # 400 on a large session is context overflow; 500 on a large session often means the # payload is too large for the API — treat it the same way. @@ -1969,7 +1955,7 @@ class GatewayTurnMixin: "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "chat_id": source.chat_id or "", - "thread_id": str(getattr(source, "thread_id", None)) if getattr(source, "thread_id", None) else "", + "thread_id": str(source.thread_id) if getattr(source, "thread_id", None) else "", "chat_type": getattr(source, "chat_type", "") or "", "session_id": session_entry.session_id, "message": message_text[:500], @@ -2064,12 +2050,10 @@ class GatewayTurnMixin: "config": "config", "default": "default — set model.context_length in config to override", }.get(resolved.context_source, "detected") - if context_length >= 1_000_000: - ctx_display = f"{context_length / 1_000_000:.1f}M" - elif context_length >= 1_000: - ctx_display = f"{context_length // 1_000}K" - else: - ctx_display = str(context_length) + ctx_display = ( + f"{context_length / 1_000_000:.1f}M" if context_length >= 1_000_000 + else f"{context_length // 1_000}K" if context_length >= 1_000 else str(context_length) + ) lines = [ f"◆ Model: `{resolved.model}`", f"◆ Provider: {resolved.provider or 'openrouter'}", @@ -2245,18 +2229,13 @@ class GatewayTurnMixin: chat_id=source.chat_id, audio_path=media_path, metadata=_thread_metadata, is_voice=_is_voice, ) - elif _ext in _VIDEO_EXTS: - await adapter.send_video( - chat_id=source.chat_id, video_path=media_path, metadata=_thread_metadata, - ) - elif _ext in _IMAGE_EXTS: - await adapter.send_image_file( - chat_id=source.chat_id, image_path=media_path, metadata=_thread_metadata, - ) else: - await adapter.send_document( - chat_id=source.chat_id, file_path=media_path, metadata=_thread_metadata, + sender, key = ( + (adapter.send_video, "video_path") if _ext in _VIDEO_EXTS + else (adapter.send_image_file, "image_path") if _ext in _IMAGE_EXTS + else (adapter.send_document, "file_path") ) + await sender(chat_id=source.chat_id, metadata=_thread_metadata, **{key: media_path}) except Exception as e: logger.exception("Background task %s failed", task_id) @@ -2479,14 +2458,10 @@ class GatewayTurnMixin: # Scope-aware read: the proxy key is a per-profile credential; under multiplex honor the # installed scope's verdict (Slack pattern for the unscoped default-profile loop). - try: - from agent.secret_scope import UnscopedSecretError, get_secret - try: - proxy_key = (get_secret("GATEWAY_PROXY_KEY") or "").strip() - except UnscopedSecretError: - proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() - except Exception: - proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() + proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() + with suppress(Exception): # UnscopedSecretError and import failures fall back to the env + from agent.secret_scope import get_secret + proxy_key = (get_secret("GATEWAY_PROXY_KEY") or "").strip() _run_still_current = self._run_still_current_fn(session_key, run_generation) @@ -2629,14 +2604,14 @@ class GatewayTurnMixin: _display_cfg = {} # Tool preview length (0 = no limit) and friendly tool labels (default on), per-platform. - with suppress(Exception): - from agent.display import set_tool_preview_max_len - _tpl = resolve_display_setting(user_config, platform_key, "tool_preview_length", 0) - set_tool_preview_max_len(int(_tpl) if _tpl else 0) - with suppress(Exception): - from agent.display import set_friendly_tool_labels - _ftl = resolve_display_setting(user_config, platform_key, "friendly_tool_labels", True) - set_friendly_tool_labels(bool(_ftl)) + for _setter, _setting, _default, _cast in ( + ("set_tool_preview_max_len", "tool_preview_length", 0, lambda v: int(v) if v else 0), + ("set_friendly_tool_labels", "friendly_tool_labels", True, bool), + ): + with suppress(Exception): + from agent import display as _agent_display + _val = resolve_display_setting(user_config, platform_key, _setting, _default) + getattr(_agent_display, _setter)(_cast(_val)) # Tool progress mode — per-platform, with HERMES_TOOL_PROGRESS_MODE winning only when the # config never set it. @@ -2690,9 +2665,9 @@ class GatewayTurnMixin: # Live working-state status for text-rendering typing indicators (Slack's assistant status # line). Independent of tool_progress; rides the existing _keep_typing refresh. _live_status_mode = resolve_display_setting(user_config, platform_key, "live_status", "full") - _live_status_adapter = adapter - if not getattr(_live_status_adapter, "supports_status_text", False) or _live_status_mode == "off": - _live_status_adapter = None + _live_status_adapter = ( + adapter if getattr(adapter, "supports_status_text", False) and _live_status_mode != "off" else None + ) # "log" mode: tool calls go to ~/.hermes/logs/tool_calls.log instead of the chat. Gateway-only. log_mode_enabled = progress_mode == "log" and not is_webhook # Natural assistant status messages are independent from tool progress and token streaming. @@ -3495,14 +3470,9 @@ class GatewayTurnMixin: await self._run_agent_deliver_first_response(turn_ctx, adapter, response, result, stream_task) updated_history = result.get("messages", history) - next_source = source - next_message = pending - next_message_id = None - next_channel_prompt = None - next_session_key = session_key - # Carry the pending event's message_type into the recursive call so queued voice turns can - # stream TTS and re-mark the generation for the final delivered turn. - next_message_type = None + next_source, next_message, next_session_key = source, pending, session_key + # message_type is carried into the recursive call so queued voice turns can stream TTS. + next_message_id = next_channel_prompt = next_message_type = None if pending_event is not None: next_source = getattr(pending_event, "source", None) or source if self._is_goal_continuation_event(pending_event) and not self._goal_still_active_for_session(session_id): From 159562aa02b7f3744ae116e61f67fc23cf72036b Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:49:47 -0700 Subject: [PATCH 4/6] =?UTF-8?q?refactor(gateway):=20run=5Fturn=20=E2=80=94?= =?UTF-8?q?=20compact=20restating=20comments/docstrings=20(docstring-strip?= =?UTF-8?q?ped=20AST=20identical)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gateway/run_turn.py | 437 ++++++++++++++++---------------------------- 1 file changed, 161 insertions(+), 276 deletions(-) diff --git a/gateway/run_turn.py b/gateway/run_turn.py index 2b93610155..4998cbef2c 100644 --- a/gateway/run_turn.py +++ b/gateway/run_turn.py @@ -79,8 +79,7 @@ class GatewayTurnMixin: skey or "", model, override_model, override_runtime.get("provider"), ) return override_model, override_runtime - # No api_key on the override: env-based resolution below, then the override's - # model/provider applied on top. + # No api_key on the override: env-based resolution below, override model/provider on top. logger.debug( "Session model override (no api_key, fallback): session=%s config_model=%s override_model=%s", skey or "", model, override_model, @@ -121,8 +120,8 @@ class GatewayTurnMixin: if override and skey: model, runtime_kwargs = self._apply_session_model_override(skey, model, runtime_kwargs) - # No model.default but a provider resolved (e.g. `hermes auth add openai-codex` without - # `hermes model`): fall back to the provider's first catalog model so the API call has one. + # Provider resolved but no model.default (`hermes auth add` without `hermes model`): use the + # provider's first catalog model. if not model and runtime_kwargs.get("provider"): with suppress(Exception): from hermes_cli.models import get_default_model_for_provider @@ -132,9 +131,8 @@ class GatewayTurnMixin: "No model configured — defaulting to %s for provider %s", model, runtime_kwargs["provider"], ) - # Final safety net: an empty model (e.g. transient config-cache miss on a post-interrupt - # recovery turn) makes every API call fail HTTP 400 and the session goes silent — reuse the - # last model resolved for this session, else the most recent process-wide. + # Final safety net: an empty model (transient config-cache miss) makes every API call 400 and + # the session goes silent — reuse the last model resolved for this session, else process-wide. if not model: _lr_state = self._peek_session_state(skey) if skey else None _lr_star = self._peek_session_state("*") @@ -159,8 +157,7 @@ class GatewayTurnMixin: def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwargs: dict) -> dict: """Effective model/runtime config for one turn. With `/fast` priority on, fast-mode - ``request_overrides`` are deep-merged OVER the per-provider ones so both reach the model. - """ + ``request_overrides`` are deep-merged OVER the per-provider ones so both reach the model.""" from gateway.run import _deep_merge_request_overrides from hermes_cli.models import resolve_fast_mode_overrides @@ -183,8 +180,7 @@ class GatewayTurnMixin: ), } if getattr(self, "_service_tier", None) != "priority": - # None (normal) or auto/cold — the bounded window is applied per request by - # agent.fast_mode, not pinned into request_overrides. + # None / auto / cold: the bounded window is applied per request by agent.fast_mode. route["request_overrides"] = base_request_overrides return route try: @@ -254,7 +250,7 @@ class GatewayTurnMixin: topic-binding heal). Returns ``(source, session_entry, session_key)`` or ``None`` to drop the event.""" # Topic-mode DMs: rewrite a stale/foreign thread_id to the user's last-active topic so a - # cross-topic Reply or stripped plain reply doesn't fragment the conversation. + # cross-topic Reply doesn't fragment the conversation. recovered = await asyncio.to_thread(self._recover_telegram_topic_thread_id, source) if recovered is not None: logger.info( @@ -287,9 +283,8 @@ class GatewayTurnMixin: ) return else: - # Internal wakes must observe reset policy without counting as user activity, or - # periodic Kanban/process notifications keep the routing key alive across every - # daily/idle boundary. + # Internal wakes observe reset policy without counting as user activity, or periodic + # notifications keep the routing key alive across every daily/idle boundary. session_entry = await self.async_session_store.get_or_create_session( source, touch_activity=not bool(getattr(event, "internal", False)), ) @@ -334,9 +329,8 @@ class GatewayTurnMixin: if canonical_session_id and canonical_session_id != bound_session_id: bound_session_id = canonical_session_id if bound_session_id and bound_session_id != session_entry.session_id: - # Route through SessionStore so the session_key → session_id mapping is persisted and - # the previous lane session ended cleanly (mutating session_entry in place split-brained - # the JSON index from downstream code). + # Route through SessionStore so the key → id mapping persists and the previous lane + # session ends cleanly (in-place mutation split-brained the JSON index). switched = await self.async_session_store.switch_session(session_key, bound_session_id) if switched is not None: session_entry = switched @@ -350,13 +344,11 @@ class GatewayTurnMixin: async def _hmwa_open_session(self, session_entry, session_key, source): """Consume auto-reset / fresh-reset flags and emit ``session:start`` for new sessions. Returns ``(_was_auto_reset, _is_new_session)``.""" - # Consume was_auto_reset immediately so it cannot re-fire on later messages and wipe - # model/reasoning overrides set between turns. + # Consume was_auto_reset immediately so it cannot re-fire and wipe overrides set between turns. _was_auto_reset = getattr(session_entry, "was_auto_reset", False) if _was_auto_reset: - # Full conversation boundary: one funnel call clears every conversation-scoped - # per-session dict; evict the cached agent (keyed on the stable session_key) so - # context_compressor._previous_summary cannot leak prior history into new summaries. + # Conversation boundary: the funnel clears every conversation-scoped dict; evict the cached + # agent so context_compressor._previous_summary cannot leak into new summaries. self._clear_conversation_scope(session_key, reason="auto_reset") self._evict_cached_agent(session_key) session_entry.was_auto_reset = False @@ -380,8 +372,7 @@ class GatewayTurnMixin: from gateway.run import _AUTO_RESET_CONTEXT_NOTES, _auto_reset_reason_text reset_reason = getattr(session_entry, 'auto_reset_reason', None) or 'idle' context_note = _AUTO_RESET_CONTEXT_NOTES.get(reset_reason, _AUTO_RESET_CONTEXT_NOTES["idle"]) - # Long-lived Slack/Discord channels: point the agent at the specific prior same-channel - # session so session_search recalls that context (deterministic, no extra API/DB calls). + # Long-lived channels: point the agent at the prior same-channel session for session_search. try: continuity_note = build_channel_continuity_note(session_entry, source) except Exception: @@ -464,8 +455,7 @@ class GatewayTurnMixin: timeout=_float_env("HERMES_TURN_LEASE_TIMEOUT", DEFAULT_LEASE_WAIT), ) except TurnLeaseTimeoutError: - # The broad session-context cleanup finally starts later; restore the tokens here or - # this early exit leaks task-local identity. + # The cleanup finally starts later; restore the tokens here or this exit leaks identity. self._clear_session_env(_session_env_tokens) raise if _lease_token is not None: @@ -498,8 +488,7 @@ class GatewayTurnMixin: hs.provider = _model_cfg.get("provider") or None hs.base_url = _model_cfg.get("base_url") or None - # Only the enabled flag is shared with the agent's compression config; hygiene's - # threshold is deliberately separate (runs higher). + # Only the enabled flag is shared with the agent's compression config (hygiene runs higher). _comp_cfg = data.get("compression", {}) if not isinstance(_comp_cfg, dict): return @@ -565,8 +554,7 @@ class GatewayTurnMixin: except Exception: hs.config_context_length = None - # custom_providers per-model context_length fallback (as in run_agent.py); must run - # after runtime resolution so base_url is set. + # custom_providers per-model context_length fallback (as in run_agent.py); needs base_url. if hs.config_context_length is None and hs.base_url: try: try: @@ -610,9 +598,8 @@ class GatewayTurnMixin: else: _approx_tokens, _token_source = estimate_messages_tokens_rough(history), "estimated" - # Hard safety valve: force compression at an extreme message count regardless of token - # estimates, breaking the spiral where API disconnects prevent token data → no compression - # → more disconnects. Default 5000 sits clear of legitimate 1M+ context sessions. + # Hard safety valve: force compression at an extreme message count regardless of tokens, + # breaking the disconnect → no token data → no compression spiral. 5000 clears 1M+ sessions. _needs_compress = _approx_tokens >= _compress_token_threshold or _msg_count >= hs.hard_msg_limit if _needs_compress: @@ -636,9 +623,8 @@ class GatewayTurnMixin: _needs_compress = False if _needs_compress and await self._session_has_compression_in_flight(session_key): - # A prior hygiene/agent compression still holds the durable lock (typically a shielded - # worker left behind by /stop or /restart). Another attempt would wait up to the 600s - # ceiling behind a commit the fence will refuse, while inbound messages demote to queue. + # A prior compression still holds the durable lock (e.g. a shielded worker left by /stop): + # another attempt would wait up to 600s behind a commit the fence will refuse. logger.info( "Session hygiene: skipping compression for %s; " "another compression is already in flight", session_entry.session_id, @@ -665,16 +651,14 @@ class GatewayTurnMixin: while True: if fence.is_cancelled: raise asyncio.TimeoutError - # Charge the idle budget from the LAST PROGRESS event, not from the start of this wait - # slice — otherwise silence can approach 2x the configured timeout. + # Charge the idle budget from the LAST PROGRESS event, else silence can approach 2x timeout. _hyg_waited = time.monotonic() - attempt.wait_started _slice = min( max(hs.timeout_seconds - fence.seconds_since_progress(), 0.005), max(hs.total_ceiling_seconds - _hyg_waited, 0.005), ) - # Cap the slice at the remaining turn-hold budget so it is re-checked at least that - # often — a continuously-streaming worker would otherwise keep the slice large and hold - # the turn until the ceiling. Budget exhausted → immediate timeout → abandonment path. + # Cap the slice at the remaining turn-hold budget so a continuously-streaming worker can't + # hold the turn until the ceiling. Budget exhausted → immediate timeout → abandonment. _turn_hold_remaining = hs.max_turn_hold_seconds - (time.monotonic() - attempt.wait_started) _slice = 0.005 if _turn_hold_remaining <= 0 else min(_slice, max(_turn_hold_remaining, 0.005)) # Short poll so a /stop or /restart cancel is not stuck behind a full idle window. @@ -688,9 +672,8 @@ class GatewayTurnMixin: raise _hyg_waited = time.monotonic() - attempt.wait_started _idle = fence.seconds_since_progress() - # Never hold the user's TURN past the budget even if the summary model is still - # streaming: the turn-hold path proceeds on the uncompressed transcript so the wire - # never trips a transport idle-timeout. + # Never hold the TURN past the budget even while the summary streams: proceed on the + # uncompressed transcript so the wire never trips a transport idle-timeout. if _hyg_waited >= hs.max_turn_hold_seconds: logger.info( "Session hygiene compression for session %s exceeded the turn-hold " @@ -813,9 +796,8 @@ class GatewayTurnMixin: except Exception as _rs_err: logger.debug("hygiene streak reset after deferred adoption failed: %s", _rs_err) else: - # Nothing to adopt (summary failed, fence refused the commit, or the attempt - # was superseded). Flat spacing so sustained traffic does not spawn and - # abandon a fresh compressor every turn. + # Nothing to adopt (summary failed / fence refused / superseded): flat spacing so + # sustained traffic doesn't spawn and abandon a compressor every turn. _record_hygiene_cooldown( _gw, _sid, _HYGIENE_TURNHOLD_RETRY_SECONDS, "hygiene compression deferred: turn-hold budget expired and the " @@ -831,8 +813,7 @@ class GatewayTurnMixin: _adopted = await self._hmwa_hygiene_cancel_or_adopt(attempt, "session hygiene turn-hold") if _adopted is not None: return _adopted - # Short flat retry-after: without it every turn re-spawns a compressor, holds it for - # the budget and cancels it — token burn that never commits. + # Short flat retry-after, else every turn re-spawns, holds and cancels a compressor. _record_hygiene_cooldown( self, session_entry.session_id, _HYGIENE_TURNHOLD_RETRY_SECONDS, "hygiene compression deferred: turn-hold budget expired while the " @@ -862,12 +843,10 @@ class GatewayTurnMixin: _hyg_waited = time.monotonic() - attempt.wait_started _hyg_total_exhausted = _hyg_waited >= hs.total_ceiling_seconds or fence.deadline_exceeded if _hyg_total_exhausted: - # The worker cooperatively checks this deadline between digest calls. Keep its lease - # until it exits so an unchanged session cannot overlap a retry (the release below is - # then a no-op). + # The worker checks this deadline between digest calls; keep its lease until it exits so + # an unchanged session cannot overlap a retry (the release below is then a no-op). fence.retain_compression_lock_until_worker_done() - # Capture fence state BEFORE try_cancel — that call itself sets is_cancelled, which would - # mis-label a genuine idle timeout as a fence cancel. + # Capture fence state BEFORE try_cancel (which itself sets is_cancelled). _hyg_fence_cancelled = fence.is_cancelled _adopted = await self._hmwa_hygiene_cancel_or_adopt(attempt, "session hygiene timeout") if _adopted is not None: @@ -955,13 +934,11 @@ class GatewayTurnMixin: a repoint-then-failed-rewrite would point the live entry at an empty session.""" from agent.model_metadata import estimate_messages_tokens_rough _hyg_agent = attempt.agent - # _compress_context ends the old session and creates a new session_id; compressed messages - # go into the NEW session so the old transcript stays intact and searchable. + # _compress_context rotates to a NEW session_id so the old transcript stays intact/searchable. _hyg_new_sid = _hyg_agent.session_id _hyg_rotated = _hyg_new_sid != session_entry.session_id _hyg_in_place = bool(getattr(_hyg_agent, "_last_compaction_in_place", False)) - # Anti-growth guard: refuse a compression that did not shrink the transcript (observed: - # 427K -> 598K). Compare like-for-like rough estimates. + # Anti-growth guard: refuse a compression that did not shrink the transcript (seen 427K→598K). _hyg_in_toks = estimate_messages_tokens_rough(history) _hyg_out_toks = estimate_messages_tokens_rough(_compressed) if _hyg_rotated and _hyg_out_toks > _hyg_in_toks: @@ -984,8 +961,7 @@ class GatewayTurnMixin: _hyg_in_place = False else: session_entry.session_id = _hyg_new_sid - # The held turn lease follows the rotation so an alias key resolving the fresh - # child still serializes against this turn. + # The held turn lease follows the rotation (alias keys still serialize on this turn). self._rebind_turn_lease(_quick_key, run_generation, _hyg_new_sid) await self.async_session_store._save() await asyncio.to_thread( @@ -993,8 +969,7 @@ class GatewayTurnMixin: ) if _hyg_rotated or _hyg_in_place: - # Transcript rewritten (rotation) or already persisted by archive_and_compact() - # (in-place): reset the stored token count to match the new active set. + # Rewritten (rotation) or persisted by archive_and_compact() (in-place): reset token count. session_entry.last_prompt_tokens = 0 attempt.history = _compressed _new_count = len(_compressed) @@ -1028,21 +1003,18 @@ class GatewayTurnMixin: attempt, _compressed, history, plan, session_entry=session_entry, source=source, _quick_key=_quick_key, run_generation=run_generation, ) - # Summary failure aborts the compressor entirely (messages unchanged, nothing dropped). - # Warn the gateway user visibly — agent.log is invisible on TG/Discord/etc. — so they - # know the chat is "frozen" at this size and can /compress to retry or /reset. + # Summary failure aborts the compressor (nothing dropped). Warn the user visibly — agent.log + # is invisible on TG/Discord — so they know the chat is "frozen" and can /compress or /reset. _comp = getattr(attempt.agent, "context_compressor", None) _hyg_aborted = _comp is not None and getattr(_comp, "_last_compress_aborted", False) # A fence-cancelled _compress_context returns the original transcript with - # _last_compress_aborted still False. Treat that no-op as an abort so hygiene records a - # cooldown instead of retrying into the 600s wait. A successful rotate/in-place commit is - # not an abort even if a later invalidation flipped the fence. + # _last_compress_aborted False: treat that no-op as an abort so hygiene records a cooldown + # instead of retrying into the 600s wait. A committed rotate/in-place is never an abort. _hyg_fence_cancelled = bool(attempt.commit_fence.is_cancelled and not outcome.rotated and not outcome.in_place) if _hyg_fence_cancelled: _hyg_aborted = True - # Recovery decision lives in the unit-tested predicate: the degenerate "neither rotated - # nor compacted in place" path reuses the pre-compression counts, so a numbers-only check - # would read a no-op as success and clear the streak. + # Recovery decision lives in the unit-tested predicate: the "neither rotated nor in place" + # path reuses pre-compression counts, so a numbers-only check would read a no-op as success. if not _hyg_aborted and hygiene_compaction_recovered( aborted=_hyg_aborted, rotated=outcome.rotated, in_place=outcome.in_place, msg_count=plan.msg_count, new_count=outcome.new_count, approx_tokens=plan.approx_tokens, @@ -1061,8 +1033,7 @@ class GatewayTurnMixin: ) if not _hyg_fence_cancelled: _err = getattr(_comp, "_last_summary_error", None) or "unknown error" - # Force-redact: provider exception text may contain credentials and this message - # reaches gateway users. + # Force-redact: provider exception text may contain credentials; this reaches users. from agent.redact import redact_sensitive_text _err = redact_sensitive_text(_err, force=True) await self._hmwa_hygiene_notify( @@ -1072,8 +1043,7 @@ class GatewayTurnMixin: "session, or check your auxiliary.compression model configuration.", "compression-failure warning", ) - # If the CONFIGURED aux model failed and we recovered on the main model, tell the user — - # a misconfigured auxiliary.compression.model is something only they can fix. + # Configured aux model failed, recovered on the main model: only the user can fix that config. elif _comp is not None and getattr(_comp, "_last_aux_model_failure_model", None): _aux_model = getattr(_comp, "_last_aux_model_failure_model", "") _aux_err = getattr(_comp, "_last_aux_model_failure_error", None) or "unknown error" @@ -1120,8 +1090,7 @@ class GatewayTurnMixin: "configured providers.", session_entry.session_id, exc, exc_info=True, ) _hyg_session_db = getattr(self._session_db, "_db", self._session_db) - # Hygiene is the same lossy rewrite as normal compression: with - # compression.checkpoint_required on, load the memory provider so the checkpoint exists + # With compression.checkpoint_required on, load the memory provider so the checkpoint exists # before any mutation; otherwise keep the fast path (no provider init). from hermes_cli.config import load_config as _load_cfg from utils import is_truthy_value as _is_truthy @@ -1135,8 +1104,7 @@ class GatewayTurnMixin: session_id=session_entry.session_id, session_db=_hyg_session_db, ) _seed_hygiene_system_prompt(_hyg_agent, _hyg_session_row) - # If compression must rebuild instead of retaining the cached prompt, make the persisted - # result deliberately stale for every real gateway surface. + # A rebuilt (not retained) prompt is deliberately stale for every real gateway surface. _hyg_agent.platform = _GATEWAY_HYGIENE_PLATFORM return _hyg_agent, _hyg_session_db @@ -1190,8 +1158,7 @@ class GatewayTurnMixin: run_generation=run_generation, ) finally: - # Evict the cached agent so the next turn rebuilds its system prompt from current - # SOUL.md, memory, and skills. + # Evict the cached agent so the next turn rebuilds its system prompt. self._evict_cached_agent(session_key) if not attempt.cleanup_deferred: await self._cleanup_agent_resources_off_loop(_hyg_agent, context="session hygiene") @@ -1222,9 +1189,8 @@ class GatewayTurnMixin: if str(_hyg_runtime.get("api_mode") or "").lower() == "codex_app_server": await self._hmwa_hygiene_codex_compaction(hs, plan, history, session_entry, session_key, _hyg_runtime) elif _hyg_runtime.get("api_key"): - # Pass the FULL transcript (tool results included), matching the agent loop: - # filtering to user/assistant starved the compressor — tool results are the bulk - # of context and short histories tripped the protect-first/last early-return. + # Pass the FULL transcript (tool results included) as the agent loop does: filtering + # to user/assistant starved the compressor (tool results are the bulk of context). _hyg_msgs = [m for m in history if m.get("role") in {"user", "assistant", "tool"}] if len(_hyg_msgs) >= 4: await self._hmwa_hygiene_detached_attempt( @@ -1232,8 +1198,7 @@ class GatewayTurnMixin: source, session_entry, session_key, _quick_key, run_generation, ) except HygieneTurnHoldExceeded: - # Availability boundary, not a failure — already logged at INFO by the turn-hold - # handler; the generic warning below made thinking-model deployments read as broken. + # Availability boundary, not a failure — already logged at INFO by the turn-hold handler. pass except Exception as e: logger.warning("Session hygiene auto-compress failed: %s", e) @@ -1252,9 +1217,8 @@ class GatewayTurnMixin: "Briefly introduce yourself and mention that /help shows available commands. " "Keep the introduction concise -- one or two sentences max.]" ) - # Opt-in profile-build path: when onboarding.profile_build is "ask" (default) and not - # yet offered on this install, swap the plain intro for a consent-gated directive that - # offers to build a user profile via memory(target="user"). Fires at most once. + # onboarding.profile_build == "ask" (default) and not yet offered: swap the plain intro for + # a consent-gated profile-build directive. Fires at most once. try: from agent.onboarding import ( PROFILE_BUILD_FLAG, is_seen, mark_seen, profile_build_directive, @@ -1270,14 +1234,12 @@ class GatewayTurnMixin: logger.debug("Profile-build onboarding directive failed, using plain intro: %s", _pb_err) turn_sidecar_notes.append(_intro_note) - # One-time prompt if no home channel is set for this platform. Skipped for webhooks — - # they deliver directly to configured targets (github_comment, etc.). + # One-time prompt if no home channel is set (webhooks deliver to configured targets instead). if not source.platform or source.platform in (Platform.LOCAL, Platform.WEBHOOK): return platform_name = source.platform.value env_key = _home_target_env_var(platform_name) - # Multiplex: the home channel may live only in the profile secret scope / PlatformConfig, - # not process os.environ. + # Multiplex: the home channel may live only in the profile secret scope, not os.environ. home_env = "" if env_key: with suppress(Exception): @@ -1288,8 +1250,7 @@ class GatewayTurnMixin: with suppress(Exception): if not home_env and self.config.get_home_channel(source.platform): home_env = "set" - # Secondary-profile platforms (e.g. Slack on yolo) may only exist under that profile's - # loaded config — re-read live config inside the already-installed scope. + # Secondary-profile platforms may only exist under that profile's config — re-read in scope. if not home_env: with suppress(Exception): from gateway.config import load_gateway_config as _lgc @@ -1297,8 +1258,7 @@ class GatewayTurnMixin: if prof and prof != "default" and _lgc().get_home_channel(source.platform): home_env = "set" if not home_env: - # Slack routes every Hermes command through the single parent slash command - # `/hermes`; bare `/sethome` is not registered and would fail. + # Slack routes every command through the parent `/hermes`; bare `/sethome` would fail. sethome_cmd = "/hermes sethome" if source.platform == Platform.SLACK else "/sethome" await self._deliver_platform_notice( source, f"📬 No home channel is set for {platform_name.title()}. " @@ -1330,8 +1290,7 @@ class GatewayTurnMixin: if _message_timestamps_enabled(_load_gateway_config()): message_text = _render_msg_ts(_clean_message_text, persist_user_timestamp, tz=_evt_tz) else: - # Toggle off: the model sees the clean message; the timestamp is still stored - # as metadata for later opt-in. + # Toggle off: the model sees the clean message; timestamp stored for later opt-in. message_text = _clean_message_text except Exception as _ts_err: logger.debug("Message timestamp injection failed (non-fatal): %s", _ts_err) @@ -1363,9 +1322,8 @@ class GatewayTurnMixin: _sanitize_gateway_final_response, _should_clear_resume_pending_after_turn, ) response = agent_result.get("final_response") or "" - # Hidden-reasoning-only retry exhaustion: the loop's sentinel text doubles as - # final_response and would be delivered verbatim — where peer agents can ingest it as a - # completed assistant turn. + # Hidden-reasoning-only retry exhaustion: the loop's sentinel text doubles as final_response + # and would be delivered verbatim (peer agents would ingest it as a completed turn). if _is_gateway_hidden_reasoning_incomplete_turn(agent_result): response = "" _intentional_silence = self._is_intentional_silence(agent_result, response) @@ -1383,9 +1341,8 @@ class GatewayTurnMixin: time.time() - _msg_start_time, agent_result.get("api_calls", 0), len(response), ) - # Successful turn: clear the stuck-loop counter (accumulates only across CONSECUTIVE - # restarts where the session never completed) and resume_pending (set by drain-timeout - # shutdown) so later messages don't get the restart-interruption system note. + # Successful turn: clear the consecutive-restart stuck-loop counter and resume_pending (set + # by drain-timeout shutdown) so later messages don't get the restart-interruption note. if session_key and _should_clear_resume_pending_after_turn(agent_result): await self._clear_restart_failure_count(session_key) try: @@ -1398,14 +1355,12 @@ class GatewayTurnMixin: response = _normalize_empty_agent_response(agent_result, response, history_len=len(history)) response = _sanitize_gateway_final_response(source.platform, response) - # Ordering contract: the agent thread already updated the contextvar in - # conversation_compression.py; propagate to SessionEntry + _save() — but only if the - # binding still points at the session this run was launched against. + # The agent thread already updated the contextvar; propagate to SessionEntry + _save() only + # if the binding still points at the session this run was launched against. if agent_result.get("session_id") and agent_result["session_id"] != session_entry.session_id: if session_entry.session_id == _run_start_session_id: session_entry.session_id = agent_result["session_id"] - # The held turn lease follows the rotation: transcript persistence writes to the - # NEW id, so the serialization boundary must move with it. + # The held turn lease follows the rotation (persistence writes to the NEW id). self._rebind_turn_lease(_quick_key, run_generation, session_entry.session_id) await self.async_session_store._save() await self.async_session_store._record_gateway_session_peer( @@ -1451,8 +1406,7 @@ class GatewayTurnMixin: display_reasoning = "\n".join(lines[:15]) + f"\n_... ({len(lines) - 15} more lines)_" else: display_reasoning = last_reasoning.strip() - # Render style is per-platform: Discord defaults to "-# " subtext (native small grey - # metadata text); other platforms keep the fenced code block. + # Per-platform render style: Discord defaults to "-# " subtext, others keep the code block. try: from gateway.display_config import resolve_display_setting _reasoning_style = resolve_display_setting( @@ -1496,8 +1450,7 @@ class GatewayTurnMixin: # Pending process watchers (check_interval on background processes) try: from tools.process_registry import process_registry - # Detach the current batch atomically: reassign to a fresh list so a watcher appended - # by a concurrent session during the yield isn't dropped by clear(). + # Detach the batch atomically (reassign, not clear()) so concurrent appends aren't dropped. watchers = process_registry.pending_watchers process_registry.pending_watchers = [] for i, watcher in enumerate(watchers): @@ -1507,9 +1460,8 @@ class GatewayTurnMixin: except Exception as e: logger.error("Process watcher setup error: %s", e) - # Drain watch notifications that arrived during the run. The queue also carries process - # completions (per-process watcher task above) and async-delegation completions (owned by - # _async_delegation_watcher) — inject only watch-type events, leave the rest queued. + # Drain watch notifications that arrived during the run; the queue also carries process / + # async-delegation completions owned elsewhere — inject only watch-type events. try: from tools.process_registry import process_registry as _pr await self._drain_watch_notifications(_pr.completion_queue) @@ -1534,8 +1486,7 @@ class GatewayTurnMixin: agent_failed_early = bool(agent_result.get("failed")) hidden_reasoning_incomplete = _is_gateway_hidden_reasoning_incomplete_turn(agent_result) _err = str(agent_result.get("error", "")).lower() - # Specific multi-word phrases (not bare "exceed"/"token") avoid false positives on - # transient errors such as "rate limit exceeded"; matches run_agent.py's classifier. + # Multi-word phrases (not bare "exceed"/"token") avoid matching "rate limit exceeded". is_context_overflow_failure = agent_failed_early and ( bool(agent_result.get("compression_exhausted")) or any(p in _err for p in self._CONTEXT_OVERFLOW_ERROR_PHRASES) @@ -1576,8 +1527,7 @@ class GatewayTurnMixin: logger.info("Auto-resetting session %s after compression exhaustion.", session_entry.session_id) new_entry = await self.async_session_store.reset_session(session_key) self._evict_cached_agent(session_key) - # Conversation boundary: one funnel call clears every conversation-scoped per-session - # dict (see _CONVERSATION_SCOPED_STATE). + # Conversation boundary: the funnel clears every conversation-scoped per-session dict. self._clear_conversation_scope(session_key, reason="compression_exhausted_reset") if new_entry is not None: # Re-point the Telegram topic binding at the fresh session, or the binding-heal walk @@ -1622,18 +1572,16 @@ class GatewayTurnMixin: store = self.async_session_store sid = session_entry.session_id history = prepared.history - # The agent already persisted this turn's rows via _flush_messages_to_session_db() (the - # codex app-server runtime too: it flushes its own projection and reports - # agent_persisted=True); skip the DB write to avoid duplicates. Default = a session DB - # exists; a non-persisting runtime opts in via False. + # The agent already persisted this turn's rows (codex app-server reports agent_persisted=True + # too); skip the DB write. Default = a session DB exists; non-persisting runtimes pass False. agent_persisted = agent_result.get("agent_persisted", self._session_db is not None) if is_context_overflow_failure: pass # Skip all transcript writes — don't grow a broken session else: if not history: - # Fresh session: write the full tool definitions as the first entry so the transcript - # is self-describing — the same dicts sent as tools=[...] in the API request. + # Fresh session: the tool definitions (as sent in the API request) make the transcript + # self-describing. await store.append_to_transcript(sid, { "role": "session_meta", "tools": agent_result.get("tools", []) or [], @@ -1670,9 +1618,8 @@ class GatewayTurnMixin: skip_db=agent_persisted, ) else: - # Attach the inbound platform message_id to the first user entry written this turn - # so platform-level quote-resolution (e.g. Yuanbao QuoteContextMiddleware's - # transcript fallback) can find earlier @bot messages by their original id. + # Attach the inbound platform message_id to the first user entry so platform-level + # quote-resolution (e.g. Yuanbao) can find earlier @bot messages by original id. _user_msg_id_attached = False for msg in new_messages: if msg.get("role") == "system": @@ -1688,8 +1635,7 @@ class GatewayTurnMixin: _user_msg_id_attached = True await store.append_to_transcript(sid, entry, skip_db=agent_persisted) - # The agent persists token counts and model itself; keep only last_prompt_tokens here for - # context-window tracking and compression decisions. + # The agent persists token counts/model itself; keep only last_prompt_tokens for hygiene. await store.update_session( session_entry.session_key, last_prompt_tokens=agent_result.get("last_prompt_tokens", 0), touch_activity=not bool(getattr(event, "internal", False)), @@ -1706,15 +1652,13 @@ class GatewayTurnMixin: ): """Final delivery decisions: intentional silence, voice reply, streamed-turn media/footer. Returns the text for the adapter to send, or ``None`` when already delivered.""" - # Intentional silence is a delivery decision, not a transcript mutation: the [SILENT] - # assistant turn stays persisted so later turns keep user/assistant alternation. + # Intentional silence is a delivery decision: the [SILENT] turn stays persisted (alternation). if _intentional_silence: logger.info("Suppressing intentional silence marker for session %s", session_entry.session_id) response = "" adapter = self._adapter_for_source(source) - # Auto voice reply (TTS audio before the text) unless streaming TTS already delivered - # audio for this turn. + # Auto voice reply (TTS audio before the text) unless streaming TTS already delivered audio. _streaming_tts_done = adapter is not None and bool( getattr(adapter, "_streaming_tts_turn_completed", lambda *_a, **_k: False)(session_key, run_generation) ) @@ -1723,9 +1667,8 @@ class GatewayTurnMixin: ): await self._send_voice_reply(event, response) - # Streamed responses still need MEDIA: files delivered before returning None (chunks carry - # the tags verbatim and post-processing is skipped when already_sent). Never skip when the - # agent failed: the error text is new content streaming didn't show. + # Streamed responses still need MEDIA: files delivered (chunks carry the tags verbatim). Never + # skip when the agent failed: the error text is new content streaming didn't show. if agent_result.get("already_sent") and not agent_result.get("failed"): if response and adapter: await self._deliver_media_from_response(response, event, adapter) @@ -1735,9 +1678,8 @@ class GatewayTurnMixin: await adapter.send(source.chat_id, _footer_line, metadata=self._event_thread_metadata(event, source)) except Exception as _e: logger.debug("trailing footer send failed: %s", _e) - # Return None so the adapter does not send the body twice; /loop and /goal hooks in - # _handle_message read the return value, so stash the delivered text on the event or - # those hooks never run and a /loop tick stays awaiting. + # Return None so the body isn't sent twice; stash the delivered text on the event for the + # /loop and /goal hooks that read the return value. with suppress(Exception): event._streamed_final_response = str(response or "") return None @@ -1756,9 +1698,8 @@ class GatewayTurnMixin: # Retain Slack thread/workspace routing so a failed turn cannot leave its status visible. await self._hmwa_stop_typing_for_turn(event, source) logger.exception("Agent error in session %s", session_key) - # Crash-resilience for failures before AIAgent enters run_conversation() (e.g. provider/ - # httpx client init): the agent can't persist the inbound turn there, so append the user - # message here once; skip if the latest user row already matches it. + # Failures before run_conversation() (provider/httpx init) can't persist the inbound turn: + # append the user message here once, unless the latest user row already matches it. try: if prepared.message_text is not None and session_entry is not None: try: @@ -1778,8 +1719,7 @@ class GatewayTurnMixin: ) except Exception: logger.debug("Failed to persist inbound user message after agent exception", exc_info=True) - # Log full details server-side only; never expose raw exception types or messages to end - # users (info-leakage risk). + # Never expose raw exception types/messages to end users (info-leakage risk). status_code = getattr(e, "status_code", None) status_hint = self._STATUS_HINTS.get(status_code, "") if status_code == 429: @@ -1798,8 +1738,7 @@ class GatewayTurnMixin: else: status_hint = " Your plan's usage limit has been reached. Please wait until it resets." elif status_code in {400, 500}: - # 400 on a large session is context overflow; 500 on a large session often means the - # payload is too large for the API — treat it the same way. + # 400/500 on a large session: context overflow / payload too large. if len(prepared.history) > 50: return ( "⚠️ Session too large for the model's context window.\nUse /compact to " @@ -1847,9 +1786,8 @@ class GatewayTurnMixin: with suppress(Exception): _redact_pii = bool((_load_gateway_config().get("privacy") or {}).get("redact_pii", False)) - # The context prompt render is pinned per session, keyed by a hash of the exact renderer - # inputs (_ephemeral_change_key): a hit reuses the pinned bytes so the system prompt cannot - # drift turn-over-turn; a miss (thread rename, /sethome, redact_pii flip) re-renders. + # The context prompt render is pinned per session, keyed by a hash of the renderer inputs, so + # the system prompt cannot drift turn-over-turn; a miss (thread rename, /sethome) re-renders. context_prompt = self._pinned_session_context_prompt(context, _redact_pii, session_key) # Per-turn notes ride the user message via the api_content sidecar, NOT context_prompt @@ -1858,17 +1796,15 @@ class GatewayTurnMixin: if _was_auto_reset: await self._hmwa_deliver_auto_reset_notice(session_entry, source, turn_sidecar_notes) - # Auto-load skill(s) for topic/channel bindings (single name or ordered list) — only on NEW - # sessions; ongoing conversations already carry the skill content in their history. + # Auto-load bound skill(s) only on NEW sessions; ongoing ones carry the content in history. _auto = getattr(event, "auto_skill", None) if _is_new_session and _auto: self._hmwa_auto_load_skills(event, _auto, _quick_key, session_key) await self._hmwa_acquire_turn_lease(_quick_key, run_generation, session_entry, _session_env_tokens) - # A turn only becomes durable recovery work after it owns (or has explicitly degraded past) - # the per-session lease. Marking before the await above would falsely recover an alias- - # routed message that never began processing if the gateway died while it was still waiting. + # A turn becomes durable recovery work only after it owns the per-session lease; marking + # earlier would falsely recover a message that never began processing. await self._mark_durable_active_turn(event, session_entry.session_key) # An unreadable store is not an empty conversation: stop before the agent invents continuity @@ -1895,8 +1831,7 @@ class GatewayTurnMixin: if _vc_note: turn_sidecar_notes.append(_vc_note) - # Auto-analyze user images (vision tool eagerly, image media_type only) so the model always - # gets a text description plus the local path for re-examination via vision_analyze. + # Auto-analyze user images so the model gets a description plus the local path. message_text = await self._prepare_profile_scoped_inbound_message_text( event=event, source=source, history=history, session_key=session_key, ) @@ -1907,13 +1842,13 @@ class GatewayTurnMixin: self._hmwa_apply_message_timestamp(event, message_text) ) - # Stage this turn's must-deliver notes (one-shot; consumed in run_sync) AFTER the - # message_text early-out so an aborted turn cannot leak its notes into the next turn. + # Stage the notes (one-shot; consumed in run_sync) AFTER the early-out so an aborted turn + # cannot leak them into the next turn. if turn_sidecar_notes and session_key: self._set_pending_turn_sidecar_notes(session_key, turn_sidecar_notes) - # Bind this run generation to the adapter's active-session event so deferred post-delivery - # callbacks can be released by the same run that registered them. + # Bind this run generation to the adapter so deferred post-delivery callbacks are released + # by the run that registered them. self._bind_adapter_run_generation(self._adapter_for_source(source), session_key, run_generation) return self._PreparedTurn( history, context_prompt, message_text, persist_user_message, persist_user_timestamp, @@ -1962,9 +1897,8 @@ class GatewayTurnMixin: } await self.hooks.emit("agent:start", hook_ctx) - # Capture the session id this run launches against so post-run compression publication - # can be identity-guarded; a /new or another lifecycle transition may move - # session_entry.session_id while the old run is still unwinding. + # Capture the launch session id so post-run compression publication is identity-guarded + # (a /new may move session_entry.session_id while the old run is still unwinding). _run_start_session_id = session_entry.session_id _turn_started_monotonic = time.monotonic() agent_result = await self._run_agent( @@ -2217,8 +2151,8 @@ class GatewayTurnMixin: await adapter.send_image( chat_id=source.chat_id, image_url=image_url, caption=alt_text, metadata=_thread_metadata, ) - # Route each media file by type so a TTS clip arrives as a voice bubble and a clip as a - # video rather than a generic document. Mirrors the streaming + kanban paths. + # Route each media file by type (voice bubble / video / image / document), as the + # streaming + kanban paths do. from gateway.platforms.base import should_send_media_as_audio as _should_send_media_as_audio from gateway.run_notifications import _IMAGE_EXTS, _VIDEO_EXTS for media_path, _is_voice in (media_files or []): @@ -2255,8 +2189,7 @@ class GatewayTurnMixin: _cache_lock = getattr(self, "_agent_cache_lock", None) if _cache_lock is None or not _cache: return - # Multiplex: only this profile's sessions; rebuilding another profile's agent in - # this scope would hand it this profile's tool registry. + # Multiplex: only this profile's sessions (another profile's agent would get this registry). _ns_prefix = _session_key_namespace(profile) + ":" if multiplex else None with _cache_lock: for _sess_key, _entry in list(_cache.items()): @@ -2324,8 +2257,7 @@ class GatewayTurnMixin: self._mcp_reload_refresh_cached_agents(multiplex, event.source.profile) - # Inject a message at the END of the session history so the model knows tools changed - # next turn; appending after all existing messages preserves the prompt-cache prefix. + # Append a note at the END of the history (preserves the prompt-cache prefix). change_parts = [ f"{label} servers: {', '.join(sorted(names))}" for label, names in (("Added", added), ("Removed", removed), ("Reconnected", reconnected)) @@ -2375,12 +2307,11 @@ class GatewayTurnMixin: if not _adapter_supports_edit and not _adapter_supports_native_stream and on_missing_cursor == "raise": raise RuntimeError("skip streaming for non-editable platform") _effective_cursor = scfg.cursor if _adapter_supports_edit else "" - # Some Matrix clients render the cursor as a tofu/white-box artifact: stream text, no cursor. + # Some Matrix clients render the cursor as tofu: stream text, no cursor. _buffer_only = source.platform == Platform.MATRIX if _buffer_only: _effective_cursor = "" - # Fresh-final applies to Telegram only — other platforms edit in place cheaply (Discord, Slack) - # or lack the edit-timestamp-stays-stale problem. + # Fresh-final applies to Telegram only (others edit in place cheaply). _fresh_final_secs = ( float(getattr(scfg, "fresh_final_after_seconds", 0.0) or 0.0) if source.platform == Platform.TELEGRAM else 0.0 @@ -2456,8 +2387,7 @@ class GatewayTurnMixin: if not proxy_url: return self._proxy_error_result("⚠️ Proxy URL not configured (GATEWAY_PROXY_URL or gateway.proxy_url)") - # Scope-aware read: the proxy key is a per-profile credential; under multiplex honor the - # installed scope's verdict (Slack pattern for the unscoped default-profile loop). + # The proxy key is a per-profile credential: honor the installed secret scope under multiplex. proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() with suppress(Exception): # UnscopedSecretError and import failures fall back to the env from agent.secret_scope import get_secret @@ -2475,9 +2405,8 @@ class GatewayTurnMixin: "history_offset": len(history), "session_id": session_id, "response_previewed": False, } - # OpenAI chat format. The remote api_server keeps continuity via X-Hermes-Session-Id and - # loads its own history, so send only the current message plus a compact text-only local - # history for a remote that has none yet (remote replays tools). + # OpenAI chat format. The remote keeps continuity via X-Hermes-Session-Id; send the current + # message plus a compact text-only history for a remote that has none yet. api_messages: List[Dict[str, str]] = [] if context_prompt: api_messages.append({"role": "system", "content": context_prompt}) @@ -2597,8 +2526,7 @@ class GatewayTurnMixin: platform_key = _platform_config_key(source.platform) enabled_toolsets, disabled_toolsets = self._resolve_turn_toolsets(user_config, source, platform_key) adapter = self._adapter_for_source(source) - # Per-platform display settings: display.platforms.., then display. - # global, then built-in platform defaults. + # display.platforms.. → display. → built-in platform defaults. _display_cfg = user_config.get("display", {}) if not isinstance(_display_cfg, dict): _display_cfg = {} @@ -2613,8 +2541,7 @@ class GatewayTurnMixin: _val = resolve_display_setting(user_config, platform_key, _setting, _default) getattr(_agent_display, _setter)(_cast(_val)) - # Tool progress mode — per-platform, with HERMES_TOOL_PROGRESS_MODE winning only when the - # config never set it. + # Tool progress mode; HERMES_TOOL_PROGRESS_MODE wins only when the config never set it. _resolved_tp = resolve_display_setting(user_config, platform_key, "tool_progress") _env_tp = os.getenv("HERMES_TOOL_PROGRESS_MODE") _platform_cfg = (_display_cfg.get("platforms") or {}).get(platform_key) or {} @@ -2658,21 +2585,18 @@ class GatewayTurnMixin: logger.debug("generic status phrase selection failed: %s", _phrase_err) return "still on it" if kind in {"heartbeat", "waiting", "long_running", "status"} else "one sec" - # Webhooks can't edit messages, so tool progress / log mode are off there (each progress - # line would be its own message). + # Webhooks can't edit messages, so tool progress / log mode are off there. is_webhook = source.platform == Platform.WEBHOOK tool_progress_enabled = progress_mode not in {"off", "log"} and not is_webhook - # Live working-state status for text-rendering typing indicators (Slack's assistant status - # line). Independent of tool_progress; rides the existing _keep_typing refresh. + # Live status for text-rendering typing indicators (Slack); independent of tool_progress. _live_status_mode = resolve_display_setting(user_config, platform_key, "live_status", "full") _live_status_adapter = ( adapter if getattr(adapter, "supports_status_text", False) and _live_status_mode != "off" else None ) # "log" mode: tool calls go to ~/.hermes/logs/tool_calls.log instead of the chat. Gateway-only. log_mode_enabled = progress_mode == "log" and not is_webhook - # Natural assistant status messages are independent from tool progress and token streaming. - # thinking_progress is independent too (same queue). Mattermost requires a per-platform - # opt-in for both: global scratch-text display leaks too easily into busy public threads. + # Interim assistant messages and thinking_progress are independent of tool progress (same + # queue). Mattermost requires a per-platform opt-in: scratch text leaks into public threads. interim_assistant_messages_mode = _display_surface_mode( "interim_assistant_messages", default=True, require_platform_override_for={Platform.MATTERMOST}, ) @@ -2680,8 +2604,7 @@ class GatewayTurnMixin: _thinking_enabled = _display_surface_mode( "thinking_progress", default=False, require_platform_override_for={Platform.MATTERMOST}, ) != "off" - # Slack-native task cards render tool progress via chat.startStream, so the progress queue is - # needed even though Slack keeps text tool_progress off by default. + # Slack-native task cards need the progress queue even with text tool_progress off. _native_slack_task_cards = False if source.platform == Platform.SLACK and hasattr(adapter, "native_task_cards_enabled"): try: @@ -2732,9 +2655,8 @@ class GatewayTurnMixin: _voice_ack_guild[0] = _gid break - # Auto-cleanup of temporary progress bubbles (any adapter implementing ``delete_message``; - # failed runs skip cleanup so the bubbles remain as breadcrumbs). getattr on the type, not - # attribute access: a fake adapter without delete_message means "can't delete", not a crash. + # Auto-cleanup of temporary progress bubbles needs a real ``delete_message`` (getattr on the + # type: a fake adapter without it means "can't delete", not a crash). _cleanup_progress = bool( disp.resolve_display_setting(disp.user_config, disp.platform_key, "cleanup_progress") ) @@ -2803,8 +2725,7 @@ class GatewayTurnMixin: if is_buzz: _progress_reply_in_thread = getattr(_adapter, "_reply_to_mode", "first") != "off" else: - # Relay lane: adapter owns mode resolution (nested platforms.relay.extra.slack - # subset, flat-key fallback). Native lane: read the flat extra as before. + # Relay lane: the adapter owns mode resolution; native lane: flat extra key. _mode_fn = getattr(_adapter, "_effective_reply_in_thread", None) _progress_reply_in_thread = bool( _mode_fn() if callable(_mode_fn) else _adapter.config.extra.get("reply_in_thread", True) @@ -2814,8 +2735,7 @@ class GatewayTurnMixin: _progress_thread_id = _resolve_progress_thread_id( source.platform, source.thread_id, event_message_id, reply_in_thread=_progress_reply_in_thread, ) - # Relay Discord auto-thread lane: a channel-initiating message has no thread_id at ingest; - # the connector stamps prospective_thread_id (anchor id == the thread it will create). + # Relay Discord auto-thread lane: the connector stamps prospective_thread_id at ingest. _relay_prospective_thread_id = ( str(getattr(source, "prospective_thread_id", None)) if source.platform == Platform.DISCORD @@ -2838,8 +2758,7 @@ class GatewayTurnMixin: _progress_metadata.setdefault("slack_team_id", source.scope_id) if source.user_id: _progress_metadata.setdefault("recipient_user_id", source.user_id) - # Buzz has no native thread_id; threading is always via reply-to the triggering event id - # (channel clutter otherwise), skipped when the user opted out of threaded replies. + # Buzz has no native thread_id: thread via reply-to unless the user opted out. _progress_reply_to = ( event_message_id if (source.platform in (Platform.FEISHU, Platform.MATTERMOST) and source.thread_id and event_message_id) @@ -2896,8 +2815,7 @@ class GatewayTurnMixin: _relay_prospective_thread_id: Optional[str], ) -> Optional[Dict[str, Any]]: """Thread metadata for status / approval / stream sends; Feishu topics need the triggering - message id (reply API with reply_in_thread=true) so carry it as a fallback. - """ + message id (reply API with reply_in_thread=true) so carry it as a fallback.""" if source.platform == Platform.FEISHU and source.thread_id and event_message_id: return {"thread_id": _progress_thread_id, "reply_to_message_id": event_message_id} return self._thread_metadata_for_progress( @@ -2954,8 +2872,7 @@ class GatewayTurnMixin: async def _run_agent_track_agent(self, turn_ctx: TurnContext) -> None: """Track this agent as running for the session (interrupt support) once it is created — only - if this run is still current, else leave the newer run's slot alone. - """ + if this run is still current, else leave the newer run's slot alone.""" session_key, run_generation, agent_holder = turn_ctx.session_key, turn_ctx.run_generation, turn_ctx.agent_holder while agent_holder[0] is None: await asyncio.sleep(0.05) @@ -3094,15 +3011,13 @@ class GatewayTurnMixin: _agent_timeout = _agent_timeout if _agent_timeout > 0 else None _agent_warning = _agent_warning if _agent_warning > 0 else None - # A background=true process intentionally survives a successful turn, so capture existing IDs - # and reap only children created by THIS turn if it times out. + # background=true processes survive a turn: reap only children created by THIS turn on timeout. _turn_task_id = turn_ctx.session_id or "" _turn_process_baseline = process_registry.snapshot_running_ids(_turn_task_id) turn_ctx.process_task_id = _turn_task_id turn_ctx.process_baseline = _turn_process_baseline - # task_id is session-scoped, not turn-scoped: gate the eventual reap on this exact claim still - # being current, so a replacement turn on the same session that starts before the watchdog - # fires doesn't get its own fresh process killed by this turn's stale baseline. + # task_id is session-scoped: gate the reap on this claim still being current so a replacement + # turn's fresh process isn't killed by this turn's stale baseline. worker = self._RunAgentWorker( agent_timeout=_agent_timeout, agent_warning=_agent_warning, task_id=_turn_task_id, process_baseline=_turn_process_baseline, worker_done=threading.Event(), @@ -3242,9 +3157,8 @@ class GatewayTurnMixin: while True: done, _ = await asyncio.wait({worker.executor_task}, timeout=5.0) if done: - # Prefer the real result even if the watchdog fired in the same window: the completed - # run already persisted its reply, so the "agent inactive" diagnostic would contradict - # the stored transcript. + # Prefer the real result even if the watchdog fired in the same window (the run already + # persisted its reply). return worker.executor_task.result() if worker.agent_timeout is not None: if worker.timeout_fired.is_set(): @@ -3276,9 +3190,8 @@ class GatewayTurnMixin: if _agent is None or not hasattr(_agent, 'model') or _run_failed: return _cfg_model = _resolve_gateway_model() - # Normalize _cfg_model as AIAgent.__init__ does so a vendor-prefixed config value matches - # the agent's stripped model on native providers — otherwise the cached agent is evicted - # every turn, destroying prompt caching. Aggregators keep the vendor slug. + # Normalize as AIAgent.__init__ does (vendor prefix stripped on native providers), else the + # cached agent is evicted every turn, destroying prompt caching. with suppress(Exception): from hermes_cli.model_normalize import _AGGREGATOR_PROVIDERS, normalize_model_for_provider _agent_provider = getattr(_agent, 'provider', '') or '' @@ -3335,8 +3248,7 @@ class GatewayTurnMixin: else: pending = interrupt_message elif pending_event: - # Transcribe audio on the dequeued event BEFORE it becomes the next user turn, so - # queued/interrupting voice messages drain with the real transcript, not a file path. + # Transcribe audio BEFORE it becomes the next user turn (real transcript, not a path). _pending_text = pending_event.text or "" if self._pending_event_audio_paths(pending_event): pending, _ = await self._transcribe_and_echo_pending_voice( @@ -3349,14 +3261,12 @@ class GatewayTurnMixin: if pending: logger.debug("Processing queued message after agent completion: '%s...'", pending[:40]) - # Leftover /steer: a steer arriving after the last tool batch (e.g. during the final API - # call) comes back in result["pending_steer"]; deliver it as the next user turn, not drop it. + # Leftover /steer (arrived after the last tool batch): deliver as the next user turn. if result and not pending and not pending_event and result.get("pending_steer"): pending = result.get("pending_steer") logger.debug("Delivering leftover /steer as next turn: '%s...'", pending[:40]) - # Safety net: a pending slash command (e.g. "/stop", "/new") is discarded — commands must - # never be passed to the agent as user input. + # Safety net: a pending slash command is never passed to the agent as user input. if pending and pending.strip().startswith("/"): _pending_parts = pending.strip().split(None, 1) _pending_cmd_word = _pending_parts[0][1:].lower() if _pending_parts else "" @@ -3391,16 +3301,13 @@ class GatewayTurnMixin: await self._await_stream_task(stream_task) except Exception as e: logger.debug("Stream consumer wait before queued message failed: %s", e) - # The queued branch needs raw ``result`` for interruption, history, and recursion state, but - # delivery must use the finalized task result — it carries empty/failure normalization and - # final-response processing from _run_agent_task. + # Delivery uses the finalized task result (empty/failure normalization), not raw ``result``. _delivery_result = response if isinstance(response, dict) else (result or {}) first_response = _delivery_result.get("final_response", "") _already_streamed = self._run_agent_stream_confirmed_final_delivery( _sc, first_response, previewed=bool(_delivery_result.get("response_previewed")), ) - # Same predicate as the normal completed-turn path: this direct queued-send branch predates - # intentional-silence filtering and would leak the literal marker. + # Same silence predicate as the normal path, else this branch leaks the literal marker. if self._is_intentional_silence(_delivery_result, first_response): logger.info( "Queued follow-up for session %s: suppressing intentional silence marker before continuing.", @@ -3422,8 +3329,7 @@ class GatewayTurnMixin: ) except Exception as e: logger.warning("Failed to send first response before queued message: %s", e) - # Release deferred bg-review notifications now that the first response is delivered: pop from - # the adapter's callback dict (no double-fire in base.py's finally) and call. + # Release deferred bg-review notifications: pop (no double-fire in base.py's finally) and call. _bg_cb = self._pop_post_delivery_callback(adapter, session_key, turn_ctx.run_generation) if callable(_bg_cb): with suppress(Exception): @@ -3445,13 +3351,11 @@ class GatewayTurnMixin: ) logger.debug("Processing pending message: '%s...'", pending[:40]) - # Clear the adapter's interrupt event so the next _run_agent call doesn't re-trigger the - # interrupt before the new agent's first API call (infinite loop otherwise). + # Clear the interrupt event so the recursive _run_agent isn't re-interrupted (infinite loop). if adapter and hasattr(adapter, '_active_sessions') and session_key and session_key in adapter._active_sessions: adapter._active_sessions[session_key].clear() - # Cap recursion depth: resource exhaustion when the user keeps sending messages while the - # agent keeps failing. + # Cap recursion depth (user keeps sending while the agent keeps failing). if _interrupt_depth >= self._MAX_INTERRUPT_DEPTH: logger.warning( "Interrupt recursion depth %d reached for session %s — " @@ -3464,8 +3368,7 @@ class GatewayTurnMixin: adapter.queue_message(session_key, pending) return turn_ctx.result_holder[0] or {"final_response": response, "messages": history} - # Interrupted: discard the response ("Operation interrupted." is noise; the user knows they - # sent a new message). + # Interrupted: discard the response ("Operation interrupted." is noise). if not result.get("interrupted"): await self._run_agent_deliver_first_response(turn_ctx, adapter, response, result, stream_task) @@ -3481,9 +3384,8 @@ class GatewayTurnMixin: session_key or "?", ) return result - # Resolve the follow-up's session key BEFORE preparing the inbound text: - # _prepare_inbound_message_text buffers native image paths under the key given, and the - # recursive _run_agent consumes them under next_session_key — mismatch drops them. + # Resolve the follow-up's session key BEFORE preparing the inbound text: native image + # paths are buffered under the key given and consumed under next_session_key. try: next_session_key = self._session_key_for_source(next_source) except Exception: @@ -3500,8 +3402,7 @@ class GatewayTurnMixin: next_channel_prompt = getattr(pending_event, "channel_prompt", None) next_message_type = getattr(pending_event, "message_type", None) - # Clear the prior logical turn's completed streaming marker so the recursive turn's streaming - # TTS isn't suppressed by that completion. + # Clear the prior turn's streaming-TTS completion marker so the recursive turn isn't suppressed. _clear_adapter = self._adapter_for_source(source) if _clear_adapter is not None and session_key and run_generation is not None: _completed_turns = getattr(_clear_adapter, "_streaming_tts_completed_turns", None) @@ -3511,15 +3412,13 @@ class GatewayTurnMixin: if _pk: _completed_turns.discard(_pk) - # Restart the typing indicator for the follow-up turn; the outer _process_message_background - # typing task is alive but may be stale. + # Restart the typing indicator; the outer typing task may be stale. if _clear_adapter: with suppress(Exception): await _clear_adapter.send_typing(source.chat_id, metadata=_status_thread_metadata) - # Re-baseline the cached agent's message_count before recursing into the /queue follow-up: - # the coherence guard would otherwise rebuild on OUR OWN flushed rows and destroy the - # prompt-cache prefix; _handle_message_with_agent re-baselines only after the chain ends. + # Re-baseline the cached agent's message_count before recursing, else the coherence guard + # rebuilds on OUR OWN flushed rows (the outer handler re-baselines only after the chain). await self._refresh_agent_cache_message_count(session_key, session_id) followup_result = await self._run_agent( @@ -3542,9 +3441,7 @@ class GatewayTurnMixin: task.cancel() if stream_task: - # If the agent never created a stream consumer (non-streaming path, or a test stub returning - # synchronously) there is nothing to flush — cancel now instead of waiting out the 5s - # timeout polling for a consumer that will never arrive. + # No stream consumer was created: nothing to flush, cancel instead of waiting out 5s. if not (stream_consumer_holder and stream_consumer_holder[0] is not None): stream_task.cancel() with suppress(asyncio.CancelledError): @@ -3552,8 +3449,7 @@ class GatewayTurnMixin: else: await self._await_stream_task(stream_task) - # Unconditional abort + bounded wait for the streaming-TTS consumer: covers cancellation / - # exception paths where the normal finalisation block was skipped. + # Abort + bounded wait for streaming TTS: covers paths where normal finalisation was skipped. _stts_finally = turn_ctx.streaming_tts_consumer_holder[0] if _stts_finally is not None and not _stts_finally.done: _stts_finally.abort("cleanup") @@ -3562,8 +3458,8 @@ class GatewayTurnMixin: tracking_task.cancel() if session_key: - # Release the slot only if this run's generation still owns it: a /stop or /new that bumped - # the generation while we unwound already installed its own state; keep it. + # Release the slot only if this run's generation still owns it (/stop or /new may have + # installed its own state). self._release_running_agent_state(session_key, run_generation=turn_ctx.run_generation) if self._draining: self._update_runtime_status("draining") @@ -3575,9 +3471,7 @@ class GatewayTurnMixin: except asyncio.CancelledError: pass except Exception: - # A background task that died of a non-cancellation error (transport drop in a - # progress/card publish) must not abort the cleanup path — final-delivery - # bookkeeping after this loop still runs. + # A background task that died of a real error must not abort the cleanup path. logger.debug("background turn task failed during cleanup", exc_info=True) async def _run_agent_mark_streamed_delivery(self, response: Any, turn_ctx: TurnContext) -> None: @@ -3593,8 +3487,7 @@ class GatewayTurnMixin: return _final = response.get("final_response") or "" _is_empty_sentinel = not _final or _final == "(empty)" - # response_previewed means interim_assistant_callback already saw the final text, but only - # suppress the send if that exact text was delivered — unrelated commentary/progress isn't it. + # response_previewed: only suppress if that EXACT text was delivered, not unrelated commentary. _previewed = bool(response.get("response_previewed")) _content_delivered = bool(_sc and getattr(_sc, "final_content_delivered", False)) _stale_finalized = False @@ -3607,11 +3500,9 @@ class GatewayTurnMixin: _stale_finalized = False if _stale_finalized: _content_delivered = False - # Plugin hooks (e.g. transform_llm_output) may append content after streaming finished — when - # transformed, always send the final version so the appended content reaches the client. + # Plugin hooks may append content after streaming finished — then send the final version. _transformed = bool(response.get("response_transformed")) - # Suppress the normal send only when the actual final reply reached the user (streamed, or - # interim preview of that *exact* text); commentary shown during a compression/split isn't it. + # Suppress the normal send only when the actual final reply reached the user. _streamed = self._run_agent_stream_confirmed_final_delivery(_sc, _final, previewed=_previewed) if _is_empty_sentinel: return @@ -3658,8 +3549,7 @@ class GatewayTurnMixin: _sk, ) elif _transformed and _sc is not None: - # Plugin hooks transformed the response after streaming — edit the existing streamed - # message instead of sending a duplicate. + # Transformed after streaming: edit the streamed message instead of sending a duplicate. _sc_msg_id = _sc.message_id if _sc_msg_id: try: @@ -3674,9 +3564,8 @@ class GatewayTurnMixin: except Exception as _edit_err: logger.warning("Failed to edit streamed message for session %s: %s", _sk, _edit_err) elif _sc is not None: - # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed but suppression did NOT fire, so the - # normal final-send is about to run. Log the decision inputs so a recurrence can be pinned - # to "signal never set" vs "ack-pending race". + # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed but suppression did NOT fire; log the + # decision inputs ("signal never set" vs "ack-pending race"). logger.warning( "Normal final-send NOT suppressed despite active stream consumer for session %s: " "streamed=%s previewed=%s content_delivered=%s transformed=%s final_len=%d — " @@ -3735,8 +3624,7 @@ class GatewayTurnMixin: turn_ctx._progress_metadata, turn_ctx._progress_reply_to, _progress_thread_id, _relay_prospective_thread_id, ) = self._run_agent_progress_threading(source, event_message_id, _native_slack_task_cards) - # Bridges: sync step_callback / event_callback / status_callback → async hooks.emit and - # adapter.send (TurnRunner methods bound onto the shared ctx). + # Bridges: sync step/event/status callbacks → async hooks.emit and adapter.send. turn_ctx._loop_for_step = asyncio.get_running_loop() turn_ctx._hooks_ref = self.hooks turn_ctx._step_callback_sync = turn_runner._step_callback_sync @@ -3779,8 +3667,7 @@ class GatewayTurnMixin: ): break _elapsed_mins = int((time.time() - _notify_start) // 60) - # Default heartbeat is terse (elapsed + current tool); the verbose iteration counter is - # gated on busy_ack_detail so users can opt in per platform. + # Terse heartbeat by default; the iteration counter is gated on busy_ack_detail. _status_detail = "" _want_iteration_detail = bool( disp.resolve_display_setting(disp.user_config, disp.platform_key, "busy_ack_detail", True) @@ -3861,12 +3748,10 @@ class GatewayTurnMixin: source, message_type, _status_thread_metadata, turn_ctx.streaming_tts_consumer_holder, ) - # Progress sender gates on needs_progress_queue (tool_progress OR thinking_progress), not - # tool_progress alone: it drains BOTH tool-progress lines and _thinking scratch bubbles. + # Progress sender drains BOTH tool-progress lines and thinking bubbles (needs_progress_queue). progress_task = asyncio.create_task(turn_runner.send_progress_messages()) if disp.needs_progress_queue else None log_task = asyncio.create_task(self._run_agent_write_tool_log(disp.log_queue)) if disp.log_mode_enabled else None - # The stream consumer is created inside run_sync (thread pool) after the agent is constructed; - # this task polls for it. + # The stream consumer is created inside run_sync; this task polls for it. stream_task = asyncio.create_task(self._run_agent_stream_consumer_task(turn_ctx.stream_consumer_holder)) tracking_task = asyncio.create_task(self._run_agent_track_agent(turn_ctx)) _interrupt_detected = asyncio.Event() # shared with backup check From bb9c83f47e210eb1010bbafff8bb6c6d847c4983 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:04:44 -0700 Subject: [PATCH 5/6] =?UTF-8?q?refactor(gateway):=20run=5Fturn=20=E2=80=94?= =?UTF-8?q?=20fold=20fence-cancel=20poll=20and=20status=20metadata=20into?= =?UTF-8?q?=20their=20single=20callers,=20unify=20streamed-message=20edit,?= =?UTF-8?q?=20MCP=20change=20table,=20misc=20collapses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gateway/run_turn.py | 337 +++++++++++++++++++------------------------- 1 file changed, 145 insertions(+), 192 deletions(-) diff --git a/gateway/run_turn.py b/gateway/run_turn.py index 4998cbef2c..e86972e871 100644 --- a/gateway/run_turn.py +++ b/gateway/run_turn.py @@ -160,7 +160,6 @@ class GatewayTurnMixin: ``request_overrides`` are deep-merged OVER the per-provider ones so both reach the model.""" from gateway.run import _deep_merge_request_overrides from hermes_cli.models import resolve_fast_mode_overrides - # Tests bind this method onto bare namespaces, so no class-level tables here. runtime = { k: runtime_kwargs.get(k) for k in ( @@ -302,14 +301,15 @@ class GatewayTurnMixin: async def _hmwa_heal_telegram_topic_binding(self, source, session_entry, session_key): """Follow the (chat_id, thread_id) topic binding — healed to its compression tip — or record a fresh one. Returns the (possibly switched) session entry.""" + binding = None try: - binding = (await self._session_db.get_telegram_topic_binding( - chat_id=str(source.chat_id), thread_id=str(source.thread_id), - profile_name=self._telegram_topic_profile_name(source), - )) if self._session_db else None + if self._session_db: + binding = await self._session_db.get_telegram_topic_binding( + chat_id=str(source.chat_id), thread_id=str(source.thread_id), + profile_name=self._telegram_topic_profile_name(source), + ) except Exception: logger.debug("Failed to read Telegram topic binding", exc_info=True) - binding = None if not binding: try: await asyncio.to_thread(self._record_telegram_topic_binding, source, session_entry) @@ -556,7 +556,7 @@ class GatewayTurnMixin: # custom_providers per-model context_length fallback (as in run_agent.py); needs base_url. if hs.config_context_length is None and hs.base_url: - try: + with suppress(TypeError, ValueError): try: from hermes_cli.config import ( get_compatible_custom_providers as _gw_gcp, @@ -572,8 +572,6 @@ class GatewayTurnMixin: ) if _hyg_custom_ctx: hs.config_context_length = int(_hyg_custom_ctx) - except (TypeError, ValueError): - pass except Exception: pass return hs @@ -582,7 +580,6 @@ class GatewayTurnMixin: """Decide whether hygiene compression fires this turn (token/message thresholds, DB-backed failure cooldown, in-flight compression).""" from agent.model_metadata import estimate_messages_tokens_rough, get_model_context_length_async - _hyg_context_length = await get_model_context_length_async( hs.model, base_url=hs.base_url or "", api_key=hs.api_key or "", config_context_length=hs.config_context_length, provider=hs.provider or "", @@ -606,21 +603,18 @@ class GatewayTurnMixin: # DB-backed cooldown (shared with context_compressor.py): survives gateway restarts, so a # failing compression is not re-triggered on every restart. _session_db = getattr(self, "_session_db", None) - if _session_db is not None: - _session_db = getattr(_session_db, "_db", _session_db) - _getter = getattr(_session_db, "get_compression_failure_cooldown", None) - if _getter is not None: - try: - _cooldown_state = _getter(session_entry.session_id) - except Exception: - _cooldown_state = None - if _cooldown_state and _cooldown_state.get("remaining_seconds", 0) > 0: - logger.info( - "Session hygiene: skipping compression for %s; " - "previous failure cooldown active for %.1fs", - session_entry.session_id, _cooldown_state["remaining_seconds"], - ) - _needs_compress = False + _getter = getattr(getattr(_session_db, "_db", _session_db), "get_compression_failure_cooldown", None) + if _getter is not None: + _cooldown_state = None + with suppress(Exception): + _cooldown_state = _getter(session_entry.session_id) + if _cooldown_state and _cooldown_state.get("remaining_seconds", 0) > 0: + logger.info( + "Session hygiene: skipping compression for %s; " + "previous failure cooldown active for %.1fs", + session_entry.session_id, _cooldown_state["remaining_seconds"], + ) + _needs_compress = False if _needs_compress and await self._session_has_compression_in_flight(session_key): # A prior compression still holds the durable lock (e.g. a shielded worker left by /stop): @@ -698,29 +692,25 @@ class GatewayTurnMixin: continue raise - async def _hmwa_hygiene_cancel_before_commit(self, fence): - """Cancel the detached worker at the commit fence: ``True`` when cancelled in time, ``False`` - when it already crossed into its commit (caller must consume the result). The lock-free - ``commit_in_flight`` marker keeps this loop from spinning on a hung commit.""" - while True: - if fence.commit_in_flight: - return False - cancelled = fence.try_cancel_before_commit() - if cancelled is not None: - return cancelled - await asyncio.sleep(0.025) - async def _hmwa_hygiene_cancel_or_adopt(self, attempt, context): - """Cancel the worker at the fence; on success release its lease and defer agent cleanup, - returning ``None``. When the worker already crossed into its commit, consume and return - the compressed transcript instead (a successful compaction is never treated as a timeout; - the turn may be held past the budget by up to the commit duration — by design).""" - if not await self._hmwa_hygiene_cancel_before_commit(attempt.commit_fence): - _compressed, _ = await attempt.future - return _compressed - attempt.commit_fence.release_cancelled_compression_lock() - self._hmwa_hygiene_defer_cleanup(attempt, context) - return None + """Cancel the worker at the commit fence; on success release its lease and defer agent + cleanup, returning ``None``. When the worker already crossed into its commit, consume and + return the compressed transcript instead (a successful compaction is never a timeout; the + turn may be held past the budget by up to the commit duration — by design). The lock-free + ``commit_in_flight`` marker keeps the poll from spinning on a hung commit.""" + fence = attempt.commit_fence + while not fence.commit_in_flight: + cancelled = fence.try_cancel_before_commit() + if cancelled is None: + await asyncio.sleep(0.025) + elif cancelled: + fence.release_cancelled_compression_lock() + self._hmwa_hygiene_defer_cleanup(attempt, context) + return None + else: + break + _compressed, _ = await attempt.future + return _compressed def _hmwa_hygiene_defer_cleanup(self, attempt, context): """Hand the agent's cleanup to the still-running worker future and mark it deferred.""" @@ -1032,10 +1022,9 @@ class GatewayTurnMixin: "AGENT_COMPRESSION_COOLDOWN", "hygiene compression abort activity stamp failed", ) if not _hyg_fence_cancelled: - _err = getattr(_comp, "_last_summary_error", None) or "unknown error" # Force-redact: provider exception text may contain credentials; this reaches users. from agent.redact import redact_sensitive_text - _err = redact_sensitive_text(_err, force=True) + _err = redact_sensitive_text(getattr(_comp, "_last_summary_error", None) or "unknown error", force=True) await self._hmwa_hygiene_notify( source, attempt.meta, "⚠️ Context compression aborted " f"({_err}). No messages were dropped — " @@ -1299,15 +1288,13 @@ class GatewayTurnMixin: async def _hmwa_stop_typing_for_turn(self, event, source): """Stop the typing indicator (never raises). Slack AI status is scoped to a thread/ workspace, so preserve the routing metadata used by the response delivery path.""" - try: + with suppress(Exception): _typing_adapter = self._adapter_for_source(source) _kind = type(_typing_adapter) if _typing_adapter and callable(getattr(_kind, "_stop_typing_with_metadata", None)): await _typing_adapter._stop_typing_with_metadata(source.chat_id, self._event_thread_metadata(event, source)) elif _typing_adapter and callable(getattr(_kind, "stop_typing", None)): await _typing_adapter.stop_typing(source.chat_id) - except Exception: - pass async def _hmwa_shape_agent_response( self, agent_result, source, history, session_entry, session_key, @@ -1575,6 +1562,7 @@ class GatewayTurnMixin: # The agent already persisted this turn's rows (codex app-server reports agent_persisted=True # too); skip the DB write. Default = a session DB exists; non-persisting runtimes pass False. agent_persisted = agent_result.get("agent_persisted", self._session_db is not None) + _user_row = self._hmwa_user_transcript_entry(event, prepared, ts) if is_context_overflow_failure: pass # Skip all transcript writes — don't grow a broken session @@ -1599,9 +1587,7 @@ class GatewayTurnMixin: event.message_id, sid, ) else: - await store.append_to_transcript( - sid, self._hmwa_user_transcript_entry(event, prepared, ts), skip_db=agent_persisted, - ) + await store.append_to_transcript(sid, _user_row, skip_db=agent_persisted) else: # Only the NEW messages: history_offset (what the agent saw), not len(history), which # counts session_meta entries stripped before the agent saw them. @@ -1609,9 +1595,7 @@ class GatewayTurnMixin: new_messages = agent_messages[history_len:] if len(agent_messages) > history_len else [] if not new_messages: # Edge case: fall back to simple user/assistant rows. - await store.append_to_transcript( - sid, self._hmwa_user_transcript_entry(event, prepared, ts), skip_db=agent_persisted, - ) + await store.append_to_transcript(sid, _user_row, skip_db=agent_persisted) if response: await store.append_to_transcript( sid, {"role": "assistant", "content": response, "timestamp": ts}, @@ -2046,7 +2030,6 @@ class GatewayTurnMixin: _platform_config_key, ) from run_agent import AIAgent - media_urls = media_urls or [] media_types = media_types or [] adapter = self._adapter_for_source(source) @@ -2108,13 +2091,9 @@ class GatewayTurnMixin: provider_data_collection=pr.get("data_collection"), session_id=task_id, platform=platform_key, - user_id=source.user_id, - user_id_alt=source.user_id_alt, - user_name=source.user_name, - chat_id=source.chat_id, - chat_name=source.chat_name, - chat_type=source.chat_type, - thread_id=source.thread_id, + **{k: getattr(source, k) for k in ( + "user_id", "user_id_alt", "user_name", "chat_id", "chat_name", "chat_type", "thread_id", + )}, session_db=getattr(self._session_db, "_db", self._session_db), # Reload from disk — do not reuse the startup snapshot. fallback_model=self._refresh_fallback_model(), @@ -2239,17 +2218,15 @@ class GatewayTurnMixin: from tools.mcp_tool import _mcp_tool_server_names with _lock: new_tools = [n for n in new_tools if _mcp_tool_server_names.get(n) in connected_servers] - added = connected_servers - old_servers - removed = old_servers - connected_servers - reconnected = connected_servers & old_servers - - lines = [t("gateway.reload_mcp.header")] - for names, key in ( - (reconnected, "gateway.reload_mcp.reconnected"), - (added, "gateway.reload_mcp.added"), (removed, "gateway.reload_mcp.removed"), - ): - if names: - lines.append(t(key, names=", ".join(sorted(names)))) + # (label, i18n key, names); i18n lines list reconnected first, the injected note added first. + changes = ( + ("Reconnected", "gateway.reload_mcp.reconnected", connected_servers & old_servers), + ("Added", "gateway.reload_mcp.added", connected_servers - old_servers), + ("Removed", "gateway.reload_mcp.removed", old_servers - connected_servers), + ) + lines = [t("gateway.reload_mcp.header")] + [ + t(key, names=", ".join(sorted(names))) for _label, key, names in changes if names + ] if not connected_servers: lines.append(t("gateway.reload_mcp.none_connected")) else: @@ -2260,8 +2237,7 @@ class GatewayTurnMixin: # Append a note at the END of the history (preserves the prompt-cache prefix). change_parts = [ f"{label} servers: {', '.join(sorted(names))}" - for label, names in (("Added", added), ("Removed", removed), ("Reconnected", reconnected)) - if names + for label, _key, names in (changes[1], changes[2], changes[0]) if names ] tool_summary = f"{len(new_tools)} MCP tool(s) now available" if new_tools else "No MCP tools available" change_detail = ". ".join(change_parts) + ". " if change_parts else "" @@ -2295,7 +2271,6 @@ class GatewayTurnMixin: ``"raise"`` raises ``RuntimeError`` so the caller skips streaming entirely. """ from gateway.stream_consumer import StreamConsumerConfig - _pause_typing_before_finalize = None if source.platform == Platform.TELEGRAM and hasattr(adapter, "pause_typing_for_chat"): def _pause_typing_before_finalize(_adapter=adapter, _chat_id=source.chat_id) -> None: @@ -2407,14 +2382,11 @@ class GatewayTurnMixin: # OpenAI chat format. The remote keeps continuity via X-Hermes-Session-Id; send the current # message plus a compact text-only history for a remote that has none yet. - api_messages: List[Dict[str, str]] = [] - if context_prompt: - api_messages.append({"role": "system", "content": context_prompt}) - for msg in history: - role = msg.get("role") - content = msg.get("content") - if role in {"user", "assistant"} and content: - api_messages.append({"role": role, "content": content}) + api_messages: List[Dict[str, str]] = [{"role": "system", "content": context_prompt}] if context_prompt else [] + api_messages += [ + {"role": msg.get("role"), "content": msg.get("content")} + for msg in history if msg.get("role") in {"user", "assistant"} and msg.get("content") + ] api_messages.append({"role": "user", "content": message}) headers: Dict[str, str] = {"Content-Type": "application/json"} @@ -2521,7 +2493,6 @@ class GatewayTurnMixin: ) from gateway.display_config import resolve_display_setting from gateway.status_phrases import choose_status_phrase, resolve_status_phrase_catalog - user_config = _load_gateway_config() platform_key = _platform_config_key(source.platform) enabled_toolsets, disabled_toolsets = self._resolve_turn_toolsets(user_config, source, platform_key) @@ -2546,10 +2517,9 @@ class GatewayTurnMixin: _env_tp = os.getenv("HERMES_TOOL_PROGRESS_MODE") _platform_cfg = (_display_cfg.get("platforms") or {}).get(platform_key) or {} _legacy_tp_overrides = _display_cfg.get("tool_progress_overrides") or {} - _tool_progress_configured = ( - "tool_progress" in _display_cfg - or (isinstance(_platform_cfg, dict) and "tool_progress" in _platform_cfg) - or (isinstance(_legacy_tp_overrides, dict) and platform_key in _legacy_tp_overrides) + _tool_progress_configured = "tool_progress" in _display_cfg or any( + isinstance(cfg, dict) and key in cfg + for cfg, key in ((_platform_cfg, "tool_progress"), (_legacy_tp_overrides, platform_key)) ) progress_mode = _env_tp if _env_tp and not _tool_progress_configured else (_resolved_tp or _env_tp or "all") # "accumulate" (edit one bubble) or "separate" (one msg per tool) @@ -2642,7 +2612,6 @@ class GatewayTurnMixin: cleanup_adapter)``. """ from gateway.run import TurnRunner - # Discord voice "verbal ack" on the FIRST tool call (discord.voice_fx.enabled): resolve the # guild whose voice connection is bound to this text channel (mirrors DiscordAdapter.play_tts). _voice_ack_guild: List[Optional[int]] = [None] @@ -2650,10 +2619,10 @@ class GatewayTurnMixin: _va = self.adapters.get(Platform.DISCORD) _vtc = getattr(_va, "_voice_text_channels", None) if isinstance(_vtc, dict) and hasattr(_va, "voice_mixer_active"): - for _gid, _tc in _vtc.items(): - if str(_tc) == str(source.chat_id) and _va.voice_mixer_active(_gid): - _voice_ack_guild[0] = _gid - break + _voice_ack_guild[0] = next( + (_gid for _gid, _tc in _vtc.items() if str(_tc) == str(source.chat_id) and _va.voice_mixer_active(_gid)), + None, + ) # Auto-cleanup of temporary progress bubbles needs a real ``delete_message`` (getattr on the # type: a fake adapter without it means "can't delete", not a crash). @@ -2661,9 +2630,8 @@ class GatewayTurnMixin: disp.resolve_display_setting(disp.user_config, disp.platform_key, "cleanup_progress") ) _cleanup_adapter = self._adapter_for_source(source) if _cleanup_progress else None - _cleanup_delete = getattr(type(_cleanup_adapter), "delete_message", None) if _cleanup_adapter is not None else None - if _cleanup_adapter is not None and ( - _cleanup_delete is None or _cleanup_delete is BasePlatformAdapter.delete_message + if _cleanup_adapter is not None and getattr(type(_cleanup_adapter), "delete_message", None) in ( + None, BasePlatformAdapter.delete_message, ): _cleanup_progress = False _cleanup_adapter = None @@ -2709,12 +2677,13 @@ class GatewayTurnMixin: def _run_agent_progress_threading( self, source: SessionSource, event_message_id: Optional[str], _native_slack_task_cards: bool - ) -> Tuple[Optional[dict], Optional[str], Any, Optional[str]]: + ) -> Tuple[Optional[dict], Optional[str], Optional[dict]]: """Resolve where progress bubbles are threaded (platform-specific). - Returns ``(progress_metadata, progress_reply_to, progress_thread_id, relay_prospective_thread_id)``. - Slack and Buzz honour the user's reply_in_thread opt-out: never synthesise a thread for - progress, or every later reply inherits it. + Returns ``(progress_metadata, progress_reply_to, status_thread_metadata)``; the latter is + for status / approval / stream sends (Feishu topics need the triggering message id via the + reply API, so carry it as a fallback). Slack and Buzz honour the user's reply_in_thread + opt-out: never synthesise a thread for progress, or every later reply inherits it. """ from gateway.run import _non_conversational_metadata, _resolve_progress_thread_id is_buzz = str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz" @@ -2766,7 +2735,13 @@ class GatewayTurnMixin: or _relay_prospective_thread_id else None ) - return _progress_metadata, _progress_reply_to, _progress_thread_id, _relay_prospective_thread_id + if source.platform == Platform.FEISHU and source.thread_id and event_message_id: + _status_thread_metadata = {"thread_id": _progress_thread_id, "reply_to_message_id": event_message_id} + else: + _status_thread_metadata = self._thread_metadata_for_progress( + source, event_message_id, _progress_thread_id, _relay_prospective_thread_id, + ) + return _progress_metadata, _progress_reply_to, _status_thread_metadata async def _run_agent_write_tool_log(self, log_queue: Any) -> None: """Drain log_queue and append tool-call lines to tool_calls.log (tool_progress=log). @@ -2810,18 +2785,6 @@ class GatewayTurnMixin: file_handler.flush() file_handler.close() - def _run_agent_status_thread_metadata( - self, source: SessionSource, event_message_id: Optional[str], _progress_thread_id: Any, - _relay_prospective_thread_id: Optional[str], - ) -> Optional[Dict[str, Any]]: - """Thread metadata for status / approval / stream sends; Feishu topics need the triggering - message id (reply API with reply_in_thread=true) so carry it as a fallback.""" - if source.platform == Platform.FEISHU and source.thread_id and event_message_id: - return {"thread_id": _progress_thread_id, "reply_to_message_id": event_message_id} - return self._thread_metadata_for_progress( - source, event_message_id, _progress_thread_id, _relay_prospective_thread_id, - ) - def _run_agent_start_streaming_tts( self, source: SessionSource, message_type: Optional[str], _status_thread_metadata: Optional[Dict[str, Any]], streaming_tts_consumer_holder: list, @@ -3006,10 +2969,10 @@ class GatewayTurnMixin: from gateway.run import _float_env, _watch_gateway_turn_inactivity from tools.process_registry import process_registry agent_holder, session_key, run_generation = turn_ctx.agent_holder, turn_ctx.session_key, turn_ctx.run_generation - _agent_timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) - _agent_warning = _float_env("HERMES_AGENT_TIMEOUT_WARNING", 900) - _agent_timeout = _agent_timeout if _agent_timeout > 0 else None - _agent_warning = _agent_warning if _agent_warning > 0 else None + _agent_timeout, _agent_warning = ( + v if v > 0 else None + for v in (_float_env("HERMES_AGENT_TIMEOUT", 1800), _float_env("HERMES_AGENT_TIMEOUT_WARNING", 900)) + ) # background=true processes survive a turn: reap only children created by THIS turn on timeout. _turn_task_id = turn_ctx.session_id or "" @@ -3186,8 +3149,7 @@ class GatewayTurnMixin: session_key = turn_ctx.session_key _agent = turn_ctx.agent_holder[0] _result_for_fb = turn_ctx.result_holder[0] - _run_failed = _result_for_fb.get("failed") if _result_for_fb else False - if _agent is None or not hasattr(_agent, 'model') or _run_failed: + if _agent is None or not hasattr(_agent, 'model') or (_result_for_fb and _result_for_fb.get("failed")): return _cfg_model = _resolve_gateway_model() # Normalize as AIAgent.__init__ does (vendor prefix stripped on native providers), else the @@ -3268,8 +3230,7 @@ class GatewayTurnMixin: # Safety net: a pending slash command is never passed to the agent as user input. if pending and pending.strip().startswith("/"): - _pending_parts = pending.strip().split(None, 1) - _pending_cmd_word = _pending_parts[0][1:].lower() if _pending_parts else "" + _pending_cmd_word = pending.strip().split(None, 1)[0][1:].lower() if _pending_cmd_word: with suppress(Exception): from hermes_cli.commands import resolve_command as _rc_pending @@ -3314,13 +3275,13 @@ class GatewayTurnMixin: session_key or "?", ) elif first_response: + logger.info( + "Queued follow-up for session %s: final text delivery confirmed; delivering explicit media before continuing." + if _already_streamed else + "Queued follow-up for session %s: final stream delivery not confirmed; sending first response before continuing.", + session_key or "?", + ) try: - logger.info( - "Queued follow-up for session %s: final text delivery confirmed; delivering explicit media before continuing." - if _already_streamed else - "Queued follow-up for session %s: final stream delivery not confirmed; sending first response before continuing.", - session_key or "?", - ) await self._deliver_queued_first_response( first_response, source=turn_ctx.source, adapter=adapter, metadata=turn_ctx._status_thread_metadata, event_message_id=turn_ctx.event_message_id, @@ -3352,8 +3313,9 @@ class GatewayTurnMixin: logger.debug("Processing pending message: '%s...'", pending[:40]) # Clear the interrupt event so the recursive _run_agent isn't re-interrupted (infinite loop). - if adapter and hasattr(adapter, '_active_sessions') and session_key and session_key in adapter._active_sessions: - adapter._active_sessions[session_key].clear() + _active = getattr(adapter, "_active_sessions", None) if adapter else None + if _active and session_key and session_key in _active: + _active[session_key].clear() # Cap recursion depth (user keeps sending while the agent keeps failing). if _interrupt_depth >= self._MAX_INTERRUPT_DEPTH: @@ -3404,13 +3366,12 @@ class GatewayTurnMixin: # Clear the prior turn's streaming-TTS completion marker so the recursive turn isn't suppressed. _clear_adapter = self._adapter_for_source(source) - if _clear_adapter is not None and session_key and run_generation is not None: - _completed_turns = getattr(_clear_adapter, "_streaming_tts_completed_turns", None) - _prior_key = getattr(_clear_adapter, "_streaming_tts_turn_key", None) - if _completed_turns is not None and callable(_prior_key): - _pk = _prior_key(session_key, run_generation) - if _pk: - _completed_turns.discard(_pk) + _completed_turns = getattr(_clear_adapter, "_streaming_tts_completed_turns", None) + _prior_key = getattr(_clear_adapter, "_streaming_tts_turn_key", None) + if _completed_turns is not None and callable(_prior_key) and session_key and run_generation is not None: + _pk = _prior_key(session_key, run_generation) + if _pk: + _completed_turns.discard(_pk) # Restart the typing indicator; the outer typing task may be stale. if _clear_adapter: @@ -3474,6 +3435,25 @@ class GatewayTurnMixin: # A background task that died of a real error must not abort the cleanup path. logger.debug("background turn task failed during cleanup", exc_info=True) + async def _run_agent_edit_streamed_message( + self, _sc, source, response, content, *, _sk, ok, fail_result, fail_exc, + ) -> None: + """Edit the stream consumer's message in place with ``content``; on success mark + ``response["already_sent"]`` and log ``ok``. ``fail_result`` (None = trust the call) logs a + returned failure as ``(session, error)``; ``fail_exc`` logs an exception as ``(session, exc)``.""" + try: + _res = await _sc.adapter.edit_message( + chat_id=source.chat_id, message_id=_sc.message_id, content=content, finalize=True, + ) + except Exception as _edit_err: + logger.warning(fail_exc, _sk, _edit_err) + return + if fail_result is not None and not getattr(_res, "success", True): + logger.warning(fail_result, _sk, getattr(_res, "error", None)) + return + response["already_sent"] = True + logger.info(*ok) + async def _run_agent_mark_streamed_delivery(self, response: Any, turn_ctx: TurnContext) -> None: """Set ``response["already_sent"]`` when streaming already delivered the final reply. @@ -3494,10 +3474,8 @@ class GatewayTurnMixin: if _content_delivered and not _is_empty_sentinel: _matcher = getattr(_sc, "delivered_final_matches", None) if callable(_matcher): - try: + with suppress(Exception): _stale_finalized = _matcher(_final) is False - except Exception: - _stale_finalized = False if _stale_finalized: _content_delivered = False # Plugin hooks may append content after streaming finished — then send the final version. @@ -3523,26 +3501,12 @@ class GatewayTurnMixin: _sk, ) elif _sc_msg_id and _sc_msg_id != "__no_edit__" and getattr(_sc, "adapter", None) is not None: - try: - _reconcile_res = await _sc.adapter.edit_message( - chat_id=source.chat_id, message_id=_sc_msg_id, content=_final, finalize=True, - ) - if getattr(_reconcile_res, "success", True): - response["already_sent"] = True - logger.info( - "Reconciled stale streamed finalize for session %s: edited message %s with the complete response (#71643).", - _sk, _sc_msg_id, - ) - else: - logger.warning( - "Stale-finalize reconciliation edit failed for session %s (%s); sending complete response via normal final send.", - _sk, getattr(_reconcile_res, "error", None), - ) - except Exception as _edit_err: - logger.warning( - "Stale-finalize reconciliation edit failed for session %s: %s; sending complete response via normal final send.", - _sk, _edit_err, - ) + await self._run_agent_edit_streamed_message( + _sc, source, response, _final, _sk=_sk, + ok=("Reconciled stale streamed finalize for session %s: edited message %s with the complete response (#71643).", _sk, _sc_msg_id), + fail_result="Stale-finalize reconciliation edit failed for session %s (%s); sending complete response via normal final send.", + fail_exc="Stale-finalize reconciliation edit failed for session %s: %s; sending complete response via normal final send.", + ) else: logger.info( "Stale streamed finalize detected for session %s with no editable message; delivering complete response via normal final send (#71643).", @@ -3550,19 +3514,12 @@ class GatewayTurnMixin: ) elif _transformed and _sc is not None: # Transformed after streaming: edit the streamed message instead of sending a duplicate. - _sc_msg_id = _sc.message_id - if _sc_msg_id: - try: - await _sc.adapter.edit_message( - chat_id=source.chat_id, message_id=_sc_msg_id, content=response["final_response"], finalize=True, - ) - response["already_sent"] = True - logger.info( - "Edited streamed message %s for session %s to include plugin-transformed content.", - _sc_msg_id, _sk, - ) - except Exception as _edit_err: - logger.warning("Failed to edit streamed message for session %s: %s", _sk, _edit_err) + if _sc.message_id: + await self._run_agent_edit_streamed_message( + _sc, source, response, response["final_response"], _sk=_sk, + ok=("Edited streamed message %s for session %s to include plugin-transformed content.", _sc.message_id, _sk), + fail_result=None, fail_exc="Failed to edit streamed message for session %s: %s", + ) elif _sc is not None: # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed but suppression did NOT fire; log the # decision inputs ("signal never set" vs "ack-pending race"). @@ -3620,22 +3577,18 @@ class GatewayTurnMixin: ``turn_ctx`` (the one-slot holders shared with run_sync's executor thread are TurnContext defaults). Returns ``_status_thread_metadata``. """ - ( - turn_ctx._progress_metadata, turn_ctx._progress_reply_to, _progress_thread_id, - _relay_prospective_thread_id, - ) = self._run_agent_progress_threading(source, event_message_id, _native_slack_task_cards) + turn_ctx._progress_metadata, turn_ctx._progress_reply_to, _status_thread_metadata = ( + self._run_agent_progress_threading(source, event_message_id, _native_slack_task_cards) + ) # Bridges: sync step/event/status callbacks → async hooks.emit and adapter.send. turn_ctx._loop_for_step = asyncio.get_running_loop() turn_ctx._hooks_ref = self.hooks turn_ctx._step_callback_sync = turn_runner._step_callback_sync turn_ctx._event_callback_sync = turn_runner._event_callback_sync + turn_ctx._status_callback_sync = turn_runner._status_callback_sync turn_ctx._status_adapter = self._adapter_for_source(source) - _status_thread_metadata = self._run_agent_status_thread_metadata( - source, event_message_id, _progress_thread_id, _relay_prospective_thread_id, - ) turn_ctx._status_chat_id = source.chat_id turn_ctx._status_thread_metadata = _status_thread_metadata - turn_ctx._status_callback_sync = turn_runner._status_callback_sync return _status_thread_metadata async def _run_agent_notify_long_running( @@ -3749,17 +3702,17 @@ class GatewayTurnMixin: ) # Progress sender drains BOTH tool-progress lines and thinking bubbles (needs_progress_queue). - progress_task = asyncio.create_task(turn_runner.send_progress_messages()) if disp.needs_progress_queue else None - log_task = asyncio.create_task(self._run_agent_write_tool_log(disp.log_queue)) if disp.log_mode_enabled else None + spawn = asyncio.create_task + progress_task = spawn(turn_runner.send_progress_messages()) if disp.needs_progress_queue else None + log_task = spawn(self._run_agent_write_tool_log(disp.log_queue)) if disp.log_mode_enabled else None # The stream consumer is created inside run_sync; this task polls for it. - stream_task = asyncio.create_task(self._run_agent_stream_consumer_task(turn_ctx.stream_consumer_holder)) - tracking_task = asyncio.create_task(self._run_agent_track_agent(turn_ctx)) + stream_task = spawn(self._run_agent_stream_consumer_task(turn_ctx.stream_consumer_holder)) + tracking_task = spawn(self._run_agent_track_agent(turn_ctx)) _interrupt_detected = asyncio.Event() # shared with backup check - interrupt_monitor = asyncio.create_task(self._run_agent_monitor_for_interrupt(turn_ctx, _interrupt_detected)) - + interrupt_monitor = spawn(self._run_agent_monitor_for_interrupt(turn_ctx, _interrupt_detected)) # Periodic "still working" notifications so the user knows the agent hasn't died. _executor_task_holder: list = [None] # bound once the executor future exists (see below) - _notify_task = asyncio.create_task(self._run_agent_notify_long_running(disp, turn_ctx, _executor_task_holder)) + _notify_task = spawn(self._run_agent_notify_long_running(disp, turn_ctx, _executor_task_holder)) try: # run_sync is TurnRunner.run_sync (bound method; executor call unchanged). From 413a6af6299997fba34ff7c5b5403f959668287a Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:10:30 -0700 Subject: [PATCH 6/6] =?UTF-8?q?refactor(gateway):=20run=5Fturn=20=E2=80=94?= =?UTF-8?q?=20=5Fhmwa=5Fprepare=5Fturn=20owns=20session=20open=20+=20env?= =?UTF-8?q?=20setup,=20tuple=20outcome=20replaces=20nested=20carrier,=20hu?= =?UTF-8?q?gged=20docstring=20closers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gateway/run_turn.py | 152 +++++++++++++++----------------------------- 1 file changed, 53 insertions(+), 99 deletions(-) diff --git a/gateway/run_turn.py b/gateway/run_turn.py index e86972e871..42e8870b86 100644 --- a/gateway/run_turn.py +++ b/gateway/run_turn.py @@ -49,8 +49,7 @@ class GatewayTurnMixin: """Resolve model/runtime for a session. Priority (highest first): session ``/model`` → ``channel_overrides`` → global config/env - (``_resolve_gateway_model(user_config)`` and default provider resolution). - """ + (``_resolve_gateway_model(user_config)`` and default provider resolution).""" from gateway.run import ( _credential_pool_for_provider, _get_channel_override, _resolve_gateway_model, _resolve_runtime_agent_kwargs, _resolve_runtime_agent_kwargs_for_provider, @@ -872,8 +871,7 @@ class GatewayTurnMixin: logger.warning( "Session hygiene compression for session %s made no progress for %.1fs " "(total wait %.1fs, ceiling %.1fs); continuing without compression", - session_entry.session_id, fence.seconds_since_progress(), _hyg_elapsed, - hs.total_ceiling_seconds, + session_entry.session_id, fence.seconds_since_progress(), _hyg_elapsed, hs.total_ceiling_seconds, ) await self._hmwa_hygiene_notify( source, attempt.meta, @@ -895,28 +893,19 @@ class GatewayTurnMixin: self._hmwa_hygiene_defer_cleanup(attempt, "session hygiene unwind") if hs.failure_cooldown_seconds >= 0: try: - _hyg_cooldown = _hygiene_cooldown_for_failure(self, session_key, hs.failure_cooldown_seconds) _record_hygiene_cooldown( - self, session_entry.session_id, _hyg_cooldown, + self, session_entry.session_id, + _hygiene_cooldown_for_failure(self, session_key, hs.failure_cooldown_seconds), "session hygiene compression cancelled at commit fence", ) except Exception as _cd_err: logger.debug("hygiene unwind cooldown record failed: %s", _cd_err) - @dataclasses.dataclass - class _HygieneOutcome: - """What a finished hygiene compression did to the transcript.""" - - rotated: bool - in_place: bool - new_count: int - new_tokens: int - async def _hmwa_hygiene_adopt_transcript( self, attempt, _compressed, history, plan, *, session_entry, source, _quick_key, run_generation, ): """Adopt a finished compression (rotation / in-place / refused); publishes the transcript to - continue with on ``attempt.history``. + continue with on ``attempt.history``. Returns ``(rotated, in_place, new_count, new_tokens)``. Rewrite only on rotation (NEW session id): in-place compaction already soft-archived the old rows and rewrite_transcript() would DELETE them; neither rotation nor in-place signals @@ -980,7 +969,7 @@ class GatewayTurnMixin: ) if _new_tokens >= plan.warn_token_threshold: logger.warning("Session hygiene: still ~%s tokens after compression", f"{_new_tokens:,}") - return self._HygieneOutcome(_hyg_rotated, _hyg_in_place, _new_count, _new_tokens) + return _hyg_rotated, _hyg_in_place, _new_count, _new_tokens async def _hmwa_hygiene_apply_result( self, attempt, hs, _compressed, history, plan, *, @@ -989,7 +978,7 @@ class GatewayTurnMixin: """Adopt a finished hygiene compression, rebind the session + turn lease, record streak/cooldown, and warn the user on abort.""" from gateway.run import _reset_hygiene_failure_streak, hygiene_compaction_recovered - outcome = await self._hmwa_hygiene_adopt_transcript( + _hyg_rotated, _hyg_in_place, _new_count, _new_tokens = await self._hmwa_hygiene_adopt_transcript( attempt, _compressed, history, plan, session_entry=session_entry, source=source, _quick_key=_quick_key, run_generation=run_generation, ) @@ -1000,15 +989,15 @@ class GatewayTurnMixin: # A fence-cancelled _compress_context returns the original transcript with # _last_compress_aborted False: treat that no-op as an abort so hygiene records a cooldown # instead of retrying into the 600s wait. A committed rotate/in-place is never an abort. - _hyg_fence_cancelled = bool(attempt.commit_fence.is_cancelled and not outcome.rotated and not outcome.in_place) + _hyg_fence_cancelled = bool(attempt.commit_fence.is_cancelled and not _hyg_rotated and not _hyg_in_place) if _hyg_fence_cancelled: _hyg_aborted = True # Recovery decision lives in the unit-tested predicate: the "neither rotated nor in place" # path reuses pre-compression counts, so a numbers-only check would read a no-op as success. if not _hyg_aborted and hygiene_compaction_recovered( - aborted=_hyg_aborted, rotated=outcome.rotated, in_place=outcome.in_place, - msg_count=plan.msg_count, new_count=outcome.new_count, approx_tokens=plan.approx_tokens, - new_tokens=outcome.new_tokens, + aborted=_hyg_aborted, rotated=_hyg_rotated, in_place=_hyg_in_place, + msg_count=plan.msg_count, new_count=_new_count, approx_tokens=plan.approx_tokens, + new_tokens=_new_tokens, ): await asyncio.to_thread(_reset_hygiene_failure_streak, self, session_key) if _hyg_aborted: @@ -1754,15 +1743,16 @@ class GatewayTurnMixin: persist_user_timestamp: Any persist_user_display_kind: Optional[str] - async def _hmwa_prepare_turn( - self, event, source, session_entry, session_key, _quick_key, run_generation, - _was_auto_reset, _is_new_session, context, _session_env_tokens, - ): - """Everything between session resolution and the agent run: context prompt, sidecar - notes, turn lease, transcript load + hygiene, inbound text. Returns a ``_PreparedTurn``; - a ``str`` is a reply to send instead of running (history unreadable); ``None`` drops the - turn (inbound text rejected).""" + async def _hmwa_prepare_turn(self, event, source, session_entry, session_key, _quick_key, run_generation): + """Everything between session resolution and the agent run: session open, task-local env, + context prompt, sidecar notes, turn lease, transcript load + hygiene, inbound text. Returns + ``(_PreparedTurn, env_tokens)``; a ``str`` first element is a reply to send instead of + running (history unreadable); ``None`` drops the turn (inbound text rejected).""" from gateway.run import _load_gateway_config + _was_auto_reset, _is_new_session = await self._hmwa_open_session(session_entry, session_key, source) + context = build_session_context(source, self.config, session_entry) + # Session context variables for tools (task-local, concurrency-safe) + _session_env_tokens = self._set_session_env(context) # Self-injected turns (MessageEvent(internal=True)) persist with a DB-only display_kind so # UIs render timeline notices, not user bubbles; role/content untouched. persist_user_display_kind = "internal_notification" if getattr(event, "internal", False) else None @@ -1801,7 +1791,7 @@ class GatewayTurnMixin: "⚠️ This session's history is temporarily unavailable, so this message was not " "processed. Ask the operator to inspect state.db, then resend after it is healthy. " "Use /reset only if you intentionally want to start a new conversation." - ) + ), _session_env_tokens history = await self._hmwa_run_session_hygiene( event, source, session_entry, session_key, history, _quick_key, run_generation, @@ -1820,7 +1810,7 @@ class GatewayTurnMixin: event=event, source=source, history=history, session_key=session_key, ) if message_text is None: - return None + return None, _session_env_tokens message_text, persist_user_message, persist_user_timestamp = ( self._hmwa_apply_message_timestamp(event, message_text) @@ -1837,7 +1827,7 @@ class GatewayTurnMixin: return self._PreparedTurn( history, context_prompt, message_text, persist_user_message, persist_user_timestamp, persist_user_display_kind, - ) + ), _session_env_tokens async def _handle_message_with_agent(self, event, source, _quick_key: str, run_generation: int): """Inner handler that runs under the _running_agents sentinel guard.""" @@ -1855,15 +1845,8 @@ class GatewayTurnMixin: if resolved is None: return source, session_entry, session_key = resolved - _was_auto_reset, _is_new_session = await self._hmwa_open_session(session_entry, session_key, source) - - context = build_session_context(source, self.config, session_entry) - # Session context variables for tools (task-local, concurrency-safe) - _session_env_tokens = self._set_session_env(context) - - prepared = await self._hmwa_prepare_turn( + prepared, _session_env_tokens = await self._hmwa_prepare_turn( event, source, session_entry, session_key, _quick_key, run_generation, - _was_auto_reset, _is_new_session, context, _session_env_tokens, ) if not isinstance(prepared, self._PreparedTurn): return prepared @@ -1943,8 +1926,7 @@ class GatewayTurnMixin: """``_profile_runtime_scope`` for ``source``'s profile when multiplexing, else a no-op context. Under multiplexing config/skills/memory resolve to the source profile's home AND credentials - come from its secret scope (never process-global ``os.environ``). - """ + come from its secret scope (never process-global ``os.environ``).""" from gateway.run import _profile_runtime_scope if getattr(getattr(self, "config", None), "multiplex_profiles", False): return _profile_runtime_scope(self._resolve_profile_home_for_source(source)) @@ -1954,8 +1936,7 @@ class GatewayTurnMixin: """Session-info block for the auto-reset notice, resolved inside the profile serving ``source``. Call via ``asyncio.to_thread``: resolution can block (credential refresh, context-length - probes), and the scope is entered here so contextvars behave in the worker thread. - """ + probes), and the scope is entered here so contextvars behave in the worker thread.""" with self._profile_scope_for_source(source): return self._format_session_info() @@ -1998,8 +1979,7 @@ class GatewayTurnMixin: ) -> list: """Enabled toolsets for an agent run, honoring an adapter ``toolsets_for_source()`` override validated through the SAME ``_get_platform_tools`` path (unknown / platform-restricted - toolsets dropped, not trusted). - """ + toolsets dropped, not trusted).""" from hermes_cli.tools_config import _get_platform_tools try: adapter = self._adapter_for_source(source) @@ -2184,8 +2164,7 @@ class GatewayTurnMixin: """Disconnect, reconnect, and notify MCP tool changes (shared by button / text / no-confirm paths). Under multiplex the reload runs inside the requesting profile's runtime scope (entered here - when the caller did not) and only that profile's servers are torn down and rediscovered. - """ + when the caller did not) and only that profile's servers are torn down and rediscovered.""" from gateway.run import _profile_runtime_scope multiplex = bool(getattr(self.config, "multiplex_profiles", False)) if multiplex and not get_hermes_home_override(): @@ -2268,8 +2247,7 @@ class GatewayTurnMixin: ) -> "tuple[Any, Optional[Callable[[], None]]]": """Build the shared ``StreamConsumerConfig`` and optional Telegram pause-typing closure. For non-editing adapters ``on_missing_cursor="fallback"`` streams with an empty cursor; - ``"raise"`` raises ``RuntimeError`` so the caller skips streaming entirely. - """ + ``"raise"`` raises ``RuntimeError`` so the caller skips streaming entirely.""" from gateway.stream_consumer import StreamConsumerConfig _pause_typing_before_finalize = None if source.platform == Platform.TELEGRAM and hasattr(adapter, "pause_typing_for_chat"): @@ -2350,8 +2328,7 @@ class GatewayTurnMixin: """Forward the message to a remote Hermes API server instead of running a local AIAgent. Lets a Docker container handle Matrix E2EE while the agent runs on the host with full - access to local files, memory, skills, and a unified session store. - """ + access to local files, memory, skills, and a unified session store.""" from gateway.run import _GATEWAY_PROXY_SSE_BUFFER_MAX_CHARS try: from aiohttp import ClientSession as _AioClientSession, ClientTimeout @@ -2609,8 +2586,7 @@ class GatewayTurnMixin: ) -> Tuple[TurnContext, TurnRunner, Any]: """Build the ``TurnContext`` and its ``TurnRunner``; ``turn_params`` (history, context_prompt, session_id, persist_user_*, …) are stored verbatim. Returns ``(turn_ctx, turn_runner, - cleanup_adapter)``. - """ + cleanup_adapter)``.""" from gateway.run import TurnRunner # Discord voice "verbal ack" on the FIRST tool call (discord.voice_fx.enabled): resolve the # guild whose voice connection is bound to this text channel (mirrors DiscordAdapter.play_tts). @@ -2660,8 +2636,7 @@ class GatewayTurnMixin: """Thread metadata for a progress-lane send; relay Discord auto-thread lane falls back to the reply anchor. The connector will auto-thread on the reply anchor (thread is born on its FIRST send), so - carrying it routes progress / status bubbles into the same thread as the final reply. - """ + carrying it routes progress / status bubbles into the same thread as the final reply.""" if not _progress_thread_id: metadata = None elif _progress_thread_id == source.thread_id: @@ -2683,8 +2658,7 @@ class GatewayTurnMixin: Returns ``(progress_metadata, progress_reply_to, status_thread_metadata)``; the latter is for status / approval / stream sends (Feishu topics need the triggering message id via the reply API, so carry it as a fallback). Slack and Buzz honour the user's reply_in_thread - opt-out: never synthesise a thread for progress, or every later reply inherits it. - """ + opt-out: never synthesise a thread for progress, or every later reply inherits it.""" from gateway.run import _non_conversational_metadata, _resolve_progress_thread_id is_buzz = str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz" _progress_reply_in_thread = True @@ -2746,8 +2720,7 @@ class GatewayTurnMixin: async def _run_agent_write_tool_log(self, log_queue: Any) -> None: """Drain log_queue and append tool-call lines to tool_calls.log (tool_progress=log). - RotatingFileHandler (5MB × 3) bounds the log; RedactingFormatter keeps secrets off disk. - """ + RotatingFileHandler (5MB × 3) bounds the log; RedactingFormatter keeps secrets off disk.""" from gateway.run import _hermes_home if log_queue is None: return @@ -2792,8 +2765,7 @@ class GatewayTurnMixin: """Start the streaming-TTS consumer for a voice-input turn on an auto-TTS chat. Created on the gateway loop thread (not run_sync's executor); an inactive consumer leaves - the holder None so the whole-file fallback path runs. - """ + the holder None so the whole-file fallback path runs.""" _stts_adapter = self._adapter_for_source(source) _is_voice_input = ( message_type is not None @@ -2860,8 +2832,7 @@ class GatewayTurnMixin: Peek WITHOUT consuming: the event must stay for the post-run ``_dequeue_pending_event()`` (popping races the agent finishing). Transcribe BEFORE signaling so voice interrupts carry - the real transcript. - """ + the real transcript.""" from gateway.run import _build_media_placeholder _peek_event = adapter._pending_messages.get(session_key) pending_text = None @@ -2886,8 +2857,7 @@ class GatewayTurnMixin: """Poll the adapter for interrupts (new messages) every 200ms and signal the agent. Level 1 (base.py) catches regular text before _handle_message(); the inactivity poll loop - has a BACKUP check in case this task dies. Keyed by session_key, NOT source.chat_id. - """ + has a BACKUP check in case this task dies. Keyed by session_key, NOT source.chat_id.""" source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder streaming_tts_consumer_holder = turn_ctx.streaming_tts_consumer_holder if not session_key: @@ -2939,8 +2909,7 @@ class GatewayTurnMixin: def _run_agent_stream_confirmed_final_delivery(consumer, final_text: str, *, previewed: bool = False) -> bool: """True only when the actual final reply reached the user: a finalize call may carry only the last preview snapshot, so reconcile against the recorded payload — a demonstrable mismatch - (False) overrides the flag; None keeps legacy trust. - """ + (False) overrides the flag; None keeps legacy trust.""" if consumer is None: return False if getattr(consumer, "final_response_sent", False): @@ -2964,8 +2933,7 @@ class GatewayTurnMixin: *Inactivity* timeout (agent.gateway_timeout / HERMES_AGENT_TIMEOUT, env wins; 0 = unlimited), not wall-clock. The daemon watchdog is independent of asyncio: cgroup memory reclaim can - starve the loop that runs the normal timeout poll. - """ + starve the loop that runs the normal timeout poll.""" from gateway.run import _float_env, _watch_gateway_turn_inactivity from tools.process_registry import process_registry agent_holder, session_key, run_generation = turn_ctx.agent_holder, turn_ctx.session_key, turn_ctx.run_generation @@ -3024,11 +2992,7 @@ class GatewayTurnMixin: def _reaper_kwargs(worker: "GatewayRunner._RunAgentWorker") -> dict: """Shared kwargs of the watchdog + timeout-reaper threads.""" return { - "task_id": worker.task_id, - "process_baseline": worker.process_baseline, - "worker_done": worker.worker_done, - "timeout_fired": worker.timeout_fired, - "cleanup_lock": worker.cleanup_lock, + **{k: getattr(worker, k) for k in ("task_id", "process_baseline", "worker_done", "timeout_fired", "cleanup_lock")}, "is_still_current": worker.is_current, } @@ -3046,13 +3010,12 @@ class GatewayTurnMixin: _warn_adapter = self._adapter_for_source(source) if not _warn_adapter: return - _elapsed_warn = int(worker.agent_warning // 60) or 1 - _remaining_mins = int((worker.agent_timeout - worker.agent_warning) // 60) or 1 try: await _warn_adapter.send( - source.chat_id, f"⚠️ No activity for {_elapsed_warn} min. " - f"If the agent does not respond soon, it will be timed out in {_remaining_mins} " - f"min. You can continue waiting or use /reset.", + source.chat_id, f"⚠️ No activity for {int(worker.agent_warning // 60) or 1} min. " + "If the agent does not respond soon, it will be timed out in " + f"{int((worker.agent_timeout - worker.agent_warning) // 60) or 1} min. " + "You can continue waiting or use /reset.", metadata=_interim_metadata(_status_thread_metadata), ) except Exception as _warn_err: @@ -3112,8 +3075,7 @@ class GatewayTurnMixin: ) -> Any: """Poll the executor future (inactivity timeout + backup interrupt checks); return its result, or a synthetic failed run dict on inactivity timeout. Polls even with an unlimited timeout - so the backup interrupt check runs if monitor_for_interrupt() silently died. - """ + so the backup interrupt check runs if monitor_for_interrupt() silently died.""" from gateway.run import _abandon_timed_out_gateway_turn agent_holder = turn_ctx.agent_holder _warning_fired = False @@ -3143,8 +3105,7 @@ class GatewayTurnMixin: def _run_agent_evict_on_fallback(self, turn_ctx: TurnContext) -> None: """Evict the cached agent when a fallback model activated on a SUCCESSFUL run (so /model shows the active model and the next message retries the primary). Skip failed runs: evicting - would loop bad model → fallback → evict → recreate. - """ + would loop bad model → fallback → evict → recreate.""" from gateway.run import _resolve_gateway_model session_key = turn_ctx.session_key _agent = turn_ctx.agent_holder[0] @@ -3165,8 +3126,7 @@ class GatewayTurnMixin: async def _run_agent_finalize_streaming_tts(self, turn_ctx: TurnContext, adapter: Any) -> None: """Finalize the streaming-TTS consumer on the outer event-loop thread (covers early returns from run_sync). On drain timeout abort to free the task — audible streams keep whole-file - suppression, silent streams stay eligible for the whole-file fallback. - """ + suppression, silent streams stay eligible for the whole-file fallback.""" _stts = turn_ctx.streaming_tts_consumer_holder[0] if _stts is None: return @@ -3188,8 +3148,7 @@ class GatewayTurnMixin: ) -> Tuple[Any, Optional[str]]: """Dequeue the adapter's pending / interrupt / leftover-steer follow-up as ``(pending_event, pending)``. - Keyed by session_key (not source.chat_id) to match the adapter's storage keys. - """ + Keyed by session_key (not source.chat_id) to match the adapter's storage keys.""" from gateway.run import ( _build_media_placeholder, _dequeue_pending_event, _is_control_interrupt_message ) @@ -3460,8 +3419,7 @@ class GatewayTurnMixin: Never when the agent failed (the error is unseen content) or on "(empty)". Both suppression flags reflect call success, not content, so reconcile against the recorded turn-final payload: a mismatch (False, incl. payload-less split delivery) never suppresses; None (no - record) keeps legacy trust. - """ + record) keeps legacy trust.""" _sc, source, session_key = turn_ctx.stream_consumer_holder[0], turn_ctx.source, turn_ctx.session_key if not isinstance(response, dict) or response.get("failed"): return @@ -3533,8 +3491,7 @@ class GatewayTurnMixin: def _run_agent_schedule_bubble_cleanup(self, response: Any, _cleanup_adapter: Any, turn_ctx: TurnContext) -> None: """Schedule deletion of tracked temporary progress bubbles after the final response lands. - Failed runs keep them as breadcrumbs. Only on adapters with ``delete_message``; failures swallowed. - """ + Failed runs keep them as breadcrumbs. Only on adapters with ``delete_message``; failures swallowed.""" from gateway.run import safe_schedule_threadsafe _cleanup_msg_ids, session_key = turn_ctx._cleanup_msg_ids, turn_ctx.session_key if not ( @@ -3575,8 +3532,7 @@ class GatewayTurnMixin: ) -> Optional[Dict[str, Any]]: """Resolve progress threading, then publish progress metadata and the sync→async bridges onto ``turn_ctx`` (the one-slot holders shared with run_sync's executor thread are TurnContext - defaults). Returns ``_status_thread_metadata``. - """ + defaults). Returns ``_status_thread_metadata``.""" turn_ctx._progress_metadata, turn_ctx._progress_reply_to, _status_thread_metadata = ( self._run_agent_progress_threading(source, event_message_id, _native_slack_task_cards) ) @@ -3599,8 +3555,7 @@ class GatewayTurnMixin: bound just after this task is scheduled (reads as None until then). Interval: agent.gateway_notify_interval / HERMES_AGENT_NOTIFY_INTERVAL (default 180s; 0 or - long_running_notifications=off disables). - """ + long_running_notifications=off disables).""" from gateway.run import _float_env, _interim_metadata, _non_conversational_metadata _notify_start = time.time() _NOTIFY_INTERVAL = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180) @@ -3672,8 +3627,7 @@ class GatewayTurnMixin: ) -> Dict[str, Any]: """Run the agent; returns the full run_conversation result dict. - Keys: "final_response", "messages", "api_calls", "completed". - """ + Keys: "final_response", "messages", "api_calls", "completed".""" if self._get_proxy_url(): return await self._run_agent_via_proxy( message=message, context_prompt=context_prompt, history=history, source=source,