From bcdfb61fb446bd1b661e2260bf1b3b0ff0ca6039 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:16:43 -0700 Subject: [PATCH 01/11] refactor(gateway): compact run_* docstrings/comments by hand; AST-neutral bracket packing to 110 cols --- gateway/run_adapters.py | 107 ++++++----------- gateway/run_notifications.py | 98 +++++----------- gateway/run_shutdown.py | 78 +++++-------- gateway/run_startup.py | 219 ++++++++++++----------------------- 4 files changed, 170 insertions(+), 332 deletions(-) diff --git a/gateway/run_adapters.py b/gateway/run_adapters.py index 3d62e6e6db..ba6e146dca 100644 --- a/gateway/run_adapters.py +++ b/gateway/run_adapters.py @@ -38,11 +38,9 @@ class GatewayAdapterLifecycleMixin: @staticmethod async def _wait_or_detach(task: "asyncio.Future", timeout: float) -> bool: - """Wait up to ``timeout`` for ``task``; on deadline (or our own cancellation) detach it. - - Not ``asyncio.wait_for``: that WAITS for the cancelled child to exit, so a connect()/close() - that swallows ``CancelledError`` blocks recovery forever. Returns True if it finished in time. - """ + """Wait up to ``timeout`` for ``task``; on deadline (or our own cancellation) detach it. Not + ``asyncio.wait_for``: that WAITS for the cancelled child, so a connect()/close() swallowing + ``CancelledError`` blocks recovery forever. True if it finished in time.""" try: done, _pending = await asyncio.wait({task}, timeout=timeout) except asyncio.CancelledError: @@ -67,11 +65,8 @@ class GatewayAdapterLifecycleMixin: return True async def _safe_adapter_disconnect(self, adapter, platform) -> None: - """Call adapter.disconnect() defensively (bounded, never raises). - - After a failed connect() partial resources (ClientSession, poll tasks, subprocesses) would - otherwise leak; must tolerate partial-init state. - """ + """Call adapter.disconnect() defensively (bounded, never raises, tolerates partial-init state): + after a failed connect() partial resources (ClientSession, poll tasks, subprocesses) leak.""" timeout = self._adapter_disconnect_timeout_secs() label = platform.value if platform is not None else "adapter" with _log_suppressed(logging.DEBUG, "Defensive %s disconnect after failed connect raised: %s", label): @@ -82,18 +77,14 @@ class GatewayAdapterLifecycleMixin: ) async def _bounded_adapter_teardown(self, adapter, platform, *, profile: Optional[str] = None) -> None: - """Tear down one adapter on the shutdown path with bounded awaits (never raises). - - Unbounded, a half-dead transport stalls shutdown past systemd's ``TimeoutStopSec``; the - SIGKILL skips ``atexit`` PID-file cleanup and the next start dies with "PID file race lost". - """ + """Tear down one adapter on the shutdown path with bounded awaits (never raises). Unbounded, + a half-dead transport stalls past systemd's ``TimeoutStopSec``; the SIGKILL skips ``atexit`` + PID-file cleanup and the next start dies with "PID file race lost".""" timeout = self._adapter_disconnect_timeout_secs() suffix = f" (profile: {profile})" if profile else "" started_at = time.monotonic() try: - if not await self._await_adapter_cleanup_with_timeout( - adapter.cancel_background_tasks(), timeout - ): + if not await self._await_adapter_cleanup_with_timeout(adapter.cancel_background_tasks(), timeout): logger.warning( "✗ %s background-task cancel timed out after %.1fs - forcing continue%s", platform.value, timeout, suffix, @@ -177,11 +168,9 @@ class GatewayAdapterLifecycleMixin: await self.hooks.emit(event_name, ctx) async def _handle_adapter_fatal_error(self, adapter: BasePlatformAdapter) -> None: - """React to an adapter failure after startup (retryable → background reconnect queue). - - Detached task: the notification arrives on the failing adapter's own polling task, which - the handler's disconnect can cancel mid-flight, stranding the platform half-handled. - """ + """React to an adapter failure after startup (retryable → background reconnect queue). Runs + detached: the notification arrives on the failing adapter's own polling task, which the + handler's disconnect can cancel mid-flight, stranding the platform half-handled.""" tasks = getattr(self, "_fatal_handler_tasks", None) if tasks is None: tasks = self._fatal_handler_tasks = set() @@ -363,16 +352,12 @@ class GatewayAdapterLifecycleMixin: def _spawn_supervised( self, coro_factory, name, *, restart=True, _attempt=0, on_spawn=None, on_give_up=None ): - """Launch a long-lived background task with task-level supervision. - - Catches outer-loop/pre-try exceptions a bare ``create_task`` drops silently. Restarts with - capped backoff up to ``_MAX_SUPERVISED_RESTARTS`` rapid failures; the counter resets after - ``_SUPERVISED_HEALTHY_SECS`` of healthy running. Each spawn uses a fresh ``Context`` (an - inherited delegated-child marker would make the Kanban dispatcher reject its own writes). - ``on_spawn`` fires on EVERY spawn incl. respawns — callers tracking the handle elsewhere - MUST pass it or a respawn leaves a stale handle and a SECOND watcher. ``on_give_up(name)`` - fires when the restart budget is spent. - """ + """Launch a long-lived supervised background task: exceptions a bare ``create_task`` drops + are logged, and it respawns with capped backoff up to ``_MAX_SUPERVISED_RESTARTS`` rapid + failures (counter resets after ``_SUPERVISED_HEALTHY_SECS`` healthy). Fresh ``Context`` per + spawn (an inherited delegated-child marker would make the Kanban dispatcher reject its own + writes). ``on_spawn`` fires on EVERY spawn incl. respawns — handle trackers MUST pass it or a + respawn leaves a stale handle and a SECOND watcher; ``on_give_up(name)`` fires at budget end.""" # Spawn timestamp lets ``_done`` tell a rapid crash-loop from a healthy-run-then-crash. _started = time.monotonic() @@ -434,11 +419,9 @@ class GatewayAdapterLifecycleMixin: return task async def _handoff_watcher(self, interval: float = 2.0, drain_timeout: float = 30.0) -> None: - """Process pending CLI→gateway session handoffs from ``state.db``. - - Claim atomically (pending → running), re-bind the home channel to the CLI session_id, - dispatch a synthetic ``MessageEvent``, mark ``completed``/``failed``. - """ + """Process pending CLI→gateway session handoffs from ``state.db``: claim atomically (pending + → running), re-bind the home channel to the CLI session_id, dispatch a synthetic event, mark + ``completed``/``failed``.""" from gateway.run import _async_profile_runtime_scope, _handoff_watch_scopes, _reclaim_stale await asyncio.sleep(5) # let platforms connect before dispatching through them @@ -470,12 +453,9 @@ class GatewayAdapterLifecycleMixin: inflight.pop(session_id, None) async def _tick(profile_name: Optional[str] = None) -> None: - """One poll of the CURRENTLY-SCOPED store; ``profile_name`` (None = root) routes - delivery to that profile's OWN adapter. - - A closure, not a method: tests bind ``_handoff_watcher`` onto a ``SimpleNamespace`` - with only ``_session_db``/``_running``/``_process_handoff``. - """ + """One poll of the CURRENTLY-SCOPED store; ``profile_name`` (None = root) routes delivery + to that profile's OWN adapter. A closure, not a method: tests bind ``_handoff_watcher`` onto + a ``SimpleNamespace`` with only ``_session_db``/``_running``/``_process_handoff``.""" session_db = getattr(self, "_session_db", None) if session_db is None: return @@ -637,8 +617,7 @@ class GatewayAdapterLifecycleMixin: queued_for / 3600.0, info.get("attempts", 0), ) self._update_platform_runtime_status( - platform.value, platform_state="retrying", needs_attention=True, - retrying_since=retrying_since_iso, + platform.value, platform_state="retrying", needs_attention=True, retrying_since=retrying_since_iso ) def _mark_platform_fatal(self, status_key: str, adapter) -> None: @@ -763,9 +742,7 @@ class GatewayAdapterLifecycleMixin: try: self._schedule_resume_pending_sessions(platform=platform) except Exception: - logger.debug( - "resume-pending reschedule after %s reconnect failed", platform.value, exc_info=True - ) + logger.debug("resume-pending reschedule after %s reconnect failed", platform.value, exc_info=True) async def _cancel_secondary_profile_reconnect_tasks(self) -> None: """Cancel profile-scoped reconnects before tearing down their registry, so a reconnect @@ -794,10 +771,8 @@ class GatewayAdapterLifecycleMixin: async def _start_secondary_profile_adapters(self) -> int: """Bring up adapters for every non-active profile (multiplex only); returns connected count. - - Each profile connects under its own HERMES_HOME + secret scope; credential/listener - collisions are refused here — the only point seeing every profile's credentials together. - """ + Each profile connects under its own HERMES_HOME + secret scope; credential/listener collisions + are refused here — the only point seeing every profile's credentials together.""" from gateway.run import ( MultiplexConfigError, SecondaryPortBindingConfigError, _multiplex_profile_homes ) @@ -825,9 +800,7 @@ class GatewayAdapterLifecycleMixin: except MultiplexConfigError: raise except Exception as e: - logger.error( - "Failed to start adapters for profile '%s': %s", profile_name, e, exc_info=True - ) + logger.error("Failed to start adapters for profile '%s': %s", profile_name, e, exc_info=True) self._record_served_profiles(active, profile_homes) return connected @@ -863,11 +836,9 @@ class GatewayAdapterLifecycleMixin: write_runtime_status(served_profiles=served) async def _load_secondary_profile_config(self, profile_name: str, profile_home: "Path"): - """Hydrate + enter ``profile_home``'s scope once; return its gateway config. - - Raises ``MultiplexConfigError`` (open dm/group policy) or ``SecondaryPortBindingConfigError`` - (the default profile owns the single shared HTTP listener). - """ + """Hydrate + enter ``profile_home``'s scope once; return its gateway config. Raises + ``MultiplexConfigError`` (open dm/group policy) or ``SecondaryPortBindingConfigError`` (the + default profile owns the single shared HTTP listener).""" from gateway.run import ( MultiplexConfigError, SecondaryPortBindingConfigError, _load_gateway_runtime_config, _own_policy_open_startup_violation, _profile_runtime_scope, @@ -897,8 +868,7 @@ class GatewayAdapterLifecycleMixin: raise MultiplexConfigError( f"Profile '{profile_name}' enables {violation}. " "Enable GATEWAY_ALLOW_ALL_USERS or the platform allow-all flag " - "for that profile, or change dm_policy/group_policy away from " - "'open'." + "for that profile, or change dm_policy/group_policy away from 'open'." ) port_binding_platforms = sorted( @@ -1074,11 +1044,9 @@ class GatewayAdapterLifecycleMixin: adapter._hermes_profile_name = profile_name async def _secondary_reconnect_attempt(self, profile_name: str, platform: Platform): - """One scoped attempt to rebuild+connect a secondary adapter → ``(adapter, success)``. - - ``(None, None)`` = give up for good (disabled, credential removed, adapter unavailable). - Caller tears down a RETURNED adapter; one whose configure/connect raised is torn down here. - """ + """One scoped attempt to rebuild+connect a secondary adapter → ``(adapter, success)``; + ``(None, None)`` = give up for good (disabled, credential removed, adapter unavailable). Caller + tears down a RETURNED adapter; one whose configure/connect raised is torn down here.""" from gateway.run import _platform_has_bot_credential, _profile_runtime_scope # Lazy + per-attempt: keeps test monkeypatches on these modules live. from hermes_cli.profiles import get_profile_dir @@ -1162,8 +1130,7 @@ class GatewayAdapterLifecycleMixin: attempts += 1 backoff = _reconnect_backoff(attempts) logger.info( - "Secondary %s reconnect retry in %ds (profile: %s)", platform.value, backoff, - profile_name, + "Secondary %s reconnect retry in %ds (profile: %s)", platform.value, backoff, profile_name ) await asyncio.sleep(backoff) finally: diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py index caaeed1463..df42a624c3 100644 --- a/gateway/run_notifications.py +++ b/gateway/run_notifications.py @@ -245,31 +245,24 @@ class GatewayNotificationsMixin: if switched is None: logger.warning( "Async-delegation completion could not bind routing key %s to " - "owning session %s; dropping injection.", - session_entry.session_key, target_session_id, + "owning session %s; dropping injection.", session_entry.session_key, target_session_id, ) return None logger.info( - "Pinned async-delegation completion to owning session %s " - "(was %s) for routing key %s (#57498)", + "Pinned async-delegation completion to owning session %s (was %s) for routing key %s (#57498)", target_session_id, prior_session_id, session_entry.session_key, ) return switched async def _deliver_media_from_response( - self, response: str, event: MessageEvent, adapter, - thread_metadata: Optional[Dict[str, Any]] = None, + self, response: str, event: MessageEvent, adapter, thread_metadata: Optional[Dict[str, Any]] = None ) -> None: - """Extract explicit MEDIA: tags from an already-streamed response and deliver them. - - The text is already delivered; this only handles file attachments. Unlike the non-streaming - path in ``gateway/platforms/base.py`` this rescan is EXPLICIT-ONLY: a bare local path in a - streamed reply was either shown as text or is stale inspected content, and promoting it sent - files the model never asked to deliver. MEDIA tags are NOT deduped against prior turns: a - MEDIA: directive in the final reply is a deliberate attach (incl. user-requested resends); - stale auto-appended tags are deduped upstream (_collect_auto_append_media_tags). - """ + """Deliver explicit MEDIA: tags from an already-streamed response (text already delivered). + EXPLICIT-ONLY, unlike the non-streaming path in ``gateway/platforms/base.py``: a bare local + path in a streamed reply is shown text or stale inspected content, and promoting it sent + files the model never asked for. MEDIA tags are NOT deduped against prior turns (a final-reply + directive is a deliberate attach); stale auto-appended tags are deduped upstream.""" from urllib.parse import quote as _quote with _log_suppressed(logging.WARNING, "Post-stream media extraction failed: %s"): # Capture [[as_document]] before extract_media strips it: image files then go through @@ -378,8 +371,7 @@ class GatewayNotificationsMixin: from gateway.run import _hermes_home return cls._UpdatePaths( pending=_hermes_home / ".update_pending.json", - claimed=_hermes_home / ".update_pending.claimed.json", - output=_hermes_home / ".update_output.txt", + claimed=_hermes_home / ".update_pending.claimed.json", output=_hermes_home / ".update_output.txt", exit_code=_hermes_home / ".update_exit_code", prompt=_hermes_home / ".update_prompt.json", response=_hermes_home / ".update_response", ) @@ -664,8 +656,7 @@ class GatewayNotificationsMixin: platform_cfg = self.config.platforms.get(platform) if platform_cfg is not None and not platform_cfg.gateway_restart_notification: logger.info( - "Restart notification suppressed: %s has gateway_restart_notification=false", - platform_str, + "Restart notification suppressed: %s has gateway_restart_notification=false", platform_str ) return None @@ -833,8 +824,7 @@ class GatewayNotificationsMixin: try: platform = Platform(platform_name) - # Reject arbitrary strings (dynamic pseudo-members): built-ins are always valid, plugin - # platforms must be registered in the platform registry. + # Reject dynamic pseudo-members: plugin platforms must be registered. if platform.value not in _BUILTIN_PLATFORM_VALUES: try: from gateway.platform_registry import platform_registry @@ -851,15 +841,12 @@ class GatewayNotificationsMixin: scope_id = _opt("scope_id") if scope_id is None and chat_type not in ("dm", "thread"): - # Reconstructed (non-persisted) source for a scoped chat with no scope discriminator: a - # relay connector's fail-closed tenant guard may decline the reply unless user_id resolves - # it. Don't fail — DMs/author-bound chats still route and native adapters need no - # scope_id — but warn so a post-restart egress decline isn't silent. + # Reconstructed scoped-chat source without scope_id: a relay connector's tenant guard may + # decline the reply. Warn, don't fail (native adapters need no scope_id). logger.warning( "Synthetic event source for %s chat=%s (%s) reconstructed " "without scope_id; scoped relay egress may be declined by " - "the connector's tenant guard (user_id fallback only).", - platform_name, chat_id, chat_type, + "the connector's tenant guard (user_id fallback only).", platform_name, chat_id, chat_type, ) return SessionSource( platform=platform, chat_id=chat_id, chat_type=chat_type, thread_id=_opt("thread_id"), @@ -916,15 +903,9 @@ class GatewayNotificationsMixin: return False def _resolve_injection_adapter(self, platform_name: str): - """Adapter for a synthetic-event platform: transport resolver first, literal scan as fallback. - - Alias-aware resolution (relay-plane): one adapter under Platform.RELAY fronts N logical - platforms, so a literal ``p.value == platform_name`` scan misses "slack" and drops the - completion as "no gateway route". Native adapter wins; relay is eligible only when it - advertises fronting the logical platform. The legacy literal scan is still correct for - native adapters and keeps minimal runner stubs (tests) / exotic platform strings working - when the resolver can't run. - """ + """Adapter for a synthetic-event platform: alias-aware transport resolver first (one + Platform.RELAY adapter fronts N logical platforms; native wins), literal ``p.value`` scan as + fallback for minimal runner stubs / exotic platform strings when the resolver can't run.""" from gateway.run import resolve_delivery_transport try: _transport = resolve_delivery_transport(Platform(platform_name), self.config, self.adapters) @@ -1042,15 +1023,10 @@ class GatewayNotificationsMixin: return seen async def _classify_completion_target(self, parent_session_id: str) -> str: - """Classify an async-completion delivery target before adapter acceptance. - - - ``"deliver"``: spawning session live (or compression-rotated with a live continuation); - proves deliverability only, the resolver still retargets. - - ``"terminal"``: parent gone for good (unknown / explicit user boundary like /new); drop - the durable row rather than falsely ack or replay forever. - - ``"retry"``: transient uncertainty (DB unavailable, rotation mid-flight); release the - claim for a later consumer; the attempt cap bounds churn. - """ + """Classify an async-completion target before adapter acceptance: ``"deliver"`` (spawning + session live or compression-rotated with a live continuation; the resolver still retargets), + ``"terminal"`` (parent gone for good — unknown / user boundary like /new; drop the durable row + rather than falsely ack), ``"retry"`` (DB unavailable / rotation mid-flight; release the claim).""" from gateway.run import _USER_BOUNDARY_END_REASONS session_db = getattr(self, "_session_db", None) if session_db is None: @@ -1139,8 +1115,7 @@ class GatewayNotificationsMixin: logger.warning( "Background process %s completion targets " "permanently-gone session %s (user boundary such as " - "/new); dropping notification (output remains " - "available via process(action='log')).", + "/new); dropping notification (output remains available via process(action='log')).", evt.get("session_id") or "", parent_session_id, ) claim.proceed = False @@ -1309,12 +1284,10 @@ class GatewayNotificationsMixin: # Some unit tests construct GatewayRunner with object.__new__. Keep the # batching seam lazy so those focused lifecycle tests remain valid. for attr, default in ( - ("_completion_notification_batches", dict), - ("_completion_notification_batch_tasks", dict), + ("_completion_notification_batches", dict), ("_completion_notification_batch_tasks", dict), ("_completion_notification_batch_flush_tasks", set), ("_completion_notification_batch_window", lambda: 0.1), - ("_completion_notification_batches_stopping", lambda: False), - ("_background_tasks", set), + ("_completion_notification_batches_stopping", lambda: False), ("_background_tasks", set), ): if not hasattr(self, attr): setattr(self, attr, default()) @@ -1360,14 +1333,10 @@ class GatewayNotificationsMixin: logger.debug(fail_msg, exc_info=True) async def _deliver_async_delegation_group(self, group: list[dict]) -> Optional[bool]: - """Deliver a same-session batch of async completions as ONE turn. - - Single-event groups ride the per-event path. Multi-event groups deliver the primary via - ``_deliver_completion_notification`` with consolidated text of every sibling THIS runner - claimed; sibling claims are acked only after adapter acceptance, and siblings claimed by - another consumer are excluded (no double delivery). Returns True after acceptance, False - to requeue the group, None when nothing is deliverable here (retry siblings requeued). - """ + """Deliver a same-session batch of async completions as ONE turn: the primary carries the + consolidated text of every sibling THIS runner claimed (siblings owned elsewhere are excluded; + their claims are acked only after adapter acceptance). True after acceptance, False to requeue + the group, None when nothing is deliverable here (retry siblings requeued).""" from gateway.run import _format_gateway_process_notification from tools.process_registry import process_registry as _pr deliverable: list[tuple[dict, str]] = [] @@ -1547,13 +1516,10 @@ class GatewayNotificationsMixin: ) async def _run_process_watcher(self, watcher: dict) -> None: - """Periodically check a background process and push updates to the user. - - Runs as an asyncio task. Stays silent when nothing changed. Auto-removes when the process - exits or is killed. Notification mode (``display.background_process_notifications``): - concise (default, one-line; failures append output tail) / all (running updates + final - raw output) / result (final raw only) / error (final raw only if exit != 0) / off. - """ + """Poll a background process and push updates until it exits. Mode + (``display.background_process_notifications``): concise (default one-liner; failures append + the output tail) / all (running updates + final raw) / result (final raw) / error (final raw + if exit != 0) / off.""" from tools.process_registry import format_process_notification, process_registry session_id = watcher["session_id"] interval = watcher["check_interval"] diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index 6a7263d174..90f65723f2 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -385,13 +385,10 @@ class GatewayShutdownMixin: return self.adapters.get(Platform.RELAY) async def _scale_to_zero_watcher(self, interval: float = 30.0) -> None: - """Watch for idle, drive the relay dormant, then self-suspend the machine. - - On sustained idle: status `draining` (NOT _running=False), relay go_dormant() (socket close, - NOT disconnect()), no mark_resume_pending (suspend preserves RAM), THEN suspend via the flaps - socket. The gateway owns the suspend because Fly autostop sees only INBOUND connections and - would freeze mid-job or before the relay flip. Off-Fly the watcher does not quiesce at all. - """ + """Watch for idle, drive the relay dormant, then self-suspend. On sustained idle: status + `draining` (NOT _running=False), relay go_dormant() (socket close, NOT disconnect()), no + mark_resume_pending (suspend preserves RAM), THEN suspend via the flaps socket — Fly autostop + sees only INBOUND connections and would freeze mid-job. Off-Fly: no quiesce at all.""" await asyncio.sleep(min(interval, 30.0)) # let startup settle while self._running: try: @@ -473,8 +470,7 @@ class GatewayShutdownMixin: self._external_drain_active = True logger.info( "External drain ENGAGED (.drain_request.json present) — refusing " - "new turns; %d in-flight turn(s) will finish. Process stays up.", - self._active_work_count(), + "new turns; %d in-flight turn(s) will finish. Process stays up.", self._active_work_count(), ) # Flip persisted lifecycle state so /api/status.gateway_busy / gateway_drainable track the # drain; active_agents is preserved (read-merge keeps the live count), only state changes. @@ -502,9 +498,8 @@ class GatewayShutdownMixin: from gateway.drain_control import drain_requested while self._running: try: - # drain_requested() does a synchronous read_text() on the marker file: at 1s cadence - # that is a blocking disk read on the event loop ~86k times/day, and under host I/O - # pressure one read can stall 30s+ and take every platform heartbeat down. Off-thread it. + # Off-thread: a synchronous marker read at 1s cadence can stall 30s+ under host I/O + # pressure and take every platform heartbeat down. if await asyncio.to_thread(drain_requested): self._enter_external_drain() # API and cron work live outside messaging's _running_agents map; refresh the @@ -1085,9 +1080,8 @@ class GatewayShutdownMixin: ) watcher_env = GatewayShutdownMixin._restart_watcher_env() project_root = Path(__file__).resolve().parent.parent - # Console python under CREATE_NO_WINDOW owns one hidden console inherited by the restart - # child, so nothing flashes. Do NOT swap in pythonw.exe — a console-less watcher forces - # every console-subsystem descendant to allocate a visible conhost. + # Console python under CREATE_NO_WINDOW: nothing flashes. NOT pythonw.exe — a console-less + # watcher makes every console-subsystem descendant allocate a visible conhost. watcher_python = sys.executable venv_dir = Path(watcher_env.get("VIRTUAL_ENV") or project_root / "venv") site_packages = venv_dir / "Lib" / "site-packages" @@ -1102,10 +1096,8 @@ class GatewayShutdownMixin: str(current_pid), str(restart_after_s), *hermes_cmd, "gateway", "restart", ] popen_kwargs = dict(stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=watcher_env) - # The watcher must break away from the parent CLI's job object (Desktop wrappers, Windows - # Terminal, schtasks) or it is reaped when the CLI exits. A job without - # JOB_OBJECT_LIMIT_BREAKAWAY_OK rejects CREATE_BREAKAWAY_FROM_JOB (OSError); retry once - # without the bit, preserving argv and the scrubbed env. + # Break away from the parent CLI's job object or be reaped when the CLI exits; a job without + # BREAKAWAY_OK rejects CREATE_BREAKAWAY_FROM_JOB (OSError) — retry once without the bit. try: subprocess.Popen(watcher_argv, **popen_kwargs, **windows_detach_popen_kwargs()) except OSError: @@ -1406,8 +1398,7 @@ class GatewayShutdownMixin: if _cron_at_start and _cron_timeout > timeout: logger.info( "Shutdown drain: %d in-flight cron job(s) — waiting up to " - "%.0fs for them (cron_drain_timeout=%.0fs, " - "restart_drain_timeout=%.0fs)", + "%.0fs for them (cron_drain_timeout=%.0fs, restart_drain_timeout=%.0fs)", _cron_at_start, _cron_timeout, _cron_drain_cfg, timeout, ) _drain_started_at = time.monotonic() @@ -1415,8 +1406,7 @@ class GatewayShutdownMixin: ctx.drain_elapsed = time.monotonic() - _drain_started_at logger.info( "Shutdown phase: drain done at +%.2fs (drain took %.2fs, " - "timed_out=%s, active_at_start=%d, active_now=%d, " - "cron_at_start=%d, cron_now=%d, " + "timed_out=%s, active_at_start=%d, active_now=%d, cron_at_start=%d, cron_now=%d, " "api_at_start=%d, api_now=%d, " "deferred_at_start=%d, deferred_now=%d)", ctx.elapsed(), ctx.drain_elapsed, ctx.timed_out, len(ctx.active_agents), self._running_agent_count(), _cron_at_start, @@ -1439,15 +1429,12 @@ class GatewayShutdownMixin: from gateway.run import GatewayRunner logger.warning( "Gateway drain timed out after %.1fs with %d active agent(s), " - "%d in-flight cron job(s), %d api_server run(s), and " - "%d deferred agent worker(s); " + "%d in-flight cron job(s), %d api_server run(s), and %d deferred agent worker(s); " "interrupting remaining work.", ctx.drain_elapsed, self._running_agent_count(), self._active_cron_job_count(), self._active_api_run_count(), ctx.deferred_count(), ) - # Mark forcibly-interrupted sessions resume_pending BEFORE interrupting so the next message - # auto-resumes instead of becoming a fresh session via suspend_recently_active(); stuck - # sessions still escalate via ``.restart_failure_counts`` (suspended=True wins). Uses the - # CURRENT _running_agents, not the drain-start snapshot, so cleanly finished sessions get no note. + # Mark resume_pending BEFORE interrupting so the next message auto-resumes (stuck sessions + # still escalate via .restart_failure_counts). CURRENT _running_agents, not the drain snapshot. await GatewayRunner._mark_running_sessions_resume_pending(self, "mark_resume_pending") reason = GatewayRunner._shutdown_interrupt_reason(self) self._interrupt_running_agents(reason) @@ -1568,11 +1555,9 @@ class GatewayShutdownMixin: def _stop_quiesce_and_close_session_dbs(self, timeout: float, ctx: "GatewayShutdownMixin._StopContext") -> None: """Quiesce the executor, then close SessionDB handles only if no worker is still live.""" from gateway.run import GatewayRunner, _EXECUTOR_QUIESCE_TIMEOUT - # Quiesce the thread pool BEFORE closing session DBs: otherwise a coroutine can mint a fresh - # pool (``_executor_closing`` still False) or an already-started run_in_executor future keeps - # writing after ``SessionDB.close()`` checkpointed the WAL — the late write reopens the handle - # and mints a split WAL generation (close-time corruption). Bounded and clamped to what is - # left of the watchdog leash minus a second for the close itself. + # Quiesce the thread pool BEFORE closing session DBs: a late executor write after + # SessionDB.close() checkpointed the WAL reopens the handle and splits the WAL generation + # (close-time corruption). Clamped to the remaining watchdog leash minus 1s for the close. _exec_quiesce_budget = max( 0.0, min(_EXECUTOR_QUIESCE_TIMEOUT, resolve_shutdown_watchdog_delay(timeout) - ctx.elapsed() - 1.0), ) @@ -1637,8 +1622,7 @@ class GatewayShutdownMixin: else: logger.info( "Skipping .clean_shutdown marker — drain timed out with " - "interrupted agents; next startup will suspend recently " - "active sessions." + "interrupted agents; next startup will suspend recently active sessions." ) # Stuck-loop detection: the counter increments for sessions active at each restart; at # the threshold (3 consecutive) the next startup auto-suspends the session. @@ -1661,14 +1645,12 @@ class GatewayShutdownMixin: self._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE self._exit_reason = self._exit_reason or "Gateway restart requested" self._draining = False - # Terminal gateway_state: "stopped" by default, "running" on an UNEXPECTED signal (s6 SIGTERM - # on docker restart, OOM-kill) — container_boot.py only auto-starts gateways last seen - # "running". Operator stops write a planned-stop marker BEFORE signalling; restarts persist "stopped". + # Terminal gateway_state: "stopped", or "running" on an UNEXPECTED signal (docker restart, + # OOM) — container_boot.py only auto-starts gateways last seen "running". if getattr(self, "_signal_initiated_shutdown", False) and not self._restart_requested: logger.info( "Gateway stopped by an unexpected signal — persisting " - "gateway_state=running so container_boot auto-starts on " - "the next boot (issue #42675)" + "gateway_state=running so container_boot auto-starts on the next boot (issue #42675)" ) self._update_runtime_status("running", self._exit_reason) else: @@ -1694,9 +1676,8 @@ class GatewayShutdownMixin: async def _stop_impl(self) -> None: """Run every ``_stop_*`` phase under the thread-based shutdown watchdog.""" from gateway.run import GatewayRunner - # Thread-based watchdog (asyncio timeouts cannot recover a frozen loop): if teardown never - # finishes within drain+grace it dumps faulthandler stacks and os._exit so KeepAlive/systemd - # can revive. Skipped under pytest (no delayed hard-exit in the worker). + # Thread-based watchdog (asyncio timeouts cannot recover a frozen loop): dumps stacks and + # os._exit past drain+grace so the service manager revives us. Skipped under pytest. _watchdog_done = threading.Event() self._shutdown_watchdog_done = _watchdog_done # Shutdown-path tests and third-party runner doubles may only implement the older @@ -1706,10 +1687,8 @@ class GatewayShutdownMixin: ) if not os.environ.get("PYTEST_CURRENT_TEST"): arm_shutdown_watchdog( - resolve_shutdown_watchdog_delay(self._restart_drain_timeout), - done_event=_watchdog_done, - snapshot_fn=lambda: GatewayRunner._shutdown_watchdog_snapshot(self, ctx), - exit_code=1, + resolve_shutdown_watchdog_delay(self._restart_drain_timeout), done_event=_watchdog_done, + snapshot_fn=lambda: GatewayRunner._shutdown_watchdog_snapshot(self, ctx), exit_code=1, ) try: await GatewayRunner._stop_begin_teardown(self, ctx) @@ -1725,8 +1704,7 @@ class GatewayShutdownMixin: _watchdog_done.set() async def stop( - self, *, restart: bool = False, detached_restart: bool = False, - service_restart: bool = False, + self, *, restart: bool = False, detached_restart: bool = False, service_restart: bool = False ) -> None: """Stop the gateway and disconnect all adapters.""" from gateway.run import GatewayRunner diff --git a/gateway/run_startup.py b/gateway/run_startup.py index d8d3a1d2d0..086b87dcd3 100644 --- a/gateway/run_startup.py +++ b/gateway/run_startup.py @@ -130,8 +130,7 @@ class GatewayStartupMixin: await self._wait_bounded_or_release( {task}, timeout, "Turn-machinery warm-up still running after %.0fs; opening inbound gate anyway — the " - "first turn may see lazily initialized machinery (#99373). Warm-up continues in the " - "background.", + "first turn may see lazily initialized machinery (#99373). Warm-up continues in the background.", "boot turn-machinery warm-up failed after gate release", level=logging.DEBUG, ) @@ -158,13 +157,10 @@ class GatewayStartupMixin: return done async def _finish_startup_restore(self) -> None: - """Wait (BOUNDED) for startup auto-resume, then release + drain inbound. - - Bounded by ``_startup_restore_drain_timeout_secs`` so one pathological boot-resume turn - cannot hold the gate shut for every channel; on timeout the gate opens and resume turns - finish in the background (NOT cancelled). Safe because ``_schedule_resume_pending_sessions`` - claims each ``_running_agents`` slot SYNCHRONOUSLY first, so drained inbound queues behind. - """ + """Wait (BOUNDED by ``_startup_restore_drain_timeout_secs``) for startup auto-resume, then + release + drain inbound. On timeout the gate opens and resume turns finish in the background + (NOT cancelled) — safe because ``_schedule_resume_pending_sessions`` claims each + ``_running_agents`` slot SYNCHRONOUSLY first, so drained inbound queues behind.""" from gateway.run import _startup_restore_drain_timeout_secs tasks = list(getattr(self, "_startup_restore_tasks", []) or []) if tasks: @@ -203,15 +199,11 @@ class GatewayStartupMixin: return _report async def _await_startup_boot_sends(self, *, planned_restart_notification_pending: bool) -> None: - """Run boot-path sends without letting them pin the inbound restore gate. - - Awaiting the sends inline before the gate releases lets one Telegram flood-control sleep - freeze inbound on every platform. Same bounded ``asyncio.wait`` as the resume gate: on - timeout return and let the sends finish in the background (not cancelled). The ledger claim - + ``resume_pending`` clear run INLINE before the send task exists (bounded DB work): deferring - it let a hung notification expire the gate with zero rows claimed, so answered turns were - replayed AND redelivered. - """ + """Run boot-path sends without letting them pin the inbound restore gate (one Telegram + flood-control sleep must not freeze inbound on every platform): same bounded wait as the + resume gate, sends finish in the background on timeout. The ledger claim + ``resume_pending`` + clear run INLINE before the send task exists — deferring it let a hung notification expire the + gate with zero rows claimed, so answered turns were replayed AND redelivered.""" from gateway.run import _clear_planned_restart_notification, _startup_restore_drain_timeout_secs claimed = await self._claim_pending_obligations() @@ -259,15 +251,11 @@ class GatewayStartupMixin: return sendable async def _claim_pending_obligations(self) -> list: - """Claim recoverable delivery-ledger rows and clear their ``resume_pending`` flags. - - Pure DB work, no sends. Must run INLINE at startup BEFORE ``_schedule_resume_pending_sessions`` - and before the abandonable boot-send task exists: these sessions already produced their - answer, so clearing ``resume_pending`` here stops the resume path from re-running (and - re-paying for) the turn however long the sends take. Rows that were mid-send or previously - rejected carry a visible recovered-reply marker so a possible duplicate is labeled, never - silent (gateway/delivery_ledger.py). Returns the claimed rows for redelivery. - """ + """Claim recoverable delivery-ledger rows and clear their ``resume_pending`` flags (pure DB + work, no sends). Must run INLINE BEFORE ``_schedule_resume_pending_sessions`` and the + abandonable boot-send task: these sessions already produced their answer, so the resume path + must not re-run (re-pay for) the turn however long the sends take. Mid-send / rejected rows + carry a visible recovered-reply marker (gateway/delivery_ledger.py). Returns the rows.""" try: from gateway.delivery_ledger import ledger_enabled, sweep_recoverable if not await asyncio.to_thread(ledger_enabled): @@ -333,9 +321,7 @@ class GatewayStartupMixin: try: result = await adapter.send(chat_id=row["chat_id"], content=content, metadata=metadata) except Exception as send_err: - logger.warning( - "obligation %s: redelivery send raised: %s", row["obligation_id"], send_err - ) + logger.warning("obligation %s: redelivery send raised: %s", row["obligation_id"], send_err) result = None with _log_suppressed(logging.DEBUG, "delivery ledger update failed", exc_info=True): if result is not None and getattr(result, "success", False): @@ -348,8 +334,7 @@ class GatewayStartupMixin: ) else: await asyncio.to_thread( - mark_failed, row["obligation_id"], - str(getattr(result, "error", "") or "send failed"), + mark_failed, row["obligation_id"], str(getattr(result, "error", "") or "send failed") ) return redelivered @@ -358,9 +343,7 @@ class GatewayStartupMixin: try: platform = Platform(row["platform"]) except Exception: - logger.debug( - "obligation %s: unknown platform %r", row["obligation_id"], row.get("platform"), - ) + logger.debug("obligation %s: unknown platform %r", row["obligation_id"], row.get("platform")) return None if "profile" in row: adapter = self._authorization_adapter(platform, row.get("profile")) @@ -384,12 +367,9 @@ class GatewayStartupMixin: async def _redeliver_failed_obligations_for_platform( self, platform: Platform, *, profile: Optional[str] = None, ) -> int: - """Replay one adapter identity's transient failures after reconnect. - - The startup sweep cannot claim live-owner rows, and an adapter can reconnect without the - process exiting, so ``send_path_degraded`` responses would otherwise stay failed until the - next restart. Claim/clear/send are best-effort and reuse the startup redelivery contract. - """ + """Replay one adapter identity's transient failures after reconnect: the startup sweep cannot + claim live-owner rows, so ``send_path_degraded`` responses would otherwise stay failed until + the next restart. Best-effort; reuses the startup redelivery contract.""" try: from gateway.delivery_ledger import ledger_enabled, sweep_failed_for_runtime if not await asyncio.to_thread(ledger_enabled): @@ -405,9 +385,7 @@ class GatewayStartupMixin: # Clear before any send so the reconnect path cannot both redeliver an # already-produced answer and schedule the same agent turn for resume. - sendable = await self._clear_resume_pending_for_claimed_obligations( - claimed, require_success=True - ) + sendable = await self._clear_resume_pending_for_claimed_obligations(claimed, require_success=True) sendable_ids = {row["obligation_id"] for row in sendable} for row in claimed: if row["obligation_id"] not in sendable_ids: @@ -434,10 +412,8 @@ class GatewayStartupMixin: logger.warning("Failed to enumerate resume-pending sessions: %s", exc) return None - # Defense-3: break the SIGTERM-respawn loop. Only count this boot when there are restart- - # interrupted sessions to resume — a clean boot must not accrue toward the breaker. If too - # many such boots hit the window, skip auto-resume for THIS boot only: the gateway still - # serves inbound; the session stays resume_pending so a real user message can continue it. + # Restart-loop breaker: only boots WITH restart-interrupted sessions count; when tripped, skip + # auto-resume for THIS boot only (inbound still served; sessions stay resume_pending). if candidates: try: from gateway import restart_loop_guard as _rlg @@ -460,20 +436,14 @@ class GatewayStartupMixin: "longer authorized under the current allowlist", session_key, ) except Exception as exc: - logger.warning( - "Skipping auto-resume for %s: authorization check failed: %s", session_key, exc, - ) + logger.warning("Skipping auto-resume for %s: authorization check failed: %s", session_key, exc) return False def _schedule_resume_pending_sessions(self, platform=None) -> int: - """Auto-continue fresh restart-interrupted sessions after startup. - - Synthesizes the next turn once adapters are back online; the event text is empty so the - existing ``_is_resume_pending`` injection path owns the recovery wording. Sessions whose - adapter is not in ``self.adapters`` stay ``resume_pending`` for the reconnect watcher, which - re-calls this scoped to that ``platform`` (a reconnecting platform never touches another's - recoveries); sessions with a running agent are skipped so none is resumed twice. - """ + """Auto-continue fresh restart-interrupted sessions: synthesize an empty-text turn (the + ``_is_resume_pending`` injection path owns the wording). Sessions whose adapter is offline stay + ``resume_pending`` for the reconnect watcher, which re-calls this scoped to that ``platform``; + sessions with a running agent are skipped so none is resumed twice.""" from gateway.run import _AGENT_PENDING_SENTINEL, _auto_continue_freshness_window window = _auto_continue_freshness_window() candidates = self._resume_pending_candidates(platform) @@ -678,24 +648,19 @@ class GatewayStartupMixin: def _open_faulthandler_log(self): """Open (append) ``/gateway_faulthandler.log``, creating the directory.""" from gateway.run import get_hermes_home - log_dir = getattr(self.config, "log_dir", None) or os.path.join( - str(get_hermes_home()), "logs", - ) + log_dir = getattr(self.config, "log_dir", None) or os.path.join(str(get_hermes_home()), "logs") os.makedirs(log_dir, exist_ok=True) return open(os.path.join(log_dir, "gateway_faulthandler.log"), "a", encoding="utf-8") def _start_install_faulthandler(self) -> None: """Enable faulthandler (stderr or a log file) plus the SIGUSR2 stack-dump hook.""" - # Falls back to a log file when sys.stderr is None (Windows VBS / pythonw / detached - # service) — otherwise the gateway would die here and take every adapter offline. + # sys.stderr may be None (Windows VBS / pythonw / detached service): fall back to a log file. try: faulthandler.enable() except (RuntimeError, ValueError, OSError): with _log_suppressed(logging.DEBUG, "faulthandler.enable() unavailable", exc_info=True): faulthandler.enable(file=self._open_faulthandler_log(), all_threads=True) - # Also dump stacks to a file on SIGUSR2 for off-line analysis under a service manager that - # doesn't capture stderr. faulthandler.register()/SIGUSR2 are POSIX-only: skip on Windows - # (faulthandler.enable() above still covers fatal errors). + # SIGUSR2 stack dump to file for service managers that drop stderr; POSIX-only. _sigusr2 = getattr(signal, "SIGUSR2", None) if _sigusr2 is not None and hasattr(faulthandler, "register"): with _log_suppressed(logging.DEBUG, "Could not set up faulthandler file logging", exc_info=True): @@ -711,10 +676,8 @@ class GatewayStartupMixin: self._gateway_loop = None if self._gateway_loop is not None: self._start_loop_liveness_guards(self._gateway_loop) - # Loop confirmed live: the startup-liveness watchdog is done and the loop-liveness - # watchdog (armed above) takes over. Disarm even when loop guards are config-disabled — - # the startup watchdog covers only the pre-loop window. Deliberately inside this branch: - # if the loop isn't live, startup has NOT reached the milestone and it must stay armed. + # Loop live: the loop-liveness watchdog takes over from the startup watchdog. Disarm even + # when loop guards are config-disabled; only inside this branch (no live loop = stay armed). with _log_suppressed(logging.DEBUG, "Startup watchdog disarm failed", exc_info=True): from gateway.startup_watchdog import disarm_startup_watchdog disarm_startup_watchdog() @@ -752,9 +715,7 @@ class GatewayStartupMixin: logger.info("Active profile: %s", _profile) with suppress(Exception): from gateway.status import write_runtime_status - write_runtime_status( - gateway_state="starting", exit_reason=None, clear_profile_platforms=True - ) + write_runtime_status(gateway_state="starting", exit_reason=None, clear_profile_platforms=True) with _log_suppressed(logging.DEBUG, "gateway health OTLP export startup failed", exc_info=True): from hermes_cli.config import load_config from agent.monitoring.gateway_health_export import start_gateway_health_export @@ -910,9 +871,8 @@ class GatewayStartupMixin: if recovered: logger.info("Recovered %s background process(es) from previous run", recovered) - # Recover sessions active when the gateway last exited. Exact durable turn markers cover - # long-running work; the 120s recency heuristic remains as a fallback for turns from older - # versions without markers. SKIP after a clean shutdown — the previous process already drained. + # Recover sessions active at last exit (exact turn markers + 120s recency fallback for + # marker-less older turns). SKIP after a clean exit — the previous process already drained. _clean_marker = _hermes_home / ".clean_shutdown" if _clean_marker.exists(): logger.info("Previous gateway exited cleanly — skipping session suspension") @@ -954,10 +914,8 @@ class GatewayStartupMixin: return True, enabled_platform_count, _multiplex_skipped_platforms, _pending_connects if not platform_config.enabled: continue - # Under multiplexing, a platform may be enabled on the default profile's config.yaml - # while its bot token lives only in a secondary profile's .env. Starting the primary with - # an empty token fails at once and queues a reconnect loop that can never heal; the - # secondary starts its own adapter with the real token, so skip the empty primary. + # Multiplex: a platform enabled in the shared config.yaml may hold its token only in a + # secondary profile's .env; an empty primary would queue a reconnect loop that never heals. if _multiplex_on and not _platform_has_bot_credential(platform, platform_config): logger.info( "Skipping %s on default profile: no bot credential in this " @@ -976,8 +934,7 @@ class GatewayStartupMixin: else: logger.warning( "No adapter for '%s' -- is the plugin installed? " - "(platform is enabled in config.yaml but no plugin registered it)", - platform.value, + "(platform is enabled in config.yaml but no plugin registered it)", platform.value, ) continue # Under multiplexing the default profile needs the same whole-handler runtime scope as @@ -1010,8 +967,7 @@ class GatewayStartupMixin: # restart/shutdown requested mid-connect must cancel still-pending connects, clean up the # ones already completed, and abort startup. _task_map = { - asyncio.ensure_future(_connect_one_startup(p, c, a)): (p, c, a) - for (p, c, a) in _pending_connects + asyncio.ensure_future(_connect_one_startup(p, c, a)): (p, c, a) for (p, c, a) in _pending_connects } _pending_tasks = set(_task_map) while _pending_tasks: @@ -1064,9 +1020,8 @@ class GatewayStartupMixin: continue if outcome == "exception": logger.error("\u2717 %s error: %s", platform.value, exc) - # Same defensive cleanup path for exceptions -- an adapter that raised mid-connect - # may still have a live aiohttp.ClientSession or child subprocess. Unexpected - # exceptions are typically transient -- queue for retry. + # An adapter that raised mid-connect may still hold a ClientSession/subprocess; treat + # unexpected exceptions as transient and queue for retry. await self._safe_adapter_disconnect(adapter, platform) self._startup_queue_transient_failure( platform, adapter, platform_config, str(exc), startup_retryable_errors @@ -1091,10 +1046,8 @@ class GatewayStartupMixin: platform, adapter, platform_config, "failed to connect", startup_retryable_errors ) continue - # A live foreign holder of this bot token is a single-writer ownership conflict, not a - # blip — ``_acquire_platform_lock`` emits it retryable only so a MID-RUN reconnect can - # recover. At startup route it non-retryable: with nothing connected the gateway exits - # 78 instead of sitting alive and deaf in the retry queue. + # A live foreign token holder is an ownership conflict, not a blip: retryable only for + # MID-RUN reconnects; at startup route it non-retryable so the gateway exits 78, not deaf. _retryable = adapter.fatal_error_retryable and not is_global_startup_conflict(adapter.fatal_error_code) self._update_platform_runtime_status( platform.value, platform_state="retrying" if _retryable else "fatal", @@ -1164,10 +1117,8 @@ class GatewayStartupMixin: self._startup_fail_fatal_config(reason) return True if startup_nonretryable_errors: - # Mixed failure mode (some fatal, some transient). Exiting 78 here would let exit-78 - # supervisors take the gateway PERMANENTLY down over a network blip and deny the - # retryable ones their retry. Log the fatal side loudly, then fall through to the - # degraded/retry path: the watcher recovers the retryable; the rest stay parked. + # Mixed (some fatal, some transient): exiting 78 would take the gateway PERMANENTLY down + # over a blip. Log the fatal side loudly and fall through to the degraded/retry path. logger.error( "%d platform(s) fatally misconfigured and parked: %s. " "Staying alive so retryable platforms can recover.", @@ -1238,27 +1189,22 @@ class GatewayStartupMixin: from gateway.run import _planned_restart_notification_pending, _restart_notification_pending await self._start_post_connect_services(connected_count) - # Give freshly connected adapters a brief moment to settle before sending restart/startup - # lifecycle messages; in practice this helps Discord thread deliveries after reconnect. + # Let fresh adapters settle before lifecycle sends (helps Discord thread deliveries). if connected_count > 0: await asyncio.sleep(1.0) - # Capture, before _send_restart_notification() unlinks the marker, whether this process - # booted from a chat-originated /restart. One-shot signal for the /restart redelivery - # guard (_is_stale_restart_redelivery): a missing dedup marker only suppresses a /restart - # when we KNOW we just came out of a restart cycle. + # Before _send_restart_notification() unlinks the marker: did we boot from a chat /restart? + # One-shot signal for _is_stale_restart_redelivery. if _restart_notification_pending(): self._booted_from_restart = True - # Restart notification, home-channel startup notice, and obligation redelivery all call - # adapter.send(). Those sends must not pin the inbound restore gate — a Telegram flood- - # control sleep on this path froze every platform for the full penalty. + # Boot-path adapter.send() calls must not pin the inbound restore gate (a Telegram flood- + # control sleep here once froze every platform). await self._await_startup_boot_sends( planned_restart_notification_pending=_planned_restart_notification_pending(), ) - # Auto-continue fresh sessions interrupted by the previous restart/shutdown. resume_pending - # is cleared by the normal successful-turn path, so a failed auto-resume stays visible on the - # next user message. _await_startup_boot_sends already cleared sessions answered in the ledger. + # Auto-resume restart-interrupted sessions (ledger-answered ones were cleared above); a failed + # auto-resume stays visible on the next user message. self._schedule_resume_pending_sessions() await self._finish_startup_restore() @@ -1266,10 +1212,8 @@ class GatewayStartupMixin: # is broken before losing data. await self._send_session_db_warning_notifications() - # Drain recovered process watchers (crash-recovery checkpoint). Detach the batch atomically: - # reassigning a fresh list takes ownership of exactly the watchers present now, so one - # appended concurrently during the yield below isn't dropped by a clear() on the shared - # list. Yield every 100 to avoid O(n^2) loop blocking when recovering thousands. + # Resume recovered process watchers. Detach the batch atomically (fresh list, not clear(): a + # concurrent append during the yield must not be lost); yield every 100 to keep the loop live. with _log_suppressed(logging.ERROR, "Recovered watcher setup error: %s"): from tools.process_registry import process_registry watchers = process_registry.pending_watchers @@ -1382,8 +1326,7 @@ class GatewayStartupMixin: if _aborted: return True if self._start_handle_no_connections( - connected_count, enabled_platform_count, startup_retryable_errors, - startup_nonretryable_errors, + connected_count, enabled_platform_count, startup_retryable_errors, startup_nonretryable_errors ): return True @@ -1447,9 +1390,8 @@ class GatewayStartupMixin: handoff_config, handoff_adapters = self._handoff_resolve_scope(profile_name) - # Adapter must be live. A relay-fronted gateway registers ONE adapter under Platform.RELAY - # fronting N logical platforms, so a literal adapters.get(discord) misses a deliverable - # platform; resolve_delivery_transport is the alias-aware resolver (native adapter wins). + # Alias-aware transport: a relay-fronted gateway registers ONE Platform.RELAY adapter fronting + # N logical platforms, so a literal adapters.get() would miss a deliverable one. transport = resolve_delivery_transport(platform, handoff_config, handoff_adapters) if not transport: raise RuntimeError(f"platform '{platform_name}' is not active in this gateway") @@ -1468,16 +1410,12 @@ class GatewayStartupMixin: home_chat_id, f"Hermes — {cli_title}", ) except Exception as exc: - logger.debug( - "Handoff: create_handoff_thread raised on %s: %s", platform_name, exc, exc_info=True - ) + logger.debug("Handoff: create_handoff_thread raised on %s: %s", platform_name, exc, exc_info=True) new_thread_id = None effective_thread_id = new_thread_id or (str(home.thread_id) if home.thread_id else None) - # Telegram private-chat DM topics are shaped differently from group/forum threads by the - # inbound adapter: a handoff-created topic in a positive chat_id must use the DM-topic source - # shape (real user id == chat_id, so topic-mode checks and binding persistence match later - # inbound turns), or the synthetic turn binds a `thread` key while replies arrive on `dm`. + # Telegram private-chat DM topics use the DM-topic source shape (user_id == chat_id) so the + # synthetic turn binds the same key later inbound turns arrive on (`dm`, not `thread`). is_telegram_private_chat = ( platform == Platform.TELEGRAM and looks_like_telegram_private_chat_id(home_chat_id) ) @@ -1501,14 +1439,11 @@ class GatewayStartupMixin: ) def _handoff_session_key(self, dest, profile_name: Optional[str]) -> str: - """Build the destination session_key with the adapters' own rules. - - Thread keys omit user_id (thread_sessions_per_user default) so the next message shares it. - The key is namespaced to the queuing profile: a multiplexed gateway would otherwise build - ``agent:main:...`` while the profile's adapter routes inbound on ``agent::...``. - The store resolver is only the root fallback (None when multiplexing is off; old key - unchanged). The isinstance check is load-bearing: a Mock store returns a truthy MagicMock. - """ + """Destination session_key by the adapters' own rules. Thread keys omit user_id so the next + message shares it. Namespaced to the queuing profile (else a multiplexed gateway builds + ``agent:main:...`` while the profile's adapter routes on ``agent::...``); the store + resolver is only the root fallback. The isinstance check is load-bearing: a Mock store returns + a truthy MagicMock.""" platform_cfg = dest.handoff_config.platforms.get(dest.platform) extra = platform_cfg.extra if platform_cfg else {} handoff_profile = profile_name if (profile_name and profile_name != "default") else None @@ -1524,20 +1459,13 @@ class GatewayStartupMixin: logger.debug("Handoff: could not resolve profile namespace", exc_info=True) return build_session_key( dest.source, group_sessions_per_user=extra.get("group_sessions_per_user", True), - thread_sessions_per_user=extra.get("thread_sessions_per_user", False), - profile=handoff_profile, + thread_sessions_per_user=extra.get("thread_sessions_per_user", False), profile=handoff_profile, ) - async def _process_handoff( - self, row: Dict[str, Any], profile_name: Optional[str] = None, - ) -> None: - """Execute one handoff row. Raises on failure (caller marks failed). - - ``profile_name`` (``None`` = root) is the profile whose store queued this handoff. Under - multiplex it is load-bearing: ``self.adapters``/``self.config`` are the primary's (secondaries - live in ``_profile_adapters``), and the session key must be namespaced ``agent::...`` - or it binds a key nobody reads. Passing the name beats re-deriving it from the contextvar. - """ + async def _process_handoff(self, row: Dict[str, Any], profile_name: Optional[str] = None) -> None: + """Execute one handoff row; raises on failure (caller marks failed). ``profile_name`` (None = + root) is the profile whose store queued it — load-bearing under multiplex: secondaries live in + ``_profile_adapters`` and the key must be namespaced ``agent::...`` or nobody reads it.""" cli_session_id = row["id"] dest = await self._handoff_resolve_destination(row, profile_name) session_key = self._handoff_session_key(dest, profile_name) @@ -1570,8 +1498,7 @@ class GatewayStartupMixin: logger.info( "Handoff: dispatching synthetic turn for CLI session %s → %s " "(home=%s, thread=%s, session_key=%s)", - cli_session_id, dest.platform_name, dest.home.chat_id, dest.effective_thread_id, - session_key, + cli_session_id, dest.platform_name, dest.home.chat_id, dest.effective_thread_id, session_key, ) # Dispatch through the runner directly: adapter.handle_message would spawn a background task # and lose error visibility; inline _handle_message keeps success/failure observable. From 10500e1a626b8ed0e408db768a622d493c74495c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:24:34 -0700 Subject: [PATCH 02/11] refactor(gateway): fold wait-or-detach tails, cron-count/interrupt-log/hasattr-clear ladders, connect-abort teardown loop --- gateway/run_adapters.py | 15 ++++++--------- gateway/run_shutdown.py | 37 ++++++++++++++++++------------------- gateway/run_startup.py | 21 ++++++++------------- 3 files changed, 32 insertions(+), 41 deletions(-) diff --git a/gateway/run_adapters.py b/gateway/run_adapters.py index ba6e146dca..9ace375554 100644 --- a/gateway/run_adapters.py +++ b/gateway/run_adapters.py @@ -41,17 +41,14 @@ class GatewayAdapterLifecycleMixin: """Wait up to ``timeout`` for ``task``; on deadline (or our own cancellation) detach it. Not ``asyncio.wait_for``: that WAITS for the cancelled child, so a connect()/close() swallowing ``CancelledError`` blocks recovery forever. True if it finished in time.""" + done: set = set() try: done, _pending = await asyncio.wait({task}, timeout=timeout) - except asyncio.CancelledError: - task.cancel() - task.add_done_callback(consume_detached_task_result) - raise - if task in done: - return True - task.cancel() - task.add_done_callback(consume_detached_task_result) - return False + finally: + if task not in done: # timed out, or our own cancellation + task.cancel() + task.add_done_callback(consume_detached_task_result) + return task in done async def _await_adapter_cleanup_with_timeout(self, awaitable: Awaitable[Any], timeout: float) -> bool: """Await adapter cleanup with a detach-on-deadline bound; True when it completed.""" diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index 90f65723f2..aefc59568c 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -138,11 +138,15 @@ class GatewayShutdownMixin: + self._active_deferred_agent_worker_count() ) + @staticmethod + def _running_cron_job_count() -> int: + from cron.scheduler import get_running_job_ids + return len(get_running_job_ids()) + def _active_cron_job_count(self) -> int: """Cron jobs currently executing — they run outside ``_running_agents``; 0 if cron can't import.""" try: - from cron.scheduler import get_running_job_ids - return len(get_running_job_ids()) + return self._running_cron_job_count() except Exception: return 0 @@ -341,11 +345,7 @@ class GatewayShutdownMixin: logger.debug("scale-to-zero: %s unreadable — staying awake", label, exc_info=True) return busy_sentinel - def _cron_count() -> int: - from cron.scheduler import get_running_job_ids - return len(get_running_job_ids()) - - cron_count = _read_or_awake("cron work count", _cron_count, 1) + cron_count = _read_or_awake("cron work count", self._running_cron_job_count, 1) api_count = _read_or_awake("api work count", lambda: self._api_server_hook("active_agent_work_count"), 1) # An attached dashboard/desktop/TUI client (heartbeat file mtime from the dashboard process) is # inbound activity too — folded into the inbound clock, not a conjunct, so disconnect gets the @@ -623,12 +623,12 @@ class GatewayShutdownMixin: logger.debug("Interrupted running agent for session %s during shutdown", session_key) # API-server / desk turns are adapter-owned and never enter _running_agents, so the loop above # cannot see them even though _drain_active_agents() waited for them. - interrupted_api = self._interrupt_api_server_runs(reason) - if interrupted_api: - logger.debug("Interrupted %d api_server run(s) during shutdown", interrupted_api) - interrupted_deferred = self._interrupt_deferred_agent_workers(reason) - if interrupted_deferred: - logger.debug("Interrupted %d deferred agent worker(s) during shutdown", interrupted_deferred) + for count, what in ( + (self._interrupt_api_server_runs(reason), "api_server run(s)"), + (self._interrupt_deferred_agent_workers(reason), "deferred agent worker(s)"), + ): + if count: + logger.debug("Interrupted %d %s during shutdown", count, what) def _shutdown_interrupt_reason(self) -> str: from gateway.run import _INTERRUPT_REASON_GATEWAY_RESTART, _INTERRUPT_REASON_GATEWAY_SHUTDOWN @@ -844,9 +844,9 @@ class GatewayShutdownMixin: ) continue # Home channels omit ``metadata=`` when empty (adapter doubles may not accept the kwarg). - send_kwargs = {"metadata": metadata} if metadata else {} if await self._send_shutdown_notice( - adapter, str(home.chat_id), msg, "home channel", platform.value, **send_kwargs + adapter, str(home.chat_id), msg, "home channel", platform.value, + **({"metadata": metadata} if metadata else {}), ): notified.add(dedup_key) @@ -1533,12 +1533,11 @@ class GatewayShutdownMixin: # session (never a wholesale dict swap that could lose a concurrent writer's entry). self._running_agents.clear() self._running_agents_ts.clear() - if hasattr(self, "_active_session_leases"): - self._active_session_leases.clear() self._pending_messages.clear() self._pending_approvals.clear() - if hasattr(self, "_busy_ack_ts"): - self._busy_ack_ts.clear() + for _attr in ("_active_session_leases", "_busy_ack_ts"): # absent on bare shutdown-path doubles + if hasattr(self, _attr): + getattr(self, _attr).clear() self._shutdown_event.set() # Global catch-all subprocess kill (safe to repeat): covers the graceful path and # anything respawned since the drain-timeout path's post-interrupt kill. diff --git a/gateway/run_startup.py b/gateway/run_startup.py index 086b87dcd3..00d5c21f6b 100644 --- a/gateway/run_startup.py +++ b/gateway/run_startup.py @@ -23,7 +23,7 @@ from gateway.session import SessionSource, build_session_key from gateway.restart import ( DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, GATEWAY_FATAL_CONFIG_EXIT_CODE, is_global_startup_conflict ) -from gateway.run_shutdown import _log_suppressed +from gateway.run_shutdown import _log_suppressed, _send_error from gateway.shutdown_watchdog import ( DEFAULT_HEARTBEAT_INTERVAL_S, DEFAULT_LOOP_WATCHDOG_INTERVAL_S, DEFAULT_LOOP_WATCHDOG_MAX_STRIKES, DEFAULT_LOOP_WATCHDOG_TIMEOUT_S, loop_heartbeat_forever, @@ -725,14 +725,12 @@ class GatewayStartupMixin: # Log any active supply-chain security advisories. Deliberately does NOT block startup or # surface inline to users — only the operator can act (uninstall, rotate credentials). - try: + with _log_suppressed(logging.DEBUG, "security advisory check failed at gateway startup", exc_info=True): from hermes_cli.security_advisories import detect_compromised, gateway_log_message _adv_msg = gateway_log_message(detect_compromised()) if _adv_msg: logger.warning("%s", _adv_msg) logger.warning("Run `hermes doctor` on the gateway host for full remediation steps.") - except Exception: - logger.debug("security advisory check failed at gateway startup", exc_info=True) def _start_log_systemd_timing_alignment(self) -> None: """Warn when systemd's TimeoutStopSec does not cover the drain window (a unit file from before @@ -784,9 +782,9 @@ class GatewayStartupMixin: entries = platform_registry.plugin_entries() allowed_vars += [e.allowed_users_env for e in entries if e.allowed_users_env] allow_all_vars += [e.allow_all_env for e in entries if e.allow_all_env] - _any_allowlist = any(os.getenv(v) for v in allowed_vars) - _allow_all = any(os.getenv(v, "").lower() in {"true", "1", "yes"} for v in allow_all_vars) - if not _any_allowlist and not _allow_all: + if not any(os.getenv(v) for v in allowed_vars) and not any( + os.getenv(v, "").lower() in {"true", "1", "yes"} for v in allow_all_vars + ): logger.warning( "No env user allowlists configured. Messaging platforms default to " "pairing/allowlist policies and will deny unknown senders unless you " @@ -985,12 +983,10 @@ class GatewayStartupMixin: # stop() won't reach them. _connected_ok = [ _t for _t in _task_map - if _t not in _pending_tasks and not _t.cancelled() - and _t.exception() is None and _t.result()[3] == "ok" + if _t not in _pending_tasks and not _t.cancelled() and _t.exception() is None and _t.result()[3] == "ok" ] for _t in [*_pending_tasks, *_connected_ok]: - _p, _c, _a = _task_map[_t] - await self._startup_teardown_adapter(_a, _p) + await self._startup_teardown_adapter(_task_map[_t][2], _task_map[_t][0]) await self._abort_startup_if_shutdown_requested() return None @@ -1519,5 +1515,4 @@ class GatewayStartupMixin: except Exception as exc: raise RuntimeError(f"adapter.send failed: {exc}") from exc if not getattr(result, "success", True): - err = getattr(result, "error", "send returned success=False") - raise RuntimeError(f"adapter.send failed: {err}") + raise RuntimeError(f"adapter.send failed: {_send_error(result)}") From 3d1fbd3696b4db38b9bf9f596ac966165b2e3142 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:25:26 -0700 Subject: [PATCH 03/11] refactor(gateway): drop statement-separator blank lines inside run_* function bodies (whitespace-only) --- gateway/run_adapters.py | 25 ------------- gateway/run_notifications.py | 70 ------------------------------------ gateway/run_startup.py | 45 ----------------------- 3 files changed, 140 deletions(-) diff --git a/gateway/run_adapters.py b/gateway/run_adapters.py index 9ace375554..5d5ca6573c 100644 --- a/gateway/run_adapters.py +++ b/gateway/run_adapters.py @@ -273,7 +273,6 @@ class GatewayAdapterLifecycleMixin: adapter.platform.value, adapter.fatal_error_code or "unknown", ) return - logger.error( "Fatal %s adapter error (%s): %s", adapter.platform.value, adapter.fatal_error_code or "unknown", adapter.fatal_error_message or "unknown error", @@ -288,21 +287,17 @@ class GatewayAdapterLifecycleMixin: error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) - if existing is adapter: # Claim for teardown BEFORE awaiting disconnect(), else a concurrent second fatal # notification for the same adapter would disconnect() it twice. self.adapters.pop(adapter.platform, None) self.delivery_router.adapters = self.adapters - # Queue BEFORE any disconnect await: a half-dead transport can wedge native close() (or # swallow CancelledError), and "disconnect then queue" left platforms permanently deaf. self._queue_retryable_fatal_platform(adapter) - if existing is adapter: # Bounded by the shutdown-path timeout so this always returns to the stranded check. await self._safe_adapter_disconnect(adapter, adapter.platform) - if not self.adapters and not self._failed_platforms: self._exit_reason = adapter.fatal_error_message or "All messaging adapters disconnected" if adapter.fatal_error_retryable: @@ -357,7 +352,6 @@ class GatewayAdapterLifecycleMixin: respawn leaves a stale handle and a SECOND watcher; ``on_give_up(name)`` fires at budget end.""" # Spawn timestamp lets ``_done`` tell a rapid crash-loop from a healthy-run-then-crash. _started = time.monotonic() - # No create_task kwargs (test doubles mock a narrow signature); running it from a fresh # Context gives the same isolation as create_task(..., context=Context()). task = Context().run(lambda: asyncio.create_task(coro_factory())) @@ -421,14 +415,12 @@ class GatewayAdapterLifecycleMixin: ``completed``/``failed``.""" from gateway.run import _async_profile_runtime_scope, _handoff_watch_scopes, _reclaim_stale await asyncio.sleep(5) # let platforms connect before dispatching through them - # Does _process_handoff accept the profile argument? Test stand-ins bind a one-arg callable. try: import inspect as _inspect _process_takes_profile = len(_inspect.signature(self._process_handoff).parameters) >= 2 except Exception: _process_takes_profile = False - # In-flight dispatches keyed by session id: a handoff runs a FULL agent turn, so inline # processing would let one slow handoff time out every other profile's poll. inflight: Dict[str, "asyncio.Task"] = {} @@ -480,7 +472,6 @@ class GatewayAdapterLifecycleMixin: with _log_suppressed(logging.DEBUG, "Stale-handoff reclaim failed", exc_info=True): async with _scope(_phome): await _reclaim_stale(self) - try: while self._running: try: @@ -647,7 +638,6 @@ class GatewayAdapterLifecycleMixin: self._flag_reconnect_needs_attention(platform, info, now) if now < info["next_retry"]: return # not time yet - platform_config = info["config"] attempt = info["attempts"] + 1 # Empty-token primary configs can never reconnect; drop them so multiplex setups @@ -656,14 +646,12 @@ class GatewayAdapterLifecycleMixin: self._drop_from_reconnect_queue(platform, "no bot credential on queued config") return logger.info("Reconnecting %s (attempt %d)...", platform.value, attempt) - adapter = None try: adapter = self._create_adapter(platform, platform_config) if not adapter: self._drop_from_reconnect_queue(platform, "adapter creation returned None") return - self._wire_adapter_handlers(adapter) # Reconnect after outage: keep the platform's server-side update queue so # messages sent while the bot was offline are delivered rather than dropped. @@ -717,7 +705,6 @@ class GatewayAdapterLifecycleMixin: needs_attention=False, retrying_since=None, ) logger.info("✓ %s reconnected successfully", platform.value) - # Final responses rejected while this adapter was down are still owned by this live # process, so startup recovery cannot claim them. Replay the transient subset now. try: @@ -726,14 +713,12 @@ class GatewayAdapterLifecycleMixin: logger.debug( "failed-obligation redelivery after %s reconnect failed", platform.value, exc_info=True, ) - # Rebuild channel directory with the new adapter try: from gateway.channel_directory import build_channel_directory await build_channel_directory(self.adapters) except Exception: pass - # A platform offline at gateway startup never got its restart-interrupted sessions # auto-resumed — the startup pass skips sessions whose adapter isn't connected yet. try: @@ -775,12 +760,10 @@ class GatewayAdapterLifecycleMixin: ) if not getattr(self.config, "multiplex_profiles", False): return 0 - try: from hermes_cli.profiles import get_active_profile_name except Exception: return 0 - active = get_active_profile_name() or "default" connected = 0 claimed = self._primary_resource_claims(active) @@ -798,7 +781,6 @@ class GatewayAdapterLifecycleMixin: raise except Exception as e: logger.error("Failed to start adapters for profile '%s': %s", profile_name, e, exc_info=True) - self._record_served_profiles(active, profile_homes) return connected @@ -842,22 +824,18 @@ class GatewayAdapterLifecycleMixin: ) from gateway.config import load_gateway_config from hermes_cli.env_loader import hydrate_profile_secret_sources - # Hydrate external secret sources (1Password/vault/...) off-loop ONCE: the sync hydration is # network-bound and would stall every other profile's heartbeat while this one boots. await asyncio.to_thread(hydrate_profile_secret_sources, profile_home) - with _profile_runtime_scope(profile_home, hydrate_secrets=False): profile_runtime_cfg = _load_gateway_runtime_config() from hermes_cli.plugins import discover_plugins discover_plugins() - # Register this profile's shell hooks / outbound webhooks: start() registers before any # profile scope exists, so a secondary profile's `hooks:` block would be silently inert. self._register_config_hooks( "shell-hook/webhook registration failed for profile '%s'", profile_name, level=logging.WARNING, ) - profile_cfg = load_gateway_config() violation = _own_policy_open_startup_violation(profile_cfg) self._snapshot_profile_busy_modes(profile_name, profile_runtime_cfg) @@ -867,7 +845,6 @@ class GatewayAdapterLifecycleMixin: "Enable GATEWAY_ALLOW_ALL_USERS or the platform allow-all flag " "for that profile, or change dm_policy/group_policy away from 'open'." ) - port_binding_platforms = sorted( platform.value for platform, platform_config in profile_cfg.platforms.items() @@ -1121,7 +1098,6 @@ class GatewayAdapterLifecycleMixin: "Secondary %s reconnect attempt failed (profile: %s)", platform.value, profile_name, exc_info=True, ) - if not self._running: return attempts += 1 @@ -1411,7 +1387,6 @@ class GatewayAdapterLifecycleMixin: config.extra.setdefault( "thread_sessions_per_user", getattr(self.config, "thread_sessions_per_user", False) ) - with _log_suppressed(logging.DEBUG, "Platform registry lookup for '%s' failed: %s", platform.value): from gateway.platform_registry import platform_registry if platform_registry.is_registered(platform.value): diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py index df42a624c3..d9ca12fe21 100644 --- a/gateway/run_notifications.py +++ b/gateway/run_notifications.py @@ -94,7 +94,6 @@ class GatewayNotificationsMixin: adapter = self._adapter_for_source(source) if not adapter: return - config = getattr(self, "config", None) chat_id = getattr(source, "chat_id", None) if config and getattr(source, "platform", None) == Platform.SLACK and _is_slack_ignored_channel(config, chat_id): @@ -115,7 +114,6 @@ class GatewayNotificationsMixin: "[%s] send_private_notice failed, falling back to public", getattr(source, "platform", "?"), exc_info=True, ) - await adapter.send(source.chat_id, content, metadata=metadata) async def _resolve_compression_lineage_target( @@ -127,14 +125,12 @@ class GatewayNotificationsMixin: except Exception: logger.debug("Async-delegation compression-tip lookup failed for %s", pinned_session_id, exc_info=True) target_session_id = None - if not target_session_id or target_session_id == pinned_session_id: logger.warning( "Async-delegation completion pinned to compressed session %s " "without a continuation; dropping injection.", pinned_session_id, ) return None - try: tip_row = await session_db.get_session(target_session_id) except Exception: @@ -145,7 +141,6 @@ class GatewayNotificationsMixin: target_session_id, "unknown" if tip_row is None else "ended", ) return None - route_owns_lineage = session_entry.session_id in {pinned_session_id, target_session_id} if not route_owns_lineage: # A delegation may survive several compression rotations: accept an intermediate stale @@ -162,7 +157,6 @@ class GatewayNotificationsMixin: except Exception: route_tip = None route_owns_lineage = route_tip == target_session_id - if not route_owns_lineage: logger.warning( "Async-delegation completion for compression lineage %s -> %s " @@ -189,20 +183,17 @@ class GatewayNotificationsMixin: "dropping injection (#55578 fail-closed)." ) return None - pinned_row = None try: pinned_row = await session_db.get_session(pinned_session_id) except Exception: logger.debug("Async-delegation parent lookup failed for %s", pinned_session_id, exc_info=True) - if pinned_row is None: logger.warning( "Async-delegation completion has unknown spawning session %s; " "dropping injection (#55578 fail-closed).", pinned_session_id, ) return None - target_session_id = pinned_session_id follows_compression = False if pinned_row.get("ended_at"): @@ -224,17 +215,14 @@ class GatewayNotificationsMixin: _end_reason or "idle", pinned_session_id, session_entry.session_id, ) return session_entry - follows_compression = True target_session_id = await self._resolve_compression_lineage_target( session_db, session_entry, pinned_session_id, ) if target_session_id is None: return None - if target_session_id == session_entry.session_id: return session_entry - prior_session_id = session_entry.session_id if follows_compression: switched = await self.async_session_store.advance_compression_session( @@ -248,7 +236,6 @@ class GatewayNotificationsMixin: "owning session %s; dropping injection.", session_entry.session_key, target_session_id, ) return None - logger.info( "Pinned async-delegation completion to owning session %s (was %s) for routing key %s (#57498)", target_session_id, prior_session_id, session_entry.session_key, @@ -268,20 +255,17 @@ class GatewayNotificationsMixin: # Capture [[as_document]] before extract_media strips it: image files then go through # send_document (preserving bytes), not send_multiple_images (Telegram recompresses). force_document_attachments = "[[as_document]]" in response - from gateway.platforms.base import BasePlatformAdapter, should_send_media_as_audio media_files, cleaned = adapter.extract_media(response) media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files) # Strip image URLs for parity with the non-streaming chain; no extract_local_files here. adapter.extract_images(cleaned) - _thread_meta = ( dict(thread_metadata) if thread_metadata is not None else self._thread_metadata_for_source(event.source, self._reply_anchor_for_event(event)) ) chat_id = event.source.chat_id - # Images go out as one batch (e.g. Signal's multi-attachment RPC) unless [[as_document]]. def _is_photo(media_path: str, is_voice: bool) -> bool: ext = Path(media_path).suffix.lower() @@ -289,14 +273,12 @@ class GatewayNotificationsMixin: image_paths = [p for p, v in media_files if _is_photo(p, v)] non_image_media = [(p, v) for p, v in media_files if not _is_photo(p, v)] - if image_paths: try: images = [(f"file://{_quote(p)}", "") for p in image_paths] await adapter.send_multiple_images(chat_id=chat_id, images=images, metadata=_thread_meta) except Exception as e: logger.warning("[%s] Post-stream image batch delivery failed: %s", adapter.name, e) - for media_path, is_voice in non_image_media: try: ext = Path(media_path).suffix.lower() @@ -346,12 +328,10 @@ class GatewayNotificationsMixin: logger.debug("Queued-lane reconcile edit failed (%s); falling back to send.", _qe) if not _reconciled: await adapter.send(source.chat_id, text_content, metadata=metadata) - # Failed turns deliver their (normalized failure) text but must not upload attachments as if # they succeeded — mirrors the ``not agent_result.get("failed")`` completed-turn guard. if not deliver_media: return - synthetic_event = MessageEvent(text="", source=source, message_id=event_message_id) await self._deliver_media_from_response(response, synthetic_event, adapter, thread_metadata=metadata) @@ -360,7 +340,6 @@ class GatewayNotificationsMixin: existing_task = getattr(self, "_update_notification_task", None) if existing_task and not existing_task.done(): return - try: self._update_notification_task = asyncio.create_task(self._watch_update_progress()) except RuntimeError: @@ -488,13 +467,11 @@ class GatewayNotificationsMixin: paths = self._update_paths() loop = asyncio.get_running_loop() deadline = loop.time() + timeout - target = self._resolve_update_target(paths) if target is None: await self._watch_update_completion_only(paths, deadline, poll_interval) return session_key = target.session_key - bytes_sent = 0 last_stream_time = loop.time() buffer = "" @@ -528,11 +505,9 @@ class GatewayNotificationsMixin: logger.info("Update finished (exit=%s), notified %s", exit_code, session_key) self._clear_update_markers(paths, session_key) return - _read_new_output() if buffer.strip() and (loop.time() - last_stream_time) >= stream_interval: await _flush_buffer() - # Forward a prompt only when none is still awaiting a response; otherwise the watcher # would re-read the same .update_prompt.json every poll and spam duplicate prompts. _pending_state = self._peek_session_state(session_key) if session_key else None @@ -548,9 +523,7 @@ class GatewayNotificationsMixin: await self._forward_update_prompt(target, prompt_text, prompt_data.get("default", "")) except (json.JSONDecodeError, OSError) as e: logger.debug("Failed to read update prompt: %s", e) - await asyncio.sleep(poll_interval) - if not paths.exit_code.exists(): logger.warning("Update watcher timed out after %.0fs", timeout) paths.exit_code.write_text("124", encoding="utf-8") @@ -568,7 +541,6 @@ class GatewayNotificationsMixin: paths = self._update_paths() if not paths.any_pending(): return False - cleanup = True active_pending_path = paths.claimed @@ -589,17 +561,13 @@ class GatewayNotificationsMixin: return True elif not paths.claimed.exists(): return True - pending = json.loads(paths.claimed.read_text(encoding="utf-8")) platform_str = pending.get("platform") chat_id = pending.get("chat_id") - if not paths.exit_code.exists(): return _defer("Update notification deferred: update still running") - exit_code = self._update_exit_code(paths) output = paths.output.read_bytes().decode("utf-8", errors="replace") if paths.output.exists() else "" - platform = Platform(platform_str) adapter = self.adapters.get(platform) if chat_id and not adapter: @@ -628,7 +596,6 @@ class GatewayNotificationsMixin: if cleanup: for p in (active_pending_path, paths.claimed, paths.output, paths.exit_code): p.unlink(missing_ok=True) - return True async def _send_restart_notification(self) -> Optional[tuple[str, str, Optional[str]]]: @@ -637,29 +604,24 @@ class GatewayNotificationsMixin: notify_path = _hermes_home / ".restart_notify.json" if not notify_path.exists(): return None - try: data = json.loads(notify_path.read_text(encoding="utf-8")) platform_str = data.get("platform") chat_id = data.get("chat_id") thread_id = data.get("thread_id") - if not platform_str or not chat_id: return None - platform = Platform(platform_str) transport = resolve_delivery_transport(platform, self.config, self.adapters) if transport is None: logger.debug("Restart notification skipped: no live transport for %s", platform_str) return None - platform_cfg = self.config.platforms.get(platform) if platform_cfg is not None and not platform_cfg.gateway_restart_notification: logger.info( "Restart notification suppressed: %s has gateway_restart_notification=false", platform_str ) return None - metadata = self._pending_marker_metadata(platform, chat_id, data, transport.adapter) if data.get("delivered_via_upstream_relay") is True: metadata = dict(metadata or {}) @@ -677,7 +639,6 @@ class GatewayNotificationsMixin: "Restart notification to %s:%s was not delivered: %s", platform_str, chat_id, _send_error(result), ) return None - logger.info("Sent restart notification to %s:%s", platform_str, chat_id) return str(platform_str), str(chat_id), str(thread_id) if thread_id else None except Exception as e: @@ -733,7 +694,6 @@ class GatewayNotificationsMixin: delivered: set[tuple[str, str, Optional[str]]] = set() skipped = skip_targets or set() message = "♻️ Gateway online — Hermes is back and ready." - for platform, platform_cfg, home, transport in self._home_channel_transports(): if not platform_cfg.gateway_restart_notification: logger.info( @@ -741,17 +701,14 @@ class GatewayNotificationsMixin: platform.value, ) continue - target = _notice_target_key(platform.value, home.chat_id, home.thread_id) if target in skipped or target in delivered: continue - if await self._send_home_channel_message( platform, home, transport, message, "Home-channel startup notification failed for %s:%s: %s", ): delivered.add(target) logger.info("Sent home-channel startup notification to %s:%s", platform.value, home.chat_id) - return delivered async def _send_session_db_warning_notifications(self) -> None: @@ -764,7 +721,6 @@ class GatewayNotificationsMixin: error = getattr(self, "_session_db_init_error", None) if not error: return - from hermes_state import classify_persistence_error, format_session_db_unavailable if classify_persistence_error(error) == "corrupt": message = ( @@ -779,9 +735,7 @@ class GatewayNotificationsMixin: f"⚠️ Session database unavailable — messages may not be persisted. " f"{format_session_db_unavailable()}\nRun `hermes doctor` for diagnostics." ) - logger.warning("Broadcasting state.db failure warning to home channels: %s", error) - for platform, _platform_cfg, home, transport in self._home_channel_transports(): await self._send_home_channel_message( platform, home, transport, message, "state.db warning notification failed for %s:%s: %s", @@ -795,7 +749,6 @@ class GatewayNotificationsMixin: from gateway.run import _parse_session_key session_key = str(evt.get("session_key") or "").strip() derived = {} - if session_key: try: self.session_store._ensure_loaded() @@ -804,13 +757,10 @@ class GatewayNotificationsMixin: return entry.origin except Exception as exc: logger.debug("Synthetic process-event session-store lookup failed for %s: %s", session_key, exc) - cached_source = self._get_cached_session_source(session_key) if cached_source is not None: return cached_source - derived = _parse_session_key(session_key) or {} - platform_name = str(evt.get("platform") or derived.get("platform") or "").strip().lower() chat_type = str(evt.get("chat_type") or derived.get("chat_type") or "").strip().lower() chat_id = str(evt.get("chat_id") or derived.get("chat_id") or "").strip() @@ -821,7 +771,6 @@ class GatewayNotificationsMixin: session_key, platform_name, chat_type, chat_id, evt.get("type", "?"), ) return None - try: platform = Platform(platform_name) # Reject dynamic pseudo-members: plugin platforms must be registered. @@ -863,7 +812,6 @@ class GatewayNotificationsMixin: watch_events = _drain_gateway_watch_events(completion_queue) if self._load_background_notifications_mode() == "off": return - for evt in watch_events: synth_text = _format_gateway_process_notification(evt) if not synth_text: @@ -1094,7 +1042,6 @@ class GatewayNotificationsMixin: return claim elif evt_type != "completion": return claim - # Background completions carry only session_key, so after /new the OLD session's notification # would land in the chat's NEW session. Stamped events get the same pre-flight as async # delegations; unstamped ones deliver. @@ -1139,18 +1086,15 @@ class GatewayNotificationsMixin: return claim.early_result if identity is not None and self._completion_identity_seen(identity, claim=True): return None - accepted = False try: injection_result = await self._inject_watch_notification(synth_text, evt) if injection_result is not True: return injection_result accepted = True - if identity is not None: with self._completion_delivery_lock: self._mark_completions_delivered_locked((identity,)) - # The durable async-delegation row is the authoritative replay state — ack it after # adapter acceptance; no parallel ledger here. if claim.claim_id: @@ -1224,7 +1168,6 @@ class GatewayNotificationsMixin: if not entries: return synth_text = entries[0][0] if len(entries) == 1 else self._format_coalesced_process_completions(entries) - # A duplicate primary can legitimately return None from the lifecycle dedupe seam; try the # next batch identity so a fresh sibling is never discarded with that duplicate. delivered = None @@ -1270,7 +1213,6 @@ class GatewayNotificationsMixin: task.cancel() if tasks: await asyncio.gather(*tasks, return_exceptions=True) - # Defensive cleanup for an orphaned queue with no live flush task. batches = getattr(self, "_completion_notification_batches", {}) for entries in batches.values(): @@ -1291,10 +1233,8 @@ class GatewayNotificationsMixin: ): if not hasattr(self, attr): setattr(self, attr, default()) - if self._completion_notification_batches_stopping: return False - key = self._event_route_key(evt, self._COMPLETION_BATCH_KEY_FIELDS) future = asyncio.get_running_loop().create_future() self._completion_notification_batches.setdefault(key, []).append((synth_text, evt, future)) @@ -1348,13 +1288,11 @@ class GatewayNotificationsMixin: if identity is not None and self._completion_identity_seen(identity): continue deliverable.append((evt, synth_text)) - if not deliverable: return None if len(deliverable) == 1: evt, synth_text = deliverable[0] return await self._deliver_completion_notification(synth_text, evt) - from tools.async_delegation import claim_event_delivery, complete_event_delivery, release_event_delivery primary_evt, primary_text = deliverable[0] blocks = [primary_text] @@ -1367,10 +1305,8 @@ class GatewayNotificationsMixin: continue siblings.append((evt, claim_id)) blocks.append(synth_text) - if not siblings: return await self._deliver_completion_notification(primary_text, primary_evt) - header = ( f"[IMPORTANT: {len(blocks)} background subagent delegations " "completed for this session. Treat these results as one " @@ -1528,10 +1464,8 @@ class GatewayNotificationsMixin: thread_id = watcher.get("thread_id", "") agent_notify = watcher.get("notify_on_complete", False) notify_mode = self._load_background_notifications_mode() - logger.debug("Process watcher started: %s (every %ss, notify=%s, agent_notify=%s)", session_id, interval, notify_mode, agent_notify) - silent = notify_mode == "off" and not agent_notify last_output_len = 0 while True: @@ -1547,7 +1481,6 @@ class GatewayNotificationsMixin: current_output_len = len(session.output_buffer) has_new_output = current_output_len > last_output_len last_output_len = current_output_len - if session.exited: # Agent-triggered completion: inject a synthetic message unless the agent already consumed # the result via wait/log. poll() is read-only and deliberately does NOT mark consumed — @@ -1563,7 +1496,6 @@ class GatewayNotificationsMixin: # of suppressing the result. continue break - # Normal text-only notification. Skip when the agent already consumed this completion via # wait/log (output returned inline) — the raw "finished" message would be a duplicate. # The agent_notify skip FALLS THROUGH here, hence this check. poll() is read-only. @@ -1579,7 +1511,6 @@ class GatewayNotificationsMixin: message_text = self._format_process_final_message(session_id, session, notify_mode) await self._send_watcher_message(platform_name, chat_id, thread_id, message_text) break - elif has_new_output and notify_mode == "all" and not agent_notify: # New output — deliver a status update (only in "all" mode; agent_notify watchers # only care about completion). @@ -1588,5 +1519,4 @@ class GatewayNotificationsMixin: platform_name, chat_id, thread_id, f"[Background process {session_id} is still running~ New output:\n{new_output}]", ) - logger.debug("Process watcher ended%s: %s", " (silent)" if silent else "", session_id) diff --git a/gateway/run_startup.py b/gateway/run_startup.py index 00d5c21f6b..bdca671c88 100644 --- a/gateway/run_startup.py +++ b/gateway/run_startup.py @@ -282,7 +282,6 @@ class GatewayStartupMixin: return [] if not claimed: return [] - # Clear resume_pending for EVERY claimed row before any send: claiming already spent one # redelivery attempt and the answer is in the ledger, so the resume path must never re-run. await self._clear_resume_pending_for_claimed_obligations(claimed) @@ -308,7 +307,6 @@ class GatewayStartupMixin: except Exception: logger.debug("delivery ledger import failed", exc_info=True) return 0 - redelivered = 0 for row in claimed: adapter = await self._obligation_adapter(row) @@ -382,7 +380,6 @@ class GatewayStartupMixin: return 0 if not claimed: return 0 - # Clear before any send so the reconnect path cannot both redeliver an # already-produced answer and schedule the same agent turn for resume. sendable = await self._clear_resume_pending_for_claimed_obligations(claimed, require_success=True) @@ -411,7 +408,6 @@ class GatewayStartupMixin: except Exception as exc: logger.warning("Failed to enumerate resume-pending sessions: %s", exc) return None - # Restart-loop breaker: only boots WITH restart-interrupted sessions count; when tripped, skip # auto-resume for THIS boot only (inbound still served; sessions stay resume_pending). if candidates: @@ -449,18 +445,15 @@ class GatewayStartupMixin: candidates = self._resume_pending_candidates(platform) if candidates is None: return 0 - now = datetime.now() scheduled = 0 for entry in candidates: marker = entry.last_resume_marked_at or entry.updated_at if marker is not None and (now - marker).total_seconds() > window: continue - # Already being resumed (e.g. scheduled at startup, still in-flight) — no second turn. if self._is_session_running(entry.session_key): continue - source = entry.origin adapter = self._adapter_for_source(source) if adapter is None: @@ -471,7 +464,6 @@ class GatewayStartupMixin: continue if not self._resume_owner_authorized(entry.session_key, source): continue - # Claim the session slot *before* spawning the task so an inbound message arriving # between task creation and the task's first await (where _process_message_background # sets the real sentinel) sees the slot occupied and queues, not a duplicate AIAgent. @@ -479,7 +471,6 @@ class GatewayStartupMixin: _resume_state.turn.agent = _AGENT_PENDING_SENTINEL _resume_state.turn.started_ts = time.time() self._persist_active_agents() - # Empty-text internal event: the _is_resume_pending branch in _handle_message_with_agent # prepends the reason-aware system note before the turn runs. event = MessageEvent(text="", message_type=MessageType.TEXT, source=source, internal=True) @@ -538,7 +529,6 @@ class GatewayStartupMixin: if getattr(self, "_loop_floor_timer_handle", None) is None: with _log_suppressed(logging.DEBUG, "Failed to arm gateway loop floor timer", exc_info=True): self._loop_floor_timer_handle = _arm_loop_floor_timer(loop) - watchdog = getattr(self, "_loop_liveness_watchdog", None) if watchdog is None or not watchdog.is_alive(): try: @@ -599,7 +589,6 @@ class GatewayStartupMixin: """Start the local Group Chat worker without importing the dashboard.""" import tui_gateway.server # noqa: F401 from tui_gateway import methods_groups - service = methods_groups.get_hosted_room_service() if service is None: service = methods_groups.start_hosted_room_service() @@ -722,7 +711,6 @@ class GatewayStartupMixin: self._gateway_health_export_runtime = start_gateway_health_export(load_config()) if getattr(self._gateway_health_export_runtime, "enabled", False): logger.info("Gateway health OTLP export: enabled") - # Log any active supply-chain security advisories. Deliberately does NOT block startup or # surface inline to users — only the operator can act (uninstall, rotate credentials). with _log_suppressed(logging.DEBUG, "security advisory check failed at gateway startup", exc_info=True): @@ -792,7 +780,6 @@ class GatewayStartupMixin: "or explicitly opt in with GATEWAY_ALLOW_ALL_USERS=true plus " "dm_policy/group_policy: open on the platform." ) - reason = _own_policy_open_startup_violation(self.config) if reason: platform_value = reason.split(":", 1)[0] @@ -818,7 +805,6 @@ class GatewayStartupMixin: with _log_suppressed(logging.WARNING, "plugin discovery failed at gateway startup", exc_info=True): from hermes_cli.plugins import discover_plugins discover_plugins() - # Generic relay adapter only if GATEWAY_RELAY_URL / gateway.relay_url is set; no URL -> no-op. try: from gateway.relay import ( @@ -835,7 +821,6 @@ class GatewayStartupMixin: send_relay_policy() except Exception: logger.warning("relay adapter registration failed at gateway startup", exc_info=True) - GatewayStartupMixin._register_config_hooks("shell-hook registration failed at gateway startup") @staticmethod @@ -861,14 +846,12 @@ class GatewayStartupMixin: from gateway.run import _hermes_home self._start_register_plugins_relay_hooks() self.hooks.discover_and_load() - # Recover background processes from checkpoint (crash recovery) with _log_suppressed(logging.WARNING, "Process checkpoint recovery: %s"): from tools.process_registry import process_registry recovered = process_registry.recover_from_checkpoint() if recovered: logger.info("Recovered %s background process(es) from previous run", recovered) - # Recover sessions active at last exit (exact turn markers + 120s recency fallback for # marker-less older turns). SKIP after a clean exit — the previous process already drained. _clean_marker = _hermes_home / ".clean_shutdown" @@ -891,7 +874,6 @@ class GatewayStartupMixin: "Marked %d in-flight session(s) as resumable from previous run " "(%d exact, %d legacy)", exact + fallback, exact, fallback, ) - # Stuck-loop detection: a session active across 3+ consecutive restarts is probably looping # (its history keeps hanging the agent); auto-suspend so the next message starts clean. with _log_suppressed(logging.DEBUG, "Stuck-loop detection failed: %s"): @@ -923,7 +905,6 @@ class GatewayStartupMixin: _multiplex_skipped_platforms.append(platform) continue enabled_platform_count += 1 - adapter = self._create_adapter(platform, platform_config) if not adapter: # Distinguish between missing builtin deps and missing plugin @@ -1086,7 +1067,6 @@ class GatewayStartupMixin: # Startup authority is one phase, not a persistent runner mode: from here on every # adapter retry is non-evicting. self._platform_lock_takeover_on_start = False - # A platform skipped on the primary for a missing credential should have been picked up by # a secondary profile owning the token. If none did, it is enabled in config.yaml yet # silently unserved — surface it loudly instead of leaving a quiet dead channel. @@ -1156,21 +1136,18 @@ class GatewayStartupMixin: ) self._spawn_supervised(self._hosted_room_worker_watcher, "hosted_room_worker") self._start_loop_heartbeat_task() - hook_count = len(self.hooks.loaded_hooks) if hook_count: logger.info("%s hook(s) loaded", hook_count) await self.hooks.emit("gateway:startup", {"platforms": [p.value for p in self.adapters]}) if connected_count > 0: logger.info("Gateway running with %s platform(s)", connected_count) - # Initial channel directory for send_message name resolution with _log_suppressed(logging.WARNING, "Channel directory build failed: %s"): from gateway.channel_directory import build_channel_directory directory = await build_channel_directory(self.adapters) ch_count = sum(len(chs) for chs in directory.get("platforms", {}).values()) logger.info("Channel directory built: %d target(s)", ch_count) - # Restarting after a /update? If the update is still running, keep watching so we notify # once it actually finishes. notified = await self._send_update_notification() @@ -1184,11 +1161,9 @@ class GatewayStartupMixin: """Post-connect wiring: services, boot notifications, startup restore, recovered watchers.""" from gateway.run import _planned_restart_notification_pending, _restart_notification_pending await self._start_post_connect_services(connected_count) - # Let fresh adapters settle before lifecycle sends (helps Discord thread deliveries). if connected_count > 0: await asyncio.sleep(1.0) - # Before _send_restart_notification() unlinks the marker: did we boot from a chat /restart? # One-shot signal for _is_stale_restart_redelivery. if _restart_notification_pending(): @@ -1198,16 +1173,13 @@ class GatewayStartupMixin: await self._await_startup_boot_sends( planned_restart_notification_pending=_planned_restart_notification_pending(), ) - # Auto-resume restart-interrupted sessions (ledger-answered ones were cleared above); a failed # auto-resume stays visible on the next user message. self._schedule_resume_pending_sessions() await self._finish_startup_restore() - # Surface state.db init failures to the user's messaging platforms so they know persistence # is broken before losing data. await self._send_session_db_warning_notifications() - # Resume recovered process watchers. Detach the batch atomically (fresh list, not clear(): a # concurrent append during the yield must not be lost); yield every 100 to keep the loop live. with _log_suppressed(logging.ERROR, "Recovered watcher setup error: %s"): @@ -1237,7 +1209,6 @@ class GatewayStartupMixin: """Spawn the long-lived supervised background watchers.""" for method in self._PRE_RECONNECT_WATCHERS: self._spawn_supervised(getattr(self, method), method[1:]) - if self._failed_platforms: logger.info( "Starting reconnection watcher for %d failed platform(s): %s", @@ -1249,10 +1220,8 @@ class GatewayStartupMixin: # arrival). ``on_spawn`` keeps ``_reconnect_watcher_task`` on the CURRENT live task across # backoff respawns so a superseded handle never looks like a dead watcher. self._spawn_reconnect_watcher() - for method in self._POST_RECONNECT_WATCHERS: self._spawn_supervised(getattr(self, method), method[1:]) - # Scale-to-zero idle watcher ONLY when opted in (HERMES_SCALE_TO_ZERO stamp), messaging is # relay-only/absent, and a wakeUrl is registered. When armed it drives the relay dormant on # sustained idle, then suspends via flaps — Fly autostop is inbound-only, job-blind. @@ -1269,7 +1238,6 @@ class GatewayStartupMixin: self._log_scale_to_zero_not_armed_reason() except Exception: # noqa: BLE001 - arming must never block startup logger.debug("scale-to-zero: arm check failed at startup", exc_info=True) - # Drain-control watcher: reconciles the gateway's new-turn accept-state with the external # ``.drain_request.json`` marker the dashboard begin/cancel-drain endpoint writes. A marker # from a prior instantiation (durable-volume restart) is ignored via its epoch. @@ -1285,7 +1253,6 @@ class GatewayStartupMixin: if self._start_check_access_policy(): return True await self._start_recover_previous_run() - # Serialize startup restore against inbound dispatch: adapters can receive messages as soon # as they connect, but restart-interrupted sessions are not auto-resumed until all startup # wiring below completes, so inbound queues until every synthetic resume turn has finished. @@ -1296,7 +1263,6 @@ class GatewayStartupMixin: # while the turn machinery is still cold, so a message in that window got a skeleton system # prompt. Warm NOW to overlap the connects below; _finish_startup_restore awaits it (bounded). self._start_startup_warmup() - startup_nonretryable_errors: list[str] = [] startup_retryable_errors: list[str] = [] ( @@ -1304,7 +1270,6 @@ class GatewayStartupMixin: ) = await self._start_prefilter_platforms() if _aborted: return True - if await self._abort_startup_if_shutdown_requested(): return True _raw = await self._start_connect_pending(_pending_connects) @@ -1313,7 +1278,6 @@ class GatewayStartupMixin: connected_count = await self._start_aggregate_connect_results( _raw, startup_retryable_errors, startup_nonretryable_errors ) - if await self._abort_startup_if_shutdown_requested(): return True _aborted, connected_count = await self._start_secondary_profiles( @@ -1325,12 +1289,10 @@ class GatewayStartupMixin: connected_count, enabled_platform_count, startup_retryable_errors, startup_nonretryable_errors ): return True - if await self._abort_startup_if_shutdown_requested(): return True self.delivery_router.adapters = self.adapters self._wire_teams_pipeline_runtime() - self._running = True self._install_plugin_message_injector() self._update_runtime_status("running") @@ -1383,9 +1345,7 @@ class GatewayStartupMixin: platform = Platform(platform_name) except (ValueError, KeyError): raise RuntimeError(f"unknown platform '{platform_name}'") - handoff_config, handoff_adapters = self._handoff_resolve_scope(profile_name) - # Alias-aware transport: a relay-fronted gateway registers ONE Platform.RELAY adapter fronting # N logical platforms, so a literal adapters.get() would miss a deliverable one. transport = resolve_delivery_transport(platform, handoff_config, handoff_adapters) @@ -1397,7 +1357,6 @@ class GatewayStartupMixin: f"no home channel configured for {platform_name}; run /sethome on the desired chat first" ) home_chat_id = str(home.chat_id) - # Fresh thread on the destination so the handoff has its own scrollback. Adapter returns # None if threading is unsupported (Matrix/WhatsApp/Signal/SMS) or creation failed. cli_title = row.get("title") or cli_session_id[:8] @@ -1409,7 +1368,6 @@ class GatewayStartupMixin: logger.debug("Handoff: create_handoff_thread raised on %s: %s", platform_name, exc, exc_info=True) new_thread_id = None effective_thread_id = new_thread_id or (str(home.thread_id) if home.thread_id else None) - # Telegram private-chat DM topics use the DM-topic source shape (user_id == chat_id) so the # synthetic turn binds the same key later inbound turns arrive on (`dm`, not `thread`). is_telegram_private_chat = ( @@ -1465,7 +1423,6 @@ class GatewayStartupMixin: cli_session_id = row["id"] dest = await self._handoff_resolve_destination(row, profile_name) session_key = self._handoff_session_key(dest, profile_name) - # Ensure a session_store entry exists for this key (get_or_create_session creates one for a # never-used home channel); switch_session then re-points it. await self.async_session_store.get_or_create_session(dest.source) @@ -1479,7 +1436,6 @@ class GatewayStartupMixin: # synthetic turn isn't queued behind it. self._evict_cached_agent(session_key) self._release_running_agent_state(session_key) - cli_title = row.get("title") or cli_session_id[:8] synthetic_event = MessageEvent( text=( @@ -1503,7 +1459,6 @@ class GatewayStartupMixin: # Streaming may have already delivered the response inline; the agent ran without # raising either way — count as success. return - # Send the reply to the new thread if we created one, else the configured home channel # (which may carry a thread_id). Use the resolved transport (not adapter.send) so a # relay-fronted logical platform is stamped on the outbound frame (send_for_platform). From 64b5c7ab38d502f1309ccffb3de942b915b34fa9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:36:51 -0700 Subject: [PATCH 04/11] refactor(gateway): second try/except-log fold pass (12 sites), suppress() for bare passes, fused log-call literals --- gateway/run_adapters.py | 65 +++++++++++++++--------------------- gateway/run_notifications.py | 18 ++++------ gateway/run_shutdown.py | 20 ++++------- gateway/run_startup.py | 23 ++++++------- 4 files changed, 49 insertions(+), 77 deletions(-) diff --git a/gateway/run_adapters.py b/gateway/run_adapters.py index 5d5ca6573c..fe12fd0496 100644 --- a/gateway/run_adapters.py +++ b/gateway/run_adapters.py @@ -11,6 +11,7 @@ import logging from typing import TYPE_CHECKING import asyncio import contextlib +from contextlib import suppress import functools import os import time @@ -88,7 +89,10 @@ class GatewayAdapterLifecycleMixin: ) except Exception as e: logger.debug("✗ %s background-task cancel error%s: %s", platform.value, suffix, e) - try: + with _log_suppressed( + logging.ERROR, "✗ %s disconnect error after %.2fs%s: %s", + platform.value, time.monotonic() - started_at, suffix, + ): if await self._await_adapter_cleanup_with_timeout(adapter.disconnect(), timeout): logger.info( "✓ %s disconnected (%.2fs)%s", platform.value, time.monotonic() - started_at, suffix, @@ -98,11 +102,6 @@ class GatewayAdapterLifecycleMixin: "✗ %s disconnect timed out after %.1fs - forcing continue%s", platform.value, timeout, suffix, ) - except Exception as e: - logger.error( - "✗ %s disconnect error after %.2fs%s: %s", - platform.value, time.monotonic() - started_at, suffix, e, - ) @staticmethod def _env_timeout_override(name: str) -> Optional[float]: @@ -256,12 +255,11 @@ class GatewayAdapterLifecycleMixin: await self.stop() def _queue_retryable_best_effort(self, adapter: BasePlatformAdapter, why: str) -> None: - try: + with _log_suppressed( + logging.DEBUG, "Failed to queue %s after fatal-handler %s", + adapter.platform.value, why, exc_info=True, + ): self._queue_retryable_fatal_platform(adapter) - except Exception: - logger.debug( - "Failed to queue %s after fatal-handler %s", adapter.platform.value, why, exc_info=True, - ) async def _handle_adapter_fatal_error_impl(self, adapter: BasePlatformAdapter) -> None: # Snapshot the slot owner first: a stale notification must not overwrite a healthy @@ -512,10 +510,9 @@ class GatewayAdapterLifecycleMixin: """Bounded slow-tier respawn of the reconnect watcher.""" if attempt >= self._MAX_SLOW_WATCHER_RESPAWNS: logger.error( - "Reconnect watcher could not be kept alive after %d slow " - "respawns; %d platform(s) remain queued and unattended: %s. " - "Manual intervention or a gateway restart is required.", attempt, - len(self._failed_platforms), ", ".join(str(p) for p in self._failed_platforms), + "Reconnect watcher could not be kept alive after %d slow respawns; %d platform(s) remain " + "queued and unattended: %s. Manual intervention or a gateway restart is required.", + attempt, len(self._failed_platforms), ", ".join(str(p) for p in self._failed_platforms), ) return @@ -529,9 +526,8 @@ class GatewayAdapterLifecycleMixin: ): return logger.warning( - "Reconnect watcher still down with %d platform(s) queued — " - "slow respawn %d/%d", len(self._failed_platforms), attempt + 1, - self._MAX_SLOW_WATCHER_RESPAWNS, + "Reconnect watcher still down with %d platform(s) queued — slow respawn %d/%d", + len(self._failed_platforms), attempt + 1, self._MAX_SLOW_WATCHER_RESPAWNS, ) self._spawn_reconnect_watcher( on_give_up=lambda _name: self._schedule_slow_reconnect_watcher_respawn(attempt=attempt + 1) @@ -597,12 +593,10 @@ class GatewayAdapterLifecycleMixin: queued_for = now - info.get("queued_at", now) retrying_since_iso = (datetime.now(timezone.utc) - timedelta(seconds=queued_for)).isoformat() logger.warning( - "%s has been failing/reconnecting continuously for " - "%.1f hours (%d attempts) — flagging NEEDS_ATTENTION. " - "Retries continue, but this usually means a permanent " - "problem (revoked credentials, missing intents, broken " - "sidecar). Check `hermes status` / `/platform list`.", platform.value, - queued_for / 3600.0, info.get("attempts", 0), + "%s has been failing/reconnecting continuously for %.1f hours (%d attempts) — flagging " + "NEEDS_ATTENTION. Retries continue, but this usually means a permanent problem (revoked " + "credentials, missing intents, broken sidecar). Check `hermes status` / `/platform list`.", + platform.value, queued_for / 3600.0, info.get("attempts", 0), ) self._update_platform_runtime_status( platform.value, platform_state="retrying", needs_attention=True, retrying_since=retrying_since_iso @@ -707,18 +701,15 @@ class GatewayAdapterLifecycleMixin: logger.info("✓ %s reconnected successfully", platform.value) # Final responses rejected while this adapter was down are still owned by this live # process, so startup recovery cannot claim them. Replay the transient subset now. - try: + with _log_suppressed( + logging.DEBUG, "failed-obligation redelivery after %s reconnect failed", + platform.value, exc_info=True, + ): await self._redeliver_failed_obligations_for_platform(platform) - except Exception: - logger.debug( - "failed-obligation redelivery after %s reconnect failed", platform.value, exc_info=True, - ) # Rebuild channel directory with the new adapter - try: + with suppress(Exception): from gateway.channel_directory import build_channel_directory await build_channel_directory(self.adapters) - except Exception: - pass # A platform offline at gateway startup never got its restart-interrupted sessions # auto-resumed — the startup pass skips sessions whose adapter isn't connected yet. try: @@ -1221,11 +1212,9 @@ class GatewayAdapterLifecycleMixin: @staticmethod def _stamp_event_profile(event, profile_name: str) -> None: """Best-effort: stamp ``source.profile`` on an inbound event that has none yet.""" - try: + with suppress(Exception): if getattr(event, "source", None) is not None and not event.source.profile: event.source.profile = profile_name - except Exception: - pass def _make_profile_message_handler(self, profile_name: str): """Message handler that stamps source.profile, then delegates under the profile scope @@ -1292,13 +1281,11 @@ class GatewayAdapterLifecycleMixin: async def _handle_gateway_platform_event(self, event: dict, source) -> None: """Authorize and publish one normalized adapter event to plugin hooks.""" - try: + # Observer failures must never break the adapter's update loop. + with _log_suppressed(logging.DEBUG, "gateway_platform_event hook dispatch failed", exc_info=True): from hermes_cli.lifecycle import has_hook, invoke_hook if has_hook("gateway_platform_event") and self._is_user_authorized_for_source(source): invoke_hook("gateway_platform_event", **event) - except Exception: - # Observer failures must never break the adapter's update loop. - logger.debug("gateway_platform_event hook dispatch failed", exc_info=True) def _make_profile_platform_event_handler(self, profile_name: str): """Bind platform-event auth and hook dispatch to one multiplex profile.""" diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py index d9ca12fe21..828b386c23 100644 --- a/gateway/run_notifications.py +++ b/gateway/run_notifications.py @@ -105,15 +105,13 @@ class GatewayNotificationsMixin: ) metadata = self._thread_metadata_for_source(source) if notice_delivery == "private" and getattr(source, "user_id", None): - try: + with _log_suppressed( + logging.DEBUG, "[%s] send_private_notice failed, falling back to public", + getattr(source, "platform", "?"), exc_info=True, + ): result = await adapter.send_private_notice(source.chat_id, source.user_id, content, metadata=metadata) if getattr(result, "success", False): return - except Exception: - logger.debug( - "[%s] send_private_notice failed, falling back to public", - getattr(source, "platform", "?"), exc_info=True, - ) await adapter.send(source.chat_id, content, metadata=metadata) async def _resolve_compression_lineage_target( @@ -360,7 +358,7 @@ class GatewayNotificationsMixin: for path in (paths.claimed, paths.pending): if not path.exists(): continue - try: + with suppress(Exception): pending = json.loads(path.read_text(encoding="utf-8")) platform_str = pending.get("platform") chat_id = pending.get("chat_id") @@ -376,8 +374,6 @@ class GatewayNotificationsMixin: return self._UpdateTarget( adapter, chat_id, session_key or f"{platform_str}:{chat_id}", metadata, platform, ) - except Exception: - pass return None def _pending_marker_metadata(self, platform, chat_id, data: dict, adapter): @@ -486,11 +482,9 @@ class GatewayNotificationsMixin: def _read_new_output() -> None: nonlocal buffer, bytes_sent if paths.output.exists(): - try: + with suppress(OSError): chunk, bytes_sent = self._read_update_output_since(paths.output, bytes_sent) buffer += chunk - except OSError: - pass while loop.time() < deadline: if paths.exit_code.exists(): diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index aefc59568c..294d74aeab 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -407,9 +407,8 @@ class GatewayShutdownMixin: if not self._scale_to_zero_no_suspend_logged: self._scale_to_zero_no_suspend_logged = True logger.info( - "scale-to-zero: idle, but this platform suspends on " - "its own timer (no in-machine suspend API); staying " - "connected rather than quiescing" + "scale-to-zero: idle, but this platform suspends on its own timer (no " + "in-machine suspend API); staying connected rather than quiescing" ) continue logger.info( @@ -544,9 +543,8 @@ class GatewayShutdownMixin: platform.value, platform_state="paused", error_code=None, error_message=info["pause_reason"], ) logger.warning( - "%s paused after %d consecutive failures (%s) — " - "fix the underlying issue then run `/platform resume %s` " - "to retry, or `hermes gateway restart` to restart the gateway.", + "%s paused after %d consecutive failures (%s) — fix the underlying issue then run `/platform " + "resume %s` to retry, or `hermes gateway restart` to restart the gateway.", platform.value, info.get("attempts", 0), info["pause_reason"], platform.value, ) @@ -815,16 +813,14 @@ class GatewayShutdownMixin: # Suppress ONLY the home-channel broadcast when the drain asked to be quiet (routine fleet # auto-update); per-session pings above stay (empty on a drained shutdown, useful on a # force-interrupt). Current-epoch marker only, so an orphaned marker can't silence a fresh gateway. - try: + # Never let the suppression check block the broadcast — fail toward the louder behaviour. + with _log_suppressed(logging.DEBUG, "drain_notification_suppressed check failed: %s"): from gateway.drain_control import drain_notification_suppressed if drain_notification_suppressed(): logger.info( "Home-channel shutdown broadcast suppressed by drain marker (suppress_notification=true)" ) return - except Exception as e: - # Never let the suppression check block the broadcast — fail toward the louder behaviour. - logger.debug("drain_notification_suppressed check failed: %s", e) # Snapshot adapters: adapter.send() can hit a fatal path (_handle_fatal) that pops the adapter # from self.adapters -> ``RuntimeError: dictionary changed size during iteration``. for platform, adapter in list(self.adapters.items()): @@ -1027,7 +1023,7 @@ class GatewayShutdownMixin: return 0 suspended = 0 for session_key in [k for k, v in counts.items() if v >= self._STUCK_LOOP_THRESHOLD]: - try: + with suppress(Exception): entry = self.session_store._entries.get(session_key) if entry and not entry.suspended: entry.suspended = True @@ -1036,8 +1032,6 @@ class GatewayShutdownMixin: "Auto-suspended stuck session %s (active across %d consecutive restarts — likely a stuck loop)", session_key, counts[session_key], ) - except Exception: - pass if suspended: with suppress(Exception): self.session_store._save() diff --git a/gateway/run_startup.py b/gateway/run_startup.py index bdca671c88..1b4849d9a4 100644 --- a/gateway/run_startup.py +++ b/gateway/run_startup.py @@ -104,7 +104,10 @@ class GatewayStartupMixin: """Initialize turn machinery on an executor thread before the gate opens. Never raises: a failed warm-up degrades to lazy init and must not block startup.""" from gateway.run import _warm_turn_machinery_sync - try: + with _log_suppressed( + logging.WARNING, "Turn-machinery warm-up failed; first inbound turn will initialize lazily", + exc_info=True, + ): loop = asyncio.get_running_loop() t0 = time.monotonic() tool_count = await loop.run_in_executor(None, _warm_turn_machinery_sync) @@ -112,10 +115,6 @@ class GatewayStartupMixin: "Turn machinery warmed in %.1fs (%d tool schema(s) materialized)", time.monotonic() - t0, tool_count, ) - except Exception: - logger.warning( - "Turn-machinery warm-up failed; first inbound turn will initialize lazily", exc_info=True, - ) async def _await_startup_warmup(self) -> None: """Bounded wait for the boot warm-up. On timeout the gate opens anyway (availability outranks @@ -326,9 +325,8 @@ class GatewayStartupMixin: await asyncio.to_thread(mark_delivered, row["obligation_id"]) redelivered += 1 logger.info( - "Redelivered recovered final response to %s:%s " - "(obligation %s, attempt %d)", row["platform"], row["chat_id"], - row["obligation_id"], row["attempts"], + "Redelivered recovered final response to %s:%s (obligation %s, attempt %d)", + row["platform"], row["chat_id"], row["obligation_id"], row["attempts"], ) else: await asyncio.to_thread( @@ -774,11 +772,10 @@ class GatewayStartupMixin: os.getenv(v, "").lower() in {"true", "1", "yes"} for v in allow_all_vars ): logger.warning( - "No env user allowlists configured. Messaging platforms default to " - "pairing/allowlist policies and will deny unknown senders unless you " - "configure platform allowlists (e.g., TELEGRAM_ALLOWED_USERS=your_id) " - "or explicitly opt in with GATEWAY_ALLOW_ALL_USERS=true plus " - "dm_policy/group_policy: open on the platform." + "No env user allowlists configured. Messaging platforms default to pairing/allowlist " + "policies and will deny unknown senders unless you configure platform allowlists (e.g., " + "TELEGRAM_ALLOWED_USERS=your_id) or explicitly opt in with GATEWAY_ALLOW_ALL_USERS=true " + "plus dm_policy/group_policy: open on the platform." ) reason = _own_policy_open_startup_violation(self.config) if reason: From d7797770273ace13a082670d732c2eca753a4f5e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:43:34 -0700 Subject: [PATCH 05/11] refactor(gateway): shutdown-notice message ladder, count-step/probe loops in stop+scale-to-zero, update-notification message fold --- gateway/run_adapters.py | 21 +++++++------ gateway/run_notifications.py | 12 ++++---- gateway/run_shutdown.py | 57 ++++++++++++++++++------------------ 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/gateway/run_adapters.py b/gateway/run_adapters.py index fe12fd0496..42eca0eb49 100644 --- a/gateway/run_adapters.py +++ b/gateway/run_adapters.py @@ -908,20 +908,19 @@ class GatewayAdapterLifecycleMixin: # one connection fans out by route); a secondary would only retry-loop and stall startup. if multiplex and platform in (Platform.RELAY, Platform.WHATSAPP): continue - try: + adapter = None + with _log_suppressed( + logging.ERROR, "[MULTIPLEX] Profile '%s': _create_adapter('%s') raised %s", profile_name, + platform.value, exc_info=True, + ): with _profile_runtime_scope(profile_home, hydrate_secrets=False): adapter = self._create_adapter(platform, platform_config) - except Exception as e: - logger.error( - "[MULTIPLEX] Profile '%s': _create_adapter('%s') raised %s", profile_name, - platform.value, e, exc_info=True, - ) - continue + if not adapter: + logger.warning( + "[MULTIPLEX] Profile '%s': skipping platform '%s' - adapter creation returned None", + profile_name, platform.value, + ) if not adapter: - logger.warning( - "[MULTIPLEX] Profile '%s': skipping platform '%s' - adapter creation returned None", - profile_name, platform.value, - ) continue # Same-token / same-listener conflict detection — refuse a duplicate poll or bind. credential_claim = self._adapter_credential_claim(platform, adapter) diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py index 828b386c23..c75dbc21ed 100644 --- a/gateway/run_notifications.py +++ b/gateway/run_notifications.py @@ -505,9 +505,8 @@ class GatewayNotificationsMixin: # Forward a prompt only when none is still awaiting a response; otherwise the watcher # would re-read the same .update_prompt.json every poll and spam duplicate prompts. _pending_state = self._peek_session_state(session_key) if session_key else None - if ( - paths.prompt.exists() and session_key - and not (_pending_state is not None and _pending_state.persistent.update_prompt_pending) + if paths.prompt.exists() and session_key and not getattr( + getattr(_pending_state, "persistent", None), "update_prompt_pending", False ): try: prompt_data = json.loads(paths.prompt.read_text(encoding="utf-8")) @@ -578,10 +577,11 @@ class GatewayNotificationsMixin: output = "…" + output[-3500:] status = "✅ Hermes update finished." if exit_code == 0 else "❌ Hermes update failed." msg = f"{status}\n\n```\n{output}\n```" - elif exit_code == 0: - msg = "✅ Hermes update finished successfully." else: - msg = "❌ Hermes update failed. Check the gateway logs or run `hermes update` manually for details." + msg = ( + "✅ Hermes update finished successfully." if exit_code == 0 else + "❌ Hermes update failed. Check the gateway logs or run `hermes update` manually for details." + ) await adapter.send(chat_id, msg, metadata=_non_conversational_metadata(metadata, platform=platform)) logger.info("Sent post-update notification to %s:%s (exit=%s)", platform_str, chat_id, exit_code) except Exception as e: diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index 294d74aeab..2a15f7c95b 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -229,18 +229,18 @@ class GatewayShutdownMixin: for t in self._background_tasks ): return True - try: + def _delegations_active() -> bool: from tools.async_delegation import active_count - if active_count() > 0: - return True - except Exception: # noqa: BLE001 - never let the idle check raise - logger.debug("scale-to-zero async-delegation check failed", exc_info=True) - try: + return active_count() > 0 + + def _processes_active() -> bool: from tools.process_registry import process_registry - if process_registry.has_any_active() or process_registry.pending_watchers: - return True - except Exception: # noqa: BLE001 - never let the idle check raise - logger.debug("scale-to-zero bg-work check failed", exc_info=True) + return bool(process_registry.has_any_active() or process_registry.pending_watchers) + + for label, probe in (("async-delegation", _delegations_active), ("bg-work", _processes_active)): + with _log_suppressed(logging.DEBUG, f"scale-to-zero {label} check failed", exc_info=True): + if probe(): + return True return False @staticmethod @@ -765,14 +765,12 @@ class GatewayShutdownMixin: Called at the start of stop() while adapters are connected; send failures never block shutdown. """ restart_source = self._restart_command_source if self._restart_requested else None - action = "restarting" if self._restart_requested else "shutting down" - hint = ( - "Your current task will be interrupted. " - "Send any message after restart and I'll try to resume where you left off." - if self._restart_requested - else "Your current task will be interrupted." - ) - msg = f"⚠️ Gateway {action} — {hint}" + msg = "⚠️ Gateway shutting down — Your current task will be interrupted." + if self._restart_requested: + msg = ( + "⚠️ Gateway restarting — Your current task will be interrupted. " + "Send any message after restart and I'll try to resume where you left off." + ) restart_key = None if restart_source is not None: with suppress(Exception): @@ -998,7 +996,7 @@ class GatewayShutdownMixin: @staticmethod def _read_json_counts(path: Path) -> Optional[dict]: - """Parsed counter dict, or None when the file is missing/unreadable.""" + """Parsed counter dict, or None when the file is missing/unreadable (no exists() pre-check needed).""" try: return json.loads(path.read_text(encoding="utf-8")) except Exception: @@ -1008,10 +1006,9 @@ class GatewayShutdownMixin: """Increment persisted restart-failure counters for active sessions; drop the rest (loop broken).""" from gateway.run import atomic_json_write path = self._stuck_loop_counts_path() - counts = (self._read_json_counts(path) if path.exists() else None) or {} - new_counts = {key: counts.get(key, 0) + 1 for key in active_session_keys} + counts = self._read_json_counts(path) or {} with suppress(Exception): - atomic_json_write(path, new_counts, indent=None) + atomic_json_write(path, {key: counts.get(key, 0) + 1 for key in active_session_keys}, indent=None) def _suspend_stuck_loop_sessions(self) -> int: """Suspend sessions active across too many restarts (startup, AFTER suspend_recently_active()).""" @@ -1301,11 +1298,14 @@ class GatewayShutdownMixin: def _step(label: str, fn: Callable[[], Any]) -> Any: return GatewayShutdownMixin._quiet_step(f"{label} ({phase}) error", fn) + def _count_step(fmt: str, fn: Callable[[], int]) -> None: + n = fn() + if n: + logger.info(fmt, phase, n) + def _kill_processes() -> None: from tools.process_registry import process_registry - _killed = process_registry.kill_all() - if _killed: - logger.info("Shutdown (%s): killed %d tool subprocess(es)", phase, _killed) + _count_step("Shutdown (%s): killed %d tool subprocess(es)", process_registry.kill_all) def _mark_cron_interrupted() -> list: # kill_all() is a global sweep, so any cron job dispatched right now lost its tool @@ -1324,9 +1324,10 @@ class GatewayShutdownMixin: def _interrupt_delegations() -> None: from tools.async_delegation import interrupt_all as _interrupt_async - _async_n = _interrupt_async(reason=f"gateway shutdown ({phase})") - if _async_n: - logger.info("Shutdown (%s): interrupted %d background delegation(s)", phase, _async_n) + _count_step( + "Shutdown (%s): interrupted %d background delegation(s)", + lambda: _interrupt_async(reason=f"gateway shutdown ({phase})"), + ) _step("process_registry.kill_all", _kill_processes) _marked_cron_jobs = _step("mark_running_jobs_interrupted", _mark_cron_interrupted) or [] From 158fcfe13ada8b8061a4ce4be6e1fb1f24d8d7f0 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:50:14 -0700 Subject: [PATCH 06/11] refactor(gateway): route quiet runtime-status writes through gateway.run._write_runtime_status_quiet --- gateway/run_shutdown.py | 21 ++++++++++----------- gateway/run_startup.py | 10 ++++------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index 2a15f7c95b..19379df62f 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -517,17 +517,16 @@ class GatewayShutdownMixin: error_code: Optional[str] = None, error_message: Optional[str] = None, needs_attention: Optional[bool] = None, retrying_since: Any = _UNSET, ) -> None: - with suppress(Exception): - from gateway.status import write_runtime_status - extra: Dict[str, Any] = {} - if needs_attention is not None: - extra["needs_attention"] = needs_attention - if retrying_since is not _UNSET: - extra["retrying_since"] = retrying_since - write_runtime_status( - platform=platform, platform_state=platform_state, error_code=error_code, - error_message=error_message, **extra, - ) + from gateway.run import _write_runtime_status_quiet + extra: Dict[str, Any] = {} + if needs_attention is not None: + extra["needs_attention"] = needs_attention + if retrying_since is not _UNSET: + extra["retrying_since"] = retrying_since + _write_runtime_status_quiet( + platform=platform, platform_state=platform_state, error_code=error_code, + error_message=error_message, **extra, + ) # Per-platform circuit breaker (pause/resume): reconnect watcher + /platform pause|resume. def _pause_failed_platform(self, platform, *, reason: str = "") -> None: diff --git a/gateway/run_startup.py b/gateway/run_startup.py index 1b4849d9a4..36e9eb746a 100644 --- a/gateway/run_startup.py +++ b/gateway/run_startup.py @@ -657,6 +657,7 @@ class GatewayStartupMixin: def _start_log_startup_environment(self) -> None: """Bind the gateway loop, disarm the startup watchdog, and log the startup environment.""" + from gateway.run import _write_runtime_status_quiet try: self._gateway_loop = asyncio.get_running_loop() except RuntimeError: @@ -700,9 +701,7 @@ class GatewayStartupMixin: _profile = get_active_profile_name() if _profile and _profile != "default": logger.info("Active profile: %s", _profile) - with suppress(Exception): - from gateway.status import write_runtime_status - write_runtime_status(gateway_state="starting", exit_reason=None, clear_profile_platforms=True) + _write_runtime_status_quiet(gateway_state="starting", exit_reason=None, clear_profile_platforms=True) with _log_suppressed(logging.DEBUG, "gateway health OTLP export startup failed", exc_info=True): from hermes_cli.config import load_config from agent.monitoring.gateway_health_export import start_gateway_health_export @@ -1082,6 +1081,7 @@ class GatewayStartupMixin: startup_nonretryable_errors: list, ) -> bool: """Log/degrade when nothing connected; return True when startup must exit.""" + from gateway.run import _write_runtime_status_quiet if connected_count != 0: return False if startup_nonretryable_errors and not startup_retryable_errors: @@ -1109,9 +1109,7 @@ class GatewayStartupMixin: "Gateway started with no connected platforms — %d platform(s) queued for retry: %s", len(self._failed_platforms), "; ".join(startup_retryable_errors), ) - with suppress(Exception): - from gateway.status import write_runtime_status - write_runtime_status(gateway_state="degraded", exit_reason=None) + _write_runtime_status_quiet(gateway_state="degraded", exit_reason=None) # (Or: no adapter for any enabled platform — missing library or credentials.) Fleet nodes # share one config.yaml but hold credentials for only a subset, so degrade gracefully. logger.warning( From 308ae97d507038168aff8c2cc1826939c8a59351 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:54:36 -0700 Subject: [PATCH 07/11] refactor(gateway): _multiplex_on predicate, inline single-use locals in adapter/notification/shutdown seams --- gateway/run_adapters.py | 28 ++++++++++++++-------------- gateway/run_notifications.py | 6 ++++-- gateway/run_shutdown.py | 3 +-- gateway/run_startup.py | 2 +- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/gateway/run_adapters.py b/gateway/run_adapters.py index 42eca0eb49..ba6265181b 100644 --- a/gateway/run_adapters.py +++ b/gateway/run_adapters.py @@ -170,10 +170,11 @@ class GatewayAdapterLifecycleMixin: tasks = getattr(self, "_fatal_handler_tasks", None) if tasks is None: tasks = self._fatal_handler_tasks = set() - task = self._track_task_in(tasks, asyncio.create_task(self._handle_adapter_fatal_error_detached(adapter))) # shield(): a plain `await task` would tunnel the caller's cancellation into the detached # task; with shield the caller sees CancelledError and the handler runs to completion. - await asyncio.shield(task) + await asyncio.shield( + self._track_task_in(tasks, asyncio.create_task(self._handle_adapter_fatal_error_detached(adapter))) + ) def _reconnect_queue_entry( self, platform, adapter, platform_config, *, attempts: int, delay: float, queued: bool = True @@ -591,7 +592,6 @@ class GatewayAdapterLifecycleMixin: return info["attention_flagged"] = True queued_for = now - info.get("queued_at", now) - retrying_since_iso = (datetime.now(timezone.utc) - timedelta(seconds=queued_for)).isoformat() logger.warning( "%s has been failing/reconnecting continuously for %.1f hours (%d attempts) — flagging " "NEEDS_ATTENTION. Retries continue, but this usually means a permanent problem (revoked " @@ -599,7 +599,8 @@ class GatewayAdapterLifecycleMixin: platform.value, queued_for / 3600.0, info.get("attempts", 0), ) self._update_platform_runtime_status( - platform.value, platform_state="retrying", needs_attention=True, retrying_since=retrying_since_iso + platform.value, platform_state="retrying", needs_attention=True, + retrying_since=(datetime.now(timezone.utc) - timedelta(seconds=queued_for)).isoformat(), ) def _mark_platform_fatal(self, status_key: str, adapter) -> None: @@ -749,7 +750,7 @@ class GatewayAdapterLifecycleMixin: from gateway.run import ( MultiplexConfigError, SecondaryPortBindingConfigError, _multiplex_profile_homes ) - if not getattr(self.config, "multiplex_profiles", False): + if not self._multiplex_on(): return 0 try: from hermes_cli.profiles import get_active_profile_name @@ -890,7 +891,7 @@ class GatewayAdapterLifecycleMixin: """Create+connect one profile's adapters under its runtime scope.""" from gateway.run import _platform_has_bot_credential, _profile_runtime_scope profile_cfg = await self._load_secondary_profile_config(profile_name, profile_home) - multiplex = getattr(self.config, "multiplex_profiles", False) + multiplex = self._multiplex_on() profile_map = self._profile_adapters.setdefault(profile_name, {}) connected = 0 for platform, platform_config in profile_cfg.platforms.items(): @@ -1232,8 +1233,7 @@ class GatewayAdapterLifecycleMixin: """Stamp an owning adapter's profile before resolving busy policy.""" async def _handler(event, _session_key): self._stamp_event_profile(event, profile_name) - routed_session_key = self._session_key_for_source(event.source) - return await self._handle_active_session_busy_message(event, routed_session_key) + return await self._handle_active_session_busy_message(event, self._session_key_for_source(event.source)) return _handler @@ -1274,9 +1274,10 @@ class GatewayAdapterLifecycleMixin: def _primary_message_handler(self): """Return the correctly scoped handler for a primary adapter.""" - if getattr(self.config, "multiplex_profiles", False): - return self._make_default_profile_message_handler() - return self._handle_message + return self._make_default_profile_message_handler() if self._multiplex_on() else self._handle_message + + def _multiplex_on(self) -> bool: + return bool(getattr(self.config, "multiplex_profiles", False)) async def _handle_gateway_platform_event(self, event: dict, source) -> None: """Authorize and publish one normalized adapter event to plugin hooks.""" @@ -1312,7 +1313,7 @@ class GatewayAdapterLifecycleMixin: return _handler def _primary_platform_event_handler(self): - if getattr(self.config, "multiplex_profiles", False): + if self._multiplex_on(): return self._make_default_profile_platform_event_handler() return self._handle_gateway_platform_event @@ -1393,8 +1394,7 @@ class GatewayAdapterLifecycleMixin: its scope; for the shared primary (None) the routed profile is stamped so its pairing store is consulted while allowlist reads stay under the transport home.""" from gateway.run import get_hermes_home - multiplex = bool(getattr(self.config, "multiplex_profiles", False)) - transport_home = (Path(get_hermes_home()) if multiplex and profile_name is None else None) + transport_home = Path(get_hermes_home()) if self._multiplex_on() and profile_name is None else None def check( user_id: str, chat_type: Optional[str] = None, chat_id: Optional[str] = None, *, diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py index c75dbc21ed..23cd75ae65 100644 --- a/gateway/run_notifications.py +++ b/gateway/run_notifications.py @@ -330,8 +330,10 @@ class GatewayNotificationsMixin: # they succeeded — mirrors the ``not agent_result.get("failed")`` completed-turn guard. if not deliver_media: return - synthetic_event = MessageEvent(text="", source=source, message_id=event_message_id) - await self._deliver_media_from_response(response, synthetic_event, adapter, thread_metadata=metadata) + await self._deliver_media_from_response( + response, MessageEvent(text="", source=source, message_id=event_message_id), adapter, + thread_metadata=metadata, + ) def _schedule_update_notification_watch(self) -> None: """Ensure a background task is watching for update completion.""" diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index 19379df62f..ec2a0d79aa 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -155,8 +155,7 @@ class GatewayShutdownMixin: Only the primary API server owns the HTTP listener, so only it is a source of this work. """ - adapter = getattr(self, "adapters", {}).get(Platform.API_SERVER) - helper = getattr(adapter, name, None) + helper = getattr(getattr(self, "adapters", {}).get(Platform.API_SERVER), name, None) return max(0, int(helper(*args))) if callable(helper) else 0 def _active_api_run_count(self) -> int: diff --git a/gateway/run_startup.py b/gateway/run_startup.py index 36e9eb746a..4046e09e8b 100644 --- a/gateway/run_startup.py +++ b/gateway/run_startup.py @@ -882,7 +882,7 @@ class GatewayStartupMixin: (aborted, enabled_platform_count, multiplex_skipped_platforms, pending_connects).""" from gateway.run import _platform_has_bot_credential enabled_platform_count = 0 - _multiplex_on = bool(getattr(self.config, "multiplex_profiles", False)) + _multiplex_on = self._multiplex_on() _multiplex_skipped_platforms: list[Platform] = [] _pending_connects = [] # (platform, platform_config, adapter); connected concurrently later for platform, platform_config in self.config.platforms.items(): From 506796fc81d20c06b88388276f7b71fa8f5bb484 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:56:12 -0700 Subject: [PATCH 08/11] refactor(gateway): fold secondary-reconnect release branches; drop redundant fast path in startup-reconnect handoff --- gateway/run_adapters.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/gateway/run_adapters.py b/gateway/run_adapters.py index ba6265181b..b491fc0523 100644 --- a/gateway/run_adapters.py +++ b/gateway/run_adapters.py @@ -1078,13 +1078,11 @@ class GatewayAdapterLifecycleMixin: and not getattr(adapter, "fatal_error_retryable", True) ): return - except asyncio.CancelledError: - if adapter is not None: - await self._safe_adapter_disconnect(adapter, platform) - raise - except Exception: + except BaseException as exc: if adapter is not None: await self._safe_adapter_disconnect(adapter, platform) + if not isinstance(exc, Exception): + raise # CancelledError (and other BaseExceptions) propagate after release logger.debug( "Secondary %s reconnect attempt failed (profile: %s)", platform.value, profile_name, exc_info=True, @@ -1138,7 +1136,7 @@ class GatewayAdapterLifecycleMixin: ) async def _await_running_then_schedule() -> None: - if self._running: + if self._running: # fast path (also the only path for bare runners without _shutdown_event) _handoff() return # Poll (startup completion has no event); bounded so a wedged startup cannot spin. From fcb0ff29d6a7201fe37e6231e5f4cdc44b84977a Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:58:47 -0700 Subject: [PATCH 09/11] refactor(gateway): AST-neutral string-fragment rewrap to 110 cols --- gateway/run_notifications.py | 9 ++++----- gateway/run_shutdown.py | 15 ++++++--------- gateway/run_startup.py | 24 ++++++++++-------------- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py index 23cd75ae65..e57ec6d393 100644 --- a/gateway/run_notifications.py +++ b/gateway/run_notifications.py @@ -720,11 +720,10 @@ class GatewayNotificationsMixin: from hermes_state import classify_persistence_error, format_session_db_unavailable if classify_persistence_error(error) == "corrupt": message = ( - "⚠️ Session database corruption detected. Messages may not be " - "persisted. Recovery options:\n1. Run `hermes doctor --fix`\n" - "2. Salvage with: sqlite3 ~/.hermes/state.db \".recover\" (then replace state.db)\n" - "3. Restore from a backup in ~/.hermes/backups/\n" - "Run `hermes doctor` for sanitized diagnostics." + "⚠️ Session database corruption detected. Messages may not be persisted. Recovery " + "options:\n1. Run `hermes doctor --fix`\n2. Salvage with: sqlite3 ~/.hermes/state.db " + "\".recover\" (then replace state.db)\n3. Restore from a backup in ~/.hermes/backups/\nRun " + "`hermes doctor` for sanitized diagnostics." ) else: message = ( diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index ec2a0d79aa..824aebb3b9 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -322,9 +322,8 @@ class GatewayShutdownMixin: return # not opted in — normal, stay quiet active = [getattr(p, "value", p) for p in self._scale_to_zero_active_messaging_platforms()] logger.info( - "scale-to-zero: NOT armed despite opt-in — " - "relay_only_or_absent=%s (enabled platforms=%s), wake_url=%s. " - "Need relay-only messaging + a registered wake URL.", + "scale-to-zero: NOT armed despite opt-in — relay_only_or_absent=%s (enabled platforms=%s), " + "wake_url=%s. Need relay-only messaging + a registered wake URL.", messaging_is_relay_only_or_absent(active), active or "none", "set" if self._relay_wake_url_or_none() else "MISSING", ) @@ -1398,9 +1397,8 @@ class GatewayShutdownMixin: ctx.active_agents, ctx.timed_out = await self._drain_active_agents(timeout, _cron_timeout) ctx.drain_elapsed = time.monotonic() - _drain_started_at logger.info( - "Shutdown phase: drain done at +%.2fs (drain took %.2fs, " - "timed_out=%s, active_at_start=%d, active_now=%d, cron_at_start=%d, cron_now=%d, " - "api_at_start=%d, api_now=%d, " + "Shutdown phase: drain done at +%.2fs (drain took %.2fs, timed_out=%s, active_at_start=%d, " + "active_now=%d, cron_at_start=%d, cron_now=%d, api_at_start=%d, api_now=%d, " "deferred_at_start=%d, deferred_now=%d)", ctx.elapsed(), ctx.drain_elapsed, ctx.timed_out, len(ctx.active_agents), self._running_agent_count(), _cron_at_start, self._active_cron_job_count(), _api_at_start, self._active_api_run_count(), @@ -1559,9 +1557,8 @@ class GatewayShutdownMixin: # so skip the close and let SQLite recover from its WAL on next open (a transient # "database is locked" on an immediate --replace at worst, not a corrupt file). logger.warning( - "Shutdown phase: %d executor worker(s) still running after " - "a %.2fs quiesce — skipping the SessionDB close/checkpoint " - "to avoid racing a live write (#101093); handles are left " + "Shutdown phase: %d executor worker(s) still running after a %.2fs quiesce — skipping the " + "SessionDB close/checkpoint to avoid racing a live write (#101093); handles are left " "open for SQLite to recover on next open", _exec_live, _exec_quiesce_budget, ) return diff --git a/gateway/run_startup.py b/gateway/run_startup.py index 4046e09e8b..76e42961d9 100644 --- a/gateway/run_startup.py +++ b/gateway/run_startup.py @@ -222,9 +222,8 @@ class GatewayStartupMixin: return await self._wait_bounded_or_release( {boot_task}, timeout, - "Boot-path sends still running after %.0fs; releasing inbound gate so other " - "platforms are not frozen. Restart notification / obligation redelivery continue " - "in the background.", + "Boot-path sends still running after %.0fs; releasing inbound gate so other platforms are not " + "frozen. Restart notification / obligation redelivery continue in the background.", "background boot-path send failed after gate release: see traceback", track=True, ) @@ -691,9 +690,8 @@ class GatewayStartupMixin: ) else: logger.warning( - "Secret redaction: DISABLED (HERMES_REDACT_SECRETS=%s). " - "API keys and tokens may appear verbatim in chat output, " - "session JSONs, and logs. Set security.redact_secrets: true " + "Secret redaction: DISABLED (HERMES_REDACT_SECRETS=%s). API keys and tokens may appear " + "verbatim in chat output, session JSONs, and logs. Set security.redact_secrets: true " "in config.yaml to re-enable.", _redact_raw, ) with suppress(Exception): @@ -728,11 +726,10 @@ class GatewayStartupMixin: ) if _alignment is not None and _alignment.get("mismatch"): logger.warning( - "Stale systemd unit detected: %s has TimeoutStopSec=%.0fs but " - "drain_timeout=%.0fs cron_drain_timeout=%.0fs (expected >=%.0fs). " - "systemd may SIGKILL the gateway mid-drain. Run " - "`hermes gateway install --force` to regenerate the unit, or " - "shorten agent.restart_drain_timeout / agent.cron_drain_timeout.", + "Stale systemd unit detected: %s has TimeoutStopSec=%.0fs but drain_timeout=%.0fs " + "cron_drain_timeout=%.0fs (expected >=%.0fs). systemd may SIGKILL the gateway " + "mid-drain. Run `hermes gateway install --force` to regenerate the unit, or shorten " + "agent.restart_drain_timeout / agent.cron_drain_timeout.", _alignment.get("unit", "(unknown)"), _alignment["timeout_stop_sec"], _alignment["drain_timeout"], _alignment.get("cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT), @@ -1069,9 +1066,8 @@ class GatewayStartupMixin: for _skipped in _multiplex_skipped_platforms: if not any(_skipped in _profile_map for _profile_map in self._profile_adapters.values()): logger.warning( - "%s is enabled but no profile (default or secondary) " - "provided a bot credential for it — the platform is not " - "being served. Add its token to the profile that should " + "%s is enabled but no profile (default or secondary) provided a bot credential for it — " + "the platform is not being served. Add its token to the profile that should " "own it, or disable the platform.", _skipped.value, ) return False, connected_count From b3ba9d628522b6339aba2088ae800e2c61131569 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:03:17 -0700 Subject: [PATCH 10/11] refactor(gateway): shared logged-send helper for shutdown and cron-interrupt notices --- gateway/run_shutdown.py | 48 ++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index 824aebb3b9..e79c246b08 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -702,17 +702,13 @@ class GatewayShutdownMixin: dedup_key = (job_id, *_notice_target_key(platform.value, chat_id, thread_id)) if dedup_key in notified: continue - try: + with _log_suppressed(logging.DEBUG, "Cron interrupt notice to %s:%s raised: %s", platform.value, chat_id): metadata = self._thread_metadata_for_target(platform, chat_id, thread_id, adapter=adapter) - result = await adapter.send(chat_id, msg, metadata=metadata) - if _send_failed(result): - logger.debug( - "Cron interrupt notice to %s:%s failed: %s", platform.value, chat_id, _send_error(result), - ) - continue - notified.add(dedup_key) - except Exception as e: - logger.debug("Cron interrupt notice to %s:%s raised: %s", platform.value, chat_id, e) + if await self._send_notice_logged( + adapter, chat_id, msg, platform.value, "Cron interrupt notice to %s:%s failed: %s", + "Cron interrupt notice to %s:%s raised: %s", metadata=metadata, + ): + notified.add(dedup_key) if notified: logger.info("Shutdown: delivered %d interrupted-cron-job notice(s)", len(notified)) return len(notified) @@ -742,19 +738,27 @@ class GatewayShutdownMixin: ) -> bool: """Send one shutdown notice; True when delivered. Failures are debug-logged, never raised.""" where = "home channel " if kind == "home channel" else "" - try: - result = await adapter.send(chat_id, msg, **send_kwargs) - if _send_failed(result): - logger.debug( - "Failed to send shutdown notification to %s%s:%s: %s", where, platform_str, chat_id, - _send_error(result), - ) - return False - logger.info("Sent shutdown notification to %s %s:%s", kind, platform_str, chat_id) - return True - except Exception as e: - logger.debug("Failed to send shutdown notification to %s%s:%s: %s", where, platform_str, chat_id, e) + fail_fmt = f"Failed to send shutdown notification to {where}%s:%s: %s" + if not await self._send_notice_logged(adapter, chat_id, msg, platform_str, fail_fmt, **send_kwargs): return False + logger.info("Sent shutdown notification to %s %s:%s", kind, platform_str, chat_id) + return True + + @staticmethod + async def _send_notice_logged( + adapter, chat_id: str, msg: str, platform_str: str, fail_fmt: str, raise_fmt: Optional[str] = None, **kw + ) -> bool: + """``adapter.send`` whose failure is debug-logged as ``fmt % (platform, chat, error)`` — ``fail_fmt`` + for success=False, ``raise_fmt`` (default ``fail_fmt``) for a raise; True only on a delivered send.""" + try: + result = await adapter.send(chat_id, msg, **kw) + except Exception as e: + logger.debug(raise_fmt or fail_fmt, platform_str, chat_id, e) + return False + if _send_failed(result): + logger.debug(fail_fmt, platform_str, chat_id, _send_error(result)) + return False + return True async def _notify_active_sessions_of_shutdown(self) -> None: """Send shutdown/restart notifications to active chats and home channels. From 792960cba1cadd977bf11c37bd9dd136c992aee9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:09:38 -0700 Subject: [PATCH 11/11] refactor(gateway): compact run_* inline comments by hand (keep every WHY/invariant) --- gateway/run_adapters.py | 101 +++++++------------ gateway/run_notifications.py | 122 +++++++++------------- gateway/run_shutdown.py | 142 ++++++++++---------------- gateway/run_startup.py | 190 ++++++++++++----------------------- 4 files changed, 198 insertions(+), 357 deletions(-) diff --git a/gateway/run_adapters.py b/gateway/run_adapters.py index b491fc0523..3ed61e4cbe 100644 --- a/gateway/run_adapters.py +++ b/gateway/run_adapters.py @@ -199,9 +199,8 @@ class GatewayAdapterLifecycleMixin: if not platform_config: return False if adapter.platform in self._failed_platforms: - # Already queued is exactly when the watcher may have died (_spawn_supervised gives up - # after _MAX_SUPERVISED_RESTARTS); without this backstop the platform is a silent - # permanent outage — nothing retries and the stranded check treats "queued" as safe. + # Already queued is exactly when the watcher may have died (supervision gave up); without + # this backstop nothing retries and the stranded check treats "queued" as safe. self._ensure_reconnect_watcher_running() return False self._failed_platforms[adapter.platform] = self._reconnect_queue_entry( @@ -215,8 +214,7 @@ class GatewayAdapterLifecycleMixin: """Run the fatal handler; a platform left stranded (not reconnected, not queued, not intentionally disabled) exits the gateway with failure so the service manager restarts it.""" try: - # Outer hard deadline: a hang anywhere in the impl must not wedge this task — the - # stranded check in ``finally`` only runs when we return. + # Outer hard deadline: the stranded check in ``finally`` only runs when we return. timeout = self._adapter_disconnect_timeout_secs() if timeout <= 0: await self._handle_adapter_fatal_error_impl(adapter) @@ -263,8 +261,7 @@ class GatewayAdapterLifecycleMixin: self._queue_retryable_fatal_platform(adapter) async def _handle_adapter_fatal_error_impl(self, adapter: BasePlatformAdapter) -> None: - # Snapshot the slot owner first: a stale notification must not overwrite a healthy - # platform's runtime status or re-queue it. + # Snapshot the slot owner first: a stale notification must not touch a healthy platform. existing = self.adapters.get(adapter.platform) if existing is not None and existing is not adapter: logger.debug( @@ -287,12 +284,10 @@ class GatewayAdapterLifecycleMixin: error_message=adapter.fatal_error_message, ) if existing is adapter: - # Claim for teardown BEFORE awaiting disconnect(), else a concurrent second fatal - # notification for the same adapter would disconnect() it twice. + # Claim for teardown BEFORE awaiting disconnect(), else a second fatal disconnects it twice. self.adapters.pop(adapter.platform, None) self.delivery_router.adapters = self.adapters - # Queue BEFORE any disconnect await: a half-dead transport can wedge native close() (or - # swallow CancelledError), and "disconnect then queue" left platforms permanently deaf. + # Queue BEFORE any disconnect await: a wedged close() once left platforms permanently deaf. self._queue_retryable_fatal_platform(adapter) if existing is adapter: # Bounded by the shutdown-path timeout so this always returns to the stranded check. @@ -306,8 +301,7 @@ class GatewayAdapterLifecycleMixin: logger.error("No connected messaging platforms remain. Shutting down gateway cleanly.") await self.stop() elif not self.adapters and self._failed_platforms: - # Everything is down but queued: stay alive (cron still runs, watcher recovers) rather - # than turn a transient outage into a state-killing systemd restart loop. + # All down but queued: stay alive (cron runs, watcher recovers) rather than restart-loop. logger.warning( "No connected messaging platforms remain, but %d platform(s) " "queued for reconnection — gateway staying alive, watcher will " @@ -351,11 +345,9 @@ class GatewayAdapterLifecycleMixin: respawn leaves a stale handle and a SECOND watcher; ``on_give_up(name)`` fires at budget end.""" # Spawn timestamp lets ``_done`` tell a rapid crash-loop from a healthy-run-then-crash. _started = time.monotonic() - # No create_task kwargs (test doubles mock a narrow signature); running it from a fresh - # Context gives the same isolation as create_task(..., context=Context()). + # No create_task kwargs (test doubles mock a narrow signature); Context().run isolates instead. task = Context().run(lambda: asyncio.create_task(coro_factory())) - # Marks a PERMANENT watcher so the scale-to-zero idle check ignores it (else the gateway - # counts itself busy forever); transient _background_tasks stay counted. + # PERMANENT watcher: the scale-to-zero idle check ignores it (else busy forever). task._hermes_supervised_watcher = True # type: ignore[attr-defined] self._retain_background_task(task) if on_spawn is not None: @@ -376,8 +368,7 @@ class GatewayAdapterLifecycleMixin: logger.error("Supervised task %s died: %r", name, exc, exc_info=exc) if not (restart and self._running): return - # A healthy run before the crash is a FRESH failure, not a crash-loop: reset so a - # daemon crashing a few times over days is never abandoned. + # A healthy run before the crash is a FRESH failure, not a crash-loop: reset the counter. healthy = time.monotonic() - _started >= self._SUPERVISED_HEALTHY_SECS effective_attempt = 0 if healthy else _attempt if effective_attempt >= self._MAX_SUPERVISED_RESTARTS: @@ -420,8 +411,7 @@ class GatewayAdapterLifecycleMixin: _process_takes_profile = len(_inspect.signature(self._process_handoff).parameters) >= 2 except Exception: _process_takes_profile = False - # In-flight dispatches keyed by session id: a handoff runs a FULL agent turn, so inline - # processing would let one slow handoff time out every other profile's poll. + # In-flight dispatches by session id: a handoff is a FULL agent turn, so never process inline. inflight: Dict[str, "asyncio.Task"] = {} async def _dispatch(row, session_id, session_db, profile_name) -> None: @@ -430,8 +420,7 @@ class GatewayAdapterLifecycleMixin: await self._process_handoff(*((row, profile_name) if _process_takes_profile else (row,))) await session_db.complete_handoff(session_id) except asyncio.CancelledError: - # Gateway shutting down: leave the row 'running' so the next - # start's reclaim marks it failed with a clear reason. + # Leave the row 'running' so the next start's reclaim marks it failed with a clear reason. raise except Exception as exc: logger.warning("Handoff for session %s failed: %s", session_id, exc, exc_info=True) @@ -455,15 +444,13 @@ class GatewayAdapterLifecycleMixin: if not await session_db.claim_handoff(session_id): # Another tick or another gateway already claimed it. continue - # INVARIANT (do not weaken): this task is created inside _profile_runtime_scope but - # typically RUNS after it exits; it sees the profile's home/secret scope only because - # those seams are ContextVar-based and ensure_future copies the Context. + # INVARIANT (do not weaken): created inside _profile_runtime_scope but RUNS after it + # exits; it sees the profile scope only because ensure_future copies the Context. inflight[session_id] = asyncio.ensure_future( _dispatch(row, session_id, session_db, profile_name) ) - # A row still 'running' at startup belongs to a gateway that died mid-dispatch; it blocks - # request_handoff until reclaimed. + # A row still 'running' at startup died mid-dispatch and blocks request_handoff until reclaimed. def _scope(profile_home): # local: tests bind this watcher onto bare SimpleNamespace runners return GatewayAdapterLifecycleMixin._scope_or_null(_async_profile_runtime_scope, profile_home) @@ -648,8 +635,7 @@ class GatewayAdapterLifecycleMixin: self._drop_from_reconnect_queue(platform, "adapter creation returned None") return self._wire_adapter_handlers(adapter) - # Reconnect after outage: keep the platform's server-side update queue so - # messages sent while the bot was offline are delivered rather than dropped. + # is_reconnect keeps the server-side update queue so offline-period messages are delivered. success = await self._connect_adapter_with_timeout(adapter, platform, is_reconnect=True) if success: await self._install_reconnected_adapter(platform, adapter) @@ -659,13 +645,11 @@ class GatewayAdapterLifecycleMixin: "Reconnect %s: non-retryable error (%s), removing from retry queue", platform.value, adapter.fatal_error_message, ) - # Never installed on self.adapters, so nothing else will disconnect() it: dispose or - # the resource owners built in __init__ leak ~2 fds each (fd limit in ~12h at the cap). + # Never installed on self.adapters: dispose here or its __init__ resources leak ~2 fds each. await _dispose_unused_adapter(adapter) del self._failed_platforms[platform] else: - # Retryable failures retry at the backoff cap forever; never auto-pause them — a - # transient outage must not need `/platform resume`. Same fd-leak dispose as above. + # Retryable failures retry at the cap forever (never auto-pause). Same fd-leak dispose. backoff = self._bump_reconnect_backoff( platform, info, attempt, adapter.fatal_error_code, adapter.fatal_error_message or "failed to reconnect", @@ -700,8 +684,7 @@ class GatewayAdapterLifecycleMixin: needs_attention=False, retrying_since=None, ) logger.info("✓ %s reconnected successfully", platform.value) - # Final responses rejected while this adapter was down are still owned by this live - # process, so startup recovery cannot claim them. Replay the transient subset now. + # Responses rejected while down are owned by this live process (startup recovery cannot claim them). with _log_suppressed( logging.DEBUG, "failed-obligation redelivery after %s reconnect failed", platform.value, exc_info=True, @@ -711,8 +694,7 @@ class GatewayAdapterLifecycleMixin: with suppress(Exception): from gateway.channel_directory import build_channel_directory await build_channel_directory(self.adapters) - # A platform offline at gateway startup never got its restart-interrupted sessions - # auto-resumed — the startup pass skips sessions whose adapter isn't connected yet. + # A platform offline at startup skipped its restart-interrupted sessions; resume them now. try: self._schedule_resume_pending_sessions(platform=platform) except Exception: @@ -816,15 +798,13 @@ class GatewayAdapterLifecycleMixin: ) from gateway.config import load_gateway_config from hermes_cli.env_loader import hydrate_profile_secret_sources - # Hydrate external secret sources (1Password/vault/...) off-loop ONCE: the sync hydration is - # network-bound and would stall every other profile's heartbeat while this one boots. + # Hydrate external secret sources off-loop ONCE: sync hydration would stall every heartbeat. await asyncio.to_thread(hydrate_profile_secret_sources, profile_home) with _profile_runtime_scope(profile_home, hydrate_secrets=False): profile_runtime_cfg = _load_gateway_runtime_config() from hermes_cli.plugins import discover_plugins discover_plugins() - # Register this profile's shell hooks / outbound webhooks: start() registers before any - # profile scope exists, so a secondary profile's `hooks:` block would be silently inert. + # This profile's `hooks:` block: start() registered before any profile scope existed. self._register_config_hooks( "shell-hook/webhook registration failed for profile '%s'", profile_name, level=logging.WARNING, ) @@ -897,16 +877,14 @@ class GatewayAdapterLifecycleMixin: for platform, platform_config in profile_cfg.platforms.items(): if not platform_config.enabled: continue - # Enabled in the shared YAML but no credential in THIS profile's scope: building an - # adapter anyway would fan one inbound message out across every credential-less profile. + # No credential in THIS profile's scope: an adapter would fan inbound across every such profile. if multiplex and not _platform_has_bot_credential(platform, platform_config): logger.info( "[MULTIPLEX] Profile '%s': skipping %s - no bot credential " "in this profile's secrets", profile_name, platform.value, ) continue - # Relay/WhatsApp are shared process-level ingress under multiplex (the active profile's - # one connection fans out by route); a secondary would only retry-loop and stall startup. + # Relay/WhatsApp are shared process-level ingress under multiplex; a secondary would retry-loop. if multiplex and platform in (Platform.RELAY, Platform.WHATSAPP): continue adapter = None @@ -978,12 +956,10 @@ class GatewayAdapterLifecycleMixin: self, adapter: BasePlatformAdapter, profile_name: str, platform: Platform ) -> None: """Install the profile-scoped handlers shared by startup and reconnect.""" - # Runtime status is process-scoped while message/config work is profile-scoped. Keep both - # dimensions in the key so dashboard/NAS health aggregation sees which secondary failed. + # Runtime status is process-scoped: key on profile:platform so health shows WHICH secondary failed. adapter._runtime_status_platform_key = f"{profile_name}:{platform.value}" - # Declare ownership BEFORE any inbound event: adapter-level session keys are derived at - # ingress, before the handler stamps source.profile — without this every secondary bot - # would key into the default profile's `agent:main:` lane. + # Declare ownership BEFORE any inbound event: adapter-level session keys are derived at ingress, + # before the handler stamps source.profile (else every secondary keys into `agent:main:`). _set_owner = getattr(adapter, "set_owner_profile", None) if callable(_set_owner): _set_owner(profile_name) @@ -1001,11 +977,9 @@ class GatewayAdapterLifecycleMixin: else self._busy_text_mode ), ) - # Voice transcripts from this bot's channels dispatch through THIS adapter (primary - # wiring lives at connect time). + # Voice transcripts from this bot's channels dispatch through THIS adapter. self._bind_voice_input_callback(adapter) - # Secondary adapters always carry the profile they serve so prune paths namespace topic - # bindings correctly under multiplex. + # Secondary adapters carry their profile so prune paths namespace topic bindings correctly. adapter._hermes_profile_name = profile_name async def _secondary_reconnect_attempt(self, profile_name: str, platform: Platform): @@ -1069,9 +1043,8 @@ class GatewayAdapterLifecycleMixin: platform, profile=profile_name ) return - # Not installed: a newer reconnect won the slot mid-connect (never replace it), - # shutdown began mid-connect (never republish after the registry drained), or - # connect failed. Release partial resources; stop only for a non-retryable fatal. + # Not installed (newer reconnect won the slot, shutdown began, or connect failed): + # release partial resources; stop only for a non-retryable fatal. await self._safe_adapter_disconnect(adapter, platform) if success or ( getattr(adapter, "has_fatal_error", False) @@ -1114,8 +1087,7 @@ class GatewayAdapterLifecycleMixin: if not getattr(adapter, "fatal_error_retryable", True): return if is_global_startup_conflict(getattr(adapter, "fatal_error_code", None)): - # A live foreign holder of this profile's token is an ownership conflict, not a blip: - # park it fatal (like ``duplicate_credential``) instead of retry-storming. + # A live foreign token holder is an ownership conflict, not a blip: park it fatal. logger.error( "[MULTIPLEX] Profile '%s': %s credential is held by another " "gateway (%s) — parked, not retried. %s", profile_name, platform.value, @@ -1128,8 +1100,7 @@ class GatewayAdapterLifecycleMixin: try: self._schedule_secondary_profile_reconnect(profile_name, platform, adapter) except Exception: - # The handoff touches live registries; if it raises, the parked task dies as an - # unretrieved-task exception logged only at GC. Surface it where operators look. + # A raise here would die as an unretrieved-task exception logged only at GC; surface it. logger.exception( "secondary-startup-reconnect handoff failed (profile=%s platform=%s)", profile_name, platform.value, @@ -1339,8 +1310,7 @@ class GatewayAdapterLifecycleMixin: def _adapter_credential_fingerprint(adapter: Any) -> Optional[str]: """Salted, log-safe hash of an adapter's credential; None when none is discoverable (conflict detection is then skipped).""" - # Many adapters (e.g. Discord) keep the token on `config`; without that fallback the - # same-token check is silently skipped and every profile polls the same bot. + # Many adapters (Discord) keep the token on `config`; without that fallback the check is skipped. candidates = [ (adapter, attr) for attr in ( "token", "bot_token", "_token", "api_token", "_bot_token", @@ -1404,8 +1374,7 @@ class GatewayAdapterLifecycleMixin: platform=platform, chat_id=chat_id or "", chat_type=chat_type or "group", user_id=user_id, thread_id=thread_id, is_bot=bool(is_bot), profile=profile_name, ) - # Same transport provenance ``build_source`` retains, so adapter-level policy reads - # resolve the receiving adapter even once the routed profile is stamped below. + # Same transport provenance as ``build_source``, so policy reads resolve the receiving adapter. registry = ( (getattr(self, "_profile_adapters", None) or {}).get(profile_name) if profile_name else getattr(self, "adapters", None) diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py index e57ec6d393..494fc934a7 100644 --- a/gateway/run_notifications.py +++ b/gateway/run_notifications.py @@ -39,8 +39,7 @@ _DURABLE_CLAIM_OPS = { class GatewayNotificationsMixin: """Process/completion/update notifications, media delivery and async-delegation delivery for GatewayRunner.""" - # Routing-complete coalescing keys: process completions (short-window fan-in) and async - # delegations (originating session + parent session + route). + # Coalescing keys: process completions (short-window fan-in) and async delegations (+ parent session). _COMPLETION_BATCH_KEY_FIELDS = ("session_key", "platform", "chat_type", "chat_id", "thread_id", "user_id") _ASYNC_GROUP_KEY_FIELDS = ("session_key", "parent_session_id", *_COMPLETION_BATCH_KEY_FIELDS[1:]) @@ -141,8 +140,7 @@ class GatewayNotificationsMixin: return None route_owns_lineage = session_entry.session_id in {pinned_session_id, target_session_id} if not route_owns_lineage: - # A delegation may survive several compression rotations: accept an intermediate stale - # route only when its own verified compression tip is the same live target. + # Across several rotations, accept a stale route only when its own tip is the same live target. try: route_row = await session_db.get_session(session_entry.session_id) route_tip = ( @@ -204,9 +202,8 @@ class GatewayNotificationsMixin: ) return None if _end_reason != "compression": - # Idle/timeout/lifecycle end (scale-to-zero norm): the chat route is still valid and - # ``session_entry`` is its current session, so deliver here rather than drop — otherwise - # the row is acked at adapter acceptance then silently lost. + # Idle/timeout end (scale-to-zero norm): the chat route is still valid, so deliver to its + # current session rather than drop (the row would be acked then silently lost). logger.info( "Async-delegation completion pinned to %s-ended session %s; " "retargeting to the chat's current session %s.", @@ -250,13 +247,12 @@ class GatewayNotificationsMixin: directive is a deliberate attach); stale auto-appended tags are deduped upstream.""" from urllib.parse import quote as _quote with _log_suppressed(logging.WARNING, "Post-stream media extraction failed: %s"): - # Capture [[as_document]] before extract_media strips it: image files then go through - # send_document (preserving bytes), not send_multiple_images (Telegram recompresses). + # Capture [[as_document]] before extract_media strips it: images then go via send_document. force_document_attachments = "[[as_document]]" in response from gateway.platforms.base import BasePlatformAdapter, should_send_media_as_audio media_files, cleaned = adapter.extract_media(response) media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files) - # Strip image URLs for parity with the non-streaming chain; no extract_local_files here. + # Strip image URLs (parity with the non-streaming chain); no extract_local_files here. adapter.extract_images(cleaned) _thread_meta = ( dict(thread_metadata) @@ -302,9 +298,8 @@ class GatewayNotificationsMixin: if not text_already_delivered: text_content = _strip_response_attachments_for_direct_send(response, adapter) if text_content: - # Reconcile-by-edit first: a stream-sealed message whose recorded payload didn't confirm - # the final (post-stream mutation) already carries most of the answer; a plain send here - # would duplicate it. + # Reconcile-by-edit first: a stream-sealed message already carries most of the answer; + # a plain send here would duplicate it. _reconciled = False _sc_msg_id = getattr(stream_consumer, "message_id", None) if ( @@ -388,8 +383,7 @@ class GatewayNotificationsMixin: async def _watch_update_completion_only(self, paths: "_UpdatePaths", deadline: float, poll_interval: float) -> None: """Fallback when no adapter/chat can be resolved: wait for the exit code, then notify.""" logger.warning("Update watcher: cannot resolve adapter/chat_id, falling back to completion-only") - # Keep polling until _send_update_notification actually delivers (True) — it re-resolves the - # adapter each call and returns False (markers kept) while the platform is still reconnecting. + # Poll until _send_update_notification delivers (it returns False while the platform reconnects). loop = asyncio.get_running_loop() while paths.any_pending() and loop.time() < deadline: if paths.exit_code.exists() and await self._send_update_notification(): @@ -443,8 +437,7 @@ class GatewayNotificationsMixin: f"⚕ **Update needs your input:**\n\n{prompt_text}{default_hint}\n\n" f"Reply `{_p}approve` (yes) or `{_p}deny` (no), or type your answer directly." ) - # Keep the prompt marker on disk until the user answers so a watcher after a mid-prompt - # gateway restart can recover by re-forwarding it. + # Keep the prompt marker on disk until answered so a restarted watcher can re-forward it. self._session_state(target.session_key).persistent.update_prompt_pending = True logger.info("Forwarded update prompt to %s: %s", target.session_key, prompt_text[:80]) @@ -504,8 +497,7 @@ class GatewayNotificationsMixin: _read_new_output() if buffer.strip() and (loop.time() - last_stream_time) >= stream_interval: await _flush_buffer() - # Forward a prompt only when none is still awaiting a response; otherwise the watcher - # would re-read the same .update_prompt.json every poll and spam duplicate prompts. + # Forward a prompt only when none is pending, else every poll re-forwards the same prompt. _pending_state = self._peek_session_state(session_key) if session_key else None if paths.prompt.exists() and session_key and not getattr( getattr(_pending_state, "persistent", None), "update_prompt_pending", False @@ -566,9 +558,8 @@ class GatewayNotificationsMixin: platform = Platform(platform_str) adapter = self.adapters.get(platform) if chat_id and not adapter: - # The update finished, but the target platform has not reconnected yet (common right - # after the restart that `hermes update` triggers). A definitive skip would delete the - # markers and silently lose the notification; preserve them for a later retry. + # Target platform not reconnected yet (common right after the update's restart): keep the + # markers for a later retry instead of silently losing the notification. return _defer("Update notification deferred: %s adapter not connected yet", platform_str) if chat_id: metadata = self._pending_marker_metadata(platform, chat_id, pending, adapter) @@ -869,8 +860,7 @@ class GatewayNotificationsMixin: from gateway.wake import adapter_supports_push source = await asyncio.to_thread(self._build_process_event_source, evt) if not source: - # API-server sessions bind the RAW X-Hermes-Session-Id key (_bind_api_server_session), not a - # structured ``agent:main:...`` key, so _build_process_event_source returned None above. + # API-server sessions bind the RAW X-Hermes-Session-Id key, not a structured ``agent:...`` key. raw_sid = str(evt.get("origin_session_id") or "").strip() _sk = str(evt.get("session_key") or "").strip() if not raw_sid and _sk and _parse_session_key(_sk) is None: @@ -894,9 +884,8 @@ class GatewayNotificationsMixin: if not adapter: return None if not adapter_supports_push(adapter): - # Non-push adapter (api_server) resolved WITH routing metadata: its chat_id is the raw session - # id (_bind_api_server_session binds chat_id = session_id), so handle_message would run the - # wake under a build_session_key() key that never matches the raw session — self-post. + # Non-push adapter (api_server): its chat_id IS the raw session id, so handle_message would + # key the wake under a build_session_key() that never matches — self-post instead. raw_sid = str(evt.get("origin_session_id") or "").strip() or str(source.chat_id or "") return await self._self_post_api_server(adapter, synth_text, raw_sid, evt) try: @@ -912,9 +901,8 @@ class GatewayNotificationsMixin: "Watch pattern notification — injecting for %s chat=%s thread=%s", platform_name, source.chat_id, source.thread_id, ) - # Relay-plane egress priming: a synthetic turn injected right after a restart reaches a relay - # adapter whose per-chat routing caches are cold (they warm only on inbound), so its replies - # egress without tenant discriminators and the connector's fail-closed guard declines them. + # Relay egress priming: post-restart routing caches are cold (they warm only on inbound), so + # replies would egress without tenant discriminators and be declined by the connector. _prime = getattr(adapter, "prime_routing_cache", None) if callable(_prime): _prime(synth_event) @@ -985,15 +973,13 @@ class GatewayNotificationsMixin: return "deliver" end_reason = str(parent.get("end_reason") or "") if end_reason != "compression": - # Only a USER-closed thread of work (/new, user_exit, session_switch) is unreachable; - # idle/timeout ends are normal on scale-to-zero relays — the chat stays routable and the - # resolver retargets. Boundary set shared with the resolver: no drift. + # Only a USER-closed session (/new, user_exit, session_switch) is unreachable; idle/timeout + # ends stay routable and the resolver retargets. Boundary set shared with the resolver. return "terminal" if end_reason in _USER_BOUNDARY_END_REASONS else "deliver" try: tip_session_id = await session_db.get_compression_tip(parent_session_id) if not tip_session_id or tip_session_id == parent_session_id: - # Rotation caught mid-flight: parent is compression-ended but its continuation isn't - # visible yet. Retry, don't drop. + # Rotation mid-flight: continuation not visible yet. Retry, don't drop. return "retry" tip = await session_db.get_session(tip_session_id) except Exception: @@ -1038,8 +1024,7 @@ class GatewayNotificationsMixin: elif evt_type != "completion": return claim # Background completions carry only session_key, so after /new the OLD session's notification - # would land in the chat's NEW session. Stamped events get the same pre-flight as async - # delegations; unstamped ones deliver. + # would land in the NEW one. Stamped events get the async-delegation pre-flight; unstamped deliver. parent_session_id = str(evt.get("parent_session_id") or "").strip() if not parent_session_id: return claim @@ -1062,8 +1047,7 @@ class GatewayNotificationsMixin: ) claim.proceed = False elif verdict == "retry": - # Transient uncertainty (session DB down / compression rotation mid-flight): tell the - # watcher to re-poll and retry rather than drop or misroute the result. + # Transient uncertainty: tell the watcher to re-poll rather than drop or misroute. if claim.claim_id: self._settle_durable_claim("release", claim.delegation_id, claim.claim_id) claim.proceed, claim.early_result = False, False @@ -1090,8 +1074,7 @@ class GatewayNotificationsMixin: if identity is not None: with self._completion_delivery_lock: self._mark_completions_delivered_locked((identity,)) - # The durable async-delegation row is the authoritative replay state — ack it after - # adapter acceptance; no parallel ledger here. + # The durable row is the authoritative replay state — ack it after adapter acceptance. if claim.claim_id: try: from tools.async_delegation import complete_completion_delivery @@ -1124,8 +1107,7 @@ class GatewayNotificationsMixin: session_id = str(evt.get("session_id") or "unknown") exit_code = evt.get("exit_code") reason = str(evt.get("completion_reason") or "exited") - # Completion output normally passes the terminal redactor at the producer seam, but that is - # configurable and this is user-facing, so keep the unconditional gateway floor. Redact + # Unconditional gateway redaction floor (the producer-seam redactor is configurable). Redact # BEFORE slicing: truncating first can leave a credential fragment the patterns miss. output = _redact_gateway_user_facing_secrets(str(evt.get("output") or "")).strip() if len(output) > 800: @@ -1156,15 +1138,14 @@ class GatewayNotificationsMixin: try: await asyncio.sleep(self._completion_notification_batch_window) entries = self._completion_notification_batches.pop(key, []) - # Detach before adapter delivery: a completion arriving while this batch is in flight - # must be able to schedule the next flush. + # Detach before delivery so a completion arriving mid-flight can schedule the next flush. if self._completion_notification_batch_tasks.get(key) is current_task: self._completion_notification_batch_tasks.pop(key, None) if not entries: return synth_text = entries[0][0] if len(entries) == 1 else self._format_coalesced_process_completions(entries) - # A duplicate primary can legitimately return None from the lifecycle dedupe seam; try the - # next batch identity so a fresh sibling is never discarded with that duplicate. + # A duplicate primary returns None from the dedupe seam; try the next identity so a fresh + # sibling is never discarded with it. delivered = None for _text, candidate_evt, _future in entries: delivered = await self._deliver_completion_notification(synth_text, candidate_evt) @@ -1173,8 +1154,7 @@ class GatewayNotificationsMixin: if delivered is True and len(entries) > 1: self._record_coalesced_completion_siblings([evt for _text, evt, _future in entries]) except asyncio.CancelledError: - # Shutdown may cancel us mid fan-in or while adapter delivery is blocked: recover entries not - # yet detached and resolve every waiter as retryable before adapters are torn down. + # Shutdown cancellation: recover undetached entries and resolve every waiter as retryable. delivered = False if not entries: entries = self._completion_notification_batches.pop(key, []) @@ -1183,8 +1163,7 @@ class GatewayNotificationsMixin: logger.exception("Coalesced process completion delivery failed") delivered = False finally: - # Never strand watcher futures when formatting, delivery, or cancellation interrupts a batch: - # False follows the existing watcher retry path; None remains the ordinary dedupe result. + # Never strand watcher futures: False = watcher retry path; None = ordinary dedupe result. self._settle_batch_waiters(entries, delivered) # Do not remove a newer flush task that reused the same route key. if self._completion_notification_batch_tasks.get(key) is current_task: @@ -1218,8 +1197,7 @@ class GatewayNotificationsMixin: async def _enqueue_process_completion_notification(self, synth_text: str, evt: dict) -> Optional[bool]: """Fan in concurrent process completions that share one conversation.""" - # Some unit tests construct GatewayRunner with object.__new__. Keep the - # batching seam lazy so those focused lifecycle tests remain valid. + # Lazy defaults: lifecycle tests build GatewayRunner via object.__new__. for attr, default in ( ("_completion_notification_batches", dict), ("_completion_notification_batch_tasks", dict), ("_completion_notification_batch_flush_tasks", set), @@ -1295,8 +1273,7 @@ class GatewayNotificationsMixin: for evt, synth_text in deliverable[1:]: claim_id = claim_event_delivery(evt, f"gateway-batch:{id(self)}") if claim_id is None: - # Another consumer owns this row's delivery; keep its result out of our - # consolidated text so it is never double-injected. + # Another consumer owns this row: keep it out of our text so it is never double-injected. continue siblings.append((evt, claim_id)) blocks.append(synth_text) @@ -1319,12 +1296,10 @@ class GatewayNotificationsMixin: ) self._record_coalesced_completion_siblings([evt for evt, _claim_id in siblings]) else: - # Not delivered — release every sibling claim so a retry or another consumer can claim - # it, honestly leaving the durable rows pending. + # Not delivered: release every sibling claim so a retry or another consumer can take it. self._settle_sibling_claims(siblings, release_event_delivery, "Could not release coalesced durable claim") if delivered is None: - # The primary was dropped/owned elsewhere but the siblings still need delivery — - # requeue just them for the next tick. + # Primary dropped/owned elsewhere: requeue just the siblings for the next tick. for evt, _claim_id in siblings: _pr.completion_queue.put(evt) return delivered @@ -1339,8 +1314,7 @@ class GatewayNotificationsMixin: from tools.process_registry import process_registry as _pr while self._running: with _log_suppressed(logging.DEBUG, "Async delegation watcher error: %s"): - # Peek for async-delegation events only; watch/completion events belong to other drains, - # so requeue anything that isn't ours. + # Take async-delegation events only; requeue watch/completion events for their own drains. requeue = [] async_events = [] while not _pr.completion_queue.empty(): @@ -1351,9 +1325,8 @@ class GatewayNotificationsMixin: (async_events if evt.get("type") == "async_delegation" else requeue).append(evt) for evt in requeue: _pr.completion_queue.put(evt) - # A same-tick drain often carries several completions for the SAME session (a fan-out - # finishing together); delivering each individually floods it with N synthetic turns. - # Group by full gateway route + parent session: one consolidated turn per group. + # A fan-out finishing together yields N completions for one session; group by full route + + # parent session so each group becomes ONE consolidated turn. groups: dict[tuple[str, ...], list[dict]] = {} for evt in async_events: self._enrich_async_delegation_routing(evt) @@ -1378,9 +1351,8 @@ class GatewayNotificationsMixin: if new_output: from agent.redact import redact_terminal_output new_output = redact_terminal_output(new_output, getattr(session, "command", "") or "") - # redact_terminal_output() is unforced, so it returns raw text when security.redact_secrets - # is off. This goes straight to the platform adapter, so it needs the same unconditional - # floor as agent-notify. + # redact_terminal_output() is unforced (raw when security.redact_secrets is off); this goes + # straight to the adapter, so apply the same unconditional floor as agent-notify. new_output = _redact_gateway_user_facing_secrets(new_output) return new_output @@ -1403,8 +1375,7 @@ class GatewayNotificationsMixin: _command = getattr(session, "command", "") or "" _raw = strip_ansi(session.output_buffer) if session.output_buffer else "" _raw = redact_terminal_output(_raw, _command) - # Truncate on line boundaries (never start mid-line): keep the last ~2000 chars snapped to - # the preceding newline, prepending a marker when output was cut. + # Keep the last ~2000 chars snapped to a line boundary, with a marker when cut. _LIMIT = 2000 if len(_raw) > _LIMIT: _tail = _raw[-_LIMIT:] @@ -1424,8 +1395,7 @@ class GatewayNotificationsMixin: "completion_reason": getattr(session, "completion_reason", "exited"), "termination_source": getattr(session, "termination_source", ""), "output": _redact_gateway_user_facing_secrets(_out), - # Spawning conversation's session-db id (stamped in terminal_tool); lets delivery - # pre-flight drop this completion if the user closed that session (/new) first. + # Spawning session-db id: lets pre-flight drop this completion if the user /new'd first. "parent_session_id": ( watcher.get("parent_session_id") or getattr(session, "parent_session_id", "") or "" ), @@ -1477,9 +1447,8 @@ class GatewayNotificationsMixin: has_new_output = current_output_len > last_output_len last_output_len = current_output_len if session.exited: - # Agent-triggered completion: inject a synthetic message unless the agent already consumed - # the result via wait/log. poll() is read-only and deliberately does NOT mark consumed — - # a status check must not suppress this delivery turn. + # Agent-notify: inject a synthetic message unless the agent already consumed the result via + # wait/log (poll() is read-only and deliberately does NOT mark consumed). if agent_notify and not process_registry.is_completion_consumed(session_id): completion_evt = self._build_process_completion_event(watcher, session, session_id) synth_text = format_process_notification(completion_evt) @@ -1491,9 +1460,8 @@ class GatewayNotificationsMixin: # of suppressing the result. continue break - # Normal text-only notification. Skip when the agent already consumed this completion via - # wait/log (output returned inline) — the raw "finished" message would be a duplicate. - # The agent_notify skip FALLS THROUGH here, hence this check. poll() is read-only. + # Text-only notification; skip when already consumed via wait/log (the agent_notify branch + # FALLS THROUGH here, hence the re-check). if process_registry.is_completion_consumed(session_id): logger.debug( "Process watcher: completion for %s already consumed " diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index e79c246b08..75121e9025 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -332,9 +332,8 @@ class GatewayShutdownMixin: def _scale_to_zero_is_idle(self) -> bool: from gateway.scale_to_zero import is_idle - # FULL work aggregate (cron + API runs live outside _running_agents) with fail-AWAKE reads: - # the drain counters swallow errors to 0, which for a suspend predicate would look idle, so - # an unreadable source counts as work here. + # FULL work aggregate with fail-AWAKE reads: the drain counters swallow errors to 0, which a + # suspend predicate would read as idle, so an unreadable source counts as work here. def _read_or_awake(label: str, fn: Callable[[], Any], busy_sentinel: Any) -> Any: try: @@ -345,9 +344,8 @@ class GatewayShutdownMixin: cron_count = _read_or_awake("cron work count", self._running_cron_job_count, 1) api_count = _read_or_awake("api work count", lambda: self._api_server_hook("active_agent_work_count"), 1) - # An attached dashboard/desktop/TUI client (heartbeat file mtime from the dashboard process) is - # inbound activity too — folded into the inbound clock, not a conjunct, so disconnect gets the - # same idle_timeout grace as a message and a lingering marker cannot pin the box. + # An attached dashboard/desktop/TUI client (heartbeat mtime) is inbound activity — folded into + # the inbound clock, not a conjunct, so a lingering marker cannot pin the box. last_inbound = self._last_inbound_at from gateway.scale_to_zero import dashboard_client_last_seen seen = _read_or_awake("dashboard heartbeat", dashboard_client_last_seen, time.time()) @@ -426,9 +424,8 @@ class GatewayShutdownMixin: # After a wake the drained inbound updates _last_inbound_at; give it a window so we # don't immediately re-go-dormant on the same idle reading before traffic lands. self._scale_to_zero_cooldown_until = time.time() + max(interval, 60.0) - # Self-suspend ONLY after a clean quiesce: the relay flip (buffered delivery + wake - # poke armed) must be set before the freeze, or inbound black-holes while we sleep. - # Re-check idle one last time — inbound may have landed during the quiesce await. + # Self-suspend ONLY after a clean quiesce (else inbound black-holes while we sleep), and + # re-check idle — inbound may have landed during the quiesce await. if not dormant_ok: continue if not self._scale_to_zero_is_idle(): @@ -457,9 +454,8 @@ class GatewayShutdownMixin: except Exception: # noqa: BLE001 - suspend is best-effort, never crash logger.debug("scale-to-zero: self-suspend failed", exc_info=True) - # External drain control: the dashboard's begin/cancel-drain endpoint writes/removes the - # ``.drain_request.json`` marker (gateway/drain_control.py); the watcher flips the gateway - # between accepting and refusing NEW turns WITHOUT exiting (reversible on marker removal). + # External drain control: the dashboard writes/removes ``.drain_request.json`` (gateway/drain_control.py); + # the watcher flips between accepting and refusing NEW turns WITHOUT exiting (reversible). def _enter_external_drain(self) -> None: """Begin external drain: refuse NEW turns (in-flight ones are NOT interrupted). Idempotent.""" if self._external_drain_active: @@ -469,8 +465,7 @@ class GatewayShutdownMixin: "External drain ENGAGED (.drain_request.json present) — refusing " "new turns; %d in-flight turn(s) will finish. Process stays up.", self._active_work_count(), ) - # Flip persisted lifecycle state so /api/status.gateway_busy / gateway_drainable track the - # drain; active_agents is preserved (read-merge keeps the live count), only state changes. + # Persist "draining" so /api/status tracks it; active_agents is read-merged, only state changes. self._update_runtime_status("draining") def _exit_external_drain(self) -> None: @@ -534,7 +529,7 @@ class GatewayShutdownMixin: return info["paused"] = True info["pause_reason"] = reason or "auto-paused after repeated failures" - # Push next_retry far enough out that a stale code path missing "paused" still never fires. + # next_retry=inf: a stale code path missing "paused" still never fires. info["next_retry"] = float("inf") self._update_platform_runtime_status( platform.value, platform_state="paused", error_code=None, error_message=info["pause_reason"], @@ -582,14 +577,12 @@ class GatewayShutdownMixin: self._update_runtime_status("draining") last_counts, last_status_at = counts, now - # Cron/API-server/deferred work lives outside ``self._running_agents`` — fold it into this - # wait, or a cron job's tool work is killed without warning once it's the only thing running. + # Cron/API/deferred work lives outside ``_running_agents``; fold it in or it is killed unwarned. _cron0, _api0, _deferred0 = last_counts[1:] _maybe_update_status(force=True) if not self._running_agents and not (_cron0 or _api0 or _deferred0): return snapshot, False - # Cron has its own deadline: ``restart_drain_timeout`` defaults to 0 (an interrupted chat turn - # is announced and resumable) but a cron run killed mid-flight is a permanent failure. + # Cron has its own deadline: a chat turn is announced+resumable; a killed cron run is a permanent failure. started = loop.time() deadline = started + timeout cron_deadline = started + (timeout if cron_timeout is None else cron_timeout) @@ -599,8 +592,7 @@ class GatewayShutdownMixin: agents, cron, api, deferred = self._drain_work_counts() return bool(((agents or api or deferred) and now < deadline) or (cron and now < cron_deadline)) - # Both budgets at 0 leave this loop unentered ("interrupt immediately") as an expired deadline, - # not a special case, so timed_out below is always computed from real state. + # Both budgets at 0 = an expired deadline (loop unentered), so timed_out still comes from real state. while _still_draining(): _maybe_update_status() await asyncio.sleep(0.1) @@ -677,9 +669,8 @@ class GatewayShutdownMixin: job = get_job(job_id) if not job: continue - # deliver=local jobs, and deliver=origin jobs with no resolvable origin, resolve to zero - # targets and must stay silent rather than fall back to a home channel. Interrupted - # notices are failure-category engine status, so they honor failure_deliver. + # deliver=local / unresolvable-origin jobs resolve to zero targets and stay silent (no home- + # channel fallback). Interrupted notices are failure-category status: honor failure_deliver. targets = _resolve_delivery_targets(job, for_failure=True) except Exception as e: logger.debug("Cron interrupt targets unresolved for %s: %s", job_id, e) @@ -809,10 +800,8 @@ class GatewayShutdownMixin: if self._restart_requested and restart_source is not None: logger.debug("Skipping home-channel shutdown notifications for in-chat restart") return - # Suppress ONLY the home-channel broadcast when the drain asked to be quiet (routine fleet - # auto-update); per-session pings above stay (empty on a drained shutdown, useful on a - # force-interrupt). Current-epoch marker only, so an orphaned marker can't silence a fresh gateway. - # Never let the suppression check block the broadcast — fail toward the louder behaviour. + # A quiet drain (routine fleet auto-update) suppresses ONLY the home-channel broadcast; per-session + # pings above stay. Current-epoch marker only; a failing check fails toward the louder behaviour. with _log_suppressed(logging.DEBUG, "drain_notification_suppressed check failed: %s"): from gateway.drain_control import drain_notification_suppressed if drain_notification_suppressed(): @@ -966,9 +955,8 @@ class GatewayShutdownMixin: return with suppress(Exception): if hasattr(agent, "shutdown_memory_provider"): - # Drain queued memory writes BEFORE teardown: shutdown_all() gives the memory worker - # only ~5s, so a /reset or rotation could drop handed-off writes (stale memory next - # session). Bounded, mirrors CLI exit; a failure never blocks teardown. + # Drain queued memory writes BEFORE teardown (shutdown_all() gives the worker only ~5s, so a + # /reset or rotation could drop them). Bounded; a failure never blocks teardown. _mm = getattr(agent, "_memory_manager", None) if _mm is not None and hasattr(_mm, "flush_pending"): with suppress(Exception): @@ -1236,22 +1224,19 @@ class GatewayShutdownMixin: self._restart_detached = detached self._restart_via_service = via_service self._restart_task_started = True - # Refuse new turns while in-flight work finishes. Keep ``_running`` True so adapters stay - # connected and the active turn can still deliver its final response. + # Refuse new turns; keep ``_running`` True so the active turn can still deliver its final response. self._draining = True async def _run_restart() -> None: await self._await_active_work_before_restart() - # Launch the detached helper only AFTER the after-turn wait: its drain_timeout+5 deadline - # covers stop() teardown; earlier it would fire the restart mid-turn. + # Detached helper only AFTER the after-turn wait, or its drain_timeout+5 deadline fires mid-turn. if detached: with _log_suppressed(logging.ERROR, "Failed to launch detached gateway restart helper: %s"): await self._launch_detached_restart_command() await asyncio.sleep(0.05) await self.stop(restart=True, detached_restart=detached, service_restart=via_service) - # NOT in _background_tasks: _stop_impl cancels every entry there, which would propagate - # CancelledError into _stop_impl and skip _shutdown_event.set() / _exit_code = 75. + # NOT in _background_tasks: _stop_impl cancels those, which would skip _shutdown_event.set() / exit 75. self._restart_task = asyncio.create_task(_run_restart()) return True @@ -1309,9 +1294,8 @@ class GatewayShutdownMixin: _count_step("Shutdown (%s): killed %d tool subprocess(es)", process_registry.kill_all) def _mark_cron_interrupted() -> list: - # kill_all() is a global sweep, so any cron job dispatched right now lost its tool - # subprocess; its agent thread may still emit a plausible response from truncated - # output. Mark the run interrupted so it can never be reported as success. + # kill_all() is global: a cron job mid-dispatch lost its tool subprocess and its agent thread may + # still emit a plausible response from truncated output — mark it interrupted, never success. from cron.scheduler import mark_running_jobs_interrupted _interrupted = mark_running_jobs_interrupted( f"Gateway shutdown ({phase}) killed the job's tool subprocess before the run finished." @@ -1376,16 +1360,14 @@ class GatewayShutdownMixin: async def _stop_drain_active_work(self, timeout: float, ctx: "GatewayShutdownMixin._StopContext") -> None: """Pre-mark resume_pending, drain agents/cron/API work into ``ctx``.""" from gateway.run import GatewayRunner - # Pre-mark resume_pending BEFORE the drain: if the service manager kills us mid-drain, the - # durable marker already lets the next boot recover the sessions. + # Pre-mark resume_pending BEFORE the drain so a mid-drain SIGKILL still leaves a durable marker. _pre_drain_keys = await GatewayRunner._mark_running_sessions_resume_pending( self, "pre-drain mark_resume_pending" ) _cron_at_start = self._active_cron_job_count() _api_at_start = self._active_api_run_count() _deferred_at_start = ctx.deferred_count() - # Cron floor clamped to the watchdog leash so it never costs the post-drain cleanup window. - # getattr-guard: shutdown-path tests drive the phases from bare doubles lacking the class default. + # Cron floor clamped to the watchdog leash; getattr-guard for bare shutdown-path doubles. _cron_drain_cfg = getattr(self, "_cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT) _cron_timeout = resolve_cron_drain_budget( timeout, _cron_drain_cfg, watchdog_delay=resolve_shutdown_watchdog_delay(timeout), @@ -1441,23 +1423,19 @@ class GatewayShutdownMixin: def _work_live() -> bool: return bool(self._running_agents or self._active_api_run_count() or ctx.deferred_count()) - # Wait on API-server work too: the interrupt is cooperative; closing the settle window as soon - # as _running_agents is empty would kill an API turn's tool subprocesses before it unwinds. + # Wait on API-server work too, or an API turn's tool subprocesses are killed before it unwinds. while _work_live() and loop.time() < interrupt_deadline: self._update_runtime_status("draining") await asyncio.sleep(0.1) - # Work can materialize AFTER the one-shot interrupt (a /v1/runs task registers only when - # _create_agent returns; a pending sentinel is promoted by track_agent() later). Re-signal so - # it gets a cooperative interrupt instead of a bare tool-subprocess kill. + # Work can materialize AFTER the one-shot interrupt (/v1/runs registers on _create_agent return; + # pending sentinels promote later). Re-signal for a cooperative interrupt, not a bare kill. if _work_live(): self._interrupt_running_agents(reason) logger.debug("Re-signaled interrupt for work still live at settle-window exit") - # Kill lingering tool subprocesses NOW, before adapter disconnect / DB close: under systemd - # (TimeoutStopSec ≈ drain_timeout + headroom) deferring risks the cgroup SIGKILL reaping them. + # Kill tool subprocesses NOW: deferring past adapter/DB teardown risks the systemd cgroup SIGKILL. _interrupted_cron_jobs = GatewayRunner._stop_kill_tool_subprocesses("post-interrupt") logger.info("Shutdown phase: post-interrupt tool kill done at +%.2fs", ctx.elapsed()) - # Last window with the transport up: the cron worker's own "interrupted" notice arrives - # after adapter teardown and is lost. + # Last window with the transport up (the cron worker's own notice arrives after teardown). with _log_suppressed(logging.DEBUG, "Cron interrupt notification failed: %s"): await self._notify_interrupted_cron_jobs(_interrupted_cron_jobs) logger.info("Shutdown phase: cron interrupt notices done at +%.2fs", ctx.elapsed()) @@ -1468,8 +1446,7 @@ class GatewayShutdownMixin: with _log_suppressed(logging.ERROR, "Failed to launch detached gateway restart: %s"): await self._launch_detached_restart_command() await self._finalize_shutdown_agents(ctx.active_agents) - # Idle cached agents too: _finalize_shutdown_agents only covers agents mid-turn at drain - # time, but _agent_cache may hold agents whose MemoryProviders never got on_session_end(). + # Idle cached agents too: their MemoryProviders may never have seen on_session_end(). _cache_lock = getattr(self, "_agent_cache_lock", None) _cache = getattr(self, "_agent_cache", None) if _cache_lock is not None and _cache is not None: @@ -1477,13 +1454,11 @@ class GatewayShutdownMixin: _idle_agents = list(_cache.values()) _cache.clear() for _entry in _idle_agents: - # Bounded + off-loop so a wedged memory provider can't hang shutdown forever - # (this path is why SIGTERM once failed to kill the process). + # Bounded + off-loop: a wedged memory provider here once made SIGTERM hang forever. await self._cleanup_agent_resources_off_loop( _entry[0] if isinstance(_entry, tuple) else _entry, context="shutdown idle-cache" ) - # Cancel + await completion flush tasks (fan-in sleep / adapter delivery) while adapters are - # still alive so every watcher gets a retryable result before platform teardown. + # Settle completion flush tasks while adapters are alive so every watcher gets a retryable result. cancel_completion_batches = getattr(self, "_cancel_process_completion_batch_tasks", None) if cancel_completion_batches is not None: await cancel_completion_batches() @@ -1502,8 +1477,7 @@ class GatewayShutdownMixin: """Cancel background tasks, flush pending messages, clear per-session state, final tool kill.""" from gateway.run import GatewayRunner for _task in list(self._background_tasks): - # _restart_task is awaiting _stop_task right now; cancelling it would propagate - # CancelledError into _stop_impl and skip _shutdown_event.set(). It self-terminates anyway. + # _restart_task awaits _stop_task: cancelling it would tunnel into _stop_impl and skip _shutdown_event.set(). if _task is self._stop_task or _task is self._restart_task: continue _task.cancel() @@ -1511,21 +1485,18 @@ class GatewayShutdownMixin: self.adapters.clear() for _session_key in list(self._running_agents): self._release_running_agent_state(_session_key) - # Flush pending messages to disk before clearing: under FTS5 corruption the in-memory - # pending text is the only surviving copy; clearing unflushed loses it permanently. + # Flush pending messages before clearing: under FTS5 corruption they are the only surviving copy. with suppress(Exception): from gateway.shutdown_flush import flush_pending_to_file flush_pending_to_file(dict(self._pending_messages), reason="shutdown") - # The FIFO tail lives in SessionState.conversation.queued_events, not the slot dict - # above — flush it too or every follow-up parked in overflow at restart time is lost. + # The overflow FIFO tail lives in SessionState.conversation.queued_events — flush it too. with suppress(Exception): from gateway.shutdown_flush import flush_overflow_to_file flush_overflow_to_file( {_k: list(_v) for _k, _v in dict(getattr(self, "_queued_events", None) or {}).items() if _v}, reason="shutdown", ) - # On the real runner these are live SessionState views whose clear() resets one field per - # session (never a wholesale dict swap that could lose a concurrent writer's entry). + # Live SessionState views: clear() resets one field per session (never a wholesale dict swap). self._running_agents.clear() self._running_agents_ts.clear() self._pending_messages.clear() @@ -1534,12 +1505,10 @@ class GatewayShutdownMixin: if hasattr(self, _attr): getattr(self, _attr).clear() self._shutdown_event.set() - # Global catch-all subprocess kill (safe to repeat): covers the graceful path and - # anything respawned since the drain-timeout path's post-interrupt kill. + # Global catch-all subprocess kill (safe to repeat) for the graceful path and late respawns. GatewayRunner._stop_kill_tool_subprocesses("final-cleanup") logger.info("Shutdown phase: final-cleanup tool kill done at +%.2fs", ctx.elapsed()) - # Reap the process-global auxiliary-client cache: per-turn cleanup misses clients bound to - # worker-thread loops that died with their executor (cron ticks) → httpx transports leak to EMFILE. + # Reap the auxiliary-client cache: clients bound to dead worker-thread loops leak httpx transports. def _reap_aux_clients() -> None: from agent.auxiliary_client import shutdown_cached_clients shutdown_cached_clients() @@ -1557,9 +1526,8 @@ class GatewayShutdownMixin: ) _exec_live = GatewayRunner._shutdown_executor(self, drain_timeout=_exec_quiesce_budget) if _exec_live: - # A live worker may be mid-write: closing now is the exact #101093 corruption sequence, - # so skip the close and let SQLite recover from its WAL on next open (a transient - # "database is locked" on an immediate --replace at worst, not a corrupt file). + # A live worker may be mid-write (the #101093 corruption sequence): skip the close and let + # SQLite recover from its WAL on next open (at worst a transient "database is locked"). logger.warning( "Shutdown phase: %d executor worker(s) still running after a %.2fs quiesce — skipping the " "SessionDB close/checkpoint to avoid racing a live write (#101093); handles are left " @@ -1568,8 +1536,7 @@ class GatewayShutdownMixin: return logger.info("Shutdown phase: executor quiesced at +%.2fs", ctx.elapsed()) _step = GatewayShutdownMixin._quiet_step - # Close SQLite session DBs so the WAL lock is released; otherwise --replace leaves the old - # connection holding it until exit and the new gateway gets 'database is locked'. + # Close SQLite session DBs so --replace's new gateway does not hit 'database is locked'. # ``_session_db`` is an AsyncSessionDB facade — unwrap; ``session_store`` holds ``_db``. _self_db = getattr(self, "_session_db", None) _self_db = getattr(_self_db, "_db", _self_db) @@ -1577,18 +1544,15 @@ class GatewayShutdownMixin: for _db in (_self_db, getattr(store, "_db", None)): if _db is not None and hasattr(_db, "close"): _step("SessionDB close error", _db.close) - # A multiplexed session_store caches one SessionDB per profile; ``_db`` above only covered - # the root scope. Sweep the rest so secondary WAL locks are released before --replace. + # Multiplexed session_store caches one SessionDB per profile; sweep the secondary WAL locks too. _sweep = getattr(store, "close_all_db_handles", None) if _sweep is not None: _step("SessionDB handle sweep error", _sweep) - # Same sweep for the runner's own per-profile session_search handles (slash commands - # resolve them under profile scopes). + # Same sweep for the runner's own per-profile session_search handles. _step("Runner SessionDB handle sweep error", lambda: GatewayRunner.close_all_session_db_handles(self)) def _close_shared() -> None: - # Shared SessionDB instances still held by the process-wide registry (tools, cron, - # mirror, etc. opened via get_shared_session_db but not released above). + # Shared SessionDB instances still held by the process-wide registry (tools, cron, mirror). from hermes_state import close_shared_session_dbs closed = close_shared_session_dbs() if closed: @@ -1606,9 +1570,8 @@ class GatewayShutdownMixin: from gateway.status import remove_pid_file, release_gateway_runtime_lock remove_pid_file() release_gateway_runtime_lock() - # Clean-shutdown marker: suspend_recently_active() need only run after unexpected exits. - # If the drain timed out and agents were force-interrupted, sessions may be half-finished - # — skip the marker so the next startup suspends them. + # Clean-shutdown marker skips suspend_recently_active() next boot; a timed-out drain left + # half-finished sessions, so no marker — the next startup suspends them. if not ctx.timed_out: with suppress(Exception): (_hermes_home / ".clean_shutdown").touch() @@ -1617,8 +1580,7 @@ class GatewayShutdownMixin: "Skipping .clean_shutdown marker — drain timed out with " "interrupted agents; next startup will suspend recently active sessions." ) - # Stuck-loop detection: the counter increments for sessions active at each restart; at - # the threshold (3 consecutive) the next startup auto-suspends the session. + # Stuck-loop counter: sessions active across 3 consecutive restarts are auto-suspended next boot. if ctx.active_agents: self._increment_restart_failure_counts(set(ctx.active_agents.keys())) if self._restart_requested and self._restart_command_source is None: @@ -1633,8 +1595,7 @@ class GatewayShutdownMixin: indent=None, ) if self._restart_requested and self._restart_via_service: - # Service manager owns restarts: exit 75 + ``RestartForceExitStatus=75`` has systemd - # replace this process without a second helper racing the unit's stop/start job. + # Exit 75 + ``RestartForceExitStatus=75``: systemd replaces us without a racing helper. self._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE self._exit_reason = self._exit_reason or "Gateway restart requested" self._draining = False @@ -1673,8 +1634,7 @@ class GatewayShutdownMixin: # os._exit past drain+grace so the service manager revives us. Skipped under pytest. _watchdog_done = threading.Event() self._shutdown_watchdog_done = _watchdog_done - # Shutdown-path tests and third-party runner doubles may only implement the older - # drain-count surface (no deferred-worker counter). + # Shutdown-path doubles may lack the deferred-worker counter. ctx = GatewayShutdownMixin._StopContext( deferred_count=getattr(self, "_active_deferred_agent_worker_count", lambda: 0) ) diff --git a/gateway/run_startup.py b/gateway/run_startup.py index 76e42961d9..3ff27838ee 100644 --- a/gateway/run_startup.py +++ b/gateway/run_startup.py @@ -50,8 +50,7 @@ class GatewayStartupMixin: if task is not None: await asyncio.shield(task) finally: - # The runner slot was pre-claimed before this task spawned; release it if handle_message - # raises before _handle_message takes ownership, else the real run's cleanup owns it. + # Release the pre-claimed slot if handle_message raised before _handle_message took ownership. _pre_state = self._peek_session_state(session_key) if (_pre_state.turn.agent if _pre_state else None) is _AGENT_PENDING_SENTINEL: self._release_running_agent_state(session_key) @@ -83,8 +82,7 @@ class GatewayStartupMixin: getattr(getattr(source, "platform", None), "value", None), ) continue - # Mark this replay so _handle_message does not queue it again while - # the restore gate remains closed for any fresh inbound arrivals. + # Mark the replay so _handle_message does not re-queue it while the restore gate is closed. with suppress(Exception): setattr(event, "_hermes_startup_restore_replay", True) await adapter.handle_message(event) @@ -163,8 +161,7 @@ class GatewayStartupMixin: from gateway.run import _startup_restore_drain_timeout_secs tasks = list(getattr(self, "_startup_restore_tasks", []) or []) if tasks: - # Tasks that outlive the gate get a late-failure callback: their normal done-callback - # only discards them from _background_tasks, so a LATER failure would be swallowed. + # Tasks outliving the gate get a late-failure callback (their done-callback only discards them). done = await self._wait_bounded_or_release( set(tasks), _startup_restore_drain_timeout_secs(), "Startup-restore gate released after %.0fs with %d boot auto-resume turn(s) " @@ -176,8 +173,7 @@ class GatewayStartupMixin: for task in done: report(task) self._startup_restore_tasks = [] - # Warm the turn machinery BEFORE the queue drains: replayed (and - # fresh) inbound turns must not build skeleton prompts (#99373). + # Warm the turn machinery BEFORE the queue drains: inbound turns must not build skeleton prompts. await self._await_startup_warmup() drained = await self._drain_startup_restore_queue() self._startup_restore_in_progress = False @@ -258,14 +254,12 @@ class GatewayStartupMixin: from gateway.delivery_ledger import ledger_enabled, sweep_recoverable if not await asyncio.to_thread(ledger_enabled): return [] - # Only claim rows whose exact transport owner is connected this boot. A multiplexed - # gateway can host several bot identities for one platform; platform-only filtering - # would spend a disconnected bot's retry budget merely because another bot is online. + # Claim only rows whose exact transport owner is connected: platform-only filtering would spend + # a disconnected bot's retry budget because another bot on that platform is online. _profile_adapters = getattr(self, "_profile_adapters", None) or {} _pval = lambda p: getattr(p, "value", str(p)) # noqa: E731 _deliverable_targets = {(_pval(p), "default") for p in self.adapters} - # Legacy rows predate adapter_profile. They are unambiguous only in a non-multiplexed - # gateway; fail closed when multiple bot identities share the process. + # Legacy rows (no adapter_profile) are unambiguous only without multiplexing; else fail closed. if not _profile_adapters: _deliverable_targets.update((_pval(p), None) for p in self.adapters) for _profile, _adapters in _profile_adapters.items(): @@ -280,8 +274,7 @@ class GatewayStartupMixin: return [] if not claimed: return [] - # Clear resume_pending for EVERY claimed row before any send: claiming already spent one - # redelivery attempt and the answer is in the ledger, so the resume path must never re-run. + # Clear resume_pending for EVERY claimed row before any send: the answer is in the ledger. await self._clear_resume_pending_for_claimed_obligations(claimed) return claimed @@ -345,9 +338,8 @@ class GatewayStartupMixin: else: # Startup rows preserve the historical default-adapter route. adapter = self.adapters.get(platform) - # Runtime claims have not reached a transport yet. If the reconnect vanished before - # dispatch, release the claim without spending an attempt so the next reconnect can retry - # it. Startup claims keep their state; attempts cap + stale cutoff bound later retries. + # A runtime claim whose reconnect vanished before dispatch is released without spending an + # attempt; startup claims keep their state (attempts cap + stale cutoff bound retries). if adapter is None and row.get("runtime_recovery"): await self._release_runtime_claim_quiet( row["obligation_id"], "failed to release undispatched runtime obligation %s" @@ -377,8 +369,7 @@ class GatewayStartupMixin: return 0 if not claimed: return 0 - # Clear before any send so the reconnect path cannot both redeliver an - # already-produced answer and schedule the same agent turn for resume. + # Clear before any send so the reconnect path cannot both redeliver AND resume the same turn. sendable = await self._clear_resume_pending_for_claimed_obligations(claimed, require_success=True) sendable_ids = {row["obligation_id"] for row in sendable} for row in claimed: @@ -461,15 +452,13 @@ class GatewayStartupMixin: continue if not self._resume_owner_authorized(entry.session_key, source): continue - # Claim the session slot *before* spawning the task so an inbound message arriving - # between task creation and the task's first await (where _process_message_background - # sets the real sentinel) sees the slot occupied and queues, not a duplicate AIAgent. + # Claim the slot *before* spawning so an inbound message arriving before the task's first + # await queues instead of building a duplicate AIAgent. _resume_state = self._session_state(entry.session_key) _resume_state.turn.agent = _AGENT_PENDING_SENTINEL _resume_state.turn.started_ts = time.time() self._persist_active_agents() - # Empty-text internal event: the _is_resume_pending branch in _handle_message_with_agent - # prepends the reason-aware system note before the turn runs. + # Empty-text internal event: the _is_resume_pending branch prepends the reason-aware note. event = MessageEvent(text="", message_type=MessageType.TEXT, source=source, internal=True) task = self._retain_background_task( asyncio.create_task(self._run_startup_resume_event(adapter, event, entry.session_key)) @@ -529,8 +518,7 @@ class GatewayStartupMixin: watchdog = getattr(self, "_loop_liveness_watchdog", None) if watchdog is None or not watchdog.is_alive(): try: - # getattr defaults cover the config=None / bare-object test path; config-loaded - # values are already validated+clamped by GatewayConfig.from_dict; no re-clamping. + # getattr defaults cover config=None test paths; loaded values are already clamped. self._loop_liveness_watchdog = start_loop_liveness_watchdog( loop, probe_interval=float(getattr( @@ -623,9 +611,7 @@ class GatewayStartupMixin: start_time=getattr(self, "_gateway_started_at", 0.0), ) ) - # PERMANENT for the process lifetime, same as a _spawn_supervised watcher — tag it so - # _scale_to_zero_has_live_background_work() doesn't treat an armed, otherwise-idle - # gateway as busy forever. + # PERMANENT watcher tag so the scale-to-zero idle check doesn't count it as busy forever. task._hermes_supervised_watcher = True # type: ignore[attr-defined] _bg = getattr(self, "_background_tasks", None) if _bg is not None: @@ -670,17 +656,15 @@ class GatewayStartupMixin: disarm_startup_watchdog() logger.info("Session storage: %s", self.config.sessions_dir) self._start_log_systemd_timing_alignment() - # Log the resolved max_iterations budget so operators can verify the config.yaml → env - # bridge at a glance (instead of silently running at a stale .env value for weeks). + # Log the resolved max_iterations so operators can verify the config.yaml → env bridge. with suppress(Exception): logger.info( "Agent budget: max_iterations=%d (agent.max_turns from config.yaml, " "or HERMES_MAX_ITERATIONS from .env, or default 500)", int(os.getenv("HERMES_MAX_ITERATIONS", "500")), ) - # Redaction is ON by default; warn prominently when an operator has explicitly opted out so - # the downgrade isn't forgotten. The redactor snapshots its state at import time, so this - # log line is the source of truth for the process lifetime. + # Warn prominently when redaction is opted out; the redactor snapshots its state at import time, + # so this line is the source of truth for the process lifetime. with suppress(Exception): _redact_raw = os.getenv("HERMES_REDACT_SECRETS", "true") if _redact_raw.lower() in {"1", "true", "yes", "on"}: @@ -706,8 +690,7 @@ class GatewayStartupMixin: self._gateway_health_export_runtime = start_gateway_health_export(load_config()) if getattr(self._gateway_health_export_runtime, "enabled", False): logger.info("Gateway health OTLP export: enabled") - # Log any active supply-chain security advisories. Deliberately does NOT block startup or - # surface inline to users — only the operator can act (uninstall, rotate credentials). + # Supply-chain advisories: log only (never block startup or surface to users; only the operator can act). with _log_suppressed(logging.DEBUG, "security advisory check failed at gateway startup", exc_info=True): from hermes_cli.security_advisories import detect_compromised, gateway_log_message _adv_msg = gateway_log_message(detect_compromised()) @@ -736,9 +719,8 @@ class GatewayStartupMixin: _alignment["expected_min"], ) - # Builtin platforms whose ``

_ALLOWED_USERS`` / ``

_ALLOW_ALL_USERS`` env vars count as - # "an allowlist is configured" / "open access opted in"; plugin platforms are appended at - # check time. + # Builtin platforms whose ``

_ALLOWED_USERS`` / ``

_ALLOW_ALL_USERS`` env vars count as an + # allowlist / open-access opt-in; plugin platforms are appended at check time. _ALLOWLIST_ENV_PLATFORMS = ( "TELEGRAM", "DISCORD", "WHATSAPP", "WHATSAPP_CLOUD", "SLACK", "SIGNAL", "EMAIL", "SMS", "MATTERMOST", "MATRIX", "DINGTALK", "FEISHU", "WECOM", "WECOM_CALLBACK", "WEIXIN", @@ -755,8 +737,7 @@ class GatewayStartupMixin: from gateway.run import ( _OWN_POLICY_OPEN_ENV, _own_policy_open_startup_violation, _write_runtime_status_quiet ) - # Plugin-registered platforms declare their own allowed_users_env / allow_all_env, so the - # warning stays accurate as plugins (IRC) arrive. + # Plugin platforms declare their own allowed_users_env / allow_all_env. allowed_vars = list(self._BUILTIN_ALLOWED_USERS_VARS) allow_all_vars = ["GATEWAY_ALLOW_ALL_USERS", *self._BUILTIN_ALLOW_ALL_VARS] with suppress(Exception): @@ -792,9 +773,8 @@ class GatewayStartupMixin: @staticmethod def _start_register_plugins_relay_hooks() -> None: """Plugin discovery, relay registration and shell-hook/webhook registration. Never raises.""" - # Discover Python plugins before shell hooks so plugin block decisions take precedence in - # tie cases. Explicit here because the gateway lazily imports run_agent per request, so - # the discover_plugins() side-effect in model_tools.py is NOT guaranteed to have run yet. + # Discover plugins before shell hooks (plugin block decisions win ties). Explicit: the gateway + # lazily imports run_agent, so model_tools' discover_plugins() side-effect may not have run. with _log_suppressed(logging.WARNING, "plugin discovery failed at gateway startup", exc_info=True): from hermes_cli.plugins import discover_plugins discover_plugins() @@ -803,14 +783,11 @@ class GatewayStartupMixin: from gateway.relay import ( register_relay_adapter, relay_url, self_provision_relay, send_relay_policy ) - # Boot-time relay self-provision: resolve the agent's NAS token -> POST /relay/provision - # -> set GATEWAY_RELAY_* in os.environ BEFORE registration reads them. Never raises. + # Relay self-provision sets GATEWAY_RELAY_* in os.environ BEFORE registration reads them. self_provision_relay() if register_relay_adapter(): logger.info("relay adapter registered (connector at %s)", relay_url()) - # Declare this gateway's relevance policy (mention-gating / free-response / allow- - # bots) to the connector so the SAME behavior governs relay delivery. Runs after - # the secret is resolved; never raises, never blocks boot. + # Declare the relevance policy to the connector so the SAME behavior governs relay delivery. send_relay_policy() except Exception: logger.warning("relay adapter registration failed at gateway startup", exc_info=True) @@ -867,8 +844,7 @@ class GatewayStartupMixin: "Marked %d in-flight session(s) as resumable from previous run " "(%d exact, %d legacy)", exact + fallback, exact, fallback, ) - # Stuck-loop detection: a session active across 3+ consecutive restarts is probably looping - # (its history keeps hanging the agent); auto-suspend so the next message starts clean. + # Stuck-loop detection: a session active across 3+ consecutive restarts is auto-suspended. with _log_suppressed(logging.DEBUG, "Stuck-loop detection failed: %s"): stuck = self._suspend_stuck_loop_sessions() if stuck: @@ -909,9 +885,8 @@ class GatewayStartupMixin: "(platform is enabled in config.yaml but no plugin registered it)", platform.value, ) continue - # Under multiplexing the default profile needs the same whole-handler runtime scope as - # a secondary profile: authorization and prompt rendering both run before the narrower - # agent-turn scope is installed. + # Under multiplexing the default profile needs the same whole-handler runtime scope as a + # secondary (authorization and prompt rendering run before the agent-turn scope). self._wire_adapter_handlers(adapter) _pending_connects.append((platform, platform_config, adapter)) return False, enabled_platform_count, _multiplex_skipped_platforms, _pending_connects @@ -935,9 +910,8 @@ class GatewayStartupMixin: if not _pending_connects: return [] - # Abort-aware concurrent wait (parity with the serial loop's between-platforms check): a - # restart/shutdown requested mid-connect must cancel still-pending connects, clean up the - # ones already completed, and abort startup. + # Abort-aware concurrent wait: a restart/shutdown mid-connect cancels pending connects, tears down + # completed ones, and aborts startup. _task_map = { asyncio.ensure_future(_connect_one_startup(p, c, a)): (p, c, a) for (p, c, a) in _pending_connects } @@ -948,13 +922,11 @@ class GatewayStartupMixin: break else: return [_t.exception() or _t.result() for _t in _task_map] - # Cancel and fully settle the in-flight connects FIRST, so a completed adapter's - # disconnect cannot unblock a sibling's connect() before the sibling is cancelled. + # Settle in-flight connects FIRST so a completed adapter's disconnect cannot unblock a sibling. for _t in _pending_tasks: _t.cancel() await asyncio.gather(*_pending_tasks, return_exceptions=True) - # Then tear down adapters whose connect already succeeded — they were never registered, so - # stop() won't reach them. + # Then tear down adapters whose connect succeeded — never registered, so stop() won't reach them. _connected_ok = [ _t for _t in _task_map if _t not in _pending_tasks and not _t.cancelled() and _t.exception() is None and _t.result()[3] == "ok" @@ -1007,8 +979,7 @@ class GatewayStartupMixin: continue # outcome == "failed" logger.warning("\u2717 %s failed to connect", platform.value) - # Defensive cleanup: a failed connect() may have allocated resources - # (aiohttp.ClientSession, poll tasks, bridge subprocesses) before giving up. + # A failed connect() may have allocated ClientSessions / poll tasks / subprocesses. await self._safe_adapter_disconnect(adapter, platform) if not adapter.has_fatal_error: # No fatal error info means likely a transient issue -- queue for retry @@ -1044,25 +1015,21 @@ class GatewayStartupMixin: ) -> Tuple[bool, int]: """Bring up multiplexed secondary-profile adapters. Returns (aborted, connected_count).""" from gateway.run import MultiplexConfigError - # Adapters for every OTHER profile this gateway serves connect under that profile's home + - # credential scope and stamp inbound events with the profile so the agent turn resolves. + # Secondary-profile adapters connect under their own home + credential scope. try: connected_count += await self._start_secondary_profile_adapters() except MultiplexConfigError as e: - # Invalid multiplexer config — abort startup cleanly so the operator fixes config.yaml - # rather than running a half-wired gateway. + # Invalid multiplexer config — abort cleanly rather than run a half-wired gateway. logger.error("Gateway multiplexer config error: %s", str(e)) self._startup_fail_fatal_config(str(e)) return True, connected_count except Exception as e: logger.error("Secondary-profile adapter startup failed: %s", e, exc_info=True) finally: - # Startup authority is one phase, not a persistent runner mode: from here on every - # adapter retry is non-evicting. + # Startup authority is one phase: from here on every adapter retry is non-evicting. self._platform_lock_takeover_on_start = False - # A platform skipped on the primary for a missing credential should have been picked up by - # a secondary profile owning the token. If none did, it is enabled in config.yaml yet - # silently unserved — surface it loudly instead of leaving a quiet dead channel. + # A platform skipped on the primary should have been picked up by a secondary owning the token; + # if none did it is enabled yet silently unserved — say so loudly. for _skipped in _multiplex_skipped_platforms: if not any(_skipped in _profile_map for _profile_map in self._profile_adapters.values()): logger.warning( @@ -1098,16 +1065,14 @@ class GatewayStartupMixin: logger.info("Gateway will continue running for cron job execution.") return False if startup_retryable_errors: - # All enabled platforms hit retryable failures. Keep the gateway alive so cron jobs - # still run and the reconnect watcher can recover them once the cause is fixed; - # exiting would turn one misconfigured platform into an infinite systemd restart loop. + # All retryable: stay alive (cron runs, watcher recovers) rather than systemd restart-loop. logger.warning( "Gateway started with no connected platforms — %d platform(s) queued for retry: %s", len(self._failed_platforms), "; ".join(startup_retryable_errors), ) _write_runtime_status_quiet(gateway_state="degraded", exit_reason=None) - # (Or: no adapter for any enabled platform — missing library or credentials.) Fleet nodes - # share one config.yaml but hold credentials for only a subset, so degrade gracefully. + # No adapter for any enabled platform: fleet nodes share one config.yaml but hold a subset of + # credentials, so degrade gracefully. logger.warning( "No adapter could be created for any of the %d configured platform(s). " "Check that required dependencies are installed and credentials are set. " @@ -1139,8 +1104,7 @@ class GatewayStartupMixin: directory = await build_channel_directory(self.adapters) ch_count = sum(len(chs) for chs in directory.get("platforms", {}).values()) logger.info("Channel directory built: %d target(s)", ch_count) - # Restarting after a /update? If the update is still running, keep watching so we notify - # once it actually finishes. + # Restarting after a /update still in progress: keep watching so we notify when it finishes. notified = await self._send_update_notification() if not notified and any( (_hermes_home / name).exists() @@ -1168,8 +1132,7 @@ class GatewayStartupMixin: # auto-resume stays visible on the next user message. self._schedule_resume_pending_sessions() await self._finish_startup_restore() - # Surface state.db init failures to the user's messaging platforms so they know persistence - # is broken before losing data. + # Surface state.db init failures to messaging platforms before the user loses data. await self._send_session_db_warning_notifications() # Resume recovered process watchers. Detach the batch atomically (fresh list, not clear(): a # concurrent append during the yield must not be lost); yield every 100 to keep the loop live. @@ -1186,10 +1149,8 @@ class GatewayStartupMixin: if i % 100 == 99: await asyncio.sleep(0) - # Long-lived supervised watchers spawned at the end of start(), in order; the supervised name is - # the method name without its leading underscore. Each watcher's docstring explains its job - # (session expiry, /model catalog TTL refresh, stall warnings, kanban notifier/dispatcher, - # CLI handoff, async-delegation completions, /loop wakeups). + # Long-lived supervised watchers spawned at the end of start(), in order; supervised name = method + # name minus the leading underscore. _PRE_RECONNECT_WATCHERS = ( "_session_expiry_watcher", "_model_catalog_refresh_watcher", "_session_stall_watcher", "_kanban_notifier_watcher", "_kanban_dispatcher_watcher", @@ -1205,17 +1166,12 @@ class GatewayStartupMixin: "Starting reconnection watcher for %d failed platform(s): %s", len(self._failed_platforms), ", ".join(p.value for p in self._failed_platforms), ) - # Spawned via _spawn_supervised so an exception escaping the watcher's OUTER loop is caught, - # logged, and restarted with backoff instead of silently killing it (else a platform already - # queued in _failed_platforms stays stranded: the ensure hook only runs on a NEW fatal-error - # arrival). ``on_spawn`` keeps ``_reconnect_watcher_task`` on the CURRENT live task across - # backoff respawns so a superseded handle never looks like a dead watcher. + # Supervised: an escaping exception is restarted with backoff instead of stranding queued + # platforms (the ensure hook only runs on a NEW fatal). ``on_spawn`` keeps the handle current. self._spawn_reconnect_watcher() for method in self._POST_RECONNECT_WATCHERS: self._spawn_supervised(getattr(self, method), method[1:]) - # Scale-to-zero idle watcher ONLY when opted in (HERMES_SCALE_TO_ZERO stamp), messaging is - # relay-only/absent, and a wakeUrl is registered. When armed it drives the relay dormant on - # sustained idle, then suspends via flaps — Fly autostop is inbound-only, job-blind. + # Scale-to-zero watcher ONLY when opted in, messaging is relay-only/absent, and a wakeUrl exists. try: if self._scale_to_zero_should_arm(): logger.info( @@ -1224,14 +1180,12 @@ class GatewayStartupMixin: ) self._spawn_supervised(self._scale_to_zero_watcher, "scale_to_zero_watcher") else: - # Surface WHY an OPTED-IN instance didn't arm (non-opted not arming is normal — - # stay silent); otherwise a failed arm is invisible and needs a box-dive. + # Say WHY an OPTED-IN instance didn't arm (non-opted stays silent). self._log_scale_to_zero_not_armed_reason() except Exception: # noqa: BLE001 - arming must never block startup logger.debug("scale-to-zero: arm check failed at startup", exc_info=True) - # Drain-control watcher: reconciles the gateway's new-turn accept-state with the external - # ``.drain_request.json`` marker the dashboard begin/cancel-drain endpoint writes. A marker - # from a prior instantiation (durable-volume restart) is ignored via its epoch. + # Drain-control watcher: reconciles new-turn acceptance with the dashboard's ``.drain_request.json`` + # marker (prior-instantiation markers are ignored via epoch). self._spawn_supervised(self._drain_control_watcher, "drain_control_watcher") async def start(self) -> bool: @@ -1244,15 +1198,13 @@ class GatewayStartupMixin: if self._start_check_access_policy(): return True await self._start_recover_previous_run() - # Serialize startup restore against inbound dispatch: adapters can receive messages as soon - # as they connect, but restart-interrupted sessions are not auto-resumed until all startup - # wiring below completes, so inbound queues until every synthetic resume turn has finished. + # Serialize startup restore against inbound: adapters receive as soon as they connect, so inbound + # queues until every synthetic resume turn has finished. self._startup_restore_in_progress = True self._startup_restore_queue = [] self._startup_restore_tasks = [] - # Fresh-boot readiness: with no resume_pending sessions the gate opens almost immediately - # while the turn machinery is still cold, so a message in that window got a skeleton system - # prompt. Warm NOW to overlap the connects below; _finish_startup_restore awaits it (bounded). + # Fresh boot: the gate opens while the turn machinery is still cold (skeleton prompts). Warm NOW + # to overlap the connects; _finish_startup_restore awaits it (bounded). self._start_startup_warmup() startup_nonretryable_errors: list[str] = [] startup_retryable_errors: list[str] = [] @@ -1348,8 +1300,7 @@ class GatewayStartupMixin: f"no home channel configured for {platform_name}; run /sethome on the desired chat first" ) home_chat_id = str(home.chat_id) - # Fresh thread on the destination so the handoff has its own scrollback. Adapter returns - # None if threading is unsupported (Matrix/WhatsApp/Signal/SMS) or creation failed. + # Fresh thread for the handoff's own scrollback; None when unsupported or creation failed. cli_title = row.get("title") or cli_session_id[:8] try: new_thread_id = await transport.adapter.create_handoff_thread( @@ -1365,8 +1316,7 @@ class GatewayStartupMixin: platform == Platform.TELEGRAM and looks_like_telegram_private_chat_id(home_chat_id) ) is_thread = bool(new_thread_id) and not is_telegram_private_chat - # Discord (unlike Slack/Telegram) builds in-thread messages with ``chat_id == thread id``, - # so key on the thread's OWN id; keying on the parent would make the next reply spawn anew. + # Discord builds in-thread messages with ``chat_id == thread id``: key on the thread's OWN id. dest_source = SessionSource( platform=platform, chat_id=str(effective_thread_id) if ( @@ -1414,17 +1364,14 @@ class GatewayStartupMixin: cli_session_id = row["id"] dest = await self._handoff_resolve_destination(row, profile_name) session_key = self._handoff_session_key(dest, profile_name) - # Ensure a session_store entry exists for this key (get_or_create_session creates one for a - # never-used home channel); switch_session then re-points it. + # Ensure a session_store entry exists for this key; switch_session then re-points it. await self.async_session_store.get_or_create_session(dest.source) - # Re-bind the destination key to the CLI session_id: switch_session ends the prior session - # in SQLite and reopens the CLI session under the new key; its transcript is now active. + # switch_session ends the prior session and reopens the CLI session under the new key. switched = await self.async_session_store.switch_session(session_key, cli_session_id) if switched is None: raise RuntimeError(f"could not switch session key {session_key} → {cli_session_id}") - # Evict any cached AIAgent for this key so the next dispatch rebuilds it against the CLI - # session_id (mirrors /resume / /branch), and clear stale running-agent state so the - # synthetic turn isn't queued behind it. + # Evict the cached AIAgent (rebuild against the CLI session_id, like /resume) and clear stale + # running-agent state so the synthetic turn isn't queued behind it. self._evict_cached_agent(session_key) self._release_running_agent_state(session_key) cli_title = row.get("title") or cli_session_id[:8] @@ -1443,16 +1390,13 @@ class GatewayStartupMixin: "(home=%s, thread=%s, session_key=%s)", cli_session_id, dest.platform_name, dest.home.chat_id, dest.effective_thread_id, session_key, ) - # Dispatch through the runner directly: adapter.handle_message would spawn a background task - # and lose error visibility; inline _handle_message keeps success/failure observable. + # Inline _handle_message keeps success/failure observable (handle_message would detach it). response_text = await self._handle_message(synthetic_event) if not response_text: - # Streaming may have already delivered the response inline; the agent ran without - # raising either way — count as success. + # Streaming may have delivered inline; the agent ran without raising — success. return - # Send the reply to the new thread if we created one, else the configured home channel - # (which may carry a thread_id). Use the resolved transport (not adapter.send) so a - # relay-fronted logical platform is stamped on the outbound frame (send_for_platform). + # Reply into the new thread (else the home channel) via the resolved transport, so a relay-fronted + # logical platform is stamped on the outbound frame. send_metadata = {"thread_id": dest.effective_thread_id} if dest.effective_thread_id else None try: result = await dest.transport.send(