import os import sys # Stop a ``utils/`` (or ``proxy/``, ``ui/``) package in the launch directory from # shadowing Hermes's own top-level modules. ``hermes_bootstrap`` lives at the repo # root (its name can't collide with a user package), so importing it first is safe. import hermes_bootstrap hermes_bootstrap.harden_import_path() import json import logging import signal import threading import time import traceback from contextlib import suppress from tui_gateway._env import env_float from tui_gateway._stdin_recovery import handle_spurious_eof from tui_gateway import server from tui_gateway.event_replay import replay_epoch from tui_gateway.server import _CRASH_LOG, dispatch, resolve_skin, write_json from tui_gateway.transport import TeeTransport logger = logging.getLogger(__name__) # Discovery thread spawned by THIS module; None when delegated to the shared owner in # hermes_cli.mcp_startup (current path). The wait/in-flight/join helpers consult both. _mcp_discovery_thread = None # Set once MCP servers are found configured, so wait_for_mcp_discovery can re-invoke # the idempotent spawn on later builds (retry-after-zero-connected) without a config # re-probe — non-MCP sessions never pay the tools.mcp_tool import per build. _mcp_discovery_enabled = False def _install_sidecar_publisher() -> None: """Mirror every dispatcher emit to the dashboard sidebar via WS when `HERMES_TUI_SIDECAR_URL` is set (best-effort: a dropped WS falls back to stdio-only).""" url = os.environ.get("HERMES_TUI_SIDECAR_URL") if not url: return from tui_gateway.event_publisher import WsPublisherTransport server._stdio_transport = TeeTransport(server._stdio_transport, WsPublisherTransport(url)) # Grace for orderly shutdown before ``os._exit(0)`` so a worker wedged mid-flush can't # strand the process; ``HERMES_TUI_GATEWAY_SHUTDOWN_GRACE_S`` overrides (a longer grace # also means a longer wait on a real deadlock). _DEFAULT_SHUTDOWN_GRACE_S = 1.0 def _shutdown_grace_seconds() -> float: value = env_float("HERMES_TUI_GATEWAY_SHUTDOWN_GRACE_S", _DEFAULT_SHUTDOWN_GRACE_S) return value if value > 0 else _DEFAULT_SHUTDOWN_GRACE_S def _stamp() -> str: return time.strftime("%Y-%m-%d %H:%M:%S") def _mcp_startup_call(name: str, *args, default=None, log=None, **kwargs): """Call ``hermes_cli.mcp_startup.`` (lazy import); ``default`` on any failure, optionally logged as ``(level, message)``.""" try: from hermes_cli import mcp_startup return getattr(mcp_startup, name)(*args, **kwargs) except Exception: if log: getattr(logger, log[0])(log[1], exc_info=True) return default def _spawn_discovery(log: tuple) -> None: _mcp_startup_call( "start_background_mcp_discovery", logger=logger, thread_name="tui-mcp-discovery", log=log) def _append_crash_log(header: str, dump=None) -> None: """Best-effort ``=== header ===`` entry in the crash log; ``dump(f)`` adds detail.""" with suppress(Exception): os.makedirs(os.path.dirname(_CRASH_LOG), exist_ok=True) with open(_CRASH_LOG, "a", encoding="utf-8") as f: f.write(f"\n=== {header} ===\n") if dump is not None: dump(f) def _log_signal(signum: int, frame) -> None: """Capture WHICH thread and WHERE a termination signal hit us, then exit. ``sys.exit(0)`` alone raced the worker pool (a thread holding ``_stdout_lock`` mid-flush blocks interpreter shutdown), so: log all thread stacks, give the configured grace to drain, then ``os._exit(0)``.""" # SIGPIPE/SIGHUP don't exist on Windows — only look up attributes present. names = {int(sig): attr for attr in ("SIGPIPE", "SIGTERM", "SIGHUP", "SIGINT", "SIGBREAK") if (sig := getattr(signal, attr, None)) is not None} name = names.get(signum, f"signal {signum}") def _dump(f): if frame is not None: f.write("main-thread stack at signal delivery:\n") traceback.print_stack(frame, file=f) # All live threads — the signal may have come from a background writer. for tid, th in threading._active.items(): f.write(f"\n--- thread {th.name} (id={tid}) ---\n") f.write("".join(traceback.format_stack(sys._current_frames().get(tid)))) _append_crash_log(f"{name} received · {_stamp()}", _dump) print(f"[gateway-signal] {name}", file=sys.stderr, flush=True) # ``os._exit`` skips atexit but breaks the mid-flush deadlock; the crash log # + stderr line above are the forensic trail. timer = threading.Timer(_shutdown_grace_seconds(), lambda: os._exit(0)) timer.daemon = True timer.start() # atexit (_shutdown_sessions) can be blocked past the grace window by a worker # holding the GIL/_stdout_lock; finalize explicitly so unpersisted messages reach # state.db before the hard-exit timer fires. with suppress(Exception): from tui_gateway.server import _shutdown_sessions _shutdown_sessions() # Unwind the main thread so atexit + finalisers run inside the grace window; # the daemon timer is the safety net if that unwind hangs. sys.exit(0) def _install_signal(signame, handler): """Install a signal handler if legal here: signal.signal() raises off the main thread (Desktop build path: server._build imports entry from a worker), and Windows lacks SIGPIPE/SIGHUP — both are skipped. Handlers are process-global.""" sig = getattr(signal, signame, None) if sig is None or threading.current_thread() is not threading.main_thread(): return # Off the main thread despite the check, or handler rejected by the platform. with suppress(ValueError, OSError, RuntimeError): signal.signal(sig, handler) # SIGPIPE: ignore, don't exit — SIG_DFL killed the process silently whenever a # *background* thread (TTS, beep) wrote to a pipe the TUI had gone quiet on; ignoring # lets write_json see BrokenPipeError and exit cleanly via _log_exit. Terminal signals # route through _log_signal so kills/hangups are diagnosable (SIGBREAK = Windows SIGHUP). _install_signal("SIGPIPE", signal.SIG_IGN) _install_signal("SIGTERM", _log_signal) if hasattr(signal, "SIGHUP"): _install_signal("SIGHUP", _log_signal) elif hasattr(signal, "SIGBREAK"): _install_signal("SIGBREAK", _log_signal) _install_signal("SIGINT", signal.SIG_IGN) def _log_exit(reason: str) -> None: """Record why the gateway exits: every path collapses into a silent sys.exit(0), and without this trail the TUI can't tell WHICH broken pipe triggered it.""" _append_crash_log(f"gateway exit · {_stamp()} · reason={reason}") print(f"[gateway-exit] {reason}", file=sys.stderr, flush=True) def wait_for_mcp_discovery(timeout: "float | None" = None) -> None: """Block until background MCP discovery finishes, up to the resolved bound (``mcp_discovery_timeout`` from config; ``timeout`` overrides). The agent snapshots its tool list ONCE at build time, so this bounded join lets already-spawning servers land without re-introducing the startup hang.""" thread = _mcp_discovery_thread if thread is not None and thread.is_alive(): fallback = timeout if timeout is not None else 0.75 bound = _mcp_startup_call("_resolve_discovery_timeout", timeout, default=fallback) thread.join(timeout=bound) return # Shared-owner path: re-invoke the idempotent spawn first so a zero-connected run # gets its retry instead of latching the process MCP-less. Runs under the CALLER's # profile context (agent build binds the session profile's HERMES_HOME first). if not _mcp_discovery_enabled: return _spawn_discovery(("debug", "TUI MCP discovery retry-spawn failed")) _mcp_startup_call("wait_for_mcp_discovery", timeout) def mcp_discovery_in_flight() -> bool: """True if ANY background MCP discovery thread is still running. Two owners by surface (stdio thread here, ``hermes_cli.mcp_startup`` for desktop/dashboard); the late-refresh scheduler calls this regardless of surface, so it MUST consult both or slow MCP servers' tools never surface on desktop.""" thread = _mcp_discovery_thread if thread is not None and thread.is_alive(): return True return _mcp_startup_call("mcp_discovery_in_flight", default=False) def join_mcp_discovery(timeout: float | None = None) -> bool: """Join both discovery owners; True once neither is alive. Unlike ``wait_for_mcp_discovery`` this accepts an unbounded wait (off-critical-path late-refresh waiter); ``timeout`` bounds EACH join, entry thread first.""" entry_done = True thread = _mcp_discovery_thread if thread is not None: thread.join(timeout=timeout) entry_done = not thread.is_alive() return entry_done and _mcp_startup_call("join_mcp_discovery", timeout=timeout, default=True) # Spurious stdin-EOF recovery tracker (shared open-file-description O_NONBLOCK flip). _recovery_times: list[float] = [] def _has_configured_mcp_servers() -> bool: """Delegate to the shared native and portable MCP startup gate.""" from hermes_cli.mcp_startup import _has_configured_mcp_servers as configured return configured() def ensure_mcp_discovery_started() -> None: """Start background MCP discovery for the current profile context, once. ``main()`` calls this for stdio; WS/Desktop skip ``main()``, so ``server._start_agent_build`` also calls it AFTER binding the session profile's HERMES_HOME so discovery reads the SELECTED profile's ``mcp_servers``. MCP registration is process-global: the FIRST profile to build an agent wins.""" global _mcp_discovery_enabled if not _has_configured_mcp_servers(): return _mcp_discovery_enabled = True _spawn_discovery(("warning", "Background MCP tool discovery failed to start")) def _write_or_exit(payload: dict, reason: str) -> None: if not write_json(payload): _log_exit(reason) sys.exit(0) def main(): _install_sidecar_publisher() # Heartbeat row lets the orphan sweep tell "live but idle" from "truly orphaned"; # it must run BEFORE the sweep. The sweep is once-per-process and config-gated. for start, what in ( (server._start_backend_heartbeat_refresher, "backend heartbeat refresher start"), (server._schedule_startup_orphan_sweep, "startup orphan sweep scheduling"), ): try: start() except Exception: logger.warning("%s failed", what, exc_info=True) # Backgrounded so a dead MCP server (~7s of retries) can't freeze startup; # _make_agent briefly joins it. The config gate keeps the MCP SDK import off # the no-mcp_servers path. ensure_mcp_discovery_started() _write_or_exit({ "jsonrpc": "2.0", "method": "event", "params": { "type": "gateway.ready", # change_events: clients demote legacy polls (see tui_gateway/ws.py). # replay_epoch: WS restart detection; the stdio TUI ignores it. "payload": { "skin": resolve_skin(), "change_events": True, "replay_epoch": replay_epoch(), }, }, }, "startup write failed (broken stdout pipe before first event)") # Live-apply skins Hermes activates mid-conversation. server._ensure_skin_watcher() # Warm the /model picker's provider-models cache in this idle window, else the # first /model open blocks on serial /v1/models fetches. Fire-and-forget. try: from hermes_cli.model_switch import prewarm_picker_cache_async prewarm_picker_cache_async() except Exception: logger.debug("picker cache prewarm (tui) failed to start", exc_info=True) while True: raw = sys.stdin.readline() if not raw: # Spurious (child flipped O_NONBLOCK on the shared description) or genuine EOF? if not handle_spurious_eof(_recovery_times, _log_exit): break continue line = raw.strip() if not line: continue try: req = json.loads(line) except json.JSONDecodeError: _write_or_exit( {"jsonrpc": "2.0", "error": {"code": -32700, "message": "parse error"}, "id": None}, "parse-error-response write failed (broken stdout pipe)") continue method = req.get("method") if isinstance(req, dict) else None resp = dispatch(req) if resp is not None: _write_or_exit( resp, f"response write failed for method={method!r} (broken stdout pipe)") if __name__ == "__main__": main()