diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 8c1733d551..8ee4e5c931 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -3223,6 +3223,16 @@ DEFAULT_CONFIG = { "loop_watchdog_probe_timeout_s": 10.0, "loop_watchdog_max_strikes": 3, + # Startup-liveness watchdog (OOF-298): plain daemon thread armed at + # process entry for gateway runs, hard-exits 75 if the event loop is + # not confirmed live within the deadline. The watchdog module itself + # is stdlib-only and armed before config can load, so these keys are + # BRIDGED to the internal HERMES_STARTUP_WATCHDOG / + # HERMES_STARTUP_WATCHDOG_TIMEOUT_S env vars by the gateway + # launchers — config.yaml is the user-facing surface. + "startup_watchdog": True, + "startup_watchdog_timeout_seconds": 300, + # Whether the gateway keeps writing the legacy sessions.json mirror of # its routing index. The primary copy lives in state.db (the # gateway_routing table). Default True for backward compatibility with diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index bf111ac3d7..d81ff4bfe0 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -6448,8 +6448,32 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo # process-conflict guards: a --replace loser exiting above must not have # armed a watchdog first. Disarmed by GatewayRunner once the event loop # is confirmed live. + # + # config.yaml is the user-facing surface (gateway.startup_watchdog / + # gateway.startup_watchdog_timeout_seconds); the env vars are the + # internal bridge, needed because the argv fast-path arms before config + # can load. Explicit env values (operator override) are respected. try: - from hermes_startup_watchdog import arm_startup_watchdog + from hermes_startup_watchdog import ( + ENV_STARTUP_WATCHDOG, + ENV_STARTUP_WATCHDOG_TIMEOUT_S, + arm_startup_watchdog, + ) + try: + from hermes_cli.config import load_config as _sw_load_config + _gw_cfg = (_sw_load_config() or {}).get("gateway", {}) or {} + if ENV_STARTUP_WATCHDOG not in os.environ and not _gw_cfg.get( + "startup_watchdog", True + ): + os.environ[ENV_STARTUP_WATCHDOG] = "0" + _sw_timeout = _gw_cfg.get("startup_watchdog_timeout_seconds") + if ( + ENV_STARTUP_WATCHDOG_TIMEOUT_S not in os.environ + and _sw_timeout is not None + ): + os.environ[ENV_STARTUP_WATCHDOG_TIMEOUT_S] = str(_sw_timeout) + except Exception: + pass arm_startup_watchdog() except Exception: pass diff --git a/hermes_cli/main.py b/hermes_cli/main.py index d1f084d5b7..82e3933255 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -107,11 +107,21 @@ except Exception: # init, contended import lock) is exactly the "wedged before the event loop, # no logs, live PID" class this watchdog exists for. ``hermes_startup_watchdog`` # is stdlib-only, so importing it here cannot itself wedge on application -# code. argv sniffing is deliberately crude: over-arming is harmless (any -# non-gateway command either exits well inside the deadline or... should be -# covered anyway if it wedges), while under-arming recreates OOF-298. -# GatewayRunner disarms once the event loop is confirmed live. -if "gateway" in sys.argv[1:] and "run" in sys.argv[1:]: +# code. The match requires the ADJACENT token pair ``gateway run`` (the +# subcommand shape, wherever global flags like ``-p `` put it) so +# unrelated commands that merely mention both words in different arguments +# never arm a 300s hard-exit timer, while flag-carrying invocations still +# do — under-arming recreates OOF-298. Foreground `hermes gateway run` +# still arms — a pre-loop wedge is just as dead without a supervisor, and +# the stack dump plus exit beats a silent hang; GatewayRunner disarms once +# the event loop is confirmed live. +def _argv_is_gateway_run(argv: list) -> bool: + return any( + a == "gateway" and b == "run" for a, b in zip(argv, argv[1:]) + ) + + +if _argv_is_gateway_run(sys.argv[1:]): try: from hermes_startup_watchdog import arm_startup_watchdog as _arm_sw