fix(gateway): config.yaml surface for the startup watchdog + precise argv arming

Review follow-ups on the salvaged #89750:

- gateway.startup_watchdog / gateway.startup_watchdog_timeout_seconds in
  config_defaults, bridged to the internal HERMES_STARTUP_WATCHDOG env
  vars in run_gateway() (the argv fast-path arms before config can load,
  so env remains the mechanism; config.yaml is the user-facing surface
  per policy — explicit env values still win as operator override).
- hermes_cli/main.py argv sniff now requires the ADJACENT token pair
  'gateway run' instead of independent membership, so unrelated commands
  mentioning both words can't arm a 300s hard-exit timer; profile-flagged
  invocations (-p work gateway run) still arm.
This commit is contained in:
Kshitij Kapoor
2026-08-22 18:11:30 +05:30
committed by Teknium
parent 852db61abe
commit d2c3c38e98
3 changed files with 50 additions and 6 deletions
+10
View File
@@ -3223,6 +3223,16 @@ DEFAULT_CONFIG = {
"loop_watchdog_probe_timeout_s": 10.0,
"loop_watchdog_max_strikes": 3,
# Startup-liveness watchdog (OOF-298): plain daemon thread armed at
# process entry for gateway runs, hard-exits 75 if the event loop is
# not confirmed live within the deadline. The watchdog module itself
# is stdlib-only and armed before config can load, so these keys are
# BRIDGED to the internal HERMES_STARTUP_WATCHDOG /
# HERMES_STARTUP_WATCHDOG_TIMEOUT_S env vars by the gateway
# launchers — config.yaml is the user-facing surface.
"startup_watchdog": True,
"startup_watchdog_timeout_seconds": 300,
# Whether the gateway keeps writing the legacy sessions.json mirror of
# its routing index. The primary copy lives in state.db (the
# gateway_routing table). Default True for backward compatibility with
+25 -1
View File
@@ -6448,8 +6448,32 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo
# process-conflict guards: a --replace loser exiting above must not have
# armed a watchdog first. Disarmed by GatewayRunner once the event loop
# is confirmed live.
#
# config.yaml is the user-facing surface (gateway.startup_watchdog /
# gateway.startup_watchdog_timeout_seconds); the env vars are the
# internal bridge, needed because the argv fast-path arms before config
# can load. Explicit env values (operator override) are respected.
try:
from hermes_startup_watchdog import arm_startup_watchdog
from hermes_startup_watchdog import (
ENV_STARTUP_WATCHDOG,
ENV_STARTUP_WATCHDOG_TIMEOUT_S,
arm_startup_watchdog,
)
try:
from hermes_cli.config import load_config as _sw_load_config
_gw_cfg = (_sw_load_config() or {}).get("gateway", {}) or {}
if ENV_STARTUP_WATCHDOG not in os.environ and not _gw_cfg.get(
"startup_watchdog", True
):
os.environ[ENV_STARTUP_WATCHDOG] = "0"
_sw_timeout = _gw_cfg.get("startup_watchdog_timeout_seconds")
if (
ENV_STARTUP_WATCHDOG_TIMEOUT_S not in os.environ
and _sw_timeout is not None
):
os.environ[ENV_STARTUP_WATCHDOG_TIMEOUT_S] = str(_sw_timeout)
except Exception:
pass
arm_startup_watchdog()
except Exception:
pass
+15 -5
View File
@@ -107,11 +107,21 @@ except Exception:
# init, contended import lock) is exactly the "wedged before the event loop,
# no logs, live PID" class this watchdog exists for. ``hermes_startup_watchdog``
# is stdlib-only, so importing it here cannot itself wedge on application
# code. argv sniffing is deliberately crude: over-arming is harmless (any
# non-gateway command either exits well inside the deadline or... should be
# covered anyway if it wedges), while under-arming recreates OOF-298.
# GatewayRunner disarms once the event loop is confirmed live.
if "gateway" in sys.argv[1:] and "run" in sys.argv[1:]:
# code. The match requires the ADJACENT token pair ``gateway run`` (the
# subcommand shape, wherever global flags like ``-p <profile>`` put it) so
# unrelated commands that merely mention both words in different arguments
# never arm a 300s hard-exit timer, while flag-carrying invocations still
# do — under-arming recreates OOF-298. Foreground `hermes gateway run`
# still arms — a pre-loop wedge is just as dead without a supervisor, and
# the stack dump plus exit beats a silent hang; GatewayRunner disarms once
# the event loop is confirmed live.
def _argv_is_gateway_run(argv: list) -> bool:
return any(
a == "gateway" and b == "run" for a, b in zip(argv, argv[1:])
)
if _argv_is_gateway_run(sys.argv[1:]):
try:
from hermes_startup_watchdog import arm_startup_watchdog as _arm_sw