fix(gateway): config.yaml surface for the startup watchdog + precise argv arming
Review follow-ups on the salvaged #89750: - gateway.startup_watchdog / gateway.startup_watchdog_timeout_seconds in config_defaults, bridged to the internal HERMES_STARTUP_WATCHDOG env vars in run_gateway() (the argv fast-path arms before config can load, so env remains the mechanism; config.yaml is the user-facing surface per policy — explicit env values still win as operator override). - hermes_cli/main.py argv sniff now requires the ADJACENT token pair 'gateway run' instead of independent membership, so unrelated commands mentioning both words can't arm a 300s hard-exit timer; profile-flagged invocations (-p work gateway run) still arm.
This commit is contained in:
@@ -3223,6 +3223,16 @@ DEFAULT_CONFIG = {
|
||||
"loop_watchdog_probe_timeout_s": 10.0,
|
||||
"loop_watchdog_max_strikes": 3,
|
||||
|
||||
# Startup-liveness watchdog (OOF-298): plain daemon thread armed at
|
||||
# process entry for gateway runs, hard-exits 75 if the event loop is
|
||||
# not confirmed live within the deadline. The watchdog module itself
|
||||
# is stdlib-only and armed before config can load, so these keys are
|
||||
# BRIDGED to the internal HERMES_STARTUP_WATCHDOG /
|
||||
# HERMES_STARTUP_WATCHDOG_TIMEOUT_S env vars by the gateway
|
||||
# launchers — config.yaml is the user-facing surface.
|
||||
"startup_watchdog": True,
|
||||
"startup_watchdog_timeout_seconds": 300,
|
||||
|
||||
# Whether the gateway keeps writing the legacy sessions.json mirror of
|
||||
# its routing index. The primary copy lives in state.db (the
|
||||
# gateway_routing table). Default True for backward compatibility with
|
||||
|
||||
+25
-1
@@ -6448,8 +6448,32 @@ def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, fo
|
||||
# process-conflict guards: a --replace loser exiting above must not have
|
||||
# armed a watchdog first. Disarmed by GatewayRunner once the event loop
|
||||
# is confirmed live.
|
||||
#
|
||||
# config.yaml is the user-facing surface (gateway.startup_watchdog /
|
||||
# gateway.startup_watchdog_timeout_seconds); the env vars are the
|
||||
# internal bridge, needed because the argv fast-path arms before config
|
||||
# can load. Explicit env values (operator override) are respected.
|
||||
try:
|
||||
from hermes_startup_watchdog import arm_startup_watchdog
|
||||
from hermes_startup_watchdog import (
|
||||
ENV_STARTUP_WATCHDOG,
|
||||
ENV_STARTUP_WATCHDOG_TIMEOUT_S,
|
||||
arm_startup_watchdog,
|
||||
)
|
||||
try:
|
||||
from hermes_cli.config import load_config as _sw_load_config
|
||||
_gw_cfg = (_sw_load_config() or {}).get("gateway", {}) or {}
|
||||
if ENV_STARTUP_WATCHDOG not in os.environ and not _gw_cfg.get(
|
||||
"startup_watchdog", True
|
||||
):
|
||||
os.environ[ENV_STARTUP_WATCHDOG] = "0"
|
||||
_sw_timeout = _gw_cfg.get("startup_watchdog_timeout_seconds")
|
||||
if (
|
||||
ENV_STARTUP_WATCHDOG_TIMEOUT_S not in os.environ
|
||||
and _sw_timeout is not None
|
||||
):
|
||||
os.environ[ENV_STARTUP_WATCHDOG_TIMEOUT_S] = str(_sw_timeout)
|
||||
except Exception:
|
||||
pass
|
||||
arm_startup_watchdog()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
+15
-5
@@ -107,11 +107,21 @@ except Exception:
|
||||
# init, contended import lock) is exactly the "wedged before the event loop,
|
||||
# no logs, live PID" class this watchdog exists for. ``hermes_startup_watchdog``
|
||||
# is stdlib-only, so importing it here cannot itself wedge on application
|
||||
# code. argv sniffing is deliberately crude: over-arming is harmless (any
|
||||
# non-gateway command either exits well inside the deadline or... should be
|
||||
# covered anyway if it wedges), while under-arming recreates OOF-298.
|
||||
# GatewayRunner disarms once the event loop is confirmed live.
|
||||
if "gateway" in sys.argv[1:] and "run" in sys.argv[1:]:
|
||||
# code. The match requires the ADJACENT token pair ``gateway run`` (the
|
||||
# subcommand shape, wherever global flags like ``-p <profile>`` put it) so
|
||||
# unrelated commands that merely mention both words in different arguments
|
||||
# never arm a 300s hard-exit timer, while flag-carrying invocations still
|
||||
# do — under-arming recreates OOF-298. Foreground `hermes gateway run`
|
||||
# still arms — a pre-loop wedge is just as dead without a supervisor, and
|
||||
# the stack dump plus exit beats a silent hang; GatewayRunner disarms once
|
||||
# the event loop is confirmed live.
|
||||
def _argv_is_gateway_run(argv: list) -> bool:
|
||||
return any(
|
||||
a == "gateway" and b == "run" for a, b in zip(argv, argv[1:])
|
||||
)
|
||||
|
||||
|
||||
if _argv_is_gateway_run(sys.argv[1:]):
|
||||
try:
|
||||
from hermes_startup_watchdog import arm_startup_watchdog as _arm_sw
|
||||
|
||||
|
||||
Reference in New Issue
Block a user