fix(gateway): restart after startup SIGTERM
This commit is contained in:
+15
-4
@@ -5096,6 +5096,17 @@ def _exit_with_failure_verdict(runner) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _unexpected_signal_requires_restart(runner, signal_initiated_shutdown: bool) -> bool:
|
||||
"""True when an unplanned signal should make the supervisor revive the gateway."""
|
||||
if not signal_initiated_shutdown or runner._restart_requested:
|
||||
return False
|
||||
logger.info(
|
||||
"Exiting with code 1 (signal-initiated shutdown without restart "
|
||||
"request) so the service manager can revive the gateway."
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def _start_gateway_shutdown_tail(
|
||||
runner, _control_server, cron_stop: threading.Event, cron_provider,
|
||||
cron_thread: threading.Thread, housekeeping_thread: threading.Thread,
|
||||
@@ -5142,10 +5153,8 @@ async def _start_gateway_shutdown_tail(
|
||||
if runner.exit_code is not None:
|
||||
raise SystemExit(runner.exit_code)
|
||||
|
||||
# Unplanned SIGTERM exits non-zero so systemd Restart=on-failure revives us; planned stops must not.
|
||||
if _signal_initiated_shutdown[0] and not runner._restart_requested:
|
||||
logger.info("Exiting with code 1 (signal-initiated shutdown without restart "
|
||||
"request) so systemd Restart=on-failure can revive the gateway.")
|
||||
# Unplanned SIGTERM exits non-zero so the service manager revives us; planned stops must not.
|
||||
if _unexpected_signal_requires_restart(runner, _signal_initiated_shutdown[0]):
|
||||
return False # → sys.exit(1) in the caller
|
||||
|
||||
# Older restart paths may reach here without ``runner.exit_code``; keep the non-zero fallback.
|
||||
@@ -5301,6 +5310,8 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =
|
||||
await _shutdown_mcp_servers_nonblocking()
|
||||
if runner.exit_code is not None:
|
||||
raise SystemExit(runner.exit_code)
|
||||
if _unexpected_signal_requires_restart(runner, _signal_initiated_shutdown[0]):
|
||||
return False
|
||||
return True
|
||||
finally:
|
||||
_shutdown_gateway_health_export(runner)
|
||||
|
||||
@@ -216,3 +216,67 @@ async def test_start_gateway_does_not_start_cron_after_aborted_startup(tmp_path,
|
||||
assert exc.value.code == GATEWAY_SERVICE_RESTART_EXIT_CODE
|
||||
assert cron_started is False
|
||||
assert export_shutdown_calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("unexpected_signal", "expected_success"),
|
||||
[(True, False), (False, True)],
|
||||
ids=["unexpected-sigterm", "planned-stop"],
|
||||
)
|
||||
async def test_start_gateway_classifies_startup_signal_exit(
|
||||
tmp_path, monkeypatch, unexpected_signal, expected_success
|
||||
):
|
||||
"""A startup SIGTERM is restartable unless a planned-stop marker classified it as intentional."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
signal_state = None
|
||||
cron_started = False
|
||||
|
||||
class AbortedStartupRunner:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.adapters = {}
|
||||
self._running = False
|
||||
self._restart_requested = False
|
||||
self.should_exit_cleanly = False
|
||||
self.should_exit_with_failure = False
|
||||
self.exit_reason = None
|
||||
self.exit_code = None
|
||||
|
||||
async def start(self):
|
||||
if unexpected_signal:
|
||||
signal_state[0] = True
|
||||
return True
|
||||
|
||||
async def wait_for_shutdown(self):
|
||||
return None
|
||||
|
||||
def capture_signal_state(runner, state):
|
||||
nonlocal signal_state
|
||||
signal_state = state
|
||||
return lambda received_signal=None: None
|
||||
|
||||
def fail_if_cron_starts(*args, **kwargs):
|
||||
nonlocal cron_started
|
||||
cron_started = True
|
||||
|
||||
monkeypatch.setattr("gateway.status.get_running_pid", lambda: None)
|
||||
monkeypatch.setattr("gateway.status.acquire_gateway_runtime_lock", lambda: True)
|
||||
monkeypatch.setattr("gateway.status.write_pid_file", lambda: None)
|
||||
monkeypatch.setattr("gateway.status.remove_pid_file", lambda: None)
|
||||
monkeypatch.setattr("gateway.status.release_gateway_runtime_lock", lambda: None)
|
||||
monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None)
|
||||
monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: None)
|
||||
monkeypatch.setattr("gateway.run.GatewayRunner", AbortedStartupRunner)
|
||||
monkeypatch.setattr(
|
||||
"gateway.run._start_gateway_make_shutdown_signal_handler", capture_signal_state
|
||||
)
|
||||
monkeypatch.setattr("gateway.run._start_cron_ticker", fail_if_cron_starts)
|
||||
monkeypatch.setattr("tools.mcp_tool_lifecycle.shutdown_mcp_servers", lambda: None)
|
||||
|
||||
result = await gateway_run.start_gateway(
|
||||
config=GatewayConfig(), replace=False, verbosity=None
|
||||
)
|
||||
|
||||
assert result is expected_success
|
||||
assert cron_started is False
|
||||
|
||||
Reference in New Issue
Block a user