fix(gateway): restart after startup SIGTERM

This commit is contained in:
YuYigeng
2026-09-05 05:29:44 +08:00
committed by kshitij
parent 6d9c166455
commit 51ddff0bb1
2 changed files with 79 additions and 4 deletions
+15 -4
View File
@@ -5096,6 +5096,17 @@ def _exit_with_failure_verdict(runner) -> bool:
return True
def _unexpected_signal_requires_restart(runner, signal_initiated_shutdown: bool) -> bool:
"""True when an unplanned signal should make the supervisor revive the gateway."""
if not signal_initiated_shutdown or runner._restart_requested:
return False
logger.info(
"Exiting with code 1 (signal-initiated shutdown without restart "
"request) so the service manager can revive the gateway."
)
return True
async def _start_gateway_shutdown_tail(
runner, _control_server, cron_stop: threading.Event, cron_provider,
cron_thread: threading.Thread, housekeeping_thread: threading.Thread,
@@ -5142,10 +5153,8 @@ async def _start_gateway_shutdown_tail(
if runner.exit_code is not None:
raise SystemExit(runner.exit_code)
# Unplanned SIGTERM exits non-zero so systemd Restart=on-failure revives us; planned stops must not.
if _signal_initiated_shutdown[0] and not runner._restart_requested:
logger.info("Exiting with code 1 (signal-initiated shutdown without restart "
"request) so systemd Restart=on-failure can revive the gateway.")
# Unplanned SIGTERM exits non-zero so the service manager revives us; planned stops must not.
if _unexpected_signal_requires_restart(runner, _signal_initiated_shutdown[0]):
return False # → sys.exit(1) in the caller
# Older restart paths may reach here without ``runner.exit_code``; keep the non-zero fallback.
@@ -5301,6 +5310,8 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =
await _shutdown_mcp_servers_nonblocking()
if runner.exit_code is not None:
raise SystemExit(runner.exit_code)
if _unexpected_signal_requires_restart(runner, _signal_initiated_shutdown[0]):
return False
return True
finally:
_shutdown_gateway_health_export(runner)
@@ -216,3 +216,67 @@ async def test_start_gateway_does_not_start_cron_after_aborted_startup(tmp_path,
assert exc.value.code == GATEWAY_SERVICE_RESTART_EXIT_CODE
assert cron_started is False
assert export_shutdown_calls == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
("unexpected_signal", "expected_success"),
[(True, False), (False, True)],
ids=["unexpected-sigterm", "planned-stop"],
)
async def test_start_gateway_classifies_startup_signal_exit(
tmp_path, monkeypatch, unexpected_signal, expected_success
):
"""A startup SIGTERM is restartable unless a planned-stop marker classified it as intentional."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
signal_state = None
cron_started = False
class AbortedStartupRunner:
def __init__(self, config):
self.config = config
self.adapters = {}
self._running = False
self._restart_requested = False
self.should_exit_cleanly = False
self.should_exit_with_failure = False
self.exit_reason = None
self.exit_code = None
async def start(self):
if unexpected_signal:
signal_state[0] = True
return True
async def wait_for_shutdown(self):
return None
def capture_signal_state(runner, state):
nonlocal signal_state
signal_state = state
return lambda received_signal=None: None
def fail_if_cron_starts(*args, **kwargs):
nonlocal cron_started
cron_started = True
monkeypatch.setattr("gateway.status.get_running_pid", lambda: None)
monkeypatch.setattr("gateway.status.acquire_gateway_runtime_lock", lambda: True)
monkeypatch.setattr("gateway.status.write_pid_file", lambda: None)
monkeypatch.setattr("gateway.status.remove_pid_file", lambda: None)
monkeypatch.setattr("gateway.status.release_gateway_runtime_lock", lambda: None)
monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None)
monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: None)
monkeypatch.setattr("gateway.run.GatewayRunner", AbortedStartupRunner)
monkeypatch.setattr(
"gateway.run._start_gateway_make_shutdown_signal_handler", capture_signal_state
)
monkeypatch.setattr("gateway.run._start_cron_ticker", fail_if_cron_starts)
monkeypatch.setattr("tools.mcp_tool_lifecycle.shutdown_mcp_servers", lambda: None)
result = await gateway_run.start_gateway(
config=GatewayConfig(), replace=False, verbosity=None
)
assert result is expected_success
assert cron_started is False