diff --git a/gateway/run.py b/gateway/run.py index f3cffd185a..50b2925d71 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -15218,8 +15218,9 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # The service manager is the sole restart owner. Exit 75 # paired with ``RestartForceExitStatus=75`` asks systemd to # replace this process without a second helper racing the - # unit's stop/start job. launchd likewise treats the planned - # non-zero exit as restartable. + # unit's stop/start job. The generated launchd plist's + # unconditional ``KeepAlive`` likewise replaces the process + # after this planned exit. self._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE self._exit_reason = self._exit_reason or "Gateway restart requested" diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index fed0e98c7f..66d574ecf3 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -1245,13 +1245,15 @@ def _wait_for_systemd_service_restart( *, system: bool = False, previous_pid: int | None = None, - timeout: float = 60.0, + timeout: float | None = None, ) -> bool: """Wait for the gateway service to become active after a restart handoff.""" import time svc = get_service_name() scope_label = _service_scope_label(system).capitalize() + if timeout is None: + timeout = _systemd_restart_wait_timeout(system=system) deadline = time.monotonic() + timeout printed_runtime_wait = False @@ -1308,6 +1310,25 @@ def _wait_for_systemd_service_restart( return False +def _systemd_restart_wait_timeout(system: bool = False) -> float: + """Cover systemd's relaunch delays before applying the runtime wait floor.""" + from gateway.shutdown_forensics import _parse_systemd_duration_to_us + + props = _read_systemd_unit_properties( + system=system, + properties=("RestartUSec", "TimeoutStartUSec"), + ) + supervisor_budget = 0.0 + for name in ("RestartUSec", "TimeoutStartUSec"): + raw = props.get(name, "") + duration_us = ( + int(raw) if raw.isdigit() else _parse_systemd_duration_to_us(raw) + ) + if duration_us is not None: + supervisor_budget += duration_us / 1_000_000 + return 60.0 + supervisor_budget + + def _systemd_unit_is_start_limited(props: dict[str, str]) -> bool: result = props.get("Result", "").lower() sub_state = props.get("SubState", "").lower() @@ -4124,13 +4145,32 @@ def systemd_restart(system: bool = False): # Exit 75 transfers restart ownership to systemd. Observe that # single replacement instead of issuing another restart that can # stop the process systemd has already brought up. - _wait_for_systemd_service_restart(system=system, previous_pid=pid) - return + if _wait_for_systemd_service_restart(system=system, previous_pid=pid): + return + if _systemd_service_is_start_limited(system=system): + return + + # A replacement may have started but not reached gateway runtime + # readiness before the wait expired. Never stop that generation. + props = _read_systemd_unit_properties(system=system) + replacement_pid = _systemd_main_pid_from_props(props) + if ( + props.get("ActiveState") in {"active", "activating", "reloading"} + or props.get("SubState") == "auto-restart" + or (replacement_pid is not None and replacement_pid != pid) + ): + return + + print( + "⚠ Systemd did not relaunch the gateway after its graceful exit; " + "forcing a service restart..." + ) + else: + print( + f"⚠ Graceful restart did not complete within {int(wait_budget)}s; " + "forcing a service restart..." + ) - print( - f"⚠ Graceful restart did not complete within {int(wait_budget)}s; " - "forcing a service restart..." - ) _run_systemctl( ["reset-failed", svc], system=system, diff --git a/tests/gateway/test_gateway_shutdown.py b/tests/gateway/test_gateway_shutdown.py index 3aaeb51696..eb7ae47090 100644 --- a/tests/gateway/test_gateway_shutdown.py +++ b/tests/gateway/test_gateway_shutdown.py @@ -1,4 +1,5 @@ import asyncio +import subprocess from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -147,6 +148,26 @@ async def test_gateway_stop_settles_completion_batch_before_adapter_disconnect() assert runner._completion_notification_batch_flush_tasks == set() +@pytest.mark.asyncio +async def test_planned_service_exit_issues_no_restart_of_its_own(monkeypatch): + runner, adapter = make_restart_runner() + adapter.disconnect = AsyncMock() + runner._restart_requested = True + runner._restart_via_service = True + monkeypatch.setattr( + subprocess, + "Popen", + lambda *args, **kwargs: pytest.fail( + f"planned service exit must not spawn a restart helper: {args}" + ), + ) + + with patch("gateway.status.remove_pid_file"), patch("gateway.status.write_runtime_status"): + await runner.stop() + + assert runner._exit_code == GATEWAY_SERVICE_RESTART_EXIT_CODE + + @pytest.mark.asyncio async def test_in_chat_restart_skips_home_shutdown_even_with_active_session(): runner, adapter = make_restart_runner() diff --git a/tests/hermes_cli/test_gateway_service.py b/tests/hermes_cli/test_gateway_service.py index 02e4566e48..1e38d90786 100644 --- a/tests/hermes_cli/test_gateway_service.py +++ b/tests/hermes_cli/test_gateway_service.py @@ -756,6 +756,81 @@ class TestGatewaySystemServiceRouting: assert "21627" not in out # must use the mocked budget, not live defaults assert "27" in out + def test_systemd_restart_forces_recovery_only_when_handoff_has_no_replacement( + self, monkeypatch, capsys + ): + calls = [] + + monkeypatch.setattr(gateway_cli, "_select_systemd_scope", lambda system=False: False) + monkeypatch.setattr(gateway_cli, "_require_service_installed", lambda action, system=False: None) + monkeypatch.setattr(gateway_cli, "_preflight_user_systemd", lambda **kwargs: None) + monkeypatch.setattr(gateway_cli, "refresh_systemd_unit_if_needed", lambda system=False: None) + monkeypatch.setattr(gateway_cli, "_get_restart_exit_wait_budget", lambda: 27.0) + monkeypatch.setattr("gateway.status.get_running_pid", lambda: 654) + monkeypatch.setattr(gateway_cli, "_graceful_restart_via_sigusr1", lambda pid, timeout: True) + waits = iter((False, True)) + monkeypatch.setattr( + gateway_cli, + "_wait_for_systemd_service_restart", + lambda system=False, previous_pid=None: next(waits), + ) + monkeypatch.setattr(gateway_cli, "_systemd_service_is_start_limited", lambda system=False: False) + monkeypatch.setattr( + gateway_cli, + "_read_systemd_unit_properties", + lambda system=False, properties=None: {"ActiveState": "inactive", "MainPID": "0"}, + ) + monkeypatch.setattr( + gateway_cli, + "_run_systemctl", + lambda args, **kwargs: calls.append((args, kwargs)) + or SimpleNamespace(returncode=0, stdout="", stderr=""), + ) + + gateway_cli.systemd_restart() + + assert [call[0][0] for call in calls] == ["reset-failed", "restart"] + assert "did not relaunch" in capsys.readouterr().out + + def test_systemd_restart_does_not_force_an_unready_replacement(self, monkeypatch): + calls = [] + + monkeypatch.setattr(gateway_cli, "_select_systemd_scope", lambda system=False: False) + monkeypatch.setattr(gateway_cli, "_require_service_installed", lambda action, system=False: None) + monkeypatch.setattr(gateway_cli, "_preflight_user_systemd", lambda **kwargs: None) + monkeypatch.setattr(gateway_cli, "refresh_systemd_unit_if_needed", lambda system=False: None) + monkeypatch.setattr(gateway_cli, "_get_restart_exit_wait_budget", lambda: 27.0) + monkeypatch.setattr("gateway.status.get_running_pid", lambda: 654) + monkeypatch.setattr(gateway_cli, "_graceful_restart_via_sigusr1", lambda pid, timeout: True) + monkeypatch.setattr( + gateway_cli, + "_wait_for_systemd_service_restart", + lambda system=False, previous_pid=None: False, + ) + monkeypatch.setattr(gateway_cli, "_systemd_service_is_start_limited", lambda system=False: False) + monkeypatch.setattr( + gateway_cli, + "_read_systemd_unit_properties", + lambda system=False, properties=None: {"ActiveState": "active", "MainPID": "777"}, + ) + monkeypatch.setattr(gateway_cli, "_run_systemctl", lambda args, **kwargs: calls.append(args)) + + gateway_cli.systemd_restart() + + assert calls == [] + + def test_systemd_restart_wait_timeout_includes_supervisor_budgets(self, monkeypatch): + monkeypatch.setattr( + gateway_cli, + "_read_systemd_unit_properties", + lambda system=False, properties=None: { + "RestartUSec": "5s", + "TimeoutStartUSec": "1min 30s", + }, + ) + + assert gateway_cli._systemd_restart_wait_timeout() == 155.0 +