fix(gateway): distinguish failed systemd replacements

hermes_cli/gateway.py

@@ -1246,6 +1246,7 @@ def _wait_for_systemd_service_restart(
     system: bool = False,
     previous_pid: int | None = None,
     timeout: float | None = None,
+    replacement_observed: list[bool] | None = None,
 ) -> bool:
     """Wait for the gateway service to become active after a restart handoff."""
     import time
@@ -1270,6 +1271,14 @@ def _wait_for_systemd_service_restart(
             new_pid = None
         if not new_pid:
             new_pid = _systemd_main_pid_from_props(props)
+        if (
+            new_pid
+            and previous_pid is not None
+            and new_pid != previous_pid
+            and replacement_observed is not None
+            and not replacement_observed
+        ):
+            replacement_observed.append(True)
 
         if active_state == "active":
             if new_pid and (previous_pid is None or new_pid != previous_pid):
@@ -4146,7 +4155,14 @@ def systemd_restart(system: bool = False):
             # Exit 75 transfers restart ownership to systemd.  Observe that
             # single replacement instead of issuing another restart that can
             # stop the process systemd has already brought up.
-            if _wait_for_systemd_service_restart(system=system, previous_pid=pid):
+            replacement_observed: list[bool] = []
+            if _wait_for_systemd_service_restart(
+                system=system,
+                previous_pid=pid,
+                replacement_observed=replacement_observed,
+            ):
+                return
+            if replacement_observed:
                 return
             if _systemd_service_is_start_limited(system=system):
                 return

tests/hermes_cli/test_gateway_service.py

@@ -744,7 +744,10 @@ class TestGatewaySystemServiceRouting:
         monkeypatch.setattr(
             gateway_cli,
             "_wait_for_systemd_service_restart",
-            lambda system=False, previous_pid=None: calls.append(("wait", system, previous_pid)) or True,
+            lambda system=False, previous_pid=None, replacement_observed=None: calls.append(
+                ("wait", system, previous_pid)
+            )
+            or True,
         )
 
         gateway_cli.systemd_restart()
@@ -772,7 +775,7 @@ class TestGatewaySystemServiceRouting:
         monkeypatch.setattr(
             gateway_cli,
             "_wait_for_systemd_service_restart",
-            lambda system=False, previous_pid=None: next(waits),
+            lambda system=False, previous_pid=None, replacement_observed=None: next(waits),
         )
         monkeypatch.setattr(gateway_cli, "_systemd_service_is_start_limited", lambda system=False: False)
         monkeypatch.setattr(
@@ -805,7 +808,7 @@ class TestGatewaySystemServiceRouting:
         monkeypatch.setattr(
             gateway_cli,
             "_wait_for_systemd_service_restart",
-            lambda system=False, previous_pid=None: False,
+            lambda system=False, previous_pid=None, replacement_observed=None: False,
         )
         monkeypatch.setattr(gateway_cli, "_systemd_service_is_start_limited", lambda system=False: False)
         monkeypatch.setattr(
@@ -819,6 +822,34 @@ class TestGatewaySystemServiceRouting:
 
         assert calls == []
 
+    def test_systemd_restart_does_not_recover_a_failed_replacement(self, monkeypatch):
+        calls = []
+
+        monkeypatch.setattr(gateway_cli, "_select_systemd_scope", lambda system=False: False)
+        monkeypatch.setattr(gateway_cli, "_require_service_installed", lambda action, system=False: None)
+        monkeypatch.setattr(gateway_cli, "_preflight_user_systemd", lambda **kwargs: None)
+        monkeypatch.setattr(gateway_cli, "refresh_systemd_unit_if_needed", lambda system=False: None)
+        monkeypatch.setattr(gateway_cli, "_get_restart_exit_wait_budget", lambda: 27.0)
+        monkeypatch.setattr("gateway.status.get_running_pid", lambda: 654)
+        monkeypatch.setattr(gateway_cli, "_graceful_restart_via_sigusr1", lambda pid, timeout: True)
+
+        def failed_replacement_wait(
+            system=False, previous_pid=None, replacement_observed=None
+        ):
+            replacement_observed.append(True)
+            return False
+
+        monkeypatch.setattr(
+            gateway_cli,
+            "_wait_for_systemd_service_restart",
+            failed_replacement_wait,
+        )
+        monkeypatch.setattr(gateway_cli, "_run_systemctl", lambda args, **kwargs: calls.append(args))
+
+        gateway_cli.systemd_restart()
+
+        assert calls == []
+
     def test_systemd_restart_does_not_recover_when_handoff_state_is_unknown(
         self, monkeypatch
     ):
@@ -834,7 +865,7 @@ class TestGatewaySystemServiceRouting:
         monkeypatch.setattr(
             gateway_cli,
             "_wait_for_systemd_service_restart",
-            lambda system=False, previous_pid=None: False,
+            lambda system=False, previous_pid=None, replacement_observed=None: False,
         )
         monkeypatch.setattr(gateway_cli, "_systemd_service_is_start_limited", lambda system=False: False)
         monkeypatch.setattr(