diff --git a/hermes_cli/main.py b/hermes_cli/main.py index dcba3617ab..6d11237fe8 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -5082,6 +5082,7 @@ _LAZY_COMMAND_EXPORTS = { "_for_each_systemd_gateway_unit", "_format_concurrent_instances_message", "_format_time_ago", + "_gateway_service_matches_profile", "_gateway_restart_recovery_profiles", "_handoff_reapable_backend_pids", "_ledger_reapable_backend_pids", diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 373386a6ca..3813467479 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -5592,6 +5592,29 @@ def _surviving_gateway_pids_after_failed_restart(): _FRESH_RESTART_SUPERVISORS = frozenset({"systemd", "launchd", "service", "s6"}) +def _gateway_service_matches_profile(profile: str, service: object) -> bool: + """Match an exact gateway service/label to a profile. + + Profile names must not be matched as substrings: ``foo`` must not claim + that ``hermes-gateway-foobar.service`` was already restarted. These are + the service/label shapes produced by the existing systemd, launchd, and + s6 lifecycle implementations. + """ + name = str(service).removesuffix(".service") + if profile == "default": + return name in { + "hermes-gateway", + "ai.hermes.gateway", + "gateway", + "gateway-default", + } + return name in { + f"hermes-gateway-{profile}", + f"ai.hermes.gateway-{profile}", + f"gateway-{profile}", + } + + def _gateway_restart_recovery_profiles( plan, *, skip_profiles: set[str] | None = None ) -> list[str]: @@ -5621,7 +5644,7 @@ def _gateway_restart_recovery_profiles( def _recover_gateway_restart_after_abort( plan, *, gateway_mode: bool, skip_profiles: set[str] | None = None -) -> bool: +) -> dict[str, list[str]]: """Retry supervised gateway restarts from a clean Python process. ``hermes update`` normally performs the fleet restart in the interpreter @@ -5635,12 +5658,12 @@ def _recover_gateway_restart_after_abort( Only profiles classified as supervisor-owned by the pre-update inventory are handed off. A manual gateway must remain running and be reported for explicit operator action rather than being killed without a relaunch - authority. Returns True only when the recovery child reports success for - every handed-off profile. + authority. The returned protocol is persisted in the update receipt so + operators can distinguish a spawn failure from a per-profile failure. """ profiles = _gateway_restart_recovery_profiles(plan, skip_profiles=skip_profiles) if not profiles: - return False + return {"requested": [], "succeeded": [], "failed": []} command = [ sys.executable, @@ -5662,7 +5685,7 @@ def _recover_gateway_restart_after_abort( systemd_run = shutil.which("systemd-run") if not systemd_run: logger.warning("Cannot isolate fresh gateway recovery from the gateway cgroup") - return False + return {"requested": profiles, "succeeded": [], "failed": profiles} command = [ systemd_run, "--user", @@ -5695,11 +5718,11 @@ def _recover_gateway_restart_after_abort( result = subprocess.run(command, **kwargs) except (OSError, subprocess.TimeoutExpired) as exc: logger.warning("Fresh gateway restart recovery failed: %s", exc) - return False + return {"requested": profiles, "succeeded": [], "failed": profiles} if result.returncode != 0: logger.warning("Fresh gateway restart recovery exited %s", result.returncode) - return False + return {"requested": profiles, "succeeded": [], "failed": profiles} try: recovery_result = json.loads(result.stdout or "") @@ -5707,7 +5730,7 @@ def _recover_gateway_restart_after_abort( failed = recovery_result.get("failed") except (AttributeError, TypeError, ValueError): logger.warning("Fresh gateway restart recovery returned invalid JSON") - return False + return {"requested": profiles, "succeeded": [], "failed": profiles} if ( not isinstance(succeeded, list) @@ -5717,13 +5740,17 @@ def _recover_gateway_restart_after_abort( or failed ): logger.warning("Fresh gateway restart recovery returned incomplete profiles") - return False + return { + "requested": profiles, + "succeeded": succeeded if isinstance(succeeded, list) else [], + "failed": failed if isinstance(failed, list) else profiles, + } print( " ✓ Retried supervised gateway restart(s) in a fresh process: " + ", ".join(profiles) ) - return True + return {"requested": profiles, "succeeded": succeeded, "failed": []} def _warn_gateway_restart_phase_aborted(exc: BaseException, pids) -> None: @@ -8766,24 +8793,25 @@ def _cmd_update_impl(args, gateway_mode: bool): if not isinstance(profile, str): continue if any( - profile in str(service) - or (profile == "default" and "hermes-gateway" in str(service)) + _gateway_service_matches_profile(profile, service) for service in restarted_services ): _already_restarted_profiles.add(profile) - _recovery_profiles = _gateway_restart_recovery_profiles( - _pre_update_plan, - skip_profiles=_already_restarted_profiles, - ) - _recovered = _recover_gateway_restart_after_abort( + _recovery_result = _recover_gateway_restart_after_abort( _pre_update_plan, gateway_mode=gateway_mode, skip_profiles=_already_restarted_profiles, ) - if _recovered: + _recovery_profiles = list(_recovery_result.get("requested") or []) + _recovery_succeeded = set(_recovery_result.get("succeeded") or []) + _recovered = not _recovery_profiles or ( + _recovery_succeeded == set(_recovery_profiles) + and not _recovery_result.get("failed") + ) + if _recovery_succeeded: relaunched_profiles.extend( profile - for profile in _recovery_profiles + for profile in sorted(_recovery_succeeded) if profile not in relaunched_profiles ) _planned_gateway_runtimes = [ @@ -8796,11 +8824,11 @@ def _cmd_update_impl(args, gateway_mode: bool): runtime.profile for runtime in _planned_gateway_runtimes } _covered_gateway_profiles = _already_restarted_profiles | set( - _recovery_profiles + _recovery_succeeded ) _all_gateway_profiles_are_safe_to_recover = all( runtime.profile in _already_restarted_profiles - or runtime.supervisor in _FRESH_RESTART_SUPERVISORS + or getattr(runtime, "supervisor", None) in _FRESH_RESTART_SUPERVISORS for runtime in _planned_gateway_runtimes ) _recovery_complete = bool(_planned_gateway_profiles) and ( @@ -8835,6 +8863,7 @@ def _cmd_update_impl(args, gateway_mode: bool): failed_units=failed_or_stale_units, incomplete=gateway_fleet_restart_incomplete, phase_error=str(e), + fresh_recovery=_recovery_result, ) except Exception: pass diff --git a/tests/hermes_cli/test_update_restart_recovery.py b/tests/hermes_cli/test_update_restart_recovery.py index b6f0e9b65e..9e66119fd2 100644 --- a/tests/hermes_cli/test_update_restart_recovery.py +++ b/tests/hermes_cli/test_update_restart_recovery.py @@ -59,7 +59,12 @@ def test_abort_recovery_hands_managed_profiles_to_a_fresh_process(monkeypatch): ] ) - assert update_cmd._recover_gateway_restart_after_abort(plan, gateway_mode=False) + result = update_cmd._recover_gateway_restart_after_abort(plan, gateway_mode=False) + assert result == { + "requested": ["coder", "default"], + "succeeded": ["coder", "default"], + "failed": [], + } assert len(calls) == 1 argv, kwargs = calls[0] @@ -81,7 +86,10 @@ def test_abort_recovery_does_not_claim_success_when_fresh_process_fails(monkeypa ) plan = SimpleNamespace(runtimes=[_runtime("default", "systemd")]) - assert update_cmd._recover_gateway_restart_after_abort(plan, gateway_mode=False) is False + result = update_cmd._recover_gateway_restart_after_abort(plan, gateway_mode=False) + assert result["requested"] == ["default"] + assert result["succeeded"] == [] + assert result["failed"] == ["default"] def test_abort_recovery_skips_profiles_already_restarted_by_the_phase(monkeypatch): @@ -96,11 +104,12 @@ def test_abort_recovery_skips_profiles_already_restarted_by_the_phase(monkeypatc runtimes=[_runtime("default", "systemd"), _runtime("coder", "systemd")] ) - assert update_cmd._recover_gateway_restart_after_abort( + result = update_cmd._recover_gateway_restart_after_abort( plan, gateway_mode=False, skip_profiles={"default"}, ) + assert result == {"requested": ["coder"], "succeeded": ["coder"], "failed": []} assert json.loads(calls[0][1]["input"]) == {"profiles": ["coder"]} @@ -117,7 +126,10 @@ def test_abort_recovery_rejects_partial_json_success(monkeypatch): runtimes=[_runtime("default", "systemd"), _runtime("coder", "systemd")] ) - assert update_cmd._recover_gateway_restart_after_abort(plan, gateway_mode=False) is False + result = update_cmd._recover_gateway_restart_after_abort(plan, gateway_mode=False) + assert result["requested"] == ["coder", "default"] + assert result["succeeded"] == ["default"] + assert result["failed"] == ["coder"] def test_abort_recovery_rejects_malformed_json_success(monkeypatch): @@ -128,7 +140,10 @@ def test_abort_recovery_rejects_malformed_json_success(monkeypatch): ) plan = SimpleNamespace(runtimes=[_runtime("default", "systemd")]) - assert update_cmd._recover_gateway_restart_after_abort(plan, gateway_mode=False) is False + result = update_cmd._recover_gateway_restart_after_abort(plan, gateway_mode=False) + assert result["requested"] == ["default"] + assert result["succeeded"] == [] + assert result["failed"] == ["default"] def test_abort_recovery_does_not_restart_manual_only_fleet(monkeypatch): @@ -140,10 +155,26 @@ def test_abort_recovery_does_not_restart_manual_only_fleet(monkeypatch): ) plan = SimpleNamespace(runtimes=[_runtime("manual-box", "manual")]) - assert update_cmd._recover_gateway_restart_after_abort(plan, gateway_mode=False) is False + result = update_cmd._recover_gateway_restart_after_abort(plan, gateway_mode=False) + assert result == {"requested": [], "succeeded": [], "failed": []} assert calls == [] +def test_service_matching_is_exact_for_overlapping_profile_names(): + assert update_cmd._gateway_service_matches_profile( + "foo", "hermes-gateway-foo.service" + ) + assert not update_cmd._gateway_service_matches_profile( + "foo", "hermes-gateway-foobar.service" + ) + assert update_cmd._gateway_service_matches_profile( + "default", "ai.hermes.gateway" + ) + assert not update_cmd._gateway_service_matches_profile( + "default", "ai.hermes.gateway-foo" + ) + + def test_recovery_child_restarts_each_profile_with_a_fresh_main(monkeypatch): recovery = importlib.import_module("hermes_cli.update_restart_recovery") calls = []