diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index e495274c2f..5c3d95e8a9 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -1069,6 +1069,9 @@ class _GatewayRestartOutcome: #: the summary tells the user to restart them by hand, so the fleet probe must not expect #: a row for them. stopped_unmapped_pids: set = field(default_factory=set) + #: ``scope/name`` of every settled systemd unit; the fleet probe stops waiting for a state stamp + #: once none of them is active or activating any more (the successor died, nothing will publish). + restarted_scoped_units: set = field(default_factory=set) def fleet_probe_signals(self) -> tuple: """``(pre_restart_pids, killed_pids)`` with the unmapped stops removed — the signals that @@ -1347,6 +1350,7 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): e, _pre_update_plan, out, gateway_mode=gateway_mode, restarted_scoped_units=restarted_scoped_units ) + out.restarted_scoped_units = set(restarted_scoped_units) return out @@ -1389,10 +1393,29 @@ def _collect_fleet_snapshot(restart, rows_expected: bool) -> list: snapshot = collect_fleet_versions(pre_restart_pids=restart.pre_restart_gateway_pids) if snapshot and not any(row.get("state") == "down" for row in snapshot): return snapshot - if _time.monotonic() >= _fleet_deadline: + if _time.monotonic() >= _fleet_deadline or _restarted_units_gone( + getattr(restart, "restarted_scoped_units", ())): return snapshot +def _restarted_units_gone(scoped_units) -> bool: + """True when every restarted systemd unit is neither active nor activating: the successor died, + nothing will publish a state stamp, so the settle poll should fail closed now instead of at the + deadline. Unknown (no units, systemctl missing/slow) keeps waiting.""" + if not scoped_units: + return False + scope_cmds = dict(_SYSTEMD_SCOPES) + for scoped in scoped_units: + scope, _, name = scoped.partition("/") + try: + state = _systemctl(scope_cmds[scope] + ["is-active", name], timeout=5).stdout.strip() + except (KeyError, FileNotFoundError, subprocess.TimeoutExpired): + return False + if state in ("active", "activating", "reloading"): + return False + return True + + def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_resume, node_failures, update_complete): """Post-restart verification: legacy-unit warning, dashboard cleanup, stale serve probe, fleet version matrix, plan-vs-execution reconciliation, receipt finalize. diff --git a/tests/hermes_cli/test_update_fleet_snapshot_settle.py b/tests/hermes_cli/test_update_fleet_snapshot_settle.py index 54b9ff5adc..835b57eb9e 100644 --- a/tests/hermes_cli/test_update_fleet_snapshot_settle.py +++ b/tests/hermes_cli/test_update_fleet_snapshot_settle.py @@ -50,7 +50,21 @@ def test_snapshot_waits_for_late_current_gateway_state(monkeypatch) -> None: assert result == [expected] assert clock.now > 30.0 assert clock.now <= update_cmd_fleet._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS - assert len(clock.sleeps) > 15 + + +def test_snapshot_stops_waiting_once_the_restarted_unit_is_dead(monkeypatch) -> None: + """A successor that exits (unit failed/inactive) fails closed at once, not at the 120s deadline.""" + clock = _FakeClock() + monkeypatch.setattr(update_cmd_fleet._time, "monotonic", clock.monotonic) + monkeypatch.setattr(update_cmd_fleet._time, "sleep", clock.sleep) + monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda **_kwargs: []) + monkeypatch.setattr( + update_cmd_fleet, "_systemctl", + lambda cmd, *, timeout: SimpleNamespace(stdout="failed\n", stderr="", returncode=3)) + + restart = SimpleNamespace(pre_restart_gateway_pids=[101], restarted_scoped_units={"user/hermes-gateway.service"}) + assert update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) == [] + assert clock.now < 30.0 def test_verifier_clears_marker_after_late_current_gateway_state(monkeypatch) -> None: