fix(update): stop the fleet settle poll early when the restarted unit is dead
Salvage of #111385 (@JoaoMarcos44): the 30s -> 120s settle window is kept so a slow host's gateway can publish its state stamp. This commit bounds the other side of that trade: when every restarted systemd unit reports neither active nor activating, the successor has died and nothing will ever publish, so the poll fails closed at once instead of spending the full 120s. Unknown states (no units, systemctl missing or slow) keep waiting. Also drops the test assertion pinning the 2s poll cadence.
This commit is contained in:
@@ -1069,6 +1069,9 @@ class _GatewayRestartOutcome:
|
||||
#: the summary tells the user to restart them by hand, so the fleet probe must not expect
|
||||
#: a row for them.
|
||||
stopped_unmapped_pids: set = field(default_factory=set)
|
||||
#: ``scope/name`` of every settled systemd unit; the fleet probe stops waiting for a state stamp
|
||||
#: once none of them is active or activating any more (the successor died, nothing will publish).
|
||||
restarted_scoped_units: set = field(default_factory=set)
|
||||
|
||||
def fleet_probe_signals(self) -> tuple:
|
||||
"""``(pre_restart_pids, killed_pids)`` with the unmapped stops removed — the signals that
|
||||
@@ -1347,6 +1350,7 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool):
|
||||
e, _pre_update_plan, out, gateway_mode=gateway_mode, restarted_scoped_units=restarted_scoped_units
|
||||
)
|
||||
|
||||
out.restarted_scoped_units = set(restarted_scoped_units)
|
||||
return out
|
||||
|
||||
|
||||
@@ -1389,10 +1393,29 @@ def _collect_fleet_snapshot(restart, rows_expected: bool) -> list:
|
||||
snapshot = collect_fleet_versions(pre_restart_pids=restart.pre_restart_gateway_pids)
|
||||
if snapshot and not any(row.get("state") == "down" for row in snapshot):
|
||||
return snapshot
|
||||
if _time.monotonic() >= _fleet_deadline:
|
||||
if _time.monotonic() >= _fleet_deadline or _restarted_units_gone(
|
||||
getattr(restart, "restarted_scoped_units", ())):
|
||||
return snapshot
|
||||
|
||||
|
||||
def _restarted_units_gone(scoped_units) -> bool:
|
||||
"""True when every restarted systemd unit is neither active nor activating: the successor died,
|
||||
nothing will publish a state stamp, so the settle poll should fail closed now instead of at the
|
||||
deadline. Unknown (no units, systemctl missing/slow) keeps waiting."""
|
||||
if not scoped_units:
|
||||
return False
|
||||
scope_cmds = dict(_SYSTEMD_SCOPES)
|
||||
for scoped in scoped_units:
|
||||
scope, _, name = scoped.partition("/")
|
||||
try:
|
||||
state = _systemctl(scope_cmds[scope] + ["is-active", name], timeout=5).stdout.strip()
|
||||
except (KeyError, FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return False
|
||||
if state in ("active", "activating", "reloading"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_resume, node_failures, update_complete):
|
||||
"""Post-restart verification: legacy-unit warning, dashboard cleanup, stale serve
|
||||
probe, fleet version matrix, plan-vs-execution reconciliation, receipt finalize.
|
||||
|
||||
@@ -50,7 +50,21 @@ def test_snapshot_waits_for_late_current_gateway_state(monkeypatch) -> None:
|
||||
assert result == [expected]
|
||||
assert clock.now > 30.0
|
||||
assert clock.now <= update_cmd_fleet._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS
|
||||
assert len(clock.sleeps) > 15
|
||||
|
||||
|
||||
def test_snapshot_stops_waiting_once_the_restarted_unit_is_dead(monkeypatch) -> None:
|
||||
"""A successor that exits (unit failed/inactive) fails closed at once, not at the 120s deadline."""
|
||||
clock = _FakeClock()
|
||||
monkeypatch.setattr(update_cmd_fleet._time, "monotonic", clock.monotonic)
|
||||
monkeypatch.setattr(update_cmd_fleet._time, "sleep", clock.sleep)
|
||||
monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda **_kwargs: [])
|
||||
monkeypatch.setattr(
|
||||
update_cmd_fleet, "_systemctl",
|
||||
lambda cmd, *, timeout: SimpleNamespace(stdout="failed\n", stderr="", returncode=3))
|
||||
|
||||
restart = SimpleNamespace(pre_restart_gateway_pids=[101], restarted_scoped_units={"user/hermes-gateway.service"})
|
||||
assert update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) == []
|
||||
assert clock.now < 30.0
|
||||
|
||||
|
||||
def test_verifier_clears_marker_after_late_current_gateway_state(monkeypatch) -> None:
|
||||
|
||||
Reference in New Issue
Block a user