fix(update): stop the fleet settle poll early when the restarted unit is dead

Salvage of #111385 (@JoaoMarcos44): the 30s -> 120s settle window is kept so a
slow host's gateway can publish its state stamp. This commit bounds the other
side of that trade: when every restarted systemd unit reports neither active
nor activating, the successor has died and nothing will ever publish, so the
poll fails closed at once instead of spending the full 120s. Unknown states
(no units, systemctl missing or slow) keep waiting. Also drops the test
assertion pinning the 2s poll cadence.
This commit is contained in:
teknium1
2026-09-15 12:12:49 -07:00
committed by Teknium
parent c99fee4e0a
commit f81c33cb6a
2 changed files with 39 additions and 2 deletions
+24 -1
View File
@@ -1069,6 +1069,9 @@ class _GatewayRestartOutcome:
#: the summary tells the user to restart them by hand, so the fleet probe must not expect
#: a row for them.
stopped_unmapped_pids: set = field(default_factory=set)
#: ``scope/name`` of every settled systemd unit; the fleet probe stops waiting for a state stamp
#: once none of them is active or activating any more (the successor died, nothing will publish).
restarted_scoped_units: set = field(default_factory=set)
def fleet_probe_signals(self) -> tuple:
"""``(pre_restart_pids, killed_pids)`` with the unmapped stops removed — the signals that
@@ -1347,6 +1350,7 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool):
e, _pre_update_plan, out, gateway_mode=gateway_mode, restarted_scoped_units=restarted_scoped_units
)
out.restarted_scoped_units = set(restarted_scoped_units)
return out
@@ -1389,10 +1393,29 @@ def _collect_fleet_snapshot(restart, rows_expected: bool) -> list:
snapshot = collect_fleet_versions(pre_restart_pids=restart.pre_restart_gateway_pids)
if snapshot and not any(row.get("state") == "down" for row in snapshot):
return snapshot
if _time.monotonic() >= _fleet_deadline:
if _time.monotonic() >= _fleet_deadline or _restarted_units_gone(
getattr(restart, "restarted_scoped_units", ())):
return snapshot
def _restarted_units_gone(scoped_units) -> bool:
"""True when every restarted systemd unit is neither active nor activating: the successor died,
nothing will publish a state stamp, so the settle poll should fail closed now instead of at the
deadline. Unknown (no units, systemctl missing/slow) keeps waiting."""
if not scoped_units:
return False
scope_cmds = dict(_SYSTEMD_SCOPES)
for scoped in scoped_units:
scope, _, name = scoped.partition("/")
try:
state = _systemctl(scope_cmds[scope] + ["is-active", name], timeout=5).stdout.strip()
except (KeyError, FileNotFoundError, subprocess.TimeoutExpired):
return False
if state in ("active", "activating", "reloading"):
return False
return True
def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_resume, node_failures, update_complete):
"""Post-restart verification: legacy-unit warning, dashboard cleanup, stale serve
probe, fleet version matrix, plan-vs-execution reconciliation, receipt finalize.
@@ -50,7 +50,21 @@ def test_snapshot_waits_for_late_current_gateway_state(monkeypatch) -> None:
assert result == [expected]
assert clock.now > 30.0
assert clock.now <= update_cmd_fleet._FLEET_PROBE_SETTLE_TIMEOUT_SECONDS
assert len(clock.sleeps) > 15
def test_snapshot_stops_waiting_once_the_restarted_unit_is_dead(monkeypatch) -> None:
"""A successor that exits (unit failed/inactive) fails closed at once, not at the 120s deadline."""
clock = _FakeClock()
monkeypatch.setattr(update_cmd_fleet._time, "monotonic", clock.monotonic)
monkeypatch.setattr(update_cmd_fleet._time, "sleep", clock.sleep)
monkeypatch.setattr("hermes_cli.update_receipt.collect_fleet_versions", lambda **_kwargs: [])
monkeypatch.setattr(
update_cmd_fleet, "_systemctl",
lambda cmd, *, timeout: SimpleNamespace(stdout="failed\n", stderr="", returncode=3))
restart = SimpleNamespace(pre_restart_gateway_pids=[101], restarted_scoped_units={"user/hermes-gateway.service"})
assert update_cmd_fleet._collect_fleet_snapshot(restart, rows_expected=True) == []
assert clock.now < 30.0
def test_verifier_clears_marker_after_late_current_gateway_state(monkeypatch) -> None: