From 1cce7c6dd8db736c2e4e27583a236195c1d3dad7 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:43:13 -0700 Subject: [PATCH] fix(update): unmapped gateway stops no longer fail the fleet check with "no rows" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A gateway with no profile mapping (or one whose relaunch could not be armed) is SIGTERMed and listed under "Restart manually" — by design it has no successor and publishes no fleet-matrix row. It still counted in ``killed_pids`` and the pre-restart snapshot, so ``_fleet_probe_expected_runtimes`` demanded rows that could not exist and a fully successful update exited 1 with "Fleet version check returned no rows even though gateway runtimes were expected", leaving the fleet_restart_pending marker behind and every later CLI start warning about it. Track the unmapped stops on the restart outcome and subtract them from the row-predicting signals (``fleet_probe_signals``); relaunched/systemd gateways still predict rows exactly as before. --- hermes_cli/update_cmd_fleet.py | 22 ++++++++++++++++--- .../test_update_fleet_check_fail_closed.py | 19 ++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index 193d624ee3..e9f1c92875 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -11,7 +11,7 @@ import os import subprocess import sys import time as _time -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from hermes_cli.update_cmd_common import _best_effort @@ -850,6 +850,18 @@ class _GatewayRestartOutcome: relaunched_profiles: list externally_supervised_profiles: list killed_pids: set + #: Gateways stopped with NO successor (no profile mapping / relaunch could not be armed); + #: the summary tells the user to restart them by hand, so the fleet probe must not expect + #: a row for them. + stopped_unmapped_pids: set = field(default_factory=set) + + def fleet_probe_signals(self) -> tuple: + """``(pre_restart_pids, killed_pids)`` with the unmapped stops removed — the signals that + legitimately predict a fleet-matrix row.""" + pre = self.pre_restart_gateway_pids + if pre is not None: + pre = [pid for pid in pre if pid not in self.stopped_unmapped_pids] + return pre, self.killed_pids - self.stopped_unmapped_pids def record_receipt(self, **extra) -> None: """Best-effort ``record_gateway_restart`` from the current bookkeeping.""" @@ -921,6 +933,7 @@ def _restart_manual_gateways(out: _GatewayRestartOutcome, _drain_budget) -> None with suppress(ProcessLookupError, PermissionError): os.kill(pid, _signal.SIGTERM) out.killed_pids.add(pid) + out.stopped_unmapped_pids.add(pid) if out.restarted_services or out.killed_pids: print() @@ -1213,9 +1226,12 @@ def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_re # never fires on Windows (pause/resume populates neither), so a healthy # resumed gateway yielded zero rows and exit 0. # See #93406. + # A gateway stopped WITHOUT a successor ("Restart manually") publishes no row by design, + # so it must not count as an expected one — otherwise an update whose only live gateways + # were unmapped exits 1 with "no rows" after correctly stopping them. + _pre_restart, _killed = restart.fleet_probe_signals() _fleet_rows_expected = _m()._fleet_probe_expected_runtimes( - _pre_update_plan, restart.pre_restart_gateway_pids, _windows_gateway_resume, restart.restarted_services, - restart.killed_pids, + _pre_update_plan, _pre_restart, _windows_gateway_resume, restart.restarted_services, _killed, ) _fleet_snapshot = _collect_fleet_snapshot(restart, _fleet_rows_expected) if print_fleet_version_matrix(_fleet_snapshot): diff --git a/tests/hermes_cli/test_update_fleet_check_fail_closed.py b/tests/hermes_cli/test_update_fleet_check_fail_closed.py index 88b8f25147..862746f922 100644 --- a/tests/hermes_cli/test_update_fleet_check_fail_closed.py +++ b/tests/hermes_cli/test_update_fleet_check_fail_closed.py @@ -201,3 +201,22 @@ class TestCallSiteWiring: "elif not _fleet_snapshot and (restarted_services or killed_pids):" not in src ) + + +def test_unmapped_stops_are_not_expected_rows(): + # A gateway stopped WITHOUT a successor is listed under "Restart manually" and never + # publishes a row; counting it made the probe demand rows that cannot exist and the + # update exited 1 after correctly stopping every unmapped gateway. + from hermes_cli.update_cmd_fleet import _GatewayRestartOutcome + + out = _GatewayRestartOutcome( + incomplete=False, phase_errors=[], pre_restart_gateway_pids=[101, 102], restarted_services=[], + failed_or_stale_units=[], relaunched_profiles=[], externally_supervised_profiles=[], + killed_pids={101, 102}, stopped_unmapped_pids={101, 102}, + ) + pre, killed = out.fleet_probe_signals() + assert not _fleet_probe_expected_runtimes(_plan([]), pre, None, out.restarted_services, killed) + # A relaunched profile gateway (not unmapped) still predicts a row. + out.stopped_unmapped_pids.discard(102) + pre, killed = out.fleet_probe_signals() + assert _fleet_probe_expected_runtimes(_plan([]), pre, None, out.restarted_services, killed)