fix(cli): allow hermes update without gateway restart for cron (rebased on upstream/main)

(cherry picked from commit e70f78e54a96f2e8037f7e385fc31563bbeaf392)
(cherry picked from commit 225f56ab29e91977f1fef2e743c50dd6e2e89b60)

hermes_cli/subcommands/update.py

@@ -66,4 +66,11 @@ def build_update_parser(subparsers, *, cmd_update: Callable) -> None:
         "--force-venv", action="store_true", default=False,
         help="Windows: mutate the venv even while other processes are running from its interpreter (desktop backend, gateway, terminals). Those processes keep native .pyd files locked, so the dependency sync will likely fail partway and strand the install half-updated. Use only if you know the detected holders are false positives.",
     )
+    update_parser.add_argument(
+        "--no-gateway-restart", action="store_true", default=False,
+        help="Update code and dependencies but skip the final gateway restart. "
+            "Use for cron/automated updates that run inside the gateway process: "
+            "the gateway would otherwise restart its own cgroup and kill the updater. "
+            "Pair with a separate restart step (e.g. a cron that runs 10-15 min later).",
+    )
     update_parser.set_defaults(func=cmd_update)

hermes_cli/update_cmd.py

@@ -43,7 +43,7 @@ from hermes_cli.update_cmd_windows import (  # noqa: F401
 from hermes_cli.update_cmd_fleet import (  # noqa: F401
     _FLEET_RESTART_PENDING_NAME, _FRESH_RESTART_SUPERVISORS, _GatewayRestartOutcome,
     _apply_pending_fleet_restart_catchup, _clear_fleet_restart_pending_marker,
-    _current_checkout_sha, _drain_or_signal_gateway_for_update, _fleet_probe_expected_runtimes,
+    _current_checkout_sha, _defer_fleet_restart_after_update, _drain_or_signal_gateway_for_update, _fleet_probe_expected_runtimes,
     _fleet_restart_pending_marker_path, _for_each_systemd_gateway_unit,
     _gateway_recovery_partition, _gateway_service_matches_profile, _pending_fleet_restart_needed,
     _receipt_looks_unfinished, _receipt_reports_stale_runtime, _resolve_manage_cmd,
@@ -972,6 +972,7 @@ class _UpdateOptions:
     keep_stash: bool
     switch_branch: bool
     discard_local_changes: bool
+    no_gateway_restart: bool = False
 
 
 def _resolve_update_options(args, gateway_mode: bool) -> _UpdateOptions:
@@ -997,6 +998,10 @@ def _resolve_update_options(args, gateway_mode: bool) -> _UpdateOptions:
     # branch's history; only meaningful with parked_branch_strategy "update_in_place".
     # See #89507.
     switch_branch = bool(getattr(args, "switch_branch", False))
+    # --no-gateway-restart (cron inside the gateway's own cgroup): update code
+    # and dependencies but defer the fleet restart so the updater is not killed
+    # by its own restart. The pending-restart marker is kept for catch-up.
+    no_gateway_restart = bool(getattr(args, "no_gateway_restart", False))
 
     # Interactive terminals always stash-and-ask; only non-interactive updates consult
     # updates.non_interactive_local_changes (auto-restore vs discard).
@@ -1010,7 +1015,8 @@ def _resolve_update_options(args, gateway_mode: bool) -> _UpdateOptions:
         active_lazy_features=active_lazy_features,
         active_tool_dependencies=active_tool_dependencies, pre_update_version=pre_update_version,
         gw_input_fn=gw_input_fn, assume_yes=assume_yes, keep_stash=keep_stash,
-        switch_branch=switch_branch, discard_local_changes=discard_local_changes)
+        switch_branch=switch_branch, discard_local_changes=discard_local_changes,
+        no_gateway_restart=no_gateway_restart)
 
 
 def _begin_update_receipt_and_plan(args):
@@ -1182,7 +1188,8 @@ def _finalize_receipt(status: str, debug_message: str) -> None:
 def _finish_already_up_to_date(
     git_cmd, branch: str, current_branch: str, _plan, *, assume_yes: bool, gateway_mode: bool,
     gw_input_fn, pre_update_snapshot_id, had_desktop_app_before_update: bool,
-    active_lazy_features, active_tool_dependencies, _windows_gateway_resume) -> None:
+    active_lazy_features, active_tool_dependencies, _windows_gateway_resume,
+    no_gateway_restart: bool = False) -> None:
     """"Already up to date" path: restore stash/branch, repair the checkout, catch up the fleet.
     ``sys.exit(1)`` when the repair is incomplete (after gateway exit code + partial receipt)."""
     _invalidate_update_cache()
@@ -1217,8 +1224,10 @@ def _finish_already_up_to_date(
     # Catch up even on the "Already up to date" path — that early return is what left the gateway on stale
     # code for two days. Runs BEFORE the runtime-verification exit gate below: a vulnerable SQLite runtime
     # demotes the outcome to partial, but must not strand the fleet on stale code (#91277 fleet contract —
-    # the pending-restart check always executes).
-    _apply_pending_fleet_restart_catchup()
+    # the pending-restart check always executes). Under --no-gateway-restart the
+    # catch-up is deferred instead (executing it would kill the cron's own gateway).
+    _apply_pending_fleet_restart_catchup(
+        respect_no_gateway_restart=True, no_gateway_restart=no_gateway_restart)
     if not current_checkout_complete:
         if gateway_mode:
             _write_gateway_update_exit_code(False)
@@ -1276,6 +1285,28 @@ def _apply_pulled_update(
     if gateway_mode:
         _write_gateway_update_exit_code(update_complete)
 
+    if opts.no_gateway_restart:
+        # Cron inside the gateway's own cgroup: restarting the fleet now would
+        # SIGUSR1-drain this updater's own gateway and systemd would SIGKILL the
+        # updater with it. Defer instead — the pending marker written above is
+        # kept for the next normal update. Skipping the restart phase also skips
+        # its stale-module purge: this live interpreter still serves pre-update
+        # code and must not have its sys.modules graph mutated mid-flight.
+        # Windows pause/resume still runs (paused gateways must be resumed onto
+        # pre-update code); only the fleet restart + verification are deferred.
+        # The resume outcome is RETAINED (not discarded): a failed resume must
+        # still make this update partial, exactly as on the normal path.
+        resume_outcome = _GatewayRestartOutcome(
+            incomplete=False, phase_errors=[], pre_restart_gateway_pids=[],
+            restarted_services=[], failed_or_stale_units=[], relaunched_profiles=[],
+            externally_supervised_profiles=[], killed_pids=set(),
+        )
+        _resume_windows_gateways_and_merge_outcome(
+            resume_outcome, _windows_gateway_resume, gateway_mode)
+        _defer_fleet_restart_after_update(
+            update_complete=update_complete, resume_incomplete=resume_outcome.incomplete)
+        return
+
     _restart = _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode)
     _resume_windows_gateways_and_merge_outcome(_restart, _windows_gateway_resume, gateway_mode)
     _verify_fleet_after_update(
@@ -1385,7 +1416,8 @@ def _cmd_update_impl(args, gateway_mode: bool):
                 had_desktop_app_before_update=had_desktop_app_before_update,
                 active_lazy_features=opts.active_lazy_features,
                 active_tool_dependencies=opts.active_tool_dependencies,
-                _windows_gateway_resume=_windows_gateway_resume)
+                _windows_gateway_resume=_windows_gateway_resume,
+                no_gateway_restart=opts.no_gateway_restart)
             return
 
         if commit_count > 0:

hermes_cli/update_cmd_fleet.py

@@ -428,15 +428,62 @@ def _run_pending_fleet_restart() -> bool:
         return False
 
 
-def _apply_pending_fleet_restart_catchup() -> None:
+def _defer_fleet_restart_after_update(*, update_complete: bool, resume_incomplete: bool = False) -> None:
+    """Record a deliberately deferred fleet restart and return/exit on outcome.
+
+    ``hermes update --no-gateway-restart`` (cron running inside the gateway's
+    own cgroup) updated code and dependencies but must not restart the fleet:
+    the SIGUSR1 drain + systemd restart would kill the updater itself. The
+    ``fleet_restart_pending`` marker written before the pull is KEPT so the
+    next normal update (or ``hermes gateway restart``) catches up.
+
+    Outcome contract (same success/partial meaning as the normal path):
+    a STALE fleet caused only by this deliberate deferral is expected and
+    does NOT make the update partial — exit 0, receipt "success". The receipt
+    is "partial" (and the process exits 1, marker kept) when the update
+    itself did not complete (``update_complete`` False) or the Windows
+    pause/resume reconciliation reported incomplete.
+
+    The live interpreter still serves pre-update code here, so callers must
+    also skip stale-module purge/reload work: mutating this process's
+    sys.modules graph mid-flight risks breaking the serving gateway.
+    """
+    print()
+    print("→ Gateway restart skipped (--no-gateway-restart).")
+    print("  Code and dependencies are updated; gateways still serve pre-update code.")
+    print("  Restart them separately: `hermes gateway restart` or a daily-restart cron.")
+    print("  (fleet restart deferred — marker kept for catch-up)")
+    with suppress(Exception):
+        from hermes_cli.update_receipt import record_skip
+        record_skip("gateway_restart", "--no-gateway-restart: deferred, marker kept")
+    partial = (not update_complete) or resume_incomplete
+    with suppress(Exception):
+        from hermes_cli.update_receipt import finalize_update_receipt
+        finalize_update_receipt("partial" if partial else "success")
+    if partial:
+        sys.exit(1)
+
+
+def _apply_pending_fleet_restart_catchup(*, respect_no_gateway_restart: bool = False, no_gateway_restart: bool = False) -> None:
     """On an already-up-to-date ``hermes update``, finish a skipped restart.
 
     No-op when nothing is pending; exits 1 on incomplete catch-up so automation
     does not treat the fleet as healthy.
+
+    When called from a cron's ``--no-gateway-restart`` flow (both flags true),
+    the pending restart is deferred instead of executed: running it here would
+    kill the cron's own gateway mid-flight. The marker is kept for a later
+    out-of-cron update.
     """
     from hermes_cli.update_cmd import _run_pending_fleet_restart
     if not _pending_fleet_restart_needed():
         return
+    if respect_no_gateway_restart and no_gateway_restart:
+        print()
+        _warn_pending_fleet_restart()
+        print("  (fleet restart deferred — --no-gateway-restart; marker kept)")
+        print("  Restart separately: `hermes gateway restart` or next non-cron update.")
+        return
     print()
     _warn_pending_fleet_restart()
     print("→ Running the pending fleet restart...")

tests/hermes_cli/test_update_no_gateway_restart.py

@@ -0,0 +1,285 @@
+"""Focused coverage for `hermes update --no-gateway-restart`.
+
+A cron running inside the gateway's own cgroup cannot survive the fleet
+restart phase (SIGUSR1 drain + systemd KillMode=mixed kills the updater
+itself). The flag runs the full update pipeline but defers the restart;
+the pending-restart marker is kept so a later normal update catches up.
+"""
+import argparse
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from hermes_cli.subcommands.update import build_update_parser
+from hermes_cli import update_cmd as uc
+from hermes_cli import update_cmd_fleet as fleet
+
+
+def _h(name):
+    def handler(args):  # pragma: no cover - identity only
+        return name
+    handler.__name__ = f"cmd_{name}"
+    return handler
+
+
+def _parse(argv):
+    parser = argparse.ArgumentParser(prog="hermes")
+    sub = parser.add_subparsers(dest="command")
+    build_update_parser(sub, cmd_update=_h("update"))
+    return parser.parse_args(argv)
+
+
+def _opts(**overrides):
+    base = dict(
+        assume_yes=True, gw_input_fn=None, active_lazy_features=[],
+        active_tool_dependencies=[], pre_update_version="1.0",
+        discard_local_changes=False, keep_stash=False, switch_branch=False,
+        no_gateway_restart=False,
+    )
+    base.update(overrides)
+    return SimpleNamespace(**base)
+
+
+# ── flag parsing ──────────────────────────────────────────────────────
+
+
+def test_parser_defaults_to_restart():
+    ns = _parse(["update"])
+    assert ns.no_gateway_restart is False
+
+
+def test_parser_accepts_no_gateway_restart():
+    ns = _parse(["update", "--no-gateway-restart"])
+    assert ns.no_gateway_restart is True
+
+
+def test_resolve_options_threads_flag():
+    with (
+        patch.object(uc, "_m") as mock_m,
+        patch.object(uc, "_read_project_version", return_value="1.0"),
+        patch.object(uc, "_updates_config", return_value={}),
+    ):
+        mock_m.return_value._capture_active_lazy_features.return_value = []
+        mock_m.return_value._capture_active_tool_dependencies.return_value = []
+        assert uc._resolve_update_options(
+            SimpleNamespace(no_gateway_restart=True), False).no_gateway_restart is True
+        assert uc._resolve_update_options(
+            SimpleNamespace(), False).no_gateway_restart is False
+
+
+# ── pending catch-up deferral ─────────────────────────────────────────
+
+
+def test_catchup_deferred_under_flag_when_pending():
+    """Already-up-to-date + pending marker + flag: no restart, no exit, marker kept."""
+    with (
+        patch.object(fleet, "_pending_fleet_restart_needed", return_value=True),
+        patch.object(fleet, "_warn_pending_fleet_restart"),
+        patch.object(uc, "_run_pending_fleet_restart") as mock_run,
+        patch.object(fleet, "_clear_fleet_restart_pending_marker") as mock_clear,
+    ):
+        fleet._apply_pending_fleet_restart_catchup(
+            respect_no_gateway_restart=True, no_gateway_restart=True)
+    mock_run.assert_not_called()
+    mock_clear.assert_not_called()
+
+
+def test_catchup_default_path_unchanged():
+    """Without the flag the pending restart still runs and clears the marker."""
+    with (
+        patch.object(fleet, "_pending_fleet_restart_needed", return_value=True),
+        patch.object(fleet, "_warn_pending_fleet_restart"),
+        patch.object(uc, "_run_pending_fleet_restart", return_value=True) as mock_run,
+        patch.object(fleet, "_clear_fleet_restart_pending_marker") as mock_clear,
+    ):
+        fleet._apply_pending_fleet_restart_catchup()
+    mock_run.assert_called_once()
+    mock_clear.assert_called_once()
+
+
+def test_catchup_noop_when_nothing_pending():
+    with (
+        patch.object(fleet, "_pending_fleet_restart_needed", return_value=False),
+        patch.object(uc, "_run_pending_fleet_restart") as mock_run,
+    ):
+        fleet._apply_pending_fleet_restart_catchup(
+            respect_no_gateway_restart=True, no_gateway_restart=True)
+    mock_run.assert_not_called()
+
+
+# ── defer helper: outcome contract ────────────────────────────────────
+
+
+def test_defer_success_when_update_complete_and_resume_clean():
+    """Successful update + deliberate defer => success, exit 0, marker kept."""
+    with (
+        patch("hermes_cli.update_receipt.record_skip") as mock_skip,
+        patch("hermes_cli.update_receipt.finalize_update_receipt") as mock_final,
+        patch.object(fleet, "_clear_fleet_restart_pending_marker") as mock_clear,
+    ):
+        fleet._defer_fleet_restart_after_update(update_complete=True)
+    mock_skip.assert_called_once()
+    assert mock_skip.call_args[0][0] == "gateway_restart"
+    mock_final.assert_called_once_with("success")
+    mock_clear.assert_not_called()
+
+
+def test_defer_partial_when_update_incomplete():
+    """update_complete=False + defer => partial + exit 1, marker kept."""
+    with (
+        patch("hermes_cli.update_receipt.record_skip"),
+        patch("hermes_cli.update_receipt.finalize_update_receipt") as mock_final,
+        patch.object(fleet, "_clear_fleet_restart_pending_marker") as mock_clear,
+    ):
+        with pytest.raises(SystemExit) as exc:
+            fleet._defer_fleet_restart_after_update(update_complete=False)
+    assert exc.value.code == 1
+    mock_final.assert_called_once_with("partial")
+    mock_clear.assert_not_called()
+
+
+def test_defer_partial_when_resume_incomplete():
+    """Clean update but failed Windows resume + defer => partial + exit 1."""
+    with (
+        patch("hermes_cli.update_receipt.record_skip"),
+        patch("hermes_cli.update_receipt.finalize_update_receipt") as mock_final,
+        patch.object(fleet, "_clear_fleet_restart_pending_marker") as mock_clear,
+    ):
+        with pytest.raises(SystemExit) as exc:
+            fleet._defer_fleet_restart_after_update(
+                update_complete=True, resume_incomplete=True)
+    assert exc.value.code == 1
+    mock_final.assert_called_once_with("partial")
+    mock_clear.assert_not_called()
+
+
+# ── pulled-update orchestration ───────────────────────────────────────
+
+
+def test_pulled_update_skips_restart_phase_under_flag():
+    with (
+        patch.object(uc, "_invalidate_update_cache"),
+        patch.object(uc, "_verify_head_after_pull", return_value="newsha"),
+        patch.object(uc, "_write_fleet_restart_pending_marker") as mock_marker,
+        patch.object(uc, "_clear_fleet_restart_pending_marker") as mock_clear,
+        patch.object(uc, "_sweep_bytecode_after_update"),
+        patch.object(uc, "_sync_python_dependencies_after_pull"),
+        patch.object(uc, "_update_node_dependencies", return_value=[]),
+        patch.object(uc, "_rebuild_desktop_after_update", return_value=True),
+        patch.object(uc, "_run_post_update_maintenance", return_value=True),
+        patch.object(uc, "_branch_head_suffix", return_value=""),
+        patch.object(uc, "_m", return_value=MagicMock()),
+        patch.object(uc, "_restart_gateway_fleet_after_update") as mock_restart,
+        patch.object(uc, "_resume_windows_gateways_and_merge_outcome") as mock_resume,
+        patch.object(uc, "_verify_fleet_after_update") as mock_verify,
+        patch.object(uc, "_defer_fleet_restart_after_update") as mock_defer,
+    ):
+        uc._apply_pulled_update(
+            "git", "main", "oldsha", SimpleNamespace(in_place_update=False),
+            _opts(no_gateway_restart=True), gateway_mode=False,
+            is_fork=False, desktop_dir="/tmp", had_desktop_app_before_update=False,
+            pre_update_snapshot_id=None, _pre_update_plan=None,
+            _windows_gateway_resume=None,
+        )
+    mock_restart.assert_not_called()
+    mock_verify.assert_not_called()
+    mock_defer.assert_called_once_with(update_complete=True, resume_incomplete=False)
+    mock_resume.assert_called_once()  # Windows pause/resume still runs
+    mock_marker.assert_called_once()  # pending marker kept for catch-up
+    mock_clear.assert_not_called()  # deferred stale fleet alone is not partial
+
+
+def test_pulled_update_defer_reports_incomplete_maintenance():
+    """update_complete=False flows into the deferral (=> partial downstream)."""
+    with (
+        patch.object(uc, "_invalidate_update_cache"),
+        patch.object(uc, "_verify_head_after_pull", return_value="newsha"),
+        patch.object(uc, "_write_fleet_restart_pending_marker"),
+        patch.object(uc, "_sweep_bytecode_after_update"),
+        patch.object(uc, "_sync_python_dependencies_after_pull"),
+        patch.object(uc, "_update_node_dependencies", return_value=[]),
+        patch.object(uc, "_rebuild_desktop_after_update", return_value=True),
+        patch.object(uc, "_run_post_update_maintenance", return_value=False),
+        patch.object(uc, "_branch_head_suffix", return_value=""),
+        patch.object(uc, "_m", return_value=MagicMock()),
+        patch.object(uc, "_restart_gateway_fleet_after_update") as mock_restart,
+        patch.object(uc, "_resume_windows_gateways_and_merge_outcome"),
+        patch.object(uc, "_verify_fleet_after_update") as mock_verify,
+        patch.object(uc, "_defer_fleet_restart_after_update") as mock_defer,
+    ):
+        uc._apply_pulled_update(
+            "git", "main", "oldsha", SimpleNamespace(in_place_update=False),
+            _opts(no_gateway_restart=True), gateway_mode=False,
+            is_fork=False, desktop_dir="/tmp", had_desktop_app_before_update=False,
+            pre_update_snapshot_id=None, _pre_update_plan=None,
+            _windows_gateway_resume=None,
+        )
+    mock_restart.assert_not_called()
+    mock_verify.assert_not_called()
+    mock_defer.assert_called_once_with(update_complete=False, resume_incomplete=False)
+
+
+def test_pulled_update_defer_retains_failed_resume_outcome():
+    """A resume phase that marks its outcome incomplete reaches the deferral."""
+    def _fail_resume(outcome, token, gw_mode):
+        outcome.incomplete = True
+
+    with (
+        patch.object(uc, "_invalidate_update_cache"),
+        patch.object(uc, "_verify_head_after_pull", return_value="newsha"),
+        patch.object(uc, "_write_fleet_restart_pending_marker"),
+        patch.object(uc, "_sweep_bytecode_after_update"),
+        patch.object(uc, "_sync_python_dependencies_after_pull"),
+        patch.object(uc, "_update_node_dependencies", return_value=[]),
+        patch.object(uc, "_rebuild_desktop_after_update", return_value=True),
+        patch.object(uc, "_run_post_update_maintenance", return_value=True),
+        patch.object(uc, "_branch_head_suffix", return_value=""),
+        patch.object(uc, "_m", return_value=MagicMock()),
+        patch.object(uc, "_restart_gateway_fleet_after_update") as mock_restart,
+        patch.object(
+            uc, "_resume_windows_gateways_and_merge_outcome",
+            side_effect=_fail_resume,
+        ),
+        patch.object(uc, "_verify_fleet_after_update") as mock_verify,
+        patch.object(uc, "_defer_fleet_restart_after_update") as mock_defer,
+    ):
+        uc._apply_pulled_update(
+            "git", "main", "oldsha", SimpleNamespace(in_place_update=False),
+            _opts(no_gateway_restart=True), gateway_mode=False,
+            is_fork=False, desktop_dir="/tmp", had_desktop_app_before_update=False,
+            pre_update_snapshot_id=None, _pre_update_plan=None,
+            _windows_gateway_resume=None,
+        )
+    mock_restart.assert_not_called()
+    mock_verify.assert_not_called()
+    mock_defer.assert_called_once_with(update_complete=True, resume_incomplete=True)
+
+
+def test_pulled_update_default_path_unchanged():
+    with (
+        patch.object(uc, "_invalidate_update_cache"),
+        patch.object(uc, "_verify_head_after_pull", return_value="newsha"),
+        patch.object(uc, "_write_fleet_restart_pending_marker"),
+        patch.object(uc, "_sweep_bytecode_after_update"),
+        patch.object(uc, "_sync_python_dependencies_after_pull"),
+        patch.object(uc, "_update_node_dependencies", return_value=[]),
+        patch.object(uc, "_rebuild_desktop_after_update", return_value=True),
+        patch.object(uc, "_run_post_update_maintenance", return_value=True),
+        patch.object(uc, "_branch_head_suffix", return_value=""),
+        patch.object(uc, "_m", return_value=MagicMock()),
+        patch.object(uc, "_restart_gateway_fleet_after_update") as mock_restart,
+        patch.object(uc, "_resume_windows_gateways_and_merge_outcome"),
+        patch.object(uc, "_verify_fleet_after_update") as mock_verify,
+        patch.object(uc, "_defer_fleet_restart_after_update") as mock_defer,
+    ):
+        uc._apply_pulled_update(
+            "git", "main", "oldsha", SimpleNamespace(in_place_update=False),
+            _opts(no_gateway_restart=False), gateway_mode=False,
+            is_fork=False, desktop_dir="/tmp", had_desktop_app_before_update=False,
+            pre_update_snapshot_id=None, _pre_update_plan=None,
+            _windows_gateway_resume=None,
+        )
+    mock_restart.assert_called_once()
+    mock_verify.assert_called_once()
+    mock_defer.assert_not_called()

tests/hermes_cli/test_update_obligation_identity.py

@@ -57,7 +57,8 @@ def test_every_owed_identity_requires_current_evidence(monkeypatch, bad):
     directory = home / "logs" / "update_receipts"
     directory.mkdir(parents=True)
     if bad == "marker":
-        (home / "fleet_restart_pending").write_text("expected_sha=new\n")
+        # Obligation for an SHA the fleet does not serve: no verified discharge.
+        (home / "fleet_restart_pending").write_text("expected_sha=future\n")
     owed = {"kind": "gateway", "profile": "beta", "code_sha": "old"}
     if bad == "wrong-kind":
         owed["kind"] = "serve"