diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index e2ab9ecd8c..aed2d5f2c6 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -1,26 +1,10 @@ -"""Hermes update pipeline — extracted from ``hermes_cli/main.py``. +"""Hermes update pipeline: dispatchers (``_cmd_update_impl``/``_cmd_update_check``) + git plumbing. -Mechanical move (main.py decomposition): ``_cmd_update_impl``, ``_cmd_update_check`` -and every module-level helper used only by the update path, plus the update-only -constants they read. Function bodies are lifted verbatim; the only mechanical -change is that references to helpers/constants that STAY in ``hermes_cli.main`` -(and to moved-but-test-patched siblings) are routed through ``_m()`` — a lazy -``hermes_cli.main`` reference — so existing call sites and test monkeypatches -that target ``hermes_cli.main.`` (``PROJECT_ROOT``, ``_is_windows``, -``_run_pre_update_backup``, ...) keep working unchanged. ``main.py`` re-imports -every public-ish name from here (``# noqa: F401``) so the argparse wiring and -the test-patch surface still resolve on ``hermes_cli.main``. - -The closures that used to be nested inside ``_cmd_update_impl`` (``_print_items``, -``_wait_for_service_active``, ``_service_restart_sec``, ``_resolve_manage_cmd``, -``_restart_one_systemd_gateway_unit``) now live at module level or inside the -phase helpers ``_cmd_update_impl`` calls in order: ``_pull_updates`` -> -``_sync_python_dependencies_after_pull`` -> ``_run_post_update_maintenance`` -> -``_restart_gateway_fleet_after_update`` -> ``_verify_fleet_after_update``. - -Imports are one-way: ``hermes_cli.main`` imports this module, never the reverse -at import time (``_m()`` resolves lazily at call time, when main.py is fully -loaded, so there is no import cycle). +Each concern lives in ``update_cmd_.py`` and is re-imported here so +``hermes_cli.update_cmd.`` keeps resolving (and stays monkeypatchable). +``_m()`` is the lazy ``hermes_cli.main`` handle kept for main-side test patches. +Imports are one-way: main -> update_cmd -> update_cmd_* (never the reverse at +import time; ``_m()`` resolves at call time, so no cycle). """ import logging @@ -34,15 +18,12 @@ import time as _time from dataclasses import dataclass from pathlib import Path -# Both re-exported: the split update_cmd_* modules import them lazily from -# here so tests that monkeypatch ``update_cmd.get_hermes_home`` stay effective. +# Re-exported: update_cmd_* modules import lazily from here so patches stick. from hermes_cli.config import get_hermes_home # noqa: F401 from hermes_cli.update_cmd_common import _best_effort from hermes_constants import get_default_hermes_root, venv_python_path -# Abort recovery lives in its own bounded module (review on #96235). Re-exported -# here because `hermes_cli.main` and the split update modules address these -# names through `update_cmd` (tests patch them here). +# Re-exported: main and the split modules address these via update_cmd (tests patch here). from hermes_cli.update_abort_recovery import ( # noqa: F401 _abort_recovery_is_complete, _qualified_serve_skips, @@ -263,35 +244,24 @@ logger = logging.getLogger(__name__) def _m(): - """Lazy ``hermes_cli.main`` reference. - - Keeps ``hermes_cli.main.`` test patches effective in this code - path and keeps the ``main`` -> ``update_cmd`` import one-way at import time. - """ + """Lazy ``hermes_cli.main`` handle: keeps main-side test patches effective, import one-way.""" from hermes_cli import main return main def _no_prompt_git_kwargs() -> dict: - """``subprocess.run`` kwargs for the updater's network git calls. - - GitHub answers anonymous fetches with HTTP 401 during outages (and for - unreachable repos); git then prompts ``Username for 'https://github.com':`` - on the inherited terminal and the update sits there forever. Disable the - prompt so the fetch fails fast into ``_classify_fetch_failure``. Only the - *prompt* is disabled — a configured credential helper / askpass still - runs, so a private-fork origin keeps authenticating non-interactively. - """ + """``subprocess.run`` kwargs for network git calls: GitHub answers anonymous fetches with + 401 during outages and git would then block forever on ``Username for ...``; disable only + the *prompt* (credential helpers/askpass still run) so it fails fast into ``_classify_fetch_failure``.""" env = dict(os.environ) env["GIT_TERMINAL_PROMPT"] = "0" env["GCM_INTERACTIVE"] = "Never" return {"stdin": subprocess.DEVNULL, "env": env} -# Files that must parse right after an update/install (CLI startup imports; -# ``web_server.py`` is the desktop backend a fresh Windows install launches). -# The post-pull syntax guard validates these and auto-rolls-back on failure. +# CLI-startup imports (+ web_server.py, the desktop backend a fresh Windows install +# launches) that must parse post-update; the syntax guard auto-rolls-back on failure. _UPDATE_CRITICAL_FILES = ( "hermes_cli/main.py", "hermes_cli/config.py", @@ -314,11 +284,8 @@ def _record_update_step(step: str, ok: bool, detail: str = "") -> None: def _git_run(git_cmd, args, cwd=None, *, check=False, network=False): - """Run ``git_cmd + args`` (default cwd: the checkout), capturing utf-8 text. - - ``network=True`` (fetch/pull/push) disables git's terminal prompt so an - HTTP 401 fails fast instead of hanging on ``Username for ...``. - """ + """Run git capturing utf-8 text (default cwd: checkout); ``network=True`` disables the + terminal prompt so an HTTP 401 fails fast instead of hanging.""" return subprocess.run( git_cmd + args, cwd=_m().PROJECT_ROOT if cwd is None else cwd, @@ -362,30 +329,18 @@ def _validate_python_files_syntax( def _validate_critical_files_syntax(root) -> tuple[bool, str | None, str | None]: - """Compile each file in ``_UPDATE_CRITICAL_FILES`` to catch SyntaxErrors. + """Compile ``_UPDATE_CRITICAL_FILES`` -> ``(ok, failing_path, error_message)``. - These are imported on every ``hermes`` startup; a syntax error (orphan - conflict markers, etc.) means the CLI can't bootstrap, so we validate - after ``git pull`` and auto-roll-back instead of leaving a bricked install. - - The ``.pyc`` goes to a temp dir, not the tree's ``__pycache__/``: avoids - racing concurrent test workers and leaving a stale pyc behind when the - next interpreter run uses a different Python. Only the compile-or-not - signal matters. - - Returns ``(ok, failing_path, error_message)``. + A syntax error there means the CLI can't bootstrap, so validate post-pull and + auto-roll-back. The .pyc goes to a temp dir, not ``__pycache__/``: avoids racing + concurrent test workers and leaving a stale pyc for a different interpreter. """ return _validate_python_files_syntax(root, _UPDATE_CRITICAL_FILES) def _gateway_prompt(prompt_text: str, default: str = "", timeout: float = 300.0) -> str: - """File-based IPC prompt for gateway mode. - - Writes a prompt marker file for the gateway to forward to the user, then - polls for a response file; falls back to *default* on timeout. Lets - ``hermes update --gateway`` forward prompts (stash restore, config - migration) to the messenger instead of silently skipping them. - """ + """File-based IPC prompt for ``--gateway``: write a marker the gateway forwards to the + messenger, poll for a response file, fall back to *default* on timeout.""" import json as _json import uuid as _uuid from hermes_constants import get_hermes_home # noqa: F811 (deliberate: constants variant) @@ -440,8 +395,7 @@ def _called_process_error_is_git(exc: subprocess.CalledProcessError) -> bool: parts = _called_process_error_cmd_parts(exc) if not parts: return False - # Windows argv may use backslashes; basename() on POSIX would otherwise - # keep the whole path. Normalize separators before taking the name. + # Windows argv may use backslashes; POSIX basename() would keep the whole path. name = os.path.basename(parts[0].replace("\\", "/")).lower() return name in {"git", "git.exe"} @@ -464,14 +418,9 @@ def _called_process_error_is_python_dep_install( def _format_update_failure_stage(exc: subprocess.CalledProcessError) -> str: - """Name the update stage that actually failed. - - The git pull and the Python-dependency install share one ``try`` in - ``_cmd_update_impl``. Calling every ``CalledProcessError`` a git failure - (the historical Windows message) sent users hunting in the wrong place - and, worse, keyed the ZIP overlay on exception *type* rather than on git - actually having failed (#87304, #85840). - """ + """Name the failed stage: git pull and dep install share one ``try``, and calling every + CalledProcessError a git failure misled users and keyed the ZIP overlay on exception + *type* rather than on git actually failing.""" if _called_process_error_is_python_dep_install(exc): return "Python dependency install failed" if _called_process_error_is_git(exc): @@ -480,11 +429,8 @@ def _format_update_failure_stage(exc: subprocess.CalledProcessError) -> str: def _shim_quarantine_error_type() -> "type[BaseException]": - """The strict-quarantine refusal type, resolved lazily through ``_m()``. - - Falls back to a never-raised private type when main.py lacks it (torn - mid-update tree), so the ``except`` clause stays valid. - """ + """Strict-quarantine refusal type via ``_m()``; falls back to a never-raised private + type when main.py lacks it (torn mid-update tree) so the ``except`` stays valid.""" cls = getattr(_m(), "ShimQuarantineError", None) if isinstance(cls, type) and issubclass(cls, BaseException): return cls @@ -496,16 +442,10 @@ def _shim_quarantine_error_type() -> "type[BaseException]": def _refuse_update_for_contended_shims(exc: BaseException) -> None: - """Refuse the dependency sync when live shims could not be quarantined. - - #87331 fail-closed half: a shim rename that failed every retry proves a - process holds the venv without FILE_SHARE_DELETE — running the installer - anyway is exactly how the venv ends up stranded between versions. The - code swap (when one happened) is already committed; only the dependency - install is deferred, via the update-incomplete marker, to the next fresh - launch after the holder exits. Exits 2 (refused) so the command-boundary - receipt net records it as a refusal, not a failure. - """ + """Fail closed when live shims could not be quarantined: a rename failing every retry + proves a holder without FILE_SHARE_DELETE, and installing anyway strands the venv between + versions. The code swap is already committed; only the dep install is deferred (via the + update-incomplete marker). Exits 2 so the receipt records a refusal, not a failure.""" print("✗ Cannot continue the update: live Hermes launcher(s) could not be") print(" moved aside:") for name in getattr(exc, "failed_shims", []) or ["hermes.exe"]: @@ -515,21 +455,15 @@ def _refuse_update_for_contended_shims(exc: BaseException) -> None: print(" now would strand it half-updated.") print(" The dependency install has been deferred: close the process(es)") print(" above, then run any `hermes` command to finish it automatically.") - # Idempotent: the git path already dropped the marker before the sync; - # this covers the ZIP/repair paths so the deferral is never silent. + # Idempotent (git path already dropped it); covers ZIP/repair paths so the deferral is never silent. _write_update_incomplete_marker() sys.exit(2) def _should_zip_fallback_on_update_error(exc: BaseException) -> bool: - """ZIP fallback is for Windows git file-I/O breakage, not later stages. - - A dependency-install failure (locked ``hermes.exe`` / ``uv pip install`` - exit 2) is not a git failure. The pull has already succeeded by then, so - re-downloading the source ZIP cannot fix the install and would replace - every top-level entry except ``venv`` / ``node_modules`` / ``.git`` / - ``.env`` — permanently deleting uncommitted edits and untracked files. - """ + """ZIP fallback is only for Windows git file-I/O breakage: after a dep-install failure the + pull already succeeded, so a ZIP overlay can't fix it and would replace every top-level + entry except venv/node_modules/.git/.env, deleting uncommitted and untracked files.""" return ( isinstance(exc, subprocess.CalledProcessError) and _m()._is_windows() @@ -553,17 +487,11 @@ def _print_called_process_error_tail( def _invalidate_update_cache(): - """Delete the update-check cache for ALL profiles. - - The git repo is shared, so one profile's update makes every profile - current; a per-profile cache would show a stale "commits behind" banner. - """ + """Delete the update-check cache for ALL profiles: the repo is shared, so one profile's + update makes every profile current and a stale "commits behind" banner would linger.""" homes = [] - # Default profile home (Docker-aware — uses /opt/data in Docker) - default_home = get_default_hermes_root() homes.append(default_home) - # Named profiles under /profiles/ profiles_root = default_home / "profiles" if profiles_root.is_dir(): for entry in profiles_root.iterdir(): @@ -626,18 +554,11 @@ def _format_concurrent_instances_message( def _classify_concurrent_instance(pid: int) -> str: - """Return ``"gateway"`` when ``pid``'s command line is a gateway runtime. + """Classify ``pid`` as "gateway" / "non-gateway" / "unknown" (psutil can't read it). - Delegates to ``_is_pausable_gateway`` — the same canonical ``gateway run`` - matcher used by the Desktop preflight exemption and the venv-holder guard - — so a PID classified ``"gateway"`` here is exactly the set the downstream - pause/kill+restart machinery will stop. That symmetry lets the pre-update - concurrent gate skip the abort for gateway-only matches instead of making - the user kill a gateway that is about to be paused anyway. - - Returns ``"non-gateway"`` when the cmdline doesn't match and ``"unknown"`` - when psutil can't read it; the gate treats ``"unknown"`` as non-gateway - (better to block than proceed against an unidentified process). + Uses ``_is_pausable_gateway``, the same matcher as the Desktop preflight exemption and + venv-holder guard, so "gateway" is exactly the set the downstream pause/restart machinery + stops. The gate treats "unknown" as non-gateway (better block than proceed blind). """ try: import psutil # noqa: PLC0415 @@ -661,13 +582,8 @@ def _classify_concurrent_instance(pid: int) -> str: def _filter_non_gateway_concurrent_instances( matches: list[tuple[int, str]], ) -> list[tuple[int, str]]: - """Return only the concurrent-instance matches that are NOT the gateway. - - If every concurrent instance is a gateway, the pause machinery and the - post-update kill+restart handle it and the update proceeds. Anything else - (TUI shell, Desktop backend child, another ``hermes`` REPL) has no pause - machinery downstream, so the gate still aborts. - """ + """Drop gateway matches (the pause + post-update restart machinery handles them); + anything else (TUI, Desktop backend child, another REPL) has no pause path, so the gate aborts.""" non_gateway: list[tuple[int, str]] = [] for pid, name in matches: if _classify_concurrent_instance(pid) != "gateway": @@ -676,13 +592,8 @@ def _filter_non_gateway_concurrent_instances( def _log_only_write(text: str) -> None: - """Write ``text`` to ``~/.hermes/logs/update.log`` only, never the terminal. - - During ``hermes update`` ``sys.stdout`` is an ``_UpdateOutputStream`` - mirroring to terminal and log; this reaches past it to the log handle so - loud, low-signal subprocess output (npm, Electron/vite, cua-driver "Next - steps") stays debuggable without flooding the terminal. - """ + """Write to update.log only: reaches past the ``_UpdateOutputStream`` stdout mirror so + loud, low-signal subprocess output stays debuggable without flooding the terminal.""" if not text: return stream = _m().sys.stdout @@ -695,11 +606,8 @@ def _log_only_write(text: str) -> None: def _run_logged_subprocess(cmd, *, cwd=None, env=None): - """Run ``cmd`` capturing combined output into update.log (not the terminal). - - Returns the ``CompletedProcess`` (with ``stdout`` populated) so the caller - can decide whether to surface the captured output on failure. - """ + """Run ``cmd`` with combined output captured into update.log only; returns the + ``CompletedProcess`` so the caller can surface the output on failure.""" result = subprocess.run( cmd, cwd=cwd, @@ -716,19 +624,11 @@ def _run_logged_subprocess(cmd, *, cwd=None, env=None): def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): - """Implement ``hermes update --check``: fetch and report without installing. - - ``branch`` selects which branch the check compares against. Default is - "main"; callers can pass another branch to ask "are there new commits - on origin/?" without performing the update. - - ``branch_explicit`` is True iff the caller passed --branch on the CLI. - Installs that can't honor non-default branches (e.g. Docker) surface a - one-line notice instead of silently dropping the flag. - """ - # Shared admission gate (#91277 Phase 3): same marker-first decision as - # the apply path, so --check can never report git state for an install - # whose real update mechanism is an image pull. + """``hermes update --check``: fetch and report without installing. ``branch_explicit`` is + True iff --branch was passed; installs that can't honor it (Docker) print a notice + instead of silently dropping the flag.""" + # Same marker-first admission gate as the apply path, so --check never reports git + # state for an install whose real update mechanism is an image pull. from hermes_cli.update_contract import ( evaluate_update_admission, record_refusal_receipt, @@ -749,26 +649,22 @@ def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): if sys.platform == "win32": git_cmd = ["git", "-c", "windows.appendAtomically=false"] - # An interrupted fetch can leave .git/shallow.lock (or another lock) behind, - # making every later fetch fail with "File exists". Self-heal before fetching. + # Interrupted fetches leave .git/*.lock behind ("File exists" forever); self-heal first. from hermes_cli.gitlock import clear_stale_git_locks, clear_stale_tmp_packs cleared = clear_stale_git_locks(_m().PROJECT_ROOT) for lock_path in cleared: print(f" (removed stale git lock: {lock_path})") - # Aborted fetches on flaky lines also strand tmp_pack_* debris in - # .git/objects/pack — unchecked it reached 6 GB and corrupted the pack - # dir outright (#93732). Same age+process safety contract as the locks. + # Aborted fetches also strand tmp_pack_* debris (has reached 6 GB and corrupted the + # pack dir); same age+process safety contract as the locks. swept = clear_stale_tmp_packs(_m().PROJECT_ROOT) if swept: print(f" (removed {len(swept)} aborted-fetch pack temp file(s))") - # Fetch only : a bare `git fetch ` pulls thousands of - # auto-generated branches. Prefer upstream as canonical, but only for main - # (a fork's non-default branch has no upstream counterpart). Installer - # checkouts are shallow (`--depth 1`); a plain fetch would unshallow them - # and rev-list would report a huge bogus "behind" count, so fetch with - # --depth 1 and report presence-only. + # Fetch only (a bare fetch pulls thousands of auto-generated branches). Prefer + # upstream only for main (a fork's other branches have no upstream counterpart). Installer + # checkouts are shallow: a plain fetch would unshallow them and rev-list would report a + # bogus huge "behind" count, so fetch --depth 1 and report presence-only. is_shallow = ( _git_run(git_cmd, ["rev-parse", "--is-shallow-repository"]).stdout.strip() == "true" @@ -776,8 +672,7 @@ def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): depth_args = ["--depth", "1"] if is_shallow else [] if branch == "main": - # Probe locally (~6 ms) for an 'upstream' remote before spending a - # network fetch (~0.3-1 s) that non-fork installs would always fail. + # Probe locally for an 'upstream' remote before a network fetch non-forks always fail. has_upstream_remote = ( _git_run(git_cmd, ["remote", "get-url", "upstream"]).returncode == 0 @@ -789,12 +684,10 @@ def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): if fetch_result is not None and fetch_result.returncode == 0: compare_branch = f"upstream/{branch}" else: - # No upstream remote, or the upstream fetch failed — use origin. print("→ Fetching from origin...") fetch_result = _git_run(git_cmd, ["fetch"] + depth_args + ["origin", branch], network=True) compare_branch = f"origin/{branch}" else: - # Non-default branch: compare against origin/ directly. print("→ Fetching from origin...") fetch_result = _git_run(git_cmd, ["fetch"] + depth_args + ["origin", branch], network=True) compare_branch = f"origin/{branch}" @@ -803,17 +696,15 @@ def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): _print_fetch_failure(fetch_result.stderr) sys.exit(1) - # Verify the compare ref exists first: rev-list on a bogus ref exits 128 - # and (with check=True) would surface a Python traceback. + # rev-list on a bogus ref exits 128 and (check=True) would traceback; verify first. verify_result = _git_run(git_cmd, ["rev-parse", "--verify", "--quiet", compare_branch]) if verify_result.returncode != 0: print(f"✗ Branch '{branch}' not found on {compare_branch.split('/', 1)[0]}.") sys.exit(1) if is_shallow: - # No history across the shallow boundary: compare tip SHAs (like the - # banner's _check_via_local_git), then recover the exact count via the - # GitHub compare API, whose graph is complete. + # No history across the shallow boundary: compare tip SHAs, then recover the + # exact count via the GitHub compare API (complete graph). head_sha = _git_run(git_cmd, ["rev-parse", "HEAD"]).stdout.strip() target_sha = _git_run(git_cmd, ["rev-parse", compare_branch]).stdout.strip() if head_sha and target_sha and head_sha == target_sha: @@ -823,8 +714,7 @@ def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): from hermes_cli.config import recommended_update_command counted = _github_compare_behind(head_sha, target_sha) - if counted == 0: - # Local commits on top of the remote tip — not behind. + if counted == 0: # local-ahead, not behind print("✓ Already up to date.") return if counted is not None: @@ -860,17 +750,10 @@ def _repair_current_checkout( upstream_checked, _windows_gateway_resume, ) -> bool: - """Already-up-to-date path: keep the managed runtime current and repair a broken venv. - - A current checkout does not imply a healthy install (a prior dependency - sync may have died partway), and the Windows shim hand-off child lands - here BY DESIGN to run the sync its parent could not. Returns whether the - checkout can be reported as complete. - """ - # "No new commits" does not mean the managed interpreter is safe. - # uv can retain the same CPython patch while python-build-standalone - # refreshes the embedded SQLite underneath it. Keep the existing - # update-boundary hook active on this retry path too. + """Already-up-to-date path: keep the managed runtime current, repair a broken venv. + Returns whether the checkout can be reported complete.""" + # "No new commits" != safe interpreter: uv can keep the same CPython patch while + # python-build-standalone refreshes the embedded SQLite; keep the boundary hook here too. from hermes_cli.managed_uv import ensure_uv, update_managed_uv runtime_repairs = [] @@ -881,15 +764,11 @@ def _repair_current_checkout( None, ) - # A current checkout does NOT imply a healthy install: a prior sync may - # have died partway (Windows: locked .pyd → uv/pip access-denied, venv - # stranded between versions). Probe core imports and repair, or - # "Already up to date!" hides a bricked install. + # A current checkout does NOT imply a healthy install (a prior sync may have died + # partway, e.g. Windows locked .pyd); probe or "Already up to date!" hides a bricked venv. healthy, detail = _venv_core_imports_healthy() - # The Windows shim hand-off child exists to run the sync its parent - # could not; the checkout is current BY DESIGN, so the pending sync — - # not venv health — is the question. Without this it would print - # "Already up to date!" and skip its one job. + # The Windows shim hand-off child is current BY DESIGN; its one job is the pending sync, + # not venv health — without this it would print "Already up to date!" and skip it. handed_off_sync = os.environ.get(_m()._UPDATE_REEXEC_ENV) == "1" current_checkout_complete = True if handed_off_sync: @@ -899,16 +778,13 @@ def _repair_current_checkout( print(f" {detail}") print("→ Repairing Python dependencies...") if handed_off_sync or not healthy: - # Self-lock deferral (#86735): the repair rewrites the venv - # too — same mapped-extension hazard as the update sync. + # Self-lock deferral: the repair rewrites the venv too (same mapped-extension hazard). _m()._abort_dependency_sync_if_self_locked(_windows_gateway_resume) _write_update_incomplete_marker() from hermes_cli.managed_uv import ensure_uv repair_uv = ensure_uv() - # A managed install whose venv is gone entirely (interrupted - # repair after the old venv was moved aside) needs the venv - # recreated before dependencies can be installed into it. + # Venv gone entirely (repair interrupted after the old one was moved aside): recreate. venv_python_missing = not ( venv_python_path( _m().PROJECT_ROOT / "venv", windows=_m()._is_windows() @@ -922,8 +798,7 @@ def _repair_current_checkout( check=False, ) if repair_uv: - # Isolated from third-party UV env vars (#83914), same as - # the main-path and git-path dependency syncs. + # Isolated from third-party UV env vars, like the other dependency syncs. from hermes_cli.managed_uv import managed_python_env repair_env = managed_python_env() @@ -962,8 +837,7 @@ def _repair_current_checkout( gateway_mode=gateway_mode, pre_update_snapshot_id=pre_update_snapshot_id, ) - # The hand-off child never reaches the commits-pulled rebuild, - # so rebuild the Desktop app here or it stays on the old build (#97343). + # The hand-off child never reaches the commits-pulled rebuild; do it here. if _rebuild_desktop_after_update( desktop_dir, had_desktop_app_before_update=had_desktop_app_before_update, @@ -1017,33 +891,19 @@ def _pull_updates( discard_local_changes, keep_stash, ): - """Fast-forward the checkout onto ``origin/`` and settle the autostash. - - Divergence is handled by shape (custom branch -> merge, same branch -> - reset, orphan history -> rescue ref first); a post-pull syntax error in a - critical file rolls back to the pre-pull SHA. Exits the process on - failure. Returns the pre-pull HEAD SHA (or None). - """ + """Fast-forward onto ``origin/`` and settle the autostash. Divergence by shape: + custom branch -> merge, same branch -> reset, orphan history -> rescue ref first; a + post-pull syntax error in a critical file rolls back. Exits on failure; returns pre-pull SHA.""" update_succeeded = False - # Pre-pull SHA for auto-rollback when pulled code has a syntax error in - # a critical file (PR #28452: stray conflict markers in config.py - # bricked every updater for 7 minutes). + # Pre-pull SHA for auto-rollback (stray conflict markers once bricked every updater). pre_pull_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT) try: - # Merge the ref we already fetched above (→ Fetching updates...) - # instead of `git pull`, which performs a SECOND network fetch of - # the same branch (~0.5-1.5 s of redundant round-trip per update). - # `merge --ff-only origin/` is byte-identical in effect to - # `pull --ff-only origin ` given the fresh tracking ref; - # the divergence fallback below is unchanged. + # merge --ff-only the already-fetched ref instead of `git pull`, which would do a + # SECOND network fetch; identical in effect given the fresh tracking ref. pull_result = _git_run(git_cmd, ["merge", "--ff-only", f"origin/{branch}"]) if pull_result.returncode != 0: - # ff-only failed — local and remote have diverged. Before - # assuming an upstream force-push, check WHY: a checkout on a - # custom branch (local commits on top of origin/) also - # cannot fast-forward, and `reset --hard` here would silently - # discard that work. Merge instead and stop cleanly on - # conflict — an update must never destroy local commits. + # Diverged. A custom branch (local commits atop origin/) also can't ff, + # and reset --hard would discard that work: merge instead, stop on conflict. _cur_branch = ( _git_run(git_cmd, ["branch", "--show-current"]).stdout or "" @@ -1053,7 +913,7 @@ def _pull_updates( f" ⚠ Checkout is on custom branch '{_cur_branch}' — " f"merging origin/{branch} instead of resetting so local commits survive..." ) - # Best-effort safety tag; recovery anchor if anything goes wrong. + # Best-effort safety tag as a recovery anchor. subprocess.run( git_cmd + ["tag", f"pre-update-{_time.strftime('%Y%m%d-%H%M%S')}"], @@ -1082,11 +942,9 @@ def _pull_updates( ) sys.exit(1) else: - # Same branch as the target — a true upstream force-push/ - # rebase; local changes are stashed, so reset to the remote. - # Orphan divergence (no common ancestor: corrupted HEAD, repo - # re-init — #87694) would lose the whole local commit graph, - # so park pre_pull_sha behind a rescue ref first. + # Same branch: a true upstream force-push/rebase; local changes are stashed, so + # reset. Orphan divergence (no common ancestor: corrupted HEAD, re-init) would + # lose the whole local graph, so park pre_pull_sha behind a rescue ref first. merge_base_result = _git_run(git_cmd, ["merge-base", "HEAD", f"origin/{branch}"]) has_common_ancestor = bool( merge_base_result.returncode == 0 @@ -1095,8 +953,7 @@ def _pull_updates( if not has_common_ancestor and pre_pull_sha: from datetime import datetime as _dt, timezone - # SHA suffix (not just a 1s timestamp) so two updates in - # the same second get distinct refs instead of overwriting. + # SHA suffix so two updates in the same second get distinct refs. rescue_ref = ( f"refs/hermes-update-backups/orphan-{branch}-" f"{_dt.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}" @@ -1112,9 +969,7 @@ def _pull_updates( f"{_ORPHAN_RESCUE_REF_MAX_AGE_DAYS} days." ) else: - # update-ref's return code is intentionally not - # fatal (disk full, permissions) — but don't tell - # the user a backup exists when the write failed. + # update-ref failure is intentionally non-fatal, but never claim a backup exists. print( " ⚠ Local history shares no common ancestor with " f"origin/{branch} (orphan divergence) — attempted " @@ -1136,9 +991,7 @@ def _pull_updates( ) sys.exit(1) - # Post-pull syntax guard: a bad commit that slipped past CI - # (admin-merge bypass) is caught here and rolled back so the CLI - # stays bootable until a fix lands. + # Post-pull syntax guard: a bad commit past CI (admin-merge) is rolled back so the CLI stays bootable. syntax_ok, failing_path, syntax_error = _validate_critical_files_syntax( _m().PROJECT_ROOT ) @@ -1147,8 +1000,7 @@ def _pull_updates( print("✗ Pulled code has a syntax error in a critical file:") print(f" {failing_path}") if syntax_error: - # py_compile errors can be multi-line; show the first - # ~6 lines so the user sees the actual SyntaxError text. + # py_compile errors can be multi-line; show enough for the SyntaxError text. for line in str(syntax_error).splitlines()[:6]: print(f" {line}") if pre_pull_sha: @@ -1172,26 +1024,21 @@ def _pull_updates( update_succeeded = True finally: if auto_stash_ref is not None: - # Don't attempt stash restore if the code update itself failed — - # working tree is in an unknown state. + # No stash restore if the update failed — tree state is unknown. if not update_succeeded: print( f" ℹ️ Local changes preserved in stash (ref: {auto_stash_ref})" ) print(" Restore manually with: git stash apply") elif discard_local_changes: - # Non-interactive update + user opted into discarding local - # source edits (updates.non_interactive_local_changes: - # discard). Throw the stash away instead of re-applying it. + # Non-interactive + updates.non_interactive_local_changes: discard. _m()._discard_stashed_changes( git_cmd, _m().PROJECT_ROOT, auto_stash_ref, ) elif keep_stash: - # --keep-stash (desktop updater): the update landed; leave - # local edits parked in the stash instead of silently - # re-applying them onto the updated code. + # --keep-stash (desktop updater): leave edits parked rather than re-apply silently. _m()._park_stashed_changes(auto_stash_ref) else: _m()._restore_stashed_changes( @@ -1229,26 +1076,17 @@ def _prepare_checkout_for_update( switch_branch, _windows_gateway_resume, ): - """Apply the parked-branch guard, land on the update target, stash, and count new commits. - - Exits the process when the checkout is unsafe to move or the target branch - does not exist. ``commit_count`` is 0 when up to date, -1 when tips differ - but the shallow count is unrecoverable. - """ + """Parked-branch guard, land on the target, stash, count new commits. Exits when the + checkout is unsafe to move or the target is missing. ``commit_count`` is 0 when up to + date, -1 when tips differ but the shallow count is unrecoverable.""" switch_block_reason = None # only meaningful when parked_branch_switched - # Parked-branch guard: a checkout parked on a stale feature branch - # used to stash-switch-pull-switch-back, "updating" main while the - # running code stayed behind. Routing by branch contents + - # updates.parked_branch_strategy: + # Parked-branch guard (stash-switch-pull-switch-back used to "update" main while the + # running code stayed behind). By branch contents + updates.parked_branch_strategy: # fully merged -> switch back to the target. - # unmerged: N -> "switch" (default): switch anyway (commits are - # safe on the branch) with a loud "kept" notice; - # deterministic for non-interactive callers. - # "update_in_place": merge origin/ INTO - # the branch — checkout never moves, local commits - # survive. --switch-branch overrides for one run. - # anything else -> dirty/unverifiable/opted out: touch nothing, - # warn, mark the code update SKIPPED, stop. + # unmerged: N -> "switch" (default): switch anyway with a loud "kept" notice; + # "update_in_place": merge origin/ INTO the branch (checkout + # never moves). --switch-branch overrides for one run. + # anything else -> dirty/unverifiable/opted out: touch nothing, warn, mark SKIPPED, stop. parked_branch_switched = False in_place_update = False if current_branch != branch and current_branch != "HEAD": @@ -1284,9 +1122,7 @@ def _prepare_checkout_for_update( == "update_in_place" ) if _in_place_configured and not switch_branch: - # The merge source must exist upstream; --branch typos - # previously surfaced through the checkout failing, which - # does not run on this path. + # --branch typos used to surface via the checkout failing, which this path skips. verify_ref = _git_run(git_cmd, ["rev-parse", "--verify", "--quiet", f"origin/{branch}"]) if verify_ref.returncode != 0: print(f"✗ Branch '{branch}' does not exist locally or on origin.") @@ -1316,15 +1152,12 @@ def _prepare_checkout_for_update( f" ⚠ Currently on detached HEAD — switching to {branch} " "for update..." ) - # Stash before checkout so uncommitted work isn't lost auto_stash_ref = _m()._stash_local_changes_if_needed(git_cmd, _m().PROJECT_ROOT) checkout_result = _git_run(git_cmd, ["checkout", branch]) if checkout_result.returncode != 0: - # Branch not local yet — set it up tracking origin/. track_result = _git_run(git_cmd, ["checkout", "-B", branch, f"origin/{branch}"]) if track_result.returncode != 0: - # Restore the user's prior stash before bailing - # so we don't leave them stranded in a weird state. + # Restore the stash before bailing so the user isn't stranded. if auto_stash_ref is not None: _m()._restore_stashed_changes( git_cmd, @@ -1346,12 +1179,9 @@ def _prepare_checkout_for_update( and (gateway_mode or (sys.stdin.isatty() and sys.stdout.isatty())) ) - # Check if there are updates. On shallow checkouts `rev-list --count` - # walks the truncated graph and can report the entire remote ancestry - # (e.g. "Found 9980 new commit(s)" on a depth-1 install — #53479). - # The zero/nonzero gate is still sound (HEAD == origin/ counts - # 0), so keep it, but treat the shallow NUMBER as unknown and recover - # the real one via the GitHub compare API when possible. + # On shallow checkouts `rev-list --count` can report the entire remote ancestry. The + # zero/nonzero gate is still sound; treat the shallow NUMBER as unknown and recover it + # via the GitHub compare API when possible. result = _git_run(git_cmd, ["rev-list", f"HEAD..origin/{branch}", "--count"], check=True) commit_count = int(result.stdout.strip()) @@ -1365,15 +1195,12 @@ def _prepare_checkout_for_update( head_sha = _git_run(git_cmd, ["rev-parse", "HEAD"]).stdout.strip() target_sha = _git_run(git_cmd, ["rev-parse", f"origin/{branch}"]).stdout.strip() counted = _github_compare_behind(head_sha, target_sha) - # counted == 0 means local-ahead (remote tip reachable from HEAD): - # not behind, fall through to the up-to-date path. + # counted == 0 means local-ahead: falls through to the up-to-date path. commit_count = counted if counted is not None else -1 - # A fork can match origin yet trail upstream, so the upstream sync can - # move HEAD with commit_count == 0. Detect that BEFORE the no-update - # return so deps, restarts AND the fleet matrix still run (#73108 — - # the sync used to live inside the early-return branch and verified - # nothing). Non-forks have no upstream question. + # A fork can match origin yet trail upstream, so the sync can move HEAD with + # commit_count == 0; detect that BEFORE the no-update return so deps, restarts AND the + # fleet matrix still run (it used to live in the early-return branch and verified nothing). upstream_checked = True if commit_count == 0 and is_fork and branch == "main": pre_sync_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT) @@ -1391,8 +1218,7 @@ def _prepare_checkout_for_update( pre_sync_sha, post_sync_sha, ) - # HEAD moving is itself proof of an update. Keep the update - # path active even if the informational count cannot be read. + # HEAD moving is proof of an update even if the count can't be read. commit_count = max(1, synced_count) return _CheckoutPlan( @@ -1422,38 +1248,28 @@ class _UpdateOptions: def _resolve_update_options(args, gateway_mode: bool) -> _UpdateOptions: """Snapshot pre-update state and resolve the flags/config ``_cmd_update_impl`` runs on.""" - # A managed-runtime refresh can replace site-packages before the normal - # ``.[all]`` install runs. Snapshot while the old environment can still - # prove which optional backends the user had activated. + # Snapshot before a managed-runtime refresh can replace site-packages, while the old + # environment can still prove which optional backends were active. active_lazy_features = _m()._capture_active_lazy_features() active_tool_dependencies = _m()._capture_active_tool_dependencies() - # Snapshot the pre-update version before any code is pulled so the - # completion line can report the transition (prime-agent#630 port). + # Captured before any pull so the completion line can report the transition. pre_update_version = _read_project_version() - # In gateway mode, use file-based IPC for prompts instead of stdin gw_input_fn = ( (lambda prompt, default="": _gateway_prompt(prompt, default)) if gateway_mode else None ) assume_yes = bool(getattr(args, "yes", False)) - # --keep-stash (desktop updater): stash local changes so the update can - # proceed, but never re-apply them afterward — they stay parked in git - # stash. Only applies when an update actually landed; abort/no-op paths - # still restore, since the tree they restore onto is unchanged. + # --keep-stash (desktop updater): never re-apply the autostash; only when an update + # landed — abort/no-op paths still restore since the tree is unchanged. keep_stash = bool(getattr(args, "keep_stash", False)) - # --switch-branch: on a branch carrying unmerged commits, prefer switching - # to the update target over an in-place merge, so the branch's history is - # never written to by an update (#89507 review feedback). Only meaningful - # when updates.parked_branch_strategy is "update_in_place". + # --switch-branch: prefer switching over an in-place merge so an update never writes the + # branch's history; only meaningful with parked_branch_strategy "update_in_place". switch_branch = bool(getattr(args, "switch_branch", False)) - # Whether this update is running without a human at the keyboard. - # Interactive terminal updates always stash-and-ask (unchanged behavior); - # only non-interactive updates (desktop/chat app, gateway, `--yes`) consult - # the `updates.non_interactive_local_changes` config setting to decide - # whether to auto-restore stashed local source changes or throw them away. + # Interactive terminals always stash-and-ask; only non-interactive updates consult + # updates.non_interactive_local_changes (auto-restore vs discard). _non_interactive_update = ( gateway_mode or assume_yes @@ -1469,7 +1285,7 @@ def _resolve_update_options(args, gateway_mode: bool) -> _UpdateOptions: _mode = str(_update_cfg.get("non_interactive_local_changes", "stash")).lower() discard_local_changes = _mode == "discard" except Exception as exc: - # Never let a config read failure change the safe default. + # A config read failure must never change the safe default. logger.debug("Could not read updates.non_interactive_local_changes: %s", exc) discard_local_changes = False return _UpdateOptions( @@ -1485,23 +1301,17 @@ def _resolve_update_options(args, gateway_mode: bool) -> _UpdateOptions: def _begin_update_receipt_and_plan(args): - """Open the update receipt, snapshot the running fleet, refuse on Windows shim holders. - - Returns the pre-update runtime plan (or None when the probe failed); - ``sys.exit(2)`` when a non-gateway hermes.exe still holds the venv shim. - """ - # Phase 1 (#91277): structured update receipt — record what this run - # discovers, does, and skips, so silent-failure classes (#88848, - # #74973, #85753, #81193) become diagnosable from disk. + """Open the receipt, snapshot the fleet, refuse on Windows shim holders. Returns the + pre-update plan (None if the probe failed); ``sys.exit(2)`` when a non-gateway hermes.exe + holds the venv shim.""" + # Structured receipt: record what this run discovers/does/skips so silent failures are diagnosable. with _best_effort('Update receipt unavailable: %s'): from hermes_cli.update_receipt import begin_update_receipt begin_update_receipt() - # Plan phase (#91277): snapshot every running runtime, supervisor and - # code version into the receipt (read-only; probe failure records - # nothing). Re-read AFTER the restart phase to reconcile planned - # runtimes against bookkeeping — the plan is the worklist. + # Plan phase: snapshot runtimes/supervisors/version (read-only; probe failure records + # nothing). Re-read AFTER the restart phase to reconcile — the plan is the worklist. _pre_update_plan = None with _best_effort('Update plan phase failed: %s'): from hermes_cli.update_inventory import ( @@ -1518,12 +1328,9 @@ def _begin_update_receipt_and_plan(args): ) print(f"→ Fleet: {_n} running service(s) across profiles: {_profiles}") - # Windows: abort if another hermes.exe holds the venv shim — continuing - # yields WinError 32 spam and a deferred-rename leftover or silent ZIP - # fallback (#26670). Exception (#37039): instances positively identified - # as gateways are paused by ``_pause_windows_gateways_for_update`` below - # and restarted afterwards; anything else (TUI, Desktop backend, - # unreadable cmdline) still aborts. + # Windows: another hermes.exe holding the venv shim means WinError 32 spam and a + # deferred-rename leftover or silent ZIP fallback. Positively identified gateways are + # paused/restarted by the update instead; anything else still aborts. if _m()._is_windows() and not getattr(args, "force", False): scripts_dir = _m()._venv_scripts_dir() if scripts_dir is not None: @@ -1543,13 +1350,8 @@ def _begin_update_receipt_and_plan(args): def _prepare_git_command() -> tuple[bool, list, bool]: - """Decide git-vs-ZIP, build the git argv and detect a fork origin. - - Returns ``(use_zip_update, git_cmd, is_fork)``; ``sys.exit(1)`` when the - checkout is not a git repo on a non-Windows host. - """ - # Try git-based update first, fall back to ZIP download on Windows - # when git file I/O is broken (antivirus, NTFS filter drivers, etc.) + """Return ``(use_zip_update, git_cmd, is_fork)``; ``sys.exit(1)`` when not a git repo + on a non-Windows host (Windows falls back to ZIP: broken git file I/O, AV, NTFS filters).""" use_zip_update = False git_dir = _m().PROJECT_ROOT / ".git" @@ -1563,8 +1365,7 @@ def _prepare_git_command() -> tuple[bool, list, bool]: ) sys.exit(1) - # On Windows, git can fail with "unable to write loose object file: Invalid argument" - # due to filesystem atomicity issues. Set the recommended workaround. + # Windows git can fail "unable to write loose object file: Invalid argument" (fs atomicity). if sys.platform == "win32" and git_dir.exists(): subprocess.run( [ @@ -1580,26 +1381,18 @@ def _prepare_git_command() -> tuple[bool, list, bool]: capture_output=True, ) - # Build git command once — reused for fork detection and the update itself. git_cmd = ["git"] if sys.platform == "win32": git_cmd = ["git", "-c", "windows.appendAtomically=false"] - # A broken Git-for-Windows trampoline refuses every git call with a - # "BUG (fork bomb)" guard instead of running; swap in a real binary up - # front so the normal git path survives instead of degrading to ZIP - # (#87876). + # A broken Git-for-Windows trampoline refuses every call with a "BUG (fork bomb)" guard; + # swap in a real binary up front so git survives instead of degrading to ZIP. git_cmd = _ensure_non_trampoline_git(git_cmd) - # Discard npm lockfile churn before stash/branch logic: npm rewrites - # package-lock.json non-deterministically, which is never an intentional - # edit on a managed install but forces an autostash every update. + # Before stash/branch logic: npm rewrites package-lock.json non-deterministically and + # line-ending churn is machine-made dirt; both would otherwise force an autostash every update. _discard_lockfile_churn(git_cmd, _m().PROJECT_ROOT) - # Same rationale, different generator: line-ending churn is machine-made - # dirt on a managed checkout, so clear it (and stop generating it) before - # the stash/branch logic rather than autostashing the entire tree. _normalize_managed_eol(git_cmd, _m().PROJECT_ROOT) - # Detect if we're updating from a fork (before any branch logic) origin_url = _m()._get_origin_url(git_cmd, _m().PROJECT_ROOT) is_fork = _is_fork(origin_url) @@ -1614,10 +1407,8 @@ def _verify_head_after_pull( git_cmd, branch: str, pre_pull_sha, *, in_place_update: bool, _windows_gateway_resume ) -> str | None: """Return the post-pull HEAD SHA; ``sys.exit(1)`` if the pull was a no-op or landed off-branch.""" - # Verify HEAD moved (#79678): a detached checkout pinned to a SHA can - # report "N new commit(s)" and a successful ``merge --ff-only`` yet - # stay on the old commit, so the old code reinstalled deps and - # claimed "✓ Code updated!". Surface the no-op instead. + # A detached checkout pinned to a SHA can report "N new commit(s)" and a successful + # merge --ff-only yet stay put; surface the no-op instead of claiming "Code updated!". post_pull_sha = _capture_head_sha(git_cmd, _m().PROJECT_ROOT) if pre_pull_sha and post_pull_sha == pre_pull_sha: print() @@ -1633,10 +1424,8 @@ def _verify_head_after_pull( _m()._resume_windows_gateways_after_update(_windows_gateway_resume) sys.exit(1) - # Verify HEAD is on the target branch; otherwise "✓ Code updated!" - # would be a lie. An IN-PLACE update is the one legitimate way to end - # elsewhere: origin/ was merged INTO the checked-out branch, - # so the running code *is* current. + # HEAD must be on the target or "Code updated!" is a lie; an IN-PLACE update is the one + # legitimate exception (origin/ merged INTO the checked-out branch). post_pull_branch = _git_run(git_cmd, ["rev-parse", "--abbrev-ref", "HEAD"]).stdout.strip() if ( not in_place_update @@ -1710,11 +1499,8 @@ def _finish_already_up_to_date( active_tool_dependencies, _windows_gateway_resume, ) -> None: - """"Already up to date" path: restore stash/branch, repair the current checkout, catch up the fleet. - - ``sys.exit(1)`` when the current-checkout repair is incomplete (after - writing the gateway exit code and finalizing the receipt as partial). - """ + """"Already up to date" path: restore stash/branch, repair the checkout, catch up the fleet. + ``sys.exit(1)`` when the repair is incomplete (after gateway exit code + partial receipt).""" auto_stash_ref = _plan.auto_stash_ref parked_branch_switched = _plan.parked_branch_switched prompt_for_restore = _plan.prompt_for_restore @@ -1722,10 +1508,8 @@ def _finish_already_up_to_date( upstream_checked = _plan.upstream_checked _invalidate_update_cache() - # Restore stash and switch back to original branch if we moved. - # EXCEPTION: a parked feature branch we verified clean + fully - # merged stays on the target — re-parking the checkout on the - # stale branch is the 2026-08-17 incident all over again. + # Restore stash and switch back if we moved. EXCEPTION: a parked branch verified clean + + # fully merged stays on the target — re-parking on the stale branch recreates the incident. if auto_stash_ref is not None: _m()._restore_stashed_changes( git_cmd, @@ -1762,10 +1546,8 @@ def _finish_already_up_to_date( _windows_gateway_resume=_windows_gateway_resume, ) _m()._resume_windows_gateways_after_update(_windows_gateway_resume) - # A prior pull may still owe the fleet a restart (#95294); catch - # up even on the "Already up to date" path, and BEFORE the exit - # gate below so a partial outcome can't strand the fleet on stale - # code (#91277 fleet contract). + # A prior pull may still owe the fleet a restart; catch up here too, BEFORE the exit + # gate so a partial outcome can't strand the fleet on stale code. _apply_pending_fleet_restart_catchup() if not current_checkout_complete: if gateway_mode: @@ -1795,10 +1577,8 @@ def _cmd_update_impl(args, gateway_mode: bool): _pre_update_plan = _begin_update_receipt_and_plan(args) - # Pre-update backup — runs before any git/file mutation so users can - # always roll back to the exact state they had before this update. - # Returns the quick-snapshot id (or None when disabled/failed); the - # post-update cron-jobs safety net uses it to detect job loss. + # Backup before any git/file mutation; the snapshot id (None if disabled/failed) feeds + # the post-update cron-jobs safety net. pre_update_snapshot_id = _m()._run_pre_update_backup(args) _record_update_step( "pre_update_backup", @@ -1815,36 +1595,24 @@ def _cmd_update_impl(args, gateway_mode: bool): _windows_gateway_resume, ) - # With gateways paused, any venv python still running (typically the - # Desktop `hermes serve` backend) keeps .pyd files locked and would - # corrupt the sync; refuse rather than race (the app respawns a killed - # backend). NOT bypassed by --force: the desktop updater passes it to - # skip the shim guard but only probes the shim and app.asar. - # --force-venv is the explicit escape hatch. + # Any venv python still running (typically the Desktop `hermes serve` backend) keeps .pyd + # locked and would corrupt the sync; refuse rather than race (the app respawns a killed + # backend). NOT bypassed by --force (desktop updater, shim guard only); --force-venv is. if _m()._is_windows() and not getattr(args, "force_venv", False): _clear_windows_venv_holders_or_exit(args, gateway_mode, _windows_gateway_resume) - # Self-lock deferral moved: the venv-holder sweep above excludes this - # process by design (a CLI `hermes update` IS the venv python), and an - # updater that has imported a native venv extension cannot rewrite its - # own mapped .pyd (#83569). That check used to run HERE — before the - # fetch — but firing pre-fetch meant a deferral stranded the user on the - # OLD checkout, and any startup path that eagerly loaded cryptography - # turned every Windows update into an exit-2 loop (#86735/#86780/#86781). - # It now runs via _abort_dependency_sync_if_self_locked() after the code - # swap, immediately before the dependency sync — the only phase the lock - # can actually break — and only when the sync would truly rewrite the - # loaded distribution. + # Self-lock deferral (this process IS the venv python and can't rewrite its own mapped + # .pyd) deliberately does NOT run here: pre-fetch it stranded users on the OLD checkout + # and eager cryptography imports made every Windows update an exit-2 loop. It runs via + # _abort_dependency_sync_if_self_locked() right before the dependency sync instead. - # Capture this after every fail-closed venv guard, but before either - # update path can remove the ignored release tree. + # After every fail-closed venv guard, before either path can remove the release tree. desktop_dir = _m().PROJECT_ROOT / "apps" / "desktop" had_desktop_app_before_update = _desktop_app_present(desktop_dir) use_zip_update, git_cmd, is_fork = _prepare_git_command() if use_zip_update: - # ZIP-based update for Windows when git is broken try: desktop_build_ok = _update_via_zip( args, @@ -1856,17 +1624,11 @@ def _cmd_update_impl(args, gateway_mode: bool): _write_gateway_update_exit_code(desktop_build_ok) return - # Fetch and pull try: - - # Resolve the branch first so the fetch is scoped: a bare `git fetch - # origin` pulls thousands of auto-generated branches and can stall for minutes. + # Scoped fetch: a bare `git fetch origin` pulls thousands of branches and can stall. branch = _m()._resolve_update_branch(args) - # Self-heal abandoned git lock files (e.g. .git/shallow.lock left by a - # crashed fetch) before the fetch — otherwise the update fails with - # "Unable to create .../shallow.lock: File exists" and never reaches - # the network. + # Self-heal abandoned .git/*.lock files (crashed fetch) or the fetch fails "File exists". from hermes_cli.gitlock import clear_stale_git_locks, clear_stale_tmp_packs cleared = clear_stale_git_locks(_m().PROJECT_ROOT) @@ -1876,9 +1638,7 @@ def _cmd_update_impl(args, gateway_mode: bool): if swept: print(" (removed %d aborted-fetch pack temp file(s))" % len(swept)) - # Surface autostash entries left behind by earlier updates (#63717 - # problem 6) — parked --keep-stash runs and failed restores preserve - # the stash but nothing ever mentioned it again. + # Surface autostashes left by earlier updates (--keep-stash, failed restores). _m()._warn_orphaned_update_autostashes(git_cmd, _m().PROJECT_ROOT) print("→ Fetching updates...") @@ -1887,7 +1647,7 @@ def _cmd_update_impl(args, gateway_mode: bool): _print_fetch_failure(fetch_result.stderr) sys.exit(1) - # Get current branch (returns literal "HEAD" when detached) + # literal "HEAD" when detached result = _git_run(git_cmd, ["rev-parse", "--abbrev-ref", "HEAD"], check=True) current_branch = result.stdout.strip() @@ -1928,8 +1688,7 @@ def _cmd_update_impl(args, gateway_mode: bool): if commit_count > 0: print(f"→ Found {commit_count} new commit(s)") else: - # Shallow checkout, exact count unrecoverable (offline/rate-limited - # compare API) — the tips differ, so there IS an update. + # Shallow, exact count unrecoverable — but the tips differ, so there IS an update. print("→ Updates available (commit count unknown on this shallow checkout)") print("→ Pulling updates...") @@ -1953,19 +1712,14 @@ def _cmd_update_impl(args, gateway_mode: bool): _windows_gateway_resume=_windows_gateway_resume, ) - # #95294: HEAD advanced; running gateways still serve pre-pull - # modules until the restart phase below. Any interrupt between here - # and a completed (or no-op) restart leaves this marker so the next - # ``hermes update`` can catch up even when git is already up to date. - # Distinct from ``.update-incomplete`` (venv/install repair). + # Gateways still serve pre-pull modules until the restart phase; an interrupt before a + # completed restart leaves this marker so the next update catches up even when git is + # current. Distinct from ``.update-incomplete`` (venv/install repair). _write_fleet_restart_pending_marker(expected_sha=post_pull_sha or "") - # Clear stale .pyc bytecode cache — prevents ImportError on gateway - # restart when updated source references names that didn't exist in - # the old bytecode (e.g. get_hermes_home added to hermes_constants). + # Stale .pyc would ImportError on gateway restart when new source references new names. _sweep_bytecode_after_update(branch) - # Fork upstream sync logic (only for main branch on forks) if is_fork and branch == "main": _m()._sync_with_upstream_if_needed( git_cmd, @@ -1974,10 +1728,8 @@ def _cmd_update_impl(args, gateway_mode: bool): input_fn=gw_input_fn, ) - # Reinstall deps: .[all], falling back to base + remaining extras - # individually so a broken extra doesn't strip working capabilities. - # Ownership preflight (#83529) refuses first if the venv has - # foreign-owned (sudo-pip) files that would brick the install mid-mutation. + # .[all], falling back to base + extras individually so one broken extra doesn't strip + # the rest; the ownership preflight refuses first on foreign-owned (sudo-pip) venv files. _sync_python_dependencies_after_pull( git_cmd, branch, @@ -2008,13 +1760,9 @@ def _cmd_update_impl(args, gateway_mode: bool): pre_update_version=pre_update_version, ) - # Write the exit code *before* the restart attempt: under ``update - # --gateway`` this process lives in the gateway's systemd cgroup, and - # the ``systemctl restart`` fallback SIGKILLs the cgroup (KillMode= - # mixed) — us and the wrapping shell included — so the marker would - # never be written and the new gateway's watcher would poll 30 min - # and report a spurious timeout. The verified summary already folds - # in Desktop and SQLite-runtime health (gateway/run.py). + # Exit code *before* the restart: under --gateway this process lives in the gateway's + # systemd cgroup and the systemctl-restart fallback SIGKILLs it (KillMode=mixed), so + # the marker would never land and the new gateway's watcher would time out spuriously. if gateway_mode: _write_gateway_update_exit_code(update_complete) @@ -2029,12 +1777,7 @@ def _cmd_update_impl(args, gateway_mode: bool): ) except _shim_quarantine_error_type() as e: - # Fail-closed shim contention (#87331): strict quarantine refused - # BEFORE any installer ran — defer via marker, exit 2, no ZIP. + # Strict quarantine refused BEFORE any installer ran — defer via marker, exit 2, no ZIP. _refuse_update_for_contended_shims(e) except subprocess.CalledProcessError as e: _handle_update_called_process_error(e, args, gateway_mode, had_desktop_app_before_update) - -# --- Hoisted from the body of _cmd_update_impl (self-contained, no closure state) --- - - diff --git a/hermes_cli/update_cmd_fleet.py b/hermes_cli/update_cmd_fleet.py index a94201a837..544ba28abf 100644 --- a/hermes_cli/update_cmd_fleet.py +++ b/hermes_cli/update_cmd_fleet.py @@ -1,9 +1,8 @@ -"""Gateway fleet restart + post-update verification for ``hermes update``: systemd/launchd unit restarts, pending-restart marker, fleet probe. +"""Gateway fleet restart + post-update verification for ``hermes update``. -Split out of ``hermes_cli/update_cmd.py``; every moved name is re-imported there, so -``hermes_cli.update_cmd.`` keeps resolving (and monkeypatching) as before. -Origin-internal helpers are imported lazily inside each function (no import cycle; -test patches on ``hermes_cli.update_cmd.`` stay effective). +Split out of ``hermes_cli/update_cmd.py``; every name is re-imported there so +``hermes_cli.update_cmd.`` keeps resolving/monkeypatching. Origin helpers are +imported lazily inside each function (no import cycle; test patches stay effective). """ import logging @@ -28,9 +27,8 @@ def _write_gateway_update_exit_code(ok: bool) -> None: path.write_text("0" if ok else "1", encoding="utf-8") -# Lives under HERMES_HOME (not next to the venv). Unlike the venv-repair -# markers, this records the fleet-restart obligation after a pull advanced -# HEAD (#95294); cleared only when the restart completes or nothing was running. +# Under HERMES_HOME (not next to the venv): records the fleet-restart obligation +# after a pull advanced HEAD; cleared only when the restart completes or nothing ran. _FLEET_RESTART_PENDING_NAME = "fleet_restart_pending" @@ -95,10 +93,9 @@ def _receipt_looks_unfinished(receipt: dict) -> bool: def _receipt_reports_stale_runtime(expected_sha: str | None = None) -> bool: """True when ``update_receipts/latest.json`` records a runtime SHA skew. - Prefer the post-restart ``fleet`` matrix. ``plan.runtimes[].code_sha`` is - captured *before* the pull, so a finished update's plan always shows stale - SHAs and must not retrigger a restart; consult it only for an unfinished - receipt (#95294). + Prefer the post-restart ``fleet`` matrix. ``plan.runtimes[].code_sha`` is captured + *before* the pull, so a finished update's plan always looks stale and must not + retrigger a restart; consult it only for an unfinished receipt. """ from hermes_cli.update_cmd import _current_checkout_sha try: @@ -140,7 +137,7 @@ def _receipt_reports_stale_runtime(expected_sha: str | None = None) -> bool: def _pending_fleet_restart_needed() -> bool: - """True when a prior pull still owes the fleet a restart (#95294).""" + """True when a prior pull still owes the fleet a restart.""" with suppress(OSError): if _fleet_restart_pending_marker_path().is_file(): return True @@ -212,10 +209,9 @@ def _restart_systemd_gateway_units_best_effort(failed: list) -> None: def _run_pending_fleet_restart() -> bool: - """Catch-up restart for gateways left on pre-update code (#95294). + """Catch-up restart for gateways left on pre-update code. Never raises. - Returns True when restart completed or no services were running. - Returns False if restart was incomplete. Never raises. + True when the restart completed or nothing was running; False if incomplete. """ from hermes_cli.update_cmd import _m print("→ Restarting gateways left on pre-update code...") @@ -291,8 +287,8 @@ def _run_pending_fleet_restart() -> bool: def _apply_pending_fleet_restart_catchup() -> None: """On an already-up-to-date ``hermes update``, finish a skipped restart. - No-op when nothing is pending. Exits 1 when the catch-up restart is - incomplete so automation does not treat the fleet as healthy. + No-op when nothing is pending; exits 1 on incomplete catch-up so automation + does not treat the fleet as healthy. """ from hermes_cli.update_cmd import _run_pending_fleet_restart if not _pending_fleet_restart_needed(): @@ -318,9 +314,8 @@ def _systemctl(cmd: list, *, timeout: float): def _systemctl_reset_and_restart(manage_cmd: list, svc_name: str): - """``reset-failed`` then ``restart`` a unit. Always clear failed state first: if - systemd's own auto-restart attempts already parked the unit in a failed state, - a plain ``restart`` can wedge against the RestartSec backoff and leave it dead.""" + """``reset-failed`` then ``restart``: a unit parked in failed state by systemd's own + auto-restart can wedge a plain ``restart`` against RestartSec backoff and stay dead.""" _systemctl(manage_cmd + ["reset-failed", svc_name], timeout=10) return _systemctl(manage_cmd + ["restart", svc_name], timeout=15) @@ -331,12 +326,10 @@ def _for_each_systemd_gateway_unit( process_unit, on_unit_timeout, ) -> None: - """Process each ``hermes-gateway*.service``/``hermes-serve*.service`` unit - from ``systemctl list-units``. + """Process each hermes-gateway*/hermes-serve* unit from ``systemctl list-units``. - ``subprocess.TimeoutExpired`` raised by ``process_unit`` is isolated to - that unit via ``on_unit_timeout`` so one wedged systemctl call cannot - abort the rest of the fleet (#68523). + ``TimeoutExpired`` from ``process_unit`` is isolated per unit via ``on_unit_timeout`` + so one wedged systemctl call cannot abort the rest of the fleet. """ for line in (list_units_stdout or "").strip().splitlines(): parts = line.split() @@ -345,10 +338,8 @@ def _for_each_systemd_gateway_unit( unit = parts[0] if not unit.endswith(".service"): continue - # list-units is already pattern-filtered, but keep the name gate so a - # stray line cannot enter the restart path. Require the exact base unit - # or hyphenated profile family: ``startswith("hermes-serve")`` would - # also accept the unrelated ``hermes-server.service`` (#83595). + # Name gate against stray lines. Exact base unit or hyphenated profile family + # only: ``startswith("hermes-serve")`` would accept ``hermes-server.service``. if not ( unit == "hermes-gateway.service" or unit.startswith("hermes-gateway-") @@ -366,14 +357,10 @@ def _for_each_systemd_gateway_unit( def _service_unit_supports_graceful_sigusr1_restart(svc_name: str) -> bool: """Whether *svc_name* wires SIGUSR1 to a graceful drain-then-restart. - Only ``hermes-gateway*`` units run ``gateway/run.py`` (the SIGUSR1 - handler). ``hermes-serve*`` units (#83438) don't: SIGUSR1 would just - terminate them and burn the full drain budget, so they go straight to the - blunt ``systemctl restart`` path. - - Same strict exact/hyphenated shape as the unit-name gate in - ``_for_each_systemd_gateway_unit``, so a near-prefix unit like - ``hermes-gatewayd`` can't be sent a SIGUSR1 it doesn't handle. + Only ``hermes-gateway*`` runs ``gateway/run.py`` (the handler); SIGUSR1 would just + kill ``hermes-serve*`` and burn the drain budget, so those go straight to the blunt + restart. Same exact/hyphenated shape as ``_for_each_systemd_gateway_unit`` so a + near-prefix unit like ``hermes-gatewayd`` is never signalled. """ return svc_name == "hermes-gateway" or svc_name.startswith("hermes-gateway-") @@ -397,9 +384,8 @@ def _warn_incomplete_gateway_fleet_restart(failed_units: list) -> None: for name in ordered: print(f" - {name}") if is_macos(): - # A launchd label lands here when launchd was not supervising a live - # process after the restart (#88848) — very likely deregistered, which - # `launchctl kickstart` cannot revive. + # A label lands here when launchd wasn't supervising a live process after + # the restart — likely deregistered, which `launchctl kickstart` can't revive. print(" Listed services may be deregistered from launchd, or still") print(" running pre-update code (mixed sys.modules). Recover with:") print(" hermes gateway status") @@ -422,19 +408,14 @@ def _restart_launchd_gateway_after_update( ) -> tuple[list, list]: """Restart the invoking profile's launchd gateway after an update. - No ``launchctl list``-based classification (#74973): a *booted-out* job - (plist present, definition deregistered — crashed helper, manual bootout, - failed prior update) fails that check, and ``launchctl list`` is - session-scoped and can exit non-zero while the job is alive, so gating on - it silently skipped the restart while still printing "Update complete!". - When the plist exists, ``launchd_restart()`` always runs — it drains a - live PID, kickstarts with ``-k``, and owns the bootout/bootstrap/kickstart - ladder for the unloaded state. Every failure path is loud and names the - manual recovery command. - - Returns ``(restarted_labels, failed_labels)``. With ``supervision_verify`` - (the update path), success additionally requires launchd reporting a fresh - supervised PID (#88848 — "the call returned" is not "supervised"). + No ``launchctl list`` gating: a booted-out job (plist present, definition + deregistered) fails it, and it is session-scoped and can exit non-zero while the + job is alive — gating on it silently skipped the restart yet printed "Update + complete!". When the plist exists ``launchd_restart()`` always runs (drain, -k + kickstart, bootout/bootstrap ladder); every failure path is loud with a manual + recovery command. Returns ``(restarted_labels, failed_labels)``; with + ``supervision_verify`` success also requires a fresh supervised PID ("the call + returned" is not "supervised"). """ from hermes_cli.gateway import ( get_launchd_label, @@ -458,9 +439,8 @@ def _restart_launchd_gateway_after_update( ) return [], [current_label] except (FileNotFoundError, subprocess.TimeoutExpired) as e: - # A plist exists, so a gateway is SUPPOSED to be supervised; a broken/ - # wedged launchctl is not proof nothing needs restarting (#74973's - # second silent variant). Count it and tell the operator. + # A plist exists, so a gateway is SUPPOSED to be supervised; a broken/wedged + # launchctl is not proof nothing needs restarting. Count it, tell the operator. print( " ⚠ Could not restart the gateway " f"({e.__class__.__name__}: {e}).\n" @@ -471,12 +451,10 @@ def _restart_launchd_gateway_after_update( if not supervision_verify: return [current_label], [] - # launchd_restart() returning only means "restart REQUESTED" — both the - # self-restart branch and a plist reload are asynchronous. A helper that - # dies before its first bootstrap (#88848), or a bootstrap that exits 0 - # without registering (seen on macOS 26.6.1), would otherwise reach "Update - # complete!" unsupervised. Verified domain-agnostically: a domain locate - # fails on macOS-26 hosts whose per-user domains reject service management. + # launchd_restart() returning only means "restart REQUESTED" (both branches are + # async). A helper dying before first bootstrap, or a bootstrap exiting 0 without + # registering (macOS 26.6.1), would otherwise reach "Update complete!" unsupervised. + # Verified domain-agnostically: domain locate fails on macOS-26 per-user domains. if wait_for_launchd_gateway_supervision(label=current_label): return [current_label], [] print( @@ -493,16 +471,11 @@ def _restart_macos_launchd_gateways( ) -> None: """Restart every launchd-managed gateway after an update (macOS). - The git pull is shared across profiles, so every ``ai.hermes.gateway*`` - LaunchAgent must reload it; restarting only the invoking profile leaves - siblings on pre-update ``sys.modules`` (#41403). Parity with systemd. - - The invoking profile keeps ``launchd_restart()`` (self-restart request → - drain → kickstart). Siblings get the same drain-first sequence with their - launchd domain resolved per label (``gui/`` vs ``user/``) so - none is kickstarted in the wrong domain. ``subprocess.TimeoutExpired`` is - isolated per label so one wedged launchctl call cannot strand the fleet - on old code (#68523). + The pull is shared across profiles, so every ``ai.hermes.gateway*`` LaunchAgent + must reload it or siblings stay on pre-update ``sys.modules`` (systemd parity). + Invoking profile uses ``launchd_restart()``; siblings get the same drain-first + sequence with their domain (``gui/`` vs ``user/``) resolved per label so + none is kickstarted in the wrong domain. ``TimeoutExpired`` is isolated per label. """ from hermes_cli.gateway import ( get_launchd_label, @@ -513,7 +486,7 @@ def _restart_macos_launchd_gateways( _wait_for_launchd_service_pid, ) - # --- Current profile: unchanged single-service path --------------------- + # --- Current profile --- _restarted, _failed = _restart_launchd_gateway_after_update( supervision_verify=True ) @@ -521,18 +494,16 @@ def _restart_macos_launchd_gateways( failed_or_stale_units.extend(_failed) current_label = get_launchd_label() - # --- Sibling profiles --------------------------------------------------- + # --- Sibling profiles --- for label in launchd_gateway_labels_for_install(): if label == current_label: continue try: - # Locate = liveness + domain in one probe; the kickstart and - # fresh-PID checks below reuse that domain so a sibling is never - # probed in one gui/user domain and restarted in another. + # Locate = liveness + domain in one probe; kickstart and fresh-PID checks + # reuse that domain so a sibling is never probed in one and restarted in another. domain, old_pid = _locate_launchd_gateway_service(label) if domain is None: - # Installed but not bootstrapped (stopped/uninstalled - # mid-way) — nothing is running old code here. + # Installed but not bootstrapped — nothing running old code here. continue graceful_ok = False if old_pid is not None and old_pid > 0: @@ -543,8 +514,7 @@ def _restart_macos_launchd_gateways( if graceful_ok and _wait_for_launchd_service_pid( label, old_pid=old_pid, timeout=10.0, domain=domain ): - # Unconditional KeepAlive already respawned it on the new - # code — a hard kickstart now would kill the fresh process. + # KeepAlive already respawned it on new code — a kickstart would kill it. restarted_services.append(label) continue try: @@ -578,11 +548,9 @@ def _restart_macos_launchd_gateways( def _surviving_gateway_pids_after_failed_restart(): """Best-effort PIDs of gateways still running after the restart phase died. - ``None`` when undeterminable — notably when ``hermes_cli.gateway`` no - longer imports, one of the ways the restart phase aborts (the checkout was - replaced under a process holding old modules). The caller treats ``None`` - and a non-empty list as "assume stale"; only a positive empty result proves - nothing needs restarting. + ``None`` when undeterminable (notably ``hermes_cli.gateway`` no longer importing + under the replaced checkout). Callers treat ``None`` and non-empty as "assume + stale"; only a positive empty result proves nothing needs restarting. """ try: from hermes_cli.gateway import find_gateway_pids @@ -597,12 +565,9 @@ _FRESH_RESTART_SUPERVISORS = frozenset({"systemd", "launchd", "service", "s6"}) def _gateway_service_matches_profile(profile: str, service: object) -> bool: - """Match an exact gateway service/label to a profile. + """Match an exact gateway service/label (systemd/launchd/s6 shapes) to a profile. - Profile names must not be matched as substrings: ``foo`` must not claim - that ``hermes-gateway-foobar.service`` was already restarted. These are - the service/label shapes produced by the existing systemd, launchd, and - s6 lifecycle implementations. + Never substring-match: ``foo`` must not claim ``hermes-gateway-foobar.service``. """ name = str(service).removesuffix(".service") if profile == "default": @@ -624,21 +589,15 @@ def _gateway_recovery_partition( ) -> tuple[dict[str, str], list[dict]]: """Partition pre-update runtimes into fresh-restart candidates and skips. - Uses only the inventory captured before the checkout changed: re-importing - ``hermes_cli.gateway`` in the failing interpreter is exactly what can raise - the original ``ImportError``. - - Returns ``(candidates, skipped)``: ``candidates`` maps profile → supervisor - for supervised gateway runtimes the fresh process may restart; ``skipped`` - lists every other inventoried runtime, each with an explicit reason, so - nothing from the spawn ledger vanishes silently (manual gateways have no - relaunch authority; serve/dashboard runtimes have no per-profile command). - - A ``skipped`` serve/dashboard entry does NOT mean unrecoverable: the fresh - child runs a separate ``hermes-serve*`` systemd pass enumerating units from - systemd, because the ledger collector cannot classify a systemd-launched - ``hermes serve`` (no spawner ⇒ ``manual-serve``). Leftovers are caught by - :func:`_surviving_pre_update_serve_runtimes` (#92145). + Uses only the pre-checkout inventory: re-importing ``hermes_cli.gateway`` in the + failing interpreter is what raises the original ``ImportError``. Returns + ``(candidates, skipped)``: profile → supervisor for supervised gateways the fresh + process may restart; every other inventoried runtime with an explicit reason so + nothing vanishes silently (manual gateways have no relaunch authority; + serve/dashboard have no per-profile command). Skipped serve/dashboard is NOT + unrecoverable: the fresh child's ``hermes-serve*`` systemd pass enumerates units + from systemd (the ledger reads a systemd-launched serve as ``manual-serve``); + leftovers are caught by :func:`_surviving_pre_update_serve_runtimes`. """ skip_profiles = skip_profiles or set() candidates: dict[str, str] = {} @@ -676,12 +635,10 @@ def _gateway_recovery_partition( " its supervisor" ) else: - # NOT a claim that no supervisor exists: a systemd-launched - # `hermes serve` sets neither HERMES_SPAWN nor - # HERMES_PARENT_PID, so it reads as "manual-serve". - # Unit-backed serves are recovered by the fresh child's - # systemd pass (enumerated from systemd); survivors are - # reported by _surviving_pre_update_serve_runtimes (#92145). + # NOT a claim that no supervisor exists: a systemd-launched serve + # sets neither HERMES_SPAWN nor HERMES_PARENT_PID ("manual-serve"). + # Unit-backed serves are recovered by the fresh child's systemd pass; + # survivors reported by _surviving_pre_update_serve_runtimes. reason = ( "no per-profile relaunch command reaches a serve/" "dashboard runtime; recovered by the fresh systemd" @@ -702,11 +659,9 @@ def _gateway_recovery_partition( def _warn_gateway_restart_phase_aborted(exc: BaseException, pids) -> None: """Print a recovery warning when the whole restart phase raised. - #78574: the phase was wrapped in a blanket ``except Exception`` logged at - debug level, so an early failure (e.g. importing ``hermes_cli.gateway`` - from the fresh checkout) erased every drain/restart line; the update - printed "Update complete!" and exited 0 while the gateway kept serving - pre-update modules and died on the next turn with an ImportError. + Previously a blanket debug-logged ``except Exception`` erased every drain/restart + line, so "Update complete!" exited 0 while the gateway kept serving pre-update + modules and died on the next turn with an ImportError. """ print() print(f"⚠ Update incomplete — gateway auto-restart failed: {exc}") @@ -726,25 +681,16 @@ def _drain_or_signal_gateway_for_update( drain_budget: float, label: str, ) -> bool: - """Decide how ``hermes update`` hands a running gateway over to new code. + """Three-way triage (shared by systemd and bare-process paths) for handing a + running gateway over to new code. Returns True when signalled/stopped. - Three-way triage shared by the systemd and bare-process restart paths: - - 1. **Gateway is an ancestor of this process** — deadlock break (#100179). - When ``hermes update`` runs INSIDE the gateway's process tree (the - hermes-auto-update cron job), waiting for the gateway is circular: - gateway waits on in-flight work units (#77184) → cron session waits on - ``hermes update`` → ``hermes update`` waits on the gateway. The - wedged-loop probe can't break it (the cron session posts activity every - ~180s, so it is never marked wedged) and the gateway burns the full - 1800s force-drain cap. Fire-and-forget instead: signal the restart and - return; the gateway's own restart completes once THIS process exits. - 2. **Event loop provably wedged** (#81642) — SIGUSR1 can never drain it; - bounded escalation (SIGTERM grace → SIGKILL) instead. - 3. **Live, out-of-tree gateway** — normal graceful SIGUSR1 drain, waiting - up to ``drain_budget`` (including the #86684 cron floor). - - Returns True when the gateway was signalled/stopped successfully. + 1. Gateway is an ancestor of this process (auto-update cron inside the gateway + tree): waiting is circular (gateway waits on in-flight work → cron session + waits on update → update waits on gateway); the wedged probe can't break it + (cron posts activity every ~180s) and the 1800s force-drain cap burns. So + fire-and-forget: signal restart and return; it completes once THIS process exits. + 2. Event loop provably wedged: SIGUSR1 can never drain it; bounded SIGTERM→SIGKILL. + 3. Live out-of-tree gateway: graceful SIGUSR1 drain up to ``drain_budget``. """ from hermes_cli.gateway import ( GATEWAY_LOOP_WEDGED, @@ -774,21 +720,14 @@ def _drain_or_signal_gateway_for_update( def _resolve_manage_cmd(cache: dict, scope_: str, scope_cmd_: list, svc_name_: str): - """Resolve the command prefix for manage-units operations. + """Resolve the command prefix for manage-units verbs (None ⇒ no privilege path). - Read-only systemctl calls work unprivileged, but manage-units verbs - (``reset-failed``, ``start``, ``restart``) on a *system* service trigger a - polkit auth prompt for non-root users. That prompt runs inside our captured - 10-15s subprocess — it flashes and dies before the user can answer, and the - TimeoutExpired used to be swallowed silently. - - Strategy: root → plain systemctl. Otherwise try ``sudo -n`` — a blanket - probe, then a targeted ``systemctl reset-failed`` probe so a least-privilege - sudoers entry scoped to ``systemctl ... hermes-gateway*`` also qualifies - (``reset-failed`` is an idempotent no-op we run before every privileged - restart anyway). If neither works return None: the caller must SKIP the - restart (without draining the gateway first!) and print manual steps. - ``--no-ask-password`` guarantees polkit can never hang this path. + Manage-units verbs on a *system* service trigger a polkit prompt for non-root + users, which flashes and dies inside our captured 10-15s subprocess. Root → plain + systemctl; else ``sudo -n`` blanket probe, then a targeted ``reset-failed`` probe + so a least-privilege sudoers entry scoped to hermes-gateway* qualifies (idempotent + no-op we run before every privileged restart anyway). On None the caller must SKIP + the restart (without draining first!). ``--no-ask-password`` prevents polkit hangs. """ if scope_ in cache: return cache[scope_] @@ -808,9 +747,7 @@ def _resolve_manage_cmd(cache: dict, scope_: str, scope_cmd_: list, svc_name_: s ) sudo_ok = _probe.returncode == 0 if not sudo_ok: - # Blanket sudo refused — a targeted sudoers entry - # (NOPASSWD for systemctl ... hermes-gateway*) - # may still allow the exact commands we need. + # Blanket sudo refused — a targeted NOPASSWD sudoers entry may still work. _probe = subprocess.run( sudo_cmd + ["reset-failed", svc_name_], capture_output=True, @@ -827,19 +764,16 @@ def _resolve_manage_cmd(cache: dict, scope_: str, scope_cmd_: list, svc_name_: s def _restart_systemd_gateway_units( restarted_services, failed_or_stale_units, restarted_scoped_units, drain_budget ): - """Restart every active hermes-gateway*/hermes-serve* systemd unit (user + system scope). + """Restart every active hermes-gateway*/hermes-serve* systemd unit (user + system). - Appends settled units to ``restarted_services`` (bare names) and - ``restarted_scoped_units`` (``scope/name``), failures to ``failed_or_stale_units``. - Per-unit timeouts are isolated so one wedged unit never aborts the fleet. + Settled units → ``restarted_services`` (bare) and ``restarted_scoped_units`` + (``scope/name``); failures → ``failed_or_stale_units``. Per-unit timeouts isolated. """ from hermes_cli.gateway import supports_systemd_services, _ensure_user_systemd_env _manage_cmd_cache: dict = {} - # --- Systemd services (Linux) --- - # Discover all hermes-gateway* units (default + profiles) plus - # hermes-serve* units (the Desktop app's backend, #83438). + # hermes-gateway* (default + profiles) plus hermes-serve* (Desktop backend). if supports_systemd_services(): with suppress(Exception): _ensure_user_systemd_env() @@ -871,19 +805,15 @@ def _restart_systemd_gateway_units( if check.stdout.strip() != "active": return - # Resolve how we may run manage-units verbs for this scope. - # None ⇒ no non-interactive privilege path; avoid those verbs - # entirely or polkit throws an auth prompt inside our captured - # 10-15s subprocess (it flashes and "exits directly"). - _manage_cmd = _resolve_manage_cmd(_manage_cmd_cache, + # None ⇒ no non-interactive privilege path; avoid manage-units verbs + # entirely or polkit prompts inside the captured subprocess. + _manage_cmd = _resolve_manage_cmd(_manage_cmd_cache, scope, scope_cmd, svc_name ) - # Prefer a graceful SIGUSR1 restart so in-flight agent runs - # drain instead of being SIGKILLed: the handler calls - # request_restart(via_service=True) → drain → exit, and - # Restart=always respawns the unit. hermes-serve has no such - # handler, so it skips straight to the blunt restart below. + # Graceful SIGUSR1 first so in-flight runs drain: handler → + # request_restart(via_service=True) → drain → exit, Restart=always + # respawns. hermes-serve has no handler → blunt restart below. _main_pid = 0 if _service_unit_supports_graceful_sigusr1_restart(svc_name): try: @@ -898,29 +828,21 @@ def _restart_systemd_gateway_units( _graceful_ok = False if _main_pid > 0: - # Three-way triage (#100179 ancestor / #81642 wedged / - # graceful drain), shared with the bare-process path. + # Three-way triage (ancestor / wedged / graceful drain). _graceful_ok = _drain_or_signal_gateway_for_update( _main_pid, drain_budget, svc_name ) if _graceful_ok: - # Gateway exited after a planned restart. ``Restart=always`` - # respawns the unit only after ``RestartSec`` (60s on our - # unit file) — a crash-loop guard that is dead time for a - # voluntary update restart. ``reset-failed`` + ``start`` - # skips RestartSec (we initiate the unit manually), taking - # ~1-3s on a warm box; if RestartSec already elapsed while - # draining, ``start`` is a no-op and we fall through to the - # poll below. Either way the 60s+ delay collapses to ~5s. - # - # The shortcut needs manage-units privileges; without them - # skip it — systemd's auto-restart still relaunches the - # unit after RestartSec. + # ``Restart=always`` respawns only after RestartSec (60s in our unit; + # dead time for a voluntary restart). ``reset-failed`` + ``start`` + # skips it (~1-3s); if RestartSec already elapsed, ``start`` is a + # no-op and we fall through to the poll. Needs manage-units + # privileges; without them auto-restart still fires after RestartSec. if _manage_cmd is not None: _systemctl(_manage_cmd + ["reset-failed", svc_name], timeout=10) _systemctl(_manage_cmd + ["start", svc_name], timeout=15) - # Short poll: RestartSec was bypassed, so it should be up in seconds. + # RestartSec bypassed — should be up in seconds. if _wait_for_service_active( scope_cmd, svc_name, @@ -928,9 +850,8 @@ def _restart_systemd_gateway_units( ): restarted_services.append(svc_name) return - # Passive poll: systemd's auto-restart fires after - # RestartSec regardless of privileges — the primary path - # when _manage_cmd is None, the fallback otherwise. + # Passive poll: auto-restart fires after RestartSec regardless of + # privileges — primary when _manage_cmd is None, fallback otherwise. _restart_sec = _service_restart_sec( scope_cmd, svc_name, @@ -953,17 +874,14 @@ def _restart_systemd_gateway_units( ): restarted_services.append(svc_name) return - # Exited but not respawned (older unit without - # Restart=on-failure / RestartForceExitStatus=75); fall - # through to systemctl start/restart. + # Exited but not respawned (older unit without Restart=on-failure / + # RestartForceExitStatus=75); fall through to forced restart. print( f" ⚠ {svc_name} drained but didn't relaunch — forcing restart" ) - # Forcing a restart needs manage-units privileges. Without a - # non-interactive path, systemctl would spawn a polkit prompt - # inside a captured 10-15s subprocess (flashes and dies before - # the user can answer) — skip with clear instructions. + # Forcing needs manage-units privileges; without a non-interactive path + # polkit would prompt inside the captured subprocess — skip, instruct. if _manage_cmd is None: failed_or_stale_units.append(svc_name) print( @@ -975,14 +893,12 @@ def _restart_systemd_gateway_units( ) return - # Fallback: blunt systemctl restart — only reached when the graceful - # path failed (unit missing SIGUSR1 wiring, drain exceeded the budget, - # restart-policy mismatch). Mirrors `hermes gateway restart` - # (`systemd_restart()`, PR #20949). + # Blunt restart — only when the graceful path failed (no SIGUSR1 + # wiring, drain over budget, restart-policy mismatch). Mirrors + # `hermes gateway restart` (`systemd_restart()`). restart = _systemctl_reset_and_restart(_manage_cmd, svc_name) if restart.returncode == 0: - # systemctl restart returns 0 even if the new process - # crashes immediately — verify it survived. + # restart returns 0 even if the new process crashes at once — verify. if _wait_for_service_active( scope_cmd, svc_name, @@ -990,9 +906,8 @@ def _restart_systemd_gateway_units( ): restarted_services.append(svc_name) else: - # Retry once — transient startup failures (stale module - # cache, import race) often resolve on the second try. - # Clear failed state first so the retry isn't blocked. + # Retry once — transient startup failures (stale module cache, + # import race) often clear; reset-failed so the retry isn't blocked. print( f" ⚠ {svc_name} died after restart, retrying..." ) @@ -1022,8 +937,7 @@ def _restart_systemd_gateway_units( ) def _on_unit_timeout(svc_name: str, exc: subprocess.TimeoutExpired) -> None: - # Isolate the timeout to this unit and keep going - # (#68523). A scope-wide handler used to abort every + # Isolate to this unit; a scope-wide handler used to abort every # later gateway and leave the fleet on mixed code. failed_or_stale_units.append(svc_name) print( @@ -1032,9 +946,8 @@ def _restart_systemd_gateway_units( f"continuing with remaining gateways" ) - # Qualify everything this scope appended to ``restarted_services`` - # before the next scope can add a same-named unit; ``finally`` so a - # mid-scope abort still carries the units it settled. + # Scope-qualify this scope's additions before the next scope can add a + # same-named unit; ``finally`` so a mid-scope abort keeps settled units. _scope_mark = len(restarted_services) try: _for_each_systemd_gateway_unit( @@ -1051,12 +964,8 @@ def _restart_systemd_gateway_units( @dataclass class _GatewayRestartOutcome: - """Bookkeeping the post-update gateway restart phase hands back to the update flow. - - ``restarted_services`` keeps bare unit names (the fleet probe, receipt and - operator summary all read it); ``incomplete`` means at least one gateway - may still be serving pre-update code. - """ + """Restart-phase bookkeeping. ``restarted_services`` keeps bare unit names (fleet + probe, receipt, summary read it); ``incomplete`` ⇒ a gateway may still be stale.""" incomplete: bool phase_errors: list @@ -1083,14 +992,11 @@ def _restart_manual_gateways( ): """Drain/stop every manual (non-service) gateway and print the restart summary. - Mutates ``killed_pids`` / ``relaunched_profiles`` / - ``externally_supervised_profiles`` in place; raises like the inline - code did so the caller's phase-abort recovery still fires. + Mutates the list/set args in place; raises so the caller's abort recovery fires. """ import signal as _signal - # --- Manual (non-service) gateways --- excluding PIDs of - # just-restarted services so we don't kill what systemd/launchd spawned. + # Exclude just-restarted service PIDs so we don't kill what systemd/launchd spawned. service_pids = _get_service_pids(all_profiles=True) manual_pids = find_gateway_pids( exclude_pids=service_pids, all_profiles=True @@ -1100,18 +1006,15 @@ def _restart_manual_gateways( for proc in find_profile_gateway_processes(exclude_pids=service_pids) if proc.pid in manual_pids } - # Profile gateways we could not arm a relaunch for must NOT keep - # running on pre-update modules (#88654): hand them to the unmapped - # sweep below, which stops them and lists them under "Restart manually". + # Profile gateways we couldn't arm a relaunch for must NOT keep running stale: + # the unmapped sweep below stops them and lists them under "Restart manually". unrestartable_pids = set() for pid, proc in profile_processes.items(): restart_mode = _prepare_profile_gateway_update_restart( proc.profile, pid ) if restart_mode is None: - # Previously a bare ``continue``: the gateway was neither - # relaunched nor stopped nor mentioned, so it kept serving - # from stale modules with no operator signal at all. + # A bare ``continue`` here left it serving stale modules with no signal. print( f" ⚠ {proc.profile}: could not arm an automatic " f"gateway restart for PID {pid} — stopping it instead " @@ -1119,23 +1022,18 @@ def _restart_manual_gateways( ) unrestartable_pids.add(pid) continue - # Graceful SIGUSR1 drain first (in-flight runs finish), SIGTERM - # fallback if unsupported or over budget — the watcher relaunches - # either way. Three-way triage (ancestor fire-and-forget #100179 / - # wedged escalation #81642 / normal drain) shared with the systemd - # path; the helper announces its choice first because a silent - # full-budget wait reads as a hung update (#44515). + # SIGUSR1 drain first, SIGTERM fallback if unsupported/over budget — the + # watcher relaunches either way. The helper announces its choice first + # because a silent full-budget wait reads as a hung update. drained = _drain_or_signal_gateway_for_update( pid, _drain_budget, proc.profile ) if not drained: with suppress(ProcessLookupError, PermissionError): os.kill(pid, _signal.SIGTERM) - # Wait up to 5s for the old process to exit before the watcher - # respawns. Telegram keeps the old getUpdates session alive ~30s; - # a new gateway connecting inside that window gets a 409 that - # _handle_polling_conflict() retries through, but a brief wait - # avoids that path on fast machines (watcher restarts in <1s). + # Wait ≤5s for exit: Telegram keeps the old getUpdates session ~30s; a new + # gateway inside that window gets a 409 (_handle_polling_conflict retries, + # but a brief wait avoids it on fast machines). _wait_for_gateway_exit(timeout=5.0, force_after=None) killed_pids.add(pid) if restart_mode == "external-supervisor": @@ -1178,11 +1076,9 @@ def _restart_manual_gateways( def _force_kill_stuck_gateways(killed_pids, *, find_gateway_pids, _get_service_pids): - # --- Post-restart survivor sweep (#17648) --------------------- - # Gateways that ignore SIGTERM (stuck drain, blocked I/O, zombie) - # never exit, so the 120s profile watcher never respawns and the - # user keeps hitting ImportError on stale sys.modules. Give graceful - # paths a moment, then SIGKILL remaining pre-update PIDs. + # Survivor sweep: gateways ignoring SIGTERM (stuck drain, blocked I/O, zombie) + # never exit, so the watcher never respawns and ImportErrors persist. Give + # graceful paths a moment, then SIGKILL remaining pre-update PIDs. with _best_effort('Post-restart survivor sweep failed: %s'): _time.sleep(3.0) _service_pids_after = _get_service_pids(all_profiles=True) @@ -1190,8 +1086,7 @@ def _force_kill_stuck_gateways(killed_pids, *, find_gateway_pids, _get_service_p exclude_pids=_service_pids_after, all_profiles=True, ) - # Only PIDs we already tried to kill; anything newer started - # AFTER our restart attempt and is left alone. + # Only PIDs we already tried to kill; newer ones are left alone. _stuck = [pid for pid in _surviving if pid in killed_pids] if _stuck: print() @@ -1204,15 +1099,13 @@ def _force_kill_stuck_gateways(killed_pids, *, find_gateway_pids, _get_service_p ) for pid in _stuck: with suppress(ProcessLookupError, PermissionError, OSError): - # taskkill /T /F on Windows, SIGKILL on POSIX — - # _signal.SIGKILL doesn't exist on Windows. + # taskkill /T /F on Windows (no SIGKILL there), SIGKILL on POSIX. _terminate_pid( pid, force=True, expected_start_time=_get_process_start_time(pid), ) - # Give the OS a beat to reap the processes so the - # watchers see them exit and respawn. + # Let the OS reap so watchers see the exit and respawn. _time.sleep(1.5) @@ -1233,8 +1126,7 @@ def _recover_after_restart_phase_abort( ) -> bool: """Phase-abort recovery: fresh-child restart + fail-closed verdict. - Returns the new ``incomplete`` flag; extends ``relaunched_profiles`` and - ``gateway_restart_phase_errors`` in place. + Returns the new ``incomplete`` flag; extends the list args in place. """ from hermes_cli.update_cmd import ( _abort_recovery_is_complete, @@ -1246,11 +1138,9 @@ def _recover_after_restart_phase_abort( logger.debug("Gateway restart during update failed: %s", e) gateway_restart_phase_errors.append(str(e)) - # An escaped exception means the restart output never printed. Treat - # the fleet as stale unless we can positively prove no gateway runs - # (#78574). An empty ``_surviving`` proves safety only if nothing was - # running beforehand; a pre-restart gateway that is gone now was - # stopped without a verified replacement, so ``[]`` still fails closed. + # Restart output never printed: assume stale unless provably no gateway runs. + # Empty ``_surviving`` proves safety only if nothing ran beforehand; a gone + # pre-restart gateway was stopped without verified replacement → fail closed. _surviving = _surviving_gateway_pids_after_failed_restart() _already_restarted_profiles = set(relaunched_profiles) _already_restarted_profiles.update(externally_supervised_profiles) @@ -1273,21 +1163,17 @@ def _recover_after_restart_phase_abort( ) _recovery_serve_units = _recovery_result.get("serve_units") or {} _serve_units_failed = list(_recovery_serve_units.get("failed") or []) - # Deliberately NOT merged into ``restarted_services`` (gateway-phase - # vocabulary feeding the fleet-probe expectation); serve coverage - # lives in the recovery result and receipt. A serve/dashboard runtime - # that is still the SAME pre-update process is live on old code - # (#92145, e.g. tui_gateway under `hermes serve`, unreachable by any - # `gateway restart`). Recovery may not claim success while one - # remains, and must never kill one: manual/Desktop-owned serves have - # no relaunch authority. + # Deliberately NOT merged into ``restarted_services`` (gateway vocabulary feeding + # the fleet probe); serve coverage lives in the recovery result/receipt. A + # serve/dashboard still the SAME pre-update process is live on old code + # (unreachable by `gateway restart`): recovery may not claim success while one + # remains, and must never kill one (manual/Desktop serves have no relaunch authority). _stale_runtime_rows = _surviving_pre_update_serve_runtimes( _pre_update_plan ) _recovery_result["stale_runtimes"] = _stale_runtime_rows - # Only systemd-VERIFIED outcomes may claim supervisor coverage. - # A relaunch that merely exited 0 ("relaunch_attempted") was never - # observed by the code and must not clear the incomplete flag. + # Only systemd-VERIFIED outcomes claim coverage; a relaunch that merely exited 0 + # ("relaunch_attempted") was never observed and must not clear incomplete. _recovery_verified = set(_recovery_result.get("verified") or []) if _recovery_verified: relaunched_profiles.extend( @@ -1314,9 +1200,8 @@ def _recover_after_restart_phase_abort( stale_runtime_rows=_stale_runtime_rows, ) if _recovery_complete: - # The fresh child is the recovery terminal result. Leave the - # final fleet-version matrix below as the authoritative - # read-back before the update is declared successful. + # Fresh child is terminal; the fleet-version matrix stays the authoritative + # read-back before success is declared. gateway_fleet_restart_incomplete = False elif ( _restart_phase_failure_is_incomplete( @@ -1347,43 +1232,34 @@ def _recover_after_restart_phase_abort( def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): - """Restart every running gateway (systemd, launchd, manual) so it picks up the pulled code. + """Restart every running gateway (systemd, launchd, manual) onto the pulled code. - Never raises: a phase abort runs the fresh-child recovery and fails closed - (``incomplete=True``) unless every planned gateway is verifiably covered. + Never raises: a phase abort runs fresh-child recovery and fails closed unless + every planned gateway is verifiably covered. """ from hermes_cli.update_cmd import _m, _write_gateway_update_exit_code gateway_fleet_restart_incomplete = False gateway_restart_phase_errors: list[str] = [] - # Gateways running before we touch anything. Stays empty until the probe - # is imported and we are about to stop/drain, so an early exception has - # nothing to fail closed on, while a failure after stopping a discovered - # gateway fails closed on an empty survivor probe (#78574). + # Empty until we are about to stop/drain, so an early exception has nothing to + # fail closed on, while a failure after stopping a discovered gateway fails + # closed on an empty survivor probe. _pre_restart_gateway_pids: list | None = [] - # Declared outside the try/except (never reset to None) so it is safe to - # read even if the block raises early — already-restarted units are - # forwarded to ``_finish_dashboard_update_cleanup`` (#83595). + # All bookkeeping is declared before the try (never reset to None) so abort + # recovery and fleet reconciliation can read it even if the phase raises early. restarted_services: list = [] - # Scope-qualified twin of ``restarted_services`` (``user/hermes-serve`` - # vs ``system/hermes-serve`` are different processes; abort recovery - # needs to know WHICH settled). ``restarted_services`` keeps bare names - # for the fleet probe, receipt and summary (#92145). + # Scope-qualified twin (``user/hermes-serve`` vs ``system/hermes-serve`` are + # different processes; abort recovery needs WHICH settled). Bare names stay in + # ``restarted_services`` for the fleet probe, receipt and summary. restarted_scoped_units: set = set() - # Defined up front so abort recovery and fleet reconciliation can read - # them even when the phase raises before its imports initialize them. failed_or_stale_units: list = [] relaunched_profiles: list = [] externally_supervised_profiles: list = [] - # Same treatment: the fleet version check uses killed_pids to decide - # whether to wait for settle, and the except path forwards it to the receipt. killed_pids: set = set() - # The pulled code is shared across profiles, so EVERY running gateway - # restarts. Purge stale cached Hermes modules FIRST: the import below - # loads new gateway source into this pre-update interpreter, and a - # cached sibling (cli_output, status, ...) missing a symbol the new - # source expects would ImportError and abort the whole phase. + # Purge stale cached Hermes modules FIRST: the import below loads new gateway + # source into this pre-update interpreter, and a cached sibling missing a + # symbol the new source expects would ImportError and abort the whole phase. _m()._purge_stale_hermes_modules() try: from hermes_cli.gateway import ( @@ -1395,11 +1271,9 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): _wait_for_gateway_exit, ) - # Wait budget for graceful SIGUSR1 restarts: covers both the - # ``restart_after_turn_timeout`` deferral (#77184) and the - # ``restart_drain_timeout`` inside stop(), so we don't hard-kill a - # gateway still waiting on a turn. Units without SIGUSR1 wiring - # just time out and fall back to ``systemctl restart``. + # Drain budget covers ``restart_after_turn_timeout`` and stop()'s + # ``restart_drain_timeout`` so a gateway waiting on a turn isn't hard-killed; + # units without SIGUSR1 wiring just time out into ``systemctl restart``. try: from hermes_cli.gateway import _get_restart_exit_wait_budget @@ -1412,9 +1286,8 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): relaunched_profiles = [] externally_supervised_profiles = [] - # Snapshot running gateways before any stop/drain so an empty - # survivor probe later reads as "stopped and never came back", not - # "nothing was running" (#78574). If the probe raises, None fails closed. + # Snapshot before any stop/drain so an empty survivor probe reads as "stopped + # and never came back", not "nothing was running"; None fails closed. try: _pre_restart_gateway_pids = list(find_gateway_pids(all_profiles=True)) except Exception: @@ -1424,8 +1297,7 @@ def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool): restarted_services, failed_or_stale_units, restarted_scoped_units, _drain_budget ) - # --- Launchd services (macOS): EVERY ai.hermes.gateway* LaunchAgent, - # parity with systemd (#41403). Per-label TimeoutExpired isolation inside. + # macOS: EVERY ai.hermes.gateway* LaunchAgent (systemd parity). if is_macos(): with suppress(FileNotFoundError, ImportError): _restart_macos_launchd_gateways( @@ -1508,9 +1380,8 @@ def _verify_fleet_after_update( """Post-restart verification: legacy-unit warning, dashboard cleanup, stale serve probe, fleet version matrix, plan-vs-execution reconciliation, receipt finalize. - Exits 1 (leaving ``fleet_restart_pending`` in place for the next catch-up) - when any gateway may still be serving pre-update code; otherwise clears - the pending marker. + Exits 1 (leaving ``fleet_restart_pending`` for the next catch-up) when any gateway + may still be stale; otherwise clears the marker. """ from hermes_cli.update_cmd import ( _finish_dashboard_update_cleanup, @@ -1518,8 +1389,8 @@ def _verify_fleet_after_update( _surviving_pre_update_serve_runtimes, _warn_stale_serve_runtimes, ) - # Legacy hermes.service + hermes-gateway.service SIGTERM-fight over the - # same bot token (PR #11909); warn on every update until migrated. + # Legacy hermes.service fights hermes-gateway.service over the bot token; warn + # on every update until migrated. with _best_effort('Legacy unit check during update failed: %s'): from hermes_cli.gateway import ( has_legacy_hermes_units, @@ -1542,21 +1413,17 @@ def _verify_fleet_after_update( print() print(" (add `sudo` if any are in system scope)") - # Restart a managed dashboard via systemd or stop stale manual ones - # (raw-killing a systemd-owned PID reads as a clean stop and leaves the - # Cloudflare origin dead). A failed Node refresh leaves the running - # dashboard untouched. Already-restarted units (incl. hermes-serve*, - # #83438) are forwarded so they aren't restarted twice (#83595). + # Restart a managed dashboard via systemd or stop stale manual ones (raw-killing + # a systemd-owned PID reads as clean stop and leaves the Cloudflare origin dead). + # Failed Node refresh leaves it untouched; already-restarted units aren't redone. _finish_dashboard_update_cleanup( node_failures, already_restarted_units=set(restart.restarted_services) ) - # Success-path twin of the abort-recovery probe (#100479): the restart - # phase only touches units, so a unit-less `hermes serve` keeps its - # pre-update sys.modules and its cron ticker ImportErrors. Runs AFTER - # dashboard cleanup so a respawned manual dashboard isn't a survivor. - # Rows feed the reconciliation below (survivor → exit 1); ``None`` - # means the probe failed and reconciliation stays fail-closed. + # Success-path twin of the abort-recovery probe: the restart phase only touches + # units, so a unit-less `hermes serve` keeps stale sys.modules. Runs AFTER + # dashboard cleanup so a respawned manual dashboard isn't a survivor. Rows feed + # reconciliation (survivor → exit 1); ``None`` = probe failed, stays fail-closed. _stale_serve_rows: "list | None" = None with _best_effort('Failed to check for surviving serve runtimes: %s'): _stale_serve_rows = _surviving_pre_update_serve_runtimes(_pre_update_plan) @@ -1567,10 +1434,8 @@ def _verify_fleet_after_update( print("Tip: You can now select a provider and model:") print(" hermes model # Select provider and model") - # Phase 1 (#91277): post-update fleet version verification. Compare - # every live gateway's stamped code_sha against the freshly-updated - # checkout and surface any gateway still serving pre-update code — - # instead of assuming the restart phase worked (#88654, #69754). + # Compare every live gateway's stamped code_sha against the fresh checkout + # instead of assuming the restart phase worked. _fleet_snapshot: list = [] with _best_effort('Fleet version verification failed: %s'): from hermes_cli.update_receipt import ( @@ -1578,10 +1443,9 @@ def _verify_fleet_after_update( print_fleet_version_matrix, ) - # Cross-platform "we expected fleet rows" signal (#93406). The - # old (restart.restarted_services or restart.killed_pids) condition never fires - # on Windows: the pause/resume phase populates neither list, so - # a healthy resumed gateway yielded zero rows and exit 0. + # Cross-platform "rows expected" signal: (restarted_services or killed_pids) + # never fires on Windows (pause/resume populates neither), so a healthy + # resumed gateway yielded zero rows and exit 0. _fleet_rows_expected = _m()._fleet_probe_expected_runtimes( _pre_update_plan, restart.pre_restart_gateway_pids, @@ -1589,26 +1453,21 @@ def _verify_fleet_after_update( restart.restarted_services, restart.killed_pids, ) - # Settle window (skipped when nothing was running): restarted - # gateways need time to rewrite gateway_state.json. Windows resumes - # DETACHED and may take ~10s to boot, so a single 2s sleep reported - # "no rows" (exit 1) on healthy resumes and the retry re-killed the - # new gateway. Poll a bounded window instead. + # Settle window (skipped when nothing ran): gateways need time to rewrite + # gateway_state.json. Windows resumes DETACHED (~10s boot), so a single 2s + # sleep reported "no rows" on healthy resumes; poll a bounded window instead. _fleet_snapshot = [] if _fleet_rows_expected: _fleet_deadline = _time.monotonic() + 30.0 while True: _time.sleep(2.0) - # Pass the pre-restart PID snapshot so a gateway the - # restart phase stopped WITHOUT a verified replacement - # shows as a DOWN row (exit 1) instead of silently - # producing no row at all. + # Pre-restart PIDs make a gateway stopped WITHOUT verified + # replacement a DOWN row (exit 1) instead of no row at all. _fleet_snapshot = collect_fleet_versions( pre_restart_pids=restart.pre_restart_gateway_pids ) - # A "down" row may just be a detached replacement still - # booting; keep polling until no "down" rows remain or the - # deadline passes, so a slow gateway isn't misread. + # A "down" row may be a detached replacement still booting; + # poll until no "down" rows remain or the deadline passes. if _fleet_snapshot and not any( row.get("state") == "down" for row in _fleet_snapshot ): @@ -1622,19 +1481,16 @@ def _verify_fleet_after_update( if print_fleet_version_matrix(_fleet_snapshot): restart.incomplete = True elif not _fleet_snapshot and _fleet_rows_expected: - # Zero rows although a gateway was (or may have been) live - # pre-update. collect_fleet_versions() swallows every failure, - # so an empty list is indistinguishable from a healthy fleet — - # treat it as verification failure (receipt "partial", exit 1) (#93406). + # collect_fleet_versions() swallows every failure, so zero rows with + # expected runtimes is indistinguishable from health — fail (partial, exit 1). print( "\n⚠ Fleet version check returned no rows even though" " gateway runtimes were expected — verification incomplete." ) restart.incomplete = True - # Plan-vs-execution reconciliation (#91277): every runtime the PLAN saw - # must appear in the restart bookkeeping; an unaccounted one is a - # silent miss and escalates like a STALE/DOWN row. + # Every runtime the PLAN saw must appear in restart bookkeeping; an + # unaccounted one is a silent miss and escalates like a STALE/DOWN row. _runtime_outcomes: list = [] with _best_effort('Runtime-outcome reconciliation failed: %s'): if _pre_update_plan is not None and _pre_update_plan.runtimes: @@ -1650,8 +1506,7 @@ def _verify_fleet_after_update( externally_supervised_profiles=restart.externally_supervised_profiles, killed_pids=restart.killed_pids, failed_units=restart.failed_or_stale_units, - # Serve/dashboard runtimes reconcile by incarnation - # liveness, not by the gateway's unit names (#100479). + # Serve/dashboard reconcile by incarnation liveness, not unit names. stale_serve_pids=( {row.get("pid") for row in _stale_serve_rows} if _stale_serve_rows is not None @@ -1681,28 +1536,21 @@ def _verify_fleet_after_update( logger.info("Update receipt written: %s", _receipt_path) if restart.incomplete: - # Code update itself succeeded, but at least one gateway still - # runs pre-update modules — surface that as a failed update so - # automation / operators do not treat the fleet as healthy. - # Leave ``fleet_restart_pending`` in place so the next - # ``hermes update`` still runs the catch-up restart. + # Code updated but a gateway may still run stale modules: fail so automation + # doesn't treat the fleet as healthy; leave the pending marker for catch-up. sys.exit(1) _clear_fleet_restart_pending_marker() def _restart_phase_failure_is_incomplete(surviving, pre_restart_pids) -> bool: - """Whether an escaped gateway-restart-phase exception must fail the update. + """Whether an escaped restart-phase exception must fail the update. - Fail closed unless the fleet is provably safe: ``surviving is None`` - (probe couldn't determine state, e.g. new ``hermes_cli.gateway`` no - longer imports) or non-empty -> stale. ``surviving == []`` is proof of - safety ONLY if nothing ran beforehand; a pre-restart gateway - (``pre_restart_pids`` non-empty, or ``None`` = unreadable) that is gone - now was stopped without a verified replacement (#78574). + Fail closed unless provably safe: ``surviving`` None (unprobeable) or non-empty → + stale. ``[]`` proves safety ONLY if nothing ran beforehand; a pre-restart gateway + (``pre_restart_pids`` non-empty or None) now gone was stopped unverified. """ if surviving is None or surviving: return True - # surviving == []: safe only if we know nothing was running beforehand. return pre_restart_pids is None or bool(pre_restart_pids) @@ -1715,50 +1563,22 @@ def _fleet_probe_expected_runtimes( ) -> bool: """Whether the post-update fleet probe should have produced rows. - The zero-rows fail-open (#93406): ``collect_fleet_versions()`` swallows - every probe failure via ``logger.debug()`` and ``print_fleet_version_matrix([])`` - early-returns ``False``, so an empty snapshot reads as \"healthy fleet\" and - the update exits 0. An empty snapshot is only proof-of-safety when NOTHING - says a gateway existed before the update. Any of these signals means at - least one runtime was (or may have been) live pre-update, so zero rows is - verification failure, not health: + ``collect_fleet_versions()`` swallows every failure and an empty matrix prints + as healthy, so zero rows is only proof-of-safety when NOTHING says a gateway + existed pre-update. Signals: ``restarted_services``/``killed_pids`` (POSIX phase + touched gateways); ``pre_restart_pids`` non-empty or None (unreadable, same + contract as ``_restart_phase_failure_is_incomplete``); plan inventoried ≥1 runtime. - * ``restarted_services`` / ``killed_pids`` — the POSIX restart phase - touched live gateways. - * ``pre_restart_pids`` non-empty, or ``None`` (pre-state unreadable — - cannot prove nothing was running; same contract as - ``_restart_phase_failure_is_incomplete``, #78574). - * the pre-update plan inventoried ≥1 runtime. - - ``windows_resume_token`` is deliberately EXCLUDED (#93406 residual). The - pause/resume token is bookkeeping for ``_pause_windows_gateways_for_update`` - / ``_resume_windows_gateways_after_update`` — it is not a runtime - inventory, and its entries do not correspond to rows - ``collect_fleet_versions()`` is capable of returning: - - * ``unmapped`` entries (Scheduled-Task gateways) never publish - ``gateway_state.json`` rows at all, and - * a paused profile gateway is resumed as a DETACHED relaunch that may not - republish its identity within the probe window. - - Counting the token therefore made ``_fleet_rows_expected`` True on every - Windows update that had paused a gateway, the probe's polling window ran - out with zero rows on a perfectly healthy update, and verification - reported "no rows … verification incomplete" and exited 1 after a long - silent wait. Expected-runtimes must key only on signals that map to rows - the probe can actually see; a genuinely live pre-update Windows gateway - is already covered by ``pre_restart_pids`` and the plan inventory. The - parameter stays in the signature so the call site keeps passing the token - (cheap, explicit, and the docstring is where the exclusion is explained). - - The same condition gates the 2.0s settle sleep: a freshly restarted - gateway needs the settle window to rewrite ``gateway_state.json``. - - Note this keys ONLY on zero-rows-despite-expected-runtimes. A non-empty - snapshot — including rows in ``unknown`` state — is still judged solely by - ``print_fleet_version_matrix``. + ``windows_resume_token`` is deliberately EXCLUDED: it is pause/resume bookkeeping, + not an inventory, and its entries don't map to probe rows (``unmapped`` + Scheduled-Task gateways never publish gateway_state.json; a paused profile resumes + DETACHED and may not republish within the window). Counting it made every + Windows update that paused a gateway exit 1 after a long silent wait. A live + pre-update Windows gateway is already covered by ``pre_restart_pids`` and the plan. + The same condition gates the settle sleep. Non-empty snapshots (incl. ``unknown`` + rows) are judged solely by ``print_fleet_version_matrix``. """ - del windows_resume_token # excluded on purpose — see docstring (#93406) + del windows_resume_token # excluded on purpose — see docstring if restarted_services or killed_pids: return True if pre_restart_pids is None or pre_restart_pids: @@ -1774,13 +1594,8 @@ def _wait_for_service_active( svc_name_: str, timeout: float = 10.0, ) -> bool: - """Poll ``systemctl is-active`` until the unit reports active. - - systemd's Stopped -> Started transition after a graceful exit - (or a hard restart) is not instantaneous; a one-shot check - races that window and falsely reports the unit as down. - Poll every 0.5s up to ``timeout`` seconds before giving up. - """ + """Poll ``systemctl is-active`` (0.5s) up to ``timeout``: the Stopped -> Started + transition isn't instantaneous, so a one-shot check falsely reports down.""" deadline = _time.monotonic() + max(timeout, 0.5) while True: with suppress(FileNotFoundError, subprocess.TimeoutExpired): @@ -1797,21 +1612,14 @@ def _service_restart_sec( svc_name_: str, default: float = 0.0, ) -> float: - """Read the unit's ``RestartUSec`` (RestartSec) in seconds. - - After a graceful exit-75, systemd waits ``RestartSec`` before - respawning the unit. Callers that poll for ``is-active`` - must use a timeout >= ``RestartSec`` + transition slack, or - they'll give up *during* the cooldown window and wrongly - conclude the unit didn't relaunch. - """ + """Read the unit's ``RestartUSec`` in seconds. ``is-active`` pollers must wait + >= RestartSec + slack or they give up *during* the cooldown and misreport.""" try: _show = _systemctl(scope_cmd_ + ["show", svc_name_, "--property=RestartUSec", "--value"], timeout=5) except (FileNotFoundError, subprocess.TimeoutExpired): return default raw = (_show.stdout or "").strip() - # systemd emits values like "30s", "100ms", "1min 30s", or - # "infinity". Parse conservatively; on any miss return default. + # Values like "30s", "100ms", "1min 30s", "infinity"; on any miss return default. if not raw or raw == "infinity": return default total = 0.0 diff --git a/hermes_cli/update_cmd_zip.py b/hermes_cli/update_cmd_zip.py index 71d8025d01..30c033b6c8 100644 --- a/hermes_cli/update_cmd_zip.py +++ b/hermes_cli/update_cmd_zip.py @@ -1,9 +1,7 @@ -"""ZIP-download fallback update path for ``hermes update`` (Windows with broken git): two-phase stage/commit directory swap, dirty-tree guard. +"""ZIP-download fallback for ``hermes update`` (Windows with broken git): two-phase stage/commit swap, dirty-tree guard. -Split out of ``hermes_cli/update_cmd.py``; every moved name is re-imported there, so -``hermes_cli.update_cmd.`` keeps resolving (and monkeypatching) as before. -Origin-internal helpers are imported lazily inside each function (no import cycle; -test patches on ``hermes_cli.update_cmd.`` stay effective). +Split out of ``update_cmd.py``; every name is re-imported there so ``hermes_cli.update_cmd.`` keeps +resolving/monkeypatching. Origin helpers are imported lazily per function (no cycle; test patches stay effective). """ import logging @@ -22,32 +20,21 @@ logger = logging.getLogger("hermes_cli.update_cmd") def _atomic_replace_dir(src: str, dst: str) -> None: - """Replace directory *dst* with *src* without leaving *dst* half-deleted. - - Naive ``rmtree(dst); copytree(src, dst)`` has a destructive window: a - copy that fails partway (common on the Windows ZIP path, which only runs - because file I/O is already flaky) leaves the old tree gone and nothing - in its place (#49145: ``ui-tui/`` vanished and broke the TUI). - - Now a thin alias over the two-phase helpers below (#76104); retained as - part of the ``hermes_cli.main`` re-export surface and the #49145 guard. + """Replace *dst* with *src* without a half-deleted window: naive ``rmtree; copytree`` loses the old + tree when the copy fails partway (likely here, since the ZIP path only runs when file I/O is flaky). + Thin alias over the two-phase helpers; retained for the ``hermes_cli.main`` re-export surface. """ _commit_staged_replacements([(_stage_replacement(src, dst), dst)]) def _stage_replacement(src: str, dst: str) -> str: - """Copy *src* to a sibling staging path for *dst*; return the staging path. - - Phase 1 of the two-phase replace. Handles both directories and plain - files. Touches nothing live, so a failure here leaves the whole install - untouched. + """Phase 1: copy *src* (dir or file) to a sibling staging path for *dst*; return it. + Touches nothing live, so a failure here leaves the install untouched. """ staging = f"{dst}.hermes-update-staging" backup = f"{dst}.hermes-update-old" - # A previous run may have died between "move dst aside" and "move staging - # in", leaving the backup as the ONLY copy. Restore it BEFORE clearing - # leftovers: deleting it and then failing to stage (disk exhaustion is - # likely here) would leave a hole with nothing to roll back to. + # A prior run may have died mid-swap leaving the backup as the ONLY copy. Restore it BEFORE + # clearing leftovers, else deleting it then failing to stage (disk exhaustion) leaves a hole. if not os.path.exists(dst) and os.path.exists(backup): os.rename(backup, dst) for leftover in (staging, backup): @@ -63,11 +50,8 @@ def _stage_replacement(src: str, dst: str) -> str: def _discard_staged(staged) -> None: - """Remove staging paths for entries that were never committed. - - Otherwise a phase-1 failure (typically disk exhaustion) orphans one - staging copy per processed entry — up to a full second tree — and the - advised "re-run `hermes update`" retry fails harder with less free space. + """Remove staging paths for never-committed entries; otherwise a phase-1 failure (disk exhaustion) + orphans up to a full second tree and the advised retry fails harder with less free space. """ for staging, _dst in staged: try: @@ -82,20 +66,11 @@ def _discard_staged(staged) -> None: def _commit_staged_replacements(staged) -> None: """Phase 2: swap every staged entry into place, rolling back all on failure. - ``_atomic_replace_dir`` made each *individual* swap safe, but the ZIP - update loops over ~90 top-level entries and nothing made the loop atomic - *as a whole*: a partway failure left a mixed-version tree — every file - valid, the combination unbootable (#76104; also #76091, #63717). - - Covers plain files too: the repo root holds 20 first-party modules, so a - files-only failure reproduces the same bug class. Every swap is an - ``os.rename`` onto a just-moved-aside path — atomic on POSIX and NTFS — - so a file swap can't leave a half-written module the way ``copy2`` onto - a live path can. - - Stage-all-then-swap-all shrinks the failure window from "a full tree - copy" to "N renames" and makes it recoverable: a failed swap restores - every entry already swapped, so the tree lands wholly new or wholly old. + Per-entry safety wasn't enough: a partway failure over ~90 entries left a mixed-version tree (every + file valid, combination unbootable). Covers plain files too (repo root holds 20 first-party modules). + Each swap is an ``os.rename`` onto a just-moved-aside path — atomic on POSIX and NTFS, unlike + ``copy2`` onto a live path. Stage-all-then-swap-all shrinks the failure window to N renames and a + failed swap restores every entry already swapped, so the tree lands wholly new or wholly old. """ swapped: list[tuple[str, str]] = [] # (dst, backup) in swap order; "" = absent try: @@ -118,9 +93,7 @@ def _commit_staged_replacements(staged) -> None: if backup and os.path.exists(backup): os.rename(backup, dst) except OSError as exc: - # Keep restoring the rest — a silent failure here is the one - # thing that turns a recoverable rollback into a mixed tree, - # so say so rather than swallowing it. + # Keep restoring the rest; a silent failure here turns a recoverable rollback into a mixed tree. logger.warning("rollback failed for %s: %s", dst, exc) raise # All swaps succeeded — drop the backups (best-effort, never fatal). @@ -137,13 +110,10 @@ def _zip_overlay_block_reason( ) -> Optional[str]: """Why overlaying a ZIP onto ``root`` would destroy work, or None if safe. - The ZIP path swaps every top-level entry (minus a tiny preserve set) and - deletes the backups, so uncommitted edits and untracked files are gone. - Fails closed when git status cannot run (#87304). - - ``ignore_staging_artifacts`` is for the pre-swap re-check: phase 1 leaves - ``*.hermes-update-staging`` siblings that git reports as untracked; they - are our own artifacts, and without the filter the re-check always refuses. + The swap replaces every top-level entry (minus a tiny preserve set) and deletes backups, so uncommitted + edits and untracked files are gone. Fails closed when git status cannot run. ``ignore_staging_artifacts`` + is for the pre-swap re-check: phase 1 leaves our own ``*.hermes-update-staging`` siblings that git + reports as untracked; without the filter the re-check always refuses. """ if not (root / ".git").exists(): return None @@ -151,12 +121,9 @@ def _zip_overlay_block_reason( if sys.platform == "win32": git_cmd = ["git", "-c", "windows.appendAtomically=false"] result = subprocess.run( - # -uall: a user-level ``status.showUntrackedFiles = no`` must not - # blind this guard. --ignored=matching: gitignored files are still - # USER DATA the overlay would delete (#87392); ``matching`` reports an - # ignored dir as one ``dir/`` line (cheaper, same verdict below). - # NOTE: ``--ignored=all`` is NOT a valid git mode — exits 128 and - # would fail-close every ZIP update. + # -uall: a user-level ``status.showUntrackedFiles = no`` must not blind this guard. --ignored=matching: + # gitignored files are still USER DATA the overlay would delete; ``matching`` reports an ignored dir + # as one ``dir/`` line. ``--ignored=all`` is NOT a valid git mode (exits 128, would fail-close every update). git_cmd + ["status", "--porcelain", "--untracked-files=all", "--ignored=matching"], cwd=root, capture_output=True, @@ -169,10 +136,8 @@ def _zip_overlay_block_reason( suffix = f" ({detail[0]})" if detail else "" return f"could not check the working tree{suffix}" lines = [line for line in (result.stdout or "").splitlines() if line.strip()] - # --ignored=all reports the ZIP path's own preserved entries (venv, - # node_modules are gitignored on every normal install). The swap never - # touches those top-level entries, so they must not turn into a false - # dirty-tree refusal. Everything else — including ignored files — blocks. + # Preserved entries (venv, node_modules are gitignored on every normal install) are never touched by the + # swap, so they must not cause a false refusal. Everything else — including ignored files — blocks. lines = [line for line in lines if not _is_zip_preserved_entry_status_line(line)] if ignore_staging_artifacts: lines = [ @@ -186,20 +151,16 @@ def _zip_overlay_block_reason( _ZIP_STAGING_ARTIFACT_SUFFIXES = (".hermes-update-staging", ".hermes-update-old") -# Single source of truth for the top-level entries the ZIP swap preserves — -# consumed by both the dirty-tree filter below and _update_via_zip's swap loop. +# Single source of truth for entries the ZIP swap preserves — used by the dirty-tree filter and the swap loop. _ZIP_PRESERVED_TOP_LEVEL = {"venv", "node_modules", ".git", ".env"} def _is_zip_preserved_entry_status_line(line: str) -> bool: - """True when every path on a porcelain status line sits under a top-level - entry the ZIP swap preserves. + """True when every path on a porcelain status line sits under a preserved top-level entry. - The ``" -> "`` split applies ONLY to rename/copy codes (R/C): porcelain - v1 doesn't quote plain filenames with spaces, so an ignored file named - ``venv -> node_modules`` on a ``!!``/``??`` line is ONE path — splitting - would fail-open into the destructive swap. Requiring EVERY path preserved - keeps renames out of a preserved dir (``R venv/x -> src/x``) blocking. + The ``" -> "`` split applies ONLY to R/C codes: porcelain v1 doesn't quote plain names with spaces, so + ``venv -> node_modules`` on a ``!!``/``??`` line is ONE path and splitting would fail-open. Requiring + EVERY path preserved keeps renames out of a preserved dir (``R venv/x -> src/x``) blocking. """ status, payload = (line[:2], line[3:]) if len(line) >= 3 else ("", line) is_rename = any(code in "RC" for code in status) @@ -223,7 +184,7 @@ def _is_zip_staging_artifact_status_line(line: str) -> bool: def _abort_zip_update_if_dirty_tree() -> None: - """Refuse to overlay a ZIP onto a dirty git checkout (#87304).""" + """Refuse to overlay a ZIP onto a dirty git checkout.""" from hermes_cli.update_cmd import _m reason = _zip_overlay_block_reason(_m().PROJECT_ROOT) if reason is None: @@ -240,9 +201,7 @@ def _abort_zip_update_if_dirty_tree() -> None: def _download_and_swap_zip(branch: str, zip_url: str) -> None: """Download the source ZIP for *branch* and two-phase swap it into the checkout. - - Exits the process (``sys.exit(1)``) on any failure; the two-phase replace - guarantees the install is either fully updated or fully rolled back. + ``sys.exit(1)`` on any failure; the install ends fully updated or fully rolled back. """ from hermes_cli.update_cmd import _m @@ -259,8 +218,7 @@ def _download_and_swap_zip(branch: str, zip_url: str) -> None: print("→ Extracting...") import stat as _stat with zipfile.ZipFile(zip_path, "r") as zf: - # Reject zip-slip (path traversal) AND symlink members: a - # hermes-agent source ZIP never legitimately contains symlinks, + # Reject zip-slip AND symlink members: a source ZIP never legitimately contains symlinks, # and a compromised mirror could use them to plant files anywhere. tmp_dir_real = os.path.realpath(tmp_dir) for member in zf.infolist(): @@ -272,8 +230,7 @@ def _download_and_swap_zip(branch: str, zip_url: str) -> None: raise ValueError( f"Zip-slip detected: {member.filename} escapes extraction directory" ) - # Unix mode lives in the upper 16 bits of external_attr; - # mask to the file-type bits. + # Unix mode lives in the upper 16 bits of external_attr; mask to the file-type bits. mode = (member.external_attr >> 16) & 0o170000 if _stat.S_ISLNK(mode): raise ValueError( @@ -293,12 +250,9 @@ def _download_and_swap_zip(branch: str, zip_url: str) -> None: preserve = _ZIP_PRESERVED_TOP_LEVEL entries = [i for i in os.listdir(extracted) if i not in preserve] - # Two-phase replace (#76104): phase 1 stages every entry (dirs AND - # top-level files — the repo root holds 20 first-party modules) beside - # its target; phase 2 swaps all in with same-filesystem renames and - # rolls back on any failure. One-at-a-time replacement left `agent/` - # new and `tools/` stale on interruption: all files valid, tree - # unbootable. Staging costs one extra tree copy — check space up front. + # Two-phase replace: stage every entry (dirs AND top-level files) beside its target, then swap all + # in with same-filesystem renames, rolling back on failure — one-at-a-time replacement left a + # mixed, unbootable tree on interruption. Staging costs one extra tree copy: check space up front. need = sum( os.path.getsize(os.path.join(dirpath, f)) for entry in entries @@ -309,9 +263,8 @@ def _download_and_swap_zip(branch: str, zip_url: str) -> None: for e in entries if os.path.isfile(os.path.join(extracted, e)) ) - # Swaps are renames, so only the staging copy is new: require it plus - # 20% headroom, not 2x — which would block updates on exactly the - # space-constrained machines most likely to hit this path. + # Swaps are renames, so only the staging copy is new: require it plus 20% headroom, not 2x, + # which would block updates on exactly the space-constrained machines that hit this path. required = int(need * 1.2) free = shutil.disk_usage(str(_m().PROJECT_ROOT)).free if free < required: @@ -327,10 +280,8 @@ def _download_and_swap_zip(branch: str, zip_url: str) -> None: src = os.path.join(extracted, item) dst = os.path.join(str(_m().PROJECT_ROOT), item) staged.append((_stage_replacement(src, dst), dst)) - # #70337/#87331: the source ZIP lacks apps/desktop/release/ - # (the BUILT desktop app); swapping `apps` without it deletes - # the build and breaks the shortcut. Graft the live release - # dir into the staged copy BEFORE the swap. + # The source ZIP lacks apps/desktop/release/ (the BUILT desktop app); swapping `apps` without + # it deletes the build and breaks the shortcut. Graft the live release dir in BEFORE the swap. if item == "apps": live_release = os.path.join(dst, "desktop", "release") staged_release = os.path.join( @@ -342,15 +293,13 @@ def _download_and_swap_zip(branch: str, zip_url: str) -> None: os.makedirs(os.path.dirname(staged_release), exist_ok=True) shutil.copytree(live_release, staged_release) except Exception: - # Nothing is live yet; drop the partial staging copies so a retry - # starts from the same free space this attempt did. + # Nothing is live yet; drop partial staging copies so a retry starts from the same free space. _discard_staged(staged) raise try: - # Re-check right before the swap (#87304 TOCTOU): download + - # extract + staging can take minutes, and work created meanwhile - # would be destroyed. Our own staging siblings are filtered out. + # TOCTOU re-check right before the swap: download + extract + staging can take minutes and + # work created meanwhile would be destroyed. Our own staging siblings are filtered out. recheck_reason = _zip_overlay_block_reason( _m().PROJECT_ROOT, ignore_staging_artifacts=True ) @@ -365,11 +314,9 @@ def _download_and_swap_zip(branch: str, zip_url: str) -> None: _m().sys.exit(1) _commit_staged_replacements(staged) except Exception: - # Rollback restored the swapped entries, but staging copies for - # the rest (possibly most of a tree) remain. Drop them, or the - # retry's up-front free-space check (which runs BEFORE per-entry - # leftover cleanup) fails on our litter. Safe post-rollback: - # _discard_staged skips paths that no longer exist. + # Rollback restored swapped entries but staging copies for the rest remain; drop them or the + # retry's up-front free-space check (runs BEFORE per-entry leftover cleanup) fails on our litter. + # Safe post-rollback: _discard_staged skips paths that no longer exist. _discard_staged(staged) raise update_count = len(staged) @@ -378,9 +325,7 @@ def _download_and_swap_zip(branch: str, zip_url: str) -> None: except Exception as e: print(f"✗ ZIP update failed: {e}") - # The two-phase replace either commits every entry or rolls them all - # back, so a failure here does not leave a mixed-version tree — don't - # scare the user toward a reinstall they don't need. + # Two-phase replace commits all or rolls all back, so no mixed tree here — don't push a needless reinstall. print(" Your existing install was left in place.") print( " Re-run `hermes update` to retry; if the agent won't start, " @@ -412,9 +357,8 @@ def _reinstall_python_deps_after_zip(active_tool_dependencies) -> None: if not uv_bin: uv_bin = _ensure_uv_for_termux(pip_cmd) if uv_bin: - # Same third-party UV-env isolation as the main update path (#83914): - # a user-level UV_PYTHON_INSTALL_DIR / UV_PYTHON from unrelated - # software must not steer which interpreter uv resolves here. + # Same UV-env isolation as the main update path: a user-level UV_PYTHON_INSTALL_DIR / UV_PYTHON + # from unrelated software must not steer which interpreter uv resolves here. from hermes_cli.managed_uv import managed_python_env uv_env = managed_python_env() @@ -425,9 +369,8 @@ def _reinstall_python_deps_after_zip(active_tool_dependencies) -> None: try: _m()._install_python_dependencies_with_optional_fallback([uv_bin, "pip"], env=uv_env) except _shim_quarantine_error_type() as _sqe: - # #87331: this runs inside the ZIP-fallback error handler, so the - # boundary except clause in cmd_update cannot catch it — refuse - # here with the same defer-via-marker contract. + # Runs inside the ZIP-fallback error handler, so cmd_update's boundary except cannot catch + # it — refuse here with the same defer-via-marker contract. _refuse_update_for_contended_shims(_sqe) else: # sys.executable -m pip avoids PEP 668 'externally-managed-environment' errors. @@ -442,19 +385,13 @@ def _reinstall_python_deps_after_zip(active_tool_dependencies) -> None: env=install_env, ) - # ZIP path parity: heal the active memory provider's bridge packages - # after the dependency reinstall, same as the git-pull path (#53272, - # #70636). + # Parity with git-pull path: heal the active memory provider's bridge packages after the reinstall. _m()._refresh_active_memory_provider_dependencies() def _update_via_zip(args, *, had_desktop_app_before_update: bool = False) -> bool: - """Update Hermes Agent by downloading a ZIP archive. - - Used on Windows when git file I/O is broken (antivirus, NTFS filter - drivers causing 'Invalid argument' errors on file creation). - - Returns ``False`` when a Desktop rebuild ran and failed; ``True`` otherwise. + """Update via ZIP archive; used on Windows when git file I/O is broken (antivirus / NTFS filter + drivers causing 'Invalid argument'). Returns ``False`` when a Desktop rebuild ran and failed. """ from hermes_cli.update_cmd import ( _finish_dashboard_update_cleanup, @@ -472,12 +409,10 @@ def _update_via_zip(args, *, had_desktop_app_before_update: bool = False) -> boo ) active_tool_dependencies = _m()._capture_active_tool_dependencies() - # Snapshot the pre-update version before files are replaced so the - # completion line can report the transition (prime-agent#630 port). + # Snapshot the pre-update version before files are replaced so the completion line can report it. pre_update_version = _read_project_version() - # The static GitHub archive is fine for "main" but would silently ignore - # --branch — the exact silent-divergence bug --branch was added to + # The static archive would silently ignore --branch — the exact silent-divergence bug it exists to # prevent. Refuse rather than lie. branch = _m()._resolve_update_branch(args) if branch != "main": @@ -501,21 +436,17 @@ def _update_via_zip(args, *, had_desktop_app_before_update: bool = False) -> boo _sweep_bytecode_after_update(branch) - # Reinstall Python deps: prefer .[all]; if one extra breaks, keep base - # deps and retry the remaining extras individually so working - # capabilities aren't silently stripped. Self-lock deferral (#86735): the - # code swap is committed; defer only the dependency sync when this - # process holds a native extension the sync must rewrite. + # Prefer .[all]; if one extra breaks, keep base deps and retry remaining extras individually so + # capabilities aren't silently stripped. Self-lock deferral: the code swap is committed; defer only + # the dependency sync when this process holds a native extension the sync must rewrite. _m()._abort_dependency_sync_if_self_locked() print("→ Updating Python dependencies...") _reinstall_python_deps_after_zip(active_tool_dependencies) - # Verify the tree actually imports (catches the parse-OK-but-skewed tree - # an interrupted copy leaves). Placed *after* the dependency reinstall so - # a genuinely-new third-party requirement isn't misreported as a partial - # copy. No SHA to roll back to here — surface a concrete recovery step - # instead of reporting success over a bricked install. + # Verify the tree imports (catches the parse-OK-but-skewed tree an interrupted copy leaves). Runs + # *after* the dep reinstall so a genuinely-new third-party requirement isn't misreported as a partial + # copy. No SHA to roll back to — surface a concrete recovery step instead of success over a bricked install. import_ok, failing_module, import_error = _validate_critical_modules_import( _m().PROJECT_ROOT ) @@ -539,16 +470,14 @@ def _update_via_zip(args, *, had_desktop_app_before_update: bool = False) -> boo print("→ Syncing bundled skills...") _print_bundled_skills_sync_report() - # Seed the model-catalog disk cache from the freshly-unpacked checkout - # (same rationale as the git-pull path in _cmd_update_impl). Non-fatal. + # Seed the model-catalog disk cache from the fresh checkout (same rationale as _cmd_update_impl). Non-fatal. with _best_effort('Model catalog seed during zip update failed: %s'): from hermes_cli.model_catalog import seed_cache_from_checkout if seed_cache_from_checkout(_m().PROJECT_ROOT): print(" ✓ Model catalog cache refreshed from checkout") - # Post-update state.db integrity guard (#68474, #97994): root home AND - # every sibling profile, each auto-restored from its own snapshot. + # state.db integrity guard: root home AND every sibling profile, each auto-restored from its own snapshot. with _best_effort('Post-update state.db integrity check (zip path) failed: %s'): _verify_and_restore_state_dbs_post_update() @@ -561,8 +490,7 @@ def _update_via_zip(args, *, had_desktop_app_before_update: bool = False) -> boo _print_curator_first_run_notice() with _best_effort('Curator recent-run notice failed: %s'): _print_curator_recent_run_notice() - # Don't stop a working dashboard when the Node refresh failed — see the - # git-update path for rationale (#30271). + # Don't stop a working dashboard when the Node refresh failed — see the git-update path for rationale. _finish_dashboard_update_cleanup(node_failures) with _best_effort('Update receipt finalize (zip path) failed: %s'): from hermes_cli.update_receipt import finalize_update_receipt