From 07cd3ba7c9ae51ca134a737b15c30a0f456e20d2 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:08:08 -0700 Subject: [PATCH] refactor(hermes_cli/main): extract install/update recovery cluster to main_install_repair.py --- hermes_cli/main.py | 1672 +-------------------- hermes_cli/main_install_repair.py | 1677 ++++++++++++++++++++++ tests/test_managed_runtime_resolution.py | 2 +- 3 files changed, 1730 insertions(+), 1621 deletions(-) create mode 100644 hermes_cli/main_install_repair.py diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 55cd6fa733..8b9f3e2f2f 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -873,6 +873,58 @@ from hermes_cli.model_setup_flows import ( _model_flow_ai_gateway, ) logger = logging.getLogger(__name__) +from hermes_cli.main_install_repair import ( # noqa: E402,F401 (re-exported; tests patch hermes_cli.main.) + ShimQuarantineError, + _LAZY_REFRESH_IMPORT_PROBES, + _LAZY_REFRESH_REPAIR_PACKAGES, + _PENDING_RENAME_KEY, + _PENDING_RENAME_VALUE, + _QUARANTINE_GRACE_SECONDS, + _UPDATE_REEXEC_ENV, + _cleanup_pending_shim_renames, + _cleanup_quarantined_exes, + _clear_lazy_refresh_incomplete_marker, + _clear_marker_file, + _clear_update_incomplete_marker, + _default_venv_install_target, + _detect_broken_lazy_refresh_imports, + _filter_pending_shim_renames, + _hermes_exe_shims, + _insert_python_pin, + _install_python_dependencies_with_optional_fallback, + _interpreter_scripts_dir, + _is_termux_env, + _is_uv_command, + _is_windows, + _is_windows_npm_path, + _lazy_refresh_marker_path, + _lazy_refresh_repair_specs, + _load_console_script_names, + _load_installable_optional_extras, + _norm_exe_path, + _pytest_owns_live_checkout, + _quarantine_running_hermes_exe, + _quarantine_stamp_ms, + _recover_core_update_marker_locked, + _recover_from_interrupted_install, + _recover_lazy_refresh_marker_locked, + _reexec_dependency_sync_off_windows_shim, + _repair_broken_lazy_refresh_imports, + _repair_venv_via_import_probes, + _resolve_install_target_python, + _resolve_node_runtime_npm, + _resolve_update_branch, + _restore_quarantined_exes, + _run_install_with_heartbeat, + _run_package_only_install, + _run_quarantined_install, + _update_marker_path, + _venv_scripts_dir, + _verify_console_scripts_installed, + _verify_core_dependencies_installed, + _windows_running_hermes_launcher_locked, + _windows_shim_in_process_chain, +) from hermes_cli.main_desktop import ( # noqa: E402,F401 (re-exported; tests patch hermes_cli.main.) _DESKTOP_PREVIOUS_SUFFIX, _DESKTOP_STAGING_PREFIX, @@ -4949,1615 +5001,6 @@ def _respawn_dashboard_processes(commands: list[list[str]]) -> list[list[str]]: # ========================================================================= -def _load_installable_optional_extras(group: str = "all") -> list[str]: - """Return optional extras referenced by a dependency group. - - ``group`` is usually ``all`` (desktop/server broad install) or - ``termux-all`` (Termux-compatible broad install). - """ - try: - import tomllib - - with (PROJECT_ROOT / "pyproject.toml").open("rb") as handle: - project = tomllib.load(handle).get("project", {}) - except Exception: - return [] - - optional_deps = project.get("optional-dependencies", {}) - if not isinstance(optional_deps, dict): - return [] - - refs = optional_deps.get(group, []) - referenced: list[str] = [] - for ref in refs: - if "[" in ref and "]" in ref: - name = ref.split("[", 1)[1].split("]", 1)[0] - if name in optional_deps: - referenced.append(name) - - return referenced - - -# Install-scoped breadcrumbs live next to the venv (not under $HERMES_HOME) -# because the venv is shared across profiles. -# -# ``.update-incomplete`` — generic core ``.[all]`` install was interrupted. -# Cleared only after a confirmed full dependency reinstall/recovery. -# -# ``.lazy-refresh-incomplete`` — lazy-backend refresh phase may have corrupted -# packages. Cleared only after import-probe repair confirms healthy (not when -# probes are unavailable/indeterminate). Narrow lazy probes must NEVER clear -# the generic core marker (#58004 review). -def _update_marker_path() -> Path: - return PROJECT_ROOT / ".update-incomplete" - - -def _lazy_refresh_marker_path() -> Path: - return PROJECT_ROOT / ".lazy-refresh-incomplete" - - -def _pytest_owns_live_checkout(root: Path) -> bool: - """True when running under pytest AND ``root`` is this checkout itself. - - Tests that drive update/recovery without sandboxing ``PROJECT_ROOT`` - must neither litter the live repo root with recovery breadcrumbs - (a leftover ``.lazy-refresh-incomplete`` / ``.update-incomplete`` - false-arms recovery on the developer's next real launch) nor run a real - reinstall against the executing venv. Sandboxed tests point at a - tmp_path and are unaffected (same posture as - ``managed_scope._under_pytest``).""" - return ( - "PYTEST_CURRENT_TEST" in os.environ - and root == Path(__file__).resolve().parent.parent - ) - - -def _clear_marker_file(path: Path, *, label: str) -> None: - """Remove an update-recovery breadcrumb. Never raises.""" - try: - path.unlink() - except FileNotFoundError: - pass - except OSError as exc: - logger.debug("Could not clear %s marker: %s", label, exc) - - -def _clear_update_incomplete_marker() -> None: - """Remove the interrupted core-install breadcrumb. Never raises.""" - _clear_marker_file(_update_marker_path(), label="update-incomplete") - - -def _clear_lazy_refresh_incomplete_marker() -> None: - """Remove the interrupted lazy-refresh breadcrumb. Never raises.""" - _clear_marker_file(_lazy_refresh_marker_path(), label="lazy-refresh-incomplete") - - -def _recover_from_interrupted_install() -> None: - """Finish update work left half-done by a prior ``hermes update``. - - Handles two independent breadcrumbs: - - - ``.update-incomplete`` — core ``.[all]`` install interrupted. Recovers - via full quarantined reinstall. Never cleared by the narrow lazy-refresh - import probes alone. - - ``.lazy-refresh-incomplete`` — lazy-backend refresh may have corrupted - packages. Recovers via package-only import probes; cleared only when - probes confirm healthy/repaired (indeterminate keeps the marker). - - Never raises: a recovery failure must not block launch. If it can't - self-heal it prints the manual command and leaves the relevant marker so - the next launch tries again. - - Concurrency: markers live next to the shared venv, so a gateway start - plus a CLI launch (or two profiles starting at once) can both see them. - An ``O_EXCL`` lockfile ensures only one process runs recovery; the - others skip and let the winner clear markers. - - Output: everything — our status lines AND the streamed pip/uv install - (which inherits fd 1) — is routed to stderr. Launches whose stdout is a - protocol stream (``hermes acp`` speaks JSON-RPC on stdout) must never get - install noise on stdout. - """ - if _pytest_owns_live_checkout(PROJECT_ROOT): - return - core_marker = _update_marker_path().exists() - lazy_marker = _lazy_refresh_marker_path().exists() - if not core_marker and not lazy_marker: - return - - # Skip in managed/Docker installs and on PyPI installs with no git checkout: - # those don't run the source-tree update path, so a stray marker is not ours - # to act on. Just clear it. - if not (PROJECT_ROOT / "pyproject.toml").is_file(): - _clear_update_incomplete_marker() - _clear_lazy_refresh_incomplete_marker() - return - - # Single-flight guard: atomically claim the recovery lock. If another - # process holds it, skip — it is running the same reinstall into the same - # shared venv right now. A crashed holder leaves a stale lock; break it - # after an hour (well past any realistic install) so recovery can't be - # wedged forever. - lock_path = PROJECT_ROOT / ".update-incomplete.lock" - try: - fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) - os.write(fd, f"{os.getpid()}\n".encode()) - os.close(fd) - except FileExistsError: - try: - if _time.time() - lock_path.stat().st_mtime > 3600: - lock_path.unlink() - except OSError: - pass - return - except OSError as exc: - # Couldn't create the lock (read-only fs, perms). Proceed unlocked — - # the install itself will surface the real problem. - logger.debug("Could not create install-recovery lock: %s", exc) - - saved_stdout_fd = None - saved_sys_stdout = sys.stdout - try: - # Route Python-level prints AND subprocess-inherited fd 1 to stderr - # for the duration of recovery (see docstring: ACP stdout safety). - try: - saved_stdout_fd = os.dup(1) - os.dup2(2, 1) - except OSError: - saved_stdout_fd = None - sys.stdout = sys.stderr - - if lazy_marker: - _recover_lazy_refresh_marker_locked() - - if _update_marker_path().exists(): - _recover_core_update_marker_locked() - finally: - sys.stdout = saved_sys_stdout - if saved_stdout_fd is not None: - try: - os.dup2(saved_stdout_fd, 1) - os.close(saved_stdout_fd) - except OSError: - pass - try: - lock_path.unlink() - except OSError: - pass - - -def _recover_lazy_refresh_marker_locked() -> None: - """Heal ``.lazy-refresh-incomplete`` via confirmed import-probe repair.""" - print( - "⚠ A previous lazy-backend refresh may have left the venv unhealthy — " - "running import-based package repair..." - ) - install_prefix, install_env = _default_venv_install_target() - status = _repair_venv_via_import_probes(install_prefix, env=install_env) - if status in ("healthy", "repaired"): - _clear_lazy_refresh_incomplete_marker() - print("✓ Lazy-refresh venv recovery confirmed — install is healthy again.") - return - if status == "indeterminate": - print( - " ⚠ Import probes unavailable — cannot confirm venv health. " - "Leaving `.lazy-refresh-incomplete` for the next launch." - ) - else: - print( - " ⚠ Lazy-refresh package repair incomplete. " - "Leaving `.lazy-refresh-incomplete` for the next launch." - ) - print(" Recover manually with:") - all_specs = _lazy_refresh_repair_specs( - sorted(set(_LAZY_REFRESH_REPAIR_PACKAGES.values())) - ) - print( - f" {' '.join(install_prefix)} install --force-reinstall " - + " ".join(shlex.quote(s) for s in all_specs) - ) - - -def _recover_core_update_marker_locked() -> None: - """Heal ``.update-incomplete`` via full ``.[all]`` reinstall only. - - Narrow lazy-refresh import probes are not sufficient proof that a generic - interrupted core install finished — a missing dep outside that probe set - would otherwise look healthy and clear the breadcrumb too early. - """ - print( - "⚠ A previous `hermes update` was interrupted mid-install — " - "finishing dependency installation now..." - ) - - # Windows: a normal ``hermes.exe`` launch always has the launcher as an - # ancestor. Full editable reinstall uses quarantine so the live shim can - # still be replaced. Package-only import repair may help as first aid but - # must NEVER clear this core marker on its own (#58004 review). - self_locked = _windows_running_hermes_launcher_locked() - if self_locked: - install_prefix, install_env = _default_venv_install_target() - print( - " → Running from hermes.exe; applying package-only first aid, " - "then quarantined full reinstall (core marker stays until that " - "succeeds)..." - ) - _repair_venv_via_import_probes(install_prefix, env=install_env) - - try: - from hermes_cli import _install_repair as _ir - - # ensure_uv bootstraps the installer itself when missing (the early - # pass's stdlib-only lookup cannot); keeping it here means the late - # path still self-heals a venv whose uv vanished mid-update. - from hermes_cli.managed_uv import ensure_uv - - ensure_uv() - - # Delegate the install itself to the shared stdlib executor so both - # this late path and the pre-import early pass run exactly the same - # reinstall. Called inside the same stdout→stderr redirect already - # established by _recover_from_interrupted_install, so - # run_core_install's own redirect nests harmlessly. - _ir.run_core_install(PROJECT_ROOT) - - _clear_update_incomplete_marker() - print("✓ Dependency installation recovered — your install is healthy again.") - except Exception as exc: - # Leave the marker in place so the next launch retries. Give the user - # the exact manual recovery command in the meantime. - logger.debug("Interrupted-install recovery failed: %s", exc) - print("✗ Could not auto-recover the interrupted install.") - if self_locked: - print( - " Hermes is still running from the launcher that needs " - "replacing. Close other Hermes windows, restart from a " - "different terminal, then run:" - ) - print(f' cd /d "{PROJECT_ROOT}"') - print( - f' "{sys.executable}" -m pip install -e ".[all]"' - ) - else: - print(" Recover manually with:") - print(f" cd {PROJECT_ROOT}") - print(f" {sys.executable} -m ensurepip --upgrade") - print(f" {sys.executable} -m pip install -e '.[all]'") - - -def _norm_exe_path(path) -> str: - """Case-folded resolved path, for comparing executables on Windows.""" - try: - return str(Path(path).resolve()).lower() - except OSError: - return str(path).lower() - - -def _windows_shim_in_process_chain() -> Path | None: - """The venv console shim this process runs from or under, if any. - - ``venv\\Scripts\\hermes.exe`` is a launcher that runs the interpreter with - the shim itself as its script, and that keeps the shim open — without - ``FILE_SHARE_DELETE`` — for the whole process lifetime. So every - ``hermes ...`` command holds its own shim, and an editable install run - from one can never rewrite it (#88838, #89599). - - Two independent probes, because either can come up empty. Process - ancestry finds the launcher when it is a separate parent process, but - needs psutil. This process's own launch paths (``sys.argv[0]``, - ``__main__.__file__``, the module spec origin) cover the rest — the - runpy/zipapp launch puts ``\\__main__.py`` there, which a plain - argv[0] check misses. - - Candidates are intersected with the project venv's own shims, so a - ``hermes.exe`` belonging to some other install never matches. - """ - if not _is_windows(): - return None - scripts_dir = _venv_scripts_dir() - if scripts_dir is None: - return None - shims = {_norm_exe_path(shim): shim for shim in _hermes_exe_shims(scripts_dir)} - if not shims: - return None - - def _match(candidate) -> Path | None: - path = Path(candidate) - if path.name.lower() == "__main__.py": - path = path.parent - return shims.get(_norm_exe_path(path)) - - candidates: list[str] = list(sys.argv[:1]) - main_mod = sys.modules.get("__main__") - for attr in (getattr(main_mod, "__file__", None), - getattr(getattr(main_mod, "__spec__", None), "origin", None)): - if attr: - candidates.append(attr) - for candidate in candidates: - matched = _match(candidate) - if matched is not None: - return matched - - try: - import psutil - - me = psutil.Process() - for proc in [me] + list(me.parents()): - try: - matched = _match(proc.exe()) - except Exception: - continue - if matched is not None: - return matched - except Exception: - return None - return None - - -def _windows_running_hermes_launcher_locked() -> bool: - """True when a venv ``hermes*.exe`` shim is this process or an ancestor. - - Best-effort: returns False when psutil is unavailable or inspection fails. - """ - return _windows_shim_in_process_chain() is not None - - -# Set on the re-exec'd child so it can never spawn another one. -_UPDATE_REEXEC_ENV = "HERMES_UPDATE_REEXEC" - - -def _reexec_dependency_sync_off_windows_shim() -> bool: - """Hand the dependency sync to the venv interpreter, off the console shim. - - Returns True when a child was spawned and the caller must exit at once, - releasing the shim before the child reaches ``pip install -e .``. Returns - False to continue the sync in-process. - - Called at the dependency-sync boundary, NOT at the top of the command — - the same placement rule as the native-module deferral beside it, and for - the same reason (#86735): a hand-off that fires before the fetch detaches - every run, including the ``Already up to date!`` no-op that never touches - the venv at all, and it takes the interactive prompts with it. By the time - we reach here the code swap is done and every question — stash, branch - switch, config migration — has already been asked and answered in the - user's own console. Only the venv rewrite is left, and that is the single - step that genuinely cannot run from inside the shim. - - ``venv\\Scripts\\hermes.exe`` is a launcher that runs the interpreter with - the shim as its script and holds it open without ``FILE_SHARE_DELETE`` for - the whole command, so the quarantine rename is refused and uv fails to - replace it with os error 32 (#88838, #89599). - - A child is required, and waiting on it cannot work: this process holds the - handle the child needs released, so a parent that waits deadlocks against - the work it is waiting for. Windows has no exec to escape with either. - The shell therefore returns while the install runs on; the child keeps the - console and prints its own result, and ``--gateway`` writes the true exit - code to ``.update_exit_code`` for the gateway watcher. - - The child re-runs ``hermes update``, so the whole remaining flow — the - dependency sync and the node/web/lazy-refresh tail behind it — still - happens exactly once. ``_UPDATE_REEXEC_ENV`` marks it so it cannot spawn - another child, and so the "already up to date" early return does not - swallow the sync it was spawned to perform (the checkout is current by - now; that is the point). - - The caller has already written ``.update-incomplete``, so a child that - dies mid-install is finished by the next launch's recovery instead of - leaving a half-synced venv. Anything that stops the hand-off (no venv - python, spawn refused) returns False and syncs in-process, where the - pre-existing os-error-32 path and its marker recovery still apply. - """ - if os.environ.get(_UPDATE_REEXEC_ENV) == "1": - return False - shim = _windows_shim_in_process_chain() - if shim is None: - return False - - from hermes_constants import venv_python_path - - python_exe = venv_python_path(shim.parent.parent, windows=True) - cmd = [str(python_exe), "-m", "hermes_cli.main", *sys.argv[1:]] - if python_exe.is_file(): - try: - subprocess.Popen( - cmd, - env={**os.environ, _UPDATE_REEXEC_ENV: "1"}, - stdin=subprocess.DEVNULL, - ) - print( - f"→ Windows: {shim.name} cannot replace itself while it runs; " - "finishing the dependency install under the venv Python." - ) - print( - " The code update is already applied. The install continues " - "below and this shell returns right away." - ) - return True - except OSError as exc: - logger.debug("Dependency-sync hand-off via %s failed: %s", python_exe, exc) - print(f" ⚠ Could not hand the dependency install off {shim.name}.") - print(" Continuing in-process; if it cannot replace the shim, run:") - print(f" {subprocess.list2cmdline(cmd)}") - return False - - -def _default_venv_install_target() -> tuple[list[str], dict[str, str] | None]: - """Return ``(install_cmd_prefix, env)`` for the project venv when possible.""" - try: - from hermes_cli.managed_uv import ensure_uv - - uv_bin = ensure_uv() - except Exception: - uv_bin = None - if uv_bin: - from hermes_constants import project_venv_dir - - venv_dir = project_venv_dir(PROJECT_ROOT) or PROJECT_ROOT / "venv" - env = {**os.environ, "VIRTUAL_ENV": str(venv_dir)} - if _is_termux_env(env): - env.pop("PYTHONPATH", None) - env.pop("PYTHONHOME", None) - return [uv_bin, "pip"], env - return [sys.executable, "-m", "pip"], None - - -def _run_install_with_heartbeat( - cmd: list[str], - *, - env: dict[str, str] | None = None, - heartbeat_interval_seconds: int = 30, -) -> None: - """Run dependency install command with periodic heartbeat output. - - Some resolvers/build backends (especially when compiling Rust/C extensions) - can stay quiet for minutes. Emit a simple elapsed-time heartbeat so users - know ``hermes update`` is still progressing even if pip/uv itself is silent. - """ - done = threading.Event() - start = _time.time() - - def _heartbeat() -> None: - # Wait first, then print, so short installs don't emit noise. - while not done.wait(heartbeat_interval_seconds): - elapsed = int(_time.time() - start) - print( - f" … still installing dependencies ({elapsed}s elapsed)" - " — compiling Rust/C extensions can take several minutes", - flush=True, - ) - - t = threading.Thread(target=_heartbeat, daemon=True) - t.start() - try: - subprocess.run( - cmd, - cwd=PROJECT_ROOT, - check=True, - env=env, - ) - finally: - done.set() - t.join(timeout=0.2) - - -def _is_windows() -> bool: - return sys.platform == "win32" - - -def _venv_scripts_dir() -> Path | None: - """Return the venv Scripts directory if we're running inside the project venv.""" - from hermes_constants import project_venv_dir, venv_bin_dir - - venv_dir = project_venv_dir(PROJECT_ROOT) - if venv_dir is None: - return None - - scripts = venv_bin_dir(venv_dir, windows=_is_windows()) - return scripts if scripts.is_dir() else None - - -def _hermes_exe_shims(scripts_dir: Path) -> list[Path]: - """Entry-point shims that uv may try to rewrite during ``pip install -e .``. - - On Windows these are .exe launchers generated by setuptools/uv. On POSIX - they're regular Python scripts which can be replaced atomically — no - self-replacement hazard exists outside Windows. - """ - if not _is_windows(): - return [] - - names = set(_load_console_script_names()) or {"hermes", "hermes-agent", "hermes-acp"} - # The gateway shim is not a [project.scripts] entry point, but older - # update/install paths still rewrite and quarantine it. - names.add("hermes-gateway") - return [scripts_dir / f"{name}.exe" for name in sorted(names)] - - -def _quarantine_running_hermes_exe( - scripts_dir: Path, *, max_attempts: int = 4, - failed_out: list[str] | None = None, -) -> list[tuple[Path, Path]]: - """Pre-empt Windows file lock on the running ``hermes.exe``. - - Windows allows RENAMING a mapped/running executable (the kernel tracks the - file by handle, not path), but blocks DELETE/REPLACE while it's loaded. uv - needs to overwrite the entry-point shims during ``pip install -e .``; - when ``hermes update`` runs, ``hermes.exe`` IS the live process, and uv - fails with ``Access is denied. (os error 5)``. - - We rename live shims to ``hermes.exe.old.`` first. uv then writes - fresh shims at the original paths. The ``.old`` files are cleaned up on - the next hermes invocation by ``_cleanup_quarantined_exes``. - - Rename can still fail when *another* process has opened the .exe without - ``FILE_SHARE_DELETE`` — typically AV real-time scanners with transient - handles (recovers in <1s), or the Hermes Desktop backend child process - (won't recover until the user closes it). We mitigate: - - 1. Retry up to ``max_attempts`` times with exponential backoff - (100/250/500/1000 ms). Handles the AV-scanner case. - 2. If all retries fail, print a clear warning naming the most likely - culprit (running Hermes Desktop / gateway / REPL). - - The updater's own launcher is no longer one of those culprits: an update - started from ``hermes.exe`` re-runs itself under the venv Python before - reaching here (``_reexec_dependency_sync_off_windows_shim``). - - Returns the list of (original, quarantined) pairs so the caller can roll - back if the install itself fails before uv writes a replacement. - - ``failed_out``: when provided, the names of shims whose rename failed on - every attempt are appended — callers that must not mutate a contended - venv (the update dependency sync, #87331) check it and refuse instead of - letting the install run into a half-broken state. - """ - moved: list[tuple[Path, Path]] = [] - if not _is_windows(): - return moved - - import time - - stamp = int(time.time() * 1000) - # Backoff schedule: first attempt is immediate, subsequent ones sleep. - # 100ms / 250ms / 500ms covers the typical AV scanner re-scan window. - backoff_ms = [0, 100, 250, 500, 1000] - attempts = max(1, min(max_attempts, len(backoff_ms))) - - for shim in _hermes_exe_shims(scripts_dir): - if not shim.exists(): - continue - target = shim.with_suffix(shim.suffix + f".old.{stamp}") - - last_exc: OSError | None = None - for attempt in range(attempts): - delay = backoff_ms[attempt] / 1000.0 - if delay: - time.sleep(delay) - try: - shim.rename(target) - moved.append((shim, target)) - last_exc = None - break - except OSError as e: - last_exc = e - continue - - if last_exc is None: - continue - - # Every rename failed. Deferring one to next boot via - # MOVEFILE_DELAY_UNTIL_REBOOT used to be the fallback here, but it - # cannot help: it needs elevation we don't have, and when it does - # land it frees nothing for the install running right now while - # queueing an operation that will move a later, freshly repaired shim - # aside at next boot. Report and let uv try its luck instead — - # sometimes its own retry handling pulls through. - print( - f" ⚠ Could not quarantine {shim.name} ({last_exc.__class__.__name__}: " - f"another process is holding it open)." - ) - print( - " Close Hermes Desktop, exit other `hermes` REPLs, stop the " - "gateway, or pause AV scanning, then re-run `hermes update`." - ) - if failed_out is not None: - failed_out.append(shim.name) - - return moved - - -_PENDING_RENAME_KEY = r"SYSTEM\CurrentControlSet\Control\Session Manager" -_PENDING_RENAME_VALUE = "PendingFileRenameOperations" - - -def _filter_pending_shim_renames( - entries: list[str], shims: list[Path] -) -> tuple[list[str], int]: - """Drop shim-quarantine pairs from a PendingFileRenameOperations value. - - The value is a flat REG_MULTI_SZ of (source, target) pairs, and other - installers share it, so only pairs matching our own - ```` -> ``.old.`` naming are removed. Returns the - entries to keep and how many pairs were dropped. - """ - import ntpath - - def _norm(value: str) -> str: - path = str(value).lstrip("!") - if path.startswith("\\??\\"): - path = path[4:] - return ntpath.normcase(ntpath.normpath(path)) - - shim_paths = {_norm(str(shim)) for shim in shims} - kept: list[str] = [] - removed = 0 - for index in range(0, len(entries) - 1, 2): - source, target = entries[index], entries[index + 1] - source_norm = _norm(source) - if source_norm in shim_paths and _norm(target).startswith(f"{source_norm}.old."): - removed += 1 - else: - kept.extend((source, target)) - if len(entries) % 2: - kept.append(entries[-1]) - return kept, removed - - -def _cleanup_pending_shim_renames(scripts_dir: Path) -> int: - """Drop reboot renames older Hermes versions queued for our shims. - - Hermes used to fall back to ``MoveFileExW(MOVEFILE_DELAY_UNTIL_REBOOT)`` - when the quarantine rename failed. Those entries outlive the update that - queued them, so at the next boot they move away whatever now sits at the - shim path — including a shim a later repair just wrote. Needs elevation - to remove (same as it needed to create); a no-op otherwise. - """ - if not _is_windows(): - return 0 - try: - import winreg - - with winreg.OpenKey( - winreg.HKEY_LOCAL_MACHINE, - _PENDING_RENAME_KEY, - 0, - winreg.KEY_QUERY_VALUE | winreg.KEY_SET_VALUE, - ) as key: - entries, value_type = winreg.QueryValueEx(key, _PENDING_RENAME_VALUE) - if value_type != winreg.REG_MULTI_SZ or not isinstance(entries, list): - return 0 - kept, removed = _filter_pending_shim_renames( - entries, _hermes_exe_shims(scripts_dir) - ) - if not removed: - return 0 - if kept: - winreg.SetValueEx(key, _PENDING_RENAME_VALUE, 0, winreg.REG_MULTI_SZ, kept) - else: - winreg.DeleteValue(key, _PENDING_RENAME_VALUE) - return removed - except (OSError, ValueError): - return 0 - - -def _restore_quarantined_exes(moved: list[tuple[Path, Path]]) -> None: - """Roll back ``_quarantine_running_hermes_exe`` if uv didn't write replacements. - - This is the safety-critical direction. A failed *quarantine* only aborts an - update; a failed *restore* leaves the install with no ``hermes`` on PATH, - and therefore no way to run the command that would repair it (#75584). The - outbound rename already retries a lock, so this one must too rather than - swallow the first ``OSError`` in silence. - - Delegates to the stdlib-only helper that the early-recovery copy in - ``_install_repair`` also uses, so the two cannot drift apart. - """ - _early_recovery_mod.restore_quarantined_shims(moved) - - -class ShimQuarantineError(RuntimeError): - """A live ``hermes*.exe`` shim could not be renamed aside (#87331). - - Raised by :func:`_run_quarantined_install` in ``strict_quarantine`` mode - BEFORE the install command runs. A shim that cannot even be renamed means - another process holds the venv hard enough that the dependency sync would - die partway and strand the install half-updated — the update must refuse, - not warn-and-continue. - """ - - def __init__(self, failed_shims: list[str]): - self.failed_shims = list(failed_shims) - super().__init__( - "could not quarantine live shim(s): " + ", ".join(self.failed_shims) - ) - - -def _run_quarantined_install( - cmd: list[str], - *, - env: dict[str, str] | None = None, - scripts_dir: Path | None = None, - strict_quarantine: bool = False, -) -> None: - """Run an editable install, quarantining the running ``hermes.exe`` first. - - Any ``pip install -e .`` (or ``--reinstall``) rewrites the entry-point - shims, and on Windows the live ``hermes.exe`` is the running process — - pip can neither delete nor overwrite it, so without quarantine the shim - is left missing and ``hermes`` drops off PATH. This wraps - :func:`_run_install_with_heartbeat` with the same rename-out-of-the-way / - restore-on-failure dance that the primary install path uses, so EVERY - install that touches the shims is protected — including the - verification-repair reinstalls in - :func:`_verify_core_dependencies_installed`, which previously called - ``_run_install_with_heartbeat`` directly and bypassed quarantine. - - ``strict_quarantine=True`` (the update dependency sync, #87331): a shim - whose rename failed every retry means a process is holding the venv - without ``FILE_SHARE_DELETE`` — the install WILL hit the same lock on - .pyd files and strand the venv between versions. Roll the successful - renames back and raise :class:`ShimQuarantineError` WITHOUT running the - install. Non-strict callers (post-sync entry-point repair) keep the old - warn-and-try behavior: their venv is already mutated, so refusing buys - nothing. - - Off-Windows (``scripts_dir is None``) this is a thin pass-through. - """ - moved: list[tuple[Path, Path]] = [] - failed: list[str] = [] - if scripts_dir is not None: - moved = _quarantine_running_hermes_exe(scripts_dir, failed_out=failed) - if strict_quarantine and failed: - _restore_quarantined_exes(moved) - raise ShimQuarantineError(failed) - try: - _run_install_with_heartbeat(cmd, env=env) - finally: - # Restore shims when the installer didn't write replacements — on - # FAILURE (install died before the entry-points step) and on SUCCESS - # too: uv audits an already-satisfied editable install as a no-op and - # rewrites no entry points, which would otherwise leave the shims - # quarantined aside and `hermes` missing from PATH after a green - # install (#75584). _restore_quarantined_exes skips any shim the - # installer actually replaced, so this never clobbers fresh output. - # Errors are not swallowed — the finally re-raises whatever escaped. - if scripts_dir is not None: - _restore_quarantined_exes(moved) - - -# A quarantine file younger than this may belong to an update running RIGHT -# NOW in another process, whose restore step still needs it. Deleting one -# mid-flight destroys the only copy of that shim. -_QUARANTINE_GRACE_SECONDS = 15 * 60 - - -def _quarantine_stamp_ms(stale: Path) -> int | None: - """The ``.old.`` stamp in a quarantine filename, or ``None``. - - ``None`` means the name was not produced by - :func:`_quarantine_running_hermes_exe`. We neither rescue nor delete those: - the sweep should not destroy files whose provenance it cannot establish, and - they are not ours to put back. - - Parsed from the NAME rather than ``st_mtime`` because ``rename`` preserves - the original shim's mtime, which records when uv wrote the shim — days - earlier, in general — not when it was quarantined. - """ - try: - return int(stale.name.rsplit(".old.", 1)[1]) - except (IndexError, ValueError): - return None - - -def _cleanup_quarantined_exes(scripts_dir: Path | None = None) -> None: - """Sweep — and where necessary RESCUE — ``hermes.exe.old.*`` from updates. - - Called early on every hermes invocation. Two cases the old unconditional - ``unlink()`` got wrong, both ending with ``hermes`` gone from PATH: - - 1. **Orphan rescue.** If ``hermes.exe`` is missing while - ``hermes.exe.old.*`` is present, that .old file is the ONLY surviving - copy of the shim — an update died, or its restore failed, between - the rename and uv writing a replacement (#75584). Deleting it converts a - one-rename recovery into a full reinstall. Put it back instead, through - the same retry-and-report helper the update-time restore uses. - 2. **Concurrency.** A fresh quarantine file may belong to an update in - flight in another process (the desktop update button racing a shell - ``hermes update`` does exactly this). Leave anything inside the grace - window alone; a later run sweeps it. - - Silent no-op on non-Windows, when there is nothing to do, or on - file-locked / permission errors. - """ - if not _is_windows(): - return - if scripts_dir is None: - scripts_dir = _venv_scripts_dir() - if scripts_dir is None: - return - _cleanup_pending_shim_renames(scripts_dir) - - now = _time.time() - - try: - candidates = [ - (stamp, stale) - for stale, stamp in ( - (p, _quarantine_stamp_ms(p)) for p in scripts_dir.glob("*.exe.old.*") - ) - if stamp is not None - ] - except OSError: - return - - # Newest first by PARSED stamp. Sorting the raw filenames lexicographically - # only tracks recency while every stamp shares a digit width: a stray - # ``.old.999`` sorts above a 13-digit epoch-ms stamp and would be the copy - # rescued onto the live shim name. - candidates.sort(key=lambda pair: pair[0], reverse=True) - - for stamp, stale in candidates: - try: - original = stale.with_name(stale.name.rsplit(".old.", 1)[0]) - - if not original.exists(): - # Orphan rescue: this is the last copy of the shim, so it gets - # the retry ladder and the recovery message, not a bare rename. - _early_recovery_mod.restore_quarantined_shims([(original, stale)]) - continue - - if now - stamp / 1000.0 < _QUARANTINE_GRACE_SECONDS: - continue # may be a live quarantine from a concurrent update - - stale.unlink() - except OSError: - pass # still locked or in use — try again next run - - -# Import probes for venv corruption after a failed lazy ``uv pip install``. -# Metadata can look fine while ``.py`` files were removed mid-install (#57828). -# Canonical tables live in the stdlib-only ``_early_recovery`` module (which -# also probes/repairs BEFORE this module's third-party imports can run) so the -# early and full recovery layers can never drift apart. -_LAZY_REFRESH_IMPORT_PROBES: tuple[tuple[str, str], ...] = ( - _early_recovery_mod.LAZY_REFRESH_IMPORT_PROBES -) - -_LAZY_REFRESH_REPAIR_PACKAGES: dict[str, str] = ( - _early_recovery_mod.LAZY_REFRESH_REPAIR_PACKAGES -) - - -def _run_package_only_install( - cmd: list[str], - *, - env: dict[str, str] | None = None, -) -> None: - """Run a package-only pip/uv install without quarantining entry-point shims. - - ``pip install --upgrade pip`` and ``--force-reinstall `` do not - rewrite ``hermes.exe``. The editable-install quarantine path would rename - shims without uv recreating them on Windows (#57828). - """ - _run_install_with_heartbeat(cmd, env=env) - - -def _lazy_refresh_repair_specs(packages: list[str]) -> list[str]: - """Map repair package names to their declared pin specs in pyproject.toml.""" - try: - import tomllib # Python 3.11+ - except ImportError: # pragma: no cover - return packages - - pyproject = PROJECT_ROOT / "pyproject.toml" - if not pyproject.is_file(): - return packages - - try: - with open(pyproject, "rb") as f: - raw_deps = tomllib.load(f).get("project", {}).get("dependencies", []) or [] - except Exception as exc: - logger.debug("lazy refresh repair spec lookup failed: %s", exc) - return packages - - name_to_spec: dict[str, str] = {} - try: - from packaging.requirements import Requirement # type: ignore - - for spec in raw_deps: - try: - req = Requirement(spec) - name_to_spec[req.name.lower()] = spec.split(";", 1)[0].strip() - except Exception: - continue - except Exception: - for spec in raw_deps: - head = spec.split(";", 1)[0].strip() - bare = head - for op in ("==", ">=", "<=", "~=", ">", "<", "!="): - if op in bare: - bare = bare.split(op, 1)[0] - break - key = bare.strip().split("[", 1)[0].strip().lower() - if key: - name_to_spec[key] = head - - return [name_to_spec.get(pkg.lower(), pkg) for pkg in packages] - - -def _detect_broken_lazy_refresh_imports( - install_cmd_prefix: list[str], - *, - env: dict[str, str] | None = None, -) -> list[str] | None: - """Probe lazy-refresh packages via real imports. - - Returns: - - ``[]`` when probes ran and every package imported cleanly - - ``[dist, ...]`` when probes ran and some packages failed - - ``None`` when the probe could not run (missing venv Python, subprocess - failure, non-zero probe exit) — this is *indeterminate*, not healthy - """ - venv_python = _resolve_install_target_python(install_cmd_prefix, env) - if venv_python is None: - return None - - probe_lines = "\n".join( - f" ({mod!r}, {attr!r})," for mod, attr in _LAZY_REFRESH_IMPORT_PROBES - ) - check_script = ( - "import os\n" - "import sys\n" - "probes = [\n" - f"{probe_lines}\n" - "]\n" - "broken = []\n" - "for mod, attr in probes:\n" - " try:\n" - " imported = __import__(mod)\n" - " if not hasattr(imported, attr):\n" - " broken.append(mod)\n" - " elif mod == 'certifi':\n" - " # The module can import cleanly while cacert.pem is\n" - " # missing/corrupt (brew Python upgrade, interrupted venv\n" - " # rebuild) - every TLS call then fails (#29866).\n" - " bundle = imported.where()\n" - " if not os.path.isfile(bundle) or os.path.getsize(bundle) < 1024:\n" - " broken.append(mod)\n" - " except Exception:\n" - " broken.append(mod)\n" - "print('\\n'.join(broken))\n" - ) - try: - result = subprocess.run( - [str(venv_python), "-c", check_script], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=False, - env=env, - ) - except Exception as exc: - logger.debug("lazy refresh import probe failed: %s", exc) - return None - - if result.returncode != 0: - logger.debug( - "lazy refresh import probe exited %s: %s", - result.returncode, - (result.stderr or "")[:200], - ) - return None - - broken_modules = [ - line.strip() for line in result.stdout.splitlines() if line.strip() - ] - packages: list[str] = [] - seen: set[str] = set() - for mod in broken_modules: - pkg = _LAZY_REFRESH_REPAIR_PACKAGES.get(mod) - if pkg and pkg not in seen: - seen.add(pkg) - packages.append(pkg) - return packages - - -def _repair_broken_lazy_refresh_imports( - install_cmd_prefix: list[str], - packages: list[str], - *, - env: dict[str, str] | None = None, -) -> bool: - """Force-reinstall ``packages`` and re-probe imports. Never raises.""" - if not packages: - return True - - specs = _lazy_refresh_repair_specs(packages) - try: - _run_package_only_install( - install_cmd_prefix + ["install", "--force-reinstall", *specs], - env=env, - ) - except subprocess.CalledProcessError as exc: - logger.warning("lazy refresh venv repair failed: %s", exc) - return False - - after = _detect_broken_lazy_refresh_imports(install_cmd_prefix, env=env) - # Indeterminate re-probe is not confirmed success. - return after == [] - - -def _repair_venv_via_import_probes( - install_cmd_prefix: list[str], - *, - env: dict[str, str] | None = None, -) -> str: - """Probe imports and force-reinstall any broken lazy-refresh packages. - - Uses real ``import`` checks (not distribution metadata) so a venv where - METADATA remains but ``.py`` files were wiped mid-install is still - detected (#57828). Package-only reinstall — never rewrites ``hermes.exe``. - - Never raises. Returns one of: - - ``"healthy"`` — probes ran and found nothing broken - - ``"repaired"`` — probes found breakage and force-reinstall confirmed clean - - ``"failed"`` — probes found breakage and repair did not confirm clean - - ``"indeterminate"`` — probes could not run; do NOT treat as healthy - """ - broken = _detect_broken_lazy_refresh_imports(install_cmd_prefix, env=env) - if broken is None: - print( - " ⚠ Import probes unavailable — cannot confirm venv package health." - ) - return "indeterminate" - if not broken: - return "healthy" - print( - " → Detected corrupted venv packages via import probes: " - f"{', '.join(broken)}; repairing..." - ) - if _repair_broken_lazy_refresh_imports( - install_cmd_prefix, broken, env=env - ): - print(" ✓ Venv repair succeeded") - return "repaired" - manual = " ".join( - shlex.quote(s) for s in _lazy_refresh_repair_specs(broken) - ) - print(" ⚠ Venv repair incomplete. Run manually, then `hermes update`:") - print( - f" {' '.join(install_cmd_prefix)} install --force-reinstall {manual}" - ) - return "failed" - - -def _is_uv_command(install_cmd_prefix: list[str]) -> bool: - """True when the install command is a uv/uvx invocation. - - Handles a bare uv binary (``uv`` / ``uvx``, any extension), a path to - one, and ``python -m uv`` / ``python -m uvx`` — the naive basename check - misses the module form and launcher wrappers whose name does not contain - "uv". - """ - if not install_cmd_prefix: - return False - first = str(install_cmd_prefix[0]).lower() - if "uv" in Path(first).name: - return True - # python -m uv / python -m uvx - if len(install_cmd_prefix) >= 3 and first.endswith(("python", "python.exe")): - return install_cmd_prefix[1] == "-m" and install_cmd_prefix[2] in ( - "uv", - "uvx", - ) - return False - - -def _insert_python_pin(args: list[str]) -> list[str]: - """Insert ``--python `` into a uv command line. - - If the caller already passed ``--python``, its value wins (uv's last-wins - semantics are ambiguous; the explicit caller intent should not be - overridden by the fallback pin). - """ - if "--python" in args: - return args - return [args[0], "--python", str(sys.executable), *args[1:]] - - -def _interpreter_scripts_dir() -> Path | None: - """Scripts/bin directory of the running interpreter (sys.executable). - - Used when pinning an install to ``sys.executable`` on a site-packages - install where ``PROJECT_ROOT / "venv"`` does not exist: the entry-point - shims uv rewrites live next to the interpreter, not under a project venv. - Layout comes from the canonical ``venv_bin_dir`` helper (#76105 — - hand-rolling Scripts/bin is lint-tested against). - """ - from hermes_constants import venv_bin_dir - - exe = Path(sys.executable) - # sys.executable lives IN the bin/Scripts dir; its parent.parent is the - # env root venv_bin_dir derives from. - cand = venv_bin_dir(exe.parent.parent, windows=_is_windows()) - if cand.is_dir(): - return cand - return exe.parent if exe.parent.is_dir() else None - - -def _install_python_dependencies_with_optional_fallback( - install_cmd_prefix: list[str], - *, - env: dict[str, str] | None = None, - group: str = "all", -) -> None: - """Install base deps plus as many optional extras as the environment supports. - - By default this targets ``.[all]``; Termux callers can pass - ``group='termux-all'`` to use the curated Android-compatible profile. - - On Windows, pre-renames live ``hermes.exe`` / ``hermes-gateway.exe`` shims - in the venv Scripts dir before each install attempt so uv can write fresh - copies (Windows blocks REPLACE on a running .exe but allows RENAME). See - ``_quarantine_running_hermes_exe`` for the rationale. - - When ``env`` carries a ``VIRTUAL_ENV`` that does not exist (a pip / - site-packages install whose ``PROJECT_ROOT`` is the interpreter's - ``site-packages`` directory, where ``PROJECT_ROOT / "venv"`` is never - created), ``uv pip`` fails with ``Failed to inspect Python interpreter from - active virtual environment`` before doing any work. Pin the install at the - running interpreter instead so the update/recovery path succeeds on those - installs (#71510 fixed the ZIP path, #83335 fixed lazy-deps; this closes the - shared helper for the remaining callers). - """ - scripts_dir = _venv_scripts_dir() if _is_windows() else None - - # A pip / site-packages install has no PROJECT_ROOT/venv; the caller still - # passes VIRTUAL_ENV=PROJECT_ROOT/venv, which does not exist. uv would fail - # before installing anything ("Failed to inspect Python interpreter from - # active virtual environment"). Detect the stale pointer and pin the target - # interpreter explicitly instead of trusting the nonexistent venv. - pin_python = False - if ( - env - and env.get("VIRTUAL_ENV") - and not Path(env["VIRTUAL_ENV"]).is_dir() - and install_cmd_prefix - and _is_uv_command(install_cmd_prefix) - ): - # Only uv needs the explicit pin; pip resolves the target from - # sys.executable itself and has no --python flag. - pin_python = True - env = {**env} - env.pop("VIRTUAL_ENV", None) - # When we pin to sys.executable, the entry-point shims that uv will - # rewrite live in that interpreter's Scripts/bin directory, NOT in - # PROJECT_ROOT/venv (which does not exist on a site-packages install). - # Quarantining the wrong dir means the running hermes.exe stays locked - # on Windows and the install fails exactly like the original bug. Only - # override when the venv-derived dir is missing; otherwise keep it. - if scripts_dir is None and _is_windows(): - scripts_dir = _interpreter_scripts_dir() - - def _install(args: list[str]) -> None: - if pin_python: - args = _insert_python_pin(args) - # strict_quarantine: this is the UPDATE dependency sync. A shim that - # cannot be renamed aside proves a hard venv hold; running uv anyway - # is how installs strand half-updated (#87331). ShimQuarantineError - # propagates to the update's sync boundary, which defers via the - # update-incomplete marker instead of mutating a contended venv. - _run_quarantined_install( - install_cmd_prefix + args, env=env, scripts_dir=scripts_dir, - strict_quarantine=True, - ) - - try: - _install(["install", "-e", f".[{group}]"]) - _verify_console_scripts_installed(install_cmd_prefix, env=env) - return - except subprocess.CalledProcessError: - print( - " ⚠ Optional extras failed, reinstalling base dependencies and retrying extras individually..." - ) - - _install(["install", "-e", "."]) - - failed_extras: list[str] = [] - installed_extras: list[str] = [] - for extra in _load_installable_optional_extras(group=group): - try: - _install(["install", "-e", f".[{extra}]"]) - installed_extras.append(extra) - except subprocess.CalledProcessError: - failed_extras.append(extra) - - if installed_extras: - print( - f" ✓ Reinstalled optional extras individually: {', '.join(installed_extras)}" - ) - if failed_extras: - print( - f" ⚠ Skipped optional extras that still failed: {', '.join(failed_extras)}" - ) - - # Belt-and-suspenders: verify every declared core dependency from - # pyproject.toml's [project.dependencies] is actually importable in the - # target venv. uv's incremental resolver has — in the wild — produced - # partial installs where a newly added base dep (e.g. ``pathspec``) - # silently fails to land on top of a half-stale venv, and the only - # symptom is a downstream subprocess crashing with ModuleNotFoundError - # hours later inside ``hermes update``'s desktop-rebuild or skill-sync - # stage. Reinstall with --reinstall to force resolution if anything is - # missing, then re-verify so the failure surfaces here instead of - # downstream. - _verify_core_dependencies_installed(install_cmd_prefix, env=env, group=group) - _verify_console_scripts_installed(install_cmd_prefix, env=env) - - -def _load_console_script_names() -> list[str]: - """Return ``[project.scripts]`` entry-point names from pyproject.toml.""" - try: - import tomllib # Python 3.11+ - except ImportError: # pragma: no cover - return [] - - pyproject = PROJECT_ROOT / "pyproject.toml" - if not pyproject.is_file(): - return [] - - try: - with open(pyproject, "rb") as f: - data = tomllib.load(f) - scripts = data.get("project", {}).get("scripts", {}) or {} - return [str(name) for name in scripts if name] - except Exception as e: - logger.debug("console script verification: failed to read pyproject.toml: %s", e) - return [] - - -def _verify_console_scripts_installed( - install_cmd_prefix: list[str], - *, - env: dict[str, str] | None = None, -) -> None: - """Ensure every declared console_script shim exists on disk after install. - - On Windows, ``uv pip install -e .`` can register ``hermes.exe`` in the - wheel RECORD while the file never lands on disk — typically when the live - ``hermes.exe`` shim is locked during ``hermes update``, or when uv/distlib - skips a launcher write. The symptom is ``hermes-agent.exe`` and - ``hermes-acp.exe`` present but ``hermes.exe`` missing, so ``hermes`` drops - off PATH even though the install reported success (issue #52931). - - If any shim is missing we reinstall with ``--reinstall -e .`` under the - same quarantine dance as the primary install path, then re-check. - """ - if not _is_windows(): - return - - scripts_dir = _venv_scripts_dir() - if scripts_dir is None: - return - - names = _load_console_script_names() - if not names: - return - - def _missing() -> list[str]: - return [ - name - for name in names - if not (scripts_dir / f"{name}.exe").is_file() - ] - - missing = _missing() - if not missing: - return - - print( - f" ⚠ Verification: {len(missing)} console script(s) missing on disk: " - f"{', '.join(missing)}" - ) - print(" → Reinstalling entry points with --reinstall...") - - try: - _run_quarantined_install( - install_cmd_prefix + ["install", "--reinstall", "-e", "."], - env=env, - scripts_dir=scripts_dir, - ) - except subprocess.CalledProcessError as e: - logger.warning("console script verification: repair install failed: %s", e) - print( - " ⚠ Entry point repair failed; try `hermes update --force` after " - "closing other hermes processes." - ) - return - - still_missing = _missing() - if still_missing: - print( - f" ⚠ Still missing after repair: {', '.join(still_missing)}. " - "Workaround: python -m hermes_cli.main " - ) - else: - print(" ✓ All console entry points restored") - - -def _verify_core_dependencies_installed( - install_cmd_prefix: list[str], - *, - env: dict[str, str] | None = None, - group: str = "all", -) -> None: - """Check that every base dep from pyproject.toml is importable; if not, retry. - - Reads ``pyproject.toml`` directly (so we don't trust the venv's stale - metadata), filters out deps gated by ``;`` environment markers that don't - apply to this platform, and runs ``importlib.metadata.version()`` in the - venv interpreter for each one. If anything is missing we reinstall the - base group with ``--reinstall`` to force uv to re-resolve, then check - again. We treat the final state as a warning rather than a hard failure - so a single broken-on-PyPI dep can't block an otherwise-successful - update — but the warning makes the partial install visible at the spot - that caused it, instead of hours later in a downstream subprocess. - """ - try: - import tomllib # Python 3.11+ - except ImportError: # pragma: no cover — Python < 3.11 unsupported but be safe - return - - pyproject = PROJECT_ROOT / "pyproject.toml" - if not pyproject.is_file(): - return - - try: - with open(pyproject, "rb") as f: - data = tomllib.load(f) - raw_deps = data.get("project", {}).get("dependencies", []) or [] - except Exception as e: - logger.debug("dep verification: failed to read pyproject.toml: %s", e) - return - - # Parse each "name OP version ; marker" string into (dist_name, marker_obj). - # We use packaging.requirements when available (it ships with pip/uv envs), - # falling back to a naive split that's good enough for the canonical - # ``name==version[; marker]`` style this repo uses. - deps: list[tuple[str, "object | None"]] = [] - try: - from packaging.requirements import Requirement # type: ignore - - for spec in raw_deps: - try: - req = Requirement(spec) - deps.append((req.name, req.marker)) - except Exception: - continue - except Exception: - for spec in raw_deps: - head = spec.split(";", 1)[0] - for op in ("==", ">=", "<=", "~=", ">", "<", "!="): - if op in head: - head = head.split(op, 1)[0] - break - name = head.strip().split("[", 1)[0].strip() - if name: - deps.append((name, None)) - - # Apply environment markers to drop deps that don't apply on this platform - # (e.g. ``ptyprocess ; sys_platform != 'win32'`` is correctly skipped on - # Windows). Without markers we'd false-positive every cross-platform exclusion. - applicable: list[str] = [] - for name, marker in deps: - if marker is None: - applicable.append(name) - continue - try: - if marker.evaluate(): # type: ignore[union-attr] - applicable.append(name) - except Exception: - applicable.append(name) - - if not applicable: - return - - # Run the check inside the venv Python — sys.executable here may be the - # outer Python that drove ``hermes update``, not the venv we just wrote - # to. The uv install_cmd_prefix encodes which environment we targeted - # (either ``[uv, pip]`` with VIRTUAL_ENV in env, or - # ``[sys.executable, -m, pip]`` for the in-process Python); resolve the - # right interpreter for the verification. - venv_python = _resolve_install_target_python(install_cmd_prefix, env) - if venv_python is None: - return - - def _missing_deps() -> list[str]: - check_script = ( - "import importlib.metadata as md, sys\n" - "missing=[]\n" - "for name in sys.argv[1:]:\n" - " try: md.version(name)\n" - " except md.PackageNotFoundError: missing.append(name)\n" - "print('\\n'.join(missing))\n" - ) - try: - result = subprocess.run( - [str(venv_python), "-c", check_script, *applicable], - capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=False, - env=env, - ) - except Exception as e: - logger.debug("dep verification: subprocess failed: %s", e) - return [] - return [line.strip() for line in result.stdout.splitlines() if line.strip()] - - missing = _missing_deps() - if not missing: - return - - print( - f" ⚠ Verification: {len(missing)} declared dep(s) missing after install: " - f"{', '.join(missing[:8])}{'...' if len(missing) > 8 else ''}" - ) - print(" → Reinstalling base group with --reinstall to repair...") - - # Reinstall base group with --reinstall so uv re-resolves from scratch - # against the current pyproject. We don't pass ``[{group}]`` here on - # purpose — the missing dep is in *base* deps; rerunning the full all- - # extras install can cost minutes and trips on whatever optional extra - # was already broken upstream. Base is fast and is what's actually wrong. - # - # Quarantine the running ``hermes.exe`` first: ``--reinstall -e .`` - # rewrites the entry-point shims, and on Windows pip can't overwrite the - # live launcher, which would leave ``hermes`` off PATH. - scripts_dir = _venv_scripts_dir() if _is_windows() else None - repair_args = ["install", "--reinstall", "-e", "."] - try: - _run_quarantined_install( - install_cmd_prefix + repair_args, env=env, scripts_dir=scripts_dir - ) - except subprocess.CalledProcessError as e: - logger.warning("dep verification: repair install failed: %s", e) - print(" ⚠ Repair install failed; check `hermes update` output above.") - return - - still_missing = _missing_deps() - if not still_missing: - print(" ✓ All declared core dependencies now installed") - return - - # Last-ditch: install each remaining missing dep with its pin directly. - # Useful when uv's resolver thinks the env is satisfied but the on-disk - # package metadata says otherwise (rare but observed). - name_to_spec = {} - for spec in raw_deps: - head = spec.split(";", 1)[0].strip() - bare = head - for op in ("==", ">=", "<=", "~=", ">", "<", "!="): - if op in bare: - bare = bare.split(op, 1)[0] - break - name_to_spec[bare.strip().split("[", 1)[0].strip()] = head - - specs = [name_to_spec.get(n, n) for n in still_missing] - print( - f" → Force-installing remaining missing dep(s): {', '.join(specs)}" - ) - try: - _run_install_with_heartbeat( - install_cmd_prefix + ["install", "--reinstall", *specs], env=env - ) - except subprocess.CalledProcessError as e: - logger.warning("dep verification: per-package repair failed: %s", e) - print( - f" ⚠ Could not install: {', '.join(still_missing)}. " - "Run `hermes update --force` after closing other hermes processes." - ) - return - - final_missing = _missing_deps() - if final_missing: - print( - f" ⚠ Still missing after repair: {', '.join(final_missing)}. " - "Run `hermes update --force` after closing other hermes processes." - ) - else: - print(" ✓ All declared core dependencies now installed") - - -def _resolve_install_target_python( - install_cmd_prefix: list[str], env: dict[str, str] | None -) -> Path | None: - """Figure out which Python interpreter the install just targeted. - - ``_install_python_dependencies_with_optional_fallback`` is called with - either ``[uv, pip]`` (and a ``VIRTUAL_ENV`` env var pointing at the - target venv) or ``[sys.executable, -m, pip]`` (the in-process Python). - The verification step needs the *resulting* environment's Python so - ``importlib.metadata`` queries the right site-packages. - """ - if env and "VIRTUAL_ENV" in env: - from hermes_constants import venv_python_path - - venv_root = Path(env["VIRTUAL_ENV"]) - candidate = venv_python_path(venv_root, windows=_is_windows()) - if candidate.exists(): - return candidate - - # Fallback: assume install_cmd_prefix[0] is the python interpreter (the - # ``[sys.executable, -m, pip]`` shape). Skip if it looks like ``uv``. - if install_cmd_prefix: - first = Path(install_cmd_prefix[0]) - if first.exists() and "uv" not in first.name.lower(): - return first - - return None - - -def _is_termux_env(env: dict[str, str] | None = None) -> bool: - return _is_termux_startup_environment(env) - - -def _is_windows_npm_path(npm_path: str) -> bool: - """Return True if ``npm_path`` points at a Windows npm shim. - - On WSL the Windows install dir is exposed through the ``/mnt/c`` drive - mount and PATH interop, so ``shutil.which("npm")`` can hand back - ``/mnt/c/Program Files/nodejs/npm`` (or the ``npm.cmd`` / ``npm.exe`` - shim). Those are detected here by their ``.exe``/``.cmd``/``.bat`` - suffix, a ``/mnt/`` drive-mount prefix, or an embedded backslash (a UNC - path). Callers use this only on a POSIX host — on native Windows an - ``npm.cmd`` shim is the correct executable. - """ - low = npm_path.lower() - return ( - low.endswith((".exe", ".cmd", ".bat")) - or low.startswith("/mnt/") - or "\\" in npm_path - ) - - -def _resolve_node_runtime_npm() -> str | None: - """Resolve an npm executable that belongs to the host's Node runtime. - - On WSL/Linux ``shutil.which("npm")`` may resolve a Windows npm exposed - through PATH interop. Running that Windows npm against the Linux checkout - operates over ``\\wsl.localhost\\...`` UNC paths and fails with EISDIR / - symlink errors in symlink-heavy trees like ``ui-tui`` (#30271). Refuse a - Windows npm on a POSIX host and re-scan PATH (skipping ``/mnt/*`` interop - entries) for a Linux-native npm. Returns the npm path, or ``None`` when - no suitable npm is reachable. - """ - from hermes_constants import find_node_executable - - npm = find_node_executable("npm") - - # On native Windows the platform npm (``npm.cmd``) is exactly what we - # want — only reject Windows shims when we're a POSIX/WSL process. - if _is_windows(): - return npm - - if not npm: - return None - - if not _is_windows_npm_path(npm): - return npm - - # The first resolution was a Windows npm. Re-scan PATH skipping the - # ``/mnt/*`` Windows drive mounts WSL injects, so a Linux-native npm that - # came later on PATH is still found. - for directory in os.environ.get("PATH", "").split(os.pathsep): - if not directory or directory.lower().startswith("/mnt/"): - continue - candidate = shutil.which("npm", path=directory) - if candidate and not _is_windows_npm_path(candidate): - return candidate - return None - - class _UpdateOutputStream: """Stream wrapper used during ``hermes update`` to survive terminal loss. @@ -6734,17 +5177,6 @@ def _finalize_update_output(state): pass -def _resolve_update_branch(args) -> str: - """Normalize ``args.branch`` into a non-empty branch name. - - Centralizes the "default to main, accept --branch override, treat empty - or whitespace-only values as the default" parsing so every consumer of - ``--branch`` (check path, git-update path, ZIP-fallback path) agrees on - the same answer. - """ - return (getattr(args, "branch", None) or "main").strip() or "main" - - def cmd_update(args): """Update Hermes Agent to the latest version. diff --git a/hermes_cli/main_install_repair.py b/hermes_cli/main_install_repair.py new file mode 100644 index 0000000000..a7fa5769d2 --- /dev/null +++ b/hermes_cli/main_install_repair.py @@ -0,0 +1,1677 @@ +"""Install/update recovery: interrupted-install markers, lazy-refresh repair, Windows shim quarantine, dependency verification. + +Split out of ``hermes_cli/main.py``; every moved name is re-imported there, so +``hermes_cli.main.`` keeps resolving (and monkeypatching) as before. +Names that stay in main are imported lazily inside the functions that use them +(call-time resolution keeps ``hermes_cli.main.`` patches effective and +avoids an import cycle). +""" + +import logging +import os +import shlex +import shutil +import subprocess +import sys +import threading +import time as _time + +from pathlib import Path +from hermes_cli import _early_recovery as _early_recovery_mod + +# Log-record parity with the origin module. +logger = logging.getLogger("hermes_cli.main") + + +def _load_installable_optional_extras(group: str = "all") -> list[str]: + """Return optional extras referenced by a dependency group. + + ``group`` is usually ``all`` (desktop/server broad install) or + ``termux-all`` (Termux-compatible broad install). + """ + from hermes_cli.main import PROJECT_ROOT + try: + import tomllib + + with (PROJECT_ROOT / "pyproject.toml").open("rb") as handle: + project = tomllib.load(handle).get("project", {}) + except Exception: + return [] + + optional_deps = project.get("optional-dependencies", {}) + if not isinstance(optional_deps, dict): + return [] + + refs = optional_deps.get(group, []) + referenced: list[str] = [] + for ref in refs: + if "[" in ref and "]" in ref: + name = ref.split("[", 1)[1].split("]", 1)[0] + if name in optional_deps: + referenced.append(name) + + return referenced + + +# Install-scoped breadcrumbs live next to the venv (not under $HERMES_HOME) +# because the venv is shared across profiles. +# +# ``.update-incomplete`` — generic core ``.[all]`` install was interrupted. +# Cleared only after a confirmed full dependency reinstall/recovery. +# +# ``.lazy-refresh-incomplete`` — lazy-backend refresh phase may have corrupted +# packages. Cleared only after import-probe repair confirms healthy (not when +# probes are unavailable/indeterminate). Narrow lazy probes must NEVER clear +# the generic core marker (#58004 review). +def _update_marker_path() -> Path: + from hermes_cli.main import PROJECT_ROOT + return PROJECT_ROOT / ".update-incomplete" + + +def _lazy_refresh_marker_path() -> Path: + from hermes_cli.main import PROJECT_ROOT + return PROJECT_ROOT / ".lazy-refresh-incomplete" + + +def _pytest_owns_live_checkout(root: Path) -> bool: + """True when running under pytest AND ``root`` is this checkout itself. + + Tests that drive update/recovery without sandboxing ``PROJECT_ROOT`` + must neither litter the live repo root with recovery breadcrumbs + (a leftover ``.lazy-refresh-incomplete`` / ``.update-incomplete`` + false-arms recovery on the developer's next real launch) nor run a real + reinstall against the executing venv. Sandboxed tests point at a + tmp_path and are unaffected (same posture as + ``managed_scope._under_pytest``).""" + return ( + "PYTEST_CURRENT_TEST" in os.environ + and root == Path(__file__).resolve().parent.parent + ) + + +def _clear_marker_file(path: Path, *, label: str) -> None: + """Remove an update-recovery breadcrumb. Never raises.""" + try: + path.unlink() + except FileNotFoundError: + pass + except OSError as exc: + logger.debug("Could not clear %s marker: %s", label, exc) + + +def _clear_update_incomplete_marker() -> None: + """Remove the interrupted core-install breadcrumb. Never raises.""" + from hermes_cli.main import _update_marker_path + _clear_marker_file(_update_marker_path(), label="update-incomplete") + + +def _clear_lazy_refresh_incomplete_marker() -> None: + """Remove the interrupted lazy-refresh breadcrumb. Never raises.""" + _clear_marker_file(_lazy_refresh_marker_path(), label="lazy-refresh-incomplete") + + +def _recover_from_interrupted_install() -> None: + """Finish update work left half-done by a prior ``hermes update``. + + Handles two independent breadcrumbs: + + - ``.update-incomplete`` — core ``.[all]`` install interrupted. Recovers + via full quarantined reinstall. Never cleared by the narrow lazy-refresh + import probes alone. + - ``.lazy-refresh-incomplete`` — lazy-backend refresh may have corrupted + packages. Recovers via package-only import probes; cleared only when + probes confirm healthy/repaired (indeterminate keeps the marker). + + Never raises: a recovery failure must not block launch. If it can't + self-heal it prints the manual command and leaves the relevant marker so + the next launch tries again. + + Concurrency: markers live next to the shared venv, so a gateway start + plus a CLI launch (or two profiles starting at once) can both see them. + An ``O_EXCL`` lockfile ensures only one process runs recovery; the + others skip and let the winner clear markers. + + Output: everything — our status lines AND the streamed pip/uv install + (which inherits fd 1) — is routed to stderr. Launches whose stdout is a + protocol stream (``hermes acp`` speaks JSON-RPC on stdout) must never get + install noise on stdout. + """ + from hermes_cli.main import PROJECT_ROOT, _clear_update_incomplete_marker, _pytest_owns_live_checkout, _recover_core_update_marker_locked, _update_marker_path + if _pytest_owns_live_checkout(PROJECT_ROOT): + return + core_marker = _update_marker_path().exists() + lazy_marker = _lazy_refresh_marker_path().exists() + if not core_marker and not lazy_marker: + return + + # Skip in managed/Docker installs and on PyPI installs with no git checkout: + # those don't run the source-tree update path, so a stray marker is not ours + # to act on. Just clear it. + if not (PROJECT_ROOT / "pyproject.toml").is_file(): + _clear_update_incomplete_marker() + _clear_lazy_refresh_incomplete_marker() + return + + # Single-flight guard: atomically claim the recovery lock. If another + # process holds it, skip — it is running the same reinstall into the same + # shared venv right now. A crashed holder leaves a stale lock; break it + # after an hour (well past any realistic install) so recovery can't be + # wedged forever. + lock_path = PROJECT_ROOT / ".update-incomplete.lock" + try: + fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + os.write(fd, f"{os.getpid()}\n".encode()) + os.close(fd) + except FileExistsError: + try: + if _time.time() - lock_path.stat().st_mtime > 3600: + lock_path.unlink() + except OSError: + pass + return + except OSError as exc: + # Couldn't create the lock (read-only fs, perms). Proceed unlocked — + # the install itself will surface the real problem. + logger.debug("Could not create install-recovery lock: %s", exc) + + saved_stdout_fd = None + saved_sys_stdout = sys.stdout + try: + # Route Python-level prints AND subprocess-inherited fd 1 to stderr + # for the duration of recovery (see docstring: ACP stdout safety). + try: + saved_stdout_fd = os.dup(1) + os.dup2(2, 1) + except OSError: + saved_stdout_fd = None + sys.stdout = sys.stderr + + if lazy_marker: + _recover_lazy_refresh_marker_locked() + + if _update_marker_path().exists(): + _recover_core_update_marker_locked() + finally: + sys.stdout = saved_sys_stdout + if saved_stdout_fd is not None: + try: + os.dup2(saved_stdout_fd, 1) + os.close(saved_stdout_fd) + except OSError: + pass + try: + lock_path.unlink() + except OSError: + pass + + +def _recover_lazy_refresh_marker_locked() -> None: + """Heal ``.lazy-refresh-incomplete`` via confirmed import-probe repair.""" + from hermes_cli.main import _default_venv_install_target, _repair_venv_via_import_probes + print( + "⚠ A previous lazy-backend refresh may have left the venv unhealthy — " + "running import-based package repair..." + ) + install_prefix, install_env = _default_venv_install_target() + status = _repair_venv_via_import_probes(install_prefix, env=install_env) + if status in ("healthy", "repaired"): + _clear_lazy_refresh_incomplete_marker() + print("✓ Lazy-refresh venv recovery confirmed — install is healthy again.") + return + if status == "indeterminate": + print( + " ⚠ Import probes unavailable — cannot confirm venv health. " + "Leaving `.lazy-refresh-incomplete` for the next launch." + ) + else: + print( + " ⚠ Lazy-refresh package repair incomplete. " + "Leaving `.lazy-refresh-incomplete` for the next launch." + ) + print(" Recover manually with:") + all_specs = _lazy_refresh_repair_specs( + sorted(set(_LAZY_REFRESH_REPAIR_PACKAGES.values())) + ) + print( + f" {' '.join(install_prefix)} install --force-reinstall " + + " ".join(shlex.quote(s) for s in all_specs) + ) + + +def _recover_core_update_marker_locked() -> None: + """Heal ``.update-incomplete`` via full ``.[all]`` reinstall only. + + Narrow lazy-refresh import probes are not sufficient proof that a generic + interrupted core install finished — a missing dep outside that probe set + would otherwise look healthy and clear the breadcrumb too early. + """ + from hermes_cli.main import PROJECT_ROOT, _clear_update_incomplete_marker, _default_venv_install_target, _repair_venv_via_import_probes + print( + "⚠ A previous `hermes update` was interrupted mid-install — " + "finishing dependency installation now..." + ) + + # Windows: a normal ``hermes.exe`` launch always has the launcher as an + # ancestor. Full editable reinstall uses quarantine so the live shim can + # still be replaced. Package-only import repair may help as first aid but + # must NEVER clear this core marker on its own (#58004 review). + self_locked = _windows_running_hermes_launcher_locked() + if self_locked: + install_prefix, install_env = _default_venv_install_target() + print( + " → Running from hermes.exe; applying package-only first aid, " + "then quarantined full reinstall (core marker stays until that " + "succeeds)..." + ) + _repair_venv_via_import_probes(install_prefix, env=install_env) + + try: + from hermes_cli import _install_repair as _ir + + # ensure_uv bootstraps the installer itself when missing (the early + # pass's stdlib-only lookup cannot); keeping it here means the late + # path still self-heals a venv whose uv vanished mid-update. + from hermes_cli.managed_uv import ensure_uv + + ensure_uv() + + # Delegate the install itself to the shared stdlib executor so both + # this late path and the pre-import early pass run exactly the same + # reinstall. Called inside the same stdout→stderr redirect already + # established by _recover_from_interrupted_install, so + # run_core_install's own redirect nests harmlessly. + _ir.run_core_install(PROJECT_ROOT) + + _clear_update_incomplete_marker() + print("✓ Dependency installation recovered — your install is healthy again.") + except Exception as exc: + # Leave the marker in place so the next launch retries. Give the user + # the exact manual recovery command in the meantime. + logger.debug("Interrupted-install recovery failed: %s", exc) + print("✗ Could not auto-recover the interrupted install.") + if self_locked: + print( + " Hermes is still running from the launcher that needs " + "replacing. Close other Hermes windows, restart from a " + "different terminal, then run:" + ) + print(f' cd /d "{PROJECT_ROOT}"') + print( + f' "{sys.executable}" -m pip install -e ".[all]"' + ) + else: + print(" Recover manually with:") + print(f" cd {PROJECT_ROOT}") + print(f" {sys.executable} -m ensurepip --upgrade") + print(f" {sys.executable} -m pip install -e '.[all]'") + + +def _norm_exe_path(path) -> str: + """Case-folded resolved path, for comparing executables on Windows.""" + try: + return str(Path(path).resolve()).lower() + except OSError: + return str(path).lower() + + +def _windows_shim_in_process_chain() -> Path | None: + """The venv console shim this process runs from or under, if any. + + ``venv\\Scripts\\hermes.exe`` is a launcher that runs the interpreter with + the shim itself as its script, and that keeps the shim open — without + ``FILE_SHARE_DELETE`` — for the whole process lifetime. So every + ``hermes ...`` command holds its own shim, and an editable install run + from one can never rewrite it (#88838, #89599). + + Two independent probes, because either can come up empty. Process + ancestry finds the launcher when it is a separate parent process, but + needs psutil. This process's own launch paths (``sys.argv[0]``, + ``__main__.__file__``, the module spec origin) cover the rest — the + runpy/zipapp launch puts ``\\__main__.py`` there, which a plain + argv[0] check misses. + + Candidates are intersected with the project venv's own shims, so a + ``hermes.exe`` belonging to some other install never matches. + """ + from hermes_cli.main import _hermes_exe_shims, _is_windows, _venv_scripts_dir + if not _is_windows(): + return None + scripts_dir = _venv_scripts_dir() + if scripts_dir is None: + return None + shims = {_norm_exe_path(shim): shim for shim in _hermes_exe_shims(scripts_dir)} + if not shims: + return None + + def _match(candidate) -> Path | None: + path = Path(candidate) + if path.name.lower() == "__main__.py": + path = path.parent + return shims.get(_norm_exe_path(path)) + + candidates: list[str] = list(sys.argv[:1]) + main_mod = sys.modules.get("__main__") + for attr in (getattr(main_mod, "__file__", None), + getattr(getattr(main_mod, "__spec__", None), "origin", None)): + if attr: + candidates.append(attr) + for candidate in candidates: + matched = _match(candidate) + if matched is not None: + return matched + + try: + import psutil + + me = psutil.Process() + for proc in [me] + list(me.parents()): + try: + matched = _match(proc.exe()) + except Exception: + continue + if matched is not None: + return matched + except Exception: + return None + return None + + +def _windows_running_hermes_launcher_locked() -> bool: + """True when a venv ``hermes*.exe`` shim is this process or an ancestor. + + Best-effort: returns False when psutil is unavailable or inspection fails. + """ + from hermes_cli.main import _windows_shim_in_process_chain + return _windows_shim_in_process_chain() is not None + + +# Set on the re-exec'd child so it can never spawn another one. +_UPDATE_REEXEC_ENV = "HERMES_UPDATE_REEXEC" + + +def _reexec_dependency_sync_off_windows_shim() -> bool: + """Hand the dependency sync to the venv interpreter, off the console shim. + + Returns True when a child was spawned and the caller must exit at once, + releasing the shim before the child reaches ``pip install -e .``. Returns + False to continue the sync in-process. + + Called at the dependency-sync boundary, NOT at the top of the command — + the same placement rule as the native-module deferral beside it, and for + the same reason (#86735): a hand-off that fires before the fetch detaches + every run, including the ``Already up to date!`` no-op that never touches + the venv at all, and it takes the interactive prompts with it. By the time + we reach here the code swap is done and every question — stash, branch + switch, config migration — has already been asked and answered in the + user's own console. Only the venv rewrite is left, and that is the single + step that genuinely cannot run from inside the shim. + + ``venv\\Scripts\\hermes.exe`` is a launcher that runs the interpreter with + the shim as its script and holds it open without ``FILE_SHARE_DELETE`` for + the whole command, so the quarantine rename is refused and uv fails to + replace it with os error 32 (#88838, #89599). + + A child is required, and waiting on it cannot work: this process holds the + handle the child needs released, so a parent that waits deadlocks against + the work it is waiting for. Windows has no exec to escape with either. + The shell therefore returns while the install runs on; the child keeps the + console and prints its own result, and ``--gateway`` writes the true exit + code to ``.update_exit_code`` for the gateway watcher. + + The child re-runs ``hermes update``, so the whole remaining flow — the + dependency sync and the node/web/lazy-refresh tail behind it — still + happens exactly once. ``_UPDATE_REEXEC_ENV`` marks it so it cannot spawn + another child, and so the "already up to date" early return does not + swallow the sync it was spawned to perform (the checkout is current by + now; that is the point). + + The caller has already written ``.update-incomplete``, so a child that + dies mid-install is finished by the next launch's recovery instead of + leaving a half-synced venv. Anything that stops the hand-off (no venv + python, spawn refused) returns False and syncs in-process, where the + pre-existing os-error-32 path and its marker recovery still apply. + """ + from hermes_cli.main import _UPDATE_REEXEC_ENV, _windows_shim_in_process_chain + if os.environ.get(_UPDATE_REEXEC_ENV) == "1": + return False + shim = _windows_shim_in_process_chain() + if shim is None: + return False + + from hermes_constants import venv_python_path + + python_exe = venv_python_path(shim.parent.parent, windows=True) + cmd = [str(python_exe), "-m", "hermes_cli.main", *sys.argv[1:]] + if python_exe.is_file(): + try: + subprocess.Popen( + cmd, + env={**os.environ, _UPDATE_REEXEC_ENV: "1"}, + stdin=subprocess.DEVNULL, + ) + print( + f"→ Windows: {shim.name} cannot replace itself while it runs; " + "finishing the dependency install under the venv Python." + ) + print( + " The code update is already applied. The install continues " + "below and this shell returns right away." + ) + return True + except OSError as exc: + logger.debug("Dependency-sync hand-off via %s failed: %s", python_exe, exc) + print(f" ⚠ Could not hand the dependency install off {shim.name}.") + print(" Continuing in-process; if it cannot replace the shim, run:") + print(f" {subprocess.list2cmdline(cmd)}") + return False + + +def _default_venv_install_target() -> tuple[list[str], dict[str, str] | None]: + """Return ``(install_cmd_prefix, env)`` for the project venv when possible.""" + from hermes_cli.main import PROJECT_ROOT, _is_termux_env + try: + from hermes_cli.managed_uv import ensure_uv + + uv_bin = ensure_uv() + except Exception: + uv_bin = None + if uv_bin: + from hermes_constants import project_venv_dir + + venv_dir = project_venv_dir(PROJECT_ROOT) or PROJECT_ROOT / "venv" + env = {**os.environ, "VIRTUAL_ENV": str(venv_dir)} + if _is_termux_env(env): + env.pop("PYTHONPATH", None) + env.pop("PYTHONHOME", None) + return [uv_bin, "pip"], env + return [sys.executable, "-m", "pip"], None + + +def _run_install_with_heartbeat( + cmd: list[str], + *, + env: dict[str, str] | None = None, + heartbeat_interval_seconds: int = 30, +) -> None: + """Run dependency install command with periodic heartbeat output. + + Some resolvers/build backends (especially when compiling Rust/C extensions) + can stay quiet for minutes. Emit a simple elapsed-time heartbeat so users + know ``hermes update`` is still progressing even if pip/uv itself is silent. + """ + from hermes_cli.main import PROJECT_ROOT + done = threading.Event() + start = _time.time() + + def _heartbeat() -> None: + # Wait first, then print, so short installs don't emit noise. + while not done.wait(heartbeat_interval_seconds): + elapsed = int(_time.time() - start) + print( + f" … still installing dependencies ({elapsed}s elapsed)" + " — compiling Rust/C extensions can take several minutes", + flush=True, + ) + + t = threading.Thread(target=_heartbeat, daemon=True) + t.start() + try: + subprocess.run( + cmd, + cwd=PROJECT_ROOT, + check=True, + env=env, + ) + finally: + done.set() + t.join(timeout=0.2) + + +def _is_windows() -> bool: + return sys.platform == "win32" + + +def _venv_scripts_dir() -> Path | None: + """Return the venv Scripts directory if we're running inside the project venv.""" + from hermes_cli.main import PROJECT_ROOT, _is_windows + from hermes_constants import project_venv_dir, venv_bin_dir + + venv_dir = project_venv_dir(PROJECT_ROOT) + if venv_dir is None: + return None + + scripts = venv_bin_dir(venv_dir, windows=_is_windows()) + return scripts if scripts.is_dir() else None + + +def _hermes_exe_shims(scripts_dir: Path) -> list[Path]: + """Entry-point shims that uv may try to rewrite during ``pip install -e .``. + + On Windows these are .exe launchers generated by setuptools/uv. On POSIX + they're regular Python scripts which can be replaced atomically — no + self-replacement hazard exists outside Windows. + """ + from hermes_cli.main import _is_windows + if not _is_windows(): + return [] + + names = set(_load_console_script_names()) or {"hermes", "hermes-agent", "hermes-acp"} + # The gateway shim is not a [project.scripts] entry point, but older + # update/install paths still rewrite and quarantine it. + names.add("hermes-gateway") + return [scripts_dir / f"{name}.exe" for name in sorted(names)] + + +def _quarantine_running_hermes_exe( + scripts_dir: Path, *, max_attempts: int = 4, + failed_out: list[str] | None = None, +) -> list[tuple[Path, Path]]: + """Pre-empt Windows file lock on the running ``hermes.exe``. + + Windows allows RENAMING a mapped/running executable (the kernel tracks the + file by handle, not path), but blocks DELETE/REPLACE while it's loaded. uv + needs to overwrite the entry-point shims during ``pip install -e .``; + when ``hermes update`` runs, ``hermes.exe`` IS the live process, and uv + fails with ``Access is denied. (os error 5)``. + + We rename live shims to ``hermes.exe.old.`` first. uv then writes + fresh shims at the original paths. The ``.old`` files are cleaned up on + the next hermes invocation by ``_cleanup_quarantined_exes``. + + Rename can still fail when *another* process has opened the .exe without + ``FILE_SHARE_DELETE`` — typically AV real-time scanners with transient + handles (recovers in <1s), or the Hermes Desktop backend child process + (won't recover until the user closes it). We mitigate: + + 1. Retry up to ``max_attempts`` times with exponential backoff + (100/250/500/1000 ms). Handles the AV-scanner case. + 2. If all retries fail, print a clear warning naming the most likely + culprit (running Hermes Desktop / gateway / REPL). + + The updater's own launcher is no longer one of those culprits: an update + started from ``hermes.exe`` re-runs itself under the venv Python before + reaching here (``_reexec_dependency_sync_off_windows_shim``). + + Returns the list of (original, quarantined) pairs so the caller can roll + back if the install itself fails before uv writes a replacement. + + ``failed_out``: when provided, the names of shims whose rename failed on + every attempt are appended — callers that must not mutate a contended + venv (the update dependency sync, #87331) check it and refuse instead of + letting the install run into a half-broken state. + """ + from hermes_cli.main import _hermes_exe_shims, _is_windows + moved: list[tuple[Path, Path]] = [] + if not _is_windows(): + return moved + + import time + + stamp = int(time.time() * 1000) + # Backoff schedule: first attempt is immediate, subsequent ones sleep. + # 100ms / 250ms / 500ms covers the typical AV scanner re-scan window. + backoff_ms = [0, 100, 250, 500, 1000] + attempts = max(1, min(max_attempts, len(backoff_ms))) + + for shim in _hermes_exe_shims(scripts_dir): + if not shim.exists(): + continue + target = shim.with_suffix(shim.suffix + f".old.{stamp}") + + last_exc: OSError | None = None + for attempt in range(attempts): + delay = backoff_ms[attempt] / 1000.0 + if delay: + time.sleep(delay) + try: + shim.rename(target) + moved.append((shim, target)) + last_exc = None + break + except OSError as e: + last_exc = e + continue + + if last_exc is None: + continue + + # Every rename failed. Deferring one to next boot via + # MOVEFILE_DELAY_UNTIL_REBOOT used to be the fallback here, but it + # cannot help: it needs elevation we don't have, and when it does + # land it frees nothing for the install running right now while + # queueing an operation that will move a later, freshly repaired shim + # aside at next boot. Report and let uv try its luck instead — + # sometimes its own retry handling pulls through. + print( + f" ⚠ Could not quarantine {shim.name} ({last_exc.__class__.__name__}: " + f"another process is holding it open)." + ) + print( + " Close Hermes Desktop, exit other `hermes` REPLs, stop the " + "gateway, or pause AV scanning, then re-run `hermes update`." + ) + if failed_out is not None: + failed_out.append(shim.name) + + return moved + + +_PENDING_RENAME_KEY = r"SYSTEM\CurrentControlSet\Control\Session Manager" + + +_PENDING_RENAME_VALUE = "PendingFileRenameOperations" + + +def _filter_pending_shim_renames( + entries: list[str], shims: list[Path] +) -> tuple[list[str], int]: + """Drop shim-quarantine pairs from a PendingFileRenameOperations value. + + The value is a flat REG_MULTI_SZ of (source, target) pairs, and other + installers share it, so only pairs matching our own + ```` -> ``.old.`` naming are removed. Returns the + entries to keep and how many pairs were dropped. + """ + import ntpath + + def _norm(value: str) -> str: + path = str(value).lstrip("!") + if path.startswith("\\??\\"): + path = path[4:] + return ntpath.normcase(ntpath.normpath(path)) + + shim_paths = {_norm(str(shim)) for shim in shims} + kept: list[str] = [] + removed = 0 + for index in range(0, len(entries) - 1, 2): + source, target = entries[index], entries[index + 1] + source_norm = _norm(source) + if source_norm in shim_paths and _norm(target).startswith(f"{source_norm}.old."): + removed += 1 + else: + kept.extend((source, target)) + if len(entries) % 2: + kept.append(entries[-1]) + return kept, removed + + +def _cleanup_pending_shim_renames(scripts_dir: Path) -> int: + """Drop reboot renames older Hermes versions queued for our shims. + + Hermes used to fall back to ``MoveFileExW(MOVEFILE_DELAY_UNTIL_REBOOT)`` + when the quarantine rename failed. Those entries outlive the update that + queued them, so at the next boot they move away whatever now sits at the + shim path — including a shim a later repair just wrote. Needs elevation + to remove (same as it needed to create); a no-op otherwise. + """ + from hermes_cli.main import _filter_pending_shim_renames, _hermes_exe_shims, _is_windows + if not _is_windows(): + return 0 + try: + import winreg + + with winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + _PENDING_RENAME_KEY, + 0, + winreg.KEY_QUERY_VALUE | winreg.KEY_SET_VALUE, + ) as key: + entries, value_type = winreg.QueryValueEx(key, _PENDING_RENAME_VALUE) + if value_type != winreg.REG_MULTI_SZ or not isinstance(entries, list): + return 0 + kept, removed = _filter_pending_shim_renames( + entries, _hermes_exe_shims(scripts_dir) + ) + if not removed: + return 0 + if kept: + winreg.SetValueEx(key, _PENDING_RENAME_VALUE, 0, winreg.REG_MULTI_SZ, kept) + else: + winreg.DeleteValue(key, _PENDING_RENAME_VALUE) + return removed + except (OSError, ValueError): + return 0 + + +def _restore_quarantined_exes(moved: list[tuple[Path, Path]]) -> None: + """Roll back ``_quarantine_running_hermes_exe`` if uv didn't write replacements. + + This is the safety-critical direction. A failed *quarantine* only aborts an + update; a failed *restore* leaves the install with no ``hermes`` on PATH, + and therefore no way to run the command that would repair it (#75584). The + outbound rename already retries a lock, so this one must too rather than + swallow the first ``OSError`` in silence. + + Delegates to the stdlib-only helper that the early-recovery copy in + ``_install_repair`` also uses, so the two cannot drift apart. + """ + _early_recovery_mod.restore_quarantined_shims(moved) + + +class ShimQuarantineError(RuntimeError): + """A live ``hermes*.exe`` shim could not be renamed aside (#87331). + + Raised by :func:`_run_quarantined_install` in ``strict_quarantine`` mode + BEFORE the install command runs. A shim that cannot even be renamed means + another process holds the venv hard enough that the dependency sync would + die partway and strand the install half-updated — the update must refuse, + not warn-and-continue. + """ + + def __init__(self, failed_shims: list[str]): + self.failed_shims = list(failed_shims) + super().__init__( + "could not quarantine live shim(s): " + ", ".join(self.failed_shims) + ) + + +def _run_quarantined_install( + cmd: list[str], + *, + env: dict[str, str] | None = None, + scripts_dir: Path | None = None, + strict_quarantine: bool = False, +) -> None: + """Run an editable install, quarantining the running ``hermes.exe`` first. + + Any ``pip install -e .`` (or ``--reinstall``) rewrites the entry-point + shims, and on Windows the live ``hermes.exe`` is the running process — + pip can neither delete nor overwrite it, so without quarantine the shim + is left missing and ``hermes`` drops off PATH. This wraps + :func:`_run_install_with_heartbeat` with the same rename-out-of-the-way / + restore-on-failure dance that the primary install path uses, so EVERY + install that touches the shims is protected — including the + verification-repair reinstalls in + :func:`_verify_core_dependencies_installed`, which previously called + ``_run_install_with_heartbeat`` directly and bypassed quarantine. + + ``strict_quarantine=True`` (the update dependency sync, #87331): a shim + whose rename failed every retry means a process is holding the venv + without ``FILE_SHARE_DELETE`` — the install WILL hit the same lock on + .pyd files and strand the venv between versions. Roll the successful + renames back and raise :class:`ShimQuarantineError` WITHOUT running the + install. Non-strict callers (post-sync entry-point repair) keep the old + warn-and-try behavior: their venv is already mutated, so refusing buys + nothing. + + Off-Windows (``scripts_dir is None``) this is a thin pass-through. + """ + from hermes_cli.main import ShimQuarantineError, _quarantine_running_hermes_exe, _restore_quarantined_exes, _run_install_with_heartbeat + moved: list[tuple[Path, Path]] = [] + failed: list[str] = [] + if scripts_dir is not None: + moved = _quarantine_running_hermes_exe(scripts_dir, failed_out=failed) + if strict_quarantine and failed: + _restore_quarantined_exes(moved) + raise ShimQuarantineError(failed) + try: + _run_install_with_heartbeat(cmd, env=env) + finally: + # Restore shims when the installer didn't write replacements — on + # FAILURE (install died before the entry-points step) and on SUCCESS + # too: uv audits an already-satisfied editable install as a no-op and + # rewrites no entry points, which would otherwise leave the shims + # quarantined aside and `hermes` missing from PATH after a green + # install (#75584). _restore_quarantined_exes skips any shim the + # installer actually replaced, so this never clobbers fresh output. + # Errors are not swallowed — the finally re-raises whatever escaped. + if scripts_dir is not None: + _restore_quarantined_exes(moved) + + +# A quarantine file younger than this may belong to an update running RIGHT +# NOW in another process, whose restore step still needs it. Deleting one +# mid-flight destroys the only copy of that shim. +_QUARANTINE_GRACE_SECONDS = 15 * 60 + + +def _quarantine_stamp_ms(stale: Path) -> int | None: + """The ``.old.`` stamp in a quarantine filename, or ``None``. + + ``None`` means the name was not produced by + :func:`_quarantine_running_hermes_exe`. We neither rescue nor delete those: + the sweep should not destroy files whose provenance it cannot establish, and + they are not ours to put back. + + Parsed from the NAME rather than ``st_mtime`` because ``rename`` preserves + the original shim's mtime, which records when uv wrote the shim — days + earlier, in general — not when it was quarantined. + """ + try: + return int(stale.name.rsplit(".old.", 1)[1]) + except (IndexError, ValueError): + return None + + +def _cleanup_quarantined_exes(scripts_dir: Path | None = None) -> None: + """Sweep — and where necessary RESCUE — ``hermes.exe.old.*`` from updates. + + Called early on every hermes invocation. Two cases the old unconditional + ``unlink()`` got wrong, both ending with ``hermes`` gone from PATH: + + 1. **Orphan rescue.** If ``hermes.exe`` is missing while + ``hermes.exe.old.*`` is present, that .old file is the ONLY surviving + copy of the shim — an update died, or its restore failed, between + the rename and uv writing a replacement (#75584). Deleting it converts a + one-rename recovery into a full reinstall. Put it back instead, through + the same retry-and-report helper the update-time restore uses. + 2. **Concurrency.** A fresh quarantine file may belong to an update in + flight in another process (the desktop update button racing a shell + ``hermes update`` does exactly this). Leave anything inside the grace + window alone; a later run sweeps it. + + Silent no-op on non-Windows, when there is nothing to do, or on + file-locked / permission errors. + """ + from hermes_cli.main import _QUARANTINE_GRACE_SECONDS, _cleanup_pending_shim_renames, _is_windows, _quarantine_stamp_ms, _venv_scripts_dir + if not _is_windows(): + return + if scripts_dir is None: + scripts_dir = _venv_scripts_dir() + if scripts_dir is None: + return + _cleanup_pending_shim_renames(scripts_dir) + + now = _time.time() + + try: + candidates = [ + (stamp, stale) + for stale, stamp in ( + (p, _quarantine_stamp_ms(p)) for p in scripts_dir.glob("*.exe.old.*") + ) + if stamp is not None + ] + except OSError: + return + + # Newest first by PARSED stamp. Sorting the raw filenames lexicographically + # only tracks recency while every stamp shares a digit width: a stray + # ``.old.999`` sorts above a 13-digit epoch-ms stamp and would be the copy + # rescued onto the live shim name. + candidates.sort(key=lambda pair: pair[0], reverse=True) + + for stamp, stale in candidates: + try: + original = stale.with_name(stale.name.rsplit(".old.", 1)[0]) + + if not original.exists(): + # Orphan rescue: this is the last copy of the shim, so it gets + # the retry ladder and the recovery message, not a bare rename. + _early_recovery_mod.restore_quarantined_shims([(original, stale)]) + continue + + if now - stamp / 1000.0 < _QUARANTINE_GRACE_SECONDS: + continue # may be a live quarantine from a concurrent update + + stale.unlink() + except OSError: + pass # still locked or in use — try again next run + + +# Import probes for venv corruption after a failed lazy ``uv pip install``. +# Metadata can look fine while ``.py`` files were removed mid-install (#57828). +# Canonical tables live in the stdlib-only ``_early_recovery`` module (which +# also probes/repairs BEFORE this module's third-party imports can run) so the +# early and full recovery layers can never drift apart. +_LAZY_REFRESH_IMPORT_PROBES: tuple[tuple[str, str], ...] = ( + _early_recovery_mod.LAZY_REFRESH_IMPORT_PROBES +) + + +_LAZY_REFRESH_REPAIR_PACKAGES: dict[str, str] = ( + _early_recovery_mod.LAZY_REFRESH_REPAIR_PACKAGES +) + + +def _run_package_only_install( + cmd: list[str], + *, + env: dict[str, str] | None = None, +) -> None: + """Run a package-only pip/uv install without quarantining entry-point shims. + + ``pip install --upgrade pip`` and ``--force-reinstall `` do not + rewrite ``hermes.exe``. The editable-install quarantine path would rename + shims without uv recreating them on Windows (#57828). + """ + from hermes_cli.main import _run_install_with_heartbeat + _run_install_with_heartbeat(cmd, env=env) + + +def _lazy_refresh_repair_specs(packages: list[str]) -> list[str]: + """Map repair package names to their declared pin specs in pyproject.toml.""" + from hermes_cli.main import PROJECT_ROOT + try: + import tomllib # Python 3.11+ + except ImportError: # pragma: no cover + return packages + + pyproject = PROJECT_ROOT / "pyproject.toml" + if not pyproject.is_file(): + return packages + + try: + with open(pyproject, "rb") as f: + raw_deps = tomllib.load(f).get("project", {}).get("dependencies", []) or [] + except Exception as exc: + logger.debug("lazy refresh repair spec lookup failed: %s", exc) + return packages + + name_to_spec: dict[str, str] = {} + try: + from packaging.requirements import Requirement # type: ignore + + for spec in raw_deps: + try: + req = Requirement(spec) + name_to_spec[req.name.lower()] = spec.split(";", 1)[0].strip() + except Exception: + continue + except Exception: + for spec in raw_deps: + head = spec.split(";", 1)[0].strip() + bare = head + for op in ("==", ">=", "<=", "~=", ">", "<", "!="): + if op in bare: + bare = bare.split(op, 1)[0] + break + key = bare.strip().split("[", 1)[0].strip().lower() + if key: + name_to_spec[key] = head + + return [name_to_spec.get(pkg.lower(), pkg) for pkg in packages] + + +def _detect_broken_lazy_refresh_imports( + install_cmd_prefix: list[str], + *, + env: dict[str, str] | None = None, +) -> list[str] | None: + """Probe lazy-refresh packages via real imports. + + Returns: + - ``[]`` when probes ran and every package imported cleanly + - ``[dist, ...]`` when probes ran and some packages failed + - ``None`` when the probe could not run (missing venv Python, subprocess + failure, non-zero probe exit) — this is *indeterminate*, not healthy + """ + from hermes_cli.main import _resolve_install_target_python + venv_python = _resolve_install_target_python(install_cmd_prefix, env) + if venv_python is None: + return None + + probe_lines = "\n".join( + f" ({mod!r}, {attr!r})," for mod, attr in _LAZY_REFRESH_IMPORT_PROBES + ) + check_script = ( + "import os\n" + "import sys\n" + "probes = [\n" + f"{probe_lines}\n" + "]\n" + "broken = []\n" + "for mod, attr in probes:\n" + " try:\n" + " imported = __import__(mod)\n" + " if not hasattr(imported, attr):\n" + " broken.append(mod)\n" + " elif mod == 'certifi':\n" + " # The module can import cleanly while cacert.pem is\n" + " # missing/corrupt (brew Python upgrade, interrupted venv\n" + " # rebuild) - every TLS call then fails (#29866).\n" + " bundle = imported.where()\n" + " if not os.path.isfile(bundle) or os.path.getsize(bundle) < 1024:\n" + " broken.append(mod)\n" + " except Exception:\n" + " broken.append(mod)\n" + "print('\\n'.join(broken))\n" + ) + try: + result = subprocess.run( + [str(venv_python), "-c", check_script], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=False, + env=env, + ) + except Exception as exc: + logger.debug("lazy refresh import probe failed: %s", exc) + return None + + if result.returncode != 0: + logger.debug( + "lazy refresh import probe exited %s: %s", + result.returncode, + (result.stderr or "")[:200], + ) + return None + + broken_modules = [ + line.strip() for line in result.stdout.splitlines() if line.strip() + ] + packages: list[str] = [] + seen: set[str] = set() + for mod in broken_modules: + pkg = _LAZY_REFRESH_REPAIR_PACKAGES.get(mod) + if pkg and pkg not in seen: + seen.add(pkg) + packages.append(pkg) + return packages + + +def _repair_broken_lazy_refresh_imports( + install_cmd_prefix: list[str], + packages: list[str], + *, + env: dict[str, str] | None = None, +) -> bool: + """Force-reinstall ``packages`` and re-probe imports. Never raises.""" + from hermes_cli.main import _detect_broken_lazy_refresh_imports, _run_package_only_install + if not packages: + return True + + specs = _lazy_refresh_repair_specs(packages) + try: + _run_package_only_install( + install_cmd_prefix + ["install", "--force-reinstall", *specs], + env=env, + ) + except subprocess.CalledProcessError as exc: + logger.warning("lazy refresh venv repair failed: %s", exc) + return False + + after = _detect_broken_lazy_refresh_imports(install_cmd_prefix, env=env) + # Indeterminate re-probe is not confirmed success. + return after == [] + + +def _repair_venv_via_import_probes( + install_cmd_prefix: list[str], + *, + env: dict[str, str] | None = None, +) -> str: + """Probe imports and force-reinstall any broken lazy-refresh packages. + + Uses real ``import`` checks (not distribution metadata) so a venv where + METADATA remains but ``.py`` files were wiped mid-install is still + detected (#57828). Package-only reinstall — never rewrites ``hermes.exe``. + + Never raises. Returns one of: + - ``"healthy"`` — probes ran and found nothing broken + - ``"repaired"`` — probes found breakage and force-reinstall confirmed clean + - ``"failed"`` — probes found breakage and repair did not confirm clean + - ``"indeterminate"`` — probes could not run; do NOT treat as healthy + """ + from hermes_cli.main import _detect_broken_lazy_refresh_imports, _repair_broken_lazy_refresh_imports + broken = _detect_broken_lazy_refresh_imports(install_cmd_prefix, env=env) + if broken is None: + print( + " ⚠ Import probes unavailable — cannot confirm venv package health." + ) + return "indeterminate" + if not broken: + return "healthy" + print( + " → Detected corrupted venv packages via import probes: " + f"{', '.join(broken)}; repairing..." + ) + if _repair_broken_lazy_refresh_imports( + install_cmd_prefix, broken, env=env + ): + print(" ✓ Venv repair succeeded") + return "repaired" + manual = " ".join( + shlex.quote(s) for s in _lazy_refresh_repair_specs(broken) + ) + print(" ⚠ Venv repair incomplete. Run manually, then `hermes update`:") + print( + f" {' '.join(install_cmd_prefix)} install --force-reinstall {manual}" + ) + return "failed" + + +def _is_uv_command(install_cmd_prefix: list[str]) -> bool: + """True when the install command is a uv/uvx invocation. + + Handles a bare uv binary (``uv`` / ``uvx``, any extension), a path to + one, and ``python -m uv`` / ``python -m uvx`` — the naive basename check + misses the module form and launcher wrappers whose name does not contain + "uv". + """ + if not install_cmd_prefix: + return False + first = str(install_cmd_prefix[0]).lower() + if "uv" in Path(first).name: + return True + # python -m uv / python -m uvx + if len(install_cmd_prefix) >= 3 and first.endswith(("python", "python.exe")): + return install_cmd_prefix[1] == "-m" and install_cmd_prefix[2] in ( + "uv", + "uvx", + ) + return False + + +def _insert_python_pin(args: list[str]) -> list[str]: + """Insert ``--python `` into a uv command line. + + If the caller already passed ``--python``, its value wins (uv's last-wins + semantics are ambiguous; the explicit caller intent should not be + overridden by the fallback pin). + """ + if "--python" in args: + return args + return [args[0], "--python", str(sys.executable), *args[1:]] + + +def _interpreter_scripts_dir() -> Path | None: + """Scripts/bin directory of the running interpreter (sys.executable). + + Used when pinning an install to ``sys.executable`` on a site-packages + install where ``PROJECT_ROOT / "venv"`` does not exist: the entry-point + shims uv rewrites live next to the interpreter, not under a project venv. + Layout comes from the canonical ``venv_bin_dir`` helper (#76105 — + hand-rolling Scripts/bin is lint-tested against). + """ + from hermes_cli.main import _is_windows + from hermes_constants import venv_bin_dir + + exe = Path(sys.executable) + # sys.executable lives IN the bin/Scripts dir; its parent.parent is the + # env root venv_bin_dir derives from. + cand = venv_bin_dir(exe.parent.parent, windows=_is_windows()) + if cand.is_dir(): + return cand + return exe.parent if exe.parent.is_dir() else None + + +def _install_python_dependencies_with_optional_fallback( + install_cmd_prefix: list[str], + *, + env: dict[str, str] | None = None, + group: str = "all", +) -> None: + """Install base deps plus as many optional extras as the environment supports. + + By default this targets ``.[all]``; Termux callers can pass + ``group='termux-all'`` to use the curated Android-compatible profile. + + On Windows, pre-renames live ``hermes.exe`` / ``hermes-gateway.exe`` shims + in the venv Scripts dir before each install attempt so uv can write fresh + copies (Windows blocks REPLACE on a running .exe but allows RENAME). See + ``_quarantine_running_hermes_exe`` for the rationale. + + When ``env`` carries a ``VIRTUAL_ENV`` that does not exist (a pip / + site-packages install whose ``PROJECT_ROOT`` is the interpreter's + ``site-packages`` directory, where ``PROJECT_ROOT / "venv"`` is never + created), ``uv pip`` fails with ``Failed to inspect Python interpreter from + active virtual environment`` before doing any work. Pin the install at the + running interpreter instead so the update/recovery path succeeds on those + installs (#71510 fixed the ZIP path, #83335 fixed lazy-deps; this closes the + shared helper for the remaining callers). + """ + from hermes_cli.main import _insert_python_pin, _interpreter_scripts_dir, _is_windows, _load_installable_optional_extras, _run_quarantined_install, _venv_scripts_dir, _verify_console_scripts_installed, _verify_core_dependencies_installed + scripts_dir = _venv_scripts_dir() if _is_windows() else None + + # A pip / site-packages install has no PROJECT_ROOT/venv; the caller still + # passes VIRTUAL_ENV=PROJECT_ROOT/venv, which does not exist. uv would fail + # before installing anything ("Failed to inspect Python interpreter from + # active virtual environment"). Detect the stale pointer and pin the target + # interpreter explicitly instead of trusting the nonexistent venv. + pin_python = False + if ( + env + and env.get("VIRTUAL_ENV") + and not Path(env["VIRTUAL_ENV"]).is_dir() + and install_cmd_prefix + and _is_uv_command(install_cmd_prefix) + ): + # Only uv needs the explicit pin; pip resolves the target from + # sys.executable itself and has no --python flag. + pin_python = True + env = {**env} + env.pop("VIRTUAL_ENV", None) + # When we pin to sys.executable, the entry-point shims that uv will + # rewrite live in that interpreter's Scripts/bin directory, NOT in + # PROJECT_ROOT/venv (which does not exist on a site-packages install). + # Quarantining the wrong dir means the running hermes.exe stays locked + # on Windows and the install fails exactly like the original bug. Only + # override when the venv-derived dir is missing; otherwise keep it. + if scripts_dir is None and _is_windows(): + scripts_dir = _interpreter_scripts_dir() + + def _install(args: list[str]) -> None: + if pin_python: + args = _insert_python_pin(args) + # strict_quarantine: this is the UPDATE dependency sync. A shim that + # cannot be renamed aside proves a hard venv hold; running uv anyway + # is how installs strand half-updated (#87331). ShimQuarantineError + # propagates to the update's sync boundary, which defers via the + # update-incomplete marker instead of mutating a contended venv. + _run_quarantined_install( + install_cmd_prefix + args, env=env, scripts_dir=scripts_dir, + strict_quarantine=True, + ) + + try: + _install(["install", "-e", f".[{group}]"]) + _verify_console_scripts_installed(install_cmd_prefix, env=env) + return + except subprocess.CalledProcessError: + print( + " ⚠ Optional extras failed, reinstalling base dependencies and retrying extras individually..." + ) + + _install(["install", "-e", "."]) + + failed_extras: list[str] = [] + installed_extras: list[str] = [] + for extra in _load_installable_optional_extras(group=group): + try: + _install(["install", "-e", f".[{extra}]"]) + installed_extras.append(extra) + except subprocess.CalledProcessError: + failed_extras.append(extra) + + if installed_extras: + print( + f" ✓ Reinstalled optional extras individually: {', '.join(installed_extras)}" + ) + if failed_extras: + print( + f" ⚠ Skipped optional extras that still failed: {', '.join(failed_extras)}" + ) + + # Belt-and-suspenders: verify every declared core dependency from + # pyproject.toml's [project.dependencies] is actually importable in the + # target venv. uv's incremental resolver has — in the wild — produced + # partial installs where a newly added base dep (e.g. ``pathspec``) + # silently fails to land on top of a half-stale venv, and the only + # symptom is a downstream subprocess crashing with ModuleNotFoundError + # hours later inside ``hermes update``'s desktop-rebuild or skill-sync + # stage. Reinstall with --reinstall to force resolution if anything is + # missing, then re-verify so the failure surfaces here instead of + # downstream. + _verify_core_dependencies_installed(install_cmd_prefix, env=env, group=group) + _verify_console_scripts_installed(install_cmd_prefix, env=env) + + +def _load_console_script_names() -> list[str]: + """Return ``[project.scripts]`` entry-point names from pyproject.toml.""" + from hermes_cli.main import PROJECT_ROOT + try: + import tomllib # Python 3.11+ + except ImportError: # pragma: no cover + return [] + + pyproject = PROJECT_ROOT / "pyproject.toml" + if not pyproject.is_file(): + return [] + + try: + with open(pyproject, "rb") as f: + data = tomllib.load(f) + scripts = data.get("project", {}).get("scripts", {}) or {} + return [str(name) for name in scripts if name] + except Exception as e: + logger.debug("console script verification: failed to read pyproject.toml: %s", e) + return [] + + +def _verify_console_scripts_installed( + install_cmd_prefix: list[str], + *, + env: dict[str, str] | None = None, +) -> None: + """Ensure every declared console_script shim exists on disk after install. + + On Windows, ``uv pip install -e .`` can register ``hermes.exe`` in the + wheel RECORD while the file never lands on disk — typically when the live + ``hermes.exe`` shim is locked during ``hermes update``, or when uv/distlib + skips a launcher write. The symptom is ``hermes-agent.exe`` and + ``hermes-acp.exe`` present but ``hermes.exe`` missing, so ``hermes`` drops + off PATH even though the install reported success (issue #52931). + + If any shim is missing we reinstall with ``--reinstall -e .`` under the + same quarantine dance as the primary install path, then re-check. + """ + from hermes_cli.main import _is_windows, _run_quarantined_install, _venv_scripts_dir + if not _is_windows(): + return + + scripts_dir = _venv_scripts_dir() + if scripts_dir is None: + return + + names = _load_console_script_names() + if not names: + return + + def _missing() -> list[str]: + return [ + name + for name in names + if not (scripts_dir / f"{name}.exe").is_file() + ] + + missing = _missing() + if not missing: + return + + print( + f" ⚠ Verification: {len(missing)} console script(s) missing on disk: " + f"{', '.join(missing)}" + ) + print(" → Reinstalling entry points with --reinstall...") + + try: + _run_quarantined_install( + install_cmd_prefix + ["install", "--reinstall", "-e", "."], + env=env, + scripts_dir=scripts_dir, + ) + except subprocess.CalledProcessError as e: + logger.warning("console script verification: repair install failed: %s", e) + print( + " ⚠ Entry point repair failed; try `hermes update --force` after " + "closing other hermes processes." + ) + return + + still_missing = _missing() + if still_missing: + print( + f" ⚠ Still missing after repair: {', '.join(still_missing)}. " + "Workaround: python -m hermes_cli.main " + ) + else: + print(" ✓ All console entry points restored") + + +def _verify_core_dependencies_installed( + install_cmd_prefix: list[str], + *, + env: dict[str, str] | None = None, + group: str = "all", +) -> None: + """Check that every base dep from pyproject.toml is importable; if not, retry. + + Reads ``pyproject.toml`` directly (so we don't trust the venv's stale + metadata), filters out deps gated by ``;`` environment markers that don't + apply to this platform, and runs ``importlib.metadata.version()`` in the + venv interpreter for each one. If anything is missing we reinstall the + base group with ``--reinstall`` to force uv to re-resolve, then check + again. We treat the final state as a warning rather than a hard failure + so a single broken-on-PyPI dep can't block an otherwise-successful + update — but the warning makes the partial install visible at the spot + that caused it, instead of hours later in a downstream subprocess. + """ + from hermes_cli.main import PROJECT_ROOT, _is_windows, _resolve_install_target_python, _run_install_with_heartbeat, _run_quarantined_install, _venv_scripts_dir + try: + import tomllib # Python 3.11+ + except ImportError: # pragma: no cover — Python < 3.11 unsupported but be safe + return + + pyproject = PROJECT_ROOT / "pyproject.toml" + if not pyproject.is_file(): + return + + try: + with open(pyproject, "rb") as f: + data = tomllib.load(f) + raw_deps = data.get("project", {}).get("dependencies", []) or [] + except Exception as e: + logger.debug("dep verification: failed to read pyproject.toml: %s", e) + return + + # Parse each "name OP version ; marker" string into (dist_name, marker_obj). + # We use packaging.requirements when available (it ships with pip/uv envs), + # falling back to a naive split that's good enough for the canonical + # ``name==version[; marker]`` style this repo uses. + deps: list[tuple[str, "object | None"]] = [] + try: + from packaging.requirements import Requirement # type: ignore + + for spec in raw_deps: + try: + req = Requirement(spec) + deps.append((req.name, req.marker)) + except Exception: + continue + except Exception: + for spec in raw_deps: + head = spec.split(";", 1)[0] + for op in ("==", ">=", "<=", "~=", ">", "<", "!="): + if op in head: + head = head.split(op, 1)[0] + break + name = head.strip().split("[", 1)[0].strip() + if name: + deps.append((name, None)) + + # Apply environment markers to drop deps that don't apply on this platform + # (e.g. ``ptyprocess ; sys_platform != 'win32'`` is correctly skipped on + # Windows). Without markers we'd false-positive every cross-platform exclusion. + applicable: list[str] = [] + for name, marker in deps: + if marker is None: + applicable.append(name) + continue + try: + if marker.evaluate(): # type: ignore[union-attr] + applicable.append(name) + except Exception: + applicable.append(name) + + if not applicable: + return + + # Run the check inside the venv Python — sys.executable here may be the + # outer Python that drove ``hermes update``, not the venv we just wrote + # to. The uv install_cmd_prefix encodes which environment we targeted + # (either ``[uv, pip]`` with VIRTUAL_ENV in env, or + # ``[sys.executable, -m, pip]`` for the in-process Python); resolve the + # right interpreter for the verification. + venv_python = _resolve_install_target_python(install_cmd_prefix, env) + if venv_python is None: + return + + def _missing_deps() -> list[str]: + check_script = ( + "import importlib.metadata as md, sys\n" + "missing=[]\n" + "for name in sys.argv[1:]:\n" + " try: md.version(name)\n" + " except md.PackageNotFoundError: missing.append(name)\n" + "print('\\n'.join(missing))\n" + ) + try: + result = subprocess.run( + [str(venv_python), "-c", check_script, *applicable], + capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=False, + env=env, + ) + except Exception as e: + logger.debug("dep verification: subprocess failed: %s", e) + return [] + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + missing = _missing_deps() + if not missing: + return + + print( + f" ⚠ Verification: {len(missing)} declared dep(s) missing after install: " + f"{', '.join(missing[:8])}{'...' if len(missing) > 8 else ''}" + ) + print(" → Reinstalling base group with --reinstall to repair...") + + # Reinstall base group with --reinstall so uv re-resolves from scratch + # against the current pyproject. We don't pass ``[{group}]`` here on + # purpose — the missing dep is in *base* deps; rerunning the full all- + # extras install can cost minutes and trips on whatever optional extra + # was already broken upstream. Base is fast and is what's actually wrong. + # + # Quarantine the running ``hermes.exe`` first: ``--reinstall -e .`` + # rewrites the entry-point shims, and on Windows pip can't overwrite the + # live launcher, which would leave ``hermes`` off PATH. + scripts_dir = _venv_scripts_dir() if _is_windows() else None + repair_args = ["install", "--reinstall", "-e", "."] + try: + _run_quarantined_install( + install_cmd_prefix + repair_args, env=env, scripts_dir=scripts_dir + ) + except subprocess.CalledProcessError as e: + logger.warning("dep verification: repair install failed: %s", e) + print(" ⚠ Repair install failed; check `hermes update` output above.") + return + + still_missing = _missing_deps() + if not still_missing: + print(" ✓ All declared core dependencies now installed") + return + + # Last-ditch: install each remaining missing dep with its pin directly. + # Useful when uv's resolver thinks the env is satisfied but the on-disk + # package metadata says otherwise (rare but observed). + name_to_spec = {} + for spec in raw_deps: + head = spec.split(";", 1)[0].strip() + bare = head + for op in ("==", ">=", "<=", "~=", ">", "<", "!="): + if op in bare: + bare = bare.split(op, 1)[0] + break + name_to_spec[bare.strip().split("[", 1)[0].strip()] = head + + specs = [name_to_spec.get(n, n) for n in still_missing] + print( + f" → Force-installing remaining missing dep(s): {', '.join(specs)}" + ) + try: + _run_install_with_heartbeat( + install_cmd_prefix + ["install", "--reinstall", *specs], env=env + ) + except subprocess.CalledProcessError as e: + logger.warning("dep verification: per-package repair failed: %s", e) + print( + f" ⚠ Could not install: {', '.join(still_missing)}. " + "Run `hermes update --force` after closing other hermes processes." + ) + return + + final_missing = _missing_deps() + if final_missing: + print( + f" ⚠ Still missing after repair: {', '.join(final_missing)}. " + "Run `hermes update --force` after closing other hermes processes." + ) + else: + print(" ✓ All declared core dependencies now installed") + + +def _resolve_install_target_python( + install_cmd_prefix: list[str], env: dict[str, str] | None +) -> Path | None: + """Figure out which Python interpreter the install just targeted. + + ``_install_python_dependencies_with_optional_fallback`` is called with + either ``[uv, pip]`` (and a ``VIRTUAL_ENV`` env var pointing at the + target venv) or ``[sys.executable, -m, pip]`` (the in-process Python). + The verification step needs the *resulting* environment's Python so + ``importlib.metadata`` queries the right site-packages. + """ + from hermes_cli.main import _is_windows + if env and "VIRTUAL_ENV" in env: + from hermes_constants import venv_python_path + + venv_root = Path(env["VIRTUAL_ENV"]) + candidate = venv_python_path(venv_root, windows=_is_windows()) + if candidate.exists(): + return candidate + + # Fallback: assume install_cmd_prefix[0] is the python interpreter (the + # ``[sys.executable, -m, pip]`` shape). Skip if it looks like ``uv``. + if install_cmd_prefix: + first = Path(install_cmd_prefix[0]) + if first.exists() and "uv" not in first.name.lower(): + return first + + return None + + +def _is_termux_env(env: dict[str, str] | None = None) -> bool: + from hermes_cli.main import _is_termux_startup_environment + return _is_termux_startup_environment(env) + + +def _is_windows_npm_path(npm_path: str) -> bool: + """Return True if ``npm_path`` points at a Windows npm shim. + + On WSL the Windows install dir is exposed through the ``/mnt/c`` drive + mount and PATH interop, so ``shutil.which("npm")`` can hand back + ``/mnt/c/Program Files/nodejs/npm`` (or the ``npm.cmd`` / ``npm.exe`` + shim). Those are detected here by their ``.exe``/``.cmd``/``.bat`` + suffix, a ``/mnt/`` drive-mount prefix, or an embedded backslash (a UNC + path). Callers use this only on a POSIX host — on native Windows an + ``npm.cmd`` shim is the correct executable. + """ + low = npm_path.lower() + return ( + low.endswith((".exe", ".cmd", ".bat")) + or low.startswith("/mnt/") + or "\\" in npm_path + ) + + +def _resolve_node_runtime_npm() -> str | None: + """Resolve an npm executable that belongs to the host's Node runtime. + + On WSL/Linux ``shutil.which("npm")`` may resolve a Windows npm exposed + through PATH interop. Running that Windows npm against the Linux checkout + operates over ``\\wsl.localhost\\...`` UNC paths and fails with EISDIR / + symlink errors in symlink-heavy trees like ``ui-tui`` (#30271). Refuse a + Windows npm on a POSIX host and re-scan PATH (skipping ``/mnt/*`` interop + entries) for a Linux-native npm. Returns the npm path, or ``None`` when + no suitable npm is reachable. + """ + from hermes_cli.main import _is_windows + from hermes_constants import find_node_executable + + npm = find_node_executable("npm") + + # On native Windows the platform npm (``npm.cmd``) is exactly what we + # want — only reject Windows shims when we're a POSIX/WSL process. + if _is_windows(): + return npm + + if not npm: + return None + + if not _is_windows_npm_path(npm): + return npm + + # The first resolution was a Windows npm. Re-scan PATH skipping the + # ``/mnt/*`` Windows drive mounts WSL injects, so a Linux-native npm that + # came later on PATH is still found. + for directory in os.environ.get("PATH", "").split(os.pathsep): + if not directory or directory.lower().startswith("/mnt/"): + continue + candidate = shutil.which("npm", path=directory) + if candidate and not _is_windows_npm_path(candidate): + return candidate + return None + + +def _resolve_update_branch(args) -> str: + """Normalize ``args.branch`` into a non-empty branch name. + + Centralizes the "default to main, accept --branch override, treat empty + or whitespace-only values as the default" parsing so every consumer of + ``--branch`` (check path, git-update path, ZIP-fallback path) agrees on + the same answer. + """ + return (getattr(args, "branch", None) or "main").strip() or "main" diff --git a/tests/test_managed_runtime_resolution.py b/tests/test_managed_runtime_resolution.py index 30ae739e0c..1cfbccf8f3 100644 --- a/tests/test_managed_runtime_resolution.py +++ b/tests/test_managed_runtime_resolution.py @@ -92,7 +92,7 @@ _ALLOWED: dict[tuple[str, str], str] = { ("hermes_cli/main_tui_launch.py", "npm"): ( "Same _ensure_tui_node() gate as node." ), - ("hermes_cli/main.py", "npm"): ( + ("hermes_cli/main_install_repair.py", "npm"): ( "_resolve_node_runtime_npm()'s WSL re-scan: PATH minus /mnt/* IS the question." ), ("tools/browser_tool.py", "npx"): (