diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index 74a950db8b..1972898abd 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -143,6 +143,31 @@ def _scan_dashboard_processes( dashboard_processes = [ proc for proc in dashboard_processes if proc[0] not in exclude_pids ] + + # Spawn-ledger augmentation (#63206/#81564): the substring patterns above + # miss profiled launches — `hermes --profile p serve --host ` contains + # neither "hermes serve" nor "hermes_cli.main serve". Every serve/ + # dashboard registers itself in the machine spawn ledger at startup with + # live-verified (pid, create_time), so ledger rows are positive identity, + # not argv guessing. Add any live ledger serve/dashboard the scan missed; + # prefer the ledger's recorded argv (full launch args) over the scan's + # truncated view. + try: + from hermes_cli.process_identity import ledger_entries + + seen = {pid for pid, _ in dashboard_processes} + for entry in ledger_entries(): + if entry.get("purpose") not in ("serve", "dashboard"): + continue + pid = entry.get("pid") + if not isinstance(pid, int) or pid == self_pid or pid in seen: + continue + if exclude_pids and pid in exclude_pids: + continue + dashboard_processes.append((pid, str(entry.get("argv") or ""))) + except Exception: + pass # ledger unavailable → scan-only behavior, exactly as before + return dashboard_processes diff --git a/hermes_cli/main.py b/hermes_cli/main.py index f3d3b4f0b5..65a63f0b99 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -5094,6 +5094,9 @@ _LAZY_COMMAND_EXPORTS = { "_is_android_python", "_is_fork", "_leftover_pausable_gateway_pids", + "_ledger_manual_serve_holders", + "_relaunch_stopped_serves", + "_serve_relaunch_commands", "_log_only_write", "_mark_skip_upstream_prompt", "_npm_bin_exists", @@ -11619,30 +11622,35 @@ def _render_distribution_plan(plan) -> None: def _report_dashboard_status() -> int: - """Print live listening dashboard processes and return the count.""" + """Print live listening dashboard/serve processes and return the count. + + Serve-mode backends are INCLUDED (#81564): `--stop` kills them, so + `--status` hiding them left Desktop SSH backends invisible to the CLI — + an operator could kill what they couldn't see. Ledger-registered serves + (profiled launches the argv scan can't match) surface via the + spawn-ledger augmentation in _scan_dashboard_processes. + """ from gateway.status import _pid_exists - live: list[tuple[int, str]] = [] + live: list[tuple[int, str, str]] = [] for pid, command in _self()._scan_dashboard_processes(): runtime = _parse_dashboard_runtime(command) if runtime is None: continue mode, host, port = runtime - if mode != "dashboard": - continue if port <= 0 or not _pid_exists(pid): continue if not _dashboard_listening(host, port): continue - live.append((pid, command)) + live.append((pid, command, mode)) if not live: - print("No hermes dashboard processes running.") + print("No hermes dashboard or serve processes running.") return 0 - print(f"{len(live)} hermes dashboard process(es) running:") - for pid, command in live: - print(f" PID {pid}: {command}") + print(f"{len(live)} hermes dashboard/serve process(es) running:") + for pid, command, mode in live: + print(f" PID {pid} [{mode}]: {command}") return len(live) diff --git a/hermes_cli/process_identity.py b/hermes_cli/process_identity.py index bac5e25bc1..81deb1d42e 100644 --- a/hermes_cli/process_identity.py +++ b/hermes_cli/process_identity.py @@ -161,6 +161,13 @@ class LedgerEntry: spawner_create: Optional[float] registered_at: float argv: str + # Structured launch identity (#63206): what a relauncher needs to bring + # this runtime back after an update, without parsing argv. Empty for + # purposes that don't supply it; readers must use .get() — older ledger + # files on disk predate these keys. + host: str = "" + port: Optional[int] = None + profile: str = "" def _ledger_path() -> Path: @@ -224,13 +231,23 @@ def _pid_alive_matches(pid: int, create_time: Optional[float]) -> Optional[bool] return None -def register_self(purpose: str, *, project_root: Optional[Path] = None) -> bool: +def register_self( + purpose: str, + *, + project_root: Optional[Path] = None, + detail: Optional[dict] = None, +) -> bool: """Record this process in the machine spawn ledger. Best-effort. Called at the top of every long-lived entry point (serve/dashboard backend, gateway run loop). Dead entries — ``(pid, create_time)`` no longer live — are pruned on every write so the ledger tracks reality instead of growing forever. + + ``detail`` optionally carries structured launch identity (#63206) — + ``host``/``port``/``profile`` — so the update pipeline can relaunch a + manually-started serve with its real bind address instead of guessing + from argv. """ tag = parse_spawn_tag(os.environ.get(SPAWN_ENV_VAR)) spawner_pid: Optional[int] = tag.spawner_pid if tag else None @@ -262,10 +279,22 @@ def register_self(purpose: str, *, project_root: Optional[Path] = None) -> bool: registered_at=time.time(), argv="", ) + if detail: + try: + entry.host = str(detail.get("host") or "") + port = detail.get("port") + entry.port = int(port) if port is not None else None + entry.profile = str(detail.get("profile") or "") + except (TypeError, ValueError): + pass try: import sys as _sys - entry.argv = " ".join(_sys.argv[:6]) + # 10 tokens (was 6): enough for `hermes serve --host X --port N + # --profile P` — the relaunch shapes #63206 needs — while still + # bounding pathological argv. Structured detail above is the + # canonical identity; argv is the human-readable fallback. + entry.argv = " ".join(_sys.argv[:10]) except Exception: pass diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index f9e339aa4b..fa527fe55a 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -4613,6 +4613,113 @@ def _leftover_pausable_gateway_pids( return pids +def _ledger_manual_serve_holders( + matches: list[tuple[int, str, str]], +) -> list[dict]: + """Ledger entries for venv holders that are MANUAL serve/dashboard backends. + + Positive identity only (#63206): the process self-registered in the spawn + ledger with purpose serve/dashboard, its (pid, create_time) still matches + a live process, and its recorded spawner is NOT alive (a Desktop-owned + backend keeps its live Electron spawner and must keep the refusal — the + app would respawn what we kill; a PowerShell-launched serve has no live + Hermes spawner). Returns the full ledger entries so the relauncher can + rebuild the launch command from structured host/port/profile instead of + parsing argv. + """ + try: + from hermes_cli.process_identity import ledger_entries, spawner_is_dead + except Exception: + return [] + holder_pids = {int(pid) for pid, _name, _cmd in matches} + out: list[dict] = [] + for entry in ledger_entries(): + if entry.get("purpose") not in ("serve", "dashboard"): + continue + pid = entry.get("pid") + if not isinstance(pid, int) or pid not in holder_pids: + continue + if spawner_is_dead(entry) is False: + continue # live Desktop supervisor owns it — keep refusing + out.append(entry) + return out + + +def _serve_relaunch_commands(entries: list[dict]) -> list[list[str]]: + """Rebuild launch commands for stopped serves from structured identity. + + Uses the ledger's host/port/profile fields — never argv parsing (a + joined argv string cannot round-trip Windows paths with spaces). Entries + without a recorded port are skipped; the caller prints the manual hint + for those. + """ + commands: list[list[str]] = [] + hermes = None + try: + scripts_dir = _m()._venv_scripts_dir() + if scripts_dir is not None: + for name in ("hermes.exe", "hermes"): + candidate = scripts_dir / name + if candidate.is_file(): + hermes = str(candidate) + break + except Exception: + hermes = None + if hermes is None: + hermes = "hermes" + for entry in entries: + port = entry.get("port") + if not isinstance(port, int) or port <= 0: + continue + cmd = [hermes] + profile = str(entry.get("profile") or "") + if profile and profile != "default": + cmd += ["--profile", profile] + cmd.append(str(entry.get("purpose"))) + host = str(entry.get("host") or "") + if host: + cmd += ["--host", host] + cmd += ["--port", str(port)] + commands.append(cmd) + return commands + + +def _relaunch_stopped_serves(token: dict) -> None: + """Idempotent atexit relaunch of manual serves stopped by the venv guard. + + Mirrors the gateway resume token contract: `pending` flips False on the + first invocation so the explicit call and the atexit registration cannot + double-spawn (#63206). + """ + if not token.get("pending"): + return + token["pending"] = False + entries = token.get("entries") or [] + if not entries: + return + commands = _serve_relaunch_commands(entries) + skipped = len(entries) - len(commands) + failed: list = [] + if commands: + print(" ⟲ Relaunching stopped serve/dashboard backend(s)") + failed = _m()._respawn_dashboard_processes(commands) + if skipped or failed: + print( + " ⚠ Some stopped backends could not be relaunched automatically; " + "restart them manually (hermes serve --host --port )." + ) + try: + from hermes_cli.update_receipt import record_step + + record_step( + "serve_relaunch", + not failed and not skipped, + f"relaunched={len(commands) - len(failed)} failed={len(failed)} skipped={skipped}", + ) + except Exception: + pass + + def _orphaned_desktop_backend_pids( matches: list[tuple[int, str, str]], ) -> list[int] | None: @@ -6100,6 +6207,49 @@ def _cmd_update_impl(args, gateway_mode: bool): _m()._stop_process_trees(_orphan_backends) _time.sleep(1.0) _venv_holders = _m()._detect_venv_python_processes() + if _venv_holders: + # Manual serve/dashboard rung (#63206): a network-bound + # `hermes serve --host ` powering a REMOTE Desktop holds the + # venv and used to dead-end the update with exit 2 — the user's + # only option was killing the backend by hand, and nothing ever + # brought it back (the remote client's endpoint stayed dead). + # Positive ledger identity only: self-registered serve/dashboard + # whose recorded spawner is not alive (Desktop-owned backends + # keep the refusal — the app respawns what we kill). Stop them, + # and register an idempotent atexit relaunch built from the + # ledger's structured host/port/profile so the endpoint comes + # back on the SAME bind after the update — success or failure. + _serve_entries = _m()._ledger_manual_serve_holders(_venv_holders) + if _serve_entries: + print( + f" ⚠ {len(_serve_entries)} manual serve/dashboard " + "backend(s) hold the venv; stopping them for the update " + "(they will be relaunched on their recorded endpoints)" + ) + _m()._stop_process_trees( + [int(e["pid"]) for e in _serve_entries] + ) + _serve_resume_token = { + "pending": True, + "entries": _serve_entries, + } + try: + from hermes_cli.update_receipt import record_step + + record_step( + "serve_pause", + True, + f"stopped={len(_serve_entries)}", + ) + except Exception: + pass + import atexit as _serve_atexit + + _serve_atexit.register( + _m()._relaunch_stopped_serves, _serve_resume_token + ) + _time.sleep(1.0) + _venv_holders = _m()._detect_venv_python_processes() if _venv_holders: # Final rung before the dead-end: a GUI-updater hand-off # (`update --gateway --force` with the update-incomplete marker diff --git a/hermes_cli/update_inventory.py b/hermes_cli/update_inventory.py index c9e2bdc6d2..4b8f2234bc 100644 --- a/hermes_cli/update_inventory.py +++ b/hermes_cli/update_inventory.py @@ -109,6 +109,8 @@ def _restart_mechanism(supervisor: str, profile: str) -> str: return "launchd" if supervisor == "desktop": return "desktop" + if supervisor == "manual-serve": + return "respawn-argv" return "manual" @@ -120,6 +122,8 @@ def describe_restart_mechanism(mechanism: str, profile: str) -> str: return "launchctl kickstart -k (drain-first, per-label domain)" if mechanism == "desktop": return "Desktop app respawns its serve backend" + if mechanism == "respawn-argv": + return "stop before code swap, relaunch with recorded launch args" if profile != "default": return f"hermes -p {profile} gateway restart" return "hermes gateway restart" @@ -292,6 +296,46 @@ def collect_runtime_inventory() -> UpdatePlan: except Exception as exc: logger.debug("PID-file gateway inventory failed: %s", exc) + # Serve/dashboard backends from the spawn ledger (#63206). These are the + # runtimes the gateway collectors above can never see: a manually + # launched `hermes serve --host ` for a remote Desktop, or a + # long-lived `hermes dashboard`. Every serve/dashboard registers itself + # (with structured host/port/profile since #63206) at startup, and + # ledger_entries() live-verifies (pid, create_time) so PID reuse never + # fabricates a row. Desktop-supervised backends are classified by their + # recorded spawner still being alive — those restart via the Desktop's + # own respawn, not ours. + try: + from hermes_cli.process_identity import ledger_entries, spawner_is_dead + + for entry in ledger_entries(): + purpose = entry.get("purpose") + if purpose not in ("serve", "dashboard"): + continue + pid = entry.get("pid") + if not isinstance(pid, int) or pid in seen_pids: + continue + seen_pids.add(pid) + has_live_spawner = spawner_is_dead(entry) is False + supervisor = "desktop" if has_live_spawner else "manual-serve" + profile = str(entry.get("profile") or "default") + plan.runtimes.append( + RuntimeRecord( + kind=str(purpose), + profile=profile, + pid=pid, + supervisor=supervisor, + restart_via=_restart_mechanism(supervisor, profile), + detail={ + "argv": entry.get("argv") or "", + "host": entry.get("host") or "", + "port": entry.get("port"), + }, + ) + ) + except Exception as exc: + logger.debug("Serve/dashboard ledger inventory failed: %s", exc) + return plan diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index e4f1102a26..0751e8746f 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -19801,24 +19801,35 @@ def start_server( # for standalone `hermes serve` (no HERMES_PARENT_PID env). _start_parent_death_watchdog() + actual_port = _read_bound_port(server, fallback=port) + app.state.bound_port = actual_port + # Positive process identity: record (pid, create_time, purpose, # spawner) in the machine spawn ledger and — on Windows — attach # to a kill-on-close job so this backend's whole child tree dies # with it. Both best-effort; failures degrade to legacy behavior. + # Registered AFTER the bind so the entry carries the ACTUAL port + # (ephemeral binds included) — the structured host/port/profile + # is what lets `hermes update` relaunch a manually-started serve + # on its real endpoint instead of dropping it (#63206). try: from hermes_cli.process_identity import ( attach_self_to_kill_on_close_job, register_self, ) - register_self("serve" if headless else "dashboard") + register_self( + "serve" if headless else "dashboard", + detail={ + "host": host, + "port": actual_port, + "profile": initial_profile or "", + }, + ) attach_self_to_kill_on_close_job() except Exception as exc: _log.debug("process-identity registration skipped: %s", exc) - actual_port = _read_bound_port(server, fallback=port) - app.state.bound_port = actual_port - _write_dashboard_ready_file(actual_port) # Port-discovery sentinel parsed by the desktop spawn. `serve` is a # plain backend, not a dashboard, so it announces a neutral token; diff --git a/tests/hermes_cli/test_serve_runtime_inventory.py b/tests/hermes_cli/test_serve_runtime_inventory.py new file mode 100644 index 0000000000..ce3ad99cc3 --- /dev/null +++ b/tests/hermes_cli/test_serve_runtime_inventory.py @@ -0,0 +1,218 @@ +"""Serve-kind runtime inventory + stop/relaunch rung (#63206, campaign #91277). + +A network-bound `hermes serve --host ` powering a remote Desktop used to +be invisible to the update pipeline: not in the inventory, a dead-end at the +venv-holder guard, and never relaunched after `hermes update` killed it. The +fix threads the spawn ledger's structured launch identity (host/port/profile, +registered at serve startup) through inventory → guard rung → relaunch. +""" + +from __future__ import annotations + +import sys +from types import SimpleNamespace +from unittest.mock import patch # noqa: F401 - kept for parity with siblings + +import hermes_cli.update_cmd as update_cmd +import hermes_cli.update_inventory as update_inventory +from hermes_cli import main as cli_main + + +def _ledger_entry(**over): + entry = { + "pid": 4321, + "create_time": 111.0, + "purpose": "serve", + "install": "inst", + "spawner_pid": None, + "spawner_create": None, + "registered_at": 222.0, + "argv": "hermes serve --host 100.94.65.93 --port 9119", + "host": "100.94.65.93", + "port": 9119, + "profile": "", + } + entry.update(over) + return entry + + +# --------------------------------------------------------------------------- +# process_identity: structured detail round-trip +# --------------------------------------------------------------------------- + + +def test_register_self_records_structured_detail(tmp_path, monkeypatch): + from hermes_cli import process_identity as pi + + monkeypatch.setattr(pi, "_ledger_path", lambda: tmp_path / "ledger.json") + monkeypatch.setattr(pi, "install_id", lambda *a, **k: "inst") + assert pi.register_self( + "serve", detail={"host": "100.94.65.93", "port": 9119, "profile": "work"} + ) + entries = [ + e + for e in pi._read_ledger(tmp_path / "ledger.json") + if e["purpose"] == "serve" + ] + assert entries, "serve entry must be written" + e = entries[-1] + assert e["host"] == "100.94.65.93" + assert e["port"] == 9119 + assert e["profile"] == "work" + + +def test_register_self_without_detail_stays_backward_compatible( + tmp_path, monkeypatch +): + from hermes_cli import process_identity as pi + + monkeypatch.setattr(pi, "_ledger_path", lambda: tmp_path / "ledger.json") + monkeypatch.setattr(pi, "install_id", lambda *a, **k: "inst") + assert pi.register_self("gateway") + e = pi._read_ledger(tmp_path / "ledger.json")[-1] + assert e["host"] == "" and e["port"] is None and e["profile"] == "" + + +# --------------------------------------------------------------------------- +# update_inventory: serve collector +# --------------------------------------------------------------------------- + + +def test_inventory_includes_manual_serve_from_ledger(monkeypatch): + entry = _ledger_entry() + fake_pi = SimpleNamespace( + ledger_entries=lambda **k: [entry], + spawner_is_dead=lambda e: None, # no spawner recorded → manual + ) + monkeypatch.setitem(sys.modules, "hermes_cli.process_identity", fake_pi) + plan = update_inventory.collect_runtime_inventory() + serves = [r for r in plan.runtimes if r.kind == "serve"] + assert serves, "manual serve must appear in the inventory" + row = serves[0] + assert row.pid == 4321 + assert row.supervisor == "manual-serve" + assert row.restart_via == "respawn-argv" + assert row.detail["host"] == "100.94.65.93" + assert row.detail["port"] == 9119 + + +def test_inventory_classifies_desktop_owned_serve(monkeypatch): + entry = _ledger_entry(spawner_pid=999, spawner_create=1.0) + fake_pi = SimpleNamespace( + ledger_entries=lambda **k: [entry], + spawner_is_dead=lambda e: False, # Electron parent alive + ) + monkeypatch.setitem(sys.modules, "hermes_cli.process_identity", fake_pi) + plan = update_inventory.collect_runtime_inventory() + serves = [r for r in plan.runtimes if r.kind == "serve"] + assert serves and serves[0].supervisor == "desktop" + assert serves[0].restart_via == "desktop" + + +def test_describe_restart_mechanism_respawn_argv(): + text = update_inventory.describe_restart_mechanism("respawn-argv", "default") + assert "relaunch" in text + + +# --------------------------------------------------------------------------- +# update_cmd: guard rung helpers +# --------------------------------------------------------------------------- + + +def test_ledger_manual_serve_holders_filters_correctly(monkeypatch): + manual = _ledger_entry(pid=100) + desktop_owned = _ledger_entry(pid=200, spawner_pid=999, spawner_create=1.0) + gateway = _ledger_entry(pid=300, purpose="gateway") + not_a_holder = _ledger_entry(pid=400) + + fake_pi = SimpleNamespace( + ledger_entries=lambda **k: [manual, desktop_owned, gateway, not_a_holder], + spawner_is_dead=lambda e: False if e["pid"] == 200 else None, + ) + monkeypatch.setitem(sys.modules, "hermes_cli.process_identity", fake_pi) + holders = [(100, "python.exe", "..."), (200, "python.exe", "..."), (300, "python.exe", "...")] + + result = update_cmd._ledger_manual_serve_holders(holders) + pids = [e["pid"] for e in result] + assert pids == [100], ( + "only the manual serve holder qualifies: desktop-owned keeps the " + "refusal, gateways belong to the pause machinery, non-holders skipped" + ) + + +def test_serve_relaunch_commands_built_from_structured_identity(monkeypatch): + monkeypatch.setattr(cli_main, "_venv_scripts_dir", lambda: None) + entries = [ + _ledger_entry(), # default profile + _ledger_entry(pid=5000, profile="work", port=9200, host=""), + _ledger_entry(pid=6000, port=None), # no port → skipped + _ledger_entry(pid=7000, purpose="dashboard", host="0.0.0.0", port=9300), + ] + cmds = update_cmd._serve_relaunch_commands(entries) + assert ["hermes", "serve", "--host", "100.94.65.93", "--port", "9119"] in cmds + assert ["hermes", "--profile", "work", "serve", "--port", "9200"] in cmds + assert ["hermes", "dashboard", "--host", "0.0.0.0", "--port", "9300"] in cmds + assert len(cmds) == 3 # the port-less entry is skipped + + +def test_relaunch_stopped_serves_is_idempotent(monkeypatch): + calls = [] + monkeypatch.setattr( + cli_main, "_respawn_dashboard_processes", lambda cmds: calls.append(cmds) or [] + ) + monkeypatch.setattr(cli_main, "_venv_scripts_dir", lambda: None) + token = {"pending": True, "entries": [_ledger_entry()]} + + update_cmd._relaunch_stopped_serves(token) + update_cmd._relaunch_stopped_serves(token) # atexit double-fire + + assert len(calls) == 1, "relaunch must fire exactly once" + assert token["pending"] is False + + +def test_relaunch_stopped_serves_untriggered_token_noop(monkeypatch): + calls = [] + monkeypatch.setattr( + cli_main, "_respawn_dashboard_processes", lambda cmds: calls.append(cmds) or [] + ) + update_cmd._relaunch_stopped_serves({"pending": False, "entries": [_ledger_entry()]}) + assert calls == [] + + +# --------------------------------------------------------------------------- +# dashboard_procs: ledger augmentation of the scan (#81564 half) +# --------------------------------------------------------------------------- + + +def test_scan_dashboard_processes_includes_ledger_only_serves(monkeypatch): + """A profiled serve (`hermes --profile p serve ...`) matches no scan + pattern; the ledger row must still surface it.""" + import hermes_cli.dashboard_procs as dp + + profiled = _ledger_entry( + pid=8123, + argv="hermes --profile work serve --host 100.94.65.93 --port 9119", + profile="work", + ) + fake_pi = SimpleNamespace(ledger_entries=lambda **k: [profiled]) + monkeypatch.setitem(sys.modules, "hermes_cli.process_identity", fake_pi) + + # Force the ps/wmic scan itself to find nothing. + fake_run = SimpleNamespace(returncode=0, stdout="") + monkeypatch.setattr( + dp.subprocess, "run", lambda *a, **k: fake_run + ) + result = dp._scan_dashboard_processes() + assert (8123, profiled["argv"]) in result + + +def test_scan_dashboard_processes_ledger_respects_exclusions(monkeypatch): + import hermes_cli.dashboard_procs as dp + + entry = _ledger_entry(pid=8124) + fake_pi = SimpleNamespace(ledger_entries=lambda **k: [entry]) + monkeypatch.setitem(sys.modules, "hermes_cli.process_identity", fake_pi) + fake_run = SimpleNamespace(returncode=0, stdout="") + monkeypatch.setattr(dp.subprocess, "run", lambda *a, **k: fake_run) + + assert dp._scan_dashboard_processes(exclude_pids={8124}) == []