feat(update): network-bound serve backends survive hermes update on their recorded endpoints (#63206)

A manually-launched `hermes serve --host <ip>` powering a remote Desktop
was invisible to the entire update pipeline: not in the runtime
inventory, a permanent exit-2 dead-end at the Windows venv-holder guard,
and — when anything killed it — never relaunched, stranding the remote
client on a dead endpoint (#63206). Serve backends were also visible to
`hermes dashboard --stop` but hidden from `--status` (#81564's
asymmetry), so operators could kill what they couldn't see.

Built on the spawn ledger (positive identity, never argv guessing):

- process_identity.py: LedgerEntry gains structured host/port/profile
  (backward-compatible — readers .get()); register_self accepts detail=;
  argv capture widened 6→10 tokens so profiled launches survive.
- web_server.py: serve/dashboard registration moved AFTER the bind and
  now records the ACTUAL bound host/port/profile.
- update_inventory.py: serve/dashboard collector reading the ledger —
  manual backends inventory as supervisor=manual-serve with
  restart_via=respawn-argv; Desktop-owned ones (live recorded spawner)
  as desktop. Plan/receipts/fleet matrix see them for free.
- update_cmd.py: new venv-guard rung — manual serve/dashboard holders
  are stopped for the update and relaunched via an idempotent atexit
  token built from structured identity (same contract as the gateway
  pause/resume); receipts record serve_pause/serve_relaunch.
  Desktop-owned backends keep the refusal (the app respawns what we
  kill).
- dashboard_procs.py: the process scan is augmented with live ledger
  rows, so profiled launches (`hermes --profile p serve ...`) that match
  no substring pattern are finally visible to kill/respawn.
- main.py: `--status` now lists serve-mode backends too, tagged [serve]
  — closing the #81564 status/stop asymmetry.

Salvage note: detection deliberately does NOT reuse #70742's psutil
cmdline-pattern scan (the argv-guessing class this campaign retires);
its resume-token lifecycle (atexit + idempotent flag) and don't-replay
guard shaped the relaunch contract here — credit @Tranquil-Flow.

Co-authored-by: Tranquil-Flow <66773372+Tranquil-Flow@users.noreply.github.com>
This commit is contained in:
Teknium
2026-08-26 06:49:11 -07:00
parent 64424a16a2
commit 27385e586b
7 changed files with 500 additions and 15 deletions
+25
View File
@@ -143,6 +143,31 @@ def _scan_dashboard_processes(
dashboard_processes = [
proc for proc in dashboard_processes if proc[0] not in exclude_pids
]
# Spawn-ledger augmentation (#63206/#81564): the substring patterns above
# miss profiled launches — `hermes --profile p serve --host <ip>` contains
# neither "hermes serve" nor "hermes_cli.main serve". Every serve/
# dashboard registers itself in the machine spawn ledger at startup with
# live-verified (pid, create_time), so ledger rows are positive identity,
# not argv guessing. Add any live ledger serve/dashboard the scan missed;
# prefer the ledger's recorded argv (full launch args) over the scan's
# truncated view.
try:
from hermes_cli.process_identity import ledger_entries
seen = {pid for pid, _ in dashboard_processes}
for entry in ledger_entries():
if entry.get("purpose") not in ("serve", "dashboard"):
continue
pid = entry.get("pid")
if not isinstance(pid, int) or pid == self_pid or pid in seen:
continue
if exclude_pids and pid in exclude_pids:
continue
dashboard_processes.append((pid, str(entry.get("argv") or "")))
except Exception:
pass # ledger unavailable → scan-only behavior, exactly as before
return dashboard_processes
+17 -9
View File
@@ -5094,6 +5094,9 @@ _LAZY_COMMAND_EXPORTS = {
"_is_android_python",
"_is_fork",
"_leftover_pausable_gateway_pids",
"_ledger_manual_serve_holders",
"_relaunch_stopped_serves",
"_serve_relaunch_commands",
"_log_only_write",
"_mark_skip_upstream_prompt",
"_npm_bin_exists",
@@ -11619,30 +11622,35 @@ def _render_distribution_plan(plan) -> None:
def _report_dashboard_status() -> int:
"""Print live listening dashboard processes and return the count."""
"""Print live listening dashboard/serve processes and return the count.
Serve-mode backends are INCLUDED (#81564): `--stop` kills them, so
`--status` hiding them left Desktop SSH backends invisible to the CLI —
an operator could kill what they couldn't see. Ledger-registered serves
(profiled launches the argv scan can't match) surface via the
spawn-ledger augmentation in _scan_dashboard_processes.
"""
from gateway.status import _pid_exists
live: list[tuple[int, str]] = []
live: list[tuple[int, str, str]] = []
for pid, command in _self()._scan_dashboard_processes():
runtime = _parse_dashboard_runtime(command)
if runtime is None:
continue
mode, host, port = runtime
if mode != "dashboard":
continue
if port <= 0 or not _pid_exists(pid):
continue
if not _dashboard_listening(host, port):
continue
live.append((pid, command))
live.append((pid, command, mode))
if not live:
print("No hermes dashboard processes running.")
print("No hermes dashboard or serve processes running.")
return 0
print(f"{len(live)} hermes dashboard process(es) running:")
for pid, command in live:
print(f" PID {pid}: {command}")
print(f"{len(live)} hermes dashboard/serve process(es) running:")
for pid, command, mode in live:
print(f" PID {pid} [{mode}]: {command}")
return len(live)
+31 -2
View File
@@ -161,6 +161,13 @@ class LedgerEntry:
spawner_create: Optional[float]
registered_at: float
argv: str
# Structured launch identity (#63206): what a relauncher needs to bring
# this runtime back after an update, without parsing argv. Empty for
# purposes that don't supply it; readers must use .get() — older ledger
# files on disk predate these keys.
host: str = ""
port: Optional[int] = None
profile: str = ""
def _ledger_path() -> Path:
@@ -224,13 +231,23 @@ def _pid_alive_matches(pid: int, create_time: Optional[float]) -> Optional[bool]
return None
def register_self(purpose: str, *, project_root: Optional[Path] = None) -> bool:
def register_self(
purpose: str,
*,
project_root: Optional[Path] = None,
detail: Optional[dict] = None,
) -> bool:
"""Record this process in the machine spawn ledger. Best-effort.
Called at the top of every long-lived entry point (serve/dashboard
backend, gateway run loop). Dead entries — ``(pid, create_time)`` no
longer live — are pruned on every write so the ledger tracks reality
instead of growing forever.
``detail`` optionally carries structured launch identity (#63206) —
``host``/``port``/``profile`` — so the update pipeline can relaunch a
manually-started serve with its real bind address instead of guessing
from argv.
"""
tag = parse_spawn_tag(os.environ.get(SPAWN_ENV_VAR))
spawner_pid: Optional[int] = tag.spawner_pid if tag else None
@@ -262,10 +279,22 @@ def register_self(purpose: str, *, project_root: Optional[Path] = None) -> bool:
registered_at=time.time(),
argv="",
)
if detail:
try:
entry.host = str(detail.get("host") or "")
port = detail.get("port")
entry.port = int(port) if port is not None else None
entry.profile = str(detail.get("profile") or "")
except (TypeError, ValueError):
pass
try:
import sys as _sys
entry.argv = " ".join(_sys.argv[:6])
# 10 tokens (was 6): enough for `hermes serve --host X --port N
# --profile P` — the relaunch shapes #63206 needs — while still
# bounding pathological argv. Structured detail above is the
# canonical identity; argv is the human-readable fallback.
entry.argv = " ".join(_sys.argv[:10])
except Exception:
pass
+150
View File
@@ -4613,6 +4613,113 @@ def _leftover_pausable_gateway_pids(
return pids
def _ledger_manual_serve_holders(
matches: list[tuple[int, str, str]],
) -> list[dict]:
"""Ledger entries for venv holders that are MANUAL serve/dashboard backends.
Positive identity only (#63206): the process self-registered in the spawn
ledger with purpose serve/dashboard, its (pid, create_time) still matches
a live process, and its recorded spawner is NOT alive (a Desktop-owned
backend keeps its live Electron spawner and must keep the refusal — the
app would respawn what we kill; a PowerShell-launched serve has no live
Hermes spawner). Returns the full ledger entries so the relauncher can
rebuild the launch command from structured host/port/profile instead of
parsing argv.
"""
try:
from hermes_cli.process_identity import ledger_entries, spawner_is_dead
except Exception:
return []
holder_pids = {int(pid) for pid, _name, _cmd in matches}
out: list[dict] = []
for entry in ledger_entries():
if entry.get("purpose") not in ("serve", "dashboard"):
continue
pid = entry.get("pid")
if not isinstance(pid, int) or pid not in holder_pids:
continue
if spawner_is_dead(entry) is False:
continue # live Desktop supervisor owns it — keep refusing
out.append(entry)
return out
def _serve_relaunch_commands(entries: list[dict]) -> list[list[str]]:
"""Rebuild launch commands for stopped serves from structured identity.
Uses the ledger's host/port/profile fields — never argv parsing (a
joined argv string cannot round-trip Windows paths with spaces). Entries
without a recorded port are skipped; the caller prints the manual hint
for those.
"""
commands: list[list[str]] = []
hermes = None
try:
scripts_dir = _m()._venv_scripts_dir()
if scripts_dir is not None:
for name in ("hermes.exe", "hermes"):
candidate = scripts_dir / name
if candidate.is_file():
hermes = str(candidate)
break
except Exception:
hermes = None
if hermes is None:
hermes = "hermes"
for entry in entries:
port = entry.get("port")
if not isinstance(port, int) or port <= 0:
continue
cmd = [hermes]
profile = str(entry.get("profile") or "")
if profile and profile != "default":
cmd += ["--profile", profile]
cmd.append(str(entry.get("purpose")))
host = str(entry.get("host") or "")
if host:
cmd += ["--host", host]
cmd += ["--port", str(port)]
commands.append(cmd)
return commands
def _relaunch_stopped_serves(token: dict) -> None:
"""Idempotent atexit relaunch of manual serves stopped by the venv guard.
Mirrors the gateway resume token contract: `pending` flips False on the
first invocation so the explicit call and the atexit registration cannot
double-spawn (#63206).
"""
if not token.get("pending"):
return
token["pending"] = False
entries = token.get("entries") or []
if not entries:
return
commands = _serve_relaunch_commands(entries)
skipped = len(entries) - len(commands)
failed: list = []
if commands:
print(" ⟲ Relaunching stopped serve/dashboard backend(s)")
failed = _m()._respawn_dashboard_processes(commands)
if skipped or failed:
print(
" ⚠ Some stopped backends could not be relaunched automatically; "
"restart them manually (hermes serve --host <ip> --port <port>)."
)
try:
from hermes_cli.update_receipt import record_step
record_step(
"serve_relaunch",
not failed and not skipped,
f"relaunched={len(commands) - len(failed)} failed={len(failed)} skipped={skipped}",
)
except Exception:
pass
def _orphaned_desktop_backend_pids(
matches: list[tuple[int, str, str]],
) -> list[int] | None:
@@ -6100,6 +6207,49 @@ def _cmd_update_impl(args, gateway_mode: bool):
_m()._stop_process_trees(_orphan_backends)
_time.sleep(1.0)
_venv_holders = _m()._detect_venv_python_processes()
if _venv_holders:
# Manual serve/dashboard rung (#63206): a network-bound
# `hermes serve --host <ip>` powering a REMOTE Desktop holds the
# venv and used to dead-end the update with exit 2 — the user's
# only option was killing the backend by hand, and nothing ever
# brought it back (the remote client's endpoint stayed dead).
# Positive ledger identity only: self-registered serve/dashboard
# whose recorded spawner is not alive (Desktop-owned backends
# keep the refusal — the app respawns what we kill). Stop them,
# and register an idempotent atexit relaunch built from the
# ledger's structured host/port/profile so the endpoint comes
# back on the SAME bind after the update — success or failure.
_serve_entries = _m()._ledger_manual_serve_holders(_venv_holders)
if _serve_entries:
print(
f" ⚠ {len(_serve_entries)} manual serve/dashboard "
"backend(s) hold the venv; stopping them for the update "
"(they will be relaunched on their recorded endpoints)"
)
_m()._stop_process_trees(
[int(e["pid"]) for e in _serve_entries]
)
_serve_resume_token = {
"pending": True,
"entries": _serve_entries,
}
try:
from hermes_cli.update_receipt import record_step
record_step(
"serve_pause",
True,
f"stopped={len(_serve_entries)}",
)
except Exception:
pass
import atexit as _serve_atexit
_serve_atexit.register(
_m()._relaunch_stopped_serves, _serve_resume_token
)
_time.sleep(1.0)
_venv_holders = _m()._detect_venv_python_processes()
if _venv_holders:
# Final rung before the dead-end: a GUI-updater hand-off
# (`update --gateway --force` with the update-incomplete marker
+44
View File
@@ -109,6 +109,8 @@ def _restart_mechanism(supervisor: str, profile: str) -> str:
return "launchd"
if supervisor == "desktop":
return "desktop"
if supervisor == "manual-serve":
return "respawn-argv"
return "manual"
@@ -120,6 +122,8 @@ def describe_restart_mechanism(mechanism: str, profile: str) -> str:
return "launchctl kickstart -k (drain-first, per-label domain)"
if mechanism == "desktop":
return "Desktop app respawns its serve backend"
if mechanism == "respawn-argv":
return "stop before code swap, relaunch with recorded launch args"
if profile != "default":
return f"hermes -p {profile} gateway restart"
return "hermes gateway restart"
@@ -292,6 +296,46 @@ def collect_runtime_inventory() -> UpdatePlan:
except Exception as exc:
logger.debug("PID-file gateway inventory failed: %s", exc)
# Serve/dashboard backends from the spawn ledger (#63206). These are the
# runtimes the gateway collectors above can never see: a manually
# launched `hermes serve --host <ip>` for a remote Desktop, or a
# long-lived `hermes dashboard`. Every serve/dashboard registers itself
# (with structured host/port/profile since #63206) at startup, and
# ledger_entries() live-verifies (pid, create_time) so PID reuse never
# fabricates a row. Desktop-supervised backends are classified by their
# recorded spawner still being alive — those restart via the Desktop's
# own respawn, not ours.
try:
from hermes_cli.process_identity import ledger_entries, spawner_is_dead
for entry in ledger_entries():
purpose = entry.get("purpose")
if purpose not in ("serve", "dashboard"):
continue
pid = entry.get("pid")
if not isinstance(pid, int) or pid in seen_pids:
continue
seen_pids.add(pid)
has_live_spawner = spawner_is_dead(entry) is False
supervisor = "desktop" if has_live_spawner else "manual-serve"
profile = str(entry.get("profile") or "default")
plan.runtimes.append(
RuntimeRecord(
kind=str(purpose),
profile=profile,
pid=pid,
supervisor=supervisor,
restart_via=_restart_mechanism(supervisor, profile),
detail={
"argv": entry.get("argv") or "",
"host": entry.get("host") or "",
"port": entry.get("port"),
},
)
)
except Exception as exc:
logger.debug("Serve/dashboard ledger inventory failed: %s", exc)
return plan
+15 -4
View File
@@ -19801,24 +19801,35 @@ def start_server(
# for standalone `hermes serve` (no HERMES_PARENT_PID env).
_start_parent_death_watchdog()
actual_port = _read_bound_port(server, fallback=port)
app.state.bound_port = actual_port
# Positive process identity: record (pid, create_time, purpose,
# spawner) in the machine spawn ledger and — on Windows — attach
# to a kill-on-close job so this backend's whole child tree dies
# with it. Both best-effort; failures degrade to legacy behavior.
# Registered AFTER the bind so the entry carries the ACTUAL port
# (ephemeral binds included) — the structured host/port/profile
# is what lets `hermes update` relaunch a manually-started serve
# on its real endpoint instead of dropping it (#63206).
try:
from hermes_cli.process_identity import (
attach_self_to_kill_on_close_job,
register_self,
)
register_self("serve" if headless else "dashboard")
register_self(
"serve" if headless else "dashboard",
detail={
"host": host,
"port": actual_port,
"profile": initial_profile or "",
},
)
attach_self_to_kill_on_close_job()
except Exception as exc:
_log.debug("process-identity registration skipped: %s", exc)
actual_port = _read_bound_port(server, fallback=port)
app.state.bound_port = actual_port
_write_dashboard_ready_file(actual_port)
# Port-discovery sentinel parsed by the desktop spawn. `serve` is a
# plain backend, not a dashboard, so it announces a neutral token;
@@ -0,0 +1,218 @@
"""Serve-kind runtime inventory + stop/relaunch rung (#63206, campaign #91277).
A network-bound `hermes serve --host <ip>` powering a remote Desktop used to
be invisible to the update pipeline: not in the inventory, a dead-end at the
venv-holder guard, and never relaunched after `hermes update` killed it. The
fix threads the spawn ledger's structured launch identity (host/port/profile,
registered at serve startup) through inventory → guard rung → relaunch.
"""
from __future__ import annotations
import sys
from types import SimpleNamespace
from unittest.mock import patch # noqa: F401 - kept for parity with siblings
import hermes_cli.update_cmd as update_cmd
import hermes_cli.update_inventory as update_inventory
from hermes_cli import main as cli_main
def _ledger_entry(**over):
entry = {
"pid": 4321,
"create_time": 111.0,
"purpose": "serve",
"install": "inst",
"spawner_pid": None,
"spawner_create": None,
"registered_at": 222.0,
"argv": "hermes serve --host 100.94.65.93 --port 9119",
"host": "100.94.65.93",
"port": 9119,
"profile": "",
}
entry.update(over)
return entry
# ---------------------------------------------------------------------------
# process_identity: structured detail round-trip
# ---------------------------------------------------------------------------
def test_register_self_records_structured_detail(tmp_path, monkeypatch):
from hermes_cli import process_identity as pi
monkeypatch.setattr(pi, "_ledger_path", lambda: tmp_path / "ledger.json")
monkeypatch.setattr(pi, "install_id", lambda *a, **k: "inst")
assert pi.register_self(
"serve", detail={"host": "100.94.65.93", "port": 9119, "profile": "work"}
)
entries = [
e
for e in pi._read_ledger(tmp_path / "ledger.json")
if e["purpose"] == "serve"
]
assert entries, "serve entry must be written"
e = entries[-1]
assert e["host"] == "100.94.65.93"
assert e["port"] == 9119
assert e["profile"] == "work"
def test_register_self_without_detail_stays_backward_compatible(
tmp_path, monkeypatch
):
from hermes_cli import process_identity as pi
monkeypatch.setattr(pi, "_ledger_path", lambda: tmp_path / "ledger.json")
monkeypatch.setattr(pi, "install_id", lambda *a, **k: "inst")
assert pi.register_self("gateway")
e = pi._read_ledger(tmp_path / "ledger.json")[-1]
assert e["host"] == "" and e["port"] is None and e["profile"] == ""
# ---------------------------------------------------------------------------
# update_inventory: serve collector
# ---------------------------------------------------------------------------
def test_inventory_includes_manual_serve_from_ledger(monkeypatch):
entry = _ledger_entry()
fake_pi = SimpleNamespace(
ledger_entries=lambda **k: [entry],
spawner_is_dead=lambda e: None, # no spawner recorded → manual
)
monkeypatch.setitem(sys.modules, "hermes_cli.process_identity", fake_pi)
plan = update_inventory.collect_runtime_inventory()
serves = [r for r in plan.runtimes if r.kind == "serve"]
assert serves, "manual serve must appear in the inventory"
row = serves[0]
assert row.pid == 4321
assert row.supervisor == "manual-serve"
assert row.restart_via == "respawn-argv"
assert row.detail["host"] == "100.94.65.93"
assert row.detail["port"] == 9119
def test_inventory_classifies_desktop_owned_serve(monkeypatch):
entry = _ledger_entry(spawner_pid=999, spawner_create=1.0)
fake_pi = SimpleNamespace(
ledger_entries=lambda **k: [entry],
spawner_is_dead=lambda e: False, # Electron parent alive
)
monkeypatch.setitem(sys.modules, "hermes_cli.process_identity", fake_pi)
plan = update_inventory.collect_runtime_inventory()
serves = [r for r in plan.runtimes if r.kind == "serve"]
assert serves and serves[0].supervisor == "desktop"
assert serves[0].restart_via == "desktop"
def test_describe_restart_mechanism_respawn_argv():
text = update_inventory.describe_restart_mechanism("respawn-argv", "default")
assert "relaunch" in text
# ---------------------------------------------------------------------------
# update_cmd: guard rung helpers
# ---------------------------------------------------------------------------
def test_ledger_manual_serve_holders_filters_correctly(monkeypatch):
manual = _ledger_entry(pid=100)
desktop_owned = _ledger_entry(pid=200, spawner_pid=999, spawner_create=1.0)
gateway = _ledger_entry(pid=300, purpose="gateway")
not_a_holder = _ledger_entry(pid=400)
fake_pi = SimpleNamespace(
ledger_entries=lambda **k: [manual, desktop_owned, gateway, not_a_holder],
spawner_is_dead=lambda e: False if e["pid"] == 200 else None,
)
monkeypatch.setitem(sys.modules, "hermes_cli.process_identity", fake_pi)
holders = [(100, "python.exe", "..."), (200, "python.exe", "..."), (300, "python.exe", "...")]
result = update_cmd._ledger_manual_serve_holders(holders)
pids = [e["pid"] for e in result]
assert pids == [100], (
"only the manual serve holder qualifies: desktop-owned keeps the "
"refusal, gateways belong to the pause machinery, non-holders skipped"
)
def test_serve_relaunch_commands_built_from_structured_identity(monkeypatch):
monkeypatch.setattr(cli_main, "_venv_scripts_dir", lambda: None)
entries = [
_ledger_entry(), # default profile
_ledger_entry(pid=5000, profile="work", port=9200, host=""),
_ledger_entry(pid=6000, port=None), # no port → skipped
_ledger_entry(pid=7000, purpose="dashboard", host="0.0.0.0", port=9300),
]
cmds = update_cmd._serve_relaunch_commands(entries)
assert ["hermes", "serve", "--host", "100.94.65.93", "--port", "9119"] in cmds
assert ["hermes", "--profile", "work", "serve", "--port", "9200"] in cmds
assert ["hermes", "dashboard", "--host", "0.0.0.0", "--port", "9300"] in cmds
assert len(cmds) == 3 # the port-less entry is skipped
def test_relaunch_stopped_serves_is_idempotent(monkeypatch):
calls = []
monkeypatch.setattr(
cli_main, "_respawn_dashboard_processes", lambda cmds: calls.append(cmds) or []
)
monkeypatch.setattr(cli_main, "_venv_scripts_dir", lambda: None)
token = {"pending": True, "entries": [_ledger_entry()]}
update_cmd._relaunch_stopped_serves(token)
update_cmd._relaunch_stopped_serves(token) # atexit double-fire
assert len(calls) == 1, "relaunch must fire exactly once"
assert token["pending"] is False
def test_relaunch_stopped_serves_untriggered_token_noop(monkeypatch):
calls = []
monkeypatch.setattr(
cli_main, "_respawn_dashboard_processes", lambda cmds: calls.append(cmds) or []
)
update_cmd._relaunch_stopped_serves({"pending": False, "entries": [_ledger_entry()]})
assert calls == []
# ---------------------------------------------------------------------------
# dashboard_procs: ledger augmentation of the scan (#81564 half)
# ---------------------------------------------------------------------------
def test_scan_dashboard_processes_includes_ledger_only_serves(monkeypatch):
"""A profiled serve (`hermes --profile p serve ...`) matches no scan
pattern; the ledger row must still surface it."""
import hermes_cli.dashboard_procs as dp
profiled = _ledger_entry(
pid=8123,
argv="hermes --profile work serve --host 100.94.65.93 --port 9119",
profile="work",
)
fake_pi = SimpleNamespace(ledger_entries=lambda **k: [profiled])
monkeypatch.setitem(sys.modules, "hermes_cli.process_identity", fake_pi)
# Force the ps/wmic scan itself to find nothing.
fake_run = SimpleNamespace(returncode=0, stdout="")
monkeypatch.setattr(
dp.subprocess, "run", lambda *a, **k: fake_run
)
result = dp._scan_dashboard_processes()
assert (8123, profiled["argv"]) in result
def test_scan_dashboard_processes_ledger_respects_exclusions(monkeypatch):
import hermes_cli.dashboard_procs as dp
entry = _ledger_entry(pid=8124)
fake_pi = SimpleNamespace(ledger_entries=lambda **k: [entry])
monkeypatch.setitem(sys.modules, "hermes_cli.process_identity", fake_pi)
fake_run = SimpleNamespace(returncode=0, stdout="")
monkeypatch.setattr(dp.subprocess, "run", lambda *a, **k: fake_run)
assert dp._scan_dashboard_processes(exclude_pids={8124}) == []