"""Serve-process lifecycle helpers: parent start markers and death watchdog, port-conflict preflight, ready-file/sentinel announcement, browser auto-open, forwarded-IP resolution. Split out of ``hermes_cli.web_server``; every externally used name is re-imported there, so ``web_server.`` keeps resolving (and monkeypatching) as before. Helpers that tests patch on ``web_server`` are reached lazily through it. """ import logging import ipaddress import json import os import subprocess import sys import tempfile import threading import time from pathlib import Path from typing import TYPE_CHECKING, Any, Optional if TYPE_CHECKING: # pragma: no cover - annotation only import uvicorn # Same logger the code used before extraction (record parity). _log = logging.getLogger("hermes_cli.web_server") def _process_start_marker(pid: int) -> str: """Return a cross-runtime marker for the current incarnation of ``pid``. ``ProcessLookupError`` means the process is absent. Other failures are left distinct so callers can fail safe rather than killing a healthy backend. """ if sys.platform == "linux": try: stat_line = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8") except FileNotFoundError as exc: raise ProcessLookupError(pid) from exc # The command in field 2 may contain spaces or parentheses. Splitting # after its final ')' leaves field 3 at index zero and field 22 at 19. fields = stat_line.rsplit(")", 1)[1].strip().split() if len(fields) < 20 or not fields[19].isdigit(): raise OSError(f"invalid /proc stat data for PID {pid}") return f"linux:{fields[19]}" if os.name == "nt": import ctypes from ctypes import wintypes process_query_limited_information = 0x1000 kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] kernel32.OpenProcess.restype = wintypes.HANDLE kernel32.GetProcessTimes.argtypes = [ wintypes.HANDLE, ctypes.POINTER(wintypes.FILETIME), ctypes.POINTER(wintypes.FILETIME), ctypes.POINTER(wintypes.FILETIME), ctypes.POINTER(wintypes.FILETIME), ] kernel32.GetProcessTimes.restype = wintypes.BOOL kernel32.CloseHandle.argtypes = [wintypes.HANDLE] kernel32.CloseHandle.restype = wintypes.BOOL handle = kernel32.OpenProcess(process_query_limited_information, False, pid) if not handle: error = ctypes.get_last_error() if error in (87, 1168): # invalid parameter / not found raise ProcessLookupError(pid) raise OSError(error, f"OpenProcess failed for PID {pid}") creation = wintypes.FILETIME() exit_time = wintypes.FILETIME() kernel = wintypes.FILETIME() user = wintypes.FILETIME() try: if not kernel32.GetProcessTimes( handle, ctypes.byref(creation), ctypes.byref(exit_time), ctypes.byref(kernel), ctypes.byref(user), ): error = ctypes.get_last_error() raise OSError(error, f"GetProcessTimes failed for PID {pid}") finally: kernel32.CloseHandle(handle) filetime = (creation.dwHighDateTime << 32) | creation.dwLowDateTime return f"win:{filetime + 504911232000000000}" result = subprocess.run( ["ps", "-p", str(pid), "-o", "lstart="], capture_output=True, text=True, check=False, ) marker = result.stdout.strip() if result.returncode == 0 and marker: return f"ps:{marker}" if result.returncode == 1 and not marker: raise ProcessLookupError(pid) raise OSError(f"ps could not inspect PID {pid}: {result.stderr.strip()}") def _valid_parent_start_marker(marker: str) -> bool: prefix, separator, value = marker.partition(":") if not separator or not value or value != value.strip(): return False if prefix in ("linux", "win", "winms"): return value.isdigit() return prefix == "ps" def _parent_start_markers_match(actual: str, expected: str) -> bool: """Compare parent markers across Desktop protocol generations. Older Windows Desktop builds send .NET ticks (``win:``). New builds use Electron's native process creation time in Unix milliseconds (``winms:``) so startup does not need to launch PowerShell. The backend still reads the exact FILETIME and normalizes it only when the expected marker is ``winms``. """ if actual == expected: return True if not actual.startswith("win:") or not expected.startswith("winms:"): return False try: dotnet_ticks = int(actual.removeprefix("win:")) expected_unix_ms = int(expected.removeprefix("winms:")) except ValueError: return False dotnet_ticks_at_unix_epoch = 621_355_968_000_000_000 actual_unix_ms = (dotnet_ticks - dotnet_ticks_at_unix_epoch) // 10_000 return actual_unix_ms == expected_unix_ms def _warm_gateway_module() -> None: """Pre-import heavy modules so the event loop is not stalled on first use. On a cold Windows install, importing these module chains triggers .pyc compilation and Defender real-time scans that can stall the event loop for 15-30s. The original fix (pre-#60800) only warmed ``hermes_cli.gateway``. But the first WS connection and its initial RPC burst (``setup.status``, ``setup.runtime_check``, ``gateway.ready``→``resolve_skin``) pull in several *other* heavy chains that were still imported on the loop thread, contributing to the ~14s cold-start stall (#60800). Warm them all here so the cost is paid in a worker thread while the server socket is already open. """ for mod in ( "hermes_cli.gateway", # setup.status / setup.runtime_check resolve provider auth state, # which imports copilot_auth (→ subprocess module) and scans # credential files. First import is noticeably slow on Windows. "hermes_cli.auth", "hermes_cli.copilot_auth", "hermes_cli.runtime_provider", # resolve_skin() reads config + initialises the skin engine. # Even though handle_ws now calls it via asyncio.to_thread # (see tui_gateway/ws.py), warming it here avoids the first-call # import cost inside that thread. "hermes_cli.skin_engine", # model.options / picker context — parses provider catalogs and # the models.dev cache on first use. "hermes_cli.inventory", "hermes_cli.model_switch", ): try: __import__(mod) except Exception: pass def _resolve_restart_drain_timeout() -> float: try: from hermes_cli.gateway import _get_restart_drain_timeout return _get_restart_drain_timeout() except ImportError: from gateway.restart import DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT return DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT def _eager_reconcile_own_session_db() -> None: """One writable open of this process's own state.db at startup. ``SessionDB.__init__`` runs ``_init_schema`` → ``_reconcile_columns``, bringing a store left behind by `hermes update` current before the dashboard's first session-list poll, with the open-time lock patience (jittered retries) absorbing transient contention. Never raises: a store this cannot fix is still served through the read-probe heal in :func:`_open_session_db_at_path`, which retries on every poll. """ try: from hermes_state import SessionDB, _default_db_path SessionDB(db_path=Path(_default_db_path()), read_only=False).close() except Exception as exc: _log.warning( "startup schema reconcile of state.db failed (%s); session " "reads will retry the heal per poll", exc, ) def _read_bound_port(server: "uvicorn.Server", fallback: int) -> int: """Read the OS-assigned port from a live uvicorn server socket. After ``server.startup()`` the socket is bound. Returns the actual port so ephemeral (port-0) discovery works without a pre-bind TOCTOU. Falls back to *fallback* if the socket list is empty (shouldn't happen but guards against uvicorn internals changing). """ if server.servers and server.servers[0].sockets: return server.servers[0].sockets[0].getsockname()[1] return fallback def _write_dashboard_ready_file(actual_port: int) -> None: """Optionally publish the dashboard port through an atomic ready file. Windows Desktop can launch dashboard backends with ``pythonw.exe`` to avoid console flashes. That path cannot rely on stdout for the port announcement, so Electron passes ``HERMES_DESKTOP_READY_FILE`` and waits for this JSON. Normal CLI/dashboard launches still use the stdout READY line below. """ target = os.environ.get("HERMES_DESKTOP_READY_FILE") if not target: return tmp_name = "" try: path = Path(target) path.parent.mkdir(parents=True, exist_ok=True) payload = json.dumps({"port": int(actual_port)}, separators=(",", ":")) with tempfile.NamedTemporaryFile( "w", encoding="utf-8", dir=str(path.parent), prefix=f"{path.name}.", suffix=".tmp", delete=False, ) as fh: fh.write(payload) fh.flush() os.fsync(fh.fileno()) tmp_name = fh.name os.replace(tmp_name, path) except Exception as exc: if tmp_name: try: Path(tmp_name).unlink(missing_ok=True) except Exception: pass _log.warning("Failed to write dashboard ready file %r: %s", target, exc) def _maybe_open_browser( host: str, actual_port: int, open_browser: bool, initial_profile: str ) -> None: """Open the dashboard URL in the user's browser if appropriate. Skips on headless Linux (no ``DISPLAY`` / ``WAYLAND_DISPLAY``) to avoid TUI browsers (links, lynx) that would SIGHUP the server process. Maps ``0.0.0.0`` / ``::`` binds to ``127.0.0.1`` so the browser opens a reachable URL. """ if not open_browser: return import webbrowser _has_display = ( sys.platform != "linux" or bool(os.environ.get("DISPLAY")) or bool(os.environ.get("WAYLAND_DISPLAY")) ) if not _has_display: _log.debug( "Skipping browser-open: no DISPLAY or WAYLAND_DISPLAY detected " "(headless Linux). Pass --no-open to suppress this detection." ) return _display_host = host if host not in ("0.0.0.0", "::") else "127.0.0.1" _open_url = f"http://{_display_host}:{actual_port}" if initial_profile: from urllib.parse import quote _open_url += f"/?profile={quote(initial_profile)}" def _open(): try: time.sleep(1.0) webbrowser.open(_open_url) except Exception: pass threading.Thread(target=_open, daemon=True).start() def _is_serve_orphaned( desktop_pid: int, expected_start_marker: Optional[str] = None, *, pid_exists=None, process_start_marker=None, ) -> bool: """True when the exact Desktop process that owns this backend is gone. ``HERMES_PARENT_PID`` is the Electron Desktop PID, not necessarily this Python process's immediate PPID. On Windows the venv ``hermes.exe`` launcher introduces one or more shim processes, so comparing ``os.getppid()`` to the Electron PID incorrectly treats a healthy backend as orphaned and exits 0. New Desktop versions also provide the owner's process-start marker. This prevents a recycled PID from keeping an orphan alive. Older versions remain compatible through the PID-only probe. Any inconclusive probe failure is fail-safe: keep serving rather than killing a backend whose owner could not be conclusively shown to be dead. """ try: if expected_start_marker is not None: probe = process_start_marker or _process_start_marker return not _parent_start_markers_match( probe(int(desktop_pid)), expected_start_marker ) if pid_exists is None: from gateway.status import _pid_exists pid_exists = _pid_exists return not bool(pid_exists(int(desktop_pid))) except ProcessLookupError: return True except Exception: return False def _start_parent_death_watchdog() -> None: """Exit when the exact desktop parent that spawned this backend dies. The desktop passes its PID and, in newer versions, its process-start marker plus a per-spawn nonce. The marker distinguishes a live owner from PID reuse; the nonce makes partial/mixed-version identity plumbing fail safe. Legacy Desktop versions that provide only ``HERMES_PARENT_PID`` retain PID-only tracking. """ raw_pid = os.environ.get("HERMES_PARENT_PID") start_marker = os.environ.get("HERMES_PARENT_START_MARKER") nonce = os.environ.get("HERMES_PARENT_NONCE") try: desktop_pid = int(raw_pid or "") except (TypeError, ValueError): return if desktop_pid <= 0: return has_marker = start_marker is not None has_nonce = nonce is not None if has_marker != has_nonce: return if has_marker and ( not _valid_parent_start_marker(start_marker or "") or not nonce or nonce != nonce.strip() ): return try: poll = max(0.5, float(os.environ.get("HERMES_SERVE_WATCHDOG_POLL_S", "2.0"))) except (TypeError, ValueError): poll = 2.0 def _loop() -> None: while not _is_serve_orphaned(desktop_pid, start_marker): time.sleep(poll) os._exit(0) threading.Thread(target=_loop, daemon=True, name="serve-parent-watchdog").start() # ── Port-conflict sentinel (#93608) ───────────────────────────────────────── # When the requested port is already bound, uvicorn's ``bind_socket()`` # catches the OSError itself and does ``logger.error(exc); sys.exit(1)`` — a # bare ERROR line plus the same exit 1 as any real backend crash. The desktop # spawn (and any script wrapping ``hermes serve``) cannot tell "port occupied" # from "backend broken". So we probe the exact bind before handing the socket # to uvicorn and, on conflict, emit ONE machine-readable stdout sentinel plus # a human hint, then exit with a distinct code. # # 75 == BSD ``EX_TEMPFAIL`` (sysexits.h) — the codebase's existing convention # for "transient environmental condition, not a code failure" (see # gateway/restart.py and kanban_db.py's quota-wall sentinel). PORT_IN_USE_EXIT_CODE = 75 # One line, stable format, parsed by machines — mirrors the shape of the # HERMES_BACKEND_READY sentinel (which is NOT changed by any of this). _PORT_IN_USE_SENTINEL = "BACKEND_PORT_IN_USE port={port}" def _is_addr_in_use_error(exc: OSError) -> bool: """True when ``exc`` is the platform's address-in-use bind failure.""" import errno codes = {errno.EADDRINUSE, 98, 48, 10048} # POSIX, Linux, macOS, WinSock if exc.errno in codes: return True return getattr(exc, "winerror", None) == 10048 # WSAEADDRINUSE def _port_bind_conflict(host: str, port: int) -> bool: """Probe whether binding ``host:port`` would fail with EADDRINUSE. ``port == 0`` (ephemeral) can never conflict — the kernel picks a free port — so the probe is skipped and ``--port 0`` behaves exactly as before. Any probe error other than address-in-use returns ``False`` so uvicorn surfaces it with its normal diagnostics (bad host, EACCES, …). """ if not port: return False import socket as _socket family = _socket.AF_INET6 if ":" in host else _socket.AF_INET try: probe = _socket.socket(family, _socket.SOCK_STREAM) except OSError: return False try: import sys as _sys_mod _exclusive = getattr(_socket, "SO_EXCLUSIVEADDRUSE", None) if _sys_mod.platform == "win32" and _exclusive is not None: # Windows: SO_REUSEADDR means "bind over anyone" — a probe (or # uvicorn bind) with it SUCCEEDS on top of a live LISTEN socket, # so it can never detect a conflict. SO_EXCLUSIVEADDRUSE makes # the probe fail with WSAEADDRINUSE exactly when another socket # holds the port (the reporter's 10048 shape in #93608). probe.setsockopt(_socket.SOL_SOCKET, _exclusive, 1) else: # POSIX: match uvicorn's bind flags (uvicorn/config.py # bind_socket) so the probe conflicts exactly when uvicorn's own # bind would: SO_REUSEADDR lets TIME_WAIT remnants pass while a # live LISTEN socket still fails. probe.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) probe.bind((host, port)) except OSError as exc: return _is_addr_in_use_error(exc) except Exception: return False finally: probe.close() return False def _write_machine_sentinel_line(line: str) -> None: """Write a machine-parsed sentinel line to the REAL stdout (fd 1). The serve startup path imports ``tui_gateway.server`` (flush-on-SIGTERM handlers, #94724) which redirects ``sys.stdout`` to ``sys.stderr`` at import time to keep stray prints off the JSON-RPC protocol stream. Any machine-readable sentinel printed after that import via ``print()`` lands on stderr — invisible to consumers that parse the child's stdout pipe (the Desktop spawn, scripts). fd 1 is untouched by the Python-level redirect, so write there. Best-effort by design: if fd 1 is unwritable (closed; invalid under pythonw.exe), fall back to ``print()`` for human visibility only — the redirected stream can't reach stdout-parsing consumers, and pythonw Desktop spawns rely on ``_write_dashboard_ready_file()`` (the HERMES_DESKTOP_READY_FILE channel) for port discovery instead. Never raises: a sentinel-delivery failure must not kill a healthy serve. """ try: os.write(1, (line + "\n").encode()) except OSError: try: print(line, flush=True) except Exception: pass def _report_port_in_use(host: str, port: int) -> None: """Print the machine sentinel + a human hint naming likely holders.""" from hermes_cli.web_server import _write_machine_sentinel_line _write_machine_sentinel_line(_PORT_IN_USE_SENTINEL.format(port=port)) print( f" Port {port} on {host} is already in use — likely another " "'hermes serve' / 'hermes dashboard' backend or the Hermes gateway. " "Stop the other process, or pass --port " "(--port 0 picks a free ephemeral port).", flush=True, ) _DEFAULT_DASHBOARD_FORWARDED_ALLOW_IPS = ("127.0.0.1", "::1") def _dashboard_forwarded_allow_ips(dashboard_config: dict[str, Any]) -> list[str]: """Return the bounded proxy addresses uvicorn may trust. Uvicorn's default trusts loopback. Preserve that behavior and extend it only with explicit IP addresses or CIDR networks from config. Invalid or unbounded entries fail closed instead of turning arbitrary client-supplied forwarding headers into request metadata. """ configured = dashboard_config.get("trusted_proxies", []) if configured in (None, ""): configured = [] elif isinstance(configured, str): configured = [configured] elif not isinstance(configured, (list, tuple)): _log.warning( "dashboard.trusted_proxies must be a list of IP addresses or CIDR networks; " "ignoring %r", configured, ) configured = [] trusted = list(_DEFAULT_DASHBOARD_FORWARDED_ALLOW_IPS) for raw_entry in configured: if not isinstance(raw_entry, str) or not raw_entry.strip(): _log.warning( "Ignoring invalid dashboard.trusted_proxies entry %r; expected an IP " "address or CIDR network", raw_entry, ) continue entry = raw_entry.strip() try: if "/" in entry: network = ipaddress.ip_network(entry, strict=False) if network.prefixlen == 0: raise ValueError("unbounded network") normalized = str(network) else: normalized = str(ipaddress.ip_address(entry)) except ValueError: _log.warning( "Ignoring unsafe dashboard.trusted_proxies entry %r; use a bounded IP " "address or CIDR network, never '*' or a /0 network", raw_entry, ) continue if normalized not in trusted: trusted.append(normalized) if trusted != list(_DEFAULT_DASHBOARD_FORWARDED_ALLOW_IPS): _log.info("Dashboard trusted proxies: %s", ", ".join(trusted)) return trusted