fix(local-runtime): contain Windows runtimes and safely stop orphans

This commit is contained in:
emozilla
2026-09-14 12:56:59 -04:00
parent 8c3a35b69d
commit ef1cfbbbf9
10 changed files with 1071 additions and 59 deletions
+14 -18
View File
@@ -32,27 +32,23 @@ def _pid_alive(pid: int) -> bool:
def _state_endpoint() -> dict | None:
from hermes_cli.local_runtime.supervisor import state_path
from hermes_cli.local_runtime.recovery import is_modern, read_state, recorded_process
path = state_path()
if not path.exists():
return None
try:
state = json.loads(path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return None
state = read_state()
base_url = state.get("base_url", "")
if not base_url:
return None
# Ownership proof: on the stable port a SECOND install (different HERMES_HOME) can own
# 127.0.0.1:18434 with a different api key while this install's state file still points
# there. /health is public and answers 200 for ANYONE's server — trusting it sent every
# request at a server that 401s our key, silently — so the recorded supervisor pid is the
# ONLY tiebreaker: a live pid is ours (healthy, or STARTING — state is written at spawn, and
# readiness probes racing the boot must see a configured provider, not missing credentials);
# a dead pid is a crashed-without-cleanup leftover, ignored so requests don't blackhole.
if not _pid_alive(int(state.get("pid") or 0)):
if not isinstance(base_url, str) or not base_url:
return None
if is_modern(state):
if recorded_process(state) is None:
return None
else:
# Preserve the legacy endpoint shape, with malformed PID values rejected.
try:
pid = state.get("pid")
if isinstance(pid, bool) or not _pid_alive(int(pid or 0)):
return None
except (TypeError, ValueError, OverflowError):
return None
return {"base_url": base_url, "api_key": state.get("api_key", "")}
+121
View File
@@ -0,0 +1,121 @@
"""Contain each managed Windows router tree without adopting the owner process."""
from __future__ import annotations
import ctypes
from ctypes import wintypes
import subprocess
import sys
import threading
import psutil
class _BasicLimits(ctypes.Structure):
_fields_ = [
("PerProcessUserTimeLimit", ctypes.c_longlong),
("PerJobUserTimeLimit", ctypes.c_longlong),
("LimitFlags", wintypes.DWORD),
("MinimumWorkingSetSize", ctypes.c_size_t),
("MaximumWorkingSetSize", ctypes.c_size_t),
("ActiveProcessLimit", wintypes.DWORD),
("Affinity", ctypes.c_size_t),
("PriorityClass", wintypes.DWORD),
("SchedulingClass", wintypes.DWORD),
]
class _IoCounters(ctypes.Structure):
_fields_ = [(name, ctypes.c_ulonglong) for name in (
"ReadOperationCount", "WriteOperationCount", "OtherOperationCount",
"ReadTransferCount", "WriteTransferCount", "OtherTransferCount",
)]
class _ExtendedLimits(ctypes.Structure):
_fields_ = [
("BasicLimitInformation", _BasicLimits),
("IoInfo", _IoCounters),
("ProcessMemoryLimit", ctypes.c_size_t),
("JobMemoryLimit", ctypes.c_size_t),
("PeakProcessMemoryUsed", ctypes.c_size_t),
("PeakJobMemoryUsed", ctypes.c_size_t),
]
class _WindowsJob:
def __init__(self):
self._lock = threading.Lock()
self._api = ctypes.WinDLL("kernel32", use_last_error=True)
for name, args, result in (
("CreateJobObjectW", [ctypes.c_void_p, wintypes.LPCWSTR], wintypes.HANDLE),
("SetInformationJobObject", [wintypes.HANDLE, ctypes.c_int,
ctypes.c_void_p, wintypes.DWORD], wintypes.BOOL),
("AssignProcessToJobObject", [wintypes.HANDLE, wintypes.HANDLE], wintypes.BOOL),
("CloseHandle", [wintypes.HANDLE], wintypes.BOOL),
):
fn = getattr(self._api, name)
fn.argtypes = args
fn.restype = result
# NULL security attributes create a non-inheritable, unnamed owner handle.
self._handle = self._api.CreateJobObjectW(None, None)
if not self._handle:
raise ctypes.WinError(ctypes.get_last_error())
try:
limits = _ExtendedLimits()
# Neither BREAKAWAY_OK nor SILENT_BREAKAWAY_OK: descendants stay contained.
limits.BasicLimitInformation.LimitFlags = 0x2000 # KILL_ON_JOB_CLOSE
if not self._api.SetInformationJobObject(
self._handle, 9, ctypes.byref(limits), ctypes.sizeof(limits)):
raise ctypes.WinError(ctypes.get_last_error())
except BaseException:
self.close()
raise
def assign(self, proc: subprocess.Popen) -> None:
# Popen retains the original process handle, avoiding a PID-reuse race.
if not self._api.AssignProcessToJobObject(self._handle, int(proc._handle)):
raise ctypes.WinError(ctypes.get_last_error())
def close(self) -> None:
"""Terminate the contained tree; repeated closes are harmless."""
with self._lock:
if self._handle is not None:
if not self._api.CloseHandle(self._handle):
raise ctypes.WinError(ctypes.get_last_error())
self._handle = None
def spawn_server(cmd, **kwargs) -> tuple[subprocess.Popen, _WindowsJob | None]:
"""Start a router, returning its process and an owner-held containment handle.
Keep the job until shutdown and call close() to terminate the entire tree.
Windows closes it automatically if the owner dies. Other hosts retain Popen's
ordinary behavior. Assignment happens before the child's first instruction.
"""
if sys.platform != "win32":
return subprocess.Popen(cmd, **kwargs), None
job = _WindowsJob()
proc = None
try:
kwargs["creationflags"] = kwargs.get("creationflags", 0) | 0x00000004 # CREATE_SUSPENDED
proc = subprocess.Popen(cmd, **kwargs)
job.assign(proc)
psutil.Process(proc.pid).resume()
return proc, job
except BaseException:
try:
if proc is not None:
# Assignment may have failed: closing an empty job is not enough.
proc.kill()
proc.wait(timeout=10)
finally:
try:
job.close()
finally:
if proc is not None:
for stream in (proc.stdin, proc.stdout, proc.stderr):
if stream is not None:
stream.close()
proc._handle.Close()
raise
+159
View File
@@ -0,0 +1,159 @@
"""Conservative recovery of the one router recorded by this managed runtime."""
from __future__ import annotations
import json
import logging
import math
import os
from pathlib import Path
import psutil
logger = logging.getLogger(__name__)
def read_state() -> dict:
from hermes_cli.local_runtime.supervisor import state_path
try:
state = json.loads(state_path().read_text(encoding="utf-8"))
return state if isinstance(state, dict) else {}
except (OSError, ValueError):
return {}
_MODERN_FIELDS = ("create_time", "executable", "owner_pid", "owner_create_time")
def is_modern(state: dict) -> bool:
return any(key in state for key in _MODERN_FIELDS)
def _valid_pid(value) -> bool:
return type(value) is int and value > 0
def _valid_birth(value) -> bool:
return type(value) in (int, float) and math.isfinite(value) and value > 0
def recorded_process(state: dict):
"""Match a modern process incarnation, never just its reusable PID."""
try:
pid, created = state["pid"], state["create_time"]
owner_pid, owner_created = state["owner_pid"], state["owner_create_time"]
exe = state["executable"]
if (not _valid_pid(pid) or not _valid_birth(created)
or not _valid_pid(owner_pid) or not _valid_birth(owner_created)
or owner_created > created or not isinstance(exe, str) or not exe):
return None
proc = psutil.Process(pid)
if (not proc.is_running() or proc.create_time() != created
or Path(proc.exe()) != Path(exe)):
return None
# Windows retains the original parent PID; POSIX reparents orphans.
if os.name == "nt" and proc.ppid() != owner_pid:
return None
parent = proc.parent()
if parent is not None:
if parent.pid == owner_pid:
if parent.create_time() != owner_created:
return None
elif os.name == "nt" or not _owner_is_dead(state):
return None # POSIX may reparent a router whose recorded owner exited.
return proc
except (KeyError, TypeError, ValueError, OverflowError, OSError, psutil.Error):
return None
def _owner_is_dead(state: dict) -> bool:
try:
pid, created = state["owner_pid"], state["owner_create_time"]
if not _valid_pid(pid) or not _valid_birth(created):
return False
try:
owner = psutil.Process(pid)
# A newer incarnation proves the recorded owner has exited.
return owner.create_time() > created or not owner.is_running()
except psutil.NoSuchProcess:
return True
except (KeyError, TypeError, ValueError, OverflowError, OSError, psutil.Error):
return False
def _legacy_orphan_process(state: dict):
"""Older state lacks birth times: require the exact installed binary and launch arguments."""
from urllib.parse import urlsplit
from hermes_cli.local_runtime.bootstrap import models_dir
from hermes_cli.local_runtime.supervisor import state_path
# A damaged new record must not fall back to weaker legacy evidence.
if os.name != "nt" or is_modern(state):
return None
try:
if not _valid_pid(state["pid"]):
return None
proc = psutil.Process(state["pid"])
root = state_path().parent
exe = Path(proc.exe())
relative = exe.relative_to(root)
if len(relative.parts) != 3 or exe.name.lower() != "llama-server.exe":
return None
if proc.parent() is not None or proc.ppid() <= 0:
return None
if proc.create_time() > state_path().stat().st_mtime:
return None # the PID was reused after this record was written
url = urlsplit(state["base_url"])
if url.scheme != "http" or url.hostname != "127.0.0.1" or not url.port or not state["api_key"]:
return None
argv = proc.cmdline()
def value(flag):
if argv.count(flag) != 1:
return None
index = argv.index(flag) + 1
return argv[index] if index < len(argv) else None
if (value("--host") != "127.0.0.1" or value("--port") != str(url.port)
or value("--api-key") != state["api_key"]):
return None
preset, directory = value("--models-preset"), value("--models-dir")
if not ((preset and Path(preset) == root / "presets.ini" and directory is None)
or (directory and Path(directory) == models_dir() and preset is None)):
return None
return proc
except (KeyError, TypeError, ValueError, OSError, psutil.Error):
return None
def stop_recorded_orphan() -> bool:
"""Explicit user stop only. Refuse uncertain identity or a living recorded owner."""
from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor, state_path
try:
state = json.loads(state_path().read_text(encoding="utf-8"))
except FileNotFoundError:
return True # already stopped and no record
except (OSError, ValueError):
return False
if not isinstance(state, dict) or not _valid_pid(state.get("pid")):
return False
try:
if not psutil.pid_exists(state["pid"]):
return True # keep the dead record; endpoint resolution rejects it
if is_modern(state):
proc = recorded_process(state)
if proc is None or not _owner_is_dead(state):
return False
else:
proc = _legacy_orphan_process(state)
if proc is None or read_state() != state or not proc.is_running():
return False
# Retain the psutil incarnation object: destructive methods guard PID reuse.
LlamaServerSupervisor._terminate_tree(proc, verified_root=True)
proc.wait(timeout=5)
logger.info("stopped orphaned managed llama-server pid=%s", proc.pid)
return True
except (OSError, psutil.Error):
logger.warning("could not verify or stop recorded managed llama-server")
return False
+73 -28
View File
@@ -15,6 +15,7 @@ import logging
import secrets
import socket
import subprocess
import sys
import threading
import time
import urllib.error
@@ -22,6 +23,7 @@ import urllib.request
from pathlib import Path
from hermes_cli.local_runtime.binaries import server_binary, runtimes_root
from hermes_cli.local_runtime.processes import spawn_server
logger = logging.getLogger(__name__)
@@ -113,9 +115,13 @@ class LlamaServerSupervisor:
self.log_path = log_path or (self.models_dir.parent / "logs" / "llama-server.log")
self.preset_path = preset_path
self.proc: subprocess.Popen | None = None
self._job = None
self.primary_model: str | None = None
self._restarts = 0
self._stopping = False
self._stop_event = threading.Event()
self._lifecycle_lock = threading.RLock()
self._state: dict | None = None
self._watchdog: threading.Thread | None = None
self._log_handle = None
self._idle_since: dict[str, float] = {}
@@ -177,8 +183,8 @@ class LlamaServerSupervisor:
self._log_handle.write(f"\n# spawn: {cmd}\n")
self._log_handle.flush()
# list-args, never a shell: spaced paths (user homes) must survive.
self.proc = subprocess.Popen(cmd, stdout=self._log_handle,
stderr=subprocess.STDOUT, cwd=str(exe.parent))
self.proc, self._job = spawn_server(cmd, stdout=self._log_handle,
stderr=subprocess.STDOUT, cwd=str(exe.parent))
logger.info("llama-server router spawned pid=%s port=%s", self.proc.pid, self.port)
# State goes down at SPAWN, not after health: endpoint resolution treats a
# live-pid-but-not-yet-healthy server as "starting" rather than "unconfigured", so a
@@ -186,22 +192,33 @@ class LlamaServerSupervisor:
self._write_state()
def start(self, timeout_s: int = 120) -> None:
self._stopping = False
self._spawn()
with self._lifecycle_lock:
self._stopping = False
self._stop_event.clear()
self._spawn()
self._wait_health(timeout_s)
self._write_state()
self._watchdog = threading.Thread(target=self._watch, daemon=True, name="llamacpp-supervisor")
self._watchdog.start()
def _write_state(self) -> None:
import os
import psutil
from utils import atomic_json_write
proc = psutil.Process(self.proc.pid)
self._state = {"base_url": self.base_url, "api_key": self.api_key,
"pid": proc.pid, "create_time": proc.create_time(),
"executable": proc.exe(), "owner_pid": os.getpid(),
"owner_create_time": psutil.Process().create_time()}
path = state_path()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"base_url": self.base_url, "api_key": self.api_key,
"pid": self.proc.pid if self.proc else None}), encoding="utf-8")
atomic_json_write(path, self._state, mode=0o600)
def _wait_health(self, timeout_s: int) -> None:
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
if self._stop_event.is_set():
raise RuntimeError("llama-server startup cancelled")
if self.proc and self.proc.poll() is not None:
raise RuntimeError(f"llama-server exited rc={self.proc.returncode} during startup "
f"(log: {self.log_path})")
@@ -226,11 +243,15 @@ class LlamaServerSupervisor:
return
backoff = _RESTART_BACKOFF_S[min(self._restarts, len(_RESTART_BACKOFF_S) - 1)]
logger.warning("llama-server exited rc=%s; restart #%s in %ss", rc, self._restarts + 1, backoff)
time.sleep(backoff)
if self._stop_event.wait(backoff):
return
self._restarts += 1
try:
self._reap_orphaned_children()
self._spawn()
with self._lifecycle_lock:
if self._stopping:
return
self._reap_orphaned_children()
self._spawn()
self._wait_health(120)
if self.primary_model:
self.ensure_model_ready(self.primary_model)
@@ -238,16 +259,23 @@ class LlamaServerSupervisor:
logger.error("llama-server restart failed: %s", exc)
def stop(self) -> None:
self._stopping = True
state_path().unlink(missing_ok=True)
if self.proc and self.proc.poll() is None:
self._terminate_tree(self.proc)
if self._log_handle:
self._log_handle.close()
self._log_handle = None
with self._lifecycle_lock:
self._stopping = True
self._stop_event.set()
try:
if self.proc and self.proc.poll() is None:
self._terminate_tree(self.proc)
finally:
if self._job is not None:
self._job.close()
self._job = None
# Retain state: deleting it could race a replacement publication.
if self._log_handle:
self._log_handle.close()
self._log_handle = None
@staticmethod
def _terminate_tree(proc: subprocess.Popen) -> None:
def _terminate_tree(proc: subprocess.Popen, *, verified_root: bool = False) -> None:
"""Terminate the router AND its model children.
Each child holds gigabytes of VRAM; terminating only the router (TerminateProcess on
@@ -255,19 +283,30 @@ class LlamaServerSupervisor:
FIRST (the parent must be alive to walk them), terminate all, escalate to kill.
"""
children: list = []
with suppress(Exception): # no psutil view; still stop the router
timeouts = (subprocess.TimeoutExpired,)
with suppress(ImportError):
import psutil
children = psutil.Process(proc.pid).children(recursive=True)
proc.terminate()
for child in children:
_quiet(child.terminate)
timeouts += (psutil.TimeoutExpired,)
if verified_root:
# Recovery retains the birth identity; never rebuild it from a PID.
children = proc.children(recursive=True)
if not proc.is_running():
raise psutil.NoSuchProcess(proc.pid)
else:
with suppress(psutil.Error):
children = psutil.Process(proc.pid).children(recursive=True)
try:
proc.wait(timeout=15)
except subprocess.TimeoutExpired:
proc.kill()
for child in children:
_quiet(lambda: child.is_running() and child.kill())
for child in children:
_quiet(child.terminate)
proc.terminate()
try:
proc.wait(timeout=15)
except timeouts:
proc.kill()
finally:
for child in children:
_quiet(lambda: child.is_running() and child.kill())
def _reap_orphaned_children(self) -> None:
"""Kill model children orphaned by a router crash, before respawn.
@@ -276,6 +315,12 @@ class LlamaServerSupervisor:
llama-server binary whose parent is gone is an orphan of a previous router. Its VRAM must
come back before the new router loads models next to the ghosts.
"""
if self._job is not None:
self._job.close()
self._job = None
return
if sys.platform == "win32":
return # Unrecorded processes are not ours merely because the binary matches.
try:
import psutil
+12 -10
View File
@@ -773,21 +773,19 @@ async def local_models_quickstart(body: QuickstartBody):
# ── server lifecycle: turn the engine on/off ─────────────────
def _terminate_state_pid() -> None:
"""Server owned by another process (or an orphan): terminate via the state file's pid, then clear the state."""
import psutil # type: ignore
"""Explicit recovery, never raw-PID termination of another live owner."""
from hermes_cli.local_runtime.recovery import stop_recorded_orphan
state = json.loads(supervisor.state_path().read_text(encoding="utf-8"))
pid = int(state.get("pid") or 0)
if pid > 0 and psutil.pid_exists(pid):
psutil.Process(pid).terminate()
supervisor.state_path().unlink(missing_ok=True)
if not stop_recorded_orphan():
raise HTTPException(status_code=409, detail=(
"Another Hermes process owns this server, or its ownership could not be verified"))
def _stop_server() -> None:
if bootstrap.get_supervisor() is not None:
bootstrap.shutdown_local_runtime()
elif _state_endpoint() is not None:
_quiet(_terminate_state_pid, None) # best-effort
else:
_terminate_state_pid()
_set_runtime_enabled(False)
@@ -805,8 +803,12 @@ async def local_models_server(body: ServerActionBody):
action = (body.action or "").strip().lower()
if action not in _SERVER_ACTIONS:
raise HTTPException(status_code=400, detail="action must be 'stop' or 'start'")
with _http_error(502):
try:
await asyncio.to_thread(_SERVER_ACTIONS[action])
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
return {"ok": True, "action": action}