fix(local-runtime): contain Windows runtimes and safely stop orphans
This commit is contained in:
@@ -32,27 +32,23 @@ def _pid_alive(pid: int) -> bool:
|
||||
|
||||
|
||||
def _state_endpoint() -> dict | None:
|
||||
from hermes_cli.local_runtime.supervisor import state_path
|
||||
from hermes_cli.local_runtime.recovery import is_modern, read_state, recorded_process
|
||||
|
||||
path = state_path()
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
state = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
state = read_state()
|
||||
base_url = state.get("base_url", "")
|
||||
if not base_url:
|
||||
return None
|
||||
# Ownership proof: on the stable port a SECOND install (different HERMES_HOME) can own
|
||||
# 127.0.0.1:18434 with a different api key while this install's state file still points
|
||||
# there. /health is public and answers 200 for ANYONE's server — trusting it sent every
|
||||
# request at a server that 401s our key, silently — so the recorded supervisor pid is the
|
||||
# ONLY tiebreaker: a live pid is ours (healthy, or STARTING — state is written at spawn, and
|
||||
# readiness probes racing the boot must see a configured provider, not missing credentials);
|
||||
# a dead pid is a crashed-without-cleanup leftover, ignored so requests don't blackhole.
|
||||
if not _pid_alive(int(state.get("pid") or 0)):
|
||||
if not isinstance(base_url, str) or not base_url:
|
||||
return None
|
||||
if is_modern(state):
|
||||
if recorded_process(state) is None:
|
||||
return None
|
||||
else:
|
||||
# Preserve the legacy endpoint shape, with malformed PID values rejected.
|
||||
try:
|
||||
pid = state.get("pid")
|
||||
if isinstance(pid, bool) or not _pid_alive(int(pid or 0)):
|
||||
return None
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
return {"base_url": base_url, "api_key": state.get("api_key", "")}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Contain each managed Windows router tree without adopting the owner process."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import psutil
|
||||
|
||||
|
||||
class _BasicLimits(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("PerProcessUserTimeLimit", ctypes.c_longlong),
|
||||
("PerJobUserTimeLimit", ctypes.c_longlong),
|
||||
("LimitFlags", wintypes.DWORD),
|
||||
("MinimumWorkingSetSize", ctypes.c_size_t),
|
||||
("MaximumWorkingSetSize", ctypes.c_size_t),
|
||||
("ActiveProcessLimit", wintypes.DWORD),
|
||||
("Affinity", ctypes.c_size_t),
|
||||
("PriorityClass", wintypes.DWORD),
|
||||
("SchedulingClass", wintypes.DWORD),
|
||||
]
|
||||
|
||||
|
||||
class _IoCounters(ctypes.Structure):
|
||||
_fields_ = [(name, ctypes.c_ulonglong) for name in (
|
||||
"ReadOperationCount", "WriteOperationCount", "OtherOperationCount",
|
||||
"ReadTransferCount", "WriteTransferCount", "OtherTransferCount",
|
||||
)]
|
||||
|
||||
|
||||
class _ExtendedLimits(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("BasicLimitInformation", _BasicLimits),
|
||||
("IoInfo", _IoCounters),
|
||||
("ProcessMemoryLimit", ctypes.c_size_t),
|
||||
("JobMemoryLimit", ctypes.c_size_t),
|
||||
("PeakProcessMemoryUsed", ctypes.c_size_t),
|
||||
("PeakJobMemoryUsed", ctypes.c_size_t),
|
||||
]
|
||||
|
||||
|
||||
class _WindowsJob:
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._api = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
for name, args, result in (
|
||||
("CreateJobObjectW", [ctypes.c_void_p, wintypes.LPCWSTR], wintypes.HANDLE),
|
||||
("SetInformationJobObject", [wintypes.HANDLE, ctypes.c_int,
|
||||
ctypes.c_void_p, wintypes.DWORD], wintypes.BOOL),
|
||||
("AssignProcessToJobObject", [wintypes.HANDLE, wintypes.HANDLE], wintypes.BOOL),
|
||||
("CloseHandle", [wintypes.HANDLE], wintypes.BOOL),
|
||||
):
|
||||
fn = getattr(self._api, name)
|
||||
fn.argtypes = args
|
||||
fn.restype = result
|
||||
# NULL security attributes create a non-inheritable, unnamed owner handle.
|
||||
self._handle = self._api.CreateJobObjectW(None, None)
|
||||
if not self._handle:
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
try:
|
||||
limits = _ExtendedLimits()
|
||||
# Neither BREAKAWAY_OK nor SILENT_BREAKAWAY_OK: descendants stay contained.
|
||||
limits.BasicLimitInformation.LimitFlags = 0x2000 # KILL_ON_JOB_CLOSE
|
||||
if not self._api.SetInformationJobObject(
|
||||
self._handle, 9, ctypes.byref(limits), ctypes.sizeof(limits)):
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def assign(self, proc: subprocess.Popen) -> None:
|
||||
# Popen retains the original process handle, avoiding a PID-reuse race.
|
||||
if not self._api.AssignProcessToJobObject(self._handle, int(proc._handle)):
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
|
||||
def close(self) -> None:
|
||||
"""Terminate the contained tree; repeated closes are harmless."""
|
||||
with self._lock:
|
||||
if self._handle is not None:
|
||||
if not self._api.CloseHandle(self._handle):
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
self._handle = None
|
||||
|
||||
|
||||
def spawn_server(cmd, **kwargs) -> tuple[subprocess.Popen, _WindowsJob | None]:
|
||||
"""Start a router, returning its process and an owner-held containment handle.
|
||||
|
||||
Keep the job until shutdown and call close() to terminate the entire tree.
|
||||
Windows closes it automatically if the owner dies. Other hosts retain Popen's
|
||||
ordinary behavior. Assignment happens before the child's first instruction.
|
||||
"""
|
||||
if sys.platform != "win32":
|
||||
return subprocess.Popen(cmd, **kwargs), None
|
||||
job = _WindowsJob()
|
||||
proc = None
|
||||
try:
|
||||
kwargs["creationflags"] = kwargs.get("creationflags", 0) | 0x00000004 # CREATE_SUSPENDED
|
||||
proc = subprocess.Popen(cmd, **kwargs)
|
||||
job.assign(proc)
|
||||
psutil.Process(proc.pid).resume()
|
||||
return proc, job
|
||||
except BaseException:
|
||||
try:
|
||||
if proc is not None:
|
||||
# Assignment may have failed: closing an empty job is not enough.
|
||||
proc.kill()
|
||||
proc.wait(timeout=10)
|
||||
finally:
|
||||
try:
|
||||
job.close()
|
||||
finally:
|
||||
if proc is not None:
|
||||
for stream in (proc.stdin, proc.stdout, proc.stderr):
|
||||
if stream is not None:
|
||||
stream.close()
|
||||
proc._handle.Close()
|
||||
raise
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Conservative recovery of the one router recorded by this managed runtime."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import psutil
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def read_state() -> dict:
|
||||
from hermes_cli.local_runtime.supervisor import state_path
|
||||
|
||||
try:
|
||||
state = json.loads(state_path().read_text(encoding="utf-8"))
|
||||
return state if isinstance(state, dict) else {}
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
_MODERN_FIELDS = ("create_time", "executable", "owner_pid", "owner_create_time")
|
||||
|
||||
|
||||
def is_modern(state: dict) -> bool:
|
||||
return any(key in state for key in _MODERN_FIELDS)
|
||||
|
||||
|
||||
def _valid_pid(value) -> bool:
|
||||
return type(value) is int and value > 0
|
||||
|
||||
|
||||
def _valid_birth(value) -> bool:
|
||||
return type(value) in (int, float) and math.isfinite(value) and value > 0
|
||||
|
||||
|
||||
def recorded_process(state: dict):
|
||||
"""Match a modern process incarnation, never just its reusable PID."""
|
||||
try:
|
||||
pid, created = state["pid"], state["create_time"]
|
||||
owner_pid, owner_created = state["owner_pid"], state["owner_create_time"]
|
||||
exe = state["executable"]
|
||||
if (not _valid_pid(pid) or not _valid_birth(created)
|
||||
or not _valid_pid(owner_pid) or not _valid_birth(owner_created)
|
||||
or owner_created > created or not isinstance(exe, str) or not exe):
|
||||
return None
|
||||
proc = psutil.Process(pid)
|
||||
if (not proc.is_running() or proc.create_time() != created
|
||||
or Path(proc.exe()) != Path(exe)):
|
||||
return None
|
||||
# Windows retains the original parent PID; POSIX reparents orphans.
|
||||
if os.name == "nt" and proc.ppid() != owner_pid:
|
||||
return None
|
||||
parent = proc.parent()
|
||||
if parent is not None:
|
||||
if parent.pid == owner_pid:
|
||||
if parent.create_time() != owner_created:
|
||||
return None
|
||||
elif os.name == "nt" or not _owner_is_dead(state):
|
||||
return None # POSIX may reparent a router whose recorded owner exited.
|
||||
return proc
|
||||
except (KeyError, TypeError, ValueError, OverflowError, OSError, psutil.Error):
|
||||
return None
|
||||
|
||||
|
||||
def _owner_is_dead(state: dict) -> bool:
|
||||
try:
|
||||
pid, created = state["owner_pid"], state["owner_create_time"]
|
||||
if not _valid_pid(pid) or not _valid_birth(created):
|
||||
return False
|
||||
try:
|
||||
owner = psutil.Process(pid)
|
||||
# A newer incarnation proves the recorded owner has exited.
|
||||
return owner.create_time() > created or not owner.is_running()
|
||||
except psutil.NoSuchProcess:
|
||||
return True
|
||||
except (KeyError, TypeError, ValueError, OverflowError, OSError, psutil.Error):
|
||||
return False
|
||||
|
||||
|
||||
def _legacy_orphan_process(state: dict):
|
||||
"""Older state lacks birth times: require the exact installed binary and launch arguments."""
|
||||
from urllib.parse import urlsplit
|
||||
from hermes_cli.local_runtime.bootstrap import models_dir
|
||||
from hermes_cli.local_runtime.supervisor import state_path
|
||||
|
||||
# A damaged new record must not fall back to weaker legacy evidence.
|
||||
if os.name != "nt" or is_modern(state):
|
||||
return None
|
||||
try:
|
||||
if not _valid_pid(state["pid"]):
|
||||
return None
|
||||
proc = psutil.Process(state["pid"])
|
||||
root = state_path().parent
|
||||
exe = Path(proc.exe())
|
||||
relative = exe.relative_to(root)
|
||||
if len(relative.parts) != 3 or exe.name.lower() != "llama-server.exe":
|
||||
return None
|
||||
if proc.parent() is not None or proc.ppid() <= 0:
|
||||
return None
|
||||
if proc.create_time() > state_path().stat().st_mtime:
|
||||
return None # the PID was reused after this record was written
|
||||
url = urlsplit(state["base_url"])
|
||||
if url.scheme != "http" or url.hostname != "127.0.0.1" or not url.port or not state["api_key"]:
|
||||
return None
|
||||
argv = proc.cmdline()
|
||||
|
||||
def value(flag):
|
||||
if argv.count(flag) != 1:
|
||||
return None
|
||||
index = argv.index(flag) + 1
|
||||
return argv[index] if index < len(argv) else None
|
||||
|
||||
if (value("--host") != "127.0.0.1" or value("--port") != str(url.port)
|
||||
or value("--api-key") != state["api_key"]):
|
||||
return None
|
||||
preset, directory = value("--models-preset"), value("--models-dir")
|
||||
if not ((preset and Path(preset) == root / "presets.ini" and directory is None)
|
||||
or (directory and Path(directory) == models_dir() and preset is None)):
|
||||
return None
|
||||
return proc
|
||||
except (KeyError, TypeError, ValueError, OSError, psutil.Error):
|
||||
return None
|
||||
|
||||
|
||||
def stop_recorded_orphan() -> bool:
|
||||
"""Explicit user stop only. Refuse uncertain identity or a living recorded owner."""
|
||||
from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor, state_path
|
||||
|
||||
try:
|
||||
state = json.loads(state_path().read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return True # already stopped and no record
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
if not isinstance(state, dict) or not _valid_pid(state.get("pid")):
|
||||
return False
|
||||
try:
|
||||
if not psutil.pid_exists(state["pid"]):
|
||||
return True # keep the dead record; endpoint resolution rejects it
|
||||
if is_modern(state):
|
||||
proc = recorded_process(state)
|
||||
if proc is None or not _owner_is_dead(state):
|
||||
return False
|
||||
else:
|
||||
proc = _legacy_orphan_process(state)
|
||||
if proc is None or read_state() != state or not proc.is_running():
|
||||
return False
|
||||
# Retain the psutil incarnation object: destructive methods guard PID reuse.
|
||||
LlamaServerSupervisor._terminate_tree(proc, verified_root=True)
|
||||
proc.wait(timeout=5)
|
||||
logger.info("stopped orphaned managed llama-server pid=%s", proc.pid)
|
||||
return True
|
||||
except (OSError, psutil.Error):
|
||||
logger.warning("could not verify or stop recorded managed llama-server")
|
||||
return False
|
||||
@@ -15,6 +15,7 @@ import logging
|
||||
import secrets
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -22,6 +23,7 @@ import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from hermes_cli.local_runtime.binaries import server_binary, runtimes_root
|
||||
from hermes_cli.local_runtime.processes import spawn_server
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -113,9 +115,13 @@ class LlamaServerSupervisor:
|
||||
self.log_path = log_path or (self.models_dir.parent / "logs" / "llama-server.log")
|
||||
self.preset_path = preset_path
|
||||
self.proc: subprocess.Popen | None = None
|
||||
self._job = None
|
||||
self.primary_model: str | None = None
|
||||
self._restarts = 0
|
||||
self._stopping = False
|
||||
self._stop_event = threading.Event()
|
||||
self._lifecycle_lock = threading.RLock()
|
||||
self._state: dict | None = None
|
||||
self._watchdog: threading.Thread | None = None
|
||||
self._log_handle = None
|
||||
self._idle_since: dict[str, float] = {}
|
||||
@@ -177,8 +183,8 @@ class LlamaServerSupervisor:
|
||||
self._log_handle.write(f"\n# spawn: {cmd}\n")
|
||||
self._log_handle.flush()
|
||||
# list-args, never a shell: spaced paths (user homes) must survive.
|
||||
self.proc = subprocess.Popen(cmd, stdout=self._log_handle,
|
||||
stderr=subprocess.STDOUT, cwd=str(exe.parent))
|
||||
self.proc, self._job = spawn_server(cmd, stdout=self._log_handle,
|
||||
stderr=subprocess.STDOUT, cwd=str(exe.parent))
|
||||
logger.info("llama-server router spawned pid=%s port=%s", self.proc.pid, self.port)
|
||||
# State goes down at SPAWN, not after health: endpoint resolution treats a
|
||||
# live-pid-but-not-yet-healthy server as "starting" rather than "unconfigured", so a
|
||||
@@ -186,22 +192,33 @@ class LlamaServerSupervisor:
|
||||
self._write_state()
|
||||
|
||||
def start(self, timeout_s: int = 120) -> None:
|
||||
self._stopping = False
|
||||
self._spawn()
|
||||
with self._lifecycle_lock:
|
||||
self._stopping = False
|
||||
self._stop_event.clear()
|
||||
self._spawn()
|
||||
self._wait_health(timeout_s)
|
||||
self._write_state()
|
||||
self._watchdog = threading.Thread(target=self._watch, daemon=True, name="llamacpp-supervisor")
|
||||
self._watchdog.start()
|
||||
|
||||
def _write_state(self) -> None:
|
||||
import os
|
||||
import psutil
|
||||
from utils import atomic_json_write
|
||||
|
||||
proc = psutil.Process(self.proc.pid)
|
||||
self._state = {"base_url": self.base_url, "api_key": self.api_key,
|
||||
"pid": proc.pid, "create_time": proc.create_time(),
|
||||
"executable": proc.exe(), "owner_pid": os.getpid(),
|
||||
"owner_create_time": psutil.Process().create_time()}
|
||||
path = state_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"base_url": self.base_url, "api_key": self.api_key,
|
||||
"pid": self.proc.pid if self.proc else None}), encoding="utf-8")
|
||||
atomic_json_write(path, self._state, mode=0o600)
|
||||
|
||||
def _wait_health(self, timeout_s: int) -> None:
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
if self._stop_event.is_set():
|
||||
raise RuntimeError("llama-server startup cancelled")
|
||||
if self.proc and self.proc.poll() is not None:
|
||||
raise RuntimeError(f"llama-server exited rc={self.proc.returncode} during startup "
|
||||
f"(log: {self.log_path})")
|
||||
@@ -226,11 +243,15 @@ class LlamaServerSupervisor:
|
||||
return
|
||||
backoff = _RESTART_BACKOFF_S[min(self._restarts, len(_RESTART_BACKOFF_S) - 1)]
|
||||
logger.warning("llama-server exited rc=%s; restart #%s in %ss", rc, self._restarts + 1, backoff)
|
||||
time.sleep(backoff)
|
||||
if self._stop_event.wait(backoff):
|
||||
return
|
||||
self._restarts += 1
|
||||
try:
|
||||
self._reap_orphaned_children()
|
||||
self._spawn()
|
||||
with self._lifecycle_lock:
|
||||
if self._stopping:
|
||||
return
|
||||
self._reap_orphaned_children()
|
||||
self._spawn()
|
||||
self._wait_health(120)
|
||||
if self.primary_model:
|
||||
self.ensure_model_ready(self.primary_model)
|
||||
@@ -238,16 +259,23 @@ class LlamaServerSupervisor:
|
||||
logger.error("llama-server restart failed: %s", exc)
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stopping = True
|
||||
state_path().unlink(missing_ok=True)
|
||||
if self.proc and self.proc.poll() is None:
|
||||
self._terminate_tree(self.proc)
|
||||
if self._log_handle:
|
||||
self._log_handle.close()
|
||||
self._log_handle = None
|
||||
with self._lifecycle_lock:
|
||||
self._stopping = True
|
||||
self._stop_event.set()
|
||||
try:
|
||||
if self.proc and self.proc.poll() is None:
|
||||
self._terminate_tree(self.proc)
|
||||
finally:
|
||||
if self._job is not None:
|
||||
self._job.close()
|
||||
self._job = None
|
||||
# Retain state: deleting it could race a replacement publication.
|
||||
if self._log_handle:
|
||||
self._log_handle.close()
|
||||
self._log_handle = None
|
||||
|
||||
@staticmethod
|
||||
def _terminate_tree(proc: subprocess.Popen) -> None:
|
||||
def _terminate_tree(proc: subprocess.Popen, *, verified_root: bool = False) -> None:
|
||||
"""Terminate the router AND its model children.
|
||||
|
||||
Each child holds gigabytes of VRAM; terminating only the router (TerminateProcess on
|
||||
@@ -255,19 +283,30 @@ class LlamaServerSupervisor:
|
||||
FIRST (the parent must be alive to walk them), terminate all, escalate to kill.
|
||||
"""
|
||||
children: list = []
|
||||
with suppress(Exception): # no psutil view; still stop the router
|
||||
timeouts = (subprocess.TimeoutExpired,)
|
||||
with suppress(ImportError):
|
||||
import psutil
|
||||
|
||||
children = psutil.Process(proc.pid).children(recursive=True)
|
||||
proc.terminate()
|
||||
for child in children:
|
||||
_quiet(child.terminate)
|
||||
timeouts += (psutil.TimeoutExpired,)
|
||||
if verified_root:
|
||||
# Recovery retains the birth identity; never rebuild it from a PID.
|
||||
children = proc.children(recursive=True)
|
||||
if not proc.is_running():
|
||||
raise psutil.NoSuchProcess(proc.pid)
|
||||
else:
|
||||
with suppress(psutil.Error):
|
||||
children = psutil.Process(proc.pid).children(recursive=True)
|
||||
try:
|
||||
proc.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
for child in children:
|
||||
_quiet(lambda: child.is_running() and child.kill())
|
||||
for child in children:
|
||||
_quiet(child.terminate)
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=15)
|
||||
except timeouts:
|
||||
proc.kill()
|
||||
finally:
|
||||
for child in children:
|
||||
_quiet(lambda: child.is_running() and child.kill())
|
||||
|
||||
def _reap_orphaned_children(self) -> None:
|
||||
"""Kill model children orphaned by a router crash, before respawn.
|
||||
@@ -276,6 +315,12 @@ class LlamaServerSupervisor:
|
||||
llama-server binary whose parent is gone is an orphan of a previous router. Its VRAM must
|
||||
come back before the new router loads models next to the ghosts.
|
||||
"""
|
||||
if self._job is not None:
|
||||
self._job.close()
|
||||
self._job = None
|
||||
return
|
||||
if sys.platform == "win32":
|
||||
return # Unrecorded processes are not ours merely because the binary matches.
|
||||
try:
|
||||
import psutil
|
||||
|
||||
|
||||
@@ -773,21 +773,19 @@ async def local_models_quickstart(body: QuickstartBody):
|
||||
|
||||
# ── server lifecycle: turn the engine on/off ─────────────────
|
||||
def _terminate_state_pid() -> None:
|
||||
"""Server owned by another process (or an orphan): terminate via the state file's pid, then clear the state."""
|
||||
import psutil # type: ignore
|
||||
"""Explicit recovery, never raw-PID termination of another live owner."""
|
||||
from hermes_cli.local_runtime.recovery import stop_recorded_orphan
|
||||
|
||||
state = json.loads(supervisor.state_path().read_text(encoding="utf-8"))
|
||||
pid = int(state.get("pid") or 0)
|
||||
if pid > 0 and psutil.pid_exists(pid):
|
||||
psutil.Process(pid).terminate()
|
||||
supervisor.state_path().unlink(missing_ok=True)
|
||||
if not stop_recorded_orphan():
|
||||
raise HTTPException(status_code=409, detail=(
|
||||
"Another Hermes process owns this server, or its ownership could not be verified"))
|
||||
|
||||
|
||||
def _stop_server() -> None:
|
||||
if bootstrap.get_supervisor() is not None:
|
||||
bootstrap.shutdown_local_runtime()
|
||||
elif _state_endpoint() is not None:
|
||||
_quiet(_terminate_state_pid, None) # best-effort
|
||||
else:
|
||||
_terminate_state_pid()
|
||||
_set_runtime_enabled(False)
|
||||
|
||||
|
||||
@@ -805,8 +803,12 @@ async def local_models_server(body: ServerActionBody):
|
||||
action = (body.action or "").strip().lower()
|
||||
if action not in _SERVER_ACTIONS:
|
||||
raise HTTPException(status_code=400, detail="action must be 'stop' or 'start'")
|
||||
with _http_error(502):
|
||||
try:
|
||||
await asyncio.to_thread(_SERVER_ACTIONS[action])
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||
return {"ok": True, "action": action}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user