ouyangbo/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

refactor(local_runtime): compact supervisor/catalog/hardware — shared _RESIDENT status tuple, collapsed integrated verdict, tightened rationale comments

Browse Source
This commit is contained in:
Teknium
2026-09-02 20:22:10 -07:00
parent 113f04616b
commit cb1b44aacd
3 changed files with 160 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hermes_cli/local_runtime/catalog.py
+60 -97
View File
@@ -2,11 +2,10 @@
Small and honest: every entry carries the estimator inputs (measured on real GGUFs) so the picker
can price a model BEFORE the user downloads gigabytes. Once a file is on disk, profile_from_gguf()
is the authority and the catalog numbers are only used for the download decision.
is the authority; the catalog numbers only drive the download decision.
Validation lifecycle: builds proven end-to-end on real hardware are marked validated. Day-0 entries
ship before that proof (they simply lack the validated flag) — ensure_model_ready's touch generation
still gates every first load at runtime.
Builds proven end-to-end on real hardware are marked validated; day-0 entries ship without the flag
— ensure_model_ready's touch generation still gates every first load at runtime.
"""
from __future__ import annotations
@@ -40,11 +39,11 @@ _PART_SUFFIX = re.compile(r"-\d{5}-of-\d{5}$")
@dataclass(frozen=True)
class AssetFile:
"""One downloadable file: repo-relative path and exact bytes (the size feeds the estimator and the
download progress bar; there is no download-time integrity check by design — a corrupt file
surfaces as a llama.cpp load error). ``local`` overrides the on-disk name (repos reuse generic
names like mmproj-BF16.gguf across models). Non-model extras live under the models dir's assets/
subdirectory so the router never lists them.
"""One downloadable file: repo-relative path and exact bytes (feeds the estimator and the
progress bar; no download-time integrity check by design — a corrupt file surfaces as a
llama.cpp load error). ``local`` overrides the on-disk name (repos reuse generic names like
mmproj-BF16.gguf). Non-model extras live under the models dir's assets/ subdirectory so the
router never lists them.
"""
path: str # repo-relative (may include a subdir)
@@ -58,8 +57,8 @@ class AssetFile:
@dataclass(frozen=True)
class QuantVariant:
"""One downloadable build of a model. Split GGUFs list every part in
files; the model loads from the first part."""
"""One downloadable build. Split GGUFs list every part in files; the model loads from the
first part."""
quant: str # e.g. "UD-Q4_K_M"
files: tuple # AssetFile, first = the load target
@@ -76,9 +75,8 @@ class QuantVariant:
@property
def weights_bytes(self) -> int:
"""Pre-download weights estimate: GGUF bytes ≈ tensor bytes + a small header (<2%) — a safe,
slightly conservative stand-in until profile_from_gguf reads the real table.
"""
"""Pre-download weights estimate: GGUF bytes ≈ tensor bytes + a <2% header — slightly
conservative until profile_from_gguf reads the real table."""
return self.size_bytes
@@ -89,9 +87,9 @@ class CatalogEntry:
description: str # one line, plain language
repo: str # HF repo
variants: tuple # QuantVariant (exactly one, Q4-class)
# Estimator inputs (measured or config-derived; quant changes weights,
# never KV). Entries with gated upstream configs carry a conservative
# same-family prior — the GGUF header is the authority after download.
# Estimator inputs (measured or config-derived; quant changes weights, never KV). Gated
# upstream configs carry a conservative same-family prior — the GGUF header is the authority
# after download.
n_ctx_train: int
full_layers: int
recurrent_layers: int
@@ -100,32 +98,25 @@ class CatalogEntry:
swa_window: int = 0
moe: bool = False
mtp: bool = False # ships MTP heads (spec decode when loaded)
# Speculative draft depth for MTP models. Per-model and measured:
# deeper drafting pays only while draft acceptance holds, and the
# break-even depth differs by model.
# Speculative draft depth for MTP models: per-model and measured — deeper drafting pays only
# while draft acceptance holds, and the break-even depth differs by model.
mtp_draft_depth: int = 3
# Vocab size prices the GPU logits buffers (ubatch x vocab x fp32,
# doubled under MTP backend sampling) — a multi-GiB term at large
# vocab sizes that a weights-only fit would miss.
# Vocab size prices the GPU logits buffers (ubatch x vocab x fp32, doubled under MTP backend
# sampling) — a multi-GiB term at large vocabs that a weights-only fit would miss.
n_vocab: int = 0
mmproj: "AssetFile | None" = None # vision projector, downloads with model
draft: "AssetFile | None" = None # spec-decode draft model (e.g. DSpark)
sampling: dict = field(default_factory=dict) # INI long-form launch defaults
# Oldest llama.cpp release tag that can load this model (day-0
# architectures need the release where their support landed). Empty
# means any installed engine. The pane gates download/activate on it.
# Oldest llama.cpp release tag that can load this model (day-0 architectures need the release
# where their support landed). Empty means any installed engine.
min_engine: str = ""
# Editorial quality ordering (higher = smarter), authored once,
# globally, at catalog-authoring time — Artificial Analysis-informed
# where they cover the model (scripts/aa_quality_sync.py proposes,
# the commit decides), editorial elsewhere. Ranks entries for the
# per-machine recommendation; never displayed as a score (it grades
# the full-precision model, not our Q4 build).
# Editorial quality ordering (higher = smarter), authored once at catalog time — Artificial
# Analysis-informed where covered (scripts/aa_quality_sync.py proposes, the commit decides).
# Ranks entries for the per-machine recommendation; never displayed as a score (it grades the
# full-precision model, not our Q4 build).
quality: int = 0
# Fraction of the build's bytes read per decoded token: 1.0 for dense
# models (every weight streams every token), the active slice for MoE
# (attention + shared + routed experts over total). With memory
# bandwidth this predicts decode speed — the physics half of the
# Fraction of the build's bytes read per decoded token: 1.0 for dense, the active slice for
# MoE. With memory bandwidth this predicts decode speed — the physics half of the
# recommendation.
decode_fraction: float = 1.0
@@ -151,8 +142,8 @@ class CatalogEntry:
@dataclass(frozen=True)
class VariantChoice:
"""Selection result: which build this machine should download and why.
reason_key is a UI-copy discriminator, not display text."""
"""Which build this machine should download and why. reason_key is a UI-copy
discriminator, not display text."""
variant: QuantVariant
zero_spill: bool
@@ -160,14 +151,11 @@ class VariantChoice:
def select_variant(entry: CatalogEntry, budget: HardwareBudget) -> VariantChoice | None:
"""Fit the entry's one build (Q4-class) to this machine.
"""Fit the entry's one Q4-class build to this machine; headroom buys a bigger window, never a
bigger quant.
Every entry ships exactly one variant (see the module docstring for why there is no quant
ladder); headroom buys a bigger window, never a bigger quant. The fit shapes:
- "best-large-window": zero-spills at TARGET_WINDOW - "best-fits": zero-spills at the 64K floor
- "smallest-fits-spilled": weights spill to host RAM, priced honestly - None: even spilled,
physics refuses (the machine can't run it)
"best-large-window": zero-spill at TARGET_WINDOW; "best-fits": zero-spill at the 64K floor;
"smallest-fits-spilled": weights spill to host RAM, priced honestly; None: physics refuses.
"""
overhead = (RUNTIME_OVERHEAD_BYTES
+ (entry.mmproj.size_bytes if entry.mmproj else 0)
@@ -189,30 +177,20 @@ def select_variant(entry: CatalogEntry, budget: HardwareBudget) -> VariantChoice
# ── recommendation: best quality that fits and isn't miserably slow ──
#
# Two axes, each living where it belongs. QUALITY is a judgment made once,
# globally, at authoring time (entry.quality — AA-informed, editorially
# owned). SPEED is physics computed per machine: decode is memory-bound,
# so predicted tok/s ≈ bandwidth / bytes-read-per-token, and the bytes per
# token are the build's size scaled by its decode fraction (dense reads
# everything; MoE reads the active slice). The pick: highest quality among
# entries that run resident and clear a pleasant speed floor; else the
# fastest resident entry; else the least-painful spilled one.
#
# The bandwidth axis is the `uma` flag for now: every discrete card that
# matters is 900+ GB/s GDDR while the unified-memory class measures ~1/5th
# of that, so the flag IS the high/low split. A measured per-machine
# bandwidth (one cached memcpy probe) can replace these class constants
# without touching the rule; predictions order candidates and gate the
# floor — they are not display values.
# QUALITY is a judgment made once at authoring time (entry.quality). SPEED is physics per machine:
# decode is memory-bound, so predicted tok/s ≈ bandwidth / bytes-read-per-token (build size scaled
# by decode fraction). The bandwidth axis is the `uma` flag: every discrete card that matters is
# 900+ GB/s GDDR while the unified-memory class measures ~1/5th of that. A measured per-machine
# bandwidth could replace these class constants without touching the rule; predictions order
# candidates and gate the floor — they are not display values.
_DISCRETE_BANDWIDTH_GB_S = 1000.0 # representative GDDR6X/GDDR7 class
_UMA_BANDWIDTH_GB_S = 210.0 # measured on unified-memory NVIDIA
_HOST_BANDWIDTH_GB_S = 80.0 # spilled weights stream over host DRAM
# The one editorial constant in the tree: below this predicted decode
# speed a model stops feeling pleasant for agentic use (roughly reading
# speed with headroom for tool-call bursts). Distinct from the growth
# policy's 6 tok/s compress floor, which marks unusable, not unpleasant.
# Below this predicted decode speed a model stops feeling pleasant for agentic use (roughly
# reading speed with headroom for tool-call bursts). Distinct from the growth policy's 6 tok/s
# compress floor, which marks unusable, not unpleasant.
PLEASANT_FLOOR_TOK_S = 20.0
@@ -230,16 +208,13 @@ def predicted_decode_tok_s(entry: CatalogEntry, variant: QuantVariant,
def recommended_entry(budget: HardwareBudget,
entries: "tuple[CatalogEntry, ...] | None" = None
) -> "tuple[CatalogEntry, str] | None":
"""The catalog's default pick for THIS machine, with its reason.
"""The catalog's default pick for THIS machine, with its reason key.
Callers pass pre-filtered entries when some are ineligible for reasons the catalog can't know
(engine too old); default is the full catalog.
best-quality-resident quality won among resident entries that clear the pleasant floor speed-
gated-quality same, but the floor eliminated a HIGHER quality candidate — the exact 'why not the
big model?' a unified-memory owner asks fastest-resident nothing resident clears the floor; the
quickest resident entry wins least-painful-spilled nothing runs resident; fastest from host
memory (MoE by construction)
(engine too old). Reasons: best-quality-resident (quality won among resident entries clearing
the pleasant floor); speed-gated-quality (same, but the floor eliminated a HIGHER quality
candidate); fastest-resident (nothing resident clears the floor); least-painful-spilled
(nothing runs resident; fastest from host memory — MoE by construction).
"""
pool = CATALOG if entries is None else entries
fitting = [(e, c) for e in pool if (c := select_variant(e, budget)) is not None]
@@ -257,22 +232,16 @@ def recommended_entry(budget: HardwareBudget,
return (pick, "speed-gated-quality" if floor_gated else "best-quality-resident")
if resident:
return (max(resident, key=speed)[0], "fastest-resident")
# Everything spills: take the least painful — fastest predicted decode
# from host memory (MoE wins here by construction; a dense spill
# streams every weight over the host bus).
return (max(fitting, key=lambda t: speed(t, spilled=True))[0], "least-painful-spilled")
# ── catalog data: packaged JSON, refreshed from GitHub in memory ─
#
# The catalog DATA lives in catalog.json (checked in beside this module
# and shipped as package data); this module keeps all policy. At import
# we load the packaged copy — no network on the import path. A TTL-gated
# background refresh fetches the same file from the repo's main branch
# and swaps it in memory only: nothing on disk changes, so a git
# checkout never sees a dirty tracked file and the packaged copy remains
# the offline truth. A reverted commit on main heals every install on
# its next fetch, and day-0 entries reach users without an app release.
# catalog.json ships as package data and is loaded at import (no network on the import path). A
# TTL-gated background refresh fetches the same file from the repo's main branch and swaps it in
# MEMORY only: nothing on disk changes, so a git checkout never sees a dirty tracked file and the
# packaged copy remains the offline truth. A reverted commit on main heals every install on its
# next fetch, and day-0 entries reach users without an app release.
_CATALOG_URL = ("https://raw.githubusercontent.com/NousResearch/hermes-agent"
"/main/hermes_cli/local_runtime/catalog.json")
@@ -289,8 +258,7 @@ def _asset_from(d: "dict | None") -> "AssetFile | None":
local=d.get("local"))
# Scalar CatalogEntry fields parsed from JSON: key -> (coerce, default);
# a None default means the key is required.
# Scalar CatalogEntry fields parsed from JSON: key -> (coerce, default); None default = required.
_SCALAR_FIELDS = {
"n_ctx_train": (int, None), "full_layers": (int, None),
"recurrent_layers": (int, None), "per_layer_f16": (int, None),
@@ -302,9 +270,8 @@ _SCALAR_FIELDS = {
def _load_catalog(doc: dict) -> "tuple[CatalogEntry, ...]":
"""Parse a catalog document into entries. Unknown fields are ignored
(newer catalogs stay readable by older apps); a major schema bump is
the signal that they wouldn't be, and the caller skips the document."""
"""Parse a catalog document. Unknown fields are ignored (newer catalogs stay readable by older
apps); a major schema bump is the signal that they wouldn't be, and the caller skips it."""
if int(doc.get("schema_version", 0)) != _SCHEMA_VERSION:
raise ValueError(f"catalog schema {doc.get('schema_version')!r} "
f"(this build reads {_SCHEMA_VERSION})")
@@ -337,12 +304,9 @@ CATALOG: "tuple[CatalogEntry, ...]" = _packaged_catalog()
def refresh_catalog(force: bool = False) -> bool:
"""Fetch the current catalog from the repo and swap it in memory.
Best-effort by design: any failure (offline, GitHub down, unreadable schema) leaves the running
catalog untouched and retries after the TTL. Returns True when a fetched document replaced the
catalog.
"""
"""Fetch the current catalog from the repo and swap it in memory. Best-effort: any failure
(offline, GitHub down, unreadable schema) leaves the running catalog untouched and retries
after the TTL. Returns True when a fetched document replaced the catalog."""
global CATALOG, _last_refresh_attempt
now = time.monotonic()
@@ -365,9 +329,8 @@ def refresh_catalog(force: bool = False) -> bool:
def refresh_catalog_soon() -> None:
"""TTL-gated background refresh; returns immediately. The caller's current request serves the
catalog it already has — the refresh lands for the next one.
"""
"""TTL-gated background refresh; returns immediately. The current request serves the catalog
it already has — the refresh lands for the next one."""
if time.monotonic() - _last_refresh_attempt < _REFRESH_TTL_S:
return
threading.Thread(target=refresh_catalog, daemon=True,
hermes_cli/local_runtime/hardware.py
+50 -76
View File
@@ -24,44 +24,35 @@ from hermes_cli.local_runtime.estimator import HardwareBudget
logger = logging.getLogger(__name__)
_GIB = 1 << 30
# Reserve carved off the card before any grant: the desktop's own
# co-residents (compositor, browser, Electron) measure ~2-2.5 GiB on a
# working machine, and a window granted into that space demotes silently
# under WDDM. 7% covers big cards; the 2 GiB floor is what the margin's
# old 512 MiB floor failed to cover in practice (a 221K grant measured
# 31.9/32.6 GiB with the desktop running — 'fits' by the math, demoted
# in reality). Small cards give up window to this; spill mode is their
# path to big models regardless.
# Reserve carved off the card before any grant: the desktop's own co-residents (compositor,
# browser, Electron) measure ~2-2.5 GiB, and a window granted into that space demotes silently
# under WDDM. 7% covers big cards; the 2 GiB floor is what a 512 MiB floor failed to cover (a
# 221K grant measured 31.9/32.6 GiB with the desktop running — 'fits' by the math, demoted in
# reality). Small cards give up window to this; spill mode is their path to big models.
_MARGIN_FLOOR = 2 << 30
_MARGIN_FRACTION = 0.09
# UMA headroom: on unified-memory machines (Apple Silicon, unified-memory
# NVIDIA) the model shares physical memory with the OS and every app, so
# budget from RAM minus this fraction.
# UMA headroom: on unified-memory machines the model shares physical memory with the OS and every
# app, so budget from RAM minus this fraction.
_UMA_HEADROOM_FRACTION = 0.20
# Engine-fallback gates for the unified-pool quirk — BOTH must hold, and
# no discrete card can meet either: (1) the allocator's pool exceeds the
# smi report by well past rounding/ECC slack (discrete cards agree within
# ~2%; carve-out disagreement runs to whole multiples), and (2) the pool is
# system-RAM-sized — a workstation card in a RAM-matched box fails (1)
# because its smi and allocator AGREE, and a big discrete card in a
# bigger box fails (2). The driver's INTEGRATED attribute, when
# readable, bypasses both gates in whichever direction it points.
# Engine-fallback gates for the unified-pool quirk — BOTH must hold, and no discrete card can
# meet either: (1) the allocator's pool exceeds the smi report by well past rounding/ECC slack
# (discrete cards agree within ~2%; carve-out disagreement runs to whole multiples), and (2) the
# pool is system-RAM-sized. The driver's INTEGRATED attribute, when readable, bypasses both gates
# in whichever direction it points.
_POOL_DISAGREEMENT_FACTOR = 1.5
_POOL_RAM_FRACTION = 0.75
# cuDeviceGetAttribute enum: device is integrated with host memory.
_CU_DEVICE_ATTRIBUTE_INTEGRATED = 18
# One probe per process once a device answers (silicon doesn't change);
# a miss retries after this long so a runtime installed mid-session gets
# picked up by the engine fallback.
# One probe per process once a device answers (silicon doesn't change); a miss retries after this
# long so a runtime installed mid-session gets picked up by the engine fallback.
_POOL_NEGATIVE_TTL_S = 60.0
_pool_probe_cache: tuple[float, "tuple[int, bool | None] | None"] | None = None
# ' CUDA0: NVIDIA Example Device (1234-core Example GPU) (46464 MiB, 46284 MiB free)'
# — greedy .* pins the LAST parenthesized group, so device names carrying
# their own parentheses parse correctly.
# — greedy .* pins the LAST parenthesized group, so device names with parentheses parse.
_DEVICE_LINE_RE = re.compile(r"CUDA\d+:.*\((\d+)\s*MiB,\s*\d+\s*MiB free\)\s*$")
@@ -91,9 +82,8 @@ def _ram_bytes() -> tuple[int, int]:
pass
try:
if sys.platform == "darwin":
# macOS getconf has no _PHYS_PAGES/_AVPHYS_PAGES (exit 64, "no such
# configuration parameter") — the POSIX branch below returns (0, 0)
# and every model reads unavailable. sysctl is the platform truth.
# macOS getconf has no _PHYS_PAGES/_AVPHYS_PAGES (exit 64) — the POSIX branch would
# return (0, 0) and every model would read unavailable. sysctl is the platform truth.
total = int(_stdout("/usr/sbin/sysctl", "-n", "hw.memsize").strip() or 0)
if total <= 0:
return 0, 0
@@ -102,8 +92,8 @@ def _ram_bytes() -> tuple[int, int]:
out = _stdout("/usr/bin/vm_stat")
page_m = re.search(r"page size of (\d+)", out)
page = int(page_m.group(1)) if page_m else 16384
# free + inactive + purgeable ≈ reclaimable-on-demand; the
# speculative pool is dropped by the OS under pressure too.
# free + inactive + purgeable ≈ reclaimable-on-demand; the speculative pool is
# dropped by the OS under pressure too.
pages = sum(int(m.group(1)) for key in (
"Pages free", "Pages inactive", "Pages purgeable", "Pages speculative")
if (m := re.search(rf"{key}:\s+(\d+)\.", out)))
@@ -125,17 +115,15 @@ def _ram_bytes() -> tuple[int, int]:
return 0, 0
# nvidia-smi lives at a fixed path under the driver install; PATH presence
# varies by session type (services and gateways often run with a minimal
# environment) and by driver generation (legacy NVSMI dir was never on
# PATH). Resolution result is cached: the driver doesn't move mid-process.
# nvidia-smi lives at a fixed path under the driver install; PATH presence varies by session type
# (services and gateways often run minimal environments) and by driver generation (the legacy
# NVSMI dir was never on PATH). Cached: the driver doesn't move mid-process.
_smi_path_cache: "tuple[str | None] | None" = None
def _nvidia_smi_path() -> str | None:
"""Absolute path to nvidia-smi, or None. PATH first (respects user overrides), then the driver's
known install locations on Windows; on Linux/WSL the PATH lookup is the whole ladder.
"""
"""Absolute path to nvidia-smi, or None. PATH first (respects user overrides), then the
driver's known Windows install locations; on Linux/WSL the PATH lookup is the whole ladder."""
global _smi_path_cache
if _smi_path_cache is not None:
return _smi_path_cache[0]
@@ -173,11 +161,10 @@ def _nvidia_vram() -> tuple[int, int] | None:
def _cuda_driver_pool() -> "tuple[int, bool | None] | None":
"""(allocator_total_bytes, integrated_or_None) from the CUDA driver API, or None when unreachable.
ctypes against the driver's own DLL/SO — no toolkit, no subprocess, ~ms. INTEGRATED is the
vendor's own unified-memory declaration; total is the pool the allocator will actually hand out
(on carve-out devices, several times what nvidia-smi reports).
"""
"""(allocator_total_bytes, integrated_or_None) from the CUDA driver API via ctypes against the
driver's own DLL/SO — no toolkit, no subprocess, ~ms. INTEGRATED is the vendor's own
unified-memory declaration; total is the pool the allocator will actually hand out (on
carve-out devices, several times what nvidia-smi reports)."""
import ctypes
for name in ("nvcuda.dll", "libcuda.so.1", "libcuda.so"):
@@ -210,9 +197,8 @@ def _cuda_driver_pool() -> "tuple[int, bool | None] | None":
def _engine_device_pool() -> "tuple[int, bool | None] | None":
"""(engine_total_bytes, None) from the installed runtime's own --list-devices, or None. The
fallback truth source when the driver API is unreachable: asks the exact binary that will do the
allocating. Carries no integrated verdict — callers must gate it.
"""
fallback when the driver API is unreachable: asks the exact binary that will do the
allocating. Carries no integrated verdict — callers must gate it."""
try:
from hermes_cli.local_runtime.binaries import (
installed_tags,
@@ -242,9 +228,8 @@ def _engine_device_pool() -> "tuple[int, bool | None] | None":
def _device_pool_view() -> "tuple[int, bool | None] | None":
"""Best available allocator-side view, cached: a hit is permanent for the process, a miss retries
after a short TTL (the engine binary can appear mid-session via a pane install).
"""
"""Best available allocator-side view, cached: a hit is permanent for the process, a miss
retries after a short TTL (the engine binary can appear mid-session via a pane install)."""
global _pool_probe_cache
now = time.monotonic()
if _pool_probe_cache is not None:
@@ -257,20 +242,17 @@ def _device_pool_view() -> "tuple[int, bool | None] | None":
def _unified_pool_bytes(smi_total: int, ram_total: int) -> int | None:
"""The real pool size when this NVIDIA device is unified memory behind
"""The real pool size when this NVIDIA device is unified memory behind a carve-out, else None.
The driver's INTEGRATED attribute decides in BOTH directions when readable (0 pins discrete
even if numbers look odd; a declared-integrated driver is believed at modest pool sizes). Only
the attribute-less engine fallback needs the two numeric gates, both of which must hold.
The driver's INTEGRATED attribute decides in BOTH directions when readable; only the
attribute-less engine fallback needs the two numeric gates, both of which must hold.
"""
view = _device_pool_view()
if view is None:
return None
pool, integrated = view
if integrated is False:
return None
if integrated is True:
return pool
if integrated is not None:
return pool if integrated else None
if (smi_total > 0 and pool >= int(smi_total * _POOL_DISAGREEMENT_FACTOR)
and ram_total > 0 and pool >= int(ram_total * _POOL_RAM_FRACTION)):
return pool
@@ -287,23 +269,18 @@ def probe_budget(*, planning: bool = False) -> HardwareBudget:
"""Construct the budget per the source rules above.
``planning=False``: LIVE budget (free VRAM now) for launch-time fit and growth re-grants.
``planning=True``: CAPACITY budget (total minus margin) for catalog pricing and quant selection;
pricing against live-free while a model was loaded made every row read as too large and
degraded quant picks. The managed server unloads/relaunches itself, so capacity is real.
``planning=True``: CAPACITY budget (total minus margin) for catalog pricing and quant
selection — pricing against live-free while a model was loaded made every row read as too
large. The managed server unloads/relaunches itself, so capacity is real.
"""
ram_total, ram_avail = _ram_bytes()
vram = _nvidia_vram()
# Unified-memory NVIDIA: the CUDA allocator pool is the real
# capacity. Classification comes from the driver API/engine — it
# must not require nvidia-smi (stripped-PATH sessions lose smi but
# nvcuda loads via the system loader regardless). Crossing the
# carve-out costs nothing (effective bandwidth is flat through the
# boundary; smi's used/total merely saturate at it) — the carve-out
# is an OS accounting knob, not a GPU limit. Deliberately NOT
# clamped to OS RAM: carved-out memory is invisible to
# GlobalMemoryStatusEx (the OS reports correspondingly less total
# RAM), so a RAM clamp would throw away exactly the carved capacity.
# Unified-memory NVIDIA: the CUDA allocator pool is the real capacity. Classification comes
# from the driver API/engine and must not require nvidia-smi (stripped-PATH sessions lose smi
# but nvcuda loads via the system loader). Crossing the carve-out costs nothing — it is an OS
# accounting knob, not a GPU limit. Deliberately NOT clamped to OS RAM: carved-out memory is
# invisible to GlobalMemoryStatusEx, so a RAM clamp would throw away exactly that capacity.
unified = _unified_pool_bytes(vram[0] if vram else 0, ram_total)
if unified is not None:
logger.info(
@@ -314,19 +291,16 @@ def probe_budget(*, planning: bool = False) -> HardwareBudget:
if planning:
base = unified
else:
# Live: dedicated-free plus what the OS can still give. smi's
# free saturates at the carve-out so this under-counts a bit —
# the safe direction (the pool edge is a measured soft cliff:
# decode collapses ~3.5x when concurrent demand hits it).
# Without smi, OS-available alone is the honest floor.
# Live: dedicated-free plus what the OS can still give. smi's free saturates at the
# carve-out so this under-counts a bit — the safe direction (the pool edge is a
# measured soft cliff: decode collapses ~3.5x when concurrent demand hits it).
live = (vram[1] + ram_avail) if vram else ram_avail
base = min(unified, live)
return _uma_budget(base, unified)
if vram is None:
# No NVIDIA device visible: Metal/Vulkan/CPU paths budget from RAM
# as UMA (Apple Silicon) — conservative for discrete AMD until a
# vendor probe lands (E3 hardware).
# No NVIDIA device visible: Metal/Vulkan/CPU paths budget from RAM as UMA (Apple
# Silicon) — conservative for discrete AMD until a vendor probe lands.
return _uma_budget(ram_total if planning else ram_avail, ram_total)
total, free = vram
hermes_cli/local_runtime/supervisor.py
+50 -71
View File
@@ -3,9 +3,9 @@
The router process is ours (restart with backoff on crash); router children are its problem — child
failures surface via GET /models exit_code, never auto-retried here.
Readiness rules (each learned the hard way on real hardware): - health-200 is NOT readiness; every
Readiness rules (each learned the hard way on real hardware): health-200 is NOT readiness; every
readiness claim requires a touch generation (temp-0, expected token, generous budget,
reasoning_content scanned). - Always dial 127.0.0.1 — resolving localhost adds ~2s per request on
reasoning_content scanned). Always dial 127.0.0.1 — resolving localhost adds ~2s per request on
Windows via IPv6 fallback.
"""
@@ -29,11 +29,17 @@ logger = logging.getLogger(__name__)
TOUCH_PROMPT = "Reply with exactly one word: the capital of France."
TOUCH_EXPECT = "paris"
_RESTART_BACKOFF_S = (1, 5, 15, 60)
_RESIDENT = ("loaded", "ready")
# Chosen once and reused across restarts: sessions persist the resolved base_url, so an ephemeral
# port would strand every resumed session after each restart. Deliberately NOT 8080 so we never
# collide with a user's own llama-server/Ollama-adjacent stack.
_DEFAULT_PORT = 18434
def state_path() -> Path:
"""Endpoint state for other Hermes processes (provider resolution reads
this to route llamacpp-alias requests at the managed server)."""
"""Endpoint state for other Hermes processes (provider resolution routes llamacpp-alias
requests at the managed server from this)."""
return runtimes_root() / "server.json"
@@ -43,18 +49,9 @@ def _free_port() -> int:
return s.getsockname()[1]
# Default port for the managed server, chosen once and reused across
# restarts. Sessions persist the resolved base_url; an ephemeral port
# would strand every resumed session on a dead endpoint after each
# restart. Deliberately NOT 8080 so we never collide with a user's own
# llama-server/Ollama-adjacent stack.
_DEFAULT_PORT = 18434
def _stable_port() -> int:
"""The stable default port, falling back to an ephemeral one only when something else already
listens there (and it isn't a leftover managed server, which stop() would have cleaned up).
"""
"""The stable default port, or an ephemeral one only when something else already listens
there (a leftover managed server would have been cleaned up by stop())."""
try:
with socket.socket() as s:
s.bind(("127.0.0.1", _DEFAULT_PORT))
@@ -69,9 +66,9 @@ def _stable_port() -> int:
def _stable_api_key() -> str:
"""One key for the life of the install, persisted beside the runtimes.
Endpoint identity must survive restarts as a UNIT — sessions persist the resolved base_url +
api_key, so a per-boot key strands every resumed session on HTTP 401 exactly the way a per-boot
port would strand them on connection errors.
Endpoint identity must survive restarts as a UNIT — sessions persist base_url + api_key, so a
per-boot key strands every resumed session on HTTP 401 exactly as a per-boot port would on
connection errors.
"""
key_path = runtimes_root() / ".api_key"
try:
@@ -93,6 +90,12 @@ def _stable_api_key() -> str:
class LlamaServerSupervisor:
"""Own one llama-server router process for the life of a Hermes session."""
# A model that has gone quiet gets its VRAM back after this long. A constant, not a knob:
# long enough that an active conversation never trips it, short enough that a wandered-off
# session frees ~20 GiB within the hour. No exemptions: demand reloads anything the user
# comes back to.
IDLE_UNLOAD_S = 15 * 60
def __init__(self, install_dir: Path, models_dir: Path, *,
models_max: int = 4, port: int | None = None,
extra_args: list[str] | None = None,
@@ -151,19 +154,15 @@ class LlamaServerSupervisor:
"--api-key", self.api_key,
"--models-dir", str(self.models_dir),
"--models-max", str(self.models_max),
# The residency contract at the layer that sees every message:
# a chat request to a staged-but-unloaded model loads it (slow
# first token) instead of failing with 'model not found' —
# without this flag, chat after an eject is a bare 400/404.
# Residency contract: a chat request to a staged-but-unloaded model loads it (slow
# first token) instead of a bare 400/404 after an eject.
"--models-autoload",
"--metrics", # opt-in flag; supervisor telemetry needs it
"--slots", # /slots endpoint is also opt-in; is_idle reads it
"--no-webui",
"--jinja",
# Direct I/O on model load: bypasses the page cache, so a
# multi-GB load doesn't evict half the OS cache — measured
# faster loads on NVMe, and our router bounces (download/
# delete/activate) reload models often enough to care.
# Direct I/O on model load bypasses the page cache so a multi-GB load doesn't evict
# half the OS cache — measured faster on NVMe, and our router bounces reload often.
"-dio",
]
if self.preset_path and self.preset_path.exists():
@@ -171,8 +170,7 @@ class LlamaServerSupervisor:
cmd += self.extra_args
self.log_path.parent.mkdir(parents=True, exist_ok=True)
if self._log_handle is not None:
# The crash-restart loop calls _spawn repeatedly; without
# closing the prior handle each restart leaks one fd.
# The crash-restart loop calls _spawn repeatedly; each restart would leak one fd.
try:
self._log_handle.close()
except Exception: # noqa: BLE001 — best-effort
@@ -184,10 +182,9 @@ class LlamaServerSupervisor:
self.proc = subprocess.Popen(cmd, stdout=self._log_handle,
stderr=subprocess.STDOUT, cwd=str(exe.parent))
logger.info("llama-server router spawned pid=%s port=%s", self.proc.pid, self.port)
# State goes down at SPAWN, not after health: endpoint resolution
# treats a live-pid-but-not-yet-healthy server as "starting" rather
# than "unconfigured", so a readiness probe racing the boot doesn't
# throw the app back to onboarding (observed on first restart test).
# State goes down at SPAWN, not after health: endpoint resolution treats a
# live-pid-but-not-yet-healthy server as "starting" rather than "unconfigured", so a
# readiness probe racing the boot doesn't throw the app back to onboarding.
self._write_state()
def start(self, timeout_s: int = 120) -> None:
@@ -263,12 +260,10 @@ class LlamaServerSupervisor:
def _terminate_tree(proc: subprocess.Popen) -> None:
"""Terminate the router AND its model children.
The router spawns one child llama-server per loaded model, each holding gigabytes of VRAM.
Terminating only the router (on Windows, TerminateProcess with no cleanup) orphans them: the
port goes quiet but the weights stay resident. Enumerate children FIRST (the parent must be
alive to walk them), then terminate all together, escalating to kill for stragglers.
Each child holds gigabytes of VRAM; terminating only the router (TerminateProcess on
Windows does no cleanup) orphans them with the weights still resident. Enumerate children
FIRST (the parent must be alive to walk them), terminate all, escalate to kill.
"""
children: list = []
try:
import psutil
@@ -295,10 +290,9 @@ class LlamaServerSupervisor:
def _reap_orphaned_children(self) -> None:
"""Kill model children orphaned by a router crash, before respawn.
A crashed router can't clean up its children, and a dead parent can't be walked — so match
by identity instead: any process running OUR llama-server binary whose parent is gone is an
orphan of a previous router. Their VRAM must come back before the new router loads models
next to the ghosts.
A dead parent can't be walked, so match by identity: any process running OUR
llama-server binary whose parent is gone is an orphan of a previous router. Its VRAM must
come back before the new router loads models next to the ghosts.
"""
try:
import psutil
@@ -332,33 +326,21 @@ class LlamaServerSupervisor:
self._request("/models/load", {"model": model_id}, timeout_s=timeout_s)
def unload_model(self, model_id: str) -> None:
"""Free the child's VRAM now. Route existence verified empirically on b10290 (POST
/models/unload; bogus name -> 400 'model is not found'). Momentary action: never touches
primary_model — the declaration is durable, an eject is not (residency design).
"""
"""Free the child's VRAM now (POST /models/unload; bogus name -> 400). Momentary: never
touches primary_model — the declaration is durable, an eject is not."""
self._request("/models/unload", {"model": model_id}, timeout_s=120)
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
try:
if self.models().get(model_id) not in ("loaded", "ready", "unloading"):
if self.models().get(model_id) not in (*_RESIDENT, "unloading"):
return
except Exception: # noqa: BLE001
return
time.sleep(0.3)
# ── idle residency (non-primary models) ──────────────────
# A model that has gone quiet gets its VRAM back after this long. A
# constant, not a knob: long enough that an active conversation never
# trips it, short enough that a wandered-off session frees ~20 GiB
# within the hour. No exemptions (residency v2): demand reloads
# anything the user comes back to.
IDLE_UNLOAD_S = 15 * 60
def sweep_idle(self, now: float | None = None) -> list[str]:
"""Unload models idle past IDLE_UNLOAD_S. Returns the model ids
unloaded. Idle means no busy slots and no queued work, tracked
per model across calls; a model seen busy resets its clock."""
"""Unload models idle past IDLE_UNLOAD_S; returns their ids. Idle = no busy slots and no
queued work, tracked per model across calls; a model seen busy resets its clock."""
now = time.monotonic() if now is None else now
unloaded: list[str] = []
try:
@@ -366,7 +348,7 @@ class LlamaServerSupervisor:
except Exception: # noqa: BLE001
return unloaded
for model_id, status in statuses.items():
if status not in ("loaded", "ready") or not self.is_idle(model_id):
if status not in _RESIDENT or not self.is_idle(model_id):
self._idle_since.pop(model_id, None)
continue
first_idle = self._idle_since.setdefault(model_id, now)
@@ -383,8 +365,7 @@ class LlamaServerSupervisor:
def touch_generate(self, model_id: str, timeout_s: int = 300) -> bool:
"""The readiness proof. Generous budget + reasoning_content scan — small token budgets
false-fail reasoning models, which spend their first tokens thinking.
"""
false-fail reasoning models, which spend their first tokens thinking."""
try:
resp = self._request("/v1/chat/completions", {
"model": model_id,
@@ -403,23 +384,21 @@ class LlamaServerSupervisor:
status = self.models().get(model_id)
if status is None:
raise KeyError(f"model {model_id} not present in models dir")
if status not in ("loaded", "ready"):
if status not in _RESIDENT:
self.load_model(model_id, timeout_s=timeout_s)
return self.touch_generate(model_id)
# ── telemetry ────────────────────────────────────────────
def is_idle(self, model_id: str | None = None) -> bool:
"""No processing requests and no busy slots. Router quirk: /slots and /metrics are per-child
and require ?model= (bare calls 400), and no KV-usage metric exists. With ``model_id``
checks that one child; without, every loaded child.
"""
"""No processing requests and no busy slots. Router quirk: /slots and /metrics are
per-child and require ?model= (bare calls 400). With ``model_id`` checks that one child;
without, every loaded child."""
try:
if model_id is not None:
loaded = [model_id]
else:
loaded = [m for m, status in self.models().items()
if status in ("loaded", "ready")]
loaded = [m for m, status in self.models().items() if status in _RESIDENT]
for mid in loaded:
slots = self._request(f"/slots?model={mid}")
if any(s.get("is_processing") for s in slots):
@@ -427,9 +406,9 @@ class LlamaServerSupervisor:
with self._open(f"/metrics?model={mid}", timeout_s=10, json_type=False) as r:
text = r.read().decode()
for line in text.splitlines():
if line.startswith("llamacpp:requests_processing"):
if float(line.split()[-1]) != 0.0:
return False
if (line.startswith("llamacpp:requests_processing")
and float(line.split()[-1]) != 0.0):
return False
return True
except Exception: # noqa: BLE001
return False