From cb1b44aacd7967bb1e68d69f4dfde78a3cfd5b32 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:22:10 -0700 Subject: [PATCH] =?UTF-8?q?refactor(local=5Fruntime):=20compact=20supervis?= =?UTF-8?q?or/catalog/hardware=20=E2=80=94=20shared=20=5FRESIDENT=20status?= =?UTF-8?q?=20tuple,=20collapsed=20integrated=20verdict,=20tightened=20rat?= =?UTF-8?q?ionale=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/local_runtime/catalog.py | 157 ++++++++++--------------- hermes_cli/local_runtime/hardware.py | 126 ++++++++------------ hermes_cli/local_runtime/supervisor.py | 121 ++++++++----------- 3 files changed, 160 insertions(+), 244 deletions(-) diff --git a/hermes_cli/local_runtime/catalog.py b/hermes_cli/local_runtime/catalog.py index 09beef35f1..8d96cd2560 100644 --- a/hermes_cli/local_runtime/catalog.py +++ b/hermes_cli/local_runtime/catalog.py @@ -2,11 +2,10 @@ Small and honest: every entry carries the estimator inputs (measured on real GGUFs) so the picker can price a model BEFORE the user downloads gigabytes. Once a file is on disk, profile_from_gguf() -is the authority and the catalog numbers are only used for the download decision. +is the authority; the catalog numbers only drive the download decision. -Validation lifecycle: builds proven end-to-end on real hardware are marked validated. Day-0 entries -ship before that proof (they simply lack the validated flag) — ensure_model_ready's touch generation -still gates every first load at runtime. +Builds proven end-to-end on real hardware are marked validated; day-0 entries ship without the flag +— ensure_model_ready's touch generation still gates every first load at runtime. """ from __future__ import annotations @@ -40,11 +39,11 @@ _PART_SUFFIX = re.compile(r"-\d{5}-of-\d{5}$") @dataclass(frozen=True) class AssetFile: - """One downloadable file: repo-relative path and exact bytes (the size feeds the estimator and the - download progress bar; there is no download-time integrity check by design — a corrupt file - surfaces as a llama.cpp load error). ``local`` overrides the on-disk name (repos reuse generic - names like mmproj-BF16.gguf across models). Non-model extras live under the models dir's assets/ - subdirectory so the router never lists them. + """One downloadable file: repo-relative path and exact bytes (feeds the estimator and the + progress bar; no download-time integrity check by design — a corrupt file surfaces as a + llama.cpp load error). ``local`` overrides the on-disk name (repos reuse generic names like + mmproj-BF16.gguf). Non-model extras live under the models dir's assets/ subdirectory so the + router never lists them. """ path: str # repo-relative (may include a subdir) @@ -58,8 +57,8 @@ class AssetFile: @dataclass(frozen=True) class QuantVariant: - """One downloadable build of a model. Split GGUFs list every part in - files; the model loads from the first part.""" + """One downloadable build. Split GGUFs list every part in files; the model loads from the + first part.""" quant: str # e.g. "UD-Q4_K_M" files: tuple # AssetFile, first = the load target @@ -76,9 +75,8 @@ class QuantVariant: @property def weights_bytes(self) -> int: - """Pre-download weights estimate: GGUF bytes ≈ tensor bytes + a small header (<2%) — a safe, - slightly conservative stand-in until profile_from_gguf reads the real table. - """ + """Pre-download weights estimate: GGUF bytes ≈ tensor bytes + a <2% header — slightly + conservative until profile_from_gguf reads the real table.""" return self.size_bytes @@ -89,9 +87,9 @@ class CatalogEntry: description: str # one line, plain language repo: str # HF repo variants: tuple # QuantVariant (exactly one, Q4-class) - # Estimator inputs (measured or config-derived; quant changes weights, - # never KV). Entries with gated upstream configs carry a conservative - # same-family prior — the GGUF header is the authority after download. + # Estimator inputs (measured or config-derived; quant changes weights, never KV). Gated + # upstream configs carry a conservative same-family prior — the GGUF header is the authority + # after download. n_ctx_train: int full_layers: int recurrent_layers: int @@ -100,32 +98,25 @@ class CatalogEntry: swa_window: int = 0 moe: bool = False mtp: bool = False # ships MTP heads (spec decode when loaded) - # Speculative draft depth for MTP models. Per-model and measured: - # deeper drafting pays only while draft acceptance holds, and the - # break-even depth differs by model. + # Speculative draft depth for MTP models: per-model and measured — deeper drafting pays only + # while draft acceptance holds, and the break-even depth differs by model. mtp_draft_depth: int = 3 - # Vocab size prices the GPU logits buffers (ubatch x vocab x fp32, - # doubled under MTP backend sampling) — a multi-GiB term at large - # vocab sizes that a weights-only fit would miss. + # Vocab size prices the GPU logits buffers (ubatch x vocab x fp32, doubled under MTP backend + # sampling) — a multi-GiB term at large vocabs that a weights-only fit would miss. n_vocab: int = 0 mmproj: "AssetFile | None" = None # vision projector, downloads with model draft: "AssetFile | None" = None # spec-decode draft model (e.g. DSpark) sampling: dict = field(default_factory=dict) # INI long-form launch defaults - # Oldest llama.cpp release tag that can load this model (day-0 - # architectures need the release where their support landed). Empty - # means any installed engine. The pane gates download/activate on it. + # Oldest llama.cpp release tag that can load this model (day-0 architectures need the release + # where their support landed). Empty means any installed engine. min_engine: str = "" - # Editorial quality ordering (higher = smarter), authored once, - # globally, at catalog-authoring time — Artificial Analysis-informed - # where they cover the model (scripts/aa_quality_sync.py proposes, - # the commit decides), editorial elsewhere. Ranks entries for the - # per-machine recommendation; never displayed as a score (it grades - # the full-precision model, not our Q4 build). + # Editorial quality ordering (higher = smarter), authored once at catalog time — Artificial + # Analysis-informed where covered (scripts/aa_quality_sync.py proposes, the commit decides). + # Ranks entries for the per-machine recommendation; never displayed as a score (it grades the + # full-precision model, not our Q4 build). quality: int = 0 - # Fraction of the build's bytes read per decoded token: 1.0 for dense - # models (every weight streams every token), the active slice for MoE - # (attention + shared + routed experts over total). With memory - # bandwidth this predicts decode speed — the physics half of the + # Fraction of the build's bytes read per decoded token: 1.0 for dense, the active slice for + # MoE. With memory bandwidth this predicts decode speed — the physics half of the # recommendation. decode_fraction: float = 1.0 @@ -151,8 +142,8 @@ class CatalogEntry: @dataclass(frozen=True) class VariantChoice: - """Selection result: which build this machine should download and why. - reason_key is a UI-copy discriminator, not display text.""" + """Which build this machine should download and why. reason_key is a UI-copy + discriminator, not display text.""" variant: QuantVariant zero_spill: bool @@ -160,14 +151,11 @@ class VariantChoice: def select_variant(entry: CatalogEntry, budget: HardwareBudget) -> VariantChoice | None: - """Fit the entry's one build (Q4-class) to this machine. + """Fit the entry's one Q4-class build to this machine; headroom buys a bigger window, never a + bigger quant. - Every entry ships exactly one variant (see the module docstring for why there is no quant - ladder); headroom buys a bigger window, never a bigger quant. The fit shapes: - - - "best-large-window": zero-spills at TARGET_WINDOW - "best-fits": zero-spills at the 64K floor - - "smallest-fits-spilled": weights spill to host RAM, priced honestly - None: even spilled, - physics refuses (the machine can't run it) + "best-large-window": zero-spill at TARGET_WINDOW; "best-fits": zero-spill at the 64K floor; + "smallest-fits-spilled": weights spill to host RAM, priced honestly; None: physics refuses. """ overhead = (RUNTIME_OVERHEAD_BYTES + (entry.mmproj.size_bytes if entry.mmproj else 0) @@ -189,30 +177,20 @@ def select_variant(entry: CatalogEntry, budget: HardwareBudget) -> VariantChoice # ── recommendation: best quality that fits and isn't miserably slow ── # -# Two axes, each living where it belongs. QUALITY is a judgment made once, -# globally, at authoring time (entry.quality — AA-informed, editorially -# owned). SPEED is physics computed per machine: decode is memory-bound, -# so predicted tok/s ≈ bandwidth / bytes-read-per-token, and the bytes per -# token are the build's size scaled by its decode fraction (dense reads -# everything; MoE reads the active slice). The pick: highest quality among -# entries that run resident and clear a pleasant speed floor; else the -# fastest resident entry; else the least-painful spilled one. -# -# The bandwidth axis is the `uma` flag for now: every discrete card that -# matters is 900+ GB/s GDDR while the unified-memory class measures ~1/5th -# of that, so the flag IS the high/low split. A measured per-machine -# bandwidth (one cached memcpy probe) can replace these class constants -# without touching the rule; predictions order candidates and gate the -# floor — they are not display values. +# QUALITY is a judgment made once at authoring time (entry.quality). SPEED is physics per machine: +# decode is memory-bound, so predicted tok/s ≈ bandwidth / bytes-read-per-token (build size scaled +# by decode fraction). The bandwidth axis is the `uma` flag: every discrete card that matters is +# 900+ GB/s GDDR while the unified-memory class measures ~1/5th of that. A measured per-machine +# bandwidth could replace these class constants without touching the rule; predictions order +# candidates and gate the floor — they are not display values. _DISCRETE_BANDWIDTH_GB_S = 1000.0 # representative GDDR6X/GDDR7 class _UMA_BANDWIDTH_GB_S = 210.0 # measured on unified-memory NVIDIA _HOST_BANDWIDTH_GB_S = 80.0 # spilled weights stream over host DRAM -# The one editorial constant in the tree: below this predicted decode -# speed a model stops feeling pleasant for agentic use (roughly reading -# speed with headroom for tool-call bursts). Distinct from the growth -# policy's 6 tok/s compress floor, which marks unusable, not unpleasant. +# Below this predicted decode speed a model stops feeling pleasant for agentic use (roughly +# reading speed with headroom for tool-call bursts). Distinct from the growth policy's 6 tok/s +# compress floor, which marks unusable, not unpleasant. PLEASANT_FLOOR_TOK_S = 20.0 @@ -230,16 +208,13 @@ def predicted_decode_tok_s(entry: CatalogEntry, variant: QuantVariant, def recommended_entry(budget: HardwareBudget, entries: "tuple[CatalogEntry, ...] | None" = None ) -> "tuple[CatalogEntry, str] | None": - """The catalog's default pick for THIS machine, with its reason. + """The catalog's default pick for THIS machine, with its reason key. Callers pass pre-filtered entries when some are ineligible for reasons the catalog can't know - (engine too old); default is the full catalog. - - best-quality-resident quality won among resident entries that clear the pleasant floor speed- - gated-quality same, but the floor eliminated a HIGHER quality candidate — the exact 'why not the - big model?' a unified-memory owner asks fastest-resident nothing resident clears the floor; the - quickest resident entry wins least-painful-spilled nothing runs resident; fastest from host - memory (MoE by construction) + (engine too old). Reasons: best-quality-resident (quality won among resident entries clearing + the pleasant floor); speed-gated-quality (same, but the floor eliminated a HIGHER quality + candidate); fastest-resident (nothing resident clears the floor); least-painful-spilled + (nothing runs resident; fastest from host memory — MoE by construction). """ pool = CATALOG if entries is None else entries fitting = [(e, c) for e in pool if (c := select_variant(e, budget)) is not None] @@ -257,22 +232,16 @@ def recommended_entry(budget: HardwareBudget, return (pick, "speed-gated-quality" if floor_gated else "best-quality-resident") if resident: return (max(resident, key=speed)[0], "fastest-resident") - # Everything spills: take the least painful — fastest predicted decode - # from host memory (MoE wins here by construction; a dense spill - # streams every weight over the host bus). return (max(fitting, key=lambda t: speed(t, spilled=True))[0], "least-painful-spilled") # ── catalog data: packaged JSON, refreshed from GitHub in memory ─ # -# The catalog DATA lives in catalog.json (checked in beside this module -# and shipped as package data); this module keeps all policy. At import -# we load the packaged copy — no network on the import path. A TTL-gated -# background refresh fetches the same file from the repo's main branch -# and swaps it in memory only: nothing on disk changes, so a git -# checkout never sees a dirty tracked file and the packaged copy remains -# the offline truth. A reverted commit on main heals every install on -# its next fetch, and day-0 entries reach users without an app release. +# catalog.json ships as package data and is loaded at import (no network on the import path). A +# TTL-gated background refresh fetches the same file from the repo's main branch and swaps it in +# MEMORY only: nothing on disk changes, so a git checkout never sees a dirty tracked file and the +# packaged copy remains the offline truth. A reverted commit on main heals every install on its +# next fetch, and day-0 entries reach users without an app release. _CATALOG_URL = ("https://raw.githubusercontent.com/NousResearch/hermes-agent" "/main/hermes_cli/local_runtime/catalog.json") @@ -289,8 +258,7 @@ def _asset_from(d: "dict | None") -> "AssetFile | None": local=d.get("local")) -# Scalar CatalogEntry fields parsed from JSON: key -> (coerce, default); -# a None default means the key is required. +# Scalar CatalogEntry fields parsed from JSON: key -> (coerce, default); None default = required. _SCALAR_FIELDS = { "n_ctx_train": (int, None), "full_layers": (int, None), "recurrent_layers": (int, None), "per_layer_f16": (int, None), @@ -302,9 +270,8 @@ _SCALAR_FIELDS = { def _load_catalog(doc: dict) -> "tuple[CatalogEntry, ...]": - """Parse a catalog document into entries. Unknown fields are ignored - (newer catalogs stay readable by older apps); a major schema bump is - the signal that they wouldn't be, and the caller skips the document.""" + """Parse a catalog document. Unknown fields are ignored (newer catalogs stay readable by older + apps); a major schema bump is the signal that they wouldn't be, and the caller skips it.""" if int(doc.get("schema_version", 0)) != _SCHEMA_VERSION: raise ValueError(f"catalog schema {doc.get('schema_version')!r} " f"(this build reads {_SCHEMA_VERSION})") @@ -337,12 +304,9 @@ CATALOG: "tuple[CatalogEntry, ...]" = _packaged_catalog() def refresh_catalog(force: bool = False) -> bool: - """Fetch the current catalog from the repo and swap it in memory. - - Best-effort by design: any failure (offline, GitHub down, unreadable schema) leaves the running - catalog untouched and retries after the TTL. Returns True when a fetched document replaced the - catalog. - """ + """Fetch the current catalog from the repo and swap it in memory. Best-effort: any failure + (offline, GitHub down, unreadable schema) leaves the running catalog untouched and retries + after the TTL. Returns True when a fetched document replaced the catalog.""" global CATALOG, _last_refresh_attempt now = time.monotonic() @@ -365,9 +329,8 @@ def refresh_catalog(force: bool = False) -> bool: def refresh_catalog_soon() -> None: - """TTL-gated background refresh; returns immediately. The caller's current request serves the - catalog it already has — the refresh lands for the next one. - """ + """TTL-gated background refresh; returns immediately. The current request serves the catalog + it already has — the refresh lands for the next one.""" if time.monotonic() - _last_refresh_attempt < _REFRESH_TTL_S: return threading.Thread(target=refresh_catalog, daemon=True, diff --git a/hermes_cli/local_runtime/hardware.py b/hermes_cli/local_runtime/hardware.py index e6bcd25e2e..0d3ca542c0 100644 --- a/hermes_cli/local_runtime/hardware.py +++ b/hermes_cli/local_runtime/hardware.py @@ -24,44 +24,35 @@ from hermes_cli.local_runtime.estimator import HardwareBudget logger = logging.getLogger(__name__) _GIB = 1 << 30 -# Reserve carved off the card before any grant: the desktop's own -# co-residents (compositor, browser, Electron) measure ~2-2.5 GiB on a -# working machine, and a window granted into that space demotes silently -# under WDDM. 7% covers big cards; the 2 GiB floor is what the margin's -# old 512 MiB floor failed to cover in practice (a 221K grant measured -# 31.9/32.6 GiB with the desktop running — 'fits' by the math, demoted -# in reality). Small cards give up window to this; spill mode is their -# path to big models regardless. +# Reserve carved off the card before any grant: the desktop's own co-residents (compositor, +# browser, Electron) measure ~2-2.5 GiB, and a window granted into that space demotes silently +# under WDDM. 7% covers big cards; the 2 GiB floor is what a 512 MiB floor failed to cover (a +# 221K grant measured 31.9/32.6 GiB with the desktop running — 'fits' by the math, demoted in +# reality). Small cards give up window to this; spill mode is their path to big models. _MARGIN_FLOOR = 2 << 30 _MARGIN_FRACTION = 0.09 -# UMA headroom: on unified-memory machines (Apple Silicon, unified-memory -# NVIDIA) the model shares physical memory with the OS and every app, so -# budget from RAM minus this fraction. +# UMA headroom: on unified-memory machines the model shares physical memory with the OS and every +# app, so budget from RAM minus this fraction. _UMA_HEADROOM_FRACTION = 0.20 -# Engine-fallback gates for the unified-pool quirk — BOTH must hold, and -# no discrete card can meet either: (1) the allocator's pool exceeds the -# smi report by well past rounding/ECC slack (discrete cards agree within -# ~2%; carve-out disagreement runs to whole multiples), and (2) the pool is -# system-RAM-sized — a workstation card in a RAM-matched box fails (1) -# because its smi and allocator AGREE, and a big discrete card in a -# bigger box fails (2). The driver's INTEGRATED attribute, when -# readable, bypasses both gates in whichever direction it points. +# Engine-fallback gates for the unified-pool quirk — BOTH must hold, and no discrete card can +# meet either: (1) the allocator's pool exceeds the smi report by well past rounding/ECC slack +# (discrete cards agree within ~2%; carve-out disagreement runs to whole multiples), and (2) the +# pool is system-RAM-sized. The driver's INTEGRATED attribute, when readable, bypasses both gates +# in whichever direction it points. _POOL_DISAGREEMENT_FACTOR = 1.5 _POOL_RAM_FRACTION = 0.75 # cuDeviceGetAttribute enum: device is integrated with host memory. _CU_DEVICE_ATTRIBUTE_INTEGRATED = 18 -# One probe per process once a device answers (silicon doesn't change); -# a miss retries after this long so a runtime installed mid-session gets -# picked up by the engine fallback. +# One probe per process once a device answers (silicon doesn't change); a miss retries after this +# long so a runtime installed mid-session gets picked up by the engine fallback. _POOL_NEGATIVE_TTL_S = 60.0 _pool_probe_cache: tuple[float, "tuple[int, bool | None] | None"] | None = None # ' CUDA0: NVIDIA Example Device (1234-core Example GPU) (46464 MiB, 46284 MiB free)' -# — greedy .* pins the LAST parenthesized group, so device names carrying -# their own parentheses parse correctly. +# — greedy .* pins the LAST parenthesized group, so device names with parentheses parse. _DEVICE_LINE_RE = re.compile(r"CUDA\d+:.*\((\d+)\s*MiB,\s*\d+\s*MiB free\)\s*$") @@ -91,9 +82,8 @@ def _ram_bytes() -> tuple[int, int]: pass try: if sys.platform == "darwin": - # macOS getconf has no _PHYS_PAGES/_AVPHYS_PAGES (exit 64, "no such - # configuration parameter") — the POSIX branch below returns (0, 0) - # and every model reads unavailable. sysctl is the platform truth. + # macOS getconf has no _PHYS_PAGES/_AVPHYS_PAGES (exit 64) — the POSIX branch would + # return (0, 0) and every model would read unavailable. sysctl is the platform truth. total = int(_stdout("/usr/sbin/sysctl", "-n", "hw.memsize").strip() or 0) if total <= 0: return 0, 0 @@ -102,8 +92,8 @@ def _ram_bytes() -> tuple[int, int]: out = _stdout("/usr/bin/vm_stat") page_m = re.search(r"page size of (\d+)", out) page = int(page_m.group(1)) if page_m else 16384 - # free + inactive + purgeable ≈ reclaimable-on-demand; the - # speculative pool is dropped by the OS under pressure too. + # free + inactive + purgeable ≈ reclaimable-on-demand; the speculative pool is + # dropped by the OS under pressure too. pages = sum(int(m.group(1)) for key in ( "Pages free", "Pages inactive", "Pages purgeable", "Pages speculative") if (m := re.search(rf"{key}:\s+(\d+)\.", out))) @@ -125,17 +115,15 @@ def _ram_bytes() -> tuple[int, int]: return 0, 0 -# nvidia-smi lives at a fixed path under the driver install; PATH presence -# varies by session type (services and gateways often run with a minimal -# environment) and by driver generation (legacy NVSMI dir was never on -# PATH). Resolution result is cached: the driver doesn't move mid-process. +# nvidia-smi lives at a fixed path under the driver install; PATH presence varies by session type +# (services and gateways often run minimal environments) and by driver generation (the legacy +# NVSMI dir was never on PATH). Cached: the driver doesn't move mid-process. _smi_path_cache: "tuple[str | None] | None" = None def _nvidia_smi_path() -> str | None: - """Absolute path to nvidia-smi, or None. PATH first (respects user overrides), then the driver's - known install locations on Windows; on Linux/WSL the PATH lookup is the whole ladder. - """ + """Absolute path to nvidia-smi, or None. PATH first (respects user overrides), then the + driver's known Windows install locations; on Linux/WSL the PATH lookup is the whole ladder.""" global _smi_path_cache if _smi_path_cache is not None: return _smi_path_cache[0] @@ -173,11 +161,10 @@ def _nvidia_vram() -> tuple[int, int] | None: def _cuda_driver_pool() -> "tuple[int, bool | None] | None": - """(allocator_total_bytes, integrated_or_None) from the CUDA driver API, or None when unreachable. - ctypes against the driver's own DLL/SO — no toolkit, no subprocess, ~ms. INTEGRATED is the - vendor's own unified-memory declaration; total is the pool the allocator will actually hand out - (on carve-out devices, several times what nvidia-smi reports). - """ + """(allocator_total_bytes, integrated_or_None) from the CUDA driver API via ctypes against the + driver's own DLL/SO — no toolkit, no subprocess, ~ms. INTEGRATED is the vendor's own + unified-memory declaration; total is the pool the allocator will actually hand out (on + carve-out devices, several times what nvidia-smi reports).""" import ctypes for name in ("nvcuda.dll", "libcuda.so.1", "libcuda.so"): @@ -210,9 +197,8 @@ def _cuda_driver_pool() -> "tuple[int, bool | None] | None": def _engine_device_pool() -> "tuple[int, bool | None] | None": """(engine_total_bytes, None) from the installed runtime's own --list-devices, or None. The - fallback truth source when the driver API is unreachable: asks the exact binary that will do the - allocating. Carries no integrated verdict — callers must gate it. - """ + fallback when the driver API is unreachable: asks the exact binary that will do the + allocating. Carries no integrated verdict — callers must gate it.""" try: from hermes_cli.local_runtime.binaries import ( installed_tags, @@ -242,9 +228,8 @@ def _engine_device_pool() -> "tuple[int, bool | None] | None": def _device_pool_view() -> "tuple[int, bool | None] | None": - """Best available allocator-side view, cached: a hit is permanent for the process, a miss retries - after a short TTL (the engine binary can appear mid-session via a pane install). - """ + """Best available allocator-side view, cached: a hit is permanent for the process, a miss + retries after a short TTL (the engine binary can appear mid-session via a pane install).""" global _pool_probe_cache now = time.monotonic() if _pool_probe_cache is not None: @@ -257,20 +242,17 @@ def _device_pool_view() -> "tuple[int, bool | None] | None": def _unified_pool_bytes(smi_total: int, ram_total: int) -> int | None: - """The real pool size when this NVIDIA device is unified memory behind + """The real pool size when this NVIDIA device is unified memory behind a carve-out, else None. - The driver's INTEGRATED attribute decides in BOTH directions when readable (0 pins discrete - even if numbers look odd; a declared-integrated driver is believed at modest pool sizes). Only - the attribute-less engine fallback needs the two numeric gates, both of which must hold. + The driver's INTEGRATED attribute decides in BOTH directions when readable; only the + attribute-less engine fallback needs the two numeric gates, both of which must hold. """ view = _device_pool_view() if view is None: return None pool, integrated = view - if integrated is False: - return None - if integrated is True: - return pool + if integrated is not None: + return pool if integrated else None if (smi_total > 0 and pool >= int(smi_total * _POOL_DISAGREEMENT_FACTOR) and ram_total > 0 and pool >= int(ram_total * _POOL_RAM_FRACTION)): return pool @@ -287,23 +269,18 @@ def probe_budget(*, planning: bool = False) -> HardwareBudget: """Construct the budget per the source rules above. ``planning=False``: LIVE budget (free VRAM now) for launch-time fit and growth re-grants. - ``planning=True``: CAPACITY budget (total minus margin) for catalog pricing and quant selection; - pricing against live-free while a model was loaded made every row read as too large and - degraded quant picks. The managed server unloads/relaunches itself, so capacity is real. + ``planning=True``: CAPACITY budget (total minus margin) for catalog pricing and quant + selection — pricing against live-free while a model was loaded made every row read as too + large. The managed server unloads/relaunches itself, so capacity is real. """ ram_total, ram_avail = _ram_bytes() vram = _nvidia_vram() - # Unified-memory NVIDIA: the CUDA allocator pool is the real - # capacity. Classification comes from the driver API/engine — it - # must not require nvidia-smi (stripped-PATH sessions lose smi but - # nvcuda loads via the system loader regardless). Crossing the - # carve-out costs nothing (effective bandwidth is flat through the - # boundary; smi's used/total merely saturate at it) — the carve-out - # is an OS accounting knob, not a GPU limit. Deliberately NOT - # clamped to OS RAM: carved-out memory is invisible to - # GlobalMemoryStatusEx (the OS reports correspondingly less total - # RAM), so a RAM clamp would throw away exactly the carved capacity. + # Unified-memory NVIDIA: the CUDA allocator pool is the real capacity. Classification comes + # from the driver API/engine and must not require nvidia-smi (stripped-PATH sessions lose smi + # but nvcuda loads via the system loader). Crossing the carve-out costs nothing — it is an OS + # accounting knob, not a GPU limit. Deliberately NOT clamped to OS RAM: carved-out memory is + # invisible to GlobalMemoryStatusEx, so a RAM clamp would throw away exactly that capacity. unified = _unified_pool_bytes(vram[0] if vram else 0, ram_total) if unified is not None: logger.info( @@ -314,19 +291,16 @@ def probe_budget(*, planning: bool = False) -> HardwareBudget: if planning: base = unified else: - # Live: dedicated-free plus what the OS can still give. smi's - # free saturates at the carve-out so this under-counts a bit — - # the safe direction (the pool edge is a measured soft cliff: - # decode collapses ~3.5x when concurrent demand hits it). - # Without smi, OS-available alone is the honest floor. + # Live: dedicated-free plus what the OS can still give. smi's free saturates at the + # carve-out so this under-counts a bit — the safe direction (the pool edge is a + # measured soft cliff: decode collapses ~3.5x when concurrent demand hits it). live = (vram[1] + ram_avail) if vram else ram_avail base = min(unified, live) return _uma_budget(base, unified) if vram is None: - # No NVIDIA device visible: Metal/Vulkan/CPU paths budget from RAM - # as UMA (Apple Silicon) — conservative for discrete AMD until a - # vendor probe lands (E3 hardware). + # No NVIDIA device visible: Metal/Vulkan/CPU paths budget from RAM as UMA (Apple + # Silicon) — conservative for discrete AMD until a vendor probe lands. return _uma_budget(ram_total if planning else ram_avail, ram_total) total, free = vram diff --git a/hermes_cli/local_runtime/supervisor.py b/hermes_cli/local_runtime/supervisor.py index 23d2fb258b..2fd5a2e80f 100644 --- a/hermes_cli/local_runtime/supervisor.py +++ b/hermes_cli/local_runtime/supervisor.py @@ -3,9 +3,9 @@ The router process is ours (restart with backoff on crash); router children are its problem — child failures surface via GET /models exit_code, never auto-retried here. -Readiness rules (each learned the hard way on real hardware): - health-200 is NOT readiness; every +Readiness rules (each learned the hard way on real hardware): health-200 is NOT readiness; every readiness claim requires a touch generation (temp-0, expected token, generous budget, -reasoning_content scanned). - Always dial 127.0.0.1 — resolving localhost adds ~2s per request on +reasoning_content scanned). Always dial 127.0.0.1 — resolving localhost adds ~2s per request on Windows via IPv6 fallback. """ @@ -29,11 +29,17 @@ logger = logging.getLogger(__name__) TOUCH_PROMPT = "Reply with exactly one word: the capital of France." TOUCH_EXPECT = "paris" _RESTART_BACKOFF_S = (1, 5, 15, 60) +_RESIDENT = ("loaded", "ready") + +# Chosen once and reused across restarts: sessions persist the resolved base_url, so an ephemeral +# port would strand every resumed session after each restart. Deliberately NOT 8080 so we never +# collide with a user's own llama-server/Ollama-adjacent stack. +_DEFAULT_PORT = 18434 def state_path() -> Path: - """Endpoint state for other Hermes processes (provider resolution reads - this to route llamacpp-alias requests at the managed server).""" + """Endpoint state for other Hermes processes (provider resolution routes llamacpp-alias + requests at the managed server from this).""" return runtimes_root() / "server.json" @@ -43,18 +49,9 @@ def _free_port() -> int: return s.getsockname()[1] -# Default port for the managed server, chosen once and reused across -# restarts. Sessions persist the resolved base_url; an ephemeral port -# would strand every resumed session on a dead endpoint after each -# restart. Deliberately NOT 8080 so we never collide with a user's own -# llama-server/Ollama-adjacent stack. -_DEFAULT_PORT = 18434 - - def _stable_port() -> int: - """The stable default port, falling back to an ephemeral one only when something else already - listens there (and it isn't a leftover managed server, which stop() would have cleaned up). - """ + """The stable default port, or an ephemeral one only when something else already listens + there (a leftover managed server would have been cleaned up by stop()).""" try: with socket.socket() as s: s.bind(("127.0.0.1", _DEFAULT_PORT)) @@ -69,9 +66,9 @@ def _stable_port() -> int: def _stable_api_key() -> str: """One key for the life of the install, persisted beside the runtimes. - Endpoint identity must survive restarts as a UNIT — sessions persist the resolved base_url + - api_key, so a per-boot key strands every resumed session on HTTP 401 exactly the way a per-boot - port would strand them on connection errors. + Endpoint identity must survive restarts as a UNIT — sessions persist base_url + api_key, so a + per-boot key strands every resumed session on HTTP 401 exactly as a per-boot port would on + connection errors. """ key_path = runtimes_root() / ".api_key" try: @@ -93,6 +90,12 @@ def _stable_api_key() -> str: class LlamaServerSupervisor: """Own one llama-server router process for the life of a Hermes session.""" + # A model that has gone quiet gets its VRAM back after this long. A constant, not a knob: + # long enough that an active conversation never trips it, short enough that a wandered-off + # session frees ~20 GiB within the hour. No exemptions: demand reloads anything the user + # comes back to. + IDLE_UNLOAD_S = 15 * 60 + def __init__(self, install_dir: Path, models_dir: Path, *, models_max: int = 4, port: int | None = None, extra_args: list[str] | None = None, @@ -151,19 +154,15 @@ class LlamaServerSupervisor: "--api-key", self.api_key, "--models-dir", str(self.models_dir), "--models-max", str(self.models_max), - # The residency contract at the layer that sees every message: - # a chat request to a staged-but-unloaded model loads it (slow - # first token) instead of failing with 'model not found' — - # without this flag, chat after an eject is a bare 400/404. + # Residency contract: a chat request to a staged-but-unloaded model loads it (slow + # first token) instead of a bare 400/404 after an eject. "--models-autoload", "--metrics", # opt-in flag; supervisor telemetry needs it "--slots", # /slots endpoint is also opt-in; is_idle reads it "--no-webui", "--jinja", - # Direct I/O on model load: bypasses the page cache, so a - # multi-GB load doesn't evict half the OS cache — measured - # faster loads on NVMe, and our router bounces (download/ - # delete/activate) reload models often enough to care. + # Direct I/O on model load bypasses the page cache so a multi-GB load doesn't evict + # half the OS cache — measured faster on NVMe, and our router bounces reload often. "-dio", ] if self.preset_path and self.preset_path.exists(): @@ -171,8 +170,7 @@ class LlamaServerSupervisor: cmd += self.extra_args self.log_path.parent.mkdir(parents=True, exist_ok=True) if self._log_handle is not None: - # The crash-restart loop calls _spawn repeatedly; without - # closing the prior handle each restart leaks one fd. + # The crash-restart loop calls _spawn repeatedly; each restart would leak one fd. try: self._log_handle.close() except Exception: # noqa: BLE001 — best-effort @@ -184,10 +182,9 @@ class LlamaServerSupervisor: self.proc = subprocess.Popen(cmd, stdout=self._log_handle, stderr=subprocess.STDOUT, cwd=str(exe.parent)) logger.info("llama-server router spawned pid=%s port=%s", self.proc.pid, self.port) - # State goes down at SPAWN, not after health: endpoint resolution - # treats a live-pid-but-not-yet-healthy server as "starting" rather - # than "unconfigured", so a readiness probe racing the boot doesn't - # throw the app back to onboarding (observed on first restart test). + # State goes down at SPAWN, not after health: endpoint resolution treats a + # live-pid-but-not-yet-healthy server as "starting" rather than "unconfigured", so a + # readiness probe racing the boot doesn't throw the app back to onboarding. self._write_state() def start(self, timeout_s: int = 120) -> None: @@ -263,12 +260,10 @@ class LlamaServerSupervisor: def _terminate_tree(proc: subprocess.Popen) -> None: """Terminate the router AND its model children. - The router spawns one child llama-server per loaded model, each holding gigabytes of VRAM. - Terminating only the router (on Windows, TerminateProcess with no cleanup) orphans them: the - port goes quiet but the weights stay resident. Enumerate children FIRST (the parent must be - alive to walk them), then terminate all together, escalating to kill for stragglers. + Each child holds gigabytes of VRAM; terminating only the router (TerminateProcess on + Windows does no cleanup) orphans them with the weights still resident. Enumerate children + FIRST (the parent must be alive to walk them), terminate all, escalate to kill. """ - children: list = [] try: import psutil @@ -295,10 +290,9 @@ class LlamaServerSupervisor: def _reap_orphaned_children(self) -> None: """Kill model children orphaned by a router crash, before respawn. - A crashed router can't clean up its children, and a dead parent can't be walked — so match - by identity instead: any process running OUR llama-server binary whose parent is gone is an - orphan of a previous router. Their VRAM must come back before the new router loads models - next to the ghosts. + A dead parent can't be walked, so match by identity: any process running OUR + llama-server binary whose parent is gone is an orphan of a previous router. Its VRAM must + come back before the new router loads models next to the ghosts. """ try: import psutil @@ -332,33 +326,21 @@ class LlamaServerSupervisor: self._request("/models/load", {"model": model_id}, timeout_s=timeout_s) def unload_model(self, model_id: str) -> None: - """Free the child's VRAM now. Route existence verified empirically on b10290 (POST - /models/unload; bogus name -> 400 'model is not found'). Momentary action: never touches - primary_model — the declaration is durable, an eject is not (residency design). - """ + """Free the child's VRAM now (POST /models/unload; bogus name -> 400). Momentary: never + touches primary_model — the declaration is durable, an eject is not.""" self._request("/models/unload", {"model": model_id}, timeout_s=120) deadline = time.monotonic() + 15 while time.monotonic() < deadline: try: - if self.models().get(model_id) not in ("loaded", "ready", "unloading"): + if self.models().get(model_id) not in (*_RESIDENT, "unloading"): return except Exception: # noqa: BLE001 return time.sleep(0.3) - # ── idle residency (non-primary models) ────────────────── - - # A model that has gone quiet gets its VRAM back after this long. A - # constant, not a knob: long enough that an active conversation never - # trips it, short enough that a wandered-off session frees ~20 GiB - # within the hour. No exemptions (residency v2): demand reloads - # anything the user comes back to. - IDLE_UNLOAD_S = 15 * 60 - def sweep_idle(self, now: float | None = None) -> list[str]: - """Unload models idle past IDLE_UNLOAD_S. Returns the model ids - unloaded. Idle means no busy slots and no queued work, tracked - per model across calls; a model seen busy resets its clock.""" + """Unload models idle past IDLE_UNLOAD_S; returns their ids. Idle = no busy slots and no + queued work, tracked per model across calls; a model seen busy resets its clock.""" now = time.monotonic() if now is None else now unloaded: list[str] = [] try: @@ -366,7 +348,7 @@ class LlamaServerSupervisor: except Exception: # noqa: BLE001 return unloaded for model_id, status in statuses.items(): - if status not in ("loaded", "ready") or not self.is_idle(model_id): + if status not in _RESIDENT or not self.is_idle(model_id): self._idle_since.pop(model_id, None) continue first_idle = self._idle_since.setdefault(model_id, now) @@ -383,8 +365,7 @@ class LlamaServerSupervisor: def touch_generate(self, model_id: str, timeout_s: int = 300) -> bool: """The readiness proof. Generous budget + reasoning_content scan — small token budgets - false-fail reasoning models, which spend their first tokens thinking. - """ + false-fail reasoning models, which spend their first tokens thinking.""" try: resp = self._request("/v1/chat/completions", { "model": model_id, @@ -403,23 +384,21 @@ class LlamaServerSupervisor: status = self.models().get(model_id) if status is None: raise KeyError(f"model {model_id} not present in models dir") - if status not in ("loaded", "ready"): + if status not in _RESIDENT: self.load_model(model_id, timeout_s=timeout_s) return self.touch_generate(model_id) # ── telemetry ──────────────────────────────────────────── def is_idle(self, model_id: str | None = None) -> bool: - """No processing requests and no busy slots. Router quirk: /slots and /metrics are per-child - and require ?model= (bare calls 400), and no KV-usage metric exists. With ``model_id`` - checks that one child; without, every loaded child. - """ + """No processing requests and no busy slots. Router quirk: /slots and /metrics are + per-child and require ?model= (bare calls 400). With ``model_id`` checks that one child; + without, every loaded child.""" try: if model_id is not None: loaded = [model_id] else: - loaded = [m for m, status in self.models().items() - if status in ("loaded", "ready")] + loaded = [m for m, status in self.models().items() if status in _RESIDENT] for mid in loaded: slots = self._request(f"/slots?model={mid}") if any(s.get("is_processing") for s in slots): @@ -427,9 +406,9 @@ class LlamaServerSupervisor: with self._open(f"/metrics?model={mid}", timeout_s=10, json_type=False) as r: text = r.read().decode() for line in text.splitlines(): - if line.startswith("llamacpp:requests_processing"): - if float(line.split()[-1]) != 0.0: - return False + if (line.startswith("llamacpp:requests_processing") + and float(line.split()[-1]) != 0.0): + return False return True except Exception: # noqa: BLE001 return False