refactor(hermes_cli/web_routers): local_models — _http_error ctx, _step/_download_job/_catalog_row/_quickstart_target helpers, status phase helpers (1174->1142 LOC)

This commit is contained in:
Teknium
2026-09-02 20:53:08 -07:00
parent 6e578041ae
commit 448e1fa50c
+202 -234
View File
@@ -10,6 +10,7 @@ job pattern: start-POST -> {job_id} -> GET poll with byte progress.
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
import os
@@ -53,12 +54,26 @@ _GIB = 1 << 30
_JOBS: Dict[str, Dict[str, Any]] = {}
_JOBS_LOCK = threading.Lock()
_LLAMACPP_PROVIDERS = ("llamacpp", "llama.cpp", "llama-cpp")
_SPLIT_PART_RE = r"-\d{5}-of-\d{5}"
def _human_gb(n: int | float) -> str:
return f"{n / _GIB:.1f} GB"
def _k_label(tokens: int) -> str:
return f"{tokens // 1024}K"
@contextlib.contextmanager
def _http_error(status: int, prefix: str = ""):
"""Map any exception to ``HTTPException(status, f"{prefix}{exc}")``."""
try:
yield
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=status, detail=f"{prefix}{exc}") from exc
def _job(kind: str, target: str, model_id: str | None = None) -> Dict[str, Any]:
job = {
"job_id": uuid.uuid4().hex[:12], "kind": kind, "target": target,
@@ -80,12 +95,16 @@ def _job_view(job: Dict[str, Any]) -> Dict[str, Any]:
return out
def _finish(job: Dict[str, Any], detail: str) -> None:
job["phase"] = "done"
job["status"] = "done"
def _step(job: Dict[str, Any], phase: str, detail: str) -> None:
job["phase"] = phase
job["detail"] = detail
def _finish(job: Dict[str, Any], detail: str) -> None:
_step(job, "done", detail)
job["status"] = "done"
def _spawn_job(job: Dict[str, Any], name: str, body: Callable[[], None], *,
fail_msg: str | None = None,
on_exit: Callable[[], None] | None = None) -> None:
@@ -110,7 +129,6 @@ def _refresh_runtime(skip_msg: str) -> None:
"""Bounce a running router so it rescans the models dir (it only scans at
spawn). Never raises — the file operation already succeeded."""
try:
bootstrap.refresh_local_runtime()
except Exception: # noqa: BLE001
logger.debug(skip_msg, exc_info=True)
@@ -164,13 +182,12 @@ def _probe_range_support(url: str) -> int:
def _model_id_for(gguf: Path) -> str:
"""Variant model id for a staged file (strips split-part suffixes)."""
return re.sub(r"-\d{5}-of-\d{5}$", "", gguf.stem)
return re.sub(_SPLIT_PART_RE + "$", "", gguf.stem)
def _variant_files_on_disk(model_id: str) -> "list[Path]":
"""Every local file belonging to a staged model: all split parts plus
its catalog-declared assets (mmproj/draft) when present."""
files = [p for p in _models_dir().glob("*.gguf") if _model_id_for(p) == model_id]
hit = catalog.find_entry_for_model(model_id)
if hit is not None:
@@ -198,18 +215,15 @@ def download_file(url: str, dest: Path, job: Dict[str, Any],
file_done = [0]
progress_lock = threading.Lock()
def bump(n: int) -> None:
with progress_lock:
file_done[0] += n
job["done_bytes"] = base_done + file_done[0]
def pump(r, f) -> None:
while True:
chunk = r.read(_CHUNK)
if not chunk:
break
f.write(chunk)
bump(len(chunk))
with progress_lock:
file_done[0] += len(chunk)
job["done_bytes"] = base_done + file_done[0]
try:
# The probe and the preallocation both take real seconds on a 20+ GB
@@ -270,7 +284,6 @@ def download_file(url: str, dest: Path, job: Dict[str, Any],
def _models_dir() -> Path:
return bootstrap.models_dir()
@@ -281,7 +294,6 @@ def _hf_url(repo: str, path: str) -> str:
def _download_plan(entry, variant) -> list:
"""Everything a variant needs: split parts + mmproj/draft assets, as
(url, dest, bytes) tuples."""
plan = [(_hf_url(entry.repo, a.path), _models_dir() / a.local_name, a.size_bytes)
for a in variant.files]
plan += [(_hf_url(entry.repo, a.path), bootstrap.assets_dir() / a.local_name, a.size_bytes)
@@ -293,8 +305,7 @@ def _run_download_plan(job: Dict[str, Any], plan: list, label: str) -> None:
"""Download every missing file in ``plan``; already-present files count
toward progress without a transfer."""
total = sum(p[2] for p in plan)
job["phase"] = "downloading"
job["detail"] = f"{label} — {_human_gb(total)}"
_step(job, "downloading", f"{label} — {_human_gb(total)}")
done_before = 0
for url, dest, size in plan:
if not dest.exists():
@@ -311,7 +322,6 @@ def _engine_too_old(min_engine: str) -> bool:
if not min_engine:
return False
try:
tags = binaries.installed_tags() or [binaries.default_tag()]
newest = max(int(t.lstrip("b")) for t in tags if t.lstrip("b").isdigit())
return newest < int(min_engine.lstrip("b"))
@@ -320,7 +330,6 @@ def _engine_too_old(min_engine: str) -> bool:
def _load_config() -> dict:
try:
return config_mod.load_config()
except Exception: # noqa: BLE001
@@ -333,7 +342,6 @@ def _runtime_section() -> dict:
def _set_runtime_enabled(enabled: bool) -> dict:
"""Persist ``local_runtime.enabled`` and return the config written."""
config = config_mod.load_config()
config.setdefault("local_runtime", {})["enabled"] = enabled
config_mod.save_config(config)
@@ -341,7 +349,6 @@ def _set_runtime_enabled(enabled: bool) -> dict:
def _resolve_backend(section: dict, requested: str | None = None) -> str:
backend = requested or section.get("backend", "auto")
return binaries.select_backend(bootstrap._detect_gpu_vendor()) if backend == "auto" else backend
@@ -349,22 +356,32 @@ def _resolve_backend(section: dict, requested: str | None = None) -> str:
def _eligible_entries():
"""Catalog entries this engine can activate today (engine-gated ones
can't be the recommendation either)."""
return tuple(e for e in catalog.CATALOG if not _engine_too_old(e.min_engine))
def _resolve_assets_or_400(tag: str, backend: str):
"""Resolve first so an impossible combination fails the POST, not the job."""
with _http_error(400):
return binaries.resolve_assets(tag, backend)
def _start_local_server(config: dict, fail_detail: str):
"""Start the local server (force) and return the supervisor; raise
``fail_detail`` when neither we nor another process ended up serving."""
sup = bootstrap.ensure_local_runtime(config, force=True)
if sup is None and _state_endpoint() is None:
raise RuntimeError(fail_detail)
return sup
def _ensure_server(job: Dict[str, Any], config: dict, model_id: str, *,
fail_detail: str, skip_msg: str) -> None:
"""Start the local server if needed and self-heal a stale router: the
model list is spawn-only, so a server started before ``model_id``
finished downloading can't serve it — bounce it when it doesn't know
the model."""
job["phase"] = "starting-server"
job["detail"] = "Starting the local server"
sup = bootstrap.ensure_local_runtime(config, force=True)
if sup is None and _state_endpoint() is None:
raise RuntimeError(fail_detail)
_step(job, "starting-server", "Starting the local server")
sup = _start_local_server(config, fail_detail)
if sup is not None:
try:
if model_id not in sup.models():
@@ -377,9 +394,7 @@ def _ensure_server(job: Dict[str, Any], config: dict, model_id: str, *,
def _assign_default(job: Dict[str, Any], model_id: str) -> None:
"""Make ``model_id`` the main model through the same machinery as
/api/model/set (late-bound so tests can stub web_deps.late)."""
job["phase"] = "setting-default"
job["detail"] = "Making it your default"
_step(job, "setting-default", "Making it your default")
web_deps.late("_apply_model_assignment_sync")("main", "llamacpp", model_id, "", "", "")
@@ -390,7 +405,6 @@ def _loaded_models(running: Dict[str, Any]) -> "tuple[Dict[str, str], Dict[str,
"""Resident models right now, plus how each is placed (granted window
from the child, spill facts from the preset decision) — the difference
between 'fast' and 'why is my CPU busy', so it must be inspectable."""
data = _router_request(running, "/models", timeout=3)
# Everything resident or becoming resident: 'loading' renders as its own
# state in the pane (a 20-GB load in flight is the most important thing
@@ -407,7 +421,7 @@ def _loaded_models(running: Dict[str, Any]) -> "tuple[Dict[str, str], Dict[str,
plan = decisions.get(model_id)
if plan is not None:
facts["window"] = plan.window
facts["window_label"] = f"{plan.window // 1024}K"
facts["window_label"] = _k_label(plan.window)
facts["spilled"] = plan.spilled
if state in ("loaded", "ready"):
try:
@@ -415,7 +429,7 @@ def _loaded_models(running: Dict[str, Any]) -> "tuple[Dict[str, str], Dict[str,
n_ctx = props.get("default_generation_settings", {}).get("n_ctx")
if n_ctx:
facts["granted_window"] = int(n_ctx)
facts["granted_window_label"] = f"{int(n_ctx) // 1024}K"
facts["granted_window_label"] = _k_label(int(n_ctx))
except Exception: # noqa: BLE001
pass
if facts:
@@ -423,12 +437,37 @@ def _loaded_models(running: Dict[str, Any]) -> "tuple[Dict[str, str], Dict[str,
return loaded, placement
def _installed_backend(tag: str) -> str | None:
"""Name of the first backend dir under ``tag`` with a working server binary."""
root = binaries.runtimes_root() / tag
if not root.exists():
return None
for backend_dir in sorted(p for p in root.iterdir() if p.is_dir()):
try:
binaries.server_binary(backend_dir)
return backend_dir.name
except Exception: # noqa: BLE001
continue
return None
def _active_llamacpp_model_id() -> str | None:
"""The active main model when it is one of ours (config authority: the
same model.provider + model.default that /api/model/set writes)."""
try:
model_section = (_load_config() or {}).get("model") or {}
if str(model_section.get("provider", "")).strip().lower() in _LLAMACPP_PROVIDERS:
return str(model_section.get("default") or model_section.get("name") or "").strip() or None
except Exception: # noqa: BLE001
pass
return None
@router.get("/api/local-models/status")
def local_models_status():
"""Cheap, immediate, never blocks on probes: config state + installed
runtime + staged models + supervisor state (GPU facts live in /hardware).
Sync def on purpose: blocking urlopen/scans run in the threadpool."""
section = _runtime_section()
configured_tag = section.get("tag") or binaries.default_tag()
have = binaries.installed_tags()
@@ -440,24 +479,12 @@ def local_models_status():
# Update pending = engine in use (enabled + something installed) and the
# configured tag (pinned or release default) isn't on disk. The download
# is a button click, never automatic.
update_available = bool(
section.get("enabled") and have and configured_tag not in have)
runtime_backend = None
root = binaries.runtimes_root() / tag
if root.exists():
for backend_dir in sorted(p for p in root.iterdir() if p.is_dir()):
try:
binaries.server_binary(backend_dir)
runtime_backend = backend_dir.name
break
except Exception: # noqa: BLE001
continue
update_available = bool(section.get("enabled") and have and configured_tag not in have)
runtime_backend = _installed_backend(tag)
staged = []
mdir = _models_dir()
if mdir.exists():
for gguf in bootstrap.staged_models():
model_id = _model_id_for(gguf)
# Split models: report the whole variant's bytes, not one part's.
@@ -479,18 +506,6 @@ def local_models_status():
logger.warning("loaded-models read failed: %r", exc)
loaded = {}
# The active main model, when it is one of ours (config authority: the
# same model.provider + model.default that /api/model/set writes).
active_model_id = None
try:
model_section = (_load_config() or {}).get("model") or {}
if str(model_section.get("provider", "")).strip().lower() in _LLAMACPP_PROVIDERS:
active_model_id = str(
model_section.get("default") or model_section.get("name") or ""
).strip() or None
except Exception: # noqa: BLE001
pass
return {
"enabled": bool(section.get("enabled")),
"tag": tag,
@@ -500,7 +515,7 @@ def local_models_status():
"runtime_backend": runtime_backend,
"server_running": running is not None,
"server_base_url": (running or {}).get("base_url"),
"active_model_id": active_model_id,
"active_model_id": _active_llamacpp_model_id(),
"loaded_models": loaded,
# Live load progress per model (SSE-fed): {model_id: {stage, value,
# percent}}. The chat's loading bar and the picker rows poll this.
@@ -513,7 +528,6 @@ def local_models_status():
def _loading_progress() -> Dict[str, Any]:
try:
return load_progress.get_loading_progress()
except Exception: # noqa: BLE001 — progress is garnish, never a 500
return {}
@@ -526,7 +540,6 @@ def _loading_progress() -> Dict[str, Any]:
def local_models_hardware():
"""The budget as plain facts, polled by the pane and statusbar. Sync def
on purpose: shells out to nvidia-smi — threadpool, not loop."""
budget = hardware.probe_budget()
ram_total, ram_avail = hardware._ram_bytes()
out = {
@@ -538,7 +551,6 @@ def local_models_hardware():
# GPU identity + live utilization (NVIDIA; other vendors degrade to None
# and the UI hides those readouts).
try:
smi_exe = hardware._nvidia_smi_path()
smi = subprocess.run(
[smi_exe, "--query-gpu=name,utilization.gpu,memory.used",
@@ -567,6 +579,70 @@ _QUANT_REASON_COMPACT = ("Compact build sized for this machine ({quant}) — "
"larger than GPU memory, runs slower")
def _catalog_row(entry, budget, recommended, recommended_reason, staged_ids) -> Dict[str, Any]:
choice = catalog.select_variant(entry, budget)
# Any variant of this family on disk counts as downloaded.
dl = next((v for v in entry.variants if v.model_id in staged_ids), None)
row: Dict[str, Any] = {
"id": entry.id, "display_name": entry.display_name, "description": entry.description,
"native_context": entry.n_ctx_train,
"native_context_label": _k_label(entry.n_ctx_train),
"recommended": entry.id == recommended,
"recommended_reason": recommended_reason if entry.id == recommended else None,
"downloaded": dl is not None,
"downloaded_model_id": dl.model_id if dl else None,
"downloaded_quant": dl.quant if dl else None,
"mtp": entry.mtp, "vision": entry.mmproj is not None,
# Day-0 architectures need the llama.cpp release where their support
# landed: True gates download/activate until the engine updates, but
# the row still renders (visible + explained beats hidden).
"needs_engine": _engine_too_old(entry.min_engine),
"min_engine": entry.min_engine or None,
}
if choice is None:
smallest = min(entry.variants, key=lambda v: v.size_bytes)
smallest_total = entry.download_bytes(smallest)
row.update({
"fits": False, "size_bytes": smallest_total, "size_label": _human_gb(smallest_total),
"fit_summary": "Needs more memory than this machine has",
"fit_detail": (f"even the most compact build ({smallest.quant}, "
f"{_human_gb(smallest_total)}) exceeds GPU + system memory"),
})
return row
variant = choice.variant
# Same overhead the launch decision prices (runtime buffers + vision
# projector + microbatch/MTP logits): the row must advertise the window
# the model will actually get, not a paper number.
overhead = (context_policy.RUNTIME_OVERHEAD_BYTES
+ (entry.mmproj.size_bytes if entry.mmproj else 0)
+ context_policy.ub_logits_bytes(entry.n_vocab, mtp_capable=entry.mtp))
decision = context_policy.initial_window(entry.profile(variant), budget, overhead_bytes=overhead)
download_total = entry.download_bytes(variant)
row.update({
"fits": True, "model_id": variant.model_id, "quant": variant.quant,
"quant_validated": variant.validated, "size_bytes": download_total,
"size_label": _human_gb(download_total), "variant_count": len(entry.variants),
"quant_reason": _QUANT_REASONS.get(
choice.reason_key, _QUANT_REASON_COMPACT).format(quant=variant.quant),
})
if isinstance(decision, estimator.PhysicsRefusal):
row["fit_summary"] = row["quant_reason"]
return row
row["start_window"] = decision.window
row["start_window_label"] = _k_label(decision.window)
row["spilled"] = decision.spilled
if decision.window >= entry.n_ctx_train:
shape = f"runs at its full {row['native_context_label']} context"
else:
shape = (f"starts at {row['start_window_label']} and grows toward "
f"{row['native_context_label']} as you use it")
if decision.spilled:
shape += " (larger than your GPU memory — runs slower)"
row["fit_summary"] = shape
return row
@router.get("/api/local-models/catalog")
def local_models_catalog():
"""Every entry answers up front: how big is the download, will it fit,
@@ -574,7 +650,6 @@ def local_models_catalog():
build for this machine (highest quality fully on GPU at the 64K floor;
else the smallest that works, spilled and priced). No entry is hidden;
unaffordable models show WHY. Sync def: blocking I/O -> threadpool."""
# Serve the in-memory catalog; a TTL-gated background fetch lands new
# entries for the next call (day-0 models without an app release).
catalog.refresh_catalog_soon()
@@ -589,71 +664,8 @@ def local_models_catalog():
# Completeness-checked staging (split parts all present) — same answer the
# picker and router see, so a mid-download model never reads as downloaded.
staged_ids = set(bootstrap.staged_model_ids())
entries = []
for entry in catalog.CATALOG:
choice = catalog.select_variant(entry, budget)
# Any variant of this family on disk counts as downloaded.
dl = next((v for v in entry.variants if v.model_id in staged_ids), None)
row: Dict[str, Any] = {
"id": entry.id, "display_name": entry.display_name, "description": entry.description,
"native_context": entry.n_ctx_train,
"native_context_label": f"{entry.n_ctx_train // 1024}K",
"recommended": entry.id == recommended,
"recommended_reason": recommended_reason if entry.id == recommended else None,
"downloaded": dl is not None,
"downloaded_model_id": dl.model_id if dl else None,
"downloaded_quant": dl.quant if dl else None,
"mtp": entry.mtp, "vision": entry.mmproj is not None,
# Day-0 architectures need the llama.cpp release where their support
# landed: True gates download/activate until the engine updates, but
# the row still renders (visible + explained beats hidden).
"needs_engine": _engine_too_old(entry.min_engine),
"min_engine": entry.min_engine or None,
}
if choice is None:
smallest = min(entry.variants, key=lambda v: v.size_bytes)
smallest_total = entry.download_bytes(smallest)
row.update({
"fits": False, "size_bytes": smallest_total, "size_label": _human_gb(smallest_total),
"fit_summary": "Needs more memory than this machine has",
"fit_detail": (f"even the most compact build ({smallest.quant}, "
f"{_human_gb(smallest_total)}) exceeds GPU + system memory"),
})
entries.append(row)
continue
variant = choice.variant
# Same overhead the launch decision prices (runtime buffers + vision
# projector + microbatch/MTP logits): the row must advertise the window
# the model will actually get, not a paper number.
overhead = (context_policy.RUNTIME_OVERHEAD_BYTES
+ (entry.mmproj.size_bytes if entry.mmproj else 0)
+ context_policy.ub_logits_bytes(entry.n_vocab, mtp_capable=entry.mtp))
decision = context_policy.initial_window(entry.profile(variant), budget, overhead_bytes=overhead)
download_total = entry.download_bytes(variant)
row.update({
"fits": True, "model_id": variant.model_id, "quant": variant.quant,
"quant_validated": variant.validated, "size_bytes": download_total,
"size_label": _human_gb(download_total), "variant_count": len(entry.variants),
"quant_reason": _QUANT_REASONS.get(
choice.reason_key, _QUANT_REASON_COMPACT).format(quant=variant.quant),
})
if not isinstance(decision, estimator.PhysicsRefusal):
row["start_window"] = decision.window
row["start_window_label"] = f"{decision.window // 1024}K"
row["spilled"] = decision.spilled
if decision.window >= entry.n_ctx_train:
shape = f"runs at its full {row['native_context_label']} context"
else:
shape = (f"starts at {row['start_window_label']} and grows toward "
f"{row['native_context_label']} as you use it")
if decision.spilled:
shape += " (larger than your GPU memory — runs slower)"
row["fit_summary"] = shape
else:
row["fit_summary"] = row["quant_reason"]
entries.append(row)
return {"models": entries}
return {"models": [_catalog_row(e, budget, recommended, recommended_reason, staged_ids)
for e in catalog.CATALOG]}
# ── runtime install (job) ────────────────────────────────────
@@ -693,35 +705,25 @@ def _runtime_progress_hook(job: Dict[str, Any]):
job["done_bytes"] = plan_done
job["total_bytes"] = plan_total or None
elif stage == "extract":
job["phase"] = "unpacking-runtime"
pct = f" — {min(100, round(done / total * 100))}%" if total else ""
job["detail"] = f"Unpacking the engine{suffix}{pct}"
_step(job, "unpacking-runtime", f"Unpacking the engine{suffix}{pct}")
else: # verify
job["phase"] = "verifying-runtime"
job["detail"] = f"Verifying the engine{suffix}"
_step(job, "verifying-runtime", f"Verifying the engine{suffix}")
return hook
@router.post("/api/local-models/runtime/install")
async def local_models_runtime_install(body: RuntimeInstallBody):
section = _runtime_section()
tag = section.get("tag") or binaries.default_tag()
backend = _resolve_backend(section, body.backend)
# Resolve first so an impossible combination fails the POST, not the job.
try:
plan = binaries.resolve_assets(tag, backend)
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=400, detail=str(exc))
plan = _resolve_assets_or_400(tag, backend)
job = _job("runtime-install", f"llama.cpp {tag} ({backend})")
def _run():
previous = binaries.installed_tags()
job["phase"] = "downloading"
job["detail"] = f"Fetching {len(plan.assets)} package(s) for {backend}"
_step(job, "downloading", f"Fetching {len(plan.assets)} package(s) for {backend}")
binaries.ensure_runtime_installed(tag, backend, progress=_runtime_progress_hook(job))
# Engine update path: a server already running on an older tag moves
@@ -729,10 +731,8 @@ async def local_models_runtime_install(body: RuntimeInstallBody):
# server) skip this; Use/boot handles their start.
restarted = False
try:
if bootstrap.get_supervisor() is not None and previous and tag not in previous:
job["phase"] = "restarting"
job["detail"] = "Switching the running server to the new build"
_step(job, "restarting", "Switching the running server to the new build")
bootstrap.shutdown_local_runtime()
bootstrap.ensure_local_runtime(_load_config(), force=True)
restarted = True
@@ -761,11 +761,22 @@ class ModelDownloadBody(BaseModel):
model_id: str
def _download_job(job: Dict[str, Any], body: Callable[[], None], name: str, label: str,
fail_msg: str | None = None) -> None:
"""Spawn a download job: ``body`` fetches, then the job finishes as
"<label> ready" and the router is bounced to pick the file up."""
def _run():
body()
_finish(job, f"{label} ready")
_refresh_runtime("post-download runtime refresh skipped")
_spawn_job(job, name, _run, fail_msg=fail_msg)
@router.post("/api/local-models/download")
async def local_models_download(body: ModelDownloadBody):
"""Accepts either a family id (downloads this machine's selected
variant) or an exact variant model_id."""
entry = catalog.catalog_by_id().get(body.model_id)
variant = None
if entry is not None:
@@ -795,13 +806,8 @@ async def local_models_download(body: ModelDownloadBody):
job = _job("model-download", f"{entry.display_name} ({variant.quant})",
model_id=entry.id)
job["total_bytes"] = sum(p[2] for p in plan)
def _run():
_run_download_plan(job, plan, entry.display_name)
_finish(job, f"{entry.display_name} ready")
_refresh_runtime("post-download runtime refresh skipped")
_spawn_job(job, "lr-model-download", _run, fail_msg="model download failed: %s")
_download_job(job, lambda: _run_download_plan(job, plan, entry.display_name),
"lr-model-download", entry.display_name, fail_msg="model download failed: %s")
return {"job_id": job["job_id"], "model_id": variant.model_id}
@@ -818,7 +824,6 @@ async def local_models_delete(model_id: str):
# Growth state dies with the model: a re-download starts back at its
# zero-spill window instead of inheriting a stale grown one.
try:
growth.clear_window_override(model_id)
except Exception: # noqa: BLE001
logger.debug("window-override clear skipped", exc_info=True)
@@ -828,13 +833,6 @@ async def local_models_delete(model_id: str):
return {"ok": True}
# ── server lifecycle: turn the engine on/off ─────────────────
class ServerActionBody(BaseModel):
action: str # "stop" | "start"
# ── quickstart: one click from nothing to a working default ──
@@ -848,18 +846,9 @@ class QuickstartBody(BaseModel):
_QUICKSTART_LOCK = threading.Lock()
@router.post("/api/local-models/quickstart")
async def local_models_quickstart(body: QuickstartBody):
"""One job: install the runtime (if missing), download this machine's
build of the recommended model (if missing), make it the default. Each
leg is the same code the individual routes run, so 'Configure' and
quickstart can never disagree. Preflight rejects (no servable entry,
engine too old) fail the POST synchronously so the button can explain
itself; everything slow runs in the job with phase/byte progress."""
# Resolve the target entry: explicit id, else this machine's
# recommendation, else the first catalog entry this machine can serve.
budget = hardware.probe_budget(planning=True)
def _quickstart_target(body: QuickstartBody, budget):
"""(entry, variant) to set up: explicit id, else this machine's
recommendation, else the first catalog entry this machine can serve."""
if body.model_id:
entry = catalog.catalog_by_id().get(body.model_id)
if entry is None:
@@ -871,29 +860,32 @@ async def local_models_quickstart(body: QuickstartBody):
best = picked[0] if picked is not None else None
candidates = ([best] if best is not None else []) + [
e for e in catalog.CATALOG if best is None or e.id != best.id]
chosen = None
for candidate in candidates:
choice = catalog.select_variant(candidate, budget)
if choice is not None and not _engine_too_old(candidate.min_engine):
chosen = (candidate, choice.variant)
break
if chosen is None:
raise HTTPException(
status_code=409,
detail="no catalog model fits this machine — open Local Models "
"to browse for a smaller build")
entry, variant = chosen
return candidate, choice.variant
raise HTTPException(
status_code=409,
detail="no catalog model fits this machine — open Local Models "
"to browse for a smaller build")
@router.post("/api/local-models/quickstart")
async def local_models_quickstart(body: QuickstartBody):
"""One job: install the runtime (if missing), download this machine's
build of the recommended model (if missing), make it the default. Each
leg is the same code the individual routes run, so 'Configure' and
quickstart can never disagree. Preflight rejects (no servable entry,
engine too old) fail the POST synchronously so the button can explain
itself; everything slow runs in the job with phase/byte progress."""
entry, variant = _quickstart_target(body, hardware.probe_budget(planning=True))
section = _runtime_section()
tag = section.get("tag") or binaries.default_tag()
backend = _resolve_backend(section)
need_runtime = not binaries.installed_tags()
if need_runtime:
# Same preflight as /runtime/install: impossible combos fail the POST.
try:
binaries.resolve_assets(tag, backend)
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=400, detail=str(exc))
_resolve_assets_or_400(tag, backend)
need_download = variant.model_id not in bootstrap.staged_model_ids()
download_plan = _download_plan(entry, variant) if need_download else []
@@ -908,9 +900,7 @@ async def local_models_quickstart(body: QuickstartBody):
def _run():
if need_runtime:
job["phase"] = "installing-runtime"
job["detail"] = "Installing the local engine"
_step(job, "installing-runtime", "Installing the local engine")
binaries.ensure_runtime_installed(tag, backend, progress=_runtime_progress_hook(job))
if need_download:
@@ -935,8 +925,14 @@ async def local_models_quickstart(body: QuickstartBody):
"download_bytes": download_bytes}
def _stop_server() -> None:
# ── server lifecycle: turn the engine on/off ─────────────────
class ServerActionBody(BaseModel):
action: str # "stop" | "start"
def _stop_server() -> None:
if bootstrap.get_supervisor() is not None:
bootstrap.shutdown_local_runtime()
elif _state_endpoint() is not None:
@@ -945,7 +941,6 @@ def _stop_server() -> None:
try:
import psutil # type: ignore
state = json.loads(supervisor.state_path().read_text(encoding="utf-8"))
pid = int(state.get("pid") or 0)
if pid > 0 and psutil.pid_exists(pid):
@@ -957,11 +952,8 @@ def _stop_server() -> None:
def _start_server() -> None:
sup = bootstrap.ensure_local_runtime(_set_runtime_enabled(True), force=True)
if sup is None and _state_endpoint() is None:
raise RuntimeError("The local server could not start — check the "
"runtime is installed")
_start_local_server(_set_runtime_enabled(True),
"The local server could not start — check the runtime is installed")
_SERVER_ACTIONS = {"stop": _stop_server, "start": _start_server}
@@ -975,10 +967,8 @@ async def local_models_server(body: ServerActionBody):
action = (body.action or "").strip().lower()
if action not in _SERVER_ACTIONS:
raise HTTPException(status_code=400, detail="action must be 'stop' or 'start'")
try:
with _http_error(502):
await asyncio.to_thread(_SERVER_ACTIONS[action])
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=502, detail=str(exc)) from exc
return {"ok": True, "action": action}
@@ -994,26 +984,21 @@ def local_models_eject(body: ModelEjectBody):
"""Free a loaded model's GPU memory now; only demand (the next message)
reloads it — residency v2 has no automatic loading anywhere. Sync def:
the fallback path blocks on a 120s urlopen — threadpool, never the loop."""
sup = bootstrap.get_supervisor()
if sup is not None:
try:
with _http_error(502):
sup.unload_model(body.model_id)
return {"ok": True}
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=502, detail=str(exc)) from exc
return {"ok": True}
# Server owned by another process (or state-file only): drive the
# router directly with the persisted endpoint.
endpoint = _state_endpoint()
if endpoint is None:
raise HTTPException(status_code=409, detail="local server is not running")
try:
with _http_error(502):
_router_request(endpoint, "/models/unload", timeout=120,
payload={"model": body.model_id})
return {"ok": True}
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=502, detail=str(exc)) from exc
return {"ok": True}
class ModelActivateBody(BaseModel):
@@ -1027,20 +1012,17 @@ async def local_models_activate(body: ModelActivateBody):
NO model loading (residency v2: models load on first inference; an empty
router costs nothing). Kept as a job for UI continuity."""
# Split variants stage under their first part — resolve like the other routes.
if body.model_id not in bootstrap.staged_model_ids():
raise HTTPException(status_code=404, detail=f"{body.model_id} is not downloaded")
job = _job("model-activate", body.model_id, model_id=body.model_id)
def _run():
_ensure_server(
job, config_mod.load_config(), body.model_id,
fail_detail="The local server could not start — check the runtime is installed",
skip_msg="activate rescan check skipped")
job["phase"] = "setting-default"
job["detail"] = "Making it your default"
_step(job, "setting-default", "Making it your default")
_set_runtime_enabled(True)
_assign_default(job, body.model_id)
_finish(job, f"{body.model_id} is the default for new chats")
@@ -1078,15 +1060,10 @@ async def local_models_job(job_id: str):
async def local_models_search(q: str, limit: int = 20):
"""Full-text HF search over GGUF models — the firehose behind the curated
catalog; per-quant fit pills come from the repo-files call."""
if not q.strip():
return {"hits": []}
try:
with _http_error(502, "Hugging Face search unavailable: "):
hits = await run_in_threadpool(hf_browse.search_models, q, limit)
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=502,
detail=f"Hugging Face search unavailable: {exc}") from exc
return {"hits": [h.__dict__ for h in hits]}
@@ -1094,14 +1071,9 @@ async def local_models_search(q: str, limit: int = 20):
async def local_models_search_files(repo: str):
"""Servable GGUFs in one HF repo with a rough pre-download fit verdict per
quant (file size + conservative fill-ins; the GGUF header refines it)."""
try:
with _http_error(502, f"Could not list {repo}: "):
groups = await run_in_threadpool(
hf_browse.priced_repo_files, repo, hardware.probe_budget(planning=True))
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=502,
detail=f"Could not list {repo}: {exc}") from exc
return {"files": [dict(g.__dict__, paths=list(g.paths)) for g in groups]}
@@ -1116,12 +1088,11 @@ async def local_models_download_browsed(body: BrowsedDownloadBody):
it is a normal staged model (the post-download bounce regenerates presets
from its real header); with no catalog entry it serves 'unverified',
capabilities answered from the live server only."""
paths = [p for p in (body.paths or []) if p.lower().endswith(".gguf")]
if not paths:
raise HTTPException(status_code=422, detail="no .gguf files given")
first = paths[0].rsplit("/", 1)[-1]
model_id = re.sub(r"-\d{5}-of-\d{5}\.gguf$", "", first, flags=re.IGNORECASE)
model_id = re.sub(_SPLIT_PART_RE + r"\.gguf$", "", first, flags=re.IGNORECASE)
model_id = model_id[:-5] if model_id.lower().endswith(".gguf") else model_id
if model_id in bootstrap.staged_model_ids():
return {"job_id": None, "already_downloaded": True, "model_id": model_id}
@@ -1129,7 +1100,7 @@ async def local_models_download_browsed(body: BrowsedDownloadBody):
job = _job("model-download", f"{model_id} (from {body.repo})",
model_id=model_id)
def _run():
def _fetch():
job["phase"] = "downloading"
for p in paths:
dest = _models_dir() / p.rsplit("/", 1)[-1]
@@ -1139,10 +1110,8 @@ async def local_models_download_browsed(body: BrowsedDownloadBody):
base_done=int(job.get("done_bytes") or 0),
keep_totals=bool(job.get("total_bytes")))
job["phase"] = "downloading"
_finish(job, f"{model_id} ready")
_refresh_runtime("post-download runtime refresh skipped")
_spawn_job(job, "lm-download-browsed", _run)
_download_job(job, _fetch, "lm-download-browsed", model_id)
return {"job_id": job["job_id"], "model_id": model_id}
@@ -1155,7 +1124,6 @@ async def local_models_sideload(body: SideloadBody):
"""Register a GGUF already on this machine: link it into the managed
models dir (copy only when linking is impossible) and bounce the router.
The original stays put; delete-from-Hermes removes only our link."""
src = Path(body.path)
if not src.is_file() or src.suffix.lower() != ".gguf":
raise HTTPException(status_code=422, detail="Pick a .gguf model file")