fix(local-models): keep automatic recommendations GPU-resident
Stop recommending a system-RAM spill when no curated model fits resident. Preserve explicit model selection and the existing resident quality/speed ranking, including the separate unified-memory policy. Require a recommendation for automatic quickstart, expose Browse when none exists, and rename Configure to Let me choose. Keep policy copy and reason keys consistent across the four translated local-model sections. Cover automatic refusal and explicit spilled setup against one budget, plus the Browse, Download and Use interactions in the desktop pane.
This commit is contained in:
@@ -113,10 +113,10 @@ function renderPane() {
|
||||
|
||||
// The fresh-machine states these tests exercise now lead with the
|
||||
// quickstart card; the full pane (runtime rows, model list, browser)
|
||||
// is one 'Configure…' click away. Render and click through.
|
||||
// is one 'Let me choose' click away. Render and click through.
|
||||
async function renderFullPane() {
|
||||
const result = renderPane()
|
||||
const configure = await screen.findByRole('button', { name: /configure/i })
|
||||
const configure = await screen.findByRole('button', { name: /let me choose/i })
|
||||
|
||||
fireEvent.click(configure)
|
||||
|
||||
@@ -202,9 +202,11 @@ describe('LocalModelsSettings', () => {
|
||||
}
|
||||
|
||||
mocked.getLocalCatalog.mockResolvedValue({ models: [spilledFull] })
|
||||
await renderFullPane()
|
||||
renderPane()
|
||||
await screen.findByText('Spilled Full')
|
||||
|
||||
expect(screen.queryByRole('button', { name: /set up for me/i })).toBeNull()
|
||||
expect(screen.getByRole('button', { name: /browse models/i })).toBeTruthy()
|
||||
expect(screen.getByText('Full 256K context').className).not.toContain('emerald')
|
||||
})
|
||||
|
||||
@@ -393,7 +395,7 @@ describe('quickstart', () => {
|
||||
renderPane()
|
||||
|
||||
expect(await screen.findByText('Qwen3.6 27B — 17.6 GB')).toBeTruthy()
|
||||
// One job, one view: no Set up / Configure buttons while it runs.
|
||||
// One job, one view: no setup or model-choice buttons while it runs.
|
||||
expect(screen.queryByRole('button', { name: /set up for me/i })).toBeNull()
|
||||
})
|
||||
|
||||
@@ -413,6 +415,47 @@ describe('quickstart', () => {
|
||||
})
|
||||
|
||||
describe('BrowseSection', () => {
|
||||
it('keeps manual spill selection and HF browsing available without an automatic recommendation', async () => {
|
||||
const stagedId = 'Spilled-Model-Q4_K_M'
|
||||
mocked.getLocalModelsStatus.mockResolvedValue({ ...BASE_STATUS, runtime_installed: true })
|
||||
mocked.getLocalCatalog.mockResolvedValue({ models: [SPILLED_MODEL] })
|
||||
renderPane()
|
||||
|
||||
await screen.findByText('Spilled Model')
|
||||
expect(screen.getByText('No automatic recommendation for this machine')).toBeTruthy()
|
||||
expect(screen.getByText('Uses system RAM')).toBeTruthy()
|
||||
expect(screen.queryByRole('button', { name: /set up for me/i })).toBeNull()
|
||||
|
||||
// Attach the browser-only scroll method to the real search container,
|
||||
// so a missing or misdirected click handler cannot satisfy the assertion.
|
||||
const search = screen.getByPlaceholderText(/search models/i)
|
||||
const browse = search.closest('#local-model-browse')
|
||||
expect(browse).not.toBeNull()
|
||||
const scroll = vi.fn()
|
||||
Object.defineProperty(browse, 'scrollIntoView', { configurable: true, value: scroll })
|
||||
fireEvent.click(screen.getByRole('button', { name: /browse models/i }))
|
||||
expect(scroll).toHaveBeenCalledWith({ behavior: 'smooth', block: 'start' })
|
||||
expect(mocked.downloadLocalModel).not.toHaveBeenCalled()
|
||||
|
||||
// The backend reports the completed download on refresh. Use must send
|
||||
// the staged variant id, not the catalog family id or an automatic pick.
|
||||
mocked.downloadLocalModel.mockResolvedValue({ already_downloaded: true, job_id: null })
|
||||
mocked.getLocalModelsStatus.mockResolvedValue({
|
||||
...BASE_STATUS,
|
||||
runtime_installed: true,
|
||||
models: [{ id: stagedId, size_bytes: SPILLED_MODEL.size_bytes, size_label: SPILLED_MODEL.size_label }]
|
||||
})
|
||||
mocked.getLocalCatalog.mockResolvedValue({
|
||||
models: [{ ...SPILLED_MODEL, downloaded: true, downloaded_model_id: stagedId }]
|
||||
})
|
||||
fireEvent.click(screen.getByRole('button', { name: /download ·/i }))
|
||||
await waitFor(() => expect(mocked.downloadLocalModel).toHaveBeenCalledWith(SPILLED_MODEL.id))
|
||||
mocked.activateLocalModel.mockResolvedValue({ job_id: 'explicit-spill' })
|
||||
fireEvent.click(await screen.findByRole('button', { name: /^use$/i }))
|
||||
await waitFor(() => expect(mocked.activateLocalModel).toHaveBeenCalledWith(stagedId))
|
||||
expect(mocked.quickstartLocalModels).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('searches HF after a pause and shows fit-priced files on demand', async () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
@@ -438,7 +481,7 @@ describe('BrowseSection', () => {
|
||||
await vi.runOnlyPendingTimersAsync()
|
||||
})
|
||||
// Fresh machine leads with the quickstart card — enter the full pane.
|
||||
fireEvent.click(screen.getByRole('button', { name: /configure/i }))
|
||||
fireEvent.click(screen.getByRole('button', { name: /let me choose/i }))
|
||||
|
||||
const box = screen.getByPlaceholderText(/search models/i)
|
||||
fireEvent.change(box, { target: { value: 'qwen' } })
|
||||
|
||||
@@ -299,12 +299,15 @@ export function LocalModelsSettings() {
|
||||
// ── Quickstart: the dummy-proof front door ──
|
||||
// Until something is servable (runtime + at least one model), the pane
|
||||
// leads with a hero that does everything in one click; the full pane
|
||||
// stays one 'Configure…' click away. A running quickstart pins this
|
||||
// stays one 'Let me choose' click away. A running quickstart pins this
|
||||
// view so its progress has a home even after a remount.
|
||||
const qJob = runningQuickstart ?? null
|
||||
|
||||
const needsSetup = !status.runtime_installed || status.models.length === 0
|
||||
const heroModel = catalog.find(c => c.recommended && c.fits) ?? catalog.find(c => c.fits) ?? null
|
||||
// The setup hero is reserved for an automatic recommendation. A
|
||||
// spilled model remains visible below, but setup must not silently choose it.
|
||||
const heroModel = catalog.find(c => c.recommended && c.fits) ?? null
|
||||
const hasRecommendation = catalog.some(c => c.recommended)
|
||||
|
||||
if (qJob || (needsSetup && !configure && heroModel)) {
|
||||
// Stage rail derived from the job phase: engine -> model -> finish.
|
||||
@@ -551,6 +554,22 @@ export function LocalModelsSettings() {
|
||||
|
||||
{/* ── Models ── */}
|
||||
<SettingsSection icon={Download} meta={`${catalog.length}`} title={copy.modelsTitle}>
|
||||
{!hasRecommendation && (
|
||||
<ListRow
|
||||
action={
|
||||
<Button
|
||||
onClick={() => document.getElementById('local-model-browse')?.scrollIntoView({ behavior: 'smooth', block: 'start' })}
|
||||
size="sm"
|
||||
>
|
||||
<Search />
|
||||
{copy.noRecommendationAction}
|
||||
</Button>
|
||||
}
|
||||
description={copy.noRecommendationDetail}
|
||||
title={copy.noRecommendationTitle}
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="grid gap-1">
|
||||
{sortedCatalog.map(model => {
|
||||
const dJob = runningDownloadFor(jobs, model.id)
|
||||
@@ -982,7 +1001,8 @@ function BrowseSection({ onChanged }: { onChanged: () => void }) {
|
||||
icon={Search}
|
||||
title={copy.browseTitle}
|
||||
>
|
||||
<p className="text-[0.75rem] text-muted-foreground">{copy.browseHint}</p>
|
||||
<div id="local-model-browse">
|
||||
<p className="text-[0.75rem] text-muted-foreground">{copy.browseHint}</p>
|
||||
|
||||
<div className="relative">
|
||||
<Search className="pointer-events-none absolute left-2.5 top-1/2 size-3.5 -translate-y-1/2 text-muted-foreground" />
|
||||
@@ -1100,6 +1120,7 @@ function BrowseSection({ onChanged }: { onChanged: () => void }) {
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</SettingsSection>
|
||||
)
|
||||
|
||||
@@ -1313,9 +1313,12 @@ export const en: Translations = {
|
||||
'speed-gated-quality':
|
||||
'A higher-quality model fits this machine but would respond too slowly on its memory bandwidth — this is the best model that stays fast.',
|
||||
'fastest-resident':
|
||||
'No model reaches full speed on this hardware; this one comes closest while running entirely in GPU memory.',
|
||||
'least-painful-spilled': 'No model fits entirely in GPU memory here — this one runs best from system RAM.'
|
||||
'No model reaches full speed on this hardware; this one comes closest while running entirely in GPU memory.'
|
||||
} as Record<string, string>,
|
||||
noRecommendationTitle: 'No automatic recommendation for this machine',
|
||||
noRecommendationDetail:
|
||||
'Automatic setup requires a curated model that fits entirely in GPU or unified memory. You can still choose a model below or browse more models.',
|
||||
noRecommendationAction: 'Browse models',
|
||||
downloaded: 'Downloaded',
|
||||
downloadAction: size => `Download · ${size}`,
|
||||
downloadProgress: (done, total) => `Downloading ${done} of ${total}`,
|
||||
@@ -1327,7 +1330,7 @@ export const en: Translations = {
|
||||
quickstartDetailReady: model =>
|
||||
`One click makes ${model} your default for new chats. Everything runs on this machine.`,
|
||||
quickstartAction: 'Set up for me',
|
||||
quickstartConfigure: 'Configure…',
|
||||
quickstartConfigure: 'Let me choose',
|
||||
quickstartDoneToast: model => `${model} is set up — new chats run on this machine.`,
|
||||
quickstartFailed: 'Local model setup failed',
|
||||
quickstartStageEngine: 'Engine',
|
||||
|
||||
@@ -1182,10 +1182,13 @@ export const ja = defineLocale({
|
||||
'speed-gated-quality':
|
||||
'より高品質なモデルもこのマシンに載りますが、メモリ帯域の制約で応答が遅くなります — これは速度を保てる最良のモデルです。',
|
||||
'fastest-resident':
|
||||
'このハードウェアでフルスピードに達するモデルはありません。GPU メモリ内で動くものの中で最速です。',
|
||||
'least-painful-spilled':
|
||||
'GPU メモリに完全に収まるモデルはありません — システム RAM からの実行で最も快適なモデルです。'
|
||||
'このハードウェアでフルスピードに達するモデルはありません。GPU メモリ内で動くものの中で最速です。'
|
||||
} as Record<string, string>,
|
||||
noRecommendationTitle: 'このマシン向けの自動推奨モデルはありません',
|
||||
noRecommendationDetail:
|
||||
'自動セットアップには、GPU メモリまたはユニファイドメモリに完全に収まる厳選モデルが必要です。下の一覧から選ぶか、ほかのモデルを探すこともできます。',
|
||||
noRecommendationAction: 'モデルを探す',
|
||||
quickstartConfigure: '自分で選ぶ',
|
||||
downloaded: 'ダウンロード済み',
|
||||
downloadAction: size => `ダウンロード · ${size}`,
|
||||
downloadProgress: (done, total) => `ダウンロード中 ${done} / ${total}`,
|
||||
|
||||
@@ -1308,6 +1308,9 @@ export interface Translations {
|
||||
/** Recommended-badge tooltip by resolver branch; unknown keys (newer
|
||||
* backend) simply show no tooltip. */
|
||||
recommendedReason: Record<string, string>
|
||||
noRecommendationTitle: string
|
||||
noRecommendationDetail: string
|
||||
noRecommendationAction: string
|
||||
downloaded: string
|
||||
downloadAction: (size: string) => string
|
||||
downloadProgress: (done: string, total: string) => string
|
||||
|
||||
@@ -1135,9 +1135,13 @@ export const zhHant = defineLocale({
|
||||
'best-quality-resident': '在完全駐留 GPU 且保持全速的模型中品質最高。推薦會在品質與該硬體的預計速度之間權衡。',
|
||||
'speed-gated-quality':
|
||||
'有更高品質的模型可以裝入這台機器,但受記憶體頻寬限制回應會太慢——這是保持流暢的最佳模型。',
|
||||
'fastest-resident': '沒有模型能在該硬體上達到全速;這是完全駐留 GPU 記憶體中最快的一個。',
|
||||
'least-painful-spilled': '沒有模型能完全裝入 GPU 記憶體——這是從系統記憶體執行表現最好的一個。'
|
||||
'fastest-resident': '沒有模型能在該硬體上達到全速;這是完全駐留 GPU 記憶體中最快的一個。'
|
||||
} as Record<string, string>,
|
||||
noRecommendationTitle: '此裝置暫無自動推薦模型',
|
||||
noRecommendationDetail:
|
||||
'自動設定需要一個可完全放入 GPU 記憶體或統一記憶體的精選模型。你仍可在下方自行選擇,或瀏覽更多模型。',
|
||||
noRecommendationAction: '瀏覽模型',
|
||||
quickstartConfigure: '讓我選擇',
|
||||
downloaded: '已下載',
|
||||
downloadAction: size => `下載 · ${size}`,
|
||||
downloadProgress: (done, total) => `正在下載 ${done} / ${total}`,
|
||||
|
||||
@@ -1485,7 +1485,7 @@ export const zh: Translations = {
|
||||
`一键完成所有设置:本地引擎、${model}(需下载 ${size}),并设为新会话的默认模型。数据不会离开这台电脑。`,
|
||||
quickstartDetailReady: model => `一键将 ${model} 设为新会话的默认模型。所有内容都在本机运行。`,
|
||||
quickstartAction: '为我设置',
|
||||
quickstartConfigure: '自定义…',
|
||||
quickstartConfigure: '让我选择',
|
||||
quickstartDoneToast: model => `${model} 已就绪——新会话将在本机运行。`,
|
||||
quickstartFailed: '本地模型设置失败',
|
||||
quickstartStageEngine: '引擎',
|
||||
@@ -1501,9 +1501,12 @@ export const zh: Translations = {
|
||||
recommendedReason: {
|
||||
'best-quality-resident': '在完全驻留 GPU 且保持全速的模型中质量最高。推荐会在质量与该硬件的预计速度之间权衡。',
|
||||
'speed-gated-quality': '有更高质量的模型可以装入这台机器,但受内存带宽限制响应会太慢——这是保持流畅的最佳模型。',
|
||||
'fastest-resident': '没有模型能在该硬件上达到全速;这是完全驻留 GPU 内存中最快的一个。',
|
||||
'least-painful-spilled': '没有模型能完全装入 GPU 内存——这是从系统内存运行表现最好的一个。'
|
||||
'fastest-resident': '没有模型能在该硬件上达到全速;这是完全驻留 GPU 内存中最快的一个。'
|
||||
} as Record<string, string>,
|
||||
noRecommendationTitle: '此设备暂无自动推荐模型',
|
||||
noRecommendationDetail:
|
||||
'自动设置需要一个可完全放入显存或统一内存的精选模型。你仍可在下方自行选择,或浏览更多模型。',
|
||||
noRecommendationAction: '浏览模型',
|
||||
downloaded: '已下载',
|
||||
downloadAction: size => `下载 · ${size}`,
|
||||
downloadProgress: (done, total) => `正在下载 ${done} / ${total}`,
|
||||
|
||||
@@ -1400,9 +1400,7 @@ export interface LocalCatalogModel {
|
||||
native_context: number
|
||||
native_context_label: string
|
||||
recommended: boolean
|
||||
/** Why the resolver picked this entry (recommended rows only):
|
||||
* best-quality-resident | speed-gated-quality | fastest-resident |
|
||||
* least-painful-spilled. Renders as the Recommended badge's tooltip. */
|
||||
/** Why the resolver picked this entry (recommended rows only). */
|
||||
recommended_reason?: string | null
|
||||
downloaded: boolean
|
||||
downloaded_model_id?: string | null
|
||||
|
||||
@@ -194,8 +194,8 @@ def recommended_entry(budget: HardwareBudget,
|
||||
Callers pass pre-filtered entries when some are ineligible for reasons the catalog can't know
|
||||
(engine too old). Reasons: best-quality-resident (quality won among resident entries clearing
|
||||
the pleasant floor); speed-gated-quality (same, but the floor eliminated a HIGHER quality
|
||||
candidate); fastest-resident (nothing resident clears the floor); least-painful-spilled
|
||||
(nothing runs resident; fastest from host memory — MoE by construction).
|
||||
candidate); fastest-resident (nothing resident clears the floor). Returns None when no
|
||||
eligible entry runs resident; spilled models remain available for explicit selection.
|
||||
"""
|
||||
pool = CATALOG if entries is None else entries
|
||||
fitting = [(e, c) for e in pool if (c := select_variant(e, budget)) is not None]
|
||||
@@ -213,7 +213,9 @@ def recommended_entry(budget: HardwareBudget,
|
||||
return (pick, "speed-gated-quality" if floor_gated else "best-quality-resident")
|
||||
if resident:
|
||||
return (max(resident, key=speed)[0], "fastest-resident")
|
||||
return (max(fitting, key=lambda t: speed(t, spilled=True))[0], "least-painful-spilled")
|
||||
# A spilled model may be usable, but it is not a recommendation. Keep it
|
||||
# discoverable through Browse so the user can opt in with the degradation visible.
|
||||
return None
|
||||
|
||||
|
||||
# ── catalog data: packaged JSON, refreshed from GitHub in memory ─
|
||||
|
||||
@@ -708,12 +708,20 @@ async def local_models_delete(model_id: str):
|
||||
|
||||
# ── quickstart: one click from nothing to a working default ──
|
||||
def _quickstart_target(body: QuickstartBody, budget):
|
||||
"""(entry, variant) to set up: explicit id, else this machine's recommendation, else the first servable entry."""
|
||||
"""Resolve an explicit model, or start with the machine's automatic recommendation.
|
||||
|
||||
With no recommendation, require an explicit choice before starting setup.
|
||||
"""
|
||||
if body.model_id:
|
||||
candidates = [_entry_or_404(body.model_id)]
|
||||
else:
|
||||
picked = catalog.recommended_entry(budget, _eligible_entries())
|
||||
candidates = ([picked[0]] if picked else []) + [e for e in catalog.CATALOG if not picked or e.id != picked[0].id]
|
||||
if picked is None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="No automatic recommendation for this machine — open Local Models to browse or choose a model explicitly",
|
||||
)
|
||||
candidates = [picked[0]] + [e for e in catalog.CATALOG if e.id != picked[0].id]
|
||||
for candidate in candidates:
|
||||
choice = catalog.select_variant(candidate, budget)
|
||||
if choice is not None and not _engine_too_old(candidate.min_engine):
|
||||
@@ -725,8 +733,8 @@ def _quickstart_target(body: QuickstartBody, budget):
|
||||
@router.post("/api/local-models/quickstart")
|
||||
async def local_models_quickstart(body: QuickstartBody):
|
||||
"""One job: install the runtime (if missing), download this machine's build of the recommended model (if
|
||||
missing), make it the default. Each leg is the same code the individual routes run, so 'Configure' and
|
||||
quickstart can never disagree. Preflight rejects (no servable entry, engine too old) fail the POST
|
||||
missing), make it the default. Each leg uses the same code as the individual setup routes.
|
||||
Preflight rejects (no automatic recommendation or no servable choice) fail the POST
|
||||
synchronously so the button can explain itself; everything slow runs in the job with phase/byte progress."""
|
||||
entry, variant = _quickstart_target(body, hardware.probe_budget(planning=True))
|
||||
tag, backend = _runtime_target()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Quickstart route: one POST from nothing to a working local default.
|
||||
|
||||
Contract, not implementation: the route must (a) preflight-fail
|
||||
synchronously when nothing fits, (b) report which legs the job will run
|
||||
synchronously when automatic setup has no recommendation, (b) report which legs the job will run
|
||||
(runtime install / model download), skipping legs already satisfied,
|
||||
and (c) run install -> download -> activate through the same code paths
|
||||
the individual routes use. The slow legs are stubbed at their module
|
||||
@@ -42,6 +42,72 @@ def test_quickstart_unknown_model_404s(client):
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
def test_quickstart_without_recommendation_requires_explicit_choice(client, monkeypatch):
|
||||
"""One budget: automatic setup refuses; an explicit spilled choice reaches activation."""
|
||||
from hermes_cli.local_runtime.estimator import HardwareBudget
|
||||
import hermes_cli.web_routers.local_models as lm
|
||||
|
||||
gib = 1 << 30
|
||||
budget = HardwareBudget(
|
||||
usable_vram_bytes=14 * gib, total_device_bytes=16 * gib,
|
||||
ram_available_bytes=64 * gib, uma=False,
|
||||
)
|
||||
monkeypatch.setattr(lm.hardware, "probe_budget", lambda **kw: budget)
|
||||
monkeypatch.setattr(lm.catalog, "refresh_catalog_soon", lambda: None)
|
||||
monkeypatch.setattr(lm.binaries, "installed_tags", lambda: [lm.binaries.default_tag()])
|
||||
monkeypatch.setattr(lm.bootstrap, "staged_model_ids", lambda: set())
|
||||
config = lm.config_mod.load_config()
|
||||
config.setdefault("local_runtime", {})["backend"] = "cpu"
|
||||
config["local_runtime"]["enabled"] = False
|
||||
lm.config_mod.save_config(config)
|
||||
|
||||
calls: list[tuple] = []
|
||||
|
||||
def download(job, plan, label):
|
||||
calls.append(("download", label))
|
||||
|
||||
class Server:
|
||||
def models(self):
|
||||
return [chosen["model_id"]]
|
||||
|
||||
def start_server(config, force=False):
|
||||
calls.append(("server", config["local_runtime"]["enabled"], force))
|
||||
return Server()
|
||||
|
||||
# Stub only slow external legs. Catalog, HTTP preflight, job sequencing,
|
||||
# runtime-enabled persistence and assignment dispatch remain real.
|
||||
monkeypatch.setattr(lm, "_run_download_plan", download)
|
||||
monkeypatch.setattr(lm.bootstrap, "ensure_local_runtime", start_server)
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.web_server_config._apply_model_assignment_sync",
|
||||
lambda *args: calls.append(("assign", *args)),
|
||||
)
|
||||
rows = client.get("/api/local-models/catalog").json()["models"]
|
||||
assert not any(row["recommended"] for row in rows)
|
||||
chosen = next(row for row in rows if row["id"] == "qwen3.8-27b")
|
||||
assert chosen["fits"] and chosen["spilled"]
|
||||
|
||||
automatic = client.post("/api/local-models/quickstart", json={})
|
||||
assert automatic.status_code == 409
|
||||
assert "no automatic recommendation" in automatic.json()["detail"].lower()
|
||||
assert calls == []
|
||||
assert lm.config_mod.load_config()["local_runtime"]["enabled"] is False
|
||||
|
||||
explicit = client.post("/api/local-models/quickstart", json={"model_id": chosen["id"]})
|
||||
assert explicit.status_code == 200, explicit.text
|
||||
result = explicit.json()
|
||||
assert result["model_id"] == chosen["id"]
|
||||
assert result["needs_download"] and not result["needs_runtime"]
|
||||
job = _wait_job(client, result["job_id"])
|
||||
assert job["status"] == "done", job["error"]
|
||||
assert calls == [
|
||||
("download", chosen["display_name"]),
|
||||
("server", True, True),
|
||||
("assign", "main", "llamacpp", chosen["model_id"], "", "", ""),
|
||||
]
|
||||
assert lm.config_mod.load_config()["local_runtime"]["enabled"] is True
|
||||
|
||||
|
||||
def test_quickstart_refuses_when_nothing_fits(client, monkeypatch):
|
||||
"""Preflight is synchronous: a machine no catalog entry fits gets a 409
|
||||
with guidance, not a doomed background job."""
|
||||
|
||||
@@ -2,12 +2,10 @@
|
||||
|
||||
The recommendation itself is DERIVED (catalog.recommended_entry: best
|
||||
quality among resident entries clearing the pleasant speed floor, else
|
||||
fastest resident, else least-painful spilled), so nobody hand-maintains
|
||||
per-hardware-class picks. This table is the editorial control on that
|
||||
derivation: it enumerates the real memory size classes x {discrete,
|
||||
unified} and pins every cell. A catalog change (new model, quality
|
||||
re-rank, quant swap) flips cells HERE, and the diff of this file in
|
||||
review IS the sign-off on what each machine class gets.
|
||||
fastest resident). Spilled models stay browseable but are never automatic
|
||||
recommendations, so nobody hand-maintains per-hardware-class picks.
|
||||
This table pins the model and reason across discrete and unified memory
|
||||
classes so changes to the recommendation remain reviewable.
|
||||
|
||||
These are decision pins, not change-detectors: each cell is a choice a
|
||||
human approved, exactly like a golden file. When a cell flips on
|
||||
@@ -55,8 +53,8 @@ def _unified(size_gb: int) -> HardwareBudget:
|
||||
#
|
||||
# VRAM | discrete | unified
|
||||
# -----+-------------------------+------------------------
|
||||
# 8 | qwen3.6-35b-a3b spilled | (none fits)
|
||||
# 16 | qwen3.6-35b-a3b spilled | (none fits)
|
||||
# 8 | (no recommendation) | (none fits)
|
||||
# 16 | (no recommendation) | (none fits)
|
||||
# 24 | qwen3.8-27b | (none fits)
|
||||
# 32 | qwen3.8-27b | qwen3.6-35b-a3b
|
||||
# 48 | qwen3.8-27b | qwen3.6-35b-a3b
|
||||
@@ -66,9 +64,8 @@ def _unified(size_gb: int) -> HardwareBudget:
|
||||
# 512 | qwen3.8-flash-next | qwen3.8-flash-next
|
||||
#
|
||||
# Reading guide for reviewers:
|
||||
# - Discrete <=16 GB: nothing runs resident; the 35B MoE is the least
|
||||
# painful spill (active slice streams from host; a dense spill reads
|
||||
# every weight over the bus).
|
||||
# - Discrete <=16 GB: nothing runs resident; no automatic recommendation.
|
||||
# Browse remains available for explicit spill choices.
|
||||
# - Discrete 24-96 GB: the 27B is the flagship experience — dense reads
|
||||
# at ~1 TB/s clear the floor easily, so quality decides.
|
||||
# - Discrete/unified where Flash Next fits resident (128 GB discrete,
|
||||
@@ -83,9 +80,9 @@ def _unified(size_gb: int) -> HardwareBudget:
|
||||
# the RAM). The pane's browse flow is the path for those machines
|
||||
# until a small catalog entry lands (revisit when one does).
|
||||
DECISION_TABLE = [
|
||||
(8, "discrete", "qwen3.6-35b-a3b", "least-painful-spilled"),
|
||||
(8, "discrete", None, None),
|
||||
(8, "unified", None, None),
|
||||
(16, "discrete", "qwen3.6-35b-a3b", "least-painful-spilled"),
|
||||
(16, "discrete", None, None),
|
||||
(16, "unified", None, None),
|
||||
(24, "discrete", "qwen3.8-27b", "best-quality-resident"),
|
||||
(24, "unified", None, None),
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Automatic recommendations require residency; manual choices may spill."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hermes_cli.local_runtime.catalog import recommended_entry, select_variant
|
||||
from hermes_cli.local_runtime.estimator import HardwareBudget
|
||||
|
||||
_GIB = 1 << 30
|
||||
|
||||
|
||||
def _discrete(size_gb: int, ram_gb: int = 64) -> HardwareBudget:
|
||||
total = size_gb * _GIB
|
||||
margin = max(2 * _GIB, int(total * 0.09))
|
||||
return HardwareBudget(
|
||||
usable_vram_bytes=max(0, total - margin),
|
||||
total_device_bytes=total,
|
||||
ram_available_bytes=ram_gb * _GIB,
|
||||
uma=False,
|
||||
)
|
||||
|
||||
|
||||
def _unified(size_gb: int) -> HardwareBudget:
|
||||
total = size_gb * _GIB
|
||||
return HardwareBudget(
|
||||
usable_vram_bytes=int(total * 0.80),
|
||||
total_device_bytes=total,
|
||||
ram_available_bytes=0,
|
||||
uma=True,
|
||||
)
|
||||
|
||||
|
||||
def test_small_discrete_cards_have_no_automatic_recommendation():
|
||||
for size_gb in (8, 16):
|
||||
assert recommended_entry(_discrete(size_gb)) is None
|
||||
|
||||
|
||||
def test_24gb_discrete_card_recommends_resident_qwen_27b():
|
||||
picked = recommended_entry(_discrete(24))
|
||||
assert picked is not None
|
||||
assert picked[0].id == "qwen3.8-27b"
|
||||
assert picked[1] == "best-quality-resident"
|
||||
choice = select_variant(picked[0], _discrete(24))
|
||||
assert choice is not None and choice.zero_spill
|
||||
|
||||
|
||||
def test_spilled_model_remains_explicitly_browseable_on_16gb():
|
||||
from hermes_cli.local_runtime.catalog import CATALOG
|
||||
|
||||
entry = next(e for e in CATALOG if e.id == "qwen3.8-27b")
|
||||
choice = select_variant(entry, _discrete(16))
|
||||
assert choice is not None
|
||||
assert not choice.zero_spill
|
||||
assert recommended_entry(_discrete(16)) is None
|
||||
|
||||
|
||||
def test_unified_memory_policy_does_not_inherit_discrete_recommendations():
|
||||
assert recommended_entry(_unified(24)) is None
|
||||
Reference in New Issue
Block a user