Teknium
72b7c6c8d1
PR #67934 marked auto-discovered catalogs by writing two sentinel keys INSIDE the user-facing ``models`` mapping of custom provider entries: ``__discovered_model_catalog__`` (written by _save_discovered_models_to_config) and ``__explicit_model_allowlist__`` (injected by _normalize_custom_provider_entry). Every consumer of that mapping — pickers, selectors, gateway/agent readers, and the user's own config.yaml — had to know to filter those keys, and any site that didn't listed them as phantom model IDs (``__discovered_model_catalog__`` showing up as a selectable "model"). The v11→v12 config migration and the ACP session-state test caught exactly that leak on main. Replace the in-mapping sentinels with a single entry-level flag: - ``models_discovered: true`` now sits next to ``models``/``base_url`` on the provider entry; the models mapping stays a clean ``{model_id: metadata}`` dict with no reserved keys. - _save_discovered_models_to_config writes the new shape and refreshes catalogs it previously discovered (entry-level flag or legacy sentinel) instead of treating them as user-curated metadata. - _normalize_custom_provider_entry no longer injects ``__explicit_model_allowlist__``; a dict-shaped models mapping counts as an explicit allowlist exactly when the entry is NOT marked models_discovered. - _models_config_is_allowlist takes the discovered flag as a parameter (new helper _entry_models_discovered resolves it, including the legacy in-mapping sentinel); all call sites updated (model_switch.py, model_setup_flows.py, acp_adapter/server.py). - Backward compat, no config version bump: configs written by a pre-fix Hermes (sentinels inside models) still read correctly — ``__discovered_model_catalog__: true`` is treated as models_discovered, both sentinel keys are stripped from model listings, and the next discovery save migrates the entry to the clean shape. Covered by a new regression test. Also restore ``except Exception:`` on the pre-existing guards this PR had narrowed to specific exception tuples (the resolve_runtime_provider fallback in switch_model, the picker discovery/cache guards in list_authenticated_providers, _get_model_config_dict, and _credential_fingerprint). Those guards were intentionally broad on main — a failed resolution or probe must degrade to the fallback path, never crash the model switch. Guards the PR introduced for its own new probe code keep their authored tuples. The ACP new_session payload also goes back to probe_current_custom_provider=False, matching the contract main's test_new_session_returns_authenticated_cross_provider_model_state pins (session opens must not block on live-probing the current custom endpoint).
2171 lines
78 KiB
Python
2171 lines
78 KiB
Python
"""Regression tests for /model support of config.yaml custom_providers.
|
|
|
|
The terminal `hermes model` flow already exposes `custom_providers`, but the
|
|
shared slash-command pipeline (`/model` in CLI/gateway/Telegram) historically
|
|
only looked at `providers:`.
|
|
|
|
Fixtures that set ``discover_models=False`` deliberately opt out of live
|
|
endpoint probing; they test saved/configured-provider behavior rather than a
|
|
local model server. The live-probe cases set it separately when discovery is
|
|
the behavior under test.
|
|
"""
|
|
|
|
import time
|
|
|
|
import hermes_cli.providers as providers_mod
|
|
import pytest
|
|
import yaml
|
|
from hermes_cli.model_switch import (
|
|
_fetch_picker_live_models,
|
|
_save_discovered_models_to_config,
|
|
list_authenticated_providers,
|
|
switch_model,
|
|
)
|
|
from hermes_cli.providers import resolve_provider_full
|
|
|
|
|
|
_MOCK_VALIDATION = {
|
|
"accepted": True,
|
|
"persist": True,
|
|
"recognized": True,
|
|
"message": None,
|
|
}
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _disable_live_custom_provider_model_probe(monkeypatch):
|
|
"""Keep custom-provider picker fixtures independent of local model servers."""
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", lambda *_a, **_kw: None)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.cached_provider_model_ids", lambda *_a, **_kw: []
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.provider_model_ids", lambda *_a, **_kw: []
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.fetch_ollama_local_models", lambda *_a, **_kw: None
|
|
)
|
|
|
|
|
|
def test_picker_native_probe_failure_falls_back_to_openai_catalog(monkeypatch):
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.should_use_ollama_native_catalog", lambda *a, **k: True
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models._get_ollama_native_headers", lambda *a, **k: {}
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.fetch_ollama_local_models", lambda *a, **k: None
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.fetch_api_models", lambda *a, **k: ["fallback-model"]
|
|
)
|
|
|
|
assert _fetch_picker_live_models(
|
|
"key", "http://127.0.0.1:11434/v1", "ollama", False
|
|
) == ["fallback-model"]
|
|
|
|
|
|
def test_picker_generic_discovery_preserves_api_mode(monkeypatch):
|
|
calls = []
|
|
|
|
def cached(*args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
return ["model-a"]
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.should_use_ollama_native_catalog", lambda *a, **k: False
|
|
)
|
|
monkeypatch.setattr("hermes_cli.models.cached_fetch_api_models", cached)
|
|
|
|
assert _fetch_picker_live_models(
|
|
"key",
|
|
"https://proxy.example/anthropic/v1",
|
|
"custom",
|
|
False,
|
|
api_mode="anthropic_messages",
|
|
) == ["model-a"]
|
|
assert calls[0][1]["api_mode"] == "anthropic_messages"
|
|
|
|
|
|
def test_list_authenticated_providers_includes_custom_providers(monkeypatch):
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", lambda *a, **k: [])
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openai-codex",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local (127.0.0.1:4141)",
|
|
"base_url": "http://127.0.0.1:4141/v1",
|
|
"model": "rotator-openrouter-coding",
|
|
}
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
assert any(
|
|
p["slug"] == "custom:local-(127.0.0.1:4141)"
|
|
and p["name"] == "Local (127.0.0.1:4141)"
|
|
and p["models"] == ["rotator-openrouter-coding"]
|
|
and p["api_url"] == "http://127.0.0.1:4141/v1"
|
|
for p in providers
|
|
)
|
|
|
|
|
|
|
|
def test_providers_singular_model_does_not_suppress_ollama_native_discovery(monkeypatch):
|
|
"""A saved selection in ``providers:`` is not an explicit catalog."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.fetch_ollama_local_models",
|
|
lambda *a, **k: ["qwen3:latest", "llama3.2:latest"],
|
|
)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openai-codex",
|
|
user_providers={
|
|
"ollama": {
|
|
"base_url": "http://localhost:11434/v1",
|
|
"model": "qwen3:latest",
|
|
}
|
|
},
|
|
custom_providers=[],
|
|
max_models=50,
|
|
)
|
|
|
|
ollama = next(p for p in providers if p["slug"] == "ollama")
|
|
assert ollama["models"] == ["qwen3:latest", "llama3.2:latest"]
|
|
|
|
|
|
def test_list_authenticated_providers_can_skip_custom_provider_live_probe(monkeypatch):
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
fetch = lambda *a, **k: (_ for _ in ()).throw(AssertionError("unexpected probe"))
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
providers = list_authenticated_providers(
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Slow Local",
|
|
"base_url": "http://127.0.0.1:8080/v1",
|
|
"api_key": "sk-local",
|
|
"model": "local-model",
|
|
}
|
|
],
|
|
probe_custom_providers=False,
|
|
)
|
|
|
|
row = next(p for p in providers if p["slug"] == "custom:slow-local")
|
|
assert row["models"] == ["local-model"]
|
|
assert row["total_models"] == 1
|
|
|
|
|
|
|
|
|
|
|
|
def test_resolve_provider_full_finds_named_custom_provider():
|
|
"""Explicit /model --provider should resolve saved custom_providers entries."""
|
|
resolved = resolve_provider_full(
|
|
"custom:local-(127.0.0.1:4141)",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local (127.0.0.1:4141)",
|
|
"base_url": "http://127.0.0.1:4141/v1",
|
|
}
|
|
],
|
|
)
|
|
|
|
assert resolved is not None
|
|
assert resolved.id == "custom:local-(127.0.0.1:4141)"
|
|
assert resolved.name == "Local (127.0.0.1:4141)"
|
|
assert resolved.base_url == "http://127.0.0.1:4141/v1"
|
|
assert resolved.source == "user-config"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"requested",
|
|
[
|
|
"Local Ollama",
|
|
"local-ollama",
|
|
"local-127.0.0.1:11434",
|
|
"custom:local-ollama",
|
|
"custom:local-127.0.0.1:11434",
|
|
],
|
|
)
|
|
def test_keyed_custom_provider_legacy_aliases_resolve_to_stable_key(requested):
|
|
"""Every historical identity resolves, but keyed providers return one ID."""
|
|
resolved = resolve_provider_full(
|
|
requested,
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local Ollama",
|
|
"provider_key": "local-127.0.0.1:11434",
|
|
"base_url": "http://127.0.0.1:11434/v1",
|
|
}
|
|
],
|
|
)
|
|
|
|
assert resolved is not None
|
|
assert resolved.id == "custom:local-127.0.0.1:11434"
|
|
assert resolved.name == "Local Ollama"
|
|
|
|
|
|
def test_keyed_custom_provider_bare_custom_fallback_uses_stable_key():
|
|
resolved = resolve_provider_full(
|
|
"custom",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local Ollama",
|
|
"provider_key": "local-127.0.0.1:11434",
|
|
"base_url": "http://127.0.0.1:11434/v1",
|
|
}
|
|
],
|
|
)
|
|
|
|
assert resolved is not None
|
|
assert resolved.id == "custom:local-127.0.0.1:11434"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"requested",
|
|
["foo", "custom:foo", "custom:custom:foo"],
|
|
)
|
|
def test_prefixed_provider_key_does_not_accumulate_custom_prefixes(requested):
|
|
"""Accept the historical doubled form without writing a third identity."""
|
|
resolved = resolve_provider_full(
|
|
requested,
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Foo Relay",
|
|
"provider_key": "custom:foo",
|
|
"base_url": "https://foo.example/v1",
|
|
}
|
|
],
|
|
)
|
|
|
|
assert resolved is not None
|
|
assert resolved.id == "custom:foo"
|
|
|
|
|
|
def test_list_authenticated_providers_includes_active_bare_custom_endpoint(monkeypatch):
|
|
"""Bare model.provider=custom + model.base_url should still populate /model.
|
|
|
|
Users can configure a one-off OpenAI-compatible endpoint directly under
|
|
``model:`` without a named ``providers:`` or ``custom_providers:`` row.
|
|
The gateway picker receives only the current model/base_url slice, so it
|
|
must surface that active endpoint rather than looking like config was
|
|
ignored.
|
|
"""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom",
|
|
current_base_url="https://www.ccsub.net/v1",
|
|
current_model="gpt-4o",
|
|
user_providers={},
|
|
custom_providers=[],
|
|
max_models=50,
|
|
)
|
|
|
|
bare_custom = next((p for p in providers if p["slug"] == "custom"), None)
|
|
assert bare_custom is not None
|
|
assert bare_custom["name"] == "Custom endpoint"
|
|
assert bare_custom["is_current"] is True
|
|
assert bare_custom["is_user_defined"] is True
|
|
assert bare_custom["models"] == ["gpt-4o"]
|
|
assert bare_custom["api_url"] == "https://www.ccsub.net/v1"
|
|
|
|
|
|
def test_list_authenticated_providers_can_probe_active_bare_custom_endpoint(monkeypatch):
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.fetch_api_models",
|
|
lambda api_key, api_url, **kwargs: ["gpt-4o", "gpt-4o-mini"],
|
|
)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom",
|
|
current_base_url="https://www.ccsub.net/v1",
|
|
current_model="gpt-4o",
|
|
user_providers={},
|
|
custom_providers=[],
|
|
probe_custom_providers=False,
|
|
probe_current_custom_provider=True,
|
|
)
|
|
|
|
bare_custom = next(p for p in providers if p["slug"] == "custom")
|
|
assert bare_custom["is_current"] is True
|
|
assert bare_custom["models"] == ["gpt-4o", "gpt-4o-mini"]
|
|
|
|
|
|
def test_switch_model_accepts_explicit_bare_custom_current_endpoint(monkeypatch):
|
|
"""Picker selections for bare custom endpoints should route to current base_url."""
|
|
monkeypatch.setattr("hermes_cli.models.validate_requested_model", lambda *a, **k: _MOCK_VALIDATION)
|
|
monkeypatch.setattr("hermes_cli.model_switch.get_model_info", lambda *a, **k: None)
|
|
monkeypatch.setattr("hermes_cli.model_switch.get_model_capabilities", lambda *a, **k: None)
|
|
|
|
result = switch_model(
|
|
raw_input="gpt-4o-mini",
|
|
current_provider="custom",
|
|
current_model="gpt-4o",
|
|
current_base_url="https://www.ccsub.net/v1",
|
|
current_api_key="sk-test",
|
|
explicit_provider="custom",
|
|
user_providers={},
|
|
custom_providers=[],
|
|
)
|
|
|
|
assert result.success is True
|
|
assert result.target_provider == "custom"
|
|
assert result.provider_label == "Custom endpoint"
|
|
assert result.new_model == "gpt-4o-mini"
|
|
assert result.base_url == "https://www.ccsub.net/v1"
|
|
assert result.api_key == "sk-test"
|
|
|
|
|
|
def test_is_aggregator_recognizes_named_custom_provider():
|
|
assert providers_mod.is_aggregator("custom:hpc-ai") is True
|
|
assert providers_mod.is_aggregator("custom:litellm") is True
|
|
|
|
|
|
def test_is_aggregator_leaves_unknown_provider_non_aggregator():
|
|
assert providers_mod.is_aggregator("not-a-provider") is False
|
|
|
|
|
|
def test_switch_model_does_not_send_ollama_headers_to_unrelated_custom_endpoint(monkeypatch):
|
|
"""A custom endpoint must not inherit headers from configured Ollama."""
|
|
seen_headers = []
|
|
validation_headers = []
|
|
|
|
def fake_native_detection(provider, base_url, headers=None):
|
|
seen_headers.append(headers)
|
|
return True
|
|
|
|
def fake_validation(*args, **kwargs):
|
|
validation_headers.append(kwargs.get("headers"))
|
|
return _MOCK_VALIDATION
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.should_use_ollama_native_catalog",
|
|
fake_native_detection,
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models._get_ollama_request_headers",
|
|
lambda: {"Authorization": "Bearer configured-ollama-secret"},
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models._get_provider_config_dict",
|
|
lambda provider: (
|
|
{"base_url": "https://trusted-ollama.example:11434"}
|
|
if provider == "ollama"
|
|
else {}
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
|
lambda **kwargs: {
|
|
"api_key": "custom-key",
|
|
"base_url": "https://attacker.example:11434/v1",
|
|
"api_mode": "chat_completions",
|
|
},
|
|
)
|
|
monkeypatch.setattr("hermes_cli.models.validate_requested_model", fake_validation)
|
|
monkeypatch.setattr("hermes_cli.model_switch.get_model_info", lambda *a, **k: None)
|
|
monkeypatch.setattr("hermes_cli.model_switch.get_model_capabilities", lambda *a, **k: None)
|
|
|
|
result = switch_model(
|
|
raw_input="new-model",
|
|
current_provider="custom",
|
|
current_model="old-model",
|
|
current_base_url="https://attacker.example:11434/v1",
|
|
current_api_key="custom-key",
|
|
explicit_provider="",
|
|
user_providers={},
|
|
custom_providers=[],
|
|
)
|
|
|
|
assert result.success is True
|
|
assert seen_headers == [{}]
|
|
assert validation_headers == [None]
|
|
|
|
|
|
|
|
def test_is_routing_aggregator_excludes_flat_namespace_resellers():
|
|
"""opencode-go / opencode-zen stay ``is_aggregator=True`` (model-switch
|
|
relies on it to search their flat bare-name catalog), but they are NOT
|
|
routing aggregators — their models are first-party, so the picker dedup
|
|
must not strip them. (#47077)"""
|
|
# Still aggregators for model-switch flat-catalog resolution.
|
|
assert providers_mod.is_aggregator("opencode-go") is True
|
|
assert providers_mod.is_aggregator("opencode-zen") is True
|
|
# But NOT routing aggregators for picker-dedup purposes.
|
|
assert providers_mod.is_routing_aggregator("opencode-go") is False
|
|
assert providers_mod.is_routing_aggregator("opencode-zen") is False
|
|
# True routers and custom proxies remain routing aggregators.
|
|
assert providers_mod.is_routing_aggregator("openrouter") is True
|
|
assert providers_mod.is_routing_aggregator("custom:litellm") is True
|
|
assert providers_mod.is_routing_aggregator("not-a-provider") is False
|
|
|
|
|
|
def test_picker_selection_resolves_named_custom_provider_model_id(monkeypatch):
|
|
"""Picker prefixes must not leak into a named custom provider API model id."""
|
|
monkeypatch.setattr(
|
|
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
|
lambda **kwargs: {
|
|
"api_key": "test-key",
|
|
"base_url": "https://token.sensenova.cn/v1",
|
|
"api_mode": "chat_completions",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.validate_requested_model",
|
|
lambda *a, **k: _MOCK_VALIDATION,
|
|
)
|
|
monkeypatch.setattr("hermes_cli.model_switch.get_model_info", lambda *a, **k: None)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.model_switch.get_model_capabilities",
|
|
lambda *a, **k: None,
|
|
)
|
|
|
|
result = switch_model(
|
|
raw_input="sensenova/deepseek-v4-flash",
|
|
current_provider="openai-codex",
|
|
current_model="gpt-5.4",
|
|
explicit_provider="custom:sensenova",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "sensenova",
|
|
"base_url": "https://token.sensenova.cn/v1",
|
|
"models": [
|
|
{"id": "deepseek-v4-flash", "name": "deepseek-v4-flash"}
|
|
],
|
|
}
|
|
],
|
|
)
|
|
|
|
assert result.success is True
|
|
assert result.target_provider == "custom:sensenova"
|
|
assert result.new_model == "deepseek-v4-flash"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_list_groups_same_name_custom_providers_into_one_row(monkeypatch):
|
|
"""Multiple custom_providers entries sharing a name should produce one row
|
|
with all models collected, not N duplicate rows."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", lambda *a, **k: [])
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openrouter",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "qwen3-coder:480b-cloud"},
|
|
{"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "glm-5.1:cloud"},
|
|
{"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "kimi-k2.5"},
|
|
{"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "minimax-m2.7:cloud"},
|
|
{"name": "Moonshot", "base_url": "https://api.moonshot.ai/v1", "model": "kimi-k2-thinking"},
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
ollama_rows = [p for p in providers if p["name"] == "Ollama Cloud"]
|
|
assert len(ollama_rows) == 1, f"Expected 1 Ollama Cloud row, got {len(ollama_rows)}"
|
|
assert ollama_rows[0]["models"] == [
|
|
"qwen3-coder:480b-cloud", "glm-5.1:cloud", "kimi-k2.5", "minimax-m2.7:cloud"
|
|
]
|
|
assert ollama_rows[0]["total_models"] == 4
|
|
|
|
moonshot_rows = [p for p in providers if p["name"] == "Moonshot"]
|
|
assert len(moonshot_rows) == 1
|
|
assert moonshot_rows[0]["models"] == ["kimi-k2-thinking"]
|
|
|
|
|
|
def test_list_deduplicates_same_model_in_group(monkeypatch):
|
|
"""Duplicate model entries under the same provider name should not produce
|
|
duplicate entries in the models list."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", lambda *a, **k: [])
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openrouter",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{"name": "MyProvider", "base_url": "http://localhost:11434/v1", "model": "llama3", "discover_models": False},
|
|
{"name": "MyProvider", "base_url": "http://localhost:11434/v1", "model": "llama3", "discover_models": False},
|
|
{"name": "MyProvider", "base_url": "http://localhost:11434/v1", "model": "mistral", "discover_models": False},
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
my_rows = [p for p in providers if p["name"] == "MyProvider"]
|
|
assert len(my_rows) == 1
|
|
assert my_rows[0]["models"] == ["llama3", "mistral"]
|
|
assert my_rows[0]["total_models"] == 2
|
|
|
|
|
|
def test_custom_provider_no_key_singular_model_still_probes_live_models(monkeypatch):
|
|
"""A singular ``model:`` is the active selection, not an explicit catalog.
|
|
|
|
No-key local OpenAI-compatible endpoints such as llama.cpp should still be
|
|
probed so /model matches the terminal ``hermes model`` flow. Ollama-native
|
|
discovery is covered separately with a fake ``/api/tags`` server.
|
|
"""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
calls = []
|
|
|
|
def fake_fetch_api_models(api_key, base_url, **kwargs):
|
|
calls.append((api_key, base_url, kwargs))
|
|
return ["llama3", "mistral", "qwen3-coder"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openai-codex",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
# Keep this generic and off Ollama's default :11434: this case
|
|
# covers /v1/models probing, not native /api/tags discovery.
|
|
"name": "Local llama.cpp",
|
|
"base_url": "http://localhost:8080/v1",
|
|
"model": "llama3",
|
|
}
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
assert calls == [
|
|
("", "http://localhost:8080/v1", {
|
|
"timeout": 5.0,
|
|
"api_mode": None,
|
|
"headers": None,
|
|
})
|
|
]
|
|
row = next(p for p in providers if p["name"] == "Local llama.cpp")
|
|
assert row["models"] == ["llama3", "mistral", "qwen3-coder"]
|
|
assert row["total_models"] == 3
|
|
|
|
|
|
def test_custom_provider_model_metadata_dict_still_probes(monkeypatch):
|
|
"""Dict-shaped ``models:`` is metadata, not an explicit allowlist."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
calls = []
|
|
|
|
def fetch(*args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
return ["unexpected-live-model"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:local-ollama",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local Ollama",
|
|
"base_url": "http://localhost:11434/v1",
|
|
"model": "llama3",
|
|
"models": {"llama3": {}},
|
|
}
|
|
],
|
|
)
|
|
|
|
row = next(p for p in providers if p["name"] == "Local Ollama")
|
|
assert calls == [
|
|
(
|
|
("", "http://localhost:11434/v1"),
|
|
{"timeout": 5.0, "api_mode": None, "headers": None},
|
|
)
|
|
]
|
|
assert row["models"] == ["unexpected-live-model"]
|
|
|
|
|
|
def test_custom_provider_group_explicit_duplicate_skips_probe(monkeypatch):
|
|
"""A later grouped entry can explicitly narrow to an existing model."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
calls = []
|
|
|
|
def fetch(*args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
return ["unexpected-live-model"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:local-ollama",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local Ollama",
|
|
"base_url": "http://localhost:11434/v1",
|
|
"model": "llama3",
|
|
},
|
|
{
|
|
"name": "Local Ollama",
|
|
"base_url": "http://localhost:11434/v1",
|
|
"models": ["llama3"],
|
|
},
|
|
],
|
|
)
|
|
|
|
row = next(p for p in providers if p["name"] == "Local Ollama")
|
|
assert calls == []
|
|
assert row["models"] == ["llama3"]
|
|
|
|
|
|
def test_custom_provider_current_only_probe_respects_explicit_catalog(monkeypatch):
|
|
"""Normal GUI opens probe only the active singular-only provider."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
calls = []
|
|
|
|
def fetch(api_key, base_url, **kwargs):
|
|
calls.append((api_key, base_url, kwargs))
|
|
return ["live-a", "live-b"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:active",
|
|
current_base_url="http://active.local/v1",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Active",
|
|
"base_url": "http://active.local/v1",
|
|
"model": "seed",
|
|
},
|
|
{
|
|
"name": "Offline",
|
|
"base_url": "http://offline.local/v1",
|
|
"model": "offline-seed",
|
|
},
|
|
{
|
|
"name": "Static",
|
|
"base_url": "http://static.local/v1",
|
|
"model": "only",
|
|
"models": ["only"],
|
|
},
|
|
],
|
|
probe_custom_providers=False,
|
|
probe_current_custom_provider=True,
|
|
)
|
|
|
|
assert calls == [
|
|
(
|
|
"",
|
|
"http://active.local/v1",
|
|
{"timeout": 5.0, "api_mode": None, "headers": None},
|
|
)
|
|
]
|
|
rows = {row["name"]: row for row in providers if row.get("is_user_defined")}
|
|
assert rows["Active"]["models"] == ["live-a", "live-b"]
|
|
assert rows["Offline"]["models"] == ["offline-seed"]
|
|
assert rows["Static"]["models"] == ["only"]
|
|
|
|
|
|
def test_custom_provider_current_explicit_catalog_skips_probe(monkeypatch):
|
|
"""Current-only GUI probing must still honor an explicit catalog."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
calls = []
|
|
|
|
def fetch(*args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
return ["unexpected-live-model"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:static",
|
|
current_base_url="http://static.local/v1",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Static",
|
|
"base_url": "http://static.local/v1",
|
|
"model": "only",
|
|
"models": ["only"],
|
|
}
|
|
],
|
|
probe_custom_providers=False,
|
|
probe_current_custom_provider=True,
|
|
)
|
|
|
|
assert calls == []
|
|
row = next(p for p in providers if p["name"] == "Static")
|
|
assert row["is_current"] is True
|
|
assert row["models"] == ["only"]
|
|
|
|
|
|
def test_custom_provider_empty_explicit_list_allows_probe(monkeypatch):
|
|
"""An empty ``models:`` declaration is not an explicit catalog."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
calls = []
|
|
|
|
def fetch(api_key, base_url, **kwargs):
|
|
calls.append((api_key, base_url, kwargs))
|
|
return ["live-a", "live-b"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:local",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local",
|
|
"base_url": "http://local.test/v1",
|
|
"model": "seed",
|
|
"models": [],
|
|
}
|
|
],
|
|
)
|
|
|
|
assert calls == [
|
|
(
|
|
"",
|
|
"http://local.test/v1",
|
|
{"timeout": 5.0, "api_mode": None, "headers": None},
|
|
)
|
|
]
|
|
row = next(p for p in providers if p["name"] == "Local")
|
|
assert row["models"] == ["live-a", "live-b"]
|
|
|
|
|
|
def test_list_enumerates_dict_format_models_alongside_default(monkeypatch):
|
|
"""custom_providers entry with dict-format ``models:`` plus singular
|
|
``model:`` should surface the default and every dict key.
|
|
|
|
Regression: Hermes's own writer stores configured models as a dict
|
|
keyed by model id, but the /model picker previously only honored the
|
|
singular ``model:`` field, so multi-model custom providers appeared
|
|
to have only the active model.
|
|
"""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openai-codex",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "DeepSeek",
|
|
"base_url": "https://api.deepseek.com",
|
|
"api_mode": "chat_completions",
|
|
"model": "deepseek-chat",
|
|
"models": {
|
|
"deepseek-chat": {"context_length": 128000},
|
|
"deepseek-reasoner": {"context_length": 128000},
|
|
},
|
|
}
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
ds_rows = [p for p in providers if p["name"] == "DeepSeek"]
|
|
assert len(ds_rows) == 1
|
|
assert ds_rows[0]["models"] == ["deepseek-chat", "deepseek-reasoner"]
|
|
assert ds_rows[0]["total_models"] == 2
|
|
|
|
|
|
def test_list_enumerates_dict_format_models_without_singular_model(monkeypatch):
|
|
"""Dict-format ``models:`` with no singular ``model:`` should still
|
|
enumerate every dict key (previously the picker reported 0 models)."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openai-codex",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Thor",
|
|
"base_url": "http://thor.lab:8337/v1",
|
|
"models": {
|
|
"gemma-4-26B-A4B-it-MXFP4_MOE": {"context_length": 262144},
|
|
"Qwen3.5-35B-A3B-MXFP4_MOE": {"context_length": 262144},
|
|
"gemma-4-31B-it-Q4_K_M": {"context_length": 262144},
|
|
},
|
|
}
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
thor_rows = [p for p in providers if p["name"] == "Thor"]
|
|
assert len(thor_rows) == 1
|
|
assert set(thor_rows[0]["models"]) == {
|
|
"gemma-4-26B-A4B-it-MXFP4_MOE",
|
|
"Qwen3.5-35B-A3B-MXFP4_MOE",
|
|
"gemma-4-31B-it-Q4_K_M",
|
|
}
|
|
assert thor_rows[0]["total_models"] == 3
|
|
|
|
|
|
def test_list_dedupes_dict_model_matching_singular_default(monkeypatch):
|
|
"""When the singular ``model:`` is also a key in the ``models:`` dict,
|
|
it must appear exactly once in the picker."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openai-codex",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "DeepSeek",
|
|
"base_url": "https://api.deepseek.com",
|
|
"model": "deepseek-chat",
|
|
"models": {
|
|
"deepseek-chat": {"context_length": 128000},
|
|
"deepseek-reasoner": {"context_length": 128000},
|
|
},
|
|
}
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
ds_rows = [p for p in providers if p["name"] == "DeepSeek"]
|
|
assert ds_rows[0]["models"].count("deepseek-chat") == 1
|
|
assert ds_rows[0]["models"] == ["deepseek-chat", "deepseek-reasoner"]
|
|
|
|
|
|
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# #9210: group custom_providers by (base_url, api_key) in /model picker
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_list_authenticated_providers_groups_same_endpoint(monkeypatch):
|
|
"""Multiple custom_providers entries sharing a base_url+api_key must be
|
|
returned as a single picker row with all their models merged."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom",
|
|
current_base_url="http://localhost:11434/v1",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{"name": "Ollama — MiniMax M2.7", "base_url": "http://localhost:11434/v1",
|
|
"api_key": "ollama", "model": "minimax-m2.7", "discover_models": False},
|
|
{"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1",
|
|
"api_key": "ollama", "model": "glm-5.1", "discover_models": False},
|
|
{"name": "Ollama — Qwen3-coder", "base_url": "http://localhost:11434/v1",
|
|
"api_key": "ollama", "model": "qwen3-coder", "discover_models": False},
|
|
],
|
|
max_models=50,
|
|
probe_custom_providers=False,
|
|
)
|
|
|
|
custom_groups = [p for p in providers if p.get("is_user_defined")]
|
|
assert len(custom_groups) == 1, (
|
|
"Expected 1 group for shared endpoint, got "
|
|
f"{[p['slug'] for p in custom_groups]}"
|
|
)
|
|
group = custom_groups[0]
|
|
assert set(group["models"]) == {"minimax-m2.7", "glm-5.1", "qwen3-coder"}
|
|
assert group["total_models"] == 3
|
|
# Per-model suffix stripped from display name
|
|
assert group["name"] == "Ollama"
|
|
|
|
|
|
def test_list_authenticated_providers_current_endpoint_uses_current_slug(monkeypatch):
|
|
"""When current_base_url matches the grouped endpoint, the slug must
|
|
equal current_provider so picker selection routes through the live
|
|
credential pipeline — provided current_provider is a real slug, not
|
|
the corrupt bare "custom" (see #17478)."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:ollama",
|
|
current_base_url="http://localhost:11434/v1",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1",
|
|
"api_key": "ollama", "model": "glm-5.1"},
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
matches = [p for p in providers if p.get("is_user_defined")]
|
|
assert len(matches) == 1
|
|
group = matches[0]
|
|
assert group["slug"] == "custom:ollama"
|
|
assert group["is_current"] is True
|
|
|
|
|
|
def test_picker_endpoint_authorization_overrides_inferred_bearer(monkeypatch):
|
|
from hermes_cli.model_switch import _fetch_picker_live_models
|
|
|
|
captured: dict[str, str] = {}
|
|
|
|
def fake_native(_url, *, timeout, headers):
|
|
captured.update(headers or {})
|
|
return ["model-a"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.should_use_ollama_native_catalog", lambda *a, **k: True)
|
|
monkeypatch.setattr("hermes_cli.models.fetch_ollama_local_models", fake_native)
|
|
result = _fetch_picker_live_models(
|
|
"endpoint-key",
|
|
"http://127.0.0.1:11434/v1",
|
|
"ollama",
|
|
False,
|
|
headers={"authorization": "Api-Key explicit"},
|
|
)
|
|
|
|
assert result == ["model-a"]
|
|
auth_headers = {
|
|
key: value
|
|
for key, value in captured.items()
|
|
if key.lower() == "authorization"
|
|
}
|
|
assert auth_headers == {"authorization": "Api-Key explicit"}
|
|
|
|
|
|
def test_list_authenticated_providers_bare_custom_slug_recovers(monkeypatch):
|
|
"""Regression for #17478: when a prior failed switch left the bare
|
|
literal "custom" in model.provider, the picker must NOT propagate
|
|
that broken slug. It must fall back to the canonical
|
|
``custom:<name>`` form so the picker stays usable."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom",
|
|
current_base_url="http://localhost:11434/v1",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1",
|
|
"api_key": "ollama", "model": "glm-5.1"},
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
matches = [p for p in providers if p.get("is_user_defined")]
|
|
assert len(matches) == 1
|
|
group = matches[0]
|
|
# Canonical slug, NOT the bare "custom" that caused #17478
|
|
assert group["slug"] == "custom:ollama"
|
|
assert group["is_current"] is True
|
|
|
|
|
|
def test_compatible_keyed_provider_uses_stable_key_and_accepts_legacy_current_name(
|
|
monkeypatch,
|
|
):
|
|
"""The merged providers view keeps the config key while old IDs stay current."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:local-ollama",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local Ollama",
|
|
"provider_key": "local-127.0.0.1:11434",
|
|
"base_url": "http://127.0.0.1:11434/v1",
|
|
"model": "qwen3.5:9b",
|
|
}
|
|
],
|
|
max_models=50,
|
|
probe_custom_providers=False,
|
|
)
|
|
|
|
row = next(p for p in providers if p.get("is_user_defined"))
|
|
assert row["slug"] == "custom:local-127.0.0.1:11434"
|
|
assert row["is_current"] is True
|
|
|
|
|
|
def test_user_provider_row_recognizes_stable_custom_key_as_current(monkeypatch):
|
|
"""Section 3 keeps its legacy row slug but recognizes the stable ID."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:local-127.0.0.1:11434",
|
|
user_providers={
|
|
"local-127.0.0.1:11434": {
|
|
"name": "Local Ollama",
|
|
"base_url": "http://127.0.0.1:11434/v1",
|
|
"default_model": "qwen3.5:9b",
|
|
}
|
|
},
|
|
custom_providers=[],
|
|
max_models=50,
|
|
probe_custom_providers=False,
|
|
)
|
|
|
|
row = next(p for p in providers if p.get("is_user_defined"))
|
|
assert row["slug"] == "local-127.0.0.1:11434"
|
|
assert row["is_current"] is True
|
|
|
|
|
|
def test_list_authenticated_providers_distinct_endpoints_stay_separate(monkeypatch):
|
|
"""Entries with different base_urls must produce separate picker rows
|
|
even if some display names happen to be similar."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
user_providers={},
|
|
custom_providers=[
|
|
{"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1",
|
|
"api_key": "ollama", "model": "glm-5.1"},
|
|
{"name": "Moonshot", "base_url": "https://api.moonshot.cn/v1",
|
|
"api_key": "sk-m", "model": "moonshot-v1"},
|
|
{"name": "Ollama — Qwen3-coder", "base_url": "http://localhost:11434/v1",
|
|
"api_key": "ollama", "model": "qwen3-coder"},
|
|
],
|
|
max_models=50,
|
|
probe_custom_providers=False,
|
|
)
|
|
|
|
custom_groups = [p for p in providers if p.get("is_user_defined")]
|
|
assert len(custom_groups) == 2
|
|
# Ollama endpoint collapses to one row with both models
|
|
ollama = next(p for p in custom_groups if p["name"] == "Ollama")
|
|
assert set(ollama["models"]) == {"glm-5.1", "qwen3-coder"}
|
|
moonshot = next(p for p in custom_groups if p["name"] == "Moonshot")
|
|
assert moonshot["models"] == ["moonshot-v1"]
|
|
|
|
|
|
def test_list_authenticated_providers_same_url_different_keys_disambiguated(monkeypatch):
|
|
"""Two custom_providers entries with the same base_url but different
|
|
api_keys (and identical cleaned names) must both stay visible in the
|
|
picker — slug is suffixed to disambiguate."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
user_providers={},
|
|
custom_providers=[
|
|
{"name": "OpenAI — key A", "base_url": "https://api.openai.com/v1",
|
|
"api_key": "sk-AAA", "model": "gpt-5.4"},
|
|
{"name": "OpenAI — key B", "base_url": "https://api.openai.com/v1",
|
|
"api_key": "sk-BBB", "model": "gpt-4.6"},
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
custom_groups = [p for p in providers if p.get("is_user_defined")]
|
|
assert len(custom_groups) == 2
|
|
slugs = sorted(p["slug"] for p in custom_groups)
|
|
# First group keeps the base slug, second gets a numeric suffix
|
|
assert slugs == ["custom:openai", "custom:openai-2"]
|
|
# Each row has a distinct model
|
|
models = {p["slug"]: p["models"] for p in custom_groups}
|
|
assert models["custom:openai"] == ["gpt-5.4"]
|
|
assert models["custom:openai-2"] == ["gpt-4.6"]
|
|
|
|
|
|
def test_list_authenticated_providers_same_url_different_key_env_and_api_mode_stay_separate(monkeypatch):
|
|
"""Same gateway host but different key_env/api_mode entries are distinct providers."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:gpt",
|
|
current_base_url="https://gateway.example.com",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "gpt",
|
|
"base_url": "https://gateway.example.com",
|
|
"key_env": "GPT_KEY",
|
|
"api_mode": "codex_responses",
|
|
"model": "gpt-5.5",
|
|
},
|
|
{
|
|
"name": "claude",
|
|
"base_url": "https://gateway.example.com",
|
|
"key_env": "CLAUDE_KEY",
|
|
"api_mode": "anthropic_messages",
|
|
"model": "claude-opus-4-8",
|
|
},
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
custom = [p for p in providers if p.get("is_user_defined")]
|
|
by_slug = {p["slug"]: p for p in custom}
|
|
|
|
assert set(by_slug) == {"custom:gpt", "custom:claude"}
|
|
assert by_slug["custom:gpt"]["models"] == ["gpt-5.5"]
|
|
assert by_slug["custom:claude"]["models"] == ["claude-opus-4-8"]
|
|
assert by_slug["custom:gpt"]["is_current"] is True
|
|
assert by_slug["custom:claude"]["is_current"] is False
|
|
|
|
|
|
def test_list_authenticated_providers_total_models_reflects_grouped_count(monkeypatch):
|
|
"""After grouping six entries into one row, total_models must reflect
|
|
the full count, and every grouped model appears in the list."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
entries = [
|
|
{"name": f"Ollama \u2014 Model {i}", "base_url": "http://localhost:11434/v1",
|
|
"api_key": "ollama", "model": f"model-{i}"}
|
|
for i in range(6)
|
|
]
|
|
providers = list_authenticated_providers(
|
|
user_providers={},
|
|
custom_providers=entries,
|
|
max_models=4,
|
|
probe_custom_providers=False,
|
|
)
|
|
|
|
groups = [p for p in providers if p.get("is_user_defined")]
|
|
assert len(groups) == 1
|
|
group = groups[0]
|
|
assert group["total_models"] == 6
|
|
# All six models are preserved in the grouped row.
|
|
assert sorted(group["models"]) == sorted(f"model-{i}" for i in range(6))
|
|
|
|
|
|
def test_lmstudio_picker_probes_active_config_base_url(monkeypatch):
|
|
"""When `provider: lmstudio` is saved with a remote base_url and no
|
|
LM_BASE_URL env var, the picker must probe the saved base_url — not
|
|
127.0.0.1. Regression: prior behavior always probed localhost, so users
|
|
with LM Studio on a lab box saw the wrong (or empty) model list.
|
|
"""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
monkeypatch.delenv("LM_BASE_URL", raising=False)
|
|
monkeypatch.delenv("LM_API_KEY", raising=False)
|
|
|
|
captured: dict = {}
|
|
|
|
def _fake_fetch(api_key=None, base_url=None, timeout=5.0):
|
|
captured["base_url"] = base_url
|
|
captured["api_key"] = api_key
|
|
return ["qwen/qwen3-coder-30b"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_lmstudio_models", _fake_fetch)
|
|
|
|
list_authenticated_providers(
|
|
current_provider="lmstudio",
|
|
current_base_url="http://192.168.1.10:1234/v1",
|
|
current_model="qwen/qwen3-coder-30b",
|
|
)
|
|
|
|
assert captured["base_url"] == "http://192.168.1.10:1234/v1"
|
|
|
|
|
|
def test_lmstudio_picker_lm_base_url_env_wins_over_active_config(monkeypatch):
|
|
"""LM_BASE_URL env var must still take precedence over the saved
|
|
base_url so users can temporarily redirect the picker without editing
|
|
config.yaml.
|
|
"""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
monkeypatch.setenv("LM_BASE_URL", "http://override.local:9999/v1")
|
|
monkeypatch.delenv("LM_API_KEY", raising=False)
|
|
|
|
captured: dict = {}
|
|
|
|
def _fake_fetch(api_key=None, base_url=None, timeout=5.0):
|
|
captured["base_url"] = base_url
|
|
return []
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_lmstudio_models", _fake_fetch)
|
|
|
|
list_authenticated_providers(
|
|
current_provider="lmstudio",
|
|
current_base_url="http://192.168.1.10:1234/v1",
|
|
)
|
|
|
|
assert captured["base_url"] == "http://override.local:9999/v1"
|
|
|
|
|
|
def test_lmstudio_picker_skips_probe_when_not_configured(monkeypatch):
|
|
"""If the user has never configured LM Studio (no LM_API_KEY / LM_BASE_URL
|
|
and not on lmstudio), the picker must not pay the localhost probe cost
|
|
just to discover LM Studio is unavailable.
|
|
"""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
monkeypatch.delenv("LM_BASE_URL", raising=False)
|
|
monkeypatch.delenv("LM_API_KEY", raising=False)
|
|
|
|
captured: dict = {}
|
|
|
|
def _fake_fetch(api_key=None, base_url=None, timeout=5.0):
|
|
captured["base_url"] = base_url
|
|
return []
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_lmstudio_models", _fake_fetch)
|
|
|
|
list_authenticated_providers(
|
|
current_provider="openrouter",
|
|
current_base_url="https://openrouter.ai/api/v1",
|
|
)
|
|
|
|
assert "base_url" not in captured
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_custom_providers_uses_live_models_for_multi_model_endpoint(monkeypatch):
|
|
"""Custom providers with api_key + base_url should prefer live /models.
|
|
|
|
Custom providers (section 4 of list_authenticated_providers) point at
|
|
gateways like Bifrost that expose hundreds of models. Reading only the
|
|
static ``models:`` dict from config.yaml leaves the /model picker with
|
|
a stale subset. Live discovery fills the picker with all available
|
|
models from the endpoint.
|
|
"""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
|
|
|
|
calls = []
|
|
|
|
def fake_fetch_api_models(api_key, base_url, **kwargs):
|
|
calls.append((api_key, base_url, kwargs))
|
|
return ["gateway-model-a", "gateway-model-b", "gateway-model-c"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
|
|
|
|
custom_providers = [
|
|
{
|
|
"name": "my-gateway",
|
|
"api_key": "sk-gateway-key",
|
|
"base_url": "https://gateway.example.com/v1",
|
|
"model": "gateway-model-a",
|
|
"models": {
|
|
"gateway-model-a": {"context_length": 128000},
|
|
"gateway-model-b": {"context_length": 128000},
|
|
},
|
|
}
|
|
]
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openrouter",
|
|
current_base_url="https://openrouter.ai/api/v1",
|
|
custom_providers=custom_providers,
|
|
max_models=50,
|
|
)
|
|
|
|
gateway_prov = next(
|
|
(
|
|
p
|
|
for p in providers
|
|
if p.get("api_url") == "https://gateway.example.com/v1"
|
|
),
|
|
None,
|
|
)
|
|
|
|
assert gateway_prov is not None, "Custom provider group not found in results"
|
|
assert calls == [
|
|
(
|
|
"sk-gateway-key",
|
|
"https://gateway.example.com/v1",
|
|
{"timeout": 5.0, "api_mode": None, "headers": None},
|
|
)
|
|
], "fetch_api_models must be called with the custom provider's credentials"
|
|
assert gateway_prov["models"] == [
|
|
"gateway-model-a",
|
|
"gateway-model-b",
|
|
"gateway-model-c",
|
|
], "Live models must replace the static subset"
|
|
assert gateway_prov["total_models"] == 3
|
|
|
|
|
|
def test_same_endpoint_different_extra_headers_not_collapsed(monkeypatch):
|
|
"""Entries sharing (api_url, credential, api_mode) but declaring different
|
|
extra_headers must NOT collapse into one picker row — each is a distinct
|
|
header-authenticated endpoint (e.g. per-tenant routing behind one proxy)
|
|
and must probe /models with its own headers."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
|
|
|
|
calls = []
|
|
|
|
def fake_fetch_api_models(api_key, base_url, **kwargs):
|
|
calls.append((api_key, base_url, kwargs.get("headers")))
|
|
# Return a per-tenant model list keyed by the routing header so we can
|
|
# assert each row got its OWN probe rather than a shared one.
|
|
tenant = (kwargs.get("headers") or {}).get("X-Tenant", "none")
|
|
return [f"model-{tenant}"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openrouter",
|
|
current_base_url="https://openrouter.ai/api/v1",
|
|
custom_providers=[
|
|
{
|
|
"name": "Proxy Tenant A",
|
|
"api_key": "shared-key",
|
|
"base_url": "http://localhost:8081/v1",
|
|
"extra_headers": {"X-Tenant": "a"},
|
|
},
|
|
{
|
|
"name": "Proxy Tenant B",
|
|
"api_key": "shared-key",
|
|
"base_url": "http://localhost:8081/v1",
|
|
"extra_headers": {"X-Tenant": "b"},
|
|
},
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
rows = [
|
|
p for p in providers if p.get("api_url") == "http://localhost:8081/v1"
|
|
]
|
|
# Two distinct rows, not one collapsed row.
|
|
assert len(rows) == 2, f"expected 2 rows, got {len(rows)}: {rows}"
|
|
|
|
# Each tenant was probed with its OWN header set (order-independent).
|
|
assert ("shared-key", "http://localhost:8081/v1", {"X-Tenant": "a"}) in calls
|
|
assert ("shared-key", "http://localhost:8081/v1", {"X-Tenant": "b"}) in calls
|
|
|
|
# Each row surfaces the model list its own headers unlocked.
|
|
models_by_row = {tuple(r["models"]) for r in rows}
|
|
assert models_by_row == {("model-a",), ("model-b",)}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_resolve_custom_provider_passes_key_env():
|
|
"""resolve_custom_provider should propagate key_env into api_key_env_vars.
|
|
|
|
Regression: previously api_key_env_vars was always (), silently dropping
|
|
the configured env var and causing 401s on every request.
|
|
"""
|
|
from hermes_cli.providers import resolve_custom_provider
|
|
|
|
resolved = resolve_custom_provider(
|
|
"custom:token-plan",
|
|
custom_providers=[
|
|
{
|
|
"name": "token-plan",
|
|
"base_url": "https://token-plan-sgp.xiaomimimo.com/v1",
|
|
"key_env": "XIAOMI_MIMO_API_KEY",
|
|
"model": "mimo-v2-pro",
|
|
}
|
|
],
|
|
)
|
|
|
|
assert resolved is not None
|
|
assert resolved.api_key_env_vars == ("XIAOMI_MIMO_API_KEY",)
|
|
assert resolved.base_url == "https://token-plan-sgp.xiaomimimo.com/v1"
|
|
|
|
|
|
def test_discovered_models_auto_saved_to_cache(monkeypatch):
|
|
"""Discovered models are persisted to config so ``discover_models: false``
|
|
has a populated cache on the next read (#65652).
|
|
|
|
When a successful probe returns live models, ``_save_discovered_models_to_config``
|
|
must be called with the provider's base_url and the discovered model list.
|
|
"""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
|
|
|
|
save_calls = []
|
|
|
|
def fake_fetch_api_models(api_key, base_url, **kwargs):
|
|
return ["discovered-a", "discovered-b", "discovered-c"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.model_switch._save_discovered_models_to_config",
|
|
lambda api_url, model_ids, **kwargs: save_calls.append((api_url, model_ids)),
|
|
)
|
|
|
|
custom_providers = [
|
|
{
|
|
"name": "my-gateway",
|
|
"api_key": "***",
|
|
"base_url": "https://gateway.example.com/v1",
|
|
"discover_models": True,
|
|
"model": "only-model",
|
|
"models": {"only-model": {"context_length": 128000}},
|
|
}
|
|
]
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="my-gateway",
|
|
current_base_url="https://gateway.example.com/v1",
|
|
custom_providers=custom_providers,
|
|
max_models=50,
|
|
probe_custom_providers=True,
|
|
)
|
|
|
|
assert len(save_calls) == 1, (
|
|
"_save_discovered_models_to_config must be called after a successful probe"
|
|
)
|
|
assert save_calls[0][0] == "https://gateway.example.com/v1"
|
|
assert save_calls[0][1] == ["discovered-a", "discovered-b", "discovered-c"]
|
|
|
|
gateway_prov = next(
|
|
(p for p in providers if p.get("api_url") == "https://gateway.example.com/v1"),
|
|
None,
|
|
)
|
|
assert gateway_prov is not None
|
|
assert gateway_prov["models"] == ["discovered-a", "discovered-b", "discovered-c"]
|
|
|
|
|
|
|
|
|
|
def test_save_discovered_models_preserves_dict_form(monkeypatch):
|
|
"""``_save_discovered_models_to_config`` must not replace a dict-form
|
|
``models`` mapping (per-model metadata like ``context_length``) with
|
|
a flat list of strings (#67841)."""
|
|
from hermes_cli.model_switch import _save_discovered_models_to_config
|
|
|
|
save_calls = []
|
|
|
|
def fake_save(config):
|
|
save_calls.append(dict(config))
|
|
|
|
monkeypatch.setattr("hermes_cli.config.save_config", fake_save)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.load_config",
|
|
lambda: {
|
|
"custom_providers": [
|
|
{
|
|
"name": "my-gateway",
|
|
"base_url": "https://gateway.example.com/v1",
|
|
"models": {
|
|
"configured-model": {"context_length": 8192},
|
|
},
|
|
}
|
|
]
|
|
},
|
|
)
|
|
|
|
# Dict-form models must NOT be overwritten by discovered models
|
|
_save_discovered_models_to_config(
|
|
"https://gateway.example.com/v1",
|
|
["configured-model", "discovered-model"],
|
|
)
|
|
assert save_calls == [], (
|
|
"Dict-form models must not be replaced with a flat list"
|
|
)
|
|
|
|
|
|
def test_model_flow_named_custom_persists_discovered_models(monkeypatch):
|
|
"""The ``hermes model`` named-custom-provider flow persists the discovered
|
|
catalog back to the entry's ``models:`` list.
|
|
|
|
No-probe surfaces (dashboard, desktop, ACP) call
|
|
``build_models_payload(..., probe_custom_providers=False)`` and only show
|
|
the configured ``models:`` list. The CLI flow probes and shows the full
|
|
catalog but (before this fix) never saved it, so a provider added via
|
|
``hermes model`` collapsed to the single ``model:`` default everywhere but
|
|
the CLI. It must persist discovered models the same way the picker path in
|
|
``_save_discovered_models_to_config`` does.
|
|
"""
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.fetch_api_models",
|
|
lambda api_key, base_url, **kw: [
|
|
"discovered-a",
|
|
"discovered-b",
|
|
"discovered-c",
|
|
],
|
|
)
|
|
# Non-interactive model selection.
|
|
monkeypatch.setattr(
|
|
"hermes_cli.curses_ui.curses_radiolist", lambda *a, **k: 0
|
|
)
|
|
# No-op downstream writes so the test never touches a real config.
|
|
monkeypatch.setattr("hermes_cli.main._save_custom_provider", lambda *a, **k: None)
|
|
monkeypatch.setattr("hermes_cli.auth._save_model_choice", lambda *a, **k: None)
|
|
monkeypatch.setattr("hermes_cli.auth.deactivate_provider", lambda *a, **k: None)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.load_config",
|
|
lambda: {"model": {}, "providers": {}, "custom_providers": []},
|
|
)
|
|
monkeypatch.setattr("hermes_cli.config.save_config", lambda cfg: None)
|
|
|
|
save_calls = []
|
|
monkeypatch.setattr(
|
|
"hermes_cli.model_switch._save_discovered_models_to_config",
|
|
lambda api_url, model_ids, **kwargs: save_calls.append(
|
|
(api_url, model_ids, kwargs)
|
|
),
|
|
)
|
|
|
|
from hermes_cli.model_setup_flows import _model_flow_named_custom
|
|
|
|
_model_flow_named_custom(
|
|
{},
|
|
{
|
|
"name": "Dragomes",
|
|
"base_url": "http://example.com/v1",
|
|
"api_mode": "anthropic_messages",
|
|
"extra_headers": {"X-Tenant": "dragomes"},
|
|
"api_key": "sk-test",
|
|
"key_env": "",
|
|
"model": "MiniMax-M3",
|
|
"provider_key": "",
|
|
"discover_models": True,
|
|
"models": {},
|
|
},
|
|
)
|
|
|
|
assert save_calls == [
|
|
(
|
|
"http://example.com/v1",
|
|
["discovered-a", "discovered-b", "discovered-c"],
|
|
{
|
|
"api_mode": "anthropic_messages",
|
|
"headers": {"X-Tenant": "dragomes"},
|
|
},
|
|
)
|
|
], (
|
|
"_model_flow_named_custom must persist each live catalog with its "
|
|
"base URL, API mode, and endpoint headers"
|
|
)
|
|
|
|
|
|
def test_shared_url_different_display_names_are_separate_rows(monkeypatch):
|
|
"""Multiple custom_providers entries sharing base_url + api_key + api_mode
|
|
but with *different* display-name prefixes (e.g. a proxy fronting
|
|
cerebras, groq and perplexity at one URL) must each get their own picker
|
|
row, not collapse into one."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
# Stub live discovery so the test is deterministic regardless of network.
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.fetch_api_models",
|
|
lambda api_key, base_url, **kwargs: [],
|
|
)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="openrouter",
|
|
current_base_url="https://openrouter.ai/api/v1",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{"name": "Cerebras", "base_url": "https://proxy.example.com/v1",
|
|
"api_key": "proxy-key", "model": "llama-4-scout"},
|
|
{"name": "Groq", "base_url": "https://proxy.example.com/v1",
|
|
"api_key": "proxy-key", "model": "llama-4-scout"},
|
|
{"name": "Perplexity", "base_url": "https://proxy.example.com/v1",
|
|
"api_key": "proxy-key", "model": "sonar-pro"},
|
|
],
|
|
max_models=50,
|
|
)
|
|
|
|
custom = [p for p in providers if p.get("is_user_defined")]
|
|
names = sorted(p["name"] for p in custom)
|
|
assert names == ["Cerebras", "Groq", "Perplexity"], (
|
|
f"expected three separate rows, got {names}"
|
|
)
|
|
# Each row carries only its own model (no cross-contamination).
|
|
by_name = {p["name"]: p["models"] for p in custom}
|
|
assert by_name["Cerebras"] == ["llama-4-scout"]
|
|
assert by_name["Groq"] == ["llama-4-scout"]
|
|
assert by_name["Perplexity"] == ["sonar-pro"]
|
|
|
|
|
|
def test_excluded_providers_hides_builtin_row(monkeypatch):
|
|
"""``excluded_providers`` must hide a built-in provider row that would
|
|
otherwise surface when its credentials are present."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test")
|
|
|
|
baseline = list_authenticated_providers(
|
|
current_provider="openrouter",
|
|
current_base_url="https://openrouter.ai/api/v1",
|
|
user_providers={},
|
|
custom_providers=[],
|
|
max_models=50,
|
|
)
|
|
assert any(p["slug"] == "openrouter" for p in baseline), (
|
|
"sanity: openrouter row must appear when OPENROUTER_API_KEY is set"
|
|
)
|
|
|
|
filtered = list_authenticated_providers(
|
|
current_provider="openrouter",
|
|
current_base_url="https://openrouter.ai/api/v1",
|
|
user_providers={},
|
|
custom_providers=[],
|
|
max_models=50,
|
|
excluded_providers=["openrouter"],
|
|
)
|
|
assert not any(p["slug"] == "openrouter" for p in filtered), (
|
|
"excluded_providers=['openrouter'] must hide the openrouter row"
|
|
)
|
|
|
|
|
|
def test_custom_provider_context_length_models_dict_still_probes(monkeypatch):
|
|
"""Dict-shaped ``models:`` from ``_save_custom_provider`` is metadata.
|
|
|
|
``hermes model`` writes ``models: {default: {context_length: N}}`` for
|
|
local Ollama. That must not suppress live /v1/models discovery — otherwise
|
|
Desktop/Telegram only show the saved default and Refresh does nothing.
|
|
"""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
calls = []
|
|
|
|
def fetch(api_key, base_url, **kwargs):
|
|
calls.append((api_key, base_url, kwargs))
|
|
return ["qwen3.6:35b-mlx", "gemma4:31b", "llama3"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:local-ollama",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local Ollama",
|
|
"base_url": "http://localhost:11434/v1",
|
|
"model": "qwen3.6:35b-mlx",
|
|
"models": {"qwen3.6:35b-mlx": {"context_length": 32768}},
|
|
}
|
|
],
|
|
# GUI picker path: probe current custom provider only.
|
|
probe_custom_providers=False,
|
|
probe_current_custom_provider=True,
|
|
current_base_url="http://localhost:11434/v1",
|
|
)
|
|
|
|
assert len(calls) == 1
|
|
assert calls[0][0] == ""
|
|
assert calls[0][1] == "http://localhost:11434/v1"
|
|
row = next(p for p in providers if p["name"] == "Local Ollama")
|
|
assert row["models"] == ["qwen3.6:35b-mlx", "gemma4:31b", "llama3"]
|
|
assert row["total_models"] == 3
|
|
|
|
|
|
def test_custom_provider_dict_models_pin_requires_discover_false(monkeypatch):
|
|
"""Dict-shaped catalogs pin only when ``discover_models: false``."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
calls = []
|
|
|
|
def fetch(*args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
return ["unexpected-live-model"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom:local-ollama",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local Ollama",
|
|
"base_url": "http://localhost:11434/v1",
|
|
"model": "llama3",
|
|
"models": {"llama3": {}},
|
|
"discover_models": False,
|
|
}
|
|
],
|
|
)
|
|
|
|
row = next(p for p in providers if p["name"] == "Local Ollama")
|
|
assert calls == []
|
|
assert row["models"] == ["llama3"]
|
|
|
|
|
|
# ─── No-probe picker opens still serve the cached catalog ───────────────
|
|
#
|
|
# #58183 stopped GUI picker opens from live-probing saved custom endpoints so
|
|
# a stopped local server could not stall the picker. It skipped the cached
|
|
# read along with the network one, so a non-current endpoint collapsed to the
|
|
# one model named in config even with a full catalog already on disk. These
|
|
# pin both halves: the cache is served, the network is not touched.
|
|
|
|
|
|
_LOCAL_ENDPOINT = "http://127.0.0.1:8000/v1"
|
|
_LOCAL_CATALOG = [f"omlx-model-{i}" for i in range(1, 9)]
|
|
_SHARED_PROXY_URL = "https://proxy.example.com/v1"
|
|
|
|
|
|
def _seed_custom_model_cache(monkeypatch, models, *, age_seconds=10):
|
|
"""Put *models* on disk for ``_LOCAL_ENDPOINT`` under the no-credential
|
|
fingerprint the picker probes local endpoints with."""
|
|
import hermes_cli.models as models_mod
|
|
|
|
fp = models_mod._custom_endpoint_fingerprint("", None, None)
|
|
cache = {
|
|
f"custom:{_LOCAL_ENDPOINT}": {
|
|
"fp": fp,
|
|
"at": time.time() - age_seconds,
|
|
"models": list(models),
|
|
}
|
|
}
|
|
monkeypatch.setattr(models_mod, "_load_provider_models_cache", lambda: cache)
|
|
|
|
|
|
def _no_probe_local_row(monkeypatch, *, custom_providers=None, user_providers=None,
|
|
current_provider="nous", **kwargs):
|
|
"""Run the GUI picker path (no live probing) and return the local row
|
|
plus every base_url a live fetch was attempted against."""
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
fetched = []
|
|
|
|
def fetch(_api_key, base_url, **_kwargs):
|
|
fetched.append(base_url)
|
|
return ["should-not-be-reached"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider=current_provider,
|
|
user_providers=user_providers or {},
|
|
custom_providers=custom_providers or [],
|
|
for_picker=True,
|
|
refresh=False,
|
|
probe_custom_providers=False,
|
|
probe_current_custom_provider=True,
|
|
**kwargs,
|
|
)
|
|
row = next(
|
|
(p for p in providers if _LOCAL_ENDPOINT in str(p.get("api_url", ""))), None
|
|
)
|
|
return row, fetched
|
|
|
|
|
|
def test_no_probe_open_serves_cached_catalog_for_custom_provider(monkeypatch):
|
|
"""A ``custom_providers`` endpoint that is not the current provider still
|
|
shows its full discovered catalog, from cache, with no network call."""
|
|
_seed_custom_model_cache(monkeypatch, _LOCAL_CATALOG)
|
|
|
|
row, fetched = _no_probe_local_row(
|
|
monkeypatch,
|
|
custom_providers=[
|
|
{
|
|
"name": "Local (127.0.0.1:8000)",
|
|
"base_url": _LOCAL_ENDPOINT,
|
|
"model": "omlx-model-1",
|
|
}
|
|
],
|
|
)
|
|
|
|
assert row is not None
|
|
assert row["is_current"] is False
|
|
assert row["models"] == _LOCAL_CATALOG
|
|
assert row["total_models"] == len(_LOCAL_CATALOG)
|
|
assert fetched == []
|
|
|
|
|
|
def test_no_probe_open_serves_cached_catalog_for_user_provider(monkeypatch):
|
|
"""Same contract for a ``providers:`` entry (section 3)."""
|
|
_seed_custom_model_cache(monkeypatch, _LOCAL_CATALOG)
|
|
|
|
row, fetched = _no_probe_local_row(
|
|
monkeypatch,
|
|
user_providers={
|
|
"local-8000": {
|
|
"name": "Local (127.0.0.1:8000)",
|
|
"base_url": _LOCAL_ENDPOINT,
|
|
"default_model": "omlx-model-1",
|
|
}
|
|
},
|
|
)
|
|
|
|
assert row is not None
|
|
assert row["models"] == _LOCAL_CATALOG
|
|
assert fetched == []
|
|
|
|
|
|
def test_no_probe_open_serves_cached_catalog_for_bare_custom_endpoint(monkeypatch):
|
|
"""Same contract for the bare ``provider: custom`` shape (section 3b),
|
|
where the fallback would otherwise be the single active model."""
|
|
_seed_custom_model_cache(monkeypatch, _LOCAL_CATALOG)
|
|
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
fetched = []
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.fetch_api_models",
|
|
lambda _k, base_url, **_kw: (fetched.append(base_url), None)[1],
|
|
)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="custom",
|
|
current_base_url=_LOCAL_ENDPOINT,
|
|
current_model="omlx-model-1",
|
|
user_providers={},
|
|
custom_providers=[],
|
|
for_picker=True,
|
|
refresh=False,
|
|
probe_custom_providers=False,
|
|
probe_current_custom_provider=False,
|
|
)
|
|
|
|
row = next(p for p in providers if p["slug"] == "custom")
|
|
assert row["models"] == _LOCAL_CATALOG
|
|
assert fetched == []
|
|
|
|
|
|
def test_no_probe_open_without_cache_keeps_configured_models_and_stays_offline(
|
|
monkeypatch,
|
|
):
|
|
"""The #58183 guarantee: a cold cache must not trigger a live probe. The
|
|
row degrades to its configured list rather than stalling on a dead port."""
|
|
_seed_custom_model_cache(monkeypatch, [], age_seconds=10)
|
|
|
|
row, fetched = _no_probe_local_row(
|
|
monkeypatch,
|
|
custom_providers=[
|
|
{
|
|
"name": "Local (127.0.0.1:8000)",
|
|
"base_url": _LOCAL_ENDPOINT,
|
|
"model": "omlx-model-1",
|
|
}
|
|
],
|
|
)
|
|
|
|
assert row is not None
|
|
assert row["models"] == ["omlx-model-1"]
|
|
assert fetched == []
|
|
|
|
|
|
def test_no_probe_open_respects_discover_models_false(monkeypatch):
|
|
"""A user who pinned their catalog must not have it replaced from cache."""
|
|
_seed_custom_model_cache(monkeypatch, _LOCAL_CATALOG)
|
|
|
|
row, fetched = _no_probe_local_row(
|
|
monkeypatch,
|
|
custom_providers=[
|
|
{
|
|
"name": "Local (127.0.0.1:8000)",
|
|
"base_url": _LOCAL_ENDPOINT,
|
|
"model": "pinned-model",
|
|
"models": ["pinned-model"],
|
|
"discover_models": False,
|
|
}
|
|
],
|
|
)
|
|
|
|
assert row is not None
|
|
assert row["models"] == ["pinned-model"]
|
|
assert fetched == []
|
|
|
|
|
|
def test_cached_catalog_is_not_written_back_to_config(monkeypatch):
|
|
"""Only a real probe persists discovered models; a cache hit is already
|
|
the product of the probe that saved it."""
|
|
_seed_custom_model_cache(monkeypatch, _LOCAL_CATALOG)
|
|
saves = []
|
|
monkeypatch.setattr(
|
|
"hermes_cli.model_switch._save_discovered_models_to_config",
|
|
lambda api_url, model_ids, **kwargs: saves.append((api_url, model_ids)),
|
|
)
|
|
|
|
row, _ = _no_probe_local_row(
|
|
monkeypatch,
|
|
custom_providers=[
|
|
{
|
|
"name": "Local (127.0.0.1:8000)",
|
|
"base_url": _LOCAL_ENDPOINT,
|
|
"model": "omlx-model-1",
|
|
}
|
|
],
|
|
)
|
|
|
|
assert row["models"] == _LOCAL_CATALOG
|
|
assert saves == []
|
|
|
|
|
|
def test_keyless_endpoint_with_saved_catalog_still_reads_cache(monkeypatch):
|
|
"""A keyless local server must not be pinned by Hermes' own auto-save.
|
|
|
|
``_save_discovered_models_to_config()`` writes a plain list into
|
|
``models:``, which ``_models_config_is_allowlist()`` reads back as an
|
|
explicit allowlist. Combined with the no-key discovery gate, a keyless
|
|
endpoint (the common local-model-server shape) froze on the catalog of
|
|
its first probe and could never widen again — the exact "lineup changes
|
|
after config was written" case. The cache read must not be subject to the
|
|
probe's network-cost gate.
|
|
"""
|
|
_seed_custom_model_cache(monkeypatch, _LOCAL_CATALOG)
|
|
|
|
row, fetched = _no_probe_local_row(
|
|
monkeypatch,
|
|
custom_providers=[
|
|
{
|
|
"name": "Local (127.0.0.1:8000)",
|
|
"base_url": _LOCAL_ENDPOINT,
|
|
"model": "omlx-model-1",
|
|
# No api_key, and a models: list of the shape our own
|
|
# auto-save writes after a successful probe.
|
|
"models": ["omlx-model-1"],
|
|
}
|
|
],
|
|
)
|
|
|
|
assert row is not None
|
|
assert row["models"] == _LOCAL_CATALOG
|
|
assert fetched == []
|
|
|
|
|
|
def test_keyless_endpoint_with_saved_catalog_is_still_not_probed(monkeypatch):
|
|
"""...but the network-cost gate it rides on must survive intact.
|
|
|
|
The no-key + declared-catalog combination exists to keep Hermes from
|
|
probing an endpoint it cannot authenticate to. Serving that endpoint from
|
|
a warm cache is free; hitting the network is not. With a cold cache and
|
|
live probing fully enabled, this row must still make zero fetches.
|
|
"""
|
|
_seed_custom_model_cache(monkeypatch, []) # cold: only a probe could answer
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
fetched = []
|
|
|
|
def fetch(_api_key, base_url, **_kwargs):
|
|
fetched.append(base_url)
|
|
return ["should-not-be-reached"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
providers = list_authenticated_providers(
|
|
current_provider="nous",
|
|
user_providers={},
|
|
custom_providers=[
|
|
{
|
|
"name": "Local (127.0.0.1:8000)",
|
|
"base_url": _LOCAL_ENDPOINT,
|
|
"model": "omlx-model-1",
|
|
"models": ["omlx-model-1"],
|
|
}
|
|
],
|
|
for_picker=True,
|
|
refresh=False,
|
|
probe_custom_providers=True, # live probing fully enabled
|
|
)
|
|
row = next(
|
|
(p for p in providers if _LOCAL_ENDPOINT in str(p.get("api_url", ""))), None
|
|
)
|
|
|
|
assert row is not None
|
|
assert row["models"] == ["omlx-model-1"]
|
|
assert fetched == []
|
|
|
|
|
|
def test_api_mode_rows_do_not_share_a_cached_catalog(monkeypatch):
|
|
"""Two rows differing only by ``api_mode`` must not share a cache entry.
|
|
|
|
``api_mode`` selects the wire protocol — ``x-api-key`` +
|
|
``anthropic-version`` versus ``Authorization: Bearer`` — so it is part of
|
|
both the picker's group identity and
|
|
``_custom_endpoint_fingerprint()``. The cache read has to pass it through
|
|
or an ``anthropic_messages`` row renders whatever the OpenAI-mode row
|
|
cached against the same base_url.
|
|
"""
|
|
import hermes_cli.models as models_mod
|
|
|
|
openai_catalog = ["gpt-oss-a", "gpt-oss-b"]
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {})
|
|
|
|
fetched = []
|
|
|
|
def fetch(_api_key, base_url, **_kwargs):
|
|
fetched.append(base_url)
|
|
return ["should-not-be-reached"]
|
|
|
|
monkeypatch.setattr("hermes_cli.models.fetch_api_models", fetch)
|
|
|
|
# Only the OpenAI-mode probe (api_mode=None) is on disk.
|
|
fp = models_mod._custom_endpoint_fingerprint("sk-shared", None, None)
|
|
cache = {
|
|
f"custom:{_SHARED_PROXY_URL}": {
|
|
"fp": fp,
|
|
"at": time.time() - 10,
|
|
"models": list(openai_catalog),
|
|
}
|
|
}
|
|
monkeypatch.setattr(models_mod, "_load_provider_models_cache", lambda: cache)
|
|
|
|
def _row(entry):
|
|
providers = list_authenticated_providers(
|
|
current_provider="nous",
|
|
user_providers={},
|
|
custom_providers=[entry],
|
|
for_picker=True,
|
|
refresh=False,
|
|
probe_custom_providers=False,
|
|
probe_current_custom_provider=True,
|
|
)
|
|
return next(
|
|
(p for p in providers if _SHARED_PROXY_URL in str(p.get("api_url", ""))),
|
|
None,
|
|
)
|
|
|
|
anthropic_row = _row(
|
|
{
|
|
"name": "Proxy Anthropic",
|
|
"base_url": _SHARED_PROXY_URL,
|
|
"api_key": "sk-shared",
|
|
"api_mode": "anthropic_messages",
|
|
"model": "claude-via-proxy",
|
|
}
|
|
)
|
|
openai_row = _row(
|
|
{
|
|
"name": "Proxy OpenAI",
|
|
"base_url": _SHARED_PROXY_URL,
|
|
"api_key": "sk-shared",
|
|
"model": "gpt-via-proxy",
|
|
}
|
|
)
|
|
|
|
assert anthropic_row is not None and openai_row is not None
|
|
assert anthropic_row["models"] == ["claude-via-proxy"], (
|
|
"an anthropic_messages row must not render the OpenAI-mode catalog "
|
|
"cached against the same base_url"
|
|
)
|
|
# ...while the row the entry actually belongs to still resolves.
|
|
assert openai_row["models"] == openai_catalog
|
|
assert fetched == []
|
|
|
|
|
|
def test_auto_saved_catalog_round_trips_without_pinning(tmp_path, monkeypatch):
|
|
"""End-to-end: the shape we persist must not read back as a user pin.
|
|
|
|
Guards the whole chain rather than one branch — probe saves a catalog,
|
|
config is reloaded, and the endpoint must still be discoverable. If a
|
|
future change makes the saved shape look like an intentional allowlist
|
|
again, this fails even if the gate logic above is refactored away.
|
|
"""
|
|
import hermes_cli.config as config_mod
|
|
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
cfg_path = tmp_path / "config.yaml"
|
|
cfg_path.write_text(
|
|
"custom_providers:\n"
|
|
f" - name: Local MLX\n base_url: {_LOCAL_ENDPOINT}\n"
|
|
" model: omlx-model-1\n"
|
|
)
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", str(cfg_path), raising=False)
|
|
|
|
_save_discovered_models_to_config(_LOCAL_ENDPOINT, list(_LOCAL_CATALOG))
|
|
|
|
saved = yaml.safe_load(cfg_path.read_text())["custom_providers"][0]
|
|
assert saved["models_discovered"] is True
|
|
assert list(saved["models"]) == _LOCAL_CATALOG
|
|
assert not any(m.startswith("__") for m in saved["models"]), (
|
|
"sentinel keys must never appear inside the user-facing models mapping"
|
|
)
|
|
|
|
# The persisted shape is what the picker will read on the next open. It
|
|
# must not, on a keyless entry, suppress discovery of a wider catalog.
|
|
_seed_custom_model_cache(monkeypatch, [*_LOCAL_CATALOG, "omlx-model-9"])
|
|
row, fetched = _no_probe_local_row(
|
|
monkeypatch, custom_providers=[saved]
|
|
)
|
|
|
|
assert row is not None
|
|
assert row["models"] == [*_LOCAL_CATALOG, "omlx-model-9"], (
|
|
"an auto-saved catalog must not pin the endpoint against a newer "
|
|
"cached lineup"
|
|
)
|
|
assert fetched == []
|
|
|
|
|
|
def test_legacy_sentinel_catalog_still_resolves_and_migrates(tmp_path, monkeypatch):
|
|
"""Old-shape configs (sentinels inside ``models``) keep working.
|
|
|
|
Pre-fix Hermes wrote ``__discovered_model_catalog__: true`` (and
|
|
``__explicit_model_allowlist__``) inside the user-facing ``models``
|
|
mapping. Reading such a config must (a) recognize the catalog as
|
|
discovered — not a user pin, (b) never list the sentinels as model IDs,
|
|
and (c) migrate to the clean entry-level ``models_discovered`` shape on
|
|
the next discovery save.
|
|
"""
|
|
import hermes_cli.config as config_mod
|
|
from hermes_cli.model_switch import (
|
|
_declared_model_ids,
|
|
_entry_models_discovered,
|
|
_models_config_is_allowlist,
|
|
)
|
|
|
|
legacy_entry = {
|
|
"name": "Local MLX",
|
|
"base_url": _LOCAL_ENDPOINT,
|
|
"model": "omlx-model-1",
|
|
"models": {
|
|
"__discovered_model_catalog__": True,
|
|
**{m: {} for m in _LOCAL_CATALOG},
|
|
},
|
|
}
|
|
|
|
# (a) recognized as a discovered catalog, not an allowlist.
|
|
assert _entry_models_discovered(legacy_entry) is True
|
|
assert not _models_config_is_allowlist(
|
|
legacy_entry["models"], _entry_models_discovered(legacy_entry)
|
|
)
|
|
|
|
# (b) sentinels never surface as model IDs.
|
|
assert _declared_model_ids(legacy_entry["models"]) == _LOCAL_CATALOG
|
|
normalized = config_mod._normalize_custom_provider_entry(dict(legacy_entry))
|
|
assert normalized is not None
|
|
assert normalized["models_discovered"] is True
|
|
assert list(normalized["models"]) == _LOCAL_CATALOG
|
|
assert not any(m.startswith("__") for m in normalized["models"])
|
|
|
|
# ...and the picker row built from the legacy entry lists no phantoms.
|
|
_seed_custom_model_cache(monkeypatch, [])
|
|
row, fetched = _no_probe_local_row(
|
|
monkeypatch, custom_providers=[legacy_entry]
|
|
)
|
|
assert row is not None
|
|
assert not any(str(m).startswith("__") for m in row["models"])
|
|
assert fetched == []
|
|
|
|
# (c) the next discovery save rewrites to the clean shape.
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
cfg_path = tmp_path / "config.yaml"
|
|
cfg_path.write_text(
|
|
yaml.safe_dump({"custom_providers": [legacy_entry]})
|
|
)
|
|
monkeypatch.setattr(config_mod, "CONFIG_PATH", str(cfg_path), raising=False)
|
|
|
|
_save_discovered_models_to_config(_LOCAL_ENDPOINT, list(_LOCAL_CATALOG))
|
|
|
|
saved = yaml.safe_load(cfg_path.read_text())["custom_providers"][0]
|
|
assert saved["models_discovered"] is True
|
|
assert list(saved["models"]) == _LOCAL_CATALOG
|
|
assert "__discovered_model_catalog__" not in saved["models"]
|
|
assert "__explicit_model_allowlist__" not in saved["models"]
|