test: prune low-value tests suite-wide (wave 1) — 46,820 → 28,106 test functions
Systematic prune per AGENTS.md test policy, one pass over every major test tree (gateway, hermes_cli, tools, agent, run_agent, plugins, cli, cron, tui_gateway, honcho/openviking, root-level): - DELETE: source-reading tests (read_text/getsource on prod files), change-detector tests (exact catalog counts, model-name snapshots, config version literals), mock-echo tests (assert a mock returns what it was told), assertion-free/trivial tests, near-duplicate parametrizations (boundaries + one representative kept), async/sync twin duplicates, cosmetic within-file variations. - KEEP (mandatory): security/redaction/approval guards, message-role alternation invariants, prompt-caching/deterministic-call-id invariants, issue-number regression tests (deduped), E2E tests. - 6 test files deleted outright (script-style/no-assert or fully redundant); conftest.py, fakes/, fixtures/ untouched. - tests/acp/conftest.py added: autouse fixture stubs the live models.dev/GitHub/Copilot/Anthropic inventory fetches that ACP server tests performed on every session create — test_server.py 147s → 3.4s, and the tests are now genuinely hermetic. - Sleep-based slowness shrunk where safe (codex_ttfb_watchdog, compression_concurrent_fork, etc.); no wall-clock assertion tightened. Verification: full hermetic suite via scripts/run_tests.sh — 2439 files, 31,130 tests passed, 0 failed, 0 flaky retries, 315s wall (baseline: 583s wall, 13,564s subprocess CPU).
This commit is contained in:
@@ -13,56 +13,9 @@ class TestGetDefaultModelForProvider:
|
||||
assert result
|
||||
assert isinstance(result, str)
|
||||
|
||||
def test_openrouter_returns_preferred_silent_default(self):
|
||||
"""OpenRouter has no static catalog (live fetch), but the silent
|
||||
default must still resolve — to the cost-safe preferred model, never
|
||||
the curated list's Anthropic flagship (claude-fable-5)."""
|
||||
from hermes_cli.models import (
|
||||
PREFERRED_SILENT_DEFAULT_MODEL,
|
||||
get_default_model_for_provider,
|
||||
)
|
||||
result = get_default_model_for_provider("openrouter")
|
||||
assert result == PREFERRED_SILENT_DEFAULT_MODEL
|
||||
assert "claude" not in result.lower()
|
||||
|
||||
def test_unknown_provider_returns_empty(self):
|
||||
from hermes_cli.models import get_default_model_for_provider
|
||||
assert get_default_model_for_provider("nonexistent-provider") == ""
|
||||
|
||||
def test_custom_provider_returns_empty(self):
|
||||
"""Custom provider has no model catalog — should return empty."""
|
||||
from hermes_cli.models import get_default_model_for_provider
|
||||
# Custom providers don't have entries in _PROVIDER_MODELS
|
||||
assert get_default_model_for_provider("some-random-custom") == ""
|
||||
|
||||
def test_nous_silent_default_is_not_the_expensive_flagship(self):
|
||||
"""Nous Portal is a metered aggregator whose curated list is ordered
|
||||
most-capable-first, so entry [0] is the priciest flagship
|
||||
(anthropic/claude-fable-5). The silent fallback (provider set, no model)
|
||||
must NOT escalate to it — otherwise an unconfigured profile silently
|
||||
bills the most expensive model. Regression for the billing footgun.
|
||||
"""
|
||||
from hermes_cli.models import (
|
||||
_PROVIDER_MODELS,
|
||||
get_default_model_for_provider,
|
||||
get_preferred_silent_default_model,
|
||||
)
|
||||
|
||||
result = get_default_model_for_provider("nous")
|
||||
assert result, "nous must resolve to a usable default model"
|
||||
assert "opus" not in result.lower(), (
|
||||
f"silent default escalated to an expensive flagship: {result!r}"
|
||||
)
|
||||
assert "claude" not in result.lower(), (
|
||||
f"silent default escalated to an expensive flagship: {result!r}"
|
||||
)
|
||||
assert result != _PROVIDER_MODELS["nous"][0], (
|
||||
"silent default must not be the most-capable/priciest catalog entry"
|
||||
)
|
||||
# The default must resolve through the catalog-label helper and point
|
||||
# at a model that actually exists in the curated catalog.
|
||||
assert result == get_preferred_silent_default_model("nous")
|
||||
assert result in _PROVIDER_MODELS["nous"]
|
||||
|
||||
def test_catalog_label_overrides_constant(self):
|
||||
"""A ``"default": true`` label in the cached catalog manifest wins over
|
||||
@@ -86,75 +39,9 @@ class TestGetDefaultModelForProvider:
|
||||
== "qwen/qwen3.7-max"
|
||||
)
|
||||
|
||||
def test_no_catalog_cache_falls_back_to_constant(self):
|
||||
"""With no cached manifest (fresh install / offline), the in-repo
|
||||
constant is the silent default."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from hermes_cli import models as models_mod
|
||||
|
||||
with patch(
|
||||
"hermes_cli.model_catalog.get_default_model_from_cache",
|
||||
return_value=None,
|
||||
):
|
||||
assert (
|
||||
models_mod.get_preferred_silent_default_model("openrouter")
|
||||
== models_mod.PREFERRED_SILENT_DEFAULT_MODEL
|
||||
)
|
||||
|
||||
def test_stale_label_not_in_catalog_falls_back(self):
|
||||
"""If the labeled default model is no longer in the provider's curated
|
||||
catalog, fall back to entry [0] rather than returning an absent id."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from hermes_cli import models as models_mod
|
||||
|
||||
with patch(
|
||||
"hermes_cli.model_catalog.get_default_model_from_cache",
|
||||
return_value="does-not-exist-model",
|
||||
):
|
||||
result = models_mod.get_default_model_for_provider("nous")
|
||||
assert result == models_mod._PROVIDER_MODELS["nous"][0]
|
||||
|
||||
|
||||
class TestDetectStaticProviderCostSafeDefault:
|
||||
"""detect_static_provider_for_model must apply the same cost-safe default
|
||||
as get_default_model_for_provider when a bare provider name is typed as a
|
||||
model (e.g. ``/model nous``)."""
|
||||
|
||||
def test_bare_nous_does_not_escalate_to_flagship(self):
|
||||
from hermes_cli.models import (
|
||||
_PROVIDER_MODELS,
|
||||
get_default_model_for_provider,
|
||||
detect_static_provider_for_model,
|
||||
)
|
||||
|
||||
result = detect_static_provider_for_model("nous", "openrouter")
|
||||
assert result is not None
|
||||
provider, model = result
|
||||
assert provider == "nous"
|
||||
# Must match the cost-safe silent default, NOT the priciest catalog
|
||||
# entry [0]. Regression: this path returned _PROVIDER_MODELS["nous"][0]
|
||||
# directly, re-introducing the billing footgun on the interactive
|
||||
# ``/model nous`` path.
|
||||
assert model == get_default_model_for_provider("nous")
|
||||
assert "opus" not in model.lower()
|
||||
assert model != _PROVIDER_MODELS["nous"][0]
|
||||
|
||||
def test_provider_without_override_still_uses_first_model(self):
|
||||
"""Providers outside _SILENT_DEFAULT_PROVIDERS are unchanged."""
|
||||
from hermes_cli.models import (
|
||||
_PROVIDER_MODELS,
|
||||
_SILENT_DEFAULT_PROVIDERS,
|
||||
detect_static_provider_for_model,
|
||||
)
|
||||
|
||||
for provider in ("anthropic", "xai"):
|
||||
if provider in _SILENT_DEFAULT_PROVIDERS:
|
||||
continue
|
||||
result = detect_static_provider_for_model(provider, "openrouter")
|
||||
assert result is not None
|
||||
assert result[1] == _PROVIDER_MODELS[provider][0]
|
||||
|
||||
|
||||
class TestGatewayEmptyModelFallback:
|
||||
|
||||
Reference in New Issue
Block a user