m4
bb9bed82e1
ModelRole collapses to "primary": every role (main, tool selector, memory
agents, subagents, summarizer) resolves to the snapshot's frozen primary
model, per design 6.1/8.3 — users typically configure a single usable LLM,
so compile-time auxiliary bindings were bypassing run snapshots and
mis-attributing usage. Legacy auxiliary keys in stored snapshots, registry
JSON, and thread metadata are tolerated on read and dropped.
BREAKING CHANGE: ThreadModelSelection no longer carries an auxiliary ref;
snapshot selection_hash is computed over {primary, reasoning_effort} only;
ConfigurableModelMiddleware(role="auxiliary") is rejected.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
520 lines
20 KiB
Python
520 lines
20 KiB
Python
"""Tests for ModelRegistryResolver (design doc 4.3, 6.1, 6.4, 6.5, 8.1).
|
|
|
|
Covers ``resolve``/``resolve_for_test`` (provider, model, contract,
|
|
verification five-tuple, limits, and budget validation producing a frozen
|
|
``ResolvedModelConfig``) and ``compute_availability`` (the section 4.3
|
|
six-state judgement order, the single availability computation).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from EvoScientist.model_registry.adapters import adapter_specs, find_adapter_spec
|
|
from EvoScientist.model_registry.errors import (
|
|
ADAPTER_NOT_SUPPORTED,
|
|
AUTH_MODE_UNSUPPORTED,
|
|
CONTEXT_BUDGET_UNSATISFIABLE,
|
|
CREDENTIAL_NOT_CONFIGURED,
|
|
MODEL_DISABLED,
|
|
MODEL_LIMITS_UNCONFIRMED,
|
|
MODEL_NOT_AVAILABLE,
|
|
MODEL_NOT_FOUND,
|
|
ModelRegistryError,
|
|
)
|
|
from EvoScientist.model_registry.hashing import configuration_hash
|
|
from EvoScientist.model_registry.resolver import ModelRegistryResolver
|
|
from EvoScientist.model_registry.schemas import (
|
|
CredentialWrite,
|
|
ModelRef,
|
|
RegistryV4,
|
|
)
|
|
from EvoScientist.model_registry.store import ModelRuntimeStore
|
|
|
|
ZHIPU_REF = ModelRef(provider_id="zhipu-glm", model_key="glm-5.2")
|
|
OLLAMA_REF = ModelRef(provider_id="local-ollama", model_key="qwen3")
|
|
SECRET = "sk-live-9876abcd"
|
|
VERIFIED_AT = 1_750_000_000 # 2025-06-15T15:06:40Z
|
|
|
|
|
|
def _model_runtime(**overrides):
|
|
payload = {
|
|
"limit_mode": "combined",
|
|
"context_window_tokens": 1048576,
|
|
"max_input_tokens": None,
|
|
"max_output_tokens": 32768,
|
|
"min_effective_input_tokens": 8192,
|
|
"fixed_system_reserve_tokens": 4096,
|
|
"fixed_tools_reserve_tokens": 8192,
|
|
"fixed_attachments_reserve_tokens": 4096,
|
|
"limits_status": "confirmed",
|
|
"limits_source": "provider",
|
|
"temperature": None,
|
|
"top_p": None,
|
|
"reasoning_effort": "auto",
|
|
"declared_capabilities": {
|
|
"tools": True,
|
|
"vision": False,
|
|
"structured_output": True,
|
|
},
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
def _zhipu_provider(**overrides):
|
|
payload = {
|
|
"id": "zhipu-glm",
|
|
"name": "Zhipu GLM",
|
|
"adapter": "openai-compatible",
|
|
"base_url": "https://open.bigmodel.cn/api/paas/v4",
|
|
"auth": {"mode": "api_key", "credential_id": "zhipu-primary"},
|
|
"enabled": True,
|
|
"runtime": {
|
|
"timeout_seconds": 120,
|
|
"max_retries": 2,
|
|
"default_temperature": 0.7,
|
|
"default_top_p": 0.95,
|
|
"default_reasoning_effort": "auto",
|
|
},
|
|
"models": [
|
|
{
|
|
"key": "glm-5.2",
|
|
"name": "GLM-5.2",
|
|
"upstream_model_id": "glm-5.2",
|
|
"enabled": True,
|
|
"runtime": _model_runtime(),
|
|
}
|
|
],
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
def _ollama_provider(**overrides):
|
|
payload = {
|
|
"id": "local-ollama",
|
|
"name": "Local Ollama",
|
|
"adapter": "ollama",
|
|
"base_url": "http://localhost:11434",
|
|
"auth": {"mode": "none", "credential_id": None},
|
|
"enabled": True,
|
|
"runtime": {"timeout_seconds": 120, "max_retries": 2},
|
|
"models": [
|
|
{
|
|
"key": "qwen3",
|
|
"name": "Qwen3",
|
|
"upstream_model_id": "qwen3",
|
|
"enabled": True,
|
|
"runtime": _model_runtime(),
|
|
}
|
|
],
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
def _registry(providers=None, **overrides) -> RegistryV4:
|
|
payload = {
|
|
"version": 4,
|
|
"revision": 1,
|
|
"state": "bootstrap",
|
|
"defaults": {"primary": None},
|
|
"providers": [_zhipu_provider()] if providers is None else providers,
|
|
}
|
|
payload.update(overrides)
|
|
return RegistryV4.model_validate(payload)
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path):
|
|
return ModelRuntimeStore(config_dir=tmp_path)
|
|
|
|
|
|
def _activate(store, registry, *, credential=True):
|
|
writes = (
|
|
[CredentialWrite(credential_id="zhipu-primary", secret_value=SECRET)]
|
|
if credential
|
|
else None
|
|
)
|
|
return store.save_registry(
|
|
expected_revision=1, registry=registry, credential_writes=writes
|
|
)
|
|
|
|
|
|
def _verify(
|
|
store,
|
|
registry,
|
|
provider_id="zhipu-glm",
|
|
model_key="glm-5.2",
|
|
*,
|
|
result="passed",
|
|
config_hash=None,
|
|
credential_revision=None,
|
|
adapter_spec_revision=None,
|
|
caps=None,
|
|
):
|
|
provider = registry.find_provider(provider_id)
|
|
model = provider.find_model(model_key)
|
|
spec = find_adapter_spec(provider.adapter, model.upstream_model_id)
|
|
if credential_revision is None:
|
|
credential_revision = 1 if provider.auth.credential_id else 0
|
|
store.record_model_verification(
|
|
provider_id=provider_id,
|
|
model_key=model_key,
|
|
configuration_hash=config_hash or configuration_hash(provider, model),
|
|
credential_revision=credential_revision,
|
|
adapter_spec_revision=adapter_spec_revision or spec.spec_revision,
|
|
result=result,
|
|
verified_capabilities=caps
|
|
or {"tools": True, "vision": False, "structured_output": True},
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def active_store(store):
|
|
registry = _activate(
|
|
store, _registry(providers=[_zhipu_provider(), _ollama_provider()])
|
|
)
|
|
_verify(store, registry)
|
|
return store
|
|
|
|
|
|
@pytest.fixture
|
|
def resolver(active_store):
|
|
return ModelRegistryResolver(active_store)
|
|
|
|
|
|
class TestResolve:
|
|
def test_freezes_complete_resolved_config(self, resolver):
|
|
config = resolver.resolve(ZHIPU_REF, "primary")
|
|
assert config.model_ref == ZHIPU_REF
|
|
assert config.role == "primary"
|
|
assert config.adapter_id == "openai-compatible"
|
|
# The glm-5.2 model-specific contract wins over the generic one.
|
|
assert config.adapter_spec_revision == 1
|
|
assert config.upstream_model_id == "glm-5.2"
|
|
assert config.base_url == "https://open.bigmodel.cn/api/paas/v4"
|
|
assert config.auth_ref.mode == "api_key"
|
|
assert config.auth_ref.credential_id == "zhipu-primary"
|
|
assert config.auth_ref.credential_revision == 1
|
|
assert config.client_options.timeout_seconds == 120
|
|
assert config.client_options.max_retries == 2
|
|
assert config.request_options.max_output_tokens == 32768
|
|
assert config.request_options.temperature == 0.7
|
|
assert config.request_options.top_p == 0.95
|
|
assert config.request_options.reasoning_effort == "auto"
|
|
# Section 6.5: combined mode subtracts the output budget.
|
|
assert config.budget.resolved_input_limit == 1048576 - 32768
|
|
assert config.budget.fixed_reserves.fixed_system_reserve_tokens == 4096
|
|
assert config.budget.fixed_reserves.fixed_tools_reserve_tokens == 8192
|
|
assert config.budget.fixed_reserves.fixed_attachments_reserve_tokens == 4096
|
|
assert config.budget.message_budget == 1048576 - 32768 - 4096
|
|
# protocol AND declared AND verified (glm-5.2 has no vision).
|
|
assert config.effective_capabilities.tools is True
|
|
assert config.effective_capabilities.vision is False
|
|
assert config.effective_capabilities.structured_output is True
|
|
|
|
def test_resolved_config_never_carries_secret(self, resolver):
|
|
config = resolver.resolve(ZHIPU_REF, "primary")
|
|
assert SECRET not in config.model_dump_json()
|
|
|
|
def test_primary_role_is_stamped(self, resolver):
|
|
assert resolver.resolve(ZHIPU_REF, "primary").role == "primary"
|
|
|
|
def test_unknown_role_rejected(self, resolver):
|
|
with pytest.raises(ValueError, match="role"):
|
|
resolver.resolve(ZHIPU_REF, "planner")
|
|
|
|
def test_unknown_provider_is_not_found(self, resolver):
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ModelRef(provider_id="missing", model_key="glm-5.2"))
|
|
assert excinfo.value.code == MODEL_NOT_FOUND
|
|
|
|
def test_unknown_model_is_not_found(self, resolver):
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ModelRef(provider_id="zhipu-glm", model_key="missing"))
|
|
assert excinfo.value.code == MODEL_NOT_FOUND
|
|
|
|
def test_disabled_provider_is_unavailable(self, store):
|
|
registry = _activate(
|
|
store, _registry(providers=[_zhipu_provider(enabled=False)])
|
|
)
|
|
_verify(store, registry)
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ZHIPU_REF)
|
|
assert excinfo.value.code == MODEL_NOT_AVAILABLE
|
|
|
|
def test_disabled_model_is_disabled(self, store):
|
|
provider = _zhipu_provider()
|
|
provider["models"][0]["enabled"] = False
|
|
registry = _activate(store, _registry(providers=[provider]))
|
|
_verify(store, registry)
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ZHIPU_REF)
|
|
assert excinfo.value.code == MODEL_DISABLED
|
|
|
|
def test_unopened_adapter_rejected(self, store):
|
|
provider = _zhipu_provider(adapter="google-genai")
|
|
_activate(store, _registry(providers=[provider]))
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ZHIPU_REF)
|
|
assert excinfo.value.code == ADAPTER_NOT_SUPPORTED
|
|
|
|
def test_no_matching_contract_is_unavailable(self, store):
|
|
registry = _activate(store, _registry(providers=[_ollama_provider()]))
|
|
_verify(store, registry, "local-ollama", "qwen3")
|
|
# Only the glm-5.2 model-specific contract: qwen3 matches nothing.
|
|
specs = tuple(spec for spec in adapter_specs() if spec.model_selector != "*")
|
|
resolver = ModelRegistryResolver(store, specs=specs)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(OLLAMA_REF)
|
|
assert excinfo.value.code == MODEL_NOT_AVAILABLE
|
|
|
|
def test_missing_verification_record_is_unavailable(self, active_store):
|
|
# The active_store fixture recorded no verification for qwen3.
|
|
resolver = ModelRegistryResolver(active_store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(OLLAMA_REF)
|
|
assert excinfo.value.code == MODEL_NOT_AVAILABLE
|
|
|
|
def test_failed_verification_is_unavailable(self, store):
|
|
registry = _activate(store, _registry())
|
|
_verify(store, registry, result="failed")
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ZHIPU_REF)
|
|
assert excinfo.value.code == MODEL_NOT_AVAILABLE
|
|
|
|
def test_stale_configuration_hash_is_unavailable(self, store):
|
|
registry = _activate(store, _registry())
|
|
_verify(store, registry, config_hash="0" * 64)
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ZHIPU_REF)
|
|
assert excinfo.value.code == MODEL_NOT_AVAILABLE
|
|
|
|
def test_credential_rotation_invalidates_verification(self, active_store):
|
|
# Verifed at revision 1; rotating the key moves the pointer to 2.
|
|
active_store.write_credential_version("zhipu-primary", "sk-rotated-2222")
|
|
resolver = ModelRegistryResolver(active_store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ZHIPU_REF)
|
|
assert excinfo.value.code == MODEL_NOT_AVAILABLE
|
|
|
|
def test_unconfirmed_limits_rejected(self, store):
|
|
provider = _zhipu_provider()
|
|
provider["models"][0]["runtime"]["limits_status"] = "needs_confirmation"
|
|
registry = _activate(store, _registry(providers=[provider]))
|
|
_verify(store, registry)
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ZHIPU_REF)
|
|
assert excinfo.value.code == MODEL_LIMITS_UNCONFIRMED
|
|
|
|
def test_budget_unsatisfiable_rejected(self, store):
|
|
provider = _zhipu_provider()
|
|
provider["models"][0]["runtime"] = _model_runtime(
|
|
context_window_tokens=16384,
|
|
max_output_tokens=8192,
|
|
min_effective_input_tokens=8192,
|
|
)
|
|
registry = _activate(store, _registry(providers=[provider]))
|
|
_verify(store, registry)
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ZHIPU_REF)
|
|
assert excinfo.value.code == CONTEXT_BUDGET_UNSATISFIABLE
|
|
|
|
def test_missing_credential_is_rejected(self, store):
|
|
_activate(store, _registry(), credential=False)
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ZHIPU_REF)
|
|
assert excinfo.value.code == CREDENTIAL_NOT_CONFIGURED
|
|
|
|
def test_auth_mode_not_in_contract_rejected(self, store):
|
|
provider = _zhipu_provider(
|
|
auth={"mode": "bearer", "credential_id": "zhipu-primary"}
|
|
)
|
|
_activate(store, _registry(providers=[provider]))
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve(ZHIPU_REF)
|
|
assert excinfo.value.code == AUTH_MODE_UNSUPPORTED
|
|
|
|
def test_mode_none_resolves_without_credential(self, store):
|
|
registry = _activate(store, _registry(providers=[_ollama_provider()]))
|
|
_verify(store, registry, "local-ollama", "qwen3")
|
|
resolver = ModelRegistryResolver(store)
|
|
config = resolver.resolve(OLLAMA_REF, "primary")
|
|
assert config.auth_ref.mode == "none"
|
|
assert config.auth_ref.credential_id is None
|
|
assert config.auth_ref.credential_revision is None
|
|
|
|
|
|
class TestResolveForTest:
|
|
def test_allows_disabled_model(self, store):
|
|
provider = _zhipu_provider()
|
|
provider["models"][0]["enabled"] = False
|
|
registry = _activate(store, _registry(providers=[provider]))
|
|
_verify(store, registry)
|
|
resolver = ModelRegistryResolver(store)
|
|
config = resolver.resolve_for_test(ZHIPU_REF)
|
|
assert config.model_ref == ZHIPU_REF
|
|
|
|
def test_still_enforces_credential(self, store):
|
|
_activate(store, _registry(), credential=False)
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve_for_test(ZHIPU_REF)
|
|
assert excinfo.value.code == CREDENTIAL_NOT_CONFIGURED
|
|
|
|
def test_still_enforces_disabled_provider(self, store):
|
|
registry = _activate(
|
|
store, _registry(providers=[_zhipu_provider(enabled=False)])
|
|
)
|
|
_verify(store, registry)
|
|
resolver = ModelRegistryResolver(store)
|
|
with pytest.raises(ModelRegistryError) as excinfo:
|
|
resolver.resolve_for_test(ZHIPU_REF)
|
|
assert excinfo.value.code == MODEL_NOT_AVAILABLE
|
|
|
|
|
|
def _record(
|
|
registry,
|
|
provider_id="zhipu-glm",
|
|
model_key="glm-5.2",
|
|
*,
|
|
result="passed",
|
|
config_hash=None,
|
|
credential_revision=1,
|
|
adapter_spec_revision=1,
|
|
verified_at=VERIFIED_AT,
|
|
caps=None,
|
|
error_code=None,
|
|
):
|
|
provider = registry.find_provider(provider_id)
|
|
model = provider.find_model(model_key)
|
|
return {
|
|
"provider_id": provider_id,
|
|
"model_key": model_key,
|
|
"configuration_hash": config_hash or configuration_hash(provider, model),
|
|
"credential_revision": credential_revision,
|
|
"adapter_spec_revision": adapter_spec_revision,
|
|
"result": result,
|
|
"verified_capabilities": caps
|
|
or {"tools": True, "vision": False, "structured_output": True},
|
|
"verified_at": verified_at,
|
|
"error_code": error_code,
|
|
}
|
|
|
|
|
|
class TestComputeAvailability:
|
|
def test_enabled_state(self, store):
|
|
registry = _registry()
|
|
resolver = ModelRegistryResolver(store)
|
|
records = [_record(registry)]
|
|
(availability,) = resolver.compute_availability(
|
|
registry, records, credential_revisions={"zhipu-primary": 1}
|
|
)
|
|
assert availability.model_ref == ZHIPU_REF
|
|
assert availability.state == "enabled"
|
|
assert availability.selectable is True
|
|
assert availability.reason_code is None
|
|
assert availability.verification.status == "passed"
|
|
assert availability.verification.verified_at == "2025-06-15T15:06:40Z"
|
|
assert availability.verification.adapter_spec_revision == 1
|
|
assert availability.effective_capabilities.tools is True
|
|
assert availability.effective_capabilities.vision is False
|
|
assert availability.effective_capabilities.structured_output is True
|
|
|
|
def test_verified_when_model_disabled(self, store):
|
|
provider = _zhipu_provider()
|
|
provider["models"][0]["enabled"] = False
|
|
registry = _registry(providers=[provider])
|
|
resolver = ModelRegistryResolver(store)
|
|
(availability,) = resolver.compute_availability(
|
|
registry, [_record(registry)], credential_revisions={"zhipu-primary": 1}
|
|
)
|
|
assert availability.state == "verified"
|
|
assert availability.selectable is False
|
|
assert availability.verification.status == "passed"
|
|
|
|
def test_unavailable_when_provider_disabled(self, store):
|
|
registry = _registry(providers=[_zhipu_provider(enabled=False)])
|
|
resolver = ModelRegistryResolver(store)
|
|
(availability,) = resolver.compute_availability(
|
|
registry, [_record(registry)], credential_revisions={"zhipu-primary": 1}
|
|
)
|
|
assert availability.state == "unavailable"
|
|
assert availability.selectable is False
|
|
assert availability.reason_code == "PROVIDER_DISABLED"
|
|
|
|
def test_unavailable_without_matching_contract(self, store):
|
|
registry = _registry(providers=[_ollama_provider()])
|
|
specs = tuple(spec for spec in adapter_specs() if spec.model_selector != "*")
|
|
resolver = ModelRegistryResolver(store, specs=specs)
|
|
(availability,) = resolver.compute_availability(
|
|
registry,
|
|
[_record(registry, "local-ollama", "qwen3", credential_revision=0)],
|
|
)
|
|
assert availability.state == "unavailable"
|
|
assert availability.reason_code == "NO_ADAPTER_CONTRACT"
|
|
|
|
def test_verification_failed(self, store):
|
|
registry = _registry()
|
|
resolver = ModelRegistryResolver(store)
|
|
(availability,) = resolver.compute_availability(
|
|
registry,
|
|
[_record(registry, result="failed", error_code="PROVIDER_UNREACHABLE")],
|
|
credential_revisions={"zhipu-primary": 1},
|
|
)
|
|
assert availability.state == "verification_failed"
|
|
assert availability.selectable is False
|
|
assert availability.reason_code == "PROVIDER_UNREACHABLE"
|
|
assert availability.verification.status == "failed"
|
|
assert availability.effective_capabilities.tools is False
|
|
|
|
def test_stale_when_only_old_records_exist(self, store):
|
|
registry = _registry()
|
|
resolver = ModelRegistryResolver(store)
|
|
records = [_record(registry, config_hash="0" * 64)]
|
|
(availability,) = resolver.compute_availability(
|
|
registry, records, credential_revisions={"zhipu-primary": 1}
|
|
)
|
|
# Stale wins over configured when records exist for other five-tuples.
|
|
assert availability.state == "verification_stale"
|
|
assert availability.selectable is False
|
|
assert availability.reason_code == "VERIFICATION_STALE"
|
|
assert availability.verification.status == "stale"
|
|
assert availability.verification.verified_at == "2025-06-15T15:06:40Z"
|
|
|
|
def test_stale_after_credential_rotation(self, store):
|
|
registry = _registry()
|
|
resolver = ModelRegistryResolver(store)
|
|
records = [_record(registry, credential_revision=1)]
|
|
(availability,) = resolver.compute_availability(
|
|
registry, records, credential_revisions={"zhipu-primary": 2}
|
|
)
|
|
assert availability.state == "verification_stale"
|
|
|
|
def test_configured_without_any_records(self, store):
|
|
registry = _registry()
|
|
resolver = ModelRegistryResolver(store)
|
|
(availability,) = resolver.compute_availability(registry, [])
|
|
assert availability.state == "configured"
|
|
assert availability.selectable is False
|
|
assert availability.verification.status == "none"
|
|
assert availability.effective_capabilities.tools is False
|
|
|
|
def test_covers_every_model_of_every_provider(self, store):
|
|
registry = _registry(providers=[_zhipu_provider(), _ollama_provider()])
|
|
resolver = ModelRegistryResolver(store)
|
|
availabilities = resolver.compute_availability(registry, [])
|
|
refs = {availability.model_ref for availability in availabilities}
|
|
assert refs == {ZHIPU_REF, OLLAMA_REF}
|