"""Tests for ModelRegistryResolver (design doc 4.3, 6.1, 6.4, 6.5, 8.1). Covers ``resolve``/``resolve_for_test`` (provider, model, contract, verification five-tuple, limits, and budget validation producing a frozen ``ResolvedModelConfig``) and ``compute_availability`` (the section 4.3 six-state judgement order, the single availability computation). """ from __future__ import annotations import pytest from EvoScientist.model_registry.adapters import adapter_specs, find_adapter_spec from EvoScientist.model_registry.errors import ( ADAPTER_NOT_SUPPORTED, AUTH_MODE_UNSUPPORTED, CONTEXT_BUDGET_UNSATISFIABLE, CREDENTIAL_NOT_CONFIGURED, MODEL_DISABLED, MODEL_LIMITS_UNCONFIRMED, MODEL_NOT_AVAILABLE, MODEL_NOT_FOUND, ModelRegistryError, ) from EvoScientist.model_registry.hashing import configuration_hash from EvoScientist.model_registry.resolver import ModelRegistryResolver from EvoScientist.model_registry.schemas import ( CredentialWrite, ModelRef, RegistryV4, ) from EvoScientist.model_registry.store import ModelRuntimeStore ZHIPU_REF = ModelRef(provider_id="zhipu-glm", model_key="glm-5.2") OLLAMA_REF = ModelRef(provider_id="local-ollama", model_key="qwen3") SECRET = "sk-live-9876abcd" VERIFIED_AT = 1_750_000_000 # 2025-06-15T15:06:40Z def _model_runtime(**overrides): payload = { "limit_mode": "combined", "context_window_tokens": 1048576, "max_input_tokens": None, "max_output_tokens": 32768, "min_effective_input_tokens": 8192, "fixed_system_reserve_tokens": 4096, "fixed_tools_reserve_tokens": 8192, "fixed_attachments_reserve_tokens": 4096, "limits_status": "confirmed", "limits_source": "provider", "temperature": None, "top_p": None, "reasoning_effort": "auto", "declared_capabilities": { "tools": True, "vision": False, "structured_output": True, }, } payload.update(overrides) return payload def _zhipu_provider(**overrides): payload = { "id": "zhipu-glm", "name": "Zhipu GLM", "adapter": "openai-compatible", "base_url": "https://open.bigmodel.cn/api/paas/v4", "auth": {"mode": "api_key", "credential_id": "zhipu-primary"}, "enabled": True, "runtime": { "timeout_seconds": 120, "max_retries": 2, "default_temperature": 0.7, "default_top_p": 0.95, "default_reasoning_effort": "auto", }, "models": [ { "key": "glm-5.2", "name": "GLM-5.2", "upstream_model_id": "glm-5.2", "enabled": True, "runtime": _model_runtime(), } ], } payload.update(overrides) return payload def _ollama_provider(**overrides): payload = { "id": "local-ollama", "name": "Local Ollama", "adapter": "ollama", "base_url": "http://localhost:11434", "auth": {"mode": "none", "credential_id": None}, "enabled": True, "runtime": {"timeout_seconds": 120, "max_retries": 2}, "models": [ { "key": "qwen3", "name": "Qwen3", "upstream_model_id": "qwen3", "enabled": True, "runtime": _model_runtime(), } ], } payload.update(overrides) return payload def _registry(providers=None, **overrides) -> RegistryV4: payload = { "version": 4, "revision": 1, "state": "bootstrap", "defaults": {"primary": None}, "providers": [_zhipu_provider()] if providers is None else providers, } payload.update(overrides) return RegistryV4.model_validate(payload) @pytest.fixture def store(tmp_path): return ModelRuntimeStore(config_dir=tmp_path) def _activate(store, registry, *, credential=True): writes = ( [CredentialWrite(credential_id="zhipu-primary", secret_value=SECRET)] if credential else None ) return store.save_registry( expected_revision=1, registry=registry, credential_writes=writes ) def _verify( store, registry, provider_id="zhipu-glm", model_key="glm-5.2", *, result="passed", config_hash=None, credential_revision=None, adapter_spec_revision=None, caps=None, ): provider = registry.find_provider(provider_id) model = provider.find_model(model_key) spec = find_adapter_spec(provider.adapter, model.upstream_model_id) if credential_revision is None: credential_revision = 1 if provider.auth.credential_id else 0 store.record_model_verification( provider_id=provider_id, model_key=model_key, configuration_hash=config_hash or configuration_hash(provider, model), credential_revision=credential_revision, adapter_spec_revision=adapter_spec_revision or spec.spec_revision, result=result, verified_capabilities=caps or {"tools": True, "vision": False, "structured_output": True}, ) @pytest.fixture def active_store(store): registry = _activate( store, _registry(providers=[_zhipu_provider(), _ollama_provider()]) ) _verify(store, registry) return store @pytest.fixture def resolver(active_store): return ModelRegistryResolver(active_store) class TestResolve: def test_freezes_complete_resolved_config(self, resolver): config = resolver.resolve(ZHIPU_REF, "primary") assert config.model_ref == ZHIPU_REF assert config.role == "primary" assert config.adapter_id == "openai-compatible" # The glm-5.2 model-specific contract wins over the generic one. assert config.adapter_spec_revision == 1 assert config.upstream_model_id == "glm-5.2" assert config.base_url == "https://open.bigmodel.cn/api/paas/v4" assert config.auth_ref.mode == "api_key" assert config.auth_ref.credential_id == "zhipu-primary" assert config.auth_ref.credential_revision == 1 assert config.client_options.timeout_seconds == 120 assert config.client_options.max_retries == 2 assert config.request_options.max_output_tokens == 32768 assert config.request_options.temperature == 0.7 assert config.request_options.top_p == 0.95 assert config.request_options.reasoning_effort == "auto" # Section 6.5: combined mode subtracts the output budget. assert config.budget.resolved_input_limit == 1048576 - 32768 assert config.budget.fixed_reserves.fixed_system_reserve_tokens == 4096 assert config.budget.fixed_reserves.fixed_tools_reserve_tokens == 8192 assert config.budget.fixed_reserves.fixed_attachments_reserve_tokens == 4096 assert config.budget.message_budget == 1048576 - 32768 - 4096 # protocol AND declared AND verified (glm-5.2 has no vision). assert config.effective_capabilities.tools is True assert config.effective_capabilities.vision is False assert config.effective_capabilities.structured_output is True def test_resolved_config_never_carries_secret(self, resolver): config = resolver.resolve(ZHIPU_REF, "primary") assert SECRET not in config.model_dump_json() def test_primary_role_is_stamped(self, resolver): assert resolver.resolve(ZHIPU_REF, "primary").role == "primary" def test_unknown_role_rejected(self, resolver): with pytest.raises(ValueError, match="role"): resolver.resolve(ZHIPU_REF, "planner") def test_unknown_provider_is_not_found(self, resolver): with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ModelRef(provider_id="missing", model_key="glm-5.2")) assert excinfo.value.code == MODEL_NOT_FOUND def test_unknown_model_is_not_found(self, resolver): with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ModelRef(provider_id="zhipu-glm", model_key="missing")) assert excinfo.value.code == MODEL_NOT_FOUND def test_disabled_provider_is_unavailable(self, store): registry = _activate( store, _registry(providers=[_zhipu_provider(enabled=False)]) ) _verify(store, registry) resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ZHIPU_REF) assert excinfo.value.code == MODEL_NOT_AVAILABLE def test_disabled_model_is_disabled(self, store): provider = _zhipu_provider() provider["models"][0]["enabled"] = False registry = _activate(store, _registry(providers=[provider])) _verify(store, registry) resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ZHIPU_REF) assert excinfo.value.code == MODEL_DISABLED def test_unopened_adapter_rejected(self, store): provider = _zhipu_provider(adapter="google-genai") _activate(store, _registry(providers=[provider])) resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ZHIPU_REF) assert excinfo.value.code == ADAPTER_NOT_SUPPORTED def test_no_matching_contract_is_unavailable(self, store): registry = _activate(store, _registry(providers=[_ollama_provider()])) _verify(store, registry, "local-ollama", "qwen3") # Only the glm-5.2 model-specific contract: qwen3 matches nothing. specs = tuple(spec for spec in adapter_specs() if spec.model_selector != "*") resolver = ModelRegistryResolver(store, specs=specs) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(OLLAMA_REF) assert excinfo.value.code == MODEL_NOT_AVAILABLE def test_missing_verification_record_is_unavailable(self, active_store): # The active_store fixture recorded no verification for qwen3. resolver = ModelRegistryResolver(active_store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(OLLAMA_REF) assert excinfo.value.code == MODEL_NOT_AVAILABLE def test_failed_verification_is_unavailable(self, store): registry = _activate(store, _registry()) _verify(store, registry, result="failed") resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ZHIPU_REF) assert excinfo.value.code == MODEL_NOT_AVAILABLE def test_stale_configuration_hash_is_unavailable(self, store): registry = _activate(store, _registry()) _verify(store, registry, config_hash="0" * 64) resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ZHIPU_REF) assert excinfo.value.code == MODEL_NOT_AVAILABLE def test_credential_rotation_invalidates_verification(self, active_store): # Verifed at revision 1; rotating the key moves the pointer to 2. active_store.write_credential_version("zhipu-primary", "sk-rotated-2222") resolver = ModelRegistryResolver(active_store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ZHIPU_REF) assert excinfo.value.code == MODEL_NOT_AVAILABLE def test_unconfirmed_limits_rejected(self, store): provider = _zhipu_provider() provider["models"][0]["runtime"]["limits_status"] = "needs_confirmation" registry = _activate(store, _registry(providers=[provider])) _verify(store, registry) resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ZHIPU_REF) assert excinfo.value.code == MODEL_LIMITS_UNCONFIRMED def test_budget_unsatisfiable_rejected(self, store): provider = _zhipu_provider() provider["models"][0]["runtime"] = _model_runtime( context_window_tokens=16384, max_output_tokens=8192, min_effective_input_tokens=8192, ) registry = _activate(store, _registry(providers=[provider])) _verify(store, registry) resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ZHIPU_REF) assert excinfo.value.code == CONTEXT_BUDGET_UNSATISFIABLE def test_missing_credential_is_rejected(self, store): _activate(store, _registry(), credential=False) resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ZHIPU_REF) assert excinfo.value.code == CREDENTIAL_NOT_CONFIGURED def test_auth_mode_not_in_contract_rejected(self, store): provider = _zhipu_provider( auth={"mode": "bearer", "credential_id": "zhipu-primary"} ) _activate(store, _registry(providers=[provider])) resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve(ZHIPU_REF) assert excinfo.value.code == AUTH_MODE_UNSUPPORTED def test_mode_none_resolves_without_credential(self, store): registry = _activate(store, _registry(providers=[_ollama_provider()])) _verify(store, registry, "local-ollama", "qwen3") resolver = ModelRegistryResolver(store) config = resolver.resolve(OLLAMA_REF, "primary") assert config.auth_ref.mode == "none" assert config.auth_ref.credential_id is None assert config.auth_ref.credential_revision is None class TestResolveForTest: def test_allows_disabled_model(self, store): provider = _zhipu_provider() provider["models"][0]["enabled"] = False registry = _activate(store, _registry(providers=[provider])) _verify(store, registry) resolver = ModelRegistryResolver(store) config = resolver.resolve_for_test(ZHIPU_REF) assert config.model_ref == ZHIPU_REF def test_still_enforces_credential(self, store): _activate(store, _registry(), credential=False) resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve_for_test(ZHIPU_REF) assert excinfo.value.code == CREDENTIAL_NOT_CONFIGURED def test_still_enforces_disabled_provider(self, store): registry = _activate( store, _registry(providers=[_zhipu_provider(enabled=False)]) ) _verify(store, registry) resolver = ModelRegistryResolver(store) with pytest.raises(ModelRegistryError) as excinfo: resolver.resolve_for_test(ZHIPU_REF) assert excinfo.value.code == MODEL_NOT_AVAILABLE def _record( registry, provider_id="zhipu-glm", model_key="glm-5.2", *, result="passed", config_hash=None, credential_revision=1, adapter_spec_revision=1, verified_at=VERIFIED_AT, caps=None, error_code=None, ): provider = registry.find_provider(provider_id) model = provider.find_model(model_key) return { "provider_id": provider_id, "model_key": model_key, "configuration_hash": config_hash or configuration_hash(provider, model), "credential_revision": credential_revision, "adapter_spec_revision": adapter_spec_revision, "result": result, "verified_capabilities": caps or {"tools": True, "vision": False, "structured_output": True}, "verified_at": verified_at, "error_code": error_code, } class TestComputeAvailability: def test_enabled_state(self, store): registry = _registry() resolver = ModelRegistryResolver(store) records = [_record(registry)] (availability,) = resolver.compute_availability( registry, records, credential_revisions={"zhipu-primary": 1} ) assert availability.model_ref == ZHIPU_REF assert availability.state == "enabled" assert availability.selectable is True assert availability.reason_code is None assert availability.verification.status == "passed" assert availability.verification.verified_at == "2025-06-15T15:06:40Z" assert availability.verification.adapter_spec_revision == 1 assert availability.effective_capabilities.tools is True assert availability.effective_capabilities.vision is False assert availability.effective_capabilities.structured_output is True def test_verified_when_model_disabled(self, store): provider = _zhipu_provider() provider["models"][0]["enabled"] = False registry = _registry(providers=[provider]) resolver = ModelRegistryResolver(store) (availability,) = resolver.compute_availability( registry, [_record(registry)], credential_revisions={"zhipu-primary": 1} ) assert availability.state == "verified" assert availability.selectable is False assert availability.verification.status == "passed" def test_unavailable_when_provider_disabled(self, store): registry = _registry(providers=[_zhipu_provider(enabled=False)]) resolver = ModelRegistryResolver(store) (availability,) = resolver.compute_availability( registry, [_record(registry)], credential_revisions={"zhipu-primary": 1} ) assert availability.state == "unavailable" assert availability.selectable is False assert availability.reason_code == "PROVIDER_DISABLED" def test_unavailable_without_matching_contract(self, store): registry = _registry(providers=[_ollama_provider()]) specs = tuple(spec for spec in adapter_specs() if spec.model_selector != "*") resolver = ModelRegistryResolver(store, specs=specs) (availability,) = resolver.compute_availability( registry, [_record(registry, "local-ollama", "qwen3", credential_revision=0)], ) assert availability.state == "unavailable" assert availability.reason_code == "NO_ADAPTER_CONTRACT" def test_verification_failed(self, store): registry = _registry() resolver = ModelRegistryResolver(store) (availability,) = resolver.compute_availability( registry, [_record(registry, result="failed", error_code="PROVIDER_UNREACHABLE")], credential_revisions={"zhipu-primary": 1}, ) assert availability.state == "verification_failed" assert availability.selectable is False assert availability.reason_code == "PROVIDER_UNREACHABLE" assert availability.verification.status == "failed" assert availability.effective_capabilities.tools is False def test_stale_when_only_old_records_exist(self, store): registry = _registry() resolver = ModelRegistryResolver(store) records = [_record(registry, config_hash="0" * 64)] (availability,) = resolver.compute_availability( registry, records, credential_revisions={"zhipu-primary": 1} ) # Stale wins over configured when records exist for other five-tuples. assert availability.state == "verification_stale" assert availability.selectable is False assert availability.reason_code == "VERIFICATION_STALE" assert availability.verification.status == "stale" assert availability.verification.verified_at == "2025-06-15T15:06:40Z" def test_stale_after_credential_rotation(self, store): registry = _registry() resolver = ModelRegistryResolver(store) records = [_record(registry, credential_revision=1)] (availability,) = resolver.compute_availability( registry, records, credential_revisions={"zhipu-primary": 2} ) assert availability.state == "verification_stale" def test_configured_without_any_records(self, store): registry = _registry() resolver = ModelRegistryResolver(store) (availability,) = resolver.compute_availability(registry, []) assert availability.state == "configured" assert availability.selectable is False assert availability.verification.status == "none" assert availability.effective_capabilities.tools is False def test_covers_every_model_of_every_provider(self, store): registry = _registry(providers=[_zhipu_provider(), _ollama_provider()]) resolver = ModelRegistryResolver(store) availabilities = resolver.compute_availability(registry, []) refs = {availability.model_ref for availability in availabilities} assert refs == {ZHIPU_REF, OLLAMA_REF}