From af4ae1aef5c43ea50fff606defad9dc278f58fae Mon Sep 17 00:00:00 2001 From: m4 Date: Mon, 20 Jul 2026 22:17:11 +0800 Subject: [PATCH] feat(model-registry): add adapter parameter contracts and build_chat_model factory Add the Task 3 parameter contract layer (design doc 6.1-6.4): - adapters.py: versioned built-in contracts for the five phase-1 adapters plus the openai-compatible/glm-5.2 model-specific contract (verbatim section 6.2 values); exact > longest glob > generic matching with spec_revision pinning; resolve_parameters implementing the section 6.1 inherit/omit semantics, contract validation with stable error codes, and named normalizers (identity, clamp_to_model_limit, omit_when_none, omit_when_auto, reject_non_auto); Adapter.build_request as the single entry point mapping ResolvedModelConfig to {client_options, request_options}; compute_effective_capabilities (protocol AND declared AND verified). - factory.py: build_chat_model(resolved_config, http_client, *, credential=None) with no **kwargs and no setdefault merging; injects the safe HTTP client into ChatOpenAI/ChatAnthropic/ChatOllama, never reads provider API-key environment variables, and strips the OLLAMA_API_KEY authorization header for mode=none adapters. - tests: per-adapter request-capturing fakes plus an httpx.MockTransport outbound capture proving registry resolution matches the wire request. --- EvoScientist/model_registry/__init__.py | 20 + EvoScientist/model_registry/adapters.py | 794 ++++++++++++++++++++++++ EvoScientist/model_registry/factory.py | 135 ++++ tests/test_adapter_contracts.py | 652 +++++++++++++++++++ tests/test_model_factory.py | 312 ++++++++++ 5 files changed, 1913 insertions(+) create mode 100644 EvoScientist/model_registry/adapters.py create mode 100644 EvoScientist/model_registry/factory.py create mode 100644 tests/test_adapter_contracts.py create mode 100644 tests/test_model_factory.py diff --git a/EvoScientist/model_registry/__init__.py b/EvoScientist/model_registry/__init__.py index 13b79ae..18305de 100644 --- a/EvoScientist/model_registry/__init__.py +++ b/EvoScientist/model_registry/__init__.py @@ -9,8 +9,19 @@ for adapters. from __future__ import annotations +from .adapters import ( + Adapter, + BuiltRequest, + ResolvedParameters, + adapter_specs, + compute_effective_capabilities, + find_adapter_spec, + get_adapter, + resolve_parameters, +) from .endpoint_policy import EndpointPolicy from .errors import ERROR_HTTP_STATUS, ErrorDetail, ErrorPayload, ModelRegistryError +from .factory import build_chat_model from .hashing import configuration_hash from .safe_transport import ( AsyncSafeHttpTransport, @@ -45,12 +56,14 @@ from .store import ModelRuntimeStore, SharedStorageError __all__ = [ "ERROR_HTTP_STATUS", + "Adapter", "AdapterParameterSpec", "AsyncSafeHttpTransport", "AsyncSafeNetworkBackend", "AuthConfig", "AuthRef", "AuthSpec", + "BuiltRequest", "Capabilities", "CredentialStatus", "CredentialWrite", @@ -70,11 +83,18 @@ __all__ = [ "ProviderRuntimeConfig", "RegistryV4", "ResolvedModelConfig", + "ResolvedParameters", "SafeHttpTransport", "SafeNetworkBackend", "SharedStorageError", "VerificationInfo", + "adapter_specs", + "build_chat_model", "build_safe_async_http_client", "build_safe_http_client", + "compute_effective_capabilities", "configuration_hash", + "find_adapter_spec", + "get_adapter", + "resolve_parameters", ] diff --git a/EvoScientist/model_registry/adapters.py b/EvoScientist/model_registry/adapters.py new file mode 100644 index 0000000..56ebe8c --- /dev/null +++ b/EvoScientist/model_registry/adapters.py @@ -0,0 +1,794 @@ +"""Adapter parameter contracts and request mapping (design doc 6.1-6.3). + +This module is the single authority for how unified registry parameters are +validated, normalized, and mapped onto LangChain constructor options and +provider request fields: + +- ``adapter_specs`` holds the versioned built-in contracts: a generic + ``model_selector: "*"`` contract for each phase-1 adapter plus the + ``openai-compatible``/``glm-5.2`` model-specific contract. +- ``find_adapter_spec`` implements the matching order — exact + ``upstream_model_id`` first, longest glob next, generic contract last. +- ``resolve_parameters`` applies the section 6.1 inheritance semantics and + the save-time contract checks (stable error codes from section 9.5). +- ``Adapter.build_request`` is the only entry point that turns a frozen + ``ResolvedModelConfig`` into ``{client_options, request_options}``; + parameters the contract does not declare never enter the request. + +A ``target_name`` of ``""`` declares a parameter the contract validates but +enforces outside the request payload (for example ollama retries, which live +in the SafeHttpTransport). Dotted ``client_option`` target names (for example +``client_kwargs.timeout``) build nested option dicts. +""" + +from __future__ import annotations + +import fnmatch +from typing import Any + +from pydantic import BaseModel, ConfigDict + +from .errors import ( + ADAPTER_NOT_SUPPORTED, + AUTH_MODE_UNSUPPORTED, + CAPABILITY_UNSUPPORTED_BY_ADAPTER, + CREDENTIAL_NOT_CONFIGURED, + UNSUPPORTED_RUNTIME_PARAMETER, + ModelRegistryError, +) +from .schemas import ( + AdapterParameterSpec, + AuthSpec, + Capabilities, + ConnectionSpec, + ModelConfig, + ParameterRule, + ProviderConfig, + ReasoningEffort, + ResolvedModelConfig, +) + +SUPPORTED_ADAPTER_IDS = ( + "openai", + "anthropic", + "openai-compatible", + "anthropic-compatible", + "ollama", +) + +# Known adapter IDs that phase 1 deliberately does not open (section 6.2). +UNOPENED_ADAPTER_IDS = ("google-genai", "grok", "openrouter", "nvidia", "antigravity") + +# Unified parameter keys the mapping engine knows how to source (section 6.3). +_UNIFIED_PARAMETERS = ( + "timeout_seconds", + "max_retries", + "max_output_tokens", + "temperature", + "top_p", + "reasoning_effort", +) + +_CAPABILITY_NAMES = ("tools", "vision", "structured_output") + +_REASONING_VALUES = ["low", "medium", "high"] + + +class BuiltRequest(BaseModel): + """The output of ``Adapter.build_request`` (section 6.4).""" + + model_config = ConfigDict(frozen=True) + + client_options: dict[str, Any] + request_options: dict[str, Any] + + +class ResolvedParameters(BaseModel): + """Save-time resolution result; the Resolver freezes it into a snapshot.""" + + model_config = ConfigDict(frozen=True) + + timeout_seconds: int + max_retries: int + max_output_tokens: int + temperature: float | None + top_p: float | None + # "auto" means the field is omitted from the provider request. + reasoning_effort: ReasoningEffort + + +# --- Normalizers (named server-side functions referenced by contracts) --- + + +class _Omit: + """Sentinel: the parameter is omitted from the outbound request.""" + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return "OMIT" + + +OMIT = _Omit() + + +def _reject_parameter(name: str, message: str) -> ModelRegistryError: + return ModelRegistryError( + UNSUPPORTED_RUNTIME_PARAMETER, + message, + details=[{"path": f"runtime.{name}", "code": UNSUPPORTED_RUNTIME_PARAMETER}], + ) + + +def _normalize_identity(name: str, rule: ParameterRule, value: Any) -> Any: + return value + + +def _normalize_clamp_to_model_limit(name: str, rule: ParameterRule, value: Any) -> Any: + if value is None: + return OMIT + if rule.maximum is not None and value > rule.maximum: + return rule.maximum + if rule.minimum is not None and value < rule.minimum: + raise _reject_parameter( + name, + f"{name}={value} is below the contract minimum {rule.minimum}.", + ) + return value + + +def _normalize_omit_when_none(name: str, rule: ParameterRule, value: Any) -> Any: + return OMIT if value is None else value + + +def _normalize_omit_when_auto(name: str, rule: ParameterRule, value: Any) -> Any: + return OMIT if value is None or value == "auto" else value + + +def _normalize_reject_non_auto(name: str, rule: ParameterRule, value: Any) -> Any: + if value is None or value == "auto": + return OMIT + raise _reject_parameter( + name, + f"{name} is not supported by this adapter contract; only the " + "inheriting 'auto' value is accepted.", + ) + + +_NORMALIZERS = { + "identity": _normalize_identity, + "clamp_to_model_limit": _normalize_clamp_to_model_limit, + "omit_when_none": _normalize_omit_when_none, + "omit_when_auto": _normalize_omit_when_auto, + "reject_non_auto": _normalize_reject_non_auto, +} + + +def _check_rule_value(name: str, rule: ParameterRule, value: Any) -> None: + """Type/range/enum validation for a non-null value against the contract. + + With the ``clamp_to_model_limit`` normalizer an out-of-range high value is + clamped by the normalizer instead of rejected, so the maximum check is + left to it. + """ + if rule.value_type == "integer": + if isinstance(value, bool) or not isinstance(value, int): + raise _reject_parameter(name, f"{name} must be an integer.") + elif rule.value_type == "number": + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise _reject_parameter(name, f"{name} must be a number.") + elif rule.value_type == "enum": + if rule.enum_values is not None and value not in rule.enum_values: + raise _reject_parameter(name, f"{name} must be one of {rule.enum_values}.") + if rule.value_type in ("integer", "number"): + if rule.minimum is not None and value < rule.minimum: + raise _reject_parameter( + name, + f"{name}={value} is below the contract minimum {rule.minimum}.", + ) + if ( + rule.normalizer != "clamp_to_model_limit" + and rule.maximum is not None + and value > rule.maximum + ): + raise _reject_parameter( + name, + f"{name}={value} exceeds the contract maximum {rule.maximum}.", + ) + + +# --- Built-in contracts (section 6.2) --- + + +def _connection(chat_model: str) -> ConnectionSpec: + return ConnectionSpec( + chat_model=chat_model, model_field="model", base_url_field="base_url" + ) + + +def _timeout_rule(target_name: str) -> ParameterRule: + return ParameterRule( + supported=True, + value_type="integer", + minimum=10, + maximum=600, + nullable="forbidden", + target="client_option", + target_name=target_name, + normalizer="identity", + ) + + +def _retries_rule(target_name: str) -> ParameterRule: + return ParameterRule( + supported=True, + value_type="integer", + minimum=0, + maximum=5, + nullable="forbidden", + target="client_option", + target_name=target_name, + normalizer="identity", + ) + + +def _max_output_tokens_rule( + target_name: str, maximum: float | None = None +) -> ParameterRule: + return ParameterRule( + supported=True, + value_type="integer", + minimum=1, + maximum=maximum, + nullable="forbidden", + target="request_option", + target_name=target_name, + normalizer="clamp_to_model_limit", + ) + + +def _temperature_rule(maximum: float) -> ParameterRule: + return ParameterRule( + supported=True, + value_type="number", + minimum=0, + maximum=maximum, + nullable="omit", + target="request_option", + target_name="temperature", + normalizer="omit_when_none", + ) + + +def _top_p_rule() -> ParameterRule: + return ParameterRule( + supported=True, + value_type="number", + minimum=0.000001, + maximum=1, + nullable="omit", + target="request_option", + target_name="top_p", + normalizer="omit_when_none", + ) + + +def _reasoning_supported_rule() -> ParameterRule: + return ParameterRule( + supported=True, + value_type="enum", + nullable="omit", + enum_values=list(_REASONING_VALUES), + target="request_option", + target_name="reasoning_effort", + normalizer="omit_when_auto", + ) + + +def _reasoning_unsupported_rule() -> ParameterRule: + return ParameterRule( + supported=False, + value_type="enum", + nullable="omit", + enum_values=list(_REASONING_VALUES), + target="request_option", + target_name="", + normalizer="reject_non_auto", + ) + + +def _api_key_auth() -> AuthSpec: + return AuthSpec( + credential_required=True, + credential_kind="api_key", + target="client_option", + target_name="api_key", + ) + + +def _bearer_auth() -> AuthSpec: + return AuthSpec( + credential_required=True, + credential_kind="bearer_token", + target="request_header", + target_name="Authorization", + ) + + +def _openai_style_parameters() -> dict[str, ParameterRule]: + return { + "timeout_seconds": _timeout_rule("timeout"), + "max_retries": _retries_rule("max_retries"), + "max_output_tokens": _max_output_tokens_rule("max_tokens"), + "temperature": _temperature_rule(2), + "top_p": _top_p_rule(), + "reasoning_effort": _reasoning_supported_rule(), + } + + +def _anthropic_style_parameters() -> dict[str, ParameterRule]: + return { + "timeout_seconds": _timeout_rule("timeout"), + "max_retries": _retries_rule("max_retries"), + "max_output_tokens": _max_output_tokens_rule("max_tokens"), + "temperature": _temperature_rule(1), + "top_p": _top_p_rule(), + "reasoning_effort": _reasoning_unsupported_rule(), + } + + +def _build_builtin_specs() -> tuple[AdapterParameterSpec, ...]: + openai_generic = AdapterParameterSpec( + adapter_id="openai", + spec_revision=1, + model_selector="*", + auth_specs={"api_key": _api_key_auth()}, + parameters=_openai_style_parameters(), + protocol_capabilities=Capabilities( + tools=True, vision=True, structured_output=True + ), + connection=_connection("ChatOpenAI"), + ) + openai_compatible_generic = AdapterParameterSpec( + adapter_id="openai-compatible", + spec_revision=1, + model_selector="*", + auth_specs={"api_key": _api_key_auth(), "bearer": _bearer_auth()}, + parameters=_openai_style_parameters(), + protocol_capabilities=Capabilities( + tools=True, vision=True, structured_output=True + ), + connection=_connection("ChatOpenAI"), + ) + anthropic_generic = AdapterParameterSpec( + adapter_id="anthropic", + spec_revision=1, + model_selector="*", + auth_specs={"api_key": _api_key_auth()}, + parameters=_anthropic_style_parameters(), + protocol_capabilities=Capabilities( + tools=True, vision=True, structured_output=False + ), + connection=_connection("ChatAnthropic"), + ) + anthropic_compatible_generic = anthropic_generic.model_copy( + update={"adapter_id": "anthropic-compatible"} + ) + ollama_generic = AdapterParameterSpec( + adapter_id="ollama", + spec_revision=1, + model_selector="*", + auth_specs={ + "none": AuthSpec( + credential_required=False, + credential_kind="none", + target="adapter_internal", + target_name=None, + ) + }, + parameters={ + "timeout_seconds": _timeout_rule("client_kwargs.timeout"), + # Retries are enforced by the SafeHttpTransport built in Task 2, + # not by an ollama client option; the contract still validates + # the registry value (empty target_name = validated, not sent). + "max_retries": _retries_rule(""), + "max_output_tokens": _max_output_tokens_rule("num_predict"), + "temperature": _temperature_rule(2), + "top_p": _top_p_rule(), + "reasoning_effort": _reasoning_unsupported_rule(), + }, + protocol_capabilities=Capabilities( + tools=True, vision=True, structured_output=True + ), + connection=_connection("ChatOllama"), + ) + # The first model-specific contract (section 6.2 YAML, verbatim values). + glm_52 = AdapterParameterSpec( + adapter_id="openai-compatible", + spec_revision=1, + model_selector="glm-5.2", + auth_specs={"api_key": _api_key_auth()}, + parameters={ + "timeout_seconds": _timeout_rule("timeout"), + "max_retries": _retries_rule("max_retries"), + "max_output_tokens": _max_output_tokens_rule("max_tokens", maximum=32768), + "temperature": _temperature_rule(1), + "top_p": _top_p_rule(), + "reasoning_effort": _reasoning_unsupported_rule(), + }, + protocol_capabilities=Capabilities( + tools=True, vision=False, structured_output=True + ), + connection=_connection("ChatOpenAI"), + ) + return ( + openai_generic, + openai_compatible_generic, + anthropic_generic, + anthropic_compatible_generic, + ollama_generic, + glm_52, + ) + + +_BUILTIN_SPECS: tuple[AdapterParameterSpec, ...] = _build_builtin_specs() + + +def adapter_specs() -> tuple[AdapterParameterSpec, ...]: + """Return the built-in versioned adapter contracts.""" + return _BUILTIN_SPECS + + +def find_adapter_spec( + adapter_id: str, + upstream_model_id: str, + *, + spec_revision: int | None = None, + specs: list[AdapterParameterSpec] | tuple[AdapterParameterSpec, ...] | None = None, +) -> AdapterParameterSpec | None: + """Match a contract: exact ID first, longest glob next, ``*`` last. + + Unopened or unknown adapter IDs raise ``ADAPTER_NOT_SUPPORTED``. ``None`` + is returned when no contract matches (or the pinned ``spec_revision`` is + gone); such configurations may only be saved as ``configured``. + """ + if adapter_id not in SUPPORTED_ADAPTER_IDS: + note = ( + "a known but not yet opened adapter" + if adapter_id in UNOPENED_ADAPTER_IDS + else "an unknown adapter" + ) + raise ModelRegistryError( + ADAPTER_NOT_SUPPORTED, + f"Adapter {adapter_id!r} is {note}; phase 1 supports " + f"{list(SUPPORTED_ADAPTER_IDS)}.", + ) + candidates = [ + spec + for spec in (adapter_specs() if specs is None else specs) + if spec.adapter_id == adapter_id + and (spec_revision is None or spec.spec_revision == spec_revision) + ] + for spec in candidates: + if spec.model_selector == upstream_model_id: + return spec + globs = [ + spec + for spec in candidates + if spec.model_selector != "*" + and fnmatch.fnmatchcase(upstream_model_id, spec.model_selector) + ] + if globs: + return max(globs, key=lambda spec: len(spec.model_selector)) + for spec in candidates: + if spec.model_selector == "*": + return spec + return None + + +def get_adapter( + adapter_id: str, + upstream_model_id: str, + *, + spec_revision: int | None = None, +) -> Adapter: + """Return the matched Adapter, failing loudly when no contract applies.""" + spec = find_adapter_spec(adapter_id, upstream_model_id, spec_revision=spec_revision) + if spec is None: + revision_note = ( + f" at spec_revision {spec_revision}" if spec_revision is not None else "" + ) + raise ModelRegistryError( + ADAPTER_NOT_SUPPORTED, + f"No adapter contract matches {adapter_id!r}/{upstream_model_id!r}" + f"{revision_note}; the configuration may only remain 'configured'.", + ) + return Adapter(spec) + + +def compute_effective_capabilities( + protocol_capabilities: Capabilities, + declared_capabilities: Capabilities, + verified_capabilities: Capabilities, +) -> Capabilities: + """The single capability rule: protocol AND declared AND verified.""" + return Capabilities( + **{ + name: ( + getattr(protocol_capabilities, name) + and getattr(declared_capabilities, name) + and getattr(verified_capabilities, name) + ) + for name in _CAPABILITY_NAMES + } + ) + + +def _resolve_nullable_parameter( + name: str, + rule: ParameterRule, + value: Any, +) -> Any: + """Validate one already-inherited value; returns the value or ``OMIT``.""" + if not rule.supported: + if value is None or value == "auto": + return OMIT + raise _reject_parameter( + name, f"{name} is not supported by this adapter contract." + ) + if value is None or (name == "reasoning_effort" and value == "auto"): + if rule.nullable == "forbidden": + raise _reject_parameter(name, f"{name} must have a concrete value.") + return OMIT + _check_rule_value(name, rule, value) + return _NORMALIZERS[rule.normalizer](name, rule, value) + + +def _resolve_required_parameter( + name: str, + rule: ParameterRule, + value: Any, +) -> Any: + """Validate a registry-sourced value that must always resolve.""" + if not rule.supported: + raise _reject_parameter( + name, f"{name} is not supported by this adapter contract." + ) + if value is None: + raise _reject_parameter(name, f"{name} must have a concrete value.") + _check_rule_value(name, rule, value) + resolved = _NORMALIZERS[rule.normalizer](name, rule, value) + if resolved is OMIT: + raise _reject_parameter(name, f"{name} must have a concrete value.") + return resolved + + +def _parameter_rule(spec: AdapterParameterSpec, name: str) -> ParameterRule: + rule = spec.parameters.get(name) + if rule is None: + raise _reject_parameter( + name, f"Adapter contract {spec.adapter_id!r} does not declare {name}." + ) + return rule + + +def resolve_parameters( + provider: ProviderConfig, + model: ModelConfig, + spec: AdapterParameterSpec, +) -> ResolvedParameters: + """Resolve a model's runtime parameters against the matched contract. + + Applies the section 6.1 inheritance semantics (model value overrides the + provider default; provider ``null`` means the field is omitted, never + zero; ``reasoning_effort=auto`` inherits and is omitted when still auto) + and the save-time contract checks. Raises ``ModelRegistryError`` with a + stable section 9.5 code on any violation. + """ + auth_spec = spec.auth_specs.get(provider.auth.mode) + if auth_spec is None: + raise ModelRegistryError( + AUTH_MODE_UNSUPPORTED, + f"Adapter {spec.adapter_id!r} does not support auth mode " + f"{provider.auth.mode!r}.", + details=[{"path": "auth.mode", "code": AUTH_MODE_UNSUPPORTED}], + ) + if auth_spec.credential_required and provider.auth.credential_id is None: + raise ModelRegistryError( + CREDENTIAL_NOT_CONFIGURED, + f"Auth mode {provider.auth.mode!r} requires a credential reference.", + details=[{"path": "auth.credential_id", "code": CREDENTIAL_NOT_CONFIGURED}], + ) + + for name in _CAPABILITY_NAMES: + if getattr(model.runtime.declared_capabilities, name) and not getattr( + spec.protocol_capabilities, name + ): + raise ModelRegistryError( + CAPABILITY_UNSUPPORTED_BY_ADAPTER, + f"Adapter {spec.adapter_id!r} protocol does not support the " + f"declared capability {name!r}.", + details=[ + { + "path": f"runtime.declared_capabilities.{name}", + "code": CAPABILITY_UNSUPPORTED_BY_ADAPTER, + } + ], + ) + + timeout_seconds = _resolve_required_parameter( + "timeout_seconds", + _parameter_rule(spec, "timeout_seconds"), + provider.runtime.timeout_seconds, + ) + max_retries = _resolve_required_parameter( + "max_retries", + _parameter_rule(spec, "max_retries"), + provider.runtime.max_retries, + ) + max_output_tokens = _resolve_required_parameter( + "max_output_tokens", + _parameter_rule(spec, "max_output_tokens"), + model.runtime.max_output_tokens, + ) + + temperature = model.runtime.temperature + if temperature is None: + temperature = provider.runtime.default_temperature + temperature = _resolve_nullable_parameter( + "temperature", _parameter_rule(spec, "temperature"), temperature + ) + + top_p = model.runtime.top_p + if top_p is None: + top_p = provider.runtime.default_top_p + top_p = _resolve_nullable_parameter("top_p", _parameter_rule(spec, "top_p"), top_p) + + reasoning_effort: ReasoningEffort = model.runtime.reasoning_effort + if reasoning_effort == "auto": + reasoning_effort = provider.runtime.default_reasoning_effort + reasoning_effort = _resolve_nullable_parameter( + "reasoning_effort", + _parameter_rule(spec, "reasoning_effort"), + reasoning_effort, + ) + + resolved = { + "timeout_seconds": timeout_seconds, + "max_retries": max_retries, + "max_output_tokens": max_output_tokens, + "temperature": None if temperature is OMIT else temperature, + "top_p": None if top_p is OMIT else top_p, + "reasoning_effort": ("auto" if reasoning_effort is OMIT else reasoning_effort), + } + _check_conflicts(spec, resolved) + return ResolvedParameters(**resolved) + + +def _check_conflicts(spec: AdapterParameterSpec, resolved: dict[str, Any]) -> None: + present = { + name + for name, value in resolved.items() + if value is not None and value != "auto" + } + for name in present: + rule = spec.parameters.get(name) + if rule is None: + continue + for other in rule.conflicts_with: + if other in present: + raise _reject_parameter( + name, f"{name} conflicts with {other} in this contract." + ) + + +def _assign_option(options: dict[str, Any], dotted_name: str, value: Any) -> None: + parts = dotted_name.split(".") + target = options + for part in parts[:-1]: + existing = target.get(part) + if not isinstance(existing, dict): + existing = {} + target[part] = existing + target = existing + target[parts[-1]] = value + + +class Adapter: + """A versioned parameter contract bound to one ``AdapterParameterSpec``.""" + + def __init__(self, spec: AdapterParameterSpec) -> None: + for name, rule in spec.parameters.items(): + if name not in _UNIFIED_PARAMETERS: + raise ValueError(f"Unknown unified parameter {name!r} in contract.") + if rule.normalizer not in _NORMALIZERS: + raise ValueError(f"Unknown normalizer {rule.normalizer!r}.") + self._spec = spec + + @property + def spec(self) -> AdapterParameterSpec: + return self._spec + + def build_request( + self, + resolved_config: ResolvedModelConfig, + *, + credential: str | None = None, + ) -> BuiltRequest: + """Map a frozen config into ``{client_options, request_options}``. + + This is the only entry point that constructs LangChain parameters and + provider request parameters. Contract validation is re-applied so a + stale snapshot fails loudly instead of silently sending values the + current contract would reject. + """ + spec = self._spec + auth_spec = spec.auth_specs.get(resolved_config.auth_ref.mode) + if auth_spec is None: + raise ModelRegistryError( + AUTH_MODE_UNSUPPORTED, + f"Adapter {spec.adapter_id!r} does not support auth mode " + f"{resolved_config.auth_ref.mode!r}.", + ) + if auth_spec.credential_required and credential is None: + raise ModelRegistryError( + CREDENTIAL_NOT_CONFIGURED, + "A credential is required to build the request but none was " + "resolved for this run.", + ) + + client_options: dict[str, Any] = {} + if spec.connection is not None: + client_options[spec.connection.model_field] = ( + resolved_config.upstream_model_id + ) + client_options[spec.connection.base_url_field] = resolved_config.base_url + + sources: dict[str, Any] = { + "timeout_seconds": resolved_config.client_options.timeout_seconds, + "max_retries": resolved_config.client_options.max_retries, + "max_output_tokens": resolved_config.request_options.max_output_tokens, + "temperature": resolved_config.request_options.temperature, + "top_p": resolved_config.request_options.top_p, + "reasoning_effort": resolved_config.request_options.reasoning_effort, + } + request_options: dict[str, Any] = {} + for name, rule in spec.parameters.items(): + value = _resolve_nullable_parameter(name, rule, sources[name]) + if value is OMIT or rule.target_name == "": + continue + if rule.target == "client_option": + _assign_option(client_options, rule.target_name, value) + elif rule.target == "request_option": + _assign_option(request_options, rule.target_name, value) + else: # extra_body_path + extra_body = request_options.get("extra_body") + if not isinstance(extra_body, dict): + extra_body = {} + request_options["extra_body"] = extra_body + _assign_option(extra_body, rule.target_name, value) + + if resolved_config.auth_ref.mode != "none" and credential is not None: + self._apply_auth(auth_spec, client_options, credential) + + return BuiltRequest( + client_options=client_options, request_options=request_options + ) + + @staticmethod + def _apply_auth( + auth_spec: AuthSpec, client_options: dict[str, Any], credential: str + ) -> None: + if auth_spec.target == "client_option" and auth_spec.target_name: + client_options[auth_spec.target_name] = credential + elif auth_spec.target == "request_header" and auth_spec.target_name: + header_value = ( + f"Bearer {credential}" + if auth_spec.credential_kind == "bearer_token" + else credential + ) + headers = client_options.get("default_headers") + if not isinstance(headers, dict): + headers = {} + client_options["default_headers"] = headers + headers[auth_spec.target_name] = header_value + # adapter_internal credentials are handled by the adapter itself and + # never appear in client or request options. diff --git a/EvoScientist/model_registry/factory.py b/EvoScientist/model_registry/factory.py new file mode 100644 index 0000000..2e06b72 --- /dev/null +++ b/EvoScientist/model_registry/factory.py @@ -0,0 +1,135 @@ +"""build_chat_model: the single LangChain construction entry point (6.4). + +``build_chat_model`` calls ``Adapter.build_request`` for the frozen +``ResolvedModelConfig`` and constructs the LangChain chat model from the +result. There is no second construction entry point: the interface takes no +``**kwargs`` and never merges caller values into snapshot values. + +Credential handling: the resolved secret arrives via the ``credential`` +parameter (resolved from ``auth_ref`` by the Resolver/API layer). This layer +never reads the credential store and never falls back to provider API-key +environment variables such as ``OPENAI_API_KEY`` — a missing credential +fails with ``CREDENTIAL_NOT_CONFIGURED`` before any model is constructed. + +Every builder injects the Task 2 safe HTTP client so provider egress keeps +passing the EndpointPolicy/SSRF defenses: + +- ``ChatOpenAI`` (openai, openai-compatible) takes ``http_client`` directly. +- ``ChatAnthropic`` builds its SDK client from its own ``_client_params`` + plus the safe client (LangChain exposes no constructor hook for it). +- ``ChatOllama`` routes through the safe transport via ``client_kwargs``; + the ollama SDK also reads ``OLLAMA_API_KEY`` from the environment, so the + ``Authorization`` header it may inject is stripped after construction — + a ``mode=none`` adapter must not read, write, or fabricate API keys. +""" + +from __future__ import annotations + +import copy +from collections.abc import Callable +from typing import Any + +import anthropic +import httpx +from langchain_anthropic import ChatAnthropic +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_ollama import ChatOllama +from langchain_openai import ChatOpenAI + +from .adapters import get_adapter +from .schemas import ResolvedModelConfig + +ChatModelBuilder = Callable[[dict[str, Any], httpx.Client], BaseChatModel] + + +def _build_chat_openai( + options: dict[str, Any], http_client: httpx.Client +) -> BaseChatModel: + return ChatOpenAI(**options, http_client=http_client) + + +def _build_chat_anthropic( + options: dict[str, Any], http_client: httpx.Client +) -> BaseChatModel: + model = ChatAnthropic(**options) + # ChatAnthropic exposes no http_client constructor argument; it builds + # ``_client`` (a cached_property) from ``_client_params``. Seeding the + # cached property with an SDK client wrapped around the safe transport + # keeps anthropic egress inside the EndpointPolicy defenses. + client_params = model._client_params + model.__dict__["_client"] = anthropic.Client( + **client_params, http_client=http_client + ) + return model + + +def _transport_of(http_client: httpx.Client) -> httpx.BaseTransport: + transport = getattr(http_client, "_transport", None) + if transport is None: # pragma: no cover - defensive + raise TypeError( + "http_client must be an httpx.Client built by build_safe_http_client." + ) + return transport + + +def _build_chat_ollama( + options: dict[str, Any], http_client: httpx.Client +) -> BaseChatModel: + options = copy.deepcopy(options) + client_kwargs = dict(options.get("client_kwargs") or {}) + # The ollama SDK forwards client_kwargs to its internal httpx client, so + # the safe transport (URL + IP layers) carries over; retries live in the + # transport built by Task 2. + client_kwargs["transport"] = _transport_of(http_client) + options["client_kwargs"] = client_kwargs + model = ChatOllama(**options) + # ollama-python silently adds an Authorization header from OLLAMA_API_KEY. + # Phase-1 ollama contracts only allow auth mode "none", which must not + # read, write, or fabricate API keys — strip any injected header. + for ollama_client in (model._client, model._async_client): + ollama_client._client.headers.pop("authorization", None) + return model + + +# chat_model name (from the contract's connection spec) -> builder. Tests +# substitute fakes here to capture construction arguments per adapter. +CHAT_MODEL_BUILDERS: dict[str, ChatModelBuilder] = { + "ChatOpenAI": _build_chat_openai, + "ChatAnthropic": _build_chat_anthropic, + "ChatOllama": _build_chat_ollama, +} + + +def build_chat_model( + resolved_config: ResolvedModelConfig, + http_client: httpx.Client, + *, + credential: str | None = None, +) -> BaseChatModel: + """Construct the LangChain chat model for a frozen run configuration. + + The snapshot's ``adapter_spec_revision`` pins the contract: if that + revision no longer exists the call fails with ``ADAPTER_NOT_SUPPORTED`` + instead of silently substituting a newer contract. + """ + if not isinstance(resolved_config, ResolvedModelConfig): + raise TypeError( + "resolved_config must be a ResolvedModelConfig, got " + f"{type(resolved_config).__name__}." + ) + if not isinstance(http_client, httpx.Client): + raise TypeError( + f"http_client must be an httpx.Client, got {type(http_client).__name__}." + ) + adapter = get_adapter( + resolved_config.adapter_id, + resolved_config.upstream_model_id, + spec_revision=resolved_config.adapter_spec_revision, + ) + built = adapter.build_request(resolved_config, credential=credential) + options = {**built.client_options, **built.request_options} + connection = adapter.spec.connection + if connection is None: # pragma: no cover - built-in contracts always set it + raise TypeError(f"Adapter {adapter.spec.adapter_id!r} has no connection spec.") + builder = CHAT_MODEL_BUILDERS[connection.chat_model] + return builder(options, http_client) diff --git a/tests/test_adapter_contracts.py b/tests/test_adapter_contracts.py new file mode 100644 index 0000000..433b844 --- /dev/null +++ b/tests/test_adapter_contracts.py @@ -0,0 +1,652 @@ +"""Tests for the adapter parameter contracts (design doc sections 6.1-6.3). + +Covers the built-in contract registry (five phase-1 adapters plus the +``openai-compatible``/``glm-5.2`` model-specific contract), the contract +matching order, save-time parameter resolution with the section 6.1 +inheritance semantics, and ``Adapter.build_request`` mapping. +""" + +from __future__ import annotations + +import pytest + +from EvoScientist.model_registry.adapters import ( + Adapter, + adapter_specs, + compute_effective_capabilities, + find_adapter_spec, + get_adapter, + resolve_parameters, +) +from EvoScientist.model_registry.errors import ( + ADAPTER_NOT_SUPPORTED, + AUTH_MODE_UNSUPPORTED, + CAPABILITY_UNSUPPORTED_BY_ADAPTER, + CREDENTIAL_NOT_CONFIGURED, + UNSUPPORTED_RUNTIME_PARAMETER, + ModelRegistryError, +) +from EvoScientist.model_registry.schemas import ( + AdapterParameterSpec, + Capabilities, + ModelConfig, + ModelRuntimeConfig, + ProviderConfig, + ResolvedModelConfig, +) + +PHASE_ONE_ADAPTERS = [ + "openai", + "anthropic", + "openai-compatible", + "anthropic-compatible", + "ollama", +] + +UNOPENED_ADAPTERS = ["google-genai", "grok", "openrouter", "nvidia", "antigravity"] + + +def _model_runtime(**overrides): + payload = { + "limit_mode": "combined", + "context_window_tokens": 1048576, + "max_input_tokens": None, + "max_output_tokens": 32768, + "min_effective_input_tokens": 8192, + "fixed_system_reserve_tokens": 4096, + "fixed_tools_reserve_tokens": 8192, + "fixed_attachments_reserve_tokens": 4096, + "limits_status": "confirmed", + "limits_source": "provider", + "temperature": None, + "top_p": None, + "reasoning_effort": "auto", + "declared_capabilities": { + "tools": True, + "vision": False, + "structured_output": True, + }, + } + payload.update(overrides) + return payload + + +def _provider(**overrides): + payload = { + "id": "zhipu-glm", + "name": "Zhipu GLM", + "adapter": "openai-compatible", + "base_url": "https://open.bigmodel.cn/api/paas/v4", + "auth": {"mode": "api_key", "credential_id": "zhipu-primary"}, + "enabled": True, + "runtime": { + "timeout_seconds": 120, + "max_retries": 2, + "default_temperature": 0.7, + "default_top_p": 0.95, + "default_reasoning_effort": "auto", + }, + "models": [ + { + "key": "glm-5.2", + "name": "GLM-5.2", + "upstream_model_id": "glm-5.2", + "enabled": True, + "runtime": _model_runtime(), + } + ], + } + payload.update(overrides) + return ProviderConfig.model_validate(payload) + + +def _model(provider: ProviderConfig, **runtime_overrides) -> ModelConfig: + model = provider.models[0] + if not runtime_overrides: + return model + runtime = model.runtime.model_dump() + runtime.update(runtime_overrides) + return model.model_copy( + update={"runtime": ModelRuntimeConfig.model_validate(runtime)} + ) + + +def _resolved_config(**overrides): + payload = { + "model_ref": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, + "role": "primary", + "adapter_id": "openai-compatible", + "adapter_spec_revision": 1, + "upstream_model_id": "glm-5.2", + "base_url": "https://open.bigmodel.cn/api/paas/v4", + "auth_ref": { + "mode": "api_key", + "credential_id": "zhipu-primary", + "credential_revision": 1, + }, + "client_options": {"timeout_seconds": 120, "max_retries": 2}, + "request_options": { + "max_output_tokens": 32768, + "temperature": 0.7, + "top_p": 0.95, + "reasoning_effort": "auto", + }, + "budget": { + "resolved_input_limit": 1015808, + "fixed_reserves": { + "fixed_system_reserve_tokens": 4096, + "fixed_tools_reserve_tokens": 8192, + "fixed_attachments_reserve_tokens": 4096, + }, + "message_budget": 1000000, + }, + "effective_capabilities": { + "tools": True, + "vision": False, + "structured_output": True, + }, + } + payload.update(overrides) + return ResolvedModelConfig.model_validate(payload) + + +class TestContractRegistry: + @pytest.mark.parametrize("adapter_id", PHASE_ONE_ADAPTERS) + def test_generic_contract_exists_for_each_phase_one_adapter(self, adapter_id): + spec = find_adapter_spec(adapter_id, "any-model") + assert spec is not None + assert spec.adapter_id == adapter_id + assert spec.model_selector == "*" + assert spec.spec_revision == 1 + assert spec.connection is not None + assert spec.parameters + + def test_glm_contract_matches_design_doc_verbatim(self): + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + assert spec.model_selector == "glm-5.2" + assert spec.spec_revision == 1 + assert spec.connection is not None + assert spec.connection.chat_model == "ChatOpenAI" + assert spec.connection.model_field == "model" + assert spec.connection.base_url_field == "base_url" + auth = spec.auth_specs["api_key"] + assert auth.credential_required is True + assert auth.credential_kind == "api_key" + assert auth.target == "client_option" + assert auth.target_name == "api_key" + + parameters = spec.parameters + timeout = parameters["timeout_seconds"] + assert (timeout.supported, timeout.value_type) == (True, "integer") + assert (timeout.minimum, timeout.maximum) == (10, 600) + assert timeout.nullable == "forbidden" + assert (timeout.target, timeout.target_name) == ("client_option", "timeout") + assert timeout.normalizer == "identity" + + retries = parameters["max_retries"] + assert (retries.minimum, retries.maximum) == (0, 5) + assert (retries.target, retries.target_name) == ( + "client_option", + "max_retries", + ) + + max_output = parameters["max_output_tokens"] + assert (max_output.minimum, max_output.maximum) == (1, 32768) + assert (max_output.target, max_output.target_name) == ( + "request_option", + "max_tokens", + ) + assert max_output.normalizer == "clamp_to_model_limit" + + temperature = parameters["temperature"] + assert (temperature.minimum, temperature.maximum) == (0, 1) + assert temperature.nullable == "omit" + assert temperature.normalizer == "omit_when_none" + + top_p = parameters["top_p"] + assert (top_p.minimum, top_p.maximum) == (0.000001, 1) + + reasoning = parameters["reasoning_effort"] + assert reasoning.supported is False + assert reasoning.nullable == "omit" + assert reasoning.target_name == "" + assert reasoning.normalizer == "reject_non_auto" + + assert spec.protocol_capabilities == Capabilities( + tools=True, vision=False, structured_output=True + ) + + def test_exact_match_beats_generic_contract(self): + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + assert spec.model_selector == "glm-5.2" + + def test_other_models_fall_back_to_generic_contract(self): + spec = find_adapter_spec("openai-compatible", "some-other-model") + assert spec is not None + assert spec.model_selector == "*" + + def test_longest_glob_beats_shorter_glob(self): + extra = [ + spec.model_copy(update={"model_selector": "glm-*"}) + for spec in adapter_specs() + if spec.adapter_id == "openai-compatible" + and spec.model_selector == "glm-5.2" + ] + spec = find_adapter_spec( + "openai-compatible", "glm-5.2-flash", specs=[*extra, *adapter_specs()] + ) + assert spec is not None + assert spec.model_selector == "glm-*" + + @pytest.mark.parametrize("adapter_id", UNOPENED_ADAPTERS) + def test_unopened_adapter_ids_rejected(self, adapter_id): + with pytest.raises(ModelRegistryError) as excinfo: + find_adapter_spec(adapter_id, "any-model") + assert excinfo.value.code == ADAPTER_NOT_SUPPORTED + + def test_unknown_adapter_id_rejected(self): + with pytest.raises(ModelRegistryError) as excinfo: + find_adapter_spec("made-up-adapter", "any-model") + assert excinfo.value.code == ADAPTER_NOT_SUPPORTED + + def test_pinned_spec_revision_must_exist(self): + assert ( + find_adapter_spec("openai-compatible", "glm-5.2", spec_revision=1) + is not None + ) + assert ( + find_adapter_spec("openai-compatible", "glm-5.2", spec_revision=99) is None + ) + with pytest.raises(ModelRegistryError) as excinfo: + get_adapter("openai-compatible", "glm-5.2", spec_revision=99) + assert excinfo.value.code == ADAPTER_NOT_SUPPORTED + + +class TestGlm52Resolution: + def test_design_doc_mapping(self): + provider = _provider() + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + resolved = resolve_parameters(provider, provider.models[0], spec) + assert resolved.timeout_seconds == 120 + assert resolved.max_retries == 2 + assert resolved.max_output_tokens == 32768 + assert resolved.temperature == 0.7 + assert resolved.top_p == 0.95 + assert resolved.reasoning_effort == "auto" + + adapter = Adapter(spec) + request = adapter.build_request(_resolved_config(), credential="test-secret") + assert request.client_options == { + "model": "glm-5.2", + "base_url": "https://open.bigmodel.cn/api/paas/v4", + "timeout": 120, + "max_retries": 2, + "api_key": "test-secret", + } + assert request.request_options == { + "max_tokens": 32768, + "temperature": 0.7, + "top_p": 0.95, + } + + def test_model_value_overrides_provider_default(self): + provider = _provider() + model = _model(provider, temperature=0.4) + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + resolved = resolve_parameters(provider, model, spec) + assert resolved.temperature == 0.4 + + def test_provider_null_omits_field_and_never_sends_zero(self): + provider = _provider() + provider = provider.model_copy( + update={ + "runtime": provider.runtime.model_copy( + update={"default_temperature": None, "default_top_p": None} + ) + } + ) + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + resolved = resolve_parameters(provider, provider.models[0], spec) + assert resolved.temperature is None + assert resolved.top_p is None + + adapter = Adapter(spec) + request = adapter.build_request( + _resolved_config( + request_options={ + "max_output_tokens": 32768, + "temperature": None, + "top_p": None, + "reasoning_effort": "auto", + } + ), + credential="test-secret", + ) + assert "temperature" not in request.request_options + assert "top_p" not in request.request_options + assert request.request_options.get("temperature") != 0 + + def test_reasoning_effort_auto_is_omitted(self): + adapter = get_adapter("openai-compatible", "glm-5.2") + request = adapter.build_request(_resolved_config(), credential="test-secret") + assert all("reasoning" not in key for key in request.request_options) + + def test_reasoning_effort_non_auto_rejected_at_save(self): + provider = _provider() + model = _model(provider, reasoning_effort="high") + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + with pytest.raises(ModelRegistryError) as excinfo: + resolve_parameters(provider, model, spec) + assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER + + def test_inherited_reasoning_effort_non_auto_rejected(self): + provider = _provider() + provider = provider.model_copy( + update={ + "runtime": provider.runtime.model_copy( + update={"default_reasoning_effort": "high"} + ) + } + ) + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + with pytest.raises(ModelRegistryError) as excinfo: + resolve_parameters(provider, provider.models[0], spec) + assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER + + def test_reasoning_effort_non_auto_rejected_at_run(self): + adapter = get_adapter("openai-compatible", "glm-5.2") + resolved = _resolved_config( + request_options={ + "max_output_tokens": 32768, + "temperature": 0.7, + "top_p": 0.95, + "reasoning_effort": "high", + } + ) + with pytest.raises(ModelRegistryError) as excinfo: + adapter.build_request(resolved, credential="test-secret") + assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER + + def test_temperature_above_contract_maximum_rejected(self): + provider = _provider() + model = _model(provider, temperature=1.5) + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + with pytest.raises(ModelRegistryError) as excinfo: + resolve_parameters(provider, model, spec) + assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER + + def test_top_p_below_contract_minimum_rejected(self): + provider = _provider() + model = _model(provider, top_p=0.0000001) + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + with pytest.raises(ModelRegistryError) as excinfo: + resolve_parameters(provider, model, spec) + assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER + + def test_max_output_tokens_clamped_to_model_limit(self): + provider = _provider() + model = _model(provider, max_output_tokens=40000) + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + resolved = resolve_parameters(provider, model, spec) + assert resolved.max_output_tokens == 32768 + + def test_declared_vision_capability_rejected(self): + provider = _provider() + model = _model( + provider, + declared_capabilities={ + "tools": True, + "vision": True, + "structured_output": True, + }, + ) + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + with pytest.raises(ModelRegistryError) as excinfo: + resolve_parameters(provider, model, spec) + assert excinfo.value.code == CAPABILITY_UNSUPPORTED_BY_ADAPTER + + def test_auth_mode_outside_contract_rejected(self): + provider = _provider(auth={"mode": "bearer", "credential_id": "zhipu-primary"}) + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + with pytest.raises(ModelRegistryError) as excinfo: + resolve_parameters(provider, provider.models[0], spec) + assert excinfo.value.code == AUTH_MODE_UNSUPPORTED + + def test_missing_credential_reference_rejected(self): + provider = _provider(auth={"mode": "api_key", "credential_id": None}) + spec = find_adapter_spec("openai-compatible", "glm-5.2") + assert spec is not None + with pytest.raises(ModelRegistryError) as excinfo: + resolve_parameters(provider, provider.models[0], spec) + assert excinfo.value.code == CREDENTIAL_NOT_CONFIGURED + + def test_build_request_requires_credential(self): + adapter = get_adapter("openai-compatible", "glm-5.2") + with pytest.raises(ModelRegistryError) as excinfo: + adapter.build_request(_resolved_config()) + assert excinfo.value.code == CREDENTIAL_NOT_CONFIGURED + + def test_build_request_rejects_unsupported_auth_mode(self): + adapter = get_adapter("openai-compatible", "glm-5.2") + resolved = _resolved_config( + auth_ref={ + "mode": "bearer", + "credential_id": "zhipu-primary", + "credential_revision": 1, + } + ) + with pytest.raises(ModelRegistryError) as excinfo: + adapter.build_request(resolved, credential="test-secret") + assert excinfo.value.code == AUTH_MODE_UNSUPPORTED + + +def _anthropic_models(): + """Anthropic protocol declares no structured_output in phase 1.""" + return [ + { + "key": "glm-5.2", + "name": "GLM-5.2", + "upstream_model_id": "glm-5.2", + "enabled": True, + "runtime": _model_runtime( + declared_capabilities={ + "tools": True, + "vision": False, + "structured_output": False, + } + ), + } + ] + + +class TestGenericContractMappings: + def _resolve_and_build(self, provider: ProviderConfig, credential=None): + model = provider.models[0] + spec = find_adapter_spec(provider.adapter, model.upstream_model_id) + assert spec is not None + resolved = resolve_parameters(provider, model, spec) + config = _resolved_config( + model_ref={"provider_id": provider.id, "model_key": model.key}, + adapter_id=provider.adapter, + adapter_spec_revision=spec.spec_revision, + upstream_model_id=model.upstream_model_id, + base_url=provider.base_url, + auth_ref={ + "mode": provider.auth.mode, + "credential_id": provider.auth.credential_id, + "credential_revision": ( + 1 if provider.auth.credential_id is not None else None + ), + }, + client_options={ + "timeout_seconds": resolved.timeout_seconds, + "max_retries": resolved.max_retries, + }, + request_options={ + "max_output_tokens": resolved.max_output_tokens, + "temperature": resolved.temperature, + "top_p": resolved.top_p, + "reasoning_effort": resolved.reasoning_effort, + }, + ) + return spec, Adapter(spec).build_request(config, credential=credential) + + def test_openai_generic_contract(self): + provider = _provider( + id="openai-main", + adapter="openai", + base_url="https://api.openai.com/v1", + ) + spec, request = self._resolve_and_build(provider, credential="sk-test") + assert spec.connection is not None + assert spec.connection.chat_model == "ChatOpenAI" + assert request.client_options["model"] == "glm-5.2" + assert request.client_options["timeout"] == 120 + assert request.client_options["max_retries"] == 2 + assert request.client_options["api_key"] == "sk-test" + assert request.request_options["max_tokens"] == 32768 + assert request.request_options["temperature"] == 0.7 + + def test_openai_generic_contract_supports_reasoning_effort(self): + provider = _provider( + id="openai-main", + adapter="openai", + base_url="https://api.openai.com/v1", + ) + provider.models[0].runtime.reasoning_effort = "low" + spec = find_adapter_spec("openai", "glm-5.2") + assert spec is not None + resolved = resolve_parameters(provider, provider.models[0], spec) + assert resolved.reasoning_effort == "low" + _, request = self._resolve_and_build(provider, credential="sk-test") + assert request.request_options["reasoning_effort"] == "low" + + def test_anthropic_generic_contract(self): + provider = _provider( + id="anthropic-main", + adapter="anthropic", + base_url="https://api.anthropic.com", + auth={"mode": "api_key", "credential_id": "anthropic-primary"}, + models=_anthropic_models(), + ) + spec, request = self._resolve_and_build(provider, credential="sk-ant") + assert spec.connection is not None + assert spec.connection.chat_model == "ChatAnthropic" + assert request.client_options["max_retries"] == 2 + assert request.client_options["api_key"] == "sk-ant" + assert request.request_options["max_tokens"] == 32768 + assert "reasoning_effort" not in request.request_options + + def test_anthropic_rejects_non_auto_reasoning_effort(self): + provider = _provider( + id="anthropic-main", + adapter="anthropic", + base_url="https://api.anthropic.com", + models=_anthropic_models(), + ) + provider.models[0].runtime.reasoning_effort = "medium" + spec = find_adapter_spec("anthropic", "glm-5.2") + assert spec is not None + with pytest.raises(ModelRegistryError) as excinfo: + resolve_parameters(provider, provider.models[0], spec) + assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER + + def test_openai_compatible_bearer_auth_maps_to_header(self): + provider = _provider( + id="compat", + adapter="openai-compatible", + base_url="https://gateway.example.com/v1", + auth={"mode": "bearer", "credential_id": "compat-token"}, + models=[ + { + "key": "other", + "name": "Other", + "upstream_model_id": "other-model", + "enabled": True, + "runtime": _model_runtime(), + } + ], + ) + spec, request = self._resolve_and_build(provider, credential="tok-123") + assert spec.model_selector == "*" + assert request.client_options["default_headers"] == { + "Authorization": "Bearer tok-123" + } + assert "api_key" not in request.client_options + + def test_anthropic_compatible_generic_contract(self): + provider = _provider( + id="ant-compat", + adapter="anthropic-compatible", + base_url="https://anthropic-gateway.example.com", + models=_anthropic_models(), + ) + spec, request = self._resolve_and_build(provider, credential="sk-ant") + assert spec.connection is not None + assert spec.connection.chat_model == "ChatAnthropic" + assert request.client_options["base_url"] == ( + "https://anthropic-gateway.example.com" + ) + assert request.request_options["max_tokens"] == 32768 + + def test_ollama_generic_contract_needs_no_credential(self): + provider = _provider( + id="local-ollama", + adapter="ollama", + base_url="http://localhost:11434", + auth={"mode": "none", "credential_id": None}, + ) + spec, request = self._resolve_and_build(provider) + assert spec.connection is not None + assert spec.connection.chat_model == "ChatOllama" + assert request.client_options["model"] == "glm-5.2" + assert request.client_options["base_url"] == "http://localhost:11434" + assert request.client_options["client_kwargs"] == {"timeout": 120} + assert request.request_options["num_predict"] == 32768 + assert not any( + "key" in option or "authorization" in option.lower() + for option in request.client_options + ) + + def test_ollama_mode_none_discards_credential(self): + provider = _provider( + id="local-ollama", + adapter="ollama", + base_url="http://localhost:11434", + auth={"mode": "none", "credential_id": None}, + ) + _spec, request = self._resolve_and_build(provider, credential="ignored") + assert "api_key" not in request.client_options + assert "default_headers" not in request.client_options + + +class TestEffectiveCapabilities: + def test_intersection_rule(self): + protocol = Capabilities(tools=True, vision=False, structured_output=True) + declared = Capabilities(tools=True, vision=True, structured_output=True) + verified = Capabilities(tools=True, vision=True, structured_output=False) + effective = compute_effective_capabilities(protocol, declared, verified) + assert effective.tools is True + assert effective.vision is False + assert effective.structured_output is False + + def test_all_true_when_every_layer_agrees(self): + all_true = Capabilities(tools=True, vision=True, structured_output=True) + effective = compute_effective_capabilities(all_true, all_true, all_true) + assert effective == all_true + + def test_adapter_spec_type_is_schema_model(self): + for spec in adapter_specs(): + assert isinstance(spec, AdapterParameterSpec) diff --git a/tests/test_model_factory.py b/tests/test_model_factory.py new file mode 100644 index 0000000..3be5927 --- /dev/null +++ b/tests/test_model_factory.py @@ -0,0 +1,312 @@ +"""Tests for build_chat_model (design doc section 6.4). + +``build_chat_model`` is the single model-construction entry point: it calls +``adapter.build_request`` for the resolved configuration, injects the Task 2 +safe HTTP client, and never falls back to provider API-key environment +variables. Per-adapter fakes capture the construction arguments and request +options to prove the registry resolution matches the outbound request. +""" + +from __future__ import annotations + +import json + +import httpx +import pytest +from langchain_core.messages import HumanMessage + +from EvoScientist.model_registry import factory +from EvoScientist.model_registry.errors import ( + ADAPTER_NOT_SUPPORTED, + CREDENTIAL_NOT_CONFIGURED, + ModelRegistryError, +) +from EvoScientist.model_registry.factory import build_chat_model +from EvoScientist.model_registry.schemas import ResolvedModelConfig + + +def _resolved_config(**overrides): + payload = { + "model_ref": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, + "role": "primary", + "adapter_id": "openai-compatible", + "adapter_spec_revision": 1, + "upstream_model_id": "glm-5.2", + "base_url": "https://open.bigmodel.cn/api/paas/v4", + "auth_ref": { + "mode": "api_key", + "credential_id": "zhipu-primary", + "credential_revision": 1, + }, + "client_options": {"timeout_seconds": 120, "max_retries": 2}, + "request_options": { + "max_output_tokens": 32768, + "temperature": 0.7, + "top_p": 0.95, + "reasoning_effort": "auto", + }, + "budget": { + "resolved_input_limit": 1015808, + "fixed_reserves": { + "fixed_system_reserve_tokens": 4096, + "fixed_tools_reserve_tokens": 8192, + "fixed_attachments_reserve_tokens": 4096, + }, + "message_budget": 1000000, + }, + "effective_capabilities": { + "tools": True, + "vision": False, + "structured_output": True, + }, + } + payload.update(overrides) + return ResolvedModelConfig.model_validate(payload) + + +def _http_client() -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(lambda request: None)) + + +class _FakeChatModel: + """Captures construction arguments instead of opening a connection.""" + + def __init__(self, **kwargs): + self.kwargs = kwargs + + +class _FakeBuilder: + """Records the options and HTTP client a builder receives.""" + + def __init__(self): + self.calls = [] + + def __call__(self, options, http_client): + self.calls.append((dict(options), http_client)) + return _FakeChatModel(**options) + + +@pytest.fixture +def fake_builders(monkeypatch): + builders = {} + for name in ("ChatOpenAI", "ChatAnthropic", "ChatOllama"): + builder = _FakeBuilder() + builders[name] = builder + monkeypatch.setitem(factory.CHAT_MODEL_BUILDERS, name, builder) + return builders + + +class TestInterfaceContract: + def test_rejects_extra_keyword_arguments(self): + with pytest.raises(TypeError): + build_chat_model(_resolved_config(), _http_client(), unexpected_option=True) + + def test_rejects_positional_credential(self): + with pytest.raises(TypeError): + build_chat_model(_resolved_config(), _http_client(), "secret") + + def test_rejects_non_resolved_config(self): + with pytest.raises(TypeError): + build_chat_model({"adapter_id": "openai"}, _http_client()) + + def test_rejects_non_httpx_client(self): + with pytest.raises(TypeError): + build_chat_model(_resolved_config(), object()) + + def test_unknown_adapter_spec_revision_fails_loudly(self): + resolved = _resolved_config(adapter_spec_revision=99) + with pytest.raises(ModelRegistryError) as excinfo: + build_chat_model(resolved, _http_client(), credential="secret") + assert excinfo.value.code == ADAPTER_NOT_SUPPORTED + + +class TestEnvironmentIsolation: + def test_openai_api_key_env_is_never_read(self, monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "env-key") + with pytest.raises(ModelRegistryError) as excinfo: + build_chat_model(_resolved_config(), _http_client()) + assert excinfo.value.code == CREDENTIAL_NOT_CONFIGURED + + def test_explicit_credential_wins_over_env(self, monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "env-key") + model = build_chat_model( + _resolved_config(), _http_client(), credential="real-key" + ) + assert model.openai_api_key.get_secret_value() == "real-key" + + def test_anthropic_api_key_env_is_never_read(self, monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "env-ant-key") + resolved = _resolved_config( + adapter_id="anthropic", + upstream_model_id="claude-x", + base_url="https://api.anthropic.com", + ) + with pytest.raises(ModelRegistryError) as excinfo: + build_chat_model(resolved, _http_client()) + assert excinfo.value.code == CREDENTIAL_NOT_CONFIGURED + + def test_ollama_mode_none_strips_env_authorization(self, monkeypatch): + monkeypatch.setenv("OLLAMA_API_KEY", "ollama-env-secret") + resolved = _resolved_config( + adapter_id="ollama", + upstream_model_id="llama3", + base_url="http://localhost:11434", + auth_ref={"mode": "none", "credential_id": None}, + ) + model = build_chat_model(resolved, _http_client()) + headers = model._client._client.headers + assert "authorization" not in headers + async_headers = model._async_client._client.headers + assert "authorization" not in async_headers + + +class TestPerAdapterConstruction: + def test_openai_compatible_glm_options(self, fake_builders): + client = _http_client() + build_chat_model(_resolved_config(), client, credential="test-secret") + options, seen_client = fake_builders["ChatOpenAI"].calls[0] + assert seen_client is client + assert options == { + "model": "glm-5.2", + "base_url": "https://open.bigmodel.cn/api/paas/v4", + "timeout": 120, + "max_retries": 2, + "api_key": "test-secret", + "max_tokens": 32768, + "temperature": 0.7, + "top_p": 0.95, + } + + def test_openai_adapter(self, fake_builders): + resolved = _resolved_config( + adapter_id="openai", + upstream_model_id="gpt-x", + base_url="https://api.openai.com/v1", + ) + build_chat_model(resolved, _http_client(), credential="sk-test") + options, _ = fake_builders["ChatOpenAI"].calls[0] + assert options["model"] == "gpt-x" + assert options["max_tokens"] == 32768 + assert options["api_key"] == "sk-test" + + def test_anthropic_adapter(self, fake_builders): + resolved = _resolved_config( + adapter_id="anthropic", + upstream_model_id="claude-x", + base_url="https://api.anthropic.com", + ) + client = _http_client() + build_chat_model(resolved, client, credential="sk-ant") + options, seen_client = fake_builders["ChatAnthropic"].calls[0] + assert seen_client is client + assert options["model"] == "claude-x" + assert options["max_tokens"] == 32768 + assert options["api_key"] == "sk-ant" + assert "reasoning_effort" not in options + + def test_anthropic_compatible_adapter(self, fake_builders): + resolved = _resolved_config( + adapter_id="anthropic-compatible", + upstream_model_id="claude-x", + base_url="https://gateway.example.com", + ) + build_chat_model(resolved, _http_client(), credential="sk-ant") + options, _ = fake_builders["ChatAnthropic"].calls[0] + assert options["base_url"] == "https://gateway.example.com" + assert options["max_tokens"] == 32768 + + def test_ollama_adapter(self, fake_builders): + resolved = _resolved_config( + adapter_id="ollama", + upstream_model_id="llama3", + base_url="http://localhost:11434", + auth_ref={"mode": "none", "credential_id": None}, + ) + client = _http_client() + build_chat_model(resolved, client) + options, seen_client = fake_builders["ChatOllama"].calls[0] + assert seen_client is client + assert options["model"] == "llama3" + assert options["base_url"] == "http://localhost:11434" + assert options["num_predict"] == 32768 + assert options["client_kwargs"] == {"timeout": 120} + assert "api_key" not in options + + +class TestSafeHttpClientInjection: + def test_chat_openai_receives_safe_client(self): + client = _http_client() + model = build_chat_model(_resolved_config(), client, credential="key") + assert model.http_client is client + + def test_chat_anthropic_egress_uses_safe_client(self): + client = _http_client() + resolved = _resolved_config( + adapter_id="anthropic", + upstream_model_id="claude-x", + base_url="https://api.anthropic.com", + ) + model = build_chat_model(resolved, client, credential="sk-ant") + assert model._client._client is client + + def test_chat_ollama_egress_uses_safe_transport(self): + client = _http_client() + resolved = _resolved_config( + adapter_id="ollama", + upstream_model_id="llama3", + base_url="http://localhost:11434", + auth_ref={"mode": "none", "credential_id": None}, + ) + model = build_chat_model(resolved, client) + assert model._client._client._transport is client._transport + + +class TestOutboundRequestCapture: + def test_glm_request_matches_resolved_config(self, monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + captured = [] + + def handler(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "glm-5.2", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hello"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 3, + "completion_tokens": 1, + "total_tokens": 4, + }, + }, + ) + + client = httpx.Client(transport=httpx.MockTransport(handler)) + model = build_chat_model(_resolved_config(), client, credential="real-key") + response = model.invoke([HumanMessage(content="hi")]) + assert response.content == "hello" + + assert len(captured) == 1 + request = captured[0] + assert request.url.path == "/api/paas/v4/chat/completions" + assert request.headers["authorization"] == "Bearer real-key" + body = json.loads(request.content) + assert body["model"] == "glm-5.2" + # The contract maps max_output_tokens to the LangChain `max_tokens` + # constructor option; LangChain encodes it as max_completion_tokens + # on the wire. + assert body["max_completion_tokens"] == 32768 + assert "max_tokens" not in body + assert body["temperature"] == 0.7 + assert body["top_p"] == 0.95 + assert "reasoning_effort" not in body + assert "reasoning" not in body