From bb9bed82e1e6890dc381b7cbcd73f7fd94b381a9 Mon Sep 17 00:00:00 2001 From: m4 Date: Thu, 23 Jul 2026 10:18:44 +0800 Subject: [PATCH] feat(runtime)!: remove auxiliary model role, resolve all roles from run snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ModelRole collapses to "primary": every role (main, tool selector, memory agents, subagents, summarizer) resolves to the snapshot's frozen primary model, per design 6.1/8.3 — users typically configure a single usable LLM, so compile-time auxiliary bindings were bypassing run snapshots and mis-attributing usage. Legacy auxiliary keys in stored snapshots, registry JSON, and thread metadata are tolerated on read and dropped. BREAKING CHANGE: ThreadModelSelection no longer carries an auxiliary ref; snapshot selection_hash is computed over {primary, reasoning_effort} only; ConfigurableModelMiddleware(role="auxiliary") is rejected. Co-Authored-By: Claude Opus 4.7 --- EvoScientist/EvoScientist.py | 64 ++------- EvoScientist/cli/agent.py | 2 +- EvoScientist/cli/commands.py | 2 +- EvoScientist/memory/agents/_factory.py | 8 +- EvoScientist/middleware/configurable_model.py | 41 +++++- EvoScientist/middleware/message_budget.py | 24 ++-- EvoScientist/middleware/tool_selector.py | 85 +++++++----- EvoScientist/model_registry/adapters.py | 12 +- EvoScientist/model_registry/http_api.py | 7 +- EvoScientist/model_registry/openapi.json | 54 +++----- EvoScientist/model_registry/resolver.py | 15 ++- EvoScientist/model_registry/runtime.py | 59 +++++---- EvoScientist/model_registry/schemas.py | 15 +-- EvoScientist/model_registry/snapshots.py | 103 +++++++-------- EvoScientist/model_registry/store.py | 37 +++--- EvoScientist/subagents/_factory.py | 15 +-- tests/registry_fixtures.py | 15 +-- tests/test_ask_user.py | 15 +-- tests/test_async_subagent_factory.py | 32 ++--- tests/test_auxiliary_model.py | 122 +++-------------- tests/test_cli_startup_gates.py | 6 +- tests/test_configurable_model_middleware.py | 29 +--- tests/test_context_editing_middleware.py | 14 +- tests/test_message_budget.py | 92 +------------ tests/test_model_registry_http.py | 19 +-- tests/test_model_registry_schemas.py | 7 +- tests/test_model_registry_store.py | 6 +- tests/test_provider_test_api.py | 2 +- tests/test_resolver.py | 9 +- tests/test_runtime_context_middleware.py | 9 +- tests/test_snapshot_runtime.py | 61 +++------ tests/test_snapshots.py | 124 ++++++++++-------- tests/test_tool_selector_middleware.py | 29 ++-- 33 files changed, 444 insertions(+), 690 deletions(-) diff --git a/EvoScientist/EvoScientist.py b/EvoScientist/EvoScientist.py index ad41a99..643e173 100644 --- a/EvoScientist/EvoScientist.py +++ b/EvoScientist/EvoScientist.py @@ -56,19 +56,11 @@ DEFAULT_SKILL_SOURCES = ("/skills/",) _config = None _chat_model = None # Track the (provider_id, model_key, registry_revision) binding of -# _chat_model so the cache invalidates when the registry defaults change. +# _chat_model so the cache invalidates when the registry default changes. # The compile-time binding is only a placeholder — per-run resolution # always comes from the run snapshot via ConfigurableModelMiddleware. _chat_model_key: tuple[str, str, int] | None = None -# Auxiliary model for background/helper LLM calls (memory workers + main-agent -# tool selector). Cached separately from the main model; resolved through the -# registry ``auxiliary`` role mapping and falls back to the main instance when -# no distinct auxiliary default is configured (see -# _ensure_auxiliary_chat_model). -_auxiliary_chat_model = None -_auxiliary_chat_model_key: tuple[str, str, int] | None = None - # Cache MCP tools by the effective config signature to avoid reconnecting # to MCP servers on every `/new` when config is unchanged. _MCP_TOOLS_CACHE_KEY: str | None = None @@ -148,45 +140,13 @@ def _ensure_chat_model(): from .model_registry.runtime import get_snapshot_runtime runtime = get_snapshot_runtime() - primary_ref, _, revision = runtime.registry_defaults() + primary_ref, revision = runtime.registry_default() key = (primary_ref.provider_id, primary_ref.model_key, revision) if _chat_model is None or _chat_model_key != key: _replace_chat_model(runtime.build_default_role_model("primary"), key) return _chat_model -def _ensure_auxiliary_chat_model(): - """Return the auxiliary chat model for background/helper LLM calls. - - Resolves the ``auxiliary`` role through the model registry role mapping - (design doc 6.1): ``registry.defaults.auxiliary ?? defaults.primary``, - constructed via ``build_chat_model`` from the resolved configuration. - - When no auxiliary default is configured — or it matches the primary - default — returns the main ``_ensure_chat_model()`` instance directly, - so no second client is built. Otherwise the model is cached under its - ``(provider_id, model_key, registry_revision)`` key. A bootstrap - registry propagates ``MODEL_REGISTRY_NOT_READY`` from - ``_ensure_chat_model()``. - """ - global _auxiliary_chat_model, _auxiliary_chat_model_key - from .model_registry.runtime import get_snapshot_runtime - - runtime = get_snapshot_runtime() - primary_ref, auxiliary_ref, revision = runtime.registry_defaults() - if auxiliary_ref is None or auxiliary_ref == primary_ref: - return _ensure_chat_model() - key: tuple[str, str, int] = ( - auxiliary_ref.provider_id, - auxiliary_ref.model_key, - revision, - ) - if _auxiliary_chat_model is None or _auxiliary_chat_model_key != key: - _auxiliary_chat_model = runtime.build_default_role_model("auxiliary") - _auxiliary_chat_model_key = key - return _auxiliary_chat_model - - def _compile_time_role_model(role: str = "primary"): """Return the compile-time model binding for graph construction. @@ -693,9 +653,8 @@ def _get_default_middleware( memory_source_agent: Attribution name for profile/observation writes. Async sub-agent factories pass their deployed agent name here. snapshot_role: The model role ``ConfigurableModelMiddleware`` - resolves from the run snapshot (``primary`` for the main agent - and working sub-agents, ``auxiliary`` for unattended helper - agents such as the scheduler). + resolves from the run snapshot; every role maps to the + snapshot's frozen primary (design doc 6.1). """ from .middleware import ( ConfigurableModelMiddleware, @@ -746,19 +705,12 @@ def _get_default_middleware( ), memory_scheduler=memory_scheduler, ) - # Main-agent tool selection may use the auxiliary model; async sub-agents - # keep their own model (they do real work, not a one-off helper call). + # Main-agent tool selection resolves its helper model from the run + # snapshot on every call (model=None below); async sub-agents and the + # pure path (explicit model + config) keep their threaded model. # context_editing stays on the main model — its model only sizes the # context-window trigger for the main agent's own history. - if for_async_subagent: - tool_selector_model = model - elif chat_model is None: - tool_selector_model = _compile_time_role_model("auxiliary") - else: - # Pure path (explicit model + config): the threaded model stands in - # for tool selection at build time; per-call resolution still comes - # from the run snapshot. - tool_selector_model = model + tool_selector_model = None if (not for_async_subagent and chat_model is None) else model mw = [ ConfigurableModelMiddleware(role=snapshot_role), create_message_budget_middleware(model, backend, snapshot_role=snapshot_role), diff --git a/EvoScientist/cli/agent.py b/EvoScientist/cli/agent.py index f8f42b8..9e5aa21 100644 --- a/EvoScientist/cli/agent.py +++ b/EvoScientist/cli/agent.py @@ -73,7 +73,7 @@ def current_model_label() -> str: try: from ..model_registry.runtime import get_snapshot_runtime - primary, _, _ = get_snapshot_runtime().registry_defaults() + primary, _ = get_snapshot_runtime().registry_default() except Exception: return "unconfigured" return f"{primary.provider_id}/{primary.model_key}" diff --git a/EvoScientist/cli/commands.py b/EvoScientist/cli/commands.py index 3509129..e5c1391 100644 --- a/EvoScientist/cli/commands.py +++ b/EvoScientist/cli/commands.py @@ -1374,7 +1374,7 @@ def _startup_gates() -> None: try: from ..model_registry.runtime import get_snapshot_runtime - get_snapshot_runtime().registry_defaults() + get_snapshot_runtime().registry_default() except ModelRegistryError as exc: if exc.code != MODEL_REGISTRY_NOT_READY: raise diff --git a/EvoScientist/memory/agents/_factory.py b/EvoScientist/memory/agents/_factory.py index e0987cb..fae9d7f 100644 --- a/EvoScientist/memory/agents/_factory.py +++ b/EvoScientist/memory/agents/_factory.py @@ -88,6 +88,7 @@ def build_memory_agent_graph( from ...backends import build_memory_agent_backend from ...EvoScientist import _compile_time_role_model + from ...middleware.configurable_model import ConfigurableModelMiddleware kwargs: dict[str, Any] = {} if response_format is not None: @@ -101,11 +102,14 @@ def build_memory_agent_graph( agent = create_deep_agent( name=name, - model=_compile_time_role_model("auxiliary"), + model=_compile_time_role_model("primary"), system_prompt=system_prompt, tools=list(tools), backend=backend, - middleware=list(middleware), + # The compile-time model is only a placeholder: the middleware + # re-resolves every call from the run snapshot (lazily healing + # background runs to a local snapshot of the registry default). + middleware=[ConfigurableModelMiddleware(), *middleware], subagents=[], skills=skills, **kwargs, diff --git a/EvoScientist/middleware/configurable_model.py b/EvoScientist/middleware/configurable_model.py index dda1a27..97e47f9 100644 --- a/EvoScientist/middleware/configurable_model.py +++ b/EvoScientist/middleware/configurable_model.py @@ -154,11 +154,8 @@ class ConfigurableModelMiddleware(AgentMiddleware): """Re-resolve the chat model from the run snapshot on every call. ``role`` selects which frozen configuration of the snapshot feeds this - agent's model calls: ``primary`` for the main agent and working - sub-agents, ``auxiliary`` for unattended helper agents (e.g. the - scheduler). The section 6.1 mapping (``auxiliary``/``summary``/ - ``tool_selector`` → ``snapshot.auxiliary ?? snapshot.primary``) is - applied by the snapshot layer. + agent's model calls; every role currently maps to the snapshot's frozen + primary (section 6.1). A per-instance cache keyed by snapshot ID avoids rebuilding the model on every call within a run; snapshots are immutable once created, so the @@ -266,3 +263,37 @@ class ConfigurableModelMiddleware(AgentMiddleware): # thread-pool overhead is irrelevant once warm. new_model = await asyncio.to_thread(self._resolve) return await handler(request.override(model=new_model)) + + +# --- per-call resolution for in-run helper models ----------------------------- + +# In-run helper LLM calls (the tool selector's internal selection call) do +# not pass through an agent's model request, so ``request.override`` cannot +# reach them. They resolve the run snapshot's primary model through this +# helper instead; background runs without an explicit snapshot are healed +# through the same lazy local-snapshot path as the middleware. +_helper_model_cache: dict[str, Any] = {} +_helper_model_cache_lock = threading.Lock() + + +def resolve_snapshot_model(runtime: SnapshotRuntime | None = None) -> Any: + """Resolve the active run's snapshot primary model (cached per snapshot). + + Uses the same binding rules as ``ConfigurableModelMiddleware``: the + run's explicit ``runtime_snapshot_id`` wins, otherwise a local snapshot + is lazily created for the run's own thread. Must be called from inside + a runnable context (or a test that installed one). + """ + configurable = _current_configurable() + check_no_outside_snapshot_model_config(configurable) + rt = runtime if runtime is not None else get_snapshot_runtime() + snapshot_id, thread_id = ensure_snapshot_binding(configurable, rt) + snapshot = rt.get_snapshot_for_run(snapshot_id, thread_id=thread_id) + with _helper_model_cache_lock: + cached = _helper_model_cache.get(snapshot.snapshot_id) + if cached is not None: + return cached + model = rt.build_role_model(snapshot, "primary") + with _helper_model_cache_lock: + _helper_model_cache[snapshot.snapshot_id] = model + return model diff --git a/EvoScientist/middleware/message_budget.py b/EvoScientist/middleware/message_budget.py index 06b43cc..66931e5 100644 --- a/EvoScientist/middleware/message_budget.py +++ b/EvoScientist/middleware/message_budget.py @@ -10,13 +10,12 @@ Snapshot mode (design doc 6.5, 8.3): every run carries — or lazily creates, via the section 8.1 local entry convention — a run snapshot; the input limit and the three fixed reserves come from the frozen ``ResolvedModelConfig.budget`` of the middleware's own ``snapshot_role`` -(``primary`` for the main agent and working sub-agents, ``auxiliary`` for -unattended helpers such as the scheduler). There is no 32K default and no -compile-time profile fallback: a bootstrap registry fails closed with -``MODEL_REGISTRY_NOT_READY``. The per-call message budget is recomputed on -every invocation from those frozen reserves with the current -``has_tools``/``has_attachments`` mode; it is never carried over from a -previous call: +(every role maps to the snapshot's frozen primary, section 6.1). There is +no 32K default and no compile-time profile fallback: a bootstrap registry +fails closed with ``MODEL_REGISTRY_NOT_READY``. The per-call message budget +is recomputed on every invocation from those frozen reserves with the +current ``has_tools``/``has_attachments`` mode; it is never carried over +from a previous call: message_budget = resolved_input_limit - fixed_system_reserve_tokens @@ -176,10 +175,9 @@ class MessageBudgetMiddleware: agent has tool capability and a configured toolset. Decided once here, never from a single request's bound tool count. snapshot_role: The snapshot role whose frozen limits size this - agent's budget (``primary`` for the main agent and working - sub-agents, ``auxiliary`` for unattended helper agents such - as the scheduler). The summarizer keeps using the snapshot's - ``summary`` role mapping regardless. + agent's budget; every role maps to the snapshot's frozen + primary (section 6.1). The summarizer uses the same frozen + primary. runtime: Snapshot runtime override; defaults to the shared process runtime (tests inject an isolated one). """ @@ -255,14 +253,14 @@ class MessageBudgetMiddleware: @property def model(self) -> Any: # type: ignore[override] - """Summarizer model: the snapshot's ``summary`` role mapping.""" + """Summarizer model: the snapshot's frozen primary.""" snapshot = self._snapshot() with self._model_cache_lock: cached = self._snapshot_model_cache.get(snapshot.snapshot_id) if cached is not None: return cached resolved = self._snapshot_runtime().build_role_model( - snapshot, "summary" + snapshot, "primary" ) with self._model_cache_lock: self._snapshot_model_cache[snapshot.snapshot_id] = resolved diff --git a/EvoScientist/middleware/tool_selector.py b/EvoScientist/middleware/tool_selector.py index 427ad29..1f8edce 100644 --- a/EvoScientist/middleware/tool_selector.py +++ b/EvoScientist/middleware/tool_selector.py @@ -93,15 +93,12 @@ class _ConditionalToolSelectorMiddleware(AgentMiddleware): self._threshold = threshold self._always_include = always_include or frozenset() self._track_stream_selection = track_stream_selection - # Agent tools are fixed after graph construction, so the filtered - # always-include set is stable for this middleware instance. - self._selector: AgentMiddleware | None = None def _build_selector(self, request: ModelRequest) -> AgentMiddleware: - if self._selector is None: - names = _available_always_include(request.tools, self._always_include) - self._selector = self._selector_factory(names) - return self._selector + # Built per call: the selector's helper model is resolved from the + # run snapshot, which varies from run to run. + names = _available_always_include(request.tools, self._always_include) + return self._selector_factory(names) def wrap_model_call( self, @@ -233,8 +230,10 @@ def create_tool_selector_middleware( names for the main-agent stream UI when ``track_stream_selection`` is true Args: - model: Chat model for tool selection. If *None*, the default - model is resolved via ``_ensure_chat_model()``. + model: Chat model for tool selection. If *None*, the helper model + is resolved from the active run's snapshot on every call + (``resolve_snapshot_model``); the registry default only applies + to snapshot-less background runs via the lazy local snapshot. threshold: Minimum number of tools to trigger selection. Default 26. Set to 0 to always run selection. track_stream_selection: Whether to update process-global stream/UI @@ -253,34 +252,50 @@ def create_tool_selector_middleware( from .utils import disable_thinking - if model is None: - from EvoScientist.EvoScientist import _ensure_chat_model + def tag_selector_model(base: BaseChatModel) -> BaseChatModel: + safe_model = disable_thinking(base) + selector_model = safe_model + from EvoScientist.usage.callback import usage_tracking_enabled - model = _ensure_chat_model() - safe_model = disable_thinking(model) - selector_model = safe_model - from EvoScientist.usage.callback import usage_tracking_enabled - - if usage_tracking_enabled(): - try: - selector_model = safe_model.model_copy( - update={ - "metadata": { - **(safe_model.metadata or {}), - "usage_scope": "tool_selector", + if usage_tracking_enabled(): + try: + selector_model = safe_model.model_copy( + update={ + "metadata": { + **(safe_model.metadata or {}), + "usage_scope": "tool_selector", + } } - } - ) - if not isinstance(selector_model, BaseChatModel): - raise TypeError("selector model copy is not a BaseChatModel") - except Exception: - # The model factory preflights this path before enabling its callback. - # Keep the selector operational even for an unusual third-party model. - from EvoScientist.usage.spool import mark_tracking_degraded + ) + if not isinstance(selector_model, BaseChatModel): + raise TypeError("selector model copy is not a BaseChatModel") + except Exception: + # The model factory preflights this path before enabling its + # callback. Keep the selector operational even for an unusual + # third-party model. + from EvoScientist.usage.spool import mark_tracking_degraded - mark_tracking_degraded("selector_model_copy_unsupported") - selector_model = safe_model - logger.exception("Could not attach Tool Selector usage scope") + mark_tracking_degraded("selector_model_copy_unsupported") + selector_model = safe_model + logger.exception("Could not attach Tool Selector usage scope") + return selector_model + + # The resolved snapshot model is cached per snapshot upstream, so the + # tagged copy can be cached by the base model's identity. + tagged_cache: dict[int, BaseChatModel] = {} + + def selector_model() -> BaseChatModel: + if model is not None: + base = model + else: + from .configurable_model import resolve_snapshot_model + + base = resolve_snapshot_model() + tagged = tagged_cache.get(id(base)) + if tagged is None: + tagged = tag_selector_model(base) + tagged_cache[id(base)] = tagged + return tagged system_prompt = ( "You are selecting tools for a scientific research agent. " @@ -293,7 +308,7 @@ def create_tool_selector_middleware( def selector_factory(always_include: list[str]) -> AgentMiddleware: return LLMToolSelectorMiddleware( - model=selector_model, + model=selector_model(), system_prompt=system_prompt, always_include=always_include, ) diff --git a/EvoScientist/model_registry/adapters.py b/EvoScientist/model_registry/adapters.py index 0c0893d..ed58bf8 100644 --- a/EvoScientist/model_registry/adapters.py +++ b/EvoScientist/model_registry/adapters.py @@ -581,6 +581,8 @@ def resolve_parameters( provider: ProviderConfig, model: ModelConfig, spec: AdapterParameterSpec, + *, + reasoning_effort_override: ReasoningEffort | None = None, ) -> ResolvedParameters: """Resolve a model's runtime parameters against the matched contract. @@ -589,6 +591,10 @@ def resolve_parameters( zero; ``reasoning_effort=auto`` inherits and is omitted when still auto) and the save-time contract checks. Raises ``ModelRegistryError`` with a stable section 9.5 code on any violation. + + ``reasoning_effort_override`` (snapshot creation only) replaces the + model's configured effort before inheritance; it flows through the same + contract rule, so unsupported adapters still reject non-``auto`` values. """ auth_spec = spec.auth_specs.get(provider.auth.mode) if auth_spec is None: @@ -649,7 +655,11 @@ def resolve_parameters( top_p = provider.runtime.default_top_p top_p = _resolve_nullable_parameter("top_p", _parameter_rule(spec, "top_p"), top_p) - reasoning_effort: ReasoningEffort = model.runtime.reasoning_effort + reasoning_effort: ReasoningEffort = ( + reasoning_effort_override + if reasoning_effort_override not in (None, "auto") + else model.runtime.reasoning_effort + ) if reasoning_effort == "auto": reasoning_effort = provider.runtime.default_reasoning_effort reasoning_effort = _resolve_nullable_parameter( diff --git a/EvoScientist/model_registry/http_api.py b/EvoScientist/model_registry/http_api.py index ec4b9d3..758390d 100644 --- a/EvoScientist/model_registry/http_api.py +++ b/EvoScientist/model_registry/http_api.py @@ -79,6 +79,7 @@ from .schemas import ( ModelAvailability, ModelRef, NonEmptyString, + RegistryDefaults, RegistryV4, ) from .snapshots import ( @@ -164,6 +165,9 @@ class GetSelectableModelsResponse(BaseModel): """The section 9.1 selector response: only selectable enabled models.""" models: list[SelectableModel] + # Lets the selector display the registry default before the first turn, + # when no run snapshot exists yet to resolve `inherit` against. + defaults: RegistryDefaults = Field(default_factory=RegistryDefaults) class SnapshotBindRequest(BaseModel): @@ -197,7 +201,6 @@ class SnapshotPublicResponse(BaseModel): snapshot_id: str registry_revision: PositiveInt primary: ResolvedModelPublic - auxiliary: ResolvedModelPublic | None = None # --- services bundle -------------------------------------------------------- @@ -754,7 +757,7 @@ class ModelRegistryHttpApi: effective_capabilities=item.effective_capabilities, ) ) - return GetSelectableModelsResponse(models=models) + return GetSelectableModelsResponse(models=models, defaults=registry.defaults) @staticmethod def _create_snapshot( diff --git a/EvoScientist/model_registry/openapi.json b/EvoScientist/model_registry/openapi.json index 220f6f0..c20c699 100644 --- a/EvoScientist/model_registry/openapi.json +++ b/EvoScientist/model_registry/openapi.json @@ -477,6 +477,9 @@ "GetSelectableModelsResponse": { "description": "The section 9.1 selector response: only selectable enabled models.", "properties": { + "defaults": { + "$ref": "#/components/schemas/RegistryDefaults" + }, "models": { "items": { "$ref": "#/components/schemas/SelectableModel" @@ -1034,17 +1037,6 @@ }, "RegistryDefaults": { "properties": { - "auxiliary": { - "anyOf": [ - { - "$ref": "#/components/schemas/ModelRef" - }, - { - "type": "null" - } - ], - "default": null - }, "primary": { "anyOf": [ { @@ -1221,17 +1213,6 @@ "SnapshotCreateRequest": { "description": "The section 8.2 snapshot creation request body.", "properties": { - "auxiliary": { - "anyOf": [ - { - "$ref": "#/components/schemas/ModelRef" - }, - { - "type": "null" - } - ], - "default": null - }, "deployment_id": { "minLength": 1, "title": "Deployment Id", @@ -1254,6 +1235,24 @@ ], "default": null }, + "reasoning_effort": { + "anyOf": [ + { + "enum": [ + "auto", + "low", + "medium", + "high" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Reasoning Effort" + }, "run_request_id": { "minLength": 1, "title": "Run Request Id", @@ -1276,17 +1275,6 @@ "SnapshotPublicResponse": { "description": "The section 8.2 public snapshot diagnostic subset.", "properties": { - "auxiliary": { - "anyOf": [ - { - "$ref": "#/components/schemas/ResolvedModelPublic" - }, - { - "type": "null" - } - ], - "default": null - }, "primary": { "$ref": "#/components/schemas/ResolvedModelPublic" }, diff --git a/EvoScientist/model_registry/resolver.py b/EvoScientist/model_registry/resolver.py index de7d995..ef4c7f0 100644 --- a/EvoScientist/model_registry/resolver.py +++ b/EvoScientist/model_registry/resolver.py @@ -50,6 +50,7 @@ from .schemas import ( ModelRef, ModelRole, ProviderConfig, + ReasoningEffort, RegistryV4, RequestOptions, ResolvedModelConfig, @@ -102,9 +103,16 @@ class ModelRegistryResolver: role: ModelRole = "primary", *, registry: RegistryV4 | None = None, + reasoning_effort_override: ReasoningEffort | None = None, ) -> ResolvedModelConfig: """Resolve an enabled, verified model into its frozen run config.""" - return self._resolve(model_ref, role, registry=registry, require_enabled=True) + return self._resolve( + model_ref, + role, + registry=registry, + require_enabled=True, + reasoning_effort_override=reasoning_effort_override, + ) def resolve_for_test(self, model_ref: ModelRef) -> ResolvedModelConfig: """Resolve for a provider test (section 9.4). @@ -130,6 +138,7 @@ class ModelRegistryResolver: registry: RegistryV4 | None, require_enabled: bool, require_verified: bool = True, + reasoning_effort_override: ReasoningEffort | None = None, ) -> ResolvedModelConfig: _check_role(role) if registry is None: @@ -176,7 +185,9 @@ class ModelRegistryResolver: # Save-time contract checks re-applied at resolve time: auth mode, # credential reference, declared capabilities, and parameters. - parameters = resolve_parameters(provider, model, spec) + parameters = resolve_parameters( + provider, model, spec, reasoning_effort_override=reasoning_effort_override + ) auth_spec = spec.auth_specs[provider.auth.mode] credential_revision: int | None = None diff --git a/EvoScientist/model_registry/runtime.py b/EvoScientist/model_registry/runtime.py index c08da98..f06484a 100644 --- a/EvoScientist/model_registry/runtime.py +++ b/EvoScientist/model_registry/runtime.py @@ -1,17 +1,15 @@ """Runtime-link glue for snapshot-driven model resolution (design doc 8.3). ``SnapshotRuntime`` bundles the store, resolver, and snapshot service into -the single accessor the running agent link (middleware, agent factories, -auxiliary roles) uses. It provides two resolution paths: +the single accessor the running agent link (middleware, agent factories) +uses. It provides two resolution paths: - ``build_role_model(snapshot, role)`` — per-call construction from a frozen - run snapshot (section 6.1 role mapping, section 5.2 per-call credential - resolution). This is the only path the per-call middleware uses. -- ``build_default_role_model(role)`` — build-time construction from the - active registry's ``defaults`` for auxiliary roles that bind a model when - a graph is compiled (tool selector, scheduler, memory workers). Auxiliary, - summary, and tool-selector roles map to ``defaults.auxiliary ?? - defaults.primary``; no free model/provider strings are consulted anywhere. + run snapshot (section 5.2 per-call credential resolution). Every role + resolves to the snapshot's frozen primary (section 6.1). +- ``build_default_role_model()`` — build-time construction from the active + registry's default primary, used only as the compile-time graph binding; + every run re-resolves its model from the run snapshot. Every model is constructed through ``build_chat_model`` with both safe HTTP clients (section 6.4). Ollama adapters additionally carry ``max_retries`` @@ -118,8 +116,8 @@ class SnapshotRuntime: # --- registry-default (build-time) resolution ---------------------------- - def registry_defaults(self) -> tuple[ModelRef, ModelRef | None, int]: - """Return ``(primary, auxiliary, revision)`` for an active registry.""" + def registry_default(self) -> tuple[ModelRef, int]: + """Return ``(primary, revision)`` for an active registry.""" registry = self._store.load_registry() if registry.state != "active" or registry.defaults.primary is None: raise ModelRegistryError( @@ -127,23 +125,16 @@ class SnapshotRuntime: "The model registry is in bootstrap; configure and enable a " "primary model first.", ) - return registry.defaults.primary, registry.defaults.auxiliary, registry.revision + return registry.defaults.primary, registry.revision - def resolve_role_config(self, role: ModelRole = "primary") -> ResolvedModelConfig: - """Resolve a role against the registry defaults (section 6.1). - - ``primary`` resolves ``defaults.primary``; every auxiliary role - (``auxiliary``, ``summary``, ``tool_selector``) resolves - ``defaults.auxiliary ?? defaults.primary``. - """ - primary, auxiliary, _ = self.registry_defaults() - if role == "primary" or auxiliary is None: - return self._resolver.resolve(primary, "primary") - return self._resolver.resolve(auxiliary, "auxiliary") + def resolve_default_config(self) -> ResolvedModelConfig: + """Resolve the registry default primary (section 6.1).""" + primary, _ = self.registry_default() + return self._resolver.resolve(primary, "primary") def build_default_role_model(self, role: ModelRole = "primary") -> BaseChatModel: - """Build the chat model for a role from the registry defaults.""" - config = self.resolve_role_config(role) + """Build the chat model for a role from the registry default.""" + config = self.resolve_default_config() return self._build(config, self._resolve_credential(config)) # --- local entry points (design doc 8.1) --------------------------------- @@ -182,7 +173,6 @@ class SnapshotRuntime: deployment_id=self._local_deployment_id, model_selection_revision=0, primary=None, - auxiliary=None, ) ) return creation.snapshot @@ -207,7 +197,7 @@ class SnapshotRuntime: config.client_options.max_retries if config.adapter_id == "ollama" else 0 ) timeout = config.client_options.timeout_seconds - return build_chat_model( + model = build_chat_model( config, http_client=build_safe_http_client( self._endpoint_policy, timeout=timeout, retries=retries @@ -217,6 +207,21 @@ class SnapshotRuntime: ), credential=credential, ) + # Usage tracking is inert unless the launcher supplied the complete + # sink environment; the adapter contract revision stands in for the + # removed per-profile runtime revision. + from ..usage import UsageModelIdentity, attach_usage_callback + + return attach_usage_callback( + model, + UsageModelIdentity( + provider_profile_id=config.model_ref.provider_id, + provider_revision=str(config.adapter_spec_revision), + provider_adapter=config.adapter_id, + model_alias=config.model_ref.model_key, + upstream_model_id=config.upstream_model_id, + ), + ) # --- process-shared default runtime ------------------------------------------- diff --git a/EvoScientist/model_registry/schemas.py b/EvoScientist/model_registry/schemas.py index 5d44ceb..e21fef0 100644 --- a/EvoScientist/model_registry/schemas.py +++ b/EvoScientist/model_registry/schemas.py @@ -45,7 +45,7 @@ RegistryState = Literal["bootstrap", "active"] LimitMode = Literal["combined", "input_only"] LimitsStatus = Literal["confirmed", "needs_confirmation"] LimitsSource = Literal["provider", "tested_contract", "user"] -ModelRole = Literal["primary", "auxiliary", "summary", "tool_selector"] +ModelRole = Literal["primary"] class _FrozenModel(BaseModel): @@ -61,7 +61,6 @@ class ModelRef(_FrozenModel): class RegistryDefaults(BaseModel): primary: ModelRef | None = None - auxiliary: ModelRef | None = None class Capabilities(BaseModel): @@ -175,22 +174,18 @@ class RegistryV4(BaseModel): provider_ids = [provider.id for provider in self.providers] if len(provider_ids) != len(set(provider_ids)): raise ValueError("Provider IDs must be unique.") - for role, ref in ( - ("primary", self.defaults.primary), - ("auxiliary", self.defaults.auxiliary), - ): - if ref is None: - continue + ref = self.defaults.primary + if ref is not None: target = self._locate(ref) if target is None: raise ValueError( - f"defaults.{role} references an unknown ModelRef " + f"defaults.primary references an unknown ModelRef " f"{ref.provider_id!r}/{ref.model_key!r}." ) provider, model = target if self.state == "active" and not (provider.enabled and model.enabled): raise ValueError( - f"defaults.{role} must reference an enabled model while " + "defaults.primary must reference an enabled model while " "the registry is active." ) if self.state == "active" and self.defaults.primary is None: diff --git a/EvoScientist/model_registry/snapshots.py b/EvoScientist/model_registry/snapshots.py index 5f94c08..f625cc6 100644 --- a/EvoScientist/model_registry/snapshots.py +++ b/EvoScientist/model_registry/snapshots.py @@ -2,7 +2,7 @@ ``SnapshotService`` is the single implementation shared by the HTTP snapshot API (Task 5) and the local CLI/channel/scheduler entry points (Task 7): it -freezes both roles' complete ``ResolvedModelConfig`` — adapter spec +freezes the primary role's complete ``ResolvedModelConfig`` — adapter spec revision, fixed budget reserves, capabilities, and credential revisions — into ``run_runtime_snapshots.payload_json``. Payloads and logs never carry ``secret_value``. @@ -11,9 +11,9 @@ into ``run_runtime_snapshots.payload_json``. Payloads and logs never carry maps to at most one non-terminal snapshot. A repeated request with the same ``selection_hash`` returns the original snapshot; a different hash raises ``RUN_REQUEST_CONFLICT``. ``selection_hash`` covers the - pre-resolution ``{primary, auxiliary}`` selection (inherit participates as - ``null``); ``model_selection_revision`` is an audit field and never part - of the hash. + pre-resolution ``{primary, reasoning_effort}`` selection (inherit + participates as ``null``); ``model_selection_revision`` is an audit field + and never part of the hash. - Lifecycle: ``prepared`` (TTL 15 minutes) → ``bound`` (retained 24 hours after binding) → ``expired`` (terminal, via ``cleanup_expired``); ``prepared`` → ``aborted`` on run creation failure. Terminal snapshots @@ -51,6 +51,7 @@ from .schemas import ( ModelRef, ModelRole, NonEmptyString, + ReasoningEffort, ResolvedModelConfig, ) from .store import ModelRuntimeStore @@ -69,9 +70,12 @@ class SnapshotCreateRequest(BaseModel): thread_id: NonEmptyString deployment_id: NonEmptyString model_selection_revision: NonNegativeInt = 0 - # ``None`` means inherit: the registry defaults are resolved at creation. + # ``None`` means inherit: the registry default is resolved at creation. primary: ModelRef | None = None - auxiliary: ModelRef | None = None + # Per-run reasoning-effort override (thread-level selection); + # ``None``/``auto`` keeps the registry-configured effort. Unsupported + # adapters reject it. + reasoning_effort: ReasoningEffort | None = None class SnapshotPayload(BaseModel): @@ -80,7 +84,6 @@ class SnapshotPayload(BaseModel): registry_revision: PositiveInt model_selection_revision: NonNegativeInt primary: ResolvedModelConfig - auxiliary: ResolvedModelConfig | None = None class RuntimeSnapshot(BaseModel): @@ -120,16 +123,20 @@ class SnapshotCreation(BaseModel): created: bool -def compute_selection_hash(primary: ModelRef | None, auxiliary: ModelRef | None) -> str: +def compute_selection_hash( + primary: ModelRef | None, + reasoning_effort: ReasoningEffort | None = None, +) -> str: """Hash the pre-resolution selection; inherit participates as ``null``.""" - - def entry(ref: ModelRef | None) -> dict[str, str] | None: - if ref is None: - return None - return {"provider_id": ref.provider_id, "model_key": ref.model_key} - encoded = json.dumps( - {"primary": entry(primary), "auxiliary": entry(auxiliary)}, + { + "primary": ( + None + if primary is None + else {"provider_id": primary.provider_id, "model_key": primary.model_key} + ), + "reasoning_effort": reasoning_effort, + }, sort_keys=True, separators=(",", ":"), ) @@ -137,18 +144,12 @@ def compute_selection_hash(primary: ModelRef | None, auxiliary: ModelRef | None) def config_for_role(snapshot: RuntimeSnapshot, role: ModelRole) -> ResolvedModelConfig: - """The section 6.1 role mapping; no other mapping may be guessed. - - ``primary`` → ``snapshot.primary``; ``auxiliary``, ``summary``, and - ``tool_selector`` → ``snapshot.auxiliary ?? snapshot.primary``. - """ + """Every role resolves to the snapshot's frozen primary (section 6.1).""" if role not in _MODEL_ROLES: raise ValueError( f"Unknown model role {role!r}; expected one of {list(_MODEL_ROLES)}." ) - if role == "primary" or snapshot.payload.auxiliary is None: - return snapshot.payload.primary - return snapshot.payload.auxiliary + return snapshot.payload.primary def public_snapshot_view(snapshot: RuntimeSnapshot) -> dict[str, Any]: @@ -163,17 +164,16 @@ def public_snapshot_view(snapshot: RuntimeSnapshot) -> dict[str, Any]: "max_output_tokens": config.request_options.max_output_tokens, "temperature": config.request_options.temperature, "top_p": config.request_options.top_p, + "reasoning_effort": config.request_options.reasoning_effort, "timeout_seconds": config.client_options.timeout_seconds, "max_retries": config.client_options.max_retries, }, } - auxiliary = snapshot.payload.auxiliary return { "snapshot_id": snapshot.snapshot_id, "registry_revision": snapshot.payload.registry_revision, "primary": public_config(snapshot.payload.primary), - "auxiliary": None if auxiliary is None else public_config(auxiliary), } @@ -197,7 +197,9 @@ class SnapshotService: "The model registry is in bootstrap; configure and enable a " "primary model first.", ) - selection_hash = compute_selection_hash(request.primary, request.auxiliary) + selection_hash = compute_selection_hash( + request.primary, request.reasoning_effort + ) existing = self._store.find_active_run_snapshot( deployment_id=request.deployment_id, thread_id=request.thread_id, @@ -219,33 +221,21 @@ class SnapshotService: if request.primary is not None else registry.defaults.primary ) - auxiliary_ref = ( - request.auxiliary - if request.auxiliary is not None - else registry.defaults.auxiliary - ) if primary_ref is None: # pragma: no cover - active registries set it raise ModelRegistryError( MODEL_REGISTRY_NOT_READY, "The model registry has no default primary model.", ) primary_config = self._resolver.resolve( - primary_ref, "primary", registry=registry - ) - auxiliary_config = ( - None - if auxiliary_ref is None - else self._resolver.resolve(auxiliary_ref, "auxiliary", registry=registry) + primary_ref, + "primary", + registry=registry, + reasoning_effort_override=request.reasoning_effort, ) payload = { "registry_revision": registry.revision, "model_selection_revision": request.model_selection_revision, "primary": primary_config.model_dump(mode="json"), - "auxiliary": ( - None - if auxiliary_config is None - else auxiliary_config.model_dump(mode="json") - ), } snapshot_id = f"snap-{uuid.uuid4().hex}" try: @@ -423,23 +413,20 @@ class SnapshotService: # --- internals ------------------------------------------------------------ def _ensure_frozen_specs_available(self, snapshot: RuntimeSnapshot) -> None: - configs = [snapshot.payload.primary] - if snapshot.payload.auxiliary is not None: - configs.append(snapshot.payload.auxiliary) - for config in configs: - spec = find_adapter_spec( - config.adapter_id, - config.upstream_model_id, - spec_revision=config.adapter_spec_revision, - specs=self._resolver.specs, + config = snapshot.payload.primary + spec = find_adapter_spec( + config.adapter_id, + config.upstream_model_id, + spec_revision=config.adapter_spec_revision, + specs=self._resolver.specs, + ) + if spec is None: + raise ModelRegistryError( + ADAPTER_NOT_SUPPORTED, + f"The adapter contract {config.adapter_id!r} at " + f"spec_revision {config.adapter_spec_revision} frozen by " + "this snapshot no longer exists.", ) - if spec is None: - raise ModelRegistryError( - ADAPTER_NOT_SUPPORTED, - f"The adapter contract {config.adapter_id!r} at " - f"spec_revision {config.adapter_spec_revision} frozen by " - "this snapshot no longer exists.", - ) def _snapshot_not_found() -> ModelRegistryError: diff --git a/EvoScientist/model_registry/store.py b/EvoScientist/model_registry/store.py index a78c228..2e2c0dd 100644 --- a/EvoScientist/model_registry/store.py +++ b/EvoScientist/model_registry/store.py @@ -319,26 +319,23 @@ class ModelRuntimeStore: def _validate_defaults(self, registry: RegistryV4) -> None: """Enforce section 9.2 rule 7 on every save, regardless of state.""" - for role, ref in ( - ("primary", registry.defaults.primary), - ("auxiliary", registry.defaults.auxiliary), - ): - if ref is None: - continue - located = registry._locate(ref) - if located is None: - raise ModelRegistryError( - MODEL_DISABLED, - f"defaults.{role} references an unknown model.", - details=[{"path": f"defaults.{role}", "code": MODEL_DISABLED}], - ) - provider, model = located - if not (provider.enabled and model.enabled): - raise ModelRegistryError( - MODEL_DISABLED, - f"defaults.{role} must reference an enabled model.", - details=[{"path": f"defaults.{role}", "code": MODEL_DISABLED}], - ) + ref = registry.defaults.primary + if ref is None: + return + located = registry._locate(ref) + if located is None: + raise ModelRegistryError( + MODEL_DISABLED, + "defaults.primary references an unknown model.", + details=[{"path": "defaults.primary", "code": MODEL_DISABLED}], + ) + provider, model = located + if not (provider.enabled and model.enabled): + raise ModelRegistryError( + MODEL_DISABLED, + "defaults.primary must reference an enabled model.", + details=[{"path": "defaults.primary", "code": MODEL_DISABLED}], + ) def _resolve_state( self, diff --git a/EvoScientist/subagents/_factory.py b/EvoScientist/subagents/_factory.py index 34c5b47..4132c9e 100644 --- a/EvoScientist/subagents/_factory.py +++ b/EvoScientist/subagents/_factory.py @@ -11,9 +11,8 @@ Reuses the main EvoScientist agent's backend and middleware so the deployed sub-agent has full capability parity with its in-process synchronous counterpart: same workspace files, same ``/skills/`` and ``/memories/`` routes, same error-handling and context-overflow middleware. The chat model -is resolved from the active model registry's defaults (``auxiliary`` for the -scheduler, ``primary`` otherwise) and re-resolved per run from the run -snapshot by ``ConfigurableModelMiddleware``. +is resolved from the active model registry's default primary and re-resolved +per run from the run snapshot by ``ConfigurableModelMiddleware``. """ from __future__ import annotations @@ -103,12 +102,10 @@ def build_async_subagent_graph(name: str) -> Any: # context and `/memories/profile/...` file guidance as the main agent. # # The compile-time model binding is resolved from the active registry's - # defaults — never from config.yaml free strings. Per-run calls are - # re-resolved from the run snapshot by ConfigurableModelMiddleware; the - # scheduler (an unattended timer task) binds the cheaper auxiliary role, - # working sub-agents bind primary. + # default primary — never from config.yaml free strings. Per-run calls + # are re-resolved from the run snapshot by ConfigurableModelMiddleware. # - # Bootstrap registry: no defaults exist yet, so no compile-time model + # Bootstrap registry: no default exists yet, so no compile-time model # can be built. The graph must still materialize — one failing factory # must not take down the whole langgraph dev service — so we bind a # placeholder that raises MODEL_REGISTRY_NOT_READY on the first model @@ -121,7 +118,7 @@ def build_async_subagent_graph(name: str) -> Any: from EvoScientist.model_registry.placeholder import RegistryNotReadyChatModel runtime = get_snapshot_runtime() - snapshot_role = "auxiliary" if name == "scheduler" else "primary" + snapshot_role = "primary" try: model = runtime.build_default_role_model(snapshot_role) except ModelRegistryError as exc: diff --git a/tests/registry_fixtures.py b/tests/registry_fixtures.py index 01524a9..eedb2dd 100644 --- a/tests/registry_fixtures.py +++ b/tests/registry_fixtures.py @@ -102,18 +102,13 @@ def ollama_provider_payload(**model_overrides) -> dict: } -def registry_payload(*, auxiliary_default: bool = True, **model_overrides) -> dict: +def registry_payload(**model_overrides) -> dict: return { "version": 4, "revision": 1, "state": "bootstrap", "defaults": { "primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, - "auxiliary": ( - {"provider_id": "local-ollama", "model_key": "qwen3"} - if auxiliary_default - else None - ), }, "providers": [ zhipu_provider_payload(**model_overrides), @@ -141,12 +136,12 @@ def _verify_model(store, registry, provider_id, model_key): ) -def activate_store(store, *, auxiliary_default: bool = True, **model_overrides): +def activate_store(store, **model_overrides): """Persist an active registry with verified models into *store*.""" registry = store.save_registry( expected_revision=1, registry=RegistryV4.model_validate( - registry_payload(auxiliary_default=auxiliary_default, **model_overrides) + registry_payload(**model_overrides) ), credential_writes=[ CredentialWrite(credential_id="zhipu-primary", secret_value=ZHIPU_SECRET) @@ -158,11 +153,10 @@ def activate_store(store, *, auxiliary_default: bool = True, **model_overrides): return store -def make_active_store(config_dir, *, auxiliary_default: bool = True, **model_overrides): +def make_active_store(config_dir, **model_overrides): """Persist an active registry with verified models into a fresh store.""" return activate_store( ModelRuntimeStore(config_dir=config_dir), - auxiliary_default=auxiliary_default, **model_overrides, ) @@ -174,7 +168,6 @@ def make_snapshot_request(**overrides) -> SnapshotCreateRequest: "deployment_id": "local", "model_selection_revision": 0, "primary": None, - "auxiliary": None, } payload.update(overrides) return SnapshotCreateRequest.model_validate(payload) diff --git a/tests/test_ask_user.py b/tests/test_ask_user.py index 74ae9c5..5011842 100644 --- a/tests/test_ask_user.py +++ b/tests/test_ask_user.py @@ -539,18 +539,15 @@ class TestConfig: @patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[]) -@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model") @patch("EvoScientist.EvoScientist._ensure_chat_model") @patch("EvoScientist.EvoScientist._ensure_config") def test_auto_approve_still_includes_ask_user_middleware( - mock_config, mock_model, mock_aux_model, mock_tool_selector + mock_config, mock_model, mock_tool_selector ): cfg = MagicMock() cfg.enable_ask_user = True cfg.auto_approve = True cfg.auto_mode = False - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000}) @@ -561,18 +558,15 @@ def test_auto_approve_still_includes_ask_user_middleware( @patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[]) -@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model") @patch("EvoScientist.EvoScientist._ensure_chat_model") @patch("EvoScientist.EvoScientist._ensure_config") def test_auto_mode_disables_ask_user_middleware( - mock_config, mock_model, mock_aux_model, mock_tool_selector + mock_config, mock_model, mock_tool_selector ): cfg = MagicMock() cfg.enable_ask_user = True cfg.auto_approve = True cfg.auto_mode = True - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000}) @@ -583,11 +577,10 @@ def test_auto_mode_disables_ask_user_middleware( @patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[]) -@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model") @patch("EvoScientist.EvoScientist._ensure_chat_model") @patch("EvoScientist.EvoScientist._ensure_config") def test_for_async_subagent_omits_ask_user_middleware( - mock_config, mock_model, mock_aux_model, mock_tool_selector + mock_config, mock_model, mock_tool_selector ): """``AskUserMiddleware`` uses ``interrupt()`` to wait on user input. @@ -602,8 +595,6 @@ def test_for_async_subagent_omits_ask_user_middleware( cfg.enable_ask_user = True cfg.auto_approve = False cfg.auto_mode = False - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000}) diff --git a/tests/test_async_subagent_factory.py b/tests/test_async_subagent_factory.py index 793f7be..0572328 100644 --- a/tests/test_async_subagent_factory.py +++ b/tests/test_async_subagent_factory.py @@ -123,7 +123,7 @@ def test_factory_requests_async_safe_middleware( @patch("EvoScientist.utils.load_subagents") @patch("EvoScientist.config.apply_config_to_env") @patch("EvoScientist.config.get_effective_config") -def test_scheduler_binds_auxiliary_role( +def test_scheduler_binds_primary_role( mock_get_cfg, mock_apply_env, mock_load_subs, @@ -133,7 +133,7 @@ def test_scheduler_binds_auxiliary_role( mock_mcp, mock_create, ): - """The scheduler is an unattended timer task → auxiliary role mapping.""" + """The scheduler resolves the registry's primary default (6.1).""" cfg = MagicMock() cfg.recursion_limit = 1_000_000 cfg.memory_profile_enabled = False @@ -141,7 +141,7 @@ def test_scheduler_binds_auxiliary_role( cfg.memory_observation_writer = MemoryObservationWriter.ALL cfg.memory_workers_enabled = False mock_get_cfg.return_value = cfg - model = MagicMock(name="auxiliary_model") + model = MagicMock(name="primary_model") mock_get_runtime.return_value.build_default_role_model.return_value = model mock_load_subs.return_value = [ { @@ -158,13 +158,13 @@ def test_scheduler_binds_auxiliary_role( build_async_subagent_graph("scheduler") mock_get_runtime.return_value.build_default_role_model.assert_called_once_with( - "auxiliary" + "primary" ) mock_get_mw.assert_called_once_with( for_async_subagent=True, memory_source_agent="scheduler", chat_model=model, - snapshot_role="auxiliary", + snapshot_role="primary", ) assert mock_create.call_args.kwargs["model"] is model @@ -254,8 +254,6 @@ def test_inject_subagent_omits_memory_middleware_when_memory_disabled( cfg.memory_observations_enabled = False cfg.memory_observation_writer = MemoryObservationWriter.ALL cfg.memory_workers_enabled = True - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg from EvoScientist.EvoScientist import _inject_subagent_middleware @@ -284,8 +282,6 @@ def test_inject_subagent_worker_only_observation_writer_keeps_live_tool_off( cfg.memory_observations_enabled = True cfg.memory_observation_writer = MemoryObservationWriter.WORKER cfg.memory_workers_enabled = True - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg from EvoScientist.EvoScientist import _inject_subagent_middleware @@ -312,11 +308,10 @@ def test_inject_subagent_worker_only_observation_writer_keeps_live_tool_off( "EvoScientist.middleware.create_tool_selector_middleware", return_value=[MagicMock()], ) -@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model") -@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._compile_time_role_model") @patch("EvoScientist.EvoScientist._ensure_config") def test_all_observation_writer_schedules_turn_worker_without_profile_memory( - mock_config, mock_chat, mock_aux, mock_tool_selector + mock_config, mock_chat, mock_tool_selector ): cfg = MagicMock() cfg.enable_ask_user = False @@ -327,8 +322,6 @@ def test_all_observation_writer_schedules_turn_worker_without_profile_memory( cfg.memory_observations_enabled = True cfg.memory_observation_writer = MemoryObservationWriter.ALL cfg.memory_workers_enabled = True - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000}) @@ -365,11 +358,10 @@ def test_all_observation_writer_schedules_turn_worker_without_profile_memory( "EvoScientist.middleware.create_tool_selector_middleware", return_value=[MagicMock()], ) -@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model") -@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._compile_time_role_model") @patch("EvoScientist.EvoScientist._ensure_config") def test_async_subagent_mode_filters_ask_user( - mock_config, mock_chat, mock_aux, mock_tool_selector + mock_config, mock_chat, mock_tool_selector ): """``_get_default_middleware(for_async_subagent=True)`` must drop ``AskUserMiddleware`` even when ``enable_ask_user`` is on. @@ -388,8 +380,6 @@ def test_async_subagent_mode_filters_ask_user( cfg.memory_observations_enabled = True cfg.memory_observation_writer = MemoryObservationWriter.ALL cfg.memory_workers_enabled = True - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000}) @@ -415,7 +405,7 @@ def test_async_subagent_mode_filters_ask_user( "EvoScientist.middleware.create_tool_selector_middleware", return_value=[MagicMock()], ) -@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._compile_time_role_model") @patch("EvoScientist.EvoScientist._ensure_config") def test_async_subagent_disables_tool_selector_stream_tracking( mock_config, mock_chat, mock_tool_selector @@ -430,8 +420,6 @@ def test_async_subagent_disables_tool_selector_stream_tracking( cfg.memory_observations_enabled = True cfg.memory_observation_writer = MemoryObservationWriter.ALL cfg.memory_workers_enabled = True - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000}) diff --git a/tests/test_auxiliary_model.py b/tests/test_auxiliary_model.py index 19ab866..042acef 100644 --- a/tests/test_auxiliary_model.py +++ b/tests/test_auxiliary_model.py @@ -1,88 +1,17 @@ -"""Tests for the auxiliary-model resolver and its middleware scoping. +"""Tests for the helper-model wiring in ``_get_default_middleware``. -Covers ``EvoScientist.EvoScientist._ensure_auxiliary_chat_model`` — now -resolved through the model registry role mapping (design doc 6.1: -``defaults.auxiliary ?? defaults.primary``) instead of the legacy -``cfg.auxiliary_model``/``auxiliary_provider`` free strings — and the wiring -in ``_get_default_middleware`` that routes the main agent's tool selector to -the auxiliary model while keeping context editing — and async sub-agents — -on the main model. +After the auxiliary-role removal (design doc 6.1: every role resolves to the +snapshot's frozen primary), the main agent's tool selector no longer binds a +compile-time auxiliary model — it resolves its helper model from the run +snapshot on every call (``model=None``). Async sub-agents and the pure path +keep their threaded model. Context editing always keeps the main model. """ from unittest.mock import MagicMock, patch -import pytest - import EvoScientist.EvoScientist as E -from EvoScientist.model_registry.errors import ( - MODEL_REGISTRY_NOT_READY, - ModelRegistryError, -) -from tests.registry_fixtures import activate_store -@pytest.fixture(autouse=True) -def _reset_model_caches(monkeypatch): - """Isolate the module-level model caches per test.""" - monkeypatch.setattr(E, "_chat_model", None, raising=False) - monkeypatch.setattr(E, "_chat_model_key", None, raising=False) - monkeypatch.setattr(E, "_auxiliary_chat_model", None, raising=False) - monkeypatch.setattr(E, "_auxiliary_chat_model_key", None, raising=False) - - -class TestAuxiliaryResolver: - def test_bootstrap_registry_fails_closed(self, isolated_snapshot_runtime): - """Bootstrap registry: auxiliary resolution propagates NOT_READY.""" - with pytest.raises(ModelRegistryError) as excinfo: - E._ensure_auxiliary_chat_model() - assert excinfo.value.code == MODEL_REGISTRY_NOT_READY - - def test_no_auxiliary_default_returns_main_instance( - self, monkeypatch, isolated_snapshot_runtime - ): - main = object() - activate_store(isolated_snapshot_runtime.store, auxiliary_default=False) - monkeypatch.setattr(E, "_ensure_chat_model", lambda: main) - assert E._ensure_auxiliary_chat_model() is main - - def test_auxiliary_equal_to_primary_reuses_main_instance( - self, monkeypatch, isolated_snapshot_runtime - ): - main = object() - store = activate_store(isolated_snapshot_runtime.store) - registry = store.load_registry() - registry.defaults.auxiliary = registry.defaults.primary - store.save_registry(expected_revision=registry.revision, registry=registry) - monkeypatch.setattr(E, "_ensure_chat_model", lambda: main) - assert E._ensure_auxiliary_chat_model() is main - - def test_auxiliary_default_builds_via_role_mapping( - self, monkeypatch, isolated_snapshot_runtime - ): - fake = object() - build = MagicMock(return_value=fake) - activate_store(isolated_snapshot_runtime.store) - monkeypatch.setattr( - isolated_snapshot_runtime, "build_default_role_model", build - ) - monkeypatch.setattr( - E, "_ensure_chat_model", lambda: pytest.fail("must not build main") - ) - assert E._ensure_auxiliary_chat_model() is fake - build.assert_called_once_with("auxiliary") - - def test_auxiliary_cache_reused_within_same_registry_revision( - self, monkeypatch, isolated_snapshot_runtime - ): - build = MagicMock(side_effect=[object(), object()]) - activate_store(isolated_snapshot_runtime.store) - monkeypatch.setattr( - isolated_snapshot_runtime, "build_default_role_model", build - ) - first = E._ensure_auxiliary_chat_model() - assert E._ensure_auxiliary_chat_model() is first - assert build.call_count == 1 - def _mock_cfg(): cfg = MagicMock() cfg.enable_ask_user = False @@ -91,8 +20,8 @@ def _mock_cfg(): return cfg -class TestAuxiliaryMiddlewareScope: - """``_get_default_middleware`` routes only the right components to aux.""" +class TestHelperModelScope: + """``_get_default_middleware`` routes only the right components.""" def _capture(self): cap: dict[str, object] = {} @@ -107,17 +36,13 @@ class TestAuxiliaryMiddlewareScope: return cap, fake_tool_selector, fake_context_editing - def test_main_agent_tool_selector_aux_context_editing_main(self): + def test_main_agent_tool_selector_uses_snapshot_resolution(self): cap, fake_ts, fake_ce = self._capture() - main_model, aux_model = object(), object() + main_model = object() with ( patch.object(E, "_ensure_config", return_value=_mock_cfg()), patch.object( - E, - "_compile_time_role_model", - side_effect=lambda role="primary": aux_model - if role == "auxiliary" - else main_model, + E, "_compile_time_role_model", return_value=main_model ), patch( "EvoScientist.middleware.create_tool_selector_middleware", @@ -130,20 +55,17 @@ class TestAuxiliaryMiddlewareScope: ): E._get_default_middleware() - assert cap["tool_selector"] is aux_model + # None = per-call resolution from the run snapshot. + assert cap["tool_selector"] is None assert cap["context_editing"] is main_model def test_async_subagent_tool_selector_stays_main(self): cap, fake_ts, fake_ce = self._capture() - main_model, aux_model = object(), object() + main_model = object() with ( patch.object(E, "_ensure_config", return_value=_mock_cfg()), patch.object( - E, - "_compile_time_role_model", - side_effect=lambda role="primary": aux_model - if role == "auxiliary" - else main_model, + E, "_compile_time_role_model", return_value=main_model ), patch( "EvoScientist.middleware.create_tool_selector_middleware", @@ -160,8 +82,6 @@ class TestAuxiliaryMiddlewareScope: assert cap["context_editing"] is main_model def test_pure_path_tool_selector_uses_threaded_model(self): - """The pure path never resolves auxiliary free strings (Task 7 wires - the local snapshot entry); the threaded model stands in.""" cap, fake_ts, fake_ce = self._capture() cfg = _mock_cfg() main_model = object() @@ -169,7 +89,6 @@ class TestAuxiliaryMiddlewareScope: with ( patch.object(E, "_ensure_config", side_effect=AssertionError), patch.object(E, "_ensure_chat_model", side_effect=AssertionError), - patch.object(E, "_ensure_auxiliary_chat_model", side_effect=AssertionError), patch( "EvoScientist.middleware.create_tool_selector_middleware", side_effect=fake_ts, @@ -190,22 +109,21 @@ class TestAuxiliaryMiddlewareScope: with ( patch.object(E, "_ensure_config", return_value=cfg), patch.object(E, "_ensure_chat_model", return_value=main_model), - patch.object(E, "_ensure_auxiliary_chat_model", return_value=main_model), patch( "EvoScientist.middleware.create_tool_selector_middleware", return_value=[MagicMock()], ), ): - mw = E._get_default_middleware(snapshot_role="auxiliary") + mw = E._get_default_middleware(snapshot_role="primary") configurable = next( m for m in mw if type(m).__name__ == "ConfigurableModelMiddleware" ) - assert configurable._role == "auxiliary" + assert configurable._role == "primary" -def test_memory_agent_factory_uses_auxiliary_role(monkeypatch): - """Memory workers bind the auxiliary role model at graph build.""" +def test_memory_agent_factory_binds_primary_placeholder(monkeypatch): + """Memory workers bind the primary placeholder plus snapshot middleware.""" sentinel = object() monkeypatch.setattr(E, "_compile_time_role_model", lambda role="primary": sentinel) captured: dict[str, object] = {} @@ -228,3 +146,5 @@ def test_memory_agent_factory_uses_auxiliary_role(monkeypatch): backend=MagicMock(), ) assert captured["model"] is sentinel + middleware = captured["middleware"] + assert type(middleware[0]).__name__ == "ConfigurableModelMiddleware" diff --git a/tests/test_cli_startup_gates.py b/tests/test_cli_startup_gates.py index 62a9b03..22c4a6b 100644 --- a/tests/test_cli_startup_gates.py +++ b/tests/test_cli_startup_gates.py @@ -39,7 +39,7 @@ def test_bootstrap_registry_aborts_startup(monkeypatch): legacy_artifacts, "assert_no_legacy_artifacts", MagicMock() ) runtime = MagicMock() - runtime.registry_defaults.side_effect = ModelRegistryError( + runtime.registry_default.side_effect = ModelRegistryError( MODEL_REGISTRY_NOT_READY, "bootstrap" ) monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime) @@ -53,7 +53,7 @@ def test_clean_environment_passes(monkeypatch): legacy_artifacts, "assert_no_legacy_artifacts", MagicMock() ) runtime = MagicMock() - runtime.registry_defaults.return_value = (MagicMock(), None, 2) + runtime.registry_default.return_value = (MagicMock(), 2) monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime) _startup_gates() # must not raise @@ -64,7 +64,7 @@ def test_non_bootstrap_registry_error_propagates(monkeypatch): legacy_artifacts, "assert_no_legacy_artifacts", MagicMock() ) runtime = MagicMock() - runtime.registry_defaults.side_effect = ModelRegistryError( + runtime.registry_default.side_effect = ModelRegistryError( "SNAPSHOT_EXPIRED", "expired" ) monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime) diff --git a/tests/test_configurable_model_middleware.py b/tests/test_configurable_model_middleware.py index 94810c1..d1be157 100644 --- a/tests/test_configurable_model_middleware.py +++ b/tests/test_configurable_model_middleware.py @@ -349,32 +349,9 @@ class TestSnapshotDrivenConstruction: assert isinstance(model, ChatOpenAI) assert model.model_name == "glm-5.2" - def test_auxiliary_role_maps_to_snapshot_auxiliary(self, store, runtime): - mw = ConfigurableModelMiddleware(role="auxiliary", runtime=runtime) - snapshot = make_snapshot(store) - req = _make_request() - handler = MagicMock(return_value="ok") - - with _patched_config(_configurable_for(snapshot)): - mw.wrap_model_call(req, handler) - - model = handler.call_args[0][0].model - assert isinstance(model, ChatOllama) - assert model.model == "qwen3" - - def test_summary_role_falls_back_to_snapshot_primary(self, tmp_path): - store = make_active_store(tmp_path / "db", auxiliary_default=False) - mw = ConfigurableModelMiddleware(role="summary", runtime=SnapshotRuntime(store)) - snapshot = make_snapshot(store) - req = _make_request() - handler = MagicMock(return_value="ok") - - with _patched_config(_configurable_for(snapshot)): - mw.wrap_model_call(req, handler) - - model = handler.call_args[0][0].model - assert isinstance(model, ChatOpenAI) - assert model.model_name == "glm-5.2" + def test_unknown_role_rejected(self, runtime): + with pytest.raises(ValueError, match="role"): + ConfigurableModelMiddleware(role="auxiliary", runtime=runtime) def test_deployment_id_from_configurable_is_verified(self, store, runtime): mw = ConfigurableModelMiddleware(runtime=runtime) diff --git a/tests/test_context_editing_middleware.py b/tests/test_context_editing_middleware.py index dc5a679..6596e92 100644 --- a/tests/test_context_editing_middleware.py +++ b/tests/test_context_editing_middleware.py @@ -107,16 +107,13 @@ def test_create_middleware_model_none_fallback(mock_model): "EvoScientist.middleware.create_tool_selector_middleware", return_value=[MagicMock(), MagicMock()], ) -@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model") -@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._compile_time_role_model") @patch("EvoScientist.EvoScientist._ensure_config") -def test_default_middleware_includes_context_editing(mock_config, mock_model, mock_aux, mock_ts): +def test_default_middleware_includes_context_editing(mock_config, mock_model, mock_ts): mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000}) cfg = MagicMock() cfg.enable_ask_user = False cfg.auto_approve = False - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg from EvoScientist.EvoScientist import _get_default_middleware @@ -144,16 +141,13 @@ def test_inject_subagent_includes_context_editing(mock_model): "EvoScientist.middleware.create_tool_selector_middleware", return_value=[MagicMock(), MagicMock()], ) -@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model") -@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._compile_time_role_model") @patch("EvoScientist.EvoScientist._ensure_config") -def test_context_editing_before_overflow_mapper(mock_config, mock_model, mock_aux, mock_ts): +def test_context_editing_before_overflow_mapper(mock_config, mock_model, mock_ts): mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000}) cfg = MagicMock() cfg.enable_ask_user = False cfg.auto_approve = False - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg from EvoScientist.EvoScientist import _get_default_middleware diff --git a/tests/test_message_budget.py b/tests/test_message_budget.py index 0f12587..c895a8e 100644 --- a/tests/test_message_budget.py +++ b/tests/test_message_budget.py @@ -16,7 +16,7 @@ from unittest.mock import MagicMock, patch import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage -from langchain_ollama import ChatOllama +from langchain_openai import ChatOpenAI from EvoScientist.middleware.message_budget import ( _snapshot_message_budget, @@ -30,16 +30,8 @@ from EvoScientist.model_registry.errors import ( ModelRegistryError, ) from EvoScientist.model_registry.runtime import SnapshotRuntime -from EvoScientist.model_registry.schemas import CredentialWrite, RegistryV4 from EvoScientist.model_registry.store import ModelRuntimeStore -from tests.registry_fixtures import ( - ZHIPU_SECRET, - _verify_model, - make_active_store, - make_snapshot, - ollama_provider_payload, - zhipu_provider_payload, -) +from tests.registry_fixtures import make_active_store, make_snapshot @contextmanager @@ -86,39 +78,6 @@ def _configurable_for(snapshot, **overrides): return configurable -def _make_split_limit_store(config_dir): - """Active store whose auxiliary model has a smaller input limit. - - primary (zhipu-glm/glm-5.2): context 1048576 → resolved_input_limit 1015808 - auxiliary (local-ollama/qwen3): context 65536 → resolved_input_limit 32768 - """ - store = ModelRuntimeStore(config_dir=config_dir) - payload = { - "version": 4, - "revision": 1, - "state": "bootstrap", - "defaults": { - "primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, - "auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"}, - }, - "providers": [ - zhipu_provider_payload(), - ollama_provider_payload(context_window_tokens=65536), - ], - } - registry = store.save_registry( - expected_revision=1, - registry=RegistryV4.model_validate(payload), - credential_writes=[ - CredentialWrite(credential_id="zhipu-primary", secret_value=ZHIPU_SECRET) - ], - ) - assert registry.state == "active" - _verify_model(store, registry, "zhipu-glm", "glm-5.2") - _verify_model(store, registry, "local-ollama", "qwen3") - return store - - def test_text_counter_excludes_attachment_payloads_and_counts_tool_results(): messages = [ HumanMessage( @@ -168,25 +127,6 @@ class TestSnapshotMessageBudget: assert tools_only.hard_tokens == int((1015808 - 4096 - 8192) * 0.90) assert full.hard_tokens == int((1015808 - 4096 - 8192 - 4096) * 0.90) - def test_auxiliary_role_uses_auxiliary_frozen_limits(self, tmp_path): - """T6 review fix: the budget role is threaded through, not hardcoded - to ``primary`` — a scheduler (auxiliary role) must size its budget - from the auxiliary model's frozen configuration.""" - store = _make_split_limit_store(tmp_path / "model-runtime") - snapshot = make_snapshot(store) - - primary_budget = _snapshot_message_budget( - snapshot, "primary", has_tools=False, has_attachments=False - ) - auxiliary_budget = _snapshot_message_budget( - snapshot, "auxiliary", has_tools=False, has_attachments=False - ) - - assert primary_budget.input_limit == 1015808 - assert auxiliary_budget.input_limit == 32768 - assert auxiliary_budget.hard_tokens == int((32768 - 4096) * 0.90) - - class TestSnapshotModeMiddleware: def test_budget_uses_frozen_reserves_not_model_profile(self, store, runtime): """A misleading compile-time profile must not affect snapshot mode.""" @@ -254,26 +194,6 @@ class TestSnapshotModeMiddleware: assert budget.has_tools is False assert budget.hard_tokens == int((1015808 - 4096) * 0.90) - def test_auxiliary_snapshot_role_sizes_budget_from_auxiliary(self, tmp_path): - """T6 review fix: ``snapshot_role="auxiliary"`` (scheduler) sizes the - budget from the snapshot's auxiliary frozen configuration.""" - store = _make_split_limit_store(tmp_path / "model-runtime") - runtime = SnapshotRuntime(store) - middleware = create_message_budget_middleware( - MagicMock(), - MagicMock(), - has_tools=False, - snapshot_role="auxiliary", - runtime=runtime, - ) - snapshot = make_snapshot(store) - - with _patched_config(_configurable_for(snapshot)): - budget = middleware._budget_for_request(_request([HumanMessage("hi")])) - - assert budget.input_limit == 32768 - assert budget.hard_tokens == int((32768 - 4096) * 0.90) - def test_unknown_snapshot_role_rejected(self, runtime): with pytest.raises(ValueError, match="Unknown model role"): create_message_budget_middleware( @@ -293,7 +213,7 @@ class TestSnapshotModeMiddleware: middleware._budget_for_request(_request([HumanMessage("hi")])) assert excinfo.value.code == SNAPSHOT_NOT_FOUND - def test_summarizer_model_uses_summary_role(self, store, runtime): + def test_summarizer_model_uses_snapshot_primary(self, store, runtime): middleware = create_message_budget_middleware( MagicMock(), MagicMock(), runtime=runtime ) @@ -302,9 +222,9 @@ class TestSnapshotModeMiddleware: with _patched_config(_configurable_for(snapshot)): model = middleware.model - # summary → snapshot.auxiliary ?? snapshot.primary (section 6.1) - assert isinstance(model, ChatOllama) - assert model.model == "qwen3" + # Every role maps to the snapshot's frozen primary (section 6.1). + assert isinstance(model, ChatOpenAI) + assert model.model_name == "glm-5.2" def test_summarizer_model_cached_per_snapshot(self, store, runtime): middleware = create_message_budget_middleware( diff --git a/tests/test_model_registry_http.py b/tests/test_model_registry_http.py index bedcdd9..d4b9deb 100644 --- a/tests/test_model_registry_http.py +++ b/tests/test_model_registry_http.py @@ -152,7 +152,6 @@ def _registry_payload(): "state": "bootstrap", "defaults": { "primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, - "auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"}, }, "providers": [_zhipu_provider(), _ollama_provider()], } @@ -476,7 +475,7 @@ def test_put_registry_rejects_defaults_to_disabled_model(active_client): # the store's rule-7 guard (enforced on every save) is what rejects it. registry = _current_registry(active_client)["registry"] registry["state"] = "bootstrap" - registry["providers"][1]["models"][0]["enabled"] = False + registry["providers"][0]["models"][0]["enabled"] = False response = _put(active_client, registry) assert response.status_code == 422 @@ -486,7 +485,7 @@ def test_put_registry_rejects_defaults_to_disabled_model(active_client): def test_put_registry_active_state_validates_defaults_in_schema(active_client): # The same rule fires earlier (Pydantic) when the payload claims active. registry = _current_registry(active_client)["registry"] - registry["providers"][1]["models"][0]["enabled"] = False + registry["providers"][0]["models"][0]["enabled"] = False response = _put(active_client, registry) assert response.status_code == 422 @@ -530,7 +529,7 @@ def test_put_registry_bootstrap_flow_configure_test_enable(client, store): # Step 1: save a draft registry — models disabled, defaults null, and the # credential written in the same transaction (section 10 steps 5-7). draft = _registry_payload() - draft["defaults"] = {"primary": None, "auxiliary": None} + draft["defaults"] = {"primary": None} for provider in draft["providers"]: for model in provider["models"]: model["enabled"] = False @@ -568,7 +567,6 @@ def test_put_registry_bootstrap_flow_configure_test_enable(client, store): enabled = body["registry"] enabled["defaults"] = { "primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, - "auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"}, } for provider in enabled["providers"]: for model in provider["models"]: @@ -651,11 +649,16 @@ def test_get_models_returns_only_selectable(active_client, store): "vision": False, "structured_output": True, } + # The selector echoes the registry defaults so clients can label the + # `inherit` choice before the first turn. + assert response.json()["defaults"]["primary"] == { + "provider_id": "zhipu-glm", + "model_key": "glm-5.2", + } - # Disable qwen3 (and drop the auxiliary default): it leaves the selector. + # Disable qwen3: it leaves the selector. registry = store.load_registry() payload = registry.model_dump(mode="json") - payload["defaults"]["auxiliary"] = None payload["providers"][1]["models"][0]["enabled"] = False store.save_registry( expected_revision=registry.revision, @@ -682,7 +685,6 @@ def _snapshot_body(**overrides): "deployment_id": "webui-1", "model_selection_revision": 4, "primary": None, - "auxiliary": None, } body.update(overrides) return body @@ -699,7 +701,6 @@ def test_snapshot_create_201_and_idempotent_200(active_client): assert body["primary"]["provider_id"] == "zhipu-glm" assert body["primary"]["adapter_spec_revision"] == 1 assert body["primary"]["runtime"]["max_output_tokens"] == 32768 - assert body["auxiliary"]["model_key"] == "qwen3" assert SECRET not in first.text replay = active_client.post( diff --git a/tests/test_model_registry_schemas.py b/tests/test_model_registry_schemas.py index 8f289c7..d9a840d 100644 --- a/tests/test_model_registry_schemas.py +++ b/tests/test_model_registry_schemas.py @@ -136,7 +136,6 @@ def _registry(**overrides): "state": "bootstrap", "defaults": { "primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, - "auxiliary": None, }, "providers": [_provider()], } @@ -153,7 +152,6 @@ class TestRegistryV4: assert registry.defaults.primary == ModelRef( provider_id="zhipu-glm", model_key="glm-5.2" ) - assert registry.defaults.auxiliary is None provider = registry.providers[0] assert provider.runtime.timeout_seconds == 120 model = provider.models[0] @@ -172,7 +170,7 @@ class TestRegistryV4: registry = RegistryV4.model_validate( _registry( state="bootstrap", - defaults={"primary": None, "auxiliary": None}, + defaults={"primary": None}, providers=[], ) ) @@ -184,7 +182,7 @@ class TestRegistryV4: RegistryV4.model_validate( _registry( state="active", - defaults={"primary": None, "auxiliary": None}, + defaults={"primary": None}, ) ) @@ -205,7 +203,6 @@ class TestRegistryV4: _registry( defaults={ "primary": {"provider_id": "zhipu-glm", "model_key": "missing"}, - "auxiliary": None, } ) ) diff --git a/tests/test_model_registry_store.py b/tests/test_model_registry_store.py index cedacc5..33c31bd 100644 --- a/tests/test_model_registry_store.py +++ b/tests/test_model_registry_store.py @@ -51,7 +51,6 @@ def _registry_payload(**overrides): "state": "bootstrap", "defaults": { "primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, - "auxiliary": None, }, "providers": [ { @@ -106,7 +105,6 @@ class TestBootstrap: assert registry.state == "bootstrap" assert registry.providers == [] assert registry.defaults.primary is None - assert registry.defaults.auxiliary is None def test_database_path_and_permissions(self, tmp_path): store = ModelRuntimeStore(config_dir=tmp_path) @@ -149,7 +147,7 @@ class TestRegistrySave: def test_bootstrap_save_without_ready_primary_stays_bootstrap(self, tmp_path): store = ModelRuntimeStore(config_dir=tmp_path) registry = _registry( - defaults={"primary": None, "auxiliary": None}, + defaults={"primary": None}, ) saved = store.save_registry(expected_revision=1, registry=registry) assert saved.state == "bootstrap" @@ -242,7 +240,7 @@ class TestRegistrySave: ], } ) - payload["defaults"]["auxiliary"] = { + payload["defaults"]["primary"] = { "provider_id": "local-ollama", "model_key": "qwen3", } diff --git a/tests/test_provider_test_api.py b/tests/test_provider_test_api.py index 2e338bf..1bf4ae1 100644 --- a/tests/test_provider_test_api.py +++ b/tests/test_provider_test_api.py @@ -219,7 +219,7 @@ def _registry_payload(port, **provider_overrides): "version": 4, "revision": 1, "state": "bootstrap", - "defaults": {"primary": None, "auxiliary": None}, + "defaults": {"primary": None}, "providers": [_provider_payload(port, **provider_overrides)], } diff --git a/tests/test_resolver.py b/tests/test_resolver.py index da12764..1655e1a 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -119,7 +119,7 @@ def _registry(providers=None, **overrides) -> RegistryV4: "version": 4, "revision": 1, "state": "bootstrap", - "defaults": {"primary": None, "auxiliary": None}, + "defaults": {"primary": None}, "providers": [_zhipu_provider()] if providers is None else providers, } payload.update(overrides) @@ -219,9 +219,8 @@ class TestResolve: config = resolver.resolve(ZHIPU_REF, "primary") assert SECRET not in config.model_dump_json() - def test_all_roles_are_stamped(self, resolver): - for role in ("primary", "auxiliary", "summary", "tool_selector"): - assert resolver.resolve(ZHIPU_REF, role).role == role + def test_primary_role_is_stamped(self, resolver): + assert resolver.resolve(ZHIPU_REF, "primary").role == "primary" def test_unknown_role_rejected(self, resolver): with pytest.raises(ValueError, match="role"): @@ -351,7 +350,7 @@ class TestResolve: registry = _activate(store, _registry(providers=[_ollama_provider()])) _verify(store, registry, "local-ollama", "qwen3") resolver = ModelRegistryResolver(store) - config = resolver.resolve(OLLAMA_REF, "auxiliary") + config = resolver.resolve(OLLAMA_REF, "primary") assert config.auth_ref.mode == "none" assert config.auth_ref.credential_id is None assert config.auth_ref.credential_revision is None diff --git a/tests/test_runtime_context_middleware.py b/tests/test_runtime_context_middleware.py index 7a9b626..2773b79 100644 --- a/tests/test_runtime_context_middleware.py +++ b/tests/test_runtime_context_middleware.py @@ -40,8 +40,6 @@ def _mock_config(): cfg.auto_mode = False cfg.auto_approve = False cfg.model_fallbacks = None - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" cfg.code_interpreter_timeout = 60 cfg.code_interpreter_max_result_chars = 6000 return cfg @@ -65,11 +63,10 @@ def test_runtime_context_injects_current_date_and_timezone(): "EvoScientist.middleware.create_tool_selector_middleware", return_value=[MagicMock(), MagicMock()], ) -@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model") -@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._compile_time_role_model") @patch("EvoScientist.EvoScientist._ensure_config") def test_default_middleware_includes_runtime_context( - mock_config, mock_model, mock_aux_model, mock_tool_selector + mock_config, mock_model, mock_tool_selector ): mock_config.return_value = _mock_config() mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000}) @@ -85,7 +82,7 @@ def test_default_middleware_includes_runtime_context( "EvoScientist.middleware.create_tool_selector_middleware", return_value=[MagicMock(), MagicMock()], ) -@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._compile_time_role_model") @patch("EvoScientist.EvoScientist._ensure_config") def test_async_subagent_middleware_includes_runtime_context( mock_config, mock_model, mock_tool_selector diff --git a/tests/test_snapshot_runtime.py b/tests/test_snapshot_runtime.py index 29a048a..f173c36 100644 --- a/tests/test_snapshot_runtime.py +++ b/tests/test_snapshot_runtime.py @@ -1,9 +1,9 @@ """Tests for ``EvoScientist.model_registry.runtime`` (design doc 8.3). -Covers the runtime-link glue: registry-default role resolution (auxiliary ?? -primary), snapshot role construction with per-call credential resolution, -safe-client wiring (ollama carries retries on the transports), the shared -default-runtime accessor, and the tolerant local platform field reader. +Covers the runtime-link glue: registry-default primary resolution, snapshot +role construction with per-call credential resolution, safe-client wiring +(ollama carries retries on the transports), the shared default-runtime +accessor, and the tolerant local platform field reader. """ from __future__ import annotations @@ -59,33 +59,19 @@ class TestRegistryDefaults: def test_bootstrap_registry_is_not_ready(self, tmp_path): runtime = SnapshotRuntime(ModelRuntimeStore(config_dir=tmp_path / "db")) with pytest.raises(ModelRegistryError) as excinfo: - runtime.registry_defaults() + runtime.registry_default() assert excinfo.value.code == MODEL_REGISTRY_NOT_READY assert excinfo.value.http_status == 422 - def test_active_registry_returns_defaults(self, runtime): - primary, auxiliary, revision = runtime.registry_defaults() + def test_active_registry_returns_default(self, runtime): + primary, revision = runtime.registry_default() assert primary == ZHIPU_REF - assert auxiliary == OLLAMA_REF assert revision == 2 -class TestResolveRoleConfig: - def test_primary_role_resolves_primary_default(self, runtime): - config = runtime.resolve_role_config("primary") - assert config.model_ref == ZHIPU_REF - assert config.role == "primary" - - def test_auxiliary_role_resolves_auxiliary_default(self, runtime): - for role in ("auxiliary", "summary", "tool_selector"): - config = runtime.resolve_role_config(role) - assert config.model_ref == OLLAMA_REF, role - assert config.role == "auxiliary" - - def test_auxiliary_falls_back_to_primary(self, tmp_path): - store = make_active_store(tmp_path / "db", auxiliary_default=False) - runtime = SnapshotRuntime(store) - config = runtime.resolve_role_config("auxiliary") +class TestResolveDefaultConfig: + def test_resolves_primary_default(self, runtime): + config = runtime.resolve_default_config() assert config.model_ref == ZHIPU_REF assert config.role == "primary" @@ -100,11 +86,6 @@ class TestBuildDefaultRoleModel: assert model.temperature == 0.7 assert model.top_p == 0.95 - def test_auxiliary_builds_ollama_without_credential(self, runtime): - model = runtime.build_default_role_model("auxiliary") - assert isinstance(model, ChatOllama) - assert model.model == "qwen3" - def test_openai_compatible_never_reads_env_api_key(self, runtime, monkeypatch): """The credential comes from the frozen auth_ref, never OPENAI_API_KEY.""" monkeypatch.setenv("OPENAI_API_KEY", "sk-env-must-not-leak") @@ -120,20 +101,6 @@ class TestBuildRoleModelFromSnapshot: assert model.model_name == "glm-5.2" assert model.openai_api_key.get_secret_value() == ZHIPU_SECRET - def test_auxiliary_roles_use_snapshot_auxiliary(self, store, runtime): - snapshot = make_snapshot(store) - for role in ("auxiliary", "summary", "tool_selector"): - model = runtime.build_role_model(snapshot, role) - assert isinstance(model, ChatOllama), role - assert model.model == "qwen3" - - def test_auxiliary_roles_fall_back_to_snapshot_primary(self, tmp_path): - store = make_active_store(tmp_path / "db", auxiliary_default=False) - runtime = SnapshotRuntime(store) - snapshot = make_snapshot(store) - model = runtime.build_role_model(snapshot, "summary") - assert isinstance(model, ChatOpenAI) - assert model.model_name == "glm-5.2" class TestSharedDefaultRuntime: @@ -213,7 +180,10 @@ class TestEndpointPolicyWiring: ) ) runtime = SnapshotRuntime(store, endpoint_policy=policy) - model = runtime.build_default_role_model("auxiliary") + # The loopback ollama provider builds only because its base URL is a + # registered development endpoint. + config = runtime.resolver.resolve(OLLAMA_REF, "primary") + model = runtime._build(config, "") assert isinstance(model, ChatOllama) @@ -233,9 +203,8 @@ class TestCreateLocalSnapshot: assert snapshot.thread_id == "cli-thread-1" assert snapshot.deployment_id == runtime.local_deployment_id assert snapshot.payload.model_selection_revision == 0 - # primary=None inherits — the registry defaults are frozen at creation. + # primary=None inherits — the registry default is frozen at creation. assert config_for_role(snapshot, "primary").model_ref == ZHIPU_REF - assert config_for_role(snapshot, "auxiliary").model_ref == OLLAMA_REF def test_same_run_request_id_reuses_snapshot(self, store, runtime): first = runtime.create_local_snapshot("t-1", run_request_id="req-1") diff --git a/tests/test_snapshots.py b/tests/test_snapshots.py index 1b6b8ba..e503bda 100644 --- a/tests/test_snapshots.py +++ b/tests/test_snapshots.py @@ -22,6 +22,7 @@ from EvoScientist.model_registry.errors import ( SNAPSHOT_ALREADY_BOUND, SNAPSHOT_EXPIRED, SNAPSHOT_NOT_FOUND, + UNSUPPORTED_RUNTIME_PARAMETER, ModelRegistryError, ) from EvoScientist.model_registry.hashing import configuration_hash @@ -120,7 +121,7 @@ def _ollama_provider(): } -def _registry(*, auxiliary_default=True) -> RegistryV4: +def _registry() -> RegistryV4: return RegistryV4.model_validate( { "version": 4, @@ -128,11 +129,6 @@ def _registry(*, auxiliary_default=True) -> RegistryV4: "state": "bootstrap", "defaults": { "primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, - "auxiliary": ( - {"provider_id": "local-ollama", "model_key": "qwen3"} - if auxiliary_default - else None - ), }, "providers": [_zhipu_provider(), _ollama_provider()], } @@ -190,16 +186,16 @@ def _request(**overrides): "deployment_id": "local", "model_selection_revision": 0, "primary": None, - "auxiliary": None, } payload.update(overrides) return SnapshotCreateRequest.model_validate(payload) -# SHA-256 of the canonical JSON '{"auxiliary":null,"primary":null}' — the -# inherit/inherit selection. Pinned as a literal to lock the wire contract. +# SHA-256 of the canonical JSON '{"primary":null,"reasoning_effort":null}' — +# the inherit selection with no effort override. Pinned as a literal to lock +# the wire contract. INHERIT_SELECTION_HASH = ( - "697c046214ccc3ddee1018af7eb6c21dbd5bfd01fbca9cb594ffc273deb55abc" + "03e47e7f41396b231bd558734715697a5aebfa114ccfd861a0f7e824e32dac8d" ) @@ -215,27 +211,69 @@ class TestCreate: assert payload.model_selection_revision == 4 assert payload.primary.model_ref == ZHIPU_REF assert payload.primary.role == "primary" - assert payload.auxiliary is not None - assert payload.auxiliary.model_ref == OLLAMA_REF - assert payload.auxiliary.role == "auxiliary" - def test_inherit_without_auxiliary_default(self, active_store): + def test_explicit_selection_freezes_primary(self, service): + creation = service.create(_request(primary=OLLAMA_REF)) + payload = creation.snapshot.payload + assert payload.primary.model_ref == OLLAMA_REF + + def test_reasoning_effort_override_freezes_into_primary(self, active_store): + # glm-5.2's model-specific contract rejects reasoning overrides, so + # add a generic-contract model on the same provider to exercise the + # override path (disabled → verify → enable, the standard flow). store = active_store registry = store.load_registry() - store.save_registry( + payload = registry.model_dump(mode="json") + payload["providers"][0]["models"].append( + { + "key": "glm-air", + "name": "GLM Air", + "upstream_model_id": "glm-air", + "enabled": False, + "runtime": _model_runtime(), + } + ) + registry = store.save_registry( expected_revision=registry.revision, - registry=_registry(auxiliary_default=False), + registry=RegistryV4.model_validate(payload), + ) + _verify(store, registry, "zhipu-glm", "glm-air") + payload = registry.model_dump(mode="json") + payload["providers"][0]["models"][1]["enabled"] = True + registry = store.save_registry( + expected_revision=registry.revision, + registry=RegistryV4.model_validate(payload), ) service = SnapshotService(store, ModelRegistryResolver(store)) - creation = service.create(_request()) - assert creation.snapshot.payload.auxiliary is None + creation = service.create( + _request( + primary={"provider_id": "zhipu-glm", "model_key": "glm-air"}, + reasoning_effort="high", + ) + ) + result = creation.snapshot.payload + assert result.primary.request_options.reasoning_effort == "high" - def test_explicit_selection_freezes_both_roles(self, service): - creation = service.create(_request(primary=ZHIPU_REF, auxiliary=OLLAMA_REF)) - payload = creation.snapshot.payload - assert payload.primary.model_ref == ZHIPU_REF - assert payload.auxiliary is not None - assert payload.auxiliary.model_ref == OLLAMA_REF + def test_reasoning_effort_override_rejected_by_model_contract(self, service): + # glm-5.2's model-specific contract does not accept reasoning_effort. + with pytest.raises(ModelRegistryError) as excinfo: + service.create(_request(reasoning_effort="high")) + assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER + + def test_reasoning_effort_override_changes_selection_hash(self, service): + plain = service.create(_request()) + assert compute_selection_hash(None, "low") != plain.snapshot.selection_hash + assert compute_selection_hash(None, "low") != INHERIT_SELECTION_HASH + # The hash covers the raw field, so an explicit ``auto`` differs from + # no override; both resolve to the same frozen configuration. + assert compute_selection_hash(None, "auto") != INHERIT_SELECTION_HASH + + def test_reasoning_effort_override_rejected_for_unsupported_adapter( + self, service + ): + with pytest.raises(ModelRegistryError) as excinfo: + service.create(_request(primary=OLLAMA_REF, reasoning_effort="high")) + assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER def test_bootstrap_registry_is_not_ready(self, store): service = SnapshotService(store, ModelRegistryResolver(store)) @@ -247,13 +285,13 @@ class TestCreate: def test_selection_hash_uses_pre_resolution_semantics(self, service): creation = service.create(_request()) snapshot = creation.snapshot - # The inherit/inherit hash is pinned to a known literal (computed - # offline from the canonical JSON) so the contract can't drift - # together with a reimplemented expectation. + # The inherit hash is pinned to a known literal (computed offline + # from the canonical JSON) so the contract can't drift together with + # a reimplemented expectation. assert snapshot.selection_hash == INHERIT_SELECTION_HASH - assert compute_selection_hash(None, None) == INHERIT_SELECTION_HASH - # An explicit selection equal to the defaults still hashes differently. - assert compute_selection_hash(ZHIPU_REF, OLLAMA_REF) != INHERIT_SELECTION_HASH + assert compute_selection_hash(None) == INHERIT_SELECTION_HASH + # An explicit selection equal to the default still hashes differently. + assert compute_selection_hash(ZHIPU_REF) != INHERIT_SELECTION_HASH def test_selection_hash_ignores_selection_revision(self, service): first = service.create(_request(model_selection_revision=4)) @@ -524,25 +562,9 @@ class TestCleanupExpired: class TestRoleMapping: - def test_auxiliary_roles_fall_back_to_primary(self, active_store): - store = active_store - registry = store.load_registry() - store.save_registry( - expected_revision=registry.revision, - registry=_registry(auxiliary_default=False), - ) - service = SnapshotService(store, ModelRegistryResolver(store)) - snapshot = service.create(_request()).snapshot - for role in ("primary", "auxiliary", "summary", "tool_selector"): - assert config_for_role(snapshot, role).model_ref == ZHIPU_REF - - def test_auxiliary_roles_use_frozen_auxiliary(self, service): + def test_primary_role_maps_to_frozen_primary(self, service): snapshot = service.create(_request()).snapshot assert config_for_role(snapshot, "primary").model_ref == ZHIPU_REF - for role in ("auxiliary", "summary", "tool_selector"): - config = config_for_role(snapshot, role) - assert config.model_ref == OLLAMA_REF - assert config.role == "auxiliary" def test_unknown_role_rejected(self, service): snapshot = service.create(_request()).snapshot @@ -574,12 +596,6 @@ class TestSnapshotCredentials: fresh_service = SnapshotService(fresh_store, ModelRegistryResolver(fresh_store)) assert fresh_service.resolve_snapshot_credential(snapshot, "primary") == SECRET - def test_auxiliary_role_resolves_auxiliary_credential(self, service): - snapshot = service.create(_request()).snapshot - # The frozen auxiliary is the mode=none ollama model: no secret. - assert service.resolve_snapshot_credential(snapshot, "summary") == "" - assert service.resolve_snapshot_credential(snapshot, "tool_selector") == "" - def test_mode_none_returns_empty_string(self, service): snapshot = service.create(_request(primary=OLLAMA_REF)).snapshot assert service.resolve_snapshot_credential(snapshot, "primary") == "" @@ -599,10 +615,10 @@ class TestPublicView: "max_output_tokens": 32768, "temperature": 0.7, "top_p": 0.95, + "reasoning_effort": "auto", "timeout_seconds": 120, "max_retries": 2, } - assert view["auxiliary"]["provider_id"] == "local-ollama" def test_never_contains_secret_or_base_url(self, service): snapshot = service.create(_request()).snapshot diff --git a/tests/test_tool_selector_middleware.py b/tests/test_tool_selector_middleware.py index fb43e4a..98685e8 100644 --- a/tests/test_tool_selector_middleware.py +++ b/tests/test_tool_selector_middleware.py @@ -55,7 +55,10 @@ def _factory_patches(): return_value=MagicMock(), create=True, ), - patch("EvoScientist.EvoScientist._ensure_chat_model", return_value=MagicMock()), + patch( + "EvoScientist.middleware.configurable_model.resolve_snapshot_model", + return_value=MagicMock(), + ), patch( "langchain.agents.middleware.LLMToolSelectorMiddleware", return_value=MagicMock(), @@ -252,8 +255,10 @@ def test_selector_always_includes_available_memory_tools(): ] -def test_selector_resolved_once_across_repeated_requests(): - """Agent tools are stable, so build the selector once and reuse it.""" +def test_selector_rebuilt_per_request_with_stable_always_include(): + """The selector is rebuilt per call (its helper model is resolved from + the run snapshot, which varies per run), but the always-include names + derived from the request's tools stay stable.""" mock_selector = MagicMock() mock_selector.wrap_model_call.side_effect = lambda request, handler: handler( request @@ -274,7 +279,9 @@ def test_selector_resolved_once_across_repeated_requests(): for _ in range(3): cond.wrap_model_call(_request(tools), MagicMock()) - selector_factory.assert_called_once_with(["search_observations", "think_tool"]) + assert selector_factory.call_count == 3 + for call in selector_factory.call_args_list: + assert call.args[0] == ["search_observations", "think_tool"] assert mock_selector.wrap_model_call.call_count == 3 @@ -305,16 +312,13 @@ def test_tracker_captures_tools(): "EvoScientist.middleware.create_tool_selector_middleware", side_effect=lambda *a, **kw: _patched_create(), ) -@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model") -@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._compile_time_role_model") @patch("EvoScientist.EvoScientist._ensure_config") -def test_default_middleware_includes_tool_selector(mock_config, mock_model, mock_aux, mock_ts): +def test_default_middleware_includes_tool_selector(mock_config, mock_model, mock_ts): mock_model.return_value = _mock_model() cfg = MagicMock() cfg.enable_ask_user = False cfg.auto_approve = False - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg from EvoScientist.EvoScientist import _get_default_middleware @@ -342,17 +346,14 @@ def test_subagent_no_tool_selector(mock_model): "EvoScientist.middleware.create_tool_selector_middleware", side_effect=lambda *a, **kw: _patched_create(), ) -@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model") -@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._compile_time_role_model") @patch("EvoScientist.EvoScientist._ensure_config") -def test_tool_selector_ordering(mock_config, mock_model, mock_aux, mock_ts): +def test_tool_selector_ordering(mock_config, mock_model, mock_ts): """ToolSelector should come after ToolErrorHandler and before Memory.""" mock_model.return_value = _mock_model() cfg = MagicMock() cfg.enable_ask_user = False cfg.auto_approve = False - cfg.auxiliary_model = "" - cfg.auxiliary_provider = "" mock_config.return_value = cfg from EvoScientist.EvoScientist import _get_default_middleware