feat(runtime)!: remove auxiliary model role, resolve all roles from run snapshot

ModelRole collapses to "primary": every role (main, tool selector, memory
agents, subagents, summarizer) resolves to the snapshot's frozen primary
model, per design 6.1/8.3 — users typically configure a single usable LLM,
so compile-time auxiliary bindings were bypassing run snapshots and
mis-attributing usage. Legacy auxiliary keys in stored snapshots, registry
JSON, and thread metadata are tolerated on read and dropped.

BREAKING CHANGE: ThreadModelSelection no longer carries an auxiliary ref;
snapshot selection_hash is computed over {primary, reasoning_effort} only;
ConfigurableModelMiddleware(role="auxiliary") is rejected.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
m4
2026-07-23 10:18:44 +08:00
parent 1a01fb5d74
commit bb9bed82e1
33 changed files with 444 additions and 690 deletions
+8 -56
View File
@@ -56,19 +56,11 @@ DEFAULT_SKILL_SOURCES = ("/skills/",)
_config = None
_chat_model = None
# Track the (provider_id, model_key, registry_revision) binding of
# _chat_model so the cache invalidates when the registry defaults change.
# _chat_model so the cache invalidates when the registry default changes.
# The compile-time binding is only a placeholder — per-run resolution
# always comes from the run snapshot via ConfigurableModelMiddleware.
_chat_model_key: tuple[str, str, int] | None = None
# Auxiliary model for background/helper LLM calls (memory workers + main-agent
# tool selector). Cached separately from the main model; resolved through the
# registry ``auxiliary`` role mapping and falls back to the main instance when
# no distinct auxiliary default is configured (see
# _ensure_auxiliary_chat_model).
_auxiliary_chat_model = None
_auxiliary_chat_model_key: tuple[str, str, int] | None = None
# Cache MCP tools by the effective config signature to avoid reconnecting
# to MCP servers on every `/new` when config is unchanged.
_MCP_TOOLS_CACHE_KEY: str | None = None
@@ -148,45 +140,13 @@ def _ensure_chat_model():
from .model_registry.runtime import get_snapshot_runtime
runtime = get_snapshot_runtime()
primary_ref, _, revision = runtime.registry_defaults()
primary_ref, revision = runtime.registry_default()
key = (primary_ref.provider_id, primary_ref.model_key, revision)
if _chat_model is None or _chat_model_key != key:
_replace_chat_model(runtime.build_default_role_model("primary"), key)
return _chat_model
def _ensure_auxiliary_chat_model():
"""Return the auxiliary chat model for background/helper LLM calls.
Resolves the ``auxiliary`` role through the model registry role mapping
(design doc 6.1): ``registry.defaults.auxiliary ?? defaults.primary``,
constructed via ``build_chat_model`` from the resolved configuration.
When no auxiliary default is configured — or it matches the primary
default — returns the main ``_ensure_chat_model()`` instance directly,
so no second client is built. Otherwise the model is cached under its
``(provider_id, model_key, registry_revision)`` key. A bootstrap
registry propagates ``MODEL_REGISTRY_NOT_READY`` from
``_ensure_chat_model()``.
"""
global _auxiliary_chat_model, _auxiliary_chat_model_key
from .model_registry.runtime import get_snapshot_runtime
runtime = get_snapshot_runtime()
primary_ref, auxiliary_ref, revision = runtime.registry_defaults()
if auxiliary_ref is None or auxiliary_ref == primary_ref:
return _ensure_chat_model()
key: tuple[str, str, int] = (
auxiliary_ref.provider_id,
auxiliary_ref.model_key,
revision,
)
if _auxiliary_chat_model is None or _auxiliary_chat_model_key != key:
_auxiliary_chat_model = runtime.build_default_role_model("auxiliary")
_auxiliary_chat_model_key = key
return _auxiliary_chat_model
def _compile_time_role_model(role: str = "primary"):
"""Return the compile-time model binding for graph construction.
@@ -693,9 +653,8 @@ def _get_default_middleware(
memory_source_agent: Attribution name for profile/observation writes.
Async sub-agent factories pass their deployed agent name here.
snapshot_role: The model role ``ConfigurableModelMiddleware``
resolves from the run snapshot (``primary`` for the main agent
and working sub-agents, ``auxiliary`` for unattended helper
agents such as the scheduler).
resolves from the run snapshot; every role maps to the
snapshot's frozen primary (design doc 6.1).
"""
from .middleware import (
ConfigurableModelMiddleware,
@@ -746,19 +705,12 @@ def _get_default_middleware(
),
memory_scheduler=memory_scheduler,
)
# Main-agent tool selection may use the auxiliary model; async sub-agents
# keep their own model (they do real work, not a one-off helper call).
# Main-agent tool selection resolves its helper model from the run
# snapshot on every call (model=None below); async sub-agents and the
# pure path (explicit model + config) keep their threaded model.
# context_editing stays on the main model — its model only sizes the
# context-window trigger for the main agent's own history.
if for_async_subagent:
tool_selector_model = model
elif chat_model is None:
tool_selector_model = _compile_time_role_model("auxiliary")
else:
# Pure path (explicit model + config): the threaded model stands in
# for tool selection at build time; per-call resolution still comes
# from the run snapshot.
tool_selector_model = model
tool_selector_model = None if (not for_async_subagent and chat_model is None) else model
mw = [
ConfigurableModelMiddleware(role=snapshot_role),
create_message_budget_middleware(model, backend, snapshot_role=snapshot_role),
+1 -1
View File
@@ -73,7 +73,7 @@ def current_model_label() -> str:
try:
from ..model_registry.runtime import get_snapshot_runtime
primary, _, _ = get_snapshot_runtime().registry_defaults()
primary, _ = get_snapshot_runtime().registry_default()
except Exception:
return "unconfigured"
return f"{primary.provider_id}/{primary.model_key}"
+1 -1
View File
@@ -1374,7 +1374,7 @@ def _startup_gates() -> None:
try:
from ..model_registry.runtime import get_snapshot_runtime
get_snapshot_runtime().registry_defaults()
get_snapshot_runtime().registry_default()
except ModelRegistryError as exc:
if exc.code != MODEL_REGISTRY_NOT_READY:
raise
+6 -2
View File
@@ -88,6 +88,7 @@ def build_memory_agent_graph(
from ...backends import build_memory_agent_backend
from ...EvoScientist import _compile_time_role_model
from ...middleware.configurable_model import ConfigurableModelMiddleware
kwargs: dict[str, Any] = {}
if response_format is not None:
@@ -101,11 +102,14 @@ def build_memory_agent_graph(
agent = create_deep_agent(
name=name,
model=_compile_time_role_model("auxiliary"),
model=_compile_time_role_model("primary"),
system_prompt=system_prompt,
tools=list(tools),
backend=backend,
middleware=list(middleware),
# The compile-time model is only a placeholder: the middleware
# re-resolves every call from the run snapshot (lazily healing
# background runs to a local snapshot of the registry default).
middleware=[ConfigurableModelMiddleware(), *middleware],
subagents=[],
skills=skills,
**kwargs,
+36 -5
View File
@@ -154,11 +154,8 @@ class ConfigurableModelMiddleware(AgentMiddleware):
"""Re-resolve the chat model from the run snapshot on every call.
``role`` selects which frozen configuration of the snapshot feeds this
agent's model calls: ``primary`` for the main agent and working
sub-agents, ``auxiliary`` for unattended helper agents (e.g. the
scheduler). The section 6.1 mapping (``auxiliary``/``summary``/
``tool_selector`` → ``snapshot.auxiliary ?? snapshot.primary``) is
applied by the snapshot layer.
agent's model calls; every role currently maps to the snapshot's frozen
primary (section 6.1).
A per-instance cache keyed by snapshot ID avoids rebuilding the model on
every call within a run; snapshots are immutable once created, so the
@@ -266,3 +263,37 @@ class ConfigurableModelMiddleware(AgentMiddleware):
# thread-pool overhead is irrelevant once warm.
new_model = await asyncio.to_thread(self._resolve)
return await handler(request.override(model=new_model))
# --- per-call resolution for in-run helper models -----------------------------
# In-run helper LLM calls (the tool selector's internal selection call) do
# not pass through an agent's model request, so ``request.override`` cannot
# reach them. They resolve the run snapshot's primary model through this
# helper instead; background runs without an explicit snapshot are healed
# through the same lazy local-snapshot path as the middleware.
_helper_model_cache: dict[str, Any] = {}
_helper_model_cache_lock = threading.Lock()
def resolve_snapshot_model(runtime: SnapshotRuntime | None = None) -> Any:
"""Resolve the active run's snapshot primary model (cached per snapshot).
Uses the same binding rules as ``ConfigurableModelMiddleware``: the
run's explicit ``runtime_snapshot_id`` wins, otherwise a local snapshot
is lazily created for the run's own thread. Must be called from inside
a runnable context (or a test that installed one).
"""
configurable = _current_configurable()
check_no_outside_snapshot_model_config(configurable)
rt = runtime if runtime is not None else get_snapshot_runtime()
snapshot_id, thread_id = ensure_snapshot_binding(configurable, rt)
snapshot = rt.get_snapshot_for_run(snapshot_id, thread_id=thread_id)
with _helper_model_cache_lock:
cached = _helper_model_cache.get(snapshot.snapshot_id)
if cached is not None:
return cached
model = rt.build_role_model(snapshot, "primary")
with _helper_model_cache_lock:
_helper_model_cache[snapshot.snapshot_id] = model
return model
+11 -13
View File
@@ -10,13 +10,12 @@ Snapshot mode (design doc 6.5, 8.3): every run carries — or lazily creates,
via the section 8.1 local entry convention — a run snapshot; the input
limit and the three fixed reserves come from the frozen
``ResolvedModelConfig.budget`` of the middleware's own ``snapshot_role``
(``primary`` for the main agent and working sub-agents, ``auxiliary`` for
unattended helpers such as the scheduler). There is no 32K default and no
compile-time profile fallback: a bootstrap registry fails closed with
``MODEL_REGISTRY_NOT_READY``. The per-call message budget is recomputed on
every invocation from those frozen reserves with the current
``has_tools``/``has_attachments`` mode; it is never carried over from a
previous call:
(every role maps to the snapshot's frozen primary, section 6.1). There is
no 32K default and no compile-time profile fallback: a bootstrap registry
fails closed with ``MODEL_REGISTRY_NOT_READY``. The per-call message budget
is recomputed on every invocation from those frozen reserves with the
current ``has_tools``/``has_attachments`` mode; it is never carried over
from a previous call:
message_budget = resolved_input_limit
- fixed_system_reserve_tokens
@@ -176,10 +175,9 @@ class MessageBudgetMiddleware:
agent has tool capability and a configured toolset. Decided
once here, never from a single request's bound tool count.
snapshot_role: The snapshot role whose frozen limits size this
agent's budget (``primary`` for the main agent and working
sub-agents, ``auxiliary`` for unattended helper agents such
as the scheduler). The summarizer keeps using the snapshot's
``summary`` role mapping regardless.
agent's budget; every role maps to the snapshot's frozen
primary (section 6.1). The summarizer uses the same frozen
primary.
runtime: Snapshot runtime override; defaults to the shared
process runtime (tests inject an isolated one).
"""
@@ -255,14 +253,14 @@ class MessageBudgetMiddleware:
@property
def model(self) -> Any: # type: ignore[override]
"""Summarizer model: the snapshot's ``summary`` role mapping."""
"""Summarizer model: the snapshot's frozen primary."""
snapshot = self._snapshot()
with self._model_cache_lock:
cached = self._snapshot_model_cache.get(snapshot.snapshot_id)
if cached is not None:
return cached
resolved = self._snapshot_runtime().build_role_model(
snapshot, "summary"
snapshot, "primary"
)
with self._model_cache_lock:
self._snapshot_model_cache[snapshot.snapshot_id] = resolved
+50 -35
View File
@@ -93,15 +93,12 @@ class _ConditionalToolSelectorMiddleware(AgentMiddleware):
self._threshold = threshold
self._always_include = always_include or frozenset()
self._track_stream_selection = track_stream_selection
# Agent tools are fixed after graph construction, so the filtered
# always-include set is stable for this middleware instance.
self._selector: AgentMiddleware | None = None
def _build_selector(self, request: ModelRequest) -> AgentMiddleware:
if self._selector is None:
names = _available_always_include(request.tools, self._always_include)
self._selector = self._selector_factory(names)
return self._selector
# Built per call: the selector's helper model is resolved from the
# run snapshot, which varies from run to run.
names = _available_always_include(request.tools, self._always_include)
return self._selector_factory(names)
def wrap_model_call(
self,
@@ -233,8 +230,10 @@ def create_tool_selector_middleware(
names for the main-agent stream UI when ``track_stream_selection`` is true
Args:
model: Chat model for tool selection. If *None*, the default
model is resolved via ``_ensure_chat_model()``.
model: Chat model for tool selection. If *None*, the helper model
is resolved from the active run's snapshot on every call
(``resolve_snapshot_model``); the registry default only applies
to snapshot-less background runs via the lazy local snapshot.
threshold: Minimum number of tools to trigger selection.
Default 26. Set to 0 to always run selection.
track_stream_selection: Whether to update process-global stream/UI
@@ -253,34 +252,50 @@ def create_tool_selector_middleware(
from .utils import disable_thinking
if model is None:
from EvoScientist.EvoScientist import _ensure_chat_model
def tag_selector_model(base: BaseChatModel) -> BaseChatModel:
safe_model = disable_thinking(base)
selector_model = safe_model
from EvoScientist.usage.callback import usage_tracking_enabled
model = _ensure_chat_model()
safe_model = disable_thinking(model)
selector_model = safe_model
from EvoScientist.usage.callback import usage_tracking_enabled
if usage_tracking_enabled():
try:
selector_model = safe_model.model_copy(
update={
"metadata": {
**(safe_model.metadata or {}),
"usage_scope": "tool_selector",
if usage_tracking_enabled():
try:
selector_model = safe_model.model_copy(
update={
"metadata": {
**(safe_model.metadata or {}),
"usage_scope": "tool_selector",
}
}
}
)
if not isinstance(selector_model, BaseChatModel):
raise TypeError("selector model copy is not a BaseChatModel")
except Exception:
# The model factory preflights this path before enabling its callback.
# Keep the selector operational even for an unusual third-party model.
from EvoScientist.usage.spool import mark_tracking_degraded
)
if not isinstance(selector_model, BaseChatModel):
raise TypeError("selector model copy is not a BaseChatModel")
except Exception:
# The model factory preflights this path before enabling its
# callback. Keep the selector operational even for an unusual
# third-party model.
from EvoScientist.usage.spool import mark_tracking_degraded
mark_tracking_degraded("selector_model_copy_unsupported")
selector_model = safe_model
logger.exception("Could not attach Tool Selector usage scope")
mark_tracking_degraded("selector_model_copy_unsupported")
selector_model = safe_model
logger.exception("Could not attach Tool Selector usage scope")
return selector_model
# The resolved snapshot model is cached per snapshot upstream, so the
# tagged copy can be cached by the base model's identity.
tagged_cache: dict[int, BaseChatModel] = {}
def selector_model() -> BaseChatModel:
if model is not None:
base = model
else:
from .configurable_model import resolve_snapshot_model
base = resolve_snapshot_model()
tagged = tagged_cache.get(id(base))
if tagged is None:
tagged = tag_selector_model(base)
tagged_cache[id(base)] = tagged
return tagged
system_prompt = (
"You are selecting tools for a scientific research agent. "
@@ -293,7 +308,7 @@ def create_tool_selector_middleware(
def selector_factory(always_include: list[str]) -> AgentMiddleware:
return LLMToolSelectorMiddleware(
model=selector_model,
model=selector_model(),
system_prompt=system_prompt,
always_include=always_include,
)
+11 -1
View File
@@ -581,6 +581,8 @@ def resolve_parameters(
provider: ProviderConfig,
model: ModelConfig,
spec: AdapterParameterSpec,
*,
reasoning_effort_override: ReasoningEffort | None = None,
) -> ResolvedParameters:
"""Resolve a model's runtime parameters against the matched contract.
@@ -589,6 +591,10 @@ def resolve_parameters(
zero; ``reasoning_effort=auto`` inherits and is omitted when still auto)
and the save-time contract checks. Raises ``ModelRegistryError`` with a
stable section 9.5 code on any violation.
``reasoning_effort_override`` (snapshot creation only) replaces the
model's configured effort before inheritance; it flows through the same
contract rule, so unsupported adapters still reject non-``auto`` values.
"""
auth_spec = spec.auth_specs.get(provider.auth.mode)
if auth_spec is None:
@@ -649,7 +655,11 @@ def resolve_parameters(
top_p = provider.runtime.default_top_p
top_p = _resolve_nullable_parameter("top_p", _parameter_rule(spec, "top_p"), top_p)
reasoning_effort: ReasoningEffort = model.runtime.reasoning_effort
reasoning_effort: ReasoningEffort = (
reasoning_effort_override
if reasoning_effort_override not in (None, "auto")
else model.runtime.reasoning_effort
)
if reasoning_effort == "auto":
reasoning_effort = provider.runtime.default_reasoning_effort
reasoning_effort = _resolve_nullable_parameter(
+5 -2
View File
@@ -79,6 +79,7 @@ from .schemas import (
ModelAvailability,
ModelRef,
NonEmptyString,
RegistryDefaults,
RegistryV4,
)
from .snapshots import (
@@ -164,6 +165,9 @@ class GetSelectableModelsResponse(BaseModel):
"""The section 9.1 selector response: only selectable enabled models."""
models: list[SelectableModel]
# Lets the selector display the registry default before the first turn,
# when no run snapshot exists yet to resolve `inherit` against.
defaults: RegistryDefaults = Field(default_factory=RegistryDefaults)
class SnapshotBindRequest(BaseModel):
@@ -197,7 +201,6 @@ class SnapshotPublicResponse(BaseModel):
snapshot_id: str
registry_revision: PositiveInt
primary: ResolvedModelPublic
auxiliary: ResolvedModelPublic | None = None
# --- services bundle --------------------------------------------------------
@@ -754,7 +757,7 @@ class ModelRegistryHttpApi:
effective_capabilities=item.effective_capabilities,
)
)
return GetSelectableModelsResponse(models=models)
return GetSelectableModelsResponse(models=models, defaults=registry.defaults)
@staticmethod
def _create_snapshot(
+21 -33
View File
@@ -477,6 +477,9 @@
"GetSelectableModelsResponse": {
"description": "The section 9.1 selector response: only selectable enabled models.",
"properties": {
"defaults": {
"$ref": "#/components/schemas/RegistryDefaults"
},
"models": {
"items": {
"$ref": "#/components/schemas/SelectableModel"
@@ -1034,17 +1037,6 @@
},
"RegistryDefaults": {
"properties": {
"auxiliary": {
"anyOf": [
{
"$ref": "#/components/schemas/ModelRef"
},
{
"type": "null"
}
],
"default": null
},
"primary": {
"anyOf": [
{
@@ -1221,17 +1213,6 @@
"SnapshotCreateRequest": {
"description": "The section 8.2 snapshot creation request body.",
"properties": {
"auxiliary": {
"anyOf": [
{
"$ref": "#/components/schemas/ModelRef"
},
{
"type": "null"
}
],
"default": null
},
"deployment_id": {
"minLength": 1,
"title": "Deployment Id",
@@ -1254,6 +1235,24 @@
],
"default": null
},
"reasoning_effort": {
"anyOf": [
{
"enum": [
"auto",
"low",
"medium",
"high"
],
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Reasoning Effort"
},
"run_request_id": {
"minLength": 1,
"title": "Run Request Id",
@@ -1276,17 +1275,6 @@
"SnapshotPublicResponse": {
"description": "The section 8.2 public snapshot diagnostic subset.",
"properties": {
"auxiliary": {
"anyOf": [
{
"$ref": "#/components/schemas/ResolvedModelPublic"
},
{
"type": "null"
}
],
"default": null
},
"primary": {
"$ref": "#/components/schemas/ResolvedModelPublic"
},
+13 -2
View File
@@ -50,6 +50,7 @@ from .schemas import (
ModelRef,
ModelRole,
ProviderConfig,
ReasoningEffort,
RegistryV4,
RequestOptions,
ResolvedModelConfig,
@@ -102,9 +103,16 @@ class ModelRegistryResolver:
role: ModelRole = "primary",
*,
registry: RegistryV4 | None = None,
reasoning_effort_override: ReasoningEffort | None = None,
) -> ResolvedModelConfig:
"""Resolve an enabled, verified model into its frozen run config."""
return self._resolve(model_ref, role, registry=registry, require_enabled=True)
return self._resolve(
model_ref,
role,
registry=registry,
require_enabled=True,
reasoning_effort_override=reasoning_effort_override,
)
def resolve_for_test(self, model_ref: ModelRef) -> ResolvedModelConfig:
"""Resolve for a provider test (section 9.4).
@@ -130,6 +138,7 @@ class ModelRegistryResolver:
registry: RegistryV4 | None,
require_enabled: bool,
require_verified: bool = True,
reasoning_effort_override: ReasoningEffort | None = None,
) -> ResolvedModelConfig:
_check_role(role)
if registry is None:
@@ -176,7 +185,9 @@ class ModelRegistryResolver:
# Save-time contract checks re-applied at resolve time: auth mode,
# credential reference, declared capabilities, and parameters.
parameters = resolve_parameters(provider, model, spec)
parameters = resolve_parameters(
provider, model, spec, reasoning_effort_override=reasoning_effort_override
)
auth_spec = spec.auth_specs[provider.auth.mode]
credential_revision: int | None = None
+32 -27
View File
@@ -1,17 +1,15 @@
"""Runtime-link glue for snapshot-driven model resolution (design doc 8.3).
``SnapshotRuntime`` bundles the store, resolver, and snapshot service into
the single accessor the running agent link (middleware, agent factories,
auxiliary roles) uses. It provides two resolution paths:
the single accessor the running agent link (middleware, agent factories)
uses. It provides two resolution paths:
- ``build_role_model(snapshot, role)`` — per-call construction from a frozen
run snapshot (section 6.1 role mapping, section 5.2 per-call credential
resolution). This is the only path the per-call middleware uses.
- ``build_default_role_model(role)`` — build-time construction from the
active registry's ``defaults`` for auxiliary roles that bind a model when
a graph is compiled (tool selector, scheduler, memory workers). Auxiliary,
summary, and tool-selector roles map to ``defaults.auxiliary ??
defaults.primary``; no free model/provider strings are consulted anywhere.
run snapshot (section 5.2 per-call credential resolution). Every role
resolves to the snapshot's frozen primary (section 6.1).
- ``build_default_role_model()`` — build-time construction from the active
registry's default primary, used only as the compile-time graph binding;
every run re-resolves its model from the run snapshot.
Every model is constructed through ``build_chat_model`` with both safe HTTP
clients (section 6.4). Ollama adapters additionally carry ``max_retries``
@@ -118,8 +116,8 @@ class SnapshotRuntime:
# --- registry-default (build-time) resolution ----------------------------
def registry_defaults(self) -> tuple[ModelRef, ModelRef | None, int]:
"""Return ``(primary, auxiliary, revision)`` for an active registry."""
def registry_default(self) -> tuple[ModelRef, int]:
"""Return ``(primary, revision)`` for an active registry."""
registry = self._store.load_registry()
if registry.state != "active" or registry.defaults.primary is None:
raise ModelRegistryError(
@@ -127,23 +125,16 @@ class SnapshotRuntime:
"The model registry is in bootstrap; configure and enable a "
"primary model first.",
)
return registry.defaults.primary, registry.defaults.auxiliary, registry.revision
return registry.defaults.primary, registry.revision
def resolve_role_config(self, role: ModelRole = "primary") -> ResolvedModelConfig:
"""Resolve a role against the registry defaults (section 6.1).
``primary`` resolves ``defaults.primary``; every auxiliary role
(``auxiliary``, ``summary``, ``tool_selector``) resolves
``defaults.auxiliary ?? defaults.primary``.
"""
primary, auxiliary, _ = self.registry_defaults()
if role == "primary" or auxiliary is None:
return self._resolver.resolve(primary, "primary")
return self._resolver.resolve(auxiliary, "auxiliary")
def resolve_default_config(self) -> ResolvedModelConfig:
"""Resolve the registry default primary (section 6.1)."""
primary, _ = self.registry_default()
return self._resolver.resolve(primary, "primary")
def build_default_role_model(self, role: ModelRole = "primary") -> BaseChatModel:
"""Build the chat model for a role from the registry defaults."""
config = self.resolve_role_config(role)
"""Build the chat model for a role from the registry default."""
config = self.resolve_default_config()
return self._build(config, self._resolve_credential(config))
# --- local entry points (design doc 8.1) ---------------------------------
@@ -182,7 +173,6 @@ class SnapshotRuntime:
deployment_id=self._local_deployment_id,
model_selection_revision=0,
primary=None,
auxiliary=None,
)
)
return creation.snapshot
@@ -207,7 +197,7 @@ class SnapshotRuntime:
config.client_options.max_retries if config.adapter_id == "ollama" else 0
)
timeout = config.client_options.timeout_seconds
return build_chat_model(
model = build_chat_model(
config,
http_client=build_safe_http_client(
self._endpoint_policy, timeout=timeout, retries=retries
@@ -217,6 +207,21 @@ class SnapshotRuntime:
),
credential=credential,
)
# Usage tracking is inert unless the launcher supplied the complete
# sink environment; the adapter contract revision stands in for the
# removed per-profile runtime revision.
from ..usage import UsageModelIdentity, attach_usage_callback
return attach_usage_callback(
model,
UsageModelIdentity(
provider_profile_id=config.model_ref.provider_id,
provider_revision=str(config.adapter_spec_revision),
provider_adapter=config.adapter_id,
model_alias=config.model_ref.model_key,
upstream_model_id=config.upstream_model_id,
),
)
# --- process-shared default runtime -------------------------------------------
+5 -10
View File
@@ -45,7 +45,7 @@ RegistryState = Literal["bootstrap", "active"]
LimitMode = Literal["combined", "input_only"]
LimitsStatus = Literal["confirmed", "needs_confirmation"]
LimitsSource = Literal["provider", "tested_contract", "user"]
ModelRole = Literal["primary", "auxiliary", "summary", "tool_selector"]
ModelRole = Literal["primary"]
class _FrozenModel(BaseModel):
@@ -61,7 +61,6 @@ class ModelRef(_FrozenModel):
class RegistryDefaults(BaseModel):
primary: ModelRef | None = None
auxiliary: ModelRef | None = None
class Capabilities(BaseModel):
@@ -175,22 +174,18 @@ class RegistryV4(BaseModel):
provider_ids = [provider.id for provider in self.providers]
if len(provider_ids) != len(set(provider_ids)):
raise ValueError("Provider IDs must be unique.")
for role, ref in (
("primary", self.defaults.primary),
("auxiliary", self.defaults.auxiliary),
):
if ref is None:
continue
ref = self.defaults.primary
if ref is not None:
target = self._locate(ref)
if target is None:
raise ValueError(
f"defaults.{role} references an unknown ModelRef "
f"defaults.primary references an unknown ModelRef "
f"{ref.provider_id!r}/{ref.model_key!r}."
)
provider, model = target
if self.state == "active" and not (provider.enabled and model.enabled):
raise ValueError(
f"defaults.{role} must reference an enabled model while "
"defaults.primary must reference an enabled model while "
"the registry is active."
)
if self.state == "active" and self.defaults.primary is None:
+45 -58
View File
@@ -2,7 +2,7 @@
``SnapshotService`` is the single implementation shared by the HTTP snapshot
API (Task 5) and the local CLI/channel/scheduler entry points (Task 7): it
freezes both roles' complete ``ResolvedModelConfig`` — adapter spec
freezes the primary role's complete ``ResolvedModelConfig`` — adapter spec
revision, fixed budget reserves, capabilities, and credential revisions —
into ``run_runtime_snapshots.payload_json``. Payloads and logs never carry
``secret_value``.
@@ -11,9 +11,9 @@ into ``run_runtime_snapshots.payload_json``. Payloads and logs never carry
maps to at most one non-terminal snapshot. A repeated request with the
same ``selection_hash`` returns the original snapshot; a different hash
raises ``RUN_REQUEST_CONFLICT``. ``selection_hash`` covers the
pre-resolution ``{primary, auxiliary}`` selection (inherit participates as
``null``); ``model_selection_revision`` is an audit field and never part
of the hash.
pre-resolution ``{primary, reasoning_effort}`` selection (inherit
participates as ``null``); ``model_selection_revision`` is an audit field
and never part of the hash.
- Lifecycle: ``prepared`` (TTL 15 minutes) → ``bound`` (retained 24 hours
after binding) → ``expired`` (terminal, via ``cleanup_expired``);
``prepared`` → ``aborted`` on run creation failure. Terminal snapshots
@@ -51,6 +51,7 @@ from .schemas import (
ModelRef,
ModelRole,
NonEmptyString,
ReasoningEffort,
ResolvedModelConfig,
)
from .store import ModelRuntimeStore
@@ -69,9 +70,12 @@ class SnapshotCreateRequest(BaseModel):
thread_id: NonEmptyString
deployment_id: NonEmptyString
model_selection_revision: NonNegativeInt = 0
# ``None`` means inherit: the registry defaults are resolved at creation.
# ``None`` means inherit: the registry default is resolved at creation.
primary: ModelRef | None = None
auxiliary: ModelRef | None = None
# Per-run reasoning-effort override (thread-level selection);
# ``None``/``auto`` keeps the registry-configured effort. Unsupported
# adapters reject it.
reasoning_effort: ReasoningEffort | None = None
class SnapshotPayload(BaseModel):
@@ -80,7 +84,6 @@ class SnapshotPayload(BaseModel):
registry_revision: PositiveInt
model_selection_revision: NonNegativeInt
primary: ResolvedModelConfig
auxiliary: ResolvedModelConfig | None = None
class RuntimeSnapshot(BaseModel):
@@ -120,16 +123,20 @@ class SnapshotCreation(BaseModel):
created: bool
def compute_selection_hash(primary: ModelRef | None, auxiliary: ModelRef | None) -> str:
def compute_selection_hash(
primary: ModelRef | None,
reasoning_effort: ReasoningEffort | None = None,
) -> str:
"""Hash the pre-resolution selection; inherit participates as ``null``."""
def entry(ref: ModelRef | None) -> dict[str, str] | None:
if ref is None:
return None
return {"provider_id": ref.provider_id, "model_key": ref.model_key}
encoded = json.dumps(
{"primary": entry(primary), "auxiliary": entry(auxiliary)},
{
"primary": (
None
if primary is None
else {"provider_id": primary.provider_id, "model_key": primary.model_key}
),
"reasoning_effort": reasoning_effort,
},
sort_keys=True,
separators=(",", ":"),
)
@@ -137,18 +144,12 @@ def compute_selection_hash(primary: ModelRef | None, auxiliary: ModelRef | None)
def config_for_role(snapshot: RuntimeSnapshot, role: ModelRole) -> ResolvedModelConfig:
"""The section 6.1 role mapping; no other mapping may be guessed.
``primary`` → ``snapshot.primary``; ``auxiliary``, ``summary``, and
``tool_selector`` → ``snapshot.auxiliary ?? snapshot.primary``.
"""
"""Every role resolves to the snapshot's frozen primary (section 6.1)."""
if role not in _MODEL_ROLES:
raise ValueError(
f"Unknown model role {role!r}; expected one of {list(_MODEL_ROLES)}."
)
if role == "primary" or snapshot.payload.auxiliary is None:
return snapshot.payload.primary
return snapshot.payload.auxiliary
return snapshot.payload.primary
def public_snapshot_view(snapshot: RuntimeSnapshot) -> dict[str, Any]:
@@ -163,17 +164,16 @@ def public_snapshot_view(snapshot: RuntimeSnapshot) -> dict[str, Any]:
"max_output_tokens": config.request_options.max_output_tokens,
"temperature": config.request_options.temperature,
"top_p": config.request_options.top_p,
"reasoning_effort": config.request_options.reasoning_effort,
"timeout_seconds": config.client_options.timeout_seconds,
"max_retries": config.client_options.max_retries,
},
}
auxiliary = snapshot.payload.auxiliary
return {
"snapshot_id": snapshot.snapshot_id,
"registry_revision": snapshot.payload.registry_revision,
"primary": public_config(snapshot.payload.primary),
"auxiliary": None if auxiliary is None else public_config(auxiliary),
}
@@ -197,7 +197,9 @@ class SnapshotService:
"The model registry is in bootstrap; configure and enable a "
"primary model first.",
)
selection_hash = compute_selection_hash(request.primary, request.auxiliary)
selection_hash = compute_selection_hash(
request.primary, request.reasoning_effort
)
existing = self._store.find_active_run_snapshot(
deployment_id=request.deployment_id,
thread_id=request.thread_id,
@@ -219,33 +221,21 @@ class SnapshotService:
if request.primary is not None
else registry.defaults.primary
)
auxiliary_ref = (
request.auxiliary
if request.auxiliary is not None
else registry.defaults.auxiliary
)
if primary_ref is None: # pragma: no cover - active registries set it
raise ModelRegistryError(
MODEL_REGISTRY_NOT_READY,
"The model registry has no default primary model.",
)
primary_config = self._resolver.resolve(
primary_ref, "primary", registry=registry
)
auxiliary_config = (
None
if auxiliary_ref is None
else self._resolver.resolve(auxiliary_ref, "auxiliary", registry=registry)
primary_ref,
"primary",
registry=registry,
reasoning_effort_override=request.reasoning_effort,
)
payload = {
"registry_revision": registry.revision,
"model_selection_revision": request.model_selection_revision,
"primary": primary_config.model_dump(mode="json"),
"auxiliary": (
None
if auxiliary_config is None
else auxiliary_config.model_dump(mode="json")
),
}
snapshot_id = f"snap-{uuid.uuid4().hex}"
try:
@@ -423,23 +413,20 @@ class SnapshotService:
# --- internals ------------------------------------------------------------
def _ensure_frozen_specs_available(self, snapshot: RuntimeSnapshot) -> None:
configs = [snapshot.payload.primary]
if snapshot.payload.auxiliary is not None:
configs.append(snapshot.payload.auxiliary)
for config in configs:
spec = find_adapter_spec(
config.adapter_id,
config.upstream_model_id,
spec_revision=config.adapter_spec_revision,
specs=self._resolver.specs,
config = snapshot.payload.primary
spec = find_adapter_spec(
config.adapter_id,
config.upstream_model_id,
spec_revision=config.adapter_spec_revision,
specs=self._resolver.specs,
)
if spec is None:
raise ModelRegistryError(
ADAPTER_NOT_SUPPORTED,
f"The adapter contract {config.adapter_id!r} at "
f"spec_revision {config.adapter_spec_revision} frozen by "
"this snapshot no longer exists.",
)
if spec is None:
raise ModelRegistryError(
ADAPTER_NOT_SUPPORTED,
f"The adapter contract {config.adapter_id!r} at "
f"spec_revision {config.adapter_spec_revision} frozen by "
"this snapshot no longer exists.",
)
def _snapshot_not_found() -> ModelRegistryError:
+17 -20
View File
@@ -319,26 +319,23 @@ class ModelRuntimeStore:
def _validate_defaults(self, registry: RegistryV4) -> None:
"""Enforce section 9.2 rule 7 on every save, regardless of state."""
for role, ref in (
("primary", registry.defaults.primary),
("auxiliary", registry.defaults.auxiliary),
):
if ref is None:
continue
located = registry._locate(ref)
if located is None:
raise ModelRegistryError(
MODEL_DISABLED,
f"defaults.{role} references an unknown model.",
details=[{"path": f"defaults.{role}", "code": MODEL_DISABLED}],
)
provider, model = located
if not (provider.enabled and model.enabled):
raise ModelRegistryError(
MODEL_DISABLED,
f"defaults.{role} must reference an enabled model.",
details=[{"path": f"defaults.{role}", "code": MODEL_DISABLED}],
)
ref = registry.defaults.primary
if ref is None:
return
located = registry._locate(ref)
if located is None:
raise ModelRegistryError(
MODEL_DISABLED,
"defaults.primary references an unknown model.",
details=[{"path": "defaults.primary", "code": MODEL_DISABLED}],
)
provider, model = located
if not (provider.enabled and model.enabled):
raise ModelRegistryError(
MODEL_DISABLED,
"defaults.primary must reference an enabled model.",
details=[{"path": "defaults.primary", "code": MODEL_DISABLED}],
)
def _resolve_state(
self,
+6 -9
View File
@@ -11,9 +11,8 @@ Reuses the main EvoScientist agent's backend and middleware so the deployed
sub-agent has full capability parity with its in-process synchronous
counterpart: same workspace files, same ``/skills/`` and ``/memories/``
routes, same error-handling and context-overflow middleware. The chat model
is resolved from the active model registry's defaults (``auxiliary`` for the
scheduler, ``primary`` otherwise) and re-resolved per run from the run
snapshot by ``ConfigurableModelMiddleware``.
is resolved from the active model registry's default primary and re-resolved
per run from the run snapshot by ``ConfigurableModelMiddleware``.
"""
from __future__ import annotations
@@ -103,12 +102,10 @@ def build_async_subagent_graph(name: str) -> Any:
# context and `/memories/profile/...` file guidance as the main agent.
#
# The compile-time model binding is resolved from the active registry's
# defaults — never from config.yaml free strings. Per-run calls are
# re-resolved from the run snapshot by ConfigurableModelMiddleware; the
# scheduler (an unattended timer task) binds the cheaper auxiliary role,
# working sub-agents bind primary.
# default primary — never from config.yaml free strings. Per-run calls
# are re-resolved from the run snapshot by ConfigurableModelMiddleware.
#
# Bootstrap registry: no defaults exist yet, so no compile-time model
# Bootstrap registry: no default exists yet, so no compile-time model
# can be built. The graph must still materialize — one failing factory
# must not take down the whole langgraph dev service — so we bind a
# placeholder that raises MODEL_REGISTRY_NOT_READY on the first model
@@ -121,7 +118,7 @@ def build_async_subagent_graph(name: str) -> Any:
from EvoScientist.model_registry.placeholder import RegistryNotReadyChatModel
runtime = get_snapshot_runtime()
snapshot_role = "auxiliary" if name == "scheduler" else "primary"
snapshot_role = "primary"
try:
model = runtime.build_default_role_model(snapshot_role)
except ModelRegistryError as exc:
+4 -11
View File
@@ -102,18 +102,13 @@ def ollama_provider_payload(**model_overrides) -> dict:
}
def registry_payload(*, auxiliary_default: bool = True, **model_overrides) -> dict:
def registry_payload(**model_overrides) -> dict:
return {
"version": 4,
"revision": 1,
"state": "bootstrap",
"defaults": {
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
"auxiliary": (
{"provider_id": "local-ollama", "model_key": "qwen3"}
if auxiliary_default
else None
),
},
"providers": [
zhipu_provider_payload(**model_overrides),
@@ -141,12 +136,12 @@ def _verify_model(store, registry, provider_id, model_key):
)
def activate_store(store, *, auxiliary_default: bool = True, **model_overrides):
def activate_store(store, **model_overrides):
"""Persist an active registry with verified models into *store*."""
registry = store.save_registry(
expected_revision=1,
registry=RegistryV4.model_validate(
registry_payload(auxiliary_default=auxiliary_default, **model_overrides)
registry_payload(**model_overrides)
),
credential_writes=[
CredentialWrite(credential_id="zhipu-primary", secret_value=ZHIPU_SECRET)
@@ -158,11 +153,10 @@ def activate_store(store, *, auxiliary_default: bool = True, **model_overrides):
return store
def make_active_store(config_dir, *, auxiliary_default: bool = True, **model_overrides):
def make_active_store(config_dir, **model_overrides):
"""Persist an active registry with verified models into a fresh store."""
return activate_store(
ModelRuntimeStore(config_dir=config_dir),
auxiliary_default=auxiliary_default,
**model_overrides,
)
@@ -174,7 +168,6 @@ def make_snapshot_request(**overrides) -> SnapshotCreateRequest:
"deployment_id": "local",
"model_selection_revision": 0,
"primary": None,
"auxiliary": None,
}
payload.update(overrides)
return SnapshotCreateRequest.model_validate(payload)
+3 -12
View File
@@ -539,18 +539,15 @@ class TestConfig:
@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[])
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_auto_approve_still_includes_ask_user_middleware(
mock_config, mock_model, mock_aux_model, mock_tool_selector
mock_config, mock_model, mock_tool_selector
):
cfg = MagicMock()
cfg.enable_ask_user = True
cfg.auto_approve = True
cfg.auto_mode = False
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
@@ -561,18 +558,15 @@ def test_auto_approve_still_includes_ask_user_middleware(
@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[])
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_auto_mode_disables_ask_user_middleware(
mock_config, mock_model, mock_aux_model, mock_tool_selector
mock_config, mock_model, mock_tool_selector
):
cfg = MagicMock()
cfg.enable_ask_user = True
cfg.auto_approve = True
cfg.auto_mode = True
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
@@ -583,11 +577,10 @@ def test_auto_mode_disables_ask_user_middleware(
@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[])
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_for_async_subagent_omits_ask_user_middleware(
mock_config, mock_model, mock_aux_model, mock_tool_selector
mock_config, mock_model, mock_tool_selector
):
"""``AskUserMiddleware`` uses ``interrupt()`` to wait on user input.
@@ -602,8 +595,6 @@ def test_for_async_subagent_omits_ask_user_middleware(
cfg.enable_ask_user = True
cfg.auto_approve = False
cfg.auto_mode = False
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
+10 -22
View File
@@ -123,7 +123,7 @@ def test_factory_requests_async_safe_middleware(
@patch("EvoScientist.utils.load_subagents")
@patch("EvoScientist.config.apply_config_to_env")
@patch("EvoScientist.config.get_effective_config")
def test_scheduler_binds_auxiliary_role(
def test_scheduler_binds_primary_role(
mock_get_cfg,
mock_apply_env,
mock_load_subs,
@@ -133,7 +133,7 @@ def test_scheduler_binds_auxiliary_role(
mock_mcp,
mock_create,
):
"""The scheduler is an unattended timer task → auxiliary role mapping."""
"""The scheduler resolves the registry's primary default (6.1)."""
cfg = MagicMock()
cfg.recursion_limit = 1_000_000
cfg.memory_profile_enabled = False
@@ -141,7 +141,7 @@ def test_scheduler_binds_auxiliary_role(
cfg.memory_observation_writer = MemoryObservationWriter.ALL
cfg.memory_workers_enabled = False
mock_get_cfg.return_value = cfg
model = MagicMock(name="auxiliary_model")
model = MagicMock(name="primary_model")
mock_get_runtime.return_value.build_default_role_model.return_value = model
mock_load_subs.return_value = [
{
@@ -158,13 +158,13 @@ def test_scheduler_binds_auxiliary_role(
build_async_subagent_graph("scheduler")
mock_get_runtime.return_value.build_default_role_model.assert_called_once_with(
"auxiliary"
"primary"
)
mock_get_mw.assert_called_once_with(
for_async_subagent=True,
memory_source_agent="scheduler",
chat_model=model,
snapshot_role="auxiliary",
snapshot_role="primary",
)
assert mock_create.call_args.kwargs["model"] is model
@@ -254,8 +254,6 @@ def test_inject_subagent_omits_memory_middleware_when_memory_disabled(
cfg.memory_observations_enabled = False
cfg.memory_observation_writer = MemoryObservationWriter.ALL
cfg.memory_workers_enabled = True
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
from EvoScientist.EvoScientist import _inject_subagent_middleware
@@ -284,8 +282,6 @@ def test_inject_subagent_worker_only_observation_writer_keeps_live_tool_off(
cfg.memory_observations_enabled = True
cfg.memory_observation_writer = MemoryObservationWriter.WORKER
cfg.memory_workers_enabled = True
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
from EvoScientist.EvoScientist import _inject_subagent_middleware
@@ -312,11 +308,10 @@ def test_inject_subagent_worker_only_observation_writer_keeps_live_tool_off(
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._compile_time_role_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_all_observation_writer_schedules_turn_worker_without_profile_memory(
mock_config, mock_chat, mock_aux, mock_tool_selector
mock_config, mock_chat, mock_tool_selector
):
cfg = MagicMock()
cfg.enable_ask_user = False
@@ -327,8 +322,6 @@ def test_all_observation_writer_schedules_turn_worker_without_profile_memory(
cfg.memory_observations_enabled = True
cfg.memory_observation_writer = MemoryObservationWriter.ALL
cfg.memory_workers_enabled = True
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000})
@@ -365,11 +358,10 @@ def test_all_observation_writer_schedules_turn_worker_without_profile_memory(
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._compile_time_role_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_async_subagent_mode_filters_ask_user(
mock_config, mock_chat, mock_aux, mock_tool_selector
mock_config, mock_chat, mock_tool_selector
):
"""``_get_default_middleware(for_async_subagent=True)`` must drop
``AskUserMiddleware`` even when ``enable_ask_user`` is on.
@@ -388,8 +380,6 @@ def test_async_subagent_mode_filters_ask_user(
cfg.memory_observations_enabled = True
cfg.memory_observation_writer = MemoryObservationWriter.ALL
cfg.memory_workers_enabled = True
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000})
@@ -415,7 +405,7 @@ def test_async_subagent_mode_filters_ask_user(
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._compile_time_role_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_async_subagent_disables_tool_selector_stream_tracking(
mock_config, mock_chat, mock_tool_selector
@@ -430,8 +420,6 @@ def test_async_subagent_disables_tool_selector_stream_tracking(
cfg.memory_observations_enabled = True
cfg.memory_observation_writer = MemoryObservationWriter.ALL
cfg.memory_workers_enabled = True
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000})
+21 -101
View File
@@ -1,88 +1,17 @@
"""Tests for the auxiliary-model resolver and its middleware scoping.
"""Tests for the helper-model wiring in ``_get_default_middleware``.
Covers ``EvoScientist.EvoScientist._ensure_auxiliary_chat_model`` — now
resolved through the model registry role mapping (design doc 6.1:
``defaults.auxiliary ?? defaults.primary``) instead of the legacy
``cfg.auxiliary_model``/``auxiliary_provider`` free strings — and the wiring
in ``_get_default_middleware`` that routes the main agent's tool selector to
the auxiliary model while keeping context editing — and async sub-agents —
on the main model.
After the auxiliary-role removal (design doc 6.1: every role resolves to the
snapshot's frozen primary), the main agent's tool selector no longer binds a
compile-time auxiliary model — it resolves its helper model from the run
snapshot on every call (``model=None``). Async sub-agents and the pure path
keep their threaded model. Context editing always keeps the main model.
"""
from unittest.mock import MagicMock, patch
import pytest
import EvoScientist.EvoScientist as E
from EvoScientist.model_registry.errors import (
MODEL_REGISTRY_NOT_READY,
ModelRegistryError,
)
from tests.registry_fixtures import activate_store
@pytest.fixture(autouse=True)
def _reset_model_caches(monkeypatch):
"""Isolate the module-level model caches per test."""
monkeypatch.setattr(E, "_chat_model", None, raising=False)
monkeypatch.setattr(E, "_chat_model_key", None, raising=False)
monkeypatch.setattr(E, "_auxiliary_chat_model", None, raising=False)
monkeypatch.setattr(E, "_auxiliary_chat_model_key", None, raising=False)
class TestAuxiliaryResolver:
def test_bootstrap_registry_fails_closed(self, isolated_snapshot_runtime):
"""Bootstrap registry: auxiliary resolution propagates NOT_READY."""
with pytest.raises(ModelRegistryError) as excinfo:
E._ensure_auxiliary_chat_model()
assert excinfo.value.code == MODEL_REGISTRY_NOT_READY
def test_no_auxiliary_default_returns_main_instance(
self, monkeypatch, isolated_snapshot_runtime
):
main = object()
activate_store(isolated_snapshot_runtime.store, auxiliary_default=False)
monkeypatch.setattr(E, "_ensure_chat_model", lambda: main)
assert E._ensure_auxiliary_chat_model() is main
def test_auxiliary_equal_to_primary_reuses_main_instance(
self, monkeypatch, isolated_snapshot_runtime
):
main = object()
store = activate_store(isolated_snapshot_runtime.store)
registry = store.load_registry()
registry.defaults.auxiliary = registry.defaults.primary
store.save_registry(expected_revision=registry.revision, registry=registry)
monkeypatch.setattr(E, "_ensure_chat_model", lambda: main)
assert E._ensure_auxiliary_chat_model() is main
def test_auxiliary_default_builds_via_role_mapping(
self, monkeypatch, isolated_snapshot_runtime
):
fake = object()
build = MagicMock(return_value=fake)
activate_store(isolated_snapshot_runtime.store)
monkeypatch.setattr(
isolated_snapshot_runtime, "build_default_role_model", build
)
monkeypatch.setattr(
E, "_ensure_chat_model", lambda: pytest.fail("must not build main")
)
assert E._ensure_auxiliary_chat_model() is fake
build.assert_called_once_with("auxiliary")
def test_auxiliary_cache_reused_within_same_registry_revision(
self, monkeypatch, isolated_snapshot_runtime
):
build = MagicMock(side_effect=[object(), object()])
activate_store(isolated_snapshot_runtime.store)
monkeypatch.setattr(
isolated_snapshot_runtime, "build_default_role_model", build
)
first = E._ensure_auxiliary_chat_model()
assert E._ensure_auxiliary_chat_model() is first
assert build.call_count == 1
def _mock_cfg():
cfg = MagicMock()
cfg.enable_ask_user = False
@@ -91,8 +20,8 @@ def _mock_cfg():
return cfg
class TestAuxiliaryMiddlewareScope:
"""``_get_default_middleware`` routes only the right components to aux."""
class TestHelperModelScope:
"""``_get_default_middleware`` routes only the right components."""
def _capture(self):
cap: dict[str, object] = {}
@@ -107,17 +36,13 @@ class TestAuxiliaryMiddlewareScope:
return cap, fake_tool_selector, fake_context_editing
def test_main_agent_tool_selector_aux_context_editing_main(self):
def test_main_agent_tool_selector_uses_snapshot_resolution(self):
cap, fake_ts, fake_ce = self._capture()
main_model, aux_model = object(), object()
main_model = object()
with (
patch.object(E, "_ensure_config", return_value=_mock_cfg()),
patch.object(
E,
"_compile_time_role_model",
side_effect=lambda role="primary": aux_model
if role == "auxiliary"
else main_model,
E, "_compile_time_role_model", return_value=main_model
),
patch(
"EvoScientist.middleware.create_tool_selector_middleware",
@@ -130,20 +55,17 @@ class TestAuxiliaryMiddlewareScope:
):
E._get_default_middleware()
assert cap["tool_selector"] is aux_model
# None = per-call resolution from the run snapshot.
assert cap["tool_selector"] is None
assert cap["context_editing"] is main_model
def test_async_subagent_tool_selector_stays_main(self):
cap, fake_ts, fake_ce = self._capture()
main_model, aux_model = object(), object()
main_model = object()
with (
patch.object(E, "_ensure_config", return_value=_mock_cfg()),
patch.object(
E,
"_compile_time_role_model",
side_effect=lambda role="primary": aux_model
if role == "auxiliary"
else main_model,
E, "_compile_time_role_model", return_value=main_model
),
patch(
"EvoScientist.middleware.create_tool_selector_middleware",
@@ -160,8 +82,6 @@ class TestAuxiliaryMiddlewareScope:
assert cap["context_editing"] is main_model
def test_pure_path_tool_selector_uses_threaded_model(self):
"""The pure path never resolves auxiliary free strings (Task 7 wires
the local snapshot entry); the threaded model stands in."""
cap, fake_ts, fake_ce = self._capture()
cfg = _mock_cfg()
main_model = object()
@@ -169,7 +89,6 @@ class TestAuxiliaryMiddlewareScope:
with (
patch.object(E, "_ensure_config", side_effect=AssertionError),
patch.object(E, "_ensure_chat_model", side_effect=AssertionError),
patch.object(E, "_ensure_auxiliary_chat_model", side_effect=AssertionError),
patch(
"EvoScientist.middleware.create_tool_selector_middleware",
side_effect=fake_ts,
@@ -190,22 +109,21 @@ class TestAuxiliaryMiddlewareScope:
with (
patch.object(E, "_ensure_config", return_value=cfg),
patch.object(E, "_ensure_chat_model", return_value=main_model),
patch.object(E, "_ensure_auxiliary_chat_model", return_value=main_model),
patch(
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock()],
),
):
mw = E._get_default_middleware(snapshot_role="auxiliary")
mw = E._get_default_middleware(snapshot_role="primary")
configurable = next(
m for m in mw if type(m).__name__ == "ConfigurableModelMiddleware"
)
assert configurable._role == "auxiliary"
assert configurable._role == "primary"
def test_memory_agent_factory_uses_auxiliary_role(monkeypatch):
"""Memory workers bind the auxiliary role model at graph build."""
def test_memory_agent_factory_binds_primary_placeholder(monkeypatch):
"""Memory workers bind the primary placeholder plus snapshot middleware."""
sentinel = object()
monkeypatch.setattr(E, "_compile_time_role_model", lambda role="primary": sentinel)
captured: dict[str, object] = {}
@@ -228,3 +146,5 @@ def test_memory_agent_factory_uses_auxiliary_role(monkeypatch):
backend=MagicMock(),
)
assert captured["model"] is sentinel
middleware = captured["middleware"]
assert type(middleware[0]).__name__ == "ConfigurableModelMiddleware"
+3 -3
View File
@@ -39,7 +39,7 @@ def test_bootstrap_registry_aborts_startup(monkeypatch):
legacy_artifacts, "assert_no_legacy_artifacts", MagicMock()
)
runtime = MagicMock()
runtime.registry_defaults.side_effect = ModelRegistryError(
runtime.registry_default.side_effect = ModelRegistryError(
MODEL_REGISTRY_NOT_READY, "bootstrap"
)
monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime)
@@ -53,7 +53,7 @@ def test_clean_environment_passes(monkeypatch):
legacy_artifacts, "assert_no_legacy_artifacts", MagicMock()
)
runtime = MagicMock()
runtime.registry_defaults.return_value = (MagicMock(), None, 2)
runtime.registry_default.return_value = (MagicMock(), 2)
monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime)
_startup_gates() # must not raise
@@ -64,7 +64,7 @@ def test_non_bootstrap_registry_error_propagates(monkeypatch):
legacy_artifacts, "assert_no_legacy_artifacts", MagicMock()
)
runtime = MagicMock()
runtime.registry_defaults.side_effect = ModelRegistryError(
runtime.registry_default.side_effect = ModelRegistryError(
"SNAPSHOT_EXPIRED", "expired"
)
monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime)
+3 -26
View File
@@ -349,32 +349,9 @@ class TestSnapshotDrivenConstruction:
assert isinstance(model, ChatOpenAI)
assert model.model_name == "glm-5.2"
def test_auxiliary_role_maps_to_snapshot_auxiliary(self, store, runtime):
mw = ConfigurableModelMiddleware(role="auxiliary", runtime=runtime)
snapshot = make_snapshot(store)
req = _make_request()
handler = MagicMock(return_value="ok")
with _patched_config(_configurable_for(snapshot)):
mw.wrap_model_call(req, handler)
model = handler.call_args[0][0].model
assert isinstance(model, ChatOllama)
assert model.model == "qwen3"
def test_summary_role_falls_back_to_snapshot_primary(self, tmp_path):
store = make_active_store(tmp_path / "db", auxiliary_default=False)
mw = ConfigurableModelMiddleware(role="summary", runtime=SnapshotRuntime(store))
snapshot = make_snapshot(store)
req = _make_request()
handler = MagicMock(return_value="ok")
with _patched_config(_configurable_for(snapshot)):
mw.wrap_model_call(req, handler)
model = handler.call_args[0][0].model
assert isinstance(model, ChatOpenAI)
assert model.model_name == "glm-5.2"
def test_unknown_role_rejected(self, runtime):
with pytest.raises(ValueError, match="role"):
ConfigurableModelMiddleware(role="auxiliary", runtime=runtime)
def test_deployment_id_from_configurable_is_verified(self, store, runtime):
mw = ConfigurableModelMiddleware(runtime=runtime)
+4 -10
View File
@@ -107,16 +107,13 @@ def test_create_middleware_model_none_fallback(mock_model):
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock(), MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._compile_time_role_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_default_middleware_includes_context_editing(mock_config, mock_model, mock_aux, mock_ts):
def test_default_middleware_includes_context_editing(mock_config, mock_model, mock_ts):
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
cfg = MagicMock()
cfg.enable_ask_user = False
cfg.auto_approve = False
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
from EvoScientist.EvoScientist import _get_default_middleware
@@ -144,16 +141,13 @@ def test_inject_subagent_includes_context_editing(mock_model):
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock(), MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._compile_time_role_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_context_editing_before_overflow_mapper(mock_config, mock_model, mock_aux, mock_ts):
def test_context_editing_before_overflow_mapper(mock_config, mock_model, mock_ts):
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
cfg = MagicMock()
cfg.enable_ask_user = False
cfg.auto_approve = False
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
from EvoScientist.EvoScientist import _get_default_middleware
+6 -86
View File
@@ -16,7 +16,7 @@ from unittest.mock import MagicMock, patch
import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langchain_ollama import ChatOllama
from langchain_openai import ChatOpenAI
from EvoScientist.middleware.message_budget import (
_snapshot_message_budget,
@@ -30,16 +30,8 @@ from EvoScientist.model_registry.errors import (
ModelRegistryError,
)
from EvoScientist.model_registry.runtime import SnapshotRuntime
from EvoScientist.model_registry.schemas import CredentialWrite, RegistryV4
from EvoScientist.model_registry.store import ModelRuntimeStore
from tests.registry_fixtures import (
ZHIPU_SECRET,
_verify_model,
make_active_store,
make_snapshot,
ollama_provider_payload,
zhipu_provider_payload,
)
from tests.registry_fixtures import make_active_store, make_snapshot
@contextmanager
@@ -86,39 +78,6 @@ def _configurable_for(snapshot, **overrides):
return configurable
def _make_split_limit_store(config_dir):
"""Active store whose auxiliary model has a smaller input limit.
primary (zhipu-glm/glm-5.2): context 1048576 → resolved_input_limit 1015808
auxiliary (local-ollama/qwen3): context 65536 → resolved_input_limit 32768
"""
store = ModelRuntimeStore(config_dir=config_dir)
payload = {
"version": 4,
"revision": 1,
"state": "bootstrap",
"defaults": {
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
"auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"},
},
"providers": [
zhipu_provider_payload(),
ollama_provider_payload(context_window_tokens=65536),
],
}
registry = store.save_registry(
expected_revision=1,
registry=RegistryV4.model_validate(payload),
credential_writes=[
CredentialWrite(credential_id="zhipu-primary", secret_value=ZHIPU_SECRET)
],
)
assert registry.state == "active"
_verify_model(store, registry, "zhipu-glm", "glm-5.2")
_verify_model(store, registry, "local-ollama", "qwen3")
return store
def test_text_counter_excludes_attachment_payloads_and_counts_tool_results():
messages = [
HumanMessage(
@@ -168,25 +127,6 @@ class TestSnapshotMessageBudget:
assert tools_only.hard_tokens == int((1015808 - 4096 - 8192) * 0.90)
assert full.hard_tokens == int((1015808 - 4096 - 8192 - 4096) * 0.90)
def test_auxiliary_role_uses_auxiliary_frozen_limits(self, tmp_path):
"""T6 review fix: the budget role is threaded through, not hardcoded
to ``primary`` — a scheduler (auxiliary role) must size its budget
from the auxiliary model's frozen configuration."""
store = _make_split_limit_store(tmp_path / "model-runtime")
snapshot = make_snapshot(store)
primary_budget = _snapshot_message_budget(
snapshot, "primary", has_tools=False, has_attachments=False
)
auxiliary_budget = _snapshot_message_budget(
snapshot, "auxiliary", has_tools=False, has_attachments=False
)
assert primary_budget.input_limit == 1015808
assert auxiliary_budget.input_limit == 32768
assert auxiliary_budget.hard_tokens == int((32768 - 4096) * 0.90)
class TestSnapshotModeMiddleware:
def test_budget_uses_frozen_reserves_not_model_profile(self, store, runtime):
"""A misleading compile-time profile must not affect snapshot mode."""
@@ -254,26 +194,6 @@ class TestSnapshotModeMiddleware:
assert budget.has_tools is False
assert budget.hard_tokens == int((1015808 - 4096) * 0.90)
def test_auxiliary_snapshot_role_sizes_budget_from_auxiliary(self, tmp_path):
"""T6 review fix: ``snapshot_role="auxiliary"`` (scheduler) sizes the
budget from the snapshot's auxiliary frozen configuration."""
store = _make_split_limit_store(tmp_path / "model-runtime")
runtime = SnapshotRuntime(store)
middleware = create_message_budget_middleware(
MagicMock(),
MagicMock(),
has_tools=False,
snapshot_role="auxiliary",
runtime=runtime,
)
snapshot = make_snapshot(store)
with _patched_config(_configurable_for(snapshot)):
budget = middleware._budget_for_request(_request([HumanMessage("hi")]))
assert budget.input_limit == 32768
assert budget.hard_tokens == int((32768 - 4096) * 0.90)
def test_unknown_snapshot_role_rejected(self, runtime):
with pytest.raises(ValueError, match="Unknown model role"):
create_message_budget_middleware(
@@ -293,7 +213,7 @@ class TestSnapshotModeMiddleware:
middleware._budget_for_request(_request([HumanMessage("hi")]))
assert excinfo.value.code == SNAPSHOT_NOT_FOUND
def test_summarizer_model_uses_summary_role(self, store, runtime):
def test_summarizer_model_uses_snapshot_primary(self, store, runtime):
middleware = create_message_budget_middleware(
MagicMock(), MagicMock(), runtime=runtime
)
@@ -302,9 +222,9 @@ class TestSnapshotModeMiddleware:
with _patched_config(_configurable_for(snapshot)):
model = middleware.model
# summary → snapshot.auxiliary ?? snapshot.primary (section 6.1)
assert isinstance(model, ChatOllama)
assert model.model == "qwen3"
# Every role maps to the snapshot's frozen primary (section 6.1).
assert isinstance(model, ChatOpenAI)
assert model.model_name == "glm-5.2"
def test_summarizer_model_cached_per_snapshot(self, store, runtime):
middleware = create_message_budget_middleware(
+10 -9
View File
@@ -152,7 +152,6 @@ def _registry_payload():
"state": "bootstrap",
"defaults": {
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
"auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"},
},
"providers": [_zhipu_provider(), _ollama_provider()],
}
@@ -476,7 +475,7 @@ def test_put_registry_rejects_defaults_to_disabled_model(active_client):
# the store's rule-7 guard (enforced on every save) is what rejects it.
registry = _current_registry(active_client)["registry"]
registry["state"] = "bootstrap"
registry["providers"][1]["models"][0]["enabled"] = False
registry["providers"][0]["models"][0]["enabled"] = False
response = _put(active_client, registry)
assert response.status_code == 422
@@ -486,7 +485,7 @@ def test_put_registry_rejects_defaults_to_disabled_model(active_client):
def test_put_registry_active_state_validates_defaults_in_schema(active_client):
# The same rule fires earlier (Pydantic) when the payload claims active.
registry = _current_registry(active_client)["registry"]
registry["providers"][1]["models"][0]["enabled"] = False
registry["providers"][0]["models"][0]["enabled"] = False
response = _put(active_client, registry)
assert response.status_code == 422
@@ -530,7 +529,7 @@ def test_put_registry_bootstrap_flow_configure_test_enable(client, store):
# Step 1: save a draft registry — models disabled, defaults null, and the
# credential written in the same transaction (section 10 steps 5-7).
draft = _registry_payload()
draft["defaults"] = {"primary": None, "auxiliary": None}
draft["defaults"] = {"primary": None}
for provider in draft["providers"]:
for model in provider["models"]:
model["enabled"] = False
@@ -568,7 +567,6 @@ def test_put_registry_bootstrap_flow_configure_test_enable(client, store):
enabled = body["registry"]
enabled["defaults"] = {
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
"auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"},
}
for provider in enabled["providers"]:
for model in provider["models"]:
@@ -651,11 +649,16 @@ def test_get_models_returns_only_selectable(active_client, store):
"vision": False,
"structured_output": True,
}
# The selector echoes the registry defaults so clients can label the
# `inherit` choice before the first turn.
assert response.json()["defaults"]["primary"] == {
"provider_id": "zhipu-glm",
"model_key": "glm-5.2",
}
# Disable qwen3 (and drop the auxiliary default): it leaves the selector.
# Disable qwen3: it leaves the selector.
registry = store.load_registry()
payload = registry.model_dump(mode="json")
payload["defaults"]["auxiliary"] = None
payload["providers"][1]["models"][0]["enabled"] = False
store.save_registry(
expected_revision=registry.revision,
@@ -682,7 +685,6 @@ def _snapshot_body(**overrides):
"deployment_id": "webui-1",
"model_selection_revision": 4,
"primary": None,
"auxiliary": None,
}
body.update(overrides)
return body
@@ -699,7 +701,6 @@ def test_snapshot_create_201_and_idempotent_200(active_client):
assert body["primary"]["provider_id"] == "zhipu-glm"
assert body["primary"]["adapter_spec_revision"] == 1
assert body["primary"]["runtime"]["max_output_tokens"] == 32768
assert body["auxiliary"]["model_key"] == "qwen3"
assert SECRET not in first.text
replay = active_client.post(
+2 -5
View File
@@ -136,7 +136,6 @@ def _registry(**overrides):
"state": "bootstrap",
"defaults": {
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
"auxiliary": None,
},
"providers": [_provider()],
}
@@ -153,7 +152,6 @@ class TestRegistryV4:
assert registry.defaults.primary == ModelRef(
provider_id="zhipu-glm", model_key="glm-5.2"
)
assert registry.defaults.auxiliary is None
provider = registry.providers[0]
assert provider.runtime.timeout_seconds == 120
model = provider.models[0]
@@ -172,7 +170,7 @@ class TestRegistryV4:
registry = RegistryV4.model_validate(
_registry(
state="bootstrap",
defaults={"primary": None, "auxiliary": None},
defaults={"primary": None},
providers=[],
)
)
@@ -184,7 +182,7 @@ class TestRegistryV4:
RegistryV4.model_validate(
_registry(
state="active",
defaults={"primary": None, "auxiliary": None},
defaults={"primary": None},
)
)
@@ -205,7 +203,6 @@ class TestRegistryV4:
_registry(
defaults={
"primary": {"provider_id": "zhipu-glm", "model_key": "missing"},
"auxiliary": None,
}
)
)
+2 -4
View File
@@ -51,7 +51,6 @@ def _registry_payload(**overrides):
"state": "bootstrap",
"defaults": {
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
"auxiliary": None,
},
"providers": [
{
@@ -106,7 +105,6 @@ class TestBootstrap:
assert registry.state == "bootstrap"
assert registry.providers == []
assert registry.defaults.primary is None
assert registry.defaults.auxiliary is None
def test_database_path_and_permissions(self, tmp_path):
store = ModelRuntimeStore(config_dir=tmp_path)
@@ -149,7 +147,7 @@ class TestRegistrySave:
def test_bootstrap_save_without_ready_primary_stays_bootstrap(self, tmp_path):
store = ModelRuntimeStore(config_dir=tmp_path)
registry = _registry(
defaults={"primary": None, "auxiliary": None},
defaults={"primary": None},
)
saved = store.save_registry(expected_revision=1, registry=registry)
assert saved.state == "bootstrap"
@@ -242,7 +240,7 @@ class TestRegistrySave:
],
}
)
payload["defaults"]["auxiliary"] = {
payload["defaults"]["primary"] = {
"provider_id": "local-ollama",
"model_key": "qwen3",
}
+1 -1
View File
@@ -219,7 +219,7 @@ def _registry_payload(port, **provider_overrides):
"version": 4,
"revision": 1,
"state": "bootstrap",
"defaults": {"primary": None, "auxiliary": None},
"defaults": {"primary": None},
"providers": [_provider_payload(port, **provider_overrides)],
}
+4 -5
View File
@@ -119,7 +119,7 @@ def _registry(providers=None, **overrides) -> RegistryV4:
"version": 4,
"revision": 1,
"state": "bootstrap",
"defaults": {"primary": None, "auxiliary": None},
"defaults": {"primary": None},
"providers": [_zhipu_provider()] if providers is None else providers,
}
payload.update(overrides)
@@ -219,9 +219,8 @@ class TestResolve:
config = resolver.resolve(ZHIPU_REF, "primary")
assert SECRET not in config.model_dump_json()
def test_all_roles_are_stamped(self, resolver):
for role in ("primary", "auxiliary", "summary", "tool_selector"):
assert resolver.resolve(ZHIPU_REF, role).role == role
def test_primary_role_is_stamped(self, resolver):
assert resolver.resolve(ZHIPU_REF, "primary").role == "primary"
def test_unknown_role_rejected(self, resolver):
with pytest.raises(ValueError, match="role"):
@@ -351,7 +350,7 @@ class TestResolve:
registry = _activate(store, _registry(providers=[_ollama_provider()]))
_verify(store, registry, "local-ollama", "qwen3")
resolver = ModelRegistryResolver(store)
config = resolver.resolve(OLLAMA_REF, "auxiliary")
config = resolver.resolve(OLLAMA_REF, "primary")
assert config.auth_ref.mode == "none"
assert config.auth_ref.credential_id is None
assert config.auth_ref.credential_revision is None
+3 -6
View File
@@ -40,8 +40,6 @@ def _mock_config():
cfg.auto_mode = False
cfg.auto_approve = False
cfg.model_fallbacks = None
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
cfg.code_interpreter_timeout = 60
cfg.code_interpreter_max_result_chars = 6000
return cfg
@@ -65,11 +63,10 @@ def test_runtime_context_injects_current_date_and_timezone():
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock(), MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._compile_time_role_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_default_middleware_includes_runtime_context(
mock_config, mock_model, mock_aux_model, mock_tool_selector
mock_config, mock_model, mock_tool_selector
):
mock_config.return_value = _mock_config()
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
@@ -85,7 +82,7 @@ def test_default_middleware_includes_runtime_context(
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock(), MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._compile_time_role_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_async_subagent_middleware_includes_runtime_context(
mock_config, mock_model, mock_tool_selector
+15 -46
View File
@@ -1,9 +1,9 @@
"""Tests for ``EvoScientist.model_registry.runtime`` (design doc 8.3).
Covers the runtime-link glue: registry-default role resolution (auxiliary ??
primary), snapshot role construction with per-call credential resolution,
safe-client wiring (ollama carries retries on the transports), the shared
default-runtime accessor, and the tolerant local platform field reader.
Covers the runtime-link glue: registry-default primary resolution, snapshot
role construction with per-call credential resolution, safe-client wiring
(ollama carries retries on the transports), the shared default-runtime
accessor, and the tolerant local platform field reader.
"""
from __future__ import annotations
@@ -59,33 +59,19 @@ class TestRegistryDefaults:
def test_bootstrap_registry_is_not_ready(self, tmp_path):
runtime = SnapshotRuntime(ModelRuntimeStore(config_dir=tmp_path / "db"))
with pytest.raises(ModelRegistryError) as excinfo:
runtime.registry_defaults()
runtime.registry_default()
assert excinfo.value.code == MODEL_REGISTRY_NOT_READY
assert excinfo.value.http_status == 422
def test_active_registry_returns_defaults(self, runtime):
primary, auxiliary, revision = runtime.registry_defaults()
def test_active_registry_returns_default(self, runtime):
primary, revision = runtime.registry_default()
assert primary == ZHIPU_REF
assert auxiliary == OLLAMA_REF
assert revision == 2
class TestResolveRoleConfig:
def test_primary_role_resolves_primary_default(self, runtime):
config = runtime.resolve_role_config("primary")
assert config.model_ref == ZHIPU_REF
assert config.role == "primary"
def test_auxiliary_role_resolves_auxiliary_default(self, runtime):
for role in ("auxiliary", "summary", "tool_selector"):
config = runtime.resolve_role_config(role)
assert config.model_ref == OLLAMA_REF, role
assert config.role == "auxiliary"
def test_auxiliary_falls_back_to_primary(self, tmp_path):
store = make_active_store(tmp_path / "db", auxiliary_default=False)
runtime = SnapshotRuntime(store)
config = runtime.resolve_role_config("auxiliary")
class TestResolveDefaultConfig:
def test_resolves_primary_default(self, runtime):
config = runtime.resolve_default_config()
assert config.model_ref == ZHIPU_REF
assert config.role == "primary"
@@ -100,11 +86,6 @@ class TestBuildDefaultRoleModel:
assert model.temperature == 0.7
assert model.top_p == 0.95
def test_auxiliary_builds_ollama_without_credential(self, runtime):
model = runtime.build_default_role_model("auxiliary")
assert isinstance(model, ChatOllama)
assert model.model == "qwen3"
def test_openai_compatible_never_reads_env_api_key(self, runtime, monkeypatch):
"""The credential comes from the frozen auth_ref, never OPENAI_API_KEY."""
monkeypatch.setenv("OPENAI_API_KEY", "sk-env-must-not-leak")
@@ -120,20 +101,6 @@ class TestBuildRoleModelFromSnapshot:
assert model.model_name == "glm-5.2"
assert model.openai_api_key.get_secret_value() == ZHIPU_SECRET
def test_auxiliary_roles_use_snapshot_auxiliary(self, store, runtime):
snapshot = make_snapshot(store)
for role in ("auxiliary", "summary", "tool_selector"):
model = runtime.build_role_model(snapshot, role)
assert isinstance(model, ChatOllama), role
assert model.model == "qwen3"
def test_auxiliary_roles_fall_back_to_snapshot_primary(self, tmp_path):
store = make_active_store(tmp_path / "db", auxiliary_default=False)
runtime = SnapshotRuntime(store)
snapshot = make_snapshot(store)
model = runtime.build_role_model(snapshot, "summary")
assert isinstance(model, ChatOpenAI)
assert model.model_name == "glm-5.2"
class TestSharedDefaultRuntime:
@@ -213,7 +180,10 @@ class TestEndpointPolicyWiring:
)
)
runtime = SnapshotRuntime(store, endpoint_policy=policy)
model = runtime.build_default_role_model("auxiliary")
# The loopback ollama provider builds only because its base URL is a
# registered development endpoint.
config = runtime.resolver.resolve(OLLAMA_REF, "primary")
model = runtime._build(config, "")
assert isinstance(model, ChatOllama)
@@ -233,9 +203,8 @@ class TestCreateLocalSnapshot:
assert snapshot.thread_id == "cli-thread-1"
assert snapshot.deployment_id == runtime.local_deployment_id
assert snapshot.payload.model_selection_revision == 0
# primary=None inherits — the registry defaults are frozen at creation.
# primary=None inherits — the registry default is frozen at creation.
assert config_for_role(snapshot, "primary").model_ref == ZHIPU_REF
assert config_for_role(snapshot, "auxiliary").model_ref == OLLAMA_REF
def test_same_run_request_id_reuses_snapshot(self, store, runtime):
first = runtime.create_local_snapshot("t-1", run_request_id="req-1")
+70 -54
View File
@@ -22,6 +22,7 @@ from EvoScientist.model_registry.errors import (
SNAPSHOT_ALREADY_BOUND,
SNAPSHOT_EXPIRED,
SNAPSHOT_NOT_FOUND,
UNSUPPORTED_RUNTIME_PARAMETER,
ModelRegistryError,
)
from EvoScientist.model_registry.hashing import configuration_hash
@@ -120,7 +121,7 @@ def _ollama_provider():
}
def _registry(*, auxiliary_default=True) -> RegistryV4:
def _registry() -> RegistryV4:
return RegistryV4.model_validate(
{
"version": 4,
@@ -128,11 +129,6 @@ def _registry(*, auxiliary_default=True) -> RegistryV4:
"state": "bootstrap",
"defaults": {
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
"auxiliary": (
{"provider_id": "local-ollama", "model_key": "qwen3"}
if auxiliary_default
else None
),
},
"providers": [_zhipu_provider(), _ollama_provider()],
}
@@ -190,16 +186,16 @@ def _request(**overrides):
"deployment_id": "local",
"model_selection_revision": 0,
"primary": None,
"auxiliary": None,
}
payload.update(overrides)
return SnapshotCreateRequest.model_validate(payload)
# SHA-256 of the canonical JSON '{"auxiliary":null,"primary":null}' — the
# inherit/inherit selection. Pinned as a literal to lock the wire contract.
# SHA-256 of the canonical JSON '{"primary":null,"reasoning_effort":null}' —
# the inherit selection with no effort override. Pinned as a literal to lock
# the wire contract.
INHERIT_SELECTION_HASH = (
"697c046214ccc3ddee1018af7eb6c21dbd5bfd01fbca9cb594ffc273deb55abc"
"03e47e7f41396b231bd558734715697a5aebfa114ccfd861a0f7e824e32dac8d"
)
@@ -215,27 +211,69 @@ class TestCreate:
assert payload.model_selection_revision == 4
assert payload.primary.model_ref == ZHIPU_REF
assert payload.primary.role == "primary"
assert payload.auxiliary is not None
assert payload.auxiliary.model_ref == OLLAMA_REF
assert payload.auxiliary.role == "auxiliary"
def test_inherit_without_auxiliary_default(self, active_store):
def test_explicit_selection_freezes_primary(self, service):
creation = service.create(_request(primary=OLLAMA_REF))
payload = creation.snapshot.payload
assert payload.primary.model_ref == OLLAMA_REF
def test_reasoning_effort_override_freezes_into_primary(self, active_store):
# glm-5.2's model-specific contract rejects reasoning overrides, so
# add a generic-contract model on the same provider to exercise the
# override path (disabled → verify → enable, the standard flow).
store = active_store
registry = store.load_registry()
store.save_registry(
payload = registry.model_dump(mode="json")
payload["providers"][0]["models"].append(
{
"key": "glm-air",
"name": "GLM Air",
"upstream_model_id": "glm-air",
"enabled": False,
"runtime": _model_runtime(),
}
)
registry = store.save_registry(
expected_revision=registry.revision,
registry=_registry(auxiliary_default=False),
registry=RegistryV4.model_validate(payload),
)
_verify(store, registry, "zhipu-glm", "glm-air")
payload = registry.model_dump(mode="json")
payload["providers"][0]["models"][1]["enabled"] = True
registry = store.save_registry(
expected_revision=registry.revision,
registry=RegistryV4.model_validate(payload),
)
service = SnapshotService(store, ModelRegistryResolver(store))
creation = service.create(_request())
assert creation.snapshot.payload.auxiliary is None
creation = service.create(
_request(
primary={"provider_id": "zhipu-glm", "model_key": "glm-air"},
reasoning_effort="high",
)
)
result = creation.snapshot.payload
assert result.primary.request_options.reasoning_effort == "high"
def test_explicit_selection_freezes_both_roles(self, service):
creation = service.create(_request(primary=ZHIPU_REF, auxiliary=OLLAMA_REF))
payload = creation.snapshot.payload
assert payload.primary.model_ref == ZHIPU_REF
assert payload.auxiliary is not None
assert payload.auxiliary.model_ref == OLLAMA_REF
def test_reasoning_effort_override_rejected_by_model_contract(self, service):
# glm-5.2's model-specific contract does not accept reasoning_effort.
with pytest.raises(ModelRegistryError) as excinfo:
service.create(_request(reasoning_effort="high"))
assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER
def test_reasoning_effort_override_changes_selection_hash(self, service):
plain = service.create(_request())
assert compute_selection_hash(None, "low") != plain.snapshot.selection_hash
assert compute_selection_hash(None, "low") != INHERIT_SELECTION_HASH
# The hash covers the raw field, so an explicit ``auto`` differs from
# no override; both resolve to the same frozen configuration.
assert compute_selection_hash(None, "auto") != INHERIT_SELECTION_HASH
def test_reasoning_effort_override_rejected_for_unsupported_adapter(
self, service
):
with pytest.raises(ModelRegistryError) as excinfo:
service.create(_request(primary=OLLAMA_REF, reasoning_effort="high"))
assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER
def test_bootstrap_registry_is_not_ready(self, store):
service = SnapshotService(store, ModelRegistryResolver(store))
@@ -247,13 +285,13 @@ class TestCreate:
def test_selection_hash_uses_pre_resolution_semantics(self, service):
creation = service.create(_request())
snapshot = creation.snapshot
# The inherit/inherit hash is pinned to a known literal (computed
# offline from the canonical JSON) so the contract can't drift
# together with a reimplemented expectation.
# The inherit hash is pinned to a known literal (computed offline
# from the canonical JSON) so the contract can't drift together with
# a reimplemented expectation.
assert snapshot.selection_hash == INHERIT_SELECTION_HASH
assert compute_selection_hash(None, None) == INHERIT_SELECTION_HASH
# An explicit selection equal to the defaults still hashes differently.
assert compute_selection_hash(ZHIPU_REF, OLLAMA_REF) != INHERIT_SELECTION_HASH
assert compute_selection_hash(None) == INHERIT_SELECTION_HASH
# An explicit selection equal to the default still hashes differently.
assert compute_selection_hash(ZHIPU_REF) != INHERIT_SELECTION_HASH
def test_selection_hash_ignores_selection_revision(self, service):
first = service.create(_request(model_selection_revision=4))
@@ -524,25 +562,9 @@ class TestCleanupExpired:
class TestRoleMapping:
def test_auxiliary_roles_fall_back_to_primary(self, active_store):
store = active_store
registry = store.load_registry()
store.save_registry(
expected_revision=registry.revision,
registry=_registry(auxiliary_default=False),
)
service = SnapshotService(store, ModelRegistryResolver(store))
snapshot = service.create(_request()).snapshot
for role in ("primary", "auxiliary", "summary", "tool_selector"):
assert config_for_role(snapshot, role).model_ref == ZHIPU_REF
def test_auxiliary_roles_use_frozen_auxiliary(self, service):
def test_primary_role_maps_to_frozen_primary(self, service):
snapshot = service.create(_request()).snapshot
assert config_for_role(snapshot, "primary").model_ref == ZHIPU_REF
for role in ("auxiliary", "summary", "tool_selector"):
config = config_for_role(snapshot, role)
assert config.model_ref == OLLAMA_REF
assert config.role == "auxiliary"
def test_unknown_role_rejected(self, service):
snapshot = service.create(_request()).snapshot
@@ -574,12 +596,6 @@ class TestSnapshotCredentials:
fresh_service = SnapshotService(fresh_store, ModelRegistryResolver(fresh_store))
assert fresh_service.resolve_snapshot_credential(snapshot, "primary") == SECRET
def test_auxiliary_role_resolves_auxiliary_credential(self, service):
snapshot = service.create(_request()).snapshot
# The frozen auxiliary is the mode=none ollama model: no secret.
assert service.resolve_snapshot_credential(snapshot, "summary") == ""
assert service.resolve_snapshot_credential(snapshot, "tool_selector") == ""
def test_mode_none_returns_empty_string(self, service):
snapshot = service.create(_request(primary=OLLAMA_REF)).snapshot
assert service.resolve_snapshot_credential(snapshot, "primary") == ""
@@ -599,10 +615,10 @@ class TestPublicView:
"max_output_tokens": 32768,
"temperature": 0.7,
"top_p": 0.95,
"reasoning_effort": "auto",
"timeout_seconds": 120,
"max_retries": 2,
}
assert view["auxiliary"]["provider_id"] == "local-ollama"
def test_never_contains_secret_or_base_url(self, service):
snapshot = service.create(_request()).snapshot
+15 -14
View File
@@ -55,7 +55,10 @@ def _factory_patches():
return_value=MagicMock(),
create=True,
),
patch("EvoScientist.EvoScientist._ensure_chat_model", return_value=MagicMock()),
patch(
"EvoScientist.middleware.configurable_model.resolve_snapshot_model",
return_value=MagicMock(),
),
patch(
"langchain.agents.middleware.LLMToolSelectorMiddleware",
return_value=MagicMock(),
@@ -252,8 +255,10 @@ def test_selector_always_includes_available_memory_tools():
]
def test_selector_resolved_once_across_repeated_requests():
"""Agent tools are stable, so build the selector once and reuse it."""
def test_selector_rebuilt_per_request_with_stable_always_include():
"""The selector is rebuilt per call (its helper model is resolved from
the run snapshot, which varies per run), but the always-include names
derived from the request's tools stay stable."""
mock_selector = MagicMock()
mock_selector.wrap_model_call.side_effect = lambda request, handler: handler(
request
@@ -274,7 +279,9 @@ def test_selector_resolved_once_across_repeated_requests():
for _ in range(3):
cond.wrap_model_call(_request(tools), MagicMock())
selector_factory.assert_called_once_with(["search_observations", "think_tool"])
assert selector_factory.call_count == 3
for call in selector_factory.call_args_list:
assert call.args[0] == ["search_observations", "think_tool"]
assert mock_selector.wrap_model_call.call_count == 3
@@ -305,16 +312,13 @@ def test_tracker_captures_tools():
"EvoScientist.middleware.create_tool_selector_middleware",
side_effect=lambda *a, **kw: _patched_create(),
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._compile_time_role_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_default_middleware_includes_tool_selector(mock_config, mock_model, mock_aux, mock_ts):
def test_default_middleware_includes_tool_selector(mock_config, mock_model, mock_ts):
mock_model.return_value = _mock_model()
cfg = MagicMock()
cfg.enable_ask_user = False
cfg.auto_approve = False
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
from EvoScientist.EvoScientist import _get_default_middleware
@@ -342,17 +346,14 @@ def test_subagent_no_tool_selector(mock_model):
"EvoScientist.middleware.create_tool_selector_middleware",
side_effect=lambda *a, **kw: _patched_create(),
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._compile_time_role_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_tool_selector_ordering(mock_config, mock_model, mock_aux, mock_ts):
def test_tool_selector_ordering(mock_config, mock_model, mock_ts):
"""ToolSelector should come after ToolErrorHandler and before Memory."""
mock_model.return_value = _mock_model()
cfg = MagicMock()
cfg.enable_ask_user = False
cfg.auto_approve = False
cfg.auxiliary_model = ""
cfg.auxiliary_provider = ""
mock_config.return_value = cfg
from EvoScientist.EvoScientist import _get_default_middleware