feat(runtime)!: remove auxiliary model role, resolve all roles from run snapshot
ModelRole collapses to "primary": every role (main, tool selector, memory
agents, subagents, summarizer) resolves to the snapshot's frozen primary
model, per design 6.1/8.3 — users typically configure a single usable LLM,
so compile-time auxiliary bindings were bypassing run snapshots and
mis-attributing usage. Legacy auxiliary keys in stored snapshots, registry
JSON, and thread metadata are tolerated on read and dropped.
BREAKING CHANGE: ThreadModelSelection no longer carries an auxiliary ref;
snapshot selection_hash is computed over {primary, reasoning_effort} only;
ConfigurableModelMiddleware(role="auxiliary") is rejected.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -56,19 +56,11 @@ DEFAULT_SKILL_SOURCES = ("/skills/",)
|
||||
_config = None
|
||||
_chat_model = None
|
||||
# Track the (provider_id, model_key, registry_revision) binding of
|
||||
# _chat_model so the cache invalidates when the registry defaults change.
|
||||
# _chat_model so the cache invalidates when the registry default changes.
|
||||
# The compile-time binding is only a placeholder — per-run resolution
|
||||
# always comes from the run snapshot via ConfigurableModelMiddleware.
|
||||
_chat_model_key: tuple[str, str, int] | None = None
|
||||
|
||||
# Auxiliary model for background/helper LLM calls (memory workers + main-agent
|
||||
# tool selector). Cached separately from the main model; resolved through the
|
||||
# registry ``auxiliary`` role mapping and falls back to the main instance when
|
||||
# no distinct auxiliary default is configured (see
|
||||
# _ensure_auxiliary_chat_model).
|
||||
_auxiliary_chat_model = None
|
||||
_auxiliary_chat_model_key: tuple[str, str, int] | None = None
|
||||
|
||||
# Cache MCP tools by the effective config signature to avoid reconnecting
|
||||
# to MCP servers on every `/new` when config is unchanged.
|
||||
_MCP_TOOLS_CACHE_KEY: str | None = None
|
||||
@@ -148,45 +140,13 @@ def _ensure_chat_model():
|
||||
from .model_registry.runtime import get_snapshot_runtime
|
||||
|
||||
runtime = get_snapshot_runtime()
|
||||
primary_ref, _, revision = runtime.registry_defaults()
|
||||
primary_ref, revision = runtime.registry_default()
|
||||
key = (primary_ref.provider_id, primary_ref.model_key, revision)
|
||||
if _chat_model is None or _chat_model_key != key:
|
||||
_replace_chat_model(runtime.build_default_role_model("primary"), key)
|
||||
return _chat_model
|
||||
|
||||
|
||||
def _ensure_auxiliary_chat_model():
|
||||
"""Return the auxiliary chat model for background/helper LLM calls.
|
||||
|
||||
Resolves the ``auxiliary`` role through the model registry role mapping
|
||||
(design doc 6.1): ``registry.defaults.auxiliary ?? defaults.primary``,
|
||||
constructed via ``build_chat_model`` from the resolved configuration.
|
||||
|
||||
When no auxiliary default is configured — or it matches the primary
|
||||
default — returns the main ``_ensure_chat_model()`` instance directly,
|
||||
so no second client is built. Otherwise the model is cached under its
|
||||
``(provider_id, model_key, registry_revision)`` key. A bootstrap
|
||||
registry propagates ``MODEL_REGISTRY_NOT_READY`` from
|
||||
``_ensure_chat_model()``.
|
||||
"""
|
||||
global _auxiliary_chat_model, _auxiliary_chat_model_key
|
||||
from .model_registry.runtime import get_snapshot_runtime
|
||||
|
||||
runtime = get_snapshot_runtime()
|
||||
primary_ref, auxiliary_ref, revision = runtime.registry_defaults()
|
||||
if auxiliary_ref is None or auxiliary_ref == primary_ref:
|
||||
return _ensure_chat_model()
|
||||
key: tuple[str, str, int] = (
|
||||
auxiliary_ref.provider_id,
|
||||
auxiliary_ref.model_key,
|
||||
revision,
|
||||
)
|
||||
if _auxiliary_chat_model is None or _auxiliary_chat_model_key != key:
|
||||
_auxiliary_chat_model = runtime.build_default_role_model("auxiliary")
|
||||
_auxiliary_chat_model_key = key
|
||||
return _auxiliary_chat_model
|
||||
|
||||
|
||||
def _compile_time_role_model(role: str = "primary"):
|
||||
"""Return the compile-time model binding for graph construction.
|
||||
|
||||
@@ -693,9 +653,8 @@ def _get_default_middleware(
|
||||
memory_source_agent: Attribution name for profile/observation writes.
|
||||
Async sub-agent factories pass their deployed agent name here.
|
||||
snapshot_role: The model role ``ConfigurableModelMiddleware``
|
||||
resolves from the run snapshot (``primary`` for the main agent
|
||||
and working sub-agents, ``auxiliary`` for unattended helper
|
||||
agents such as the scheduler).
|
||||
resolves from the run snapshot; every role maps to the
|
||||
snapshot's frozen primary (design doc 6.1).
|
||||
"""
|
||||
from .middleware import (
|
||||
ConfigurableModelMiddleware,
|
||||
@@ -746,19 +705,12 @@ def _get_default_middleware(
|
||||
),
|
||||
memory_scheduler=memory_scheduler,
|
||||
)
|
||||
# Main-agent tool selection may use the auxiliary model; async sub-agents
|
||||
# keep their own model (they do real work, not a one-off helper call).
|
||||
# Main-agent tool selection resolves its helper model from the run
|
||||
# snapshot on every call (model=None below); async sub-agents and the
|
||||
# pure path (explicit model + config) keep their threaded model.
|
||||
# context_editing stays on the main model — its model only sizes the
|
||||
# context-window trigger for the main agent's own history.
|
||||
if for_async_subagent:
|
||||
tool_selector_model = model
|
||||
elif chat_model is None:
|
||||
tool_selector_model = _compile_time_role_model("auxiliary")
|
||||
else:
|
||||
# Pure path (explicit model + config): the threaded model stands in
|
||||
# for tool selection at build time; per-call resolution still comes
|
||||
# from the run snapshot.
|
||||
tool_selector_model = model
|
||||
tool_selector_model = None if (not for_async_subagent and chat_model is None) else model
|
||||
mw = [
|
||||
ConfigurableModelMiddleware(role=snapshot_role),
|
||||
create_message_budget_middleware(model, backend, snapshot_role=snapshot_role),
|
||||
|
||||
@@ -73,7 +73,7 @@ def current_model_label() -> str:
|
||||
try:
|
||||
from ..model_registry.runtime import get_snapshot_runtime
|
||||
|
||||
primary, _, _ = get_snapshot_runtime().registry_defaults()
|
||||
primary, _ = get_snapshot_runtime().registry_default()
|
||||
except Exception:
|
||||
return "unconfigured"
|
||||
return f"{primary.provider_id}/{primary.model_key}"
|
||||
|
||||
@@ -1374,7 +1374,7 @@ def _startup_gates() -> None:
|
||||
try:
|
||||
from ..model_registry.runtime import get_snapshot_runtime
|
||||
|
||||
get_snapshot_runtime().registry_defaults()
|
||||
get_snapshot_runtime().registry_default()
|
||||
except ModelRegistryError as exc:
|
||||
if exc.code != MODEL_REGISTRY_NOT_READY:
|
||||
raise
|
||||
|
||||
@@ -88,6 +88,7 @@ def build_memory_agent_graph(
|
||||
|
||||
from ...backends import build_memory_agent_backend
|
||||
from ...EvoScientist import _compile_time_role_model
|
||||
from ...middleware.configurable_model import ConfigurableModelMiddleware
|
||||
|
||||
kwargs: dict[str, Any] = {}
|
||||
if response_format is not None:
|
||||
@@ -101,11 +102,14 @@ def build_memory_agent_graph(
|
||||
|
||||
agent = create_deep_agent(
|
||||
name=name,
|
||||
model=_compile_time_role_model("auxiliary"),
|
||||
model=_compile_time_role_model("primary"),
|
||||
system_prompt=system_prompt,
|
||||
tools=list(tools),
|
||||
backend=backend,
|
||||
middleware=list(middleware),
|
||||
# The compile-time model is only a placeholder: the middleware
|
||||
# re-resolves every call from the run snapshot (lazily healing
|
||||
# background runs to a local snapshot of the registry default).
|
||||
middleware=[ConfigurableModelMiddleware(), *middleware],
|
||||
subagents=[],
|
||||
skills=skills,
|
||||
**kwargs,
|
||||
|
||||
@@ -154,11 +154,8 @@ class ConfigurableModelMiddleware(AgentMiddleware):
|
||||
"""Re-resolve the chat model from the run snapshot on every call.
|
||||
|
||||
``role`` selects which frozen configuration of the snapshot feeds this
|
||||
agent's model calls: ``primary`` for the main agent and working
|
||||
sub-agents, ``auxiliary`` for unattended helper agents (e.g. the
|
||||
scheduler). The section 6.1 mapping (``auxiliary``/``summary``/
|
||||
``tool_selector`` → ``snapshot.auxiliary ?? snapshot.primary``) is
|
||||
applied by the snapshot layer.
|
||||
agent's model calls; every role currently maps to the snapshot's frozen
|
||||
primary (section 6.1).
|
||||
|
||||
A per-instance cache keyed by snapshot ID avoids rebuilding the model on
|
||||
every call within a run; snapshots are immutable once created, so the
|
||||
@@ -266,3 +263,37 @@ class ConfigurableModelMiddleware(AgentMiddleware):
|
||||
# thread-pool overhead is irrelevant once warm.
|
||||
new_model = await asyncio.to_thread(self._resolve)
|
||||
return await handler(request.override(model=new_model))
|
||||
|
||||
|
||||
# --- per-call resolution for in-run helper models -----------------------------
|
||||
|
||||
# In-run helper LLM calls (the tool selector's internal selection call) do
|
||||
# not pass through an agent's model request, so ``request.override`` cannot
|
||||
# reach them. They resolve the run snapshot's primary model through this
|
||||
# helper instead; background runs without an explicit snapshot are healed
|
||||
# through the same lazy local-snapshot path as the middleware.
|
||||
_helper_model_cache: dict[str, Any] = {}
|
||||
_helper_model_cache_lock = threading.Lock()
|
||||
|
||||
|
||||
def resolve_snapshot_model(runtime: SnapshotRuntime | None = None) -> Any:
|
||||
"""Resolve the active run's snapshot primary model (cached per snapshot).
|
||||
|
||||
Uses the same binding rules as ``ConfigurableModelMiddleware``: the
|
||||
run's explicit ``runtime_snapshot_id`` wins, otherwise a local snapshot
|
||||
is lazily created for the run's own thread. Must be called from inside
|
||||
a runnable context (or a test that installed one).
|
||||
"""
|
||||
configurable = _current_configurable()
|
||||
check_no_outside_snapshot_model_config(configurable)
|
||||
rt = runtime if runtime is not None else get_snapshot_runtime()
|
||||
snapshot_id, thread_id = ensure_snapshot_binding(configurable, rt)
|
||||
snapshot = rt.get_snapshot_for_run(snapshot_id, thread_id=thread_id)
|
||||
with _helper_model_cache_lock:
|
||||
cached = _helper_model_cache.get(snapshot.snapshot_id)
|
||||
if cached is not None:
|
||||
return cached
|
||||
model = rt.build_role_model(snapshot, "primary")
|
||||
with _helper_model_cache_lock:
|
||||
_helper_model_cache[snapshot.snapshot_id] = model
|
||||
return model
|
||||
|
||||
@@ -10,13 +10,12 @@ Snapshot mode (design doc 6.5, 8.3): every run carries — or lazily creates,
|
||||
via the section 8.1 local entry convention — a run snapshot; the input
|
||||
limit and the three fixed reserves come from the frozen
|
||||
``ResolvedModelConfig.budget`` of the middleware's own ``snapshot_role``
|
||||
(``primary`` for the main agent and working sub-agents, ``auxiliary`` for
|
||||
unattended helpers such as the scheduler). There is no 32K default and no
|
||||
compile-time profile fallback: a bootstrap registry fails closed with
|
||||
``MODEL_REGISTRY_NOT_READY``. The per-call message budget is recomputed on
|
||||
every invocation from those frozen reserves with the current
|
||||
``has_tools``/``has_attachments`` mode; it is never carried over from a
|
||||
previous call:
|
||||
(every role maps to the snapshot's frozen primary, section 6.1). There is
|
||||
no 32K default and no compile-time profile fallback: a bootstrap registry
|
||||
fails closed with ``MODEL_REGISTRY_NOT_READY``. The per-call message budget
|
||||
is recomputed on every invocation from those frozen reserves with the
|
||||
current ``has_tools``/``has_attachments`` mode; it is never carried over
|
||||
from a previous call:
|
||||
|
||||
message_budget = resolved_input_limit
|
||||
- fixed_system_reserve_tokens
|
||||
@@ -176,10 +175,9 @@ class MessageBudgetMiddleware:
|
||||
agent has tool capability and a configured toolset. Decided
|
||||
once here, never from a single request's bound tool count.
|
||||
snapshot_role: The snapshot role whose frozen limits size this
|
||||
agent's budget (``primary`` for the main agent and working
|
||||
sub-agents, ``auxiliary`` for unattended helper agents such
|
||||
as the scheduler). The summarizer keeps using the snapshot's
|
||||
``summary`` role mapping regardless.
|
||||
agent's budget; every role maps to the snapshot's frozen
|
||||
primary (section 6.1). The summarizer uses the same frozen
|
||||
primary.
|
||||
runtime: Snapshot runtime override; defaults to the shared
|
||||
process runtime (tests inject an isolated one).
|
||||
"""
|
||||
@@ -255,14 +253,14 @@ class MessageBudgetMiddleware:
|
||||
|
||||
@property
|
||||
def model(self) -> Any: # type: ignore[override]
|
||||
"""Summarizer model: the snapshot's ``summary`` role mapping."""
|
||||
"""Summarizer model: the snapshot's frozen primary."""
|
||||
snapshot = self._snapshot()
|
||||
with self._model_cache_lock:
|
||||
cached = self._snapshot_model_cache.get(snapshot.snapshot_id)
|
||||
if cached is not None:
|
||||
return cached
|
||||
resolved = self._snapshot_runtime().build_role_model(
|
||||
snapshot, "summary"
|
||||
snapshot, "primary"
|
||||
)
|
||||
with self._model_cache_lock:
|
||||
self._snapshot_model_cache[snapshot.snapshot_id] = resolved
|
||||
|
||||
@@ -93,15 +93,12 @@ class _ConditionalToolSelectorMiddleware(AgentMiddleware):
|
||||
self._threshold = threshold
|
||||
self._always_include = always_include or frozenset()
|
||||
self._track_stream_selection = track_stream_selection
|
||||
# Agent tools are fixed after graph construction, so the filtered
|
||||
# always-include set is stable for this middleware instance.
|
||||
self._selector: AgentMiddleware | None = None
|
||||
|
||||
def _build_selector(self, request: ModelRequest) -> AgentMiddleware:
|
||||
if self._selector is None:
|
||||
names = _available_always_include(request.tools, self._always_include)
|
||||
self._selector = self._selector_factory(names)
|
||||
return self._selector
|
||||
# Built per call: the selector's helper model is resolved from the
|
||||
# run snapshot, which varies from run to run.
|
||||
names = _available_always_include(request.tools, self._always_include)
|
||||
return self._selector_factory(names)
|
||||
|
||||
def wrap_model_call(
|
||||
self,
|
||||
@@ -233,8 +230,10 @@ def create_tool_selector_middleware(
|
||||
names for the main-agent stream UI when ``track_stream_selection`` is true
|
||||
|
||||
Args:
|
||||
model: Chat model for tool selection. If *None*, the default
|
||||
model is resolved via ``_ensure_chat_model()``.
|
||||
model: Chat model for tool selection. If *None*, the helper model
|
||||
is resolved from the active run's snapshot on every call
|
||||
(``resolve_snapshot_model``); the registry default only applies
|
||||
to snapshot-less background runs via the lazy local snapshot.
|
||||
threshold: Minimum number of tools to trigger selection.
|
||||
Default 26. Set to 0 to always run selection.
|
||||
track_stream_selection: Whether to update process-global stream/UI
|
||||
@@ -253,34 +252,50 @@ def create_tool_selector_middleware(
|
||||
|
||||
from .utils import disable_thinking
|
||||
|
||||
if model is None:
|
||||
from EvoScientist.EvoScientist import _ensure_chat_model
|
||||
def tag_selector_model(base: BaseChatModel) -> BaseChatModel:
|
||||
safe_model = disable_thinking(base)
|
||||
selector_model = safe_model
|
||||
from EvoScientist.usage.callback import usage_tracking_enabled
|
||||
|
||||
model = _ensure_chat_model()
|
||||
safe_model = disable_thinking(model)
|
||||
selector_model = safe_model
|
||||
from EvoScientist.usage.callback import usage_tracking_enabled
|
||||
|
||||
if usage_tracking_enabled():
|
||||
try:
|
||||
selector_model = safe_model.model_copy(
|
||||
update={
|
||||
"metadata": {
|
||||
**(safe_model.metadata or {}),
|
||||
"usage_scope": "tool_selector",
|
||||
if usage_tracking_enabled():
|
||||
try:
|
||||
selector_model = safe_model.model_copy(
|
||||
update={
|
||||
"metadata": {
|
||||
**(safe_model.metadata or {}),
|
||||
"usage_scope": "tool_selector",
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
if not isinstance(selector_model, BaseChatModel):
|
||||
raise TypeError("selector model copy is not a BaseChatModel")
|
||||
except Exception:
|
||||
# The model factory preflights this path before enabling its callback.
|
||||
# Keep the selector operational even for an unusual third-party model.
|
||||
from EvoScientist.usage.spool import mark_tracking_degraded
|
||||
)
|
||||
if not isinstance(selector_model, BaseChatModel):
|
||||
raise TypeError("selector model copy is not a BaseChatModel")
|
||||
except Exception:
|
||||
# The model factory preflights this path before enabling its
|
||||
# callback. Keep the selector operational even for an unusual
|
||||
# third-party model.
|
||||
from EvoScientist.usage.spool import mark_tracking_degraded
|
||||
|
||||
mark_tracking_degraded("selector_model_copy_unsupported")
|
||||
selector_model = safe_model
|
||||
logger.exception("Could not attach Tool Selector usage scope")
|
||||
mark_tracking_degraded("selector_model_copy_unsupported")
|
||||
selector_model = safe_model
|
||||
logger.exception("Could not attach Tool Selector usage scope")
|
||||
return selector_model
|
||||
|
||||
# The resolved snapshot model is cached per snapshot upstream, so the
|
||||
# tagged copy can be cached by the base model's identity.
|
||||
tagged_cache: dict[int, BaseChatModel] = {}
|
||||
|
||||
def selector_model() -> BaseChatModel:
|
||||
if model is not None:
|
||||
base = model
|
||||
else:
|
||||
from .configurable_model import resolve_snapshot_model
|
||||
|
||||
base = resolve_snapshot_model()
|
||||
tagged = tagged_cache.get(id(base))
|
||||
if tagged is None:
|
||||
tagged = tag_selector_model(base)
|
||||
tagged_cache[id(base)] = tagged
|
||||
return tagged
|
||||
|
||||
system_prompt = (
|
||||
"You are selecting tools for a scientific research agent. "
|
||||
@@ -293,7 +308,7 @@ def create_tool_selector_middleware(
|
||||
|
||||
def selector_factory(always_include: list[str]) -> AgentMiddleware:
|
||||
return LLMToolSelectorMiddleware(
|
||||
model=selector_model,
|
||||
model=selector_model(),
|
||||
system_prompt=system_prompt,
|
||||
always_include=always_include,
|
||||
)
|
||||
|
||||
@@ -581,6 +581,8 @@ def resolve_parameters(
|
||||
provider: ProviderConfig,
|
||||
model: ModelConfig,
|
||||
spec: AdapterParameterSpec,
|
||||
*,
|
||||
reasoning_effort_override: ReasoningEffort | None = None,
|
||||
) -> ResolvedParameters:
|
||||
"""Resolve a model's runtime parameters against the matched contract.
|
||||
|
||||
@@ -589,6 +591,10 @@ def resolve_parameters(
|
||||
zero; ``reasoning_effort=auto`` inherits and is omitted when still auto)
|
||||
and the save-time contract checks. Raises ``ModelRegistryError`` with a
|
||||
stable section 9.5 code on any violation.
|
||||
|
||||
``reasoning_effort_override`` (snapshot creation only) replaces the
|
||||
model's configured effort before inheritance; it flows through the same
|
||||
contract rule, so unsupported adapters still reject non-``auto`` values.
|
||||
"""
|
||||
auth_spec = spec.auth_specs.get(provider.auth.mode)
|
||||
if auth_spec is None:
|
||||
@@ -649,7 +655,11 @@ def resolve_parameters(
|
||||
top_p = provider.runtime.default_top_p
|
||||
top_p = _resolve_nullable_parameter("top_p", _parameter_rule(spec, "top_p"), top_p)
|
||||
|
||||
reasoning_effort: ReasoningEffort = model.runtime.reasoning_effort
|
||||
reasoning_effort: ReasoningEffort = (
|
||||
reasoning_effort_override
|
||||
if reasoning_effort_override not in (None, "auto")
|
||||
else model.runtime.reasoning_effort
|
||||
)
|
||||
if reasoning_effort == "auto":
|
||||
reasoning_effort = provider.runtime.default_reasoning_effort
|
||||
reasoning_effort = _resolve_nullable_parameter(
|
||||
|
||||
@@ -79,6 +79,7 @@ from .schemas import (
|
||||
ModelAvailability,
|
||||
ModelRef,
|
||||
NonEmptyString,
|
||||
RegistryDefaults,
|
||||
RegistryV4,
|
||||
)
|
||||
from .snapshots import (
|
||||
@@ -164,6 +165,9 @@ class GetSelectableModelsResponse(BaseModel):
|
||||
"""The section 9.1 selector response: only selectable enabled models."""
|
||||
|
||||
models: list[SelectableModel]
|
||||
# Lets the selector display the registry default before the first turn,
|
||||
# when no run snapshot exists yet to resolve `inherit` against.
|
||||
defaults: RegistryDefaults = Field(default_factory=RegistryDefaults)
|
||||
|
||||
|
||||
class SnapshotBindRequest(BaseModel):
|
||||
@@ -197,7 +201,6 @@ class SnapshotPublicResponse(BaseModel):
|
||||
snapshot_id: str
|
||||
registry_revision: PositiveInt
|
||||
primary: ResolvedModelPublic
|
||||
auxiliary: ResolvedModelPublic | None = None
|
||||
|
||||
|
||||
# --- services bundle --------------------------------------------------------
|
||||
@@ -754,7 +757,7 @@ class ModelRegistryHttpApi:
|
||||
effective_capabilities=item.effective_capabilities,
|
||||
)
|
||||
)
|
||||
return GetSelectableModelsResponse(models=models)
|
||||
return GetSelectableModelsResponse(models=models, defaults=registry.defaults)
|
||||
|
||||
@staticmethod
|
||||
def _create_snapshot(
|
||||
|
||||
@@ -477,6 +477,9 @@
|
||||
"GetSelectableModelsResponse": {
|
||||
"description": "The section 9.1 selector response: only selectable enabled models.",
|
||||
"properties": {
|
||||
"defaults": {
|
||||
"$ref": "#/components/schemas/RegistryDefaults"
|
||||
},
|
||||
"models": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/SelectableModel"
|
||||
@@ -1034,17 +1037,6 @@
|
||||
},
|
||||
"RegistryDefaults": {
|
||||
"properties": {
|
||||
"auxiliary": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ModelRef"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"primary": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -1221,17 +1213,6 @@
|
||||
"SnapshotCreateRequest": {
|
||||
"description": "The section 8.2 snapshot creation request body.",
|
||||
"properties": {
|
||||
"auxiliary": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ModelRef"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"deployment_id": {
|
||||
"minLength": 1,
|
||||
"title": "Deployment Id",
|
||||
@@ -1254,6 +1235,24 @@
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"reasoning_effort": {
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"auto",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Reasoning Effort"
|
||||
},
|
||||
"run_request_id": {
|
||||
"minLength": 1,
|
||||
"title": "Run Request Id",
|
||||
@@ -1276,17 +1275,6 @@
|
||||
"SnapshotPublicResponse": {
|
||||
"description": "The section 8.2 public snapshot diagnostic subset.",
|
||||
"properties": {
|
||||
"auxiliary": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ResolvedModelPublic"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"primary": {
|
||||
"$ref": "#/components/schemas/ResolvedModelPublic"
|
||||
},
|
||||
|
||||
@@ -50,6 +50,7 @@ from .schemas import (
|
||||
ModelRef,
|
||||
ModelRole,
|
||||
ProviderConfig,
|
||||
ReasoningEffort,
|
||||
RegistryV4,
|
||||
RequestOptions,
|
||||
ResolvedModelConfig,
|
||||
@@ -102,9 +103,16 @@ class ModelRegistryResolver:
|
||||
role: ModelRole = "primary",
|
||||
*,
|
||||
registry: RegistryV4 | None = None,
|
||||
reasoning_effort_override: ReasoningEffort | None = None,
|
||||
) -> ResolvedModelConfig:
|
||||
"""Resolve an enabled, verified model into its frozen run config."""
|
||||
return self._resolve(model_ref, role, registry=registry, require_enabled=True)
|
||||
return self._resolve(
|
||||
model_ref,
|
||||
role,
|
||||
registry=registry,
|
||||
require_enabled=True,
|
||||
reasoning_effort_override=reasoning_effort_override,
|
||||
)
|
||||
|
||||
def resolve_for_test(self, model_ref: ModelRef) -> ResolvedModelConfig:
|
||||
"""Resolve for a provider test (section 9.4).
|
||||
@@ -130,6 +138,7 @@ class ModelRegistryResolver:
|
||||
registry: RegistryV4 | None,
|
||||
require_enabled: bool,
|
||||
require_verified: bool = True,
|
||||
reasoning_effort_override: ReasoningEffort | None = None,
|
||||
) -> ResolvedModelConfig:
|
||||
_check_role(role)
|
||||
if registry is None:
|
||||
@@ -176,7 +185,9 @@ class ModelRegistryResolver:
|
||||
|
||||
# Save-time contract checks re-applied at resolve time: auth mode,
|
||||
# credential reference, declared capabilities, and parameters.
|
||||
parameters = resolve_parameters(provider, model, spec)
|
||||
parameters = resolve_parameters(
|
||||
provider, model, spec, reasoning_effort_override=reasoning_effort_override
|
||||
)
|
||||
|
||||
auth_spec = spec.auth_specs[provider.auth.mode]
|
||||
credential_revision: int | None = None
|
||||
|
||||
@@ -1,17 +1,15 @@
|
||||
"""Runtime-link glue for snapshot-driven model resolution (design doc 8.3).
|
||||
|
||||
``SnapshotRuntime`` bundles the store, resolver, and snapshot service into
|
||||
the single accessor the running agent link (middleware, agent factories,
|
||||
auxiliary roles) uses. It provides two resolution paths:
|
||||
the single accessor the running agent link (middleware, agent factories)
|
||||
uses. It provides two resolution paths:
|
||||
|
||||
- ``build_role_model(snapshot, role)`` — per-call construction from a frozen
|
||||
run snapshot (section 6.1 role mapping, section 5.2 per-call credential
|
||||
resolution). This is the only path the per-call middleware uses.
|
||||
- ``build_default_role_model(role)`` — build-time construction from the
|
||||
active registry's ``defaults`` for auxiliary roles that bind a model when
|
||||
a graph is compiled (tool selector, scheduler, memory workers). Auxiliary,
|
||||
summary, and tool-selector roles map to ``defaults.auxiliary ??
|
||||
defaults.primary``; no free model/provider strings are consulted anywhere.
|
||||
run snapshot (section 5.2 per-call credential resolution). Every role
|
||||
resolves to the snapshot's frozen primary (section 6.1).
|
||||
- ``build_default_role_model()`` — build-time construction from the active
|
||||
registry's default primary, used only as the compile-time graph binding;
|
||||
every run re-resolves its model from the run snapshot.
|
||||
|
||||
Every model is constructed through ``build_chat_model`` with both safe HTTP
|
||||
clients (section 6.4). Ollama adapters additionally carry ``max_retries``
|
||||
@@ -118,8 +116,8 @@ class SnapshotRuntime:
|
||||
|
||||
# --- registry-default (build-time) resolution ----------------------------
|
||||
|
||||
def registry_defaults(self) -> tuple[ModelRef, ModelRef | None, int]:
|
||||
"""Return ``(primary, auxiliary, revision)`` for an active registry."""
|
||||
def registry_default(self) -> tuple[ModelRef, int]:
|
||||
"""Return ``(primary, revision)`` for an active registry."""
|
||||
registry = self._store.load_registry()
|
||||
if registry.state != "active" or registry.defaults.primary is None:
|
||||
raise ModelRegistryError(
|
||||
@@ -127,23 +125,16 @@ class SnapshotRuntime:
|
||||
"The model registry is in bootstrap; configure and enable a "
|
||||
"primary model first.",
|
||||
)
|
||||
return registry.defaults.primary, registry.defaults.auxiliary, registry.revision
|
||||
return registry.defaults.primary, registry.revision
|
||||
|
||||
def resolve_role_config(self, role: ModelRole = "primary") -> ResolvedModelConfig:
|
||||
"""Resolve a role against the registry defaults (section 6.1).
|
||||
|
||||
``primary`` resolves ``defaults.primary``; every auxiliary role
|
||||
(``auxiliary``, ``summary``, ``tool_selector``) resolves
|
||||
``defaults.auxiliary ?? defaults.primary``.
|
||||
"""
|
||||
primary, auxiliary, _ = self.registry_defaults()
|
||||
if role == "primary" or auxiliary is None:
|
||||
return self._resolver.resolve(primary, "primary")
|
||||
return self._resolver.resolve(auxiliary, "auxiliary")
|
||||
def resolve_default_config(self) -> ResolvedModelConfig:
|
||||
"""Resolve the registry default primary (section 6.1)."""
|
||||
primary, _ = self.registry_default()
|
||||
return self._resolver.resolve(primary, "primary")
|
||||
|
||||
def build_default_role_model(self, role: ModelRole = "primary") -> BaseChatModel:
|
||||
"""Build the chat model for a role from the registry defaults."""
|
||||
config = self.resolve_role_config(role)
|
||||
"""Build the chat model for a role from the registry default."""
|
||||
config = self.resolve_default_config()
|
||||
return self._build(config, self._resolve_credential(config))
|
||||
|
||||
# --- local entry points (design doc 8.1) ---------------------------------
|
||||
@@ -182,7 +173,6 @@ class SnapshotRuntime:
|
||||
deployment_id=self._local_deployment_id,
|
||||
model_selection_revision=0,
|
||||
primary=None,
|
||||
auxiliary=None,
|
||||
)
|
||||
)
|
||||
return creation.snapshot
|
||||
@@ -207,7 +197,7 @@ class SnapshotRuntime:
|
||||
config.client_options.max_retries if config.adapter_id == "ollama" else 0
|
||||
)
|
||||
timeout = config.client_options.timeout_seconds
|
||||
return build_chat_model(
|
||||
model = build_chat_model(
|
||||
config,
|
||||
http_client=build_safe_http_client(
|
||||
self._endpoint_policy, timeout=timeout, retries=retries
|
||||
@@ -217,6 +207,21 @@ class SnapshotRuntime:
|
||||
),
|
||||
credential=credential,
|
||||
)
|
||||
# Usage tracking is inert unless the launcher supplied the complete
|
||||
# sink environment; the adapter contract revision stands in for the
|
||||
# removed per-profile runtime revision.
|
||||
from ..usage import UsageModelIdentity, attach_usage_callback
|
||||
|
||||
return attach_usage_callback(
|
||||
model,
|
||||
UsageModelIdentity(
|
||||
provider_profile_id=config.model_ref.provider_id,
|
||||
provider_revision=str(config.adapter_spec_revision),
|
||||
provider_adapter=config.adapter_id,
|
||||
model_alias=config.model_ref.model_key,
|
||||
upstream_model_id=config.upstream_model_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# --- process-shared default runtime -------------------------------------------
|
||||
|
||||
@@ -45,7 +45,7 @@ RegistryState = Literal["bootstrap", "active"]
|
||||
LimitMode = Literal["combined", "input_only"]
|
||||
LimitsStatus = Literal["confirmed", "needs_confirmation"]
|
||||
LimitsSource = Literal["provider", "tested_contract", "user"]
|
||||
ModelRole = Literal["primary", "auxiliary", "summary", "tool_selector"]
|
||||
ModelRole = Literal["primary"]
|
||||
|
||||
|
||||
class _FrozenModel(BaseModel):
|
||||
@@ -61,7 +61,6 @@ class ModelRef(_FrozenModel):
|
||||
|
||||
class RegistryDefaults(BaseModel):
|
||||
primary: ModelRef | None = None
|
||||
auxiliary: ModelRef | None = None
|
||||
|
||||
|
||||
class Capabilities(BaseModel):
|
||||
@@ -175,22 +174,18 @@ class RegistryV4(BaseModel):
|
||||
provider_ids = [provider.id for provider in self.providers]
|
||||
if len(provider_ids) != len(set(provider_ids)):
|
||||
raise ValueError("Provider IDs must be unique.")
|
||||
for role, ref in (
|
||||
("primary", self.defaults.primary),
|
||||
("auxiliary", self.defaults.auxiliary),
|
||||
):
|
||||
if ref is None:
|
||||
continue
|
||||
ref = self.defaults.primary
|
||||
if ref is not None:
|
||||
target = self._locate(ref)
|
||||
if target is None:
|
||||
raise ValueError(
|
||||
f"defaults.{role} references an unknown ModelRef "
|
||||
f"defaults.primary references an unknown ModelRef "
|
||||
f"{ref.provider_id!r}/{ref.model_key!r}."
|
||||
)
|
||||
provider, model = target
|
||||
if self.state == "active" and not (provider.enabled and model.enabled):
|
||||
raise ValueError(
|
||||
f"defaults.{role} must reference an enabled model while "
|
||||
"defaults.primary must reference an enabled model while "
|
||||
"the registry is active."
|
||||
)
|
||||
if self.state == "active" and self.defaults.primary is None:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
``SnapshotService`` is the single implementation shared by the HTTP snapshot
|
||||
API (Task 5) and the local CLI/channel/scheduler entry points (Task 7): it
|
||||
freezes both roles' complete ``ResolvedModelConfig`` — adapter spec
|
||||
freezes the primary role's complete ``ResolvedModelConfig`` — adapter spec
|
||||
revision, fixed budget reserves, capabilities, and credential revisions —
|
||||
into ``run_runtime_snapshots.payload_json``. Payloads and logs never carry
|
||||
``secret_value``.
|
||||
@@ -11,9 +11,9 @@ into ``run_runtime_snapshots.payload_json``. Payloads and logs never carry
|
||||
maps to at most one non-terminal snapshot. A repeated request with the
|
||||
same ``selection_hash`` returns the original snapshot; a different hash
|
||||
raises ``RUN_REQUEST_CONFLICT``. ``selection_hash`` covers the
|
||||
pre-resolution ``{primary, auxiliary}`` selection (inherit participates as
|
||||
``null``); ``model_selection_revision`` is an audit field and never part
|
||||
of the hash.
|
||||
pre-resolution ``{primary, reasoning_effort}`` selection (inherit
|
||||
participates as ``null``); ``model_selection_revision`` is an audit field
|
||||
and never part of the hash.
|
||||
- Lifecycle: ``prepared`` (TTL 15 minutes) → ``bound`` (retained 24 hours
|
||||
after binding) → ``expired`` (terminal, via ``cleanup_expired``);
|
||||
``prepared`` → ``aborted`` on run creation failure. Terminal snapshots
|
||||
@@ -51,6 +51,7 @@ from .schemas import (
|
||||
ModelRef,
|
||||
ModelRole,
|
||||
NonEmptyString,
|
||||
ReasoningEffort,
|
||||
ResolvedModelConfig,
|
||||
)
|
||||
from .store import ModelRuntimeStore
|
||||
@@ -69,9 +70,12 @@ class SnapshotCreateRequest(BaseModel):
|
||||
thread_id: NonEmptyString
|
||||
deployment_id: NonEmptyString
|
||||
model_selection_revision: NonNegativeInt = 0
|
||||
# ``None`` means inherit: the registry defaults are resolved at creation.
|
||||
# ``None`` means inherit: the registry default is resolved at creation.
|
||||
primary: ModelRef | None = None
|
||||
auxiliary: ModelRef | None = None
|
||||
# Per-run reasoning-effort override (thread-level selection);
|
||||
# ``None``/``auto`` keeps the registry-configured effort. Unsupported
|
||||
# adapters reject it.
|
||||
reasoning_effort: ReasoningEffort | None = None
|
||||
|
||||
|
||||
class SnapshotPayload(BaseModel):
|
||||
@@ -80,7 +84,6 @@ class SnapshotPayload(BaseModel):
|
||||
registry_revision: PositiveInt
|
||||
model_selection_revision: NonNegativeInt
|
||||
primary: ResolvedModelConfig
|
||||
auxiliary: ResolvedModelConfig | None = None
|
||||
|
||||
|
||||
class RuntimeSnapshot(BaseModel):
|
||||
@@ -120,16 +123,20 @@ class SnapshotCreation(BaseModel):
|
||||
created: bool
|
||||
|
||||
|
||||
def compute_selection_hash(primary: ModelRef | None, auxiliary: ModelRef | None) -> str:
|
||||
def compute_selection_hash(
|
||||
primary: ModelRef | None,
|
||||
reasoning_effort: ReasoningEffort | None = None,
|
||||
) -> str:
|
||||
"""Hash the pre-resolution selection; inherit participates as ``null``."""
|
||||
|
||||
def entry(ref: ModelRef | None) -> dict[str, str] | None:
|
||||
if ref is None:
|
||||
return None
|
||||
return {"provider_id": ref.provider_id, "model_key": ref.model_key}
|
||||
|
||||
encoded = json.dumps(
|
||||
{"primary": entry(primary), "auxiliary": entry(auxiliary)},
|
||||
{
|
||||
"primary": (
|
||||
None
|
||||
if primary is None
|
||||
else {"provider_id": primary.provider_id, "model_key": primary.model_key}
|
||||
),
|
||||
"reasoning_effort": reasoning_effort,
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
@@ -137,18 +144,12 @@ def compute_selection_hash(primary: ModelRef | None, auxiliary: ModelRef | None)
|
||||
|
||||
|
||||
def config_for_role(snapshot: RuntimeSnapshot, role: ModelRole) -> ResolvedModelConfig:
|
||||
"""The section 6.1 role mapping; no other mapping may be guessed.
|
||||
|
||||
``primary`` → ``snapshot.primary``; ``auxiliary``, ``summary``, and
|
||||
``tool_selector`` → ``snapshot.auxiliary ?? snapshot.primary``.
|
||||
"""
|
||||
"""Every role resolves to the snapshot's frozen primary (section 6.1)."""
|
||||
if role not in _MODEL_ROLES:
|
||||
raise ValueError(
|
||||
f"Unknown model role {role!r}; expected one of {list(_MODEL_ROLES)}."
|
||||
)
|
||||
if role == "primary" or snapshot.payload.auxiliary is None:
|
||||
return snapshot.payload.primary
|
||||
return snapshot.payload.auxiliary
|
||||
return snapshot.payload.primary
|
||||
|
||||
|
||||
def public_snapshot_view(snapshot: RuntimeSnapshot) -> dict[str, Any]:
|
||||
@@ -163,17 +164,16 @@ def public_snapshot_view(snapshot: RuntimeSnapshot) -> dict[str, Any]:
|
||||
"max_output_tokens": config.request_options.max_output_tokens,
|
||||
"temperature": config.request_options.temperature,
|
||||
"top_p": config.request_options.top_p,
|
||||
"reasoning_effort": config.request_options.reasoning_effort,
|
||||
"timeout_seconds": config.client_options.timeout_seconds,
|
||||
"max_retries": config.client_options.max_retries,
|
||||
},
|
||||
}
|
||||
|
||||
auxiliary = snapshot.payload.auxiliary
|
||||
return {
|
||||
"snapshot_id": snapshot.snapshot_id,
|
||||
"registry_revision": snapshot.payload.registry_revision,
|
||||
"primary": public_config(snapshot.payload.primary),
|
||||
"auxiliary": None if auxiliary is None else public_config(auxiliary),
|
||||
}
|
||||
|
||||
|
||||
@@ -197,7 +197,9 @@ class SnapshotService:
|
||||
"The model registry is in bootstrap; configure and enable a "
|
||||
"primary model first.",
|
||||
)
|
||||
selection_hash = compute_selection_hash(request.primary, request.auxiliary)
|
||||
selection_hash = compute_selection_hash(
|
||||
request.primary, request.reasoning_effort
|
||||
)
|
||||
existing = self._store.find_active_run_snapshot(
|
||||
deployment_id=request.deployment_id,
|
||||
thread_id=request.thread_id,
|
||||
@@ -219,33 +221,21 @@ class SnapshotService:
|
||||
if request.primary is not None
|
||||
else registry.defaults.primary
|
||||
)
|
||||
auxiliary_ref = (
|
||||
request.auxiliary
|
||||
if request.auxiliary is not None
|
||||
else registry.defaults.auxiliary
|
||||
)
|
||||
if primary_ref is None: # pragma: no cover - active registries set it
|
||||
raise ModelRegistryError(
|
||||
MODEL_REGISTRY_NOT_READY,
|
||||
"The model registry has no default primary model.",
|
||||
)
|
||||
primary_config = self._resolver.resolve(
|
||||
primary_ref, "primary", registry=registry
|
||||
)
|
||||
auxiliary_config = (
|
||||
None
|
||||
if auxiliary_ref is None
|
||||
else self._resolver.resolve(auxiliary_ref, "auxiliary", registry=registry)
|
||||
primary_ref,
|
||||
"primary",
|
||||
registry=registry,
|
||||
reasoning_effort_override=request.reasoning_effort,
|
||||
)
|
||||
payload = {
|
||||
"registry_revision": registry.revision,
|
||||
"model_selection_revision": request.model_selection_revision,
|
||||
"primary": primary_config.model_dump(mode="json"),
|
||||
"auxiliary": (
|
||||
None
|
||||
if auxiliary_config is None
|
||||
else auxiliary_config.model_dump(mode="json")
|
||||
),
|
||||
}
|
||||
snapshot_id = f"snap-{uuid.uuid4().hex}"
|
||||
try:
|
||||
@@ -423,23 +413,20 @@ class SnapshotService:
|
||||
# --- internals ------------------------------------------------------------
|
||||
|
||||
def _ensure_frozen_specs_available(self, snapshot: RuntimeSnapshot) -> None:
|
||||
configs = [snapshot.payload.primary]
|
||||
if snapshot.payload.auxiliary is not None:
|
||||
configs.append(snapshot.payload.auxiliary)
|
||||
for config in configs:
|
||||
spec = find_adapter_spec(
|
||||
config.adapter_id,
|
||||
config.upstream_model_id,
|
||||
spec_revision=config.adapter_spec_revision,
|
||||
specs=self._resolver.specs,
|
||||
config = snapshot.payload.primary
|
||||
spec = find_adapter_spec(
|
||||
config.adapter_id,
|
||||
config.upstream_model_id,
|
||||
spec_revision=config.adapter_spec_revision,
|
||||
specs=self._resolver.specs,
|
||||
)
|
||||
if spec is None:
|
||||
raise ModelRegistryError(
|
||||
ADAPTER_NOT_SUPPORTED,
|
||||
f"The adapter contract {config.adapter_id!r} at "
|
||||
f"spec_revision {config.adapter_spec_revision} frozen by "
|
||||
"this snapshot no longer exists.",
|
||||
)
|
||||
if spec is None:
|
||||
raise ModelRegistryError(
|
||||
ADAPTER_NOT_SUPPORTED,
|
||||
f"The adapter contract {config.adapter_id!r} at "
|
||||
f"spec_revision {config.adapter_spec_revision} frozen by "
|
||||
"this snapshot no longer exists.",
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_not_found() -> ModelRegistryError:
|
||||
|
||||
@@ -319,26 +319,23 @@ class ModelRuntimeStore:
|
||||
|
||||
def _validate_defaults(self, registry: RegistryV4) -> None:
|
||||
"""Enforce section 9.2 rule 7 on every save, regardless of state."""
|
||||
for role, ref in (
|
||||
("primary", registry.defaults.primary),
|
||||
("auxiliary", registry.defaults.auxiliary),
|
||||
):
|
||||
if ref is None:
|
||||
continue
|
||||
located = registry._locate(ref)
|
||||
if located is None:
|
||||
raise ModelRegistryError(
|
||||
MODEL_DISABLED,
|
||||
f"defaults.{role} references an unknown model.",
|
||||
details=[{"path": f"defaults.{role}", "code": MODEL_DISABLED}],
|
||||
)
|
||||
provider, model = located
|
||||
if not (provider.enabled and model.enabled):
|
||||
raise ModelRegistryError(
|
||||
MODEL_DISABLED,
|
||||
f"defaults.{role} must reference an enabled model.",
|
||||
details=[{"path": f"defaults.{role}", "code": MODEL_DISABLED}],
|
||||
)
|
||||
ref = registry.defaults.primary
|
||||
if ref is None:
|
||||
return
|
||||
located = registry._locate(ref)
|
||||
if located is None:
|
||||
raise ModelRegistryError(
|
||||
MODEL_DISABLED,
|
||||
"defaults.primary references an unknown model.",
|
||||
details=[{"path": "defaults.primary", "code": MODEL_DISABLED}],
|
||||
)
|
||||
provider, model = located
|
||||
if not (provider.enabled and model.enabled):
|
||||
raise ModelRegistryError(
|
||||
MODEL_DISABLED,
|
||||
"defaults.primary must reference an enabled model.",
|
||||
details=[{"path": "defaults.primary", "code": MODEL_DISABLED}],
|
||||
)
|
||||
|
||||
def _resolve_state(
|
||||
self,
|
||||
|
||||
@@ -11,9 +11,8 @@ Reuses the main EvoScientist agent's backend and middleware so the deployed
|
||||
sub-agent has full capability parity with its in-process synchronous
|
||||
counterpart: same workspace files, same ``/skills/`` and ``/memories/``
|
||||
routes, same error-handling and context-overflow middleware. The chat model
|
||||
is resolved from the active model registry's defaults (``auxiliary`` for the
|
||||
scheduler, ``primary`` otherwise) and re-resolved per run from the run
|
||||
snapshot by ``ConfigurableModelMiddleware``.
|
||||
is resolved from the active model registry's default primary and re-resolved
|
||||
per run from the run snapshot by ``ConfigurableModelMiddleware``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -103,12 +102,10 @@ def build_async_subagent_graph(name: str) -> Any:
|
||||
# context and `/memories/profile/...` file guidance as the main agent.
|
||||
#
|
||||
# The compile-time model binding is resolved from the active registry's
|
||||
# defaults — never from config.yaml free strings. Per-run calls are
|
||||
# re-resolved from the run snapshot by ConfigurableModelMiddleware; the
|
||||
# scheduler (an unattended timer task) binds the cheaper auxiliary role,
|
||||
# working sub-agents bind primary.
|
||||
# default primary — never from config.yaml free strings. Per-run calls
|
||||
# are re-resolved from the run snapshot by ConfigurableModelMiddleware.
|
||||
#
|
||||
# Bootstrap registry: no defaults exist yet, so no compile-time model
|
||||
# Bootstrap registry: no default exists yet, so no compile-time model
|
||||
# can be built. The graph must still materialize — one failing factory
|
||||
# must not take down the whole langgraph dev service — so we bind a
|
||||
# placeholder that raises MODEL_REGISTRY_NOT_READY on the first model
|
||||
@@ -121,7 +118,7 @@ def build_async_subagent_graph(name: str) -> Any:
|
||||
from EvoScientist.model_registry.placeholder import RegistryNotReadyChatModel
|
||||
|
||||
runtime = get_snapshot_runtime()
|
||||
snapshot_role = "auxiliary" if name == "scheduler" else "primary"
|
||||
snapshot_role = "primary"
|
||||
try:
|
||||
model = runtime.build_default_role_model(snapshot_role)
|
||||
except ModelRegistryError as exc:
|
||||
|
||||
@@ -102,18 +102,13 @@ def ollama_provider_payload(**model_overrides) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def registry_payload(*, auxiliary_default: bool = True, **model_overrides) -> dict:
|
||||
def registry_payload(**model_overrides) -> dict:
|
||||
return {
|
||||
"version": 4,
|
||||
"revision": 1,
|
||||
"state": "bootstrap",
|
||||
"defaults": {
|
||||
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
|
||||
"auxiliary": (
|
||||
{"provider_id": "local-ollama", "model_key": "qwen3"}
|
||||
if auxiliary_default
|
||||
else None
|
||||
),
|
||||
},
|
||||
"providers": [
|
||||
zhipu_provider_payload(**model_overrides),
|
||||
@@ -141,12 +136,12 @@ def _verify_model(store, registry, provider_id, model_key):
|
||||
)
|
||||
|
||||
|
||||
def activate_store(store, *, auxiliary_default: bool = True, **model_overrides):
|
||||
def activate_store(store, **model_overrides):
|
||||
"""Persist an active registry with verified models into *store*."""
|
||||
registry = store.save_registry(
|
||||
expected_revision=1,
|
||||
registry=RegistryV4.model_validate(
|
||||
registry_payload(auxiliary_default=auxiliary_default, **model_overrides)
|
||||
registry_payload(**model_overrides)
|
||||
),
|
||||
credential_writes=[
|
||||
CredentialWrite(credential_id="zhipu-primary", secret_value=ZHIPU_SECRET)
|
||||
@@ -158,11 +153,10 @@ def activate_store(store, *, auxiliary_default: bool = True, **model_overrides):
|
||||
return store
|
||||
|
||||
|
||||
def make_active_store(config_dir, *, auxiliary_default: bool = True, **model_overrides):
|
||||
def make_active_store(config_dir, **model_overrides):
|
||||
"""Persist an active registry with verified models into a fresh store."""
|
||||
return activate_store(
|
||||
ModelRuntimeStore(config_dir=config_dir),
|
||||
auxiliary_default=auxiliary_default,
|
||||
**model_overrides,
|
||||
)
|
||||
|
||||
@@ -174,7 +168,6 @@ def make_snapshot_request(**overrides) -> SnapshotCreateRequest:
|
||||
"deployment_id": "local",
|
||||
"model_selection_revision": 0,
|
||||
"primary": None,
|
||||
"auxiliary": None,
|
||||
}
|
||||
payload.update(overrides)
|
||||
return SnapshotCreateRequest.model_validate(payload)
|
||||
|
||||
+3
-12
@@ -539,18 +539,15 @@ class TestConfig:
|
||||
|
||||
|
||||
@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[])
|
||||
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_auto_approve_still_includes_ask_user_middleware(
|
||||
mock_config, mock_model, mock_aux_model, mock_tool_selector
|
||||
mock_config, mock_model, mock_tool_selector
|
||||
):
|
||||
cfg = MagicMock()
|
||||
cfg.enable_ask_user = True
|
||||
cfg.auto_approve = True
|
||||
cfg.auto_mode = False
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
|
||||
@@ -561,18 +558,15 @@ def test_auto_approve_still_includes_ask_user_middleware(
|
||||
|
||||
|
||||
@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[])
|
||||
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_auto_mode_disables_ask_user_middleware(
|
||||
mock_config, mock_model, mock_aux_model, mock_tool_selector
|
||||
mock_config, mock_model, mock_tool_selector
|
||||
):
|
||||
cfg = MagicMock()
|
||||
cfg.enable_ask_user = True
|
||||
cfg.auto_approve = True
|
||||
cfg.auto_mode = True
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
|
||||
@@ -583,11 +577,10 @@ def test_auto_mode_disables_ask_user_middleware(
|
||||
|
||||
|
||||
@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[])
|
||||
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_for_async_subagent_omits_ask_user_middleware(
|
||||
mock_config, mock_model, mock_aux_model, mock_tool_selector
|
||||
mock_config, mock_model, mock_tool_selector
|
||||
):
|
||||
"""``AskUserMiddleware`` uses ``interrupt()`` to wait on user input.
|
||||
|
||||
@@ -602,8 +595,6 @@ def test_for_async_subagent_omits_ask_user_middleware(
|
||||
cfg.enable_ask_user = True
|
||||
cfg.auto_approve = False
|
||||
cfg.auto_mode = False
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
|
||||
|
||||
@@ -123,7 +123,7 @@ def test_factory_requests_async_safe_middleware(
|
||||
@patch("EvoScientist.utils.load_subagents")
|
||||
@patch("EvoScientist.config.apply_config_to_env")
|
||||
@patch("EvoScientist.config.get_effective_config")
|
||||
def test_scheduler_binds_auxiliary_role(
|
||||
def test_scheduler_binds_primary_role(
|
||||
mock_get_cfg,
|
||||
mock_apply_env,
|
||||
mock_load_subs,
|
||||
@@ -133,7 +133,7 @@ def test_scheduler_binds_auxiliary_role(
|
||||
mock_mcp,
|
||||
mock_create,
|
||||
):
|
||||
"""The scheduler is an unattended timer task → auxiliary role mapping."""
|
||||
"""The scheduler resolves the registry's primary default (6.1)."""
|
||||
cfg = MagicMock()
|
||||
cfg.recursion_limit = 1_000_000
|
||||
cfg.memory_profile_enabled = False
|
||||
@@ -141,7 +141,7 @@ def test_scheduler_binds_auxiliary_role(
|
||||
cfg.memory_observation_writer = MemoryObservationWriter.ALL
|
||||
cfg.memory_workers_enabled = False
|
||||
mock_get_cfg.return_value = cfg
|
||||
model = MagicMock(name="auxiliary_model")
|
||||
model = MagicMock(name="primary_model")
|
||||
mock_get_runtime.return_value.build_default_role_model.return_value = model
|
||||
mock_load_subs.return_value = [
|
||||
{
|
||||
@@ -158,13 +158,13 @@ def test_scheduler_binds_auxiliary_role(
|
||||
build_async_subagent_graph("scheduler")
|
||||
|
||||
mock_get_runtime.return_value.build_default_role_model.assert_called_once_with(
|
||||
"auxiliary"
|
||||
"primary"
|
||||
)
|
||||
mock_get_mw.assert_called_once_with(
|
||||
for_async_subagent=True,
|
||||
memory_source_agent="scheduler",
|
||||
chat_model=model,
|
||||
snapshot_role="auxiliary",
|
||||
snapshot_role="primary",
|
||||
)
|
||||
assert mock_create.call_args.kwargs["model"] is model
|
||||
|
||||
@@ -254,8 +254,6 @@ def test_inject_subagent_omits_memory_middleware_when_memory_disabled(
|
||||
cfg.memory_observations_enabled = False
|
||||
cfg.memory_observation_writer = MemoryObservationWriter.ALL
|
||||
cfg.memory_workers_enabled = True
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
|
||||
from EvoScientist.EvoScientist import _inject_subagent_middleware
|
||||
@@ -284,8 +282,6 @@ def test_inject_subagent_worker_only_observation_writer_keeps_live_tool_off(
|
||||
cfg.memory_observations_enabled = True
|
||||
cfg.memory_observation_writer = MemoryObservationWriter.WORKER
|
||||
cfg.memory_workers_enabled = True
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
|
||||
from EvoScientist.EvoScientist import _inject_subagent_middleware
|
||||
@@ -312,11 +308,10 @@ def test_inject_subagent_worker_only_observation_writer_keeps_live_tool_off(
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
return_value=[MagicMock()],
|
||||
)
|
||||
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._compile_time_role_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_all_observation_writer_schedules_turn_worker_without_profile_memory(
|
||||
mock_config, mock_chat, mock_aux, mock_tool_selector
|
||||
mock_config, mock_chat, mock_tool_selector
|
||||
):
|
||||
cfg = MagicMock()
|
||||
cfg.enable_ask_user = False
|
||||
@@ -327,8 +322,6 @@ def test_all_observation_writer_schedules_turn_worker_without_profile_memory(
|
||||
cfg.memory_observations_enabled = True
|
||||
cfg.memory_observation_writer = MemoryObservationWriter.ALL
|
||||
cfg.memory_workers_enabled = True
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
|
||||
@@ -365,11 +358,10 @@ def test_all_observation_writer_schedules_turn_worker_without_profile_memory(
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
return_value=[MagicMock()],
|
||||
)
|
||||
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._compile_time_role_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_async_subagent_mode_filters_ask_user(
|
||||
mock_config, mock_chat, mock_aux, mock_tool_selector
|
||||
mock_config, mock_chat, mock_tool_selector
|
||||
):
|
||||
"""``_get_default_middleware(for_async_subagent=True)`` must drop
|
||||
``AskUserMiddleware`` even when ``enable_ask_user`` is on.
|
||||
@@ -388,8 +380,6 @@ def test_async_subagent_mode_filters_ask_user(
|
||||
cfg.memory_observations_enabled = True
|
||||
cfg.memory_observation_writer = MemoryObservationWriter.ALL
|
||||
cfg.memory_workers_enabled = True
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
|
||||
@@ -415,7 +405,7 @@ def test_async_subagent_mode_filters_ask_user(
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
return_value=[MagicMock()],
|
||||
)
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._compile_time_role_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_async_subagent_disables_tool_selector_stream_tracking(
|
||||
mock_config, mock_chat, mock_tool_selector
|
||||
@@ -430,8 +420,6 @@ def test_async_subagent_disables_tool_selector_stream_tracking(
|
||||
cfg.memory_observations_enabled = True
|
||||
cfg.memory_observation_writer = MemoryObservationWriter.ALL
|
||||
cfg.memory_workers_enabled = True
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
|
||||
|
||||
+21
-101
@@ -1,88 +1,17 @@
|
||||
"""Tests for the auxiliary-model resolver and its middleware scoping.
|
||||
"""Tests for the helper-model wiring in ``_get_default_middleware``.
|
||||
|
||||
Covers ``EvoScientist.EvoScientist._ensure_auxiliary_chat_model`` — now
|
||||
resolved through the model registry role mapping (design doc 6.1:
|
||||
``defaults.auxiliary ?? defaults.primary``) instead of the legacy
|
||||
``cfg.auxiliary_model``/``auxiliary_provider`` free strings — and the wiring
|
||||
in ``_get_default_middleware`` that routes the main agent's tool selector to
|
||||
the auxiliary model while keeping context editing — and async sub-agents —
|
||||
on the main model.
|
||||
After the auxiliary-role removal (design doc 6.1: every role resolves to the
|
||||
snapshot's frozen primary), the main agent's tool selector no longer binds a
|
||||
compile-time auxiliary model — it resolves its helper model from the run
|
||||
snapshot on every call (``model=None``). Async sub-agents and the pure path
|
||||
keep their threaded model. Context editing always keeps the main model.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import EvoScientist.EvoScientist as E
|
||||
from EvoScientist.model_registry.errors import (
|
||||
MODEL_REGISTRY_NOT_READY,
|
||||
ModelRegistryError,
|
||||
)
|
||||
from tests.registry_fixtures import activate_store
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_model_caches(monkeypatch):
|
||||
"""Isolate the module-level model caches per test."""
|
||||
monkeypatch.setattr(E, "_chat_model", None, raising=False)
|
||||
monkeypatch.setattr(E, "_chat_model_key", None, raising=False)
|
||||
monkeypatch.setattr(E, "_auxiliary_chat_model", None, raising=False)
|
||||
monkeypatch.setattr(E, "_auxiliary_chat_model_key", None, raising=False)
|
||||
|
||||
|
||||
class TestAuxiliaryResolver:
|
||||
def test_bootstrap_registry_fails_closed(self, isolated_snapshot_runtime):
|
||||
"""Bootstrap registry: auxiliary resolution propagates NOT_READY."""
|
||||
with pytest.raises(ModelRegistryError) as excinfo:
|
||||
E._ensure_auxiliary_chat_model()
|
||||
assert excinfo.value.code == MODEL_REGISTRY_NOT_READY
|
||||
|
||||
def test_no_auxiliary_default_returns_main_instance(
|
||||
self, monkeypatch, isolated_snapshot_runtime
|
||||
):
|
||||
main = object()
|
||||
activate_store(isolated_snapshot_runtime.store, auxiliary_default=False)
|
||||
monkeypatch.setattr(E, "_ensure_chat_model", lambda: main)
|
||||
assert E._ensure_auxiliary_chat_model() is main
|
||||
|
||||
def test_auxiliary_equal_to_primary_reuses_main_instance(
|
||||
self, monkeypatch, isolated_snapshot_runtime
|
||||
):
|
||||
main = object()
|
||||
store = activate_store(isolated_snapshot_runtime.store)
|
||||
registry = store.load_registry()
|
||||
registry.defaults.auxiliary = registry.defaults.primary
|
||||
store.save_registry(expected_revision=registry.revision, registry=registry)
|
||||
monkeypatch.setattr(E, "_ensure_chat_model", lambda: main)
|
||||
assert E._ensure_auxiliary_chat_model() is main
|
||||
|
||||
def test_auxiliary_default_builds_via_role_mapping(
|
||||
self, monkeypatch, isolated_snapshot_runtime
|
||||
):
|
||||
fake = object()
|
||||
build = MagicMock(return_value=fake)
|
||||
activate_store(isolated_snapshot_runtime.store)
|
||||
monkeypatch.setattr(
|
||||
isolated_snapshot_runtime, "build_default_role_model", build
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
E, "_ensure_chat_model", lambda: pytest.fail("must not build main")
|
||||
)
|
||||
assert E._ensure_auxiliary_chat_model() is fake
|
||||
build.assert_called_once_with("auxiliary")
|
||||
|
||||
def test_auxiliary_cache_reused_within_same_registry_revision(
|
||||
self, monkeypatch, isolated_snapshot_runtime
|
||||
):
|
||||
build = MagicMock(side_effect=[object(), object()])
|
||||
activate_store(isolated_snapshot_runtime.store)
|
||||
monkeypatch.setattr(
|
||||
isolated_snapshot_runtime, "build_default_role_model", build
|
||||
)
|
||||
first = E._ensure_auxiliary_chat_model()
|
||||
assert E._ensure_auxiliary_chat_model() is first
|
||||
assert build.call_count == 1
|
||||
|
||||
def _mock_cfg():
|
||||
cfg = MagicMock()
|
||||
cfg.enable_ask_user = False
|
||||
@@ -91,8 +20,8 @@ def _mock_cfg():
|
||||
return cfg
|
||||
|
||||
|
||||
class TestAuxiliaryMiddlewareScope:
|
||||
"""``_get_default_middleware`` routes only the right components to aux."""
|
||||
class TestHelperModelScope:
|
||||
"""``_get_default_middleware`` routes only the right components."""
|
||||
|
||||
def _capture(self):
|
||||
cap: dict[str, object] = {}
|
||||
@@ -107,17 +36,13 @@ class TestAuxiliaryMiddlewareScope:
|
||||
|
||||
return cap, fake_tool_selector, fake_context_editing
|
||||
|
||||
def test_main_agent_tool_selector_aux_context_editing_main(self):
|
||||
def test_main_agent_tool_selector_uses_snapshot_resolution(self):
|
||||
cap, fake_ts, fake_ce = self._capture()
|
||||
main_model, aux_model = object(), object()
|
||||
main_model = object()
|
||||
with (
|
||||
patch.object(E, "_ensure_config", return_value=_mock_cfg()),
|
||||
patch.object(
|
||||
E,
|
||||
"_compile_time_role_model",
|
||||
side_effect=lambda role="primary": aux_model
|
||||
if role == "auxiliary"
|
||||
else main_model,
|
||||
E, "_compile_time_role_model", return_value=main_model
|
||||
),
|
||||
patch(
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
@@ -130,20 +55,17 @@ class TestAuxiliaryMiddlewareScope:
|
||||
):
|
||||
E._get_default_middleware()
|
||||
|
||||
assert cap["tool_selector"] is aux_model
|
||||
# None = per-call resolution from the run snapshot.
|
||||
assert cap["tool_selector"] is None
|
||||
assert cap["context_editing"] is main_model
|
||||
|
||||
def test_async_subagent_tool_selector_stays_main(self):
|
||||
cap, fake_ts, fake_ce = self._capture()
|
||||
main_model, aux_model = object(), object()
|
||||
main_model = object()
|
||||
with (
|
||||
patch.object(E, "_ensure_config", return_value=_mock_cfg()),
|
||||
patch.object(
|
||||
E,
|
||||
"_compile_time_role_model",
|
||||
side_effect=lambda role="primary": aux_model
|
||||
if role == "auxiliary"
|
||||
else main_model,
|
||||
E, "_compile_time_role_model", return_value=main_model
|
||||
),
|
||||
patch(
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
@@ -160,8 +82,6 @@ class TestAuxiliaryMiddlewareScope:
|
||||
assert cap["context_editing"] is main_model
|
||||
|
||||
def test_pure_path_tool_selector_uses_threaded_model(self):
|
||||
"""The pure path never resolves auxiliary free strings (Task 7 wires
|
||||
the local snapshot entry); the threaded model stands in."""
|
||||
cap, fake_ts, fake_ce = self._capture()
|
||||
cfg = _mock_cfg()
|
||||
main_model = object()
|
||||
@@ -169,7 +89,6 @@ class TestAuxiliaryMiddlewareScope:
|
||||
with (
|
||||
patch.object(E, "_ensure_config", side_effect=AssertionError),
|
||||
patch.object(E, "_ensure_chat_model", side_effect=AssertionError),
|
||||
patch.object(E, "_ensure_auxiliary_chat_model", side_effect=AssertionError),
|
||||
patch(
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
side_effect=fake_ts,
|
||||
@@ -190,22 +109,21 @@ class TestAuxiliaryMiddlewareScope:
|
||||
with (
|
||||
patch.object(E, "_ensure_config", return_value=cfg),
|
||||
patch.object(E, "_ensure_chat_model", return_value=main_model),
|
||||
patch.object(E, "_ensure_auxiliary_chat_model", return_value=main_model),
|
||||
patch(
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
return_value=[MagicMock()],
|
||||
),
|
||||
):
|
||||
mw = E._get_default_middleware(snapshot_role="auxiliary")
|
||||
mw = E._get_default_middleware(snapshot_role="primary")
|
||||
|
||||
configurable = next(
|
||||
m for m in mw if type(m).__name__ == "ConfigurableModelMiddleware"
|
||||
)
|
||||
assert configurable._role == "auxiliary"
|
||||
assert configurable._role == "primary"
|
||||
|
||||
|
||||
def test_memory_agent_factory_uses_auxiliary_role(monkeypatch):
|
||||
"""Memory workers bind the auxiliary role model at graph build."""
|
||||
def test_memory_agent_factory_binds_primary_placeholder(monkeypatch):
|
||||
"""Memory workers bind the primary placeholder plus snapshot middleware."""
|
||||
sentinel = object()
|
||||
monkeypatch.setattr(E, "_compile_time_role_model", lambda role="primary": sentinel)
|
||||
captured: dict[str, object] = {}
|
||||
@@ -228,3 +146,5 @@ def test_memory_agent_factory_uses_auxiliary_role(monkeypatch):
|
||||
backend=MagicMock(),
|
||||
)
|
||||
assert captured["model"] is sentinel
|
||||
middleware = captured["middleware"]
|
||||
assert type(middleware[0]).__name__ == "ConfigurableModelMiddleware"
|
||||
|
||||
@@ -39,7 +39,7 @@ def test_bootstrap_registry_aborts_startup(monkeypatch):
|
||||
legacy_artifacts, "assert_no_legacy_artifacts", MagicMock()
|
||||
)
|
||||
runtime = MagicMock()
|
||||
runtime.registry_defaults.side_effect = ModelRegistryError(
|
||||
runtime.registry_default.side_effect = ModelRegistryError(
|
||||
MODEL_REGISTRY_NOT_READY, "bootstrap"
|
||||
)
|
||||
monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime)
|
||||
@@ -53,7 +53,7 @@ def test_clean_environment_passes(monkeypatch):
|
||||
legacy_artifacts, "assert_no_legacy_artifacts", MagicMock()
|
||||
)
|
||||
runtime = MagicMock()
|
||||
runtime.registry_defaults.return_value = (MagicMock(), None, 2)
|
||||
runtime.registry_default.return_value = (MagicMock(), 2)
|
||||
monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime)
|
||||
_startup_gates() # must not raise
|
||||
|
||||
@@ -64,7 +64,7 @@ def test_non_bootstrap_registry_error_propagates(monkeypatch):
|
||||
legacy_artifacts, "assert_no_legacy_artifacts", MagicMock()
|
||||
)
|
||||
runtime = MagicMock()
|
||||
runtime.registry_defaults.side_effect = ModelRegistryError(
|
||||
runtime.registry_default.side_effect = ModelRegistryError(
|
||||
"SNAPSHOT_EXPIRED", "expired"
|
||||
)
|
||||
monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime)
|
||||
|
||||
@@ -349,32 +349,9 @@ class TestSnapshotDrivenConstruction:
|
||||
assert isinstance(model, ChatOpenAI)
|
||||
assert model.model_name == "glm-5.2"
|
||||
|
||||
def test_auxiliary_role_maps_to_snapshot_auxiliary(self, store, runtime):
|
||||
mw = ConfigurableModelMiddleware(role="auxiliary", runtime=runtime)
|
||||
snapshot = make_snapshot(store)
|
||||
req = _make_request()
|
||||
handler = MagicMock(return_value="ok")
|
||||
|
||||
with _patched_config(_configurable_for(snapshot)):
|
||||
mw.wrap_model_call(req, handler)
|
||||
|
||||
model = handler.call_args[0][0].model
|
||||
assert isinstance(model, ChatOllama)
|
||||
assert model.model == "qwen3"
|
||||
|
||||
def test_summary_role_falls_back_to_snapshot_primary(self, tmp_path):
|
||||
store = make_active_store(tmp_path / "db", auxiliary_default=False)
|
||||
mw = ConfigurableModelMiddleware(role="summary", runtime=SnapshotRuntime(store))
|
||||
snapshot = make_snapshot(store)
|
||||
req = _make_request()
|
||||
handler = MagicMock(return_value="ok")
|
||||
|
||||
with _patched_config(_configurable_for(snapshot)):
|
||||
mw.wrap_model_call(req, handler)
|
||||
|
||||
model = handler.call_args[0][0].model
|
||||
assert isinstance(model, ChatOpenAI)
|
||||
assert model.model_name == "glm-5.2"
|
||||
def test_unknown_role_rejected(self, runtime):
|
||||
with pytest.raises(ValueError, match="role"):
|
||||
ConfigurableModelMiddleware(role="auxiliary", runtime=runtime)
|
||||
|
||||
def test_deployment_id_from_configurable_is_verified(self, store, runtime):
|
||||
mw = ConfigurableModelMiddleware(runtime=runtime)
|
||||
|
||||
@@ -107,16 +107,13 @@ def test_create_middleware_model_none_fallback(mock_model):
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
return_value=[MagicMock(), MagicMock()],
|
||||
)
|
||||
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._compile_time_role_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_default_middleware_includes_context_editing(mock_config, mock_model, mock_aux, mock_ts):
|
||||
def test_default_middleware_includes_context_editing(mock_config, mock_model, mock_ts):
|
||||
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
cfg = MagicMock()
|
||||
cfg.enable_ask_user = False
|
||||
cfg.auto_approve = False
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
|
||||
from EvoScientist.EvoScientist import _get_default_middleware
|
||||
@@ -144,16 +141,13 @@ def test_inject_subagent_includes_context_editing(mock_model):
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
return_value=[MagicMock(), MagicMock()],
|
||||
)
|
||||
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._compile_time_role_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_context_editing_before_overflow_mapper(mock_config, mock_model, mock_aux, mock_ts):
|
||||
def test_context_editing_before_overflow_mapper(mock_config, mock_model, mock_ts):
|
||||
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
cfg = MagicMock()
|
||||
cfg.enable_ask_user = False
|
||||
cfg.auto_approve = False
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
|
||||
from EvoScientist.EvoScientist import _get_default_middleware
|
||||
|
||||
@@ -16,7 +16,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from langchain_ollama import ChatOllama
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
from EvoScientist.middleware.message_budget import (
|
||||
_snapshot_message_budget,
|
||||
@@ -30,16 +30,8 @@ from EvoScientist.model_registry.errors import (
|
||||
ModelRegistryError,
|
||||
)
|
||||
from EvoScientist.model_registry.runtime import SnapshotRuntime
|
||||
from EvoScientist.model_registry.schemas import CredentialWrite, RegistryV4
|
||||
from EvoScientist.model_registry.store import ModelRuntimeStore
|
||||
from tests.registry_fixtures import (
|
||||
ZHIPU_SECRET,
|
||||
_verify_model,
|
||||
make_active_store,
|
||||
make_snapshot,
|
||||
ollama_provider_payload,
|
||||
zhipu_provider_payload,
|
||||
)
|
||||
from tests.registry_fixtures import make_active_store, make_snapshot
|
||||
|
||||
|
||||
@contextmanager
|
||||
@@ -86,39 +78,6 @@ def _configurable_for(snapshot, **overrides):
|
||||
return configurable
|
||||
|
||||
|
||||
def _make_split_limit_store(config_dir):
|
||||
"""Active store whose auxiliary model has a smaller input limit.
|
||||
|
||||
primary (zhipu-glm/glm-5.2): context 1048576 → resolved_input_limit 1015808
|
||||
auxiliary (local-ollama/qwen3): context 65536 → resolved_input_limit 32768
|
||||
"""
|
||||
store = ModelRuntimeStore(config_dir=config_dir)
|
||||
payload = {
|
||||
"version": 4,
|
||||
"revision": 1,
|
||||
"state": "bootstrap",
|
||||
"defaults": {
|
||||
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
|
||||
"auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"},
|
||||
},
|
||||
"providers": [
|
||||
zhipu_provider_payload(),
|
||||
ollama_provider_payload(context_window_tokens=65536),
|
||||
],
|
||||
}
|
||||
registry = store.save_registry(
|
||||
expected_revision=1,
|
||||
registry=RegistryV4.model_validate(payload),
|
||||
credential_writes=[
|
||||
CredentialWrite(credential_id="zhipu-primary", secret_value=ZHIPU_SECRET)
|
||||
],
|
||||
)
|
||||
assert registry.state == "active"
|
||||
_verify_model(store, registry, "zhipu-glm", "glm-5.2")
|
||||
_verify_model(store, registry, "local-ollama", "qwen3")
|
||||
return store
|
||||
|
||||
|
||||
def test_text_counter_excludes_attachment_payloads_and_counts_tool_results():
|
||||
messages = [
|
||||
HumanMessage(
|
||||
@@ -168,25 +127,6 @@ class TestSnapshotMessageBudget:
|
||||
assert tools_only.hard_tokens == int((1015808 - 4096 - 8192) * 0.90)
|
||||
assert full.hard_tokens == int((1015808 - 4096 - 8192 - 4096) * 0.90)
|
||||
|
||||
def test_auxiliary_role_uses_auxiliary_frozen_limits(self, tmp_path):
|
||||
"""T6 review fix: the budget role is threaded through, not hardcoded
|
||||
to ``primary`` — a scheduler (auxiliary role) must size its budget
|
||||
from the auxiliary model's frozen configuration."""
|
||||
store = _make_split_limit_store(tmp_path / "model-runtime")
|
||||
snapshot = make_snapshot(store)
|
||||
|
||||
primary_budget = _snapshot_message_budget(
|
||||
snapshot, "primary", has_tools=False, has_attachments=False
|
||||
)
|
||||
auxiliary_budget = _snapshot_message_budget(
|
||||
snapshot, "auxiliary", has_tools=False, has_attachments=False
|
||||
)
|
||||
|
||||
assert primary_budget.input_limit == 1015808
|
||||
assert auxiliary_budget.input_limit == 32768
|
||||
assert auxiliary_budget.hard_tokens == int((32768 - 4096) * 0.90)
|
||||
|
||||
|
||||
class TestSnapshotModeMiddleware:
|
||||
def test_budget_uses_frozen_reserves_not_model_profile(self, store, runtime):
|
||||
"""A misleading compile-time profile must not affect snapshot mode."""
|
||||
@@ -254,26 +194,6 @@ class TestSnapshotModeMiddleware:
|
||||
assert budget.has_tools is False
|
||||
assert budget.hard_tokens == int((1015808 - 4096) * 0.90)
|
||||
|
||||
def test_auxiliary_snapshot_role_sizes_budget_from_auxiliary(self, tmp_path):
|
||||
"""T6 review fix: ``snapshot_role="auxiliary"`` (scheduler) sizes the
|
||||
budget from the snapshot's auxiliary frozen configuration."""
|
||||
store = _make_split_limit_store(tmp_path / "model-runtime")
|
||||
runtime = SnapshotRuntime(store)
|
||||
middleware = create_message_budget_middleware(
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
has_tools=False,
|
||||
snapshot_role="auxiliary",
|
||||
runtime=runtime,
|
||||
)
|
||||
snapshot = make_snapshot(store)
|
||||
|
||||
with _patched_config(_configurable_for(snapshot)):
|
||||
budget = middleware._budget_for_request(_request([HumanMessage("hi")]))
|
||||
|
||||
assert budget.input_limit == 32768
|
||||
assert budget.hard_tokens == int((32768 - 4096) * 0.90)
|
||||
|
||||
def test_unknown_snapshot_role_rejected(self, runtime):
|
||||
with pytest.raises(ValueError, match="Unknown model role"):
|
||||
create_message_budget_middleware(
|
||||
@@ -293,7 +213,7 @@ class TestSnapshotModeMiddleware:
|
||||
middleware._budget_for_request(_request([HumanMessage("hi")]))
|
||||
assert excinfo.value.code == SNAPSHOT_NOT_FOUND
|
||||
|
||||
def test_summarizer_model_uses_summary_role(self, store, runtime):
|
||||
def test_summarizer_model_uses_snapshot_primary(self, store, runtime):
|
||||
middleware = create_message_budget_middleware(
|
||||
MagicMock(), MagicMock(), runtime=runtime
|
||||
)
|
||||
@@ -302,9 +222,9 @@ class TestSnapshotModeMiddleware:
|
||||
with _patched_config(_configurable_for(snapshot)):
|
||||
model = middleware.model
|
||||
|
||||
# summary → snapshot.auxiliary ?? snapshot.primary (section 6.1)
|
||||
assert isinstance(model, ChatOllama)
|
||||
assert model.model == "qwen3"
|
||||
# Every role maps to the snapshot's frozen primary (section 6.1).
|
||||
assert isinstance(model, ChatOpenAI)
|
||||
assert model.model_name == "glm-5.2"
|
||||
|
||||
def test_summarizer_model_cached_per_snapshot(self, store, runtime):
|
||||
middleware = create_message_budget_middleware(
|
||||
|
||||
@@ -152,7 +152,6 @@ def _registry_payload():
|
||||
"state": "bootstrap",
|
||||
"defaults": {
|
||||
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
|
||||
"auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"},
|
||||
},
|
||||
"providers": [_zhipu_provider(), _ollama_provider()],
|
||||
}
|
||||
@@ -476,7 +475,7 @@ def test_put_registry_rejects_defaults_to_disabled_model(active_client):
|
||||
# the store's rule-7 guard (enforced on every save) is what rejects it.
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
registry["state"] = "bootstrap"
|
||||
registry["providers"][1]["models"][0]["enabled"] = False
|
||||
registry["providers"][0]["models"][0]["enabled"] = False
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
@@ -486,7 +485,7 @@ def test_put_registry_rejects_defaults_to_disabled_model(active_client):
|
||||
def test_put_registry_active_state_validates_defaults_in_schema(active_client):
|
||||
# The same rule fires earlier (Pydantic) when the payload claims active.
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
registry["providers"][1]["models"][0]["enabled"] = False
|
||||
registry["providers"][0]["models"][0]["enabled"] = False
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
@@ -530,7 +529,7 @@ def test_put_registry_bootstrap_flow_configure_test_enable(client, store):
|
||||
# Step 1: save a draft registry — models disabled, defaults null, and the
|
||||
# credential written in the same transaction (section 10 steps 5-7).
|
||||
draft = _registry_payload()
|
||||
draft["defaults"] = {"primary": None, "auxiliary": None}
|
||||
draft["defaults"] = {"primary": None}
|
||||
for provider in draft["providers"]:
|
||||
for model in provider["models"]:
|
||||
model["enabled"] = False
|
||||
@@ -568,7 +567,6 @@ def test_put_registry_bootstrap_flow_configure_test_enable(client, store):
|
||||
enabled = body["registry"]
|
||||
enabled["defaults"] = {
|
||||
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
|
||||
"auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"},
|
||||
}
|
||||
for provider in enabled["providers"]:
|
||||
for model in provider["models"]:
|
||||
@@ -651,11 +649,16 @@ def test_get_models_returns_only_selectable(active_client, store):
|
||||
"vision": False,
|
||||
"structured_output": True,
|
||||
}
|
||||
# The selector echoes the registry defaults so clients can label the
|
||||
# `inherit` choice before the first turn.
|
||||
assert response.json()["defaults"]["primary"] == {
|
||||
"provider_id": "zhipu-glm",
|
||||
"model_key": "glm-5.2",
|
||||
}
|
||||
|
||||
# Disable qwen3 (and drop the auxiliary default): it leaves the selector.
|
||||
# Disable qwen3: it leaves the selector.
|
||||
registry = store.load_registry()
|
||||
payload = registry.model_dump(mode="json")
|
||||
payload["defaults"]["auxiliary"] = None
|
||||
payload["providers"][1]["models"][0]["enabled"] = False
|
||||
store.save_registry(
|
||||
expected_revision=registry.revision,
|
||||
@@ -682,7 +685,6 @@ def _snapshot_body(**overrides):
|
||||
"deployment_id": "webui-1",
|
||||
"model_selection_revision": 4,
|
||||
"primary": None,
|
||||
"auxiliary": None,
|
||||
}
|
||||
body.update(overrides)
|
||||
return body
|
||||
@@ -699,7 +701,6 @@ def test_snapshot_create_201_and_idempotent_200(active_client):
|
||||
assert body["primary"]["provider_id"] == "zhipu-glm"
|
||||
assert body["primary"]["adapter_spec_revision"] == 1
|
||||
assert body["primary"]["runtime"]["max_output_tokens"] == 32768
|
||||
assert body["auxiliary"]["model_key"] == "qwen3"
|
||||
assert SECRET not in first.text
|
||||
|
||||
replay = active_client.post(
|
||||
|
||||
@@ -136,7 +136,6 @@ def _registry(**overrides):
|
||||
"state": "bootstrap",
|
||||
"defaults": {
|
||||
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
|
||||
"auxiliary": None,
|
||||
},
|
||||
"providers": [_provider()],
|
||||
}
|
||||
@@ -153,7 +152,6 @@ class TestRegistryV4:
|
||||
assert registry.defaults.primary == ModelRef(
|
||||
provider_id="zhipu-glm", model_key="glm-5.2"
|
||||
)
|
||||
assert registry.defaults.auxiliary is None
|
||||
provider = registry.providers[0]
|
||||
assert provider.runtime.timeout_seconds == 120
|
||||
model = provider.models[0]
|
||||
@@ -172,7 +170,7 @@ class TestRegistryV4:
|
||||
registry = RegistryV4.model_validate(
|
||||
_registry(
|
||||
state="bootstrap",
|
||||
defaults={"primary": None, "auxiliary": None},
|
||||
defaults={"primary": None},
|
||||
providers=[],
|
||||
)
|
||||
)
|
||||
@@ -184,7 +182,7 @@ class TestRegistryV4:
|
||||
RegistryV4.model_validate(
|
||||
_registry(
|
||||
state="active",
|
||||
defaults={"primary": None, "auxiliary": None},
|
||||
defaults={"primary": None},
|
||||
)
|
||||
)
|
||||
|
||||
@@ -205,7 +203,6 @@ class TestRegistryV4:
|
||||
_registry(
|
||||
defaults={
|
||||
"primary": {"provider_id": "zhipu-glm", "model_key": "missing"},
|
||||
"auxiliary": None,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
@@ -51,7 +51,6 @@ def _registry_payload(**overrides):
|
||||
"state": "bootstrap",
|
||||
"defaults": {
|
||||
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
|
||||
"auxiliary": None,
|
||||
},
|
||||
"providers": [
|
||||
{
|
||||
@@ -106,7 +105,6 @@ class TestBootstrap:
|
||||
assert registry.state == "bootstrap"
|
||||
assert registry.providers == []
|
||||
assert registry.defaults.primary is None
|
||||
assert registry.defaults.auxiliary is None
|
||||
|
||||
def test_database_path_and_permissions(self, tmp_path):
|
||||
store = ModelRuntimeStore(config_dir=tmp_path)
|
||||
@@ -149,7 +147,7 @@ class TestRegistrySave:
|
||||
def test_bootstrap_save_without_ready_primary_stays_bootstrap(self, tmp_path):
|
||||
store = ModelRuntimeStore(config_dir=tmp_path)
|
||||
registry = _registry(
|
||||
defaults={"primary": None, "auxiliary": None},
|
||||
defaults={"primary": None},
|
||||
)
|
||||
saved = store.save_registry(expected_revision=1, registry=registry)
|
||||
assert saved.state == "bootstrap"
|
||||
@@ -242,7 +240,7 @@ class TestRegistrySave:
|
||||
],
|
||||
}
|
||||
)
|
||||
payload["defaults"]["auxiliary"] = {
|
||||
payload["defaults"]["primary"] = {
|
||||
"provider_id": "local-ollama",
|
||||
"model_key": "qwen3",
|
||||
}
|
||||
|
||||
@@ -219,7 +219,7 @@ def _registry_payload(port, **provider_overrides):
|
||||
"version": 4,
|
||||
"revision": 1,
|
||||
"state": "bootstrap",
|
||||
"defaults": {"primary": None, "auxiliary": None},
|
||||
"defaults": {"primary": None},
|
||||
"providers": [_provider_payload(port, **provider_overrides)],
|
||||
}
|
||||
|
||||
|
||||
@@ -119,7 +119,7 @@ def _registry(providers=None, **overrides) -> RegistryV4:
|
||||
"version": 4,
|
||||
"revision": 1,
|
||||
"state": "bootstrap",
|
||||
"defaults": {"primary": None, "auxiliary": None},
|
||||
"defaults": {"primary": None},
|
||||
"providers": [_zhipu_provider()] if providers is None else providers,
|
||||
}
|
||||
payload.update(overrides)
|
||||
@@ -219,9 +219,8 @@ class TestResolve:
|
||||
config = resolver.resolve(ZHIPU_REF, "primary")
|
||||
assert SECRET not in config.model_dump_json()
|
||||
|
||||
def test_all_roles_are_stamped(self, resolver):
|
||||
for role in ("primary", "auxiliary", "summary", "tool_selector"):
|
||||
assert resolver.resolve(ZHIPU_REF, role).role == role
|
||||
def test_primary_role_is_stamped(self, resolver):
|
||||
assert resolver.resolve(ZHIPU_REF, "primary").role == "primary"
|
||||
|
||||
def test_unknown_role_rejected(self, resolver):
|
||||
with pytest.raises(ValueError, match="role"):
|
||||
@@ -351,7 +350,7 @@ class TestResolve:
|
||||
registry = _activate(store, _registry(providers=[_ollama_provider()]))
|
||||
_verify(store, registry, "local-ollama", "qwen3")
|
||||
resolver = ModelRegistryResolver(store)
|
||||
config = resolver.resolve(OLLAMA_REF, "auxiliary")
|
||||
config = resolver.resolve(OLLAMA_REF, "primary")
|
||||
assert config.auth_ref.mode == "none"
|
||||
assert config.auth_ref.credential_id is None
|
||||
assert config.auth_ref.credential_revision is None
|
||||
|
||||
@@ -40,8 +40,6 @@ def _mock_config():
|
||||
cfg.auto_mode = False
|
||||
cfg.auto_approve = False
|
||||
cfg.model_fallbacks = None
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
cfg.code_interpreter_timeout = 60
|
||||
cfg.code_interpreter_max_result_chars = 6000
|
||||
return cfg
|
||||
@@ -65,11 +63,10 @@ def test_runtime_context_injects_current_date_and_timezone():
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
return_value=[MagicMock(), MagicMock()],
|
||||
)
|
||||
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._compile_time_role_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_default_middleware_includes_runtime_context(
|
||||
mock_config, mock_model, mock_aux_model, mock_tool_selector
|
||||
mock_config, mock_model, mock_tool_selector
|
||||
):
|
||||
mock_config.return_value = _mock_config()
|
||||
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
@@ -85,7 +82,7 @@ def test_default_middleware_includes_runtime_context(
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
return_value=[MagicMock(), MagicMock()],
|
||||
)
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._compile_time_role_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_async_subagent_middleware_includes_runtime_context(
|
||||
mock_config, mock_model, mock_tool_selector
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
"""Tests for ``EvoScientist.model_registry.runtime`` (design doc 8.3).
|
||||
|
||||
Covers the runtime-link glue: registry-default role resolution (auxiliary ??
|
||||
primary), snapshot role construction with per-call credential resolution,
|
||||
safe-client wiring (ollama carries retries on the transports), the shared
|
||||
default-runtime accessor, and the tolerant local platform field reader.
|
||||
Covers the runtime-link glue: registry-default primary resolution, snapshot
|
||||
role construction with per-call credential resolution, safe-client wiring
|
||||
(ollama carries retries on the transports), the shared default-runtime
|
||||
accessor, and the tolerant local platform field reader.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -59,33 +59,19 @@ class TestRegistryDefaults:
|
||||
def test_bootstrap_registry_is_not_ready(self, tmp_path):
|
||||
runtime = SnapshotRuntime(ModelRuntimeStore(config_dir=tmp_path / "db"))
|
||||
with pytest.raises(ModelRegistryError) as excinfo:
|
||||
runtime.registry_defaults()
|
||||
runtime.registry_default()
|
||||
assert excinfo.value.code == MODEL_REGISTRY_NOT_READY
|
||||
assert excinfo.value.http_status == 422
|
||||
|
||||
def test_active_registry_returns_defaults(self, runtime):
|
||||
primary, auxiliary, revision = runtime.registry_defaults()
|
||||
def test_active_registry_returns_default(self, runtime):
|
||||
primary, revision = runtime.registry_default()
|
||||
assert primary == ZHIPU_REF
|
||||
assert auxiliary == OLLAMA_REF
|
||||
assert revision == 2
|
||||
|
||||
|
||||
class TestResolveRoleConfig:
|
||||
def test_primary_role_resolves_primary_default(self, runtime):
|
||||
config = runtime.resolve_role_config("primary")
|
||||
assert config.model_ref == ZHIPU_REF
|
||||
assert config.role == "primary"
|
||||
|
||||
def test_auxiliary_role_resolves_auxiliary_default(self, runtime):
|
||||
for role in ("auxiliary", "summary", "tool_selector"):
|
||||
config = runtime.resolve_role_config(role)
|
||||
assert config.model_ref == OLLAMA_REF, role
|
||||
assert config.role == "auxiliary"
|
||||
|
||||
def test_auxiliary_falls_back_to_primary(self, tmp_path):
|
||||
store = make_active_store(tmp_path / "db", auxiliary_default=False)
|
||||
runtime = SnapshotRuntime(store)
|
||||
config = runtime.resolve_role_config("auxiliary")
|
||||
class TestResolveDefaultConfig:
|
||||
def test_resolves_primary_default(self, runtime):
|
||||
config = runtime.resolve_default_config()
|
||||
assert config.model_ref == ZHIPU_REF
|
||||
assert config.role == "primary"
|
||||
|
||||
@@ -100,11 +86,6 @@ class TestBuildDefaultRoleModel:
|
||||
assert model.temperature == 0.7
|
||||
assert model.top_p == 0.95
|
||||
|
||||
def test_auxiliary_builds_ollama_without_credential(self, runtime):
|
||||
model = runtime.build_default_role_model("auxiliary")
|
||||
assert isinstance(model, ChatOllama)
|
||||
assert model.model == "qwen3"
|
||||
|
||||
def test_openai_compatible_never_reads_env_api_key(self, runtime, monkeypatch):
|
||||
"""The credential comes from the frozen auth_ref, never OPENAI_API_KEY."""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-env-must-not-leak")
|
||||
@@ -120,20 +101,6 @@ class TestBuildRoleModelFromSnapshot:
|
||||
assert model.model_name == "glm-5.2"
|
||||
assert model.openai_api_key.get_secret_value() == ZHIPU_SECRET
|
||||
|
||||
def test_auxiliary_roles_use_snapshot_auxiliary(self, store, runtime):
|
||||
snapshot = make_snapshot(store)
|
||||
for role in ("auxiliary", "summary", "tool_selector"):
|
||||
model = runtime.build_role_model(snapshot, role)
|
||||
assert isinstance(model, ChatOllama), role
|
||||
assert model.model == "qwen3"
|
||||
|
||||
def test_auxiliary_roles_fall_back_to_snapshot_primary(self, tmp_path):
|
||||
store = make_active_store(tmp_path / "db", auxiliary_default=False)
|
||||
runtime = SnapshotRuntime(store)
|
||||
snapshot = make_snapshot(store)
|
||||
model = runtime.build_role_model(snapshot, "summary")
|
||||
assert isinstance(model, ChatOpenAI)
|
||||
assert model.model_name == "glm-5.2"
|
||||
|
||||
|
||||
class TestSharedDefaultRuntime:
|
||||
@@ -213,7 +180,10 @@ class TestEndpointPolicyWiring:
|
||||
)
|
||||
)
|
||||
runtime = SnapshotRuntime(store, endpoint_policy=policy)
|
||||
model = runtime.build_default_role_model("auxiliary")
|
||||
# The loopback ollama provider builds only because its base URL is a
|
||||
# registered development endpoint.
|
||||
config = runtime.resolver.resolve(OLLAMA_REF, "primary")
|
||||
model = runtime._build(config, "")
|
||||
assert isinstance(model, ChatOllama)
|
||||
|
||||
|
||||
@@ -233,9 +203,8 @@ class TestCreateLocalSnapshot:
|
||||
assert snapshot.thread_id == "cli-thread-1"
|
||||
assert snapshot.deployment_id == runtime.local_deployment_id
|
||||
assert snapshot.payload.model_selection_revision == 0
|
||||
# primary=None inherits — the registry defaults are frozen at creation.
|
||||
# primary=None inherits — the registry default is frozen at creation.
|
||||
assert config_for_role(snapshot, "primary").model_ref == ZHIPU_REF
|
||||
assert config_for_role(snapshot, "auxiliary").model_ref == OLLAMA_REF
|
||||
|
||||
def test_same_run_request_id_reuses_snapshot(self, store, runtime):
|
||||
first = runtime.create_local_snapshot("t-1", run_request_id="req-1")
|
||||
|
||||
+70
-54
@@ -22,6 +22,7 @@ from EvoScientist.model_registry.errors import (
|
||||
SNAPSHOT_ALREADY_BOUND,
|
||||
SNAPSHOT_EXPIRED,
|
||||
SNAPSHOT_NOT_FOUND,
|
||||
UNSUPPORTED_RUNTIME_PARAMETER,
|
||||
ModelRegistryError,
|
||||
)
|
||||
from EvoScientist.model_registry.hashing import configuration_hash
|
||||
@@ -120,7 +121,7 @@ def _ollama_provider():
|
||||
}
|
||||
|
||||
|
||||
def _registry(*, auxiliary_default=True) -> RegistryV4:
|
||||
def _registry() -> RegistryV4:
|
||||
return RegistryV4.model_validate(
|
||||
{
|
||||
"version": 4,
|
||||
@@ -128,11 +129,6 @@ def _registry(*, auxiliary_default=True) -> RegistryV4:
|
||||
"state": "bootstrap",
|
||||
"defaults": {
|
||||
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
|
||||
"auxiliary": (
|
||||
{"provider_id": "local-ollama", "model_key": "qwen3"}
|
||||
if auxiliary_default
|
||||
else None
|
||||
),
|
||||
},
|
||||
"providers": [_zhipu_provider(), _ollama_provider()],
|
||||
}
|
||||
@@ -190,16 +186,16 @@ def _request(**overrides):
|
||||
"deployment_id": "local",
|
||||
"model_selection_revision": 0,
|
||||
"primary": None,
|
||||
"auxiliary": None,
|
||||
}
|
||||
payload.update(overrides)
|
||||
return SnapshotCreateRequest.model_validate(payload)
|
||||
|
||||
|
||||
# SHA-256 of the canonical JSON '{"auxiliary":null,"primary":null}' — the
|
||||
# inherit/inherit selection. Pinned as a literal to lock the wire contract.
|
||||
# SHA-256 of the canonical JSON '{"primary":null,"reasoning_effort":null}' —
|
||||
# the inherit selection with no effort override. Pinned as a literal to lock
|
||||
# the wire contract.
|
||||
INHERIT_SELECTION_HASH = (
|
||||
"697c046214ccc3ddee1018af7eb6c21dbd5bfd01fbca9cb594ffc273deb55abc"
|
||||
"03e47e7f41396b231bd558734715697a5aebfa114ccfd861a0f7e824e32dac8d"
|
||||
)
|
||||
|
||||
|
||||
@@ -215,27 +211,69 @@ class TestCreate:
|
||||
assert payload.model_selection_revision == 4
|
||||
assert payload.primary.model_ref == ZHIPU_REF
|
||||
assert payload.primary.role == "primary"
|
||||
assert payload.auxiliary is not None
|
||||
assert payload.auxiliary.model_ref == OLLAMA_REF
|
||||
assert payload.auxiliary.role == "auxiliary"
|
||||
|
||||
def test_inherit_without_auxiliary_default(self, active_store):
|
||||
def test_explicit_selection_freezes_primary(self, service):
|
||||
creation = service.create(_request(primary=OLLAMA_REF))
|
||||
payload = creation.snapshot.payload
|
||||
assert payload.primary.model_ref == OLLAMA_REF
|
||||
|
||||
def test_reasoning_effort_override_freezes_into_primary(self, active_store):
|
||||
# glm-5.2's model-specific contract rejects reasoning overrides, so
|
||||
# add a generic-contract model on the same provider to exercise the
|
||||
# override path (disabled → verify → enable, the standard flow).
|
||||
store = active_store
|
||||
registry = store.load_registry()
|
||||
store.save_registry(
|
||||
payload = registry.model_dump(mode="json")
|
||||
payload["providers"][0]["models"].append(
|
||||
{
|
||||
"key": "glm-air",
|
||||
"name": "GLM Air",
|
||||
"upstream_model_id": "glm-air",
|
||||
"enabled": False,
|
||||
"runtime": _model_runtime(),
|
||||
}
|
||||
)
|
||||
registry = store.save_registry(
|
||||
expected_revision=registry.revision,
|
||||
registry=_registry(auxiliary_default=False),
|
||||
registry=RegistryV4.model_validate(payload),
|
||||
)
|
||||
_verify(store, registry, "zhipu-glm", "glm-air")
|
||||
payload = registry.model_dump(mode="json")
|
||||
payload["providers"][0]["models"][1]["enabled"] = True
|
||||
registry = store.save_registry(
|
||||
expected_revision=registry.revision,
|
||||
registry=RegistryV4.model_validate(payload),
|
||||
)
|
||||
service = SnapshotService(store, ModelRegistryResolver(store))
|
||||
creation = service.create(_request())
|
||||
assert creation.snapshot.payload.auxiliary is None
|
||||
creation = service.create(
|
||||
_request(
|
||||
primary={"provider_id": "zhipu-glm", "model_key": "glm-air"},
|
||||
reasoning_effort="high",
|
||||
)
|
||||
)
|
||||
result = creation.snapshot.payload
|
||||
assert result.primary.request_options.reasoning_effort == "high"
|
||||
|
||||
def test_explicit_selection_freezes_both_roles(self, service):
|
||||
creation = service.create(_request(primary=ZHIPU_REF, auxiliary=OLLAMA_REF))
|
||||
payload = creation.snapshot.payload
|
||||
assert payload.primary.model_ref == ZHIPU_REF
|
||||
assert payload.auxiliary is not None
|
||||
assert payload.auxiliary.model_ref == OLLAMA_REF
|
||||
def test_reasoning_effort_override_rejected_by_model_contract(self, service):
|
||||
# glm-5.2's model-specific contract does not accept reasoning_effort.
|
||||
with pytest.raises(ModelRegistryError) as excinfo:
|
||||
service.create(_request(reasoning_effort="high"))
|
||||
assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER
|
||||
|
||||
def test_reasoning_effort_override_changes_selection_hash(self, service):
|
||||
plain = service.create(_request())
|
||||
assert compute_selection_hash(None, "low") != plain.snapshot.selection_hash
|
||||
assert compute_selection_hash(None, "low") != INHERIT_SELECTION_HASH
|
||||
# The hash covers the raw field, so an explicit ``auto`` differs from
|
||||
# no override; both resolve to the same frozen configuration.
|
||||
assert compute_selection_hash(None, "auto") != INHERIT_SELECTION_HASH
|
||||
|
||||
def test_reasoning_effort_override_rejected_for_unsupported_adapter(
|
||||
self, service
|
||||
):
|
||||
with pytest.raises(ModelRegistryError) as excinfo:
|
||||
service.create(_request(primary=OLLAMA_REF, reasoning_effort="high"))
|
||||
assert excinfo.value.code == UNSUPPORTED_RUNTIME_PARAMETER
|
||||
|
||||
def test_bootstrap_registry_is_not_ready(self, store):
|
||||
service = SnapshotService(store, ModelRegistryResolver(store))
|
||||
@@ -247,13 +285,13 @@ class TestCreate:
|
||||
def test_selection_hash_uses_pre_resolution_semantics(self, service):
|
||||
creation = service.create(_request())
|
||||
snapshot = creation.snapshot
|
||||
# The inherit/inherit hash is pinned to a known literal (computed
|
||||
# offline from the canonical JSON) so the contract can't drift
|
||||
# together with a reimplemented expectation.
|
||||
# The inherit hash is pinned to a known literal (computed offline
|
||||
# from the canonical JSON) so the contract can't drift together with
|
||||
# a reimplemented expectation.
|
||||
assert snapshot.selection_hash == INHERIT_SELECTION_HASH
|
||||
assert compute_selection_hash(None, None) == INHERIT_SELECTION_HASH
|
||||
# An explicit selection equal to the defaults still hashes differently.
|
||||
assert compute_selection_hash(ZHIPU_REF, OLLAMA_REF) != INHERIT_SELECTION_HASH
|
||||
assert compute_selection_hash(None) == INHERIT_SELECTION_HASH
|
||||
# An explicit selection equal to the default still hashes differently.
|
||||
assert compute_selection_hash(ZHIPU_REF) != INHERIT_SELECTION_HASH
|
||||
|
||||
def test_selection_hash_ignores_selection_revision(self, service):
|
||||
first = service.create(_request(model_selection_revision=4))
|
||||
@@ -524,25 +562,9 @@ class TestCleanupExpired:
|
||||
|
||||
|
||||
class TestRoleMapping:
|
||||
def test_auxiliary_roles_fall_back_to_primary(self, active_store):
|
||||
store = active_store
|
||||
registry = store.load_registry()
|
||||
store.save_registry(
|
||||
expected_revision=registry.revision,
|
||||
registry=_registry(auxiliary_default=False),
|
||||
)
|
||||
service = SnapshotService(store, ModelRegistryResolver(store))
|
||||
snapshot = service.create(_request()).snapshot
|
||||
for role in ("primary", "auxiliary", "summary", "tool_selector"):
|
||||
assert config_for_role(snapshot, role).model_ref == ZHIPU_REF
|
||||
|
||||
def test_auxiliary_roles_use_frozen_auxiliary(self, service):
|
||||
def test_primary_role_maps_to_frozen_primary(self, service):
|
||||
snapshot = service.create(_request()).snapshot
|
||||
assert config_for_role(snapshot, "primary").model_ref == ZHIPU_REF
|
||||
for role in ("auxiliary", "summary", "tool_selector"):
|
||||
config = config_for_role(snapshot, role)
|
||||
assert config.model_ref == OLLAMA_REF
|
||||
assert config.role == "auxiliary"
|
||||
|
||||
def test_unknown_role_rejected(self, service):
|
||||
snapshot = service.create(_request()).snapshot
|
||||
@@ -574,12 +596,6 @@ class TestSnapshotCredentials:
|
||||
fresh_service = SnapshotService(fresh_store, ModelRegistryResolver(fresh_store))
|
||||
assert fresh_service.resolve_snapshot_credential(snapshot, "primary") == SECRET
|
||||
|
||||
def test_auxiliary_role_resolves_auxiliary_credential(self, service):
|
||||
snapshot = service.create(_request()).snapshot
|
||||
# The frozen auxiliary is the mode=none ollama model: no secret.
|
||||
assert service.resolve_snapshot_credential(snapshot, "summary") == ""
|
||||
assert service.resolve_snapshot_credential(snapshot, "tool_selector") == ""
|
||||
|
||||
def test_mode_none_returns_empty_string(self, service):
|
||||
snapshot = service.create(_request(primary=OLLAMA_REF)).snapshot
|
||||
assert service.resolve_snapshot_credential(snapshot, "primary") == ""
|
||||
@@ -599,10 +615,10 @@ class TestPublicView:
|
||||
"max_output_tokens": 32768,
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"reasoning_effort": "auto",
|
||||
"timeout_seconds": 120,
|
||||
"max_retries": 2,
|
||||
}
|
||||
assert view["auxiliary"]["provider_id"] == "local-ollama"
|
||||
|
||||
def test_never_contains_secret_or_base_url(self, service):
|
||||
snapshot = service.create(_request()).snapshot
|
||||
|
||||
@@ -55,7 +55,10 @@ def _factory_patches():
|
||||
return_value=MagicMock(),
|
||||
create=True,
|
||||
),
|
||||
patch("EvoScientist.EvoScientist._ensure_chat_model", return_value=MagicMock()),
|
||||
patch(
|
||||
"EvoScientist.middleware.configurable_model.resolve_snapshot_model",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch(
|
||||
"langchain.agents.middleware.LLMToolSelectorMiddleware",
|
||||
return_value=MagicMock(),
|
||||
@@ -252,8 +255,10 @@ def test_selector_always_includes_available_memory_tools():
|
||||
]
|
||||
|
||||
|
||||
def test_selector_resolved_once_across_repeated_requests():
|
||||
"""Agent tools are stable, so build the selector once and reuse it."""
|
||||
def test_selector_rebuilt_per_request_with_stable_always_include():
|
||||
"""The selector is rebuilt per call (its helper model is resolved from
|
||||
the run snapshot, which varies per run), but the always-include names
|
||||
derived from the request's tools stay stable."""
|
||||
mock_selector = MagicMock()
|
||||
mock_selector.wrap_model_call.side_effect = lambda request, handler: handler(
|
||||
request
|
||||
@@ -274,7 +279,9 @@ def test_selector_resolved_once_across_repeated_requests():
|
||||
for _ in range(3):
|
||||
cond.wrap_model_call(_request(tools), MagicMock())
|
||||
|
||||
selector_factory.assert_called_once_with(["search_observations", "think_tool"])
|
||||
assert selector_factory.call_count == 3
|
||||
for call in selector_factory.call_args_list:
|
||||
assert call.args[0] == ["search_observations", "think_tool"]
|
||||
assert mock_selector.wrap_model_call.call_count == 3
|
||||
|
||||
|
||||
@@ -305,16 +312,13 @@ def test_tracker_captures_tools():
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
side_effect=lambda *a, **kw: _patched_create(),
|
||||
)
|
||||
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._compile_time_role_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_default_middleware_includes_tool_selector(mock_config, mock_model, mock_aux, mock_ts):
|
||||
def test_default_middleware_includes_tool_selector(mock_config, mock_model, mock_ts):
|
||||
mock_model.return_value = _mock_model()
|
||||
cfg = MagicMock()
|
||||
cfg.enable_ask_user = False
|
||||
cfg.auto_approve = False
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
|
||||
from EvoScientist.EvoScientist import _get_default_middleware
|
||||
@@ -342,17 +346,14 @@ def test_subagent_no_tool_selector(mock_model):
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
side_effect=lambda *a, **kw: _patched_create(),
|
||||
)
|
||||
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._compile_time_role_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_tool_selector_ordering(mock_config, mock_model, mock_aux, mock_ts):
|
||||
def test_tool_selector_ordering(mock_config, mock_model, mock_ts):
|
||||
"""ToolSelector should come after ToolErrorHandler and before Memory."""
|
||||
mock_model.return_value = _mock_model()
|
||||
cfg = MagicMock()
|
||||
cfg.enable_ask_user = False
|
||||
cfg.auto_approve = False
|
||||
cfg.auxiliary_model = ""
|
||||
cfg.auxiliary_provider = ""
|
||||
mock_config.return_value = cfg
|
||||
|
||||
from EvoScientist.EvoScientist import _get_default_middleware
|
||||
|
||||
Reference in New Issue
Block a user