Fix/async subagent model switch (#217)
* feat(middleware): add ConfigurableModelMiddleware for dynamic model resolution - Introduced ConfigurableModelMiddleware to resolve chat models from RunnableConfig.configurable on each call. - Updated middleware initialization to include ConfigurableModelMiddleware. - Enhanced context editing middleware tests to verify presence of ConfigurableModelMiddleware. - Implemented tests for ConfigurableModelMiddleware to ensure correct model overriding and caching behavior. - Added tests for deepagents model-passthrough patch to verify configuration injection in async tasks. * feat(async-subagent): update middleware handling to prevent deadlocks in async sub-agents * style: Refactor code formatting for improved readability in patches and test files * refactor: streamline middleware construction and improve async handling in ConfigurableModelMiddleware * fix: remove unused request parameter from _read_model_override function * refactor: improve async handling in _ClientProxy and enhance logging in ConfigurableModelMiddleware test: add behavior test to ensure AskUserMiddleware is excluded in async subagent mode
This commit is contained in:
@@ -308,6 +308,15 @@ def _maybe_swap_async_subagents(subs: list, middleware: list | None = None) -> l
|
||||
|
||||
middleware.append(AsyncWatcherMiddleware(agent_specs))
|
||||
|
||||
# Forward the CLI's live (model, provider) into deepagents'
|
||||
# start/update_async_task tool calls so the deployed graph can
|
||||
# re-resolve its chat model per run via ConfigurableModelMiddleware.
|
||||
# Idempotent — safe to call on every CLI startup.
|
||||
if agent_specs:
|
||||
from .llm.patches import _patch_deepagents_model_passthrough
|
||||
|
||||
_patch_deepagents_model_passthrough()
|
||||
|
||||
return out
|
||||
|
||||
|
||||
@@ -439,9 +448,23 @@ def _get_default_backend():
|
||||
)
|
||||
|
||||
|
||||
def _get_default_middleware():
|
||||
"""Build the default middleware list."""
|
||||
def _get_default_middleware(*, for_async_subagent: bool = False):
|
||||
"""Build the default middleware list.
|
||||
|
||||
Args:
|
||||
for_async_subagent: When True, omit middleware that would deadlock a
|
||||
deployed async sub-agent. Specifically: ``AskUserMiddleware`` uses
|
||||
``interrupt()`` to pause the graph waiting for a user reply, but
|
||||
async sub-agents run in the ``langgraph dev`` subprocess where
|
||||
the parent only holds a ``task_id`` and has no UI path to surface
|
||||
(or resume) an interrupt — the sub-agent would hang forever the
|
||||
first time it called ``ask_user``. This mirrors the same reason
|
||||
``subagents/_factory.py`` deliberately skips ``interrupt_on=`` on
|
||||
the deepagents level. Defaults to False (full middleware list)
|
||||
for the CLI's in-process agent.
|
||||
"""
|
||||
from .middleware import (
|
||||
ConfigurableModelMiddleware,
|
||||
ContextOverflowMapperMiddleware,
|
||||
ModelFallbackMiddleware,
|
||||
ToolErrorHandlerMiddleware,
|
||||
@@ -456,7 +479,12 @@ def _get_default_middleware():
|
||||
load_fallback_chain(cfg.model_fallbacks)
|
||||
model = _ensure_chat_model()
|
||||
memory_dir = str(_paths_mod.MEMORIES_DIR)
|
||||
# ``ConfigurableModelMiddleware`` is placed first so it wraps
|
||||
# ``ModelFallbackMiddleware``: a configurable.model override sets the
|
||||
# PRIMARY model only, leaving the fallback chain free to try its own
|
||||
# alternatives instead of re-overriding every retry to the same model.
|
||||
mw = [
|
||||
ConfigurableModelMiddleware(),
|
||||
create_context_editing_middleware(model),
|
||||
ModelFallbackMiddleware(),
|
||||
ContextOverflowMapperMiddleware(),
|
||||
@@ -465,7 +493,7 @@ def _get_default_middleware():
|
||||
create_memory_middleware(memory_dir, extraction_model=model),
|
||||
]
|
||||
|
||||
if cfg.enable_ask_user and not cfg.auto_mode:
|
||||
if cfg.enable_ask_user and not cfg.auto_mode and not for_async_subagent:
|
||||
from .middleware.ask_user import AskUserMiddleware
|
||||
|
||||
mw.insert(0, AskUserMiddleware())
|
||||
@@ -557,19 +585,8 @@ def create_cli_agent(
|
||||
|
||||
from . import paths as _paths
|
||||
from .backends import CustomSandboxBackend, MergedSkillsBackend
|
||||
from .middleware import (
|
||||
ContextOverflowMapperMiddleware,
|
||||
ModelFallbackMiddleware,
|
||||
ToolErrorHandlerMiddleware,
|
||||
create_context_editing_middleware,
|
||||
create_memory_middleware,
|
||||
create_tool_selector_middleware,
|
||||
load_fallback_chain,
|
||||
)
|
||||
|
||||
cfg = _ensure_config(config)
|
||||
if cfg.model_fallbacks:
|
||||
load_fallback_chain(cfg.model_fallbacks)
|
||||
|
||||
if checkpointer is None:
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
@@ -617,19 +634,10 @@ def create_cli_agent(
|
||||
},
|
||||
)
|
||||
|
||||
model = _ensure_chat_model()
|
||||
mw: list[AgentMiddleware] = [
|
||||
create_context_editing_middleware(model),
|
||||
ModelFallbackMiddleware(),
|
||||
ContextOverflowMapperMiddleware(),
|
||||
ToolErrorHandlerMiddleware(),
|
||||
*create_tool_selector_middleware(model=model),
|
||||
create_memory_middleware(_mem_dir, extraction_model=model),
|
||||
]
|
||||
if cfg.enable_ask_user and not cfg.auto_mode:
|
||||
from .middleware.ask_user import AskUserMiddleware
|
||||
|
||||
mw.insert(0, AskUserMiddleware())
|
||||
# Delegate middleware construction to the single source of truth so the
|
||||
# CLI agent never drifts from the default chain. Anything CLI-specific
|
||||
# (e.g. ``HumanInTheLoopMiddleware``) is appended below.
|
||||
mw: list[AgentMiddleware] = _get_default_middleware()
|
||||
|
||||
# HITL on main agent only — passing `interrupt_on=` to create_deep_agent
|
||||
# would propagate it to every subagent, breaking parallel execute calls
|
||||
|
||||
@@ -522,3 +522,186 @@ def _patch_deepseek_reasoning_passback(model: Any) -> None:
|
||||
return payload
|
||||
|
||||
model._get_request_payload = _patched
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Patch: forward CLI's live (model, model_provider) into deepagents'
|
||||
# start_async_task / update_async_task tool calls so the deployed graph
|
||||
# (running in a separate ``langgraph dev`` subprocess) re-resolves the
|
||||
# chat model per run.
|
||||
#
|
||||
# Without this, async sub-agents stay on the model their graph was compiled
|
||||
# with at langgraph dev boot — `/model` switches in the CLI never reach
|
||||
# them because they live in another process.
|
||||
#
|
||||
# Mechanism: wrap deepagents' ``_build_start_tool`` and ``_build_update_tool``
|
||||
# factories. Each wrapped factory calls the original with a proxied client
|
||||
# cache that intercepts ``runs.create(...)`` calls only and injects
|
||||
# ``config={"configurable": {"model": <cfg.model>, "model_provider": <cfg.provider>}}``.
|
||||
# All other client methods (``threads.create``, ``runs.get``, ``runs.cancel``,
|
||||
# ``runs.join_stream``) pass through unchanged. The deployed graph picks up
|
||||
# ``configurable.model`` via ``ConfigurableModelMiddleware``.
|
||||
#
|
||||
# Reads ``_ensure_config()`` at tool-call time (not patch time) so a
|
||||
# ``/model`` switch in the CLI is reflected on the very next async tool
|
||||
# call without an agent rebuild.
|
||||
#
|
||||
# Upstream PR opportunity: passing ``config`` through ``client.runs.create``
|
||||
# is generic functionality; worth contributing back to ``langchain-ai/deepagents``
|
||||
# so this patch can be retired.
|
||||
# ---------------------------------------------------------------------------
|
||||
_model_passthrough_patched = False
|
||||
|
||||
|
||||
def _read_cfg_configurable() -> dict[str, str]:
|
||||
"""Read live ``(model, provider)`` from EvoScientist config.
|
||||
|
||||
Returns a dict suitable for inserting under
|
||||
``RunnableConfig.configurable``. Empty dict on any failure (so the
|
||||
patch degrades to a no-op rather than breaking async tool calls).
|
||||
"""
|
||||
try:
|
||||
from EvoScientist.EvoScientist import _ensure_config
|
||||
|
||||
cfg = _ensure_config()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
out: dict[str, str] = {}
|
||||
model = getattr(cfg, "model", None)
|
||||
provider = getattr(cfg, "provider", None)
|
||||
if isinstance(model, str) and model:
|
||||
out["model"] = model
|
||||
if isinstance(provider, str) and provider:
|
||||
out["model_provider"] = provider
|
||||
return out
|
||||
|
||||
|
||||
def _merge_runs_config_kwargs(kwargs: dict) -> dict:
|
||||
"""Merge the live model override into ``kwargs`` for ``runs.create``.
|
||||
|
||||
Preserves any caller-supplied ``config.configurable`` keys. EvoScientist's
|
||||
keys take precedence on conflict (callers shouldn't be passing model
|
||||
overrides — the CLI is the source of truth).
|
||||
"""
|
||||
overrides = _read_cfg_configurable()
|
||||
if not overrides:
|
||||
return kwargs
|
||||
existing = kwargs.get("config")
|
||||
if not isinstance(existing, dict):
|
||||
existing = {}
|
||||
existing_configurable = existing.get("configurable")
|
||||
if not isinstance(existing_configurable, dict):
|
||||
existing_configurable = {}
|
||||
merged_configurable = {**existing_configurable, **overrides}
|
||||
kwargs = dict(kwargs)
|
||||
kwargs["config"] = {**existing, "configurable": merged_configurable}
|
||||
return kwargs
|
||||
|
||||
|
||||
class _SyncRunsProxy:
|
||||
"""Wraps a sync ``RunsClient`` and injects config into ``create`` only."""
|
||||
|
||||
def __init__(self, real: Any) -> None:
|
||||
object.__setattr__(self, "_real", real)
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self._real, name)
|
||||
|
||||
def create(self, **kwargs: Any) -> Any:
|
||||
return self._real.create(**_merge_runs_config_kwargs(kwargs))
|
||||
|
||||
|
||||
class _AsyncRunsProxy:
|
||||
"""Wraps an async ``RunsClient`` and injects config into ``create`` only."""
|
||||
|
||||
def __init__(self, real: Any) -> None:
|
||||
object.__setattr__(self, "_real", real)
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self._real, name)
|
||||
|
||||
async def create(self, **kwargs: Any) -> Any:
|
||||
return await self._real.create(**_merge_runs_config_kwargs(kwargs))
|
||||
|
||||
|
||||
class _ClientProxy:
|
||||
"""Lightweight proxy that swaps ``client.runs`` for a runs proxy.
|
||||
|
||||
Read-only forwarding: only ``__getattr__`` is overridden. Attribute
|
||||
*writes* on the proxy land on the proxy itself, NOT on the wrapped real
|
||||
client. Current deepagents only reads ``.threads`` / ``.runs``, so this
|
||||
is safe — but if a future caller tries ``client.foo = bar`` through the
|
||||
proxy, the write will be silently lost. ``object.__setattr__`` in
|
||||
``__init__`` is used solely to avoid infinite recursion when seeding
|
||||
the internal slots.
|
||||
|
||||
``client.runs`` is a stable attribute set in
|
||||
``langgraph_sdk.client.LangGraphClient.__init__`` (``self.runs =
|
||||
RunsClient(...)``) — not a property or lazy initializer — so the wrapped
|
||||
runs proxy is built once at ``__init__`` time and reused on every
|
||||
``proxy.runs`` access. This avoids the previous behavior of selecting
|
||||
sync/async and instantiating a new ``_RunsProxy`` per attribute access.
|
||||
"""
|
||||
|
||||
def __init__(self, real: Any, *, is_async: bool) -> None:
|
||||
runs_proxy_cls = _AsyncRunsProxy if is_async else _SyncRunsProxy
|
||||
object.__setattr__(self, "_real", real)
|
||||
object.__setattr__(self, "_runs_proxy", runs_proxy_cls(real.runs))
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
if name == "runs":
|
||||
return self._runs_proxy
|
||||
return getattr(self._real, name)
|
||||
|
||||
|
||||
class _ClientCacheProxy:
|
||||
"""Proxy a ``_ClientCache`` so callers receive config-injecting clients."""
|
||||
|
||||
def __init__(self, real: Any) -> None:
|
||||
self._real = real
|
||||
|
||||
def get_sync(self, name: str) -> Any:
|
||||
return _ClientProxy(self._real.get_sync(name), is_async=False)
|
||||
|
||||
def get_async(self, name: str) -> Any:
|
||||
return _ClientProxy(self._real.get_async(name), is_async=True)
|
||||
|
||||
|
||||
def _patch_deepagents_model_passthrough() -> None:
|
||||
"""Wrap deepagents' async-launch tool factories to inject CLI model.
|
||||
|
||||
Idempotent: re-invocation is a no-op once the patch is active. Safe to
|
||||
call from ``_maybe_swap_async_subagents`` on every CLI startup; both
|
||||
that hook and this patch turn on together when async sub-agents are
|
||||
enabled.
|
||||
"""
|
||||
global _model_passthrough_patched
|
||||
if _model_passthrough_patched:
|
||||
return
|
||||
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as ds_mod
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
# Defensive ``getattr`` lookups mirror the rest of this file (lines 254,
|
||||
# 266, 279, 290, 339, 351, 362, 373, 473): a deepagents update that
|
||||
# renames or removes either private helper degrades to a no-op instead
|
||||
# of raising ``AttributeError`` at CLI startup.
|
||||
orig_build_start = getattr(ds_mod, "_build_start_tool", None)
|
||||
orig_build_update = getattr(ds_mod, "_build_update_tool", None)
|
||||
if orig_build_start is None or orig_build_update is None:
|
||||
return
|
||||
|
||||
def _patched_build_start(
|
||||
agent_map: Any, clients: Any, tool_description: str
|
||||
) -> Any:
|
||||
return orig_build_start(agent_map, _ClientCacheProxy(clients), tool_description)
|
||||
|
||||
def _patched_build_update(agent_map: Any, clients: Any) -> Any:
|
||||
return orig_build_update(agent_map, _ClientCacheProxy(clients))
|
||||
|
||||
ds_mod._build_start_tool = _patched_build_start
|
||||
ds_mod._build_update_tool = _patched_build_update
|
||||
_model_passthrough_patched = True
|
||||
|
||||
@@ -11,6 +11,7 @@ from .ask_user import (
|
||||
Choice,
|
||||
Question,
|
||||
)
|
||||
from .configurable_model import ConfigurableModelMiddleware
|
||||
from .context_editing import (
|
||||
compute_context_editing_trigger,
|
||||
create_context_editing_middleware,
|
||||
@@ -32,6 +33,7 @@ __all__ = [
|
||||
"AskUserRequest",
|
||||
"AskUserWidgetResult",
|
||||
"Choice",
|
||||
"ConfigurableModelMiddleware",
|
||||
"ContextOverflowMapperMiddleware",
|
||||
"EvoMemoryMiddleware",
|
||||
"EvoMemoryState",
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Middleware that resolves the chat model from RunnableConfig.configurable per call.
|
||||
|
||||
The deployed async sub-agents run in a separate ``langgraph dev`` subprocess
|
||||
and have their model frozen into the compiled graph at subprocess boot time
|
||||
(see ``EvoScientist/subagents/_factory.py``). When the user runs ``/model``
|
||||
in the CLI, only the CLI process's model state changes — the subprocess
|
||||
graph still uses the boot-time model.
|
||||
|
||||
This middleware closes that gap by reading ``model`` / ``model_provider``
|
||||
from ``RunnableConfig.configurable`` on every model call. The CLI's patched
|
||||
``start_async_task`` / ``update_async_task`` (see ``llm/patches.py``) injects
|
||||
those fields into ``client.runs.create(config=...)``; the deployed graph
|
||||
hits this middleware and re-resolves the chat model fresh.
|
||||
|
||||
When ``configurable.model`` is absent, the middleware is a pass-through —
|
||||
safe to install on the CLI's in-process agent too.
|
||||
|
||||
The middleware mirrors the pattern used by ``ModelFallbackMiddleware``:
|
||||
``request.override(model=new_model)`` does not break tool binding, because
|
||||
the downstream model-invocation node re-binds tools per request.
|
||||
|
||||
**Reading the config**: ``Runtime`` (per its own docstring) does NOT include
|
||||
``config``. The official path to reach ``RunnableConfig`` from inside any
|
||||
runnable context (including middleware) is ``langgraph.config.get_config()``,
|
||||
which reads a context-var that LangGraph populates per node execution. That
|
||||
is what this middleware uses. ``request.runtime`` is intentionally NOT
|
||||
relied upon for config — only for diagnostics.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
from langchain.agents.middleware.types import (
|
||||
AgentMiddleware,
|
||||
ModelRequest,
|
||||
ModelResponse,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _read_model_override() -> tuple[str | None, str | None]:
|
||||
"""Pull ``(model, model_provider)`` from the active ``RunnableConfig``.
|
||||
|
||||
Reads via ``langgraph.config.get_config()`` (the documented entry point
|
||||
for accessing the per-run ``RunnableConfig`` from inside any runnable
|
||||
context — middleware, node, tool). Returns ``(None, None)`` when the
|
||||
config has no ``configurable.model`` override or when called outside a
|
||||
runnable context.
|
||||
"""
|
||||
try:
|
||||
from langgraph.config import get_config
|
||||
|
||||
cfg = get_config()
|
||||
except Exception:
|
||||
# Outside a runnable context (most common in tests) or
|
||||
# langgraph not importable — nothing to override.
|
||||
return None, None
|
||||
if not isinstance(cfg, dict):
|
||||
return None, None
|
||||
configurable = cfg.get("configurable") or {}
|
||||
if not isinstance(configurable, dict):
|
||||
return None, None
|
||||
model = configurable.get("model")
|
||||
provider = configurable.get("model_provider")
|
||||
return (
|
||||
model if isinstance(model, str) and model else None,
|
||||
provider if isinstance(provider, str) and provider else None,
|
||||
)
|
||||
|
||||
|
||||
class ConfigurableModelMiddleware(AgentMiddleware):
|
||||
"""Re-resolve the chat model from RunnableConfig.configurable on every call.
|
||||
|
||||
Reads ``model`` and ``model_provider`` from the active ``RunnableConfig``
|
||||
via ``langgraph.config.get_config()`` — the documented entry point for
|
||||
accessing per-run config from any runnable context (middleware, node, tool).
|
||||
When the override is present, calls
|
||||
``EvoScientist.llm.get_chat_model(model=..., provider=...)`` and replaces
|
||||
``request.model`` via ``request.override``. When absent, the middleware
|
||||
passes through unchanged.
|
||||
|
||||
Note: ``Runtime`` (per its own docstring) does NOT include ``config`` as a
|
||||
field — an earlier version of this middleware tried to read
|
||||
``request.runtime.config`` and silently no-op'd because that attribute does
|
||||
not exist. Stick with ``get_config()``.
|
||||
|
||||
A per-instance cache keyed by ``(model, provider)`` avoids rebuilding
|
||||
identical models within a turn. The cache is a plain dict guarded by a
|
||||
``threading.Lock`` because middleware instances are shared across
|
||||
concurrent requests in long-lived deployments (e.g. ``langgraph dev``
|
||||
workers).
|
||||
"""
|
||||
|
||||
name = "configurable_model"
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self._cache: dict[tuple[str, str | None], Any] = {}
|
||||
self._lock = threading.Lock()
|
||||
# Track the last (model, provider) pair we INFO-logged so we only
|
||||
# surface a banner on transition. Without this, every LLM call in a
|
||||
# long async run would emit an identical INFO line.
|
||||
self._last_logged_key: tuple[str, str | None] | None = None
|
||||
|
||||
def _log_override(self, model_name: str, provider: str | None) -> None:
|
||||
"""INFO on transition; DEBUG on subsequent calls with same key."""
|
||||
key = (model_name, provider)
|
||||
with self._lock:
|
||||
transitioned = key != self._last_logged_key
|
||||
if transitioned:
|
||||
self._last_logged_key = key
|
||||
if transitioned:
|
||||
logger.info(
|
||||
"ConfigurableModelMiddleware: overriding model to %s (%s)",
|
||||
model_name,
|
||||
provider,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"ConfigurableModelMiddleware: reusing override model=%s provider=%s",
|
||||
model_name,
|
||||
provider,
|
||||
)
|
||||
|
||||
def _resolve(self, model: str, provider: str | None) -> Any:
|
||||
"""Return a cached or freshly-built chat model for ``(model, provider)``."""
|
||||
key = (model, provider)
|
||||
with self._lock:
|
||||
cached = self._cache.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
# Build outside the lock (network/SDK init can be slow); two
|
||||
# concurrent first-time misses for the same key may build twice but
|
||||
# the second result simply overwrites the first — both are equivalent.
|
||||
from ..llm import get_chat_model
|
||||
|
||||
new_model = get_chat_model(model=model, provider=provider)
|
||||
with self._lock:
|
||||
self._cache[key] = new_model
|
||||
return new_model
|
||||
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelResponse:
|
||||
model_name, provider = _read_model_override()
|
||||
if model_name is None:
|
||||
return handler(request)
|
||||
try:
|
||||
new_model = self._resolve(model_name, provider)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"ConfigurableModelMiddleware failed to resolve model=%r "
|
||||
"provider=%r; falling back to compile-time model",
|
||||
model_name,
|
||||
provider,
|
||||
exc_info=True,
|
||||
)
|
||||
return handler(request)
|
||||
self._log_override(model_name, provider)
|
||||
return handler(request.override(model=new_model))
|
||||
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelResponse:
|
||||
model_name, provider = _read_model_override()
|
||||
if model_name is None:
|
||||
return await handler(request)
|
||||
try:
|
||||
# Offload first-call SDK init off the event loop. ``_resolve`` calls
|
||||
# ``get_chat_model`` on a cache miss, which can spend hundreds of ms
|
||||
# building HTTP clients. Doing this synchronously inside an
|
||||
# ``async def`` would block every other coroutine on the same
|
||||
# langgraph dev event loop. Cache hits are still fast (a dict
|
||||
# lookup); the thread-pool overhead is irrelevant once warm.
|
||||
new_model = await asyncio.to_thread(self._resolve, model_name, provider)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"ConfigurableModelMiddleware failed to resolve model=%r "
|
||||
"provider=%r; falling back to compile-time model",
|
||||
model_name,
|
||||
provider,
|
||||
exc_info=True,
|
||||
)
|
||||
return await handler(request)
|
||||
self._log_override(model_name, provider)
|
||||
return await handler(request.override(model=new_model))
|
||||
@@ -90,6 +90,12 @@ def build_async_subagent_graph(name: str) -> Any:
|
||||
# approve. The user-visible HITL boundary is the parent's
|
||||
# ``start_async_task`` decision; restrict the child's reach by limiting
|
||||
# ``tools`` in ``subagents/<name>.yaml`` instead.
|
||||
#
|
||||
# ``for_async_subagent=True`` propagates the same reasoning to the
|
||||
# middleware list — specifically, it suppresses ``AskUserMiddleware``,
|
||||
# which uses ``interrupt()`` for the same purpose (waiting on a user
|
||||
# reply) and would deadlock an async sub-agent for the same reason.
|
||||
#
|
||||
# Memory middleware is included so async sub-agents can READ
|
||||
# /memory/MEMORY.md, but the extraction trigger (20+ human messages,
|
||||
# see middleware/memory.py) never fires here — sub-agents only receive
|
||||
@@ -103,5 +109,5 @@ def build_async_subagent_graph(name: str) -> Any:
|
||||
tools=spec.get("tools", []) + agent_mcp_tools,
|
||||
skills=spec.get("skills"),
|
||||
backend=_get_default_backend(),
|
||||
middleware=_get_default_middleware(),
|
||||
middleware=_get_default_middleware(for_async_subagent=True),
|
||||
).with_config({"recursion_limit": cfg.recursion_limit})
|
||||
|
||||
@@ -91,3 +91,49 @@ def tmp_workspace(tmp_path):
|
||||
ws = tmp_path / "workspace"
|
||||
ws.mkdir()
|
||||
return str(ws)
|
||||
|
||||
|
||||
# Capture deepagents tool factories at conftest load time — BEFORE any test
|
||||
# imports EvoScientist, which can trigger ``_patch_deepagents_model_passthrough``
|
||||
# during agent construction. Once captured here, the ``restore_model_passthrough_patch``
|
||||
# fixture has a stable "truly unpatched" baseline to reset to between tests, even
|
||||
# if upstream code paths apply the patch as a side effect.
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as _ds_async_subagents
|
||||
|
||||
_DEEPAGENTS_ORIGINAL_BUILD_START = _ds_async_subagents._build_start_tool
|
||||
_DEEPAGENTS_ORIGINAL_BUILD_UPDATE = _ds_async_subagents._build_update_tool
|
||||
except Exception:
|
||||
_ds_async_subagents = None
|
||||
_DEEPAGENTS_ORIGINAL_BUILD_START = None
|
||||
_DEEPAGENTS_ORIGINAL_BUILD_UPDATE = None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def restore_model_passthrough_patch():
|
||||
"""Reset deepagents internals + ``_model_passthrough_patched`` to unpatched.
|
||||
|
||||
The model-passthrough patch wraps ``deepagents.middleware.async_subagents``
|
||||
module-level functions in place. The originals are captured at conftest
|
||||
load time (above) so this fixture can always start each test from a
|
||||
known-unpatched state regardless of what other tests / agent fixtures
|
||||
did to the module before.
|
||||
"""
|
||||
from EvoScientist.llm import patches as patches_mod
|
||||
|
||||
if _ds_async_subagents is None:
|
||||
# deepagents not importable — fixture is a no-op (the patch fn itself
|
||||
# returns early in that case).
|
||||
yield
|
||||
return
|
||||
|
||||
def _reset() -> None:
|
||||
_ds_async_subagents._build_start_tool = _DEEPAGENTS_ORIGINAL_BUILD_START
|
||||
_ds_async_subagents._build_update_tool = _DEEPAGENTS_ORIGINAL_BUILD_UPDATE
|
||||
patches_mod._model_passthrough_patched = False
|
||||
|
||||
_reset()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_reset()
|
||||
|
||||
@@ -439,6 +439,45 @@ def test_auto_mode_disables_ask_user_middleware(
|
||||
assert "AskUserMiddleware" not in type_names
|
||||
|
||||
|
||||
@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[])
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_for_async_subagent_omits_ask_user_middleware(
|
||||
mock_config, mock_model, mock_tool_selector
|
||||
):
|
||||
"""``AskUserMiddleware`` uses ``interrupt()`` to wait on user input.
|
||||
|
||||
Async sub-agents run in the langgraph dev subprocess where the parent
|
||||
only holds a ``task_id`` and has no UI path to surface or resume an
|
||||
interrupt. Including ``AskUserMiddleware`` would deadlock the
|
||||
sub-agent the first time the LLM calls ``ask_user``. The
|
||||
``for_async_subagent=True`` flag must suppress it even when the user
|
||||
has globally enabled ``ask_user``.
|
||||
"""
|
||||
cfg = MagicMock()
|
||||
cfg.enable_ask_user = True
|
||||
cfg.auto_approve = False
|
||||
cfg.auto_mode = False
|
||||
mock_config.return_value = cfg
|
||||
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
|
||||
from EvoScientist.EvoScientist import _get_default_middleware
|
||||
|
||||
# Sanity: with the default flag, ask_user IS present.
|
||||
default_names = [type(m).__name__ for m in _get_default_middleware()]
|
||||
assert "AskUserMiddleware" in default_names
|
||||
|
||||
# With for_async_subagent=True, ask_user is suppressed.
|
||||
async_names = [
|
||||
type(m).__name__ for m in _get_default_middleware(for_async_subagent=True)
|
||||
]
|
||||
assert "AskUserMiddleware" not in async_names
|
||||
# Other middleware must remain — only ask_user is filtered.
|
||||
assert "ConfigurableModelMiddleware" in async_names
|
||||
assert "ContextEditingMiddleware" in async_names
|
||||
assert "ModelFallbackMiddleware" in async_names
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rich CLI prompt (mocking input)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Tests for ``EvoScientist.subagents._factory.build_async_subagent_graph``.
|
||||
|
||||
Pins the integration contract that the factory must request middleware
|
||||
in async-safe mode (``for_async_subagent=True``). Without this, a future
|
||||
refactor that drops the keyword argument would silently re-introduce
|
||||
``AskUserMiddleware`` into the deployed graph and reproduce the
|
||||
``interrupt()``-based deadlock the flag was added to prevent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
@patch("deepagents.create_deep_agent")
|
||||
@patch("EvoScientist.EvoScientist._load_mcp_tools_cached", return_value={})
|
||||
@patch("EvoScientist.EvoScientist._get_default_middleware", return_value=[])
|
||||
@patch("EvoScientist.EvoScientist._get_default_backend")
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._build_prompt_refs", return_value={})
|
||||
@patch("EvoScientist.utils.load_subagents")
|
||||
@patch("EvoScientist.config.apply_config_to_env")
|
||||
@patch("EvoScientist.config.get_effective_config")
|
||||
def test_factory_requests_async_safe_middleware(
|
||||
mock_get_cfg,
|
||||
mock_apply_env,
|
||||
mock_load_subs,
|
||||
mock_prompt_refs,
|
||||
mock_chat,
|
||||
mock_backend,
|
||||
mock_get_mw,
|
||||
mock_mcp,
|
||||
mock_create,
|
||||
):
|
||||
"""``build_async_subagent_graph`` must call ``_get_default_middleware``
|
||||
with ``for_async_subagent=True``.
|
||||
|
||||
The bare argument call would silently include ``AskUserMiddleware`` in
|
||||
the deployed graph, which deadlocks via ``interrupt()`` (no UI in the
|
||||
langgraph dev subprocess to resume the interrupt).
|
||||
"""
|
||||
# Minimal config stub so factory's `cfg.recursion_limit` access works.
|
||||
cfg = MagicMock()
|
||||
cfg.recursion_limit = 1_000_000
|
||||
mock_get_cfg.return_value = cfg
|
||||
# Factory looks up the requested name in the loaded subagent specs;
|
||||
# any matching name is fine.
|
||||
mock_load_subs.return_value = [
|
||||
{
|
||||
"name": "writing-agent",
|
||||
"system_prompt": "",
|
||||
"tools": [],
|
||||
"skills": None,
|
||||
}
|
||||
]
|
||||
# ``create_deep_agent(...).with_config({...})`` chain — return something
|
||||
# chainable so the factory's terminal ``.with_config(...)`` doesn't blow up.
|
||||
mock_create.return_value.with_config.return_value = MagicMock()
|
||||
|
||||
from EvoScientist.subagents._factory import build_async_subagent_graph
|
||||
|
||||
build_async_subagent_graph("writing-agent")
|
||||
|
||||
# The contract: factory MUST pass ``for_async_subagent=True``.
|
||||
mock_get_mw.assert_called_once_with(for_async_subagent=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Direct behavior test for ``_get_default_middleware`` filter
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The factory test above pins the *contract* (factory passes the flag).
|
||||
# This test pins the *behavior* (the flag actually excludes
|
||||
# AskUserMiddleware), so a future refactor that renames the flag or
|
||||
# restructures the middleware list cannot silently re-introduce the
|
||||
# interrupt-based deadlock.
|
||||
|
||||
|
||||
@patch(
|
||||
"EvoScientist.middleware.create_tool_selector_middleware",
|
||||
return_value=[MagicMock()],
|
||||
)
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
@patch("EvoScientist.EvoScientist._ensure_config")
|
||||
def test_async_subagent_mode_filters_ask_user(
|
||||
mock_config, mock_chat, mock_tool_selector
|
||||
):
|
||||
"""``_get_default_middleware(for_async_subagent=True)`` must drop
|
||||
``AskUserMiddleware`` even when ``enable_ask_user`` is on.
|
||||
|
||||
Without mocking the middleware list itself: we let the real list be
|
||||
constructed and assert ``AskUserMiddleware`` is absent. Mocks here
|
||||
cover only the heavy dependencies (chat model, tool-selector) that
|
||||
the middleware list builder pulls in transitively.
|
||||
"""
|
||||
cfg = MagicMock()
|
||||
cfg.enable_ask_user = True # would normally include AskUserMiddleware
|
||||
cfg.auto_mode = False
|
||||
cfg.auto_approve = False
|
||||
cfg.model_fallbacks = None
|
||||
mock_config.return_value = cfg
|
||||
mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000})
|
||||
|
||||
from EvoScientist.EvoScientist import _get_default_middleware
|
||||
from EvoScientist.middleware.ask_user import AskUserMiddleware
|
||||
|
||||
# CLI / in-process path includes AskUserMiddleware …
|
||||
cli_mw = _get_default_middleware()
|
||||
assert any(isinstance(m, AskUserMiddleware) for m in cli_mw), (
|
||||
"Sanity check: with enable_ask_user=True and CLI mode, "
|
||||
"AskUserMiddleware should be present."
|
||||
)
|
||||
|
||||
# … but the async-subagent path filters it out.
|
||||
async_mw = _get_default_middleware(for_async_subagent=True)
|
||||
assert not any(isinstance(m, AskUserMiddleware) for m in async_mw), (
|
||||
"AskUserMiddleware leaked into async sub-agent middleware — its "
|
||||
"interrupt() call deadlocks the deployed graph (no UI to resume)."
|
||||
)
|
||||
@@ -0,0 +1,394 @@
|
||||
"""Tests for ``EvoScientist.middleware.configurable_model``.
|
||||
|
||||
Verifies that the middleware reads ``model`` / ``model_provider`` from
|
||||
the active ``RunnableConfig.configurable`` (via ``langgraph.config.get_config``)
|
||||
and overrides ``request.model`` accordingly, without breaking the no-override
|
||||
pass-through path or the per-instance cache.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from EvoScientist.middleware.configurable_model import (
|
||||
ConfigurableModelMiddleware,
|
||||
_read_model_override,
|
||||
)
|
||||
from tests.conftest import run_async as _run
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patched_config(configurable: dict | object | None):
|
||||
"""Patch ``langgraph.config.get_config`` to return a controlled value.
|
||||
|
||||
Pass ``None`` to simulate "outside a runnable context" (raises
|
||||
``RuntimeError`` like the real ``get_config()`` does).
|
||||
Pass a dict for ``configurable`` to expose just that key.
|
||||
Pass any other object to simulate a malformed config.
|
||||
"""
|
||||
import langgraph.config as _lg_cfg
|
||||
|
||||
if configurable is None:
|
||||
# Simulate get_config raising outside a runnable context.
|
||||
with patch.object(
|
||||
_lg_cfg,
|
||||
"get_config",
|
||||
side_effect=RuntimeError("Called get_config outside of a runnable context"),
|
||||
):
|
||||
yield
|
||||
elif isinstance(configurable, dict):
|
||||
with patch.object(
|
||||
_lg_cfg,
|
||||
"get_config",
|
||||
return_value={"configurable": configurable},
|
||||
):
|
||||
yield
|
||||
else:
|
||||
with patch.object(
|
||||
_lg_cfg,
|
||||
"get_config",
|
||||
return_value=configurable,
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def _make_request():
|
||||
"""Build a minimal ``ModelRequest`` stub.
|
||||
|
||||
``request.override(model=...)`` returns a new request whose ``model``
|
||||
field reflects the override.
|
||||
"""
|
||||
req = MagicMock()
|
||||
|
||||
def _override(**kwargs):
|
||||
new = MagicMock()
|
||||
new.model = kwargs.get("model", req.model)
|
||||
return new
|
||||
|
||||
req.override = MagicMock(side_effect=_override)
|
||||
return req
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 1. _read_model_override — input parsing
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestReadModelOverride:
|
||||
"""Verify the helper that pulls (model, provider) from active config."""
|
||||
|
||||
def test_returns_override_when_both_present(self):
|
||||
with _patched_config({"model": "gpt-5", "model_provider": "openai"}):
|
||||
assert _read_model_override() == ("gpt-5", "openai")
|
||||
|
||||
def test_provider_optional(self):
|
||||
with _patched_config({"model": "claude-haiku-4-5"}):
|
||||
assert _read_model_override() == ("claude-haiku-4-5", None)
|
||||
|
||||
def test_no_configurable_key(self):
|
||||
with _patched_config({}):
|
||||
assert _read_model_override() == (None, None)
|
||||
|
||||
def test_outside_runnable_context(self):
|
||||
"""``get_config`` raises outside a runnable — middleware must no-op."""
|
||||
with _patched_config(None):
|
||||
assert _read_model_override() == (None, None)
|
||||
|
||||
def test_empty_string_treated_as_absent(self):
|
||||
with _patched_config({"model": "", "model_provider": ""}):
|
||||
assert _read_model_override() == (None, None)
|
||||
|
||||
def test_non_string_ignored(self):
|
||||
with _patched_config({"model": 42, "model_provider": object()}):
|
||||
assert _read_model_override() == (None, None)
|
||||
|
||||
def test_non_dict_configurable_safe(self):
|
||||
with _patched_config({"configurable": "not-a-dict"}):
|
||||
# Inner ``configurable`` is the wrong type — patched_config
|
||||
# wraps it again so we end up with {"configurable": {"configurable": "..."}}
|
||||
# which has no model/model_provider keys → no override.
|
||||
assert _read_model_override() == (None, None)
|
||||
|
||||
def test_non_dict_config_safe(self):
|
||||
with _patched_config("garbage"):
|
||||
assert _read_model_override() == (None, None)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 2. ConfigurableModelMiddleware — pass-through behavior
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestPassThrough:
|
||||
"""When no override is present, the middleware must not touch the request."""
|
||||
|
||||
def test_sync_no_override_passes_request_unchanged(self):
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
sentinel = object()
|
||||
handler = MagicMock(return_value=sentinel)
|
||||
with _patched_config({}):
|
||||
result = mw.wrap_model_call(req, handler)
|
||||
assert result is sentinel
|
||||
handler.assert_called_once_with(req)
|
||||
req.override.assert_not_called()
|
||||
|
||||
def test_async_no_override_passes_request_unchanged(self):
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
|
||||
async def handler(r):
|
||||
assert r is req
|
||||
return "ok"
|
||||
|
||||
with _patched_config({}):
|
||||
result = _run(mw.awrap_model_call(req, handler))
|
||||
assert result == "ok"
|
||||
req.override.assert_not_called()
|
||||
|
||||
def test_outside_runnable_context_passes_through(self):
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
handler = MagicMock(return_value="ok")
|
||||
with _patched_config(None):
|
||||
mw.wrap_model_call(req, handler)
|
||||
handler.assert_called_once_with(req)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 3. ConfigurableModelMiddleware — override behavior
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestModelOverride:
|
||||
"""When override present, middleware resolves model and overrides request."""
|
||||
|
||||
def test_sync_override_calls_get_chat_model_and_overrides(self):
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
new_model = MagicMock(name="resolved_chat_model")
|
||||
handler = MagicMock(return_value="response")
|
||||
|
||||
with (
|
||||
_patched_config({"model": "gpt-5", "model_provider": "openai"}),
|
||||
patch(
|
||||
"EvoScientist.llm.get_chat_model", return_value=new_model
|
||||
) as mock_get,
|
||||
):
|
||||
mw.wrap_model_call(req, handler)
|
||||
|
||||
mock_get.assert_called_once_with(model="gpt-5", provider="openai")
|
||||
req.override.assert_called_once_with(model=new_model)
|
||||
# Handler must receive the OVERRIDDEN request, not the original.
|
||||
called_with = handler.call_args[0][0]
|
||||
assert called_with is not req
|
||||
assert called_with.model is new_model
|
||||
|
||||
def test_async_override_path_parity(self):
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
new_model = MagicMock()
|
||||
|
||||
async def handler(r):
|
||||
assert r.model is new_model
|
||||
return "ok"
|
||||
|
||||
with (
|
||||
_patched_config(
|
||||
{"model": "claude-opus-4-7", "model_provider": "anthropic"}
|
||||
),
|
||||
patch(
|
||||
"EvoScientist.llm.get_chat_model", return_value=new_model
|
||||
) as mock_get,
|
||||
):
|
||||
result = _run(mw.awrap_model_call(req, handler))
|
||||
|
||||
assert result == "ok"
|
||||
mock_get.assert_called_once_with(model="claude-opus-4-7", provider="anthropic")
|
||||
|
||||
def test_provider_omitted_passed_as_none(self):
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
new_model = MagicMock()
|
||||
handler = MagicMock(return_value="ok")
|
||||
|
||||
with (
|
||||
_patched_config({"model": "gpt-5"}),
|
||||
patch(
|
||||
"EvoScientist.llm.get_chat_model", return_value=new_model
|
||||
) as mock_get,
|
||||
):
|
||||
mw.wrap_model_call(req, handler)
|
||||
|
||||
mock_get.assert_called_once_with(model="gpt-5", provider=None)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 4. ConfigurableModelMiddleware — caching
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestCache:
|
||||
"""Two consecutive calls with same (model, provider) should hit cache."""
|
||||
|
||||
def test_cache_hit_avoids_second_get_chat_model(self):
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req1 = _make_request()
|
||||
req2 = _make_request()
|
||||
new_model = MagicMock()
|
||||
handler = MagicMock(return_value="ok")
|
||||
|
||||
with (
|
||||
_patched_config({"model": "gpt-5", "model_provider": "openai"}),
|
||||
patch(
|
||||
"EvoScientist.llm.get_chat_model", return_value=new_model
|
||||
) as mock_get,
|
||||
):
|
||||
mw.wrap_model_call(req1, handler)
|
||||
mw.wrap_model_call(req2, handler)
|
||||
|
||||
# First call resolves via factory, second hits the cache.
|
||||
assert mock_get.call_count == 1
|
||||
|
||||
def test_cache_miss_on_different_provider(self):
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
handler = MagicMock(return_value="ok")
|
||||
|
||||
with patch(
|
||||
"EvoScientist.llm.get_chat_model", side_effect=[MagicMock(), MagicMock()]
|
||||
) as mock_get:
|
||||
with _patched_config(
|
||||
{"model": "claude-sonnet-4-6", "model_provider": "anthropic"}
|
||||
):
|
||||
mw.wrap_model_call(req, handler)
|
||||
with _patched_config(
|
||||
{"model": "claude-sonnet-4-6", "model_provider": "custom-anthropic"}
|
||||
):
|
||||
mw.wrap_model_call(req, handler)
|
||||
|
||||
assert mock_get.call_count == 2
|
||||
|
||||
def test_independent_instances_have_independent_caches(self):
|
||||
mw_a = ConfigurableModelMiddleware()
|
||||
mw_b = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
handler = MagicMock(return_value="ok")
|
||||
|
||||
with (
|
||||
_patched_config({"model": "gpt-5", "model_provider": "openai"}),
|
||||
patch(
|
||||
"EvoScientist.llm.get_chat_model",
|
||||
side_effect=[MagicMock(), MagicMock()],
|
||||
) as mock_get,
|
||||
):
|
||||
mw_a.wrap_model_call(req, handler)
|
||||
mw_b.wrap_model_call(req, handler)
|
||||
|
||||
# Different instances must each resolve once.
|
||||
assert mock_get.call_count == 2
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 5. Resilience — get_chat_model raising
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestResolveFailure:
|
||||
"""If get_chat_model raises, middleware must fall back to original model."""
|
||||
|
||||
def test_sync_falls_back_when_resolve_raises(self):
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
handler = MagicMock(return_value="ok")
|
||||
|
||||
with (
|
||||
_patched_config({"model": "doesnotexist", "model_provider": "openai"}),
|
||||
patch(
|
||||
"EvoScientist.llm.get_chat_model",
|
||||
side_effect=ValueError("unknown model"),
|
||||
),
|
||||
):
|
||||
mw.wrap_model_call(req, handler)
|
||||
|
||||
# Override never happened — handler called with original request.
|
||||
handler.assert_called_once_with(req)
|
||||
req.override.assert_not_called()
|
||||
|
||||
def test_async_falls_back_when_resolve_raises(self):
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
|
||||
called = []
|
||||
|
||||
async def handler(r):
|
||||
called.append(r)
|
||||
return "ok"
|
||||
|
||||
with (
|
||||
_patched_config({"model": "doesnotexist", "model_provider": "openai"}),
|
||||
patch(
|
||||
"EvoScientist.llm.get_chat_model",
|
||||
side_effect=ValueError("unknown model"),
|
||||
),
|
||||
):
|
||||
result = _run(mw.awrap_model_call(req, handler))
|
||||
|
||||
assert result == "ok"
|
||||
assert called == [req]
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 6. Integration — real langgraph contextvar (no get_config mock)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestRunnableContextVarIntegration:
|
||||
"""Set the actual ``var_child_runnable_config`` contextvar that LangGraph
|
||||
populates per node, then verify the middleware reads through to it.
|
||||
|
||||
This catches breakage of the ``langgraph.config.get_config()`` contract
|
||||
that pure-mock tests would miss (e.g. if get_config is moved to a
|
||||
different module, or the contextvar mechanism changes).
|
||||
"""
|
||||
|
||||
def test_real_contextvar_drives_override(self):
|
||||
"""Without mocking get_config, set the contextvar and verify override."""
|
||||
from langchain_core.runnables.config import var_child_runnable_config
|
||||
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
new_model = MagicMock(name="resolved")
|
||||
handler = MagicMock(return_value="ok")
|
||||
|
||||
token = var_child_runnable_config.set(
|
||||
{"configurable": {"model": "gpt-5.5", "model_provider": "openai"}}
|
||||
)
|
||||
try:
|
||||
with patch(
|
||||
"EvoScientist.llm.get_chat_model", return_value=new_model
|
||||
) as mock_get:
|
||||
mw.wrap_model_call(req, handler)
|
||||
finally:
|
||||
var_child_runnable_config.reset(token)
|
||||
|
||||
mock_get.assert_called_once_with(model="gpt-5.5", provider="openai")
|
||||
req.override.assert_called_once_with(model=new_model)
|
||||
|
||||
def test_real_contextvar_unset_passes_through(self):
|
||||
"""When no contextvar is set, get_config() raises → no override."""
|
||||
from langchain_core.runnables.config import var_child_runnable_config
|
||||
|
||||
# Defensive: ensure no leftover contextvar from another test.
|
||||
token = var_child_runnable_config.set(None)
|
||||
try:
|
||||
mw = ConfigurableModelMiddleware()
|
||||
req = _make_request()
|
||||
handler = MagicMock(return_value="ok")
|
||||
mw.wrap_model_call(req, handler)
|
||||
finally:
|
||||
var_child_runnable_config.reset(token)
|
||||
|
||||
handler.assert_called_once_with(req)
|
||||
req.override.assert_not_called()
|
||||
@@ -119,7 +119,9 @@ def test_default_middleware_includes_context_editing(mock_config, mock_model, mo
|
||||
from EvoScientist.EvoScientist import _get_default_middleware
|
||||
|
||||
mw = _get_default_middleware()
|
||||
assert isinstance(mw[0], ContextEditingMiddleware)
|
||||
# ContextEditingMiddleware is present (its absolute position depends on
|
||||
# other leading middlewares like ConfigurableModelMiddleware).
|
||||
assert any(isinstance(m, ContextEditingMiddleware) for m in mw)
|
||||
|
||||
|
||||
@patch("EvoScientist.EvoScientist._ensure_chat_model")
|
||||
|
||||
@@ -0,0 +1,514 @@
|
||||
"""Tests for the deepagents model-passthrough patch.
|
||||
|
||||
Verifies that ``_patch_deepagents_model_passthrough`` wraps
|
||||
``_build_start_tool`` / ``_build_update_tool`` so that ``client.runs.create``
|
||||
calls inside the launched async-task tools carry
|
||||
``config={"configurable": {"model": ..., "model_provider": ...}}``,
|
||||
without affecting other client methods (``threads.create``, ``runs.get``,
|
||||
``runs.cancel``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from EvoScientist.llm import patches as patches_mod
|
||||
from tests.conftest import run_async as _run
|
||||
|
||||
# =============================================================================
|
||||
# Helpers
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def _stub_cfg(model: str = "claude-sonnet-4-6", provider: str = "anthropic"):
|
||||
"""Build a stand-in for ``_ensure_config()`` return value."""
|
||||
return SimpleNamespace(model=model, provider=provider)
|
||||
|
||||
|
||||
def _make_client_cache(
|
||||
*,
|
||||
create_run_id: str = "run-001",
|
||||
thread_id: str = "thread-001",
|
||||
):
|
||||
"""Build a fake ``_ClientCache`` with sync + async stub clients.
|
||||
|
||||
Returns ``(cache_mock, runs_create_sync_mock, runs_create_async_mock)``
|
||||
so tests can both invoke through the patched factory and inspect the
|
||||
exact call kwargs passed to ``runs.create``.
|
||||
"""
|
||||
runs_sync = MagicMock()
|
||||
runs_sync.create.return_value = {"run_id": create_run_id}
|
||||
runs_sync.cancel = MagicMock(return_value=None)
|
||||
runs_sync.get = MagicMock(return_value={"status": "success"})
|
||||
|
||||
threads_sync = MagicMock()
|
||||
threads_sync.create.return_value = {"thread_id": thread_id}
|
||||
|
||||
sync_client = MagicMock()
|
||||
sync_client.runs = runs_sync
|
||||
sync_client.threads = threads_sync
|
||||
|
||||
runs_async = MagicMock()
|
||||
runs_async.create = AsyncMock(return_value={"run_id": create_run_id})
|
||||
|
||||
threads_async = MagicMock()
|
||||
threads_async.create = AsyncMock(return_value={"thread_id": thread_id})
|
||||
|
||||
async_client = MagicMock()
|
||||
async_client.runs = runs_async
|
||||
async_client.threads = threads_async
|
||||
|
||||
cache = MagicMock()
|
||||
cache.get_sync = MagicMock(return_value=sync_client)
|
||||
cache.get_async = MagicMock(return_value=async_client)
|
||||
|
||||
return cache, runs_sync, runs_async
|
||||
|
||||
|
||||
def _runtime_stub():
|
||||
"""Minimal stand-in for ``ToolRuntime`` accepted by the inner tools."""
|
||||
return SimpleNamespace(tool_call_id="tc-001", state={})
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 1. Idempotence
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestIdempotence:
|
||||
"""The patch must be a no-op after the first application."""
|
||||
|
||||
def test_double_apply_does_not_re_wrap(self, restore_model_passthrough_patch):
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as ds_mod
|
||||
except ImportError:
|
||||
pytest.skip("deepagents not available")
|
||||
|
||||
original = ds_mod._build_start_tool
|
||||
patches_mod._patch_deepagents_model_passthrough()
|
||||
wrapped_once = ds_mod._build_start_tool
|
||||
assert wrapped_once is not original
|
||||
|
||||
patches_mod._patch_deepagents_model_passthrough()
|
||||
wrapped_twice = ds_mod._build_start_tool
|
||||
assert wrapped_twice is wrapped_once # not double-wrapped
|
||||
|
||||
def test_flag_set_after_apply(self, restore_model_passthrough_patch):
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as _ # noqa: F401
|
||||
except ImportError:
|
||||
pytest.skip("deepagents not available")
|
||||
patches_mod._model_passthrough_patched = False
|
||||
patches_mod._patch_deepagents_model_passthrough()
|
||||
assert patches_mod._model_passthrough_patched is True
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 2. start_async_task injects config into runs.create
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestStartAsyncTaskInjection:
|
||||
"""Sync and async start_async_task must inject configurable.model."""
|
||||
|
||||
def test_sync_start_injects_config(self, restore_model_passthrough_patch):
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as ds_mod
|
||||
except ImportError:
|
||||
pytest.skip("deepagents not available")
|
||||
|
||||
patches_mod._patch_deepagents_model_passthrough()
|
||||
|
||||
cache, runs_sync, _ = _make_client_cache()
|
||||
agent_map = {
|
||||
"writing-agent": {
|
||||
"name": "writing-agent",
|
||||
"description": "Draft paper",
|
||||
"graph_id": "writing-agent",
|
||||
"url": "http://localhost:6174",
|
||||
}
|
||||
}
|
||||
|
||||
tool = ds_mod._build_start_tool(agent_map, cache, "desc")
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=_stub_cfg(model="gpt-5", provider="openai"),
|
||||
):
|
||||
tool.func(
|
||||
description="hello",
|
||||
subagent_type="writing-agent",
|
||||
runtime=_runtime_stub(),
|
||||
)
|
||||
|
||||
runs_sync.create.assert_called_once()
|
||||
kwargs = runs_sync.create.call_args.kwargs
|
||||
assert kwargs["thread_id"] == "thread-001"
|
||||
assert kwargs["assistant_id"] == "writing-agent"
|
||||
assert kwargs["config"] == {
|
||||
"configurable": {"model": "gpt-5", "model_provider": "openai"}
|
||||
}
|
||||
|
||||
def test_async_start_injects_config(self, restore_model_passthrough_patch):
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as ds_mod
|
||||
except ImportError:
|
||||
pytest.skip("deepagents not available")
|
||||
|
||||
patches_mod._patch_deepagents_model_passthrough()
|
||||
|
||||
cache, _, runs_async = _make_client_cache()
|
||||
agent_map = {
|
||||
"writing-agent": {
|
||||
"name": "writing-agent",
|
||||
"description": "Draft paper",
|
||||
"graph_id": "writing-agent",
|
||||
"url": "http://localhost:6174",
|
||||
}
|
||||
}
|
||||
|
||||
tool = ds_mod._build_start_tool(agent_map, cache, "desc")
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=_stub_cfg(model="claude-haiku-4-5", provider="anthropic"),
|
||||
):
|
||||
_run(
|
||||
tool.coroutine(
|
||||
description="hi",
|
||||
subagent_type="writing-agent",
|
||||
runtime=_runtime_stub(),
|
||||
)
|
||||
)
|
||||
|
||||
runs_async.create.assert_awaited_once()
|
||||
kwargs = runs_async.create.call_args.kwargs
|
||||
assert kwargs["config"] == {
|
||||
"configurable": {
|
||||
"model": "claude-haiku-4-5",
|
||||
"model_provider": "anthropic",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 3. Live config read at tool-call time (post-/model switch behavior)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestLiveConfigRead:
|
||||
"""The patch must read cfg fresh on every tool call, not at patch time."""
|
||||
|
||||
def test_two_calls_reflect_separate_cfg(self, restore_model_passthrough_patch):
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as ds_mod
|
||||
except ImportError:
|
||||
pytest.skip("deepagents not available")
|
||||
|
||||
patches_mod._patch_deepagents_model_passthrough()
|
||||
|
||||
cache, runs_sync, _ = _make_client_cache()
|
||||
agent_map = {
|
||||
"writing-agent": {
|
||||
"name": "writing-agent",
|
||||
"description": "x",
|
||||
"graph_id": "writing-agent",
|
||||
"url": "http://localhost:6174",
|
||||
}
|
||||
}
|
||||
tool = ds_mod._build_start_tool(agent_map, cache, "desc")
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=_stub_cfg(model="model-a", provider="anthropic"),
|
||||
):
|
||||
tool.func(
|
||||
description="t1",
|
||||
subagent_type="writing-agent",
|
||||
runtime=_runtime_stub(),
|
||||
)
|
||||
first_kwargs = runs_sync.create.call_args.kwargs
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=_stub_cfg(model="model-b", provider="openai"),
|
||||
):
|
||||
tool.func(
|
||||
description="t2",
|
||||
subagent_type="writing-agent",
|
||||
runtime=_runtime_stub(),
|
||||
)
|
||||
second_kwargs = runs_sync.create.call_args.kwargs
|
||||
|
||||
assert first_kwargs["config"]["configurable"]["model"] == "model-a"
|
||||
assert second_kwargs["config"]["configurable"]["model"] == "model-b"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 4. update_async_task also injects config
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestUpdateAsyncTaskInjection:
|
||||
"""update_async_task must inject config too — not just start."""
|
||||
|
||||
def _tracked_task(self, agent_name: str = "writing-agent") -> dict:
|
||||
return {
|
||||
"task_id": "thread-001",
|
||||
"agent_name": agent_name,
|
||||
"thread_id": "thread-001",
|
||||
"run_id": "old-run",
|
||||
"status": "running",
|
||||
"created_at": "2026-05-07T00:00:00Z",
|
||||
"last_checked_at": "2026-05-07T00:00:00Z",
|
||||
"last_updated_at": "2026-05-07T00:00:00Z",
|
||||
}
|
||||
|
||||
def test_async_update_injects_config(self, restore_model_passthrough_patch):
|
||||
"""The async coroutine path must inject config too."""
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as ds_mod
|
||||
except ImportError:
|
||||
pytest.skip("deepagents not available")
|
||||
|
||||
patches_mod._patch_deepagents_model_passthrough()
|
||||
|
||||
cache, _, runs_async = _make_client_cache()
|
||||
agent_map = {
|
||||
"writing-agent": {
|
||||
"name": "writing-agent",
|
||||
"description": "x",
|
||||
"graph_id": "writing-agent",
|
||||
"url": "http://localhost:6174",
|
||||
}
|
||||
}
|
||||
runtime = SimpleNamespace(
|
||||
tool_call_id="tc-002",
|
||||
state={"async_tasks": {"thread-001": self._tracked_task()}},
|
||||
)
|
||||
|
||||
tool = ds_mod._build_update_tool(agent_map, cache)
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=_stub_cfg(model="gpt-5", provider="openai"),
|
||||
):
|
||||
_run(
|
||||
tool.coroutine(
|
||||
task_id="thread-001",
|
||||
message="follow up async",
|
||||
runtime=runtime,
|
||||
)
|
||||
)
|
||||
|
||||
runs_async.create.assert_awaited_once()
|
||||
kwargs = runs_async.create.call_args.kwargs
|
||||
assert kwargs["config"] == {
|
||||
"configurable": {"model": "gpt-5", "model_provider": "openai"}
|
||||
}
|
||||
assert kwargs.get("multitask_strategy") == "interrupt"
|
||||
|
||||
def test_sync_update_injects_config(self, restore_model_passthrough_patch):
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as ds_mod
|
||||
except ImportError:
|
||||
pytest.skip("deepagents not available")
|
||||
|
||||
patches_mod._patch_deepagents_model_passthrough()
|
||||
|
||||
cache, runs_sync, _ = _make_client_cache()
|
||||
agent_map = {
|
||||
"writing-agent": {
|
||||
"name": "writing-agent",
|
||||
"description": "x",
|
||||
"graph_id": "writing-agent",
|
||||
"url": "http://localhost:6174",
|
||||
}
|
||||
}
|
||||
# update_async_task reads tracked task from runtime.state
|
||||
tracked_task = {
|
||||
"task_id": "thread-001",
|
||||
"agent_name": "writing-agent",
|
||||
"thread_id": "thread-001",
|
||||
"run_id": "old-run",
|
||||
"status": "running",
|
||||
"created_at": "2026-05-07T00:00:00Z",
|
||||
"last_checked_at": "2026-05-07T00:00:00Z",
|
||||
"last_updated_at": "2026-05-07T00:00:00Z",
|
||||
}
|
||||
runtime = SimpleNamespace(
|
||||
tool_call_id="tc-002",
|
||||
state={"async_tasks": {"thread-001": tracked_task}},
|
||||
)
|
||||
|
||||
tool = ds_mod._build_update_tool(agent_map, cache)
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=_stub_cfg(model="gpt-5", provider="openai"),
|
||||
):
|
||||
tool.func(
|
||||
task_id="thread-001",
|
||||
message="follow up",
|
||||
runtime=runtime,
|
||||
)
|
||||
|
||||
runs_sync.create.assert_called_once()
|
||||
kwargs = runs_sync.create.call_args.kwargs
|
||||
assert kwargs["config"]["configurable"]["model"] == "gpt-5"
|
||||
assert kwargs["config"]["configurable"]["model_provider"] == "openai"
|
||||
# update preserves multitask_strategy="interrupt" — verify no regression
|
||||
assert kwargs.get("multitask_strategy") == "interrupt"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 5. Other client methods are unaffected
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestNonInterceptedMethods:
|
||||
"""``threads.create``, ``runs.get``, ``runs.cancel`` must pass through."""
|
||||
|
||||
def test_threads_create_not_modified(self, restore_model_passthrough_patch):
|
||||
"""``threads.create()`` is called by start_async_task pre-runs.create.
|
||||
|
||||
The patch should not inject config here, because thread creation
|
||||
doesn't take config (and would error out if we did).
|
||||
"""
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as ds_mod
|
||||
except ImportError:
|
||||
pytest.skip("deepagents not available")
|
||||
|
||||
patches_mod._patch_deepagents_model_passthrough()
|
||||
|
||||
cache, _runs_sync, _ = _make_client_cache()
|
||||
threads_create = cache.get_sync.return_value.threads.create
|
||||
agent_map = {
|
||||
"writing-agent": {
|
||||
"name": "writing-agent",
|
||||
"description": "x",
|
||||
"graph_id": "writing-agent",
|
||||
"url": "http://localhost:6174",
|
||||
}
|
||||
}
|
||||
tool = ds_mod._build_start_tool(agent_map, cache, "desc")
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=_stub_cfg(),
|
||||
):
|
||||
tool.func(
|
||||
description="hi",
|
||||
subagent_type="writing-agent",
|
||||
runtime=_runtime_stub(),
|
||||
)
|
||||
|
||||
# threads.create() is called with no kwargs (deepagents pattern).
|
||||
threads_create.assert_called_once_with()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 6. Empty cfg → no config kwarg added
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestEmptyCfg:
|
||||
"""If neither model nor provider is set, don't inject anything."""
|
||||
|
||||
def test_empty_cfg_no_config_kwarg(self, restore_model_passthrough_patch):
|
||||
try:
|
||||
from deepagents.middleware import async_subagents as ds_mod
|
||||
except ImportError:
|
||||
pytest.skip("deepagents not available")
|
||||
|
||||
patches_mod._patch_deepagents_model_passthrough()
|
||||
|
||||
cache, runs_sync, _ = _make_client_cache()
|
||||
agent_map = {
|
||||
"writing-agent": {
|
||||
"name": "writing-agent",
|
||||
"description": "x",
|
||||
"graph_id": "writing-agent",
|
||||
"url": "http://localhost:6174",
|
||||
}
|
||||
}
|
||||
tool = ds_mod._build_start_tool(agent_map, cache, "desc")
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=SimpleNamespace(model=None, provider=None),
|
||||
):
|
||||
tool.func(
|
||||
description="hi",
|
||||
subagent_type="writing-agent",
|
||||
runtime=_runtime_stub(),
|
||||
)
|
||||
|
||||
runs_sync.create.assert_called_once()
|
||||
kwargs = runs_sync.create.call_args.kwargs
|
||||
# No config kwarg should be added when there's nothing to override.
|
||||
assert "config" not in kwargs
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 7. Caller-supplied config keys are preserved
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestPreserveExistingConfig:
|
||||
"""If a caller already supplied config.configurable.X, our merge keeps it."""
|
||||
|
||||
def test_existing_configurable_preserved(self):
|
||||
"""Direct unit test of the merge helper (integration covered above)."""
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=_stub_cfg(model="gpt-5", provider="openai"),
|
||||
):
|
||||
merged = patches_mod._merge_runs_config_kwargs(
|
||||
{
|
||||
"thread_id": "t1",
|
||||
"config": {
|
||||
"configurable": {"thread_id": "outer-t", "extra": 42},
|
||||
"tags": ["debug"],
|
||||
},
|
||||
}
|
||||
)
|
||||
assert merged["thread_id"] == "t1"
|
||||
assert merged["config"]["tags"] == ["debug"]
|
||||
assert merged["config"]["configurable"]["thread_id"] == "outer-t"
|
||||
assert merged["config"]["configurable"]["extra"] == 42
|
||||
assert merged["config"]["configurable"]["model"] == "gpt-5"
|
||||
assert merged["config"]["configurable"]["model_provider"] == "openai"
|
||||
|
||||
def test_non_dict_config_replaced(self):
|
||||
"""Non-dict ``config`` (e.g. a Pydantic RunnableConfig) is replaced.
|
||||
|
||||
Documents the policy: callers that pass a non-dict ``config`` lose
|
||||
any other fields they may have set there. Acceptable today because
|
||||
deepagents' built-in ``runs.create`` path doesn't pass a config at
|
||||
all, but a future caller passing e.g. a Pydantic model would have
|
||||
their non-configurable fields silently dropped. If that becomes a
|
||||
real use case, ``_merge_runs_config_kwargs`` should grow a
|
||||
``dict()`` coercion or raise.
|
||||
"""
|
||||
|
||||
class _Sentinel:
|
||||
"""Stand-in for any non-dict config-shaped object."""
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=_stub_cfg(model="gpt-5", provider="openai"),
|
||||
):
|
||||
merged = patches_mod._merge_runs_config_kwargs(
|
||||
{"thread_id": "t1", "config": _Sentinel()}
|
||||
)
|
||||
assert merged["thread_id"] == "t1"
|
||||
# Non-dict input was replaced with a fresh dict carrying only our
|
||||
# injected keys.
|
||||
assert merged["config"] == {
|
||||
"configurable": {"model": "gpt-5", "model_provider": "openai"}
|
||||
}
|
||||
Reference in New Issue
Block a user