From c407d2e20f4a782fb25d8483694a27cb7476c55e Mon Sep 17 00:00:00 2001 From: Xi Zhang <106144707+X-iZhang@users.noreply.github.com> Date: Fri, 8 May 2026 23:52:24 +0200 Subject: [PATCH] Fix/async subagent model switch (#217) * feat(middleware): add ConfigurableModelMiddleware for dynamic model resolution - Introduced ConfigurableModelMiddleware to resolve chat models from RunnableConfig.configurable on each call. - Updated middleware initialization to include ConfigurableModelMiddleware. - Enhanced context editing middleware tests to verify presence of ConfigurableModelMiddleware. - Implemented tests for ConfigurableModelMiddleware to ensure correct model overriding and caching behavior. - Added tests for deepagents model-passthrough patch to verify configuration injection in async tasks. * feat(async-subagent): update middleware handling to prevent deadlocks in async sub-agents * style: Refactor code formatting for improved readability in patches and test files * refactor: streamline middleware construction and improve async handling in ConfigurableModelMiddleware * fix: remove unused request parameter from _read_model_override function * refactor: improve async handling in _ClientProxy and enhance logging in ConfigurableModelMiddleware test: add behavior test to ensure AskUserMiddleware is excluded in async subagent mode --- EvoScientist/EvoScientist.py | 62 ++- EvoScientist/llm/patches.py | 183 +++++++ EvoScientist/middleware/__init__.py | 2 + EvoScientist/middleware/configurable_model.py | 196 +++++++ EvoScientist/subagents/_factory.py | 8 +- tests/conftest.py | 46 ++ tests/test_ask_user.py | 39 ++ tests/test_async_subagent_factory.py | 119 ++++ tests/test_configurable_model_middleware.py | 394 ++++++++++++++ tests/test_context_editing_middleware.py | 4 +- tests/test_model_passthrough_patch.py | 514 ++++++++++++++++++ 11 files changed, 1538 insertions(+), 29 deletions(-) create mode 100644 EvoScientist/middleware/configurable_model.py create mode 100644 tests/test_async_subagent_factory.py create mode 100644 tests/test_configurable_model_middleware.py create mode 100644 tests/test_model_passthrough_patch.py diff --git a/EvoScientist/EvoScientist.py b/EvoScientist/EvoScientist.py index ebce27c..6467d1f 100644 --- a/EvoScientist/EvoScientist.py +++ b/EvoScientist/EvoScientist.py @@ -308,6 +308,15 @@ def _maybe_swap_async_subagents(subs: list, middleware: list | None = None) -> l middleware.append(AsyncWatcherMiddleware(agent_specs)) + # Forward the CLI's live (model, provider) into deepagents' + # start/update_async_task tool calls so the deployed graph can + # re-resolve its chat model per run via ConfigurableModelMiddleware. + # Idempotent — safe to call on every CLI startup. + if agent_specs: + from .llm.patches import _patch_deepagents_model_passthrough + + _patch_deepagents_model_passthrough() + return out @@ -439,9 +448,23 @@ def _get_default_backend(): ) -def _get_default_middleware(): - """Build the default middleware list.""" +def _get_default_middleware(*, for_async_subagent: bool = False): + """Build the default middleware list. + + Args: + for_async_subagent: When True, omit middleware that would deadlock a + deployed async sub-agent. Specifically: ``AskUserMiddleware`` uses + ``interrupt()`` to pause the graph waiting for a user reply, but + async sub-agents run in the ``langgraph dev`` subprocess where + the parent only holds a ``task_id`` and has no UI path to surface + (or resume) an interrupt — the sub-agent would hang forever the + first time it called ``ask_user``. This mirrors the same reason + ``subagents/_factory.py`` deliberately skips ``interrupt_on=`` on + the deepagents level. Defaults to False (full middleware list) + for the CLI's in-process agent. + """ from .middleware import ( + ConfigurableModelMiddleware, ContextOverflowMapperMiddleware, ModelFallbackMiddleware, ToolErrorHandlerMiddleware, @@ -456,7 +479,12 @@ def _get_default_middleware(): load_fallback_chain(cfg.model_fallbacks) model = _ensure_chat_model() memory_dir = str(_paths_mod.MEMORIES_DIR) + # ``ConfigurableModelMiddleware`` is placed first so it wraps + # ``ModelFallbackMiddleware``: a configurable.model override sets the + # PRIMARY model only, leaving the fallback chain free to try its own + # alternatives instead of re-overriding every retry to the same model. mw = [ + ConfigurableModelMiddleware(), create_context_editing_middleware(model), ModelFallbackMiddleware(), ContextOverflowMapperMiddleware(), @@ -465,7 +493,7 @@ def _get_default_middleware(): create_memory_middleware(memory_dir, extraction_model=model), ] - if cfg.enable_ask_user and not cfg.auto_mode: + if cfg.enable_ask_user and not cfg.auto_mode and not for_async_subagent: from .middleware.ask_user import AskUserMiddleware mw.insert(0, AskUserMiddleware()) @@ -557,19 +585,8 @@ def create_cli_agent( from . import paths as _paths from .backends import CustomSandboxBackend, MergedSkillsBackend - from .middleware import ( - ContextOverflowMapperMiddleware, - ModelFallbackMiddleware, - ToolErrorHandlerMiddleware, - create_context_editing_middleware, - create_memory_middleware, - create_tool_selector_middleware, - load_fallback_chain, - ) cfg = _ensure_config(config) - if cfg.model_fallbacks: - load_fallback_chain(cfg.model_fallbacks) if checkpointer is None: from langgraph.checkpoint.memory import InMemorySaver @@ -617,19 +634,10 @@ def create_cli_agent( }, ) - model = _ensure_chat_model() - mw: list[AgentMiddleware] = [ - create_context_editing_middleware(model), - ModelFallbackMiddleware(), - ContextOverflowMapperMiddleware(), - ToolErrorHandlerMiddleware(), - *create_tool_selector_middleware(model=model), - create_memory_middleware(_mem_dir, extraction_model=model), - ] - if cfg.enable_ask_user and not cfg.auto_mode: - from .middleware.ask_user import AskUserMiddleware - - mw.insert(0, AskUserMiddleware()) + # Delegate middleware construction to the single source of truth so the + # CLI agent never drifts from the default chain. Anything CLI-specific + # (e.g. ``HumanInTheLoopMiddleware``) is appended below. + mw: list[AgentMiddleware] = _get_default_middleware() # HITL on main agent only — passing `interrupt_on=` to create_deep_agent # would propagate it to every subagent, breaking parallel execute calls diff --git a/EvoScientist/llm/patches.py b/EvoScientist/llm/patches.py index 70dea49..d58d0da 100644 --- a/EvoScientist/llm/patches.py +++ b/EvoScientist/llm/patches.py @@ -522,3 +522,186 @@ def _patch_deepseek_reasoning_passback(model: Any) -> None: return payload model._get_request_payload = _patched + + +# --------------------------------------------------------------------------- +# Patch: forward CLI's live (model, model_provider) into deepagents' +# start_async_task / update_async_task tool calls so the deployed graph +# (running in a separate ``langgraph dev`` subprocess) re-resolves the +# chat model per run. +# +# Without this, async sub-agents stay on the model their graph was compiled +# with at langgraph dev boot — `/model` switches in the CLI never reach +# them because they live in another process. +# +# Mechanism: wrap deepagents' ``_build_start_tool`` and ``_build_update_tool`` +# factories. Each wrapped factory calls the original with a proxied client +# cache that intercepts ``runs.create(...)`` calls only and injects +# ``config={"configurable": {"model": , "model_provider": }}``. +# All other client methods (``threads.create``, ``runs.get``, ``runs.cancel``, +# ``runs.join_stream``) pass through unchanged. The deployed graph picks up +# ``configurable.model`` via ``ConfigurableModelMiddleware``. +# +# Reads ``_ensure_config()`` at tool-call time (not patch time) so a +# ``/model`` switch in the CLI is reflected on the very next async tool +# call without an agent rebuild. +# +# Upstream PR opportunity: passing ``config`` through ``client.runs.create`` +# is generic functionality; worth contributing back to ``langchain-ai/deepagents`` +# so this patch can be retired. +# --------------------------------------------------------------------------- +_model_passthrough_patched = False + + +def _read_cfg_configurable() -> dict[str, str]: + """Read live ``(model, provider)`` from EvoScientist config. + + Returns a dict suitable for inserting under + ``RunnableConfig.configurable``. Empty dict on any failure (so the + patch degrades to a no-op rather than breaking async tool calls). + """ + try: + from EvoScientist.EvoScientist import _ensure_config + + cfg = _ensure_config() + except Exception: + return {} + + out: dict[str, str] = {} + model = getattr(cfg, "model", None) + provider = getattr(cfg, "provider", None) + if isinstance(model, str) and model: + out["model"] = model + if isinstance(provider, str) and provider: + out["model_provider"] = provider + return out + + +def _merge_runs_config_kwargs(kwargs: dict) -> dict: + """Merge the live model override into ``kwargs`` for ``runs.create``. + + Preserves any caller-supplied ``config.configurable`` keys. EvoScientist's + keys take precedence on conflict (callers shouldn't be passing model + overrides — the CLI is the source of truth). + """ + overrides = _read_cfg_configurable() + if not overrides: + return kwargs + existing = kwargs.get("config") + if not isinstance(existing, dict): + existing = {} + existing_configurable = existing.get("configurable") + if not isinstance(existing_configurable, dict): + existing_configurable = {} + merged_configurable = {**existing_configurable, **overrides} + kwargs = dict(kwargs) + kwargs["config"] = {**existing, "configurable": merged_configurable} + return kwargs + + +class _SyncRunsProxy: + """Wraps a sync ``RunsClient`` and injects config into ``create`` only.""" + + def __init__(self, real: Any) -> None: + object.__setattr__(self, "_real", real) + + def __getattr__(self, name: str) -> Any: + return getattr(self._real, name) + + def create(self, **kwargs: Any) -> Any: + return self._real.create(**_merge_runs_config_kwargs(kwargs)) + + +class _AsyncRunsProxy: + """Wraps an async ``RunsClient`` and injects config into ``create`` only.""" + + def __init__(self, real: Any) -> None: + object.__setattr__(self, "_real", real) + + def __getattr__(self, name: str) -> Any: + return getattr(self._real, name) + + async def create(self, **kwargs: Any) -> Any: + return await self._real.create(**_merge_runs_config_kwargs(kwargs)) + + +class _ClientProxy: + """Lightweight proxy that swaps ``client.runs`` for a runs proxy. + + Read-only forwarding: only ``__getattr__`` is overridden. Attribute + *writes* on the proxy land on the proxy itself, NOT on the wrapped real + client. Current deepagents only reads ``.threads`` / ``.runs``, so this + is safe — but if a future caller tries ``client.foo = bar`` through the + proxy, the write will be silently lost. ``object.__setattr__`` in + ``__init__`` is used solely to avoid infinite recursion when seeding + the internal slots. + + ``client.runs`` is a stable attribute set in + ``langgraph_sdk.client.LangGraphClient.__init__`` (``self.runs = + RunsClient(...)``) — not a property or lazy initializer — so the wrapped + runs proxy is built once at ``__init__`` time and reused on every + ``proxy.runs`` access. This avoids the previous behavior of selecting + sync/async and instantiating a new ``_RunsProxy`` per attribute access. + """ + + def __init__(self, real: Any, *, is_async: bool) -> None: + runs_proxy_cls = _AsyncRunsProxy if is_async else _SyncRunsProxy + object.__setattr__(self, "_real", real) + object.__setattr__(self, "_runs_proxy", runs_proxy_cls(real.runs)) + + def __getattr__(self, name: str) -> Any: + if name == "runs": + return self._runs_proxy + return getattr(self._real, name) + + +class _ClientCacheProxy: + """Proxy a ``_ClientCache`` so callers receive config-injecting clients.""" + + def __init__(self, real: Any) -> None: + self._real = real + + def get_sync(self, name: str) -> Any: + return _ClientProxy(self._real.get_sync(name), is_async=False) + + def get_async(self, name: str) -> Any: + return _ClientProxy(self._real.get_async(name), is_async=True) + + +def _patch_deepagents_model_passthrough() -> None: + """Wrap deepagents' async-launch tool factories to inject CLI model. + + Idempotent: re-invocation is a no-op once the patch is active. Safe to + call from ``_maybe_swap_async_subagents`` on every CLI startup; both + that hook and this patch turn on together when async sub-agents are + enabled. + """ + global _model_passthrough_patched + if _model_passthrough_patched: + return + + try: + from deepagents.middleware import async_subagents as ds_mod + except ImportError: + return + + # Defensive ``getattr`` lookups mirror the rest of this file (lines 254, + # 266, 279, 290, 339, 351, 362, 373, 473): a deepagents update that + # renames or removes either private helper degrades to a no-op instead + # of raising ``AttributeError`` at CLI startup. + orig_build_start = getattr(ds_mod, "_build_start_tool", None) + orig_build_update = getattr(ds_mod, "_build_update_tool", None) + if orig_build_start is None or orig_build_update is None: + return + + def _patched_build_start( + agent_map: Any, clients: Any, tool_description: str + ) -> Any: + return orig_build_start(agent_map, _ClientCacheProxy(clients), tool_description) + + def _patched_build_update(agent_map: Any, clients: Any) -> Any: + return orig_build_update(agent_map, _ClientCacheProxy(clients)) + + ds_mod._build_start_tool = _patched_build_start + ds_mod._build_update_tool = _patched_build_update + _model_passthrough_patched = True diff --git a/EvoScientist/middleware/__init__.py b/EvoScientist/middleware/__init__.py index ea20d1e..f725650 100644 --- a/EvoScientist/middleware/__init__.py +++ b/EvoScientist/middleware/__init__.py @@ -11,6 +11,7 @@ from .ask_user import ( Choice, Question, ) +from .configurable_model import ConfigurableModelMiddleware from .context_editing import ( compute_context_editing_trigger, create_context_editing_middleware, @@ -32,6 +33,7 @@ __all__ = [ "AskUserRequest", "AskUserWidgetResult", "Choice", + "ConfigurableModelMiddleware", "ContextOverflowMapperMiddleware", "EvoMemoryMiddleware", "EvoMemoryState", diff --git a/EvoScientist/middleware/configurable_model.py b/EvoScientist/middleware/configurable_model.py new file mode 100644 index 0000000..4c65f4e --- /dev/null +++ b/EvoScientist/middleware/configurable_model.py @@ -0,0 +1,196 @@ +"""Middleware that resolves the chat model from RunnableConfig.configurable per call. + +The deployed async sub-agents run in a separate ``langgraph dev`` subprocess +and have their model frozen into the compiled graph at subprocess boot time +(see ``EvoScientist/subagents/_factory.py``). When the user runs ``/model`` +in the CLI, only the CLI process's model state changes — the subprocess +graph still uses the boot-time model. + +This middleware closes that gap by reading ``model`` / ``model_provider`` +from ``RunnableConfig.configurable`` on every model call. The CLI's patched +``start_async_task`` / ``update_async_task`` (see ``llm/patches.py``) injects +those fields into ``client.runs.create(config=...)``; the deployed graph +hits this middleware and re-resolves the chat model fresh. + +When ``configurable.model`` is absent, the middleware is a pass-through — +safe to install on the CLI's in-process agent too. + +The middleware mirrors the pattern used by ``ModelFallbackMiddleware``: +``request.override(model=new_model)`` does not break tool binding, because +the downstream model-invocation node re-binds tools per request. + +**Reading the config**: ``Runtime`` (per its own docstring) does NOT include +``config``. The official path to reach ``RunnableConfig`` from inside any +runnable context (including middleware) is ``langgraph.config.get_config()``, +which reads a context-var that LangGraph populates per node execution. That +is what this middleware uses. ``request.runtime`` is intentionally NOT +relied upon for config — only for diagnostics. +""" + +from __future__ import annotations + +import asyncio +import logging +import threading +from collections.abc import Awaitable, Callable +from typing import Any + +from langchain.agents.middleware.types import ( + AgentMiddleware, + ModelRequest, + ModelResponse, +) + +logger = logging.getLogger(__name__) + + +def _read_model_override() -> tuple[str | None, str | None]: + """Pull ``(model, model_provider)`` from the active ``RunnableConfig``. + + Reads via ``langgraph.config.get_config()`` (the documented entry point + for accessing the per-run ``RunnableConfig`` from inside any runnable + context — middleware, node, tool). Returns ``(None, None)`` when the + config has no ``configurable.model`` override or when called outside a + runnable context. + """ + try: + from langgraph.config import get_config + + cfg = get_config() + except Exception: + # Outside a runnable context (most common in tests) or + # langgraph not importable — nothing to override. + return None, None + if not isinstance(cfg, dict): + return None, None + configurable = cfg.get("configurable") or {} + if not isinstance(configurable, dict): + return None, None + model = configurable.get("model") + provider = configurable.get("model_provider") + return ( + model if isinstance(model, str) and model else None, + provider if isinstance(provider, str) and provider else None, + ) + + +class ConfigurableModelMiddleware(AgentMiddleware): + """Re-resolve the chat model from RunnableConfig.configurable on every call. + + Reads ``model`` and ``model_provider`` from the active ``RunnableConfig`` + via ``langgraph.config.get_config()`` — the documented entry point for + accessing per-run config from any runnable context (middleware, node, tool). + When the override is present, calls + ``EvoScientist.llm.get_chat_model(model=..., provider=...)`` and replaces + ``request.model`` via ``request.override``. When absent, the middleware + passes through unchanged. + + Note: ``Runtime`` (per its own docstring) does NOT include ``config`` as a + field — an earlier version of this middleware tried to read + ``request.runtime.config`` and silently no-op'd because that attribute does + not exist. Stick with ``get_config()``. + + A per-instance cache keyed by ``(model, provider)`` avoids rebuilding + identical models within a turn. The cache is a plain dict guarded by a + ``threading.Lock`` because middleware instances are shared across + concurrent requests in long-lived deployments (e.g. ``langgraph dev`` + workers). + """ + + name = "configurable_model" + + def __init__(self) -> None: + super().__init__() + self._cache: dict[tuple[str, str | None], Any] = {} + self._lock = threading.Lock() + # Track the last (model, provider) pair we INFO-logged so we only + # surface a banner on transition. Without this, every LLM call in a + # long async run would emit an identical INFO line. + self._last_logged_key: tuple[str, str | None] | None = None + + def _log_override(self, model_name: str, provider: str | None) -> None: + """INFO on transition; DEBUG on subsequent calls with same key.""" + key = (model_name, provider) + with self._lock: + transitioned = key != self._last_logged_key + if transitioned: + self._last_logged_key = key + if transitioned: + logger.info( + "ConfigurableModelMiddleware: overriding model to %s (%s)", + model_name, + provider, + ) + else: + logger.debug( + "ConfigurableModelMiddleware: reusing override model=%s provider=%s", + model_name, + provider, + ) + + def _resolve(self, model: str, provider: str | None) -> Any: + """Return a cached or freshly-built chat model for ``(model, provider)``.""" + key = (model, provider) + with self._lock: + cached = self._cache.get(key) + if cached is not None: + return cached + # Build outside the lock (network/SDK init can be slow); two + # concurrent first-time misses for the same key may build twice but + # the second result simply overwrites the first — both are equivalent. + from ..llm import get_chat_model + + new_model = get_chat_model(model=model, provider=provider) + with self._lock: + self._cache[key] = new_model + return new_model + + def wrap_model_call( + self, + request: ModelRequest, + handler: Callable[[ModelRequest], ModelResponse], + ) -> ModelResponse: + model_name, provider = _read_model_override() + if model_name is None: + return handler(request) + try: + new_model = self._resolve(model_name, provider) + except Exception: + logger.warning( + "ConfigurableModelMiddleware failed to resolve model=%r " + "provider=%r; falling back to compile-time model", + model_name, + provider, + exc_info=True, + ) + return handler(request) + self._log_override(model_name, provider) + return handler(request.override(model=new_model)) + + async def awrap_model_call( + self, + request: ModelRequest, + handler: Callable[[ModelRequest], Awaitable[ModelResponse]], + ) -> ModelResponse: + model_name, provider = _read_model_override() + if model_name is None: + return await handler(request) + try: + # Offload first-call SDK init off the event loop. ``_resolve`` calls + # ``get_chat_model`` on a cache miss, which can spend hundreds of ms + # building HTTP clients. Doing this synchronously inside an + # ``async def`` would block every other coroutine on the same + # langgraph dev event loop. Cache hits are still fast (a dict + # lookup); the thread-pool overhead is irrelevant once warm. + new_model = await asyncio.to_thread(self._resolve, model_name, provider) + except Exception: + logger.warning( + "ConfigurableModelMiddleware failed to resolve model=%r " + "provider=%r; falling back to compile-time model", + model_name, + provider, + exc_info=True, + ) + return await handler(request) + self._log_override(model_name, provider) + return await handler(request.override(model=new_model)) diff --git a/EvoScientist/subagents/_factory.py b/EvoScientist/subagents/_factory.py index e95ed53..5d737d5 100644 --- a/EvoScientist/subagents/_factory.py +++ b/EvoScientist/subagents/_factory.py @@ -90,6 +90,12 @@ def build_async_subagent_graph(name: str) -> Any: # approve. The user-visible HITL boundary is the parent's # ``start_async_task`` decision; restrict the child's reach by limiting # ``tools`` in ``subagents/.yaml`` instead. + # + # ``for_async_subagent=True`` propagates the same reasoning to the + # middleware list — specifically, it suppresses ``AskUserMiddleware``, + # which uses ``interrupt()`` for the same purpose (waiting on a user + # reply) and would deadlock an async sub-agent for the same reason. + # # Memory middleware is included so async sub-agents can READ # /memory/MEMORY.md, but the extraction trigger (20+ human messages, # see middleware/memory.py) never fires here — sub-agents only receive @@ -103,5 +109,5 @@ def build_async_subagent_graph(name: str) -> Any: tools=spec.get("tools", []) + agent_mcp_tools, skills=spec.get("skills"), backend=_get_default_backend(), - middleware=_get_default_middleware(), + middleware=_get_default_middleware(for_async_subagent=True), ).with_config({"recursion_limit": cfg.recursion_limit}) diff --git a/tests/conftest.py b/tests/conftest.py index 495d8a7..9994834 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -91,3 +91,49 @@ def tmp_workspace(tmp_path): ws = tmp_path / "workspace" ws.mkdir() return str(ws) + + +# Capture deepagents tool factories at conftest load time — BEFORE any test +# imports EvoScientist, which can trigger ``_patch_deepagents_model_passthrough`` +# during agent construction. Once captured here, the ``restore_model_passthrough_patch`` +# fixture has a stable "truly unpatched" baseline to reset to between tests, even +# if upstream code paths apply the patch as a side effect. +try: + from deepagents.middleware import async_subagents as _ds_async_subagents + + _DEEPAGENTS_ORIGINAL_BUILD_START = _ds_async_subagents._build_start_tool + _DEEPAGENTS_ORIGINAL_BUILD_UPDATE = _ds_async_subagents._build_update_tool +except Exception: + _ds_async_subagents = None + _DEEPAGENTS_ORIGINAL_BUILD_START = None + _DEEPAGENTS_ORIGINAL_BUILD_UPDATE = None + + +@pytest.fixture +def restore_model_passthrough_patch(): + """Reset deepagents internals + ``_model_passthrough_patched`` to unpatched. + + The model-passthrough patch wraps ``deepagents.middleware.async_subagents`` + module-level functions in place. The originals are captured at conftest + load time (above) so this fixture can always start each test from a + known-unpatched state regardless of what other tests / agent fixtures + did to the module before. + """ + from EvoScientist.llm import patches as patches_mod + + if _ds_async_subagents is None: + # deepagents not importable — fixture is a no-op (the patch fn itself + # returns early in that case). + yield + return + + def _reset() -> None: + _ds_async_subagents._build_start_tool = _DEEPAGENTS_ORIGINAL_BUILD_START + _ds_async_subagents._build_update_tool = _DEEPAGENTS_ORIGINAL_BUILD_UPDATE + patches_mod._model_passthrough_patched = False + + _reset() + try: + yield + finally: + _reset() diff --git a/tests/test_ask_user.py b/tests/test_ask_user.py index 90050b2..1725fb3 100644 --- a/tests/test_ask_user.py +++ b/tests/test_ask_user.py @@ -439,6 +439,45 @@ def test_auto_mode_disables_ask_user_middleware( assert "AskUserMiddleware" not in type_names +@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[]) +@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._ensure_config") +def test_for_async_subagent_omits_ask_user_middleware( + mock_config, mock_model, mock_tool_selector +): + """``AskUserMiddleware`` uses ``interrupt()`` to wait on user input. + + Async sub-agents run in the langgraph dev subprocess where the parent + only holds a ``task_id`` and has no UI path to surface or resume an + interrupt. Including ``AskUserMiddleware`` would deadlock the + sub-agent the first time the LLM calls ``ask_user``. The + ``for_async_subagent=True`` flag must suppress it even when the user + has globally enabled ``ask_user``. + """ + cfg = MagicMock() + cfg.enable_ask_user = True + cfg.auto_approve = False + cfg.auto_mode = False + mock_config.return_value = cfg + mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000}) + + from EvoScientist.EvoScientist import _get_default_middleware + + # Sanity: with the default flag, ask_user IS present. + default_names = [type(m).__name__ for m in _get_default_middleware()] + assert "AskUserMiddleware" in default_names + + # With for_async_subagent=True, ask_user is suppressed. + async_names = [ + type(m).__name__ for m in _get_default_middleware(for_async_subagent=True) + ] + assert "AskUserMiddleware" not in async_names + # Other middleware must remain — only ask_user is filtered. + assert "ConfigurableModelMiddleware" in async_names + assert "ContextEditingMiddleware" in async_names + assert "ModelFallbackMiddleware" in async_names + + # --------------------------------------------------------------------------- # Rich CLI prompt (mocking input) # --------------------------------------------------------------------------- diff --git a/tests/test_async_subagent_factory.py b/tests/test_async_subagent_factory.py new file mode 100644 index 0000000..62d23ea --- /dev/null +++ b/tests/test_async_subagent_factory.py @@ -0,0 +1,119 @@ +"""Tests for ``EvoScientist.subagents._factory.build_async_subagent_graph``. + +Pins the integration contract that the factory must request middleware +in async-safe mode (``for_async_subagent=True``). Without this, a future +refactor that drops the keyword argument would silently re-introduce +``AskUserMiddleware`` into the deployed graph and reproduce the +``interrupt()``-based deadlock the flag was added to prevent. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + + +@patch("deepagents.create_deep_agent") +@patch("EvoScientist.EvoScientist._load_mcp_tools_cached", return_value={}) +@patch("EvoScientist.EvoScientist._get_default_middleware", return_value=[]) +@patch("EvoScientist.EvoScientist._get_default_backend") +@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._build_prompt_refs", return_value={}) +@patch("EvoScientist.utils.load_subagents") +@patch("EvoScientist.config.apply_config_to_env") +@patch("EvoScientist.config.get_effective_config") +def test_factory_requests_async_safe_middleware( + mock_get_cfg, + mock_apply_env, + mock_load_subs, + mock_prompt_refs, + mock_chat, + mock_backend, + mock_get_mw, + mock_mcp, + mock_create, +): + """``build_async_subagent_graph`` must call ``_get_default_middleware`` + with ``for_async_subagent=True``. + + The bare argument call would silently include ``AskUserMiddleware`` in + the deployed graph, which deadlocks via ``interrupt()`` (no UI in the + langgraph dev subprocess to resume the interrupt). + """ + # Minimal config stub so factory's `cfg.recursion_limit` access works. + cfg = MagicMock() + cfg.recursion_limit = 1_000_000 + mock_get_cfg.return_value = cfg + # Factory looks up the requested name in the loaded subagent specs; + # any matching name is fine. + mock_load_subs.return_value = [ + { + "name": "writing-agent", + "system_prompt": "", + "tools": [], + "skills": None, + } + ] + # ``create_deep_agent(...).with_config({...})`` chain — return something + # chainable so the factory's terminal ``.with_config(...)`` doesn't blow up. + mock_create.return_value.with_config.return_value = MagicMock() + + from EvoScientist.subagents._factory import build_async_subagent_graph + + build_async_subagent_graph("writing-agent") + + # The contract: factory MUST pass ``for_async_subagent=True``. + mock_get_mw.assert_called_once_with(for_async_subagent=True) + + +# --------------------------------------------------------------------------- +# Direct behavior test for ``_get_default_middleware`` filter +# --------------------------------------------------------------------------- +# +# The factory test above pins the *contract* (factory passes the flag). +# This test pins the *behavior* (the flag actually excludes +# AskUserMiddleware), so a future refactor that renames the flag or +# restructures the middleware list cannot silently re-introduce the +# interrupt-based deadlock. + + +@patch( + "EvoScientist.middleware.create_tool_selector_middleware", + return_value=[MagicMock()], +) +@patch("EvoScientist.EvoScientist._ensure_chat_model") +@patch("EvoScientist.EvoScientist._ensure_config") +def test_async_subagent_mode_filters_ask_user( + mock_config, mock_chat, mock_tool_selector +): + """``_get_default_middleware(for_async_subagent=True)`` must drop + ``AskUserMiddleware`` even when ``enable_ask_user`` is on. + + Without mocking the middleware list itself: we let the real list be + constructed and assert ``AskUserMiddleware`` is absent. Mocks here + cover only the heavy dependencies (chat model, tool-selector) that + the middleware list builder pulls in transitively. + """ + cfg = MagicMock() + cfg.enable_ask_user = True # would normally include AskUserMiddleware + cfg.auto_mode = False + cfg.auto_approve = False + cfg.model_fallbacks = None + mock_config.return_value = cfg + mock_chat.return_value = MagicMock(profile={"max_input_tokens": 200_000}) + + from EvoScientist.EvoScientist import _get_default_middleware + from EvoScientist.middleware.ask_user import AskUserMiddleware + + # CLI / in-process path includes AskUserMiddleware … + cli_mw = _get_default_middleware() + assert any(isinstance(m, AskUserMiddleware) for m in cli_mw), ( + "Sanity check: with enable_ask_user=True and CLI mode, " + "AskUserMiddleware should be present." + ) + + # … but the async-subagent path filters it out. + async_mw = _get_default_middleware(for_async_subagent=True) + assert not any(isinstance(m, AskUserMiddleware) for m in async_mw), ( + "AskUserMiddleware leaked into async sub-agent middleware — its " + "interrupt() call deadlocks the deployed graph (no UI to resume)." + ) diff --git a/tests/test_configurable_model_middleware.py b/tests/test_configurable_model_middleware.py new file mode 100644 index 0000000..e9a688e --- /dev/null +++ b/tests/test_configurable_model_middleware.py @@ -0,0 +1,394 @@ +"""Tests for ``EvoScientist.middleware.configurable_model``. + +Verifies that the middleware reads ``model`` / ``model_provider`` from +the active ``RunnableConfig.configurable`` (via ``langgraph.config.get_config``) +and overrides ``request.model`` accordingly, without breaking the no-override +pass-through path or the per-instance cache. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +from EvoScientist.middleware.configurable_model import ( + ConfigurableModelMiddleware, + _read_model_override, +) +from tests.conftest import run_async as _run + + +@contextmanager +def _patched_config(configurable: dict | object | None): + """Patch ``langgraph.config.get_config`` to return a controlled value. + + Pass ``None`` to simulate "outside a runnable context" (raises + ``RuntimeError`` like the real ``get_config()`` does). + Pass a dict for ``configurable`` to expose just that key. + Pass any other object to simulate a malformed config. + """ + import langgraph.config as _lg_cfg + + if configurable is None: + # Simulate get_config raising outside a runnable context. + with patch.object( + _lg_cfg, + "get_config", + side_effect=RuntimeError("Called get_config outside of a runnable context"), + ): + yield + elif isinstance(configurable, dict): + with patch.object( + _lg_cfg, + "get_config", + return_value={"configurable": configurable}, + ): + yield + else: + with patch.object( + _lg_cfg, + "get_config", + return_value=configurable, + ): + yield + + +def _make_request(): + """Build a minimal ``ModelRequest`` stub. + + ``request.override(model=...)`` returns a new request whose ``model`` + field reflects the override. + """ + req = MagicMock() + + def _override(**kwargs): + new = MagicMock() + new.model = kwargs.get("model", req.model) + return new + + req.override = MagicMock(side_effect=_override) + return req + + +# ============================================================================= +# 1. _read_model_override — input parsing +# ============================================================================= + + +class TestReadModelOverride: + """Verify the helper that pulls (model, provider) from active config.""" + + def test_returns_override_when_both_present(self): + with _patched_config({"model": "gpt-5", "model_provider": "openai"}): + assert _read_model_override() == ("gpt-5", "openai") + + def test_provider_optional(self): + with _patched_config({"model": "claude-haiku-4-5"}): + assert _read_model_override() == ("claude-haiku-4-5", None) + + def test_no_configurable_key(self): + with _patched_config({}): + assert _read_model_override() == (None, None) + + def test_outside_runnable_context(self): + """``get_config`` raises outside a runnable — middleware must no-op.""" + with _patched_config(None): + assert _read_model_override() == (None, None) + + def test_empty_string_treated_as_absent(self): + with _patched_config({"model": "", "model_provider": ""}): + assert _read_model_override() == (None, None) + + def test_non_string_ignored(self): + with _patched_config({"model": 42, "model_provider": object()}): + assert _read_model_override() == (None, None) + + def test_non_dict_configurable_safe(self): + with _patched_config({"configurable": "not-a-dict"}): + # Inner ``configurable`` is the wrong type — patched_config + # wraps it again so we end up with {"configurable": {"configurable": "..."}} + # which has no model/model_provider keys → no override. + assert _read_model_override() == (None, None) + + def test_non_dict_config_safe(self): + with _patched_config("garbage"): + assert _read_model_override() == (None, None) + + +# ============================================================================= +# 2. ConfigurableModelMiddleware — pass-through behavior +# ============================================================================= + + +class TestPassThrough: + """When no override is present, the middleware must not touch the request.""" + + def test_sync_no_override_passes_request_unchanged(self): + mw = ConfigurableModelMiddleware() + req = _make_request() + sentinel = object() + handler = MagicMock(return_value=sentinel) + with _patched_config({}): + result = mw.wrap_model_call(req, handler) + assert result is sentinel + handler.assert_called_once_with(req) + req.override.assert_not_called() + + def test_async_no_override_passes_request_unchanged(self): + mw = ConfigurableModelMiddleware() + req = _make_request() + + async def handler(r): + assert r is req + return "ok" + + with _patched_config({}): + result = _run(mw.awrap_model_call(req, handler)) + assert result == "ok" + req.override.assert_not_called() + + def test_outside_runnable_context_passes_through(self): + mw = ConfigurableModelMiddleware() + req = _make_request() + handler = MagicMock(return_value="ok") + with _patched_config(None): + mw.wrap_model_call(req, handler) + handler.assert_called_once_with(req) + + +# ============================================================================= +# 3. ConfigurableModelMiddleware — override behavior +# ============================================================================= + + +class TestModelOverride: + """When override present, middleware resolves model and overrides request.""" + + def test_sync_override_calls_get_chat_model_and_overrides(self): + mw = ConfigurableModelMiddleware() + req = _make_request() + new_model = MagicMock(name="resolved_chat_model") + handler = MagicMock(return_value="response") + + with ( + _patched_config({"model": "gpt-5", "model_provider": "openai"}), + patch( + "EvoScientist.llm.get_chat_model", return_value=new_model + ) as mock_get, + ): + mw.wrap_model_call(req, handler) + + mock_get.assert_called_once_with(model="gpt-5", provider="openai") + req.override.assert_called_once_with(model=new_model) + # Handler must receive the OVERRIDDEN request, not the original. + called_with = handler.call_args[0][0] + assert called_with is not req + assert called_with.model is new_model + + def test_async_override_path_parity(self): + mw = ConfigurableModelMiddleware() + req = _make_request() + new_model = MagicMock() + + async def handler(r): + assert r.model is new_model + return "ok" + + with ( + _patched_config( + {"model": "claude-opus-4-7", "model_provider": "anthropic"} + ), + patch( + "EvoScientist.llm.get_chat_model", return_value=new_model + ) as mock_get, + ): + result = _run(mw.awrap_model_call(req, handler)) + + assert result == "ok" + mock_get.assert_called_once_with(model="claude-opus-4-7", provider="anthropic") + + def test_provider_omitted_passed_as_none(self): + mw = ConfigurableModelMiddleware() + req = _make_request() + new_model = MagicMock() + handler = MagicMock(return_value="ok") + + with ( + _patched_config({"model": "gpt-5"}), + patch( + "EvoScientist.llm.get_chat_model", return_value=new_model + ) as mock_get, + ): + mw.wrap_model_call(req, handler) + + mock_get.assert_called_once_with(model="gpt-5", provider=None) + + +# ============================================================================= +# 4. ConfigurableModelMiddleware — caching +# ============================================================================= + + +class TestCache: + """Two consecutive calls with same (model, provider) should hit cache.""" + + def test_cache_hit_avoids_second_get_chat_model(self): + mw = ConfigurableModelMiddleware() + req1 = _make_request() + req2 = _make_request() + new_model = MagicMock() + handler = MagicMock(return_value="ok") + + with ( + _patched_config({"model": "gpt-5", "model_provider": "openai"}), + patch( + "EvoScientist.llm.get_chat_model", return_value=new_model + ) as mock_get, + ): + mw.wrap_model_call(req1, handler) + mw.wrap_model_call(req2, handler) + + # First call resolves via factory, second hits the cache. + assert mock_get.call_count == 1 + + def test_cache_miss_on_different_provider(self): + mw = ConfigurableModelMiddleware() + req = _make_request() + handler = MagicMock(return_value="ok") + + with patch( + "EvoScientist.llm.get_chat_model", side_effect=[MagicMock(), MagicMock()] + ) as mock_get: + with _patched_config( + {"model": "claude-sonnet-4-6", "model_provider": "anthropic"} + ): + mw.wrap_model_call(req, handler) + with _patched_config( + {"model": "claude-sonnet-4-6", "model_provider": "custom-anthropic"} + ): + mw.wrap_model_call(req, handler) + + assert mock_get.call_count == 2 + + def test_independent_instances_have_independent_caches(self): + mw_a = ConfigurableModelMiddleware() + mw_b = ConfigurableModelMiddleware() + req = _make_request() + handler = MagicMock(return_value="ok") + + with ( + _patched_config({"model": "gpt-5", "model_provider": "openai"}), + patch( + "EvoScientist.llm.get_chat_model", + side_effect=[MagicMock(), MagicMock()], + ) as mock_get, + ): + mw_a.wrap_model_call(req, handler) + mw_b.wrap_model_call(req, handler) + + # Different instances must each resolve once. + assert mock_get.call_count == 2 + + +# ============================================================================= +# 5. Resilience — get_chat_model raising +# ============================================================================= + + +class TestResolveFailure: + """If get_chat_model raises, middleware must fall back to original model.""" + + def test_sync_falls_back_when_resolve_raises(self): + mw = ConfigurableModelMiddleware() + req = _make_request() + handler = MagicMock(return_value="ok") + + with ( + _patched_config({"model": "doesnotexist", "model_provider": "openai"}), + patch( + "EvoScientist.llm.get_chat_model", + side_effect=ValueError("unknown model"), + ), + ): + mw.wrap_model_call(req, handler) + + # Override never happened — handler called with original request. + handler.assert_called_once_with(req) + req.override.assert_not_called() + + def test_async_falls_back_when_resolve_raises(self): + mw = ConfigurableModelMiddleware() + req = _make_request() + + called = [] + + async def handler(r): + called.append(r) + return "ok" + + with ( + _patched_config({"model": "doesnotexist", "model_provider": "openai"}), + patch( + "EvoScientist.llm.get_chat_model", + side_effect=ValueError("unknown model"), + ), + ): + result = _run(mw.awrap_model_call(req, handler)) + + assert result == "ok" + assert called == [req] + + +# ============================================================================= +# 6. Integration — real langgraph contextvar (no get_config mock) +# ============================================================================= + + +class TestRunnableContextVarIntegration: + """Set the actual ``var_child_runnable_config`` contextvar that LangGraph + populates per node, then verify the middleware reads through to it. + + This catches breakage of the ``langgraph.config.get_config()`` contract + that pure-mock tests would miss (e.g. if get_config is moved to a + different module, or the contextvar mechanism changes). + """ + + def test_real_contextvar_drives_override(self): + """Without mocking get_config, set the contextvar and verify override.""" + from langchain_core.runnables.config import var_child_runnable_config + + mw = ConfigurableModelMiddleware() + req = _make_request() + new_model = MagicMock(name="resolved") + handler = MagicMock(return_value="ok") + + token = var_child_runnable_config.set( + {"configurable": {"model": "gpt-5.5", "model_provider": "openai"}} + ) + try: + with patch( + "EvoScientist.llm.get_chat_model", return_value=new_model + ) as mock_get: + mw.wrap_model_call(req, handler) + finally: + var_child_runnable_config.reset(token) + + mock_get.assert_called_once_with(model="gpt-5.5", provider="openai") + req.override.assert_called_once_with(model=new_model) + + def test_real_contextvar_unset_passes_through(self): + """When no contextvar is set, get_config() raises → no override.""" + from langchain_core.runnables.config import var_child_runnable_config + + # Defensive: ensure no leftover contextvar from another test. + token = var_child_runnable_config.set(None) + try: + mw = ConfigurableModelMiddleware() + req = _make_request() + handler = MagicMock(return_value="ok") + mw.wrap_model_call(req, handler) + finally: + var_child_runnable_config.reset(token) + + handler.assert_called_once_with(req) + req.override.assert_not_called() diff --git a/tests/test_context_editing_middleware.py b/tests/test_context_editing_middleware.py index 8779473..929b4ed 100644 --- a/tests/test_context_editing_middleware.py +++ b/tests/test_context_editing_middleware.py @@ -119,7 +119,9 @@ def test_default_middleware_includes_context_editing(mock_config, mock_model, mo from EvoScientist.EvoScientist import _get_default_middleware mw = _get_default_middleware() - assert isinstance(mw[0], ContextEditingMiddleware) + # ContextEditingMiddleware is present (its absolute position depends on + # other leading middlewares like ConfigurableModelMiddleware). + assert any(isinstance(m, ContextEditingMiddleware) for m in mw) @patch("EvoScientist.EvoScientist._ensure_chat_model") diff --git a/tests/test_model_passthrough_patch.py b/tests/test_model_passthrough_patch.py new file mode 100644 index 0000000..f177ea2 --- /dev/null +++ b/tests/test_model_passthrough_patch.py @@ -0,0 +1,514 @@ +"""Tests for the deepagents model-passthrough patch. + +Verifies that ``_patch_deepagents_model_passthrough`` wraps +``_build_start_tool`` / ``_build_update_tool`` so that ``client.runs.create`` +calls inside the launched async-task tools carry +``config={"configurable": {"model": ..., "model_provider": ...}}``, +without affecting other client methods (``threads.create``, ``runs.get``, +``runs.cancel``). +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from EvoScientist.llm import patches as patches_mod +from tests.conftest import run_async as _run + +# ============================================================================= +# Helpers +# ============================================================================= + + +def _stub_cfg(model: str = "claude-sonnet-4-6", provider: str = "anthropic"): + """Build a stand-in for ``_ensure_config()`` return value.""" + return SimpleNamespace(model=model, provider=provider) + + +def _make_client_cache( + *, + create_run_id: str = "run-001", + thread_id: str = "thread-001", +): + """Build a fake ``_ClientCache`` with sync + async stub clients. + + Returns ``(cache_mock, runs_create_sync_mock, runs_create_async_mock)`` + so tests can both invoke through the patched factory and inspect the + exact call kwargs passed to ``runs.create``. + """ + runs_sync = MagicMock() + runs_sync.create.return_value = {"run_id": create_run_id} + runs_sync.cancel = MagicMock(return_value=None) + runs_sync.get = MagicMock(return_value={"status": "success"}) + + threads_sync = MagicMock() + threads_sync.create.return_value = {"thread_id": thread_id} + + sync_client = MagicMock() + sync_client.runs = runs_sync + sync_client.threads = threads_sync + + runs_async = MagicMock() + runs_async.create = AsyncMock(return_value={"run_id": create_run_id}) + + threads_async = MagicMock() + threads_async.create = AsyncMock(return_value={"thread_id": thread_id}) + + async_client = MagicMock() + async_client.runs = runs_async + async_client.threads = threads_async + + cache = MagicMock() + cache.get_sync = MagicMock(return_value=sync_client) + cache.get_async = MagicMock(return_value=async_client) + + return cache, runs_sync, runs_async + + +def _runtime_stub(): + """Minimal stand-in for ``ToolRuntime`` accepted by the inner tools.""" + return SimpleNamespace(tool_call_id="tc-001", state={}) + + +# ============================================================================= +# 1. Idempotence +# ============================================================================= + + +class TestIdempotence: + """The patch must be a no-op after the first application.""" + + def test_double_apply_does_not_re_wrap(self, restore_model_passthrough_patch): + try: + from deepagents.middleware import async_subagents as ds_mod + except ImportError: + pytest.skip("deepagents not available") + + original = ds_mod._build_start_tool + patches_mod._patch_deepagents_model_passthrough() + wrapped_once = ds_mod._build_start_tool + assert wrapped_once is not original + + patches_mod._patch_deepagents_model_passthrough() + wrapped_twice = ds_mod._build_start_tool + assert wrapped_twice is wrapped_once # not double-wrapped + + def test_flag_set_after_apply(self, restore_model_passthrough_patch): + try: + from deepagents.middleware import async_subagents as _ # noqa: F401 + except ImportError: + pytest.skip("deepagents not available") + patches_mod._model_passthrough_patched = False + patches_mod._patch_deepagents_model_passthrough() + assert patches_mod._model_passthrough_patched is True + + +# ============================================================================= +# 2. start_async_task injects config into runs.create +# ============================================================================= + + +class TestStartAsyncTaskInjection: + """Sync and async start_async_task must inject configurable.model.""" + + def test_sync_start_injects_config(self, restore_model_passthrough_patch): + try: + from deepagents.middleware import async_subagents as ds_mod + except ImportError: + pytest.skip("deepagents not available") + + patches_mod._patch_deepagents_model_passthrough() + + cache, runs_sync, _ = _make_client_cache() + agent_map = { + "writing-agent": { + "name": "writing-agent", + "description": "Draft paper", + "graph_id": "writing-agent", + "url": "http://localhost:6174", + } + } + + tool = ds_mod._build_start_tool(agent_map, cache, "desc") + + with patch( + "EvoScientist.EvoScientist._ensure_config", + return_value=_stub_cfg(model="gpt-5", provider="openai"), + ): + tool.func( + description="hello", + subagent_type="writing-agent", + runtime=_runtime_stub(), + ) + + runs_sync.create.assert_called_once() + kwargs = runs_sync.create.call_args.kwargs + assert kwargs["thread_id"] == "thread-001" + assert kwargs["assistant_id"] == "writing-agent" + assert kwargs["config"] == { + "configurable": {"model": "gpt-5", "model_provider": "openai"} + } + + def test_async_start_injects_config(self, restore_model_passthrough_patch): + try: + from deepagents.middleware import async_subagents as ds_mod + except ImportError: + pytest.skip("deepagents not available") + + patches_mod._patch_deepagents_model_passthrough() + + cache, _, runs_async = _make_client_cache() + agent_map = { + "writing-agent": { + "name": "writing-agent", + "description": "Draft paper", + "graph_id": "writing-agent", + "url": "http://localhost:6174", + } + } + + tool = ds_mod._build_start_tool(agent_map, cache, "desc") + + with patch( + "EvoScientist.EvoScientist._ensure_config", + return_value=_stub_cfg(model="claude-haiku-4-5", provider="anthropic"), + ): + _run( + tool.coroutine( + description="hi", + subagent_type="writing-agent", + runtime=_runtime_stub(), + ) + ) + + runs_async.create.assert_awaited_once() + kwargs = runs_async.create.call_args.kwargs + assert kwargs["config"] == { + "configurable": { + "model": "claude-haiku-4-5", + "model_provider": "anthropic", + } + } + + +# ============================================================================= +# 3. Live config read at tool-call time (post-/model switch behavior) +# ============================================================================= + + +class TestLiveConfigRead: + """The patch must read cfg fresh on every tool call, not at patch time.""" + + def test_two_calls_reflect_separate_cfg(self, restore_model_passthrough_patch): + try: + from deepagents.middleware import async_subagents as ds_mod + except ImportError: + pytest.skip("deepagents not available") + + patches_mod._patch_deepagents_model_passthrough() + + cache, runs_sync, _ = _make_client_cache() + agent_map = { + "writing-agent": { + "name": "writing-agent", + "description": "x", + "graph_id": "writing-agent", + "url": "http://localhost:6174", + } + } + tool = ds_mod._build_start_tool(agent_map, cache, "desc") + + with patch( + "EvoScientist.EvoScientist._ensure_config", + return_value=_stub_cfg(model="model-a", provider="anthropic"), + ): + tool.func( + description="t1", + subagent_type="writing-agent", + runtime=_runtime_stub(), + ) + first_kwargs = runs_sync.create.call_args.kwargs + + with patch( + "EvoScientist.EvoScientist._ensure_config", + return_value=_stub_cfg(model="model-b", provider="openai"), + ): + tool.func( + description="t2", + subagent_type="writing-agent", + runtime=_runtime_stub(), + ) + second_kwargs = runs_sync.create.call_args.kwargs + + assert first_kwargs["config"]["configurable"]["model"] == "model-a" + assert second_kwargs["config"]["configurable"]["model"] == "model-b" + + +# ============================================================================= +# 4. update_async_task also injects config +# ============================================================================= + + +class TestUpdateAsyncTaskInjection: + """update_async_task must inject config too — not just start.""" + + def _tracked_task(self, agent_name: str = "writing-agent") -> dict: + return { + "task_id": "thread-001", + "agent_name": agent_name, + "thread_id": "thread-001", + "run_id": "old-run", + "status": "running", + "created_at": "2026-05-07T00:00:00Z", + "last_checked_at": "2026-05-07T00:00:00Z", + "last_updated_at": "2026-05-07T00:00:00Z", + } + + def test_async_update_injects_config(self, restore_model_passthrough_patch): + """The async coroutine path must inject config too.""" + try: + from deepagents.middleware import async_subagents as ds_mod + except ImportError: + pytest.skip("deepagents not available") + + patches_mod._patch_deepagents_model_passthrough() + + cache, _, runs_async = _make_client_cache() + agent_map = { + "writing-agent": { + "name": "writing-agent", + "description": "x", + "graph_id": "writing-agent", + "url": "http://localhost:6174", + } + } + runtime = SimpleNamespace( + tool_call_id="tc-002", + state={"async_tasks": {"thread-001": self._tracked_task()}}, + ) + + tool = ds_mod._build_update_tool(agent_map, cache) + + with patch( + "EvoScientist.EvoScientist._ensure_config", + return_value=_stub_cfg(model="gpt-5", provider="openai"), + ): + _run( + tool.coroutine( + task_id="thread-001", + message="follow up async", + runtime=runtime, + ) + ) + + runs_async.create.assert_awaited_once() + kwargs = runs_async.create.call_args.kwargs + assert kwargs["config"] == { + "configurable": {"model": "gpt-5", "model_provider": "openai"} + } + assert kwargs.get("multitask_strategy") == "interrupt" + + def test_sync_update_injects_config(self, restore_model_passthrough_patch): + try: + from deepagents.middleware import async_subagents as ds_mod + except ImportError: + pytest.skip("deepagents not available") + + patches_mod._patch_deepagents_model_passthrough() + + cache, runs_sync, _ = _make_client_cache() + agent_map = { + "writing-agent": { + "name": "writing-agent", + "description": "x", + "graph_id": "writing-agent", + "url": "http://localhost:6174", + } + } + # update_async_task reads tracked task from runtime.state + tracked_task = { + "task_id": "thread-001", + "agent_name": "writing-agent", + "thread_id": "thread-001", + "run_id": "old-run", + "status": "running", + "created_at": "2026-05-07T00:00:00Z", + "last_checked_at": "2026-05-07T00:00:00Z", + "last_updated_at": "2026-05-07T00:00:00Z", + } + runtime = SimpleNamespace( + tool_call_id="tc-002", + state={"async_tasks": {"thread-001": tracked_task}}, + ) + + tool = ds_mod._build_update_tool(agent_map, cache) + + with patch( + "EvoScientist.EvoScientist._ensure_config", + return_value=_stub_cfg(model="gpt-5", provider="openai"), + ): + tool.func( + task_id="thread-001", + message="follow up", + runtime=runtime, + ) + + runs_sync.create.assert_called_once() + kwargs = runs_sync.create.call_args.kwargs + assert kwargs["config"]["configurable"]["model"] == "gpt-5" + assert kwargs["config"]["configurable"]["model_provider"] == "openai" + # update preserves multitask_strategy="interrupt" — verify no regression + assert kwargs.get("multitask_strategy") == "interrupt" + + +# ============================================================================= +# 5. Other client methods are unaffected +# ============================================================================= + + +class TestNonInterceptedMethods: + """``threads.create``, ``runs.get``, ``runs.cancel`` must pass through.""" + + def test_threads_create_not_modified(self, restore_model_passthrough_patch): + """``threads.create()`` is called by start_async_task pre-runs.create. + + The patch should not inject config here, because thread creation + doesn't take config (and would error out if we did). + """ + try: + from deepagents.middleware import async_subagents as ds_mod + except ImportError: + pytest.skip("deepagents not available") + + patches_mod._patch_deepagents_model_passthrough() + + cache, _runs_sync, _ = _make_client_cache() + threads_create = cache.get_sync.return_value.threads.create + agent_map = { + "writing-agent": { + "name": "writing-agent", + "description": "x", + "graph_id": "writing-agent", + "url": "http://localhost:6174", + } + } + tool = ds_mod._build_start_tool(agent_map, cache, "desc") + + with patch( + "EvoScientist.EvoScientist._ensure_config", + return_value=_stub_cfg(), + ): + tool.func( + description="hi", + subagent_type="writing-agent", + runtime=_runtime_stub(), + ) + + # threads.create() is called with no kwargs (deepagents pattern). + threads_create.assert_called_once_with() + + +# ============================================================================= +# 6. Empty cfg → no config kwarg added +# ============================================================================= + + +class TestEmptyCfg: + """If neither model nor provider is set, don't inject anything.""" + + def test_empty_cfg_no_config_kwarg(self, restore_model_passthrough_patch): + try: + from deepagents.middleware import async_subagents as ds_mod + except ImportError: + pytest.skip("deepagents not available") + + patches_mod._patch_deepagents_model_passthrough() + + cache, runs_sync, _ = _make_client_cache() + agent_map = { + "writing-agent": { + "name": "writing-agent", + "description": "x", + "graph_id": "writing-agent", + "url": "http://localhost:6174", + } + } + tool = ds_mod._build_start_tool(agent_map, cache, "desc") + + with patch( + "EvoScientist.EvoScientist._ensure_config", + return_value=SimpleNamespace(model=None, provider=None), + ): + tool.func( + description="hi", + subagent_type="writing-agent", + runtime=_runtime_stub(), + ) + + runs_sync.create.assert_called_once() + kwargs = runs_sync.create.call_args.kwargs + # No config kwarg should be added when there's nothing to override. + assert "config" not in kwargs + + +# ============================================================================= +# 7. Caller-supplied config keys are preserved +# ============================================================================= + + +class TestPreserveExistingConfig: + """If a caller already supplied config.configurable.X, our merge keeps it.""" + + def test_existing_configurable_preserved(self): + """Direct unit test of the merge helper (integration covered above).""" + with patch( + "EvoScientist.EvoScientist._ensure_config", + return_value=_stub_cfg(model="gpt-5", provider="openai"), + ): + merged = patches_mod._merge_runs_config_kwargs( + { + "thread_id": "t1", + "config": { + "configurable": {"thread_id": "outer-t", "extra": 42}, + "tags": ["debug"], + }, + } + ) + assert merged["thread_id"] == "t1" + assert merged["config"]["tags"] == ["debug"] + assert merged["config"]["configurable"]["thread_id"] == "outer-t" + assert merged["config"]["configurable"]["extra"] == 42 + assert merged["config"]["configurable"]["model"] == "gpt-5" + assert merged["config"]["configurable"]["model_provider"] == "openai" + + def test_non_dict_config_replaced(self): + """Non-dict ``config`` (e.g. a Pydantic RunnableConfig) is replaced. + + Documents the policy: callers that pass a non-dict ``config`` lose + any other fields they may have set there. Acceptable today because + deepagents' built-in ``runs.create`` path doesn't pass a config at + all, but a future caller passing e.g. a Pydantic model would have + their non-configurable fields silently dropped. If that becomes a + real use case, ``_merge_runs_config_kwargs`` should grow a + ``dict()`` coercion or raise. + """ + + class _Sentinel: + """Stand-in for any non-dict config-shaped object.""" + + with patch( + "EvoScientist.EvoScientist._ensure_config", + return_value=_stub_cfg(model="gpt-5", provider="openai"), + ): + merged = patches_mod._merge_runs_config_kwargs( + {"thread_id": "t1", "config": _Sentinel()} + ) + assert merged["thread_id"] == "t1" + # Non-dict input was replaced with a fresh dict carrying only our + # injected keys. + assert merged["config"] == { + "configurable": {"model": "gpt-5", "model_provider": "openai"} + }