jfilipiuk
0410b40f57
* feat: inherit the caller's model for async sub-agent launch and update * docs: tighten middleware related docstrings
578 lines
21 KiB
Python
578 lines
21 KiB
Python
"""Tests for the deepagents model-passthrough patch.
|
|
|
|
Verifies that ``_patch_deepagents_model_passthrough`` wraps
|
|
``_build_start_tool`` / ``_build_update_tool`` so that ``client.runs.create``
|
|
calls inside the launched async-task tools carry
|
|
``config={"configurable": {"model": ..., "model_provider": ...}}``,
|
|
without affecting other client methods (``threads.create``, ``runs.get``,
|
|
``runs.cancel``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from EvoScientist.llm import patches as patches_mod
|
|
|
|
# =============================================================================
|
|
# Helpers
|
|
# =============================================================================
|
|
|
|
|
|
def _stub_cfg(model: str = "claude-sonnet-4-6", provider: str = "anthropic"):
|
|
"""Build a stand-in for ``_ensure_config()`` return value."""
|
|
return SimpleNamespace(model=model, provider=provider)
|
|
|
|
|
|
def _make_client_cache(
|
|
*,
|
|
create_run_id: str = "run-001",
|
|
thread_id: str = "thread-001",
|
|
):
|
|
"""Build a fake ``_ClientCache`` with sync + async stub clients.
|
|
|
|
Returns ``(cache_mock, runs_create_sync_mock, runs_create_async_mock)``
|
|
so tests can both invoke through the patched factory and inspect the
|
|
exact call kwargs passed to ``runs.create``.
|
|
"""
|
|
runs_sync = MagicMock()
|
|
runs_sync.create.return_value = {"run_id": create_run_id}
|
|
runs_sync.cancel = MagicMock(return_value=None)
|
|
runs_sync.get = MagicMock(return_value={"status": "success"})
|
|
|
|
threads_sync = MagicMock()
|
|
threads_sync.create.return_value = {"thread_id": thread_id}
|
|
|
|
sync_client = MagicMock()
|
|
sync_client.runs = runs_sync
|
|
sync_client.threads = threads_sync
|
|
|
|
runs_async = MagicMock()
|
|
runs_async.create = AsyncMock(return_value={"run_id": create_run_id})
|
|
|
|
threads_async = MagicMock()
|
|
threads_async.create = AsyncMock(return_value={"thread_id": thread_id})
|
|
|
|
async_client = MagicMock()
|
|
async_client.runs = runs_async
|
|
async_client.threads = threads_async
|
|
|
|
cache = MagicMock()
|
|
cache.get_sync = MagicMock(return_value=sync_client)
|
|
cache.get_async = MagicMock(return_value=async_client)
|
|
|
|
return cache, runs_sync, runs_async
|
|
|
|
|
|
def _runtime_stub():
|
|
"""Minimal stand-in for ``ToolRuntime`` accepted by the inner tools."""
|
|
return SimpleNamespace(tool_call_id="tc-001", state={})
|
|
|
|
|
|
# =============================================================================
|
|
# 1. Idempotence
|
|
# =============================================================================
|
|
|
|
|
|
class TestIdempotence:
|
|
"""The patch must be a no-op after the first application."""
|
|
|
|
def test_double_apply_does_not_re_wrap(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
original = ds_mod._build_start_tool
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
wrapped_once = ds_mod._build_start_tool
|
|
assert wrapped_once is not original
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
wrapped_twice = ds_mod._build_start_tool
|
|
assert wrapped_twice is wrapped_once # not double-wrapped
|
|
|
|
def test_flag_set_after_apply(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as _ # noqa: F401
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
patches_mod._model_passthrough_patched = False
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
assert patches_mod._model_passthrough_patched is True
|
|
|
|
|
|
# =============================================================================
|
|
# 2. start_async_task injects config into runs.create
|
|
# =============================================================================
|
|
|
|
|
|
class TestStartAsyncTaskInjection:
|
|
"""Sync and async start_async_task must inject configurable.model."""
|
|
|
|
def test_sync_start_injects_config(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, runs_sync, _ = _make_client_cache()
|
|
agent_map = {
|
|
"writing-agent": {
|
|
"name": "writing-agent",
|
|
"description": "Draft paper",
|
|
"graph_id": "writing-agent",
|
|
"url": "http://localhost:6174",
|
|
}
|
|
}
|
|
|
|
tool = ds_mod._build_start_tool(agent_map, cache, "desc")
|
|
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(model="gpt-5", provider="openai"),
|
|
):
|
|
tool.func(
|
|
description="hello",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
|
|
runs_sync.create.assert_called_once()
|
|
kwargs = runs_sync.create.call_args.kwargs
|
|
assert kwargs["thread_id"] == "thread-001"
|
|
assert kwargs["assistant_id"] == "writing-agent"
|
|
assert kwargs["config"] == {
|
|
"configurable": {"model": "gpt-5", "model_provider": "openai"}
|
|
}
|
|
|
|
async def test_async_start_injects_config(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, _, runs_async = _make_client_cache()
|
|
agent_map = {
|
|
"writing-agent": {
|
|
"name": "writing-agent",
|
|
"description": "Draft paper",
|
|
"graph_id": "writing-agent",
|
|
"url": "http://localhost:6174",
|
|
}
|
|
}
|
|
|
|
tool = ds_mod._build_start_tool(agent_map, cache, "desc")
|
|
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(model="claude-haiku-4-5", provider="anthropic"),
|
|
):
|
|
await tool.coroutine(
|
|
description="hi",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
|
|
runs_async.create.assert_awaited_once()
|
|
kwargs = runs_async.create.call_args.kwargs
|
|
assert kwargs["config"] == {
|
|
"configurable": {
|
|
"model": "claude-haiku-4-5",
|
|
"model_provider": "anthropic",
|
|
}
|
|
}
|
|
|
|
|
|
# =============================================================================
|
|
# 3. Live config read at tool-call time (post-/model switch behavior)
|
|
# =============================================================================
|
|
|
|
|
|
class TestLiveConfigRead:
|
|
"""The patch must read cfg fresh on every tool call, not at patch time."""
|
|
|
|
def test_two_calls_reflect_separate_cfg(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, runs_sync, _ = _make_client_cache()
|
|
agent_map = {
|
|
"writing-agent": {
|
|
"name": "writing-agent",
|
|
"description": "x",
|
|
"graph_id": "writing-agent",
|
|
"url": "http://localhost:6174",
|
|
}
|
|
}
|
|
tool = ds_mod._build_start_tool(agent_map, cache, "desc")
|
|
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(model="model-a", provider="anthropic"),
|
|
):
|
|
tool.func(
|
|
description="t1",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
first_kwargs = runs_sync.create.call_args.kwargs
|
|
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(model="model-b", provider="openai"),
|
|
):
|
|
tool.func(
|
|
description="t2",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
second_kwargs = runs_sync.create.call_args.kwargs
|
|
|
|
assert first_kwargs["config"]["configurable"]["model"] == "model-a"
|
|
assert second_kwargs["config"]["configurable"]["model"] == "model-b"
|
|
|
|
|
|
# =============================================================================
|
|
# 4. update_async_task also injects config
|
|
# =============================================================================
|
|
|
|
|
|
class TestUpdateAsyncTaskInjection:
|
|
"""update_async_task must inject config too — not just start."""
|
|
|
|
def _tracked_task(self, agent_name: str = "writing-agent") -> dict:
|
|
return {
|
|
"task_id": "thread-001",
|
|
"agent_name": agent_name,
|
|
"thread_id": "thread-001",
|
|
"run_id": "old-run",
|
|
"status": "running",
|
|
"created_at": "2026-05-07T00:00:00Z",
|
|
"last_checked_at": "2026-05-07T00:00:00Z",
|
|
"last_updated_at": "2026-05-07T00:00:00Z",
|
|
}
|
|
|
|
async def test_async_update_injects_config(self, restore_model_passthrough_patch):
|
|
"""The async coroutine path must inject config too."""
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, _, runs_async = _make_client_cache()
|
|
agent_map = {
|
|
"writing-agent": {
|
|
"name": "writing-agent",
|
|
"description": "x",
|
|
"graph_id": "writing-agent",
|
|
"url": "http://localhost:6174",
|
|
}
|
|
}
|
|
runtime = SimpleNamespace(
|
|
tool_call_id="tc-002",
|
|
state={"async_tasks": {"thread-001": self._tracked_task()}},
|
|
)
|
|
|
|
tool = ds_mod._build_update_tool(agent_map, cache)
|
|
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(model="gpt-5", provider="openai"),
|
|
):
|
|
await tool.coroutine(
|
|
task_id="thread-001",
|
|
message="follow up async",
|
|
runtime=runtime,
|
|
)
|
|
|
|
runs_async.create.assert_awaited_once()
|
|
kwargs = runs_async.create.call_args.kwargs
|
|
assert kwargs["config"] == {
|
|
"configurable": {"model": "gpt-5", "model_provider": "openai"}
|
|
}
|
|
assert kwargs.get("multitask_strategy") == "interrupt"
|
|
|
|
def test_sync_update_injects_config(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, runs_sync, _ = _make_client_cache()
|
|
agent_map = {
|
|
"writing-agent": {
|
|
"name": "writing-agent",
|
|
"description": "x",
|
|
"graph_id": "writing-agent",
|
|
"url": "http://localhost:6174",
|
|
}
|
|
}
|
|
# update_async_task reads tracked task from runtime.state
|
|
tracked_task = {
|
|
"task_id": "thread-001",
|
|
"agent_name": "writing-agent",
|
|
"thread_id": "thread-001",
|
|
"run_id": "old-run",
|
|
"status": "running",
|
|
"created_at": "2026-05-07T00:00:00Z",
|
|
"last_checked_at": "2026-05-07T00:00:00Z",
|
|
"last_updated_at": "2026-05-07T00:00:00Z",
|
|
}
|
|
runtime = SimpleNamespace(
|
|
tool_call_id="tc-002",
|
|
state={"async_tasks": {"thread-001": tracked_task}},
|
|
)
|
|
|
|
tool = ds_mod._build_update_tool(agent_map, cache)
|
|
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(model="gpt-5", provider="openai"),
|
|
):
|
|
tool.func(
|
|
task_id="thread-001",
|
|
message="follow up",
|
|
runtime=runtime,
|
|
)
|
|
|
|
runs_sync.create.assert_called_once()
|
|
kwargs = runs_sync.create.call_args.kwargs
|
|
assert kwargs["config"]["configurable"]["model"] == "gpt-5"
|
|
assert kwargs["config"]["configurable"]["model_provider"] == "openai"
|
|
# update preserves multitask_strategy="interrupt" — verify no regression
|
|
assert kwargs.get("multitask_strategy") == "interrupt"
|
|
|
|
|
|
# =============================================================================
|
|
# 5. Other client methods are unaffected
|
|
# =============================================================================
|
|
|
|
|
|
class TestNonInterceptedMethods:
|
|
"""``threads.create``, ``runs.get``, ``runs.cancel`` must pass through."""
|
|
|
|
def test_threads_create_not_modified(self, restore_model_passthrough_patch):
|
|
"""``threads.create()`` is called by start_async_task pre-runs.create.
|
|
|
|
The patch should not inject config here, because thread creation
|
|
doesn't take config (and would error out if we did).
|
|
"""
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, _runs_sync, _ = _make_client_cache()
|
|
threads_create = cache.get_sync.return_value.threads.create
|
|
agent_map = {
|
|
"writing-agent": {
|
|
"name": "writing-agent",
|
|
"description": "x",
|
|
"graph_id": "writing-agent",
|
|
"url": "http://localhost:6174",
|
|
}
|
|
}
|
|
tool = ds_mod._build_start_tool(agent_map, cache, "desc")
|
|
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(),
|
|
):
|
|
tool.func(
|
|
description="hi",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
|
|
# threads.create() is called with no kwargs (deepagents pattern).
|
|
threads_create.assert_called_once_with()
|
|
|
|
|
|
# =============================================================================
|
|
# 6. Empty cfg → no config kwarg added
|
|
# =============================================================================
|
|
|
|
|
|
class TestEmptyCfg:
|
|
"""If neither model nor provider is set, don't inject anything."""
|
|
|
|
def test_empty_cfg_no_config_kwarg(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, runs_sync, _ = _make_client_cache()
|
|
agent_map = {
|
|
"writing-agent": {
|
|
"name": "writing-agent",
|
|
"description": "x",
|
|
"graph_id": "writing-agent",
|
|
"url": "http://localhost:6174",
|
|
}
|
|
}
|
|
tool = ds_mod._build_start_tool(agent_map, cache, "desc")
|
|
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=SimpleNamespace(model=None, provider=None),
|
|
):
|
|
tool.func(
|
|
description="hi",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
|
|
runs_sync.create.assert_called_once()
|
|
kwargs = runs_sync.create.call_args.kwargs
|
|
# No config kwarg should be added when there's nothing to override.
|
|
assert "config" not in kwargs
|
|
|
|
|
|
# =============================================================================
|
|
# 7. Caller-supplied config keys are preserved
|
|
# =============================================================================
|
|
|
|
|
|
class TestPreserveExistingConfig:
|
|
"""If a caller already supplied config.configurable.X, our merge keeps it."""
|
|
|
|
def test_existing_configurable_preserved(self):
|
|
"""Direct unit test of the merge helper (integration covered above)."""
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(model="gpt-5", provider="openai"),
|
|
):
|
|
merged = patches_mod._merge_runs_config_kwargs(
|
|
{
|
|
"thread_id": "t1",
|
|
"config": {
|
|
"configurable": {"thread_id": "outer-t", "extra": 42},
|
|
"tags": ["debug"],
|
|
},
|
|
}
|
|
)
|
|
assert merged["thread_id"] == "t1"
|
|
assert merged["config"]["tags"] == ["debug"]
|
|
assert merged["config"]["configurable"]["thread_id"] == "outer-t"
|
|
assert merged["config"]["configurable"]["extra"] == 42
|
|
assert merged["config"]["configurable"]["model"] == "gpt-5"
|
|
assert merged["config"]["configurable"]["model_provider"] == "openai"
|
|
|
|
def test_non_dict_config_replaced(self):
|
|
"""Non-dict ``config`` (e.g. a Pydantic RunnableConfig) is replaced.
|
|
|
|
Documents the policy: callers that pass a non-dict ``config`` lose
|
|
any other fields they may have set there. Acceptable today because
|
|
deepagents' built-in ``runs.create`` path doesn't pass a config at
|
|
all, but a future caller passing e.g. a Pydantic model would have
|
|
their non-configurable fields silently dropped. If that becomes a
|
|
real use case, ``_merge_runs_config_kwargs`` should grow a
|
|
``dict()`` coercion or raise.
|
|
"""
|
|
|
|
class _Sentinel:
|
|
"""Stand-in for any non-dict config-shaped object."""
|
|
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(model="gpt-5", provider="openai"),
|
|
):
|
|
merged = patches_mod._merge_runs_config_kwargs(
|
|
{"thread_id": "t1", "config": _Sentinel()}
|
|
)
|
|
assert merged["thread_id"] == "t1"
|
|
# Non-dict input was replaced with a fresh dict carrying only our
|
|
# injected keys.
|
|
assert merged["config"] == {
|
|
"configurable": {"model": "gpt-5", "model_provider": "openai"}
|
|
}
|
|
|
|
|
|
# =============================================================================
|
|
# 8. Caller's per-run model (contextvar) beats the config-default
|
|
# =============================================================================
|
|
|
|
|
|
class TestExtractCallerConfigurable:
|
|
"""``_extract_caller_configurable`` pulls (model, provider) from a config."""
|
|
|
|
def test_none_config_is_empty(self):
|
|
assert patches_mod._extract_caller_configurable(None) == {}
|
|
|
|
def test_no_configurable_key_is_empty(self):
|
|
assert patches_mod._extract_caller_configurable({"tags": ["x"]}) == {}
|
|
|
|
def test_model_only(self):
|
|
cfg = {"configurable": {"model": "free"}}
|
|
assert patches_mod._extract_caller_configurable(cfg) == {"model": "free"}
|
|
|
|
def test_model_and_provider(self):
|
|
cfg = {"configurable": {"model": "free", "model_provider": "openrouter"}}
|
|
assert patches_mod._extract_caller_configurable(cfg) == {
|
|
"model": "free",
|
|
"model_provider": "openrouter",
|
|
}
|
|
|
|
def test_bare_provider_without_model_dropped(self):
|
|
# A provider with no model is meaningless to the deployed resolver.
|
|
cfg = {"configurable": {"model_provider": "openrouter"}}
|
|
assert patches_mod._extract_caller_configurable(cfg) == {}
|
|
|
|
|
|
class TestCallerConfigurableWinsOverCfg:
|
|
"""The launching run's model (contextvar) beats ``_ensure_config()``.
|
|
|
|
This is the whole point on the ``langgraph_server`` backend: the proxy
|
|
runs in the dev-server process where ``_ensure_config()`` reports the
|
|
server's config-default, so the caller's per-run model must win.
|
|
"""
|
|
|
|
def test_caller_model_overrides_config_default(self):
|
|
token = patches_mod._caller_configurable.set(
|
|
{"model": "free", "model_provider": "openrouter"}
|
|
)
|
|
try:
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(
|
|
model="gemini-3-flash-preview", provider="openrouter"
|
|
),
|
|
):
|
|
merged = patches_mod._merge_runs_config_kwargs({"thread_id": "t1"})
|
|
finally:
|
|
patches_mod._caller_configurable.reset(token)
|
|
assert merged["config"]["configurable"]["model"] == "free"
|
|
assert merged["config"]["configurable"]["model_provider"] == "openrouter"
|
|
|
|
def test_no_caller_model_falls_back_to_config_default(self):
|
|
# Contextvar unset (default {}) → config-default applies as before.
|
|
with patch(
|
|
"EvoScientist.EvoScientist._ensure_config",
|
|
return_value=_stub_cfg(
|
|
model="gemini-3-flash-preview", provider="openrouter"
|
|
),
|
|
):
|
|
merged = patches_mod._merge_runs_config_kwargs({"thread_id": "t1"})
|
|
assert merged["config"]["configurable"]["model"] == "gemini-3-flash-preview"
|