m4
57176b359a
Replace config.yaml-driven model selection with registry snapshot resolution across the runtime chain: - ConfigurableModelMiddleware reads configurable["runtime_snapshot_id"] only; model/model_provider overrides are rejected with MODEL_CONFIG_OUTSIDE_SNAPSHOT - MessageBudgetMiddleware derives budgets from snapshot reserves (system/tools/attachments) and re-resolves the summarizer per snapshot - Agent factory and subagent factory resolve models via SnapshotRuntime (auxiliary/tool_selector/scheduler -> defaults.auxiliary ?? defaults.primary) - Remove ModelFallbackMiddleware, /model-fallback command, and fallback chain - Add model_registry/runtime.py SnapshotRuntime glue layer Legacy config.yaml LLM fields, /model command, and llm/models.py remain for Task 7. Report: .superpowers/sdd/briefs/task-6-report.md
418 lines
15 KiB
Python
418 lines
15 KiB
Python
"""Tests for the deepagents async-task runs.create patch.
|
|
|
|
Verifies that ``_patch_deepagents_model_passthrough`` wraps
|
|
``_build_start_tool`` / ``_build_update_tool`` so that ``client.runs.create``
|
|
calls inside the launched async-task tools inherit workspace scope and usage
|
|
correlation context — and that NO model configuration is injected: model
|
|
resolution is snapshot-driven (``runtime_snapshot_id``), and injecting
|
|
``model``/``model_provider`` would be rejected with
|
|
``MODEL_CONFIG_OUTSIDE_SNAPSHOT`` (design doc 8.2/8.3).
|
|
|
|
Other client methods (``threads.create``, ``runs.get``, ``runs.cancel``)
|
|
must pass through unmodified.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from EvoScientist.llm import patches as patches_mod
|
|
|
|
# =============================================================================
|
|
# Helpers
|
|
# =============================================================================
|
|
|
|
|
|
def _make_client_cache(
|
|
*,
|
|
create_run_id: str = "run-001",
|
|
thread_id: str = "thread-001",
|
|
):
|
|
"""Build a fake ``_ClientCache`` with sync + async stub clients.
|
|
|
|
Returns ``(cache_mock, runs_create_sync_mock, runs_create_async_mock)``
|
|
so tests can both invoke through the patched factory and inspect the
|
|
exact call kwargs passed to ``runs.create``.
|
|
"""
|
|
runs_sync = MagicMock()
|
|
runs_sync.create.return_value = {"run_id": create_run_id}
|
|
runs_sync.cancel = MagicMock(return_value=None)
|
|
runs_sync.get = MagicMock(return_value={"status": "success"})
|
|
|
|
threads_sync = MagicMock()
|
|
threads_sync.create.return_value = {"thread_id": thread_id}
|
|
|
|
sync_client = MagicMock()
|
|
sync_client.runs = runs_sync
|
|
sync_client.threads = threads_sync
|
|
|
|
runs_async = MagicMock()
|
|
runs_async.create = AsyncMock(return_value={"run_id": create_run_id})
|
|
|
|
threads_async = MagicMock()
|
|
threads_async.create = AsyncMock(return_value={"thread_id": thread_id})
|
|
|
|
async_client = MagicMock()
|
|
async_client.runs = runs_async
|
|
async_client.threads = threads_async
|
|
|
|
cache = MagicMock()
|
|
cache.get_sync = MagicMock(return_value=sync_client)
|
|
cache.get_async = MagicMock(return_value=async_client)
|
|
|
|
return cache, runs_sync, runs_async
|
|
|
|
|
|
def _runtime_stub():
|
|
"""Minimal stand-in for ``ToolRuntime`` accepted by the inner tools."""
|
|
return SimpleNamespace(tool_call_id="tc-001", state={})
|
|
|
|
|
|
def _agent_map(name: str = "writing-agent") -> dict:
|
|
return {
|
|
name: {
|
|
"name": name,
|
|
"description": "Draft paper",
|
|
"graph_id": name,
|
|
"url": "http://localhost:6174",
|
|
}
|
|
}
|
|
|
|
|
|
_SCOPE = {
|
|
"workspace_scope_id": "11111111-1111-1111-1111-111111111111",
|
|
"workspace_scope_owner_id": "22222222-2222-2222-2222-222222222222",
|
|
"workspace_scope_revision": 3,
|
|
"workspace_deployment_id": "local",
|
|
"thread_id": "thread-001",
|
|
}
|
|
|
|
|
|
def _set_runnable_config(configurable: dict, metadata: dict | None = None):
|
|
"""Populate the real langgraph contextvar the patch reads."""
|
|
from langchain_core.runnables.config import var_child_runnable_config
|
|
|
|
return var_child_runnable_config.set(
|
|
{"configurable": configurable, "metadata": metadata or {}}
|
|
)
|
|
|
|
|
|
def _reset_runnable_config(token) -> None:
|
|
from langchain_core.runnables.config import var_child_runnable_config
|
|
|
|
var_child_runnable_config.reset(token)
|
|
|
|
|
|
# =============================================================================
|
|
# 1. Idempotence
|
|
# =============================================================================
|
|
|
|
|
|
class TestIdempotence:
|
|
"""The patch must be a no-op after the first application."""
|
|
|
|
def test_double_apply_does_not_re_wrap(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
original = ds_mod._build_start_tool
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
wrapped_once = ds_mod._build_start_tool
|
|
assert wrapped_once is not original
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
wrapped_twice = ds_mod._build_start_tool
|
|
assert wrapped_twice is wrapped_once # not double-wrapped
|
|
|
|
def test_flag_set_after_apply(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as _ # noqa: F401
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
patches_mod._model_passthrough_patched = False
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
assert patches_mod._model_passthrough_patched is True
|
|
|
|
|
|
# =============================================================================
|
|
# 2. No model configuration is injected (design doc 8.2)
|
|
# =============================================================================
|
|
|
|
|
|
class TestNoModelInjection:
|
|
"""runs.create must never carry model/model_provider overrides."""
|
|
|
|
def test_sync_start_injects_no_model_keys(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, runs_sync, _ = _make_client_cache()
|
|
tool = ds_mod._build_start_tool(_agent_map(), cache, "desc")
|
|
tool.func(
|
|
description="hello",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
|
|
runs_sync.create.assert_called_once()
|
|
kwargs = runs_sync.create.call_args.kwargs
|
|
assert kwargs["thread_id"] == "thread-001"
|
|
assert kwargs["assistant_id"] == "writing-agent"
|
|
configurable = (kwargs.get("config") or {}).get("configurable") or {}
|
|
assert "model" not in configurable
|
|
assert "model_provider" not in configurable
|
|
|
|
async def test_async_start_injects_no_model_keys(
|
|
self, restore_model_passthrough_patch
|
|
):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, _, runs_async = _make_client_cache()
|
|
tool = ds_mod._build_start_tool(_agent_map(), cache, "desc")
|
|
await tool.coroutine(
|
|
description="hi",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
|
|
runs_async.create.assert_awaited_once()
|
|
configurable = (runs_async.create.call_args.kwargs.get("config") or {}).get(
|
|
"configurable"
|
|
) or {}
|
|
assert "model" not in configurable
|
|
assert "model_provider" not in configurable
|
|
|
|
def test_sync_update_injects_no_model_keys(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, runs_sync, _ = _make_client_cache()
|
|
tracked_task = {
|
|
"task_id": "thread-001",
|
|
"agent_name": "writing-agent",
|
|
"thread_id": "thread-001",
|
|
"run_id": "old-run",
|
|
"status": "running",
|
|
"created_at": "2026-05-07T00:00:00Z",
|
|
"last_checked_at": "2026-05-07T00:00:00Z",
|
|
"last_updated_at": "2026-05-07T00:00:00Z",
|
|
}
|
|
runtime = SimpleNamespace(
|
|
tool_call_id="tc-002",
|
|
state={"async_tasks": {"thread-001": tracked_task}},
|
|
)
|
|
|
|
tool = ds_mod._build_update_tool(_agent_map(), cache)
|
|
tool.func(
|
|
task_id="thread-001",
|
|
message="follow up",
|
|
runtime=runtime,
|
|
)
|
|
|
|
runs_sync.create.assert_called_once()
|
|
kwargs = runs_sync.create.call_args.kwargs
|
|
configurable = (kwargs.get("config") or {}).get("configurable") or {}
|
|
assert "model" not in configurable
|
|
assert "model_provider" not in configurable
|
|
# update preserves multitask_strategy="interrupt" — verify no regression
|
|
assert kwargs.get("multitask_strategy") == "interrupt"
|
|
|
|
|
|
# =============================================================================
|
|
# 3. Workspace scope inheritance is preserved
|
|
# =============================================================================
|
|
|
|
|
|
class TestScopeInheritance:
|
|
def test_scope_keys_forwarded(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, runs_sync, _ = _make_client_cache()
|
|
tool = ds_mod._build_start_tool(_agent_map(), cache, "desc")
|
|
|
|
token = _set_runnable_config(dict(_SCOPE))
|
|
try:
|
|
tool.func(
|
|
description="hello",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
finally:
|
|
_reset_runnable_config(token)
|
|
|
|
configurable = runs_sync.create.call_args.kwargs["config"]["configurable"]
|
|
for key in (
|
|
"workspace_scope_id",
|
|
"workspace_scope_owner_id",
|
|
"workspace_scope_revision",
|
|
"workspace_deployment_id",
|
|
):
|
|
assert configurable[key] == _SCOPE[key]
|
|
|
|
|
|
# =============================================================================
|
|
# 4. Usage correlation metadata is preserved
|
|
# =============================================================================
|
|
|
|
|
|
class TestUsageMetadata:
|
|
def test_usage_scope_tagged(self, restore_model_passthrough_patch, monkeypatch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
monkeypatch.setenv("EVOSCIENTIST_USAGE_TRACKING", "1")
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, runs_sync, _ = _make_client_cache()
|
|
tool = ds_mod._build_start_tool(_agent_map(), cache, "desc")
|
|
|
|
token = _set_runnable_config(
|
|
{"thread_id": "thread-001"}, metadata={"turn_id": "turn-9"}
|
|
)
|
|
try:
|
|
tool.func(
|
|
description="hello",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
finally:
|
|
_reset_runnable_config(token)
|
|
|
|
metadata = runs_sync.create.call_args.kwargs["metadata"]
|
|
assert metadata["usage_scope"] == "async_subagent"
|
|
assert metadata["turn_id"] == "turn-9"
|
|
|
|
|
|
# =============================================================================
|
|
# 5. Other client methods are unaffected
|
|
# =============================================================================
|
|
|
|
|
|
class TestNonInterceptedMethods:
|
|
"""``threads.create``, ``runs.get``, ``runs.cancel`` must pass through."""
|
|
|
|
def test_threads_create_not_modified(self, restore_model_passthrough_patch):
|
|
"""``threads.create()`` is called by start_async_task pre-runs.create.
|
|
|
|
The patch should not inject config here, because thread creation
|
|
doesn't take config (and would error out if we did).
|
|
"""
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, _runs_sync, _ = _make_client_cache()
|
|
threads_create = cache.get_sync.return_value.threads.create
|
|
tool = ds_mod._build_start_tool(_agent_map(), cache, "desc")
|
|
tool.func(
|
|
description="hi",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
|
|
# threads.create() is called with no kwargs (deepagents pattern).
|
|
threads_create.assert_called_once_with()
|
|
|
|
|
|
# =============================================================================
|
|
# 6. No context to inherit → no config kwarg added
|
|
# =============================================================================
|
|
|
|
|
|
class TestNoContextNoConfig:
|
|
def test_no_scope_no_config_kwarg(self, restore_model_passthrough_patch):
|
|
try:
|
|
from deepagents.middleware import async_subagents as ds_mod
|
|
except ImportError:
|
|
pytest.skip("deepagents not available")
|
|
|
|
patches_mod._patch_deepagents_model_passthrough()
|
|
|
|
cache, runs_sync, _ = _make_client_cache()
|
|
tool = ds_mod._build_start_tool(_agent_map(), cache, "desc")
|
|
tool.func(
|
|
description="hi",
|
|
subagent_type="writing-agent",
|
|
runtime=_runtime_stub(),
|
|
)
|
|
|
|
runs_sync.create.assert_called_once()
|
|
kwargs = runs_sync.create.call_args.kwargs
|
|
# No config kwarg should be added when there's nothing to inherit.
|
|
assert "config" not in kwargs
|
|
|
|
|
|
# =============================================================================
|
|
# 7. Caller-supplied config keys are preserved
|
|
# =============================================================================
|
|
|
|
|
|
class TestPreserveExistingConfig:
|
|
"""If a caller already supplied config.configurable.X, the merge keeps it."""
|
|
|
|
def test_existing_configurable_preserved(self):
|
|
merged = patches_mod._merge_runs_config_kwargs(
|
|
{
|
|
"thread_id": "t1",
|
|
"config": {
|
|
"configurable": {"thread_id": "outer-t", "extra": 42},
|
|
"tags": ["debug"],
|
|
},
|
|
}
|
|
)
|
|
assert merged["thread_id"] == "t1"
|
|
assert merged["config"]["tags"] == ["debug"]
|
|
assert merged["config"]["configurable"]["thread_id"] == "outer-t"
|
|
assert merged["config"]["configurable"]["extra"] == 42
|
|
|
|
def test_non_dict_config_replaced(self):
|
|
"""Non-dict ``config`` (e.g. a Pydantic RunnableConfig) is replaced.
|
|
|
|
Documents the policy: callers that pass a non-dict ``config`` lose
|
|
any other fields they may have set there. Acceptable today because
|
|
deepagents' built-in ``runs.create`` path doesn't pass a config at
|
|
all, but a future caller passing e.g. a Pydantic model would have
|
|
their non-configurable fields silently dropped. If that becomes a
|
|
real use case, ``_merge_runs_config_kwargs`` should grow a
|
|
``dict()`` coercion or raise.
|
|
"""
|
|
|
|
class _Sentinel:
|
|
"""Stand-in for any non-dict config-shaped object."""
|
|
|
|
merged = patches_mod._merge_runs_config_kwargs(
|
|
{"thread_id": "t1", "config": _Sentinel()}
|
|
)
|
|
assert merged["thread_id"] == "t1"
|
|
# Non-dict input was replaced with a fresh dict; nothing is injected
|
|
# when there is no runnable context to inherit from.
|
|
assert merged["config"] == {"configurable": {}}
|