Files
EvoScientist-Multi/tests/test_serve_agent_holder.py
T
dinos 8b1451cdda refactor(runtime): centralize async bridges under an owned runtime (#376)
* feat(runtime): add application-scoped async runtime

* refactor(cli): use owned runtime for session stats

* refactor(onboard): use the owned async runtime

* docs(runtime): record async bridge ownership

* refactor(middleware): keep sync fallback synchronous

* refactor(mcp): load tools on an owned runtime

* refactor(cli): share owned runtime across entry points

* refactor(channels): make inbound sync bridge explicit

* refactor(stream): run Rich streaming on owned runtime

* chore(runtime): remove nest-asyncio dependency

* refactor(asyncio): require active loops in async code

* docs(runtime): document final event loop ownership

* fix(stream): cancel stalled owned streams

* fix(cli): recover cleanly from stream cancellation

* fix(runtime): drain executor work before shutdown

* fix(runtime): terminate cancelled shell process trees

* fix(models): let fallback bypass selector failures

* fix(cli): reset interrupt handling between turns

* docs: rm implementation spec

* fix(serve): cancel active turns during shutdown

* fix(runtime): protect settlement from waiter cancellation

* fix(backends): reject empty shell commands

* fix(runtime): terminate descendants after shell exit

* fix(mcp): keep standalone discovery off channel loop

* fix(cli): own and settle interactive prompt cancellation

* fix(serve): keep channel sends off runtime loop

* fix(stream): scope cancel context to iterator steps

* refactor(serve): require the owned async runtime

* fix(channels): keep interactive sends off runtime loop

* fix(selector): surface fallback without log spam

* test(runtime): normalize Windows shell marker

* fix(cli): serialize interactive session turns

* fix(shell): bound output drain after termination

* fix(ui): do not retry owned runtime failures

* fix(shell): allow signal-safe registry reentry

* fix(shell): avoid terminating reused process ids

* fix(channels): preserve streaming send order

* fix(cli): report runtime shutdown timeouts cleanly

* fix(mcp): guide async callers to async loader

* docs(runtime): clarify reserved async bridge APIs

* fix(runtime): bound code interpreter cleanup

* test(shell): use active Python for drain regression

---------

Co-authored-by: Xi Zhang <106144707+X-iZhang@users.noreply.github.com>
2026-07-27 14:17:57 +01:00

724 lines
23 KiB
Python

"""Tests for the serve-mode ``on_cmd_completed`` hook factory.
Regression coverage for the follow-up to issue #181 — `/model` invoked
over a channel in ``EvoSci serve`` must swap the running agent for
subsequent messages, not silently keep the stale one the while-loop
captured at startup.
"""
from __future__ import annotations
import asyncio
import time
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from langgraph.graph.state import CompiledStateGraph
from EvoScientist.cli.channel import (
ChannelMessage,
_register_channel_request,
)
from EvoScientist.cli.commands import (
ServeRuntimeState,
_make_serve_cmd_completed_hook,
_make_serve_handle_session_resume_cb,
_make_serve_start_new_session_cb,
_serve_process_message,
)
from EvoScientist.commands.base import ChannelRuntime
from EvoScientist.config import EvoScientistConfig
from EvoScientist.gateway import RuntimeGateways, ThreadStore
from EvoScientist.runtime import AsyncRuntime
from tests.fakes import FakeGraphGateway, FakeThreadStore
def _agent(name: str = "agent") -> CompiledStateGraph:
return MagicMock(name=name, spec=CompiledStateGraph)
def _config() -> EvoScientistConfig:
return EvoScientistConfig()
def _thread_store(thread_id: str = "unused") -> ThreadStore:
return FakeThreadStore(generated_thread_id=thread_id)
def _runtime_gateways(thread_store: ThreadStore | None = None) -> RuntimeGateways:
store = thread_store or _thread_store()
return RuntimeGateways(
thread_store=store,
graph_gateway=FakeGraphGateway(thread_store=store),
)
def _runtime_state(
*,
agent: CompiledStateGraph | None = None,
thread_id: str = "tid",
workspace_dir: str | None = None,
config: EvoScientistConfig | None = None,
thread_store: ThreadStore | None = None,
runtime_gateways: RuntimeGateways | None = None,
async_runtime: AsyncRuntime | None = None,
) -> ServeRuntimeState:
store = thread_store or _thread_store()
return ServeRuntimeState(
agent=agent if agent is not None else _agent(),
thread_id=thread_id,
workspace_dir=workspace_dir,
config=config,
runtime_gateways=runtime_gateways or _runtime_gateways(store),
async_runtime=async_runtime or MagicMock(spec=AsyncRuntime),
)
def test_serve_runtime_state_requires_owned_runtime():
"""Message processing cannot be constructed without its runtime owner."""
with pytest.raises(TypeError, match="async_runtime"):
ServeRuntimeState(
agent=_agent(),
thread_id="tid",
workspace_dir=None,
config=None,
runtime_gateways=_runtime_gateways(),
)
async def test_hook_updates_runtime_state_on_agent_swap():
"""``/model`` mutates ``ctx.agent`` to a new handle — the hook must
push that handle into the shared runtime state so the outer poll loop sees
it on the next message."""
original_agent = _agent("original-agent")
new_agent = _agent("new-agent")
state = _runtime_state(agent=original_agent)
hook = _make_serve_cmd_completed_hook(state)
ctx = MagicMock()
ctx.agent = new_agent
ctx.thread_id = state.thread_id
cmd = MagicMock()
cmd.name = "/model"
await hook(ctx, original_agent, cmd)
assert state.agent is new_agent
async def test_hook_syncs_channel_runtime():
"""Other readers (the bus) look at ``ChannelRuntime.agent``; the
hook keeps the runtime in sync with the runtime state update."""
original_agent = _agent("original-agent")
new_agent = _agent("new-agent")
state = _runtime_state(agent=original_agent, thread_id="t")
runtime = ChannelRuntime(agent=original_agent, thread_id="t")
hook = _make_serve_cmd_completed_hook(state, runtime)
ctx = MagicMock()
ctx.agent = new_agent
# Pin ctx.thread_id explicitly — a bare MagicMock would let the
# hook's getattr fall through to a fresh MagicMock attribute and
# silently mutate runtime.thread_id, hiding regressions.
ctx.thread_id = "t"
cmd = MagicMock()
cmd.name = "/model"
await hook(ctx, original_agent, cmd)
assert runtime.agent is new_agent
assert runtime.thread_id == "t"
async def test_hook_noop_when_agent_unchanged():
"""Commands like ``/evoskills`` don't touch ``ctx.agent`` — the
runtime state must stay put."""
original_agent = _agent("original-agent")
state = _runtime_state(agent=original_agent)
hook = _make_serve_cmd_completed_hook(state)
ctx = MagicMock()
ctx.agent = original_agent # no swap
ctx.thread_id = state.thread_id
cmd = MagicMock()
cmd.name = "/evoskills"
await hook(ctx, original_agent, cmd)
assert state.agent is original_agent
async def test_hook_noop_when_ctx_agent_is_none():
"""Guard against commands that reset ``ctx.agent`` to ``None`` —
we never want to write ``None`` into runtime state."""
original_agent = _agent("original-agent")
state = _runtime_state(agent=original_agent)
hook = _make_serve_cmd_completed_hook(state)
ctx = MagicMock()
ctx.agent = None
ctx.thread_id = state.thread_id
cmd = MagicMock()
cmd.name = "/whatever"
await hook(ctx, original_agent, cmd)
assert state.agent is original_agent
async def test_hook_updates_thread_id_on_resume():
"""``/resume`` mutates ``ctx.thread_id`` — the hook must push the
new id into runtime state so the outer poll loop runs subsequent
messages on the resumed thread."""
agent = _agent("a")
state = _runtime_state(agent=agent, thread_id="original-tid")
hook = _make_serve_cmd_completed_hook(state)
ctx = MagicMock()
ctx.agent = agent # no agent swap
ctx.thread_id = "new-tid"
ctx.workspace_dir = None
cmd = MagicMock()
cmd.name = "/resume"
await hook(ctx, agent, cmd)
assert state.thread_id == "new-tid"
async def test_hook_updates_workspace_dir_on_resume():
"""`/resume` can restore a different workspace; serve must reload for it."""
cfg = _config()
old_agent = _agent("old-agent")
reloaded_agent = _agent("reloaded-agent")
state = _runtime_state(
agent=old_agent,
thread_id="original-tid",
workspace_dir="/old-ws",
config=cfg,
)
hook = _make_serve_cmd_completed_hook(state, config=cfg)
ctx = MagicMock()
ctx.agent = old_agent
ctx.thread_id = "new-tid"
ctx.workspace_dir = "/restored-ws"
cmd = MagicMock()
cmd.name = "/resume"
with (
patch(
"EvoScientist.cli.commands._sync_background_agent_server_workspace",
new=AsyncMock(),
) as sync_server,
patch(
"EvoScientist.cli.commands._load_agent",
return_value=reloaded_agent,
) as load_agent,
):
await hook(ctx, old_agent, cmd)
sync_server.assert_awaited_once_with(cfg, workspace_dir="/restored-ws")
load_agent.assert_called_once_with(
workspace_dir="/restored-ws",
config=cfg,
runtime=state.async_runtime,
)
assert state.workspace_dir == "/restored-ws"
assert state.agent is reloaded_agent
async def test_hook_syncs_channel_runtime_thread_id():
"""The bus reads ``ChannelRuntime.thread_id``; hook must sync it
alongside the runtime state update."""
agent = _agent("a")
state = _runtime_state(agent=agent, thread_id="original-tid")
runtime = ChannelRuntime(agent=agent, thread_id="original-tid")
hook = _make_serve_cmd_completed_hook(state, runtime)
ctx = MagicMock()
ctx.agent = agent
ctx.thread_id = "new-tid"
ctx.workspace_dir = None
cmd = MagicMock()
cmd.name = "/resume"
await hook(ctx, agent, cmd)
assert runtime.thread_id == "new-tid"
async def test_hook_noop_when_thread_id_unchanged():
"""Most commands don't touch thread_id — runtime state stays put."""
agent = _agent("a")
state = _runtime_state(agent=agent, thread_id="same-tid")
hook = _make_serve_cmd_completed_hook(state)
ctx = MagicMock()
ctx.agent = agent
ctx.thread_id = "same-tid"
cmd = MagicMock()
cmd.name = "/evoskills"
await hook(ctx, agent, cmd)
assert state.thread_id == "same-tid"
async def test_hook_skips_resume_warning_when_thread_unchanged():
"""Bare ``/resume`` with no argument prints usage but leaves
``ctx.thread_id`` unchanged — the in-memory-state warning must NOT
fire because no resume actually happened."""
agent = _agent("a")
state = _runtime_state(agent=agent, thread_id="original-tid")
hook = _make_serve_cmd_completed_hook(state)
ctx = MagicMock()
ctx.agent = agent
ctx.thread_id = "original-tid" # unchanged — bare /resume case
ctx.workspace_dir = None
cmd = MagicMock()
cmd.name = "/resume"
await hook(ctx, agent, cmd)
ctx.ui.append_system.assert_not_called()
ctx.ui.flush.assert_not_called()
async def test_hook_emits_resume_warning_when_thread_changed():
"""``/resume <tid>`` that actually changes thread_id must surface
the in-memory-state warning via ``ctx.ui``."""
agent = _agent("a")
state = _runtime_state(agent=agent, thread_id="original-tid")
hook = _make_serve_cmd_completed_hook(state)
ctx = MagicMock()
# Mock out async flush so the test can synchronously run the hook.
ctx.ui.flush = AsyncMock()
ctx.agent = agent
ctx.thread_id = "abc12345-resumed-tid"
ctx.workspace_dir = None
cmd = MagicMock()
cmd.name = "/resume"
await hook(ctx, agent, cmd)
ctx.ui.append_system.assert_called_once()
warn_text, warn_kwargs = (
ctx.ui.append_system.call_args.args,
ctx.ui.append_system.call_args.kwargs,
)
assert "in-memory state" in warn_text[0]
assert "abc12345" in warn_text[0]
assert warn_kwargs.get("style") == "yellow"
ctx.ui.flush.assert_awaited_once()
async def test_start_new_session_cb_rotates_thread_id():
"""``/new`` via channel calls this callback — must generate a new
thread id, push into runtime state, and sync the channel runtime."""
agent = _agent("a")
state = _runtime_state(
agent=agent,
thread_id="old-tid",
thread_store=_thread_store("freshly-generated-tid"),
)
runtime = ChannelRuntime(agent=agent, thread_id="old-tid")
cb = _make_serve_start_new_session_cb(
state,
runtime,
)
await cb()
assert state.thread_id == "freshly-generated-tid"
assert runtime.thread_id == "freshly-generated-tid"
async def test_start_new_session_cb_leaves_agent_alone():
"""``/new`` rotates thread only — agent handle must stay put
(serve's agent is a single pre-loaded instance, not per-thread)."""
agent = _agent("a")
state = _runtime_state(
agent=agent,
thread_id="old-tid",
thread_store=_thread_store("new-tid"),
)
cb = _make_serve_start_new_session_cb(state)
await cb()
assert state.agent is agent
async def test_serve_resume_callback_syncs_reloads_and_adopts_workspace():
cfg = _config()
old_agent = _agent("old-agent")
reloaded_agent = _agent("reloaded-agent")
state = _runtime_state(
agent=old_agent,
thread_id="old-tid",
workspace_dir="/old-ws",
config=cfg,
)
runtime = ChannelRuntime(agent=old_agent, thread_id="old-tid")
cb = _make_serve_handle_session_resume_cb(state, runtime, config=cfg)
call_order: list[str] = []
def _load_agent(**_kwargs):
call_order.append("load")
return reloaded_agent
async def _sync_server(*_args, **_kwargs):
call_order.append("sync")
with (
patch(
"EvoScientist.cli.commands._sync_background_agent_server_workspace",
new=AsyncMock(side_effect=_sync_server),
) as sync_server,
patch(
"EvoScientist.cli.commands._load_agent",
side_effect=_load_agent,
) as load_agent,
):
await cb("new-tid", "/new-ws")
sync_server.assert_awaited_once_with(cfg, workspace_dir="/new-ws")
load_agent.assert_called_once_with(
workspace_dir="/new-ws",
config=cfg,
runtime=state.async_runtime,
)
assert call_order == ["load", "sync"]
assert state.thread_id == "new-tid"
assert state.workspace_dir == "/new-ws"
assert state.agent is reloaded_agent
assert runtime.thread_id == "new-tid"
assert runtime.agent is reloaded_agent
async def test_hook_emits_resume_warning_after_resume_callback_adopts_thread():
cfg = _config()
old_agent = _agent("old-agent")
reloaded_agent = _agent("reloaded-agent")
state = _runtime_state(
agent=old_agent,
thread_id="old-tid",
workspace_dir="/old-ws",
config=cfg,
)
runtime = ChannelRuntime(agent=old_agent, thread_id="old-tid")
cb = _make_serve_handle_session_resume_cb(state, runtime, config=cfg)
with (
patch(
"EvoScientist.cli.commands._sync_background_agent_server_workspace",
new=AsyncMock(),
),
patch(
"EvoScientist.cli.commands._load_agent",
return_value=reloaded_agent,
),
):
await cb("abc12345-resumed-tid", "/new-ws")
hook = _make_serve_cmd_completed_hook(state, runtime, config=cfg)
ctx = MagicMock()
ctx.ui.flush = AsyncMock()
ctx.agent = reloaded_agent
ctx.thread_id = "abc12345-resumed-tid"
ctx.workspace_dir = "/new-ws"
cmd = MagicMock()
cmd.name = "/resume"
await hook(ctx, reloaded_agent, cmd)
ctx.ui.append_system.assert_called_once()
assert "in-memory state" in ctx.ui.append_system.call_args.args[0]
ctx.ui.flush.assert_awaited_once()
async def test_serve_resume_callback_preserves_state_when_sync_fails():
cfg = _config()
old_agent = _agent("old-agent")
loaded_but_not_adopted = _agent("loaded-but-not-adopted")
state = _runtime_state(
agent=old_agent,
thread_id="old-tid",
workspace_dir="/old-ws",
config=cfg,
)
runtime = ChannelRuntime(agent=old_agent, thread_id="old-tid")
cb = _make_serve_handle_session_resume_cb(state, runtime, config=cfg)
with (
patch(
"EvoScientist.cli.commands._sync_background_agent_server_workspace",
new=AsyncMock(side_effect=RuntimeError("workspace conflict")),
),
patch(
"EvoScientist.cli.commands._load_agent",
return_value=loaded_but_not_adopted,
) as load_agent,
patch("EvoScientist.cli.commands.set_active_workspace") as set_active,
pytest.raises(RuntimeError, match="workspace conflict"),
):
await cb("new-tid", "/new-ws")
load_agent.assert_called_once_with(
workspace_dir="/new-ws",
config=cfg,
runtime=state.async_runtime,
)
set_active.assert_called_once_with("/old-ws")
assert state.agent is old_agent
assert state.resume_warning_thread_id is None
assert state.thread_id == "old-tid"
assert state.workspace_dir == "/old-ws"
assert state.config is cfg
assert runtime.agent is old_agent
assert runtime.thread_id == "old-tid"
async def test_serve_resume_callback_load_failure_does_not_sync_or_adopt():
cfg = _config()
old_agent = _agent("old-agent")
state = _runtime_state(
agent=old_agent,
thread_id="old-tid",
workspace_dir="/old-ws",
config=cfg,
)
runtime = ChannelRuntime(agent=old_agent, thread_id="old-tid")
cb = _make_serve_handle_session_resume_cb(state, runtime, config=cfg)
with (
patch(
"EvoScientist.cli.commands._load_agent",
side_effect=RuntimeError("load failed"),
) as load_agent,
patch("EvoScientist.cli.commands.set_active_workspace") as set_active,
patch(
"EvoScientist.cli.commands._sync_background_agent_server_workspace",
new=AsyncMock(),
) as sync_server,
pytest.raises(RuntimeError, match="load failed"),
):
await cb("new-tid", "/new-ws")
load_agent.assert_called_once_with(
workspace_dir="/new-ws",
config=cfg,
runtime=state.async_runtime,
)
set_active.assert_called_once_with("/old-ws")
sync_server.assert_not_awaited()
assert state.resume_warning_thread_id is None
assert state.agent is old_agent
assert state.thread_id == "old-tid"
assert state.workspace_dir == "/old-ws"
assert state.config is cfg
assert runtime.agent is old_agent
assert runtime.thread_id == "old-tid"
async def test_hook_handles_both_agent_and_thread_swap():
"""Edge case: a command that changes both (hypothetical). Both
updates must land in runtime state."""
old_agent = _agent("old-agent")
new_agent = _agent("new-agent")
state = _runtime_state(agent=old_agent, thread_id="old-tid")
hook = _make_serve_cmd_completed_hook(state)
ctx = MagicMock()
ctx.agent = new_agent
ctx.thread_id = "new-tid"
cmd = MagicMock()
await hook(ctx, old_agent, cmd)
assert state.agent is new_agent
assert state.thread_id == "new-tid"
def test_serve_process_message_reports_slash_dispatch_error_without_fallback():
"""Defensive: if ``dispatch_channel_slash_command`` ever leaks an
exception past its own wrapper, ``_serve_process_message`` must set
one error response and not fall through to ``run_streaming``.
"""
msg = ChannelMessage(
msg_id="msg-1",
content="/evoskills core",
sender="channel-user",
channel_type="imessage",
metadata={},
channel_ref=None,
bus_ref=None,
chat_id="channel-user",
message_id="ts-1",
)
thread_store = _thread_store()
state = _runtime_state(
agent=_agent(),
thread_id="tid",
thread_store=thread_store,
runtime_gateways=_runtime_gateways(thread_store),
)
with (
AsyncRuntime(thread_name="test-serve-runtime") as async_runtime,
patch(
"EvoScientist.cli.commands.dispatch_channel_slash_command",
new=AsyncMock(side_effect=RuntimeError("slash broke")),
),
patch("EvoScientist.cli.commands._set_channel_response") as mock_set_resp,
patch("EvoScientist.cli.tui_runtime.run_streaming") as mock_run_streaming,
):
state.async_runtime = async_runtime
_register_channel_request(msg)
_serve_process_message(
msg,
runtime_state=state,
model="model",
workspace_dir="/tmp",
show_thinking=False,
)
mock_set_resp.assert_called_once_with("msg-1", "Command error: slash broke")
mock_run_streaming.assert_not_called()
def test_serve_process_message_uses_runtime_workspace_from_state():
"""After `/resume`, serve should use the adopted workspace, not startup ws."""
msg = ChannelMessage(
msg_id="msg-2",
content="hello",
sender="channel-user",
channel_type="imessage",
metadata={},
channel_ref=None,
bus_ref=None,
chat_id="channel-user",
message_id="ts-2",
)
thread_store = _thread_store()
state = _runtime_state(
agent=_agent(),
thread_id="tid",
workspace_dir="/restored-workspace",
thread_store=thread_store,
runtime_gateways=_runtime_gateways(thread_store),
)
captured: dict[str, str] = {}
async def _fake_dispatch(*args, **kwargs):
captured["slash_workspace"] = kwargs["workspace_dir"]
return False
def _fake_build_metadata(workspace_dir: str, _model: str | None):
captured["meta_workspace"] = workspace_dir
return {}
with (
AsyncRuntime(thread_name="test-serve-runtime") as async_runtime,
patch(
"EvoScientist.cli.commands.dispatch_channel_slash_command",
new=AsyncMock(side_effect=_fake_dispatch),
),
patch(
"EvoScientist.cli.commands.build_metadata",
side_effect=_fake_build_metadata,
),
patch("EvoScientist.cli.tui_runtime.run_streaming", return_value="ok"),
):
state.async_runtime = async_runtime
_register_channel_request(msg)
_serve_process_message(
msg,
runtime_state=state,
model="model",
workspace_dir="/startup-workspace",
show_thinking=False,
)
assert captured["slash_workspace"] == "/restored-workspace"
assert captured["meta_workspace"] == "/restored-workspace"
def test_serve_channel_send_does_not_block_owned_runtime_loop():
"""Channel I/O is scheduled on the bus loop and settled before the reply."""
from EvoScientist.cli import channel as channel_mod
events: list[str] = []
callback_elapsed: list[float] = []
class _ChannelRef:
send_thinking = True
async def send_thinking_message(self, **_kwargs):
events.append("send-started")
await asyncio.sleep(0.05)
events.append("send-finished")
msg = ChannelMessage(
msg_id="msg-nonblocking-send",
content="hello",
sender="channel-user",
channel_type="telegram",
metadata={},
channel_ref=_ChannelRef(),
bus_ref=None,
chat_id="channel-user",
message_id="ts-send",
)
state = _runtime_state(agent=_agent(), thread_id="tid")
def _fake_run_streaming(**kwargs):
async def _invoke_callback() -> None:
started = time.monotonic()
kwargs["on_thinking"]("x" * 250)
callback_elapsed.append(time.monotonic() - started)
events.append("callback-returned")
kwargs["runtime"].run_sync(_invoke_callback)
return "ok"
def _capture_response(_msg_id: str, _response: str) -> None:
events.append("response-set")
with AsyncRuntime(thread_name="test-serve-send-runtime") as runtime:
runtime.submit(lambda: asyncio.sleep(0)).result(timeout=1)
state.async_runtime = runtime
assert runtime._loop is not None
with (
patch.object(channel_mod, "_bus_loop", runtime._loop),
patch(
"EvoScientist.cli.commands.dispatch_channel_slash_command",
new=AsyncMock(return_value=False),
),
patch(
"EvoScientist.cli.tui_runtime.run_streaming",
side_effect=_fake_run_streaming,
),
patch(
"EvoScientist.cli.commands._set_channel_response",
side_effect=_capture_response,
),
):
_register_channel_request(msg)
_serve_process_message(
msg,
runtime_state=state,
model="model",
workspace_dir="/tmp",
show_thinking=True,
)
assert callback_elapsed[0] < 0.5
assert events.index("callback-returned") < events.index("send-started")
assert events.index("send-finished") < events.index("response-set")