Files
EvoScientist-Multi/tests/test_profile_memory_middleware.py
T
dinos 690b903f85 test: standardize async tests on pytest-asyncio auto mode (#338)
* chore: add pytest-asyncio in auto mode

* test: migrate channel and stream tests to native async

Convert run_async() wrapper tests to plain 'async def test_*' under
pytest-asyncio auto mode. collect_events() in stream_v3_fakes becomes a
coroutine awaited at every call site.

* test: migrate command and model/middleware tests to native async

Convert run_async() wrappers (import, alias, and fixture forms) to plain
'async def test_*'. Multi-call tests merge onto one loop as sequential
awaits; none asserted on loop identity.

* test: migrate TUI, notifier, gateway, and session tests to native async

TUI/notifier/gateway files convert run_async wrappers to plain async
tests. test_sessions.py's unittest.TestCase classes move to
unittest.IsolatedAsyncioTestCase (pytest-asyncio does not await async
methods on plain TestCase; converting blindly would have made ~70 tests
silently vacuous). Its setUpClass keeps a one-shot asyncio.run() since
IsolatedAsyncioTestCase has no async class-level hook. TestLoadingWidget
in test_tui_widgets.py drops its TestCase base for the same reason.

* test: replace direct asyncio.run() calls with native async tests

Convert tests that called asyncio.run() (directly or via a local _run
helper) to plain 'async def test_*'; delete the local helpers.

* test: drop undeclared anyio markers and delete run_async helper

The @pytest.mark.anyio tests relied on anyio being a transitive dep of
httpx; auto-mode pytest-asyncio collects them natively. run_async() and
its fixture are unreferenced after the migration, so remove them —
pytest-asyncio's per-test loop teardown covers the pending-task
cancellation the helper existed for (verified: full suite runs with no
'Event loop is closed' errors or destroyed-task warnings).
2026-07-08 18:37:48 +00:00

628 lines
20 KiB
Python

from __future__ import annotations
import threading
from types import SimpleNamespace
from blockbuster import BlockBuster
from langchain_core.messages import SystemMessage
import EvoScientist.middleware.memory as memory_module
from EvoScientist import paths
from EvoScientist.memory.observations import (
MemoryScope,
MemorySourceType,
MemoryType,
build_observation_index_context,
list_observation_documents,
record_observation_file,
)
def _request():
request = SimpleNamespace(
state={},
runtime=object(),
system_message=SystemMessage(content="base system"),
)
request.override = lambda **kwargs: SimpleNamespace(
**{
"state": request.state,
"runtime": request.runtime,
"system_message": kwargs.get("system_message", request.system_message),
}
)
return request
def _path_project_id(workspace) -> str:
return memory_module.resolve_project_id(workspace)
def _profile_texts(memories):
return [
path.read_text(encoding="utf-8")
for path in (memories / "profile").rglob("*.md")
]
def _sorted_tool_names(middleware) -> list[str]:
return sorted(tool.name for tool in middleware.tools)
def test_profile_memory_bootstraps_profiles_without_observation_project_dirs(
tmp_path, monkeypatch
):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
middleware = memory_module.create_memory_middleware(str(memories))
middleware.modify_request(_request())
assert (
memories / "profile" / "projects" / middleware.project_id / "PROJECT_PROFILE.md"
).exists()
assert (memories / "observations" / "global").is_dir()
assert not (memories / "observations" / "projects").exists()
def test_append_to_system_message_preserves_metadata():
system_message = SystemMessage(
content="base system",
id="system-1",
name="root-system",
additional_kwargs={"cache_control": {"type": "ephemeral"}},
response_metadata={"provider": "test"},
)
updated = memory_module.append_to_system_message(
system_message,
"memory context",
)
assert updated.id == "system-1"
assert updated.name == "root-system"
assert updated.additional_kwargs == {"cache_control": {"type": "ephemeral"}}
assert updated.response_metadata == {"provider": "test"}
assert updated.content_blocks == [
{"type": "text", "text": "base system"},
{"type": "text", "text": "memory context"},
]
def test_profile_memory_can_disable_observation_tool(tmp_path, monkeypatch):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
middleware = memory_module.create_memory_middleware(
str(memories),
enable_observation_tool=False,
)
middleware.modify_request(_request())
assert _sorted_tool_names(middleware) == [
"read_memory",
"search_observations",
]
assert (memories / "profile" / "USER_PROFILE.md").exists()
def test_memory_middleware_can_disable_all_memory_injection(tmp_path, monkeypatch):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
middleware = memory_module.create_memory_middleware(
str(memories),
enable_profile_memory=False,
enable_observation_memory=False,
)
request = _request()
modified = middleware.modify_request(request)
assert modified is request
assert middleware.tools == []
assert not (memories / "profile").exists()
assert not (memories / "observations").exists()
def test_observation_memory_can_be_read_only_without_profile(tmp_path, monkeypatch):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
middleware = memory_module.create_memory_middleware(
str(memories),
enable_profile_memory=False,
enable_observation_memory=True,
enable_observation_tool=False,
)
modified = middleware.modify_request(_request())
content = str(modified.system_message.content)
assert _sorted_tool_names(middleware) == [
"read_memory",
"search_observations",
]
assert not (memories / "profile").exists()
assert (memories / "observations" / "global").is_dir()
assert "<observation_memory>" in content
assert "search_observations" in content
assert "read_memory" in content
assert "record_observation" not in content
def test_observation_index_refreshes_summary_frontmatter(tmp_path, monkeypatch):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
project_id = _path_project_id(workspace)
global_result = record_observation_file(
memory_dir=memories,
project_id=project_id,
memory_type=MemoryType.SEMANTIC,
summary="A global fact is available for future lookup.",
observation="A global fact should be indexed.",
why_it_matters="Future agents can decide whether to read it.",
scope=MemoryScope.GLOBAL,
source_type=MemorySourceType.SUBAGENT,
source_session_id="thread-1",
source_agent="research-agent",
)
project_result = record_observation_file(
memory_dir=memories,
project_id=project_id,
memory_type=MemoryType.PROCEDURAL,
summary="A project recipe is available for future lookup.",
observation="A project recipe should be indexed.",
why_it_matters="Future agents can choose it for this workspace.",
scope=MemoryScope.PROJECT,
source_type=MemorySourceType.SUBAGENT,
source_session_id="thread-1",
source_agent="code-agent",
)
(memories / "observations" / "global" / "O-old.md").write_text(
"\n".join(
[
"---",
'id: "O-old"',
"memory_type: semantic",
"scope: global",
"---",
"",
"## Observation",
"",
"Old observations without summary are not indexed.",
]
),
encoding="utf-8",
)
middleware = memory_module.create_memory_middleware(str(memories))
indexed = {
document.observation_id: (
document.memory_type,
document.scope,
document.summary,
)
for document in list_observation_documents(
memory_dir=memories,
project_id=project_id,
)
}
later_result = record_observation_file(
memory_dir=memories,
project_id=project_id,
memory_type=MemoryType.SEMANTIC,
summary="This later observation is refreshed into the index.",
observation="Observation written after middleware construction.",
why_it_matters="Prompt memory should reflect worker writes during the session.",
scope=MemoryScope.GLOBAL,
source_type=MemorySourceType.SUBAGENT,
source_session_id="thread-2",
source_agent="research-agent",
)
modified = middleware.modify_request(_request())
refreshed_ids = {
document.observation_id
for document in list_observation_documents(
memory_dir=memories,
project_id=project_id,
)
}
assert indexed == {
global_result["observation_id"]: (
MemoryType.SEMANTIC,
MemoryScope.GLOBAL,
"A global fact is available for future lookup.",
),
project_result["observation_id"]: (
MemoryType.PROCEDURAL,
MemoryScope.PROJECT,
"A project recipe is available for future lookup.",
),
}
assert refreshed_ids == {*indexed, later_result["observation_id"]}
assert "This later observation is refreshed into the index." in str(
modified.system_message.content
)
def test_observation_index_omits_summaries_when_budget_exceeded(tmp_path, monkeypatch):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
memory_module.create_memory_middleware(str(memories))
record_observation_file(
memory_dir=memories,
project_id=_path_project_id(workspace),
memory_type=MemoryType.PROCEDURAL,
summary="Do not inline this summary when the index exceeds budget.",
observation="A large-index observation exists.",
why_it_matters="Future prompts should fall back to search hints.",
scope=MemoryScope.GLOBAL,
source_type=MemorySourceType.SUBAGENT,
source_session_id="thread-1",
source_agent="research-agent",
)
context = build_observation_index_context(
memory_dir=memories,
project_id=_path_project_id(workspace),
max_inline_chars=1,
)
assert "Do not inline this summary" not in context
def test_observation_index_over_budget_keeps_entries_that_fit(tmp_path, monkeypatch):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
project_id = _path_project_id(workspace)
record_observation_file(
memory_dir=memories,
project_id=project_id,
memory_type=MemoryType.PROCEDURAL,
summary="First over-budget observation " + ("x" * 320),
observation="First large-index observation.",
why_it_matters="Index truncation should retain entries when possible.",
scope=MemoryScope.GLOBAL,
source_type=MemorySourceType.SUBAGENT,
source_session_id="thread-1",
source_agent="research-agent",
)
record_observation_file(
memory_dir=memories,
project_id=project_id,
memory_type=MemoryType.PROCEDURAL,
summary="Second over-budget observation " + ("y" * 320),
observation="Second large-index observation.",
why_it_matters="Index truncation should retain entries when possible.",
scope=MemoryScope.GLOBAL,
source_type=MemorySourceType.SUBAGENT,
source_session_id="thread-2",
source_agent="research-agent",
)
context = build_observation_index_context(
memory_dir=memories,
project_id=project_id,
max_inline_chars=1_350,
)
assert "Observation index truncated to entries that fit." in context
assert len(context) <= 1_350
assert "over-budget observation" in context
def test_profile_memory_uses_path_pointers_when_profiles_exceed_budget(
tmp_path, monkeypatch
):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
middleware = memory_module.create_memory_middleware(
str(memories), max_inline_profile_chars=10
)
middleware.modify_request(_request())
records = middleware._read_profile_records()
assert middleware._profile_context_from_records(records) == (
middleware._profile_pointer_context
)
async def test_profile_memory_async_path_bootstraps_and_injects(tmp_path, monkeypatch):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
async def _handler(request):
return request
middleware = memory_module.create_memory_middleware(str(memories))
await middleware.awrap_model_call(_request(), _handler)
assert (memories / "profile" / "USER_PROFILE.md").exists()
def test_profile_memory_write_failure_uses_path_pointers(tmp_path, monkeypatch):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
monkeypatch.setattr(
memory_module.EvoMemoryMiddleware,
"_write_text",
lambda _self, _path, _content: False,
)
middleware = memory_module.create_memory_middleware(str(memories))
middleware.modify_request(_request())
assert not (memories / "profile" / "USER_PROFILE.md").exists()
def test_profile_memory_read_failure_uses_path_pointers_without_overwriting(
tmp_path, monkeypatch
):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
profile_dir = memories / "profile"
profile_dir.mkdir(parents=True)
soul_path = profile_dir / "SOUL.md"
original_bytes = b"\xff\xfe\xfa existing profile bytes"
soul_path.write_bytes(original_bytes)
middleware = memory_module.create_memory_middleware(str(memories))
middleware.modify_request(_request())
assert soul_path.read_bytes() == original_bytes
async def test_profile_memory_async_path_inlines_content_under_blockbuster(
tmp_path, monkeypatch
):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
middleware = memory_module.create_memory_middleware(str(memories))
middleware.modify_request(_request())
user_profile = memories / "profile" / "USER_PROFILE.md"
user_profile.write_text(
user_profile.read_text(encoding="utf-8")
+ "\n\n- Async profile content should be inlined.",
encoding="utf-8",
)
call_threads = []
original_read = middleware._read_profile_memory
def tracked_read_profile_memory():
call_threads.append(threading.get_ident())
return original_read()
monkeypatch.setattr(middleware, "_read_profile_memory", tracked_read_profile_memory)
event_loop_thread = threading.get_ident()
blocker = BlockBuster(scanned_modules=memory_module)
blocker.activate()
try:
modified = await middleware.amodify_request(_request())
finally:
blocker.deactivate()
assert call_threads
assert all(thread_id != event_loop_thread for thread_id in call_threads)
assert "Async profile content should be inlined." in str(
modified.system_message.content
)
def test_profile_memory_migrates_legacy_memory_once(tmp_path, monkeypatch):
memories = tmp_path / "memories"
memories.mkdir()
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
(memories / "MEMORY.md").write_text(
"\n".join(
[
"# EvoScientist Memory",
"",
"## User Profile",
"- **Name**: Alice",
"",
"## Research Preferences",
"- **Primary Domain**: RL",
"",
"## Experiment History",
"### [2026-01-01] Baseline",
"- **Conclusion**: Worked",
"",
"## Learned Preferences",
"- Prefers concise plans.",
]
),
encoding="utf-8",
)
middleware = memory_module.create_memory_middleware(str(memories))
middleware.modify_request(_request())
middleware.modify_request(_request())
user_profile = (memories / "profile" / "USER_PROFILE.md").read_text(
encoding="utf-8"
)
research_taste = (memories / "profile" / "RESEARCH_TASTE.md").read_text(
encoding="utf-8"
)
assert user_profile.count("- **Name**: Alice") == 1
assert user_profile.count("Prefers concise plans.") == 1
assert user_profile.count("### Experiment History") == 1
assert user_profile.count("- **Conclusion**: Worked") == 1
assert research_taste.count("- **Primary Domain**: RL") == 1
assert "Migrated from /memories/MEMORY.md" not in user_profile
assert "Migrated from /memories/MEMORY.md" not in research_taste
assert not (memories / "MEMORY.md").exists()
def test_profile_memory_deletes_blank_legacy_memory(tmp_path, monkeypatch):
memories = tmp_path / "memories"
memories.mkdir()
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
legacy_path = memories / "MEMORY.md"
legacy_path.write_text(" \n\n", encoding="utf-8")
middleware = memory_module.create_memory_middleware(str(memories))
middleware.modify_request(_request())
assert not legacy_path.exists()
def test_profile_memory_uses_explicit_workspace_for_project_profile(
tmp_path, monkeypatch
):
memories = tmp_path / "memories"
global_workspace = tmp_path / "global-workspace"
active_workspace = tmp_path / "active-workspace"
global_workspace.mkdir()
active_workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", global_workspace)
middleware = memory_module.create_memory_middleware(
str(memories), workspace_dir=str(active_workspace)
)
middleware.modify_request(_request())
expected_project_id = _path_project_id(active_workspace)
wrong_project_id = _path_project_id(global_workspace)
assert (
memories / "profile" / "projects" / expected_project_id / "PROJECT_PROFILE.md"
).exists()
assert not (
memories / "profile" / "projects" / wrong_project_id / "PROJECT_PROFILE.md"
).exists()
async def test_profile_memory_resolves_project_id_once_per_middleware(
tmp_path, monkeypatch
):
memories = tmp_path / "memories"
workspace = tmp_path / "workspace"
workspace.mkdir()
calls = []
def resolve_project_id(workspace_dir):
calls.append(workspace_dir)
return "P-cached-project"
monkeypatch.setattr(memory_module, "resolve_project_id", resolve_project_id)
middleware = memory_module.create_memory_middleware(
str(memories), workspace_dir=str(workspace), max_inline_profile_chars=10
)
middleware.modify_request(_request())
await middleware.amodify_request(_request())
assert calls == [workspace]
assert middleware.project_id == "P-cached-project"
assert any(
path == "/profile/projects/P-cached-project/PROJECT_PROFILE.md"
for path, _template in middleware._profile_specs
)
def test_profile_memory_preserves_unmapped_legacy_memory(tmp_path, monkeypatch):
memories = tmp_path / "memories"
memories.mkdir()
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
legacy_path = memories / "MEMORY.md"
custom_note = "Keep this custom deployment note."
legacy_path.write_text(
"\n".join(
[
"# EvoScientist Memory",
"",
"## User Profile",
"- **Name**: Alice",
"",
"## Custom Notes",
custom_note,
]
),
encoding="utf-8",
)
middleware = memory_module.create_memory_middleware(str(memories))
middleware.modify_request(_request())
user_profile = (memories / "profile" / "USER_PROFILE.md").read_text(
encoding="utf-8"
)
assert custom_note in user_profile
assert not legacy_path.exists()
def test_profile_memory_skips_legacy_unknown_placeholders(tmp_path, monkeypatch):
memories = tmp_path / "memories"
memories.mkdir()
workspace = tmp_path / "workspace"
workspace.mkdir()
monkeypatch.setattr(paths, "WORKSPACE_ROOT", workspace)
(memories / "MEMORY.md").write_text(
"\n".join(
[
"# EvoScientist Memory",
"",
"## User Profile",
"- **Name**: (unknown)",
"- **Role**: (unknown)",
"",
"## Research Preferences",
"- **Primary Domain**: (unknown)",
"- **Preferred Methods**: (unknown)",
"",
"## Experiment History",
"(No experiments yet)",
"",
"## Learned Preferences",
"- (none yet)",
]
),
encoding="utf-8",
)
middleware = memory_module.create_memory_middleware(str(memories))
middleware.modify_request(_request())
migrated_profile_text = "\n".join(_profile_texts(memories))
assert "(unknown)" not in migrated_profile_text
assert "Imported from legacy MEMORY.md" not in migrated_profile_text
assert not (memories / "MEMORY.md").exists()