ouyangbo/EvoScientist-Multi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: bring upstream v0.3.0 (72 commits) into Ai4Sci fork

Browse Source 
Merged upstream/main (418abca, release v0.3.0) into our fork on a
dedicated branch. 21 conflicting files resolved; main worktree untouched.

Resolution policy and key decisions:
- Keep Ai4Sci runtime endpoints, durable dispatch, workspace scopes and
  the HITL/DynamicReview approval chain (approval path is product-critical).
- Adopt upstream model registry (llm/registry.py): our 136 model entries
  are a strict subset of upstream's 180, so dropping our inline table
  loses nothing and gains 44 new models.
- Adopt upstream native EvoChatDeepSeek; drop our obsolete
  _patch_deepseek_reasoning_passback monkey patch.
- Keep our six patches.py additions, ported onto upstream's new
  _OpenAICompatContent class: stable tool-call ids, tool-history
  sanitization, drop_reasoning_metadata, empty-SSE keepalive,
  extracted-document-text patch, _has_assistant_tool_protocol.
- Keep our skill-budget middleware path (skills=None) instead of passing
  skills through, to avoid double loading.
- Keep sanitized error labels (_safe_error_label) while adopting
  upstream's injected MiddlewareEventSink for fallback narration.
- Keep port 3076 and the LANGGRAPH_SERVER_URL override; adopt upstream's
  host/probe-host handling and CONFIG_DRIFT_SINCE_LAUNCH.
- Adopt upstream dependency stack: deepagents 0.7.6, langchain-quickjs
  0.3.7, langgraph-api 0.14; keep our extra deps (rfc8785, pillow,
  firecrawl-anydoc, nest-asyncio).
- Align call sites with upstream APIs: create_tool_selector_middleware
  now takes events= instead of track_stream_selection=.
This commit is contained in:
m4
2026-09-13 16:07:27 +08:00
parent d4b53bfb08 418abca4f4
commit 470cf75722
220 changed files with 36430 additions and 7256 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_model_fallback.py
+259 -28
View File
@@ -10,20 +10,27 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from langchain.agents.middleware.types import ModelResponse
from langchain_core.exceptions import ContextOverflowError
from langchain_core.messages import AIMessage, HumanMessage
from EvoScientist.middleware.events import NoOpSink
from EvoScientist.middleware.model_fallback import (
_guard_and_fallback,
_guard_and_fallback_sync,
_is_non_fallbackable,
_try_fallbacks,
_try_fallbacks_sync,
add_fallback,
clear_fallbacks,
set_ui_emit,
)
from EvoScientist.stream.sink import SessionEventSink
# ── Helpers ──────────────────────────────────────────────────────
# Silent sink for tests that don't assert on the fallback narration.
_SINK = NoOpSink()
def _fake_request():
"""Build a minimal ModelRequest stub with an .override() method."""
@@ -36,14 +43,28 @@ def _fake_request():
AI_RESPONSE = AIMessage(content="ok")
def _truncated_response() -> ModelResponse:
return ModelResponse(
result=[
AIMessage(
content="",
additional_kwargs={"reasoning_content": "still thinking"},
response_metadata={"finish_reason": "length"},
)
]
)
def _successful_response() -> ModelResponse:
return ModelResponse(result=[AIMessage(content="ok")])
@pytest.fixture(autouse=True)
def _clean_chain():
"""Ensure a clean fallback chain and no UI callback for every test."""
"""Ensure a clean fallback chain for every test."""
clear_fallbacks()
set_ui_emit(None)
yield
clear_fallbacks()
set_ui_emit(None)
# ═════════════════════════════════════════════════════════════════
@@ -155,7 +176,7 @@ class TestTryFallbacks:
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = await _try_fallbacks(req, invoke, Exception("503 boom"))
result = await _try_fallbacks(req, invoke, Exception("503 boom"), _SINK)
assert result is AI_RESPONSE
invoke.assert_awaited_once()
@@ -166,18 +187,21 @@ class TestTryFallbacks:
add_fallback("fb-model", "fb-provider")
req = _fake_request()
invoke = AsyncMock(return_value=AI_RESPONSE)
emitted: list[str] = []
set_ui_emit(lambda message, _style: emitted.append(message))
messages: list[tuple[str, str]] = []
sink = SessionEventSink(
fallback_display=lambda text, style: messages.append((text, style))
)
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
await _try_fallbacks(
req,
invoke,
Exception("provider body contains sk-live-do-not-log"),
Exception("provider body contains sk-liv...-log"),
sink,
)
output = "\n".join(emitted)
output = "\n".join(text for text, _style in messages)
assert "sk-live-do-not-log" not in output
assert "Exception" in output
@@ -198,7 +222,7 @@ class TestTryFallbacks:
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = await _try_fallbacks(req, _invoke, Exception("503 boom"))
result = await _try_fallbacks(req, _invoke, Exception("503 boom"), _SINK)
assert result is AI_RESPONSE
assert call_count == 2
@@ -223,7 +247,7 @@ class TestTryFallbacks:
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
with pytest.raises(Exception, match="429 from fb-b") as exc_info:
await _try_fallbacks(req, _invoke, Exception("503 primary"))
await _try_fallbacks(req, _invoke, Exception("503 primary"), _SINK)
assert exc_info.value is last_error
@@ -239,7 +263,7 @@ class TestTryFallbacks:
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
with pytest.raises(Exception, match="context_length_exceeded"):
await _try_fallbacks(req, _invoke, Exception("503 primary"))
await _try_fallbacks(req, _invoke, Exception("503 primary"), _SINK)
# get_chat_model should only have been called once (for fb-a),
# fb-b should never be reached.
@@ -285,7 +309,12 @@ class TestTryFallbacks:
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = fallback_model
with pytest.raises(ProviderStreamError) as exc_info:
await _try_fallbacks(req, _invoke, Exception("openai primary failed"))
await _try_fallbacks(
req,
_invoke,
Exception("openai primary failed"),
_SINK,
)
# Attribution flipped to moonshot (the failing fallback), not
# openai (the original request's model).
@@ -326,7 +355,12 @@ class TestTryFallbacks:
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = model
with pytest.raises(InvalidUpdateError) as exc_info:
await _try_fallbacks(req, _invoke, Exception("primary failed"))
await _try_fallbacks(
req,
_invoke,
Exception("primary failed"),
_SINK,
)
assert exc_info.value is raised
@@ -344,7 +378,9 @@ class TestGuardAndFallback:
invoke = AsyncMock()
with pytest.raises(ContextOverflowError):
await _guard_and_fallback(ContextOverflowError("overflow"), req, invoke)
await _guard_and_fallback(
ContextOverflowError("overflow"), req, invoke, _SINK
)
invoke.assert_not_awaited()
@@ -371,7 +407,7 @@ class TestGuardAndFallback:
raised = ContextOverflowError("context length exceeded")
with pytest.raises(ContextOverflowError) as exc_info:
await _guard_and_fallback(raised, req, invoke)
await _guard_and_fallback(raised, req, invoke, _SINK)
assert exc_info.value is raised
invoke.assert_not_awaited()
@@ -383,7 +419,7 @@ class TestGuardAndFallback:
with pytest.raises(Exception, match="invalid_request_error"):
await _guard_and_fallback(
Exception("400: invalid_request_error"), req, invoke
Exception("400: invalid_request_error"), req, invoke, _SINK
)
invoke.assert_not_awaited()
@@ -395,7 +431,9 @@ class TestGuardAndFallback:
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = await _guard_and_fallback(Exception("503 overloaded"), req, invoke)
result = await _guard_and_fallback(
Exception("503 overloaded"), req, invoke, _SINK
)
assert result is AI_RESPONSE
invoke.assert_awaited_once()
@@ -409,48 +447,241 @@ class TestGuardAndFallback:
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = await _guard_and_fallback(
Exception("400 Bad Request: invalid_api_key"), req, invoke
Exception("400 Bad Request: invalid_api_key"), req, invoke, _SINK
)
assert result is AI_RESPONSE
invoke.assert_awaited_once()
class TestSynchronousFallback:
"""The sync middleware path must not create or nest an event loop."""
def test_first_fallback_succeeds_without_async_bridge(self):
add_fallback("fb-model", "fb-provider")
req = _fake_request()
invoke = MagicMock(return_value=AI_RESPONSE)
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = _try_fallbacks_sync(req, invoke, Exception("503 boom"), _SINK)
assert result is AI_RESPONSE
invoke.assert_called_once()
def test_guard_rejects_non_fallbackable_error_before_handler(self):
add_fallback("fb", "prov")
req = _fake_request()
invoke = MagicMock()
with pytest.raises(ContextOverflowError):
_guard_and_fallback_sync(
ContextOverflowError("overflow"), req, invoke, _SINK
)
invoke.assert_not_called()
def test_middleware_sync_entrypoint_uses_native_traversal(self):
from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware
add_fallback("fb", "prov")
req = _fake_request()
response = AI_RESPONSE
handler = MagicMock(side_effect=[Exception("503 primary"), response])
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = ModelFallbackMiddleware().wrap_model_call(req, handler)
assert result is response
assert handler.call_count == 2
def test_truncated_primary_response_uses_fallback(self):
from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware
add_fallback("fb", "prov")
req = _fake_request()
response = _successful_response()
handler = MagicMock(side_effect=[_truncated_response(), response])
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = ModelFallbackMiddleware().wrap_model_call(req, handler)
assert result is response
assert handler.call_count == 2
class TestTruncatedResponseFallback:
"""Empty truncated model results must participate in the fallback chain."""
async def test_truncated_primary_response_uses_fallback(self):
from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware
add_fallback("fb", "prov")
req = _fake_request()
response = _successful_response()
handler = AsyncMock(side_effect=[_truncated_response(), response])
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = await ModelFallbackMiddleware().awrap_model_call(req, handler)
assert result is response
assert handler.await_count == 2
async def test_truncated_fallback_continues_to_next_model(self):
from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware
add_fallback("fb-a", "prov-a")
add_fallback("fb-b", "prov-b")
req = _fake_request()
response = _successful_response()
handler = AsyncMock(
side_effect=[_truncated_response(), _truncated_response(), response]
)
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = await ModelFallbackMiddleware().awrap_model_call(req, handler)
assert result is response
assert handler.await_count == 3
assert mock_gcm.call_count == 2
async def test_exhausted_truncated_fallbacks_use_last_provider(self):
from EvoScientist.llm.errors import ProviderStreamError
from EvoScientist.middleware.error_normalization import (
ModelOutputTruncatedError,
)
from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware
def _make_openai_model(base_url=None):
cls = type(
"ChatOpenAI",
(),
{"__module__": "langchain_openai.chat_models.base"},
)
model = cls()
model.openai_api_base = base_url
return model
add_fallback("moonshot-model", "moonshot")
req = _fake_request()
req.model = _make_openai_model()
fallback_model = _make_openai_model(base_url="https://api.moonshot.cn/v1")
req.override = MagicMock(
side_effect=lambda **kw: SimpleNamespace(model=kw.get("model", req.model))
)
handler = AsyncMock(return_value=_truncated_response())
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = fallback_model
with pytest.raises(ProviderStreamError) as exc_info:
await ModelFallbackMiddleware().awrap_model_call(req, handler)
assert exc_info.value.provider == "moonshot"
assert isinstance(exc_info.value.__cause__, ModelOutputTruncatedError)
assert handler.await_count == 2
# ═════════════════════════════════════════════════════════════════
# 4. UI emit callback
# ═════════════════════════════════════════════════════════════════
class TestUiEmit:
"""Verify that fallback events are surfaced via the registered callback."""
"""Verify that fallback narration reaches the injected frontend sink.
The fallback middleware sends its narration lines through the same
``fallback_display`` callback the frontend supplies, so capturing that
callback exercises the exact user-facing text.
"""
def _capturing_sink(self):
messages: list[tuple[str, str]] = []
sink = SessionEventSink(
fallback_display=lambda text, style: messages.append((text, style))
)
return sink, messages
async def test_emit_captures_messages(self):
add_fallback("fb", "prov")
req = _fake_request()
invoke = AsyncMock(return_value=AI_RESPONSE)
messages: list[tuple[str, str]] = []
set_ui_emit(lambda text, style: messages.append((text, style)))
sink, messages = self._capturing_sink()
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
await _try_fallbacks(req, invoke, Exception("503 down"))
await _try_fallbacks(req, invoke, Exception("503 down"), sink)
texts = [t for t, _ in messages]
assert any("Primary model failed" in t for t in texts)
assert any("Falling back to fb (prov)" in t for t in texts)
assert any("succeeded" in t for t in texts)
async def test_default_sink_prints_to_console(self):
add_fallback("fb", "prov")
req = _fake_request()
invoke = AsyncMock(return_value=AI_RESPONSE)
sink = SessionEventSink()
with (
patch("EvoScientist.llm.models.get_chat_model") as mock_gcm,
patch("EvoScientist.stream.sink.console.print") as mock_print,
):
mock_gcm.return_value = MagicMock()
await _try_fallbacks(req, invoke, Exception("503 down"), sink)
texts = [call.args[0] for call in mock_print.call_args_list]
assert any("Primary model failed" in t for t in texts)
assert any("Falling back to fb (prov)" in t for t in texts)
assert any("succeeded" in t for t in texts)
assert all(
call.kwargs == {"style": "yellow"} for call in mock_print.call_args_list[:2]
)
async def test_display_failure_does_not_abort_fallback(self, caplog):
add_fallback("fb", "prov")
req = _fake_request()
invoke = AsyncMock(return_value=AI_RESPONSE)
sink = SessionEventSink(
fallback_display=MagicMock(side_effect=RuntimeError("ui unavailable"))
)
with (
patch("EvoScientist.llm.models.get_chat_model") as mock_gcm,
patch("EvoScientist.stream.sink.console.print") as mock_print,
):
mock_gcm.return_value = MagicMock()
result = await _try_fallbacks(req, invoke, Exception("503 down"), sink)
assert result is AI_RESPONSE
invoke.assert_awaited_once()
assert "Fallback display callback failed" in caplog.text
texts = [call.args[0] for call in mock_print.call_args_list]
assert any("Primary model failed" in t for t in texts)
assert any("Falling back to fb (prov)" in t for t in texts)
assert any("succeeded" in t for t in texts)
def test_noopsink_keeps_fallback_notices_silent(self):
sink = NoOpSink()
with patch("EvoScientist.stream.sink.console.print") as mock_print:
sink.emit_fallback_notice("hidden")
mock_print.assert_not_called()
async def test_emit_shows_non_fallbackable_rejection(self):
add_fallback("fb", "prov")
req = _fake_request()
invoke = AsyncMock()
messages: list[tuple[str, str]] = []
set_ui_emit(lambda text, style: messages.append((text, style)))
sink, messages = self._capturing_sink()
with pytest.raises(ContextOverflowError):
await _guard_and_fallback(ContextOverflowError("overflow"), req, invoke)
await _guard_and_fallback(
ContextOverflowError("overflow"), req, invoke, sink
)
texts = [t for t, _ in messages]
assert any("not eligible for fallback" in t for t in texts)