ouyangbo/EvoScientist-Multi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(llm): bound routed reasoning and surface empty truncation (#425)

Browse Source 
* fix(llm): bound routed reasoning and surface empty truncation

Default DashScope Qwen 3.8 Max requests to low reasoning effort and forward explicit reasoning controls to custom OpenAI-compatible endpoints.

Detect length-limited responses that exhaust their budget during reasoning without producing content or tool calls, and surface a provider-aware error instead of ending the turn silently.

Add regression coverage for routed reasoning configuration, truncated empty responses, and valid content/tool-call responses.

* fix(middleware): reject empty structured text blocks

* fix(llm): address reasoning truncation review feedback

* fix(llm): validate DashScope reasoning effort

* fix(llm): document DashScope reasoning support
This commit is contained in:
LinxXu
2026-08-18 15:28:19 +08:00
committed by GitHub
parent 1b324906fd
commit 47c8da3e5b
6 changed files with 513 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
EvoScientist/llm/models.py
+58
View File
@@ -96,6 +96,63 @@ def _resolve_reasoning_effort(default: str) -> str:
return os.environ.get("EVOSCIENTIST_REASONING_EFFORT", "").strip() or default
# Qwen 3.8 Max canonical levels and documented OpenAI alias mappings:
# https://docs.qwencloud.com/api-reference/chat/openai-chat#reasoning-effort
_DASHSCOPE_QWEN38_REASONING_EFFORTS = frozenset(
{"none", "minimal", "low", "medium", "high", "xhigh", "max"}
)
def _validate_dashscope_reasoning_effort(
provider: str,
model_id: str,
effort: str,
) -> None:
"""Reject reasoning levels unsupported by DashScope Qwen 3.8 Max."""
if effort not in _DASHSCOPE_QWEN38_REASONING_EFFORTS:
choices = ", ".join(sorted(_DASHSCOPE_QWEN38_REASONING_EFFORTS))
raise ValueError(
f"Unsupported EVOSCIENTIST_REASONING_EFFORT={effort!r} for "
f"{provider} model {model_id!r}. Supported values: {choices}."
)
def _apply_openai_compat_reasoning_config(
provider: str,
model_id: str,
kwargs: dict[str, Any],
) -> None:
"""Apply reasoning controls supported by OpenAI-compatible providers.
Routed providers deliberately skip the native-OpenAI branch in
:func:`_apply_auto_config`, because most compatible endpoints reject
OpenAI-only ``reasoning`` payloads. A small subset does support the
standard ``reasoning_effort`` field, though:
* DashScope Qwen 3.8 Max supports ``low`` / ``medium`` / ``xhigh`` and
maps the OpenAI aliases (including ``none``). Its server default is
extremely large, so use the standard ``medium`` level unless the user
selected another level.
* ``custom-openai`` is user-owned. Forward an *explicit* setting only;
with no setting, preserve compatibility with endpoints that reject the
field (including many non-reasoning OpenAI-compatible APIs).
Explicit caller kwargs always win.
"""
configured = os.environ.get("EVOSCIENTIST_REASONING_EFFORT", "").strip()
short_model_id = model_id.rsplit("/", 1)[-1]
if provider == "dashscope" and short_model_id.startswith("qwen3.8-max"):
if "reasoning_effort" not in kwargs:
effort = configured or "medium"
_validate_dashscope_reasoning_effort(provider, model_id, effort)
kwargs["reasoning_effort"] = effort
return
if provider == "custom-openai" and configured:
kwargs.setdefault("reasoning_effort", configured)
def _is_deepseek_endpoint(base_url: str | None) -> bool:
"""Return whether an OpenAI-compatible endpoint is DeepSeek's API."""
if not base_url:
@@ -396,6 +453,7 @@ def get_chat_model(
api_key = os.environ.get(api_key_env, "")
if api_key:
kwargs["api_key"] = api_key
_apply_openai_compat_reasoning_config(provider, model_id, kwargs)
# SiliconFlow: disable thinking — LangChain drops reasoning_content
# from history, causing error 20015 on multi-turn requests.
if provider == "siliconflow":
EvoScientist/middleware/error_normalization.py
+106 -2
View File
@@ -46,6 +46,110 @@ if TYPE_CHECKING:
from ..llm.errors import ProviderStreamError
class ModelOutputTruncatedError(RuntimeError):
"""The provider exhausted its output budget before producing an answer."""
_TRUNCATED_FINISH_REASONS = frozenset(
{
"length",
"max_tokens",
"max_output_tokens",
"max_completion_tokens",
"incomplete",
}
)
def _has_answer_content(content: object) -> bool:
"""Return whether message content contains something beyond reasoning."""
if isinstance(content, str):
return bool(content.strip())
if not isinstance(content, list):
return content is not None
reasoning_types = {
"thinking",
"redacted_thinking",
"reasoning",
"reasoning_content",
}
text_types = {"text", "output_text"}
for block in content:
if isinstance(block, str):
if block.strip():
return True
continue
if not isinstance(block, dict):
return True
block_type = str(block.get("type", "")).lower()
if block_type in reasoning_types:
continue
if block_type in text_types:
text = block.get("text")
if isinstance(text, str):
if text.strip():
return True
elif text:
return True
continue
# Any non-reasoning block is meaningful output (text, image, refusal,
# server tool result, etc.), even when its provider-specific payload
# does not use a ``text`` key.
return True
return False
def _truncated_empty_message(response: ModelResponse):
"""Return the empty truncated AI message in *response*, if present."""
from langchain_core.messages import AIMessage
if getattr(response, "structured_response", None) is not None:
return None
messages = getattr(response, "result", None) or []
message = next(
(item for item in reversed(messages) if isinstance(item, AIMessage)), None
)
if message is None:
return None
if _has_answer_content(message.content):
return None
if message.tool_calls or getattr(message, "invalid_tool_calls", None):
return None
metadata = message.response_metadata or {}
reasons = {
str(metadata.get(key, "")).strip().lower()
for key in ("finish_reason", "stop_reason", "status")
}
incomplete_details = metadata.get("incomplete_details")
if isinstance(incomplete_details, dict):
reasons.add(str(incomplete_details.get("reason", "")).strip().lower())
if reasons.isdisjoint(_TRUNCATED_FINISH_REASONS):
return None
return message
def _check_truncated_output(response: ModelResponse) -> ModelResponse:
"""Raise a visible error instead of silently accepting an empty answer."""
message = _truncated_empty_message(response)
if message is None:
return response
metadata = message.response_metadata or {}
reason = (
metadata.get("finish_reason")
or metadata.get("stop_reason")
or metadata.get("status")
or "output limit"
)
raise ModelOutputTruncatedError(
"The model exhausted its output budget during reasoning and returned "
f"no answer (finish reason: {reason}). Lower reasoning_effort, disable "
"reasoning with none when supported, or increase the provider "
"output-token limit."
)
def _should_pass_through(exc: BaseException) -> bool:
"""True if *exc* is a LangGraph-level signal that must propagate
untouched — either a control-flow signal or a structural error
@@ -210,7 +314,7 @@ class ErrorNormalizationMiddleware(AgentMiddleware):
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelResponse:
try:
return handler(request)
return _check_truncated_output(handler(request))
except Exception as exc:
normalized = _normalize(request, exc)
if normalized is None:
@@ -223,7 +327,7 @@ class ErrorNormalizationMiddleware(AgentMiddleware):
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelResponse:
try:
return await handler(request)
return _check_truncated_output(await handler(request))
except Exception as exc:
normalized = _normalize(request, exc)
if normalized is None:
EvoScientist/middleware/model_fallback.py
+16 -4
View File
@@ -447,10 +447,16 @@ class ModelFallbackMiddleware(AgentMiddleware):
) -> ModelResponse:
if not _fallback_chain:
return handler(request)
from .error_normalization import _check_truncated_output
def invoke(current_request: ModelRequest) -> ModelResponse:
return _check_truncated_output(handler(current_request))
try:
return handler(request)
return invoke(request)
except Exception as exc:
return _guard_and_fallback_sync(exc, request, handler, self._events)
return _guard_and_fallback_sync(exc, request, invoke, self._events)
async def awrap_model_call(
self,
@@ -459,7 +465,13 @@ class ModelFallbackMiddleware(AgentMiddleware):
) -> ModelResponse:
if not _fallback_chain:
return await handler(request)
from .error_normalization import _check_truncated_output
async def invoke(current_request: ModelRequest) -> ModelResponse:
return _check_truncated_output(await handler(current_request))
try:
return await handler(request)
return await invoke(request)
except Exception as exc:
return await _guard_and_fallback(exc, request, handler, self._events)
return await _guard_and_fallback(exc, request, invoke, self._events)
tests/test_error_normalization_middleware.py
+116
View File
@@ -14,10 +14,13 @@ import dataclasses
from types import SimpleNamespace
import pytest
from langchain.agents.middleware.types import ModelResponse
from langchain_core.messages import AIMessage
from EvoScientist.llm.errors import ProviderStreamError
from EvoScientist.middleware.error_normalization import (
ErrorNormalizationMiddleware,
ModelOutputTruncatedError,
_normalize,
)
@@ -421,3 +424,116 @@ class TestMiddleware:
req = _request(_openrouter_model())
mw = ErrorNormalizationMiddleware()
assert self._run_awrap(mw, req, handler) == "ok"
def test_empty_length_response_becomes_visible_provider_error(self):
"""Reasoning-only truncation must not look like a successful idle turn."""
response = ModelResponse(
result=[
AIMessage(
content="",
additional_kwargs={"reasoning_content": "still thinking"},
response_metadata={"finish_reason": "length"},
)
]
)
def handler(_req):
return response
req = _request(_openai_model(base_url="https://internal.corp/v1"))
with pytest.raises(ProviderStreamError) as excinfo:
ErrorNormalizationMiddleware().wrap_model_call(req, handler)
assert excinfo.value.provider == "openai_compat"
assert isinstance(excinfo.value.__cause__, ModelOutputTruncatedError)
assert "reasoning_effort" in str(excinfo.value)
def test_empty_incomplete_responses_api_result_is_detected(self):
response = ModelResponse(
result=[
AIMessage(
content=[],
response_metadata={
"status": "incomplete",
"incomplete_details": {"reason": "max_output_tokens"},
},
)
]
)
async def handler(_req):
return response
req = _request(_openai_model())
with pytest.raises(ProviderStreamError):
self._run_awrap(ErrorNormalizationMiddleware(), req, handler)
def test_empty_structured_text_block_with_length_is_detected(self):
response = ModelResponse(
result=[
AIMessage(
content=[{"type": "text", "text": " "}],
response_metadata={"finish_reason": "length"},
)
]
)
def handler(_req):
return response
with pytest.raises(ProviderStreamError) as excinfo:
ErrorNormalizationMiddleware().wrap_model_call(
_request(_openai_model()), handler
)
assert isinstance(excinfo.value.__cause__, ModelOutputTruncatedError)
def test_redacted_thinking_only_with_max_tokens_is_detected(self):
response = ModelResponse(
result=[
AIMessage(
content=[{"type": "redacted_thinking", "data": "opaque-payload"}],
response_metadata={"stop_reason": "max_tokens"},
)
]
)
def handler(_req):
return response
with pytest.raises(ProviderStreamError) as excinfo:
ErrorNormalizationMiddleware().wrap_model_call(
_request(_anthropic_model()), handler
)
assert isinstance(excinfo.value.__cause__, ModelOutputTruncatedError)
@pytest.mark.parametrize(
"message",
[
AIMessage(content="answer", response_metadata={"finish_reason": "length"}),
AIMessage(
content="",
tool_calls=[{"name": "search", "args": {}, "id": "call-1"}],
response_metadata={"finish_reason": "length"},
),
AIMessage(content="", response_metadata={"finish_reason": "stop"}),
AIMessage(
content=[
{"type": "redacted_thinking", "data": "opaque-payload"},
{"type": "text", "text": "answer"},
],
response_metadata={"stop_reason": "max_tokens"},
),
],
)
def test_nonempty_tool_and_normal_stop_responses_are_not_rejected(self, message):
response = ModelResponse(result=[message])
def handler(_req):
return response
result = ErrorNormalizationMiddleware().wrap_model_call(
_request(_openai_model()), handler
)
assert result is response
tests/test_llm.py
+112
View File
@@ -1077,6 +1077,34 @@ class TestThirdPartyRouting:
assert call_kwargs["base_url"] == "https://my-llm.example.com/v1"
assert call_kwargs["api_key"] == "custom-key-789"
@patch("EvoScientist.llm.models.init_chat_model")
def test_custom_openai_forwards_explicit_reasoning_effort(
self, mock_init, monkeypatch
):
"""User-owned compatible endpoints receive an explicit effort only."""
mock_init.return_value = "mock_model"
monkeypatch.setenv("CUSTOM_OPENAI_BASE_URL", "https://opencode.example/v1")
monkeypatch.setenv("CUSTOM_OPENAI_API_KEY", "custom-key")
monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", "low")
get_chat_model("reasoning-model", provider="custom-openai")
assert mock_init.call_args[1]["reasoning_effort"] == "low"
@patch("EvoScientist.llm.models.init_chat_model")
def test_custom_openai_omits_unconfigured_reasoning_effort(
self, mock_init, monkeypatch
):
"""Unknown compatible endpoints stay compatible by default."""
mock_init.return_value = "mock_model"
monkeypatch.setenv("CUSTOM_OPENAI_BASE_URL", "https://plain.example/v1")
monkeypatch.setenv("CUSTOM_OPENAI_API_KEY", "custom-key")
monkeypatch.delenv("EVOSCIENTIST_REASONING_EFFORT", raising=False)
get_chat_model("plain-model", provider="custom-openai")
assert "reasoning_effort" not in mock_init.call_args[1]
@patch("EvoScientist.llm.models.init_chat_model")
def test_anthropic_base_url_override(self, mock_init, monkeypatch):
"""Anthropic provider should support base_url override (e.g. ccproxy)."""
@@ -1167,6 +1195,90 @@ class TestThirdPartyRouting:
== "https://dashscope.aliyuncs.com/compatible-mode/v1"
)
assert call_kwargs["api_key"] == "ds-key-456"
assert "reasoning_effort" not in call_kwargs
@patch("EvoScientist.llm.models.init_chat_model")
def test_qwen38_dashscope_uses_bounded_default(self, mock_init, monkeypatch):
"""Qwen 3.8 avoids the regular endpoint's xhigh default."""
mock_init.return_value = "mock_model"
monkeypatch.setenv("DASHSCOPE_API_KEY", "ds-key")
monkeypatch.delenv("EVOSCIENTIST_REASONING_EFFORT", raising=False)
get_chat_model("qwen3.8-max", provider="dashscope")
assert mock_init.call_args[1]["reasoning_effort"] == "medium"
@patch("EvoScientist.llm.models.init_chat_model")
def test_qwen38_dashscope_respects_configured_reasoning_effort(
self, mock_init, monkeypatch
):
mock_init.return_value = "mock_model"
monkeypatch.setenv("DASHSCOPE_API_KEY", "ds-key")
monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", "medium")
get_chat_model("qwen3.8-max", provider="dashscope")
assert mock_init.call_args[1]["reasoning_effort"] == "medium"
@pytest.mark.parametrize(
"effort",
[
"none",
"minimal",
"low",
"medium",
"high",
"xhigh",
"max",
],
)
@patch("EvoScientist.llm.models.init_chat_model")
def test_qwen38_dashscope_accepts_supported_reasoning_effort(
self, mock_init, effort, monkeypatch
):
mock_init.return_value = "mock_model"
monkeypatch.setenv("DASHSCOPE_API_KEY", "ds-key")
monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", effort)
get_chat_model("qwen3.8-max", provider="dashscope")
assert mock_init.call_args[1]["reasoning_effort"] == effort
@patch("EvoScientist.llm.models.init_chat_model")
def test_qwen38_dashscope_rejects_unsupported_reasoning_effort(
self, mock_init, monkeypatch
):
monkeypatch.setenv("DASHSCOPE_API_KEY", "ds-key")
monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", "invalid")
with pytest.raises(ValueError, match="dashscope"):
get_chat_model("qwen3.8-max", provider="dashscope")
mock_init.assert_not_called()
@patch("EvoScientist.llm.models.init_chat_model")
def test_qwen38_dashscope_explicit_effort_overrides_invalid_environment(
self, mock_init, monkeypatch
):
mock_init.return_value = "mock_model"
monkeypatch.setenv("DASHSCOPE_API_KEY", "ds-key")
monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", "invalid")
get_chat_model("qwen3.8-max", provider="dashscope", reasoning_effort="low")
assert mock_init.call_args[1]["reasoning_effort"] == "low"
@patch("EvoScientist.llm.models.init_chat_model")
def test_qwen38_dashscope_code_omits_undocumented_reasoning_effort(
self, mock_init, monkeypatch
):
mock_init.return_value = "mock_model"
monkeypatch.setenv("DASHSCOPE_API_KEY", "sk-sp-key")
monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", "medium")
get_chat_model("qwen3.8-max", provider="dashscope-code")
assert "reasoning_effort" not in mock_init.call_args[1]
@patch("EvoScientist.llm.models.init_chat_model")
def test_dashscope_code_routes_through_openai(self, mock_init, monkeypatch):
tests/test_model_fallback.py
+105
View File
@@ -10,6 +10,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from langchain.agents.middleware.types import ModelResponse
from langchain_core.exceptions import ContextOverflowError
from langchain_core.messages import AIMessage, HumanMessage
@@ -42,6 +43,22 @@ def _fake_request():
AI_RESPONSE = AIMessage(content="ok")
def _truncated_response() -> ModelResponse:
return ModelResponse(
result=[
AIMessage(
content="",
additional_kwargs={"reasoning_content": "still thinking"},
response_metadata={"finish_reason": "length"},
)
]
)
def _successful_response() -> ModelResponse:
return ModelResponse(result=[AIMessage(content="ok")])
@pytest.fixture(autouse=True)
def _clean_chain():
"""Ensure a clean fallback chain for every test."""
@@ -454,6 +471,94 @@ class TestSynchronousFallback:
assert result is response
assert handler.call_count == 2
def test_truncated_primary_response_uses_fallback(self):
from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware
add_fallback("fb", "prov")
req = _fake_request()
response = _successful_response()
handler = MagicMock(side_effect=[_truncated_response(), response])
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = ModelFallbackMiddleware().wrap_model_call(req, handler)
assert result is response
assert handler.call_count == 2
class TestTruncatedResponseFallback:
"""Empty truncated model results must participate in the fallback chain."""
async def test_truncated_primary_response_uses_fallback(self):
from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware
add_fallback("fb", "prov")
req = _fake_request()
response = _successful_response()
handler = AsyncMock(side_effect=[_truncated_response(), response])
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = await ModelFallbackMiddleware().awrap_model_call(req, handler)
assert result is response
assert handler.await_count == 2
async def test_truncated_fallback_continues_to_next_model(self):
from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware
add_fallback("fb-a", "prov-a")
add_fallback("fb-b", "prov-b")
req = _fake_request()
response = _successful_response()
handler = AsyncMock(
side_effect=[_truncated_response(), _truncated_response(), response]
)
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = MagicMock()
result = await ModelFallbackMiddleware().awrap_model_call(req, handler)
assert result is response
assert handler.await_count == 3
assert mock_gcm.call_count == 2
async def test_exhausted_truncated_fallbacks_use_last_provider(self):
from EvoScientist.llm.errors import ProviderStreamError
from EvoScientist.middleware.error_normalization import (
ModelOutputTruncatedError,
)
from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware
def _make_openai_model(base_url=None):
cls = type(
"ChatOpenAI",
(),
{"__module__": "langchain_openai.chat_models.base"},
)
model = cls()
model.openai_api_base = base_url
return model
add_fallback("moonshot-model", "moonshot")
req = _fake_request()
req.model = _make_openai_model()
fallback_model = _make_openai_model(base_url="https://api.moonshot.cn/v1")
req.override = MagicMock(
side_effect=lambda **kw: SimpleNamespace(model=kw.get("model", req.model))
)
handler = AsyncMock(return_value=_truncated_response())
with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm:
mock_gcm.return_value = fallback_model
with pytest.raises(ProviderStreamError) as exc_info:
await ModelFallbackMiddleware().awrap_model_call(req, handler)
assert exc_info.value.provider == "moonshot"
assert isinstance(exc_info.value.__cause__, ModelOutputTruncatedError)
assert handler.await_count == 2
# ═════════════════════════════════════════════════════════════════
# 4. UI emit callback