diff --git a/EvoScientist/llm/models.py b/EvoScientist/llm/models.py index 7096d29..a4d66af 100644 --- a/EvoScientist/llm/models.py +++ b/EvoScientist/llm/models.py @@ -96,6 +96,63 @@ def _resolve_reasoning_effort(default: str) -> str: return os.environ.get("EVOSCIENTIST_REASONING_EFFORT", "").strip() or default +# Qwen 3.8 Max canonical levels and documented OpenAI alias mappings: +# https://docs.qwencloud.com/api-reference/chat/openai-chat#reasoning-effort +_DASHSCOPE_QWEN38_REASONING_EFFORTS = frozenset( + {"none", "minimal", "low", "medium", "high", "xhigh", "max"} +) + + +def _validate_dashscope_reasoning_effort( + provider: str, + model_id: str, + effort: str, +) -> None: + """Reject reasoning levels unsupported by DashScope Qwen 3.8 Max.""" + if effort not in _DASHSCOPE_QWEN38_REASONING_EFFORTS: + choices = ", ".join(sorted(_DASHSCOPE_QWEN38_REASONING_EFFORTS)) + raise ValueError( + f"Unsupported EVOSCIENTIST_REASONING_EFFORT={effort!r} for " + f"{provider} model {model_id!r}. Supported values: {choices}." + ) + + +def _apply_openai_compat_reasoning_config( + provider: str, + model_id: str, + kwargs: dict[str, Any], +) -> None: + """Apply reasoning controls supported by OpenAI-compatible providers. + + Routed providers deliberately skip the native-OpenAI branch in + :func:`_apply_auto_config`, because most compatible endpoints reject + OpenAI-only ``reasoning`` payloads. A small subset does support the + standard ``reasoning_effort`` field, though: + + * DashScope Qwen 3.8 Max supports ``low`` / ``medium`` / ``xhigh`` and + maps the OpenAI aliases (including ``none``). Its server default is + extremely large, so use the standard ``medium`` level unless the user + selected another level. + * ``custom-openai`` is user-owned. Forward an *explicit* setting only; + with no setting, preserve compatibility with endpoints that reject the + field (including many non-reasoning OpenAI-compatible APIs). + + Explicit caller kwargs always win. + """ + configured = os.environ.get("EVOSCIENTIST_REASONING_EFFORT", "").strip() + short_model_id = model_id.rsplit("/", 1)[-1] + + if provider == "dashscope" and short_model_id.startswith("qwen3.8-max"): + if "reasoning_effort" not in kwargs: + effort = configured or "medium" + _validate_dashscope_reasoning_effort(provider, model_id, effort) + kwargs["reasoning_effort"] = effort + return + + if provider == "custom-openai" and configured: + kwargs.setdefault("reasoning_effort", configured) + + def _is_deepseek_endpoint(base_url: str | None) -> bool: """Return whether an OpenAI-compatible endpoint is DeepSeek's API.""" if not base_url: @@ -396,6 +453,7 @@ def get_chat_model( api_key = os.environ.get(api_key_env, "") if api_key: kwargs["api_key"] = api_key + _apply_openai_compat_reasoning_config(provider, model_id, kwargs) # SiliconFlow: disable thinking — LangChain drops reasoning_content # from history, causing error 20015 on multi-turn requests. if provider == "siliconflow": diff --git a/EvoScientist/middleware/error_normalization.py b/EvoScientist/middleware/error_normalization.py index 895646f..1ddce60 100644 --- a/EvoScientist/middleware/error_normalization.py +++ b/EvoScientist/middleware/error_normalization.py @@ -46,6 +46,110 @@ if TYPE_CHECKING: from ..llm.errors import ProviderStreamError +class ModelOutputTruncatedError(RuntimeError): + """The provider exhausted its output budget before producing an answer.""" + + +_TRUNCATED_FINISH_REASONS = frozenset( + { + "length", + "max_tokens", + "max_output_tokens", + "max_completion_tokens", + "incomplete", + } +) + + +def _has_answer_content(content: object) -> bool: + """Return whether message content contains something beyond reasoning.""" + if isinstance(content, str): + return bool(content.strip()) + if not isinstance(content, list): + return content is not None + + reasoning_types = { + "thinking", + "redacted_thinking", + "reasoning", + "reasoning_content", + } + text_types = {"text", "output_text"} + for block in content: + if isinstance(block, str): + if block.strip(): + return True + continue + if not isinstance(block, dict): + return True + block_type = str(block.get("type", "")).lower() + if block_type in reasoning_types: + continue + if block_type in text_types: + text = block.get("text") + if isinstance(text, str): + if text.strip(): + return True + elif text: + return True + continue + # Any non-reasoning block is meaningful output (text, image, refusal, + # server tool result, etc.), even when its provider-specific payload + # does not use a ``text`` key. + return True + return False + + +def _truncated_empty_message(response: ModelResponse): + """Return the empty truncated AI message in *response*, if present.""" + from langchain_core.messages import AIMessage + + if getattr(response, "structured_response", None) is not None: + return None + messages = getattr(response, "result", None) or [] + message = next( + (item for item in reversed(messages) if isinstance(item, AIMessage)), None + ) + if message is None: + return None + if _has_answer_content(message.content): + return None + if message.tool_calls or getattr(message, "invalid_tool_calls", None): + return None + + metadata = message.response_metadata or {} + reasons = { + str(metadata.get(key, "")).strip().lower() + for key in ("finish_reason", "stop_reason", "status") + } + incomplete_details = metadata.get("incomplete_details") + if isinstance(incomplete_details, dict): + reasons.add(str(incomplete_details.get("reason", "")).strip().lower()) + if reasons.isdisjoint(_TRUNCATED_FINISH_REASONS): + return None + return message + + +def _check_truncated_output(response: ModelResponse) -> ModelResponse: + """Raise a visible error instead of silently accepting an empty answer.""" + message = _truncated_empty_message(response) + if message is None: + return response + metadata = message.response_metadata or {} + reason = ( + metadata.get("finish_reason") + or metadata.get("stop_reason") + or metadata.get("status") + or "output limit" + ) + raise ModelOutputTruncatedError( + "The model exhausted its output budget during reasoning and returned " + f"no answer (finish reason: {reason}). Lower reasoning_effort, disable " + "reasoning with none when supported, or increase the provider " + "output-token limit." + ) + + def _should_pass_through(exc: BaseException) -> bool: """True if *exc* is a LangGraph-level signal that must propagate untouched — either a control-flow signal or a structural error @@ -210,7 +314,7 @@ class ErrorNormalizationMiddleware(AgentMiddleware): handler: Callable[[ModelRequest], ModelResponse], ) -> ModelResponse: try: - return handler(request) + return _check_truncated_output(handler(request)) except Exception as exc: normalized = _normalize(request, exc) if normalized is None: @@ -223,7 +327,7 @@ class ErrorNormalizationMiddleware(AgentMiddleware): handler: Callable[[ModelRequest], Awaitable[ModelResponse]], ) -> ModelResponse: try: - return await handler(request) + return _check_truncated_output(await handler(request)) except Exception as exc: normalized = _normalize(request, exc) if normalized is None: diff --git a/EvoScientist/middleware/model_fallback.py b/EvoScientist/middleware/model_fallback.py index 6481713..df07429 100644 --- a/EvoScientist/middleware/model_fallback.py +++ b/EvoScientist/middleware/model_fallback.py @@ -447,10 +447,16 @@ class ModelFallbackMiddleware(AgentMiddleware): ) -> ModelResponse: if not _fallback_chain: return handler(request) + + from .error_normalization import _check_truncated_output + + def invoke(current_request: ModelRequest) -> ModelResponse: + return _check_truncated_output(handler(current_request)) + try: - return handler(request) + return invoke(request) except Exception as exc: - return _guard_and_fallback_sync(exc, request, handler, self._events) + return _guard_and_fallback_sync(exc, request, invoke, self._events) async def awrap_model_call( self, @@ -459,7 +465,13 @@ class ModelFallbackMiddleware(AgentMiddleware): ) -> ModelResponse: if not _fallback_chain: return await handler(request) + + from .error_normalization import _check_truncated_output + + async def invoke(current_request: ModelRequest) -> ModelResponse: + return _check_truncated_output(await handler(current_request)) + try: - return await handler(request) + return await invoke(request) except Exception as exc: - return await _guard_and_fallback(exc, request, handler, self._events) + return await _guard_and_fallback(exc, request, invoke, self._events) diff --git a/tests/test_error_normalization_middleware.py b/tests/test_error_normalization_middleware.py index cbb27a2..67106ca 100644 --- a/tests/test_error_normalization_middleware.py +++ b/tests/test_error_normalization_middleware.py @@ -14,10 +14,13 @@ import dataclasses from types import SimpleNamespace import pytest +from langchain.agents.middleware.types import ModelResponse +from langchain_core.messages import AIMessage from EvoScientist.llm.errors import ProviderStreamError from EvoScientist.middleware.error_normalization import ( ErrorNormalizationMiddleware, + ModelOutputTruncatedError, _normalize, ) @@ -421,3 +424,116 @@ class TestMiddleware: req = _request(_openrouter_model()) mw = ErrorNormalizationMiddleware() assert self._run_awrap(mw, req, handler) == "ok" + + def test_empty_length_response_becomes_visible_provider_error(self): + """Reasoning-only truncation must not look like a successful idle turn.""" + response = ModelResponse( + result=[ + AIMessage( + content="", + additional_kwargs={"reasoning_content": "still thinking"}, + response_metadata={"finish_reason": "length"}, + ) + ] + ) + + def handler(_req): + return response + + req = _request(_openai_model(base_url="https://internal.corp/v1")) + with pytest.raises(ProviderStreamError) as excinfo: + ErrorNormalizationMiddleware().wrap_model_call(req, handler) + + assert excinfo.value.provider == "openai_compat" + assert isinstance(excinfo.value.__cause__, ModelOutputTruncatedError) + assert "reasoning_effort" in str(excinfo.value) + + def test_empty_incomplete_responses_api_result_is_detected(self): + response = ModelResponse( + result=[ + AIMessage( + content=[], + response_metadata={ + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + }, + ) + ] + ) + + async def handler(_req): + return response + + req = _request(_openai_model()) + with pytest.raises(ProviderStreamError): + self._run_awrap(ErrorNormalizationMiddleware(), req, handler) + + def test_empty_structured_text_block_with_length_is_detected(self): + response = ModelResponse( + result=[ + AIMessage( + content=[{"type": "text", "text": " "}], + response_metadata={"finish_reason": "length"}, + ) + ] + ) + + def handler(_req): + return response + + with pytest.raises(ProviderStreamError) as excinfo: + ErrorNormalizationMiddleware().wrap_model_call( + _request(_openai_model()), handler + ) + + assert isinstance(excinfo.value.__cause__, ModelOutputTruncatedError) + + def test_redacted_thinking_only_with_max_tokens_is_detected(self): + response = ModelResponse( + result=[ + AIMessage( + content=[{"type": "redacted_thinking", "data": "opaque-payload"}], + response_metadata={"stop_reason": "max_tokens"}, + ) + ] + ) + + def handler(_req): + return response + + with pytest.raises(ProviderStreamError) as excinfo: + ErrorNormalizationMiddleware().wrap_model_call( + _request(_anthropic_model()), handler + ) + + assert isinstance(excinfo.value.__cause__, ModelOutputTruncatedError) + + @pytest.mark.parametrize( + "message", + [ + AIMessage(content="answer", response_metadata={"finish_reason": "length"}), + AIMessage( + content="", + tool_calls=[{"name": "search", "args": {}, "id": "call-1"}], + response_metadata={"finish_reason": "length"}, + ), + AIMessage(content="", response_metadata={"finish_reason": "stop"}), + AIMessage( + content=[ + {"type": "redacted_thinking", "data": "opaque-payload"}, + {"type": "text", "text": "answer"}, + ], + response_metadata={"stop_reason": "max_tokens"}, + ), + ], + ) + def test_nonempty_tool_and_normal_stop_responses_are_not_rejected(self, message): + response = ModelResponse(result=[message]) + + def handler(_req): + return response + + result = ErrorNormalizationMiddleware().wrap_model_call( + _request(_openai_model()), handler + ) + assert result is response diff --git a/tests/test_llm.py b/tests/test_llm.py index edefe09..a8f5c4d 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -1077,6 +1077,34 @@ class TestThirdPartyRouting: assert call_kwargs["base_url"] == "https://my-llm.example.com/v1" assert call_kwargs["api_key"] == "custom-key-789" + @patch("EvoScientist.llm.models.init_chat_model") + def test_custom_openai_forwards_explicit_reasoning_effort( + self, mock_init, monkeypatch + ): + """User-owned compatible endpoints receive an explicit effort only.""" + mock_init.return_value = "mock_model" + monkeypatch.setenv("CUSTOM_OPENAI_BASE_URL", "https://opencode.example/v1") + monkeypatch.setenv("CUSTOM_OPENAI_API_KEY", "custom-key") + monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", "low") + + get_chat_model("reasoning-model", provider="custom-openai") + + assert mock_init.call_args[1]["reasoning_effort"] == "low" + + @patch("EvoScientist.llm.models.init_chat_model") + def test_custom_openai_omits_unconfigured_reasoning_effort( + self, mock_init, monkeypatch + ): + """Unknown compatible endpoints stay compatible by default.""" + mock_init.return_value = "mock_model" + monkeypatch.setenv("CUSTOM_OPENAI_BASE_URL", "https://plain.example/v1") + monkeypatch.setenv("CUSTOM_OPENAI_API_KEY", "custom-key") + monkeypatch.delenv("EVOSCIENTIST_REASONING_EFFORT", raising=False) + + get_chat_model("plain-model", provider="custom-openai") + + assert "reasoning_effort" not in mock_init.call_args[1] + @patch("EvoScientist.llm.models.init_chat_model") def test_anthropic_base_url_override(self, mock_init, monkeypatch): """Anthropic provider should support base_url override (e.g. ccproxy).""" @@ -1167,6 +1195,90 @@ class TestThirdPartyRouting: == "https://dashscope.aliyuncs.com/compatible-mode/v1" ) assert call_kwargs["api_key"] == "ds-key-456" + assert "reasoning_effort" not in call_kwargs + + @patch("EvoScientist.llm.models.init_chat_model") + def test_qwen38_dashscope_uses_bounded_default(self, mock_init, monkeypatch): + """Qwen 3.8 avoids the regular endpoint's xhigh default.""" + mock_init.return_value = "mock_model" + monkeypatch.setenv("DASHSCOPE_API_KEY", "ds-key") + monkeypatch.delenv("EVOSCIENTIST_REASONING_EFFORT", raising=False) + + get_chat_model("qwen3.8-max", provider="dashscope") + + assert mock_init.call_args[1]["reasoning_effort"] == "medium" + + @patch("EvoScientist.llm.models.init_chat_model") + def test_qwen38_dashscope_respects_configured_reasoning_effort( + self, mock_init, monkeypatch + ): + mock_init.return_value = "mock_model" + monkeypatch.setenv("DASHSCOPE_API_KEY", "ds-key") + monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", "medium") + + get_chat_model("qwen3.8-max", provider="dashscope") + + assert mock_init.call_args[1]["reasoning_effort"] == "medium" + + @pytest.mark.parametrize( + "effort", + [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + ], + ) + @patch("EvoScientist.llm.models.init_chat_model") + def test_qwen38_dashscope_accepts_supported_reasoning_effort( + self, mock_init, effort, monkeypatch + ): + mock_init.return_value = "mock_model" + monkeypatch.setenv("DASHSCOPE_API_KEY", "ds-key") + monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", effort) + + get_chat_model("qwen3.8-max", provider="dashscope") + + assert mock_init.call_args[1]["reasoning_effort"] == effort + + @patch("EvoScientist.llm.models.init_chat_model") + def test_qwen38_dashscope_rejects_unsupported_reasoning_effort( + self, mock_init, monkeypatch + ): + monkeypatch.setenv("DASHSCOPE_API_KEY", "ds-key") + monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", "invalid") + + with pytest.raises(ValueError, match="dashscope"): + get_chat_model("qwen3.8-max", provider="dashscope") + + mock_init.assert_not_called() + + @patch("EvoScientist.llm.models.init_chat_model") + def test_qwen38_dashscope_explicit_effort_overrides_invalid_environment( + self, mock_init, monkeypatch + ): + mock_init.return_value = "mock_model" + monkeypatch.setenv("DASHSCOPE_API_KEY", "ds-key") + monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", "invalid") + + get_chat_model("qwen3.8-max", provider="dashscope", reasoning_effort="low") + + assert mock_init.call_args[1]["reasoning_effort"] == "low" + + @patch("EvoScientist.llm.models.init_chat_model") + def test_qwen38_dashscope_code_omits_undocumented_reasoning_effort( + self, mock_init, monkeypatch + ): + mock_init.return_value = "mock_model" + monkeypatch.setenv("DASHSCOPE_API_KEY", "sk-sp-key") + monkeypatch.setenv("EVOSCIENTIST_REASONING_EFFORT", "medium") + + get_chat_model("qwen3.8-max", provider="dashscope-code") + + assert "reasoning_effort" not in mock_init.call_args[1] @patch("EvoScientist.llm.models.init_chat_model") def test_dashscope_code_routes_through_openai(self, mock_init, monkeypatch): diff --git a/tests/test_model_fallback.py b/tests/test_model_fallback.py index 08f0ddb..64e8bdf 100644 --- a/tests/test_model_fallback.py +++ b/tests/test_model_fallback.py @@ -10,6 +10,7 @@ from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest +from langchain.agents.middleware.types import ModelResponse from langchain_core.exceptions import ContextOverflowError from langchain_core.messages import AIMessage, HumanMessage @@ -42,6 +43,22 @@ def _fake_request(): AI_RESPONSE = AIMessage(content="ok") +def _truncated_response() -> ModelResponse: + return ModelResponse( + result=[ + AIMessage( + content="", + additional_kwargs={"reasoning_content": "still thinking"}, + response_metadata={"finish_reason": "length"}, + ) + ] + ) + + +def _successful_response() -> ModelResponse: + return ModelResponse(result=[AIMessage(content="ok")]) + + @pytest.fixture(autouse=True) def _clean_chain(): """Ensure a clean fallback chain for every test.""" @@ -454,6 +471,94 @@ class TestSynchronousFallback: assert result is response assert handler.call_count == 2 + def test_truncated_primary_response_uses_fallback(self): + from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware + + add_fallback("fb", "prov") + req = _fake_request() + response = _successful_response() + handler = MagicMock(side_effect=[_truncated_response(), response]) + + with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm: + mock_gcm.return_value = MagicMock() + result = ModelFallbackMiddleware().wrap_model_call(req, handler) + + assert result is response + assert handler.call_count == 2 + + +class TestTruncatedResponseFallback: + """Empty truncated model results must participate in the fallback chain.""" + + async def test_truncated_primary_response_uses_fallback(self): + from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware + + add_fallback("fb", "prov") + req = _fake_request() + response = _successful_response() + handler = AsyncMock(side_effect=[_truncated_response(), response]) + + with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm: + mock_gcm.return_value = MagicMock() + result = await ModelFallbackMiddleware().awrap_model_call(req, handler) + + assert result is response + assert handler.await_count == 2 + + async def test_truncated_fallback_continues_to_next_model(self): + from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware + + add_fallback("fb-a", "prov-a") + add_fallback("fb-b", "prov-b") + req = _fake_request() + response = _successful_response() + handler = AsyncMock( + side_effect=[_truncated_response(), _truncated_response(), response] + ) + + with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm: + mock_gcm.return_value = MagicMock() + result = await ModelFallbackMiddleware().awrap_model_call(req, handler) + + assert result is response + assert handler.await_count == 3 + assert mock_gcm.call_count == 2 + + async def test_exhausted_truncated_fallbacks_use_last_provider(self): + from EvoScientist.llm.errors import ProviderStreamError + from EvoScientist.middleware.error_normalization import ( + ModelOutputTruncatedError, + ) + from EvoScientist.middleware.model_fallback import ModelFallbackMiddleware + + def _make_openai_model(base_url=None): + cls = type( + "ChatOpenAI", + (), + {"__module__": "langchain_openai.chat_models.base"}, + ) + model = cls() + model.openai_api_base = base_url + return model + + add_fallback("moonshot-model", "moonshot") + req = _fake_request() + req.model = _make_openai_model() + fallback_model = _make_openai_model(base_url="https://api.moonshot.cn/v1") + req.override = MagicMock( + side_effect=lambda **kw: SimpleNamespace(model=kw.get("model", req.model)) + ) + handler = AsyncMock(return_value=_truncated_response()) + + with patch("EvoScientist.llm.models.get_chat_model") as mock_gcm: + mock_gcm.return_value = fallback_model + with pytest.raises(ProviderStreamError) as exc_info: + await ModelFallbackMiddleware().awrap_model_call(req, handler) + + assert exc_info.value.provider == "moonshot" + assert isinstance(exc_info.value.__cause__, ModelOutputTruncatedError) + assert handler.await_count == 2 + # ═════════════════════════════════════════════════════════════════ # 4. UI emit callback