From 9cffe9d457b62e2379c3af73ca3156fd75a21b88 Mon Sep 17 00:00:00 2001 From: Xi Zhang <106144707+X-iZhang@users.noreply.github.com> Date: Wed, 3 Jun 2026 01:06:46 +0100 Subject: [PATCH] Enhance multimodal handling in LLM model (#256) * Enhance multimodal handling in LLM model - Updated `_flatten_message_content` to preserve media blocks (images, files) while flattening text content. - Introduced `_sanitize_messages` to manage media hoisting for tool messages, ensuring compatibility with OpenAI APIs. - Modified `_patch_openai_compat_content` to accommodate new media handling logic, including retry mechanisms for media errors. - Added comprehensive tests for media preservation, including various scenarios with images, files, and unsupported media types. * fix: preserve order of text and media blocks in message flattening * test: add tests for _strip_media_types to ensure position preservation and deduplication --- EvoScientist/llm/models.py | 5 +- EvoScientist/llm/patches.py | 347 +++++++++++++-- tests/test_llm.py | 829 ++++++++++++++++++++++++++++++++++++ 3 files changed, 1152 insertions(+), 29 deletions(-) diff --git a/EvoScientist/llm/models.py b/EvoScientist/llm/models.py index 85a22d2..87c8164 100644 --- a/EvoScientist/llm/models.py +++ b/EvoScientist/llm/models.py @@ -464,7 +464,10 @@ def get_chat_model( if ( _is_third_party or _is_openai_proxy ) and _original_provider not in _no_patch_providers: - _patch_openai_compat_content(chat_model) + # Anthropic-routed providers accept media in tool results natively; + # only OpenAI-compatible providers need tool-media hoisting. + _hoist = _original_provider not in _ANTHROPIC_ROUTED_PROVIDERS + _patch_openai_compat_content(chat_model, hoist_tool_media=_hoist) # DeepSeek thinking mode requires reasoning_content passback in multi-turn # + tool_use scenarios. diff --git a/EvoScientist/llm/patches.py b/EvoScientist/llm/patches.py index d58d0da..07e1c72 100644 --- a/EvoScientist/llm/patches.py +++ b/EvoScientist/llm/patches.py @@ -194,62 +194,260 @@ def _is_ccproxy_codex() -> bool: # --------------------------------------------------------------------------- # Utility + Patch: Flatten list content to strings for OpenAI-compatible APIs. # DeepSeek, SiliconFlow, etc. reject assistant messages whose content is a -# list rather than a string. +# list rather than a string. Image and file (PDF/document) blocks are +# preserved, not flattened away. # --------------------------------------------------------------------------- _SKIP_CONTENT_TYPES = frozenset({"thinking", "reasoning", "reasoning_content"}) +# Media block types preserved when flattening (positive allowlist; +# thinking/reasoning still dropped). Images + files (PDF/documents): both +# serialize on OpenAI-compatible APIs and capable models read them. `video` is +# deliberately EXCLUDED (langchain-openai raises ValueError on it); `audio` is +# omitted (almost no model support). is_data_content_block is unreliable +# (False for OpenAI image_url / Anthropic image-source). Kept as separate +# image/file sets so the no-media fallback can gate per modality. +_IMAGE_CONTENT_TYPES = frozenset({"image", "image_url", "input_image"}) +_FILE_CONTENT_TYPES = frozenset({"file", "input_file", "document"}) +_MEDIA_CONTENT_TYPES = _IMAGE_CONTENT_TYPES | _FILE_CONTENT_TYPES -def _flatten_message_content(content: Any) -> str | Any: - """Convert list-of-blocks content to a plain string. + +def _flatten_message_content(content: Any) -> str | list[Any] | Any: + """Convert list-of-blocks content to a string, preserving media blocks. + + Thinking/reasoning blocks are dropped. When a media block (image or file) + is present, returns a list in the ORIGINAL order — consecutive text is + joined into a single text block, media blocks kept as-is — so captions stay + next to the attachment they describe. Otherwise returns a plain string. + Non-list input is returned unchanged. Args: - content: Message content — either a string, a list of content blocks - (dicts with ``type`` and ``text`` keys), or another type. + content: Message content — a string, a list of content blocks, or + another type. Returns: - A plain string with text blocks joined by double newlines. - Thinking/reasoning blocks are skipped. Non-list input is - returned unchanged. + A plain string for text-only content, a list of blocks when media is + present, or the input unchanged for non-list input. """ if isinstance(content, str): return content if not isinstance(content, list): return content parts: list[str] = [] + ordered_blocks: list[Any] = [] + saw_media = False + + def _flush_text() -> None: + if parts: + ordered_blocks.append({"type": "text", "text": "\n\n".join(parts)}) + parts.clear() + for block in content: if isinstance(block, dict): - if block.get("type") in _SKIP_CONTENT_TYPES: + btype = block.get("type") + if btype in _MEDIA_CONTENT_TYPES: + # Keep media as-is (never mutate; upstream copy.copy is shallow) + # and preserve its position relative to surrounding text. + _flush_text() + ordered_blocks.append(block) + saw_media = True + continue + if btype in _SKIP_CONTENT_TYPES: continue text = block.get("text") if text: parts.append(text) elif isinstance(block, str): parts.append(block) + if saw_media: + _flush_text() + return ordered_blocks return "\n\n".join(parts) if parts else "" -def _patch_openai_compat_content(model: Any) -> None: +def _sanitize_messages(messages: list[Any], hoist_tool_media: bool = True) -> list[Any]: + """Flatten list content for OpenAI-compatible APIs, preserving media. + + Text/reasoning content is flattened to a string; image blocks are + preserved. Tool messages cannot carry media + on OpenAI-compatible APIs (content must be a string), so when + ``hoist_tool_media`` is set the media in a tool result is moved into a + HumanMessage emitted after that turn's (possibly parallel) tool messages. + Anthropic-routed providers accept tool-result media natively and pass + ``hoist_tool_media=False`` to keep it inline. + """ + import copy + + from langchain_core.messages import HumanMessage + + out: list[Any] = [] + pending_media: list[Any] = [] # media hoisted out of a run of tool messages + + def _flush() -> None: + if pending_media: + out.append(HumanMessage(content=list(pending_media))) + pending_media.clear() + + for msg in messages: + is_tool = getattr(msg, "type", None) == "tool" + if not is_tool: + _flush() # emit hoisted media before any non-tool message + if not isinstance(msg.content, list): + out.append(msg) + continue + flat = _flatten_message_content(msg.content) + if hoist_tool_media and is_tool and isinstance(flat, list): + text_blocks = [ + b for b in flat if isinstance(b, dict) and b.get("type") == "text" + ] + media_blocks = [ + b for b in flat if not (isinstance(b, dict) and b.get("type") == "text") + ] + tool_msg = copy.copy(msg) + # Join ALL text runs (interleaved content can yield more than one) + # so no text is lost; tool content must be a string on OpenAI-compat. + tool_msg.content = ( + "\n\n".join(b["text"] for b in text_blocks) + if text_blocks + else "[media content provided in the following message]" + ) + out.append(tool_msg) + pending_media.extend(media_blocks) + else: + msg = copy.copy(msg) + msg.content = flat + out.append(msg) + _flush() # conversation may end with tool messages + return out + + +# Fallback for models that reject some media input (no vision / no file +# support): replace blocks of the unsupported types with a placeholder so the +# conversation can keep going instead of erroring every turn that re-sends them. +_UNSUPPORTED_MEDIA_PLACEHOLDER = ( + "[attachment omitted: this model does not support this input type]" +) +# Markers that identify WHICH modality an error rejects, so only the implicated +# type is remembered (not all media). Generic markers map to all media. +_IMAGE_ERROR_MARKERS = ("image_url", "image input", "support image") +_FILE_ERROR_MARKERS = ("file input", "pdf input", "document input") +# `expected \`text\`` keeps the backticks — the bare "expected text" phrase is +# too broad (matches non-media schema errors like "... expected text"). +_GENERIC_MEDIA_MARKERS = ("multimodal", "expected `text`") + + +def _media_types_in(messages: list[Any]) -> set[str]: + """Set of preserved-media block types present across the messages.""" + found: set[str] = set() + for msg in messages: + content = getattr(msg, "content", None) + if isinstance(content, list): + for b in content: + if isinstance(b, dict) and b.get("type") in _MEDIA_CONTENT_TYPES: + found.add(b["type"]) + return found + + +def _media_error_types(exc: Exception) -> set[str]: + """Media modalities the error text implicates (empty if not media-specific). + + Used to remember ONLY the rejected modality, so e.g. a PDF rejection never + disables images. + """ + text = str(exc).lower() + types: set[str] = set() + if any(m in text for m in _IMAGE_ERROR_MARKERS): + types |= _IMAGE_CONTENT_TYPES + if any(m in text for m in _FILE_ERROR_MARKERS): + types |= _FILE_CONTENT_TYPES + if any(m in text for m in _GENERIC_MEDIA_MARKERS): + types |= _MEDIA_CONTENT_TYPES + return types + + +def _is_http_400(exc: Exception) -> bool: + """True if the error carries an HTTP 400 status (bad request).""" + status = getattr(exc, "status_code", None) + if status is None: + status = getattr(getattr(exc, "response", None), "status_code", None) + return status == 400 + + +def _strip_media_types(messages: list[Any], types: set[str]) -> list[Any]: + """Replace blocks of the given types with a placeholder text block. + + Each stripped block is replaced IN PLACE (consecutive ones collapse into one + placeholder), so surrounding text/media keep their original positions and a + model that rejects only one modality still receives the others in order. + """ + import copy + + out: list[Any] = [] + for msg in messages: + content = getattr(msg, "content", None) + if not isinstance(content, list) or not any( + isinstance(b, dict) and b.get("type") in types for b in content + ): + out.append(msg) + continue + kept: list[Any] = [] + last_was_placeholder = False + for b in content: + if isinstance(b, dict) and b.get("type") in types: + if not last_was_placeholder: + kept.append( + {"type": "text", "text": _UNSUPPORTED_MEDIA_PLACEHOLDER} + ) + last_was_placeholder = True + continue + kept.append(b) + last_was_placeholder = False + msg = copy.copy(msg) + msg.content = kept + out.append(msg) + return out + + +def _patch_openai_compat_content(model: Any, hoist_tool_media: bool = True) -> None: """Flatten list content to strings before OpenAI-compatible API calls. - Wraps ``_generate`` / ``_agenerate`` to prevent "invalid type: sequence, - expected a string" errors from strict APIs like DeepSeek. + Wraps ``_generate`` / ``_agenerate`` / ``_stream`` / ``_astream`` to prevent + "invalid type: sequence, expected a string" errors from strict APIs like + DeepSeek, while preserving image and file (PDF/document) blocks. Tracks a + per-modality ``blocked`` set of media types the model rejects: when a request + fails with a media-looking error it is retried with the offending modalities + stripped to a placeholder, and those types are remembered ONLY after the + stripped retry succeeds — so an unrelated 400 never permanently degrades the + model, and a PDF rejection never disables images. The profile pre-seeds the + set per modality (``image_inputs``/``pdf_inputs`` ``is False``). Args: model: A LangChain chat model instance to patch in-place. + hoist_tool_media: When True (OpenAI-compatible), media in a tool result + is hoisted into a following HumanMessage (tool content must be a + string). Anthropic-routed providers pass False (native support). """ - import copy import functools from langchain_core.messages import BaseMessage - def _sanitize_messages(messages: list[BaseMessage]) -> list[BaseMessage]: - out: list[BaseMessage] = [] - for msg in messages: - if isinstance(msg.content, list): - msg = copy.copy(msg) - msg.content = _flatten_message_content(msg.content) - out.append(msg) - return out + # Media block types this model rejects. Pre-seeded per modality from the + # profile, then grown reactively — but only after a stripped retry succeeds. + profile = getattr(model, "profile", None) + blocked: set[str] = set() + if isinstance(profile, dict): + if profile.get("image_inputs") is False: + blocked |= _IMAGE_CONTENT_TYPES + if profile.get("pdf_inputs") is False: + blocked |= _FILE_CONTENT_TYPES + + def _prepare(messages: list[BaseMessage]) -> list[BaseMessage]: + msgs = _strip_media_types(messages, blocked) if blocked else messages + return _sanitize_messages(msgs, hoist_tool_media) + + def _stripped(messages: list[BaseMessage], suspects: set[str]) -> list[BaseMessage]: + return _sanitize_messages( + _strip_media_types(messages, blocked | suspects), hoist_tool_media + ) orig_generate = getattr(model, "_generate", None) if orig_generate is None: @@ -259,7 +457,23 @@ def _patch_openai_compat_content(model: Any) -> None: def _patched_generate( messages: list[BaseMessage], *args: Any, **kwargs: Any ) -> Any: - return orig_generate(_sanitize_messages(messages), *args, **kwargs) + prepared = _prepare(messages) + suspects = _media_types_in(messages) - blocked + if not suspects: + return orig_generate(prepared, *args, **kwargs) + try: + return orig_generate(prepared, *args, **kwargs) + except Exception as exc: + culprit = _media_error_types(exc) & suspects + if not culprit and not _is_http_400(exc): + raise + try: + result = orig_generate(_stripped(messages, suspects), *args, **kwargs) + except Exception: + # stripping didn't help — surface original, don't cache + raise exc from None + blocked.update(culprit) # cache only the marker-identified modality + return result model._generate = _patched_generate @@ -270,7 +484,24 @@ def _patch_openai_compat_content(model: Any) -> None: async def _patched_agenerate( messages: list[BaseMessage], *args: Any, **kwargs: Any ) -> Any: - return await orig_agenerate(_sanitize_messages(messages), *args, **kwargs) + prepared = _prepare(messages) + suspects = _media_types_in(messages) - blocked + if not suspects: + return await orig_agenerate(prepared, *args, **kwargs) + try: + return await orig_agenerate(prepared, *args, **kwargs) + except Exception as exc: + culprit = _media_error_types(exc) & suspects + if not culprit and not _is_http_400(exc): + raise + try: + result = await orig_agenerate( + _stripped(messages, suspects), *args, **kwargs + ) + except Exception: + raise exc from None + blocked.update(culprit) + return result model._agenerate = _patched_agenerate @@ -283,7 +514,38 @@ def _patch_openai_compat_content(model: Any) -> None: def _patched_stream( messages: list[BaseMessage], *args: Any, **kwargs: Any ) -> Any: - return orig_stream(_sanitize_messages(messages), *args, **kwargs) + prepared = _prepare(messages) + suspects = _media_types_in(messages) - blocked + if not suspects: + yield from orig_stream(prepared, *args, **kwargs) + return + started = False + try: + for chunk in orig_stream(prepared, *args, **kwargs): + started = True + yield chunk + return + except Exception as exc: + culprit = _media_error_types(exc) & suspects + if started or (not culprit and not _is_http_400(exc)): + raise + media_exc = exc + media_culprit = culprit + retry_started = False + try: + for chunk in orig_stream( + _stripped(messages, suspects), *args, **kwargs + ): + if not retry_started: + retry_started = True + blocked.update(media_culprit) + yield chunk + except Exception: + if retry_started: + raise + raise media_exc from None + if not retry_started: + raise media_exc from None # stripped retry produced nothing model._stream = _patched_stream @@ -294,10 +556,39 @@ def _patch_openai_compat_content(model: Any) -> None: async def _patched_astream( messages: list[BaseMessage], *args: Any, **kwargs: Any ) -> Any: - async for chunk in orig_astream( - _sanitize_messages(messages), *args, **kwargs - ): - yield chunk + prepared = _prepare(messages) + suspects = _media_types_in(messages) - blocked + if not suspects: + async for chunk in orig_astream(prepared, *args, **kwargs): + yield chunk + return + started = False + try: + async for chunk in orig_astream(prepared, *args, **kwargs): + started = True + yield chunk + return + except Exception as exc: + culprit = _media_error_types(exc) & suspects + if started or (not culprit and not _is_http_400(exc)): + raise + media_exc = exc + media_culprit = culprit + retry_started = False + try: + async for chunk in orig_astream( + _stripped(messages, suspects), *args, **kwargs + ): + if not retry_started: + retry_started = True + blocked.update(media_culprit) + yield chunk + except Exception: + if retry_started: + raise + raise media_exc from None + if not retry_started: + raise media_exc from None # stripped retry produced nothing model._astream = _patched_astream diff --git a/tests/test_llm.py b/tests/test_llm.py index bd4ddb5..9d797a3 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -729,6 +729,107 @@ class TestFlattenMessageContent: content = [{"type": "thinking", "text": "thought"}] assert _flatten_message_content(content) == "" + def test_preserves_image_block(self): + from EvoScientist.llm.patches import _flatten_message_content + + img = {"type": "image", "base64": "AAA", "mime_type": "image/png"} + assert _flatten_message_content([img]) == [img] + + def test_preserves_image_url_block(self): + from EvoScientist.llm.patches import _flatten_message_content + + img = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}} + assert _flatten_message_content([img]) == [img] + + def test_preserves_file_block(self): + # PDF/document files are preserved (capable models read them). + from EvoScientist.llm.patches import _flatten_message_content + + f = {"type": "file", "base64": "FFF", "mime_type": "application/pdf"} + assert _flatten_message_content([f]) == [f] + + def test_unsupported_media_dropped(self): + # video/audio are NOT in the allowlist -> dropped, not crashing + # (langchain-openai raises ValueError on `video`). + from EvoScientist.llm.patches import _flatten_message_content + + for block in ( + {"type": "video", "base64": "VVV", "mime_type": "video/mp4"}, + {"type": "audio", "base64": "ZZZ", "mime_type": "audio/wav"}, + ): + assert _flatten_message_content([block]) == "" + + def test_non_image_media_dropped_keeps_text(self): + from EvoScientist.llm.patches import _flatten_message_content + + content = [ + {"type": "text", "text": "hi"}, + {"type": "video", "base64": "VVV", "mime_type": "video/mp4"}, + ] + # Video dropped, text kept -> plain string (no media list). + assert _flatten_message_content(content) == "hi" + + def test_consolidates_text_and_image(self): + from EvoScientist.llm.patches import _flatten_message_content + + img = {"type": "image", "base64": "AAA", "mime_type": "image/png"} + content = [{"type": "text", "text": "a photo"}, img] + assert _flatten_message_content(content) == [ + {"type": "text", "text": "a photo"}, + img, + ] + + def test_multiple_text_blocks_with_image(self): + from EvoScientist.llm.patches import _flatten_message_content + + img = {"type": "image", "base64": "AAA", "mime_type": "image/png"} + content = [ + {"type": "text", "text": "a"}, + {"type": "text", "text": "b"}, + img, + ] + assert _flatten_message_content(content) == [ + {"type": "text", "text": "a\n\nb"}, + img, + ] + + def test_preserves_text_media_ordering(self): + # Text after an image must stay AFTER it (not consolidated to the front). + from EvoScientist.llm.patches import _flatten_message_content + + img = {"type": "image", "base64": "AAA", "mime_type": "image/png"} + content = [ + {"type": "text", "text": "before"}, + img, + {"type": "text", "text": "after"}, + ] + assert _flatten_message_content(content) == [ + {"type": "text", "text": "before"}, + img, + {"type": "text", "text": "after"}, + ] + + def test_thinking_dropped_image_kept(self): + from EvoScientist.llm.patches import _flatten_message_content + + img = {"type": "image", "base64": "AAA", "mime_type": "image/png"} + content = [{"type": "thinking", "text": "hmm"}, img] + assert _flatten_message_content(content) == [img] + + def test_pure_text_still_returns_string(self): + from EvoScientist.llm.patches import _flatten_message_content + + content = [{"type": "text", "text": "x"}, {"type": "text", "text": "y"}] + result = _flatten_message_content(content) + assert result == "x\n\ny" + assert isinstance(result, str) + + def test_unknown_nontext_block_still_dropped(self): + from EvoScientist.llm.patches import _flatten_message_content + + content = [{"type": "tool_use", "id": "1", "name": "foo"}] + assert _flatten_message_content(content) == "" + # ============================================================================= # Test _patch_openai_compat_content (all 4 paths) @@ -820,6 +921,734 @@ class TestPatchOpenAICompatContent: assert chunks == ["c1", "c2"] assert received_msgs[0].content == "hello" + def test_generate_preserves_media(self): + from langchain_core.messages import HumanMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + orig = model._generate + _patch_openai_compat_content(model) + + img = {"type": "image", "base64": "AAA", "mime_type": "image/png"} + msg = HumanMessage(content=[{"type": "text", "text": "see"}, img]) + model._generate([msg]) + + called_msgs = orig.call_args[0][0] + assert called_msgs[0].content == [{"type": "text", "text": "see"}, img] + + @pytest.mark.anyio + async def test_agenerate_preserves_media(self): + from langchain_core.messages import HumanMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + orig = model._agenerate + _patch_openai_compat_content(model) + + img = {"type": "image", "base64": "AAA", "mime_type": "image/png"} + msg = HumanMessage(content=[{"type": "text", "text": "see"}, img]) + await model._agenerate([msg]) + + called_msgs = orig.call_args[0][0] + assert called_msgs[0].content == [{"type": "text", "text": "see"}, img] + + def test_stream_preserves_media(self): + from langchain_core.messages import HumanMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + orig = model._stream + _patch_openai_compat_content(model) + + img = {"type": "image", "base64": "AAA", "mime_type": "image/png"} + msg = HumanMessage(content=[{"type": "text", "text": "see"}, img]) + list(model._stream([msg])) + + called_msgs = orig.call_args[0][0] + assert called_msgs[0].content == [{"type": "text", "text": "see"}, img] + + @pytest.mark.anyio + async def test_astream_preserves_media(self): + from langchain_core.messages import HumanMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + received_msgs = [] + + async def _fake_astream(messages, *args, **kwargs): + received_msgs.extend(messages) + for chunk in ["c1", "c2"]: + yield chunk + + model._astream = _fake_astream + _patch_openai_compat_content(model) + + img = {"type": "image", "base64": "AAA", "mime_type": "image/png"} + msg = HumanMessage(content=[{"type": "text", "text": "see"}, img]) + chunks = [] + async for c in model._astream([msg]): + chunks.append(c) + + assert chunks == ["c1", "c2"] + assert received_msgs[0].content == [{"type": "text", "text": "see"}, img] + + def test_toolmessage_image_hoisted_to_human(self): + from langchain_core.messages import ToolMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + orig = model._generate + _patch_openai_compat_content(model) # hoist_tool_media=True (OpenAI-compat) + + # deepagents read_file emits this exact shape for an image file. + tm = ToolMessage( + content_blocks=[ + {"type": "image", "base64": "AAA", "mime_type": "image/png"} + ], + tool_call_id="tc1", + name="read_file", + ) + model._generate([tm]) + + called_msgs = orig.call_args[0][0] + # Tool content becomes a string placeholder (OpenAI-compat requirement) ... + assert isinstance(called_msgs[0].content, str) + # ... and the image is hoisted into a following HumanMessage. + assert len(called_msgs) == 2 + hoisted = called_msgs[1] + assert hoisted.type == "human" + assert any( + isinstance(b, dict) and b.get("type") == "image" for b in hoisted.content + ) + + def test_toolmessage_image_kept_inline_when_no_hoist(self): + from langchain_core.messages import ToolMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + orig = model._generate + _patch_openai_compat_content(model, hoist_tool_media=False) # Anthropic-routed + + tm = ToolMessage( + content_blocks=[ + {"type": "image", "base64": "AAA", "mime_type": "image/png"} + ], + tool_call_id="tc1", + name="read_file", + ) + model._generate([tm]) + + called_msgs = orig.call_args[0][0] + # No hoisting: image stays inline in the tool message content. + assert len(called_msgs) == 1 + content = called_msgs[0].content + assert isinstance(content, list) + assert any(isinstance(b, dict) and b.get("type") == "image" for b in content) + + def test_parallel_tool_images_hoisted_after_tools(self): + from langchain_core.messages import AIMessage, ToolMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + orig = model._generate + _patch_openai_compat_content(model) + + ai = AIMessage( + content="", + tool_calls=[ + {"id": "c1", "name": "read_file", "args": {}}, + {"id": "c2", "name": "read_file", "args": {}}, + ], + ) + t1 = ToolMessage( + content_blocks=[ + {"type": "image", "base64": "AAA", "mime_type": "image/png"} + ], + tool_call_id="c1", + name="read_file", + ) + t2 = ToolMessage( + content_blocks=[ + {"type": "image", "base64": "BBB", "mime_type": "image/png"} + ], + tool_call_id="c2", + name="read_file", + ) + model._generate([ai, t1, t2]) + + called_msgs = orig.call_args[0][0] + # Tool results stay consecutive; one hoisted HumanMessage follows them. + assert [m.type for m in called_msgs] == ["ai", "tool", "tool", "human"] + assert isinstance(called_msgs[1].content, str) + assert isinstance(called_msgs[2].content, str) + imgs = [b for b in called_msgs[3].content if b.get("type") == "image"] + assert len(imgs) == 2 + + def test_assistant_text_still_flattened_to_string(self): + from langchain_core.messages import AIMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + orig = model._generate + _patch_openai_compat_content(model) + + msg = AIMessage( + content=[ + {"type": "text", "text": "hi"}, + {"type": "thinking", "text": "t"}, + ] + ) + model._generate([msg]) + + called_msgs = orig.call_args[0][0] + assert called_msgs[0].content == "hi" + + def test_tool_media_flushed_before_next_human(self): + from langchain_core.messages import HumanMessage, ToolMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + orig = model._generate + _patch_openai_compat_content(model) + + tm = ToolMessage( + content_blocks=[ + {"type": "image", "base64": "AAA", "mime_type": "image/png"} + ], + tool_call_id="tc1", + name="read_file", + ) + nxt = HumanMessage(content="thanks") + model._generate([tm, nxt]) + + called = orig.call_args[0][0] + # tool(placeholder), hoisted image (human), then the original human msg + assert [m.type for m in called] == ["tool", "human", "human"] + assert isinstance(called[0].content, str) + assert any(b.get("type") == "image" for b in called[1].content) + assert called[2].content == "thanks" + + def test_tool_message_text_and_image_split(self): + from langchain_core.messages import ToolMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + orig = model._generate + _patch_openai_compat_content(model) + + tm = ToolMessage( + content=[ + {"type": "text", "text": "chart description"}, + {"type": "image", "base64": "AAA", "mime_type": "image/png"}, + ], + tool_call_id="tc1", + name="read_file", + ) + model._generate([tm]) + + called = orig.call_args[0][0] + # Tool keeps the text as its string content; image hoisted to a human msg. + assert called[0].content == "chart description" + assert any(b.get("type") == "image" for b in called[1].content) + + def test_tool_message_interleaved_text_not_lost(self): + # Interleaved [text, image, text] in a tool result: BOTH text runs must + # survive the hoisting split (not just the first). + from langchain_core.messages import ToolMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + orig = model._generate + _patch_openai_compat_content(model) + + tm = ToolMessage( + content=[ + {"type": "text", "text": "before"}, + {"type": "image", "base64": "AAA", "mime_type": "image/png"}, + {"type": "text", "text": "after"}, + ], + tool_call_id="tc1", + name="read_file", + ) + model._generate([tm]) + + called = orig.call_args[0][0] + # both text runs preserved in the tool placeholder; image hoisted + assert "before" in called[0].content + assert "after" in called[0].content + assert any(b.get("type") == "image" for b in called[1].content) + + +# ============================================================================= +# Test no-vision fallback (models that reject image input) +# ============================================================================= + + +class TestNoVisionFallback: + """Verify image-rejecting models fall back to a text placeholder.""" + + def _img_tool(self): + from langchain_core.messages import ToolMessage + + return ToolMessage( + content_blocks=[ + {"type": "image", "base64": "AAA", "mime_type": "image/png"} + ], + tool_call_id="t1", + name="read_file", + ) + + def _make_model(self): + from unittest.mock import MagicMock + + model = MagicMock() + model._agenerate = None + model._stream = None + model._astream = None + return model + + def test_media_error_types(self): + from EvoScientist.llm.patches import ( + _FILE_CONTENT_TYPES, + _IMAGE_CONTENT_TYPES, + _is_http_400, + _media_error_types, + ) + + # marker identifies the specific modality + assert ( + _media_error_types(Exception("No endpoints found that support image input")) + >= _IMAGE_CONTENT_TYPES + ) + assert ( + _media_error_types(Exception("file input is not supported")) + == _FILE_CONTENT_TYPES + ) + # DeepSeek-style maps to all media (generic "expected text") + assert ( + _media_error_types( + Exception("unknown variant `image_url`, expected `text`") + ) + >= _IMAGE_CONTENT_TYPES + ) + # non-media errors implicate nothing + assert _media_error_types(Exception("rate limit exceeded")) == set() + assert ( + _media_error_types(Exception("No endpoints found for some/model")) == set() + ) + # bare "expected text" (non-media schema error) must NOT match + assert ( + _media_error_types( + Exception("tool schema validation failed: expected text") + ) + == set() + ) + + class _E(Exception): + status_code = 400 + + assert _is_http_400(_E("bad request")) + assert not _is_http_400(Exception("rate limit exceeded")) + + def test_media_types_in(self): + from langchain_core.messages import HumanMessage + + from EvoScientist.llm.patches import _media_types_in + + img = {"type": "image", "base64": "A", "mime_type": "image/png"} + f = {"type": "file", "base64": "F", "mime_type": "application/pdf"} + assert _media_types_in([HumanMessage(content=[img, f])]) == {"image", "file"} + assert _media_types_in([HumanMessage(content="hi")]) == set() + + def test_strip_media_types_replaces_only_given(self): + from langchain_core.messages import HumanMessage + + from EvoScientist.llm.patches import _strip_media_types + + img = {"type": "image", "base64": "AAA", "mime_type": "image/png"} + f = {"type": "file", "base64": "FFF", "mime_type": "application/pdf"} + msg = HumanMessage(content=[{"type": "text", "text": "see"}, img, f]) + # Strip only files -> image survives, file becomes a placeholder block. + out = _strip_media_types([msg], {"file"}) + types = [b.get("type") for b in out[0].content if isinstance(b, dict)] + assert "image" in types # image preserved + assert "file" not in types # file stripped + assert any( + b.get("type") == "text" and "omitted" in b.get("text", "").lower() + for b in out[0].content + ) + + def test_strip_media_types_preserves_position(self): + # Stripped block is replaced IN PLACE; surrounding text/kept media keep + # their order (placeholder where the image was, file stays last). + from langchain_core.messages import HumanMessage + + from EvoScientist.llm.patches import _strip_media_types + + img = {"type": "image", "base64": "A", "mime_type": "image/png"} + f = {"type": "file", "base64": "F", "mime_type": "application/pdf"} + msg = HumanMessage( + content=[ + {"type": "text", "text": "t1"}, + img, + {"type": "text", "text": "t2"}, + f, + ] + ) + out = _strip_media_types([msg], {"image"}) # block only image + content = out[0].content + assert all(b.get("type") != "image" for b in content) # image gone + # order preserved: t1, placeholder (where image was), t2, file + assert content[0]["text"] == "t1" + assert content[1]["type"] == "text" + assert "omitted" in content[1]["text"].lower() + assert content[2]["text"] == "t2" + assert content[3]["type"] == "file" # file kept at its original position + + def test_strip_media_types_dedups_consecutive(self): + from langchain_core.messages import HumanMessage + + from EvoScientist.llm.patches import _strip_media_types + + a = {"type": "image", "base64": "A", "mime_type": "image/png"} + b = {"type": "image", "base64": "B", "mime_type": "image/png"} + msg = HumanMessage(content=[a, b]) + out = _strip_media_types([msg], {"image"}) + # two adjacent stripped blocks collapse into ONE placeholder + assert len(out[0].content) == 1 + assert "omitted" in out[0].content[0]["text"].lower() + + def test_profile_no_vision_strips_upfront(self): + # Proactive: profile says image_inputs is False -> strip from the start, + # no failing first request. + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + model.profile = {"image_inputs": False} + calls = [] + + def _gen(msgs, *a, **k): + calls.append(msgs) + return "ok" + + model._generate = _gen + _patch_openai_compat_content(model) + + assert model._generate([self._img_tool()]) == "ok" + assert len(calls) == 1 # no failed attempt + assert all(isinstance(m.content, str) for m in calls[0]) + assert any("omitted" in m.content.lower() for m in calls[0]) + + def test_profile_with_vision_does_not_strip(self): + # Profile says image_inputs is True -> normal preserve path (no upfront strip). + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + model.profile = {"image_inputs": True} + calls = [] + + def _gen(msgs, *a, **k): + calls.append(msgs) + return "ok" + + model._generate = _gen + _patch_openai_compat_content(model) + + assert model._generate([self._img_tool()]) == "ok" + # Image preserved (hoisted), not replaced by a placeholder. + assert any( + isinstance(m.content, list) + and any(b.get("type") == "image" for b in m.content) + for m in calls[0] + ) + + def test_generate_falls_back_and_caches(self): + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + calls = [] + state = {"raised": False} + + def _gen(msgs, *a, **k): + calls.append(msgs) + if not state["raised"]: # fail exactly once, ever + state["raised"] = True + raise Exception("unknown variant `image_url`, expected `text`") + return "ok" + + model._generate = _gen + _patch_openai_compat_content(model) + + tm = self._img_tool() + # 1st turn: preserve attempt fails once -> strip -> ok + assert model._generate([tm]) == "ok" + assert len(calls) == 2 + retry = calls[1] + assert all(isinstance(m.content, str) for m in retry) + assert any("omitted" in m.content.lower() for m in retry) + + # 2nd turn: cached no-vision -> straight to stripped, single call (no failure) + calls.clear() + assert model._generate([tm]) == "ok" + assert len(calls) == 1 + assert all(isinstance(m.content, str) for m in calls[0]) + + def test_non_image_error_not_retried(self): + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + calls = [] + + def _gen(msgs, *a, **k): + calls.append(msgs) + raise Exception("rate limit exceeded") + + model._generate = _gen + _patch_openai_compat_content(model) + + with pytest.raises(Exception, match="rate limit"): + model._generate([self._img_tool()]) + assert len(calls) == 1 + + def test_stream_falls_back(self): + from unittest.mock import MagicMock + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + model._generate = MagicMock(return_value="g") + calls = [] + + def _stream(msgs, *a, **k): + calls.append(msgs) + if len(calls) == 1: + raise Exception("No endpoints found that support image input") + yield from ["x", "y"] + + model._stream = _stream + _patch_openai_compat_content(model) + + out = list(model._stream([self._img_tool()])) + assert out == ["x", "y"] + assert len(calls) == 2 + + @pytest.mark.anyio + async def test_astream_falls_back(self): + from unittest.mock import MagicMock + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + model._generate = MagicMock(return_value="g") + calls = [] + + async def _astream(msgs, *a, **k): + calls.append(msgs) + if len(calls) == 1: + raise Exception("No endpoints found that support image input") + for c in ["x", "y"]: + yield c + + model._astream = _astream + _patch_openai_compat_content(model) + + out = [c async for c in model._astream([self._img_tool()])] + assert out == ["x", "y"] + assert len(calls) == 2 + + def test_unrelated_400_retry_fails_not_cached(self): + # A non-media 400 (e.g. tool schema) whose stripped retry ALSO fails must + # surface the original error and must NOT permanently flip to no-media. + from EvoScientist.llm.patches import _patch_openai_compat_content + + class _E(Exception): + status_code = 400 + + model = self._make_model() + calls = [] + + def _gen(msgs, *a, **k): + calls.append(msgs) + raise _E("invalid tool schema") # 400, not media; fails every time + + model._generate = _gen + _patch_openai_compat_content(model) + + tm = self._img_tool() + with pytest.raises(_E): + model._generate([tm]) + assert len(calls) == 2 # preserve attempt + stripped retry (both fail) + + # Not cached: the next call attempts preserve again (not straight-to-stripped) + calls.clear() + with pytest.raises(_E): + model._generate([tm]) + assert len(calls) == 2 + + def test_pdf_rejection_does_not_disable_images(self): + # Per-modality: a PDF/file rejection caches only file types; a later + # image must still be preserved (not stripped). + from langchain_core.messages import HumanMessage, ToolMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + calls = [] + state = {"raised": False} + + def _gen(msgs, *a, **k): + calls.append(msgs) + has_file = any( + isinstance(m.content, list) + and any( + isinstance(b, dict) and b.get("type") == "file" for b in m.content + ) + for m in msgs + ) + if has_file and not state["raised"]: + state["raised"] = True + raise Exception("file input is not supported") + return "ok" + + model._generate = _gen + _patch_openai_compat_content(model) + + pdf_tm = ToolMessage( + content_blocks=[ + {"type": "file", "base64": "F", "mime_type": "application/pdf"} + ], + tool_call_id="t1", + name="read_file", + ) + assert model._generate([pdf_tm]) == "ok" # file rejected -> stripped -> ok + + # Now an image: must still be preserved (images not blocked by a PDF reject) + calls.clear() + img_msg = HumanMessage( + content=[{"type": "image", "base64": "A", "mime_type": "image/png"}] + ) + assert model._generate([img_msg]) == "ok" + assert len(calls) == 1 # single attempt, no failure + assert any( + isinstance(m.content, list) + and any(isinstance(b, dict) and b.get("type") == "image" for b in m.content) + for m in calls[0] + ) + + def test_bare_400_recovers_but_not_cached(self): + # A bare 400 with NO media marker recovers this request (stripped retry) + # but must NOT cache (no permanent degradation) — High #1. + from EvoScientist.llm.patches import _patch_openai_compat_content + + class _E(Exception): + status_code = 400 + + model = self._make_model() + calls = [] + state = {"raised": False} + + def _gen(msgs, *a, **k): + calls.append(msgs) + if not state["raised"]: + state["raised"] = True + raise _E("transient bad request") # 400, no media marker + return "ok" + + model._generate = _gen + _patch_openai_compat_content(model) + + tm = self._img_tool() + assert model._generate([tm]) == "ok" # bare 400 -> stripped retry -> ok + assert len(calls) == 2 + + # NOT cached: the next call still attempts preserve (image kept, not stripped) + calls.clear() + assert model._generate([tm]) == "ok" + assert len(calls) == 1 + assert any( + isinstance(m.content, list) + and any(isinstance(b, dict) and b.get("type") == "image" for b in m.content) + for m in calls[0] + ) + + def test_mixed_modality_caches_only_culprit(self): + # image+file message; provider rejects only the file -> cache file only, + # images stay preserved on later turns — High #2. + from langchain_core.messages import HumanMessage + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + calls = [] + state = {"raised": False} + + def _gen(msgs, *a, **k): + calls.append(msgs) + if not state["raised"]: + state["raised"] = True + raise Exception("file input is not supported") + return "ok" + + model._generate = _gen + _patch_openai_compat_content(model) + + mixed = HumanMessage( + content=[ + {"type": "image", "base64": "A", "mime_type": "image/png"}, + {"type": "file", "base64": "F", "mime_type": "application/pdf"}, + ] + ) + assert model._generate([mixed]) == "ok" # file rejected -> retry -> cache file + + # later image-only request: image must still be preserved + calls.clear() + img = HumanMessage( + content=[{"type": "image", "base64": "A", "mime_type": "image/png"}] + ) + assert model._generate([img]) == "ok" + assert len(calls) == 1 + assert any( + isinstance(m.content, list) + and any(isinstance(b, dict) and b.get("type") == "image" for b in m.content) + for m in calls[0] + ) + + def test_stream_empty_retry_raises_original(self): + # If the stripped streaming retry yields ZERO chunks, surface the + # original error instead of silently returning an empty stream. + from unittest.mock import MagicMock + + from EvoScientist.llm.patches import _patch_openai_compat_content + + model = self._make_model() + model._generate = MagicMock(return_value="g") + calls = [] + + def _stream(msgs, *a, **k): + calls.append(msgs) + if len(calls) == 1: + raise Exception("No endpoints found that support image input") + return # retry yields nothing + yield # pragma: no cover (makes this a generator) + + model._stream = _stream + _patch_openai_compat_content(model) + + with pytest.raises(Exception, match="support image"): + list(model._stream([self._img_tool()])) + assert len(calls) == 2 + # ============================================================================= # Test _patch_deepseek_reasoning_passback