Enhance multimodal handling in LLM model (#256)
* Enhance multimodal handling in LLM model - Updated `_flatten_message_content` to preserve media blocks (images, files) while flattening text content. - Introduced `_sanitize_messages` to manage media hoisting for tool messages, ensuring compatibility with OpenAI APIs. - Modified `_patch_openai_compat_content` to accommodate new media handling logic, including retry mechanisms for media errors. - Added comprehensive tests for media preservation, including various scenarios with images, files, and unsupported media types. * fix: preserve order of text and media blocks in message flattening * test: add tests for _strip_media_types to ensure position preservation and deduplication
This commit is contained in:
Xi Zhang
@@ -464,7 +464,10 @@ def get_chat_model(
|
||||
if (
|
||||
_is_third_party or _is_openai_proxy
|
||||
) and _original_provider not in _no_patch_providers:
|
||||
_patch_openai_compat_content(chat_model)
|
||||
# Anthropic-routed providers accept media in tool results natively;
|
||||
# only OpenAI-compatible providers need tool-media hoisting.
|
||||
_hoist = _original_provider not in _ANTHROPIC_ROUTED_PROVIDERS
|
||||
_patch_openai_compat_content(chat_model, hoist_tool_media=_hoist)
|
||||
|
||||
# DeepSeek thinking mode requires reasoning_content passback in multi-turn
|
||||
# + tool_use scenarios.
|
||||
|
||||
+319
-28
@@ -194,62 +194,260 @@ def _is_ccproxy_codex() -> bool:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Utility + Patch: Flatten list content to strings for OpenAI-compatible APIs.
|
||||
# DeepSeek, SiliconFlow, etc. reject assistant messages whose content is a
|
||||
# list rather than a string.
|
||||
# list rather than a string. Image and file (PDF/document) blocks are
|
||||
# preserved, not flattened away.
|
||||
# ---------------------------------------------------------------------------
|
||||
_SKIP_CONTENT_TYPES = frozenset({"thinking", "reasoning", "reasoning_content"})
|
||||
|
||||
# Media block types preserved when flattening (positive allowlist;
|
||||
# thinking/reasoning still dropped). Images + files (PDF/documents): both
|
||||
# serialize on OpenAI-compatible APIs and capable models read them. `video` is
|
||||
# deliberately EXCLUDED (langchain-openai raises ValueError on it); `audio` is
|
||||
# omitted (almost no model support). is_data_content_block is unreliable
|
||||
# (False for OpenAI image_url / Anthropic image-source). Kept as separate
|
||||
# image/file sets so the no-media fallback can gate per modality.
|
||||
_IMAGE_CONTENT_TYPES = frozenset({"image", "image_url", "input_image"})
|
||||
_FILE_CONTENT_TYPES = frozenset({"file", "input_file", "document"})
|
||||
_MEDIA_CONTENT_TYPES = _IMAGE_CONTENT_TYPES | _FILE_CONTENT_TYPES
|
||||
|
||||
def _flatten_message_content(content: Any) -> str | Any:
|
||||
"""Convert list-of-blocks content to a plain string.
|
||||
|
||||
def _flatten_message_content(content: Any) -> str | list[Any] | Any:
|
||||
"""Convert list-of-blocks content to a string, preserving media blocks.
|
||||
|
||||
Thinking/reasoning blocks are dropped. When a media block (image or file)
|
||||
is present, returns a list in the ORIGINAL order — consecutive text is
|
||||
joined into a single text block, media blocks kept as-is — so captions stay
|
||||
next to the attachment they describe. Otherwise returns a plain string.
|
||||
Non-list input is returned unchanged.
|
||||
|
||||
Args:
|
||||
content: Message content — either a string, a list of content blocks
|
||||
(dicts with ``type`` and ``text`` keys), or another type.
|
||||
content: Message content — a string, a list of content blocks, or
|
||||
another type.
|
||||
|
||||
Returns:
|
||||
A plain string with text blocks joined by double newlines.
|
||||
Thinking/reasoning blocks are skipped. Non-list input is
|
||||
returned unchanged.
|
||||
A plain string for text-only content, a list of blocks when media is
|
||||
present, or the input unchanged for non-list input.
|
||||
"""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if not isinstance(content, list):
|
||||
return content
|
||||
parts: list[str] = []
|
||||
ordered_blocks: list[Any] = []
|
||||
saw_media = False
|
||||
|
||||
def _flush_text() -> None:
|
||||
if parts:
|
||||
ordered_blocks.append({"type": "text", "text": "\n\n".join(parts)})
|
||||
parts.clear()
|
||||
|
||||
for block in content:
|
||||
if isinstance(block, dict):
|
||||
if block.get("type") in _SKIP_CONTENT_TYPES:
|
||||
btype = block.get("type")
|
||||
if btype in _MEDIA_CONTENT_TYPES:
|
||||
# Keep media as-is (never mutate; upstream copy.copy is shallow)
|
||||
# and preserve its position relative to surrounding text.
|
||||
_flush_text()
|
||||
ordered_blocks.append(block)
|
||||
saw_media = True
|
||||
continue
|
||||
if btype in _SKIP_CONTENT_TYPES:
|
||||
continue
|
||||
text = block.get("text")
|
||||
if text:
|
||||
parts.append(text)
|
||||
elif isinstance(block, str):
|
||||
parts.append(block)
|
||||
if saw_media:
|
||||
_flush_text()
|
||||
return ordered_blocks
|
||||
return "\n\n".join(parts) if parts else ""
|
||||
|
||||
|
||||
def _patch_openai_compat_content(model: Any) -> None:
|
||||
def _sanitize_messages(messages: list[Any], hoist_tool_media: bool = True) -> list[Any]:
|
||||
"""Flatten list content for OpenAI-compatible APIs, preserving media.
|
||||
|
||||
Text/reasoning content is flattened to a string; image blocks are
|
||||
preserved. Tool messages cannot carry media
|
||||
on OpenAI-compatible APIs (content must be a string), so when
|
||||
``hoist_tool_media`` is set the media in a tool result is moved into a
|
||||
HumanMessage emitted after that turn's (possibly parallel) tool messages.
|
||||
Anthropic-routed providers accept tool-result media natively and pass
|
||||
``hoist_tool_media=False`` to keep it inline.
|
||||
"""
|
||||
import copy
|
||||
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
out: list[Any] = []
|
||||
pending_media: list[Any] = [] # media hoisted out of a run of tool messages
|
||||
|
||||
def _flush() -> None:
|
||||
if pending_media:
|
||||
out.append(HumanMessage(content=list(pending_media)))
|
||||
pending_media.clear()
|
||||
|
||||
for msg in messages:
|
||||
is_tool = getattr(msg, "type", None) == "tool"
|
||||
if not is_tool:
|
||||
_flush() # emit hoisted media before any non-tool message
|
||||
if not isinstance(msg.content, list):
|
||||
out.append(msg)
|
||||
continue
|
||||
flat = _flatten_message_content(msg.content)
|
||||
if hoist_tool_media and is_tool and isinstance(flat, list):
|
||||
text_blocks = [
|
||||
b for b in flat if isinstance(b, dict) and b.get("type") == "text"
|
||||
]
|
||||
media_blocks = [
|
||||
b for b in flat if not (isinstance(b, dict) and b.get("type") == "text")
|
||||
]
|
||||
tool_msg = copy.copy(msg)
|
||||
# Join ALL text runs (interleaved content can yield more than one)
|
||||
# so no text is lost; tool content must be a string on OpenAI-compat.
|
||||
tool_msg.content = (
|
||||
"\n\n".join(b["text"] for b in text_blocks)
|
||||
if text_blocks
|
||||
else "[media content provided in the following message]"
|
||||
)
|
||||
out.append(tool_msg)
|
||||
pending_media.extend(media_blocks)
|
||||
else:
|
||||
msg = copy.copy(msg)
|
||||
msg.content = flat
|
||||
out.append(msg)
|
||||
_flush() # conversation may end with tool messages
|
||||
return out
|
||||
|
||||
|
||||
# Fallback for models that reject some media input (no vision / no file
|
||||
# support): replace blocks of the unsupported types with a placeholder so the
|
||||
# conversation can keep going instead of erroring every turn that re-sends them.
|
||||
_UNSUPPORTED_MEDIA_PLACEHOLDER = (
|
||||
"[attachment omitted: this model does not support this input type]"
|
||||
)
|
||||
# Markers that identify WHICH modality an error rejects, so only the implicated
|
||||
# type is remembered (not all media). Generic markers map to all media.
|
||||
_IMAGE_ERROR_MARKERS = ("image_url", "image input", "support image")
|
||||
_FILE_ERROR_MARKERS = ("file input", "pdf input", "document input")
|
||||
# `expected \`text\`` keeps the backticks — the bare "expected text" phrase is
|
||||
# too broad (matches non-media schema errors like "... expected text").
|
||||
_GENERIC_MEDIA_MARKERS = ("multimodal", "expected `text`")
|
||||
|
||||
|
||||
def _media_types_in(messages: list[Any]) -> set[str]:
|
||||
"""Set of preserved-media block types present across the messages."""
|
||||
found: set[str] = set()
|
||||
for msg in messages:
|
||||
content = getattr(msg, "content", None)
|
||||
if isinstance(content, list):
|
||||
for b in content:
|
||||
if isinstance(b, dict) and b.get("type") in _MEDIA_CONTENT_TYPES:
|
||||
found.add(b["type"])
|
||||
return found
|
||||
|
||||
|
||||
def _media_error_types(exc: Exception) -> set[str]:
|
||||
"""Media modalities the error text implicates (empty if not media-specific).
|
||||
|
||||
Used to remember ONLY the rejected modality, so e.g. a PDF rejection never
|
||||
disables images.
|
||||
"""
|
||||
text = str(exc).lower()
|
||||
types: set[str] = set()
|
||||
if any(m in text for m in _IMAGE_ERROR_MARKERS):
|
||||
types |= _IMAGE_CONTENT_TYPES
|
||||
if any(m in text for m in _FILE_ERROR_MARKERS):
|
||||
types |= _FILE_CONTENT_TYPES
|
||||
if any(m in text for m in _GENERIC_MEDIA_MARKERS):
|
||||
types |= _MEDIA_CONTENT_TYPES
|
||||
return types
|
||||
|
||||
|
||||
def _is_http_400(exc: Exception) -> bool:
|
||||
"""True if the error carries an HTTP 400 status (bad request)."""
|
||||
status = getattr(exc, "status_code", None)
|
||||
if status is None:
|
||||
status = getattr(getattr(exc, "response", None), "status_code", None)
|
||||
return status == 400
|
||||
|
||||
|
||||
def _strip_media_types(messages: list[Any], types: set[str]) -> list[Any]:
|
||||
"""Replace blocks of the given types with a placeholder text block.
|
||||
|
||||
Each stripped block is replaced IN PLACE (consecutive ones collapse into one
|
||||
placeholder), so surrounding text/media keep their original positions and a
|
||||
model that rejects only one modality still receives the others in order.
|
||||
"""
|
||||
import copy
|
||||
|
||||
out: list[Any] = []
|
||||
for msg in messages:
|
||||
content = getattr(msg, "content", None)
|
||||
if not isinstance(content, list) or not any(
|
||||
isinstance(b, dict) and b.get("type") in types for b in content
|
||||
):
|
||||
out.append(msg)
|
||||
continue
|
||||
kept: list[Any] = []
|
||||
last_was_placeholder = False
|
||||
for b in content:
|
||||
if isinstance(b, dict) and b.get("type") in types:
|
||||
if not last_was_placeholder:
|
||||
kept.append(
|
||||
{"type": "text", "text": _UNSUPPORTED_MEDIA_PLACEHOLDER}
|
||||
)
|
||||
last_was_placeholder = True
|
||||
continue
|
||||
kept.append(b)
|
||||
last_was_placeholder = False
|
||||
msg = copy.copy(msg)
|
||||
msg.content = kept
|
||||
out.append(msg)
|
||||
return out
|
||||
|
||||
|
||||
def _patch_openai_compat_content(model: Any, hoist_tool_media: bool = True) -> None:
|
||||
"""Flatten list content to strings before OpenAI-compatible API calls.
|
||||
|
||||
Wraps ``_generate`` / ``_agenerate`` to prevent "invalid type: sequence,
|
||||
expected a string" errors from strict APIs like DeepSeek.
|
||||
Wraps ``_generate`` / ``_agenerate`` / ``_stream`` / ``_astream`` to prevent
|
||||
"invalid type: sequence, expected a string" errors from strict APIs like
|
||||
DeepSeek, while preserving image and file (PDF/document) blocks. Tracks a
|
||||
per-modality ``blocked`` set of media types the model rejects: when a request
|
||||
fails with a media-looking error it is retried with the offending modalities
|
||||
stripped to a placeholder, and those types are remembered ONLY after the
|
||||
stripped retry succeeds — so an unrelated 400 never permanently degrades the
|
||||
model, and a PDF rejection never disables images. The profile pre-seeds the
|
||||
set per modality (``image_inputs``/``pdf_inputs`` ``is False``).
|
||||
|
||||
Args:
|
||||
model: A LangChain chat model instance to patch in-place.
|
||||
hoist_tool_media: When True (OpenAI-compatible), media in a tool result
|
||||
is hoisted into a following HumanMessage (tool content must be a
|
||||
string). Anthropic-routed providers pass False (native support).
|
||||
"""
|
||||
import copy
|
||||
import functools
|
||||
|
||||
from langchain_core.messages import BaseMessage
|
||||
|
||||
def _sanitize_messages(messages: list[BaseMessage]) -> list[BaseMessage]:
|
||||
out: list[BaseMessage] = []
|
||||
for msg in messages:
|
||||
if isinstance(msg.content, list):
|
||||
msg = copy.copy(msg)
|
||||
msg.content = _flatten_message_content(msg.content)
|
||||
out.append(msg)
|
||||
return out
|
||||
# Media block types this model rejects. Pre-seeded per modality from the
|
||||
# profile, then grown reactively — but only after a stripped retry succeeds.
|
||||
profile = getattr(model, "profile", None)
|
||||
blocked: set[str] = set()
|
||||
if isinstance(profile, dict):
|
||||
if profile.get("image_inputs") is False:
|
||||
blocked |= _IMAGE_CONTENT_TYPES
|
||||
if profile.get("pdf_inputs") is False:
|
||||
blocked |= _FILE_CONTENT_TYPES
|
||||
|
||||
def _prepare(messages: list[BaseMessage]) -> list[BaseMessage]:
|
||||
msgs = _strip_media_types(messages, blocked) if blocked else messages
|
||||
return _sanitize_messages(msgs, hoist_tool_media)
|
||||
|
||||
def _stripped(messages: list[BaseMessage], suspects: set[str]) -> list[BaseMessage]:
|
||||
return _sanitize_messages(
|
||||
_strip_media_types(messages, blocked | suspects), hoist_tool_media
|
||||
)
|
||||
|
||||
orig_generate = getattr(model, "_generate", None)
|
||||
if orig_generate is None:
|
||||
@@ -259,7 +457,23 @@ def _patch_openai_compat_content(model: Any) -> None:
|
||||
def _patched_generate(
|
||||
messages: list[BaseMessage], *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
return orig_generate(_sanitize_messages(messages), *args, **kwargs)
|
||||
prepared = _prepare(messages)
|
||||
suspects = _media_types_in(messages) - blocked
|
||||
if not suspects:
|
||||
return orig_generate(prepared, *args, **kwargs)
|
||||
try:
|
||||
return orig_generate(prepared, *args, **kwargs)
|
||||
except Exception as exc:
|
||||
culprit = _media_error_types(exc) & suspects
|
||||
if not culprit and not _is_http_400(exc):
|
||||
raise
|
||||
try:
|
||||
result = orig_generate(_stripped(messages, suspects), *args, **kwargs)
|
||||
except Exception:
|
||||
# stripping didn't help — surface original, don't cache
|
||||
raise exc from None
|
||||
blocked.update(culprit) # cache only the marker-identified modality
|
||||
return result
|
||||
|
||||
model._generate = _patched_generate
|
||||
|
||||
@@ -270,7 +484,24 @@ def _patch_openai_compat_content(model: Any) -> None:
|
||||
async def _patched_agenerate(
|
||||
messages: list[BaseMessage], *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
return await orig_agenerate(_sanitize_messages(messages), *args, **kwargs)
|
||||
prepared = _prepare(messages)
|
||||
suspects = _media_types_in(messages) - blocked
|
||||
if not suspects:
|
||||
return await orig_agenerate(prepared, *args, **kwargs)
|
||||
try:
|
||||
return await orig_agenerate(prepared, *args, **kwargs)
|
||||
except Exception as exc:
|
||||
culprit = _media_error_types(exc) & suspects
|
||||
if not culprit and not _is_http_400(exc):
|
||||
raise
|
||||
try:
|
||||
result = await orig_agenerate(
|
||||
_stripped(messages, suspects), *args, **kwargs
|
||||
)
|
||||
except Exception:
|
||||
raise exc from None
|
||||
blocked.update(culprit)
|
||||
return result
|
||||
|
||||
model._agenerate = _patched_agenerate
|
||||
|
||||
@@ -283,7 +514,38 @@ def _patch_openai_compat_content(model: Any) -> None:
|
||||
def _patched_stream(
|
||||
messages: list[BaseMessage], *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
return orig_stream(_sanitize_messages(messages), *args, **kwargs)
|
||||
prepared = _prepare(messages)
|
||||
suspects = _media_types_in(messages) - blocked
|
||||
if not suspects:
|
||||
yield from orig_stream(prepared, *args, **kwargs)
|
||||
return
|
||||
started = False
|
||||
try:
|
||||
for chunk in orig_stream(prepared, *args, **kwargs):
|
||||
started = True
|
||||
yield chunk
|
||||
return
|
||||
except Exception as exc:
|
||||
culprit = _media_error_types(exc) & suspects
|
||||
if started or (not culprit and not _is_http_400(exc)):
|
||||
raise
|
||||
media_exc = exc
|
||||
media_culprit = culprit
|
||||
retry_started = False
|
||||
try:
|
||||
for chunk in orig_stream(
|
||||
_stripped(messages, suspects), *args, **kwargs
|
||||
):
|
||||
if not retry_started:
|
||||
retry_started = True
|
||||
blocked.update(media_culprit)
|
||||
yield chunk
|
||||
except Exception:
|
||||
if retry_started:
|
||||
raise
|
||||
raise media_exc from None
|
||||
if not retry_started:
|
||||
raise media_exc from None # stripped retry produced nothing
|
||||
|
||||
model._stream = _patched_stream
|
||||
|
||||
@@ -294,10 +556,39 @@ def _patch_openai_compat_content(model: Any) -> None:
|
||||
async def _patched_astream(
|
||||
messages: list[BaseMessage], *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
async for chunk in orig_astream(
|
||||
_sanitize_messages(messages), *args, **kwargs
|
||||
):
|
||||
yield chunk
|
||||
prepared = _prepare(messages)
|
||||
suspects = _media_types_in(messages) - blocked
|
||||
if not suspects:
|
||||
async for chunk in orig_astream(prepared, *args, **kwargs):
|
||||
yield chunk
|
||||
return
|
||||
started = False
|
||||
try:
|
||||
async for chunk in orig_astream(prepared, *args, **kwargs):
|
||||
started = True
|
||||
yield chunk
|
||||
return
|
||||
except Exception as exc:
|
||||
culprit = _media_error_types(exc) & suspects
|
||||
if started or (not culprit and not _is_http_400(exc)):
|
||||
raise
|
||||
media_exc = exc
|
||||
media_culprit = culprit
|
||||
retry_started = False
|
||||
try:
|
||||
async for chunk in orig_astream(
|
||||
_stripped(messages, suspects), *args, **kwargs
|
||||
):
|
||||
if not retry_started:
|
||||
retry_started = True
|
||||
blocked.update(media_culprit)
|
||||
yield chunk
|
||||
except Exception:
|
||||
if retry_started:
|
||||
raise
|
||||
raise media_exc from None
|
||||
if not retry_started:
|
||||
raise media_exc from None # stripped retry produced nothing
|
||||
|
||||
model._astream = _patched_astream
|
||||
|
||||
|
||||
@@ -729,6 +729,107 @@ class TestFlattenMessageContent:
|
||||
content = [{"type": "thinking", "text": "thought"}]
|
||||
assert _flatten_message_content(content) == ""
|
||||
|
||||
def test_preserves_image_block(self):
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
img = {"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
assert _flatten_message_content([img]) == [img]
|
||||
|
||||
def test_preserves_image_url_block(self):
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
img = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}}
|
||||
assert _flatten_message_content([img]) == [img]
|
||||
|
||||
def test_preserves_file_block(self):
|
||||
# PDF/document files are preserved (capable models read them).
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
f = {"type": "file", "base64": "FFF", "mime_type": "application/pdf"}
|
||||
assert _flatten_message_content([f]) == [f]
|
||||
|
||||
def test_unsupported_media_dropped(self):
|
||||
# video/audio are NOT in the allowlist -> dropped, not crashing
|
||||
# (langchain-openai raises ValueError on `video`).
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
for block in (
|
||||
{"type": "video", "base64": "VVV", "mime_type": "video/mp4"},
|
||||
{"type": "audio", "base64": "ZZZ", "mime_type": "audio/wav"},
|
||||
):
|
||||
assert _flatten_message_content([block]) == ""
|
||||
|
||||
def test_non_image_media_dropped_keeps_text(self):
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
content = [
|
||||
{"type": "text", "text": "hi"},
|
||||
{"type": "video", "base64": "VVV", "mime_type": "video/mp4"},
|
||||
]
|
||||
# Video dropped, text kept -> plain string (no media list).
|
||||
assert _flatten_message_content(content) == "hi"
|
||||
|
||||
def test_consolidates_text_and_image(self):
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
img = {"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
content = [{"type": "text", "text": "a photo"}, img]
|
||||
assert _flatten_message_content(content) == [
|
||||
{"type": "text", "text": "a photo"},
|
||||
img,
|
||||
]
|
||||
|
||||
def test_multiple_text_blocks_with_image(self):
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
img = {"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
content = [
|
||||
{"type": "text", "text": "a"},
|
||||
{"type": "text", "text": "b"},
|
||||
img,
|
||||
]
|
||||
assert _flatten_message_content(content) == [
|
||||
{"type": "text", "text": "a\n\nb"},
|
||||
img,
|
||||
]
|
||||
|
||||
def test_preserves_text_media_ordering(self):
|
||||
# Text after an image must stay AFTER it (not consolidated to the front).
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
img = {"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
content = [
|
||||
{"type": "text", "text": "before"},
|
||||
img,
|
||||
{"type": "text", "text": "after"},
|
||||
]
|
||||
assert _flatten_message_content(content) == [
|
||||
{"type": "text", "text": "before"},
|
||||
img,
|
||||
{"type": "text", "text": "after"},
|
||||
]
|
||||
|
||||
def test_thinking_dropped_image_kept(self):
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
img = {"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
content = [{"type": "thinking", "text": "hmm"}, img]
|
||||
assert _flatten_message_content(content) == [img]
|
||||
|
||||
def test_pure_text_still_returns_string(self):
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
content = [{"type": "text", "text": "x"}, {"type": "text", "text": "y"}]
|
||||
result = _flatten_message_content(content)
|
||||
assert result == "x\n\ny"
|
||||
assert isinstance(result, str)
|
||||
|
||||
def test_unknown_nontext_block_still_dropped(self):
|
||||
from EvoScientist.llm.patches import _flatten_message_content
|
||||
|
||||
content = [{"type": "tool_use", "id": "1", "name": "foo"}]
|
||||
assert _flatten_message_content(content) == ""
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Test _patch_openai_compat_content (all 4 paths)
|
||||
@@ -820,6 +921,734 @@ class TestPatchOpenAICompatContent:
|
||||
assert chunks == ["c1", "c2"]
|
||||
assert received_msgs[0].content == "hello"
|
||||
|
||||
def test_generate_preserves_media(self):
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
orig = model._generate
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
img = {"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
msg = HumanMessage(content=[{"type": "text", "text": "see"}, img])
|
||||
model._generate([msg])
|
||||
|
||||
called_msgs = orig.call_args[0][0]
|
||||
assert called_msgs[0].content == [{"type": "text", "text": "see"}, img]
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_agenerate_preserves_media(self):
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
orig = model._agenerate
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
img = {"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
msg = HumanMessage(content=[{"type": "text", "text": "see"}, img])
|
||||
await model._agenerate([msg])
|
||||
|
||||
called_msgs = orig.call_args[0][0]
|
||||
assert called_msgs[0].content == [{"type": "text", "text": "see"}, img]
|
||||
|
||||
def test_stream_preserves_media(self):
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
orig = model._stream
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
img = {"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
msg = HumanMessage(content=[{"type": "text", "text": "see"}, img])
|
||||
list(model._stream([msg]))
|
||||
|
||||
called_msgs = orig.call_args[0][0]
|
||||
assert called_msgs[0].content == [{"type": "text", "text": "see"}, img]
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_astream_preserves_media(self):
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
received_msgs = []
|
||||
|
||||
async def _fake_astream(messages, *args, **kwargs):
|
||||
received_msgs.extend(messages)
|
||||
for chunk in ["c1", "c2"]:
|
||||
yield chunk
|
||||
|
||||
model._astream = _fake_astream
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
img = {"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
msg = HumanMessage(content=[{"type": "text", "text": "see"}, img])
|
||||
chunks = []
|
||||
async for c in model._astream([msg]):
|
||||
chunks.append(c)
|
||||
|
||||
assert chunks == ["c1", "c2"]
|
||||
assert received_msgs[0].content == [{"type": "text", "text": "see"}, img]
|
||||
|
||||
def test_toolmessage_image_hoisted_to_human(self):
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
orig = model._generate
|
||||
_patch_openai_compat_content(model) # hoist_tool_media=True (OpenAI-compat)
|
||||
|
||||
# deepagents read_file emits this exact shape for an image file.
|
||||
tm = ToolMessage(
|
||||
content_blocks=[
|
||||
{"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
],
|
||||
tool_call_id="tc1",
|
||||
name="read_file",
|
||||
)
|
||||
model._generate([tm])
|
||||
|
||||
called_msgs = orig.call_args[0][0]
|
||||
# Tool content becomes a string placeholder (OpenAI-compat requirement) ...
|
||||
assert isinstance(called_msgs[0].content, str)
|
||||
# ... and the image is hoisted into a following HumanMessage.
|
||||
assert len(called_msgs) == 2
|
||||
hoisted = called_msgs[1]
|
||||
assert hoisted.type == "human"
|
||||
assert any(
|
||||
isinstance(b, dict) and b.get("type") == "image" for b in hoisted.content
|
||||
)
|
||||
|
||||
def test_toolmessage_image_kept_inline_when_no_hoist(self):
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
orig = model._generate
|
||||
_patch_openai_compat_content(model, hoist_tool_media=False) # Anthropic-routed
|
||||
|
||||
tm = ToolMessage(
|
||||
content_blocks=[
|
||||
{"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
],
|
||||
tool_call_id="tc1",
|
||||
name="read_file",
|
||||
)
|
||||
model._generate([tm])
|
||||
|
||||
called_msgs = orig.call_args[0][0]
|
||||
# No hoisting: image stays inline in the tool message content.
|
||||
assert len(called_msgs) == 1
|
||||
content = called_msgs[0].content
|
||||
assert isinstance(content, list)
|
||||
assert any(isinstance(b, dict) and b.get("type") == "image" for b in content)
|
||||
|
||||
def test_parallel_tool_images_hoisted_after_tools(self):
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
orig = model._generate
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
ai = AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{"id": "c1", "name": "read_file", "args": {}},
|
||||
{"id": "c2", "name": "read_file", "args": {}},
|
||||
],
|
||||
)
|
||||
t1 = ToolMessage(
|
||||
content_blocks=[
|
||||
{"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
],
|
||||
tool_call_id="c1",
|
||||
name="read_file",
|
||||
)
|
||||
t2 = ToolMessage(
|
||||
content_blocks=[
|
||||
{"type": "image", "base64": "BBB", "mime_type": "image/png"}
|
||||
],
|
||||
tool_call_id="c2",
|
||||
name="read_file",
|
||||
)
|
||||
model._generate([ai, t1, t2])
|
||||
|
||||
called_msgs = orig.call_args[0][0]
|
||||
# Tool results stay consecutive; one hoisted HumanMessage follows them.
|
||||
assert [m.type for m in called_msgs] == ["ai", "tool", "tool", "human"]
|
||||
assert isinstance(called_msgs[1].content, str)
|
||||
assert isinstance(called_msgs[2].content, str)
|
||||
imgs = [b for b in called_msgs[3].content if b.get("type") == "image"]
|
||||
assert len(imgs) == 2
|
||||
|
||||
def test_assistant_text_still_flattened_to_string(self):
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
orig = model._generate
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
msg = AIMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "hi"},
|
||||
{"type": "thinking", "text": "t"},
|
||||
]
|
||||
)
|
||||
model._generate([msg])
|
||||
|
||||
called_msgs = orig.call_args[0][0]
|
||||
assert called_msgs[0].content == "hi"
|
||||
|
||||
def test_tool_media_flushed_before_next_human(self):
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
orig = model._generate
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
tm = ToolMessage(
|
||||
content_blocks=[
|
||||
{"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
],
|
||||
tool_call_id="tc1",
|
||||
name="read_file",
|
||||
)
|
||||
nxt = HumanMessage(content="thanks")
|
||||
model._generate([tm, nxt])
|
||||
|
||||
called = orig.call_args[0][0]
|
||||
# tool(placeholder), hoisted image (human), then the original human msg
|
||||
assert [m.type for m in called] == ["tool", "human", "human"]
|
||||
assert isinstance(called[0].content, str)
|
||||
assert any(b.get("type") == "image" for b in called[1].content)
|
||||
assert called[2].content == "thanks"
|
||||
|
||||
def test_tool_message_text_and_image_split(self):
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
orig = model._generate
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
tm = ToolMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "chart description"},
|
||||
{"type": "image", "base64": "AAA", "mime_type": "image/png"},
|
||||
],
|
||||
tool_call_id="tc1",
|
||||
name="read_file",
|
||||
)
|
||||
model._generate([tm])
|
||||
|
||||
called = orig.call_args[0][0]
|
||||
# Tool keeps the text as its string content; image hoisted to a human msg.
|
||||
assert called[0].content == "chart description"
|
||||
assert any(b.get("type") == "image" for b in called[1].content)
|
||||
|
||||
def test_tool_message_interleaved_text_not_lost(self):
|
||||
# Interleaved [text, image, text] in a tool result: BOTH text runs must
|
||||
# survive the hoisting split (not just the first).
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
orig = model._generate
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
tm = ToolMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "before"},
|
||||
{"type": "image", "base64": "AAA", "mime_type": "image/png"},
|
||||
{"type": "text", "text": "after"},
|
||||
],
|
||||
tool_call_id="tc1",
|
||||
name="read_file",
|
||||
)
|
||||
model._generate([tm])
|
||||
|
||||
called = orig.call_args[0][0]
|
||||
# both text runs preserved in the tool placeholder; image hoisted
|
||||
assert "before" in called[0].content
|
||||
assert "after" in called[0].content
|
||||
assert any(b.get("type") == "image" for b in called[1].content)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Test no-vision fallback (models that reject image input)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestNoVisionFallback:
|
||||
"""Verify image-rejecting models fall back to a text placeholder."""
|
||||
|
||||
def _img_tool(self):
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
return ToolMessage(
|
||||
content_blocks=[
|
||||
{"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
],
|
||||
tool_call_id="t1",
|
||||
name="read_file",
|
||||
)
|
||||
|
||||
def _make_model(self):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
model = MagicMock()
|
||||
model._agenerate = None
|
||||
model._stream = None
|
||||
model._astream = None
|
||||
return model
|
||||
|
||||
def test_media_error_types(self):
|
||||
from EvoScientist.llm.patches import (
|
||||
_FILE_CONTENT_TYPES,
|
||||
_IMAGE_CONTENT_TYPES,
|
||||
_is_http_400,
|
||||
_media_error_types,
|
||||
)
|
||||
|
||||
# marker identifies the specific modality
|
||||
assert (
|
||||
_media_error_types(Exception("No endpoints found that support image input"))
|
||||
>= _IMAGE_CONTENT_TYPES
|
||||
)
|
||||
assert (
|
||||
_media_error_types(Exception("file input is not supported"))
|
||||
== _FILE_CONTENT_TYPES
|
||||
)
|
||||
# DeepSeek-style maps to all media (generic "expected text")
|
||||
assert (
|
||||
_media_error_types(
|
||||
Exception("unknown variant `image_url`, expected `text`")
|
||||
)
|
||||
>= _IMAGE_CONTENT_TYPES
|
||||
)
|
||||
# non-media errors implicate nothing
|
||||
assert _media_error_types(Exception("rate limit exceeded")) == set()
|
||||
assert (
|
||||
_media_error_types(Exception("No endpoints found for some/model")) == set()
|
||||
)
|
||||
# bare "expected text" (non-media schema error) must NOT match
|
||||
assert (
|
||||
_media_error_types(
|
||||
Exception("tool schema validation failed: expected text")
|
||||
)
|
||||
== set()
|
||||
)
|
||||
|
||||
class _E(Exception):
|
||||
status_code = 400
|
||||
|
||||
assert _is_http_400(_E("bad request"))
|
||||
assert not _is_http_400(Exception("rate limit exceeded"))
|
||||
|
||||
def test_media_types_in(self):
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from EvoScientist.llm.patches import _media_types_in
|
||||
|
||||
img = {"type": "image", "base64": "A", "mime_type": "image/png"}
|
||||
f = {"type": "file", "base64": "F", "mime_type": "application/pdf"}
|
||||
assert _media_types_in([HumanMessage(content=[img, f])]) == {"image", "file"}
|
||||
assert _media_types_in([HumanMessage(content="hi")]) == set()
|
||||
|
||||
def test_strip_media_types_replaces_only_given(self):
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from EvoScientist.llm.patches import _strip_media_types
|
||||
|
||||
img = {"type": "image", "base64": "AAA", "mime_type": "image/png"}
|
||||
f = {"type": "file", "base64": "FFF", "mime_type": "application/pdf"}
|
||||
msg = HumanMessage(content=[{"type": "text", "text": "see"}, img, f])
|
||||
# Strip only files -> image survives, file becomes a placeholder block.
|
||||
out = _strip_media_types([msg], {"file"})
|
||||
types = [b.get("type") for b in out[0].content if isinstance(b, dict)]
|
||||
assert "image" in types # image preserved
|
||||
assert "file" not in types # file stripped
|
||||
assert any(
|
||||
b.get("type") == "text" and "omitted" in b.get("text", "").lower()
|
||||
for b in out[0].content
|
||||
)
|
||||
|
||||
def test_strip_media_types_preserves_position(self):
|
||||
# Stripped block is replaced IN PLACE; surrounding text/kept media keep
|
||||
# their order (placeholder where the image was, file stays last).
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from EvoScientist.llm.patches import _strip_media_types
|
||||
|
||||
img = {"type": "image", "base64": "A", "mime_type": "image/png"}
|
||||
f = {"type": "file", "base64": "F", "mime_type": "application/pdf"}
|
||||
msg = HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "t1"},
|
||||
img,
|
||||
{"type": "text", "text": "t2"},
|
||||
f,
|
||||
]
|
||||
)
|
||||
out = _strip_media_types([msg], {"image"}) # block only image
|
||||
content = out[0].content
|
||||
assert all(b.get("type") != "image" for b in content) # image gone
|
||||
# order preserved: t1, placeholder (where image was), t2, file
|
||||
assert content[0]["text"] == "t1"
|
||||
assert content[1]["type"] == "text"
|
||||
assert "omitted" in content[1]["text"].lower()
|
||||
assert content[2]["text"] == "t2"
|
||||
assert content[3]["type"] == "file" # file kept at its original position
|
||||
|
||||
def test_strip_media_types_dedups_consecutive(self):
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from EvoScientist.llm.patches import _strip_media_types
|
||||
|
||||
a = {"type": "image", "base64": "A", "mime_type": "image/png"}
|
||||
b = {"type": "image", "base64": "B", "mime_type": "image/png"}
|
||||
msg = HumanMessage(content=[a, b])
|
||||
out = _strip_media_types([msg], {"image"})
|
||||
# two adjacent stripped blocks collapse into ONE placeholder
|
||||
assert len(out[0].content) == 1
|
||||
assert "omitted" in out[0].content[0]["text"].lower()
|
||||
|
||||
def test_profile_no_vision_strips_upfront(self):
|
||||
# Proactive: profile says image_inputs is False -> strip from the start,
|
||||
# no failing first request.
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
model.profile = {"image_inputs": False}
|
||||
calls = []
|
||||
|
||||
def _gen(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
return "ok"
|
||||
|
||||
model._generate = _gen
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
assert model._generate([self._img_tool()]) == "ok"
|
||||
assert len(calls) == 1 # no failed attempt
|
||||
assert all(isinstance(m.content, str) for m in calls[0])
|
||||
assert any("omitted" in m.content.lower() for m in calls[0])
|
||||
|
||||
def test_profile_with_vision_does_not_strip(self):
|
||||
# Profile says image_inputs is True -> normal preserve path (no upfront strip).
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
model.profile = {"image_inputs": True}
|
||||
calls = []
|
||||
|
||||
def _gen(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
return "ok"
|
||||
|
||||
model._generate = _gen
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
assert model._generate([self._img_tool()]) == "ok"
|
||||
# Image preserved (hoisted), not replaced by a placeholder.
|
||||
assert any(
|
||||
isinstance(m.content, list)
|
||||
and any(b.get("type") == "image" for b in m.content)
|
||||
for m in calls[0]
|
||||
)
|
||||
|
||||
def test_generate_falls_back_and_caches(self):
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
calls = []
|
||||
state = {"raised": False}
|
||||
|
||||
def _gen(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
if not state["raised"]: # fail exactly once, ever
|
||||
state["raised"] = True
|
||||
raise Exception("unknown variant `image_url`, expected `text`")
|
||||
return "ok"
|
||||
|
||||
model._generate = _gen
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
tm = self._img_tool()
|
||||
# 1st turn: preserve attempt fails once -> strip -> ok
|
||||
assert model._generate([tm]) == "ok"
|
||||
assert len(calls) == 2
|
||||
retry = calls[1]
|
||||
assert all(isinstance(m.content, str) for m in retry)
|
||||
assert any("omitted" in m.content.lower() for m in retry)
|
||||
|
||||
# 2nd turn: cached no-vision -> straight to stripped, single call (no failure)
|
||||
calls.clear()
|
||||
assert model._generate([tm]) == "ok"
|
||||
assert len(calls) == 1
|
||||
assert all(isinstance(m.content, str) for m in calls[0])
|
||||
|
||||
def test_non_image_error_not_retried(self):
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
calls = []
|
||||
|
||||
def _gen(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
raise Exception("rate limit exceeded")
|
||||
|
||||
model._generate = _gen
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
with pytest.raises(Exception, match="rate limit"):
|
||||
model._generate([self._img_tool()])
|
||||
assert len(calls) == 1
|
||||
|
||||
def test_stream_falls_back(self):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
model._generate = MagicMock(return_value="g")
|
||||
calls = []
|
||||
|
||||
def _stream(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
if len(calls) == 1:
|
||||
raise Exception("No endpoints found that support image input")
|
||||
yield from ["x", "y"]
|
||||
|
||||
model._stream = _stream
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
out = list(model._stream([self._img_tool()]))
|
||||
assert out == ["x", "y"]
|
||||
assert len(calls) == 2
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_astream_falls_back(self):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
model._generate = MagicMock(return_value="g")
|
||||
calls = []
|
||||
|
||||
async def _astream(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
if len(calls) == 1:
|
||||
raise Exception("No endpoints found that support image input")
|
||||
for c in ["x", "y"]:
|
||||
yield c
|
||||
|
||||
model._astream = _astream
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
out = [c async for c in model._astream([self._img_tool()])]
|
||||
assert out == ["x", "y"]
|
||||
assert len(calls) == 2
|
||||
|
||||
def test_unrelated_400_retry_fails_not_cached(self):
|
||||
# A non-media 400 (e.g. tool schema) whose stripped retry ALSO fails must
|
||||
# surface the original error and must NOT permanently flip to no-media.
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
class _E(Exception):
|
||||
status_code = 400
|
||||
|
||||
model = self._make_model()
|
||||
calls = []
|
||||
|
||||
def _gen(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
raise _E("invalid tool schema") # 400, not media; fails every time
|
||||
|
||||
model._generate = _gen
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
tm = self._img_tool()
|
||||
with pytest.raises(_E):
|
||||
model._generate([tm])
|
||||
assert len(calls) == 2 # preserve attempt + stripped retry (both fail)
|
||||
|
||||
# Not cached: the next call attempts preserve again (not straight-to-stripped)
|
||||
calls.clear()
|
||||
with pytest.raises(_E):
|
||||
model._generate([tm])
|
||||
assert len(calls) == 2
|
||||
|
||||
def test_pdf_rejection_does_not_disable_images(self):
|
||||
# Per-modality: a PDF/file rejection caches only file types; a later
|
||||
# image must still be preserved (not stripped).
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
calls = []
|
||||
state = {"raised": False}
|
||||
|
||||
def _gen(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
has_file = any(
|
||||
isinstance(m.content, list)
|
||||
and any(
|
||||
isinstance(b, dict) and b.get("type") == "file" for b in m.content
|
||||
)
|
||||
for m in msgs
|
||||
)
|
||||
if has_file and not state["raised"]:
|
||||
state["raised"] = True
|
||||
raise Exception("file input is not supported")
|
||||
return "ok"
|
||||
|
||||
model._generate = _gen
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
pdf_tm = ToolMessage(
|
||||
content_blocks=[
|
||||
{"type": "file", "base64": "F", "mime_type": "application/pdf"}
|
||||
],
|
||||
tool_call_id="t1",
|
||||
name="read_file",
|
||||
)
|
||||
assert model._generate([pdf_tm]) == "ok" # file rejected -> stripped -> ok
|
||||
|
||||
# Now an image: must still be preserved (images not blocked by a PDF reject)
|
||||
calls.clear()
|
||||
img_msg = HumanMessage(
|
||||
content=[{"type": "image", "base64": "A", "mime_type": "image/png"}]
|
||||
)
|
||||
assert model._generate([img_msg]) == "ok"
|
||||
assert len(calls) == 1 # single attempt, no failure
|
||||
assert any(
|
||||
isinstance(m.content, list)
|
||||
and any(isinstance(b, dict) and b.get("type") == "image" for b in m.content)
|
||||
for m in calls[0]
|
||||
)
|
||||
|
||||
def test_bare_400_recovers_but_not_cached(self):
|
||||
# A bare 400 with NO media marker recovers this request (stripped retry)
|
||||
# but must NOT cache (no permanent degradation) — High #1.
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
class _E(Exception):
|
||||
status_code = 400
|
||||
|
||||
model = self._make_model()
|
||||
calls = []
|
||||
state = {"raised": False}
|
||||
|
||||
def _gen(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
if not state["raised"]:
|
||||
state["raised"] = True
|
||||
raise _E("transient bad request") # 400, no media marker
|
||||
return "ok"
|
||||
|
||||
model._generate = _gen
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
tm = self._img_tool()
|
||||
assert model._generate([tm]) == "ok" # bare 400 -> stripped retry -> ok
|
||||
assert len(calls) == 2
|
||||
|
||||
# NOT cached: the next call still attempts preserve (image kept, not stripped)
|
||||
calls.clear()
|
||||
assert model._generate([tm]) == "ok"
|
||||
assert len(calls) == 1
|
||||
assert any(
|
||||
isinstance(m.content, list)
|
||||
and any(isinstance(b, dict) and b.get("type") == "image" for b in m.content)
|
||||
for m in calls[0]
|
||||
)
|
||||
|
||||
def test_mixed_modality_caches_only_culprit(self):
|
||||
# image+file message; provider rejects only the file -> cache file only,
|
||||
# images stay preserved on later turns — High #2.
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
calls = []
|
||||
state = {"raised": False}
|
||||
|
||||
def _gen(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
if not state["raised"]:
|
||||
state["raised"] = True
|
||||
raise Exception("file input is not supported")
|
||||
return "ok"
|
||||
|
||||
model._generate = _gen
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
mixed = HumanMessage(
|
||||
content=[
|
||||
{"type": "image", "base64": "A", "mime_type": "image/png"},
|
||||
{"type": "file", "base64": "F", "mime_type": "application/pdf"},
|
||||
]
|
||||
)
|
||||
assert model._generate([mixed]) == "ok" # file rejected -> retry -> cache file
|
||||
|
||||
# later image-only request: image must still be preserved
|
||||
calls.clear()
|
||||
img = HumanMessage(
|
||||
content=[{"type": "image", "base64": "A", "mime_type": "image/png"}]
|
||||
)
|
||||
assert model._generate([img]) == "ok"
|
||||
assert len(calls) == 1
|
||||
assert any(
|
||||
isinstance(m.content, list)
|
||||
and any(isinstance(b, dict) and b.get("type") == "image" for b in m.content)
|
||||
for m in calls[0]
|
||||
)
|
||||
|
||||
def test_stream_empty_retry_raises_original(self):
|
||||
# If the stripped streaming retry yields ZERO chunks, surface the
|
||||
# original error instead of silently returning an empty stream.
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from EvoScientist.llm.patches import _patch_openai_compat_content
|
||||
|
||||
model = self._make_model()
|
||||
model._generate = MagicMock(return_value="g")
|
||||
calls = []
|
||||
|
||||
def _stream(msgs, *a, **k):
|
||||
calls.append(msgs)
|
||||
if len(calls) == 1:
|
||||
raise Exception("No endpoints found that support image input")
|
||||
return # retry yields nothing
|
||||
yield # pragma: no cover (makes this a generator)
|
||||
|
||||
model._stream = _stream
|
||||
_patch_openai_compat_content(model)
|
||||
|
||||
with pytest.raises(Exception, match="support image"):
|
||||
list(model._stream([self._img_tool()]))
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Test _patch_deepseek_reasoning_passback
|
||||
|
||||
Reference in New Issue
Block a user