"""Image-part handling for ``AIAgent`` API messages. Vision capability probes, non-vision text fallbacks (cached ``vision_analyze`` descriptions), tool-result image stripping, and provider quirks (Anthropic dot preservation, Qwen portal message shaping). Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged. """ import logging import asyncio import base64 import copy import hashlib import json import os import tempfile from pathlib import Path from typing import Any, List, Optional from agent.lazy_forward import forward_static as _forward_static from agent.tool_dispatch_helpers import _is_multimodal_tool_result, _multimodal_text_summary from utils import base_url_host_matches, base_url_hostname # Same logger name as the origin module so log records / caplog filters are unchanged. logger = logging.getLogger("run_agent") class VisionMessagePrepMixin: """Vision probes + image-part fallbacks for outgoing messages (see module docstring).""" @staticmethod def _content_has_image_parts(content: Any) -> bool: if not isinstance(content, list): return False for part in content: if isinstance(part, dict) and part.get("type") in {"image_url", "input_image"}: return True return False # 20 MB base64 ≈ 15 MB decoded — prevents OOM from an oversized data: URL in a shared gateway process. _MAX_DATA_URL_BASE64_BYTES = 20 * 1024 * 1024 @staticmethod def _materialize_data_url_for_vision(image_url: str) -> tuple[str, Optional[Path]]: header, _, data = str(image_url or "").partition(",") if len(data) > VisionMessagePrepMixin._MAX_DATA_URL_BASE64_BYTES: logger.warning( "data-URL payload too large (%d bytes), skipping", len(data) ) return "", None mime = "image/jpeg" if header.startswith("data:"): mime_part = header[len("data:"):].split(";", 1)[0].strip() if mime_part.startswith("image/"): mime = mime_part suffix = { "image/png": ".png", "image/gif": ".gif", "image/webp": ".webp", "image/jpeg": ".jpg", "image/jpg": ".jpg", }.get(mime, ".jpg") tmp = tempfile.NamedTemporaryFile(prefix="anthropic_image_", suffix=suffix, delete=False) try: with tmp: tmp.write(base64.b64decode(data)) except Exception: # delete=False means a corrupt/unsupported data URL would otherwise # leak a zero-byte temp file on every failed materialization. try: os.unlink(tmp.name) except OSError: pass raise path = Path(tmp.name) return str(path), path def _describe_image_for_anthropic_fallback(self, image_url: str, role: str) -> str: cache_key = hashlib.sha256(str(image_url or "").encode("utf-8")).hexdigest() cached = self._anthropic_image_fallback_cache.get(cache_key) if cached: return cached role_label = { "assistant": "assistant", "tool": "tool result", }.get(role, "user") analysis_prompt = ( "Describe everything visible in this image in thorough detail. " "Include any text, code, UI, data, objects, people, layout, colors, " "and any other notable visual information." ) vision_source = str(image_url or "") cleanup_path: Optional[Path] = None if vision_source.startswith("data:"): vision_source, cleanup_path = self._materialize_data_url_for_vision(vision_source) description = "" try: from tools.vision_tools import vision_analyze_tool result_json = asyncio.run( vision_analyze_tool(image_url=vision_source, user_prompt=analysis_prompt) ) result = json.loads(result_json) if isinstance(result_json, str) else {} description = (result.get("analysis") or "").strip() except Exception as e: description = f"Image analysis failed: {e}" finally: if cleanup_path and cleanup_path.exists(): try: cleanup_path.unlink() except OSError: pass if not description: description = "Image analysis failed." note = f"[The {role_label} attached an image. Here's what it contains:\n{description}]" if vision_source and not str(image_url or "").startswith("data:"): note += ( f"\n[If you need a closer look, use vision_analyze with image_url: {vision_source}]" ) self._anthropic_image_fallback_cache[cache_key] = note return note def _model_supports_vision(self) -> bool: """Return True if the active provider+model reports native vision. Resolution: ``model.supports_vision`` > ``providers.

.models..supports_vision`` > models.dev lookup (see ``image_routing._supports_vision_override``). Custom/local models absent from models.dev would otherwise be misclassified and have their images stripped. """ try: from hermes_cli.config import load_config from agent.image_routing import _lookup_supports_vision cfg = load_config() provider = (getattr(self, "provider", "") or "").strip() model = (getattr(self, "model", "") or "").strip() return _lookup_supports_vision(provider, model, cfg) is True except Exception: return False def _provider_supports_vision_tool_messages(self) -> bool: """Return True if the active provider accepts list-type tool content. Some providers (Xiaomi MiMo) accept multimodal user messages but 400 on list-type tool content; reads the provider profile's ``supports_vision_tool_messages``. """ try: from providers import get_provider_profile provider = (getattr(self, "provider", "") or "").strip() profile = get_provider_profile(provider) if profile is not None: return getattr(profile, "supports_vision_tool_messages", True) except Exception: pass return True # default: assume compatible def _preprocess_anthropic_content(self, content: Any, role: str) -> Any: if not self._content_has_image_parts(content): return content text_parts: List[str] = [] image_notes: List[str] = [] for part in content: if isinstance(part, str): if part.strip(): text_parts.append(part.strip()) continue if not isinstance(part, dict): continue ptype = part.get("type") if ptype in {"text", "input_text"}: text = str(part.get("text", "") or "").strip() if text: text_parts.append(text) continue if ptype in {"image_url", "input_image"}: image_data = part.get("image_url", {}) image_url = image_data.get("url", "") if isinstance(image_data, dict) else str(image_data or "") if image_url: image_notes.append(self._describe_image_for_anthropic_fallback(image_url, role)) else: image_notes.append("[An image was attached but no image source was available.]") continue text = str(part.get("text", "") or "").strip() if text: text_parts.append(text) prefix = "\n\n".join(note for note in image_notes if note).strip() suffix = "\n".join(text for text in text_parts if text).strip() if prefix and suffix: return f"{prefix}\n\n{suffix}" if prefix: return prefix if suffix: return suffix return "[A multimodal message was converted to text for Anthropic compatibility.]" def _get_transport(self, api_mode: str = None): """Return the cached transport for the given (or current) api_mode (lazy; None if unregistered).""" mode = api_mode or self.api_mode cache = getattr(self, "_transport_cache", None) if cache is None: cache = {} self._transport_cache = cache t = cache.get(mode) if t is None: from agent.transports import get_transport t = get_transport(mode) cache[mode] = t return t def _prepare_messages_for_non_vision_model(self, api_messages: list) -> list: """Replace native image parts with cached vision_analyze text when the active model lacks vision. Vision-capable models pass through unchanged (the provider adapter — including the Anthropic one — handles image parts natively). The text fallback is the historically Anthropic-named preprocessor. """ if not any( isinstance(msg, dict) and self._content_has_image_parts(msg.get("content")) for msg in api_messages ): return api_messages if self._model_supports_vision(): return api_messages transformed = copy.deepcopy(api_messages) for msg in transformed: if not isinstance(msg, dict): continue msg["content"] = self._preprocess_anthropic_content( msg.get("content"), str(msg.get("role", "user") or "user"), ) return transformed # Same transform for the Anthropic route (callers/tests patch this name independently). _prepare_anthropic_messages_for_api = _prepare_messages_for_non_vision_model def _tool_result_content_for_active_model(self, tool_name: str, result: Any) -> Any: """Return the tool message content that is safe for the active model. Text-only providers must not receive image parts: a rejected tool result becomes canonical history and can make the next user turn fail before the agent can recover. """ if not _is_multimodal_tool_result(result): return result content = result.get("content") or [] if not self._content_has_image_parts(content): return content if self._model_supports_vision(): # Vision on paper, but the provider rejects list-type tool content (or we already learned that # in-session): short-circuit to a text summary. if not self._provider_supports_vision_tool_messages(): logger.debug( "Tool %s: provider %s does not accept list-type tool " "content — sending text summary", tool_name, getattr(self, "provider", ""), ) return _multimodal_text_summary(result) key = ( (getattr(self, "provider", "") or "").strip().lower(), (getattr(self, "model", "") or "").strip(), ) no_list = getattr(self, "_no_list_tool_content_models", None) if no_list and key in no_list: logger.debug( "Tool %s: model %s/%s known to reject list-type tool " "content this session — sending text summary", tool_name, key[0], key[1], ) return _multimodal_text_summary(result) return content summary = _multimodal_text_summary(result) if tool_name == "computer_use": return json.dumps({ "error": ( "computer_use returned screenshot/image content, but the active " "model/provider does not support image input. Switch to a " "vision-capable model for desktop computer use, or use browser " "tools for browser tasks." ), "text_summary": summary, }) logger.warning( "Tool %s returned image content for non-vision model %s/%s; " "falling back to text summary", tool_name, self.provider, self.model, ) return summary _try_shrink_image_parts_in_messages = _forward_static("agent.conversation_compression", "try_shrink_image_parts_in_messages") def _try_strip_image_parts_from_tool_messages( self, api_messages: list, *, remember_model: bool = True, ) -> bool: """Downgrade list-type tool messages to text summaries in place; returns True if any were downgraded. Recovery for providers that 400 on list-type tool content (e.g. MiMo "text is not set"). By default records the (provider, model) in ``_no_list_tool_content_models`` so later results downgrade without a round-trip; 413 recovery passes ``remember_model=False`` (body too large ≠ provider rejects lists). """ if not isinstance(api_messages, list): return False if remember_model: # Record (provider, model) so we don't relearn this lesson. key = ( (getattr(self, "provider", "") or "").strip().lower(), (getattr(self, "model", "") or "").strip(), ) if not hasattr(self, "_no_list_tool_content_models"): self._no_list_tool_content_models = set() if key[1]: # only record when we actually have a model id self._no_list_tool_content_models.add(key) changed = False for msg in api_messages: if not isinstance(msg, dict) or msg.get("role") != "tool": continue content = msg.get("content") if not isinstance(content, list): continue # Salvage any text parts so the model still sees some signal. text_parts: List[str] = [] had_image = False for part in content: if not isinstance(part, dict): if isinstance(part, str) and part.strip(): text_parts.append(part.strip()) continue ptype = part.get("type") if ptype == "image_url" or ptype == "input_image": had_image = True continue if ptype in {"text", "input_text"}: text = str(part.get("text") or "").strip() if text: text_parts.append(text) if not had_image: # List content without image parts — leave alone; stripping wouldn't reduce ambiguity. continue if text_parts: msg["content"] = "\n\n".join(text_parts) else: msg["content"] = ( "[image content removed — provider does not accept " "list-type tool message content]" ) changed = True return changed def _anthropic_preserve_dots(self) -> bool: """True when using an anthropic-compatible endpoint that preserves dots in model names. DashScope, MiniMax, Xiaomi MiMo, OpenCode Go/Zen (non-Claude), ZAI/Zhipu keep dots; AWS Bedrock uses dotted inference-profile IDs and rejects the hyphenated form with HTTP 400. """ if (getattr(self, "provider", "") or "").lower() in { "alibaba", "minimax", "minimax-cn", "opencode-go", "opencode-zen", "zai", "bedrock", "xiaomi", "vertex", }: return True base = (getattr(self, "base_url", "") or "").lower() host = base_url_hostname(base) return ( "dashscope" in host or base_url_host_matches(base, "aliyuncs.com") or "minimax" in host or (base_url_host_matches(base, "opencode.ai") and "/zen/" in base) or base_url_host_matches(base, "bigmodel.cn") or base_url_host_matches(base, "xiaomimimo.com") # Vertex AI OpenAI-compat endpoint — Gemini model ids keep dots # (e.g. google/gemini-3.5-flash); the hyphenated form is wrong. or base_url_host_matches(base, "aiplatform.googleapis.com") # AWS Bedrock runtime endpoints — defense-in-depth when # ``provider`` is unset but ``base_url`` still names Bedrock. or host.startswith("bedrock-runtime.") ) def _is_qwen_portal(self) -> bool: """Return True when the base URL targets Qwen Portal.""" return base_url_host_matches(self._base_url_lower, "portal.qwen.ai") def _qwen_prepare_chat_messages(self, api_messages: list) -> list: prepared = copy.deepcopy(api_messages) if not prepared: return prepared for msg in prepared: if not isinstance(msg, dict): continue content = msg.get("content") if isinstance(content, str): msg["content"] = [{"type": "text", "text": content}] elif isinstance(content, list): # Normalize: convert bare strings to text dicts, keep dicts as-is. # deepcopy already created independent copies, no need for dict(). normalized_parts = [] for part in content: if isinstance(part, str): normalized_parts.append({"type": "text", "text": part}) elif isinstance(part, dict): normalized_parts.append(part) if normalized_parts: msg["content"] = normalized_parts # Inject cache_control on the last part of the system message. for msg in prepared: if isinstance(msg, dict) and msg.get("role") == "system": content = msg.get("content") if isinstance(content, list) and content and isinstance(content[-1], dict): content[-1]["cache_control"] = {"type": "ephemeral"} break return prepared def _qwen_prepare_chat_messages_inplace(self, messages: list) -> None: """In-place variant — mutates an already-copied message list.""" if not messages: return for msg in messages: if not isinstance(msg, dict): continue content = msg.get("content") if isinstance(content, str): msg["content"] = [{"type": "text", "text": content}] elif isinstance(content, list): normalized_parts = [] for part in content: if isinstance(part, str): normalized_parts.append({"type": "text", "text": part}) elif isinstance(part, dict): normalized_parts.append(part) if normalized_parts: msg["content"] = normalized_parts for msg in messages: if isinstance(msg, dict) and msg.get("role") == "system": content = msg.get("content") if isinstance(content, list) and content and isinstance(content[-1], dict): content[-1]["cache_control"] = {"type": "ephemeral"} break