diff --git a/run_agent.py b/run_agent.py index 6c2b000b86..ce76b3d6c0 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1305,7 +1305,8 @@ class AIAgent( goal=function_args.get("goal"), context=function_args.get("context"), tasks=_strip_model_hidden_task_fields(function_args.get("tasks")), max_iterations=function_args.get("max_iterations"), role=function_args.get("role"), - background=not (getattr(self, "_delegate_depth", 0) > 0), action=function_args.get("action"), + background=not (getattr(self, "_delegate_depth", 0) > 0), images=function_args.get("images"), + action=function_args.get("action"), subagent_id=function_args.get("subagent_id"), message=function_args.get("message"), parent_agent=self, ) diff --git a/tests/tools/test_delegate_images.py b/tests/tools/test_delegate_images.py new file mode 100644 index 0000000000..c403419b12 --- /dev/null +++ b/tests/tools/test_delegate_images.py @@ -0,0 +1,173 @@ +"""Per-task image forwarding on delegate_task. + +Port of RooCodeInc/Roomote#1796 / #1767 (Fast → coding-task attachment +forwarding): a task may carry an ``images`` list (local paths or http(s) +URLs). Vision-capable children receive the pixels as native ``image_url`` +content parts on their goal turn; non-vision children get +``[Image attached at: …]`` hint lines they can feed to ``vision_analyze``. +Forwarding is best-effort — any failure degrades to the text-only goal. +""" + +import base64 +from unittest.mock import patch + +import pytest + +from tools.delegate_tool import ( + DELEGATE_TASK_SCHEMA, + _MAX_TASK_IMAGES, + _build_child_goal_message, + _normalize_task_images, +) + + +class _FakeChild: + provider = "openrouter" + model = "some/vision-model" + + +# --------------------------------------------------------------------------- +# _normalize_task_images +# --------------------------------------------------------------------------- + + +class TestNormalizeTaskImages: + def test_absent_is_none(self): + cleaned, err = _normalize_task_images({"goal": "g"}, 0) + assert cleaned is None + assert err is None + + def test_list_passes_through_stripped(self): + cleaned, err = _normalize_task_images( + {"images": [" /tmp/a.png ", "https://x.test/b.jpg"]}, 0 + ) + assert err is None + assert cleaned == ["/tmp/a.png", "https://x.test/b.jpg"] + + def test_bare_string_wrapped(self): + cleaned, err = _normalize_task_images({"images": "/tmp/a.png"}, 0) + assert err is None + assert cleaned == ["/tmp/a.png"] + + def test_non_list_rejected(self): + cleaned, err = _normalize_task_images({"images": {"path": "x"}}, 2) + assert cleaned is None + assert err and "Task 2" in err + + def test_non_string_entry_rejected(self): + cleaned, err = _normalize_task_images({"images": ["/tmp/a.png", 42]}, 0) + assert cleaned is None + assert err + + def test_empty_string_entry_rejected(self): + cleaned, err = _normalize_task_images({"images": [" "]}, 0) + assert cleaned is None + assert err + + def test_over_limit_rejected(self): + many = [f"/tmp/img{i}.png" for i in range(_MAX_TASK_IMAGES + 1)] + cleaned, err = _normalize_task_images({"images": many}, 1) + assert cleaned is None + assert err and str(_MAX_TASK_IMAGES) in err + + def test_empty_list_normalizes_to_none(self): + cleaned, err = _normalize_task_images({"images": []}, 0) + assert cleaned is None + assert err is None + + +# --------------------------------------------------------------------------- +# _build_child_goal_message +# --------------------------------------------------------------------------- + + +def _make_png(tmp_path, name="shot.png"): + # 1x1 transparent PNG + png = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBg" + "AAAABQABh6FO1AAAAABJRU5ErkJggg==" + ) + p = tmp_path / name + p.write_bytes(png) + return str(p) + + +class TestBuildChildGoalMessage: + def test_native_mode_builds_content_parts(self, tmp_path): + path = _make_png(tmp_path) + with patch( + "agent.image_routing.decide_image_input_mode", return_value="native" + ): + msg = _build_child_goal_message("Inspect the mock", [path], _FakeChild()) + assert isinstance(msg, list) + types = [p.get("type") for p in msg] + assert types[0] == "text" + assert "image_url" in types + assert "Inspect the mock" in msg[0]["text"] + # local file embedded as data URL + img = next(p for p in msg if p.get("type") == "image_url") + assert img["image_url"]["url"].startswith("data:image/") + + def test_native_mode_passes_urls_verbatim(self): + url = "https://example.test/mock.png" + with patch( + "agent.image_routing.decide_image_input_mode", return_value="native" + ): + msg = _build_child_goal_message("Look", [url], _FakeChild()) + assert isinstance(msg, list) + img = next(p for p in msg if p.get("type") == "image_url") + assert img["image_url"]["url"] == url + + def test_native_mode_all_unreadable_falls_back_to_text(self, tmp_path): + missing = str(tmp_path / "nope.png") + with patch( + "agent.image_routing.decide_image_input_mode", return_value="native" + ): + msg = _build_child_goal_message("Goal text", [missing], _FakeChild()) + assert msg == "Goal text" + + def test_text_mode_appends_hints(self, tmp_path): + path = _make_png(tmp_path) + url = "https://example.test/a.png" + with patch( + "agent.image_routing.decide_image_input_mode", return_value="text" + ): + msg = _build_child_goal_message("Goal", [path, url], _FakeChild()) + assert isinstance(msg, str) + assert f"[Image attached at: {path}]" in msg + assert f"[Image attached: {url}]" in msg + assert "vision_analyze" in msg + + def test_text_mode_missing_path_dropped(self, tmp_path): + missing = str(tmp_path / "gone.png") + with patch( + "agent.image_routing.decide_image_input_mode", return_value="text" + ): + msg = _build_child_goal_message("Goal", [missing], _FakeChild()) + assert msg == "Goal" + + def test_any_exception_degrades_to_plain_goal(self): + with patch( + "agent.image_routing.decide_image_input_mode", + side_effect=RuntimeError("boom"), + ): + msg = _build_child_goal_message("Plain goal", ["/tmp/x.png"], _FakeChild()) + assert msg == "Plain goal" + + +# --------------------------------------------------------------------------- +# Tool-schema surface +# --------------------------------------------------------------------------- + + +class TestToolSchemaSurface: + def test_images_on_task_items(self): + item = DELEGATE_TASK_SCHEMA["parameters"]["properties"]["tasks"]["items"] + assert "images" in item["properties"] + assert item["properties"]["images"]["type"] == "array" + assert "images" not in item["required"] + + def test_images_not_a_top_level_schema_param(self): + # Handler accepts top-level `images` for the legacy single-goal shape, + # but only the per-task field is advertised. + assert "images" not in DELEGATE_TASK_SCHEMA["parameters"]["properties"] diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index b47e2fd7ac..272ae09718 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) # The delegate_tool_* siblings hold the pieces split out of this module; every name callers or patching tests reach as # ``tools.delegate_tool.`` is re-imported here. Mutable flag globals live only in their owning module. from tools.delegate_tool_child_run import ( # noqa: F401 - _ChildRun, _attach_child, _build_result_entry, _dump_subagent_timeout_diagnostic, _fabricated_entry, + _ChildRun, _attach_child, _build_child_goal_message, _build_result_entry, _dump_subagent_timeout_diagnostic, _fabricated_entry, _lease_child_credential, _merge_late_steer, _register_child, _start_heartbeat, _validate_child_output_schema, ) from tools.delegate_tool_config import ( # noqa: F401 @@ -46,7 +46,9 @@ from tools.delegate_tool_registry import ( # noqa: F401 get_subagent_attribution, interrupt_subagent, is_spawn_paused, list_active_subagents, set_spawn_paused, steer_subagent, ) -from tools.delegate_tool_tasks import _coerce_task_schemas, _normalize_task_list +from tools.delegate_tool_tasks import ( # noqa: F401 + _MAX_TASK_IMAGES, _coerce_task_images, _coerce_task_schemas, _normalize_task_images, _normalize_task_list, +) from tools.delegate_tool_toolsets import ( # noqa: F401 DELEGATE_BLOCKED_TOOLS, _expand_parent_toolsets, _resolve_child_toolsets, _strip_blocked_tools, ) @@ -360,7 +362,7 @@ def _run_single_child( def _build_children( task_list: List[Dict[str, Any]], task_schemas: List[Optional[Dict[str, Any]]], creds: Dict[str, Any], *, top_role: str, max_iterations: int, parent_agent, routing_cfg: Dict[str, Any], - live_deleg_id: Optional[str], live_writers: list, + live_deleg_id: Optional[str], live_writers: list, task_images: Optional[List[Optional[List[str]]]] = None, ) -> tuple[List[tuple], Optional[str]]: """Build every child on the main thread (construction is not thread-safe); ``(children, None)`` or ``([], error)`` on an explicit-pin preflight failure.""" @@ -392,6 +394,11 @@ def _build_children( if _task_schema is not None: with _quiet("Could not attach output schema to child %d", i): child._delegate_output_schema = _task_schema + # Validated per-task images; absent on image-less tasks, which keep the text-only goal turn. + _t_images = task_images[i] if task_images and i < len(task_images) else None + if _t_images: + with _quiet("Could not attach images to child %d", i): + child._delegate_images = _t_images # Tee progress events into the live transcript (wrapper keeps the # _flush contract and swallows writer failures). _writer = live_writers[i] if i < len(live_writers) else None @@ -410,8 +417,9 @@ def _build_children( def delegate_task( goal: Optional[str] = None, context: Optional[str] = None, tasks: Optional[List[Dict[str, Any]]] = None, max_iterations: Optional[int] = None, role: Optional[str] = None, background: Optional[bool] = None, - output_schema: Optional[Dict[str, Any]] = None, action: Optional[str] = None, subagent_id: Optional[str] = None, - message: Optional[str] = None, parent_agent=None, credentials_cfg: Optional[Dict[str, Any]] = None, + output_schema: Optional[Dict[str, Any]] = None, images: Optional[List[str]] = None, action: Optional[str] = None, + subagent_id: Optional[str] = None, message: Optional[str] = None, parent_agent=None, + credentials_cfg: Optional[Dict[str, Any]] = None, ) -> str: """Spawn child agents (single ``goal`` or ``tasks=[...]`` batch) or control running ones. ``action`` list/steer/stop run synchronously and bypass the pause gate, depth limit and async dispatch. ``role`` is legacy @@ -470,6 +478,8 @@ def delegate_task( task_list, err = _normalize_task_list(goal, context, tasks, output_schema, top_role, max_children) if not err: task_schemas, err = _coerce_task_schemas(task_list, output_schema) + if not err: + task_images, err = _coerce_task_images(task_list, images) if err: return tool_error(err) @@ -485,7 +495,7 @@ def delegate_task( children, err = _build_children( task_list, task_schemas, creds, top_role=top_role, max_iterations=default_max_iter, parent_agent=parent_agent, - routing_cfg=routing_cfg, live_deleg_id=live_deleg_id, live_writers=live_writers, + routing_cfg=routing_cfg, live_deleg_id=live_deleg_id, live_writers=live_writers, task_images=task_images, ) if err: return tool_error(err) @@ -634,6 +644,14 @@ DELEGATE_TASK_SCHEMA = { "schema_valid, plus schema_errors on failure). Keep it forgiving — require only " "fields you will read.", ), + "images": _p( + "array", + "Optional images this child must SEE (max 8): local file paths or http(s) URLs — e.g. a " + "screenshot the user sent, a design mock, a chart. Vision-capable children receive the " + "pixels on their first turn; non-vision children get path hints for vision_analyze. Text " + "files do NOT belong here — put paths in 'context' instead.", + items={"type": "string"}, + ), "group": _p( "string", "Optional result-delivery bucket within this call (only when delegation.independent_completions " @@ -698,7 +716,7 @@ registry.register( goal=args.get("goal"), context=args.get("context"), tasks=_strip_model_hidden_task_fields(args.get("tasks")), max_iterations=args.get("max_iterations"), role=args.get("role"), background=_model_background_value(args, kw.get("parent_agent")), output_schema=args.get("output_schema"), - action=args.get("action"), subagent_id=args.get("subagent_id"), message=args.get("message"), + images=args.get("images"), action=args.get("action"), subagent_id=args.get("subagent_id"), message=args.get("message"), parent_agent=kw.get("parent_agent"), ), check_fn=check_delegate_requirements, diff --git a/tools/delegate_tool_child_run.py b/tools/delegate_tool_child_run.py index 4a38995cf5..0b5c6f9248 100644 --- a/tools/delegate_tool_child_run.py +++ b/tools/delegate_tool_child_run.py @@ -6,6 +6,7 @@ from __future__ import annotations import logging import contextvars import json +import os import threading import time from concurrent.futures import TimeoutError as FuturesTimeoutError @@ -551,6 +552,51 @@ def _build_result_entry( return entry +def _is_image_url(ref: str) -> bool: + return ref.startswith(("http://", "https://", "data:image/")) + + +def _build_child_goal_message(goal: str, images: List[str], child) -> Any: + """The child's first user message when a task forwards ``images``. + + Routing reuses the inbound-image policy (``agent.image_routing``, honouring ``agent.image_input_mode``): a + vision-capable child gets an OpenAI-style content list (text part + one ``image_url`` part per image; local files + as data URLs behind the read guard, http(s)/data URLs verbatim); otherwise the goal gains ``[Image attached …]`` + hint lines for ``vision_analyze``. Any failure degrades to the text-only goal so image plumbing never breaks a + spawn — logged at warning since the caller asked for the images. + """ + try: + urls = [s for s in images if _is_image_url(s)] + paths = [s for s in images if not _is_image_url(s)] + from agent.image_routing import build_native_content_parts, decide_image_input_mode + cfg = None + with _quiet(None): + from hermes_cli.config import load_config_readonly + cfg = load_config_readonly() + mode = decide_image_input_mode( + str(getattr(child, "provider", "") or ""), str(getattr(child, "model", "") or ""), cfg, + requested_provider=str(getattr(child, "requested_provider", "") or ""), + ) + if mode == "native": + parts, skipped = build_native_content_parts(goal, paths, urls) + if skipped: + logger.warning("delegate_task: skipped %d unreadable image(s) for subagent: %s", len(skipped), ", ".join(skipped[:3])) + return parts if any(p.get("type") == "image_url" for p in parts) else goal + hints: List[str] = [] + for p in paths: + if os.path.isfile(p): + hints.append(f"[Image attached at: {p}]") + else: + logger.warning("delegate_task: image path not found, not forwarded: %s", p) + hints.extend(f"[Image attached: {u}]" for u in urls) + if not hints: + return goal + return goal + "\n\n" + "\n".join(hints) + "\nUse vision_analyze to inspect these images." + except Exception: + logger.warning("delegate_task: image forwarding failed; sending text-only goal", exc_info=True) + return goal + + @dataclass class _ChildRun: """State of one child run, shared by every phase of ``_run_single_child``. @@ -658,13 +704,16 @@ class _ChildRun: ) # Worker thread handle so the timeout diagnostic can dump its stack. worker_thread_holder: Dict[str, Optional[threading.Thread]] = {"t": None} + # Resolved after seed_workspace so a multimodal goal's text part carries the worktree note too. + _images = list(getattr(child, "_delegate_images", None) or []) + user_message: Any = _build_child_goal_message(self.goal, _images, child) if _images else self.goal def _run_with_thread_capture(): worker_thread_holder["t"] = threading.current_thread() from agent.delegation_context import delegated_child_context with delegated_child_context(str(getattr(child, "session_id", "") or "")): return child.run_conversation( - user_message=self.goal, task_id=self.child_task_id, stream_callback=self.relay_text, + user_message=user_message, task_id=self.child_task_id, stream_callback=self.relay_text, ) future = executor.submit(contextvars.copy_context().run, _run_with_thread_capture) diff --git a/tools/delegate_tool_tasks.py b/tools/delegate_tool_tasks.py index 282ceec2cd..0def5aeb80 100644 --- a/tools/delegate_tool_tasks.py +++ b/tools/delegate_tool_tasks.py @@ -1,4 +1,4 @@ -"""delegate_task input validation: tasks=[...] / legacy goal normalisation and per-task output schemas.""" +"""delegate_task input validation: tasks=[...] / legacy goal normalisation, per-task output schemas and images.""" from __future__ import annotations @@ -120,3 +120,43 @@ def _coerce_task_schemas( return [], f"Task {i} output_schema invalid: {schema_err}" task_schemas.append(coerced_schema) return task_schemas, None + +# Per-task image ceiling: enough for screenshots/mocks while keeping the child's first request small. +_MAX_TASK_IMAGES = 8 + +def _normalize_task_images(task: dict, i: int) -> tuple[Optional[List[str]], Optional[str]]: + """``(cleaned_list_or_None, None)`` for a task's optional ``images`` (local paths, http(s) or data: URLs), else + ``(None, error)``. A bare string is wrapped into a one-entry list (small models emit scalars for arrays).""" + raw = task.get("images") + if raw is None: + return None, None + if isinstance(raw, str): + raw = [raw] + if not isinstance(raw, list): + return None, f"Task {i} 'images' must be an array of local file paths or http(s) URLs." + cleaned: List[str] = [] + for item in raw: + if not isinstance(item, str) or not item.strip(): + return None, f"Task {i} 'images' entries must be non-empty strings (local file paths or http(s) URLs)." + cleaned.append(item.strip()) + if len(cleaned) > _MAX_TASK_IMAGES: + return None, ( + f"Task {i} has {len(cleaned)} images; the per-task limit is {_MAX_TASK_IMAGES}. " + "Trim to the images the child actually needs to see." + ) + return (cleaned or None), None + +def _coerce_task_images( + task_list: List[Dict[str, Any]], images: Optional[List[str]] +) -> tuple[List[Optional[List[str]]], Optional[str]]: + """Per-task validated image lists; a malformed list fails the whole call before any child spawns. The legacy + top-level ``images`` applies to a single task only, like ``output_schema``.""" + task_images: List[Optional[List[str]]] = [] + for i, task in enumerate(task_list): + if task.get("images") is None and len(task_list) == 1 and images is not None: + task = {**task, "images": images} + cleaned, err = _normalize_task_images(task, i) + if err: + return [], err + task_images.append(cleaned) + return task_images, None diff --git a/website/docs/user-guide/features/delegation.md b/website/docs/user-guide/features/delegation.md index 3d2c49bd66..0feb62d928 100644 --- a/website/docs/user-guide/features/delegation.md +++ b/website/docs/user-guide/features/delegation.md @@ -102,6 +102,26 @@ delegate_task( The subagent receives a focused system prompt built from your goal and context, instructing it to complete the task and provide a structured summary of what it did, what it found, any files modified, and any issues encountered. +### Forwarding Images to a Subagent + +Text context is not enough when the task is inherently visual — a screenshot the user sent, a design mock, a rendered chart. Each task accepts an optional `images` list (up to 8 entries; local file paths or `http(s)` URLs): + +```python +delegate_task(tasks=[{ + "goal": "Compare the rendered dashboard against the design mock and list layout deviations", + "context": "The app runs at http://localhost:3000; the repo is at /home/user/dash.", + "images": ["/home/user/mocks/dashboard-v2.png", + "https://cdn.example.com/current-render.png"], +}]) +``` + +Delivery follows the same routing as user-attached images (`agent.image_input_mode`): + +- **Vision-capable child model** — the images arrive as native multimodal content on the child's first turn: local files are embedded as data URLs, remote URLs pass through for the provider to fetch. The child sees the actual pixels. +- **Non-vision child model** — the goal gains `[Image attached at: ]` hint lines and the child is told to inspect them with `vision_analyze`. + +Forwarding is best-effort: unreadable paths are skipped with a log line, and any failure in the image plumbing falls back to the plain text goal — it can never break a spawn. Images are for things the child must *see*; put text file paths in `context` as usual. + ## Practical Examples ### Parallel Research