Port from RooCodeInc/Roomote#1796: per-task image forwarding on delegate_task

Subagents can now SEE images. Each delegate_task task accepts an optional
images list (max 8; local paths or http(s) URLs). Vision-capable children
receive native image_url content parts on their goal turn (local files as
data URLs, remote URLs verbatim); non-vision children get
[Image attached at: ...] hints plus a vision_analyze pointer. Routing
reuses agent.image_routing (decide_image_input_mode /
build_native_content_parts), so agent.image_input_mode governs delegation
exactly like inbound gateway images.

Best-effort by contract: malformed images arrays fail the call loudly
before any child spawns; unreadable paths are skipped with a log line;
any exception in the forwarding path degrades to the text-only goal.

Adapted from RooCodeInc/Roomote#1796 / #1767 (Fast agent forwards bounded
current-turn attachments to delegated coding tasks).
This commit is contained in:
Teknium
2026-08-28 22:17:06 -07:00
parent dcf17633de
commit f3f5c4f7c7
6 changed files with 311 additions and 10 deletions
+2 -1
View File
@@ -1305,7 +1305,8 @@ class AIAgent(
goal=function_args.get("goal"), context=function_args.get("context"),
tasks=_strip_model_hidden_task_fields(function_args.get("tasks")),
max_iterations=function_args.get("max_iterations"), role=function_args.get("role"),
background=not (getattr(self, "_delegate_depth", 0) > 0), action=function_args.get("action"),
background=not (getattr(self, "_delegate_depth", 0) > 0), images=function_args.get("images"),
action=function_args.get("action"),
subagent_id=function_args.get("subagent_id"), message=function_args.get("message"), parent_agent=self,
)
+173
View File
@@ -0,0 +1,173 @@
"""Per-task image forwarding on delegate_task.
Port of RooCodeInc/Roomote#1796 / #1767 (Fast → coding-task attachment
forwarding): a task may carry an ``images`` list (local paths or http(s)
URLs). Vision-capable children receive the pixels as native ``image_url``
content parts on their goal turn; non-vision children get
``[Image attached at: …]`` hint lines they can feed to ``vision_analyze``.
Forwarding is best-effort — any failure degrades to the text-only goal.
"""
import base64
from unittest.mock import patch
import pytest
from tools.delegate_tool import (
DELEGATE_TASK_SCHEMA,
_MAX_TASK_IMAGES,
_build_child_goal_message,
_normalize_task_images,
)
class _FakeChild:
provider = "openrouter"
model = "some/vision-model"
# ---------------------------------------------------------------------------
# _normalize_task_images
# ---------------------------------------------------------------------------
class TestNormalizeTaskImages:
def test_absent_is_none(self):
cleaned, err = _normalize_task_images({"goal": "g"}, 0)
assert cleaned is None
assert err is None
def test_list_passes_through_stripped(self):
cleaned, err = _normalize_task_images(
{"images": [" /tmp/a.png ", "https://x.test/b.jpg"]}, 0
)
assert err is None
assert cleaned == ["/tmp/a.png", "https://x.test/b.jpg"]
def test_bare_string_wrapped(self):
cleaned, err = _normalize_task_images({"images": "/tmp/a.png"}, 0)
assert err is None
assert cleaned == ["/tmp/a.png"]
def test_non_list_rejected(self):
cleaned, err = _normalize_task_images({"images": {"path": "x"}}, 2)
assert cleaned is None
assert err and "Task 2" in err
def test_non_string_entry_rejected(self):
cleaned, err = _normalize_task_images({"images": ["/tmp/a.png", 42]}, 0)
assert cleaned is None
assert err
def test_empty_string_entry_rejected(self):
cleaned, err = _normalize_task_images({"images": [" "]}, 0)
assert cleaned is None
assert err
def test_over_limit_rejected(self):
many = [f"/tmp/img{i}.png" for i in range(_MAX_TASK_IMAGES + 1)]
cleaned, err = _normalize_task_images({"images": many}, 1)
assert cleaned is None
assert err and str(_MAX_TASK_IMAGES) in err
def test_empty_list_normalizes_to_none(self):
cleaned, err = _normalize_task_images({"images": []}, 0)
assert cleaned is None
assert err is None
# ---------------------------------------------------------------------------
# _build_child_goal_message
# ---------------------------------------------------------------------------
def _make_png(tmp_path, name="shot.png"):
# 1x1 transparent PNG
png = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBg"
"AAAABQABh6FO1AAAAABJRU5ErkJggg=="
)
p = tmp_path / name
p.write_bytes(png)
return str(p)
class TestBuildChildGoalMessage:
def test_native_mode_builds_content_parts(self, tmp_path):
path = _make_png(tmp_path)
with patch(
"agent.image_routing.decide_image_input_mode", return_value="native"
):
msg = _build_child_goal_message("Inspect the mock", [path], _FakeChild())
assert isinstance(msg, list)
types = [p.get("type") for p in msg]
assert types[0] == "text"
assert "image_url" in types
assert "Inspect the mock" in msg[0]["text"]
# local file embedded as data URL
img = next(p for p in msg if p.get("type") == "image_url")
assert img["image_url"]["url"].startswith("data:image/")
def test_native_mode_passes_urls_verbatim(self):
url = "https://example.test/mock.png"
with patch(
"agent.image_routing.decide_image_input_mode", return_value="native"
):
msg = _build_child_goal_message("Look", [url], _FakeChild())
assert isinstance(msg, list)
img = next(p for p in msg if p.get("type") == "image_url")
assert img["image_url"]["url"] == url
def test_native_mode_all_unreadable_falls_back_to_text(self, tmp_path):
missing = str(tmp_path / "nope.png")
with patch(
"agent.image_routing.decide_image_input_mode", return_value="native"
):
msg = _build_child_goal_message("Goal text", [missing], _FakeChild())
assert msg == "Goal text"
def test_text_mode_appends_hints(self, tmp_path):
path = _make_png(tmp_path)
url = "https://example.test/a.png"
with patch(
"agent.image_routing.decide_image_input_mode", return_value="text"
):
msg = _build_child_goal_message("Goal", [path, url], _FakeChild())
assert isinstance(msg, str)
assert f"[Image attached at: {path}]" in msg
assert f"[Image attached: {url}]" in msg
assert "vision_analyze" in msg
def test_text_mode_missing_path_dropped(self, tmp_path):
missing = str(tmp_path / "gone.png")
with patch(
"agent.image_routing.decide_image_input_mode", return_value="text"
):
msg = _build_child_goal_message("Goal", [missing], _FakeChild())
assert msg == "Goal"
def test_any_exception_degrades_to_plain_goal(self):
with patch(
"agent.image_routing.decide_image_input_mode",
side_effect=RuntimeError("boom"),
):
msg = _build_child_goal_message("Plain goal", ["/tmp/x.png"], _FakeChild())
assert msg == "Plain goal"
# ---------------------------------------------------------------------------
# Tool-schema surface
# ---------------------------------------------------------------------------
class TestToolSchemaSurface:
def test_images_on_task_items(self):
item = DELEGATE_TASK_SCHEMA["parameters"]["properties"]["tasks"]["items"]
assert "images" in item["properties"]
assert item["properties"]["images"]["type"] == "array"
assert "images" not in item["required"]
def test_images_not_a_top_level_schema_param(self):
# Handler accepts top-level `images` for the legacy single-goal shape,
# but only the per-task field is advertised.
assert "images" not in DELEGATE_TASK_SCHEMA["parameters"]["properties"]
+25 -7
View File
@@ -24,7 +24,7 @@ logger = logging.getLogger(__name__)
# The delegate_tool_* siblings hold the pieces split out of this module; every name callers or patching tests reach as
# ``tools.delegate_tool.<name>`` is re-imported here. Mutable flag globals live only in their owning module.
from tools.delegate_tool_child_run import ( # noqa: F401
_ChildRun, _attach_child, _build_result_entry, _dump_subagent_timeout_diagnostic, _fabricated_entry,
_ChildRun, _attach_child, _build_child_goal_message, _build_result_entry, _dump_subagent_timeout_diagnostic, _fabricated_entry,
_lease_child_credential, _merge_late_steer, _register_child, _start_heartbeat, _validate_child_output_schema,
)
from tools.delegate_tool_config import ( # noqa: F401
@@ -46,7 +46,9 @@ from tools.delegate_tool_registry import ( # noqa: F401
get_subagent_attribution, interrupt_subagent, is_spawn_paused, list_active_subagents, set_spawn_paused,
steer_subagent,
)
from tools.delegate_tool_tasks import _coerce_task_schemas, _normalize_task_list
from tools.delegate_tool_tasks import ( # noqa: F401
_MAX_TASK_IMAGES, _coerce_task_images, _coerce_task_schemas, _normalize_task_images, _normalize_task_list,
)
from tools.delegate_tool_toolsets import ( # noqa: F401
DELEGATE_BLOCKED_TOOLS, _expand_parent_toolsets, _resolve_child_toolsets, _strip_blocked_tools,
)
@@ -360,7 +362,7 @@ def _run_single_child(
def _build_children(
task_list: List[Dict[str, Any]], task_schemas: List[Optional[Dict[str, Any]]], creds: Dict[str, Any], *,
top_role: str, max_iterations: int, parent_agent, routing_cfg: Dict[str, Any],
live_deleg_id: Optional[str], live_writers: list,
live_deleg_id: Optional[str], live_writers: list, task_images: Optional[List[Optional[List[str]]]] = None,
) -> tuple[List[tuple], Optional[str]]:
"""Build every child on the main thread (construction is not thread-safe);
``(children, None)`` or ``([], error)`` on an explicit-pin preflight failure."""
@@ -392,6 +394,11 @@ def _build_children(
if _task_schema is not None:
with _quiet("Could not attach output schema to child %d", i):
child._delegate_output_schema = _task_schema
# Validated per-task images; absent on image-less tasks, which keep the text-only goal turn.
_t_images = task_images[i] if task_images and i < len(task_images) else None
if _t_images:
with _quiet("Could not attach images to child %d", i):
child._delegate_images = _t_images
# Tee progress events into the live transcript (wrapper keeps the
# _flush contract and swallows writer failures).
_writer = live_writers[i] if i < len(live_writers) else None
@@ -410,8 +417,9 @@ def _build_children(
def delegate_task(
goal: Optional[str] = None, context: Optional[str] = None, tasks: Optional[List[Dict[str, Any]]] = None,
max_iterations: Optional[int] = None, role: Optional[str] = None, background: Optional[bool] = None,
output_schema: Optional[Dict[str, Any]] = None, action: Optional[str] = None, subagent_id: Optional[str] = None,
message: Optional[str] = None, parent_agent=None, credentials_cfg: Optional[Dict[str, Any]] = None,
output_schema: Optional[Dict[str, Any]] = None, images: Optional[List[str]] = None, action: Optional[str] = None,
subagent_id: Optional[str] = None, message: Optional[str] = None, parent_agent=None,
credentials_cfg: Optional[Dict[str, Any]] = None,
) -> str:
"""Spawn child agents (single ``goal`` or ``tasks=[...]`` batch) or control running ones. ``action``
list/steer/stop run synchronously and bypass the pause gate, depth limit and async dispatch. ``role`` is legacy
@@ -470,6 +478,8 @@ def delegate_task(
task_list, err = _normalize_task_list(goal, context, tasks, output_schema, top_role, max_children)
if not err:
task_schemas, err = _coerce_task_schemas(task_list, output_schema)
if not err:
task_images, err = _coerce_task_images(task_list, images)
if err:
return tool_error(err)
@@ -485,7 +495,7 @@ def delegate_task(
children, err = _build_children(
task_list, task_schemas, creds, top_role=top_role, max_iterations=default_max_iter, parent_agent=parent_agent,
routing_cfg=routing_cfg, live_deleg_id=live_deleg_id, live_writers=live_writers,
routing_cfg=routing_cfg, live_deleg_id=live_deleg_id, live_writers=live_writers, task_images=task_images,
)
if err:
return tool_error(err)
@@ -634,6 +644,14 @@ DELEGATE_TASK_SCHEMA = {
"schema_valid, plus schema_errors on failure). Keep it forgiving — require only "
"fields you will read.",
),
"images": _p(
"array",
"Optional images this child must SEE (max 8): local file paths or http(s) URLs — e.g. a "
"screenshot the user sent, a design mock, a chart. Vision-capable children receive the "
"pixels on their first turn; non-vision children get path hints for vision_analyze. Text "
"files do NOT belong here — put paths in 'context' instead.",
items={"type": "string"},
),
"group": _p(
"string",
"Optional result-delivery bucket within this call (only when delegation.independent_completions "
@@ -698,7 +716,7 @@ registry.register(
goal=args.get("goal"), context=args.get("context"), tasks=_strip_model_hidden_task_fields(args.get("tasks")),
max_iterations=args.get("max_iterations"), role=args.get("role"),
background=_model_background_value(args, kw.get("parent_agent")), output_schema=args.get("output_schema"),
action=args.get("action"), subagent_id=args.get("subagent_id"), message=args.get("message"),
images=args.get("images"), action=args.get("action"), subagent_id=args.get("subagent_id"), message=args.get("message"),
parent_agent=kw.get("parent_agent"),
),
check_fn=check_delegate_requirements,
+50 -1
View File
@@ -6,6 +6,7 @@ from __future__ import annotations
import logging
import contextvars
import json
import os
import threading
import time
from concurrent.futures import TimeoutError as FuturesTimeoutError
@@ -551,6 +552,51 @@ def _build_result_entry(
return entry
def _is_image_url(ref: str) -> bool:
return ref.startswith(("http://", "https://", "data:image/"))
def _build_child_goal_message(goal: str, images: List[str], child) -> Any:
"""The child's first user message when a task forwards ``images``.
Routing reuses the inbound-image policy (``agent.image_routing``, honouring ``agent.image_input_mode``): a
vision-capable child gets an OpenAI-style content list (text part + one ``image_url`` part per image; local files
as data URLs behind the read guard, http(s)/data URLs verbatim); otherwise the goal gains ``[Image attached …]``
hint lines for ``vision_analyze``. Any failure degrades to the text-only goal so image plumbing never breaks a
spawn — logged at warning since the caller asked for the images.
"""
try:
urls = [s for s in images if _is_image_url(s)]
paths = [s for s in images if not _is_image_url(s)]
from agent.image_routing import build_native_content_parts, decide_image_input_mode
cfg = None
with _quiet(None):
from hermes_cli.config import load_config_readonly
cfg = load_config_readonly()
mode = decide_image_input_mode(
str(getattr(child, "provider", "") or ""), str(getattr(child, "model", "") or ""), cfg,
requested_provider=str(getattr(child, "requested_provider", "") or ""),
)
if mode == "native":
parts, skipped = build_native_content_parts(goal, paths, urls)
if skipped:
logger.warning("delegate_task: skipped %d unreadable image(s) for subagent: %s", len(skipped), ", ".join(skipped[:3]))
return parts if any(p.get("type") == "image_url" for p in parts) else goal
hints: List[str] = []
for p in paths:
if os.path.isfile(p):
hints.append(f"[Image attached at: {p}]")
else:
logger.warning("delegate_task: image path not found, not forwarded: %s", p)
hints.extend(f"[Image attached: {u}]" for u in urls)
if not hints:
return goal
return goal + "\n\n" + "\n".join(hints) + "\nUse vision_analyze to inspect these images."
except Exception:
logger.warning("delegate_task: image forwarding failed; sending text-only goal", exc_info=True)
return goal
@dataclass
class _ChildRun:
"""State of one child run, shared by every phase of ``_run_single_child``.
@@ -658,13 +704,16 @@ class _ChildRun:
)
# Worker thread handle so the timeout diagnostic can dump its stack.
worker_thread_holder: Dict[str, Optional[threading.Thread]] = {"t": None}
# Resolved after seed_workspace so a multimodal goal's text part carries the worktree note too.
_images = list(getattr(child, "_delegate_images", None) or [])
user_message: Any = _build_child_goal_message(self.goal, _images, child) if _images else self.goal
def _run_with_thread_capture():
worker_thread_holder["t"] = threading.current_thread()
from agent.delegation_context import delegated_child_context
with delegated_child_context(str(getattr(child, "session_id", "") or "")):
return child.run_conversation(
user_message=self.goal, task_id=self.child_task_id, stream_callback=self.relay_text,
user_message=user_message, task_id=self.child_task_id, stream_callback=self.relay_text,
)
future = executor.submit(contextvars.copy_context().run, _run_with_thread_capture)
+41 -1
View File
@@ -1,4 +1,4 @@
"""delegate_task input validation: tasks=[...] / legacy goal normalisation and per-task output schemas."""
"""delegate_task input validation: tasks=[...] / legacy goal normalisation, per-task output schemas and images."""
from __future__ import annotations
@@ -120,3 +120,43 @@ def _coerce_task_schemas(
return [], f"Task {i} output_schema invalid: {schema_err}"
task_schemas.append(coerced_schema)
return task_schemas, None
# Per-task image ceiling: enough for screenshots/mocks while keeping the child's first request small.
_MAX_TASK_IMAGES = 8
def _normalize_task_images(task: dict, i: int) -> tuple[Optional[List[str]], Optional[str]]:
"""``(cleaned_list_or_None, None)`` for a task's optional ``images`` (local paths, http(s) or data: URLs), else
``(None, error)``. A bare string is wrapped into a one-entry list (small models emit scalars for arrays)."""
raw = task.get("images")
if raw is None:
return None, None
if isinstance(raw, str):
raw = [raw]
if not isinstance(raw, list):
return None, f"Task {i} 'images' must be an array of local file paths or http(s) URLs."
cleaned: List[str] = []
for item in raw:
if not isinstance(item, str) or not item.strip():
return None, f"Task {i} 'images' entries must be non-empty strings (local file paths or http(s) URLs)."
cleaned.append(item.strip())
if len(cleaned) > _MAX_TASK_IMAGES:
return None, (
f"Task {i} has {len(cleaned)} images; the per-task limit is {_MAX_TASK_IMAGES}. "
"Trim to the images the child actually needs to see."
)
return (cleaned or None), None
def _coerce_task_images(
task_list: List[Dict[str, Any]], images: Optional[List[str]]
) -> tuple[List[Optional[List[str]]], Optional[str]]:
"""Per-task validated image lists; a malformed list fails the whole call before any child spawns. The legacy
top-level ``images`` applies to a single task only, like ``output_schema``."""
task_images: List[Optional[List[str]]] = []
for i, task in enumerate(task_list):
if task.get("images") is None and len(task_list) == 1 and images is not None:
task = {**task, "images": images}
cleaned, err = _normalize_task_images(task, i)
if err:
return [], err
task_images.append(cleaned)
return task_images, None
@@ -102,6 +102,26 @@ delegate_task(
The subagent receives a focused system prompt built from your goal and context, instructing it to complete the task and provide a structured summary of what it did, what it found, any files modified, and any issues encountered.
### Forwarding Images to a Subagent
Text context is not enough when the task is inherently visual — a screenshot the user sent, a design mock, a rendered chart. Each task accepts an optional `images` list (up to 8 entries; local file paths or `http(s)` URLs):
```python
delegate_task(tasks=[{
"goal": "Compare the rendered dashboard against the design mock and list layout deviations",
"context": "The app runs at http://localhost:3000; the repo is at /home/user/dash.",
"images": ["/home/user/mocks/dashboard-v2.png",
"https://cdn.example.com/current-render.png"],
}])
```
Delivery follows the same routing as user-attached images (`agent.image_input_mode`):
- **Vision-capable child model** — the images arrive as native multimodal content on the child's first turn: local files are embedded as data URLs, remote URLs pass through for the provider to fetch. The child sees the actual pixels.
- **Non-vision child model** — the goal gains `[Image attached at: <path>]` hint lines and the child is told to inspect them with `vision_analyze`.
Forwarding is best-effort: unreadable paths are skipped with a log line, and any failure in the image plumbing falls back to the plain text goal — it can never break a spawn. Images are for things the child must *see*; put text file paths in `context` as usual.
## Practical Examples
### Parallel Research