Merge branch 'simp/r3-32-H' into simp/integration3
This commit is contained in:
+10
-22
@@ -15,8 +15,7 @@ from urllib.parse import urlencode
|
||||
def import_fal_client() -> Any:
|
||||
"""Import ``fal_client`` (via ``lazy_deps`` when available); raises ImportError if unavailable.
|
||||
|
||||
Not imported at cold start (~64 ms per CLI invocation). Callers cache the result on their
|
||||
own module global so tests can monkeypatch that module's ``fal_client`` attribute.
|
||||
Callers cache the result on their own module global so tests can monkeypatch it.
|
||||
"""
|
||||
try:
|
||||
from tools.lazy_deps import ensure as _lazy_ensure
|
||||
@@ -54,31 +53,25 @@ def _require(value: Any, what: str) -> Any:
|
||||
|
||||
|
||||
class _ManagedFalSyncClient:
|
||||
"""Per-instance wrapper driving a Nous-managed fal-queue gateway via ``fal_client.SyncClient`` primitives.
|
||||
|
||||
Carries its own ``fal_client`` reference instead of a module global so the caller decides
|
||||
which module's (possibly test-patched) ``fal_client`` is used.
|
||||
"""
|
||||
"""Drives a Nous-managed fal-queue gateway via ``fal_client.SyncClient`` primitives; carries
|
||||
its own ``fal_client`` reference so the caller decides which (possibly test-patched) module is used."""
|
||||
|
||||
def __init__(self, fal_client: Any, *, key: str, queue_run_origin: str):
|
||||
sync_client_class = _require(getattr(fal_client, "SyncClient", None), "fal_client.SyncClient")
|
||||
client_module = _require(getattr(fal_client, "client", None), "fal_client.client")
|
||||
|
||||
self._queue_url_format = _normalize_fal_queue_url_format(queue_run_origin)
|
||||
self._sync_client = sync_client_class(key=key)
|
||||
self._http_client = getattr(self._sync_client, "_client", None)
|
||||
self._http_client = _require(getattr(self._sync_client, "_client", None), "fal_client.SyncClient._client")
|
||||
self._maybe_retry_request = getattr(client_module, "_maybe_retry_request", None)
|
||||
self._raise_for_status = getattr(client_module, "_raise_for_status", None)
|
||||
self._request_handle_class = getattr(client_module, "SyncRequestHandle", None)
|
||||
if self._maybe_retry_request is None or self._raise_for_status is None:
|
||||
raise RuntimeError("fal_client.client request helpers are required for managed FAL gateway mode")
|
||||
self._request_handle_class = _require(
|
||||
getattr(client_module, "SyncRequestHandle", None), "fal_client.client.SyncRequestHandle")
|
||||
self._add_hint_header = getattr(client_module, "add_hint_header", None)
|
||||
self._add_priority_header = getattr(client_module, "add_priority_header", None)
|
||||
self._add_timeout_header = getattr(client_module, "add_timeout_header", None)
|
||||
|
||||
_require(self._http_client, "fal_client.SyncClient._client")
|
||||
if self._maybe_retry_request is None or self._raise_for_status is None:
|
||||
raise RuntimeError("fal_client.client request helpers are required for managed FAL gateway mode")
|
||||
_require(self._request_handle_class, "fal_client.client.SyncRequestHandle")
|
||||
|
||||
def submit(
|
||||
self, application: str, arguments: Dict[str, Any], *, path: str = "",
|
||||
hint: Optional[str] = None, webhook_url: Optional[str] = None, priority: Any = None,
|
||||
@@ -89,7 +82,6 @@ class _ManagedFalSyncClient:
|
||||
url += "/" + path.lstrip("/")
|
||||
if webhook_url is not None:
|
||||
url += "?" + urlencode({"fal_webhook": webhook_url})
|
||||
|
||||
request_headers = dict(headers or {})
|
||||
if hint is not None and self._add_hint_header is not None:
|
||||
self._add_hint_header(hint, request_headers)
|
||||
@@ -101,15 +93,11 @@ class _ManagedFalSyncClient:
|
||||
if self._add_timeout_header is None:
|
||||
raise RuntimeError("fal_client.client.add_timeout_header is required for timeout requests")
|
||||
self._add_timeout_header(start_timeout, request_headers)
|
||||
|
||||
response = self._maybe_retry_request(
|
||||
self._http_client, "POST", url, json=arguments,
|
||||
timeout=getattr(self._sync_client, "default_timeout", 120.0), headers=request_headers,
|
||||
)
|
||||
timeout=getattr(self._sync_client, "default_timeout", 120.0), headers=request_headers)
|
||||
self._raise_for_status(response)
|
||||
|
||||
data = response.json()
|
||||
return self._request_handle_class(
|
||||
request_id=data["request_id"], response_url=data["response_url"],
|
||||
status_url=data["status_url"], cancel_url=data["cancel_url"], client=self._http_client,
|
||||
)
|
||||
status_url=data["status_url"], cancel_url=data["cancel_url"], client=self._http_client)
|
||||
|
||||
+117
-222
@@ -1,9 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Image generation via FAL.ai (model picked in ``hermes tools``, persisted to ``image_gen.model``).
|
||||
|
||||
``FAL_MODELS`` (``tools.image_generation_catalog``) holds per-model metadata;
|
||||
``_build_fal_payload()`` / ``_build_fal_edit_payload()`` translate unified inputs into
|
||||
the model payload filtered to its ``supports`` whitelist so models never receive rejected
|
||||
``_build_fal_payload()`` / ``_build_fal_edit_payload()`` translate unified inputs into the
|
||||
``FAL_MODELS`` payload filtered to its ``supports`` whitelist so models never receive rejected
|
||||
keys. Clarity upscaling is strictly per-call opt-in: default-on degraded text/CJK/faces.
|
||||
"""
|
||||
|
||||
@@ -15,8 +14,8 @@ import threading
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
# Imported lazily by _load_fal_client(): the eager import cost ~64 ms on every CLI cold
|
||||
# start. Tests that monkeypatch this attribute keep working (loader short-circuits when truthy).
|
||||
# Imported lazily by _load_fal_client() (~64 ms on every CLI cold start); a test-monkeypatched
|
||||
# value short-circuits the loader.
|
||||
fal_client: Any = None
|
||||
|
||||
|
||||
@@ -30,11 +29,8 @@ def _load_fal_client() -> Any:
|
||||
|
||||
|
||||
from tools.debug_helpers import DebugSession
|
||||
from tools.fal_common import (
|
||||
_ManagedFalSyncClient,
|
||||
_extract_http_status,
|
||||
_normalize_fal_queue_url_format, # noqa: F401 — re-exported for tests
|
||||
)
|
||||
from tools.fal_common import ( # noqa: F401 — _normalize_fal_queue_url_format re-exported for tests
|
||||
_ManagedFalSyncClient, _extract_http_status, _normalize_fal_queue_url_format)
|
||||
from tools.image_generation_catalog import ( # noqa: F401 — re-exported (plugins/tests/tools_config)
|
||||
DEFAULT_ASPECT_RATIO, DEFAULT_MODEL, FAL_MODELS, UPSCALER_CREATIVITY, UPSCALER_DEFAULT_PROMPT,
|
||||
UPSCALER_FACTOR, UPSCALER_GUIDANCE_SCALE, UPSCALER_MODEL, UPSCALER_NEGATIVE_PROMPT,
|
||||
@@ -43,8 +39,7 @@ from tools.image_generation_catalog import ( # noqa: F401 — re-exported (plug
|
||||
from tools.managed_tool_gateway import resolve_managed_tool_gateway
|
||||
from tools.tool_backend_helpers import (
|
||||
NOUS_MANAGED_PROVIDER, fal_key_is_configured, managed_nous_tools_enabled,
|
||||
nous_tool_gateway_unavailable_message, read_selection, selection_error,
|
||||
)
|
||||
nous_tool_gateway_unavailable_message, read_selection, selection_error)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -58,10 +53,9 @@ _managed_fal_client_lock = threading.Lock()
|
||||
def _resolve_managed_fal_gateway():
|
||||
"""Managed gateway config for the stored `hermes tools` selection, or ``None`` for direct FAL.
|
||||
|
||||
``"nous"`` (or legacy ``use_gateway: true``) → managed ONLY: not entitled/unreachable is a
|
||||
selection-naming error, never a silent FAL_KEY fallback. Any other stored provider → direct
|
||||
ONLY: missing FAL_KEY is an error naming FAL_KEY and the selection, never a silent managed
|
||||
reroute. Never configured → legacy autodetect: direct if FAL_KEY, else managed if resolvable.
|
||||
``"nous"`` (or legacy ``use_gateway: true``) → managed ONLY (unreachable = selection-naming
|
||||
error, never a silent FAL_KEY fallback). Other stored provider → direct ONLY (missing FAL_KEY
|
||||
= error naming the selection). Never configured → autodetect: direct if FAL_KEY, else managed.
|
||||
"""
|
||||
selected = read_selection("image_gen")
|
||||
if selected == NOUS_MANAGED_PROVIDER:
|
||||
@@ -69,13 +63,12 @@ def _resolve_managed_fal_gateway():
|
||||
if gateway is None:
|
||||
raise ValueError(selection_error(
|
||||
"image_gen", NOUS_MANAGED_PROVIDER,
|
||||
"the Nous Tool Gateway is not available (not entitled or unreachable)",
|
||||
))
|
||||
"the Nous Tool Gateway is not available (not entitled or unreachable)"))
|
||||
return gateway
|
||||
if selected is not None:
|
||||
if not fal_key_is_configured():
|
||||
raise ValueError(selection_error("image_gen", selected, "FAL_KEY is not set"))
|
||||
return None
|
||||
if fal_key_is_configured():
|
||||
return None
|
||||
raise ValueError(selection_error("image_gen", selected, "FAL_KEY is not set"))
|
||||
# Never-configured category: legacy credential autodetect (do NOT persist).
|
||||
return None if fal_key_is_configured() else resolve_managed_tool_gateway("fal-queue")
|
||||
|
||||
@@ -83,15 +76,13 @@ def _resolve_managed_fal_gateway():
|
||||
def _get_managed_fal_client(managed_gateway):
|
||||
"""Reuse the managed FAL client so its internal httpx.Client is not leaked per call."""
|
||||
global _managed_fal_client, _managed_fal_client_config
|
||||
|
||||
client_config = (managed_gateway.gateway_origin.rstrip("/"), managed_gateway.nous_user_token)
|
||||
with _managed_fal_client_lock:
|
||||
if _managed_fal_client is None or _managed_fal_client_config != client_config:
|
||||
# Resolved on this module so monkeypatching ``image_generation_tool.fal_client`` still applies.
|
||||
_managed_fal_client = _ManagedFalSyncClient(
|
||||
_load_fal_client(), key=managed_gateway.nous_user_token,
|
||||
queue_run_origin=managed_gateway.gateway_origin,
|
||||
)
|
||||
queue_run_origin=managed_gateway.gateway_origin)
|
||||
_managed_fal_client_config = client_config
|
||||
return _managed_fal_client
|
||||
|
||||
@@ -103,28 +94,23 @@ class ImageGenerationInterrupted(Exception):
|
||||
def _wait_fal_result(handler, *, poll_seconds: float = 0.5):
|
||||
"""Interrupt-aware ``handler.get()``: the SDK blocks 30-60s, hiding user interrupts.
|
||||
|
||||
Runs the get on a daemon worker and polls the per-thread interrupt bit between join
|
||||
slices; on interrupt the worker is abandoned (remote job keeps running) and
|
||||
``ImageGenerationInterrupted`` is raised.
|
||||
The get runs on a daemon worker; the interrupt bit is polled between join slices and on
|
||||
interrupt the worker is abandoned (remote job keeps running).
|
||||
"""
|
||||
from tools.interrupt import is_interrupted
|
||||
|
||||
result_box: list = []
|
||||
error_box: list = []
|
||||
|
||||
def _get():
|
||||
try:
|
||||
result_box.append(handler.get())
|
||||
except BaseException as exc: # noqa: BLE001 — re-raised on the caller thread
|
||||
error_box.append(exc)
|
||||
|
||||
worker = threading.Thread(target=_get, daemon=True, name="fal-result-wait")
|
||||
worker.start()
|
||||
while worker.is_alive():
|
||||
if is_interrupted():
|
||||
raise ImageGenerationInterrupted(
|
||||
"Image generation interrupted by user — abandoned the in-flight FAL job."
|
||||
)
|
||||
"Image generation interrupted by user — abandoned the in-flight FAL job.")
|
||||
worker.join(timeout=poll_seconds)
|
||||
if error_box:
|
||||
raise error_box[0]
|
||||
@@ -138,7 +124,6 @@ def _submit_fal_request(model: str, arguments: Dict[str, Any]):
|
||||
managed_gateway = _resolve_managed_fal_gateway()
|
||||
if managed_gateway is None:
|
||||
return fal_client.submit(model, arguments=arguments, headers=request_headers)
|
||||
|
||||
try:
|
||||
return _get_managed_fal_client(managed_gateway).submit(
|
||||
model, arguments=arguments, headers=request_headers)
|
||||
@@ -150,16 +135,13 @@ def _submit_fal_request(model: str, arguments: Dict[str, Any]):
|
||||
gateway_message = ""
|
||||
if status in {401, 402, 403}:
|
||||
gateway_message = "\n\n" + nous_tool_gateway_unavailable_message(
|
||||
"managed FAL image generation", force_fresh=True,
|
||||
)
|
||||
"managed FAL image generation", force_fresh=True)
|
||||
raise ValueError(
|
||||
f"Nous Subscription gateway rejected model '{model}' "
|
||||
f"(HTTP {status}). This model may not yet be enabled on "
|
||||
f"the Nous Portal's FAL proxy. Either:\n"
|
||||
f"Nous Subscription gateway rejected model '{model}' (HTTP {status}). This model "
|
||||
f"may not yet be enabled on the Nous Portal's FAL proxy. Either:\n"
|
||||
f" • Set FAL_KEY in your environment to use FAL.ai directly, or\n"
|
||||
f" • Pick a different model via `hermes tools` → Image Generation."
|
||||
f"{gateway_message}"
|
||||
) from exc
|
||||
f"{gateway_message}") from exc
|
||||
raise
|
||||
|
||||
|
||||
@@ -184,12 +166,8 @@ def _read_configured_image_model():
|
||||
|
||||
|
||||
def _read_configured_image_provider():
|
||||
"""``image_gen.provider`` from config.yaml, or None.
|
||||
|
||||
The plugin registry is consulted only when this is explicitly set — unset keeps users on
|
||||
the in-tree FAL fallback even when other providers are registered. ``"fal"`` routes through
|
||||
``plugins/image_gen/fal/``, which delegates back here via call-time indirection.
|
||||
"""
|
||||
"""``image_gen.provider`` from config.yaml, or None (unset keeps the in-tree FAL fallback even
|
||||
when other providers are registered; ``"fal"`` routes via ``plugins/image_gen/fal/``)."""
|
||||
return _read_image_gen_key("provider")
|
||||
|
||||
|
||||
@@ -212,16 +190,17 @@ def _resolve_fal_model() -> tuple:
|
||||
return model_id, FAL_MODELS[model_id]
|
||||
|
||||
|
||||
_SIZE_KEY_BY_STYLE = {"image_size_preset": "image_size", "gpt_literal": "image_size", "aspect_ratio": "aspect_ratio"}
|
||||
_SIZE_KEY_BY_STYLE = {"image_size_preset": "image_size", "gpt_literal": "image_size",
|
||||
"aspect_ratio": "aspect_ratio"}
|
||||
|
||||
|
||||
def _build_payload(model_id, prompt, aspect_ratio, seed, overrides, image_urls=None) -> Dict[str, Any]:
|
||||
"""Text-to-image / edit payload (``image_urls`` selects edit mode): defaults + native size
|
||||
spec + overrides, filtered to the model whitelist.
|
||||
|
||||
Edit endpoints mostly auto-infer size from the input, so the size key is sent only when
|
||||
``edit_supports`` advertises it. ``prompt`` (and ``image_urls`` on edits) are required by
|
||||
every FAL endpoint and survive a whitelist gap so a catalog mistake can't send a broken request.
|
||||
Edit endpoints mostly auto-infer size, so the size key is sent only when ``edit_supports``
|
||||
lists it. ``prompt`` (and ``image_urls`` on edits) survive a whitelist gap: every FAL
|
||||
endpoint requires them, so a catalog mistake can't send a broken request.
|
||||
"""
|
||||
meta = FAL_MODELS[model_id]
|
||||
edit = image_urls is not None
|
||||
@@ -230,14 +209,12 @@ def _build_payload(model_id, prompt, aspect_ratio, seed, overrides, image_urls=N
|
||||
aspect = (aspect_ratio or DEFAULT_ASPECT_RATIO).lower().strip()
|
||||
if aspect not in sizes:
|
||||
aspect = DEFAULT_ASPECT_RATIO
|
||||
|
||||
payload: Dict[str, Any] = dict(meta.get("defaults", {}))
|
||||
payload["prompt"] = (prompt or "").strip()
|
||||
required = {"prompt"}
|
||||
if edit:
|
||||
payload["image_urls"] = list(image_urls)
|
||||
required.add("image_urls")
|
||||
|
||||
size_key = _SIZE_KEY_BY_STYLE.get(meta["size_style"])
|
||||
if size_key is None and not edit:
|
||||
raise ValueError(f"Unknown size_style: {meta['size_style']!r}")
|
||||
@@ -254,7 +231,8 @@ def _build_fal_payload(model_id, prompt, aspect_ratio=DEFAULT_ASPECT_RATIO, seed
|
||||
return _build_payload(model_id, prompt, aspect_ratio, seed, overrides)
|
||||
|
||||
|
||||
def _build_fal_edit_payload(model_id, prompt, image_urls, aspect_ratio=DEFAULT_ASPECT_RATIO, seed=None, overrides=None):
|
||||
def _build_fal_edit_payload(model_id, prompt, image_urls, aspect_ratio=DEFAULT_ASPECT_RATIO,
|
||||
seed=None, overrides=None):
|
||||
"""FAL *edit* (image-to-image) payload: ``image_urls`` + prompt, filtered to ``edit_supports``."""
|
||||
return _build_payload(model_id, prompt, aspect_ratio, seed, overrides, image_urls=image_urls)
|
||||
|
||||
@@ -265,16 +243,12 @@ def _upscale_image(image_url: str, original_prompt: str) -> Optional[Dict[str, A
|
||||
try:
|
||||
logger.info("Upscaling image with Clarity Upscaler...")
|
||||
handler = _submit_fal_request(UPSCALER_MODEL, arguments={
|
||||
"image_url": image_url,
|
||||
"prompt": f"{UPSCALER_DEFAULT_PROMPT}, {original_prompt}",
|
||||
"upscale_factor": UPSCALER_FACTOR,
|
||||
"negative_prompt": UPSCALER_NEGATIVE_PROMPT,
|
||||
"creativity": UPSCALER_CREATIVITY,
|
||||
"resemblance": UPSCALER_RESEMBLANCE,
|
||||
"image_url": image_url, "prompt": f"{UPSCALER_DEFAULT_PROMPT}, {original_prompt}",
|
||||
"upscale_factor": UPSCALER_FACTOR, "negative_prompt": UPSCALER_NEGATIVE_PROMPT,
|
||||
"creativity": UPSCALER_CREATIVITY, "resemblance": UPSCALER_RESEMBLANCE,
|
||||
"guidance_scale": UPSCALER_GUIDANCE_SCALE,
|
||||
"num_inference_steps": UPSCALER_NUM_INFERENCE_STEPS,
|
||||
"enable_safety_checker": UPSCALER_SAFETY_CHECKER,
|
||||
})
|
||||
"enable_safety_checker": UPSCALER_SAFETY_CHECKER})
|
||||
result = _wait_fal_result(handler)
|
||||
if result and "image" in result:
|
||||
up = result["image"]
|
||||
@@ -282,8 +256,7 @@ def _upscale_image(image_url: str, original_prompt: str) -> Optional[Dict[str, A
|
||||
up.get("width", "unknown"), up.get("height", "unknown"))
|
||||
return {
|
||||
"url": up["url"], "width": up.get("width", 0), "height": up.get("height", 0),
|
||||
"upscaled": True, "upscale_factor": UPSCALER_FACTOR,
|
||||
}
|
||||
"upscaled": True, "upscale_factor": UPSCALER_FACTOR}
|
||||
logger.error("Upscaler returned invalid response")
|
||||
return None
|
||||
except ImageGenerationInterrupted:
|
||||
@@ -296,8 +269,8 @@ def _upscale_image(image_url: str, original_prompt: str) -> Optional[Dict[str, A
|
||||
|
||||
# --- Artifact path hinting for non-local terminal backends ---
|
||||
_CONTAINER_HOME_ENVS = {"DockerEnvironment", "SingularityEnvironment", "ModalEnvironment"}
|
||||
# No environment yet: only backends with deterministic cache roots can be translated without
|
||||
# side effects. SSH uses a shell-visible tilde path; its first sync uploads the cache file.
|
||||
# No env yet: only deterministic cache roots translate side-effect free (SSH: tilde path; its
|
||||
# first sync uploads the cache file).
|
||||
_CACHE_BASE_BY_BACKEND = {"docker": "/root/.hermes", "singularity": "/root/.hermes",
|
||||
"modal": "/root/.hermes", "ssh": "~/.hermes"}
|
||||
|
||||
@@ -349,7 +322,6 @@ def _postprocess_image_generate_result(raw: str, task_id: str | None = None) ->
|
||||
image = payload.get("image")
|
||||
if not isinstance(image, str) or not _looks_like_absolute_file_path(image):
|
||||
return raw
|
||||
|
||||
env = _active_terminal_env(task_id)
|
||||
cache_base = _agent_cache_base_for_env(env)
|
||||
if not cache_base:
|
||||
@@ -388,8 +360,7 @@ def _format_images(images: list, should_upscale: bool, prompt: str) -> list:
|
||||
logger.warning("Using original image as fallback (upscale failed)")
|
||||
formatted.append({
|
||||
"url": img["url"], "width": img.get("width", 0), "height": img.get("height", 0),
|
||||
"upscaled": False,
|
||||
})
|
||||
"upscaled": False})
|
||||
return formatted
|
||||
|
||||
|
||||
@@ -397,28 +368,21 @@ def _prepare_fal_request(model_id, meta, prompt, aspect_ratio, seed, overrides,
|
||||
"""Validate inputs and return ``(endpoint, arguments)``; raises ValueError with the user-facing message."""
|
||||
if not isinstance(prompt, str) or not prompt.strip():
|
||||
raise ValueError("Prompt is required and must be a non-empty string")
|
||||
|
||||
# A stored-but-broken selection raises the selection-naming error from
|
||||
# _resolve_managed_fal_gateway(); only never-configured reports "no backend at all".
|
||||
if not (fal_key_is_configured() or _resolve_managed_fal_gateway()):
|
||||
raise ValueError(_build_no_backend_setup_message())
|
||||
|
||||
edit_endpoint = meta.get("edit_endpoint")
|
||||
display = meta.get("display", model_id)
|
||||
edit_endpoint, display = meta.get("edit_endpoint"), meta.get("display", model_id)
|
||||
# Fail clearly rather than silently dropping sources and producing an unrelated picture.
|
||||
if source_images and not edit_endpoint:
|
||||
raise ValueError(
|
||||
f"Model '{display}' ({model_id}) is not "
|
||||
f"capable of image-to-image / editing. Provide a text-only "
|
||||
f"prompt (omit image_url), or switch to an edit-capable model "
|
||||
f"via `hermes tools` → Image Generation."
|
||||
)
|
||||
|
||||
f"Model '{display}' ({model_id}) is not capable of image-to-image / editing. "
|
||||
f"Provide a text-only prompt (omit image_url), or switch to an edit-capable model "
|
||||
f"via `hermes tools` → Image Generation.")
|
||||
aspect_lc = (aspect_ratio or DEFAULT_ASPECT_RATIO).lower().strip()
|
||||
if aspect_lc not in VALID_ASPECT_RATIOS:
|
||||
logger.warning("Invalid aspect_ratio '%s', defaulting to '%s'", aspect_ratio, DEFAULT_ASPECT_RATIO)
|
||||
aspect_lc = DEFAULT_ASPECT_RATIO
|
||||
|
||||
if source_images:
|
||||
# Clamp reference count to the model's declared cap.
|
||||
max_refs = int(meta.get("max_reference_images") or 1)
|
||||
@@ -438,30 +402,26 @@ def image_generate_tool(
|
||||
num_inference_steps: Optional[int] = None, guidance_scale: Optional[float] = None,
|
||||
num_images: Optional[int] = None, output_format: Optional[str] = None,
|
||||
seed: Optional[int] = None, image_url: Optional[str] = None,
|
||||
reference_image_urls: Optional[list] = None, upscale: Optional[bool] = None,
|
||||
) -> str:
|
||||
reference_image_urls: Optional[list] = None, upscale: Optional[bool] = None) -> str:
|
||||
"""Generate (or, with source images + an ``edit_endpoint`` model, edit) an image via FAL.
|
||||
|
||||
Extra kwargs are overrides for direct Python callers, filtered per-model via the
|
||||
``supports`` / ``edit_supports`` whitelist (dropped silently so legacy callers survive
|
||||
model switches). Returns JSON ``{"success", "image", "modality", "error", "error_type"}``.
|
||||
Extra kwargs are overrides filtered per-model via ``supports`` / ``edit_supports`` (dropped
|
||||
silently so callers survive model switches). Returns JSON ``{"success", "image", "modality",
|
||||
"error", "error_type"}``.
|
||||
"""
|
||||
model_id, meta = _resolve_fal_model()
|
||||
candidates = [image_url, *(reference_image_urls if isinstance(reference_image_urls, (list, tuple)) else [])]
|
||||
source_images = [c.strip() for c in candidates if isinstance(c, str) and c.strip()]
|
||||
refs = reference_image_urls if isinstance(reference_image_urls, (list, tuple)) else []
|
||||
source_images = [c.strip() for c in (image_url, *refs) if isinstance(c, str) and c.strip()]
|
||||
use_edit = bool(source_images) and bool(meta.get("edit_endpoint"))
|
||||
modality = "image" if use_edit else "text"
|
||||
|
||||
overrides: Dict[str, Any] = {
|
||||
"num_inference_steps": num_inference_steps, "guidance_scale": guidance_scale,
|
||||
"num_images": num_images, "output_format": output_format,
|
||||
}
|
||||
"num_images": num_images, "output_format": output_format}
|
||||
debug_call_data = {
|
||||
"model": model_id,
|
||||
"parameters": {"prompt": prompt, "aspect_ratio": aspect_ratio, **overrides, "seed": seed,
|
||||
"modality": modality, "source_images": len(source_images)},
|
||||
"error": None, "success": False, "images_generated": 0, "generation_time": 0,
|
||||
}
|
||||
"error": None, "success": False, "images_generated": 0, "generation_time": 0}
|
||||
start_time = datetime.datetime.now()
|
||||
|
||||
def finish(generation_time: float, response: Dict[str, Any]) -> str:
|
||||
@@ -469,32 +429,26 @@ def image_generate_tool(
|
||||
_debug.log_call("image_generate_tool", debug_call_data)
|
||||
_debug.save()
|
||||
return json.dumps(response, indent=2, ensure_ascii=False)
|
||||
|
||||
try:
|
||||
endpoint, arguments = _prepare_fal_request(
|
||||
model_id, meta, prompt, aspect_ratio, seed,
|
||||
{k: v for k, v in overrides.items() if v is not None}, source_images)
|
||||
result = _wait_fal_result(_submit_fal_request(endpoint, arguments=arguments))
|
||||
generation_time = (datetime.datetime.now() - start_time).total_seconds()
|
||||
|
||||
if not result or "images" not in result:
|
||||
raise ValueError("Invalid response from FAL.ai API — no images returned")
|
||||
images = result.get("images", [])
|
||||
if not images:
|
||||
raise ValueError("No images were generated")
|
||||
|
||||
# An explicit ``upscale`` wins over the catalog default, including for edits. The
|
||||
# catalog default never upscales edits: Clarity is a text-to-image quality pass and
|
||||
# must not silently alter edit compositions.
|
||||
# Explicit ``upscale`` wins, including for edits; the catalog default never upscales
|
||||
# edits (Clarity is a text-to-image pass and must not silently alter compositions).
|
||||
if upscale is not None:
|
||||
should_upscale = bool(upscale)
|
||||
else:
|
||||
should_upscale = bool(meta.get("upscale", False)) and not use_edit
|
||||
|
||||
formatted_images = _format_images(images, should_upscale, prompt)
|
||||
if not formatted_images:
|
||||
raise ValueError("No valid image URLs returned from API")
|
||||
|
||||
upscaled_count = sum(1 for img in formatted_images if img.get("upscaled"))
|
||||
logger.info("Generated %s image(s) in %.1fs (%s upscaled) via %s [%s]",
|
||||
len(formatted_images), generation_time, upscaled_count, endpoint, modality)
|
||||
@@ -504,14 +458,14 @@ def image_generate_tool(
|
||||
"success": True,
|
||||
"image": formatted_images[0]["url"],
|
||||
"modality": modality,
|
||||
"upscaled": bool(formatted_images[0].get("upscaled")),
|
||||
})
|
||||
"upscaled": bool(formatted_images[0].get("upscaled"))})
|
||||
except Exception as e:
|
||||
error_msg = f"Error generating image: {str(e)}"
|
||||
logger.error("%s", error_msg, exc_info=True)
|
||||
debug_call_data["error"] = error_msg
|
||||
generation_time = (datetime.datetime.now() - start_time).total_seconds()
|
||||
return finish(generation_time, {"success": False, "image": None, "error": str(e), "error_type": type(e).__name__})
|
||||
return finish(generation_time,
|
||||
{"success": False, "image": None, "error": str(e), "error_type": type(e).__name__})
|
||||
|
||||
|
||||
def check_fal_api_key() -> bool:
|
||||
@@ -520,12 +474,11 @@ def check_fal_api_key() -> bool:
|
||||
A stored-but-broken selection reports False here (registry gating); the naming error
|
||||
surfaces at call time from ``_resolve_managed_fal_gateway``.
|
||||
"""
|
||||
selected = read_selection("image_gen")
|
||||
if selected == NOUS_MANAGED_PROVIDER:
|
||||
return bool(resolve_managed_tool_gateway("fal-queue"))
|
||||
if selected is not None:
|
||||
return fal_key_is_configured()
|
||||
return bool(fal_key_is_configured() or resolve_managed_tool_gateway("fal-queue"))
|
||||
try:
|
||||
gateway = _resolve_managed_fal_gateway()
|
||||
except ValueError:
|
||||
return False
|
||||
return bool(gateway) or fal_key_is_configured()
|
||||
|
||||
|
||||
def _build_no_backend_setup_message() -> str:
|
||||
@@ -536,25 +489,16 @@ def _build_no_backend_setup_message() -> str:
|
||||
lines.append(" - FAL_KEY is not set and the managed FAL gateway is unreachable")
|
||||
else:
|
||||
lines.append(" - FAL_KEY environment variable is not set")
|
||||
gateway_message = nous_tool_gateway_unavailable_message("managed FAL image generation")
|
||||
if gateway_message:
|
||||
if gateway_message := nous_tool_gateway_unavailable_message("managed FAL image generation"):
|
||||
lines.append(f" - {gateway_message}")
|
||||
lines += [
|
||||
"",
|
||||
"To enable image generation, do one of:",
|
||||
" 1. Get a free API key at https://fal.ai and set "
|
||||
"FAL_KEY=<your-key> (then restart the session)",
|
||||
]
|
||||
lines += ["", "To enable image generation, do one of:",
|
||||
" 1. Get a free API key at https://fal.ai and set FAL_KEY=<your-key> "
|
||||
"(then restart the session)"]
|
||||
if managed:
|
||||
lines.append(
|
||||
" 2. Sign in to a Nous account that has the managed FAL "
|
||||
"gateway enabled (`hermes setup`)"
|
||||
)
|
||||
lines.append(
|
||||
" 3. Configure a different image_gen provider via `hermes tools` "
|
||||
"→ Image Generation (run `hermes plugins list` to see installed "
|
||||
"backends)"
|
||||
)
|
||||
lines.append(" 2. Sign in to a Nous account that has the managed FAL gateway enabled "
|
||||
"(`hermes setup`)")
|
||||
lines.append(" 3. Configure a different image_gen provider via `hermes tools` → Image Generation "
|
||||
"(run `hermes plugins list` to see installed backends)")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@@ -562,7 +506,6 @@ def _get_plugin_provider(name: str, *, force: bool = False):
|
||||
"""Discover plugins (local import: importing this module must not trigger discovery) and return the named provider."""
|
||||
from agent.image_gen_registry import get_provider
|
||||
from hermes_cli.plugins import _ensure_plugins_discovered
|
||||
|
||||
if force:
|
||||
_ensure_plugins_discovered(force=True)
|
||||
else:
|
||||
@@ -579,7 +522,6 @@ def check_image_generation_requirements() -> bool:
|
||||
return True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
configured = _plugin_provider_name()
|
||||
if configured is None:
|
||||
return False
|
||||
@@ -659,14 +601,9 @@ def _add_provider_kwargs(kwargs, image_url, reference_image_urls, upscale, model
|
||||
|
||||
def _dispatch_to_plugin_provider(
|
||||
prompt: str, aspect_ratio: str, image_url: Optional[str] = None,
|
||||
reference_image_urls: Optional[list] = None, upscale: Optional[bool] = None,
|
||||
):
|
||||
"""JSON result from the selected plugin provider, or ``None`` to fall through to in-tree FAL.
|
||||
|
||||
Fires when ``image_gen.provider`` is anything but unset / ``"fal"`` / ``"nous"`` (those run
|
||||
the legacy pipeline; ``"nous"`` via the managed fal-queue gateway). Edit args are
|
||||
forwarded for the backend's edit endpoint; providers without ``upscale`` ignore it via ``**kwargs``.
|
||||
"""
|
||||
reference_image_urls: Optional[list] = None, upscale: Optional[bool] = None):
|
||||
"""JSON result from the selected plugin provider, or ``None`` to fall through to in-tree FAL
|
||||
(provider unset / ``"fal"`` / ``"nous"``). Providers without ``upscale`` ignore it via ``**kwargs``."""
|
||||
configured = _plugin_provider_name()
|
||||
if configured is None:
|
||||
return None
|
||||
@@ -684,35 +621,28 @@ def _dispatch_to_plugin_provider(
|
||||
logger.debug("image_gen plugin force-refresh skipped: %s", exc)
|
||||
if provider is None:
|
||||
return _provider_error(
|
||||
f"image_gen.provider='{configured}' is set but no plugin "
|
||||
f"registered that name. Run `hermes plugins list` to see "
|
||||
f"available image gen backends.",
|
||||
"provider_not_registered",
|
||||
)
|
||||
|
||||
f"image_gen.provider='{configured}' is set but no plugin registered that name. "
|
||||
f"Run `hermes plugins list` to see available image gen backends.", "provider_not_registered")
|
||||
pname = getattr(provider, "name", "?")
|
||||
kwargs: Dict[str, Any] = {"prompt": prompt, "aspect_ratio": aspect_ratio}
|
||||
try:
|
||||
_add_provider_kwargs(kwargs, image_url, reference_image_urls, upscale, model=_read_configured_image_model())
|
||||
_add_provider_kwargs(kwargs, image_url, reference_image_urls, upscale,
|
||||
model=_read_configured_image_model())
|
||||
result = provider.generate(**kwargs)
|
||||
except TypeError as exc:
|
||||
# generate() predating image_url support (third-party plugin not yet updated):
|
||||
# text-to-image keeps working; surface a clear note when an edit was requested.
|
||||
if "image_url" in kwargs or "reference_image_urls" in kwargs:
|
||||
except Exception as exc:
|
||||
# A TypeError from generate() predating image_url support (third-party plugin not yet
|
||||
# updated): text-to-image keeps working; surface a clear note when an edit was requested.
|
||||
is_type_error = isinstance(exc, TypeError)
|
||||
if is_type_error and ("image_url" in kwargs or "reference_image_urls" in kwargs):
|
||||
logger.warning("image_gen provider '%s' rejected image-to-image kwargs "
|
||||
"(signature too narrow): %s", pname, exc)
|
||||
return _provider_error(
|
||||
f"Provider '{pname}' does not "
|
||||
f"support image-to-image / editing (its generate() "
|
||||
f"signature is out of date with the image_generate schema). "
|
||||
f"Omit image_url for text-to-image, or pick a backend that "
|
||||
f"supports editing via `hermes tools` → Image Generation.",
|
||||
"modality_unsupported",
|
||||
)
|
||||
logger.warning("Image gen provider '%s' raised TypeError: %s", pname, exc)
|
||||
return _provider_error(f"Provider '{pname}' error: {exc}", "provider_exception")
|
||||
except Exception as exc:
|
||||
logger.warning("Image gen provider '%s' raised: %s", pname, exc)
|
||||
f"Provider '{pname}' does not support image-to-image / editing (its generate() "
|
||||
f"signature is out of date with the image_generate schema). Omit image_url for "
|
||||
f"text-to-image, or pick a backend that supports editing via `hermes tools` → "
|
||||
f"Image Generation.", "modality_unsupported")
|
||||
logger.warning("Image gen provider '%s' raised%s: %s", pname,
|
||||
" TypeError" if is_type_error else "", exc)
|
||||
return _provider_error(f"Provider '{pname}' error: {exc}", "provider_exception")
|
||||
return _provider_result(result, "Provider returned a non-dict result")
|
||||
|
||||
@@ -730,13 +660,11 @@ def _normalize_krea_model(model_id: Optional[str]) -> Optional[str]:
|
||||
|
||||
def _maybe_route_managed_krea(
|
||||
prompt: str, aspect_ratio: str, image_url: Optional[str] = None,
|
||||
reference_image_urls: Optional[list] = None, upscale: Optional[bool] = None,
|
||||
) -> Optional[str]:
|
||||
reference_image_urls: Optional[list] = None, upscale: Optional[bool] = None) -> Optional[str]:
|
||||
"""JSON result from the managed Krea gateway, or ``None`` to fall through.
|
||||
|
||||
Fires only when the configured model is a native ``krea-2-*`` id AND no
|
||||
``image_gen.provider`` other than ``"nous"`` is stored (a picker choice dispatches
|
||||
normally) AND the managed Krea gateway is resolvable.
|
||||
Fires only for a native ``krea-2-*`` model with no ``image_gen.provider`` other than
|
||||
``"nous"`` stored (a picker choice dispatches normally) and a resolvable Krea gateway.
|
||||
"""
|
||||
configured_provider = _read_configured_image_provider()
|
||||
if configured_provider is not None and configured_provider != NOUS_MANAGED_PROVIDER:
|
||||
@@ -748,17 +676,12 @@ def _maybe_route_managed_krea(
|
||||
from plugins.image_gen.krea import _resolve_managed_krea_gateway
|
||||
if _resolve_managed_krea_gateway() is None:
|
||||
return None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("Managed Krea routing probe failed: %s", exc)
|
||||
return None
|
||||
try:
|
||||
provider = _get_plugin_provider("krea")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("Managed Krea routing: provider unavailable: %s", exc)
|
||||
logger.debug("Managed Krea routing unavailable: %s", exc)
|
||||
return None
|
||||
if provider is None:
|
||||
return None
|
||||
|
||||
kwargs: Dict[str, Any] = {"prompt": prompt, "aspect_ratio": aspect_ratio, "model": normalized}
|
||||
try:
|
||||
_add_provider_kwargs(kwargs, image_url, reference_image_urls, upscale)
|
||||
@@ -772,21 +695,17 @@ def _maybe_route_managed_krea(
|
||||
def _confine_source_images(image_url, reference_image_urls, task_id, *, permitted: tuple = ("image",)):
|
||||
"""Resolve path-like sources to ``data:`` URLs under a non-local terminal backend.
|
||||
|
||||
Goes through ``tools.image_source`` (in-sandbox exec-read, media-cache host reads,
|
||||
credential guard) before any provider sees them, so generation obeys the same
|
||||
confinement boundary as vision/video analysis and sandbox-only files work as edit
|
||||
sources. URLs/data: pass through; the local backend is a no-op (providers keep host reads).
|
||||
Returns ``(image_url, reference_image_urls, error_json_or_None)``.
|
||||
Routes through ``tools.image_source`` (in-sandbox exec-read, media-cache host reads,
|
||||
credential guard) so generation obeys the same confinement as vision. URLs/data: pass
|
||||
through; local backend is a no-op. Returns ``(image_url, reference_image_urls, error_json_or_None)``.
|
||||
"""
|
||||
if (os.getenv("TERMINAL_ENV") or "local").strip().lower() in ("", "local"):
|
||||
return image_url, reference_image_urls, None
|
||||
|
||||
from model_tools import _run_async
|
||||
from tools.image_source import ImageResolutionError, resolve_local_source_to_data_url
|
||||
|
||||
def resolve(ref):
|
||||
return _run_async(resolve_local_source_to_data_url(ref, task_id, permitted=permitted))
|
||||
|
||||
try:
|
||||
if isinstance(image_url, str) and image_url.strip():
|
||||
image_url = resolve(image_url)
|
||||
@@ -805,17 +724,13 @@ def _handle_image_generate(args, **kw):
|
||||
aspect_ratio = args.get("aspect_ratio", DEFAULT_ASPECT_RATIO)
|
||||
upscale = args.get("upscale")
|
||||
task_id = kw.get("task_id")
|
||||
|
||||
# Confinement chokepoint BEFORE any dispatch: plugin, managed Krea and in-tree FAL
|
||||
# all receive sandbox-confined bytes.
|
||||
# Confinement chokepoint BEFORE any dispatch: every route receives sandbox-confined bytes.
|
||||
image_url, reference_image_urls, confine_error = _confine_source_images(
|
||||
args.get("image_url"), args.get("reference_image_urls"), task_id)
|
||||
if confine_error is not None:
|
||||
return confine_error
|
||||
|
||||
# Order matters: explicit plugin provider (incl. provider == "krea"), then
|
||||
# model-driven managed Krea interception (only when no provider is set, so
|
||||
# the BYO/direct FAL path stays untouched), then the in-tree FAL pipeline.
|
||||
# Order matters: explicit plugin provider (incl. "krea"), then model-driven managed Krea
|
||||
# interception (only when no provider is set, so BYO/direct FAL stays untouched), then FAL.
|
||||
sources = dict(image_url=image_url, reference_image_urls=reference_image_urls,
|
||||
upscale=upscale if isinstance(upscale, bool) else None)
|
||||
raw = None
|
||||
@@ -827,20 +742,18 @@ def _handle_image_generate(args, **kw):
|
||||
|
||||
|
||||
# --- Dynamic schema — reflect the active backend's image-to-image capability ---
|
||||
# Telling the model up front whether it can edit saves a wasted turn. Memoized by
|
||||
# config.yaml mtime in model_tools.get_tool_definitions(), so it rebuilds on switch.
|
||||
# Advertising edit capability up front saves a wasted turn. Memoized by config.yaml mtime in
|
||||
# model_tools.get_tool_definitions(), so it rebuilds on switch.
|
||||
_NO_CAPABILITIES = {"modalities": ["text"], "max_reference_images": 0, "supports_upscale": False}
|
||||
|
||||
|
||||
def _active_image_capabilities() -> Dict[str, Any]:
|
||||
"""Best-effort capabilities of the active backend/model; never raises.
|
||||
|
||||
Mirrors runtime dispatch: a set ``image_gen.provider`` asks that plugin, otherwise the
|
||||
FAL catalog. Fail-closed: an undeclared capability is advertised as absent (an
|
||||
under-declaring provider is that provider's bug, not a safety problem).
|
||||
Mirrors runtime dispatch: a set ``image_gen.provider`` asks that plugin, else the FAL
|
||||
catalog. Fail-closed: an undeclared capability is advertised as absent.
|
||||
"""
|
||||
info: Dict[str, Any] = dict(_NO_CAPABILITIES)
|
||||
|
||||
configured_provider = _read_configured_image_provider()
|
||||
if configured_provider and configured_provider != "fal":
|
||||
try:
|
||||
@@ -861,20 +774,15 @@ def _active_image_capabilities() -> Dict[str, Any]:
|
||||
return info
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
# In-tree FAL path (provider unset or == "fal").
|
||||
try:
|
||||
model_id, meta = _resolve_fal_model()
|
||||
can_edit = bool(meta.get("edit_endpoint"))
|
||||
info["provider"] = "FAL.ai"
|
||||
info["model"] = meta.get("display", model_id)
|
||||
info["modalities"] = ["text", "image"] if can_edit else ["text"]
|
||||
info["max_reference_images"] = int(meta.get("max_reference_images") or 1) if can_edit else 0
|
||||
# Clarity is a separate endpoint available on request for ANY catalog model
|
||||
# (the per-model ``upscale`` key is only the default flag).
|
||||
info["supports_upscale"] = True
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
# In-tree FAL path (provider unset or == "fal"); _resolve_fal_model() never raises.
|
||||
model_id, meta = _resolve_fal_model()
|
||||
can_edit = bool(meta.get("edit_endpoint"))
|
||||
info["provider"] = "FAL.ai"
|
||||
info["model"] = meta.get("display", model_id)
|
||||
info["modalities"] = ["text", "image"] if can_edit else ["text"]
|
||||
info["max_reference_images"] = int(meta.get("max_reference_images") or 1) if can_edit else 0
|
||||
# Clarity is available on request for ANY catalog model (``upscale`` is only the default).
|
||||
info["supports_upscale"] = True
|
||||
return info
|
||||
|
||||
|
||||
@@ -908,17 +816,12 @@ def _build_dynamic_image_schema() -> Dict[str, Any]:
|
||||
"file path; reference it in your response using the current "
|
||||
"platform's file-delivery convention."
|
||||
)
|
||||
try:
|
||||
info = _active_image_capabilities()
|
||||
except Exception: # noqa: BLE001
|
||||
info = dict(_NO_CAPABILITIES)
|
||||
|
||||
info = _active_image_capabilities()
|
||||
max_refs = int(info.get("max_reference_images") or 0)
|
||||
can_edit = "image" in set(info.get("modalities") or ["text"])
|
||||
static_props = IMAGE_GENERATE_SCHEMA["parameters"]["properties"]
|
||||
properties: Dict[str, Any] = {
|
||||
"prompt": static_props["prompt"], "aspect_ratio": static_props["aspect_ratio"],
|
||||
}
|
||||
"prompt": static_props["prompt"], "aspect_ratio": static_props["aspect_ratio"]}
|
||||
if can_edit:
|
||||
edit_clause = ", or edit / transform an existing image by passing image_url"
|
||||
properties["image_url"] = _IMAGE_URL_PARAM
|
||||
@@ -937,21 +840,13 @@ def _build_dynamic_image_schema() -> Dict[str, Any]:
|
||||
edit_clause = " (text-to-image only — the active model cannot edit existing images)"
|
||||
if info.get("supports_upscale"):
|
||||
properties["upscale"] = _UPSCALE_PARAM
|
||||
|
||||
return {
|
||||
"description": base_desc.format(edit_clause=edit_clause),
|
||||
"parameters": {"type": "object", "properties": properties, "required": ["prompt"]},
|
||||
}
|
||||
return {"description": base_desc.format(edit_clause=edit_clause),
|
||||
"parameters": {"type": "object", "properties": properties, "required": ["prompt"]}}
|
||||
|
||||
|
||||
registry.register(
|
||||
name="image_generate",
|
||||
toolset="image_gen",
|
||||
schema=IMAGE_GENERATE_SCHEMA,
|
||||
handler=_handle_image_generate,
|
||||
check_fn=check_image_generation_requirements,
|
||||
requires_env=[],
|
||||
name="image_generate", toolset="image_gen", schema=IMAGE_GENERATE_SCHEMA,
|
||||
handler=_handle_image_generate, check_fn=check_image_generation_requirements, requires_env=[],
|
||||
is_async=False, # sync fal_client API to avoid "Event loop is closed" in gateway
|
||||
emoji="🎨",
|
||||
dynamic_schema_overrides=_build_dynamic_image_schema,
|
||||
emoji="🎨", dynamic_schema_overrides=_build_dynamic_image_schema,
|
||||
)
|
||||
|
||||
+42
-69
@@ -1,13 +1,10 @@
|
||||
"""Single resolver for every media source -> bytes + mime.
|
||||
"""Single resolver for every media source (data:/http(s)/file/local/container) -> bytes + mime.
|
||||
|
||||
All source handling (data:/http(s)/file/local/container) funnels through
|
||||
:func:`resolve_image_source` so size and magic-byte checks are enforced exactly once.
|
||||
Images are the default; callers whose argument takes video opt in via ``permitted=("video",)``.
|
||||
|
||||
Security (terminal-backend confinement, GHSA-gpxw-6wxv-w3qq): under a non-local backend
|
||||
vision is confined like the file tools: local -> read any host path; non-local -> host-read
|
||||
only inside a media cache (bind-mounted into the sandbox), anything else is exec-read
|
||||
*inside the sandbox*, so an injected ``vision_analyze('/etc/passwd')`` never reads the host.
|
||||
Everything funnels through :func:`resolve_image_source` so size and magic-byte checks run
|
||||
exactly once. Images are the default; video callers opt in via ``permitted=("video",)``.
|
||||
Security (GHSA-gpxw-6wxv-w3qq): under a non-local backend vision is confined like the file
|
||||
tools — host-read only inside a media cache (bind-mounted into the sandbox), anything else is
|
||||
exec-read *inside the sandbox*, so ``vision_analyze('/etc/passwd')`` never reads the host.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -53,7 +50,8 @@ class ResolvedImage:
|
||||
_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*://")
|
||||
|
||||
|
||||
async def resolve_image_source(src: str, ctx: ResolveContext, *, permitted: tuple = ("image",)) -> ResolvedImage:
|
||||
async def resolve_image_source(
|
||||
src: str, ctx: ResolveContext, *, permitted: tuple = ("image",)) -> ResolvedImage:
|
||||
if not isinstance(src, str) or not src.strip():
|
||||
raise SourceNotFound("image_url is required", src=str(src))
|
||||
s = src.strip()
|
||||
@@ -69,26 +67,14 @@ async def resolve_image_source(src: str, ctx: ResolveContext, *, permitted: tupl
|
||||
raise UnsupportedScheme(
|
||||
"Unrecognized image source scheme. Use an http(s) URL, a local "
|
||||
"file path, a file:// URI, or a data: URL.",
|
||||
src=s,
|
||||
)
|
||||
|
||||
src=s)
|
||||
# Everything else is a filesystem path — including bare relative names like "pic.png"
|
||||
# (a path-shape gate here regressed them once).
|
||||
candidate = s[len("file://"):] if s.lower().startswith("file://") else s
|
||||
p = Path(os.path.expanduser(candidate))
|
||||
host_target = _permitted_host_read_target(p, ctx)
|
||||
if host_target is not None and host_target.is_file():
|
||||
# Shared credential-read guard: refuse secret-bearing files (.env, auth.json) with a
|
||||
# specific error. Guard import is best-effort; a real block always propagates.
|
||||
try:
|
||||
from agent.file_safety import raise_if_read_blocked
|
||||
except Exception: # noqa: BLE001 — guard unavailable: proceed
|
||||
raise_if_read_blocked = None
|
||||
if raise_if_read_blocked is not None:
|
||||
try:
|
||||
raise_if_read_blocked(str(host_target))
|
||||
except ValueError as exc:
|
||||
raise SourceUnsafe(str(exc), src=s, origin="file")
|
||||
_guard_credential_read(host_target, s)
|
||||
data = await asyncio.to_thread(host_target.read_bytes)
|
||||
return _finalize(data, "", "file", s, permitted)
|
||||
if _is_local_terminal_backend():
|
||||
@@ -97,6 +83,19 @@ async def resolve_image_source(src: str, ctx: ResolveContext, *, permitted: tupl
|
||||
return await _resolve_container_fallback(p, ctx, s, permitted)
|
||||
|
||||
|
||||
def _guard_credential_read(host_target: Path, src: str) -> None:
|
||||
"""Shared credential-read guard: refuse secret-bearing files (.env, auth.json) with a specific
|
||||
error. Guard import is best-effort; a real block always propagates."""
|
||||
try:
|
||||
from agent.file_safety import raise_if_read_blocked
|
||||
except Exception: # noqa: BLE001 — guard unavailable: proceed
|
||||
return
|
||||
try:
|
||||
raise_if_read_blocked(str(host_target))
|
||||
except ValueError as exc:
|
||||
raise SourceUnsafe(str(exc), src=src, origin="file")
|
||||
|
||||
|
||||
def _resolve_data_url(s: str) -> tuple[bytes, str]:
|
||||
header, _, payload = s.partition(",")
|
||||
if ";base64" not in header:
|
||||
@@ -117,11 +116,9 @@ def _http_block_reason(url: str) -> Optional[str]:
|
||||
``_download_image`` re-checks per attempt and against the final redirect target (intentional)."""
|
||||
from tools.url_safety import is_safe_url
|
||||
from tools.website_policy import check_website_access
|
||||
|
||||
if not is_safe_url(url):
|
||||
return "blocked: unsafe or private URL"
|
||||
blocked = check_website_access(url)
|
||||
if blocked:
|
||||
if blocked := check_website_access(url):
|
||||
return blocked.get("message") or "blocked by website policy"
|
||||
return None
|
||||
|
||||
@@ -129,7 +126,6 @@ def _http_block_reason(url: str) -> Optional[str]:
|
||||
async def _download_to_bytes(url: str) -> bytes:
|
||||
import tempfile
|
||||
from tools.vision_tools import _download_image
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".img", delete=False) as tf:
|
||||
tmp = Path(tf.name)
|
||||
try:
|
||||
@@ -152,8 +148,7 @@ def _is_local_terminal_backend() -> bool:
|
||||
_MEDIA_CACHE_SUBDIRS = (
|
||||
"cache", # cache/images, cache/vision, cache/video(s), cache/audio
|
||||
"images", # desktop/clipboard/PDF uploads (tui_gateway)
|
||||
"image_cache", "audio_cache", "video_cache", "temp_vision_images", "temp_video_files",
|
||||
)
|
||||
"image_cache", "audio_cache", "video_cache", "temp_vision_images", "temp_video_files")
|
||||
|
||||
|
||||
def _media_cache_roots() -> list:
|
||||
@@ -163,30 +158,23 @@ def _media_cache_roots() -> list:
|
||||
|
||||
|
||||
def _permitted_host_read_target(p: Path, ctx: ResolveContext) -> Optional[Path]:
|
||||
"""Host path to read, or ``None`` if a host read is not permitted.
|
||||
"""Host path to read, or ``None`` (caller exec-reads inside the sandbox instead).
|
||||
|
||||
Local backend: any path. Non-local: only paths resolving inside a media cache root (a
|
||||
container-visible cache path is first translated back to its host mount); anything else
|
||||
returns ``None`` so the caller exec-reads inside the sandbox instead.
|
||||
Local backend: any path. Non-local: only paths inside a media cache root (a
|
||||
container-visible cache path is first translated back to its host mount).
|
||||
"""
|
||||
if _is_local_terminal_backend():
|
||||
try:
|
||||
return p.resolve()
|
||||
except Exception: # noqa: BLE001 — unresolved path: let is_file() fail downstream
|
||||
return p
|
||||
|
||||
from tools.credential_files import from_agent_visible_cache_path
|
||||
|
||||
try:
|
||||
real = Path(from_agent_visible_cache_path(str(p))).resolve()
|
||||
except Exception: # noqa: BLE001 — cannot resolve -> not a safe host read
|
||||
return None
|
||||
for root in _media_cache_roots():
|
||||
try:
|
||||
real.relative_to(root.resolve())
|
||||
return real
|
||||
except ValueError:
|
||||
continue
|
||||
if any(real.is_relative_to(root.resolve()) for root in _media_cache_roots()):
|
||||
return real
|
||||
return None
|
||||
|
||||
|
||||
@@ -213,18 +201,15 @@ def _ensure_container_env(task_id: Optional[str]) -> None:
|
||||
|
||||
|
||||
async def _resolve_container_fallback(
|
||||
p: Path, ctx: ResolveContext, src: str, permitted: tuple = ("image",)
|
||||
) -> ResolvedImage:
|
||||
p: Path, ctx: ResolveContext, src: str, permitted: tuple = ("image",)) -> ResolvedImage:
|
||||
"""Read the bytes inside the sandbox; fail-closed when no env exists (a non-cache host
|
||||
path under a sandbox must never leak via a host fallback).
|
||||
|
||||
Cold-start retry: under Docker the first exec against a fresh container can fail (empty
|
||||
pipe) while an identical second call succeeds, so retry once after a short delay. On final
|
||||
failure the container's own output is folded into the error so "no such file" /
|
||||
"permission denied" / "never came up" are distinguishable.
|
||||
pipe) while a second succeeds. On final failure the container's output is folded into the
|
||||
error so "no such file" / "permission denied" / "never came up" are distinguishable.
|
||||
"""
|
||||
import shlex
|
||||
|
||||
_ensure_container_env(ctx.task_id)
|
||||
env = _get_active_env(ctx.task_id)
|
||||
if env is None:
|
||||
@@ -232,13 +217,10 @@ async def _resolve_container_fallback(
|
||||
f"'{p}' is not reachable inside the sandbox and no active sandbox "
|
||||
f"session is available to read it",
|
||||
src=src, origin="container")
|
||||
|
||||
# Bound the read INSIDE the sandbox: head -c caps at ingest-limit+1 so /dev/zero can't
|
||||
# stream unbounded base64 into host memory (the +1 distinguishes "at the cap" from
|
||||
# "over"). The input redirect avoids argv, so leading-dash paths can't parse as options;
|
||||
# base64 -w0 is GNU-only, hence tr -d for BusyBox. env.execute blocks — keep it off the loop.
|
||||
# Bound the read INSIDE the sandbox: head -c caps at ingest-limit+1 (+1 distinguishes "at the
|
||||
# cap" from "over") so /dev/zero can't stream unbounded base64 into host memory. The input
|
||||
# redirect avoids argv (leading-dash paths); tr -d instead of GNU-only base64 -w0 (BusyBox).
|
||||
cmd = f"head -c {_MAX_INGEST_BYTES + 1} < {shlex.quote(str(p))} | base64 | tr -d '\\n'"
|
||||
|
||||
last_res: dict = {"returncode": 1, "output": ""}
|
||||
for attempt in range(2):
|
||||
last_res = await asyncio.to_thread(env.execute, cmd)
|
||||
@@ -263,17 +245,11 @@ async def _resolve_container_fallback(
|
||||
def _finalize(
|
||||
data: bytes, declared_mime: str, origin: str, src: str, permitted: tuple = ("image",)
|
||||
) -> ResolvedImage:
|
||||
"""Chokepoint: 50MB ingest cap + type check.
|
||||
|
||||
Images are typed by magic bytes. Video (opt-in) is typed by extension plus an mp4
|
||||
container sniff — sufficient because every downstream consumer re-validates, so a wrong
|
||||
guess is a clean rejection there, not a hole here.
|
||||
"""
|
||||
"""Chokepoint: 50MB ingest cap + type check. Images by magic bytes; video (opt-in) by
|
||||
extension + mp4 sniff — enough because every downstream consumer re-validates."""
|
||||
from tools.vision_tools import _detect_image_mime_type_from_bytes
|
||||
|
||||
if len(data) > _MAX_INGEST_BYTES:
|
||||
raise SourceTooLarge("media exceeds size limit", src=src, origin=origin)
|
||||
|
||||
sniffed = _detect_image_mime_type_from_bytes(data)
|
||||
if sniffed is not None:
|
||||
if "image" not in permitted:
|
||||
@@ -295,7 +271,6 @@ def _detect_video_mime(data: bytes, src: str) -> Optional[str]:
|
||||
offset 4 (covers extensionless data: URLs / query-string URLs)."""
|
||||
from urllib.parse import urlsplit
|
||||
from tools.vision_tools import _detect_video_mime_type
|
||||
|
||||
path_part = urlsplit(src).path if _SCHEME_RE.match(src) else src
|
||||
by_extension = _detect_video_mime_type(Path(path_part))
|
||||
if by_extension is not None:
|
||||
@@ -306,14 +281,12 @@ def _detect_video_mime(data: bytes, src: str) -> Optional[str]:
|
||||
|
||||
|
||||
async def resolve_local_source_to_data_url(
|
||||
src: str, task_id: Optional[str], *, permitted: tuple = ("image",)
|
||||
) -> str:
|
||||
src: str, task_id: Optional[str], *, permitted: tuple = ("image",)) -> str:
|
||||
"""Convert a path-like media source into a ``data:`` URL via the resolver.
|
||||
|
||||
Dispatch-layer chokepoint for generation tools: providers historically read model-supplied
|
||||
local paths off the HOST regardless of backend — broken under a sandbox and inconsistent
|
||||
with vision's confinement. URL-shaped sources (http/https/data) pass through untouched.
|
||||
Callers apply this only under a non-local backend (local keeps host reads).
|
||||
Dispatch-layer chokepoint for generation tools so providers never read model-supplied paths
|
||||
off the HOST under a sandbox. URL-shaped sources pass through; callers apply this only under
|
||||
a non-local backend.
|
||||
"""
|
||||
s = (src or "").strip()
|
||||
if not s or s.lower().startswith(("http://", "https://", "data:")):
|
||||
|
||||
Reference in New Issue
Block a user