refactor(video_generate): capability-gated dynamic schema (~814 → 458/377 tok/call) (#97095)
* refactor(video_generate): capability-gated dynamic schema — 6 optional args render only when the active provider/model honors them; fleet capability declarations + declaration<->implementation contract tests * fix(video_gen): H3/Grok/Happy-Horse/Gemini audio is ALWAYS-ON native, not absent — new audio_native family key + audio_always_on capability surfaces as description line (maintainer catch) * test(video_gen): duration-span test pins the active-model contract — resolved family's real window, short families not inflated, union fallback still spans 30s
This commit is contained in:
Teknium
@@ -148,11 +148,17 @@ class VideoGenProvider(abc.ABC):
|
||||
"min_duration": 1,
|
||||
"supports_audio": True,
|
||||
"supports_negative_prompt": True,
|
||||
"supports_seed": True,
|
||||
"supports_upscale": True,
|
||||
"max_reference_images": 7,
|
||||
}
|
||||
|
||||
Used by the tool layer for soft validation and by ``hermes tools``
|
||||
for the picker. Default: text-only.
|
||||
Used by the tool layer for soft validation, for capability-gated
|
||||
param rendering in the dynamic ``video_generate`` schema (args a
|
||||
backend can't honor are not advertised), and by ``hermes tools``
|
||||
for the picker. Default fails closed: text-only, no optional
|
||||
features — a provider that doesn't declare a capability doesn't
|
||||
advertise it.
|
||||
"""
|
||||
return {
|
||||
"modalities": ["text"],
|
||||
@@ -162,6 +168,8 @@ class VideoGenProvider(abc.ABC):
|
||||
"min_duration": 1,
|
||||
"supports_audio": False,
|
||||
"supports_negative_prompt": False,
|
||||
"supports_seed": False,
|
||||
"supports_upscale": False,
|
||||
"max_reference_images": 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -67,6 +67,8 @@ class DeepInfraVideoGenProvider(OpenAICompatibleVideoGenProvider):
|
||||
"min_duration": 1,
|
||||
"supports_audio": False,
|
||||
"supports_negative_prompt": True,
|
||||
"supports_seed": True,
|
||||
"supports_upscale": False,
|
||||
"max_reference_images": 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -95,6 +95,7 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = {
|
||||
"durations": None,
|
||||
"audio": True,
|
||||
"negative": True,
|
||||
"seed": True,
|
||||
},
|
||||
"pixverse-v6": {
|
||||
"display": "Pixverse v6",
|
||||
@@ -109,6 +110,7 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = {
|
||||
"durations": (1, 15),
|
||||
"audio": True,
|
||||
"negative": True,
|
||||
"seed": True,
|
||||
},
|
||||
"seedance-2.0-mini": {
|
||||
"display": "Seedance 2.0 Mini",
|
||||
@@ -140,6 +142,7 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = {
|
||||
"duration_suffix": "s", # FAL veo3.1 wants "4s" not "4"
|
||||
"audio": True,
|
||||
"negative": True,
|
||||
"seed": True,
|
||||
},
|
||||
"seedance-2.0": {
|
||||
"display": "Seedance 2.0",
|
||||
@@ -201,7 +204,8 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = {
|
||||
"1080p": "2K", "2k": "2K", "4k": "4K", "2160p": "4K",
|
||||
},
|
||||
"durations": (5, 15),
|
||||
"audio": False, # audio is native/always-on; no generate_audio key
|
||||
"audio": False, # no generate_audio TOGGLE — audio is always on
|
||||
"audio_native": True, # native audio in every generation (fal docs) # audio is native/always-on; no generate_audio key
|
||||
"negative": False,
|
||||
"seed": False,
|
||||
},
|
||||
@@ -231,8 +235,10 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = {
|
||||
# `prompt_expansion_mode` is in the schema's required array (with a
|
||||
# "balanced" default) — always send it.
|
||||
"static_payload": {"prompt_expansion_mode": "balanced"},
|
||||
"audio": False, # audio is native/always-on; no generate_audio key
|
||||
"audio": False, # no generate_audio TOGGLE — audio is always on
|
||||
"audio_native": True, # native audio in every generation (fal docs) # audio is native/always-on; no generate_audio key
|
||||
"negative": False,
|
||||
"seed": True,
|
||||
# Unlike base H3, Max declares `seed` on both endpoints.
|
||||
},
|
||||
"flux-3": {
|
||||
@@ -266,7 +272,8 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = {
|
||||
"aspect_ratios": ("16:9", "4:3", "3:2", "1:1", "2:3", "3:4", "9:16"),
|
||||
"resolutions": ("480p", "720p", "1080p"),
|
||||
"durations": (1, 15),
|
||||
"audio": False, # audio is native; no generate_audio key
|
||||
"audio": False, # no generate_audio TOGGLE — audio is always on
|
||||
"audio_native": True, # native audio in every generation (fal docs) # audio is native; no generate_audio key
|
||||
"negative": False,
|
||||
"seed": False,
|
||||
},
|
||||
@@ -283,7 +290,8 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = {
|
||||
"aspect_ratios": ("16:9", "9:16"),
|
||||
"resolutions": None,
|
||||
"durations": (3, 10),
|
||||
"audio": False, # audio is native; no generate_audio key
|
||||
"audio": False, # no generate_audio TOGGLE — audio is always on
|
||||
"audio_native": True, # native audio in every generation (fal docs) # audio is native; no generate_audio key
|
||||
"negative": False,
|
||||
"seed": False,
|
||||
},
|
||||
@@ -303,6 +311,7 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = {
|
||||
"durations": (3, 15),
|
||||
"audio": True,
|
||||
"negative": True,
|
||||
"seed": True,
|
||||
},
|
||||
"happy-horse": {
|
||||
"display": "Happy Horse 1.0",
|
||||
@@ -317,8 +326,10 @@ FAL_FAMILIES: Dict[str, Dict[str, Any]] = {
|
||||
"aspect_ratios": None,
|
||||
"resolutions": None,
|
||||
"durations": None,
|
||||
"audio": False,
|
||||
"audio": False, # no generate_audio TOGGLE — audio is always on
|
||||
"audio_native": True, # native audio in every generation (fal docs)
|
||||
"negative": False,
|
||||
"seed": True,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -806,8 +817,47 @@ class FALVideoGenProvider(VideoGenProvider):
|
||||
}
|
||||
|
||||
def capabilities(self) -> Dict[str, Any]:
|
||||
# Union across families so the tool schema doesn't understate the
|
||||
# longest-running models (Seedance 2.5 = 30s, FLUX 3 = 20s).
|
||||
# Active-model-aware (mirrors the image_gen fal plugin, #97057):
|
||||
# report the RESOLVED family's actual surface so the dynamic tool
|
||||
# schema gates params on what the selected model honors, not a
|
||||
# union that overstates every axis. Falls back to the cross-family
|
||||
# union if resolution fails (never raises).
|
||||
try:
|
||||
_family_id, family = _resolve_family(None)
|
||||
except Exception: # noqa: BLE001
|
||||
family = None
|
||||
if family:
|
||||
modalities = []
|
||||
if family.get("text_endpoint"):
|
||||
modalities.append("text")
|
||||
if family.get("image_endpoint"):
|
||||
modalities.append("image")
|
||||
durs = family.get("durations") or (1, 1)
|
||||
if _is_duration_range(durs):
|
||||
lo, hi = durs
|
||||
else:
|
||||
lo, hi = min(durs), max(durs)
|
||||
return {
|
||||
"modalities": modalities or ["text"],
|
||||
"aspect_ratios": list(family.get("aspect_ratios") or []),
|
||||
"resolutions": list(family.get("resolutions") or []),
|
||||
"max_duration": hi,
|
||||
"min_duration": lo,
|
||||
"supports_audio": bool(family.get("audio")),
|
||||
# Always-on native audio (no toggle): surfaces as a
|
||||
# description line, not a param. Verified per-family
|
||||
# against fal model pages (H3, Grok 1.5, Happy Horse,
|
||||
# Gemini Omni Flash all return native audio every run).
|
||||
"audio_always_on": bool(family.get("audio_native")),
|
||||
"supports_negative_prompt": bool(family.get("negative")),
|
||||
# Explicit per-family key (contract-tested); absent would
|
||||
# mean a catalog bug, so fail closed here.
|
||||
"supports_seed": bool(family.get("seed", False)),
|
||||
# SeedVR upscaler chains for any FAL video family.
|
||||
"supports_upscale": True,
|
||||
"max_reference_images": 0,
|
||||
}
|
||||
# Fallback: union across families (legacy shape).
|
||||
max_dur = 1
|
||||
min_dur: Optional[int] = None
|
||||
for meta in FAL_FAMILIES.values():
|
||||
@@ -828,6 +878,8 @@ class FALVideoGenProvider(VideoGenProvider):
|
||||
"min_duration": min_dur if min_dur is not None else 1,
|
||||
"supports_audio": True,
|
||||
"supports_negative_prompt": True,
|
||||
"supports_seed": True,
|
||||
"supports_upscale": True,
|
||||
"max_reference_images": 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -423,6 +423,8 @@ class XAIVideoGenProvider(VideoGenProvider):
|
||||
"min_duration": 1,
|
||||
"supports_audio": False,
|
||||
"supports_negative_prompt": False,
|
||||
"supports_seed": True,
|
||||
"supports_upscale": False,
|
||||
"max_reference_images": MAX_REFERENCE_IMAGES,
|
||||
}
|
||||
|
||||
|
||||
@@ -268,10 +268,34 @@ class TestFamilyKeyNormalization:
|
||||
assert _normalize_family_key("blackforestlabs/flux-3") == "flux-3"
|
||||
|
||||
def test_capabilities_span_longest_family_duration(self):
|
||||
"""Provider caps must not understate Seedance 2.5's 30s ceiling."""
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
"""capabilities() is active-MODEL-aware (#95681 diet): it reports
|
||||
the resolved family's real window, so the schema doesn't overstate
|
||||
short families or understate Seedance 2.5. The union fallback
|
||||
(resolution failure) must still span the 30s ceiling."""
|
||||
from unittest.mock import patch as _patch
|
||||
|
||||
caps = FALVideoGenProvider().capabilities()
|
||||
import plugins.video_gen.fal as _fp
|
||||
from plugins.video_gen.fal import FAL_FAMILIES, FALVideoGenProvider
|
||||
|
||||
# Active model resolved → that family's actual window.
|
||||
meta = FAL_FAMILIES["seedance-2.5"]
|
||||
with _patch.object(_fp, "_resolve_family",
|
||||
return_value=("seedance-2.5", meta)):
|
||||
caps = FALVideoGenProvider().capabilities()
|
||||
assert caps["max_duration"] >= 30
|
||||
# A short family must NOT be inflated to the union ceiling.
|
||||
short = FAL_FAMILIES["pixverse-v6"]
|
||||
durs = short.get("durations")
|
||||
hi = durs[1] if isinstance(durs, tuple) else max(durs)
|
||||
with _patch.object(_fp, "_resolve_family",
|
||||
return_value=("pixverse-v6", short)):
|
||||
caps = FALVideoGenProvider().capabilities()
|
||||
assert caps["max_duration"] == hi
|
||||
|
||||
# Resolution failure → union fallback still spans the ceiling.
|
||||
with _patch.object(_fp, "_resolve_family",
|
||||
side_effect=RuntimeError("no config")):
|
||||
caps = FALVideoGenProvider().capabilities()
|
||||
assert caps["max_duration"] >= 30
|
||||
assert caps["min_duration"] <= 1
|
||||
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
"""video_generate dynamic schema — capability-gated params (#95681 diet).
|
||||
|
||||
Mirrors tests/tools/test_image_generate_schema.py (#97057). Coverage is
|
||||
guaranteed three ways:
|
||||
1. every in-tree video_gen plugin's capabilities() must declare EVERY axis
|
||||
the schema builder reads (a new axis added to the builder without fleet
|
||||
declarations fails here);
|
||||
2. every FAL video family must carry the per-family keys the fal provider's
|
||||
active-model capabilities() resolution reads;
|
||||
3. declaration⇄implementation: a provider that declares seed/upscale must
|
||||
implement it, and vice versa (source-level sweep, both directions).
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
|
||||
import tools.video_generation_tool as vt
|
||||
from tools.video_generation_tool import (
|
||||
VIDEO_GENERATE_SCHEMA,
|
||||
_build_dynamic_video_schema,
|
||||
)
|
||||
|
||||
# Every axis _build_dynamic_video_schema reads from capabilities().
|
||||
CAPABILITY_AXES = (
|
||||
"modalities",
|
||||
"aspect_ratios",
|
||||
"resolutions",
|
||||
"max_duration",
|
||||
"min_duration",
|
||||
"supports_audio",
|
||||
"supports_negative_prompt",
|
||||
"supports_seed",
|
||||
"supports_upscale",
|
||||
"max_reference_images",
|
||||
)
|
||||
|
||||
# Per-family keys the FAL provider's capabilities() resolution reads.
|
||||
FAL_FAMILY_KEYS = ("durations", "aspect_ratios", "resolutions", "audio",
|
||||
"negative", "seed")
|
||||
|
||||
|
||||
def _plugin_sources():
|
||||
import pathlib
|
||||
|
||||
plugins_dir = (pathlib.Path(__file__).resolve().parents[2]
|
||||
/ "plugins" / "video_gen")
|
||||
assert plugins_dir.is_dir(), plugins_dir
|
||||
out = {}
|
||||
for plugin in sorted(plugins_dir.iterdir()):
|
||||
src_file = plugin / "__init__.py"
|
||||
if src_file.is_file():
|
||||
out[plugin.name] = src_file.read_text(encoding="utf-8")
|
||||
return out
|
||||
|
||||
|
||||
class TestFleetCapabilityCoverage(unittest.TestCase):
|
||||
def test_every_provider_declares_every_axis(self):
|
||||
"""Instantiate each in-tree provider class and check the RETURNED
|
||||
capabilities dict — source grep can't see inherited keys."""
|
||||
checked = 0
|
||||
# fal
|
||||
from plugins.video_gen.fal import FALVideoGenProvider
|
||||
|
||||
caps = FALVideoGenProvider().capabilities()
|
||||
for axis in CAPABILITY_AXES:
|
||||
self.assertIn(axis, caps, f"fal missing {axis}")
|
||||
checked += 1
|
||||
# xai
|
||||
from plugins.video_gen.xai import XAIVideoGenProvider
|
||||
|
||||
caps = XAIVideoGenProvider().capabilities()
|
||||
for axis in CAPABILITY_AXES:
|
||||
self.assertIn(axis, caps, f"xai missing {axis}")
|
||||
checked += 1
|
||||
# deepinfra
|
||||
from plugins.video_gen.deepinfra import DeepInfraVideoGenProvider
|
||||
|
||||
caps = DeepInfraVideoGenProvider().capabilities()
|
||||
for axis in CAPABILITY_AXES:
|
||||
self.assertIn(axis, caps, f"deepinfra missing {axis}")
|
||||
checked += 1
|
||||
self.assertGreaterEqual(checked, 3)
|
||||
|
||||
def test_abc_default_fails_closed(self):
|
||||
from agent.video_gen_provider import VideoGenProvider
|
||||
|
||||
class _P(VideoGenProvider):
|
||||
name = "t"
|
||||
display_name = "T"
|
||||
def generate(self, prompt, **kw):
|
||||
return {}
|
||||
def list_models(self):
|
||||
return []
|
||||
caps = _P().capabilities()
|
||||
self.assertEqual(caps.get("modalities"), ["text"])
|
||||
for axis in ("supports_audio", "supports_negative_prompt",
|
||||
"supports_seed", "supports_upscale"):
|
||||
self.assertFalse(caps.get(axis), axis)
|
||||
for axis in CAPABILITY_AXES:
|
||||
self.assertIn(axis, caps, f"ABC default missing {axis}")
|
||||
|
||||
def test_every_fal_family_declares_per_family_keys(self):
|
||||
from plugins.video_gen.fal import FAL_FAMILIES
|
||||
|
||||
for fam, meta in FAL_FAMILIES.items():
|
||||
with self.subTest(family=fam):
|
||||
for key in FAL_FAMILY_KEYS:
|
||||
self.assertIn(key, meta, f"{fam} missing {key}")
|
||||
self.assertTrue(
|
||||
meta.get("text_endpoint") or meta.get("image_endpoint"),
|
||||
f"{fam} declares no endpoint",
|
||||
)
|
||||
# Audio truth model: "audio" means a generate_audio TOGGLE
|
||||
# exists; "audio_native" means audio is ALWAYS ON with no
|
||||
# toggle. A family may have neither (silent model) but
|
||||
# never both — that would be contradictory.
|
||||
self.assertFalse(
|
||||
meta.get("audio") and meta.get("audio_native"),
|
||||
f"{fam}: audio toggle and always-on are mutually "
|
||||
"exclusive",
|
||||
)
|
||||
|
||||
def test_always_on_audio_surfaces_in_description_not_param(self):
|
||||
"""Maintainer catch (H3 has audio!): families whose audio is
|
||||
always-on (no API toggle) must TELL the model about the audio in
|
||||
the description rather than advertise a dead `audio` param."""
|
||||
schema = TestDynamicParamGating._schema_with(
|
||||
TestDynamicParamGating(), {
|
||||
"modalities": ["text", "image"],
|
||||
"supports_audio": False, "audio_always_on": True,
|
||||
"supports_negative_prompt": False, "supports_seed": False,
|
||||
"supports_upscale": False, "max_reference_images": 0,
|
||||
})
|
||||
self.assertNotIn("audio", schema["parameters"]["properties"])
|
||||
self.assertIn("always on", schema["description"])
|
||||
|
||||
def test_declaration_matches_implementation(self):
|
||||
"""supports_seed / supports_upscale: declaration ⇄ implementation,
|
||||
source-level, both directions, every in-tree plugin.
|
||||
|
||||
deepinfra inherits generate() from OpenAICompatibleVideoGenProvider
|
||||
(agent/video_gen_provider.py), so its implementation source is the
|
||||
base class file."""
|
||||
import pathlib
|
||||
|
||||
base_src = (pathlib.Path(__file__).resolve().parents[2]
|
||||
/ "agent" / "video_gen_provider.py").read_text(encoding="utf-8")
|
||||
for name, src in _plugin_sources().items():
|
||||
with self.subTest(provider=name):
|
||||
impl_src = src if "def generate" in src else src + base_src
|
||||
declares_upscale = '"supports_upscale": True' in src
|
||||
implements_upscale = ("_upscale_video" in impl_src
|
||||
or "UPSCALER_ENDPOINT" in impl_src)
|
||||
self.assertEqual(
|
||||
declares_upscale, implements_upscale,
|
||||
f"{name}: supports_upscale declaration "
|
||||
f"({declares_upscale}) != implementation "
|
||||
f"({implements_upscale})",
|
||||
)
|
||||
declares_seed = '"supports_seed": True' in src
|
||||
implements_seed = ("seed" in impl_src
|
||||
and ("payload[\"seed\"]" in impl_src
|
||||
or "seed: Optional[int]" in impl_src
|
||||
or "\"seed\": seed" in impl_src))
|
||||
self.assertEqual(
|
||||
declares_seed, implements_seed,
|
||||
f"{name}: supports_seed declaration ({declares_seed}) "
|
||||
f"!= implementation ({implements_seed})",
|
||||
)
|
||||
|
||||
|
||||
class TestDynamicParamGating(unittest.TestCase):
|
||||
def _schema_with(self, caps, model_meta=None):
|
||||
class _Prov:
|
||||
name = "fake"
|
||||
display_name = "Fake"
|
||||
def capabilities(self):
|
||||
return caps
|
||||
def list_models(self):
|
||||
return [dict({"id": "m1"}, **(model_meta or {}))]
|
||||
def default_model(self):
|
||||
return "m1"
|
||||
with patch.object(vt, "_resolve_active_provider",
|
||||
return_value=_Prov()), \
|
||||
patch.object(vt, "_read_configured_video_model",
|
||||
return_value="m1"):
|
||||
return _build_dynamic_video_schema()
|
||||
|
||||
def test_full_featured_backend_gets_all_params(self):
|
||||
schema = self._schema_with({
|
||||
"modalities": ["text", "image"],
|
||||
"aspect_ratios": ["16:9"], "resolutions": ["720p"],
|
||||
"min_duration": 2, "max_duration": 12,
|
||||
"supports_audio": True, "supports_negative_prompt": True,
|
||||
"supports_seed": True, "supports_upscale": True,
|
||||
"max_reference_images": 7,
|
||||
})
|
||||
props = schema["parameters"]["properties"]
|
||||
for p in ("image_url", "reference_image_urls", "negative_prompt",
|
||||
"audio", "seed", "upscale"):
|
||||
self.assertIn(p, props, p)
|
||||
self.assertEqual(props["reference_image_urls"]["maxItems"], 7)
|
||||
self.assertEqual(props["duration"]["minimum"], 2)
|
||||
self.assertEqual(props["duration"]["maximum"], 12)
|
||||
self.assertEqual(props["aspect_ratio"]["enum"], ["16:9"])
|
||||
|
||||
def test_minimal_backend_gets_bare_params(self):
|
||||
schema = self._schema_with({
|
||||
"modalities": ["text"],
|
||||
"supports_audio": False, "supports_negative_prompt": False,
|
||||
"supports_seed": False, "supports_upscale": False,
|
||||
"max_reference_images": 0,
|
||||
})
|
||||
props = schema["parameters"]["properties"]
|
||||
for p in ("image_url", "reference_image_urls", "negative_prompt",
|
||||
"audio", "seed", "upscale"):
|
||||
self.assertNotIn(p, props, p)
|
||||
self.assertIn("text-to-video only", schema["description"])
|
||||
|
||||
def test_i2v_only_model_overrides_backend_union(self):
|
||||
# gemini-omni-flash case: dual-modality backend, i2v-only model.
|
||||
schema = self._schema_with(
|
||||
{"modalities": ["text", "image"], "max_reference_images": 0,
|
||||
"supports_audio": False, "supports_negative_prompt": False,
|
||||
"supports_seed": False, "supports_upscale": False},
|
||||
model_meta={"modalities": ["image"]},
|
||||
)
|
||||
self.assertIn("image_url", schema["parameters"]["properties"])
|
||||
self.assertIn("image-to-video only", schema["description"])
|
||||
|
||||
def test_no_provider_serves_prompt_only(self):
|
||||
with patch.object(vt, "_resolve_active_provider", return_value=None):
|
||||
schema = _build_dynamic_video_schema()
|
||||
self.assertEqual(sorted(schema["parameters"]["properties"]), ["prompt"])
|
||||
|
||||
def test_static_schema_carries_no_capability_args(self):
|
||||
props = VIDEO_GENERATE_SCHEMA["parameters"]["properties"]
|
||||
self.assertEqual(
|
||||
sorted(props),
|
||||
["aspect_ratio", "duration", "model", "prompt", "resolution"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -97,11 +97,16 @@ class TestUnifiedDispatch:
|
||||
|
||||
|
||||
def test_upscale_in_schema_and_forwarded(self):
|
||||
"""`upscale` is an agent-facing param, forwarded to providers when
|
||||
set and omitted (not None) when unset."""
|
||||
from tools.video_generation_tool import VIDEO_GENERATE_SCHEMA
|
||||
"""`upscale` is advertised per-capability by the dynamic builder
|
||||
(#95681 diet — static schema no longer carries it) and forwarded
|
||||
to providers when set, omitted (not None) when unset."""
|
||||
from tools.video_generation_tool import (
|
||||
VIDEO_GENERATE_SCHEMA,
|
||||
_build_dynamic_video_schema,
|
||||
)
|
||||
# Static placeholder: capability args live in the dynamic override.
|
||||
props = VIDEO_GENERATE_SCHEMA["parameters"]["properties"]
|
||||
assert props["upscale"]["type"] == "boolean"
|
||||
assert "upscale" not in props
|
||||
|
||||
provider = _RecordingProvider()
|
||||
video_gen_registry.register_provider(provider)
|
||||
|
||||
@@ -110,9 +110,13 @@ class TestDynamicSchemaBuilder:
|
||||
video_gen_registry.register_provider(_BothModalitiesProvider())
|
||||
_write_cfg(cfg_home, {"video_gen": {"provider": "both", "model": "family-a"}})
|
||||
|
||||
desc = _build_dynamic_video_schema()["description"]
|
||||
assert "supports both text-to-video" in desc
|
||||
assert "duration range: 1-15s" in desc
|
||||
schema = _build_dynamic_video_schema()
|
||||
# Dual-modality (#95681 diet): capability surfaces as PARAMS —
|
||||
# image_url advertised; duration bounds from the model window.
|
||||
props = schema["parameters"]["properties"]
|
||||
assert "image_url" in props
|
||||
assert props["duration"]["minimum"] == 1
|
||||
assert props["duration"]["maximum"] == 15
|
||||
|
||||
def test_i2v_only_model_does_not_claim_text_to_video(self, cfg_home):
|
||||
"""A dual-modality backend with an i2v-only active model must not
|
||||
@@ -154,8 +158,14 @@ class TestDynamicSchemaBuilder:
|
||||
{"video_gen": {"provider": "dual-i2v", "model": "gemini-like"}},
|
||||
)
|
||||
|
||||
desc = _build_dynamic_video_schema()["description"]
|
||||
schema = _build_dynamic_video_schema()
|
||||
desc = schema["description"]
|
||||
assert "image-to-video only" in desc
|
||||
assert "supports both text-to-video" not in desc
|
||||
# Prefer the active model's duration window over the backend union.
|
||||
assert "duration range: 3-10s" in desc
|
||||
# Prefer the active model's duration window over the backend union
|
||||
# — now expressed as param bounds, not prose.
|
||||
props = schema["parameters"]["properties"]
|
||||
assert props["duration"]["minimum"] == 3
|
||||
assert props["duration"]["maximum"] == 10
|
||||
# i2v-only still advertises image_url.
|
||||
assert "image_url" in props
|
||||
|
||||
+126
-117
@@ -61,12 +61,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
VIDEO_GENERATE_SCHEMA: Dict[str, Any] = {
|
||||
"name": "video_generate",
|
||||
# Placeholder — the real description is built dynamically at
|
||||
# get_tool_definitions() time so it reflects the active backend's
|
||||
# actual capabilities (which modalities / resolutions / duration
|
||||
# ranges the user's currently-selected model supports).
|
||||
# See _build_dynamic_video_schema() below and the dynamic-tool-schemas
|
||||
# skill at github/hermes-agent-dev/references/dynamic-tool-schemas.md.
|
||||
# Placeholder — description AND params are rebuilt dynamically at
|
||||
# get_tool_definitions() time from the active provider's declared
|
||||
# capabilities() and the active model's catalog entry. Optional args
|
||||
# (image_url, reference_image_urls, negative_prompt, audio, seed,
|
||||
# upscale) are advertised ONLY when the active backend/model honors
|
||||
# them; the handler accepts them regardless (replay compat — providers
|
||||
# clamp/ignore). See _build_dynamic_video_schema().
|
||||
"description": "(rebuilt at get_definitions() time — see _build_dynamic_video_schema)",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
@@ -78,93 +79,36 @@ VIDEO_GENERATE_SCHEMA: Dict[str, Any] = {
|
||||
"subject, style, camera movement, etc."
|
||||
),
|
||||
},
|
||||
"image_url": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Optional public HTTPS URL of a still image. When provided, "
|
||||
"the active backend routes to its image-to-video "
|
||||
"endpoint (animate the image); when omitted, it routes "
|
||||
"to text-to-video. For xAI chaining, use the `image` or "
|
||||
"`public_url` HTTPS URL from a prior Imagine result."
|
||||
),
|
||||
},
|
||||
"reference_image_urls": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": (
|
||||
"Optional list of public HTTPS reference image URLs "
|
||||
"(style or character refs). For xAI chaining, use "
|
||||
"`image` or `public_url` from prior Imagine results."
|
||||
),
|
||||
},
|
||||
"duration": {
|
||||
"type": "integer",
|
||||
"description": (
|
||||
"Desired video duration in seconds. Providers clamp to "
|
||||
"their supported range (commonly 4-15s). Omit to use the "
|
||||
"provider's default."
|
||||
"their supported range. Omit for the provider default."
|
||||
),
|
||||
},
|
||||
"aspect_ratio": {
|
||||
"type": "string",
|
||||
"enum": list(COMMON_ASPECT_RATIOS),
|
||||
"description": (
|
||||
"Output aspect ratio. Providers clamp to their supported "
|
||||
"set."
|
||||
),
|
||||
"description": "Output aspect ratio.",
|
||||
"default": DEFAULT_ASPECT_RATIO,
|
||||
},
|
||||
"resolution": {
|
||||
"type": "string",
|
||||
"enum": list(COMMON_RESOLUTIONS),
|
||||
"description": (
|
||||
"Output resolution. Providers clamp to their supported "
|
||||
"set."
|
||||
),
|
||||
"description": "Output resolution.",
|
||||
"default": DEFAULT_RESOLUTION,
|
||||
},
|
||||
"negative_prompt": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Optional negative prompt — content to avoid in the "
|
||||
"output. Supported by Pixverse, Kling, and similar; "
|
||||
"ignored by providers that do not support it."
|
||||
),
|
||||
},
|
||||
"audio": {
|
||||
"type": "boolean",
|
||||
"description": (
|
||||
"Optional audio generation toggle. Supported by Veo3 and "
|
||||
"Pixverse (affects pricing tier); ignored elsewhere."
|
||||
),
|
||||
},
|
||||
"seed": {
|
||||
"type": "integer",
|
||||
"description": (
|
||||
"Optional seed for reproducible outputs (provider-"
|
||||
"dependent)."
|
||||
),
|
||||
},
|
||||
"upscale": {
|
||||
"type": "boolean",
|
||||
"description": (
|
||||
"Optional high-resolution pass: when true, the generated "
|
||||
"video is run through the active backend's video upscaler "
|
||||
"(extra cost and latency, roughly 2x resolution). Use when "
|
||||
"the user asks for high-res / 4K output. Omit for the "
|
||||
"model's native resolution. Ignored by backends without "
|
||||
"an upscaler."
|
||||
),
|
||||
},
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Optional model override. If omitted, the user's "
|
||||
"configured ``video_gen.model`` (set via `hermes tools` "
|
||||
"→ Video Generation) is used. Models that the active "
|
||||
"provider does not know are rejected."
|
||||
"Optional model override; defaults to the configured "
|
||||
"``video_gen.model``. Unknown models are rejected."
|
||||
),
|
||||
},
|
||||
# NOTE (schema diet, #95681): image_url / reference_image_urls /
|
||||
# negative_prompt / audio / seed / upscale are added
|
||||
# per-capability by _build_dynamic_video_schema. Do not re-add
|
||||
# them statically.
|
||||
},
|
||||
"required": ["prompt"],
|
||||
},
|
||||
@@ -487,22 +431,18 @@ def _format_model_caveats(
|
||||
|
||||
|
||||
def _build_dynamic_video_schema() -> Dict[str, Any]:
|
||||
"""Build a description that reflects the active backend's actual surface.
|
||||
"""Render description AND params from the active backend's declared surface.
|
||||
|
||||
Cheap: reads config (already memoized by the caller), asks the active
|
||||
provider for `capabilities()` and the active model's catalog entry,
|
||||
and formats a few lines of prose. Falls back to the generic
|
||||
description when no provider is configured or registered.
|
||||
Optional args are advertised only when the resolved provider/model
|
||||
honors them (capabilities() + the model's catalog entry — coverage is
|
||||
contract-tested per provider); enums and duration bounds tighten to
|
||||
the active model's actual sets. The handler still accepts unadvertised
|
||||
args (replay compat): providers clamp or ignore, as before.
|
||||
"""
|
||||
static_props = VIDEO_GENERATE_SCHEMA["parameters"]["properties"]
|
||||
parts: List[str] = [_GENERIC_DESCRIPTION]
|
||||
|
||||
configured_model = _read_configured_video_model()
|
||||
|
||||
# Reflect the *resolved* active provider (same resolution the handler uses
|
||||
# in _resolve_active_provider): an explicit ``video_gen.provider``, or —
|
||||
# when unset — the single available registered backend. Keeping the
|
||||
# description in sync with execution stops the agent from being told
|
||||
# "no backend configured" while a call would actually succeed.
|
||||
provider = _resolve_active_provider()
|
||||
|
||||
if provider is None:
|
||||
@@ -510,7 +450,14 @@ def _build_dynamic_video_schema() -> Dict[str, Any]:
|
||||
"\nNo video backend is available. Calls will return an error "
|
||||
"until the user picks one via `hermes tools` → Video Generation."
|
||||
)
|
||||
return {"description": "\n".join(parts)}
|
||||
return {
|
||||
"description": "\n".join(parts),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"prompt": static_props["prompt"]},
|
||||
"required": ["prompt"],
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
caps = provider.capabilities() or {}
|
||||
@@ -527,47 +474,22 @@ def _build_dynamic_video_schema() -> Dict[str, Any]:
|
||||
{},
|
||||
)
|
||||
|
||||
backend_label = provider.display_name
|
||||
line = f"\nActive backend: {backend_label}"
|
||||
if active_model:
|
||||
line += f" · model: {active_model}"
|
||||
parts.append(line)
|
||||
|
||||
# Model-specific caveats (the high-signal stuff)
|
||||
# ---- description -------------------------------------------------
|
||||
for c in _format_model_caveats(model_meta, caps):
|
||||
parts.append(f"- {c}")
|
||||
|
||||
# Prefer the active model's modalities over the backend union. An
|
||||
# i2v-only family on a dual-modality backend (e.g. gemini-omni-flash
|
||||
# on FAL) must not also claim text-to-video support.
|
||||
model_modalities = set(model_meta.get("modalities") or [])
|
||||
modality = model_meta.get("modality")
|
||||
if modality:
|
||||
model_modalities.add(modality)
|
||||
effective_modalities = model_modalities or set(caps.get("modalities") or [])
|
||||
if "text" in effective_modalities and "image" in effective_modalities:
|
||||
parts.append(
|
||||
"- supports both text-to-video (omit image_url) and "
|
||||
"image-to-video (pass image_url) — routes automatically"
|
||||
)
|
||||
can_i2v = "image" in effective_modalities
|
||||
t2v = "text" in effective_modalities
|
||||
if can_i2v and not t2v:
|
||||
parts.append("- image-to-video only: image_url is REQUIRED")
|
||||
elif not can_i2v:
|
||||
parts.append("- text-to-video only (no image input)")
|
||||
|
||||
if caps.get("aspect_ratios"):
|
||||
parts.append(f"- aspect_ratio choices: {', '.join(caps['aspect_ratios'])}")
|
||||
if caps.get("resolutions"):
|
||||
parts.append(f"- resolution choices: {', '.join(caps['resolutions'])}")
|
||||
min_duration = model_meta.get("min_duration", caps.get("min_duration"))
|
||||
max_duration = model_meta.get("max_duration", caps.get("max_duration"))
|
||||
if min_duration and max_duration:
|
||||
parts.append(
|
||||
f"- duration range: {min_duration}-{max_duration}s"
|
||||
)
|
||||
if caps.get("supports_audio"):
|
||||
parts.append("- audio: pass `audio=true` to enable native audio (pricing tier)")
|
||||
if caps.get("supports_negative_prompt"):
|
||||
parts.append("- negative_prompt: supported")
|
||||
max_refs = caps.get("max_reference_images") or 0
|
||||
if max_refs:
|
||||
parts.append(f"- reference_image_urls: up to {max_refs} images")
|
||||
if provider.name == "xai":
|
||||
parts.append(
|
||||
"- chaining: for edit/extend pass the public HTTPS MP4 in `video` "
|
||||
@@ -584,7 +506,94 @@ def _build_dynamic_video_schema() -> Dict[str, Any]:
|
||||
if notice:
|
||||
parts.append(f"- storage: {notice}")
|
||||
|
||||
return {"description": "\n".join(parts)}
|
||||
# ---- params ------------------------------------------------------
|
||||
properties: Dict[str, Any] = {"prompt": static_props["prompt"]}
|
||||
|
||||
if can_i2v:
|
||||
properties["image_url"] = {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Public HTTPS URL of a still image to animate "
|
||||
"(image-to-video). Omit for text-to-video."
|
||||
),
|
||||
}
|
||||
max_refs = int(caps.get("max_reference_images") or 0)
|
||||
if max_refs > 0:
|
||||
properties["reference_image_urls"] = {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"maxItems": max_refs,
|
||||
"description": (
|
||||
f"Up to {max_refs} public HTTPS reference image URLs "
|
||||
"(style or character refs)."
|
||||
),
|
||||
}
|
||||
|
||||
min_duration = model_meta.get("min_duration", caps.get("min_duration"))
|
||||
max_duration = model_meta.get("max_duration", caps.get("max_duration"))
|
||||
duration_param = dict(static_props["duration"])
|
||||
if min_duration and max_duration:
|
||||
duration_param["minimum"] = int(min_duration)
|
||||
duration_param["maximum"] = int(max_duration)
|
||||
duration_param["description"] = (
|
||||
f"Video duration in seconds ({min_duration}-{max_duration}). "
|
||||
"Omit for the provider default."
|
||||
)
|
||||
properties["duration"] = duration_param
|
||||
|
||||
# Tighten enums to the active backend's actual sets when declared.
|
||||
aspect_param = dict(static_props["aspect_ratio"])
|
||||
if caps.get("aspect_ratios"):
|
||||
aspect_param["enum"] = list(caps["aspect_ratios"])
|
||||
properties["aspect_ratio"] = aspect_param
|
||||
|
||||
resolution_param = dict(static_props["resolution"])
|
||||
if caps.get("resolutions"):
|
||||
resolution_param["enum"] = list(caps["resolutions"])
|
||||
properties["resolution"] = resolution_param
|
||||
|
||||
if caps.get("supports_negative_prompt"):
|
||||
properties["negative_prompt"] = {
|
||||
"type": "string",
|
||||
"description": "Content to avoid in the output.",
|
||||
}
|
||||
if caps.get("supports_audio"):
|
||||
properties["audio"] = {
|
||||
"type": "boolean",
|
||||
"description": (
|
||||
"Enable native audio generation (affects pricing tier)."
|
||||
),
|
||||
}
|
||||
elif caps.get("audio_always_on"):
|
||||
parts.append(
|
||||
"- audio: native stereo audio is generated with every video "
|
||||
"(always on; no toggle) — describe the desired sound in the "
|
||||
"prompt"
|
||||
)
|
||||
if caps.get("supports_seed"):
|
||||
properties["seed"] = {
|
||||
"type": "integer",
|
||||
"description": "Seed for reproducible outputs.",
|
||||
}
|
||||
if caps.get("supports_upscale"):
|
||||
properties["upscale"] = {
|
||||
"type": "boolean",
|
||||
"description": (
|
||||
"High-resolution pass via the backend's video upscaler "
|
||||
"(~2x, extra cost/latency). Omit for native resolution."
|
||||
),
|
||||
}
|
||||
|
||||
properties["model"] = static_props["model"]
|
||||
|
||||
return {
|
||||
"description": "\n".join(parts),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"required": ["prompt"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user