m4
d4b53bfb08
Docker / build (push) Has been cancelled
Test / pytest (ubuntu-latest, 3.12) (push) Has been cancelled
Test / pytest (windows-latest, 3.11) (push) Has been cancelled
Test / pytest (windows-latest, 3.12) (push) Has been cancelled
Lint / ruff (push) Has been cancelled
Test / pytest (ubuntu-latest, 3.11) (push) Has been cancelled
Build / build (push) Has been cancelled
95 lines
4.6 KiB
Python
95 lines
4.6 KiB
Python
"""Schema2 tool declarations migrate without silent text-only degradation."""
|
|
|
|
from copy import deepcopy
|
|
|
|
import pytest
|
|
|
|
from EvoScientist.llm.contracts import EvoRuntimeError
|
|
from EvoScientist.llm.model_config import EvoModelConfig, convert_v2_to_v3_draft
|
|
from tests.v3_fixtures import v3_payload
|
|
|
|
|
|
def test_schema2_tools_require_explicit_native_route_contract():
|
|
# Keep this node identity: the historical non_streaming label was the only
|
|
# accepted Web tool route contract, not a measured capability or HTTP knob.
|
|
payload = v3_payload()
|
|
payload["capability_evidence"] = []
|
|
models = payload["providers"]["custom-openai"]["models"]
|
|
unknown = deepcopy(models[0])
|
|
unknown["id"] = "unreferenced-text-model"
|
|
models.append(unknown)
|
|
for transport in ("native", "non_streaming"):
|
|
payload["route_selectors"]["visible-main"]["tool_call_transport"] = transport
|
|
config = EvoModelConfig.parse(payload, require_evidence=False)
|
|
parsed = config.providers["custom-openai"].models
|
|
assert parsed[unknown["id"]].capabilities["tools"] is False
|
|
assert parsed[unknown["id"]].capabilities["text"] is True
|
|
assert parsed["model-id"].capabilities["tools"] is True
|
|
assert parsed["model-id"].capabilities["text"] is True
|
|
route = config.concrete_routes("visible-main")[0]
|
|
assert config.route_model(route).capabilities["tools"] is True
|
|
assert route.tool_call_transport == transport
|
|
assert config.capability_evidence == {}
|
|
draft, report = convert_v2_to_v3_draft(
|
|
payload, target_revision=2, config_identity_key_id="migration-test"
|
|
)
|
|
assert not report.blocking_issues
|
|
migrated = {m["model_key"]: m for m in draft["providers"][0]["models"]}
|
|
for key, expected in (("model-id", True), (unknown["id"], False)):
|
|
assert migrated[key]["capabilities"]["tools"] is expected
|
|
assert migrated[key]["invocation"]["tool_call_transport"] == (
|
|
"native" if expected else "disabled"
|
|
)
|
|
|
|
ambiguous = deepcopy(payload)
|
|
ambiguous["route_selectors"]["visible-main"]["tool_call_transport"] = "streaming"
|
|
with pytest.raises(EvoRuntimeError, match="schema2 tool capability migration"):
|
|
EvoModelConfig.parse(ambiguous, require_evidence=False)
|
|
with pytest.raises(EvoRuntimeError, match="schema2 tool capability migration"):
|
|
convert_v2_to_v3_draft(
|
|
ambiguous, target_revision=2, config_identity_key_id="migration-test"
|
|
)
|
|
|
|
# Aggregate declarations across routes, independent of insertion order.
|
|
declared = deepcopy(payload["route_selectors"]["visible-main"])
|
|
ambiguous["route_selectors"]["declared"] = declared
|
|
for reverse in (False, True):
|
|
if reverse:
|
|
ambiguous["route_selectors"] = dict(
|
|
reversed(list(ambiguous["route_selectors"].items()))
|
|
)
|
|
config = EvoModelConfig.parse(ambiguous, require_evidence=False)
|
|
route = config.concrete_routes("visible-main")[0]
|
|
assert config.route_model(route).capabilities["tools"] is True
|
|
|
|
# Same model id on another provider must not inherit the declaration.
|
|
isolated = deepcopy(payload)
|
|
isolated["providers"]["other"] = deepcopy(payload["providers"]["custom-openai"])
|
|
other = {
|
|
**declared,
|
|
"provider": "other",
|
|
"endpoint": "primary",
|
|
"tool_call_transport": "streaming",
|
|
}
|
|
other.pop("endpoint_pool")
|
|
isolated["route_selectors"]["other-route"] = other
|
|
config = EvoModelConfig.parse(isolated, require_evidence=False)
|
|
assert config.providers["other"].models["model-id"].capabilities["tools"] is False
|
|
for usage in ("main", "title", "fallback"):
|
|
referenced = deepcopy(isolated)
|
|
if usage == "main":
|
|
selectable = referenced["purpose_routes"]["main_agent"]["selectable"]
|
|
selectable["other"] = "other-route"
|
|
elif usage == "title":
|
|
referenced["purpose_routes"]["title"]["default"] = "other-route"
|
|
else:
|
|
referenced["tool_protocol_fallbacks"][0]["fallbacks"] = ["other-route"]
|
|
with pytest.raises(EvoRuntimeError, match="schema2 tool capability migration"):
|
|
EvoModelConfig.parse(referenced, require_evidence=False)
|
|
referenced["route_selectors"]["other-route"].update(
|
|
tool_call_transport="non_streaming"
|
|
)
|
|
config = EvoModelConfig.parse(referenced, require_evidence=False)
|
|
assert config.providers["other"].models["model-id"].capabilities["tools"] is True
|
|
other_models = config.providers["other"].models
|
|
assert other_models[unknown["id"]].capabilities["tools"] is False |