feat(model-registry): add delegation-JWT auth and config/snapshot HTTP API
- BFF service token (constant-time, plaintext or SHA-256 hash) plus X-Evo-Actor delegation JWT verification (ES256/RS256, iss/aud, <=60s lifetime, required claims, thread binding) with atomic jti anti-replay - Config API: GET/PUT /api/model-registry, credential rotation endpoint, GET /api/models selector; PUT runs the section 9.2 save-time checks inside the registry write transaction after credential writes - Snapshot API: create/bind/delete routes delegating to SnapshotService with thread/deployment binding checks and 9.5 unified error payloads - Platform security config loader (config.yaml fields), OpenAPI export (scripts/export_model_registry_schema.py -> model_registry/openapi.json) - Mount new routes in langgraph_dev/http.py; retire the legacy GET /api/models and POST /api/runtime-snapshots handlers - Declare PyJWT>=2.8 (previously transitive); extend the 9.5 error code table with the HTTP-layer codes (400/401/403/422/500)
This commit is contained in:
m4
@@ -0,0 +1,259 @@
|
||||
"""Delegation JWT and BFF service-token authentication tests (design doc 7.3).
|
||||
|
||||
Covers the正反例 the brief requires: missing/invalid service token,
|
||||
malformed/expired/wrongly-signed delegation JWTs, missing claims, lifetime
|
||||
over 60 seconds, wrong issuer/audience, unregistered deployments, scope
|
||||
escalation, jti replay, and the thread-claim rules for config vs
|
||||
thread-level routes. ``GET /api/model-registry`` serves as the probe route.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from starlette.applications import Starlette
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from EvoScientist.model_registry.auth import BffAuthenticator
|
||||
from EvoScientist.model_registry.endpoint_policy import EndpointPolicy
|
||||
from EvoScientist.model_registry.http_api import (
|
||||
ApiServices,
|
||||
model_registry_routes,
|
||||
)
|
||||
from EvoScientist.model_registry.platform import DelegationPublicKey
|
||||
from EvoScientist.model_registry.resolver import ModelRegistryResolver
|
||||
from EvoScientist.model_registry.snapshots import SnapshotService
|
||||
from EvoScientist.model_registry.store import ModelRuntimeStore
|
||||
|
||||
SERVICE_TOKEN = "bff-service-token"
|
||||
|
||||
_PRIVATE_KEY = ec.generate_private_key(ec.SECP256R1())
|
||||
_PRIVATE_PEM = _PRIVATE_KEY.private_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PrivateFormat.PKCS8,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
_PUBLIC_PEM = _PRIVATE_KEY.public_key().public_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PublicFormat.SubjectPublicKeyInfo,
|
||||
)
|
||||
|
||||
_OTHER_KEY = ec.generate_private_key(ec.SECP256R1())
|
||||
_OTHER_PRIVATE_PEM = _OTHER_KEY.private_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PrivateFormat.PKCS8,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
|
||||
|
||||
def _services(store: ModelRuntimeStore, **auth_overrides) -> ApiServices:
|
||||
resolver = ModelRegistryResolver(store)
|
||||
auth_kwargs = {
|
||||
"service_token": SERVICE_TOKEN,
|
||||
"service_token_hash": None,
|
||||
"delegation_keys": (
|
||||
DelegationPublicKey(deployment_id="webui-1", public_key=_PUBLIC_PEM),
|
||||
),
|
||||
"jti_store": store,
|
||||
}
|
||||
auth_kwargs.update(auth_overrides)
|
||||
return ApiServices(
|
||||
store=store,
|
||||
resolver=resolver,
|
||||
snapshot_service=SnapshotService(store, resolver),
|
||||
endpoint_policy=EndpointPolicy(),
|
||||
authenticator=BffAuthenticator(**auth_kwargs),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
return ModelRuntimeStore(config_dir=tmp_path)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(store):
|
||||
services = _services(store)
|
||||
app = Starlette(routes=model_registry_routes(lambda: services))
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _claims(**overrides):
|
||||
now = int(time.time())
|
||||
claims = {
|
||||
"iss": "WebUI",
|
||||
"aud": "EvoScientist",
|
||||
"sub": "admin-1",
|
||||
"scopes": ["model_config:read"],
|
||||
"deployment_id": "webui-1",
|
||||
"iat": now,
|
||||
"exp": now + 30,
|
||||
"jti": uuid.uuid4().hex,
|
||||
}
|
||||
claims.update(overrides)
|
||||
return claims
|
||||
|
||||
|
||||
def _encode(claims, pem=_PRIVATE_PEM):
|
||||
return jwt.encode(claims, pem, algorithm="ES256")
|
||||
|
||||
|
||||
def _headers(claims=None, *, token=SERVICE_TOKEN, pem=_PRIVATE_PEM):
|
||||
headers = {}
|
||||
if token is not None:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
if claims is not None:
|
||||
headers["X-Evo-Actor"] = _encode(claims, pem)
|
||||
return headers
|
||||
|
||||
|
||||
def test_valid_token_and_delegation_pass(client):
|
||||
response = client.get("/api/model-registry", headers=_headers(_claims()))
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
def test_config_route_does_not_require_thread_claim(client):
|
||||
claims = _claims()
|
||||
assert "thread_id" not in claims
|
||||
response = client.get("/api/model-registry", headers=_headers(claims))
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
def test_missing_authorization_header_is_401(client):
|
||||
response = client.get("/api/model-registry", headers={"X-Evo-Actor": "x"})
|
||||
assert response.status_code == 401
|
||||
body = response.json()
|
||||
assert body["code"] == "UNAUTHENTICATED"
|
||||
assert set(body) == {"code", "message", "details", "request_id"}
|
||||
assert body["request_id"]
|
||||
|
||||
|
||||
def test_wrong_service_token_is_401(client):
|
||||
response = client.get(
|
||||
"/api/model-registry",
|
||||
headers=_headers(_claims(), token="not-the-token"),
|
||||
)
|
||||
assert response.status_code == 401
|
||||
assert response.json()["code"] == "UNAUTHENTICATED"
|
||||
|
||||
|
||||
def test_service_token_hash_configuration_passes(store):
|
||||
services = _services(
|
||||
store,
|
||||
service_token=None,
|
||||
service_token_hash=hashlib.sha256(SERVICE_TOKEN.encode()).hexdigest(),
|
||||
)
|
||||
app = Starlette(routes=model_registry_routes(lambda: services))
|
||||
hashed_client = TestClient(app)
|
||||
assert (
|
||||
hashed_client.get(
|
||||
"/api/model-registry", headers=_headers(_claims())
|
||||
).status_code
|
||||
== 200
|
||||
)
|
||||
assert (
|
||||
hashed_client.get(
|
||||
"/api/model-registry", headers=_headers(_claims(), token="wrong")
|
||||
).status_code
|
||||
== 401
|
||||
)
|
||||
|
||||
|
||||
def test_missing_delegation_jwt_is_401(client):
|
||||
response = client.get(
|
||||
"/api/model-registry",
|
||||
headers={"Authorization": f"Bearer {SERVICE_TOKEN}"},
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_malformed_delegation_jwt_is_401(client):
|
||||
response = client.get(
|
||||
"/api/model-registry",
|
||||
headers=_headers(None) | {"X-Evo-Actor": "not-a-jwt"},
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"missing", ["sub", "scopes", "deployment_id", "iat", "exp", "jti"]
|
||||
)
|
||||
def test_missing_required_claim_is_401(client, missing):
|
||||
claims = _claims()
|
||||
del claims[missing]
|
||||
response = client.get("/api/model-registry", headers=_headers(claims))
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_expired_delegation_jwt_is_401(client):
|
||||
now = int(time.time())
|
||||
claims = _claims(iat=now - 90, exp=now - 60)
|
||||
response = client.get("/api/model-registry", headers=_headers(claims))
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_lifetime_over_60_seconds_is_401(client):
|
||||
now = int(time.time())
|
||||
claims = _claims(iat=now, exp=now + 120)
|
||||
response = client.get("/api/model-registry", headers=_headers(claims))
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_wrong_signature_is_401(client):
|
||||
response = client.get(
|
||||
"/api/model-registry",
|
||||
headers=_headers(_claims(), pem=_OTHER_PRIVATE_PEM),
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_wrong_issuer_is_401(client):
|
||||
response = client.get(
|
||||
"/api/model-registry", headers=_headers(_claims(iss="not-webui"))
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_wrong_audience_is_401(client):
|
||||
response = client.get(
|
||||
"/api/model-registry", headers=_headers(_claims(aud="someone-else"))
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_unregistered_deployment_is_401(client):
|
||||
response = client.get(
|
||||
"/api/model-registry", headers=_headers(_claims(deployment_id="webui-x"))
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_insufficient_scope_is_403(client):
|
||||
claims = _claims(scopes=["model:select"])
|
||||
response = client.get("/api/model-registry", headers=_headers(claims))
|
||||
assert response.status_code == 403
|
||||
body = response.json()
|
||||
assert body["code"] == "FORBIDDEN"
|
||||
assert set(body) == {"code", "message", "details", "request_id"}
|
||||
|
||||
|
||||
def test_jti_replay_is_401_delegation_replayed(client):
|
||||
headers = _headers(_claims())
|
||||
first = client.get("/api/model-registry", headers=headers)
|
||||
assert first.status_code == 200
|
||||
second = client.get("/api/model-registry", headers=headers)
|
||||
assert second.status_code == 401
|
||||
assert second.json()["code"] == "DELEGATION_REPLAYED"
|
||||
|
||||
|
||||
def test_invalid_scopes_claim_is_401(client):
|
||||
response = client.get(
|
||||
"/api/model-registry", headers=_headers(_claims(scopes="model_config:read"))
|
||||
)
|
||||
assert response.status_code == 401
|
||||
@@ -1,6 +1,11 @@
|
||||
"""Smoke test for the /api/models route mounted via langgraph.json's
|
||||
``http`` field. We test the FastAPI app directly — no need to spin up
|
||||
"""Smoke tests for the legacy admin routes mounted via langgraph.json's
|
||||
``http`` field. We test the Starlette app directly — no need to spin up
|
||||
langgraph dev.
|
||||
|
||||
``GET /api/models`` and ``POST /api/runtime-snapshots`` now belong to the
|
||||
unified model registry API (``EvoScientist.model_registry.http_api``); their
|
||||
contract tests live in ``tests/test_model_registry_http.py`` and
|
||||
``tests/test_delegation_auth.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -14,7 +19,6 @@ from starlette.testclient import TestClient
|
||||
from EvoScientist.config import EvoScientistConfig, load_config, save_config
|
||||
from EvoScientist.config.provider_profiles import (
|
||||
load_provider_profiles,
|
||||
replace_provider_profiles,
|
||||
)
|
||||
from EvoScientist.langgraph_dev.http import app
|
||||
from EvoScientist.llm.provider_operations import (
|
||||
@@ -32,78 +36,6 @@ def isolate_xdg_config(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
||||
|
||||
|
||||
def test_get_models_returns_entries_and_default():
|
||||
mock_cfg = EvoScientistConfig(
|
||||
model="claude-sonnet-4-6", provider="custom-anthropic"
|
||||
)
|
||||
with patch(
|
||||
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
||||
):
|
||||
resp = client.get("/api/models")
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert "entries" in body
|
||||
assert "default" in body
|
||||
assert body["default"] == {
|
||||
"name": "claude-sonnet-4-6",
|
||||
"provider": "custom-anthropic",
|
||||
}
|
||||
assert isinstance(body["entries"], list)
|
||||
assert len(body["entries"]) > 0
|
||||
# Every entry has the three required keys
|
||||
for entry in body["entries"]:
|
||||
assert set(entry.keys()) == {"name", "model_id", "provider"}
|
||||
assert isinstance(entry["name"], str)
|
||||
assert entry["name"]
|
||||
assert isinstance(entry["model_id"], str)
|
||||
assert entry["model_id"]
|
||||
assert isinstance(entry["provider"], str)
|
||||
assert entry["provider"]
|
||||
|
||||
|
||||
def test_get_models_only_returns_enabled_catalog_entries_when_configured():
|
||||
mock_cfg = EvoScientistConfig(
|
||||
model="chat-main",
|
||||
provider="openai",
|
||||
model_catalog=[
|
||||
{
|
||||
"provider": "openai",
|
||||
"id": "chat-main",
|
||||
"name": "Chat Main",
|
||||
"model_id": "gpt-upstream",
|
||||
"enabled": True,
|
||||
},
|
||||
{
|
||||
"provider": "anthropic",
|
||||
"id": "hidden-model",
|
||||
"name": "Hidden",
|
||||
"model_id": "claude-hidden",
|
||||
"enabled": False,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
with patch(
|
||||
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
||||
):
|
||||
body = client.get("/api/models").json()
|
||||
|
||||
assert body["entries"] == [
|
||||
{"name": "chat-main", "model_id": "gpt-upstream", "provider": "openai"}
|
||||
]
|
||||
|
||||
|
||||
def test_get_models_empty_catalog_hides_all_builtin_models():
|
||||
mock_cfg = EvoScientistConfig(model_catalog=[])
|
||||
|
||||
with patch(
|
||||
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
||||
):
|
||||
body = client.get("/api/models").json()
|
||||
|
||||
assert body["entries"] == []
|
||||
|
||||
|
||||
def test_provider_profiles_api_requires_admin_token_header(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
||||
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
|
||||
@@ -156,41 +88,6 @@ def test_provider_profiles_api_round_trip_redacts_secret(tmp_path, monkeypatch):
|
||||
assert get_response.json()["providers"][0]["models"][0]["id"] == "lab-model"
|
||||
assert "provider-secret" not in get_response.text
|
||||
|
||||
snapshot_response = client.post(
|
||||
"/api/runtime-snapshots",
|
||||
headers=headers,
|
||||
json={
|
||||
"snapshot_id": "run-request-a",
|
||||
"model": "lab-model",
|
||||
"provider": "lab-openai",
|
||||
},
|
||||
)
|
||||
assert snapshot_response.status_code == 200
|
||||
snapshot = snapshot_response.json()["snapshot"]
|
||||
assert snapshot["runtime"]["max_input_tokens"] == 28_672
|
||||
assert "provider-secret" not in snapshot_response.text
|
||||
|
||||
|
||||
def test_runtime_snapshot_reports_missing_zhipu_key_before_creating_a_run(
|
||||
monkeypatch,
|
||||
):
|
||||
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
||||
monkeypatch.delenv("ZHIPU_API_KEY", raising=False)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key")
|
||||
|
||||
response = client.post(
|
||||
"/api/runtime-snapshots",
|
||||
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
|
||||
json={
|
||||
"snapshot_id": "run-request-zhipu",
|
||||
"model": "glm-5.2",
|
||||
"provider": "glm",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"].startswith("ZHIPU_API_KEY_NOT_CONFIGURED")
|
||||
|
||||
|
||||
def test_llm_config_api_requires_admin_token_header(monkeypatch):
|
||||
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
|
||||
@@ -300,11 +197,6 @@ def test_llm_config_api_saves_builtin_registry_without_rewriting_legacy_secret(
|
||||
assert saved_config.default_workdir == "/tmp/research"
|
||||
assert saved_config.model_catalog is None
|
||||
|
||||
models = client.get("/api/models").json()["entries"]
|
||||
assert models == [
|
||||
{"name": "chat-main", "model_id": "gpt-upstream", "provider": "openai"}
|
||||
]
|
||||
|
||||
|
||||
def test_first_builtin_registry_save_migrates_legacy_default_model(monkeypatch):
|
||||
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
||||
@@ -794,158 +686,6 @@ def test_default_model_api_rejects_invalid_json(monkeypatch):
|
||||
assert response.json() == {"error": "Request body must be valid JSON."}
|
||||
|
||||
|
||||
def test_entries_preserve_registry_order():
|
||||
"""The picker uses position-in-list to rank providers per short name —
|
||||
the JSON must preserve the order returned by ``list_models_by_provider``.
|
||||
|
||||
Stubs ``get_effective_config`` to keep the assertion focused on
|
||||
registry order rather than implicitly depending on the ambient
|
||||
deploy config.
|
||||
"""
|
||||
from EvoScientist.llm.models import list_models_by_provider
|
||||
|
||||
expected = [
|
||||
{"name": n, "model_id": m, "provider": p}
|
||||
for n, m, p in list_models_by_provider()
|
||||
]
|
||||
mock_cfg = EvoScientistConfig()
|
||||
with patch(
|
||||
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
||||
):
|
||||
resp = client.get("/api/models")
|
||||
assert resp.json()["entries"] == expected
|
||||
|
||||
|
||||
def test_unavailable_default_falls_back_to_first_picker_entry():
|
||||
mock_cfg = EvoScientistConfig(model="some-retired-name", provider="some-provider")
|
||||
with patch(
|
||||
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
||||
):
|
||||
resp = client.get("/api/models")
|
||||
first = resp.json()["entries"][0]
|
||||
assert resp.json()["default"] == {
|
||||
"name": first["name"],
|
||||
"provider": first["provider"],
|
||||
}
|
||||
|
||||
|
||||
def test_custom_registry_default_does_not_reuse_stale_builtin_provider():
|
||||
replace_provider_profiles(
|
||||
{
|
||||
"providers": [
|
||||
{
|
||||
"id": "open",
|
||||
"name": "Open proxy",
|
||||
"adapter": "openai",
|
||||
"base_url": "https://proxy.example.test/v1",
|
||||
"api_key": "provider-secret",
|
||||
"enabled": True,
|
||||
"models": [
|
||||
{
|
||||
"id": "gpt-5.5",
|
||||
"name": "GPT 5.5",
|
||||
"model_id": "gpt-5.5",
|
||||
"enabled": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
mock_cfg = EvoScientistConfig(model="gpt-5.5", provider="openai")
|
||||
|
||||
with patch(
|
||||
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
||||
):
|
||||
body = client.get("/api/models").json()
|
||||
|
||||
assert body["entries"] == [
|
||||
{"name": "gpt-5.5", "model_id": "gpt-5.5", "provider": "open"}
|
||||
]
|
||||
assert body["default"] == {"name": "gpt-5.5", "provider": "open"}
|
||||
|
||||
|
||||
def test_ollama_models_appended_when_base_url_configured():
|
||||
"""Mirrors the TUI ``/model`` picker: when ``ollama_base_url`` is set,
|
||||
locally-pulled Ollama models are appended after the static registry
|
||||
as ``provider: "ollama"`` entries.
|
||||
"""
|
||||
mock_cfg = EvoScientistConfig(
|
||||
model="claude-sonnet-4-6",
|
||||
provider="custom-anthropic",
|
||||
ollama_base_url="http://localhost:11434",
|
||||
)
|
||||
|
||||
async def fake_discover(_base_url, *, timeout):
|
||||
return ["llama3:8b", "mistral:7b"]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"EvoScientist.langgraph_dev.http.get_effective_config",
|
||||
return_value=mock_cfg,
|
||||
),
|
||||
patch(
|
||||
"EvoScientist.llm.ollama_discovery.discover_ollama_models",
|
||||
new=fake_discover,
|
||||
),
|
||||
):
|
||||
body = client.get("/api/models").json()
|
||||
|
||||
# Assert the response is the static registry followed by the discovered
|
||||
# Ollama suffix — robust to future static Ollama entries in the registry.
|
||||
from EvoScientist.llm.models import list_models_by_provider
|
||||
|
||||
static_entries = [
|
||||
{"name": n, "model_id": m, "provider": p}
|
||||
for n, m, p in list_models_by_provider()
|
||||
]
|
||||
discovered_entries = [
|
||||
{"name": "llama3:8b", "model_id": "llama3:8b", "provider": "ollama"},
|
||||
{"name": "mistral:7b", "model_id": "mistral:7b", "provider": "ollama"},
|
||||
]
|
||||
assert body["entries"][: len(static_entries)] == static_entries
|
||||
assert body["entries"][len(static_entries) :] == discovered_entries
|
||||
# TUI's "Custom Ollama model…" sentinel is a widget-specific affordance —
|
||||
# it must not appear on the HTTP surface.
|
||||
assert not any(e["model_id"] == "__custom_ollama__" for e in body["entries"])
|
||||
|
||||
|
||||
def test_ollama_discovery_skipped_when_base_url_absent():
|
||||
"""No Ollama discovery should happen when ``ollama_base_url`` is unset —
|
||||
matches the ``/model`` picker's gating. The probe function should never
|
||||
be called in that case.
|
||||
"""
|
||||
mock_cfg = EvoScientistConfig(
|
||||
model="claude-sonnet-4-6", provider="custom-anthropic"
|
||||
)
|
||||
calls: list[str | None] = []
|
||||
|
||||
async def spy_discover(base_url, *, timeout):
|
||||
calls.append(base_url)
|
||||
return []
|
||||
|
||||
with (
|
||||
patch(
|
||||
"EvoScientist.langgraph_dev.http.get_effective_config",
|
||||
return_value=mock_cfg,
|
||||
),
|
||||
patch(
|
||||
"EvoScientist.llm.ollama_discovery.discover_ollama_models",
|
||||
new=spy_discover,
|
||||
),
|
||||
):
|
||||
body = client.get("/api/models").json()
|
||||
|
||||
assert calls == []
|
||||
# Response is exactly the static registry — no Ollama additions whatsoever.
|
||||
from EvoScientist.llm.models import list_models_by_provider
|
||||
|
||||
assert body["entries"] == [
|
||||
{"name": n, "model_id": m, "provider": p}
|
||||
for n, m, p in list_models_by_provider()
|
||||
]
|
||||
|
||||
|
||||
def test_final_answer_extracts_latest_ai_text_blocks():
|
||||
async def fake_metadata(_thread_id):
|
||||
return {"updated_at": "2026-07-06T14:14:53+00:00"}
|
||||
|
||||
@@ -0,0 +1,911 @@
|
||||
"""HTTP API contract tests for the model registry (design doc 9.1-9.3, 9.5).
|
||||
|
||||
Covers the Config API (GET/PUT registry, credential rotation, selector),
|
||||
the eight section 9.2 atomic save-time checks, snapshot creation/binding/
|
||||
deletion state machines, the unified error payload, and the OpenAPI export.
|
||||
Authentication itself is covered exhaustively in test_delegation_auth.py;
|
||||
these tests mint valid delegations per request.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import jwt
|
||||
import pytest
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from starlette.applications import Starlette
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from EvoScientist.model_registry.adapters import find_adapter_spec
|
||||
from EvoScientist.model_registry.auth import BffAuthenticator
|
||||
from EvoScientist.model_registry.endpoint_policy import EndpointPolicy
|
||||
from EvoScientist.model_registry.hashing import configuration_hash
|
||||
from EvoScientist.model_registry.http_api import (
|
||||
ApiServices,
|
||||
build_openapi_document,
|
||||
model_registry_routes,
|
||||
)
|
||||
from EvoScientist.model_registry.platform import (
|
||||
DelegationPublicKey,
|
||||
PlatformConfigError,
|
||||
)
|
||||
from EvoScientist.model_registry.resolver import ModelRegistryResolver
|
||||
from EvoScientist.model_registry.schemas import (
|
||||
CredentialWrite,
|
||||
DevelopmentEndpoint,
|
||||
RegistryV4,
|
||||
)
|
||||
from EvoScientist.model_registry.snapshots import SnapshotService
|
||||
from EvoScientist.model_registry.store import ModelRuntimeStore
|
||||
|
||||
SERVICE_TOKEN = "bff-service-token"
|
||||
SECRET = "sk-live-9876abcd"
|
||||
|
||||
_PRIVATE_KEY = ec.generate_private_key(ec.SECP256R1())
|
||||
_PRIVATE_PEM = _PRIVATE_KEY.private_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PrivateFormat.PKCS8,
|
||||
serialization.NoEncryption(),
|
||||
)
|
||||
_PUBLIC_PEM = _PRIVATE_KEY.public_key().public_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
serialization.PublicFormat.SubjectPublicKeyInfo,
|
||||
)
|
||||
|
||||
OPENAPI_PATH = (
|
||||
Path(__file__).resolve().parent.parent
|
||||
/ "EvoScientist"
|
||||
/ "model_registry"
|
||||
/ "openapi.json"
|
||||
)
|
||||
|
||||
|
||||
# --- registry fixtures (mirrors test_snapshots.py) ---------------------------
|
||||
|
||||
|
||||
def _model_runtime(**overrides):
|
||||
payload = {
|
||||
"limit_mode": "combined",
|
||||
"context_window_tokens": 1048576,
|
||||
"max_input_tokens": None,
|
||||
"max_output_tokens": 32768,
|
||||
"min_effective_input_tokens": 8192,
|
||||
"fixed_system_reserve_tokens": 4096,
|
||||
"fixed_tools_reserve_tokens": 8192,
|
||||
"fixed_attachments_reserve_tokens": 4096,
|
||||
"limits_status": "confirmed",
|
||||
"limits_source": "provider",
|
||||
"temperature": None,
|
||||
"top_p": None,
|
||||
"reasoning_effort": "auto",
|
||||
"declared_capabilities": {
|
||||
"tools": True,
|
||||
"vision": False,
|
||||
"structured_output": True,
|
||||
},
|
||||
}
|
||||
payload.update(overrides)
|
||||
return payload
|
||||
|
||||
|
||||
def _zhipu_provider(**overrides):
|
||||
provider = {
|
||||
"id": "zhipu-glm",
|
||||
"name": "Zhipu GLM",
|
||||
"adapter": "openai-compatible",
|
||||
"base_url": "https://open.bigmodel.cn/api/paas/v4",
|
||||
"auth": {"mode": "api_key", "credential_id": "zhipu-primary"},
|
||||
"enabled": True,
|
||||
"runtime": {
|
||||
"timeout_seconds": 120,
|
||||
"max_retries": 2,
|
||||
"default_temperature": 0.7,
|
||||
"default_top_p": 0.95,
|
||||
"default_reasoning_effort": "auto",
|
||||
},
|
||||
"models": [
|
||||
{
|
||||
"key": "glm-5.2",
|
||||
"name": "GLM-5.2",
|
||||
"upstream_model_id": "glm-5.2",
|
||||
"enabled": True,
|
||||
"runtime": _model_runtime(),
|
||||
}
|
||||
],
|
||||
}
|
||||
provider.update(overrides)
|
||||
return provider
|
||||
|
||||
|
||||
def _ollama_provider(**overrides):
|
||||
provider = {
|
||||
"id": "local-ollama",
|
||||
"name": "Local Ollama",
|
||||
"adapter": "ollama",
|
||||
"base_url": "http://localhost:11434",
|
||||
"auth": {"mode": "none", "credential_id": None},
|
||||
"enabled": True,
|
||||
"runtime": {"timeout_seconds": 120, "max_retries": 2},
|
||||
"models": [
|
||||
{
|
||||
"key": "qwen3",
|
||||
"name": "Qwen3",
|
||||
"upstream_model_id": "qwen3",
|
||||
"enabled": True,
|
||||
"runtime": _model_runtime(),
|
||||
}
|
||||
],
|
||||
}
|
||||
provider.update(overrides)
|
||||
return provider
|
||||
|
||||
|
||||
def _registry_payload():
|
||||
return {
|
||||
"version": 4,
|
||||
"revision": 1,
|
||||
"state": "bootstrap",
|
||||
"defaults": {
|
||||
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
|
||||
"auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"},
|
||||
},
|
||||
"providers": [_zhipu_provider(), _ollama_provider()],
|
||||
}
|
||||
|
||||
|
||||
def _verify(store, registry, provider_id, model_key):
|
||||
provider = registry.find_provider(provider_id)
|
||||
model = provider.find_model(model_key)
|
||||
spec = find_adapter_spec(provider.adapter, model.upstream_model_id)
|
||||
store.record_model_verification(
|
||||
provider_id=provider_id,
|
||||
model_key=model_key,
|
||||
configuration_hash=configuration_hash(provider, model),
|
||||
credential_revision=1 if provider.auth.credential_id else 0,
|
||||
adapter_spec_revision=spec.spec_revision,
|
||||
result="passed",
|
||||
verified_capabilities={
|
||||
"tools": True,
|
||||
"vision": False,
|
||||
"structured_output": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# --- app fixtures --------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
return ModelRuntimeStore(config_dir=tmp_path)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def services(store):
|
||||
resolver = ModelRegistryResolver(store)
|
||||
return ApiServices(
|
||||
store=store,
|
||||
resolver=resolver,
|
||||
snapshot_service=SnapshotService(store, resolver),
|
||||
endpoint_policy=EndpointPolicy(
|
||||
[
|
||||
DevelopmentEndpoint(
|
||||
id="local-ollama",
|
||||
url="http://localhost:11434",
|
||||
label="Local Ollama",
|
||||
)
|
||||
]
|
||||
),
|
||||
authenticator=BffAuthenticator(
|
||||
service_token=SERVICE_TOKEN,
|
||||
service_token_hash=None,
|
||||
delegation_keys=(
|
||||
DelegationPublicKey(deployment_id="webui-1", public_key=_PUBLIC_PEM),
|
||||
),
|
||||
jti_store=store,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(services):
|
||||
app = Starlette(routes=model_registry_routes(lambda: services))
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def active_store(store):
|
||||
registry = store.save_registry(
|
||||
expected_revision=1,
|
||||
registry=RegistryV4.model_validate(_registry_payload()),
|
||||
credential_writes=[
|
||||
CredentialWrite(credential_id="zhipu-primary", secret_value=SECRET)
|
||||
],
|
||||
)
|
||||
assert registry.state == "active"
|
||||
_verify(store, registry, "zhipu-glm", "glm-5.2")
|
||||
_verify(store, registry, "local-ollama", "qwen3")
|
||||
return store
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def active_client(client, active_store):
|
||||
return client
|
||||
|
||||
|
||||
# --- auth helpers --------------------------------------------------------------
|
||||
|
||||
|
||||
def _headers(scopes, *, thread_id=None):
|
||||
now = int(time.time())
|
||||
claims = {
|
||||
"iss": "WebUI",
|
||||
"aud": "EvoScientist",
|
||||
"sub": "user-1",
|
||||
"scopes": list(scopes),
|
||||
"deployment_id": "webui-1",
|
||||
"iat": now,
|
||||
"exp": now + 30,
|
||||
"jti": uuid.uuid4().hex,
|
||||
}
|
||||
if thread_id is not None:
|
||||
claims["thread_id"] = thread_id
|
||||
return {
|
||||
"Authorization": f"Bearer {SERVICE_TOKEN}",
|
||||
"X-Evo-Actor": jwt.encode(claims, _PRIVATE_PEM, algorithm="ES256"),
|
||||
}
|
||||
|
||||
|
||||
def _admin_headers(**kwargs):
|
||||
return _headers(["model_config:read", "model_config:write"], **kwargs)
|
||||
|
||||
|
||||
def _run_headers(thread_id="thread-1"):
|
||||
return _headers(["run:create"], thread_id=thread_id)
|
||||
|
||||
|
||||
# --- GET /api/model-registry ----------------------------------------------------
|
||||
|
||||
|
||||
def test_get_registry_response_structure(active_client):
|
||||
response = active_client.get("/api/model-registry", headers=_admin_headers())
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert set(body) == {
|
||||
"revision",
|
||||
"registry",
|
||||
"adapter_specs",
|
||||
"credential_status",
|
||||
"model_status",
|
||||
"endpoint_policy",
|
||||
}
|
||||
assert body["revision"] == 2
|
||||
assert body["registry"]["state"] == "active"
|
||||
assert len(body["adapter_specs"]) == 6
|
||||
status = body["credential_status"]
|
||||
assert [entry["credential_id"] for entry in status] == ["zhipu-primary"]
|
||||
assert status[0]["configured"] is True
|
||||
assert status[0]["hint"] == "...abcd"
|
||||
assert status[0]["updated_at"]
|
||||
assert SECRET not in response.text
|
||||
states = {item["model_ref"]["model_key"]: item for item in body["model_status"]}
|
||||
assert states["glm-5.2"]["state"] == "enabled"
|
||||
assert states["glm-5.2"]["selectable"] is True
|
||||
assert states["qwen3"]["state"] == "enabled"
|
||||
policy = body["endpoint_policy"]
|
||||
assert policy["public_https_allowed"] is True
|
||||
assert policy["development_endpoints"] == [
|
||||
{"id": "local-ollama", "url": "http://localhost:11434", "label": "Local Ollama"}
|
||||
]
|
||||
|
||||
|
||||
def test_get_registry_marks_model_stale_after_credential_rotation(active_client):
|
||||
rotate = active_client.put(
|
||||
"/api/model-registry/credentials/zhipu-primary",
|
||||
headers=_admin_headers(),
|
||||
json={"operation": "replace", "secret_value": "rotated-secret-9999"},
|
||||
)
|
||||
assert rotate.status_code == 200
|
||||
|
||||
body = active_client.get("/api/model-registry", headers=_admin_headers()).json()
|
||||
states = {item["model_ref"]["model_key"]: item for item in body["model_status"]}
|
||||
# The full credential_revisions mapping must reach compute_availability:
|
||||
# after rotation the glm model compares against revision 2 and goes stale.
|
||||
assert states["glm-5.2"]["state"] == "verification_stale"
|
||||
assert states["glm-5.2"]["selectable"] is False
|
||||
assert states["glm-5.2"]["verification"]["status"] == "stale"
|
||||
# mode=none models are unaffected by credential rotation.
|
||||
assert states["qwen3"]["state"] == "enabled"
|
||||
|
||||
|
||||
# --- PUT /api/model-registry -----------------------------------------------------
|
||||
|
||||
|
||||
def _current_registry(active_client):
|
||||
return active_client.get("/api/model-registry", headers=_admin_headers()).json()
|
||||
|
||||
|
||||
def _put(active_client, registry, *, expected_revision=2, credential_writes=None):
|
||||
payload = {
|
||||
"expected_revision": expected_revision,
|
||||
"registry": registry,
|
||||
"credential_writes": credential_writes or [],
|
||||
}
|
||||
return active_client.put(
|
||||
"/api/model-registry", headers=_admin_headers(), json=payload
|
||||
)
|
||||
|
||||
|
||||
def test_put_registry_success_returns_full_response(active_client):
|
||||
current = _current_registry(active_client)
|
||||
registry = current["registry"]
|
||||
registry["providers"][0]["models"][0]["name"] = "GLM-5.2 Turbo"
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["revision"] == current["revision"] + 1
|
||||
assert body["registry"]["providers"][0]["models"][0]["name"] == "GLM-5.2 Turbo"
|
||||
states = {item["model_ref"]["model_key"]: item for item in body["model_status"]}
|
||||
# The model name is not part of the configuration hash: still enabled.
|
||||
assert states["glm-5.2"]["state"] == "enabled"
|
||||
|
||||
|
||||
def test_put_registry_revision_conflict(active_client):
|
||||
current = _current_registry(active_client)
|
||||
response = _put(active_client, current["registry"], expected_revision=99)
|
||||
assert response.status_code == 409
|
||||
body = response.json()
|
||||
assert body["code"] == "REGISTRY_REVISION_CONFLICT"
|
||||
assert set(body) == {"code", "message", "details", "request_id"}
|
||||
|
||||
|
||||
def test_put_registry_rejects_ssrf_endpoint(active_client):
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
registry["providers"][0]["base_url"] = "http://127.0.0.1:9000/v1"
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
body = response.json()
|
||||
assert body["code"] == "ENDPOINT_NOT_ALLOWED"
|
||||
assert body["details"][0]["path"].startswith("providers[0].base_url")
|
||||
|
||||
|
||||
def test_put_registry_rejects_unsupported_parameter(active_client):
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
# The glm-5.2 contract caps temperature at 1.
|
||||
registry["providers"][0]["models"][0]["runtime"]["temperature"] = 1.5
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
body = response.json()
|
||||
assert body["code"] == "UNSUPPORTED_RUNTIME_PARAMETER"
|
||||
assert "providers[0].models[0]" in body["details"][0]["path"]
|
||||
|
||||
|
||||
def test_put_registry_rejects_unconfirmed_limits(active_client):
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
registry["providers"][0]["models"][0]["runtime"]["limits_status"] = (
|
||||
"needs_confirmation"
|
||||
)
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["code"] == "MODEL_LIMITS_UNCONFIRMED"
|
||||
|
||||
|
||||
def test_put_registry_rejects_enabled_model_without_verification(active_client):
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
registry["providers"][0]["models"].append(
|
||||
{
|
||||
"key": "glm-5.3",
|
||||
"name": "GLM-5.3",
|
||||
"upstream_model_id": "glm-5.3",
|
||||
"enabled": True,
|
||||
"runtime": _model_runtime(),
|
||||
}
|
||||
)
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["code"] == "MODEL_NOT_AVAILABLE"
|
||||
|
||||
|
||||
def test_put_registry_rejects_unsatisfiable_budget(active_client):
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
# A disabled draft model still must satisfy the four-mode budget rule.
|
||||
registry["providers"][0]["models"].append(
|
||||
{
|
||||
"key": "glm-draft",
|
||||
"name": "GLM Draft",
|
||||
"upstream_model_id": "glm-draft",
|
||||
"enabled": False,
|
||||
"runtime": _model_runtime(min_effective_input_tokens=1048576),
|
||||
}
|
||||
)
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["code"] == "CONTEXT_BUDGET_UNSATISFIABLE"
|
||||
|
||||
|
||||
def test_put_registry_rejects_unsupported_auth_mode(active_client):
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
registry["providers"][1]["auth"] = {"mode": "api_key", "credential_id": "some-key"}
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["code"] == "AUTH_MODE_UNSUPPORTED"
|
||||
|
||||
|
||||
def test_put_registry_rejects_defaults_to_disabled_model(active_client):
|
||||
# With a bootstrap-state payload the schema-level check does not fire, so
|
||||
# the store's rule-7 guard (enforced on every save) is what rejects it.
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
registry["state"] = "bootstrap"
|
||||
registry["providers"][1]["models"][0]["enabled"] = False
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["code"] == "MODEL_DISABLED"
|
||||
|
||||
|
||||
def test_put_registry_active_state_validates_defaults_in_schema(active_client):
|
||||
# The same rule fires earlier (Pydantic) when the payload claims active.
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
registry["providers"][1]["models"][0]["enabled"] = False
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["code"] == "VALIDATION_FAILED"
|
||||
|
||||
|
||||
def test_put_registry_atomic_rollback_with_credential_writes(active_client, store):
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
registry["providers"][0]["base_url"] = "http://127.0.0.1:9000/v1"
|
||||
|
||||
response = _put(
|
||||
active_client,
|
||||
registry,
|
||||
credential_writes=[
|
||||
{
|
||||
"credential_id": "zhipu-primary",
|
||||
"operation": "replace",
|
||||
"secret_value": "should-not-persist",
|
||||
}
|
||||
],
|
||||
)
|
||||
assert response.status_code == 422
|
||||
# Neither the credential write nor the registry update may survive.
|
||||
assert store.current_credential_revision("zhipu-primary") == 1
|
||||
assert store.load_registry().revision == 2
|
||||
|
||||
|
||||
def test_put_registry_schema_validation_details(active_client):
|
||||
registry = _current_registry(active_client)["registry"]
|
||||
registry["providers"][0]["id"] = "INVALID ID!"
|
||||
|
||||
response = _put(active_client, registry)
|
||||
assert response.status_code == 422
|
||||
body = response.json()
|
||||
assert body["code"] == "VALIDATION_FAILED"
|
||||
assert body["details"]
|
||||
assert all("path" in detail and "code" in detail for detail in body["details"])
|
||||
|
||||
|
||||
def test_put_registry_bootstrap_flow_configure_test_enable(client, store):
|
||||
# Step 1: save a draft registry — models disabled, defaults null, and the
|
||||
# credential written in the same transaction (section 10 steps 5-7).
|
||||
draft = _registry_payload()
|
||||
draft["defaults"] = {"primary": None, "auxiliary": None}
|
||||
for provider in draft["providers"]:
|
||||
for model in provider["models"]:
|
||||
model["enabled"] = False
|
||||
response = client.put(
|
||||
"/api/model-registry",
|
||||
headers=_admin_headers(),
|
||||
json={
|
||||
"expected_revision": 1,
|
||||
"registry": draft,
|
||||
"credential_writes": [
|
||||
{
|
||||
"credential_id": "zhipu-primary",
|
||||
"operation": "replace",
|
||||
"secret_value": "bootstrap-secret",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["registry"]["state"] == "bootstrap"
|
||||
assert body["revision"] == 2
|
||||
assert "bootstrap-secret" not in response.text
|
||||
states = {item["model_ref"]["model_key"]: item for item in body["model_status"]}
|
||||
assert states["glm-5.2"]["state"] == "configured"
|
||||
|
||||
# Step 2: provider tests pass (Task 5b writes these records via the test
|
||||
# endpoint; here they are seeded directly).
|
||||
registry = store.load_registry()
|
||||
_verify(store, registry, "zhipu-glm", "glm-5.2")
|
||||
_verify(store, registry, "local-ollama", "qwen3")
|
||||
|
||||
# Step 3: enabling the verified models with defaults turns the registry
|
||||
# active in the same commit.
|
||||
enabled = body["registry"]
|
||||
enabled["defaults"] = {
|
||||
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
|
||||
"auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"},
|
||||
}
|
||||
for provider in enabled["providers"]:
|
||||
for model in provider["models"]:
|
||||
model["enabled"] = True
|
||||
response = client.put(
|
||||
"/api/model-registry",
|
||||
headers=_admin_headers(),
|
||||
json={"expected_revision": 2, "registry": enabled, "credential_writes": []},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["registry"]["state"] == "active"
|
||||
states = {item["model_ref"]["model_key"]: item for item in body["model_status"]}
|
||||
assert states["glm-5.2"]["state"] == "enabled"
|
||||
|
||||
|
||||
def test_put_registry_malformed_json_is_400(active_client):
|
||||
response = active_client.put(
|
||||
"/api/model-registry",
|
||||
headers={**_admin_headers(), "Content-Type": "application/json"},
|
||||
content="{not json",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert response.json()["code"] == "INVALID_REQUEST"
|
||||
|
||||
|
||||
# --- PUT /api/model-registry/credentials/{credential_id} ------------------------
|
||||
|
||||
|
||||
def test_credential_rotation_creates_new_masked_revision(active_client):
|
||||
response = active_client.put(
|
||||
"/api/model-registry/credentials/zhipu-primary",
|
||||
headers=_admin_headers(),
|
||||
json={"operation": "replace", "secret_value": "rotated-secret-9999"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["credential_id"] == "zhipu-primary"
|
||||
assert body["revision"] == 2
|
||||
assert body["configured"] is True
|
||||
assert body["hint"] == "...9999"
|
||||
assert body["updated_at"]
|
||||
assert "rotated-secret-9999" not in response.text
|
||||
|
||||
|
||||
def test_credential_rotation_rejects_invalid_credential_id(active_client):
|
||||
response = active_client.put(
|
||||
"/api/model-registry/credentials/INVALID!!",
|
||||
headers=_admin_headers(),
|
||||
json={"operation": "replace", "secret_value": "whatever"},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
def test_credential_rotation_requires_write_scope(active_client):
|
||||
response = active_client.put(
|
||||
"/api/model-registry/credentials/zhipu-primary",
|
||||
headers=_headers(["model_config:read"]),
|
||||
json={"operation": "replace", "secret_value": "whatever"},
|
||||
)
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
# --- GET /api/models -------------------------------------------------------------
|
||||
|
||||
|
||||
def test_get_models_returns_only_selectable(active_client, store):
|
||||
response = active_client.get("/api/models", headers=_headers(["model:select"]))
|
||||
assert response.status_code == 200
|
||||
models = response.json()["models"]
|
||||
refs = {
|
||||
(m["model_ref"]["provider_id"], m["model_ref"]["model_key"]) for m in models
|
||||
}
|
||||
assert refs == {("zhipu-glm", "glm-5.2"), ("local-ollama", "qwen3")}
|
||||
glm = next(m for m in models if m["model_ref"]["model_key"] == "glm-5.2")
|
||||
assert glm["name"] == "GLM-5.2"
|
||||
assert glm["provider_name"] == "Zhipu GLM"
|
||||
assert glm["effective_capabilities"] == {
|
||||
"tools": True,
|
||||
"vision": False,
|
||||
"structured_output": True,
|
||||
}
|
||||
|
||||
# Disable qwen3 (and drop the auxiliary default): it leaves the selector.
|
||||
registry = store.load_registry()
|
||||
payload = registry.model_dump(mode="json")
|
||||
payload["defaults"]["auxiliary"] = None
|
||||
payload["providers"][1]["models"][0]["enabled"] = False
|
||||
store.save_registry(
|
||||
expected_revision=registry.revision,
|
||||
registry=RegistryV4.model_validate(payload),
|
||||
)
|
||||
models = active_client.get(
|
||||
"/api/models", headers=_headers(["model:select"])
|
||||
).json()["models"]
|
||||
assert [m["model_ref"]["model_key"] for m in models] == ["glm-5.2"]
|
||||
|
||||
|
||||
def test_get_models_requires_select_scope(active_client):
|
||||
response = active_client.get("/api/models", headers=_headers(["run:create"]))
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
# --- snapshot API -----------------------------------------------------------------
|
||||
|
||||
|
||||
def _snapshot_body(**overrides):
|
||||
body = {
|
||||
"run_request_id": "req-1",
|
||||
"thread_id": "thread-1",
|
||||
"deployment_id": "webui-1",
|
||||
"model_selection_revision": 4,
|
||||
"primary": None,
|
||||
"auxiliary": None,
|
||||
}
|
||||
body.update(overrides)
|
||||
return body
|
||||
|
||||
|
||||
def test_snapshot_create_201_and_idempotent_200(active_client):
|
||||
first = active_client.post(
|
||||
"/api/runtime-snapshots", headers=_run_headers(), json=_snapshot_body()
|
||||
)
|
||||
assert first.status_code == 201
|
||||
body = first.json()
|
||||
assert body["snapshot_id"]
|
||||
assert body["registry_revision"] == 2
|
||||
assert body["primary"]["provider_id"] == "zhipu-glm"
|
||||
assert body["primary"]["adapter_spec_revision"] == 1
|
||||
assert body["primary"]["runtime"]["max_output_tokens"] == 32768
|
||||
assert body["auxiliary"]["model_key"] == "qwen3"
|
||||
assert SECRET not in first.text
|
||||
|
||||
replay = active_client.post(
|
||||
"/api/runtime-snapshots", headers=_run_headers(), json=_snapshot_body()
|
||||
)
|
||||
assert replay.status_code == 200
|
||||
assert replay.json()["snapshot_id"] == body["snapshot_id"]
|
||||
|
||||
|
||||
def test_snapshot_create_conflicting_selection_is_409(active_client):
|
||||
active_client.post(
|
||||
"/api/runtime-snapshots", headers=_run_headers(), json=_snapshot_body()
|
||||
)
|
||||
conflict = active_client.post(
|
||||
"/api/runtime-snapshots",
|
||||
headers=_run_headers(),
|
||||
json=_snapshot_body(
|
||||
primary={"provider_id": "zhipu-glm", "model_key": "glm-5.2"}
|
||||
),
|
||||
)
|
||||
assert conflict.status_code == 409
|
||||
assert conflict.json()["code"] == "RUN_REQUEST_CONFLICT"
|
||||
|
||||
|
||||
def test_snapshot_create_requires_active_registry(client):
|
||||
response = client.post(
|
||||
"/api/runtime-snapshots", headers=_run_headers(), json=_snapshot_body()
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["code"] == "MODEL_REGISTRY_NOT_READY"
|
||||
|
||||
|
||||
def test_snapshot_create_thread_mismatch_is_403(active_client):
|
||||
response = active_client.post(
|
||||
"/api/runtime-snapshots",
|
||||
headers=_run_headers("thread-1"),
|
||||
json=_snapshot_body(thread_id="thread-2"),
|
||||
)
|
||||
assert response.status_code == 403
|
||||
assert response.json()["code"] == "FORBIDDEN"
|
||||
|
||||
|
||||
def test_snapshot_create_requires_thread_claim(active_client):
|
||||
headers = _headers(["run:create"]) # no thread_id claim
|
||||
response = active_client.post(
|
||||
"/api/runtime-snapshots", headers=headers, json=_snapshot_body()
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_snapshot_create_unknown_model_is_404(active_client):
|
||||
response = active_client.post(
|
||||
"/api/runtime-snapshots",
|
||||
headers=_run_headers(),
|
||||
json=_snapshot_body(primary={"provider_id": "zhipu-glm", "model_key": "nope"}),
|
||||
)
|
||||
assert response.status_code == 404
|
||||
assert response.json()["code"] == "MODEL_NOT_FOUND"
|
||||
|
||||
|
||||
def _create_snapshot(active_client, **overrides):
|
||||
response = active_client.post(
|
||||
"/api/runtime-snapshots",
|
||||
headers=_run_headers(),
|
||||
json=_snapshot_body(**overrides),
|
||||
)
|
||||
assert response.status_code == 201
|
||||
return response.json()["snapshot_id"]
|
||||
|
||||
|
||||
def test_snapshot_bind_state_machine(active_client):
|
||||
snapshot_id = _create_snapshot(active_client)
|
||||
bound = active_client.post(
|
||||
f"/api/runtime-snapshots/{snapshot_id}/bind",
|
||||
headers=_run_headers(),
|
||||
json={"langgraph_run_id": "run-1"},
|
||||
)
|
||||
assert bound.status_code == 200
|
||||
assert bound.json()["snapshot_id"] == snapshot_id
|
||||
|
||||
again = active_client.post(
|
||||
f"/api/runtime-snapshots/{snapshot_id}/bind",
|
||||
headers=_run_headers(),
|
||||
json={"langgraph_run_id": "run-1"},
|
||||
)
|
||||
assert again.status_code == 200
|
||||
|
||||
conflict = active_client.post(
|
||||
f"/api/runtime-snapshots/{snapshot_id}/bind",
|
||||
headers=_run_headers(),
|
||||
json={"langgraph_run_id": "run-2"},
|
||||
)
|
||||
assert conflict.status_code == 409
|
||||
assert conflict.json()["code"] == "SNAPSHOT_ALREADY_BOUND"
|
||||
|
||||
|
||||
def test_snapshot_bind_unknown_and_cross_thread_are_404(active_client):
|
||||
snapshot_id = _create_snapshot(active_client)
|
||||
missing = active_client.post(
|
||||
"/api/runtime-snapshots/snap-nope/bind",
|
||||
headers=_run_headers(),
|
||||
json={"langgraph_run_id": "run-1"},
|
||||
)
|
||||
assert missing.status_code == 404
|
||||
assert missing.json()["code"] == "SNAPSHOT_NOT_FOUND"
|
||||
|
||||
cross_thread = active_client.post(
|
||||
f"/api/runtime-snapshots/{snapshot_id}/bind",
|
||||
headers=_run_headers("thread-2"),
|
||||
json={"langgraph_run_id": "run-1"},
|
||||
)
|
||||
assert cross_thread.status_code == 404
|
||||
assert cross_thread.json()["code"] == "SNAPSHOT_NOT_FOUND"
|
||||
|
||||
|
||||
def test_snapshot_delete_state_machine(active_client):
|
||||
snapshot_id = _create_snapshot(active_client)
|
||||
deleted = active_client.delete(
|
||||
f"/api/runtime-snapshots/{snapshot_id}", headers=_run_headers()
|
||||
)
|
||||
assert deleted.status_code == 204
|
||||
assert deleted.content == b""
|
||||
|
||||
bound_id = _create_snapshot(active_client, run_request_id="req-2")
|
||||
active_client.post(
|
||||
f"/api/runtime-snapshots/{bound_id}/bind",
|
||||
headers=_run_headers(),
|
||||
json={"langgraph_run_id": "run-1"},
|
||||
)
|
||||
delete_bound = active_client.delete(
|
||||
f"/api/runtime-snapshots/{bound_id}", headers=_run_headers()
|
||||
)
|
||||
assert delete_bound.status_code == 409
|
||||
assert delete_bound.json()["code"] == "SNAPSHOT_ALREADY_BOUND"
|
||||
|
||||
|
||||
def test_snapshot_expired_rejects_bind_and_delete(active_client, services):
|
||||
snapshot_id = _create_snapshot(active_client)
|
||||
row = services.store.get_run_snapshot(snapshot_id)
|
||||
services.store.insert_run_snapshot(
|
||||
snapshot_id="snap-expired",
|
||||
deployment_id="webui-1",
|
||||
thread_id="thread-1",
|
||||
run_request_id="req-expired",
|
||||
selection_hash=row["selection_hash"],
|
||||
payload=row["payload"],
|
||||
expires_at=int(time.time()) - 1,
|
||||
)
|
||||
services.snapshot_service.cleanup_expired()
|
||||
|
||||
bind = active_client.post(
|
||||
"/api/runtime-snapshots/snap-expired/bind",
|
||||
headers=_run_headers(),
|
||||
json={"langgraph_run_id": "run-1"},
|
||||
)
|
||||
assert bind.status_code == 409
|
||||
assert bind.json()["code"] == "SNAPSHOT_EXPIRED"
|
||||
|
||||
delete = active_client.delete(
|
||||
"/api/runtime-snapshots/snap-expired", headers=_run_headers()
|
||||
)
|
||||
assert delete.status_code == 409
|
||||
assert delete.json()["code"] == "SNAPSHOT_EXPIRED"
|
||||
|
||||
# The expired triplet is free again: recreation returns 201.
|
||||
recreated = active_client.post(
|
||||
"/api/runtime-snapshots",
|
||||
headers=_run_headers(),
|
||||
json=_snapshot_body(run_request_id="req-expired"),
|
||||
)
|
||||
assert recreated.status_code == 201
|
||||
|
||||
|
||||
def test_snapshot_error_payload_structure(active_client):
|
||||
response = active_client.post(
|
||||
"/api/runtime-snapshots",
|
||||
headers=_run_headers(),
|
||||
json=_snapshot_body(primary={"provider_id": "zhipu-glm", "model_key": "nope"}),
|
||||
)
|
||||
body = response.json()
|
||||
assert set(body) == {"code", "message", "details", "request_id"}
|
||||
assert isinstance(body["details"], list)
|
||||
assert body["request_id"]
|
||||
|
||||
|
||||
# --- platform configuration ------------------------------------------------
|
||||
|
||||
|
||||
def test_missing_platform_config_refuses_service():
|
||||
def _unconfigured():
|
||||
raise PlatformConfigError("missing bff_service_token")
|
||||
|
||||
app = Starlette(routes=model_registry_routes(_unconfigured))
|
||||
response = TestClient(app).get("/api/model-registry")
|
||||
assert response.status_code == 500
|
||||
body = response.json()
|
||||
assert body["code"] == "PLATFORM_CONFIG_MISSING"
|
||||
assert set(body) == {"code", "message", "details", "request_id"}
|
||||
|
||||
|
||||
# --- OpenAPI export -----------------------------------------------------------------
|
||||
|
||||
|
||||
def test_openapi_export_file_matches_pydantic_schemas():
|
||||
assert OPENAPI_PATH.exists()
|
||||
exported = json.loads(OPENAPI_PATH.read_text(encoding="utf-8"))
|
||||
# The checked-in file is exactly what the export script regenerates.
|
||||
assert exported == build_openapi_document()
|
||||
|
||||
for path in (
|
||||
"/api/model-registry",
|
||||
"/api/model-registry/credentials/{credential_id}",
|
||||
"/api/models",
|
||||
"/api/runtime-snapshots",
|
||||
"/api/runtime-snapshots/{snapshot_id}/bind",
|
||||
"/api/runtime-snapshots/{snapshot_id}",
|
||||
):
|
||||
assert path in exported["paths"]
|
||||
|
||||
schema = exported["components"]["schemas"]["GetModelRegistryResponse"]
|
||||
assert set(schema["properties"]) == {
|
||||
"revision",
|
||||
"registry",
|
||||
"adapter_specs",
|
||||
"credential_status",
|
||||
"model_status",
|
||||
"endpoint_policy",
|
||||
}
|
||||
registry_schema = exported["components"]["schemas"]["RegistryV4"]
|
||||
assert "providers" in registry_schema["properties"]
|
||||
error_schema = exported["components"]["schemas"]["ErrorPayload"]
|
||||
assert set(error_schema["properties"]) == {
|
||||
"code",
|
||||
"message",
|
||||
"details",
|
||||
"request_id",
|
||||
}
|
||||
@@ -42,6 +42,7 @@ from EvoScientist.model_registry.schemas import (
|
||||
)
|
||||
|
||||
ALL_ERROR_CODES = {
|
||||
"INVALID_REQUEST": 400,
|
||||
"REGISTRY_REVISION_CONFLICT": 409,
|
||||
"THREAD_MODEL_SELECTION_CONFLICT": 409,
|
||||
"RUN_REQUEST_CONFLICT": 409,
|
||||
@@ -49,6 +50,8 @@ ALL_ERROR_CODES = {
|
||||
"SNAPSHOT_EXPIRED": 409,
|
||||
"MODEL_CONFIGURATION_CHANGED": 409,
|
||||
"DELEGATION_REPLAYED": 401,
|
||||
"UNAUTHENTICATED": 401,
|
||||
"FORBIDDEN": 403,
|
||||
"MODEL_NOT_FOUND": 404,
|
||||
"SNAPSHOT_NOT_FOUND": 404,
|
||||
"MODEL_REGISTRY_NOT_READY": 422,
|
||||
@@ -67,6 +70,8 @@ ALL_ERROR_CODES = {
|
||||
"MODEL_LIMITS_UNCONFIRMED": 422,
|
||||
"CONTEXT_BUDGET_UNSATISFIABLE": 422,
|
||||
"PROVIDER_UNREACHABLE": 422,
|
||||
"VALIDATION_FAILED": 422,
|
||||
"PLATFORM_CONFIG_MISSING": 500,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user