m4
aae8d0a379
In-progress work committed to unblock the config import/export plan: - prompts: FILE_REFERENCES section for workspace-relative file citation - backends: resolve quoted virtual absolute paths onto the sandbox workspace - middleware: read_file_images middleware; message_budget extensions - image_gen/model_registry: image model 'enabled' flag refactor - memory/launch, gateway/background_runs, tools/image follow-ons - scripts: dev_backend.sh, release.sh - tests for the above
360 lines
11 KiB
Python
360 lines
11 KiB
Python
"""Contract tests for the /api/image-generation endpoints."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import jwt
|
|
import pytest
|
|
import yaml
|
|
from cryptography.hazmat.primitives import serialization
|
|
from cryptography.hazmat.primitives.asymmetric import ec
|
|
from starlette.applications import Starlette
|
|
from starlette.testclient import TestClient
|
|
|
|
from EvoScientist.image_gen import config as image_config
|
|
from EvoScientist.image_gen import service as image_service
|
|
from EvoScientist.image_gen.adapters.base import ImageGenError
|
|
from EvoScientist.image_gen.config import (
|
|
ImageGenerationSettings,
|
|
ImageModelEntry,
|
|
save_image_generation_settings,
|
|
)
|
|
from EvoScientist.model_registry import http_api
|
|
from EvoScientist.model_registry.auth import BffAuthenticator
|
|
from EvoScientist.model_registry.endpoint_policy import EndpointPolicy
|
|
from EvoScientist.model_registry.hashing import configuration_hash # noqa: F401 (fixture parity)
|
|
from EvoScientist.model_registry.http_api import ApiServices, model_registry_routes
|
|
from EvoScientist.model_registry.platform import DelegationPublicKey
|
|
from EvoScientist.model_registry.resolver import ModelRegistryResolver
|
|
from EvoScientist.model_registry.snapshots import SnapshotService
|
|
from EvoScientist.model_registry.store import ModelRuntimeStore
|
|
|
|
SERVICE_TOKEN = "bff-service-token"
|
|
|
|
_PRIVATE_KEY = ec.generate_private_key(ec.SECP256R1())
|
|
_PRIVATE_PEM = _PRIVATE_KEY.private_bytes(
|
|
serialization.Encoding.PEM,
|
|
serialization.PrivateFormat.PKCS8,
|
|
serialization.NoEncryption(),
|
|
)
|
|
_PUBLIC_PEM = _PRIVATE_KEY.public_key().public_bytes(
|
|
serialization.Encoding.PEM,
|
|
serialization.PublicFormat.SubjectPublicKeyInfo,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def config_path(tmp_path):
|
|
path = tmp_path / "config.yaml"
|
|
path.write_text("other_section: {keep: true}\n", encoding="utf-8")
|
|
return path
|
|
|
|
|
|
@pytest.fixture
|
|
def services(tmp_path, config_path, monkeypatch):
|
|
store = ModelRuntimeStore(config_dir=tmp_path / "runtime")
|
|
monkeypatch.setattr(
|
|
http_api,
|
|
"load_image_generation_settings",
|
|
lambda: image_config.load_image_generation_settings(config_path=config_path),
|
|
)
|
|
monkeypatch.setattr(
|
|
http_api,
|
|
"save_image_generation_settings",
|
|
lambda settings: image_config.save_image_generation_settings(
|
|
settings, config_path=config_path
|
|
),
|
|
)
|
|
resolver = ModelRegistryResolver(store)
|
|
return ApiServices(
|
|
store=store,
|
|
resolver=resolver,
|
|
snapshot_service=SnapshotService(store, resolver),
|
|
endpoint_policy=EndpointPolicy([]),
|
|
authenticator=BffAuthenticator(
|
|
service_token=SERVICE_TOKEN,
|
|
service_token_hash=None,
|
|
delegation_keys=(
|
|
DelegationPublicKey(deployment_id="webui-1", public_key=_PUBLIC_PEM),
|
|
),
|
|
jti_store=store,
|
|
),
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def client(services):
|
|
app = Starlette(routes=model_registry_routes(lambda: services))
|
|
return TestClient(app)
|
|
|
|
|
|
def _headers(scopes):
|
|
now = int(time.time())
|
|
claims = {
|
|
"iss": "WebUI",
|
|
"aud": "EvoScientist",
|
|
"sub": "user-1",
|
|
"scopes": list(scopes),
|
|
"deployment_id": "webui-1",
|
|
"iat": now,
|
|
"exp": now + 30,
|
|
"jti": uuid.uuid4().hex,
|
|
}
|
|
return {
|
|
"Authorization": f"Bearer {SERVICE_TOKEN}",
|
|
"X-Evo-Actor": jwt.encode(claims, _PRIVATE_PEM, algorithm="ES256"),
|
|
}
|
|
|
|
|
|
def _admin_headers():
|
|
return _headers(["model_config:read", "model_config:write"])
|
|
|
|
|
|
def _settings_payload(**overrides):
|
|
payload = {
|
|
"default_model": "gpt-image-2",
|
|
"timeout_seconds": 120.0,
|
|
"models": [
|
|
{
|
|
"id": "gpt-image-2",
|
|
"name": "GPT Image 2",
|
|
"provider": "openai",
|
|
"api_key": "sk-live-9876abcd",
|
|
"base_url": "",
|
|
"enabled": True,
|
|
"default_size": "1024x1024",
|
|
"default_quality": "auto",
|
|
"params": {},
|
|
}
|
|
],
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
def test_put_then_get_masks_api_key(client, config_path):
|
|
put = client.put(
|
|
"/api/image-generation", headers=_admin_headers(), json=_settings_payload()
|
|
)
|
|
assert put.status_code == 200
|
|
body = put.json()
|
|
model = body["models"][0]
|
|
assert model["api_key"] == ""
|
|
assert model["api_key_configured"] is True
|
|
assert model["api_key_hint"] == "...abcd"
|
|
assert "sk-live-9876abcd" not in json.dumps(body)
|
|
|
|
get = client.get("/api/image-generation", headers=_admin_headers())
|
|
assert get.status_code == 200
|
|
assert get.json() == body
|
|
|
|
# The plaintext key really landed in config.yaml, other sections intact.
|
|
raw = yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
|
assert raw["other_section"] == {"keep": True}
|
|
assert raw["image_generation"]["models"][0]["api_key"] == "sk-live-9876abcd"
|
|
|
|
|
|
def test_put_blank_api_key_keeps_existing(client):
|
|
client.put(
|
|
"/api/image-generation", headers=_admin_headers(), json=_settings_payload()
|
|
)
|
|
update = _settings_payload()
|
|
update["models"][0]["api_key"] = ""
|
|
update["models"][0]["name"] = "Renamed"
|
|
response = client.put(
|
|
"/api/image-generation", headers=_admin_headers(), json=update
|
|
)
|
|
assert response.status_code == 200
|
|
model = response.json()["models"][0]
|
|
assert model["name"] == "Renamed"
|
|
assert model["api_key_configured"] is True
|
|
assert model["api_key_hint"] == "...abcd"
|
|
|
|
|
|
def test_put_new_model_without_key_is_unconfigured(client):
|
|
payload = _settings_payload(models=[_settings_payload()["models"][0] | {"api_key": ""}])
|
|
response = client.put(
|
|
"/api/image-generation", headers=_admin_headers(), json=payload
|
|
)
|
|
assert response.status_code == 200
|
|
model = response.json()["models"][0]
|
|
assert model["api_key_configured"] is False
|
|
assert model["api_key_hint"] is None
|
|
|
|
|
|
def test_short_key_never_leaks_through_hint(client):
|
|
payload = _settings_payload()
|
|
payload["models"][0]["api_key"] = "abc"
|
|
response = client.put(
|
|
"/api/image-generation", headers=_admin_headers(), json=payload
|
|
)
|
|
assert response.status_code == 200
|
|
model = response.json()["models"][0]
|
|
assert model["api_key_configured"] is True
|
|
assert model["api_key_hint"] is None
|
|
|
|
|
|
def test_get_empty_when_unconfigured(client):
|
|
response = client.get("/api/image-generation", headers=_admin_headers())
|
|
assert response.status_code == 200
|
|
assert response.json() == {
|
|
"default_model": "",
|
|
"timeout_seconds": 120.0,
|
|
"models": [],
|
|
}
|
|
|
|
|
|
def test_put_rejects_invalid_payload(client):
|
|
payload = _settings_payload()
|
|
payload["models"][0]["provider"] = "midjourney"
|
|
response = client.put(
|
|
"/api/image-generation", headers=_admin_headers(), json=payload
|
|
)
|
|
assert response.status_code == 422
|
|
body = response.json()
|
|
assert body["code"] == "VALIDATION_FAILED"
|
|
assert "midjourney" not in json.dumps(body)
|
|
|
|
|
|
def test_read_scope_cannot_write(client):
|
|
response = client.put(
|
|
"/api/image-generation",
|
|
headers=_headers(["model_config:read"]),
|
|
json=_settings_payload(),
|
|
)
|
|
assert response.status_code == 403
|
|
|
|
|
|
def test_get_corrupt_section_returns_422_envelope(client, config_path):
|
|
config_path.write_text(
|
|
"image_generation:\n models:\n - id: gpt-image-2\n provider: midjourney\n",
|
|
encoding="utf-8",
|
|
)
|
|
response = client.get("/api/image-generation", headers=_admin_headers())
|
|
assert response.status_code == 422
|
|
body = response.json()
|
|
assert body["code"] == "VALIDATION_FAILED"
|
|
assert "midjourney" not in json.dumps(body)
|
|
|
|
|
|
# --- POST /api/image-generation/test -----------------------------------------
|
|
|
|
|
|
class _FakeImageAdapter:
|
|
"""Records the entry it was built with; can be primed to fail."""
|
|
|
|
instances: list["_FakeImageAdapter"] = []
|
|
failure: str | None = None
|
|
|
|
def __init__(self, entry, *, timeout=120.0):
|
|
self.entry = entry
|
|
self.timeout = timeout
|
|
_FakeImageAdapter.instances.append(self)
|
|
|
|
async def generate(self, **kwargs):
|
|
if _FakeImageAdapter.failure is not None:
|
|
raise ImageGenError(_FakeImageAdapter.failure)
|
|
return [b"\x89PNG\r\n\x1a\nfake"]
|
|
|
|
async def edit(self, **kwargs): # pragma: no cover - not used by tests
|
|
return [b"\x89PNG\r\n\x1a\nfake"]
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_image_adapter(monkeypatch):
|
|
_FakeImageAdapter.instances = []
|
|
_FakeImageAdapter.failure = None
|
|
monkeypatch.setattr(
|
|
image_service,
|
|
"ADAPTER_CLASSES",
|
|
{"openai": _FakeImageAdapter, "gemini": _FakeImageAdapter},
|
|
)
|
|
return _FakeImageAdapter
|
|
|
|
|
|
def _test_headers():
|
|
return _headers(["model_config:test"])
|
|
|
|
|
|
def _test_payload(**overrides):
|
|
payload = _settings_payload()["models"][0]
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
|
|
def test_image_test_success(client, fake_image_adapter):
|
|
response = client.post(
|
|
"/api/image-generation/test",
|
|
headers=_test_headers(),
|
|
json=_test_payload(),
|
|
)
|
|
assert response.status_code == 200
|
|
body = response.json()
|
|
assert body["ok"] is True
|
|
assert body["latency_ms"] >= 0
|
|
entry = fake_image_adapter.instances[0].entry
|
|
assert entry.id == "gpt-image-2"
|
|
assert entry.api_key == "sk-live-9876abcd"
|
|
|
|
|
|
def test_image_test_blank_key_falls_back_to_stored(client, fake_image_adapter):
|
|
client.put(
|
|
"/api/image-generation", headers=_admin_headers(), json=_settings_payload()
|
|
)
|
|
response = client.post(
|
|
"/api/image-generation/test",
|
|
headers=_test_headers(),
|
|
json=_test_payload(api_key="", name="Renamed draft"),
|
|
)
|
|
assert response.status_code == 200
|
|
entry = fake_image_adapter.instances[0].entry
|
|
assert entry.api_key == "sk-live-9876abcd"
|
|
|
|
|
|
def test_image_test_credential_rejected(client, fake_image_adapter):
|
|
fake_image_adapter.failure = "Image provider returned HTTP 401: bad key"
|
|
response = client.post(
|
|
"/api/image-generation/test",
|
|
headers=_test_headers(),
|
|
json=_test_payload(),
|
|
)
|
|
assert response.status_code == 422
|
|
body = response.json()
|
|
assert body["code"] == "CREDENTIAL_REJECTED"
|
|
assert "bad key" in body["message"]
|
|
|
|
|
|
def test_image_test_provider_unreachable(client, fake_image_adapter):
|
|
fake_image_adapter.failure = "Image provider request failed: ConnectError"
|
|
response = client.post(
|
|
"/api/image-generation/test",
|
|
headers=_test_headers(),
|
|
json=_test_payload(),
|
|
)
|
|
assert response.status_code == 422
|
|
assert response.json()["code"] == "PROVIDER_UNREACHABLE"
|
|
|
|
|
|
def test_image_test_model_not_found(client, fake_image_adapter):
|
|
fake_image_adapter.failure = "Image provider returned HTTP 404: no such model"
|
|
response = client.post(
|
|
"/api/image-generation/test",
|
|
headers=_test_headers(),
|
|
json=_test_payload(),
|
|
)
|
|
assert response.status_code == 404
|
|
assert response.json()["code"] == "MODEL_NOT_FOUND"
|
|
|
|
|
|
def test_image_test_requires_test_scope(client, fake_image_adapter):
|
|
response = client.post(
|
|
"/api/image-generation/test",
|
|
headers=_headers(["model_config:write"]),
|
|
json=_test_payload(),
|
|
)
|
|
assert response.status_code == 403
|