m4
1043 lines
34 KiB
Python
1043 lines
34 KiB
Python
"""Smoke test for the /api/models route mounted via langgraph.json's
|
|
``http`` field. We test the FastAPI app directly — no need to spin up
|
|
langgraph dev.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
from langchain_core.messages import AIMessage, HumanMessage
|
|
from starlette.testclient import TestClient
|
|
|
|
from EvoScientist.config import EvoScientistConfig, load_config, save_config
|
|
from EvoScientist.config.provider_profiles import (
|
|
load_provider_profiles,
|
|
replace_provider_profiles,
|
|
)
|
|
from EvoScientist.langgraph_dev.http import app
|
|
from EvoScientist.llm.provider_operations import (
|
|
DiscoveredProviderModel,
|
|
ProviderModelTestResult,
|
|
ProviderOperationError,
|
|
)
|
|
|
|
client = TestClient(app)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def isolate_xdg_config(tmp_path, monkeypatch):
|
|
"""Keep HTTP tests independent from the developer's saved providers."""
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
|
|
|
|
def test_get_models_returns_entries_and_default():
|
|
mock_cfg = EvoScientistConfig(
|
|
model="claude-sonnet-4-6", provider="custom-anthropic"
|
|
)
|
|
with patch(
|
|
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
|
):
|
|
resp = client.get("/api/models")
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert "entries" in body
|
|
assert "default" in body
|
|
assert body["default"] == {
|
|
"name": "claude-sonnet-4-6",
|
|
"provider": "custom-anthropic",
|
|
}
|
|
assert isinstance(body["entries"], list)
|
|
assert len(body["entries"]) > 0
|
|
# Every entry has the three required keys
|
|
for entry in body["entries"]:
|
|
assert set(entry.keys()) == {"name", "model_id", "provider"}
|
|
assert isinstance(entry["name"], str)
|
|
assert entry["name"]
|
|
assert isinstance(entry["model_id"], str)
|
|
assert entry["model_id"]
|
|
assert isinstance(entry["provider"], str)
|
|
assert entry["provider"]
|
|
|
|
|
|
def test_get_models_only_returns_enabled_catalog_entries_when_configured():
|
|
mock_cfg = EvoScientistConfig(
|
|
model="chat-main",
|
|
provider="openai",
|
|
model_catalog=[
|
|
{
|
|
"provider": "openai",
|
|
"id": "chat-main",
|
|
"name": "Chat Main",
|
|
"model_id": "gpt-upstream",
|
|
"enabled": True,
|
|
},
|
|
{
|
|
"provider": "anthropic",
|
|
"id": "hidden-model",
|
|
"name": "Hidden",
|
|
"model_id": "claude-hidden",
|
|
"enabled": False,
|
|
},
|
|
],
|
|
)
|
|
|
|
with patch(
|
|
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
|
):
|
|
body = client.get("/api/models").json()
|
|
|
|
assert body["entries"] == [
|
|
{"name": "chat-main", "model_id": "gpt-upstream", "provider": "openai"}
|
|
]
|
|
|
|
|
|
def test_get_models_empty_catalog_hides_all_builtin_models():
|
|
mock_cfg = EvoScientistConfig(model_catalog=[])
|
|
|
|
with patch(
|
|
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
|
):
|
|
body = client.get("/api/models").json()
|
|
|
|
assert body["entries"] == []
|
|
|
|
|
|
def test_provider_profiles_api_requires_admin_token_header(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
|
|
response = client.get("/api/provider-profiles")
|
|
assert response.status_code == 403
|
|
|
|
|
|
def test_provider_profiles_api_round_trip_redacts_secret(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
payload = {
|
|
"providers": [
|
|
{
|
|
"id": "lab-openai",
|
|
"name": "Lab OpenAI",
|
|
"adapter": "openai-compatible",
|
|
"base_url": "https://llm.example.test/v1",
|
|
"api_key": "provider-secret",
|
|
"enabled": True,
|
|
"models": [
|
|
{
|
|
"id": "lab-model",
|
|
"name": "Lab Model",
|
|
"model_id": "vendor/model",
|
|
"enabled": True,
|
|
}
|
|
],
|
|
}
|
|
]
|
|
}
|
|
|
|
put_response = client.put("/api/provider-profiles", headers=headers, json=payload)
|
|
assert put_response.status_code == 200
|
|
assert "provider-secret" not in put_response.text
|
|
assert put_response.json()["providers"][0]["api_key_configured"] is True
|
|
assert "openai" in put_response.json()["reserved_provider_ids"]
|
|
|
|
get_response = client.get("/api/provider-profiles", headers=headers)
|
|
assert get_response.status_code == 200
|
|
assert get_response.json()["providers"][0]["models"][0]["id"] == "lab-model"
|
|
assert "provider-secret" not in get_response.text
|
|
|
|
|
|
def test_llm_config_api_requires_admin_token_header(monkeypatch):
|
|
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
|
|
|
|
response = client.get("/api/config")
|
|
|
|
assert response.status_code == 403
|
|
|
|
|
|
def test_llm_config_api_redacts_secrets_and_reports_env_overrides(monkeypatch):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
monkeypatch.setenv("OPENAI_API_KEY", "environment-secret")
|
|
save_config(
|
|
EvoScientistConfig(
|
|
provider="openai",
|
|
model="gpt-5.4",
|
|
openai_api_key="file-secret-1234",
|
|
default_workdir="/tmp/research",
|
|
)
|
|
)
|
|
|
|
response = client.get(
|
|
"/api/config",
|
|
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
body = response.json()
|
|
assert body["values"]["provider"] == "openai"
|
|
assert body["values"]["model"] == "gpt-5.4"
|
|
assert "openai_api_key" not in body["values"]
|
|
assert body["secrets"]["openai_api_key"] == {
|
|
"configured": True,
|
|
"hint": "...1234",
|
|
}
|
|
assert body["env_overrides"]["openai_api_key"] == "OPENAI_API_KEY"
|
|
assert "file-secret-1234" not in response.text
|
|
assert "environment-secret" not in response.text
|
|
assert "default_workdir" not in body["values"]
|
|
assert body["model_catalog"] is None
|
|
assert body["builtin_model_candidates"]
|
|
openai = next(
|
|
provider for provider in body["builtin_providers"] if provider["id"] == "openai"
|
|
)
|
|
assert openai["managed"] is False
|
|
assert openai["api_key_configured"] is True
|
|
assert "environment-secret" not in response.text
|
|
|
|
|
|
def test_llm_config_api_saves_builtin_registry_without_rewriting_legacy_secret(
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
save_config(
|
|
EvoScientistConfig(
|
|
provider="openai",
|
|
model="chat-main",
|
|
openai_api_key="legacy-secret",
|
|
default_workdir="/tmp/research",
|
|
)
|
|
)
|
|
loaded = client.get("/api/config", headers=headers).json()
|
|
openai = next(
|
|
provider
|
|
for provider in loaded["builtin_providers"]
|
|
if provider["id"] == "openai"
|
|
)
|
|
openai.update(
|
|
{
|
|
"managed": True,
|
|
"base_url": "https://proxy.example.test/v1",
|
|
"models": [
|
|
{
|
|
"id": "chat-main",
|
|
"name": "Chat Main",
|
|
"model_id": "gpt-upstream",
|
|
"enabled": True,
|
|
}
|
|
],
|
|
}
|
|
)
|
|
|
|
response = client.patch(
|
|
"/api/config",
|
|
headers=headers,
|
|
json={
|
|
"revision": loaded["revision"],
|
|
"values": loaded["values"],
|
|
"secrets": {},
|
|
"clear_secrets": [],
|
|
"builtin_providers": [openai],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
body = response.json()
|
|
assert body["restart_required"] is False
|
|
assert "builtin_providers" in body["changed_fields"]
|
|
saved_profile = load_provider_profiles().builtins[0]
|
|
assert saved_profile.id == "openai"
|
|
assert saved_profile.api_key == "legacy-secret"
|
|
assert saved_profile.base_url == "https://proxy.example.test/v1"
|
|
saved_config = load_config()
|
|
assert saved_config.openai_api_key == "legacy-secret"
|
|
assert saved_config.default_workdir == "/tmp/research"
|
|
assert saved_config.model_catalog is None
|
|
|
|
models = client.get("/api/models").json()["entries"]
|
|
assert models == [
|
|
{"name": "chat-main", "model_id": "gpt-upstream", "provider": "openai"}
|
|
]
|
|
|
|
|
|
def test_first_builtin_registry_save_migrates_legacy_default_model(monkeypatch):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://environment.example.test")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
save_config(EvoScientistConfig(provider="anthropic", model="claude-sonnet-4-6"))
|
|
loaded = client.get("/api/config", headers=headers).json()
|
|
ollama = next(
|
|
provider
|
|
for provider in loaded["builtin_providers"]
|
|
if provider["id"] == "ollama"
|
|
)
|
|
ollama.update(
|
|
{
|
|
"managed": True,
|
|
"base_url": "http://127.0.0.1:11434",
|
|
"models": [],
|
|
}
|
|
)
|
|
|
|
response = client.patch(
|
|
"/api/config",
|
|
headers=headers,
|
|
json={
|
|
"revision": loaded["revision"],
|
|
"values": loaded["values"],
|
|
"secrets": {},
|
|
"clear_secrets": [],
|
|
"builtin_providers": [ollama],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
builtins = {profile.id: profile for profile in load_provider_profiles().builtins}
|
|
assert set(builtins) == {"anthropic", "ollama"}
|
|
assert builtins["anthropic"].models[0].id == "claude-sonnet-4-6"
|
|
assert builtins["anthropic"].base_url == ""
|
|
|
|
|
|
def test_llm_config_api_persists_model_catalog(monkeypatch):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
save_config(EvoScientistConfig(provider="openai", model="chat-main"))
|
|
revision = client.get("/api/config", headers=headers).json()["revision"]
|
|
catalog = [
|
|
{
|
|
"provider": "openai",
|
|
"id": "chat-main",
|
|
"name": "Chat Main",
|
|
"model_id": "gpt-upstream",
|
|
"enabled": True,
|
|
}
|
|
]
|
|
|
|
response = client.patch(
|
|
"/api/config",
|
|
headers=headers,
|
|
json={
|
|
"revision": revision,
|
|
"values": {},
|
|
"secrets": {},
|
|
"clear_secrets": [],
|
|
"model_catalog": catalog,
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["model_catalog"] == catalog
|
|
assert response.json()["restart_required"] is False
|
|
assert load_config().model_catalog == catalog
|
|
|
|
|
|
def test_llm_config_api_rejects_default_outside_model_catalog(monkeypatch):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
save_config(EvoScientistConfig(provider="openai", model="gpt-default"))
|
|
revision = client.get("/api/config", headers=headers).json()["revision"]
|
|
|
|
response = client.patch(
|
|
"/api/config",
|
|
headers=headers,
|
|
json={
|
|
"revision": revision,
|
|
"values": {},
|
|
"secrets": {},
|
|
"clear_secrets": [],
|
|
"model_catalog": [
|
|
{
|
|
"provider": "openai",
|
|
"id": "different-model",
|
|
"name": "Different",
|
|
"model_id": "gpt-different",
|
|
"enabled": True,
|
|
}
|
|
],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 400
|
|
assert "Default model" in response.json()["error"]
|
|
assert load_config().model_catalog is None
|
|
|
|
|
|
def test_llm_config_api_patch_preserves_unrelated_fields_and_secret_by_default(
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
save_config(
|
|
EvoScientistConfig(
|
|
openai_api_key="saved-openai-key",
|
|
anthropic_api_key="saved-anthropic-key",
|
|
default_workdir="/tmp/research",
|
|
)
|
|
)
|
|
revision = client.get("/api/config", headers=headers).json()["revision"]
|
|
|
|
response = client.patch(
|
|
"/api/config",
|
|
headers=headers,
|
|
json={
|
|
"revision": revision,
|
|
"values": {
|
|
"provider": "openai",
|
|
"model": "gpt-5.4",
|
|
"ollama_base_url": "http://127.0.0.1:11434",
|
|
},
|
|
"secrets": {"openai_api_key": "replacement-openai-key"},
|
|
"clear_secrets": ["anthropic_api_key"],
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
body = response.json()
|
|
assert body["restart_required"] is True
|
|
assert set(body["changed_fields"]) == {
|
|
"anthropic_api_key",
|
|
"model",
|
|
"ollama_base_url",
|
|
"openai_api_key",
|
|
"provider",
|
|
}
|
|
assert "replacement-openai-key" not in response.text
|
|
saved = load_config()
|
|
assert saved.provider == "openai"
|
|
assert saved.model == "gpt-5.4"
|
|
assert saved.ollama_base_url == "http://127.0.0.1:11434"
|
|
assert saved.openai_api_key == "replacement-openai-key"
|
|
assert saved.anthropic_api_key == ""
|
|
assert saved.default_workdir == "/tmp/research"
|
|
|
|
|
|
def test_llm_config_api_rejects_stale_revision(monkeypatch):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
save_config(EvoScientistConfig(model="claude-sonnet-4-6"))
|
|
revision = client.get("/api/config", headers=headers).json()["revision"]
|
|
save_config(EvoScientistConfig(model="gpt-5.4", provider="openai"))
|
|
|
|
response = client.patch(
|
|
"/api/config",
|
|
headers=headers,
|
|
json={
|
|
"revision": revision,
|
|
"values": {"model": "claude-opus-4-8"},
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 409
|
|
assert "Reload and try again" in response.json()["error"]
|
|
assert load_config().model == "gpt-5.4"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("payload", "message"),
|
|
[
|
|
(
|
|
{"values": {"openai_auth_mode": "password"}},
|
|
"openai_auth_mode must be 'api_key' or 'oauth'",
|
|
),
|
|
(
|
|
{"values": {"default_workdir": "/tmp/other"}},
|
|
"Unsupported config fields: default_workdir",
|
|
),
|
|
(
|
|
{"values": {"ollama_base_url": "localhost:11434"}},
|
|
"ollama_base_url must use http:// or https://",
|
|
),
|
|
],
|
|
)
|
|
def test_llm_config_api_validates_updates(monkeypatch, payload, message):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
revision = client.get("/api/config", headers=headers).json()["revision"]
|
|
|
|
response = client.patch(
|
|
"/api/config",
|
|
headers=headers,
|
|
json={"revision": revision, **payload},
|
|
)
|
|
|
|
assert response.status_code == 400
|
|
assert message in response.json()["error"]
|
|
|
|
|
|
def test_provider_actions_api_requires_admin_token_header(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
|
|
|
|
response = client.post("/api/provider-actions", json={})
|
|
|
|
assert response.status_code == 403
|
|
|
|
|
|
def test_provider_actions_lists_models_with_saved_api_key(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
provider = {
|
|
"id": "lab-openai",
|
|
"name": "Lab OpenAI",
|
|
"adapter": "openai-compatible",
|
|
"base_url": "https://llm.example.test/v1",
|
|
"api_key": "provider-secret",
|
|
"enabled": True,
|
|
"models": [],
|
|
}
|
|
assert (
|
|
client.put(
|
|
"/api/provider-profiles",
|
|
headers=headers,
|
|
json={"providers": [provider]},
|
|
).status_code
|
|
== 200
|
|
)
|
|
provider["api_key"] = ""
|
|
|
|
async def fake_discover(profile):
|
|
assert profile.api_key == "provider-secret"
|
|
return [DiscoveredProviderModel("vendor/model", "Vendor Model")]
|
|
|
|
with patch(
|
|
"EvoScientist.langgraph_dev.http.discover_provider_models",
|
|
new=fake_discover,
|
|
):
|
|
response = client.post(
|
|
"/api/provider-actions",
|
|
headers=headers,
|
|
json={"action": "list_models", "provider": provider},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {
|
|
"models": [{"model_id": "vendor/model", "name": "Vendor Model"}]
|
|
}
|
|
|
|
|
|
def test_provider_actions_tests_model(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
|
|
async def fake_test(profile, model):
|
|
assert profile.adapter == "grok"
|
|
assert model.model_id == "grok-4"
|
|
return ProviderModelTestResult(latency_ms=123, response="OK")
|
|
|
|
with patch("EvoScientist.langgraph_dev.http.test_provider_model", new=fake_test):
|
|
response = client.post(
|
|
"/api/provider-actions",
|
|
headers=headers,
|
|
json={
|
|
"action": "test_model",
|
|
"provider": {
|
|
"id": "lab-grok",
|
|
"name": "Lab Grok",
|
|
"adapter": "grok",
|
|
"base_url": "",
|
|
"api_key": "xai-secret",
|
|
"enabled": True,
|
|
"models": [],
|
|
},
|
|
"model": {
|
|
"id": "grok",
|
|
"name": "Grok",
|
|
"model_id": "grok-4",
|
|
},
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {"ok": True, "latency_ms": 123, "response": "OK"}
|
|
|
|
|
|
def test_provider_actions_maps_provider_failures_to_bad_gateway(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
|
|
async def fake_discover(_profile):
|
|
raise ProviderOperationError("Provider returned HTTP 401")
|
|
|
|
with patch(
|
|
"EvoScientist.langgraph_dev.http.discover_provider_models",
|
|
new=fake_discover,
|
|
):
|
|
response = client.post(
|
|
"/api/provider-actions",
|
|
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
|
|
json={
|
|
"action": "list_models",
|
|
"provider": {
|
|
"id": "lab-openai",
|
|
"name": "Lab OpenAI",
|
|
"adapter": "openai-compatible",
|
|
"base_url": "https://llm.example.test/v1",
|
|
"api_key": "bad-key",
|
|
"enabled": True,
|
|
"models": [],
|
|
},
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 502
|
|
assert response.json() == {"error": "Provider returned HTTP 401"}
|
|
|
|
|
|
def test_llm_config_action_discovers_models_with_effective_builtin_secret(
|
|
monkeypatch,
|
|
):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
monkeypatch.setenv("OPENAI_API_KEY", "environment-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
|
|
async def fake_discover(profile):
|
|
assert profile.id == "openai"
|
|
assert profile.adapter == "openai"
|
|
assert profile.api_key == "environment-secret"
|
|
return [DiscoveredProviderModel("gpt-discovered", "GPT Discovered")]
|
|
|
|
with patch(
|
|
"EvoScientist.langgraph_dev.http.discover_provider_models",
|
|
new=fake_discover,
|
|
):
|
|
response = client.post(
|
|
"/api/config",
|
|
headers=headers,
|
|
json={"action": "list_models", "provider": {"id": "openai"}},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {
|
|
"models": [{"model_id": "gpt-discovered", "name": "GPT Discovered"}]
|
|
}
|
|
|
|
|
|
def test_llm_config_action_tests_builtin_model_with_draft_connection(monkeypatch):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
|
|
async def fake_test(profile, model):
|
|
assert profile.id == "custom-openai"
|
|
assert profile.adapter == "openai-compatible"
|
|
assert profile.base_url == "https://proxy.example.test/v1"
|
|
assert profile.api_key == "draft-secret"
|
|
assert model.model_id == "vendor/model"
|
|
return ProviderModelTestResult(latency_ms=42, response="OK")
|
|
|
|
with patch("EvoScientist.langgraph_dev.http.test_provider_model", new=fake_test):
|
|
response = client.post(
|
|
"/api/config",
|
|
headers=headers,
|
|
json={
|
|
"action": "test_model",
|
|
"provider": {
|
|
"id": "custom-openai",
|
|
"base_url": "https://proxy.example.test/v1",
|
|
"api_key": "draft-secret",
|
|
},
|
|
"model": {
|
|
"id": "chat-main",
|
|
"name": "Chat Main",
|
|
"model_id": "vendor/model",
|
|
},
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {"ok": True, "latency_ms": 42, "response": "OK"}
|
|
|
|
|
|
def test_default_model_api_requires_admin_token_header(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
|
|
response = client.put(
|
|
"/api/default-model",
|
|
json={"model": "gpt-5.4", "provider": "openai"},
|
|
)
|
|
assert response.status_code == 403
|
|
|
|
|
|
def test_default_model_api_persists_pair_and_preserves_config(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
save_config(EvoScientistConfig(default_workdir="/tmp/research"))
|
|
|
|
response = client.put(
|
|
"/api/default-model",
|
|
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
|
|
json={"model": "gpt-5.4", "provider": "openai"},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {"default": {"name": "gpt-5.4", "provider": "openai"}}
|
|
saved = load_config()
|
|
assert saved.model == "gpt-5.4"
|
|
assert saved.provider == "openai"
|
|
assert saved.default_workdir == "/tmp/research"
|
|
|
|
|
|
def test_default_model_api_accepts_configured_dynamic_model(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
|
|
provider_payload = {
|
|
"providers": [
|
|
{
|
|
"id": "lab-openai",
|
|
"name": "Lab OpenAI",
|
|
"adapter": "openai-compatible",
|
|
"base_url": "https://llm.example.test/v1",
|
|
"api_key": "provider-secret",
|
|
"enabled": True,
|
|
"models": [
|
|
{
|
|
"id": "lab-model",
|
|
"name": "Lab Model",
|
|
"model_id": "vendor/model",
|
|
"enabled": True,
|
|
}
|
|
],
|
|
}
|
|
]
|
|
}
|
|
assert (
|
|
client.put(
|
|
"/api/provider-profiles", headers=headers, json=provider_payload
|
|
).status_code
|
|
== 200
|
|
)
|
|
|
|
response = client.put(
|
|
"/api/default-model",
|
|
headers=headers,
|
|
json={"model": "lab-model", "provider": "lab-openai"},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
saved = load_config()
|
|
assert (saved.model, saved.provider) == ("lab-model", "lab-openai")
|
|
|
|
|
|
def test_default_model_api_rejects_unconfigured_pair(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
|
|
response = client.put(
|
|
"/api/default-model",
|
|
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
|
|
json={"model": "missing-model", "provider": "missing-provider"},
|
|
)
|
|
|
|
assert response.status_code == 400
|
|
assert "is not configured" in response.json()["error"]
|
|
|
|
|
|
def test_default_model_api_rejects_invalid_json(monkeypatch):
|
|
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
|
|
response = client.put(
|
|
"/api/default-model",
|
|
headers={
|
|
"X-EvoScientist-Admin-Token": "admin-secret",
|
|
"Content-Type": "application/json",
|
|
},
|
|
content="{invalid",
|
|
)
|
|
assert response.status_code == 400
|
|
assert response.json() == {"error": "Request body must be valid JSON."}
|
|
|
|
|
|
def test_entries_preserve_registry_order():
|
|
"""The picker uses position-in-list to rank providers per short name —
|
|
the JSON must preserve the order returned by ``list_models_by_provider``.
|
|
|
|
Stubs ``get_effective_config`` to keep the assertion focused on
|
|
registry order rather than implicitly depending on the ambient
|
|
deploy config.
|
|
"""
|
|
from EvoScientist.llm.models import list_models_by_provider
|
|
|
|
expected = [
|
|
{"name": n, "model_id": m, "provider": p}
|
|
for n, m, p in list_models_by_provider()
|
|
]
|
|
mock_cfg = EvoScientistConfig()
|
|
with patch(
|
|
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
|
):
|
|
resp = client.get("/api/models")
|
|
assert resp.json()["entries"] == expected
|
|
|
|
|
|
def test_unavailable_default_falls_back_to_first_picker_entry():
|
|
mock_cfg = EvoScientistConfig(model="some-retired-name", provider="some-provider")
|
|
with patch(
|
|
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
|
):
|
|
resp = client.get("/api/models")
|
|
first = resp.json()["entries"][0]
|
|
assert resp.json()["default"] == {
|
|
"name": first["name"],
|
|
"provider": first["provider"],
|
|
}
|
|
|
|
|
|
def test_custom_registry_default_does_not_reuse_stale_builtin_provider():
|
|
replace_provider_profiles(
|
|
{
|
|
"providers": [
|
|
{
|
|
"id": "open",
|
|
"name": "Open proxy",
|
|
"adapter": "openai",
|
|
"base_url": "https://proxy.example.test/v1",
|
|
"api_key": "provider-secret",
|
|
"enabled": True,
|
|
"models": [
|
|
{
|
|
"id": "gpt-5.5",
|
|
"name": "GPT 5.5",
|
|
"model_id": "gpt-5.5",
|
|
"enabled": True,
|
|
}
|
|
],
|
|
}
|
|
]
|
|
}
|
|
)
|
|
mock_cfg = EvoScientistConfig(model="gpt-5.5", provider="openai")
|
|
|
|
with patch(
|
|
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
|
|
):
|
|
body = client.get("/api/models").json()
|
|
|
|
assert body["entries"] == [
|
|
{"name": "gpt-5.5", "model_id": "gpt-5.5", "provider": "open"}
|
|
]
|
|
assert body["default"] == {"name": "gpt-5.5", "provider": "open"}
|
|
|
|
|
|
def test_ollama_models_appended_when_base_url_configured():
|
|
"""Mirrors the TUI ``/model`` picker: when ``ollama_base_url`` is set,
|
|
locally-pulled Ollama models are appended after the static registry
|
|
as ``provider: "ollama"`` entries.
|
|
"""
|
|
mock_cfg = EvoScientistConfig(
|
|
model="claude-sonnet-4-6",
|
|
provider="custom-anthropic",
|
|
ollama_base_url="http://localhost:11434",
|
|
)
|
|
|
|
async def fake_discover(_base_url, *, timeout):
|
|
return ["llama3:8b", "mistral:7b"]
|
|
|
|
with (
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http.get_effective_config",
|
|
return_value=mock_cfg,
|
|
),
|
|
patch(
|
|
"EvoScientist.llm.ollama_discovery.discover_ollama_models",
|
|
new=fake_discover,
|
|
),
|
|
):
|
|
body = client.get("/api/models").json()
|
|
|
|
# Assert the response is the static registry followed by the discovered
|
|
# Ollama suffix — robust to future static Ollama entries in the registry.
|
|
from EvoScientist.llm.models import list_models_by_provider
|
|
|
|
static_entries = [
|
|
{"name": n, "model_id": m, "provider": p}
|
|
for n, m, p in list_models_by_provider()
|
|
]
|
|
discovered_entries = [
|
|
{"name": "llama3:8b", "model_id": "llama3:8b", "provider": "ollama"},
|
|
{"name": "mistral:7b", "model_id": "mistral:7b", "provider": "ollama"},
|
|
]
|
|
assert body["entries"][: len(static_entries)] == static_entries
|
|
assert body["entries"][len(static_entries) :] == discovered_entries
|
|
# TUI's "Custom Ollama model…" sentinel is a widget-specific affordance —
|
|
# it must not appear on the HTTP surface.
|
|
assert not any(e["model_id"] == "__custom_ollama__" for e in body["entries"])
|
|
|
|
|
|
def test_ollama_discovery_skipped_when_base_url_absent():
|
|
"""No Ollama discovery should happen when ``ollama_base_url`` is unset —
|
|
matches the ``/model`` picker's gating. The probe function should never
|
|
be called in that case.
|
|
"""
|
|
mock_cfg = EvoScientistConfig(
|
|
model="claude-sonnet-4-6", provider="custom-anthropic"
|
|
)
|
|
calls: list[str | None] = []
|
|
|
|
async def spy_discover(base_url, *, timeout):
|
|
calls.append(base_url)
|
|
return []
|
|
|
|
with (
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http.get_effective_config",
|
|
return_value=mock_cfg,
|
|
),
|
|
patch(
|
|
"EvoScientist.llm.ollama_discovery.discover_ollama_models",
|
|
new=spy_discover,
|
|
),
|
|
):
|
|
body = client.get("/api/models").json()
|
|
|
|
assert calls == []
|
|
# Response is exactly the static registry — no Ollama additions whatsoever.
|
|
from EvoScientist.llm.models import list_models_by_provider
|
|
|
|
assert body["entries"] == [
|
|
{"name": n, "model_id": m, "provider": p}
|
|
for n, m, p in list_models_by_provider()
|
|
]
|
|
|
|
|
|
def test_final_answer_extracts_latest_ai_text_blocks():
|
|
async def fake_metadata(_thread_id):
|
|
return {"updated_at": "2026-07-06T14:14:53+00:00"}
|
|
|
|
async def fake_messages(_thread_id):
|
|
return [
|
|
HumanMessage(content="question"),
|
|
AIMessage(content="old answer"),
|
|
AIMessage(
|
|
content=[
|
|
{"type": "reasoning", "text": "internal"},
|
|
{"type": "text", "text": "Part A"},
|
|
{"type": "tool_use", "name": "search"},
|
|
{"type": "output_text", "text": "Part B"},
|
|
]
|
|
),
|
|
]
|
|
|
|
async def fake_runtime(_request, _thread_id):
|
|
return {
|
|
"found": True,
|
|
"complete": True,
|
|
"completed_at": "2026-07-06T14:15:00+00:00",
|
|
}
|
|
|
|
with (
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http._get_thread_metadata_for_http",
|
|
new=fake_metadata,
|
|
),
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http._get_thread_messages_for_http",
|
|
new=fake_messages,
|
|
),
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http._read_thread_runtime_state",
|
|
new=fake_runtime,
|
|
),
|
|
):
|
|
resp = client.get("/api/threads/thread-1/final-answer")
|
|
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {
|
|
"content": "Part A\n\nPart B",
|
|
"completed_at": "2026-07-06T14:15:00+00:00",
|
|
"complete": True,
|
|
}
|
|
|
|
|
|
def test_final_answer_skips_tool_selection_json_text():
|
|
async def fake_metadata(_thread_id):
|
|
return {"updated_at": "2026-07-06T14:14:53+00:00"}
|
|
|
|
async def fake_messages(_thread_id):
|
|
return [
|
|
HumanMessage(content="question"),
|
|
AIMessage(content="stable answer"),
|
|
AIMessage(
|
|
content=(
|
|
'{"tools":["search_papers","get_abstract"]}'
|
|
'{"tools":["web_search_exa"]}'
|
|
)
|
|
),
|
|
]
|
|
|
|
async def fake_runtime(_request, _thread_id):
|
|
return {
|
|
"found": True,
|
|
"complete": True,
|
|
"completed_at": "2026-07-06T14:15:00+00:00",
|
|
}
|
|
|
|
with (
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http._get_thread_metadata_for_http",
|
|
new=fake_metadata,
|
|
),
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http._get_thread_messages_for_http",
|
|
new=fake_messages,
|
|
),
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http._read_thread_runtime_state",
|
|
new=fake_runtime,
|
|
),
|
|
):
|
|
resp = client.get("/api/threads/thread-1/final-answer")
|
|
|
|
assert resp.status_code == 200
|
|
assert resp.json()["content"] == "stable answer"
|
|
|
|
|
|
def test_final_answer_returns_404_for_unknown_thread():
|
|
async def fake_metadata(_thread_id):
|
|
return None
|
|
|
|
with patch(
|
|
"EvoScientist.langgraph_dev.http._get_thread_metadata_for_http",
|
|
new=fake_metadata,
|
|
):
|
|
resp = client.get("/api/threads/missing/final-answer")
|
|
|
|
assert resp.status_code == 404
|
|
assert resp.json() == {"error": "thread not found"}
|
|
|
|
|
|
def test_final_answer_does_not_mark_complete_when_runtime_state_fails():
|
|
async def fake_metadata(_thread_id):
|
|
return {"updated_at": "2026-07-06T14:14:53+00:00"}
|
|
|
|
async def fake_messages(_thread_id):
|
|
return [AIMessage(content="checkpoint answer")]
|
|
|
|
async def fake_runtime(_request, _thread_id):
|
|
raise RuntimeError("langgraph runtime unavailable")
|
|
|
|
with (
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http._get_thread_metadata_for_http",
|
|
new=fake_metadata,
|
|
),
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http._get_thread_messages_for_http",
|
|
new=fake_messages,
|
|
),
|
|
patch(
|
|
"EvoScientist.langgraph_dev.http._read_thread_runtime_state",
|
|
new=fake_runtime,
|
|
),
|
|
):
|
|
resp = client.get("/api/threads/thread-1/final-answer")
|
|
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {
|
|
"content": "checkpoint answer",
|
|
"completed_at": None,
|
|
"complete": False,
|
|
}
|