ouyangbo/EvoScientist
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 3 Wiki Activity
Files
57176b359a7f578bc413a847eb4aed5d45df8018
EvoScientist/tests/test_provider_profiles.py
T
m4 8a0ab17936 chore: baseline WIP before unified model configuration implementation 
Pre-existing uncommitted work (runtime snapshots, message budget middleware) preserved as baseline.
2026-07-20 20:15:38 +08:00

486 lines
16 KiB
Python
Raw Blame History

"""Tests for runtime-editable provider profiles."""
from __future__ import annotations
import os
from unittest.mock import patch
import pytest
from EvoScientist.config import EvoScientistConfig, apply_config_to_env, save_config
from EvoScientist.config.provider_profiles import (
ProviderProfileError,
get_builtin_provider_profile,
get_provider_profile_revision,
get_provider_profiles_path,
list_configured_model_entries,
load_provider_profiles,
provider_profiles_public,
replace_provider_profiles,
resolve_provider_profile_draft,
)
from EvoScientist.llm.models import get_chat_model, list_model_picker_entries
@pytest.fixture
def provider_config_dir(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
return tmp_path / "evoscientist"
def _document(api_key: str = "secret-key") -> dict:
return {
"version": 3,
"providers": [
{
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1/",
"api_key": api_key,
"enabled": True,
"runtime": {
"timeout_seconds": 90,
"max_retries": 1,
"default_temperature": 0.3,
"default_top_p": None,
"default_reasoning_effort": "auto",
},
"models": [
{
"id": "research-model",
"name": "Research Model",
"model_id": "vendor/research-1",
"enabled": True,
"runtime": {
"limit_mode": "combined",
"context_window_tokens": 32768,
"max_output_tokens": 4096,
"min_effective_input_tokens": 4096,
"limits_status": "confirmed",
"limits_source": "user",
},
},
{
"id": "disabled-model",
"name": "Disabled Model",
"model_id": "vendor/disabled",
"enabled": False,
"runtime": {
"limit_mode": "combined",
"context_window_tokens": 32768,
"max_output_tokens": 4096,
"min_effective_input_tokens": 4096,
"limits_status": "confirmed",
"limits_source": "user",
},
},
],
}
]
}
def _builtin_document(api_key: str = "builtin-secret") -> dict:
return {
"version": 3,
"builtins": [
{
"id": "openai",
"name": "OpenAI",
"adapter": "openai",
"base_url": "https://proxy.example.test/v1",
"api_key": api_key,
"auth_mode": "api_key",
"enabled": True,
"runtime": {},
"models": [
{
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
"runtime": {
"limit_mode": "combined",
"context_window_tokens": 32768,
"max_output_tokens": 4096,
"min_effective_input_tokens": 4096,
"limits_status": "confirmed",
"limits_source": "user",
},
}
],
}
]
}
def test_replace_round_trip_and_redacts_api_key(provider_config_dir):
saved = replace_provider_profiles(_document())
assert saved.providers[0].base_url == "https://llm.example.test/v1"
assert load_provider_profiles() == saved
public = provider_profiles_public(saved)
assert "api_key" not in public["providers"][0]
assert public["providers"][0]["api_key_configured"] is True
assert public["providers"][0]["api_key_hint"] == "sec...-key"
assert "openai" in public["reserved_provider_ids"]
path = get_provider_profiles_path()
assert path == provider_config_dir / "providers.yaml"
if os.name != "nt":
assert path.parent.stat().st_mode & 0o777 == 0o700
assert path.stat().st_mode & 0o777 == 0o600
def test_v1_document_requires_reset(provider_config_dir):
path = get_provider_profiles_path()
path.parent.mkdir(parents=True)
path.write_text(
"version: 1\nproviders:\n"
" - id: lab-openai\n"
" name: Lab OpenAI\n"
" adapter: openai-compatible\n"
" base_url: https://llm.example.test/v1\n"
" api_key: old-secret\n"
" enabled: true\n"
" models: []\n",
encoding="utf-8",
)
with pytest.raises(ProviderProfileError, match="PROVIDER_PROFILE_RESET_REQUIRED"):
load_provider_profiles()
def test_builtin_profiles_round_trip_and_redact_secret(provider_config_dir):
saved = replace_provider_profiles(_builtin_document())
assert saved.builtins[0].id == "openai"
assert get_builtin_provider_profile("openai") == saved.builtins[0]
public = provider_profiles_public(saved)
assert public["builtins"][0]["api_key_configured"] is True
assert "builtin-secret" not in str(public)
def test_builtin_private_revision_changes_with_connection_secret(
provider_config_dir,
):
replace_provider_profiles(_builtin_document("first-secret"))
first = get_provider_profile_revision("openai")
replace_provider_profiles(_builtin_document("second-secret"))
second = get_provider_profile_revision("openai")
assert first
assert second
assert first != second
def test_custom_update_preserves_builtin_profiles(provider_config_dir):
replace_provider_profiles(_builtin_document())
saved = replace_provider_profiles(_document())
assert saved.builtins[0].id == "openai"
assert saved.providers[0].id == "lab-openai"
def test_builtin_update_preserves_custom_profiles(provider_config_dir):
replace_provider_profiles(_document())
saved = replace_provider_profiles(_builtin_document())
assert saved.builtins[0].id == "openai"
assert saved.providers[0].id == "lab-openai"
def test_empty_api_key_keeps_existing_secret(provider_config_dir):
replace_provider_profiles(_document("first-secret"))
update = _document("")
update["providers"][0]["name"] = "Renamed"
saved = replace_provider_profiles(update)
assert saved.providers[0].name == "Renamed"
assert saved.providers[0].api_key == "first-secret"
def test_public_revision_does_not_fingerprint_api_key(provider_config_dir):
first = provider_profiles_public(
replace_provider_profiles(_document("abc-first-1234"))
)["revision"]
second = provider_profiles_public(
replace_provider_profiles(_document("abc-second-1234"))
)["revision"]
assert first == second
def test_clear_api_key_is_explicit(provider_config_dir):
replace_provider_profiles(_document("first-secret"))
update = _document("")
update["providers"][0]["clear_api_key"] = True
saved = replace_provider_profiles(update)
assert saved.providers[0].api_key == ""
def test_draft_reuses_saved_api_key_without_exposing_it(provider_config_dir):
replace_provider_profiles(_document("saved-secret"))
draft = _document("")["providers"][0]
draft["models"] = []
resolved = resolve_provider_profile_draft(draft)
assert resolved.api_key == "saved-secret"
assert resolved.models == ()
def test_draft_can_explicitly_clear_saved_api_key(provider_config_dir):
replace_provider_profiles(_document("saved-secret"))
draft = _document("")["providers"][0]
draft.update({"models": [], "clear_api_key": True})
resolved = resolve_provider_profile_draft(draft)
assert resolved.api_key == ""
def test_reserved_provider_id_is_rejected(provider_config_dir):
raw = _document()
raw["providers"][0]["id"] = "openai"
with pytest.raises(ProviderProfileError, match="openai-custom"):
replace_provider_profiles(raw)
@pytest.mark.parametrize(
("adapter", "expected"),
[
("openai", "https://llm.example.test/v1"),
("openai-compatible", "https://llm.example.test/v1"),
("openrouter", "https://llm.example.test/api/v1"),
("google-genai", "https://llm.example.test/v1beta"),
("anthropic", "https://llm.example.test"),
],
)
def test_bare_provider_origin_gets_adapter_api_path(
provider_config_dir, adapter, expected
):
raw = _document()
raw["providers"][0].update(
{
"id": "lab-provider",
"adapter": adapter,
"base_url": "https://llm.example.test",
}
)
saved = replace_provider_profiles(raw)
assert saved.providers[0].base_url == expected
def test_dynamic_models_are_listed_and_disabled_models_are_hidden(
provider_config_dir,
):
replace_provider_profiles(_document())
assert list_configured_model_entries() == [
("research-model", "vendor/research-1", "lab-openai")
]
@pytest.mark.asyncio
async def test_custom_registry_replaces_static_picker_entries(provider_config_dir):
replace_provider_profiles(_document())
entries = await list_model_picker_entries(None, include_custom_ollama=False)
assert entries == [("research-model", "vendor/research-1", "lab-openai")]
@pytest.mark.asyncio
async def test_empty_registry_hides_static_picker_entries(provider_config_dir):
replace_provider_profiles({"providers": []})
entries = await list_model_picker_entries(None, include_custom_ollama=False)
assert entries == []
@pytest.mark.asyncio
async def test_managed_builtins_replace_static_picker_entries(provider_config_dir):
replace_provider_profiles(_builtin_document())
entries = await list_model_picker_entries(None, include_custom_ollama=False)
assert entries == [("chat-main", "gpt-upstream", "openai")]
def test_dynamic_profile_routes_through_selected_adapter(provider_config_dir):
replace_provider_profiles(_document())
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider="lab-openai")
_, kwargs = init_model.call_args
assert kwargs["model"] == "vendor/research-1"
assert kwargs["model_provider"] == "openai"
assert kwargs["base_url"] == "https://llm.example.test/v1"
assert kwargs["api_key"] == "secret-key"
assert kwargs["timeout"] == 90
assert kwargs["max_retries"] == 1
assert kwargs["max_tokens"] == 4096
assert kwargs["default_headers"]["User-Agent"] == "codex_cli_rs/0.0.0"
def test_dynamic_native_openai_adapter_uses_gateway_compatible_user_agent(
provider_config_dir,
):
raw = _document()
raw["providers"][0]["adapter"] = "openai"
replace_provider_profiles(raw)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider="lab-openai")
_, kwargs = init_model.call_args
assert kwargs["model_provider"] == "openai"
assert kwargs["default_headers"]["User-Agent"] == "codex_cli_rs/0.0.0"
def test_builtin_catalog_alias_routes_to_upstream_model(provider_config_dir):
save_config(
EvoScientistConfig(
provider="openai",
model="chat-main",
model_catalog=[
{
"provider": "openai",
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
}
],
)
)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["model"] == "gpt-upstream"
assert kwargs["model_provider"] == "openai"
def test_managed_builtin_routes_connection_without_restart(
provider_config_dir, monkeypatch
):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
replace_provider_profiles(_builtin_document())
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["model"] == "gpt-upstream"
assert kwargs["model_provider"] == "openai"
assert kwargs["base_url"] == "https://proxy.example.test/v1"
assert kwargs["api_key"] == "builtin-secret"
def test_environment_overrides_managed_builtin_connection(
provider_config_dir, monkeypatch
):
replace_provider_profiles(_builtin_document())
monkeypatch.setenv("OPENAI_API_KEY", "environment-secret")
monkeypatch.setenv("OPENAI_BASE_URL", "https://env.example.test/v1")
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["base_url"] == "https://env.example.test/v1"
assert kwargs["api_key"] == "environment-secret"
def test_managed_builtin_overrides_legacy_config_injected_environment(
provider_config_dir, monkeypatch
):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
apply_config_to_env(
EvoScientistConfig(
openai_api_key="legacy-secret",
)
)
replace_provider_profiles(_builtin_document("registry-secret"))
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["api_key"] == "registry-secret"
def test_dynamic_ollama_profile_does_not_pass_api_key(provider_config_dir):
raw = _document("unused-secret")
raw["providers"][0].update(
{
"id": "local-ollama",
"adapter": "ollama",
"base_url": "http://127.0.0.1:11434",
}
)
replace_provider_profiles(raw)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider="local-ollama")
_, kwargs = init_model.call_args
assert kwargs["model_provider"] == "ollama"
assert kwargs["base_url"] == "http://127.0.0.1:11434"
assert "api_key" not in kwargs
@pytest.mark.parametrize(
("adapter", "base_url"),
[
("grok", ""),
("antigravity", "http://127.0.0.1:8317/v1"),
],
)
def test_openai_protocol_adapters_route_through_openai(
provider_config_dir, adapter, base_url
):
raw = _document()
raw["providers"][0].update(
{
"id": f"lab-{adapter}",
"adapter": adapter,
"base_url": base_url,
}
)
replace_provider_profiles(raw)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider=f"lab-{adapter}")
_, kwargs = init_model.call_args
assert kwargs["model_provider"] == "openai"
assert kwargs["base_url"] == (base_url or "https://api.x.ai/v1")
def test_revision_changes_when_credentials_change(provider_config_dir):
replace_provider_profiles(_document("first-secret"))
first = get_provider_profile_revision("lab-openai")
replace_provider_profiles(_document("second-secret"))
second = get_provider_profile_revision("lab-openai")
assert first
assert second
assert first != second
Reference in New Issue View Git Blame Copy Permalink