"""Smoke test for the /api/models route mounted via langgraph.json's ``http`` field. We test the FastAPI app directly — no need to spin up langgraph dev. """ from __future__ import annotations from unittest.mock import patch import pytest from langchain_core.messages import AIMessage, HumanMessage from starlette.testclient import TestClient from EvoScientist.config import EvoScientistConfig, load_config, save_config from EvoScientist.config.provider_profiles import ( load_provider_profiles, replace_provider_profiles, ) from EvoScientist.langgraph_dev.http import app from EvoScientist.llm.provider_operations import ( DiscoveredProviderModel, ProviderModelTestResult, ProviderOperationError, ) client = TestClient(app) @pytest.fixture(autouse=True) def isolate_xdg_config(tmp_path, monkeypatch): """Keep HTTP tests independent from the developer's saved providers.""" monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) def test_get_models_returns_entries_and_default(): mock_cfg = EvoScientistConfig( model="claude-sonnet-4-6", provider="custom-anthropic" ) with patch( "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg ): resp = client.get("/api/models") assert resp.status_code == 200 body = resp.json() assert "entries" in body assert "default" in body assert body["default"] == { "name": "claude-sonnet-4-6", "provider": "custom-anthropic", } assert isinstance(body["entries"], list) assert len(body["entries"]) > 0 # Every entry has the three required keys for entry in body["entries"]: assert set(entry.keys()) == {"name", "model_id", "provider"} assert isinstance(entry["name"], str) assert entry["name"] assert isinstance(entry["model_id"], str) assert entry["model_id"] assert isinstance(entry["provider"], str) assert entry["provider"] def test_get_models_only_returns_enabled_catalog_entries_when_configured(): mock_cfg = EvoScientistConfig( model="chat-main", provider="openai", model_catalog=[ { "provider": "openai", "id": "chat-main", "name": "Chat Main", "model_id": "gpt-upstream", "enabled": True, }, { "provider": "anthropic", "id": "hidden-model", "name": "Hidden", "model_id": "claude-hidden", "enabled": False, }, ], ) with patch( "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg ): body = client.get("/api/models").json() assert body["entries"] == [ {"name": "chat-main", "model_id": "gpt-upstream", "provider": "openai"} ] def test_get_models_empty_catalog_hides_all_builtin_models(): mock_cfg = EvoScientistConfig(model_catalog=[]) with patch( "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg ): body = client.get("/api/models").json() assert body["entries"] == [] def test_provider_profiles_api_requires_admin_token_header(tmp_path, monkeypatch): monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False) response = client.get("/api/provider-profiles") assert response.status_code == 403 def test_provider_profiles_api_round_trip_redacts_secret(tmp_path, monkeypatch): monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} payload = { "providers": [ { "id": "lab-openai", "name": "Lab OpenAI", "adapter": "openai-compatible", "base_url": "https://llm.example.test/v1", "api_key": "provider-secret", "enabled": True, "runtime": {"timeout_seconds": 90, "max_retries": 1}, "models": [ { "id": "lab-model", "name": "Lab Model", "model_id": "vendor/model", "enabled": True, "runtime": { "limit_mode": "combined", "context_window_tokens": 32_768, "max_output_tokens": 4_096, "min_effective_input_tokens": 4_096, "limits_status": "confirmed", "limits_source": "provider", }, } ], } ] } put_response = client.put("/api/provider-profiles", headers=headers, json=payload) assert put_response.status_code == 200 assert "provider-secret" not in put_response.text assert put_response.json()["providers"][0]["api_key_configured"] is True assert "openai" in put_response.json()["reserved_provider_ids"] get_response = client.get("/api/provider-profiles", headers=headers) assert get_response.status_code == 200 assert get_response.json()["providers"][0]["models"][0]["id"] == "lab-model" assert "provider-secret" not in get_response.text snapshot_response = client.post( "/api/runtime-snapshots", headers=headers, json={ "snapshot_id": "run-request-a", "model": "lab-model", "provider": "lab-openai", }, ) assert snapshot_response.status_code == 200 snapshot = snapshot_response.json()["snapshot"] assert snapshot["runtime"]["max_input_tokens"] == 28_672 assert "provider-secret" not in snapshot_response.text def test_runtime_snapshot_reports_missing_zhipu_key_before_creating_a_run( monkeypatch, ): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") monkeypatch.delenv("ZHIPU_API_KEY", raising=False) monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") response = client.post( "/api/runtime-snapshots", headers={"X-EvoScientist-Admin-Token": "admin-secret"}, json={ "snapshot_id": "run-request-zhipu", "model": "glm-5.2", "provider": "glm", }, ) assert response.status_code == 400 assert response.json()["error"].startswith("ZHIPU_API_KEY_NOT_CONFIGURED") def test_llm_config_api_requires_admin_token_header(monkeypatch): monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False) response = client.get("/api/config") assert response.status_code == 403 def test_llm_config_api_redacts_secrets_and_reports_env_overrides(monkeypatch): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") monkeypatch.setenv("OPENAI_API_KEY", "environment-secret") save_config( EvoScientistConfig( provider="openai", model="gpt-5.4", openai_api_key="file-secret-1234", default_workdir="/tmp/research", ) ) response = client.get( "/api/config", headers={"X-EvoScientist-Admin-Token": "admin-secret"}, ) assert response.status_code == 200 body = response.json() assert body["values"]["provider"] == "openai" assert body["values"]["model"] == "gpt-5.4" assert "openai_api_key" not in body["values"] assert body["secrets"]["openai_api_key"] == { "configured": True, "hint": "...1234", } assert body["env_overrides"]["openai_api_key"] == "OPENAI_API_KEY" assert "file-secret-1234" not in response.text assert "environment-secret" not in response.text assert "default_workdir" not in body["values"] assert body["model_catalog"] is None assert body["builtin_model_candidates"] openai = next( provider for provider in body["builtin_providers"] if provider["id"] == "openai" ) assert openai["managed"] is False assert openai["api_key_configured"] is True assert "environment-secret" not in response.text def test_llm_config_api_saves_builtin_registry_without_rewriting_legacy_secret( monkeypatch, ): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") monkeypatch.delenv("OPENAI_API_KEY", raising=False) headers = {"X-EvoScientist-Admin-Token": "admin-secret"} save_config( EvoScientistConfig( provider="openai", model="chat-main", openai_api_key="legacy-secret", default_workdir="/tmp/research", ) ) loaded = client.get("/api/config", headers=headers).json() openai = next( provider for provider in loaded["builtin_providers"] if provider["id"] == "openai" ) openai.update( { "managed": True, "base_url": "https://proxy.example.test/v1", "models": [ { "id": "chat-main", "name": "Chat Main", "model_id": "gpt-upstream", "enabled": True, } ], } ) response = client.patch( "/api/config", headers=headers, json={ "revision": loaded["revision"], "values": loaded["values"], "secrets": {}, "clear_secrets": [], "builtin_providers": [openai], }, ) assert response.status_code == 200 body = response.json() assert body["restart_required"] is False assert "builtin_providers" in body["changed_fields"] saved_profile = load_provider_profiles().builtins[0] assert saved_profile.id == "openai" assert saved_profile.api_key == "legacy-secret" assert saved_profile.base_url == "https://proxy.example.test/v1" saved_config = load_config() assert saved_config.openai_api_key == "legacy-secret" assert saved_config.default_workdir == "/tmp/research" assert saved_config.model_catalog is None models = client.get("/api/models").json()["entries"] assert models == [ {"name": "chat-main", "model_id": "gpt-upstream", "provider": "openai"} ] def test_first_builtin_registry_save_migrates_legacy_default_model(monkeypatch): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://environment.example.test") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} save_config(EvoScientistConfig(provider="anthropic", model="claude-sonnet-4-6")) loaded = client.get("/api/config", headers=headers).json() ollama = next( provider for provider in loaded["builtin_providers"] if provider["id"] == "ollama" ) ollama.update( { "managed": True, "base_url": "http://127.0.0.1:11434", "models": [], } ) response = client.patch( "/api/config", headers=headers, json={ "revision": loaded["revision"], "values": loaded["values"], "secrets": {}, "clear_secrets": [], "builtin_providers": [ollama], }, ) assert response.status_code == 200 builtins = {profile.id: profile for profile in load_provider_profiles().builtins} assert set(builtins) == {"anthropic", "ollama"} assert builtins["anthropic"].models[0].id == "claude-sonnet-4-6" assert builtins["anthropic"].base_url == "" def test_llm_config_api_persists_model_catalog(monkeypatch): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} save_config(EvoScientistConfig(provider="openai", model="chat-main")) revision = client.get("/api/config", headers=headers).json()["revision"] catalog = [ { "provider": "openai", "id": "chat-main", "name": "Chat Main", "model_id": "gpt-upstream", "enabled": True, } ] response = client.patch( "/api/config", headers=headers, json={ "revision": revision, "values": {}, "secrets": {}, "clear_secrets": [], "model_catalog": catalog, }, ) assert response.status_code == 200 assert response.json()["model_catalog"] == catalog assert response.json()["restart_required"] is False assert load_config().model_catalog == catalog def test_llm_config_api_rejects_default_outside_model_catalog(monkeypatch): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} save_config(EvoScientistConfig(provider="openai", model="gpt-default")) revision = client.get("/api/config", headers=headers).json()["revision"] response = client.patch( "/api/config", headers=headers, json={ "revision": revision, "values": {}, "secrets": {}, "clear_secrets": [], "model_catalog": [ { "provider": "openai", "id": "different-model", "name": "Different", "model_id": "gpt-different", "enabled": True, } ], }, ) assert response.status_code == 400 assert "Default model" in response.json()["error"] assert load_config().model_catalog is None def test_llm_config_api_patch_preserves_unrelated_fields_and_secret_by_default( monkeypatch, ): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} save_config( EvoScientistConfig( openai_api_key="saved-openai-key", anthropic_api_key="saved-anthropic-key", default_workdir="/tmp/research", ) ) revision = client.get("/api/config", headers=headers).json()["revision"] response = client.patch( "/api/config", headers=headers, json={ "revision": revision, "values": { "provider": "openai", "model": "gpt-5.4", "ollama_base_url": "http://127.0.0.1:11434", }, "secrets": {"openai_api_key": "replacement-openai-key"}, "clear_secrets": ["anthropic_api_key"], }, ) assert response.status_code == 200 body = response.json() assert body["restart_required"] is True assert set(body["changed_fields"]) == { "anthropic_api_key", "model", "ollama_base_url", "openai_api_key", "provider", } assert "replacement-openai-key" not in response.text saved = load_config() assert saved.provider == "openai" assert saved.model == "gpt-5.4" assert saved.ollama_base_url == "http://127.0.0.1:11434" assert saved.openai_api_key == "replacement-openai-key" assert saved.anthropic_api_key == "" assert saved.default_workdir == "/tmp/research" def test_llm_config_api_rejects_stale_revision(monkeypatch): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} save_config(EvoScientistConfig(model="claude-sonnet-4-6")) revision = client.get("/api/config", headers=headers).json()["revision"] save_config(EvoScientistConfig(model="gpt-5.4", provider="openai")) response = client.patch( "/api/config", headers=headers, json={ "revision": revision, "values": {"model": "claude-opus-4-8"}, }, ) assert response.status_code == 409 assert "Reload and try again" in response.json()["error"] assert load_config().model == "gpt-5.4" @pytest.mark.parametrize( ("payload", "message"), [ ( {"values": {"openai_auth_mode": "password"}}, "openai_auth_mode must be 'api_key' or 'oauth'", ), ( {"values": {"default_workdir": "/tmp/other"}}, "Unsupported config fields: default_workdir", ), ( {"values": {"ollama_base_url": "localhost:11434"}}, "ollama_base_url must use http:// or https://", ), ], ) def test_llm_config_api_validates_updates(monkeypatch, payload, message): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} revision = client.get("/api/config", headers=headers).json()["revision"] response = client.patch( "/api/config", headers=headers, json={"revision": revision, **payload}, ) assert response.status_code == 400 assert message in response.json()["error"] def test_provider_actions_api_requires_admin_token_header(tmp_path, monkeypatch): monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False) response = client.post("/api/provider-actions", json={}) assert response.status_code == 403 def test_provider_actions_lists_models_with_saved_api_key(tmp_path, monkeypatch): monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} provider = { "id": "lab-openai", "name": "Lab OpenAI", "adapter": "openai-compatible", "base_url": "https://llm.example.test/v1", "api_key": "provider-secret", "enabled": True, "models": [], } assert ( client.put( "/api/provider-profiles", headers=headers, json={"providers": [provider]}, ).status_code == 200 ) provider["api_key"] = "" async def fake_discover(profile): assert profile.api_key == "provider-secret" return [DiscoveredProviderModel("vendor/model", "Vendor Model")] with patch( "EvoScientist.langgraph_dev.http.discover_provider_models", new=fake_discover, ): response = client.post( "/api/provider-actions", headers=headers, json={"action": "list_models", "provider": provider}, ) assert response.status_code == 200 assert response.json() == { "models": [{"model_id": "vendor/model", "name": "Vendor Model"}] } def test_provider_actions_tests_model(tmp_path, monkeypatch): monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} async def fake_test(profile, model): assert profile.adapter == "grok" assert model.model_id == "grok-4" return ProviderModelTestResult(latency_ms=123, response="OK") with patch("EvoScientist.langgraph_dev.http.test_provider_model", new=fake_test): response = client.post( "/api/provider-actions", headers=headers, json={ "action": "test_model", "provider": { "id": "lab-grok", "name": "Lab Grok", "adapter": "grok", "base_url": "", "api_key": "xai-secret", "enabled": True, "models": [], }, "model": { "id": "grok", "name": "Grok", "model_id": "grok-4", }, }, ) assert response.status_code == 200 assert response.json() == {"ok": True, "latency_ms": 123, "response": "OK"} def test_provider_actions_maps_provider_failures_to_bad_gateway(tmp_path, monkeypatch): monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") async def fake_discover(_profile): raise ProviderOperationError("Provider returned HTTP 401") with patch( "EvoScientist.langgraph_dev.http.discover_provider_models", new=fake_discover, ): response = client.post( "/api/provider-actions", headers={"X-EvoScientist-Admin-Token": "admin-secret"}, json={ "action": "list_models", "provider": { "id": "lab-openai", "name": "Lab OpenAI", "adapter": "openai-compatible", "base_url": "https://llm.example.test/v1", "api_key": "bad-key", "enabled": True, "models": [], }, }, ) assert response.status_code == 502 assert response.json() == {"error": "Provider returned HTTP 401"} def test_llm_config_action_discovers_models_with_effective_builtin_secret( monkeypatch, ): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") monkeypatch.setenv("OPENAI_API_KEY", "environment-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} async def fake_discover(profile): assert profile.id == "openai" assert profile.adapter == "openai" assert profile.api_key == "environment-secret" return [DiscoveredProviderModel("gpt-discovered", "GPT Discovered")] with patch( "EvoScientist.langgraph_dev.http.discover_provider_models", new=fake_discover, ): response = client.post( "/api/config", headers=headers, json={"action": "list_models", "provider": {"id": "openai"}}, ) assert response.status_code == 200 assert response.json() == { "models": [{"model_id": "gpt-discovered", "name": "GPT Discovered"}] } def test_llm_config_action_tests_builtin_model_with_draft_connection(monkeypatch): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} async def fake_test(profile, model): assert profile.id == "custom-openai" assert profile.adapter == "openai-compatible" assert profile.base_url == "https://proxy.example.test/v1" assert profile.api_key == "draft-secret" assert model.model_id == "vendor/model" return ProviderModelTestResult(latency_ms=42, response="OK") with patch("EvoScientist.langgraph_dev.http.test_provider_model", new=fake_test): response = client.post( "/api/config", headers=headers, json={ "action": "test_model", "provider": { "id": "custom-openai", "base_url": "https://proxy.example.test/v1", "api_key": "draft-secret", }, "model": { "id": "chat-main", "name": "Chat Main", "model_id": "vendor/model", }, }, ) assert response.status_code == 200 assert response.json() == {"ok": True, "latency_ms": 42, "response": "OK"} def test_default_model_api_requires_admin_token_header(tmp_path, monkeypatch): monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False) response = client.put( "/api/default-model", json={"model": "gpt-5.4", "provider": "openai"}, ) assert response.status_code == 403 def test_default_model_api_persists_pair_and_preserves_config(tmp_path, monkeypatch): monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") save_config(EvoScientistConfig(default_workdir="/tmp/research")) response = client.put( "/api/default-model", headers={"X-EvoScientist-Admin-Token": "admin-secret"}, json={"model": "gpt-5.4", "provider": "openai"}, ) assert response.status_code == 200 assert response.json() == {"default": {"name": "gpt-5.4", "provider": "openai"}} saved = load_config() assert saved.model == "gpt-5.4" assert saved.provider == "openai" assert saved.default_workdir == "/tmp/research" def test_default_model_api_accepts_configured_dynamic_model(tmp_path, monkeypatch): monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") headers = {"X-EvoScientist-Admin-Token": "admin-secret"} provider_payload = { "providers": [ { "id": "lab-openai", "name": "Lab OpenAI", "adapter": "openai-compatible", "base_url": "https://llm.example.test/v1", "api_key": "provider-secret", "enabled": True, "models": [ { "id": "lab-model", "name": "Lab Model", "model_id": "vendor/model", "enabled": True, } ], } ] } assert ( client.put( "/api/provider-profiles", headers=headers, json=provider_payload ).status_code == 200 ) response = client.put( "/api/default-model", headers=headers, json={"model": "lab-model", "provider": "lab-openai"}, ) assert response.status_code == 200 saved = load_config() assert (saved.model, saved.provider) == ("lab-model", "lab-openai") def test_default_model_api_rejects_unconfigured_pair(tmp_path, monkeypatch): monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") response = client.put( "/api/default-model", headers={"X-EvoScientist-Admin-Token": "admin-secret"}, json={"model": "missing-model", "provider": "missing-provider"}, ) assert response.status_code == 400 assert "is not configured" in response.json()["error"] def test_default_model_api_rejects_invalid_json(monkeypatch): monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") response = client.put( "/api/default-model", headers={ "X-EvoScientist-Admin-Token": "admin-secret", "Content-Type": "application/json", }, content="{invalid", ) assert response.status_code == 400 assert response.json() == {"error": "Request body must be valid JSON."} def test_entries_preserve_registry_order(): """The picker uses position-in-list to rank providers per short name — the JSON must preserve the order returned by ``list_models_by_provider``. Stubs ``get_effective_config`` to keep the assertion focused on registry order rather than implicitly depending on the ambient deploy config. """ from EvoScientist.llm.models import list_models_by_provider expected = [ {"name": n, "model_id": m, "provider": p} for n, m, p in list_models_by_provider() ] mock_cfg = EvoScientistConfig() with patch( "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg ): resp = client.get("/api/models") assert resp.json()["entries"] == expected def test_unavailable_default_falls_back_to_first_picker_entry(): mock_cfg = EvoScientistConfig(model="some-retired-name", provider="some-provider") with patch( "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg ): resp = client.get("/api/models") first = resp.json()["entries"][0] assert resp.json()["default"] == { "name": first["name"], "provider": first["provider"], } def test_custom_registry_default_does_not_reuse_stale_builtin_provider(): replace_provider_profiles( { "providers": [ { "id": "open", "name": "Open proxy", "adapter": "openai", "base_url": "https://proxy.example.test/v1", "api_key": "provider-secret", "enabled": True, "models": [ { "id": "gpt-5.5", "name": "GPT 5.5", "model_id": "gpt-5.5", "enabled": True, } ], } ] } ) mock_cfg = EvoScientistConfig(model="gpt-5.5", provider="openai") with patch( "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg ): body = client.get("/api/models").json() assert body["entries"] == [ {"name": "gpt-5.5", "model_id": "gpt-5.5", "provider": "open"} ] assert body["default"] == {"name": "gpt-5.5", "provider": "open"} def test_ollama_models_appended_when_base_url_configured(): """Mirrors the TUI ``/model`` picker: when ``ollama_base_url`` is set, locally-pulled Ollama models are appended after the static registry as ``provider: "ollama"`` entries. """ mock_cfg = EvoScientistConfig( model="claude-sonnet-4-6", provider="custom-anthropic", ollama_base_url="http://localhost:11434", ) async def fake_discover(_base_url, *, timeout): return ["llama3:8b", "mistral:7b"] with ( patch( "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg, ), patch( "EvoScientist.llm.ollama_discovery.discover_ollama_models", new=fake_discover, ), ): body = client.get("/api/models").json() # Assert the response is the static registry followed by the discovered # Ollama suffix — robust to future static Ollama entries in the registry. from EvoScientist.llm.models import list_models_by_provider static_entries = [ {"name": n, "model_id": m, "provider": p} for n, m, p in list_models_by_provider() ] discovered_entries = [ {"name": "llama3:8b", "model_id": "llama3:8b", "provider": "ollama"}, {"name": "mistral:7b", "model_id": "mistral:7b", "provider": "ollama"}, ] assert body["entries"][: len(static_entries)] == static_entries assert body["entries"][len(static_entries) :] == discovered_entries # TUI's "Custom Ollama model…" sentinel is a widget-specific affordance — # it must not appear on the HTTP surface. assert not any(e["model_id"] == "__custom_ollama__" for e in body["entries"]) def test_ollama_discovery_skipped_when_base_url_absent(): """No Ollama discovery should happen when ``ollama_base_url`` is unset — matches the ``/model`` picker's gating. The probe function should never be called in that case. """ mock_cfg = EvoScientistConfig( model="claude-sonnet-4-6", provider="custom-anthropic" ) calls: list[str | None] = [] async def spy_discover(base_url, *, timeout): calls.append(base_url) return [] with ( patch( "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg, ), patch( "EvoScientist.llm.ollama_discovery.discover_ollama_models", new=spy_discover, ), ): body = client.get("/api/models").json() assert calls == [] # Response is exactly the static registry — no Ollama additions whatsoever. from EvoScientist.llm.models import list_models_by_provider assert body["entries"] == [ {"name": n, "model_id": m, "provider": p} for n, m, p in list_models_by_provider() ] def test_final_answer_extracts_latest_ai_text_blocks(): async def fake_metadata(_thread_id): return {"updated_at": "2026-07-06T14:14:53+00:00"} async def fake_messages(_thread_id): return [ HumanMessage(content="question"), AIMessage(content="old answer"), AIMessage( content=[ {"type": "reasoning", "text": "internal"}, {"type": "text", "text": "Part A"}, {"type": "tool_use", "name": "search"}, {"type": "output_text", "text": "Part B"}, ] ), ] async def fake_runtime(_request, _thread_id): return { "found": True, "complete": True, "completed_at": "2026-07-06T14:15:00+00:00", } with ( patch( "EvoScientist.langgraph_dev.http._get_thread_metadata_for_http", new=fake_metadata, ), patch( "EvoScientist.langgraph_dev.http._get_thread_messages_for_http", new=fake_messages, ), patch( "EvoScientist.langgraph_dev.http._read_thread_runtime_state", new=fake_runtime, ), ): resp = client.get("/api/threads/thread-1/final-answer") assert resp.status_code == 200 assert resp.json() == { "content": "Part A\n\nPart B", "completed_at": "2026-07-06T14:15:00+00:00", "complete": True, } def test_final_answer_skips_tool_selection_json_text(): async def fake_metadata(_thread_id): return {"updated_at": "2026-07-06T14:14:53+00:00"} async def fake_messages(_thread_id): return [ HumanMessage(content="question"), AIMessage(content="stable answer"), AIMessage( content=( '{"tools":["search_papers","get_abstract"]}' '{"tools":["web_search_exa"]}' ) ), ] async def fake_runtime(_request, _thread_id): return { "found": True, "complete": True, "completed_at": "2026-07-06T14:15:00+00:00", } with ( patch( "EvoScientist.langgraph_dev.http._get_thread_metadata_for_http", new=fake_metadata, ), patch( "EvoScientist.langgraph_dev.http._get_thread_messages_for_http", new=fake_messages, ), patch( "EvoScientist.langgraph_dev.http._read_thread_runtime_state", new=fake_runtime, ), ): resp = client.get("/api/threads/thread-1/final-answer") assert resp.status_code == 200 assert resp.json()["content"] == "stable answer" def test_final_answer_returns_404_for_unknown_thread(): async def fake_metadata(_thread_id): return None with patch( "EvoScientist.langgraph_dev.http._get_thread_metadata_for_http", new=fake_metadata, ): resp = client.get("/api/threads/missing/final-answer") assert resp.status_code == 404 assert resp.json() == {"error": "thread not found"} def test_final_answer_does_not_mark_complete_when_runtime_state_fails(): async def fake_metadata(_thread_id): return {"updated_at": "2026-07-06T14:14:53+00:00"} async def fake_messages(_thread_id): return [AIMessage(content="checkpoint answer")] async def fake_runtime(_request, _thread_id): raise RuntimeError("langgraph runtime unavailable") with ( patch( "EvoScientist.langgraph_dev.http._get_thread_metadata_for_http", new=fake_metadata, ), patch( "EvoScientist.langgraph_dev.http._get_thread_messages_for_http", new=fake_messages, ), patch( "EvoScientist.langgraph_dev.http._read_thread_runtime_state", new=fake_runtime, ), ): resp = client.get("/api/threads/thread-1/final-answer") assert resp.status_code == 200 assert resp.json() == { "content": "checkpoint answer", "completed_at": None, "complete": False, }