EvoScientist/tests/test_provider_operations.py

"""Tests for provider-specific model discovery and connectivity checks."""

from __future__ import annotations

from types import SimpleNamespace
from unittest.mock import patch

import httpx
import pytest

from EvoScientist.config.provider_profiles import ProviderModel, ProviderProfile
from EvoScientist.llm.provider_operations import (
    ProviderOperationError,
    discover_provider_models,
)
from EvoScientist.llm.provider_operations import (
    test_provider_model as run_provider_model_test,
)


def _profile(
    adapter: str,
    *,
    base_url: str = "",
    api_key: str = "provider-secret",
) -> ProviderProfile:
    return ProviderProfile(
        id="test-provider",
        name="Test provider",
        adapter=adapter,
        base_url=base_url,
        api_key=api_key,
        enabled=True,
        models=(),
    )


@pytest.mark.asyncio
async def test_openai_discovery_uses_bearer_token_and_models_endpoint():
    def handler(request: httpx.Request) -> httpx.Response:
        assert request.url == "https://api.openai.com/v1/models"
        assert request.headers["Authorization"] == "Bearer provider-secret"
        return httpx.Response(
            200,
            json={
                "data": [
                    {"id": "gpt-z"},
                    {"id": "gpt-a", "name": "GPT A"},
                ]
            },
        )

    models = await discover_provider_models(
        _profile("openai"), transport=httpx.MockTransport(handler)
    )

    assert [(model.model_id, model.name) for model in models] == [
        ("gpt-a", "GPT A"),
        ("gpt-z", "gpt-z"),
    ]


@pytest.mark.asyncio
async def test_anthropic_discovery_uses_native_headers_and_pagination():
    requests: list[httpx.Request] = []

    def handler(request: httpx.Request) -> httpx.Response:
        requests.append(request)
        assert request.url.path == "/v1/models"
        assert request.headers["x-api-key"] == "provider-secret"
        assert request.headers["anthropic-version"] == "2023-06-01"
        if request.url.params.get("after_id"):
            return httpx.Response(
                200,
                json={
                    "data": [{"id": "claude-opus", "display_name": "Claude Opus"}],
                    "has_more": False,
                },
            )
        return httpx.Response(
            200,
            json={
                "data": [{"id": "claude-sonnet"}],
                "has_more": True,
                "last_id": "claude-sonnet",
            },
        )

    models = await discover_provider_models(
        _profile("anthropic"), transport=httpx.MockTransport(handler)
    )

    assert len(requests) == 2
    assert requests[1].url.params["after_id"] == "claude-sonnet"
    assert [model.model_id for model in models] == ["claude-opus", "claude-sonnet"]


@pytest.mark.asyncio
async def test_google_discovery_filters_non_generation_models_and_paginates():
    def handler(request: httpx.Request) -> httpx.Response:
        assert request.url.path == "/v1beta/models"
        assert request.headers["x-goog-api-key"] == "provider-secret"
        if request.url.params.get("pageToken"):
            return httpx.Response(
                200,
                json={
                    "models": [
                        {
                            "name": "models/gemini-2.5-pro",
                            "displayName": "Gemini 2.5 Pro",
                            "supportedGenerationMethods": ["generateContent"],
                        }
                    ]
                },
            )
        return httpx.Response(
            200,
            json={
                "models": [
                    {
                        "name": "models/text-embedding-004",
                        "supportedGenerationMethods": ["embedContent"],
                    },
                    {
                        "name": "models/gemini-2.5-flash",
                        "supportedGenerationMethods": ["generateContent"],
                    },
                ],
                "nextPageToken": "next-page",
            },
        )

    models = await discover_provider_models(
        _profile("google-genai"), transport=httpx.MockTransport(handler)
    )

    assert [(model.model_id, model.name) for model in models] == [
        ("gemini-2.5-flash", "models/gemini-2.5-flash"),
        ("gemini-2.5-pro", "Gemini 2.5 Pro"),
    ]


@pytest.mark.asyncio
async def test_grok_uses_xai_default_and_antigravity_uses_configured_proxy():
    requests: list[httpx.Request] = []

    def handler(request: httpx.Request) -> httpx.Response:
        requests.append(request)
        return httpx.Response(200, json={"data": [{"id": "grok-4"}]})

    transport = httpx.MockTransport(handler)
    await discover_provider_models(_profile("grok"), transport=transport)
    await discover_provider_models(
        _profile("antigravity", base_url="http://127.0.0.1:8317/v1"),
        transport=transport,
    )

    assert str(requests[0].url) == "https://api.x.ai/v1/models"
    assert str(requests[1].url) == "http://127.0.0.1:8317/v1/models"


@pytest.mark.asyncio
async def test_ollama_discovery_uses_tags_without_api_key():
    def handler(request: httpx.Request) -> httpx.Response:
        assert request.url == "http://127.0.0.1:11434/api/tags"
        assert "Authorization" not in request.headers
        return httpx.Response(
            200,
            json={"models": [{"name": "qwen3:8b"}, {"model": "llama3.3:latest"}]},
        )

    models = await discover_provider_models(
        _profile("ollama", api_key=""), transport=httpx.MockTransport(handler)
    )

    assert [model.model_id for model in models] == ["llama3.3:latest", "qwen3:8b"]


@pytest.mark.asyncio
async def test_discovery_requires_native_provider_api_key():
    with pytest.raises(ProviderOperationError, match="API key is required"):
        await discover_provider_models(_profile("openai", api_key=""))


@pytest.mark.asyncio
async def test_provider_http_error_surfaces_message_without_response_body_dump():
    def handler(_request: httpx.Request) -> httpx.Response:
        return httpx.Response(
            401,
            json={"error": {"message": "Invalid credential"}, "secret": "hidden"},
        )

    with pytest.raises(
        ProviderOperationError, match="Provider returned HTTP 401: Invalid credential"
    ):
        await discover_provider_models(
            _profile("openai"), transport=httpx.MockTransport(handler)
        )


@pytest.mark.asyncio
async def test_bare_openai_origin_uses_v1_and_reports_non_json_content_type():
    def handler(request: httpx.Request) -> httpx.Response:
        assert request.url == "https://api.example.test/v1/models"
        return httpx.Response(
            200,
            text="<!doctype html>",
            headers={"content-type": "text/html; charset=utf-8"},
        )

    with pytest.raises(ProviderOperationError) as raised:
        await discover_provider_models(
            _profile("openai", base_url="https://api.example.test"),
            transport=httpx.MockTransport(handler),
        )

    message = str(raised.value)
    assert "https://api.example.test/v1/models" in message
    assert "Content-Type: text/html" in message
    assert "normally end in /v1" in message


@pytest.mark.asyncio
async def test_discovery_retries_a_transient_connection_failure():
    attempts = 0

    def handler(request: httpx.Request) -> httpx.Response:
        nonlocal attempts
        attempts += 1
        if attempts == 1:
            raise httpx.ConnectTimeout("temporary timeout", request=request)
        return httpx.Response(200, json={"data": [{"id": "gpt-recovered"}]})

    models = await discover_provider_models(
        _profile("openai"), transport=httpx.MockTransport(handler)
    )

    assert attempts == 2
    assert [model.model_id for model in models] == ["gpt-recovered"]


@pytest.mark.asyncio
async def test_model_test_builds_draft_model_and_returns_text():
    class FakeChatModel:
        async def ainvoke(self, prompt: str):
            assert prompt == "Reply with exactly: OK"
            return SimpleNamespace(content=[{"type": "text", "text": "OK"}])

    profile = _profile("grok")
    model = ProviderModel(
        id="grok-test", name="Grok test", model_id="grok-4", enabled=True
    )
    with patch(
        "EvoScientist.llm.models.get_profile_chat_model", return_value=FakeChatModel()
    ) as build_model:
        result = await run_provider_model_test(profile, model)

    assert result.response == "OK"
    assert result.latency_ms >= 1
    build_model.assert_called_once_with(
        profile,
        model,
        max_tokens=64,
        skip_auto_config=True,
    )


@pytest.mark.asyncio
async def test_model_test_redacts_api_key_from_errors():
    profile = _profile("openai", api_key="secret-value")
    model = ProviderModel(id="test", name="Test", model_id="gpt-test")
    with patch(
        "EvoScientist.llm.models.get_profile_chat_model",
        side_effect=ValueError("request rejected for secret-value"),
    ):
        with pytest.raises(ProviderOperationError) as raised:
            await run_provider_model_test(profile, model)

    assert "secret-value" not in str(raised.value)
    assert "[redacted]" in str(raised.value)