Teknium
4b7b2b0049
Follow-up to #85737, which migrated five provider-identity sites onto utils.base_url_host_matches()/base_url_hostname(). This completes the class sweep (never-patch-predicates: one owner, every site) and folds in the two open contributor PRs attacking individual sites: - agent/auxiliary_client.py ZAI/Kimi OpenAI-wire rewrite (PR #85715, pierrenode): 'bigmodel'/'api.z.ai'/'api.kimi.com' substring checks rewrote proxy paths containing those markers. - hermes_cli/runtime_provider.py Azure endpoint detection (PR #74721, RelaxJonh, issue #74312): 'azure.com' substring picked the Azure key for non-Azure hosts whose path contained the text. - run_agent.py: _is_azure_openai_url, _is_copilot_url, Anthropic credential-refresh azure guard, _anthropic_preserve_dots host allowlist, OpenRouter/mistral reasoning gates. - agent/chat_completion_helpers.py: nousresearch / nvidia detection. - agent/conversation_loop.py: GitHub Models 413 hint. - agent/usage_pricing.py: localhost billing-route detection. - hermes_cli/model_switch.py: api.openai.com catalog fallback and localhost custom-provider detection. - cli.py: local-model autodetect and Ollama/LM Studio context-length hints (port-anchored instead of '11434' in URL). - tools/mcp_oauth.py: Figma remote-MCP detection. - tools/skills_hub.py: raw.githubusercontent.com source-URL check. Regression tests extend tests/hermes_cli/test_base_url_host_identity.py (azure/copilot/dotted-model/figma proxy-path + lookalike cases) and tests/agent/test_minimax_auxiliary_url.py (ZAI/Kimi path false positives). Closes #74312. Salvages #85715 and #74721 with authorship preserved.
157 lines
6.7 KiB
Python
157 lines
6.7 KiB
Python
"""Regression tests: provider-identity checks must compare URL *hostnames*,
|
|
not raw substrings.
|
|
|
|
Port of earendil-works/pi#7933's bug class (DeepSeek base-URL detection used a
|
|
substring check, missing case variants and matching lookalike URLs). Hermes
|
|
had the same class at several sites: keyless-endpoint detection, /model
|
|
catalog routing, local-endpoint detection, and Nous Portal cache-layout
|
|
detection all used ``"host" in base_url``. A proxy URL that merely *contains*
|
|
a provider host in its path (``https://proxy.internal/openrouter.ai/v1``) or
|
|
a lookalike domain (``https://openrouter.ai.evil.com``) must not be treated
|
|
as that provider, and casing must not matter.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import patch
|
|
|
|
from hermes_cli.cli_agent_setup_mixin import CLIAgentSetupMixin
|
|
|
|
|
|
class _Host(CLIAgentSetupMixin):
|
|
def __init__(self):
|
|
self.requested_provider = "auto"
|
|
self._explicit_api_key = None
|
|
self._explicit_base_url = None
|
|
|
|
|
|
def _ready_with(runtime: dict) -> bool:
|
|
host = _Host()
|
|
with patch(
|
|
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
|
return_value=runtime,
|
|
):
|
|
return host._runtime_credentials_ready()
|
|
|
|
|
|
def test_keyless_real_openrouter_not_ready():
|
|
# OpenRouter itself requires a key: keyless => not ready.
|
|
assert _ready_with({"api_key": None, "base_url": "https://openrouter.ai/api/v1"}) is False
|
|
|
|
|
|
def test_keyless_uppercase_openrouter_not_ready():
|
|
# Case variants of the real host are still the real host (pi#7933 class).
|
|
assert _ready_with({"api_key": None, "base_url": "https://OpenRouter.AI/api/v1"}) is False
|
|
|
|
|
|
def test_keyless_proxy_with_openrouter_in_path_is_ready():
|
|
# A custom proxy whose *path* contains the substring is NOT OpenRouter —
|
|
# it's a keyless custom endpoint and must count as ready.
|
|
assert _ready_with({"api_key": None, "base_url": "https://proxy.internal/openrouter.ai/v1"}) is True
|
|
|
|
|
|
def test_keyless_lookalike_domain_is_ready():
|
|
assert _ready_with({"api_key": None, "base_url": "https://openrouter.ai.evil.com/v1"}) is True
|
|
|
|
|
|
def test_keyless_local_endpoint_is_ready():
|
|
assert _ready_with({"api_key": None, "base_url": "http://localhost:11434/v1"}) is True
|
|
|
|
|
|
def test_validate_requested_model_proxy_url_routes_to_custom():
|
|
"""/model validation: an 'openrouter' provider pointed at a non-OpenRouter
|
|
host is a custom endpoint, even when the URL contains the substring."""
|
|
from utils import base_url_host_matches
|
|
|
|
assert base_url_host_matches("https://openrouter.ai/api/v1", "openrouter.ai")
|
|
assert base_url_host_matches("https://OPENROUTER.AI/api/v1", "openrouter.ai")
|
|
assert not base_url_host_matches("https://proxy.internal/openrouter.ai/v1", "openrouter.ai")
|
|
assert not base_url_host_matches("https://openrouter.ai.evil.com/v1", "openrouter.ai")
|
|
|
|
|
|
def test_local_endpoint_hostname_detection():
|
|
from utils import base_url_hostname
|
|
|
|
assert base_url_hostname("http://localhost:11434/v1") == "localhost"
|
|
assert base_url_hostname("http://127.0.0.1:1234") == "127.0.0.1"
|
|
# A remote host with "localhost" embedded in its name is not local.
|
|
assert base_url_hostname("https://my-localhost-mirror.com/v1") not in (
|
|
"localhost",
|
|
"127.0.0.1",
|
|
"0.0.0.0",
|
|
)
|
|
|
|
|
|
def test_nous_portal_host_detection():
|
|
from utils import base_url_host_matches
|
|
|
|
assert base_url_host_matches("https://inference-api.nousresearch.com/v1", "nousresearch.com")
|
|
assert base_url_host_matches("https://portal.nousresearch.com", "nousresearch.com")
|
|
assert not base_url_host_matches("https://nousresearch.com.evil.io/v1", "nousresearch.com")
|
|
assert not base_url_host_matches("https://proxy.example/nousresearch.com/v1", "nousresearch.com")
|
|
|
|
|
|
# ── Widened class coverage (follow-up to #85737) ─────────────────────────────
|
|
|
|
|
|
def test_azure_endpoint_detection_host_anchored():
|
|
"""Azure detection (runtime_provider + run_agent) must be host-anchored:
|
|
a path or lookalike containing 'azure.com'/'openai.azure.com' is not Azure."""
|
|
from utils import base_url_host_matches
|
|
|
|
assert base_url_host_matches("https://myres.openai.azure.com/openai/v1", "azure.com")
|
|
assert base_url_host_matches("https://myres.openai.azure.com/openai/v1", "openai.azure.com")
|
|
assert not base_url_host_matches("https://proxy.corp/openai.azure.com/v1", "azure.com")
|
|
assert not base_url_host_matches("https://azure.com.evil.net/v1", "azure.com")
|
|
assert not base_url_host_matches("https://notazure.com/v1", "azure.com")
|
|
|
|
|
|
def test_run_agent_azure_url_predicate():
|
|
from run_agent import AIAgent
|
|
|
|
probe = object.__new__(AIAgent)
|
|
assert probe._is_azure_openai_url("https://myres.openai.azure.com/openai/v1") is True
|
|
assert probe._is_azure_openai_url("https://proxy.internal/openai.azure.com/v1") is False
|
|
assert probe._is_azure_openai_url("https://openai.azure.com.evil.io/v1") is False
|
|
|
|
|
|
def test_run_agent_copilot_url_predicate():
|
|
from run_agent import AIAgent
|
|
|
|
probe = object.__new__(AIAgent)
|
|
probe._base_url_lower = "https://api.githubcopilot.com/v1"
|
|
assert probe._is_copilot_url() is True
|
|
probe._base_url_lower = "https://proxy.test/api.githubcopilot.com/v1"
|
|
assert probe._is_copilot_url() is False
|
|
probe._base_url_lower = "https://models.github.ai/inference"
|
|
assert probe._is_copilot_url() is True
|
|
probe._base_url_lower = "https://models.github.ai.evil.com/v1"
|
|
assert probe._is_copilot_url() is False
|
|
|
|
|
|
def test_dotted_model_name_provider_allowlist_host_anchored():
|
|
from run_agent import AIAgent
|
|
|
|
probe = object.__new__(AIAgent)
|
|
probe.provider = ""
|
|
probe.base_url = "https://open.bigmodel.cn/api/paas/v4"
|
|
assert probe._anthropic_preserve_dots() is True
|
|
probe.base_url = "https://gateway.example.com/bigmodel.cn/v4"
|
|
assert probe._anthropic_preserve_dots() is False
|
|
probe.base_url = "https://aiplatform.googleapis.com/v1"
|
|
assert probe._anthropic_preserve_dots() is True
|
|
probe.base_url = "https://evil.io/aiplatform.googleapis.com/v1"
|
|
assert probe._anthropic_preserve_dots() is False
|
|
|
|
|
|
def test_figma_remote_mcp_host_anchored():
|
|
from tools.mcp_oauth import _is_figma_remote_mcp
|
|
|
|
assert _is_figma_remote_mcp(server_url="https://mcp.figma.com/mcp") is True
|
|
assert _is_figma_remote_mcp(server_url="https://www.figma.com/mcp") is True
|
|
assert _is_figma_remote_mcp(server_url="https://evil.example/mcp.figma.com/mcp") is False
|
|
assert _is_figma_remote_mcp(server_url="https://figma.com.evil.io/mcp") is False
|
|
# Name fallback still host-checks the URL when one is present.
|
|
assert _is_figma_remote_mcp(server_name="figma", server_url="https://phish.example/figma") is False
|
|
assert _is_figma_remote_mcp(server_name="figma") is True
|