diff --git a/hermes_cli/models_validate.py b/hermes_cli/models_validate.py index ede53fa602..ceae33a2e7 100644 --- a/hermes_cli/models_validate.py +++ b/hermes_cli/models_validate.py @@ -269,23 +269,19 @@ def _validate_custom(req: _Request) -> dict[str, Any]: "Consider saving that as your base URL.") return _soft_accept(message) - message = ( - f"Note: could not reach this custom endpoint's model listing at `{probe.get('probed_url')}`. " - f"Hermes will still save `{req.requested}`, but the endpoint should expose `/models` for verification." - ) - if anthropic_style: - message += ("\n Many Anthropic-compatible proxies do not implement the Models API (GET /v1/models). " - "The model name has been accepted without verification.") - if req.api_mode == "chat_completions": - message += ( - "\n The model name was accepted without verification — inference may fail " - "if this provider does not serve it. Check the provider's model catalog " - "or model name." - ) + # Many OpenAI-compatible and Anthropic-compatible proxies (DashScope coding plan, Cline, + # MiniMax) never implement GET /models; /chat/completions works fine. Rejecting the switch + # here bricked `/model` for them (#12220), so both chat modes persist the name unverified. + accepted = req.api_mode in ("chat_completions", "anthropic_messages") + message = f"Note: could not reach this custom endpoint's model listing at `{probe.get('probed_url')}`. " + if accepted: + message += (f"`{req.requested}` was accepted without verification — if this endpoint does not " + "serve it, inference will fail; check the provider's model catalog or the model name.") + else: + message += f"`{req.requested}` was not saved; the endpoint should expose `/models` for verification." if probe.get("suggested_base_url"): message += f"\n If this server expects `/v1`, try base URL: `{probe.get('suggested_base_url')}`" - # Chat-completions and Anthropic-style proxies may both omit /v1/models. - return _verdict(req.api_mode in ("chat_completions", "anthropic_messages"), True, False, message) + return _verdict(accepted, True, False, message) def _static_catalog(normalized: str) -> list[str]: diff --git a/tests/hermes_cli/test_model_validation.py b/tests/hermes_cli/test_model_validation.py index bdd8c4ec51..88381f13e8 100644 --- a/tests/hermes_cli/test_model_validation.py +++ b/tests/hermes_cli/test_model_validation.py @@ -802,135 +802,25 @@ class TestValidateRequestedModelNousPortalRecommendations: # -- validate — custom endpoint fallback when /models is unreachable (#12220) -- class TestValidateCustomUnreachableFallback: - """Issue #12220: when a custom endpoint's /models probe returns None - (unreachable or not implemented), accept the requested model for - chat_completions exactly as the anthropic_messages fallback already - does — rejecting it bricks `/model` switches in the gateway for any - custom proxy that doesn't expose a /models listing.""" + """A custom proxy without GET /models must not brick `/model` switches (#12220).""" - def _probe(self, models=None, suggested_base_url=None): - return { - "models": models, - "probed_url": "http://localhost:8000/v1/models", - "resolved_base_url": "http://localhost:8000/v1", - "suggested_base_url": suggested_base_url, - "used_fallback": False, - } - - def _validate(self, model, provider, probe, **kw): + def _validate(self, model, provider, models, **kw): + probe = {"models": models, "probed_url": "http://localhost:8000/v1/models", + "resolved_base_url": "http://localhost:8000/v1", "suggested_base_url": None, "used_fallback": False} with patch("hermes_cli.models.probe_api_models", return_value=probe): - return validate_requested_model( - model, - provider, - api_key="test-key", - base_url="http://localhost:8000/v1", - **kw, - ) + return validate_requested_model(model, provider, api_key="k", base_url="http://localhost:8000/v1", **kw) - def test_bare_custom_chat_completions_accepted_when_unreachable(self): - result = self._validate( - "my-proxy-model", "custom", api_mode="chat_completions", - probe=self._probe(None), - ) - assert result["accepted"] is True - assert result["persist"] is True - assert result["recognized"] is False - assert "could not reach" in result["message"] + @pytest.mark.parametrize("provider", ["custom", "custom:myproxy"]) + @pytest.mark.parametrize("api_mode", ["chat_completions", "anthropic_messages"]) + def test_unreachable_catalog_persists_unverified_for_chat_modes(self, provider, api_mode): + result = self._validate("my-proxy-model", provider, models=None, api_mode=api_mode) + assert (result["accepted"], result["persist"], result["recognized"]) == (True, True, False) assert "accepted without verification" in result["message"] - assert "inference may fail" in result["message"] - assert "provider's model catalog" in result["message"] - - def test_named_custom_chat_completions_accepted_when_unreachable(self): - result = self._validate( - "my-proxy-model", "custom:myproxy", api_mode="chat_completions", - probe=self._probe(None), - ) - assert result["accepted"] is True - assert result["persist"] is True - assert result["recognized"] is False - assert "could not reach" in result["message"] - assert "accepted without verification" in result["message"] - assert "inference may fail" in result["message"] - assert "provider's model catalog" in result["message"] - - def test_unreachable_warning_includes_suggested_base_url_hint(self): - result = self._validate( - "my-proxy-model", "custom", api_mode="chat_completions", - probe=self._probe(None, suggested_base_url="http://localhost:8000"), - ) - assert result["accepted"] is True - assert "If this server expects `/v1`" in result["message"] - assert "accepted without verification" in result["message"] - assert "inference may fail" in result["message"] - assert "provider's model catalog" in result["message"] - - def test_anthropic_messages_fallback_unchanged(self): - result = self._validate( - "claude-model", "custom", api_mode="anthropic_messages", - probe=self._probe(None), - ) - assert result["accepted"] is True - assert result["persist"] is True - assert result["recognized"] is False - assert "Anthropic-compatible proxies" in result["message"] - - def test_valid_catalog_exact_match_unchanged(self): - result = self._validate( - "my-model", "custom", api_mode="chat_completions", - probe=self._probe(["my-model", "other-model"]), - ) - assert result["accepted"] is True - assert result["recognized"] is True - assert result["message"] is None - - def test_valid_catalog_unknown_model_unchanged(self): - result = self._validate( - "totally-unrelated", "custom", api_mode="chat_completions", - probe=self._probe(["my-model", "other-model"]), - ) - assert result["accepted"] is True - assert result["persist"] is True - assert result["recognized"] is False - assert result.get("corrected_model") is None - assert "was not found" in result["message"] - assert "Similar models" not in result["message"] - - def test_valid_catalog_close_match_autocorrected(self): - result = self._validate( - "my-modelx", "custom", api_mode="chat_completions", - probe=self._probe(["my-model", "other-model"]), - ) - assert result["accepted"] is True - assert result["recognized"] is True - assert result["corrected_model"] == "my-model" - assert "Auto-corrected" in result["message"] - - def test_empty_and_whitespace_unchanged(self): - empty = self._validate( - "", "custom", api_mode="chat_completions", probe=self._probe(None) - ) - assert empty["accepted"] is False - assert "empty" in empty["message"] - - spaced = self._validate( - "my model", "custom", api_mode="chat_completions", probe=self._probe(None) - ) - assert spaced["accepted"] is False - assert "spaces" in spaced["message"] - - def test_whitespace_only_model_rejected_and_not_persisted(self): - result = self._validate( - " ", "custom", api_mode="chat_completions", probe=self._probe(None) - ) - assert result["accepted"] is False - assert result["persist"] is False - assert "empty" in result["message"] @pytest.mark.parametrize("api_mode", [None, "codex_responses"]) - def test_unreachable_rejects_unsupported_api_mode(self, api_mode): - result = self._validate( - "my-proxy-model", "custom", api_mode=api_mode, - probe=self._probe(None), - ) + def test_unreachable_catalog_still_rejects_other_api_modes(self, api_mode): + result = self._validate("my-proxy-model", "custom", models=None, api_mode=api_mode) assert result["accepted"] is False - assert result["recognized"] is False + assert "was not saved" in result["message"] + # A reachable catalog keeps authoritative validation regardless of mode. + assert self._validate("my-model", "custom", models=["my-model"], api_mode="chat_completions")["recognized"] is True