"""API connectivity probes for ``hermes doctor`` (split out of ``doctor.py``). Every probe is a pure function: takes its inputs, makes one HTTP/SDK call and returns a ``ProbeResult`` carrying the row(s) to print and issue strings to append. No printing inside workers — the caller prints in submission order. """ from __future__ import annotations import concurrent.futures import os import sys from typing import NamedTuple from hermes_cli.colors import Colors, color from hermes_cli.models import _HERMES_USER_AGENT from hermes_constants import OPENROUTER_MODELS_URL from utils import base_url_host_matches _APIKEY_PROVIDERS_CACHE: list | None = None class ProbeResult(NamedTuple): label: str lines: list # [(glyph, label, detail)] issues: list _GLYPH = {"ok": ("✓", Colors.GREEN), "warn": ("⚠", Colors.YELLOW), "fail": ("✗", Colors.RED)} def _row(name: str, status: str, detail: str = "", issues: list | None = None, label: str | None = None) -> ProbeResult: glyph, col = _GLYPH[status] return ProbeResult( name, [(color(glyph, col), label if label is not None else name, color(detail, Colors.DIM) if detail else "")], list(issues or []), ) def _skip(name: str) -> ProbeResult: return ProbeResult(name, [], []) def _has_healthy_oauth_fallback_for_apikey_provider(provider_label: str) -> bool: """True when a failed direct API-key probe is non-blocking because the same provider family's OAuth runtime path is already healthy: the failed row is still shown, but not promoted into the final blocking summary.""" normalized = (provider_label or "").strip().lower() getter = {"minimax": "get_minimax_oauth_auth_status", "xai": "get_xai_oauth_auth_status"}.get(normalized) if not getter: return False try: from hermes_cli import auth return bool((getattr(auth, getter)() or {}).get("logged_in")) except Exception: return False def _build_apikey_providers_list() -> list: """Build the API-key provider health-check list once and cache it. Tuple format: (name, env_vars, default_url, base_env, supports_models_endpoint) Base list augmented with any ProviderProfile with auth_type="api_key" not already present — adding plugins/model-providers// is sufficient to get into doctor. """ _static = [ ("Z.AI / GLM", ("GLM_API_KEY", "ZAI_API_KEY", "Z_AI_API_KEY"), "https://api.z.ai/api/paas/v4/models", "GLM_BASE_URL", True), ("Kimi / Moonshot", ("KIMI_API_KEY",), "https://api.moonshot.ai/v1/models", "KIMI_BASE_URL", True), ("StepFun Step Plan", ("STEPFUN_API_KEY",), "https://api.stepfun.ai/step_plan/v1/models", "STEPFUN_BASE_URL", True), ("Kimi / Moonshot (China)", ("KIMI_CN_API_KEY",), "https://api.moonshot.cn/v1/models", None, True), ("Arcee AI", ("ARCEEAI_API_KEY",), "https://api.arcee.ai/api/v1/models", "ARCEE_BASE_URL", True), ("GMI Cloud", ("GMI_API_KEY",), "https://api.gmi-serving.com/v1/models", "GMI_BASE_URL", True), ("DeepSeek", ("DEEPSEEK_API_KEY",), "https://api.deepseek.com/v1/models", "DEEPSEEK_BASE_URL", True), ("Hugging Face", ("HF_TOKEN",), "https://router.huggingface.co/v1/models", "HF_BASE_URL", True), ("NVIDIA NIM", ("NVIDIA_API_KEY",), "https://integrate.api.nvidia.com/v1/models", "NVIDIA_BASE_URL", True), ("Alibaba/DashScope", ("DASHSCOPE_API_KEY",), "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/models", "DASHSCOPE_BASE_URL", True), # MiniMax global: /v1 endpoint supports /models. ("MiniMax", ("MINIMAX_API_KEY",), "https://api.minimax.io/v1/models", "MINIMAX_BASE_URL", True), # MiniMax CN: /v1 endpoint does NOT support /models (returns 404). ("MiniMax (China)", ("MINIMAX_CN_API_KEY",), "https://api.minimaxi.com/v1/models", "MINIMAX_CN_BASE_URL", False), ("Vercel AI Gateway", ("AI_GATEWAY_API_KEY",), "https://ai-gateway.vercel.sh/v1/models", "AI_GATEWAY_BASE_URL", True), ("Kilo Code", ("KILOCODE_API_KEY",), "https://api.kilo.ai/api/gateway/models", "KILOCODE_BASE_URL", True), ("OpenCode Zen", ("OPENCODE_ZEN_API_KEY",), "https://opencode.ai/zen/v1/models", "OPENCODE_ZEN_BASE_URL", True), # OpenCode Go has no shared /models endpoint; skip the health check. ("OpenCode Go", ("OPENCODE_GO_API_KEY",), None, "OPENCODE_GO_BASE_URL", False), ] _known_names = {t[0] for t in _static} # Canonical profile names of the static rows, so profiles without a # display_name don't duplicate providers already listed above. _known_canonical = { "zai", "kimi-coding", "stepfun", "kimi-coding-cn", "arcee", "gmi", "deepseek", "huggingface", "nvidia", "alibaba", "minimax", "minimax-cn", "ai-gateway", "kilocode", "opencode-zen", "opencode-go", } # Providers with a dedicated health check (custom headers/auth). Skip their # pluggable profiles so the generic Bearer-auth loop doesn't run a duplicate, # broken check (e.g. Anthropic native API requires x-api-key, not Bearer). _dedicated_canonical = {"anthropic", "openrouter", "bedrock"} _known_canonical.update(_dedicated_canonical) try: from providers import list_providers from providers.base import ProviderProfile as _PP try: from hermes_cli.providers import normalize_provider as _normalize_provider except Exception: # pragma: no cover - normalization is best-effort def _normalize_provider(_name: str) -> str: return (_name or "").strip().lower() for _pp in list_providers(): if not isinstance(_pp, _PP) or _pp.auth_type != "api_key" or not _pp.env_vars: continue _label = _pp.display_name or _pp.name if _label in _known_names or _pp.name in _known_canonical: continue _candidates = {_normalize_provider(_pp.name)} for _alias in (_pp.aliases or ()): _candidates.add(_normalize_provider(_alias)) if _candidates & _dedicated_canonical: continue # Separate API-key vars from base-URL override vars — the health-check # loop sends the first found value as Authorization: Bearer, so a URL # string must never be picked. _key_vars = tuple( v for v in _pp.env_vars if not v.endswith("_BASE_URL") and not v.endswith("_URL") ) _base_var = next( (v for v in _pp.env_vars if v.endswith("_BASE_URL") or v.endswith("_URL")), None, ) if not _key_vars: continue _models_url = ( (_pp.models_url or (_pp.base_url.rstrip("/") + "/models")) if _pp.base_url else None ) _hc = getattr(_pp, "supports_health_check", True) _static.append((_label, _key_vars, _models_url, _base_var, _hc)) except Exception: pass return _static def _probe_openrouter() -> ProbeResult: name = "OpenRouter API" key = os.getenv("OPENROUTER_API_KEY") if not key: return _row(name, "warn", "(not configured)") try: import httpx r = httpx.get(OPENROUTER_MODELS_URL, headers={"Authorization": f"Bearer {key}"}, timeout=10) except Exception as e: return _row(name, "fail", f"({e})", ["Check network connectivity"]) if r.status_code == 200: return _row(name, "ok") if r.status_code == 401: return _row(name, "fail", "(invalid API key)", ["Check OPENROUTER_API_KEY in .env"]) if r.status_code == 402: return _row(name, "fail", "(out of credits — payment required)", [ "OpenRouter account has insufficient credits. " "Fix: run 'hermes config set model.provider ' " "to switch providers, or fund your OpenRouter account " "at https://openrouter.ai/settings/credits"]) if r.status_code == 429: return _row(name, "fail", "(rate limited)", [ "OpenRouter rate limit hit — consider switching to a different provider or waiting"]) return _row(name, "fail", f"(HTTP {r.status_code})") def _probe_anthropic() -> ProbeResult: name = "Anthropic API" from hermes_cli.auth import get_anthropic_key key = get_anthropic_key() if not key: return _skip(name) try: import httpx from agent.anthropic_adapter import ( _is_oauth_token, _COMMON_BETAS, _OAUTH_ONLY_BETAS, _CONTEXT_1M_BETA, ) headers = {"anthropic-version": "2023-06-01"} is_oauth = _is_oauth_token(key) if is_oauth: headers["Authorization"] = f"Bearer {key}" headers["anthropic-beta"] = ",".join(_COMMON_BETAS + _OAUTH_ONLY_BETAS) else: headers["x-api-key"] = key url = "https://api.anthropic.com/v1/models" r = httpx.get(url, headers=headers, timeout=10) # Reactive recovery: OAuth subscriptions without 1M context reject the # request with 400 "long context beta is not yet available for this # subscription". Retry once with that beta stripped so the doctor # check doesn't falsely report Anthropic as unreachable. if ( is_oauth and r.status_code == 400 and "long context beta" in r.text.lower() and "not yet available" in r.text.lower() ): headers["anthropic-beta"] = ",".join( [b for b in _COMMON_BETAS if b != _CONTEXT_1M_BETA] + list(_OAUTH_ONLY_BETAS) ) r = httpx.get(url, headers=headers, timeout=10) except Exception as e: return _row(name, "warn", f"({e})") if r.status_code == 200: return _row(name, "ok") if r.status_code == 401: return _row(name, "fail", "(invalid API key)") return _row(name, "warn", "(couldn't verify)") def _probe_apikey_provider(pname, env_vars, default_url, base_env, supports_health_check) -> ProbeResult: key = "" for ev in env_vars: key = os.getenv(ev, "") if key: break if not key: return _skip(pname) label = pname.ljust(20) if not supports_health_check: return _row(pname, "ok", "(key configured)", label=label) try: import httpx base = os.getenv(base_env, "") if base_env else "" # Auto-detect Kimi Code keys (sk-kimi-) → api.kimi.com/coding/v1 # (OpenAI-compat surface, which exposes /models for health check). if not base and key.startswith("sk-kimi-"): base = "https://api.kimi.com/coding/v1" # Anthropic-compat endpoints (/anthropic, api.kimi.com/coding # with no /v1) don't support /models. Rewrite to OpenAI-compat # /v1 surface for health checks. if base and base.rstrip("/").endswith("/anthropic"): from agent.auxiliary_client import _to_openai_base_url base = _to_openai_base_url(base) if base_url_host_matches(base, "api.kimi.com") and base.rstrip("/").endswith("/coding"): base = base.rstrip("/") + "/v1" url = (base.rstrip("/") + "/models") if base else default_url headers = { "Authorization": f"Bearer {key}", "User-Agent": _HERMES_USER_AGENT, } if base_url_host_matches(base, "api.kimi.com"): headers["User-Agent"] = "claude-code/0.1.0" # Google's Generative Language API rejects ``Authorization: Bearer # `` with 401 ``ACCESS_TOKEN_TYPE_UNSUPPORTED`` — that header is # reserved for OAuth 2 access tokens; plain keys use ``x-goog-api-key``. if url and base_url_host_matches(url, "generativelanguage.googleapis.com"): headers.pop("Authorization", None) headers["x-goog-api-key"] = key r = httpx.get(url, headers=headers, timeout=10) if pname == "Alibaba/DashScope" and not base and r.status_code == 401: r = httpx.get("https://dashscope.aliyuncs.com/compatible-mode/v1/models", headers=headers, timeout=10) except Exception as e: return _row(pname, "warn", f"({e})", label=label) if r.status_code == 200: return _row(pname, "ok", label=label) if r.status_code == 401: return _row(pname, "fail", "(invalid API key)", [f"Check {env_vars[0]} in .env"], label=label) return _row(pname, "warn", f"(HTTP {r.status_code})", label=label) def _probe_bedrock() -> ProbeResult: name = "AWS Bedrock" try: from agent.bedrock_adapter import ( has_aws_credentials, resolve_aws_auth_env_var, resolve_bedrock_region, ) except ImportError: return _skip(name) if not has_aws_credentials(): return _skip(name) auth_var = resolve_aws_auth_env_var() region = resolve_bedrock_region() label = name.ljust(20) try: import boto3 from botocore.config import Config as _BotoConfig # Trim retries on the actual Bedrock API call so a transient # failure doesn't pad the doctor run by 30+ seconds. cfg = _BotoConfig(connect_timeout=5, read_timeout=10, retries={"max_attempts": 1}) client = boto3.client("bedrock", region_name=region, config=cfg) resp = client.list_foundation_models() n = len(resp.get("modelSummaries", [])) return _row(name, "ok", f"({auth_var}, {region}, {n} models)", label=label) except ImportError: pip = f"{sys.executable} -m pip install boto3" return _row(name, "warn", f"(boto3 not installed — {pip})", [f"Install boto3 for Bedrock: {pip}"], label=label) except Exception as e: err_name = type(e).__name__ return _row(name, "warn", f"({err_name}: {e})", [f"AWS Bedrock: {err_name} — check IAM permissions for bedrock:ListFoundationModels"], label=label) def _probe_azure_entra() -> ProbeResult: """Probe Azure Foundry Entra ID auth, parallel to ``_probe_bedrock``. Skipped unless the active config has ``model.provider: azure-foundry`` AND ``model.auth_mode: entra_id`` — we don't probe the token-service / CLI chain for users on plain API-key Azure. Bounded by a 10s timeout (via :func:`agent.azure_identity_adapter.describe_active_credential`). """ name = "Azure Foundry (Entra ID)" label = name.ljust(28) try: from hermes_cli.config import load_config cfg = load_config() model_cfg = cfg.get("model") if isinstance(cfg, dict) else {} if not isinstance(model_cfg, dict): return _skip(name) cfg_provider = str(model_cfg.get("provider") or "").strip().lower() auth_mode = str(model_cfg.get("auth_mode") or "").strip().lower() if cfg_provider != "azure-foundry" or auth_mode != "entra_id": return _skip(name) except Exception: return _skip(name) try: from agent.azure_identity_adapter import ( EntraIdentityConfig, SCOPE_AI_AZURE_DEFAULT, describe_active_credential, has_azure_identity_installed, ) except Exception as exc: return _row(name, "warn", f"(adapter import failed: {exc})", [f"Azure Foundry adapter import failed: {exc}"], label=label) if not has_azure_identity_installed(): return _row(name, "warn", "(azure-identity not installed)", [f"Install azure-identity: {sys.executable} -m pip install azure-identity"], label=label) entra_cfg = model_cfg.get("entra") or {} if not isinstance(entra_cfg, dict): entra_cfg = {} scope = str(entra_cfg.get("scope") or "").strip() or SCOPE_AI_AZURE_DEFAULT info = describe_active_credential(config=EntraIdentityConfig(scope=scope), timeout_seconds=10.0) if info.get("ok"): env_sources = info.get("env_sources") or [] tag = ", ".join(env_sources) if env_sources else "default credential chain" return _row(name, "ok", f"({tag}, scope={scope})", label=label) err = info.get("error") or "credential chain exhausted" hint = info.get("hint") or ( "Run `az login`, set AZURE_TENANT_ID/AZURE_CLIENT_ID/" "AZURE_CLIENT_SECRET, or attach a managed identity to this VM." ) return _row(name, "warn", f"({err})", [f"Azure Foundry Entra: {err}. {hint}"], label=label) def build_probes() -> list: """(label, callable) pairs in display order.""" global _APIKEY_PROVIDERS_CACHE if _APIKEY_PROVIDERS_CACHE is None: _APIKEY_PROVIDERS_CACHE = _build_apikey_providers_list() probes = [("OpenRouter API", _probe_openrouter), ("Anthropic API", _probe_anthropic)] for _pname, _env_vars, _default_url, _base_env, _supports in _APIKEY_PROVIDERS_CACHE: # Bind loop vars via default args so every closure keeps its own provider. probes.append((_pname, lambda p=_pname, e=_env_vars, u=_default_url, b=_base_env, s=_supports: _probe_apikey_provider(p, e, u, b, s))) probes.append(("AWS Bedrock", _probe_bedrock)) probes.append(("Azure Foundry (Entra ID)", _probe_azure_entra)) return probes def run_probes(probes: list) -> list: """Run every probe in a thread pool; results in submission order. Probes are independent HTTP calls, so running them in series cost ~5s wall (2s of it boto3's IMDS lookup). Parallel collapses the section to roughly the slowest single probe without changing the output. """ # Disable boto3's EC2 instance-metadata probe for the parallel block: the # default credential chain tries 169.254.169.254 with a multi-second # timeout off-EC2. Set on the parent thread before submitting so the env # mutation never races a worker; has_aws_credentials() already gates on # real env creds, so IMDS is never the legitimate source for doctor. _imds_prev = os.environ.get("AWS_EC2_METADATA_DISABLED") os.environ["AWS_EC2_METADATA_DISABLED"] = "true" try: # 8 workers is plenty — each probe is one HTTP call plus a TLS handshake. with concurrent.futures.ThreadPoolExecutor(max_workers=8, thread_name_prefix="doctor-probe") as ex: futures = [ex.submit(fn) for _, fn in probes] return [f.result() for f in futures] finally: if _imds_prev is None: os.environ.pop("AWS_EC2_METADATA_DISABLED", None) else: os.environ["AWS_EC2_METADATA_DISABLED"] = _imds_prev