68be67e0d0
2244 -> 1612 LOC. Classification order, pattern lists, FailoverReason values and ClassifiedError fields unchanged; _THINKING_SIG_PATTERNS had zero references.
1613 lines
72 KiB
Python
1613 lines
72 KiB
Python
"""API error classification for smart failover and recovery.
|
|
|
|
A structured taxonomy of API errors plus a priority-ordered classification
|
|
pipeline that picks the recovery action (retry, rotate credential, fall back
|
|
to another provider, compress context, or abort). The retry loop in
|
|
run_agent.py consults it for every API failure instead of inline matching.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import enum
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Dict, Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Synthetic error code for the OpenAI SDK rejecting a provider's SSE ``data:``
|
|
# field before any completion chunk arrives. Distinct from generic JSON parse
|
|
# failures so stream-specific recovery needs no invented HTTP status.
|
|
PROVIDER_STREAM_NON_JSON_ERROR_CODE = "provider_stream_non_json_data"
|
|
|
|
|
|
# ── Error taxonomy ──────────────────────────────────────────────────────
|
|
|
|
class FailoverReason(enum.Enum):
|
|
"""Why an API call failed — determines recovery strategy."""
|
|
|
|
# Authentication / authorization
|
|
auth = "auth" # Transient auth (401/403) — refresh/rotate
|
|
auth_permanent = "auth_permanent" # Auth failed after refresh — abort
|
|
|
|
# Billing / quota
|
|
billing = "billing" # 402 or confirmed credit exhaustion — rotate immediately
|
|
rate_limit = "rate_limit" # 429 or quota-based throttling — backoff then rotate
|
|
# Aggregator 429 from the upstream model: fall back to another model, NOT
|
|
# credential rotation — the user's key is healthy.
|
|
upstream_rate_limit = "upstream_rate_limit"
|
|
|
|
# Server-side
|
|
overloaded = "overloaded" # 503/529 — provider overloaded, backoff
|
|
server_error = "server_error" # 500/502 — internal server error, retry
|
|
|
|
# Transport
|
|
timeout = "timeout" # Connection/read timeout — rebuild client + retry
|
|
# TLS cert verification failure is deterministic for the host (inspecting
|
|
# proxy, missing/expired CA, self-signed) — fail fast with guidance.
|
|
ssl_cert_verification = "ssl_cert_verification"
|
|
|
|
# Context / payload
|
|
context_overflow = "context_overflow" # Context too large — compress, not failover
|
|
payload_too_large = "payload_too_large" # 413 — compress payload
|
|
image_too_large = "image_too_large" # Native image part exceeds provider's per-image limit — shrink and retry
|
|
image_corrupt = "image_corrupt" # Image bytes undecodable — shrinking won't help, strip and retry
|
|
|
|
# Model / provider policy
|
|
model_not_found = "model_not_found" # 404 or invalid model — fallback to different model
|
|
provider_policy_blocked = "provider_policy_blocked" # Aggregator (e.g. OpenRouter) blocked the only endpoint via account data/privacy policy
|
|
content_policy_blocked = "content_policy_blocked" # Provider safety filter rejected this prompt — deterministic, don't retry unchanged
|
|
|
|
# Request format
|
|
format_error = "format_error" # 400 bad request — abort or strip + retry
|
|
invalid_encrypted_content = "invalid_encrypted_content" # Responses replay blob rejected — strip replay state and retry
|
|
multimodal_tool_content_unsupported = "multimodal_tool_content_unsupported" # Provider rejected list-type tool content (e.g. Xiaomi MiMo) — downgrade to text and retry
|
|
|
|
# Provider-specific
|
|
thinking_signature = "thinking_signature" # Anthropic thinking block sig invalid
|
|
long_context_tier = "long_context_tier" # Anthropic "extra usage" tier gate
|
|
oauth_long_context_beta_forbidden = "oauth_long_context_beta_forbidden" # Anthropic OAuth subscription rejects 1M context beta — disable beta and retry
|
|
llama_cpp_grammar_pattern = "llama_cpp_grammar_pattern" # llama.cpp grammar converter rejects regex escapes in `pattern` / `format` — strip from tools and retry
|
|
|
|
# Catch-all
|
|
unknown = "unknown" # Unclassifiable — retry with backoff
|
|
|
|
|
|
# ── Classification result ───────────────────────────────────────────────
|
|
|
|
@dataclass
|
|
class ClassifiedError:
|
|
"""Structured classification of an API error with recovery hints."""
|
|
|
|
reason: FailoverReason
|
|
status_code: Optional[int] = None
|
|
provider: Optional[str] = None
|
|
model: Optional[str] = None
|
|
message: str = ""
|
|
error_context: Dict[str, Any] = field(default_factory=dict)
|
|
|
|
# Recovery hints — the retry loop checks these instead of re-classifying.
|
|
retryable: bool = True
|
|
should_compress: bool = False
|
|
should_rotate_credential: bool = False
|
|
should_fallback: bool = False
|
|
|
|
@property
|
|
def is_auth(self) -> bool:
|
|
return self.reason in {FailoverReason.auth, FailoverReason.auth_permanent}
|
|
|
|
@property
|
|
def billing_unverified(self) -> bool:
|
|
"""True when a ``billing`` verdict rests on an ambiguous body.
|
|
|
|
Anthropic's "out of extra usage" 400 can also be a content-filter
|
|
rejection (#82154); surfaces must hedge rather than assert exhaustion.
|
|
"""
|
|
return bool(self.error_context.get("billing_unverified"))
|
|
|
|
|
|
# ── Provider-specific patterns ──────────────────────────────────────────
|
|
|
|
# Billing exhaustion (not transient rate limit)
|
|
_BILLING_PATTERNS = [
|
|
"insufficient credits",
|
|
"insufficient_quota",
|
|
"insufficient balance",
|
|
"credit balance",
|
|
"credits exhausted",
|
|
"credits have been exhausted",
|
|
"requires available credits",
|
|
"account balance is too low",
|
|
"no usable credits",
|
|
"top up your credits",
|
|
"payment required",
|
|
"billing hard limit",
|
|
"exceeded your current quota",
|
|
"account is deactivated",
|
|
"plan does not include",
|
|
"out of extra usage", # Anthropic OAuth Pro/Max overage bucket depleted (HTTP 400)
|
|
"out of funds",
|
|
"run out of funds",
|
|
"balance_depleted",
|
|
"model_not_supported_on_free_tier",
|
|
"not available on the free tier",
|
|
]
|
|
|
|
# Billing matches that are NOT proof of exhaustion. Anthropic returns the same
|
|
# "out of extra usage" body on a subscription OAuth token both when the overage
|
|
# bucket is depleted AND when its content filter rejects the request (#82154).
|
|
# Classification stays ``billing`` (rotation + fallback are right either way),
|
|
# but ``error_context`` carries the ambiguity so surfaces hedge and the pool
|
|
# applies a short cooldown instead of the one-hour billing bench.
|
|
_UNVERIFIED_BILLING_PATTERNS = ("out of extra usage",)
|
|
|
|
|
|
def _billing_ambiguity_context(error_msg: str) -> Dict[str, Any]:
|
|
"""error_context marking a billing verdict as unverified (see above)."""
|
|
if any(p in error_msg for p in _UNVERIFIED_BILLING_PATTERNS):
|
|
return {"billing_unverified": True, "possible_content_filter": True}
|
|
return {}
|
|
|
|
|
|
# xAI's explicit Grok credit-exhaustion code, returned as HTTP 403 not 402.
|
|
# Kept provider-scoped: other providers' billing codes stay auth on 403.
|
|
_XAI_SPENDING_LIMIT_ERROR_CODE = "personal-team-blocked:spending-limit"
|
|
|
|
# Structured codes meaning the account cannot serve paid traffic until
|
|
# credits/subscription capacity is restored.
|
|
_BILLING_ERROR_CODES = frozenset({
|
|
"insufficient_quota",
|
|
"billing_not_active",
|
|
"payment_required",
|
|
"insufficient_credits",
|
|
"no_usable_credits",
|
|
"balance_depleted",
|
|
"model_not_supported_on_free_tier",
|
|
"member_spend_cap_exceeded",
|
|
_XAI_SPENDING_LIMIT_ERROR_CODE,
|
|
})
|
|
|
|
# Rate limiting (transient, will resolve)
|
|
_RATE_LIMIT_PATTERNS = [
|
|
"rate limit",
|
|
"rate_limit",
|
|
"too many requests",
|
|
"throttled",
|
|
"requests per minute",
|
|
"tokens per minute",
|
|
"requests per day",
|
|
"try again in",
|
|
"please retry after",
|
|
"resource_exhausted",
|
|
"rate increased too quickly", # Alibaba/DashScope throttling
|
|
# AWS Bedrock throttling
|
|
"throttlingexception",
|
|
"too many concurrent requests",
|
|
"servicequotaexceededexception",
|
|
# Generic throttle prefix — Bedrock/proxies say "Throttling error: Too many
|
|
# tokens...". Without it the message hits the context-overflow list ("too
|
|
# many tokens") and a healthy session gets compressed instead of backing
|
|
# off. Rate-limit runs BEFORE overflow in the message-only path so the
|
|
# throttle wins. (port of anomalyco/opencode#37848)
|
|
"throttling",
|
|
]
|
|
|
|
# Provider-side overload, NOT a per-credential rate limit or billing problem.
|
|
# The key is valid, so recover by "back off and retry the same key", never
|
|
# rotate (rotation exhausts the pool while the endpoint is busy; single-key
|
|
# users have nothing to rotate to). Z.AI/Zhipu reuse HTTP 429 for overload, so
|
|
# the 429 path checks this list before defaulting to rate_limit. Phrases are
|
|
# kept overload-flavoured so "you have been rate-limited" doesn't land here.
|
|
# (#14038, #15297)
|
|
_OVERLOADED_PATTERNS = [
|
|
"overloaded",
|
|
"temporarily overloaded",
|
|
"service is temporarily overloaded",
|
|
"service may be temporarily overloaded",
|
|
"server is overloaded",
|
|
"server overloaded",
|
|
"service overloaded",
|
|
"service is overloaded",
|
|
"upstream overloaded",
|
|
"currently overloaded",
|
|
"at capacity",
|
|
"over capacity",
|
|
]
|
|
|
|
# Usage-limit patterns needing disambiguation (billing OR rate_limit)
|
|
_USAGE_LIMIT_PATTERNS = [
|
|
"usage limit",
|
|
"quota",
|
|
"limit exceeded",
|
|
"key limit exceeded",
|
|
]
|
|
|
|
# Signals that a usage limit is transient (not billing)
|
|
_USAGE_LIMIT_TRANSIENT_SIGNALS = [
|
|
"try again",
|
|
"retry",
|
|
"resets at",
|
|
"reset in",
|
|
"resets in",
|
|
"reset after",
|
|
"available in",
|
|
"wait",
|
|
"requests remaining",
|
|
"periodic",
|
|
"window",
|
|
"per minute",
|
|
"per second",
|
|
]
|
|
|
|
# Payload-too-large from message text (no status_code attr) — proxies and some
|
|
# backends embed the HTTP status in the message.
|
|
_PAYLOAD_TOO_LARGE_PATTERNS = [
|
|
"request entity too large",
|
|
"payload too large",
|
|
"error code: 413",
|
|
# Anthropic's structured 413 type; proxies may re-wrap it into a plain
|
|
# message with no status — same compression recovery. (port of anomalyco/opencode#37848)
|
|
"request_too_large",
|
|
"request exceeds the maximum size",
|
|
]
|
|
|
|
# Image-size patterns, matched against 400 bodies (not 413): most providers
|
|
# return a specific image-too-big 400 before the request hits the 413 limit.
|
|
# Anthropic's hard 5 MB per-image cap matters most.
|
|
_IMAGE_TOO_LARGE_PATTERNS = [
|
|
"image exceeds", # Anthropic: "image exceeds 5 MB maximum"
|
|
"image too large", # generic
|
|
"image_too_large", # error_code variant
|
|
"image size exceeds", # variant
|
|
"image dimensions exceed", # Anthropic: "image dimensions exceed max allowed size: 8000 pixels"
|
|
"dimensions exceed max allowed size", # Anthropic dimension-cap (wording variant)
|
|
"max allowed size: 8000", # Anthropic dimension-cap (explicit pixel ceiling)
|
|
# MiniMax's Anthropic-compatible endpoint: "media exceeds size limit: max
|
|
# 10485760 bytes (2013)" for an oversized native image (#76039). A
|
|
# non-image media rejection landing here is safe: the shrink pass finds no
|
|
# image parts and the caller surfaces the original error unchanged.
|
|
"media exceeds",
|
|
"media too large",
|
|
# "request_too_large" on a request with an image → image is the likely
|
|
# culprit; the shrink path is still tried first.
|
|
]
|
|
|
|
# Image corruption — the provider decodes the request but not the image bytes
|
|
# (e.g. a re-serialized part in replayed history). Shrinking can't fix that,
|
|
# so this routes to strip-and-retry (image_corrupt), never the shrink path.
|
|
# xAI has two wordings for truncated bytes ("Invalid PNG image." aligned,
|
|
# "base64 string ... cannot be decoded" unaligned) and a third for URL images
|
|
# it downloads itself. The last is matched as the full sentence on purpose —
|
|
# shorter fragments also match non-image download failures. (#69078)
|
|
_IMAGE_CORRUPT_PATTERNS = [
|
|
"invalid png image",
|
|
"invalid jpeg image",
|
|
"base64 string of provided image cannot be decoded",
|
|
"downloaded response does not contain a valid jpg, png, webp, or ico image",
|
|
]
|
|
|
|
# Strict OpenAI-spec providers require tool message ``content`` to be a string;
|
|
# some (Anthropic native, Codex, Gemini, first-party OpenAI) accept a parts
|
|
# list so computer_use screenshots survive, others (Xiaomi MiMo, some Alibaba
|
|
# endpoints, a long tail) reject it with a 400. Recovery: strip image parts
|
|
# from tool messages, remember (provider, model) for the session, retry. (#27344)
|
|
_MULTIMODAL_TOOL_CONTENT_PATTERNS = [
|
|
# Xiaomi MiMo: {"error":{"code":"400","message":"Param Incorrect","param":"text is not set"}}
|
|
"text is not set",
|
|
# Generic "tool message must be string" shapes
|
|
"tool message content must be a string",
|
|
"tool content must be a string",
|
|
"tool message must be a string",
|
|
# OpenAI-compat schema-validation messages for list-type tool content
|
|
"expected string, got list",
|
|
"expected string, got array",
|
|
# Alibaba/DashScope variant
|
|
"tool_call.content must be string",
|
|
]
|
|
|
|
# Context overflow patterns
|
|
_CONTEXT_OVERFLOW_PATTERNS = [
|
|
"context length",
|
|
"context size",
|
|
"maximum context",
|
|
"token limit",
|
|
"too many tokens",
|
|
"reduce the length",
|
|
"exceeds the limit",
|
|
"context window",
|
|
"prompt is too long",
|
|
"prompt exceeds max length",
|
|
# Bare "max_tokens" is load-bearing — the output-cap-retry path keys off it
|
|
# ("max_tokens: 65536 > context_window: 200000"). Do NOT remove. Empty-
|
|
# response advisories also say "very low max_tokens" but are intercepted by
|
|
# _EMPTY_PROVIDER_RESPONSE_PATTERNS BEFORE this list is consulted.
|
|
"max_tokens",
|
|
"maximum number of tokens",
|
|
# vLLM / local inference server patterns
|
|
"exceeds the max_model_len",
|
|
"max_model_len",
|
|
"prompt length", # "engine prompt length X exceeds"
|
|
"input is too long",
|
|
"maximum model length",
|
|
# Ollama patterns
|
|
"context length exceeded",
|
|
"truncating input",
|
|
# llama.cpp / llama-server patterns
|
|
"slot context", # "slot context: N tokens, prompt N tokens"
|
|
"n_ctx_slot",
|
|
# Chinese error messages (some providers return these)
|
|
"超过最大长度",
|
|
"上下文长度",
|
|
# Z.AI / Zhipu GLM pattern (English form; error code 1210)
|
|
"tokens in request more than max tokens allowed",
|
|
# AWS Bedrock Converse API error patterns
|
|
"input is too long",
|
|
"max input token",
|
|
"input token",
|
|
"exceeds the maximum number of input tokens",
|
|
# Together/Fireworks: "Input length 131393 exceeds the maximum allowed
|
|
# input length of 131040 tokens." (port of anomalyco/opencode#37848)
|
|
"maximum allowed input length",
|
|
]
|
|
|
|
# Model not found patterns
|
|
_MODEL_NOT_FOUND_PATTERNS = [
|
|
"is not a valid model",
|
|
"invalid model",
|
|
"model not found",
|
|
"model_not_found",
|
|
"does not exist",
|
|
"no such model",
|
|
"unknown model",
|
|
"unsupported model",
|
|
# OpenRouter 404 when no endpoint for the model supports tool calling.
|
|
# model_not_found triggers fallback to a model/provider that does; without
|
|
# it the error is ``unknown``/retryable and burns every attempt on the
|
|
# same deterministic rejection. (PR #58446)
|
|
"no endpoints found that support tool use",
|
|
]
|
|
|
|
|
|
def _model_id_missing_known_prefix(model: str, provider: str) -> bool:
|
|
"""True when a bare model id is only known to the provider as ``vendor/id``.
|
|
|
|
Some providers answer a malformed id with a naked 404 (NVIDIA NIM: ``404
|
|
page not found`` for a bare ``nemotron-3-ultra-550b-a55b``), indistinguishable
|
|
from a bad endpoint path. If the id has no ``/`` but the curated catalogue
|
|
has exactly one entry ending in ``/<id>``, the prefix was dropped and the
|
|
failure is deterministic. Never guesses: an id absent from the catalogue
|
|
(local NIM container, proxied model) returns False so genuine endpoint
|
|
problems keep their retryable ``unknown`` classification.
|
|
"""
|
|
name = (model or "").strip()
|
|
if not name or "/" in name:
|
|
return False
|
|
try:
|
|
from hermes_cli.model_normalize import suggest_prefixed_model_id
|
|
|
|
return bool(suggest_prefixed_model_id((provider or "").strip(), name))
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
# Qwen/vLLM chat-template raise_exception("No user query found in messages").
|
|
# Shared by _INVALID_MESSAGE_BODY_PATTERNS (→ format_error) and the llama.cpp
|
|
# grammar exclusion guard so the two sites cannot drift.
|
|
_NO_USER_QUERY_SIGNAL = "no user query found"
|
|
|
|
# Malformed-message-array 400s: deterministic rejections of the *transcript*
|
|
# shape, not a parameter. Canonical case: a stream dies mid-response, Hermes
|
|
# persists a content-less assistant stub, and next turn Anthropic (or the
|
|
# litellm/Bedrock proxies in front of it) reject "all messages must have
|
|
# non-empty content ..." / errorCode INVALID_REQUEST_BODY. NOT context overflow
|
|
# — the input may be tiny — but the "400 + large session" heuristic used to
|
|
# route them into compression, ending in "Cannot compress further" every retry.
|
|
# Match explicitly and fail fast as format_error. (Stub creation is fixed in
|
|
# chat_completion_helpers; this covers transcripts already poisoned.)
|
|
_INVALID_MESSAGE_BODY_PATTERNS = [
|
|
"must have non-empty content",
|
|
"messages must have non-empty",
|
|
"invalid_request_body",
|
|
"text content blocks must be non-empty",
|
|
"content field is required",
|
|
"messages: at least one message is required",
|
|
# Qwen/vLLM templates raise this when no non-empty user turn survives
|
|
# (truncation, compression that dropped the only user message, a lineage
|
|
# opening with assistant/tool). Compression cannot invent a user query, so
|
|
# fail fast rather than thrash — and don't mis-route into llama.cpp grammar
|
|
# recovery when local engines wrap it as "Unable to generate parser ...".
|
|
_NO_USER_QUERY_SIGNAL,
|
|
]
|
|
|
|
# Request-validation patterns — malformed request, fails identically on every
|
|
# retry. Some OpenAI-compatible gateways (codex.nekos.me) return these as 5xx,
|
|
# so the generic "5xx → retryable server_error" rule would hammer the same
|
|
# rejection, reset the counter via transport recovery, and flood. A 5xx body
|
|
# carrying one of these is classified as non-retryable format_error.
|
|
_REQUEST_VALIDATION_PATTERNS = [
|
|
"unknown parameter",
|
|
"unsupported parameter",
|
|
"unrecognized request argument",
|
|
"invalid_request_error",
|
|
"unknown_parameter",
|
|
"unsupported_parameter",
|
|
]
|
|
|
|
# Params Hermes sends on SOME routes only → the providers/hosts where sending
|
|
# them is deliberate. When a host NOT in the set rejects one, the client never
|
|
# sent it — the provider's own gateway injected it — so the 400 is a server
|
|
# flake, not a deterministic request-shape error (see
|
|
# _is_server_injected_param_rejection and its branch in _classify_400).
|
|
# ``prompt_cache_retention`` is only sent for api.meta.ai and bedrock-mantle
|
|
# (agent/transports/codex.py::_default_prompt_cache_retention_for_request);
|
|
# the Codex OAuth backend rejects it spontaneously on requests that never
|
|
# carried it.
|
|
_SERVER_INJECTED_PARAM_SENDERS: Dict[str, tuple] = {
|
|
"prompt_cache_retention": ("meta", "muse", "msl", "model-api", "bedrock", "mantle"),
|
|
}
|
|
|
|
|
|
def _is_server_injected_param_rejection(error_msg: str, provider: str) -> bool:
|
|
"""True when a 400 blames a parameter this route never sends.
|
|
|
|
``error_msg`` is lowercased concatenated text; ``provider`` the lowercased
|
|
slug. A match means the rejection isn't our request shape, so retrying
|
|
the identical request is the correct recovery. Deliberately conservative:
|
|
fires only for known one-route-only params AND only when the provider is
|
|
not a route that sends them, so a genuine bad parameter (``max_tokens``
|
|
on GPT-5) still fails fast as ``format_error``.
|
|
"""
|
|
if not error_msg:
|
|
return False
|
|
provider_slug = (provider or "").strip().lower()
|
|
for param, senders in _SERVER_INJECTED_PARAM_SENDERS.items():
|
|
if param not in error_msg:
|
|
continue
|
|
# Must be a rejection of that parameter, not an incidental mention.
|
|
if not (
|
|
"not supported" in error_msg
|
|
or "unsupported" in error_msg
|
|
or "unknown" in error_msg
|
|
or "unrecognized" in error_msg
|
|
):
|
|
continue
|
|
# This route sends the field on purpose — a real request error.
|
|
return not any(sender in provider_slug for sender in senders)
|
|
return False
|
|
|
|
|
|
# OpenRouter aggregator policy block: an account privacy setting (or a
|
|
# per-request ``provider.data_collection: deny``) excludes the only endpoint
|
|
# serving a model → 404 "No endpoints available matching your guardrail
|
|
# restrictions and data policy. Configure: https://openrouter.ai/settings/privacy".
|
|
# Classified ``provider_policy_blocked`` not ``model_not_found``: the model
|
|
# exists, fallback won't help (account-level setting applies to every call),
|
|
# and the body already carries the fix URL.
|
|
_PROVIDER_POLICY_BLOCKED_PATTERNS = [
|
|
"no endpoints available matching your guardrail",
|
|
"no endpoints available matching your data policy",
|
|
"no endpoints found matching your data policy",
|
|
]
|
|
|
|
# Provider content-policy / safety-filter blocks — *per-prompt* decisions by
|
|
# the upstream model provider (unlike the OpenRouter account-level block
|
|
# above). Deterministic for the unchanged request, so retrying burns paid
|
|
# attempts on a refusal; switch to a fallback immediately or surface guidance.
|
|
# Patterns are verbatim strings from specific safety pipelines, never generic
|
|
# words ("policy", "violation") that collide with billing/auth/format errors.
|
|
_CONTENT_POLICY_BLOCKED_PATTERNS = [
|
|
# OpenAI Codex (#18028) — message may arrive without an HTTP status
|
|
"flagged for possible cybersecurity risk",
|
|
"trusted access for cyber",
|
|
# OpenAI moderation — chat completions / responses ("usage policies"
|
|
# disambiguates from billing's "exceeded ... policy")
|
|
"violates our usage policies",
|
|
"violates openai's usage policies",
|
|
"your request was flagged by",
|
|
# Anthropic safety system
|
|
"prompt was flagged by our safety",
|
|
"responses cannot be generated due to safety",
|
|
# Azure / OpenAI Responses: ``content_filter`` is the OpenAI-standard
|
|
# error/finish token; ``responsibleaipolicyviolation`` is Azure's code.
|
|
# Deliberately NOT the space variant ("content filter") — it appears in
|
|
# benign config/tooltip text that providers echo back.
|
|
"content_filter",
|
|
"responsibleaipolicyviolation",
|
|
# MiniMax output-layer safety filter, surfaced verbatim as "output
|
|
# new_sensitive (1027)" when the model's *output* (often a big tool-call
|
|
# block) trips the filter and the SSE stream is truncated. Narrow enough
|
|
# not to collide with billing/format/auth strings. (#32421)
|
|
"new_sensitive",
|
|
]
|
|
|
|
# Auth patterns (non-status-code signals)
|
|
_AUTH_PATTERNS = [
|
|
"invalid api key",
|
|
"invalid_api_key",
|
|
"gateway_auth_failed",
|
|
"authentication",
|
|
"unauthorized",
|
|
"forbidden",
|
|
"invalid token",
|
|
"token expired",
|
|
"token revoked",
|
|
"access denied",
|
|
]
|
|
|
|
# Provider empty-response advisories (OpenRouter / nano-gpt / similar).
|
|
# Checked before context-overflow matching: the advisory often mentions
|
|
# "max_tokens" as a cause, which sent healthy sessions into a compression
|
|
# death spiral ending in "Cannot compress further".
|
|
_EMPTY_PROVIDER_RESPONSE_PATTERNS = [
|
|
"returned an empty response",
|
|
"empty response despite retries",
|
|
"provider returned an empty response",
|
|
"model returning empty responses",
|
|
"empty response stream",
|
|
]
|
|
|
|
# Provider-side timeout wording when the exception type is generic (e.g.
|
|
# RuntimeError from a local shim wrapping a subprocess timeout). Checked
|
|
# before the type-based transport heuristics so these don't fall into the
|
|
# unknown bucket and get misreported as empty responses.
|
|
_TIMEOUT_MESSAGE_PATTERNS = [
|
|
"timed out",
|
|
"turn timed out",
|
|
"request timed out",
|
|
"deadline exceeded",
|
|
"operation timed out",
|
|
"upstream timed out",
|
|
]
|
|
|
|
# Connection-establishment / DNS failure wording for generic exception types
|
|
# (local shim, MCP bridge, SDK re-raising without chaining) where
|
|
# _TRANSPORT_ERROR_TYPES never fires and there is no HTTP status. Without
|
|
# message matching these become ``unknown`` and miss the transport
|
|
# eager-fallback path. (ported from anomalyco/opencode#40707)
|
|
# Deliberately EXCLUDES mid-stream disconnect strings ("connection reset by
|
|
# peer", "unexpected eof", ...) — those belong to _SERVER_DISCONNECT_PATTERNS,
|
|
# which runs later and routes large sessions to compression. A connection
|
|
# never established cannot be a server-side overflow rejection.
|
|
_CONNECTION_MESSAGE_PATTERNS = [
|
|
# TCP connect failures
|
|
"connection refused",
|
|
"econnrefused",
|
|
"no route to host",
|
|
"network is unreachable",
|
|
"network unreachable",
|
|
# DNS resolution failures (Python, glibc, macOS, Node bridge phrasings)
|
|
"name or service not known",
|
|
"temporary failure in name resolution",
|
|
"nodename nor servname provided",
|
|
"getaddrinfo failed",
|
|
"getaddrinfo enotfound",
|
|
"eai_again",
|
|
# Node/undici bridge generic network failure (MCP servers, local shims)
|
|
"fetch failed",
|
|
"failed to fetch",
|
|
# Envoy/proxy upstream connect failure (cloud gateways)
|
|
"upstream connect error",
|
|
]
|
|
|
|
# Transport error type names
|
|
_TRANSPORT_ERROR_TYPES = frozenset({
|
|
"ReadTimeout", "ConnectTimeout", "PoolTimeout",
|
|
"ConnectError", "RemoteProtocolError",
|
|
"ConnectionError", "ConnectionResetError",
|
|
"ConnectionAbortedError", "BrokenPipeError",
|
|
"TimeoutError", "ReadError",
|
|
"ServerDisconnectedError",
|
|
# Transient mid-stream SSL/TLS failures. ssl.SSLError is an OSError (caught
|
|
# by isinstance) but the names are listed so provider-wrapped SSL errors
|
|
# that lose the exception chain still classify as transport.
|
|
"SSLError", "SSLZeroReturnError", "SSLWantReadError",
|
|
"SSLWantWriteError", "SSLEOFError", "SSLSyscallError",
|
|
# OpenAI SDK errors (not subclasses of Python builtins)
|
|
"APIConnectionError",
|
|
"APITimeoutError",
|
|
})
|
|
|
|
# Server disconnect patterns (no status code, transport-level). Ambiguous: a
|
|
# plain close may be a transient hiccup OR a server-side context-overflow
|
|
# rejection (gateways disconnect instead of returning an HTTP error for
|
|
# oversized requests). Large session + one of these → compression recovery.
|
|
_SERVER_DISCONNECT_PATTERNS = [
|
|
"server disconnected",
|
|
"peer closed connection",
|
|
"connection reset by peer",
|
|
"connection was closed",
|
|
"network connection lost",
|
|
"unexpected eof",
|
|
"incomplete chunked read",
|
|
]
|
|
|
|
# SSL certificate verification failures — deterministic, NOT transient (TLS-
|
|
# inspecting proxy, missing CA, expired/self-signed cert). Burning the retry
|
|
# budget hides the actionable fix for minutes. Must be checked BEFORE
|
|
# _SSL_TRANSIENT_PATTERNS: these messages usually also contain "[SSL:", which
|
|
# would otherwise match the transient list and retry forever.
|
|
_SSL_CERT_VERIFY_PATTERNS = [
|
|
"certificate verify failed", # Python ssl module canonical text
|
|
"certificate_verify_failed", # OpenSSL error token
|
|
"unable to get local issuer certificate",
|
|
"self-signed certificate",
|
|
"self signed certificate",
|
|
"certificate has expired",
|
|
"hostname mismatch, certificate is not valid",
|
|
"unable to verify the first certificate", # Node/undici phrasing (MCP bridges)
|
|
]
|
|
|
|
# SSL/TLS transient failures — distinct from _SERVER_DISCONNECT_PATTERNS. A
|
|
# mid-stream SSL alert is a transport hiccup (flaky network, renegotiation,
|
|
# LB drop), NOT an overflow signal: retry but never compress. OpenSSL 3.x
|
|
# changed the code separator (SSLV3_ALERT_BAD_RECORD_MAC → SSL/TLS_ALERT_...),
|
|
# so match stable substrings that survive format churn.
|
|
_SSL_TRANSIENT_PATTERNS = [
|
|
# Space-separated (human-readable form, Python ssl module, most SDKs)
|
|
"bad record mac",
|
|
"ssl alert",
|
|
"tls alert",
|
|
"ssl handshake failure",
|
|
"tlsv1 alert",
|
|
"sslv3 alert",
|
|
# Underscore-separated OpenSSL tokens (ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC)
|
|
"bad_record_mac",
|
|
"ssl_alert",
|
|
"tls_alert",
|
|
"tls_alert_internal_error",
|
|
# Python ssl module prefix, e.g. "[SSL: BAD_RECORD_MAC]"
|
|
"[ssl:",
|
|
]
|
|
|
|
|
|
# ── Classification pipeline ─────────────────────────────────────────────
|
|
|
|
def _billing_verdict(result_fn, **overrides) -> ClassifiedError:
|
|
"""Non-retryable billing: rotate credential and fall back."""
|
|
return result_fn(
|
|
FailoverReason.billing,
|
|
retryable=False,
|
|
should_rotate_credential=True,
|
|
should_fallback=True,
|
|
**overrides,
|
|
)
|
|
|
|
|
|
def _rate_limit_verdict(result_fn) -> ClassifiedError:
|
|
"""Retryable rate limit: rotate credential and fall back."""
|
|
return result_fn(
|
|
FailoverReason.rate_limit,
|
|
retryable=True,
|
|
should_rotate_credential=True,
|
|
should_fallback=True,
|
|
)
|
|
|
|
|
|
def _overflow_or_empty_response(error_msg: str, result_fn) -> Optional[ClassifiedError]:
|
|
"""Shared body-refinement for 400/5xx: empty-response advisory vs overflow.
|
|
|
|
Empty-provider-response advisories often mention "max_tokens" and must
|
|
NOT enter compression (they used to thrash until "Cannot compress
|
|
further" on a healthy session); an explicit overflow signal routes to
|
|
compress-and-retry. Order matters: advisories first.
|
|
"""
|
|
if any(p in error_msg for p in _EMPTY_PROVIDER_RESPONSE_PATTERNS):
|
|
return result_fn(FailoverReason.server_error, retryable=True, should_compress=False)
|
|
if any(p in error_msg for p in _CONTEXT_OVERFLOW_PATTERNS):
|
|
return result_fn(FailoverReason.context_overflow, retryable=True, should_compress=True)
|
|
return None
|
|
|
|
|
|
def _policy_or_model_not_found(error_msg: str, result_fn) -> Optional[ClassifiedError]:
|
|
"""Aggregator policy block (checked first so it isn't mislabelled as a
|
|
missing model), then model-not-found."""
|
|
if any(p in error_msg for p in _PROVIDER_POLICY_BLOCKED_PATTERNS):
|
|
return result_fn(FailoverReason.provider_policy_blocked, retryable=False, should_fallback=False)
|
|
if any(p in error_msg for p in _MODEL_NOT_FOUND_PATTERNS):
|
|
return result_fn(FailoverReason.model_not_found, retryable=False, should_fallback=True)
|
|
return None
|
|
|
|
|
|
def _body_error_message(body: dict) -> str:
|
|
"""Lowercased message from ``error.message``, then flat ``message``.
|
|
|
|
Also reads litellm/Bedrock proxy shapes ({"errorMessage", "errorArgs":
|
|
{"reason"}}): without them a long descriptive rejection looks like a bare
|
|
"generic" error and, on a large session, mis-routes into compression.
|
|
"""
|
|
if not isinstance(body, dict):
|
|
return ""
|
|
err_obj = body.get("error", {})
|
|
msg = ""
|
|
if isinstance(err_obj, dict):
|
|
msg = str(err_obj.get("message") or "").strip().lower()
|
|
if not msg:
|
|
msg = str(body.get("message") or "").strip().lower()
|
|
if not msg:
|
|
msg = str(body.get("errorMessage") or "").strip().lower()
|
|
if not msg:
|
|
args = body.get("errorArgs")
|
|
if isinstance(args, dict):
|
|
msg = str(args.get("reason") or "").strip().lower()
|
|
return msg
|
|
|
|
|
|
def classify_api_error(
|
|
error: Exception,
|
|
*,
|
|
provider: str = "",
|
|
model: str = "",
|
|
approx_tokens: int = 0,
|
|
context_length: int = 200000,
|
|
num_messages: int = 0,
|
|
) -> ClassifiedError:
|
|
"""Classify an API error into a structured recovery recommendation.
|
|
|
|
Priority-ordered pipeline:
|
|
0. Plugin ``transform_api_error_classification`` hooks (first valid result wins)
|
|
1. Special-case provider-specific patterns (thinking sigs, tier gates)
|
|
2. HTTP status code + message-aware refinement
|
|
3. Error code classification (from body)
|
|
4. Message pattern matching (billing vs rate_limit vs context vs auth)
|
|
5. SSL cert-verify → fail fast; SSL/TLS transient alerts → retry as timeout
|
|
6. Server disconnect + large session → context overflow
|
|
7. Stale-call circuit breaker → failover; transport error heuristics
|
|
8. Fallback: unknown (retryable with backoff)
|
|
|
|
``approx_tokens``/``context_length``/``num_messages`` feed the large-session
|
|
heuristics; ``provider``/``model`` are echoed into the result.
|
|
"""
|
|
status_code = _extract_status_code(error)
|
|
error_type = type(error).__name__
|
|
# Copilot/GitHub Models RateLimitError may not set .status_code; force 429
|
|
# so rate-limit handling (reason, pool rotation, fallback gating) fires.
|
|
if status_code is None and error_type == "RateLimitError":
|
|
status_code = 429
|
|
body = _extract_error_body(error)
|
|
error_code = _extract_error_code(body)
|
|
response_headers = _extract_response_headers(error)
|
|
|
|
# Build the pattern-matching text. str(error) alone may omit the body
|
|
# message (OpenAI SDK APIStatusError.__str__ returns the first arg), so
|
|
# append it; also unwrap OpenRouter's metadata.raw, where the real upstream
|
|
# message ("context length exceeded") lives inside an inner JSON string.
|
|
_raw_msg = str(error).lower()
|
|
_body_msg = ""
|
|
_metadata_msg = ""
|
|
if isinstance(body, dict):
|
|
_err_obj = body.get("error", {})
|
|
if isinstance(_err_obj, dict):
|
|
_body_msg = str(_err_obj.get("message") or "").lower()
|
|
_metadata = _err_obj.get("metadata", {})
|
|
if isinstance(_metadata, dict):
|
|
_raw_json = _metadata.get("raw") or ""
|
|
if isinstance(_raw_json, str) and _raw_json.strip():
|
|
try:
|
|
import json
|
|
_inner = json.loads(_raw_json)
|
|
if isinstance(_inner, dict):
|
|
_inner_err = _inner.get("error", {})
|
|
if isinstance(_inner_err, dict):
|
|
_metadata_msg = str(_inner_err.get("message") or "").lower()
|
|
except (json.JSONDecodeError, TypeError):
|
|
pass
|
|
if not _body_msg:
|
|
_body_msg = str(body.get("message") or "").lower()
|
|
parts = [_raw_msg]
|
|
if _body_msg and _body_msg not in _raw_msg:
|
|
parts.append(_body_msg)
|
|
if _metadata_msg and _metadata_msg not in _raw_msg and _metadata_msg not in _body_msg:
|
|
parts.append(_metadata_msg)
|
|
error_msg = " ".join(parts)
|
|
provider_lower = (provider or "").strip().lower()
|
|
model_lower = (model or "").strip().lower()
|
|
|
|
def _result(reason: FailoverReason, **overrides) -> ClassifiedError:
|
|
defaults = {
|
|
"reason": reason,
|
|
"status_code": status_code,
|
|
"provider": provider,
|
|
"model": model,
|
|
"message": _extract_message(error, body),
|
|
}
|
|
defaults.update(overrides)
|
|
return ClassifiedError(**defaults)
|
|
|
|
# ── 0. Plugin classifiers (first valid result wins) ─────────────
|
|
# Consulted BEFORE the built-in pipeline so a provider plugin can add or
|
|
# correct classifications (contract: ``transform_api_error_classification``
|
|
# in hermes_cli.plugins.VALID_HOOKS). Callback exceptions and malformed
|
|
# returns are handled inside the helper; this guard only covers
|
|
# import/dispatch failure, so a broken plugin never breaks classification.
|
|
try:
|
|
from hermes_cli.plugins import get_plugin_error_classification
|
|
plugin_classification = get_plugin_error_classification(
|
|
provider=provider,
|
|
model=model,
|
|
status_code=status_code,
|
|
error_type=error_type,
|
|
error_code=error_code,
|
|
error_message=error_msg,
|
|
error_body=body,
|
|
error=error,
|
|
approx_tokens=approx_tokens,
|
|
context_length=context_length,
|
|
num_messages=num_messages,
|
|
)
|
|
except Exception as exc:
|
|
logger.debug("Plugin error classification unavailable: %s", exc)
|
|
plugin_classification = None
|
|
if plugin_classification is not None:
|
|
reason = plugin_classification.pop("reason")
|
|
logger.info(
|
|
"API error classified by plugin hook: %s (provider=%s, status=%s)",
|
|
reason.value, provider, status_code,
|
|
)
|
|
return _result(reason, **plugin_classification)
|
|
|
|
# ── 1. Provider-specific patterns (highest priority) ────────────
|
|
|
|
# Content-policy / safety block: deterministic refusal of THIS prompt.
|
|
# Runs before status classification so a 400 block isn't downgraded to
|
|
# ``format_error`` and a status-less block (OpenAI Codex SDK) isn't left
|
|
# retryable in ``unknown``. (#18028)
|
|
if any(p in error_msg for p in _CONTENT_POLICY_BLOCKED_PATTERNS):
|
|
return _result(FailoverReason.content_policy_blocked, retryable=False, should_fallback=True)
|
|
|
|
# Anthropic thinking-block 400s. Two failure modes, one recovery (strip
|
|
# reasoning_details and retry without thinking blocks — see the
|
|
# thinking_signature handler in conversation_loop.py):
|
|
# 1. Signature mismatch — blocks are signed against the full turn; any
|
|
# mutation (compression, truncation, merging) invalidates it.
|
|
# Pattern: "signature" + "thinking".
|
|
# 2. Frozen-block mutation — "`thinking` ... blocks in the latest
|
|
# assistant message cannot be modified ... must remain as they were".
|
|
# No "signature" token, so it used to hard-abort as a client error.
|
|
# Not gated on provider: OpenRouter proxies Anthropic errors verbatim.
|
|
if (
|
|
status_code == 400
|
|
and "thinking" in error_msg
|
|
and (
|
|
"signature" in error_msg
|
|
or "cannot be modified" in error_msg
|
|
or "must remain as they were" in error_msg
|
|
)
|
|
):
|
|
return _result(FailoverReason.thinking_signature, retryable=True, should_compress=False)
|
|
|
|
# Anthropic long-context tier gate (429 "extra usage" + "long context")
|
|
if status_code == 429 and "extra usage" in error_msg and "long context" in error_msg:
|
|
return _result(FailoverReason.long_context_tier, retryable=True, should_compress=True)
|
|
|
|
# Anthropic OAuth subscription rejects the 1M-context beta header (400 "The
|
|
# long context beta is not yet available for this subscription."). The
|
|
# recovery in run_agent.py rebuilds the client without the beta and retries
|
|
# once. Different status + phrase from the 429 tier gate above.
|
|
if status_code == 400 and "long context beta" in error_msg and "not yet available" in error_msg:
|
|
return _result(FailoverReason.oauth_long_context_beta_forbidden, retryable=True, should_compress=False)
|
|
|
|
# llama.cpp's json-schema-to-grammar rejects regex escapes (``\d``/``\w``)
|
|
# and most ``format`` values that MCP servers routinely emit in ``pattern``.
|
|
# On match the retry loop strips pattern/format from the tools and retries
|
|
# once; cloud providers accept these keywords and never hit this branch.
|
|
# Exclude the Qwen/vLLM "No user query found" template failure that local
|
|
# engines wrap as "Unable to generate parser for this template" — that is a
|
|
# poisoned transcript (→ _INVALID_MESSAGE_BODY_PATTERNS / format_error),
|
|
# and stripping keywords would retry uselessly.
|
|
if status_code == 400:
|
|
_llama_cpp_grammar_hit = (
|
|
"error parsing grammar" in error_msg
|
|
or "json-schema-to-grammar" in error_msg
|
|
or ("unable to generate parser" in error_msg and "template" in error_msg)
|
|
)
|
|
else:
|
|
_llama_cpp_grammar_hit = False
|
|
if _llama_cpp_grammar_hit and _NO_USER_QUERY_SIGNAL not in error_msg:
|
|
return _result(FailoverReason.llama_cpp_grammar_pattern, retryable=True, should_compress=False)
|
|
|
|
# xAI Grok subscription entitlement ("run out of available resources or do
|
|
# not have an active Grok subscription") arrives two ways: HTTP 403, which
|
|
# _classify_by_status routes to auth and _is_entitlement_failure then
|
|
# stops the refresh loop; or an SSE ``type=error`` frame with
|
|
# status_code=None, which skips the status path and matches no message
|
|
# list — without this guard it falls to retryable ``unknown`` and
|
|
# _is_entitlement_failure never runs (it only fires under auth).
|
|
if (
|
|
"do not have an active grok subscription" in error_msg
|
|
or ("out of available resources" in error_msg and "grok" in error_msg)
|
|
):
|
|
return _result(FailoverReason.auth, retryable=False, should_fallback=True)
|
|
|
|
# ── 2. HTTP status code classification ──────────────────────────
|
|
|
|
if status_code is not None:
|
|
classified = _classify_by_status(
|
|
status_code, error_msg, error_code, body,
|
|
provider=provider_lower, model=model_lower,
|
|
approx_tokens=approx_tokens, context_length=context_length,
|
|
num_messages=num_messages,
|
|
response_headers=response_headers,
|
|
result_fn=_result,
|
|
)
|
|
if classified is not None:
|
|
return classified
|
|
|
|
# Local MoA streaming compatibility errors are adapter-shape bugs, not a
|
|
# provider outage; falling back would silently swap the MoA route for a
|
|
# single-model answer (#55933 follow-up).
|
|
if provider_lower == "moa" and (
|
|
"'types.SimpleNamespace' object is not iterable" in str(error)
|
|
or "'types.SimpleNamespace' object has no attribute 'index'" in str(error)
|
|
):
|
|
return _result(FailoverReason.format_error, retryable=False, should_fallback=False)
|
|
|
|
# Local MoA config drift (persisted session names a renamed/deleted preset)
|
|
# is deterministic — retrying makes a config error look like an outage.
|
|
from agent.errors import MoAPresetNotFoundError
|
|
|
|
if isinstance(error, MoAPresetNotFoundError):
|
|
return _result(FailoverReason.model_not_found, retryable=False)
|
|
|
|
# ── 3. Error code classification ────────────────────────────────
|
|
|
|
if error_code:
|
|
classified = _classify_by_error_code(error_code, error_msg, _result)
|
|
if classified is not None:
|
|
return classified
|
|
|
|
# ── 4. Message pattern matching (no status code) ────────────────
|
|
|
|
classified = _classify_by_message(error_msg, _result)
|
|
if classified is not None:
|
|
return classified
|
|
|
|
# ── 5. SSL certificate verification failures → fail fast ────────
|
|
# Deterministic for the host; checked BEFORE the transient-SSL patterns
|
|
# because cert-verify messages also contain "[ssl:".
|
|
if any(p in error_msg for p in _SSL_CERT_VERIFY_PATTERNS):
|
|
return _result(FailoverReason.ssl_cert_verification, retryable=False, should_fallback=False)
|
|
|
|
# ── 5b. SSL/TLS transient errors → retry as timeout (not compression) ──
|
|
# Mid-stream SSL alerts are transport hiccups, not overflow signals; checked
|
|
# before the disconnect step so a large session isn't compressed for a
|
|
# flaky handshake. Also catches SDK re-raises that lose the ssl.SSLError type.
|
|
if any(p in error_msg for p in _SSL_TRANSIENT_PATTERNS):
|
|
return _result(FailoverReason.timeout, retryable=True)
|
|
|
|
# ── 6. Server disconnect + large session → context overflow ─────
|
|
# Must precede the generic transport catch: a disconnect on a large
|
|
# session is more likely overflow than a hiccup; otherwise
|
|
# RemoteProtocolError always maps to timeout regardless of size.
|
|
is_disconnect = any(p in error_msg for p in _SERVER_DISCONNECT_PATTERNS)
|
|
if is_disconnect and not status_code:
|
|
# Reasoning-model override: a disconnect here is far more likely the
|
|
# upstream proxy idle-killing a multi-minute thinking stream (NVIDIA NIM
|
|
# ~120s, NVIDIA/NemoClaw#4846; OpenAI/Anthropic similar) than a true
|
|
# overflow, even on large sessions — compressing would silently delete
|
|
# history on a phantom context-length error. The per-model stale-timeout
|
|
# floor in agent/reasoning_timeouts.py makes a real transport failure
|
|
# recoverable via retry, so reclassify as timeout. (Part 1 of #52310)
|
|
from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
|
|
if get_reasoning_stale_timeout_floor(model) is not None:
|
|
return _result(FailoverReason.timeout, retryable=True)
|
|
# Absolute token/message thresholds only proxy for smaller windows;
|
|
# large-context sessions can hold hundreds of messages well under budget.
|
|
is_large = approx_tokens > context_length * 0.6 or (
|
|
context_length <= 256000 and (approx_tokens > 120000 or num_messages > 200)
|
|
)
|
|
if is_large:
|
|
return _result(FailoverReason.context_overflow, retryable=True, should_compress=True)
|
|
return _result(FailoverReason.timeout, retryable=True)
|
|
|
|
# ── 7b. Stale-call circuit breaker → failover immediately ──────
|
|
# _check_stale_giveup() (agent/chat_completion_helpers.py) raises
|
|
# RuntimeError after N consecutive stale attempts, *before* any network
|
|
# call. Not a transport timeout: as ``unknown``/retryable it would burn
|
|
# all max_retries instantly against the same dead provider before
|
|
# fallback. Non-retryable + should_fallback activates fallback on first hit.
|
|
if (
|
|
error_type == "RuntimeError"
|
|
and "consecutive stale attempts" in error_msg
|
|
and "aborting this call" in error_msg
|
|
):
|
|
return _result(FailoverReason.timeout, retryable=False, should_fallback=True)
|
|
|
|
# ── 8. Transport / timeout heuristics ───────────────────────────
|
|
|
|
if error_type in _TRANSPORT_ERROR_TYPES or isinstance(error, (TimeoutError, ConnectionError, OSError)):
|
|
return _result(FailoverReason.timeout, retryable=True)
|
|
|
|
# ── 9. Fallback: unknown ────────────────────────────────────────
|
|
|
|
return _result(FailoverReason.unknown, retryable=True)
|
|
|
|
|
|
# ── Status code classification ──────────────────────────────────────────
|
|
|
|
def _classify_by_status(
|
|
status_code: int,
|
|
error_msg: str,
|
|
error_code: str,
|
|
body: dict,
|
|
*,
|
|
provider: str,
|
|
model: str,
|
|
approx_tokens: int,
|
|
context_length: int,
|
|
num_messages: int = 0,
|
|
response_headers=None,
|
|
result_fn,
|
|
) -> Optional[ClassifiedError]:
|
|
"""Classify based on HTTP status code with message-aware refinement."""
|
|
|
|
if status_code == 401:
|
|
# Not retryable on its own — pool rotation and provider refresh (Codex,
|
|
# Anthropic, Nous) run before the retryability check in run_agent.py
|
|
# and ``continue`` on success; on failure retryable=False hits the
|
|
# client-error abort path (which tries fallback first).
|
|
return result_fn(
|
|
FailoverReason.auth,
|
|
retryable=False,
|
|
should_rotate_credential=True,
|
|
should_fallback=True,
|
|
)
|
|
|
|
if status_code == 403:
|
|
# OpenRouter 403 "key limit exceeded" is billing; other providers also
|
|
# use 403 for account-plan or credit exhaustion.
|
|
if (
|
|
(provider == "xai-oauth" and error_code.lower() == _XAI_SPENDING_LIMIT_ERROR_CODE)
|
|
or "key limit exceeded" in error_msg
|
|
or "spending limit" in error_msg
|
|
or any(p in error_msg for p in _BILLING_PATTERNS)
|
|
):
|
|
return _billing_verdict(result_fn)
|
|
return result_fn(FailoverReason.auth, retryable=False, should_fallback=True)
|
|
|
|
if status_code == 402:
|
|
return _classify_402(error_msg, result_fn)
|
|
|
|
if status_code == 404:
|
|
# Nous API surfaces HA/NAS credit depletion as a paid model becoming
|
|
# unavailable on the Free Tier (404, not 402) — billing, not a missing
|
|
# model, so the loop can show top-up guidance.
|
|
if any(p in error_msg for p in _BILLING_PATTERNS):
|
|
return _billing_verdict(result_fn)
|
|
# OpenRouter policy-block 404 first (model exists; fallback won't help;
|
|
# body carries the fix URL), then model-not-found.
|
|
classified = _policy_or_model_not_found(error_msg, result_fn)
|
|
if classified is not None:
|
|
return classified
|
|
# A bare id the catalogue only knows prefixed is a malformed model id:
|
|
# NVIDIA NIM answers with a naked "404 page not found", so the generic
|
|
# branch below would burn retries and report an outage (#78796).
|
|
if _model_id_missing_known_prefix(model, provider):
|
|
return result_fn(FailoverReason.model_not_found, retryable=False, should_fallback=True)
|
|
# Generic 404 with no model signal: wrong endpoint path (local
|
|
# llama.cpp/Ollama/vLLM misconfig), proxy glitch, or transient backend
|
|
# issue. model_not_found would silently fall back and tell the model
|
|
# it is missing — wrong and wastes a turn. Let the loop surface it.
|
|
return result_fn(FailoverReason.unknown, retryable=True)
|
|
|
|
if status_code == 413:
|
|
return result_fn(FailoverReason.payload_too_large, retryable=True, should_compress=True)
|
|
|
|
if status_code == 429:
|
|
# long_context_tier already handled upstream. Z.AI/Zhipu reuse 429 for
|
|
# server-wide overload: the credential is valid, so back off and retry
|
|
# the same key rather than rotating (burns the pool, useless for a
|
|
# single-key user). (#14038)
|
|
if any(p in error_msg for p in _OVERLOADED_PATTERNS):
|
|
return result_fn(FailoverReason.overloaded, retryable=True)
|
|
# OpenRouter-wrapped upstream 429 ("Provider returned error"): the
|
|
# upstream model is throttled, the user's key is healthy — rotating
|
|
# would bench it ~24min for nothing. Fall back to a different model.
|
|
if _is_openrouter_upstream_error(body, provider):
|
|
upstream_provider = _extract_upstream_provider_name(body)
|
|
ctx = {"upstream_provider": upstream_provider} if upstream_provider else {}
|
|
return result_fn(
|
|
FailoverReason.upstream_rate_limit,
|
|
retryable=True,
|
|
should_rotate_credential=False,
|
|
should_fallback=True,
|
|
error_context=ctx,
|
|
)
|
|
# Account/subscription usage exhaustion is a quota wall, not a rate
|
|
# throttle; Anthropic returns it as 429 (``usage_limit_reached``,
|
|
# #93419) and the generic branch used to retry it. Also covers
|
|
# _USAGE_LIMIT_PATTERNS for other providers' hard quota walls, but ONLY
|
|
# when the message isn't an explicit rate-limit phrase — otherwise
|
|
# "Rate limit exceeded" ("limit exceeded") would promote to billing.
|
|
# Periodic quotas with an explicit reset/retry signal stay rate_limit.
|
|
# Explicit billing phrases in a 429 body are a hard wall too (#39441).
|
|
has_usage_limit = (
|
|
error_code.lower() == "usage_limit_reached"
|
|
or "usage_limit_reached" in error_msg
|
|
or any(p in error_msg for p in _USAGE_LIMIT_PATTERNS)
|
|
)
|
|
has_billing = any(p in error_msg for p in _BILLING_PATTERNS)
|
|
has_explicit_rate_limit = any(p in error_msg for p in _RATE_LIMIT_PATTERNS)
|
|
has_transient_signal = _has_usage_limit_transient_signal(error_msg, body, response_headers)
|
|
if (has_billing or has_usage_limit) and not has_explicit_rate_limit and not has_transient_signal:
|
|
return _billing_verdict(result_fn)
|
|
return _rate_limit_verdict(result_fn)
|
|
|
|
if status_code == 400:
|
|
return _classify_400(
|
|
error_msg, error_code, body,
|
|
provider=provider,
|
|
approx_tokens=approx_tokens,
|
|
context_length=context_length,
|
|
num_messages=num_messages,
|
|
result_fn=result_fn,
|
|
)
|
|
|
|
if status_code in {500, 502}:
|
|
# Some OpenAI-compatible gateways (codex.nekos.me → 502) return
|
|
# request-validation errors as 5xx. Deterministic, so the generic
|
|
# "5xx → retryable" rule would turn one bad request into a retry
|
|
# flood: detect the signals (message or structured code) and fail fast.
|
|
# Exception: a param WE never sent on this route was injected by the
|
|
# provider itself, so retrying is correct (mirrors _classify_400).
|
|
if (
|
|
any(p in error_msg for p in _REQUEST_VALIDATION_PATTERNS)
|
|
or error_code.lower() in {"invalid_request_error", "unknown_parameter", "unsupported_parameter"}
|
|
) and not _is_server_injected_param_rejection(error_msg, provider):
|
|
return result_fn(FailoverReason.format_error, retryable=False, should_fallback=True)
|
|
# llama.cpp/llama-server report context overflow as HTTP 500 instead of
|
|
# 400/413; route explicit overflow into compression (mirroring
|
|
# _classify_400) instead of blind server_error retries that drop the turn.
|
|
classified = _overflow_or_empty_response(error_msg, result_fn)
|
|
if classified is not None:
|
|
return classified
|
|
return result_fn(FailoverReason.server_error, retryable=True)
|
|
|
|
if status_code in {503, 529}:
|
|
# Same overflow-as-5xx variant (server busy / model-load OOM, or a
|
|
# Cloudflare/Tailscale hop relabeling the status); otherwise transient
|
|
# overload.
|
|
classified = _overflow_or_empty_response(error_msg, result_fn)
|
|
if classified is not None:
|
|
return classified
|
|
return result_fn(FailoverReason.overloaded, retryable=True)
|
|
|
|
# 408 Request Timeout is retry-safe by definition (RFC 9110 §15.5.9), not a
|
|
# malformed request; reverse proxies in front of llama.cpp/Ollama/vLLM emit
|
|
# it when a long generation outruns the read window. Route to ``timeout``
|
|
# rather than the generic 4xx abort below.
|
|
if status_code == 408:
|
|
return result_fn(FailoverReason.timeout, retryable=True)
|
|
|
|
# Other 4xx — non-retryable
|
|
if 400 <= status_code < 500:
|
|
return result_fn(FailoverReason.format_error, retryable=False, should_fallback=True)
|
|
|
|
# Other 5xx — retryable
|
|
if 500 <= status_code < 600:
|
|
return result_fn(FailoverReason.server_error, retryable=True)
|
|
|
|
return None
|
|
|
|
|
|
def _has_usage_limit_transient_signal(error_msg: str, body: dict, response_headers) -> bool:
|
|
"""Return whether a usage-limit response identifies a reset window."""
|
|
if any(pattern in error_msg for pattern in _USAGE_LIMIT_TRANSIENT_SIGNALS):
|
|
return True
|
|
|
|
payloads = [body]
|
|
if isinstance(body, dict) and isinstance(body.get("error"), dict):
|
|
payloads.append(body["error"])
|
|
reset_fields = ("resets_in_seconds", "resets_at", "reset_at", "retry_after")
|
|
for payload in payloads:
|
|
if not isinstance(payload, dict):
|
|
continue
|
|
if any(payload.get(f) is not None and payload.get(f) != "" for f in reset_fields):
|
|
return True
|
|
|
|
if response_headers and hasattr(response_headers, "get"):
|
|
for header in ("retry-after", "Retry-After", "x-ratelimit-reset", "X-RateLimit-Reset"):
|
|
value = response_headers.get(header)
|
|
if value is not None and value != "":
|
|
return True
|
|
return False
|
|
|
|
|
|
def _classify_402(error_msg: str, result_fn) -> ClassifiedError:
|
|
"""Disambiguate 402: billing exhaustion vs transient usage limit.
|
|
|
|
Some 402s are rate limits disguised as payment errors — "Usage limit, try
|
|
again in 5 minutes" is a periodic quota that resets, not a billing problem.
|
|
"""
|
|
has_usage_limit = any(p in error_msg for p in _USAGE_LIMIT_PATTERNS)
|
|
has_transient_signal = any(p in error_msg for p in _USAGE_LIMIT_TRANSIENT_SIGNALS)
|
|
if has_usage_limit and has_transient_signal:
|
|
return _rate_limit_verdict(result_fn)
|
|
return _billing_verdict(result_fn)
|
|
|
|
|
|
def _classify_400(
|
|
error_msg: str,
|
|
error_code: str,
|
|
body: dict,
|
|
*,
|
|
provider: str,
|
|
approx_tokens: int,
|
|
context_length: int,
|
|
num_messages: int = 0,
|
|
result_fn,
|
|
) -> ClassifiedError:
|
|
"""Classify 400 Bad Request — context overflow, format error, or generic.
|
|
|
|
Ordering is load-bearing; every early branch below explains why it must
|
|
precede context_overflow.
|
|
"""
|
|
|
|
# Multimodal tool content rejected. BEFORE image_too_large (different
|
|
# recovery: strip image parts from tool messages, mark the model
|
|
# no-list-tool-content for the session) and BEFORE context_overflow
|
|
# because "text is not set" is ambiguous alone but specific for a 400 on a
|
|
# request known to carry multimodal tool content.
|
|
if any(p in error_msg for p in _MULTIMODAL_TOOL_CONTENT_PATTERNS):
|
|
return result_fn(FailoverReason.multimodal_tool_content_unsupported, retryable=True)
|
|
|
|
# Image corruption (xAI's undecodable-image check). BEFORE image_too_large:
|
|
# corrupt bytes need strip-and-retry — shrinking can't repair a bad PNG.
|
|
if any(p in error_msg for p in _IMAGE_CORRUPT_PATTERNS):
|
|
return result_fn(FailoverReason.image_corrupt, retryable=True)
|
|
|
|
# Image too large (Anthropic's 5 MB per-image check). BEFORE context_overflow:
|
|
# messages can trip both ("exceeds" + "image") and shrinking is cheaper.
|
|
if any(p in error_msg for p in _IMAGE_TOO_LARGE_PATTERNS):
|
|
return result_fn(FailoverReason.image_too_large, retryable=True)
|
|
|
|
# Invalid encrypted reasoning replay blob (OpenAI Responses). BEFORE
|
|
# context_overflow: "encrypted content ... could not be verified" can trip
|
|
# the overflow heuristics. ``error_msg`` is lowercased upstream.
|
|
error_code_lower = (error_code or "").lower()
|
|
if (
|
|
error_code_lower == "invalid_encrypted_content"
|
|
or "invalid_encrypted_content" in error_msg
|
|
or ("encrypted content for item" in error_msg and "could not be verified" in error_msg)
|
|
or "could not decrypt the provided encrypted_content" in error_msg
|
|
):
|
|
return result_fn(FailoverReason.invalid_encrypted_content, retryable=True, should_fallback=False)
|
|
|
|
# Server-injected parameter rejection: a 400 blaming a field the client
|
|
# never sent. MUST precede the request-validation branch, which would abort
|
|
# the turn as a deterministic format_error. Observed live on the Codex
|
|
# OAuth backend: it intermittently adds ``prompt_cache_retention`` to its
|
|
# own upstream call and rejects it (~20% over n=20 on a 1-message request
|
|
# with no cache params); a byte-identical retry succeeds.
|
|
if _is_server_injected_param_rejection(error_msg, provider):
|
|
# The request shape was fine — never route this into compression.
|
|
return result_fn(FailoverReason.server_error, retryable=True, should_compress=False)
|
|
|
|
# Request-validation (unsupported/unknown parameter) MUST precede
|
|
# context_overflow: GPT-5 rejecting max_tokens says "Unsupported parameter:
|
|
# 'max_tokens' ... Use 'max_completion_tokens'", and "max_tokens" is an
|
|
# overflow pattern — without this guard the 400 enters the compression
|
|
# loop and ends in "Cannot compress further". Deterministic → fail fast.
|
|
# NOTE: deliberately NOT keyed off generic ``invalid_request_error`` —
|
|
# OpenAI stamps that code on genuine overflow 400s too.
|
|
if (
|
|
any(p in error_msg for p in _REQUEST_VALIDATION_PATTERNS if p != "invalid_request_error")
|
|
or error_code_lower in {"unknown_parameter", "unsupported_parameter"}
|
|
):
|
|
return result_fn(FailoverReason.format_error, retryable=False, should_fallback=True)
|
|
|
|
# Malformed message array (empty-content assistant stub, etc.). BEFORE
|
|
# context_overflow: the input can be tiny, so the "400 + large session"
|
|
# heuristic would thrash compression on an unchanged request. Checked
|
|
# against message text AND the structured code (litellm/Bedrock surface
|
|
# errorCode=INVALID_REQUEST_BODY).
|
|
if (
|
|
any(p in error_msg for p in _INVALID_MESSAGE_BODY_PATTERNS)
|
|
or error_code_lower == "invalid_request_body"
|
|
):
|
|
logger.warning(
|
|
"Malformed message array 400 (invalid request body) classified as "
|
|
"format_error, NOT context overflow — failing fast + falling back "
|
|
"instead of entering the compression loop. This usually means an "
|
|
"empty-content assistant stub is in the transcript; num_messages=%s "
|
|
"approx_tokens=%s. error=%.200s",
|
|
num_messages, approx_tokens, error_msg,
|
|
)
|
|
return result_fn(FailoverReason.format_error, retryable=False, should_fallback=True)
|
|
|
|
# Empty-response advisory (no compression) / explicit context overflow.
|
|
classified = _overflow_or_empty_response(error_msg, result_fn)
|
|
if classified is not None:
|
|
return classified
|
|
|
|
# Some providers return policy-block / model-not-found as 400 instead of 404.
|
|
classified = _policy_or_model_not_found(error_msg, result_fn)
|
|
if classified is not None:
|
|
return classified
|
|
|
|
# Some providers return rate limit / billing as 400 instead of 429/402.
|
|
if any(p in error_msg for p in _RATE_LIMIT_PATTERNS):
|
|
return _rate_limit_verdict(result_fn)
|
|
if any(p in error_msg for p in _BILLING_PATTERNS):
|
|
# "out of extra usage" on a 400 may be a content-filter rejection
|
|
# (#82154): mark unverified so surfaces hedge and the pool skips the bench.
|
|
return _billing_verdict(result_fn, error_context=_billing_ambiguity_context(error_msg))
|
|
|
|
# Generic 400 + large session → probable context overflow (Anthropic can
|
|
# return a bare "Error" message when context is too large).
|
|
err_body_msg = _body_error_message(body)
|
|
is_generic = len(err_body_msg) < 30 or err_body_msg in {"error", ""}
|
|
# Absolute token/message thresholds only proxy for smaller windows;
|
|
# large-context sessions can hold many messages well under budget.
|
|
is_large = approx_tokens > context_length * 0.4 or (
|
|
context_length <= 256000 and (approx_tokens > 80000 or num_messages > 80)
|
|
)
|
|
if is_generic and is_large:
|
|
return result_fn(FailoverReason.context_overflow, retryable=True, should_compress=True)
|
|
|
|
# Non-retryable format error
|
|
return result_fn(FailoverReason.format_error, retryable=False, should_fallback=True)
|
|
|
|
|
|
# ── Error code classification ───────────────────────────────────────────
|
|
|
|
def _classify_by_error_code(error_code: str, error_msg: str, result_fn) -> Optional[ClassifiedError]:
|
|
"""Classify by structured error codes from the response body."""
|
|
code_lower = error_code.lower()
|
|
|
|
# Some OpenAI-compatible endpoints encode deterministic request-validation
|
|
# failures as plain-text ``event: error`` SSE data behind HTTP 200. The
|
|
# unchanged request cannot succeed, but a configured fallback still may.
|
|
if code_lower == PROVIDER_STREAM_NON_JSON_ERROR_CODE and "request validation failed:" in error_msg:
|
|
return result_fn(FailoverReason.format_error, retryable=False, should_fallback=True)
|
|
|
|
if code_lower in {"resource_exhausted", "throttled", "rate_limit_exceeded"}:
|
|
return result_fn(FailoverReason.rate_limit, retryable=True, should_rotate_credential=True)
|
|
|
|
if code_lower in _BILLING_ERROR_CODES:
|
|
return _billing_verdict(result_fn)
|
|
|
|
if code_lower in {"model_not_found", "model_not_available", "invalid_model"}:
|
|
return result_fn(FailoverReason.model_not_found, retryable=False, should_fallback=True)
|
|
|
|
if code_lower in {"context_length_exceeded", "max_tokens_exceeded"}:
|
|
return result_fn(FailoverReason.context_overflow, retryable=True, should_compress=True)
|
|
|
|
if code_lower == "invalid_encrypted_content":
|
|
return result_fn(FailoverReason.invalid_encrypted_content, retryable=True, should_fallback=False)
|
|
|
|
return None
|
|
|
|
|
|
# ── Message pattern classification ──────────────────────────────────────
|
|
|
|
def _classify_by_message(error_msg: str, result_fn) -> Optional[ClassifiedError]:
|
|
"""Classify on message patterns when no status code is available.
|
|
|
|
Mirrors the 400 path's ordering (payload → multimodal → image corrupt →
|
|
image too large → usage limit → overloaded → billing → rate limit →
|
|
empty-response/overflow → auth → policy/model → timeout → connection).
|
|
"""
|
|
if any(p in error_msg for p in _PAYLOAD_TOO_LARGE_PATTERNS):
|
|
return result_fn(FailoverReason.payload_too_large, retryable=True, should_compress=True)
|
|
|
|
if any(p in error_msg for p in _MULTIMODAL_TOOL_CONTENT_PATTERNS):
|
|
return result_fn(FailoverReason.multimodal_tool_content_unsupported, retryable=True)
|
|
|
|
if any(p in error_msg for p in _IMAGE_CORRUPT_PATTERNS):
|
|
return result_fn(FailoverReason.image_corrupt, retryable=True)
|
|
|
|
if any(p in error_msg for p in _IMAGE_TOO_LARGE_PATTERNS):
|
|
return result_fn(FailoverReason.image_too_large, retryable=True)
|
|
|
|
# Usage-limit needs the same disambiguation as 402: a transient signal
|
|
# ("try again", "resets at") means a periodic quota, not exhaustion.
|
|
if any(p in error_msg for p in _USAGE_LIMIT_PATTERNS):
|
|
if any(p in error_msg for p in _USAGE_LIMIT_TRANSIENT_SIGNALS):
|
|
return _rate_limit_verdict(result_fn)
|
|
return _billing_verdict(result_fn)
|
|
|
|
# Overloaded BEFORE rate_limit/billing so a message-only "overloaded" (no
|
|
# 503/529, e.g. Anthropic-compatible proxies) backs off instead of falling
|
|
# to ``unknown`` or rotating credentials.
|
|
if any(p in error_msg for p in _OVERLOADED_PATTERNS):
|
|
return result_fn(FailoverReason.overloaded, retryable=True)
|
|
|
|
if any(p in error_msg for p in _BILLING_PATTERNS):
|
|
# Adapters can strip the status from Anthropic's "out of extra usage"
|
|
# 400, so the same ambiguity marking applies here (#82154).
|
|
return _billing_verdict(result_fn, error_context=_billing_ambiguity_context(error_msg))
|
|
|
|
if any(p in error_msg for p in _RATE_LIMIT_PATTERNS):
|
|
return _rate_limit_verdict(result_fn)
|
|
|
|
classified = _overflow_or_empty_response(error_msg, result_fn)
|
|
if classified is not None:
|
|
return classified
|
|
|
|
# Auth is never retried directly — the key is invalid and will fail again;
|
|
# retryable=False makes the caller rotate or fall back instead.
|
|
if any(p in error_msg for p in _AUTH_PATTERNS):
|
|
return result_fn(
|
|
FailoverReason.auth,
|
|
retryable=False,
|
|
should_rotate_credential=True,
|
|
should_fallback=True,
|
|
)
|
|
|
|
classified = _policy_or_model_not_found(error_msg, result_fn)
|
|
if classified is not None:
|
|
return classified
|
|
|
|
# Timeout wording from generic exception types (local shims / custom
|
|
# providers wrapping a subprocess/HTTP timeout): transport timeout so the
|
|
# loop rebuilds the client instead of treating the turn as empty output.
|
|
if any(p in error_msg for p in _TIMEOUT_MESSAGE_PATTERNS):
|
|
return result_fn(FailoverReason.timeout, retryable=True)
|
|
|
|
# Connection/DNS failure wording — same generic-type problem; classified
|
|
# as timeout (the transport bucket) so eager transport fallback and client
|
|
# rebuild apply. Never compression: nothing was ever sent.
|
|
if any(p in error_msg for p in _CONNECTION_MESSAGE_PATTERNS):
|
|
return result_fn(FailoverReason.timeout, retryable=True)
|
|
|
|
return None
|
|
|
|
|
|
# ── Helpers ─────────────────────────────────────────────────────────────
|
|
|
|
def _cause_chain(error: Exception, depth: int = 5):
|
|
"""Yield ``error`` then its __cause__/__context__ chain, bounded to ``depth``."""
|
|
current = error
|
|
for _ in range(depth):
|
|
yield current
|
|
cause = getattr(current, "__cause__", None) or getattr(current, "__context__", None)
|
|
if cause is None or cause is current:
|
|
break
|
|
current = cause
|
|
|
|
|
|
def _extract_status_code(error: Exception) -> Optional[int]:
|
|
"""Walk the error and its cause chain to find an HTTP status code."""
|
|
for current in _cause_chain(error):
|
|
code = getattr(current, "status_code", None)
|
|
if isinstance(code, int):
|
|
return code
|
|
# Some SDKs use .status instead of .status_code
|
|
code = getattr(current, "status", None)
|
|
if isinstance(code, int) and 100 <= code < 600:
|
|
return code
|
|
return None
|
|
|
|
|
|
def _extract_error_body(error: Exception) -> dict:
|
|
"""Extract the structured error body from an SDK exception or its cause chain."""
|
|
for current in _cause_chain(error):
|
|
body = getattr(current, "body", None)
|
|
if isinstance(body, dict):
|
|
return body
|
|
# Some errors have .response.json()
|
|
response = getattr(current, "response", None)
|
|
if response is not None:
|
|
try:
|
|
json_body = response.json()
|
|
if isinstance(json_body, dict):
|
|
return json_body
|
|
except Exception:
|
|
pass
|
|
return {}
|
|
|
|
|
|
def _extract_response_headers(error: Exception):
|
|
"""Walk the error and its cause chain to find response headers."""
|
|
for current in _cause_chain(error):
|
|
headers = getattr(getattr(current, "response", None), "headers", None)
|
|
if headers and hasattr(headers, "get"):
|
|
return headers
|
|
return {}
|
|
|
|
|
|
def _clean_code(code) -> str:
|
|
"""Stringify a code/type value; drop blanks and the useless literal "400"."""
|
|
if isinstance(code, (str, int)):
|
|
text = str(code).strip()
|
|
if text and text != "400":
|
|
return text
|
|
return ""
|
|
|
|
|
|
def _extract_error_code(body: dict) -> str:
|
|
"""Extract an error code string from the response body."""
|
|
if not body:
|
|
return ""
|
|
|
|
def _code_from_payload(payload) -> str:
|
|
"""Code/type from a nested error payload dict (defensive)."""
|
|
if not isinstance(payload, dict):
|
|
return ""
|
|
payload_error = payload.get("error", {})
|
|
if isinstance(payload_error, dict):
|
|
nested = payload_error.get("code") or payload_error.get("type") or ""
|
|
if isinstance(nested, str) and _clean_code(nested):
|
|
return _clean_code(nested)
|
|
return _clean_code(payload.get("code") or payload.get("error_code") or "")
|
|
|
|
error_obj = body.get("error", {})
|
|
if isinstance(error_obj, dict):
|
|
code = error_obj.get("code") or error_obj.get("type") or ""
|
|
if isinstance(code, str) and _clean_code(code):
|
|
return _clean_code(code)
|
|
|
|
# Some providers wrap the real JSON error body as a string inside
|
|
# error.message — peek for a nested code (Responses API surfaces
|
|
# ``invalid_encrypted_content`` this way).
|
|
message = error_obj.get("message")
|
|
if isinstance(message, str) and message.strip().startswith("{"):
|
|
import json
|
|
try:
|
|
inner = json.loads(message)
|
|
except (json.JSONDecodeError, TypeError):
|
|
inner = None
|
|
nested_code = _code_from_payload(inner)
|
|
if nested_code:
|
|
return nested_code
|
|
|
|
# Top-level code
|
|
return _clean_code(body.get("code") or body.get("error_code") or body.get("errorCode") or "")
|
|
|
|
|
|
def _extract_message(error: Exception, body: dict) -> str:
|
|
"""Extract the most informative error message (structured body first)."""
|
|
if body:
|
|
error_obj = body.get("error", {})
|
|
candidates = []
|
|
if isinstance(error_obj, dict):
|
|
candidates.append(error_obj.get("message", ""))
|
|
candidates.append(body.get("message", ""))
|
|
# litellm / Bedrock proxy shape: {"errorMessage": ..., "errorArgs": {"reason": ...}}
|
|
candidates.append(body.get("errorMessage", ""))
|
|
args = body.get("errorArgs")
|
|
if isinstance(args, dict):
|
|
candidates.append(args.get("reason", ""))
|
|
for msg in candidates:
|
|
if isinstance(msg, str) and msg.strip():
|
|
return msg.strip()[:500]
|
|
return str(error)[:500]
|
|
|
|
|
|
def _is_openrouter_upstream_error(body: Any, provider: str) -> bool:
|
|
"""Detect OpenRouter's aggregator-wrapped upstream provider errors.
|
|
|
|
OpenRouter wraps upstream errors (DeepSeek, Anthropic, ...) with the outer
|
|
message "Provider returned error" and the real error in ``metadata.raw``.
|
|
The user's OpenRouter key is healthy, so credential rotation is wrong.
|
|
"""
|
|
if not isinstance(body, dict):
|
|
return False
|
|
err = body.get("error")
|
|
if not isinstance(err, dict):
|
|
return False
|
|
if str(err.get("message") or "").strip().lower() != "provider returned error":
|
|
return False
|
|
# Require the explicit OpenRouter provider OR the metadata shape only
|
|
# OpenRouter produces (metadata.raw / metadata.provider_name).
|
|
if (provider or "").strip().lower() == "openrouter":
|
|
return True
|
|
metadata = err.get("metadata")
|
|
return isinstance(metadata, dict) and ("raw" in metadata or "provider_name" in metadata)
|
|
|
|
|
|
def _extract_upstream_provider_name(body: Any) -> Optional[str]:
|
|
"""Pull the upstream provider name out of OpenRouter's error metadata."""
|
|
if not isinstance(body, dict):
|
|
return None
|
|
err = body.get("error")
|
|
if not isinstance(err, dict):
|
|
return None
|
|
metadata = err.get("metadata")
|
|
if not isinstance(metadata, dict):
|
|
return None
|
|
name = metadata.get("provider_name")
|
|
if isinstance(name, str) and name.strip():
|
|
return name.strip()
|
|
return None
|