fix: every Bedrock client rebuild lands on the startup wire (Claude SDK, Converse region, guardrails)
Follow-up to 564aef2946 (Mantle SigV4 on /model). The same class of bug covered the
other two Bedrock wires and two more rebuild paths:
- Claude on Bedrock (anthropic_messages): startup builds an AnthropicBedrock SDK
client (SigV4 via boto3). /model, fallback-to-Bedrock and fallback restore built a
plain Anthropic client with api_key="aws-sdk" against bedrock-runtime → 401/403.
- Converse models (bedrock_converse: Nova, DeepSeek, Llama): only agent_init set
_bedrock_region / _bedrock_guardrail_config. After a rebuild the transport fell
back to us-east-1 and guardrail_config=None, so eu-/ap- users hit the wrong region
and configured Guardrails silently dropped. switch_model also built a pointless
OpenAI client against bedrock-runtime.
Introduce bedrock_adapter.bind_bedrock_runtime(agent, base_url, api_mode) as the one
place that puts an agent on a non-Mantle Bedrock wire, and call it from agent_init
(replacing the two duplicated bodies), _build_switched_client, _rebuild_primary_client
and _swap_fallback_clients. try_recover_primary_transport had a hand-copied version of
_rebuild_primary_client's ladder (and would have hit the same gap); it now calls the
shared helper. The region/guardrail parsing that lived in agent_init and
runtime_provider_backends is now bedrock_region_from_runtime_url /
bedrock_guardrail_config in the adapter.
Live probe on origin/main across switch_model, restore_primary_runtime and
_swap_fallback_clients for both wires: 0/6 correct before, 6/6 after.
This commit is contained in:
+5
-29
@@ -689,12 +689,6 @@ def _setup_logging(agent):
|
||||
# would starve the root file handlers. Noise reduction belongs in hermes_logging.
|
||||
|
||||
|
||||
def _bedrock_region_from_url(base_url) -> str:
|
||||
"""AWS region from a bedrock-runtime.<region>.amazonaws.com URL (default us-east-1)."""
|
||||
m = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url or "")
|
||||
return m.group(1) if m else "us-east-1"
|
||||
|
||||
|
||||
def _print_key_banner(key, label: str, warn_missing: bool = False) -> None:
|
||||
"""Masked credential line. ``key`` may be a callable Entra ID bearer provider (Azure
|
||||
Foundry) — never invoke or inspect it. Keys ≤ 12 chars (incl. "dummy-key") are not shown."""
|
||||
@@ -716,14 +710,10 @@ def _init_anthropic_client(agent, api_key, base_url, _provider_timeout):
|
||||
agent._anthropic_base_url = base_url
|
||||
if agent.provider == "bedrock":
|
||||
# AnthropicBedrock SDK for full feature parity (prompt caching, thinking budgets).
|
||||
from agent.anthropic_adapter import build_anthropic_bedrock_client
|
||||
_br_region = agent._bedrock_region = _bedrock_region_from_url(base_url)
|
||||
agent._anthropic_client = build_anthropic_bedrock_client(_br_region)
|
||||
agent._anthropic_api_key = "aws-sdk"
|
||||
agent._is_anthropic_oauth = False
|
||||
agent.api_key = "aws-sdk"
|
||||
from agent.bedrock_adapter import bind_bedrock_runtime
|
||||
bind_bedrock_runtime(agent, base_url, "anthropic_messages")
|
||||
if not agent.quiet_mode:
|
||||
print(f"🤖 AI Agent initialized with model: {agent.model} (AWS Bedrock + AnthropicBedrock SDK, {_br_region})")
|
||||
print(f"🤖 AI Agent initialized with model: {agent.model} (AWS Bedrock + AnthropicBedrock SDK, {agent._bedrock_region})")
|
||||
return
|
||||
# ANTHROPIC_TOKEN fallback only for native Anthropic — other anthropic_messages providers
|
||||
# must use their own key or Anthropic credentials leak to third-party endpoints.
|
||||
@@ -785,22 +775,8 @@ def _init_moa_client(agent, api_key):
|
||||
|
||||
def _init_bedrock_client(agent, base_url):
|
||||
"""bedrock_converse: boto3 directly, no OpenAI client."""
|
||||
agent._bedrock_region = _bedrock_region_from_url(base_url)
|
||||
# Guardrail config — read from config.yaml at init time.
|
||||
agent._bedrock_guardrail_config = None
|
||||
with suppress(Exception):
|
||||
from hermes_cli.config import load_config_readonly as _load_br_cfg
|
||||
_gr = _load_br_cfg().get("bedrock", {}).get("guardrail", {})
|
||||
if _gr.get("guardrail_identifier") and _gr.get("guardrail_version"):
|
||||
agent._bedrock_guardrail_config = {
|
||||
"guardrailIdentifier": _gr["guardrail_identifier"],
|
||||
"guardrailVersion": _gr["guardrail_version"],
|
||||
}
|
||||
for _src, _dst in (("stream_processing_mode", "streamProcessingMode"), ("trace", "trace")):
|
||||
if _gr.get(_src):
|
||||
agent._bedrock_guardrail_config[_dst] = _gr[_src]
|
||||
agent.client = None
|
||||
agent._client_kwargs = {}
|
||||
from agent.bedrock_adapter import bind_bedrock_runtime
|
||||
bind_bedrock_runtime(agent, base_url, "bedrock_converse")
|
||||
if not agent.quiet_mode:
|
||||
_gr_label = " + Guardrails" if agent._bedrock_guardrail_config else ""
|
||||
print(f"🤖 AI Agent initialized with model: {agent.model} (AWS Bedrock, {agent._bedrock_region}{_gr_label})")
|
||||
|
||||
@@ -924,6 +924,9 @@ def _rebuild_primary_client(agent, rt: Dict[str, Any], *, reason: str) -> None:
|
||||
# factory so the restored facade keeps the reference_callback relay wired at init — a bare
|
||||
# MoAClient() would silently stop emitting moa.reference/moa.aggregating display events (#53802).
|
||||
agent._anthropic_client = None
|
||||
elif agent.provider == "bedrock" and agent.api_mode in ("anthropic_messages", "bedrock_converse"):
|
||||
from agent.bedrock_adapter import bind_bedrock_runtime
|
||||
bind_bedrock_runtime(agent, agent.base_url, agent.api_mode)
|
||||
elif agent.api_mode == "anthropic_messages":
|
||||
_build_anthropic_client_from_runtime(agent, rt)
|
||||
else:
|
||||
@@ -958,16 +961,7 @@ def try_recover_primary_transport(
|
||||
agent._retire_shared_openai_client(agent.client, reason="primary_recovery")
|
||||
rt = agent._primary_runtime
|
||||
_apply_primary_runtime_fields(agent, rt)
|
||||
if agent.api_mode == "anthropic_messages":
|
||||
_build_anthropic_client_from_runtime(agent, rt)
|
||||
elif (agent.provider or "").strip().lower() == "moa":
|
||||
# MoA is a virtual provider with empty client_kwargs — rebuilding via _create_openai_client
|
||||
# would raise "api_key client option must be set". Recreate the facade through the shared
|
||||
# factory so the reference_callback relay survives recovery (#53802).
|
||||
from agent.moa_loop import build_moa_facade
|
||||
agent.client = build_moa_facade(agent, agent.model)
|
||||
else:
|
||||
agent.client = agent._create_openai_client(dict(rt["client_kwargs"]), reason="primary_recovery", shared=True)
|
||||
_rebuild_primary_client(agent, rt, reason="primary_recovery")
|
||||
wait_time = min(3 + retry_count, 8)
|
||||
agent._vprint(
|
||||
f"{agent.log_prefix}🔁 Transient {error_type} on {agent.provider} — "
|
||||
@@ -1903,6 +1897,12 @@ def _build_switched_client(agent, new_provider, api_key, base_url, api_mode, new
|
||||
agent._client_kwargs = {}
|
||||
agent.client = build_moa_facade(agent, agent.model)
|
||||
return
|
||||
if new_provider == "bedrock" and api_mode in ("anthropic_messages", "bedrock_converse"):
|
||||
# Non-Mantle Bedrock wires authenticate through boto3, never through the generic
|
||||
# Anthropic/OpenAI builders (which would ship the ``aws-sdk`` sentinel as a credential).
|
||||
from agent.bedrock_adapter import bind_bedrock_runtime
|
||||
bind_bedrock_runtime(agent, base_url or agent.base_url, api_mode)
|
||||
return
|
||||
if api_mode == "anthropic_messages":
|
||||
from agent.anthropic_adapter import build_anthropic_client
|
||||
from agent.anthropic_credentials import resolve_anthropic_token, _is_oauth_token
|
||||
|
||||
@@ -298,6 +298,49 @@ def resolve_bedrock_runtime_region(config: Optional[Dict[str, Any]] = None) -> s
|
||||
return cfg_region or resolve_bedrock_region()
|
||||
|
||||
|
||||
def bedrock_region_from_runtime_url(base_url: str) -> str:
|
||||
"""AWS region from a ``bedrock-runtime.<region>.amazonaws.com`` URL (default us-east-1)."""
|
||||
m = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url or "")
|
||||
return m.group(1) if m else "us-east-1"
|
||||
|
||||
|
||||
def bedrock_guardrail_config(config: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Converse ``guardrailConfig`` from ``bedrock.guardrail`` in config.yaml (None when unset)."""
|
||||
if config is None:
|
||||
config = {}
|
||||
with suppress(Exception):
|
||||
from hermes_cli.config import load_config_readonly
|
||||
config = load_config_readonly()
|
||||
gr = ((config or {}).get("bedrock") or {}).get("guardrail") or {}
|
||||
if not (gr.get("guardrail_identifier") and gr.get("guardrail_version")):
|
||||
return None
|
||||
out = {"guardrailIdentifier": gr["guardrail_identifier"], "guardrailVersion": gr["guardrail_version"]}
|
||||
for src, dst in (("stream_processing_mode", "streamProcessingMode"), ("trace", "trace")):
|
||||
if gr.get(src):
|
||||
out[dst] = gr[src]
|
||||
return out
|
||||
|
||||
|
||||
def bind_bedrock_runtime(agent, base_url: str, api_mode: str) -> None:
|
||||
"""Point *agent* at a non-Mantle Bedrock wire: ``bedrock_converse`` (boto3 direct, no SDK client) or
|
||||
``anthropic_messages`` (AnthropicBedrock SDK, SigV4 via the boto3 chain). ``aws-sdk`` is a sentinel,
|
||||
never a credential, so the generic Anthropic/OpenAI client builders must not see it. Startup and every
|
||||
later rebuild (/model switch, fallback restore, fallback-to-Bedrock) share this so region and guardrail
|
||||
state never lag the active endpoint."""
|
||||
agent._bedrock_region = bedrock_region_from_runtime_url(base_url)
|
||||
agent._bedrock_guardrail_config = bedrock_guardrail_config()
|
||||
agent.client = None
|
||||
agent._client_kwargs = {}
|
||||
agent.api_key = agent._anthropic_api_key = "aws-sdk"
|
||||
agent._anthropic_base_url = base_url
|
||||
agent._is_anthropic_oauth = False
|
||||
if api_mode == "anthropic_messages":
|
||||
from agent.anthropic_adapter import build_anthropic_bedrock_client
|
||||
agent._anthropic_client = build_anthropic_bedrock_client(agent._bedrock_region)
|
||||
else:
|
||||
agent._anthropic_client = None
|
||||
|
||||
|
||||
def bedrock_model_ids_or_none() -> Optional[List[str]]:
|
||||
"""Live-discover Bedrock model IDs; None on failure/empty so callers use the static list."""
|
||||
with suppress(Exception):
|
||||
|
||||
@@ -67,6 +67,11 @@ def _valid_credential_pair(api_key: Any, base_url: Any) -> bool:
|
||||
def _swap_fallback_clients(agent, fb_client, fb_provider: str, fb_model: str, fb_base_url: str, fb_api_mode: str) -> None:
|
||||
"""Install the fallback client(s) in place, honoring request_timeout_seconds (None = SDK default)."""
|
||||
timeout = get_provider_request_timeout(fb_provider, fb_model)
|
||||
if fb_provider == "bedrock" and fb_api_mode in ("anthropic_messages", "bedrock_converse"):
|
||||
# Non-Mantle Bedrock: boto3-chain auth, no OpenAI/Anthropic SDK client to carry over.
|
||||
from agent.bedrock_adapter import bind_bedrock_runtime
|
||||
bind_bedrock_runtime(agent, fb_base_url, fb_api_mode)
|
||||
return
|
||||
# The SDK exposes an empty/stale api_key when a rotating source is installed.
|
||||
key_provider = vars(fb_client).get("_api_key_provider")
|
||||
credential = key_provider if callable(key_provider) else fb_client.api_key
|
||||
|
||||
@@ -169,17 +169,6 @@ def _resolve_openrouter_runtime(
|
||||
# ── AWS Bedrock ────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _bedrock_guardrail_config(bedrock_cfg: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
gr = bedrock_cfg.get("guardrail", {})
|
||||
if not (gr.get("guardrail_identifier") and gr.get("guardrail_version")):
|
||||
return None
|
||||
config = {"guardrailIdentifier": gr["guardrail_identifier"], "guardrailVersion": gr["guardrail_version"]}
|
||||
for src_key, dst_key in (("stream_processing_mode", "streamProcessingMode"), ("trace", "trace")):
|
||||
if gr.get(src_key):
|
||||
config[dst_key] = gr[src_key]
|
||||
return config
|
||||
|
||||
|
||||
def _resolve_bedrock_runtime(requested_provider: str, model_cfg: Dict[str, Any], target_model: Optional[str]) -> Dict[str, Any]:
|
||||
"""AWS Bedrock with triple-path routing: OpenAI models → Bedrock Mantle's Responses endpoint;
|
||||
Claude → AnthropicBedrock SDK (prompt caching, thinking budgets); others → Converse API.
|
||||
@@ -187,7 +176,7 @@ def _resolve_bedrock_runtime(requested_provider: str, model_cfg: Dict[str, Any],
|
||||
go through Converse regardless of model."""
|
||||
from agent.bedrock_adapter import (bedrock_openai_base_url, has_aws_credentials, is_anthropic_bedrock_model,
|
||||
is_openai_bedrock_model, resolve_aws_auth_env_var, resolve_bedrock_bearer_token,
|
||||
resolve_bedrock_runtime_region)
|
||||
resolve_bedrock_runtime_region, bedrock_guardrail_config)
|
||||
from hermes_cli.config import load_config # direct (not the origin delegate), as before
|
||||
rp = _rp()
|
||||
# Explicitly selected bedrock trusts boto3's credential chain (IMDS, ECS/Lambda roles, SSO)
|
||||
@@ -206,7 +195,7 @@ def _resolve_bedrock_runtime(requested_provider: str, model_cfg: Dict[str, Any],
|
||||
# Region priority (config.yaml bedrock.region → env → us-east-1) lives in the adapter.
|
||||
region = resolve_bedrock_runtime_region({"bedrock": bedrock_cfg})
|
||||
auth_source = resolve_aws_auth_env_var() or "aws-sdk-default-chain"
|
||||
guardrail_config = _bedrock_guardrail_config(bedrock_cfg)
|
||||
guardrail_config = bedrock_guardrail_config({"bedrock": bedrock_cfg})
|
||||
current_model = str(target_model or model_cfg.get("default") or "").strip()
|
||||
has_bearer_token = bool(os.environ.get("AWS_BEARER_TOKEN_BEDROCK", "").strip())
|
||||
runtime = rp._runtime("bedrock", "bedrock_converse", f"https://bedrock-runtime.{region}.amazonaws.com", "aws-sdk",
|
||||
|
||||
@@ -1,15 +1,19 @@
|
||||
"""A /model switch onto a Bedrock Mantle (OpenAI-wire) model must rebuild the client with SigV4
|
||||
auth. ``aws-sdk`` is a Hermes sentinel for IAM-chain auth; agent_init installed the SigV4
|
||||
http_client but switch_model rebuilt bare ``{api_key, base_url}`` kwargs, so the SDK sent
|
||||
``Authorization: Bearer aws-sdk`` and Mantle answered 401."""
|
||||
"""Every non-startup Bedrock client rebuild (/model switch, fallback-to-Bedrock, fallback restore)
|
||||
must land on the same wire startup uses: AnthropicBedrock SDK for Claude, boto3-direct for
|
||||
Converse, with region and guardrail state derived from the active endpoint. Before the fix these
|
||||
paths handed the ``aws-sdk`` sentinel to the generic Anthropic/OpenAI builders (401) and left
|
||||
``_bedrock_region`` unset (requests to the wrong region; guardrails dropped)."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.bedrock_adapter import BedrockOpenAISigV4Auth
|
||||
from agent.context_compressor import ContextCompressor
|
||||
from run_agent import AIAgent
|
||||
|
||||
MANTLE = "https://bedrock-mantle.us-east-1.api.aws/openai/v1"
|
||||
RUNTIME_EU = "https://bedrock-runtime.eu-west-1.amazonaws.com"
|
||||
|
||||
|
||||
def _agent() -> AIAgent:
|
||||
@@ -25,11 +29,15 @@ def _agent() -> AIAgent:
|
||||
return agent
|
||||
|
||||
|
||||
@patch("agent.model_metadata.get_model_context_length", return_value=272_000)
|
||||
def test_switch_to_bedrock_mantle_installs_sigv4_http_client(_ctx, monkeypatch):
|
||||
@pytest.fixture
|
||||
def aws_env(monkeypatch):
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAFAKEFAKEFAKEFAKE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret")
|
||||
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
|
||||
|
||||
|
||||
@patch("agent.model_metadata.get_model_context_length", return_value=272_000)
|
||||
def test_switch_to_bedrock_mantle_installs_sigv4_http_client(_ctx, aws_env):
|
||||
agent = _agent()
|
||||
|
||||
agent.switch_model("openai.gpt-5.6-terra", "bedrock", api_key="aws-sdk", base_url=MANTLE, api_mode="codex_responses")
|
||||
@@ -37,3 +45,34 @@ def test_switch_to_bedrock_mantle_installs_sigv4_http_client(_ctx, monkeypatch):
|
||||
auth = getattr(agent.client._client, "auth", None)
|
||||
assert isinstance(auth, BedrockOpenAISigV4Auth), f"switched client would send Bearer aws-sdk (auth={auth!r})"
|
||||
assert auth.region == "us-east-1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, api_mode, anthropic_client_type",
|
||||
[("us.anthropic.claude-opus-4-6-v1", "anthropic_messages", "AnthropicBedrock"),
|
||||
("us.amazon.nova-pro-v1:0", "bedrock_converse", "NoneType")],
|
||||
)
|
||||
@patch("agent.model_metadata.get_model_context_length", return_value=200_000)
|
||||
def test_switch_to_bedrock_runtime_wires_binds_region_and_sdk(_ctx, aws_env, model, api_mode, anthropic_client_type):
|
||||
agent = _agent()
|
||||
|
||||
agent.switch_model(model, "bedrock", api_key="aws-sdk", base_url=RUNTIME_EU, api_mode=api_mode)
|
||||
|
||||
assert agent.client is None, "no OpenAI client belongs on a bedrock-runtime wire"
|
||||
assert type(agent._anthropic_client).__name__ == anthropic_client_type
|
||||
assert agent._bedrock_region == "eu-west-1"
|
||||
assert hasattr(agent, "_bedrock_guardrail_config")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("api_mode", ["anthropic_messages", "bedrock_converse"])
|
||||
def test_fallback_to_bedrock_binds_runtime_not_generic_client(aws_env, api_mode):
|
||||
from agent.client_lifecycle import _swap_fallback_clients
|
||||
agent = _agent()
|
||||
fb_client = MagicMock(base_url=RUNTIME_EU, api_key="aws-sdk")
|
||||
agent.model, agent.provider, agent.base_url, agent.api_mode = "us.amazon.nova-pro-v1:0", "bedrock", RUNTIME_EU, api_mode
|
||||
|
||||
_swap_fallback_clients(agent, fb_client, "bedrock", agent.model, RUNTIME_EU, api_mode)
|
||||
|
||||
assert agent.client is None and agent._client_kwargs == {}
|
||||
assert agent._bedrock_region == "eu-west-1"
|
||||
assert type(agent._anthropic_client).__name__ == ("AnthropicBedrock" if api_mode == "anthropic_messages" else "NoneType")
|
||||
|
||||
Reference in New Issue
Block a user