fix: every Bedrock client rebuild lands on the startup wire (Claude SDK, Converse region, guardrails)

Follow-up to 564aef2946 (Mantle SigV4 on /model). The same class of bug covered the
other two Bedrock wires and two more rebuild paths:

- Claude on Bedrock (anthropic_messages): startup builds an AnthropicBedrock SDK
  client (SigV4 via boto3). /model, fallback-to-Bedrock and fallback restore built a
  plain Anthropic client with api_key="aws-sdk" against bedrock-runtime → 401/403.
- Converse models (bedrock_converse: Nova, DeepSeek, Llama): only agent_init set
  _bedrock_region / _bedrock_guardrail_config. After a rebuild the transport fell
  back to us-east-1 and guardrail_config=None, so eu-/ap- users hit the wrong region
  and configured Guardrails silently dropped. switch_model also built a pointless
  OpenAI client against bedrock-runtime.

Introduce bedrock_adapter.bind_bedrock_runtime(agent, base_url, api_mode) as the one
place that puts an agent on a non-Mantle Bedrock wire, and call it from agent_init
(replacing the two duplicated bodies), _build_switched_client, _rebuild_primary_client
and _swap_fallback_clients. try_recover_primary_transport had a hand-copied version of
_rebuild_primary_client's ladder (and would have hit the same gap); it now calls the
shared helper. The region/guardrail parsing that lived in agent_init and
runtime_provider_backends is now bedrock_region_from_runtime_url /
bedrock_guardrail_config in the adapter.

Live probe on origin/main across switch_model, restore_primary_runtime and
_swap_fallback_clients for both wires: 0/6 correct before, 6/6 after.
This commit is contained in:
Teknium
2026-09-10 12:40:45 -07:00
parent 6c3d4a4af7
commit 942973ae90
6 changed files with 110 additions and 58 deletions
+5 -29
View File
@@ -689,12 +689,6 @@ def _setup_logging(agent):
# would starve the root file handlers. Noise reduction belongs in hermes_logging.
def _bedrock_region_from_url(base_url) -> str:
"""AWS region from a bedrock-runtime.<region>.amazonaws.com URL (default us-east-1)."""
m = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url or "")
return m.group(1) if m else "us-east-1"
def _print_key_banner(key, label: str, warn_missing: bool = False) -> None:
"""Masked credential line. ``key`` may be a callable Entra ID bearer provider (Azure
Foundry) — never invoke or inspect it. Keys ≤ 12 chars (incl. "dummy-key") are not shown."""
@@ -716,14 +710,10 @@ def _init_anthropic_client(agent, api_key, base_url, _provider_timeout):
agent._anthropic_base_url = base_url
if agent.provider == "bedrock":
# AnthropicBedrock SDK for full feature parity (prompt caching, thinking budgets).
from agent.anthropic_adapter import build_anthropic_bedrock_client
_br_region = agent._bedrock_region = _bedrock_region_from_url(base_url)
agent._anthropic_client = build_anthropic_bedrock_client(_br_region)
agent._anthropic_api_key = "aws-sdk"
agent._is_anthropic_oauth = False
agent.api_key = "aws-sdk"
from agent.bedrock_adapter import bind_bedrock_runtime
bind_bedrock_runtime(agent, base_url, "anthropic_messages")
if not agent.quiet_mode:
print(f"🤖 AI Agent initialized with model: {agent.model} (AWS Bedrock + AnthropicBedrock SDK, {_br_region})")
print(f"🤖 AI Agent initialized with model: {agent.model} (AWS Bedrock + AnthropicBedrock SDK, {agent._bedrock_region})")
return
# ANTHROPIC_TOKEN fallback only for native Anthropic — other anthropic_messages providers
# must use their own key or Anthropic credentials leak to third-party endpoints.
@@ -785,22 +775,8 @@ def _init_moa_client(agent, api_key):
def _init_bedrock_client(agent, base_url):
"""bedrock_converse: boto3 directly, no OpenAI client."""
agent._bedrock_region = _bedrock_region_from_url(base_url)
# Guardrail config — read from config.yaml at init time.
agent._bedrock_guardrail_config = None
with suppress(Exception):
from hermes_cli.config import load_config_readonly as _load_br_cfg
_gr = _load_br_cfg().get("bedrock", {}).get("guardrail", {})
if _gr.get("guardrail_identifier") and _gr.get("guardrail_version"):
agent._bedrock_guardrail_config = {
"guardrailIdentifier": _gr["guardrail_identifier"],
"guardrailVersion": _gr["guardrail_version"],
}
for _src, _dst in (("stream_processing_mode", "streamProcessingMode"), ("trace", "trace")):
if _gr.get(_src):
agent._bedrock_guardrail_config[_dst] = _gr[_src]
agent.client = None
agent._client_kwargs = {}
from agent.bedrock_adapter import bind_bedrock_runtime
bind_bedrock_runtime(agent, base_url, "bedrock_converse")
if not agent.quiet_mode:
_gr_label = " + Guardrails" if agent._bedrock_guardrail_config else ""
print(f"🤖 AI Agent initialized with model: {agent.model} (AWS Bedrock, {agent._bedrock_region}{_gr_label})")
+10 -10
View File
@@ -924,6 +924,9 @@ def _rebuild_primary_client(agent, rt: Dict[str, Any], *, reason: str) -> None:
# factory so the restored facade keeps the reference_callback relay wired at init — a bare
# MoAClient() would silently stop emitting moa.reference/moa.aggregating display events (#53802).
agent._anthropic_client = None
elif agent.provider == "bedrock" and agent.api_mode in ("anthropic_messages", "bedrock_converse"):
from agent.bedrock_adapter import bind_bedrock_runtime
bind_bedrock_runtime(agent, agent.base_url, agent.api_mode)
elif agent.api_mode == "anthropic_messages":
_build_anthropic_client_from_runtime(agent, rt)
else:
@@ -958,16 +961,7 @@ def try_recover_primary_transport(
agent._retire_shared_openai_client(agent.client, reason="primary_recovery")
rt = agent._primary_runtime
_apply_primary_runtime_fields(agent, rt)
if agent.api_mode == "anthropic_messages":
_build_anthropic_client_from_runtime(agent, rt)
elif (agent.provider or "").strip().lower() == "moa":
# MoA is a virtual provider with empty client_kwargs — rebuilding via _create_openai_client
# would raise "api_key client option must be set". Recreate the facade through the shared
# factory so the reference_callback relay survives recovery (#53802).
from agent.moa_loop import build_moa_facade
agent.client = build_moa_facade(agent, agent.model)
else:
agent.client = agent._create_openai_client(dict(rt["client_kwargs"]), reason="primary_recovery", shared=True)
_rebuild_primary_client(agent, rt, reason="primary_recovery")
wait_time = min(3 + retry_count, 8)
agent._vprint(
f"{agent.log_prefix}🔁 Transient {error_type} on {agent.provider} — "
@@ -1903,6 +1897,12 @@ def _build_switched_client(agent, new_provider, api_key, base_url, api_mode, new
agent._client_kwargs = {}
agent.client = build_moa_facade(agent, agent.model)
return
if new_provider == "bedrock" and api_mode in ("anthropic_messages", "bedrock_converse"):
# Non-Mantle Bedrock wires authenticate through boto3, never through the generic
# Anthropic/OpenAI builders (which would ship the ``aws-sdk`` sentinel as a credential).
from agent.bedrock_adapter import bind_bedrock_runtime
bind_bedrock_runtime(agent, base_url or agent.base_url, api_mode)
return
if api_mode == "anthropic_messages":
from agent.anthropic_adapter import build_anthropic_client
from agent.anthropic_credentials import resolve_anthropic_token, _is_oauth_token
+43
View File
@@ -298,6 +298,49 @@ def resolve_bedrock_runtime_region(config: Optional[Dict[str, Any]] = None) -> s
return cfg_region or resolve_bedrock_region()
def bedrock_region_from_runtime_url(base_url: str) -> str:
"""AWS region from a ``bedrock-runtime.<region>.amazonaws.com`` URL (default us-east-1)."""
m = re.search(r"bedrock-runtime\.([a-z0-9-]+)\.", base_url or "")
return m.group(1) if m else "us-east-1"
def bedrock_guardrail_config(config: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
"""Converse ``guardrailConfig`` from ``bedrock.guardrail`` in config.yaml (None when unset)."""
if config is None:
config = {}
with suppress(Exception):
from hermes_cli.config import load_config_readonly
config = load_config_readonly()
gr = ((config or {}).get("bedrock") or {}).get("guardrail") or {}
if not (gr.get("guardrail_identifier") and gr.get("guardrail_version")):
return None
out = {"guardrailIdentifier": gr["guardrail_identifier"], "guardrailVersion": gr["guardrail_version"]}
for src, dst in (("stream_processing_mode", "streamProcessingMode"), ("trace", "trace")):
if gr.get(src):
out[dst] = gr[src]
return out
def bind_bedrock_runtime(agent, base_url: str, api_mode: str) -> None:
"""Point *agent* at a non-Mantle Bedrock wire: ``bedrock_converse`` (boto3 direct, no SDK client) or
``anthropic_messages`` (AnthropicBedrock SDK, SigV4 via the boto3 chain). ``aws-sdk`` is a sentinel,
never a credential, so the generic Anthropic/OpenAI client builders must not see it. Startup and every
later rebuild (/model switch, fallback restore, fallback-to-Bedrock) share this so region and guardrail
state never lag the active endpoint."""
agent._bedrock_region = bedrock_region_from_runtime_url(base_url)
agent._bedrock_guardrail_config = bedrock_guardrail_config()
agent.client = None
agent._client_kwargs = {}
agent.api_key = agent._anthropic_api_key = "aws-sdk"
agent._anthropic_base_url = base_url
agent._is_anthropic_oauth = False
if api_mode == "anthropic_messages":
from agent.anthropic_adapter import build_anthropic_bedrock_client
agent._anthropic_client = build_anthropic_bedrock_client(agent._bedrock_region)
else:
agent._anthropic_client = None
def bedrock_model_ids_or_none() -> Optional[List[str]]:
"""Live-discover Bedrock model IDs; None on failure/empty so callers use the static list."""
with suppress(Exception):
+5
View File
@@ -67,6 +67,11 @@ def _valid_credential_pair(api_key: Any, base_url: Any) -> bool:
def _swap_fallback_clients(agent, fb_client, fb_provider: str, fb_model: str, fb_base_url: str, fb_api_mode: str) -> None:
"""Install the fallback client(s) in place, honoring request_timeout_seconds (None = SDK default)."""
timeout = get_provider_request_timeout(fb_provider, fb_model)
if fb_provider == "bedrock" and fb_api_mode in ("anthropic_messages", "bedrock_converse"):
# Non-Mantle Bedrock: boto3-chain auth, no OpenAI/Anthropic SDK client to carry over.
from agent.bedrock_adapter import bind_bedrock_runtime
bind_bedrock_runtime(agent, fb_base_url, fb_api_mode)
return
# The SDK exposes an empty/stale api_key when a rotating source is installed.
key_provider = vars(fb_client).get("_api_key_provider")
credential = key_provider if callable(key_provider) else fb_client.api_key
+2 -13
View File
@@ -169,17 +169,6 @@ def _resolve_openrouter_runtime(
# ── AWS Bedrock ────────────────────────────────────────────────────────────────────────────
def _bedrock_guardrail_config(bedrock_cfg: Dict[str, Any]) -> Optional[Dict[str, Any]]:
gr = bedrock_cfg.get("guardrail", {})
if not (gr.get("guardrail_identifier") and gr.get("guardrail_version")):
return None
config = {"guardrailIdentifier": gr["guardrail_identifier"], "guardrailVersion": gr["guardrail_version"]}
for src_key, dst_key in (("stream_processing_mode", "streamProcessingMode"), ("trace", "trace")):
if gr.get(src_key):
config[dst_key] = gr[src_key]
return config
def _resolve_bedrock_runtime(requested_provider: str, model_cfg: Dict[str, Any], target_model: Optional[str]) -> Dict[str, Any]:
"""AWS Bedrock with triple-path routing: OpenAI models → Bedrock Mantle's Responses endpoint;
Claude → AnthropicBedrock SDK (prompt caching, thinking budgets); others → Converse API.
@@ -187,7 +176,7 @@ def _resolve_bedrock_runtime(requested_provider: str, model_cfg: Dict[str, Any],
go through Converse regardless of model."""
from agent.bedrock_adapter import (bedrock_openai_base_url, has_aws_credentials, is_anthropic_bedrock_model,
is_openai_bedrock_model, resolve_aws_auth_env_var, resolve_bedrock_bearer_token,
resolve_bedrock_runtime_region)
resolve_bedrock_runtime_region, bedrock_guardrail_config)
from hermes_cli.config import load_config # direct (not the origin delegate), as before
rp = _rp()
# Explicitly selected bedrock trusts boto3's credential chain (IMDS, ECS/Lambda roles, SSO)
@@ -206,7 +195,7 @@ def _resolve_bedrock_runtime(requested_provider: str, model_cfg: Dict[str, Any],
# Region priority (config.yaml bedrock.region → env → us-east-1) lives in the adapter.
region = resolve_bedrock_runtime_region({"bedrock": bedrock_cfg})
auth_source = resolve_aws_auth_env_var() or "aws-sdk-default-chain"
guardrail_config = _bedrock_guardrail_config(bedrock_cfg)
guardrail_config = bedrock_guardrail_config({"bedrock": bedrock_cfg})
current_model = str(target_model or model_cfg.get("default") or "").strip()
has_bearer_token = bool(os.environ.get("AWS_BEARER_TOKEN_BEDROCK", "").strip())
runtime = rp._runtime("bedrock", "bedrock_converse", f"https://bedrock-runtime.{region}.amazonaws.com", "aws-sdk",
@@ -1,15 +1,19 @@
"""A /model switch onto a Bedrock Mantle (OpenAI-wire) model must rebuild the client with SigV4
auth. ``aws-sdk`` is a Hermes sentinel for IAM-chain auth; agent_init installed the SigV4
http_client but switch_model rebuilt bare ``{api_key, base_url}`` kwargs, so the SDK sent
``Authorization: Bearer aws-sdk`` and Mantle answered 401."""
"""Every non-startup Bedrock client rebuild (/model switch, fallback-to-Bedrock, fallback restore)
must land on the same wire startup uses: AnthropicBedrock SDK for Claude, boto3-direct for
Converse, with region and guardrail state derived from the active endpoint. Before the fix these
paths handed the ``aws-sdk`` sentinel to the generic Anthropic/OpenAI builders (401) and left
``_bedrock_region`` unset (requests to the wrong region; guardrails dropped)."""
from unittest.mock import MagicMock, patch
import pytest
from agent.bedrock_adapter import BedrockOpenAISigV4Auth
from agent.context_compressor import ContextCompressor
from run_agent import AIAgent
MANTLE = "https://bedrock-mantle.us-east-1.api.aws/openai/v1"
RUNTIME_EU = "https://bedrock-runtime.eu-west-1.amazonaws.com"
def _agent() -> AIAgent:
@@ -25,11 +29,15 @@ def _agent() -> AIAgent:
return agent
@patch("agent.model_metadata.get_model_context_length", return_value=272_000)
def test_switch_to_bedrock_mantle_installs_sigv4_http_client(_ctx, monkeypatch):
@pytest.fixture
def aws_env(monkeypatch):
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAFAKEFAKEFAKEFAKE")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret")
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
@patch("agent.model_metadata.get_model_context_length", return_value=272_000)
def test_switch_to_bedrock_mantle_installs_sigv4_http_client(_ctx, aws_env):
agent = _agent()
agent.switch_model("openai.gpt-5.6-terra", "bedrock", api_key="aws-sdk", base_url=MANTLE, api_mode="codex_responses")
@@ -37,3 +45,34 @@ def test_switch_to_bedrock_mantle_installs_sigv4_http_client(_ctx, monkeypatch):
auth = getattr(agent.client._client, "auth", None)
assert isinstance(auth, BedrockOpenAISigV4Auth), f"switched client would send Bearer aws-sdk (auth={auth!r})"
assert auth.region == "us-east-1"
@pytest.mark.parametrize(
"model, api_mode, anthropic_client_type",
[("us.anthropic.claude-opus-4-6-v1", "anthropic_messages", "AnthropicBedrock"),
("us.amazon.nova-pro-v1:0", "bedrock_converse", "NoneType")],
)
@patch("agent.model_metadata.get_model_context_length", return_value=200_000)
def test_switch_to_bedrock_runtime_wires_binds_region_and_sdk(_ctx, aws_env, model, api_mode, anthropic_client_type):
agent = _agent()
agent.switch_model(model, "bedrock", api_key="aws-sdk", base_url=RUNTIME_EU, api_mode=api_mode)
assert agent.client is None, "no OpenAI client belongs on a bedrock-runtime wire"
assert type(agent._anthropic_client).__name__ == anthropic_client_type
assert agent._bedrock_region == "eu-west-1"
assert hasattr(agent, "_bedrock_guardrail_config")
@pytest.mark.parametrize("api_mode", ["anthropic_messages", "bedrock_converse"])
def test_fallback_to_bedrock_binds_runtime_not_generic_client(aws_env, api_mode):
from agent.client_lifecycle import _swap_fallback_clients
agent = _agent()
fb_client = MagicMock(base_url=RUNTIME_EU, api_key="aws-sdk")
agent.model, agent.provider, agent.base_url, agent.api_mode = "us.amazon.nova-pro-v1:0", "bedrock", RUNTIME_EU, api_mode
_swap_fallback_clients(agent, fb_client, "bedrock", agent.model, RUNTIME_EU, api_mode)
assert agent.client is None and agent._client_kwargs == {}
assert agent._bedrock_region == "eu-west-1"
assert type(agent._anthropic_client).__name__ == ("AnthropicBedrock" if api_mode == "anthropic_messages" else "NoneType")