fix(cache): scope prompt_cache_key by session to stop cross-session bucket sharing

Cherry-picked from PR #78959 by @JoaoMarcos44 with authorship preserved.
Follow-up: hoist _cache_scope_from_session_id(session_id) to a local in
build_kwargs so it's computed once instead of 4 times per call.

Closes #78941. Closes #79012. Closes #79013. Closes #79014. Closes #79015.

Co-authored-by: JoaoMarcos44 <joaomarcosdias444@gmail.com>
This commit is contained in:
joaomarcos
2026-08-05 12:38:35 +05:30
committed by kshitij
parent 2221010717
commit 34c3f06f91
10 changed files with 291 additions and 55 deletions
+12 -1
View File
@@ -1316,6 +1316,7 @@ class _CodexCompletionsAdapter:
# out of cache-key routing entirely — for those hosts, skip it here.
try:
from agent.transports.codex import (
_cache_scope_from_session_id,
_content_cache_key,
_default_prompt_cache_retention_for_request,
)
@@ -1328,7 +1329,15 @@ class _CodexCompletionsAdapter:
or base_url_host_matches(_host_src, "models.github.ai")
)
if not _is_xai and not _is_github and "prompt_cache_key" not in resp_kwargs:
_cache_key = _content_cache_key(instructions, resp_kwargs.get("tools"))
# Scope by the owning turn's session so two unrelated sessions
# with the same instructions/tools (e.g. compression, MoA,
# flush_memories firing back-to-back on different sessions)
# don't bucket-share a prompt cache slot (#78941). The main
# transport (agent/transports/codex.py::build_kwargs) does the
# same; this adapter had no session handle before
# set_runtime_main() started threading one through.
_scope = _cache_scope_from_session_id(_runtime_main_value("session_id"))
_cache_key = _content_cache_key(instructions, resp_kwargs.get("tools"), _scope)
if _cache_key:
resp_kwargs["prompt_cache_key"] = _cache_key
if "prompt_cache_retention" not in resp_kwargs:
@@ -3049,6 +3058,7 @@ def set_runtime_main(
api_key: Any = "",
api_mode: str = "",
auth_mode: str = "",
session_id: str = "",
) -> contextvars.Token:
"""Record the current context's live main runtime for auxiliary routing.
@@ -3070,6 +3080,7 @@ def set_runtime_main(
),
"api_mode": (api_mode or "").strip(),
"auth_mode": (auth_mode or "").strip().lower(),
"session_id": (session_id or "").strip(),
}
# Publish authoritative context before updating locked compatibility
# mirrors; concurrent sessions never read those mirrors at runtime.
+13 -4
View File
@@ -47,6 +47,7 @@ def _add_prompt_cache_key(
messages: list[dict[str, Any]],
tools: list[dict[str, Any]] | None,
supports_prompt_cache_key: bool,
session_id: str | None = None,
) -> None:
"""Add a content-addressed key only for an explicitly capable endpoint."""
if not supports_prompt_cache_key:
@@ -60,11 +61,17 @@ def _add_prompt_cache_key(
):
return
# Reuse the Responses transport's single authoritative hash algorithm so
# equivalent static prefixes route to the same cache bucket across modes.
from agent.transports.codex import _content_cache_key
# Reuse the Responses transport's single authoritative hash algorithm and
# session-scope normalization so equivalent static prefixes route to the
# same cache bucket across modes, without concentrating unrelated
# sessions into one shared bucket (see #78941).
from agent.transports.codex import _cache_scope_from_session_id, _content_cache_key
cache_key = _content_cache_key(_static_prompt_instructions(messages), tools)
cache_key = _content_cache_key(
_static_prompt_instructions(messages),
tools,
_cache_scope_from_session_id(session_id),
)
if cache_key:
api_kwargs["prompt_cache_key"] = cache_key
@@ -584,6 +591,7 @@ class ChatCompletionsTransport(ProviderTransport):
tools=api_kwargs.get("tools"),
supports_prompt_cache_key=bool(params.get("supports_prompt_cache_key"))
or _is_openai_api_base_url(params.get("base_url")),
session_id=params.get("session_id"),
)
return api_kwargs
@@ -733,6 +741,7 @@ class ChatCompletionsTransport(ProviderTransport):
messages=sanitized,
tools=api_kwargs.get("tools"),
supports_prompt_cache_key=bool(profile.supports_prompt_cache_key),
session_id=params.get("session_id"),
)
return api_kwargs
+71 -32
View File
@@ -10,6 +10,23 @@ import json
import re
from typing import Any, Dict, List, Optional
# Cron fires build session_id as ``cron_<job_id>_<YYYYMMDD_HHMMSS>`` (see
# cron/scheduler.py). The trailing timestamp is per-fire noise; stripped so
# repeat fires of the same job share a cache scope (see #51395/#52295).
_CRON_SESSION_ID_RE = re.compile(r"^(cron_.+)_\d{8}_\d{6}$")
def _cache_scope_from_session_id(session_id: Optional[str]) -> str:
"""Normalize a physical session_id into a stable logical cache scope.
Every non-cron session_id already identifies one conversation/agent
instance (main run, a specific child/subagent, a sibling child, ...),
so it is used unchanged. Only cron's per-fire timestamp needs stripping.
"""
sid = str(session_id or "")
match = _CRON_SESSION_ID_RE.match(sid)
return match.group(1) if match else sid
from agent.transports.base import ProviderTransport
from agent.transports.types import NormalizedResponse, ToolCall
@@ -117,20 +134,26 @@ def _default_prompt_cache_retention_for_request(
return None
def _content_cache_key(instructions: str, tools: Optional[List[Dict[str, Any]]]) -> Optional[str]:
"""Content-address the prompt cache key from the static request prefix.
def _content_cache_key(
instructions: str,
tools: Optional[List[Dict[str, Any]]],
scope_id: str = "",
) -> Optional[str]:
"""Content-address the prompt cache key within a logical cache scope.
Returns ``pck_<sha256[:24]>`` of (instructions + sorted tool schemas), or
None when there is nothing static to key on. The cache key is a routing
hint only — never a correctness boundary — so two requests sharing a system
prompt and tool set intentionally resolve to the same warm prefix bucket.
Returns ``pck_<sha256[:24]>`` of (scope_id + instructions + sorted tool
schemas), or None when there is nothing static to key on. The cache key
is a routing hint only — never a correctness boundary — so two requests
sharing a scope, system prompt, and tool set intentionally resolve to the
same warm prefix bucket.
The fix this exists for: recurring cron jobs build session_id as
``cron_<id>_<timestamp>``, so using session_id as the cache key made every
fire cache-cold. The static prefix (identity + tools) is identical across
fires, so hashing it gives a stable key that stays warm within the
provider's cache TTL. Sorting tools by name keeps the hash insertion-order
independent.
``scope_id`` (pass ``_cache_scope_from_session_id(session_id)``) keeps
unrelated sessions — independent conversations, main vs. child/subagent,
sibling children — from concentrating onto the same bucket merely because
their static prefix matches (see #78941), while still letting recurring
cron fires of one job share a stable key across their timestamped
session_ids (the original #51395/#52295 fix this built on). Sorting tools
by name keeps the hash insertion-order independent.
"""
if not instructions and not tools:
return None
@@ -143,9 +166,9 @@ def _content_cache_key(instructions: str, tools: Optional[List[Dict[str, Any]]])
tools_part = json.dumps(
sorted_tools, sort_keys=True, ensure_ascii=False, separators=(",", ":")
)
# \x00 separator so instructions ending in the tool JSON can't collide with
# a request whose instructions contain that JSON and whose tools are empty.
content = f"{instructions or ''}\x00{tools_part}"
# \x00 separators so a scope/instructions/tools boundary can't be forged
# by content that happens to contain the same bytes.
content = f"{scope_id}\x00{instructions or ''}\x00{tools_part}"
digest = hashlib.sha256(content.encode("utf-8", errors="replace")).hexdigest()[:24]
return f"pck_{digest}"
@@ -341,12 +364,17 @@ class ResponsesApiTransport(ProviderTransport):
session_id = params.get("session_id")
# prompt_cache_key is content-addressed from the static prefix
# (instructions + tools), NOT session_id — recurring cron jobs carry a
# per-fire timestamp in session_id (cron_<id>_<ts>) that made every run
# cache-cold. session_id is left untouched for transcript isolation and
# the cache-scope routing headers below. Falls back to session_id when
# there is no static content to hash.
cache_key = _content_cache_key(instructions, response_tools) or session_id
# (instructions + tools) scoped by session, NOT the raw session_id —
# recurring cron jobs carry a per-fire timestamp in session_id
# (cron_<id>_<ts>) that made every run cache-cold, so the scope strips
# that suffix (see _cache_scope_from_session_id). session_id is left
# untouched for transcript isolation and the cache-scope routing
# headers below. Falls back to session_id when there is no static
# content to hash.
_cache_scope = _cache_scope_from_session_id(session_id)
cache_key = _content_cache_key(
instructions, response_tools, _cache_scope
) or _cache_scope
# xAI Responses takes prompt_cache_key in extra_body (set further
# down); GitHub Models opts out of cache-key routing entirely.
if not is_github_responses and not is_xai_responses and cache_key:
@@ -428,13 +456,13 @@ class ResponsesApiTransport(ProviderTransport):
if is_codex_backend:
# The Codex backend rejects body-level ``extra_headers`` with
# HTTP 400, but the OpenAI SDK's ``extra_headers`` kwarg maps
# to actual HTTP request headers (not body fields). We need
# these headers for cache-scope routing so prompt cache hits
# remain high. Send session_id / x-client-request-id as HTTP
# headers while keeping ``prompt_cache_key`` in the body for
# standard OpenAI routing as a belt-and-braces fallback.
cache_scope_id = _bounded_prompt_cache_key(session_id)
if cache_scope_id:
# to actual HTTP request headers (not body fields). ``session_id``
# carries the raw physical session id — transcript/identity, per
# the #57012 contract — while ``x-client-request-id`` mirrors the
# body's effective ``prompt_cache_key`` so header and body always
# agree on the same routing bucket instead of diverging (#78941).
final_cache_key = kwargs.get("prompt_cache_key") or _bounded_prompt_cache_key(_cache_scope)
if session_id or final_cache_key:
existing_extra_headers = kwargs.get("extra_headers")
merged_extra_headers: Dict[str, str] = {}
if isinstance(existing_extra_headers, dict):
@@ -445,8 +473,10 @@ class ResponsesApiTransport(ProviderTransport):
if key and value is not None
}
)
merged_extra_headers["session_id"] = cache_scope_id
merged_extra_headers["x-client-request-id"] = cache_scope_id
if session_id:
merged_extra_headers["session_id"] = str(session_id)
if final_cache_key:
merged_extra_headers["x-client-request-id"] = final_cache_key
kwargs["extra_headers"] = merged_extra_headers
max_tokens = params.get("max_tokens")
@@ -464,18 +494,27 @@ class ResponsesApiTransport(ProviderTransport):
if key and value is not None
}
)
merged_extra_headers["x-grok-conv-id"] = session_id
# Scoped like the body cache key below — otherwise cron's
# per-fire timestamp in session_id (cron_<id>_<ts>) pins every
# fire of the same job to a different xAI backend server (#78941).
merged_extra_headers["x-grok-conv-id"] = _cache_scope
kwargs["extra_headers"] = merged_extra_headers
# xAI Responses cache-routing — body-level field per
# https://docs.x.ai/developers/advanced-api-usage/prompt-caching/maximizing-cache-hits.
# Sent via extra_body (not the typed kwarg) so it survives openai
# SDK builds whose Responses.stream() signature has dropped the field.
# A caller's request_overrides={"prompt_cache_key": ...} lands on
# the top-level kwarg set above — read it back here so an explicit
# override actually governs the field xAI reads, instead of being
# silently outrun by the auto-derived cache_key (#78941).
existing_extra_body = kwargs.get("extra_body")
merged_extra_body: Dict[str, Any] = {}
if isinstance(existing_extra_body, dict):
merged_extra_body.update(existing_extra_body)
merged_extra_body.setdefault("prompt_cache_key", cache_key)
merged_extra_body.setdefault(
"prompt_cache_key", kwargs.get("prompt_cache_key", cache_key)
)
kwargs["extra_body"] = merged_extra_body
extra_body = kwargs.get("extra_body")
+1
View File
@@ -393,6 +393,7 @@ def build_turn_context(
api_key=getattr(agent, "api_key", "") or "",
api_mode=getattr(agent, "api_mode", "") or "",
auth_mode=getattr(agent, "auth_mode", "") or "",
session_id=getattr(agent, "session_id", "") or "",
)
except Exception:
pass
+2 -1
View File
@@ -3,6 +3,7 @@
from typing import Any
from agent.portal_tags import get_conversation_context, nous_portal_tags
from agent.transports.codex import _cache_scope_from_session_id
from providers import register_provider
from providers.base import ProviderProfile
@@ -40,7 +41,7 @@ class NousProfile(ProviderProfile):
# session id; the ambient root additionally keeps the key stable for
# installs that opt back into rotating compaction, and across
# delegate-subagent trees.
sticky_key = get_conversation_context() or session_id
sticky_key = _cache_scope_from_session_id(get_conversation_context() or session_id)
if sticky_key:
body["session_id"] = sticky_key
provider_preferences = context.get("provider_preferences")
@@ -4,6 +4,7 @@ import logging
from typing import Any
from agent.portal_tags import get_conversation_context
from agent.transports.codex import _cache_scope_from_session_id
from providers import register_provider
from providers.base import ProviderProfile
@@ -95,7 +96,7 @@ class OpenRouterProfile(ProviderProfile):
# (f2f4df064d). The ambient value is the session-lineage ROOT, so it
# also stays stable for installs that opt out of the default
# ``compression.in_place: true`` and across delegate-subagent trees.
sticky_key = get_conversation_context() or session_id
sticky_key = _cache_scope_from_session_id(get_conversation_context() or session_id)
if sticky_key:
body["session_id"] = sticky_key
prefs = context.get("provider_preferences")
@@ -182,7 +183,7 @@ class OpenRouterProfile(ProviderProfile):
# backend server via this header, and aux calls pass no session_id, so
# reading the ambient conversation keeps compression/vision/MoA traffic
# on the same Grok backend as the conversation it belongs to.
grok_conv_id = get_conversation_context() or session_id
grok_conv_id = _cache_scope_from_session_id(get_conversation_context() or session_id)
if grok_conv_id and model and model.startswith(("x-ai/grok-", "xai/grok-")):
extra_headers["x-grok-conv-id"] = grok_conv_id
if extra_headers:
+63
View File
@@ -3187,6 +3187,69 @@ class TestCodexAuxiliaryAdapterTimeout:
assert time.monotonic() - started < 0.14
class TestCodexAuxiliaryAdapterCacheScope:
"""Regression for issue #78941: auxiliary Codex calls (compression,
flush_memories, MoA, session_search) must not bucket-share a prompt
cache slot across unrelated sessions just because their instructions
and tools happen to match.
"""
def _create_and_capture(self, *, session_id):
import agent.auxiliary_client as aux
class _FakeCreateStream:
def __iter__(self):
return iter([
SimpleNamespace(
type="response.output_item.done",
item=SimpleNamespace(
type="message",
content=[SimpleNamespace(type="output_text", text="ok")],
),
),
SimpleNamespace(type="response.completed", response=SimpleNamespace(
status="completed", id="r1", usage=None,
)),
])
def close(self):
pass
class FakeResponses:
def __init__(self):
self.kwargs = None
def create(self, **kwargs):
self.kwargs = kwargs
return _FakeCreateStream()
fake_client = SimpleNamespace(responses=FakeResponses(), base_url="")
adapter = aux._CodexCompletionsAdapter(fake_client, "gpt-5.5")
token = aux.set_runtime_main("openai", "gpt-5.5", session_id=session_id)
try:
adapter.create(
messages=[
{"role": "system", "content": "You are a memory summarizer."},
{"role": "user", "content": "Summarize the last turn."},
],
)
finally:
aux.reset_runtime_main(token)
return fake_client.responses.kwargs["prompt_cache_key"]
def test_different_sessions_get_different_cache_keys(self):
key_a = self._create_and_capture(session_id="session-A")
key_b = self._create_and_capture(session_id="session-B")
assert key_a != key_b
def test_cron_refires_of_the_same_job_share_a_cache_key(self):
first = self._create_and_capture(session_id="cron_job42_20260801_090000")
second = self._create_and_capture(session_id="cron_job42_20260802_090000")
other_job = self._create_and_capture(session_id="cron_job99_20260801_090000")
assert first == second
assert first != other_job
class TestCodexAuxiliaryToolMessageConversion:
"""Regression for issue #5709.
@@ -725,9 +725,28 @@ class TestPromptCacheKeyCapability:
supports_prompt_cache_key=True,
)["prompt_cache_key"]
first = key("cron_job_2026-07-15T10:00:00Z")
second = key("cron_job_2026-07-15T10:05:00Z")
first = key("cron_job_20260715_100000")
second = key("cron_job_20260715_100500")
assert first == second
assert first != key("cron_job_2026-07-15T10:05:00Z", instructions="You are different.")
assert first != key("cron_job_2026-07-15T10:05:00Z", tool_name="search")
assert first != key("cron_job_20260715_100500", instructions="You are different.")
assert first != key("cron_job_20260715_100500", tool_name="search")
def test_unrelated_sessions_get_distinct_keys(self, transport):
"""#78941: identical static prefix across unrelated (non-cron) sessions
must not collapse onto one shared prompt_cache_key."""
kw1 = transport.build_kwargs(
model="cache-model",
messages=self._messages("You are stable."),
tools=self._tools("lookup"),
session_id="session_alice_1",
supports_prompt_cache_key=True,
)
kw2 = transport.build_kwargs(
model="cache-model",
messages=self._messages("You are stable."),
tools=self._tools("lookup"),
session_id="session_bob_1",
supports_prompt_cache_key=True,
)
assert kw1["prompt_cache_key"] != kw2["prompt_cache_key"]
+71 -11
View File
@@ -74,6 +74,21 @@ class TestCodexBuildKwargs:
)
assert kw1["prompt_cache_key"] == kw2["prompt_cache_key"]
def test_cache_key_differs_across_unrelated_sessions(self, transport):
"""#78941: two unrelated sessions (different users/conversations)
sharing the same static prefix must NOT collapse onto the same
prompt_cache_key — session_id scopes the hash unless it is a cron
per-fire id, which is normalized to its stable job prefix instead."""
messages = [{"role": "user", "content": "Hi"}]
kw1 = transport.build_kwargs(
model="gpt-5.4", messages=messages, tools=[],
session_id="session_alice_1",
)
kw2 = transport.build_kwargs(
model="gpt-5.4", messages=messages, tools=[],
session_id="session_bob_1",
)
assert kw1["prompt_cache_key"] != kw2["prompt_cache_key"]
def test_github_responses_drops_message_item_id_end_to_end(self, transport):
# #32716: Copilot binds codex_message_items ids to a backend
@@ -235,6 +250,21 @@ class TestCodexBuildKwargs:
assert eb.get("prompt_cache_key") == "caller-override"
assert eb.get("other_field") == 42
def test_xai_top_level_override_also_governs_extra_body(self, transport):
"""A caller's top-level request_overrides={"prompt_cache_key": ...}
must win in extra_body.prompt_cache_key too -- the field xAI actually
reads -- instead of being silently outrun by the auto-derived
content-hash cache_key (#78941)."""
messages = [{"role": "user", "content": "Hi"}]
kw = transport.build_kwargs(
model="grok-4.3", messages=messages, tools=[],
session_id="conv-xai-1",
is_xai_responses=True,
request_overrides={"prompt_cache_key": "caller-top-level"},
)
assert kw["prompt_cache_key"] == "caller-top-level"
assert kw["extra_body"]["prompt_cache_key"] == "caller-top-level"
@@ -242,25 +272,55 @@ class TestCodexBuildKwargs:
@pytest.mark.parametrize("length", [64, 65])
def test_codex_cache_scope_boundary(self, transport, length):
session_id = "s" * length
scope = transport.build_kwargs(
kw = transport.build_kwargs(
model="gpt-5.4",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
session_id=session_id,
is_codex_backend=True,
request_overrides={"extra_headers": {"x-test": "1"}},
)
headers = kw["extra_headers"]
assert headers["x-test"] == "1"
# session_id header carries the raw physical id untouched regardless
# of length (#57012); x-client-request-id mirrors the body's
# effective (already-bounded) prompt_cache_key.
assert headers["session_id"] == session_id
assert headers["x-client-request-id"] == kw["prompt_cache_key"]
assert len(headers["x-client-request-id"]) <= 64
def test_codex_cache_scope_headers_normalize_cron_session_id(self, transport):
"""x-client-request-id shares a cache scope across cron re-fires of the
same job (cron per-fire timestamp stripped, same as prompt_cache_key),
while session_id stays the raw per-fire physical id (#57012)."""
first_run = transport.build_kwargs(
model="gpt-5.4",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
session_id="cron_job42_20260801_090000",
is_codex_backend=True,
)["extra_headers"]
second_run = transport.build_kwargs(
model="gpt-5.4",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
session_id="cron_job42_20260802_090000",
is_codex_backend=True,
)["extra_headers"]
other_job = transport.build_kwargs(
model="gpt-5.4",
messages=[{"role": "user", "content": "Hi"}],
tools=[],
session_id="cron_job99_20260801_090000",
is_codex_backend=True,
)["extra_headers"]
assert scope["x-test"] == "1"
assert len(scope["session_id"]) <= 64
assert scope["x-client-request-id"] == scope["session_id"]
if length == 64:
assert scope["session_id"] == session_id
else:
assert scope["session_id"].startswith("pck_")
assert scope["session_id"] != session_id
assert first_run["session_id"] == "cron_job42_20260801_090000"
assert second_run["session_id"] == "cron_job42_20260802_090000"
assert first_run["x-client-request-id"].startswith("pck_")
assert first_run["x-client-request-id"] == second_run["x-client-request-id"]
assert first_run["x-client-request-id"] != other_job["x-client-request-id"]
+32
View File
@@ -52,6 +52,14 @@ class TestOpenRouterProfile:
body = p.build_extra_body(provider_preferences={"allow": ["anthropic"]})
assert body["provider"] == {"allow": ["anthropic"]}
def test_sticky_session_id_normalizes_cron_timestamp(self):
"""Cron re-fires of the same job keep the same sticky routing key."""
p = get_provider_profile("openrouter")
first = p.build_extra_body(session_id="cron_job42_20260801_090000")
second = p.build_extra_body(session_id="cron_job42_20260802_090000")
assert first["session_id"] == "cron_job42"
assert first["session_id"] == second["session_id"]
@@ -86,6 +94,22 @@ class TestOpenRouterProfile:
)
assert tl["extra_headers"]["x-grok-conv-id"] == "sess-abc123"
def test_grok_conv_id_normalizes_cron_timestamp(self):
"""Cron re-fires of the same job must pin to the same xAI backend,
same as the body.session_id sticky key (#78941)."""
p = get_provider_profile("openrouter")
_, first = p.build_api_kwargs_extras(
model="x-ai/grok-4", session_id="cron_job42_20260801_090000",
)
_, second = p.build_api_kwargs_extras(
model="x-ai/grok-4", session_id="cron_job42_20260802_090000",
)
assert first["extra_headers"]["x-grok-conv-id"] == "cron_job42"
assert (
first["extra_headers"]["x-grok-conv-id"]
== second["extra_headers"]["x-grok-conv-id"]
)
@@ -133,6 +157,14 @@ class TestNousProfile:
body = p.build_extra_body()
assert body["tags"] == nous_portal_tags()
def test_sticky_session_id_normalizes_cron_timestamp(self):
"""Cron re-fires of the same job keep the same sticky routing key."""
p = get_provider_profile("nous")
first = p.build_extra_body(session_id="cron_job42_20260801_090000")
second = p.build_extra_body(session_id="cron_job42_20260802_090000")
assert first["session_id"] == "cron_job42"
assert first["session_id"] == second["session_id"]