fix(gateway): hygiene compaction keeps the profile secret scope under multiplexing

Session-hygiene compaction ran _compress_context on a bare
loop.run_in_executor(None, ...) worker. Under gateway.multiplex_profiles the
profile secret scope and HERMES_HOME override are ContextVars installed by
the per-turn _profile_runtime_scope, and a bare worker starts with an empty
Context — so the summary model's get_secret(<PROVIDER>_API_KEY) failed
closed with UnscopedSecretError on EVERY hygiene pass and compaction
silently degraded to a lossy truncation (#100849 debug bundle:
'Failed to generate context summary: get_secret(SURPLUS_API_KEY) called
with no profile secret scope active').

- gateway/run.py: run both hygiene executor hops (detached-agent path and
  codex app-server path) inside copy_context().run, keeping the default
  executor so a fence-cancelled hung summary never occupies a gateway
  agent-work slot.
- agent/context_compressor.py: UnscopedSecretError is a missing-credential
  class failure — abort and preserve the session instead of dropping the
  middle window for a placeholder summary (same carve-out as 401/402/403).
- tools/daemon_pool.py: correct the salvaged docstrings — stdlib
  ThreadPoolExecutor only propagates contextvars from 3.14; nothing is
  stripped from the bundled runtime.
- tests: hygiene worker inherits caller ContextVars (fails on bare
  run_in_executor); UnscopedSecretError classified as access failure.

Live A/B (real get_secret in a run_in_executor worker, multiplex on, profile
.env scope installed): main -> UnscopedSecretError; fixed -> scoped value.
This commit is contained in:
Teknium
2026-09-01 22:02:29 -07:00
parent 6f625f7381
commit c5c9aa8d44
5 changed files with 106 additions and 21 deletions
+14
View File
@@ -210,6 +210,20 @@ _TRUNCATED_SUMMARY_MARKER = "finish_reason=length"
def _is_summary_access_or_quota_error(exc: Exception) -> bool:
"""Return True for non-retryable summary auth, permission, or quota errors."""
# A credential read that failed closed because no profile secret scope
# was active (multiplexed gateway, worker thread without the caller's
# ContextVars) is a missing-credential failure of our own making: the
# summary model cannot be reached until the spawn site is fixed, and a
# placeholder summary would only destroy the middle window for nothing.
# Classify it with the credential class so compress() preserves the
# session unchanged (#100849 bundle: every hygiene pass truncated).
try:
from agent.secret_scope import UnscopedSecretError
except Exception: # pragma: no cover - import guard
UnscopedSecretError = () # type: ignore[assignment]
if UnscopedSecretError and isinstance(exc, UnscopedSecretError):
return True
classified = classify_api_error(exc)
if classified.reason is FailoverReason.rate_limit:
return False
+18
View File
@@ -392,8 +392,12 @@ async def run_codex_hygiene_compaction(
count_before = getattr(compressor, "compression_count", 0)
try:
await asyncio.wait_for(
# copy_context().run: keep the caller's profile secret scope /
# HERMES_HOME override in the worker (multiplex_profiles) — same
# class as the detached-agent hygiene path below.
loop.run_in_executor(
None,
copy_context().run,
lambda: agent._compress_context(
history,
"",
@@ -21305,8 +21309,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
_hyg_commit_fence = CompressionCommitFence(
total_ceiling_seconds=_hyg_total_ceiling_seconds
)
# Default executor (NOT self._get_executor):
# a fence-cancelled hung summary must never
# occupy one of the gateway's agent-work
# slots. But it MUST run inside the caller's
# contextvars: under multiplex_profiles the
# profile secret scope / HERMES_HOME override
# live in ContextVars, and a bare
# run_in_executor worker starts with an empty
# Context — the summary model's
# get_secret(<PROVIDER>_API_KEY) then fails
# closed (UnscopedSecretError) and every
# hygiene compaction silently degrades to a
# lossy truncation (#100849 bundle).
_hyg_future = loop.run_in_executor(
None,
copy_context().run,
lambda: _hyg_agent._compress_context(
_hyg_msgs, "",
approx_tokens=_approx_tokens,
+13
View File
@@ -855,6 +855,19 @@ class TestAuthFailureAborts:
)
assert _is_summary_access_or_quota_error(err) is True
def test_unscoped_secret_read_is_terminal_access_failure(self):
# Multiplexed gateway: a credential read reached get_secret() from a
# worker thread without the profile scope. The summary model is
# unreachable until the spawn site is fixed — abort and preserve the
# session rather than truncating the middle window (#100849 bundle).
from agent.secret_scope import UnscopedSecretError
err = UnscopedSecretError(
"get_secret('SURPLUS_API_KEY') called with no profile secret scope "
"active while multiplexing is on."
)
assert _is_summary_access_or_quota_error(err) is True
@@ -336,3 +336,46 @@ def test_manual_compress_without_live_thread_reports_honestly():
host._compress_codex_app_server_session("tg:123", "sess-1")
)
assert "Nothing to compact" in reply
# ---------------------------------------------------------------------------
# Multiplexed gateway: the hygiene worker must see the caller's ContextVars
# (profile secret scope / HERMES_HOME override). A bare run_in_executor worker
# starts with an EMPTY Context, so get_secret(<PROVIDER>_API_KEY) inside the
# summary path fails closed and every hygiene compaction degrades to a lossy
# truncation (#100849 bundle).
# ---------------------------------------------------------------------------
def test_hygiene_worker_inherits_caller_contextvars(tmp_path):
import contextvars
import threading
marker = contextvars.ContextVar("hygiene_scope_marker", default=None)
seen = {}
class ScopeProbeAgent(LiveCodexAgent):
def _compress_context(self, messages, system_message, **kwargs):
seen["value"] = marker.get()
seen["thread"] = threading.current_thread().name
return super()._compress_context(messages, system_message, **kwargs)
agent = ScopeProbeAgent(mode="hermes")
key = "tg:ctx"
gw, _db = _gateway(tmp_path, key, agent)
async def _scoped():
token = marker.set("profile-scope")
try:
return await run_codex_hygiene_compaction(
gw, key, agent.session_id, auto_mode="hermes",
history=_history(), approx_tokens=345_000, timeout_seconds=30.0,
)
finally:
marker.reset(token)
assert asyncio.run(_scoped()) == "compacted"
assert seen["thread"] != "MainThread", "compaction must still run off-loop"
assert seen["value"] == "profile-scope", (
"hygiene worker lost the caller's ContextVars — under multiplex_profiles "
"this is the UnscopedSecretError / lossy-truncation regression"
)
+18 -21
View File
@@ -16,16 +16,16 @@ exit hook insists on joining.
- the interpreter's non-daemon thread join at shutdown skips them.
Semantics are otherwise identical (initializer/initargs, work queue,
idle-thread reuse) and, since #95119, so is context propagation:
``submit`` snapshots the submitting context with ``copy_context()`` and
runs each work item inside it, matching stdlib ``ThreadPoolExecutor``.
That matters because some bundled CPython runtime builds omit stdlib's
context propagation entirely, which silently drops contextvar-based state
(profile secret scope, HERMES_HOME override) in pool workers — e.g. the
context-compression timeout fence resolved auxiliary provider keys with
``UnscopedSecretError`` under the multiplexed gateway. Use it for any
pool whose work is best-effort or
independently interruptible and must never hold the process open:
idle-thread reuse), plus context propagation: ``submit`` snapshots the
submitting context with ``copy_context()`` and runs each work item inside
it. Stdlib ``ThreadPoolExecutor`` only does this from Python 3.14; on the
3.11-3.13 runtimes Hermes ships, a bare pool worker starts with an EMPTY
Context and silently drops contextvar-based state (profile secret scope,
HERMES_HOME override) — under the multiplexed gateway a credential read in
such a worker fails closed with ``UnscopedSecretError``. Propagating by
default makes every consumer safe even when it forgets
``propagate_context_to_thread``. Use it for any pool whose work is
best-effort or independently interruptible and must never hold the process open:
concurrent tool execution, background memory sync, catalog fan-out,
subagent timeout wrappers. Do NOT use it for work that must complete
before exit (durable writes) — those belong on foreground threads with
@@ -49,17 +49,14 @@ class DaemonThreadPoolExecutor(ThreadPoolExecutor):
def submit(self, fn, /, *args, **kwargs):
"""Submit a callable, propagating the caller's contextvars.
Stdlib ``ThreadPoolExecutor`` snapshots the submitting context with
``copy_context()`` and runs each work item inside it, so pool
workers inherit contextvar state such as the multiplexed profile
secret scope. Some bundled CPython runtime builds strip that
propagation from the stdlib executor (their ``_WorkItem.run`` calls
the callable directly), which broke auxiliary LLM key resolution
from the context-compression timeout fence with
``UnscopedSecretError``. Restore the stdlib behavior explicitly so
the daemon pool behaves identically on every runtime; on runtimes
that already propagate, the inner ``ctx.run`` re-applies the same
immutable context and is a no-op.
Python 3.14's ``ThreadPoolExecutor`` snapshots the submitting
context with ``copy_context()`` and runs each work item inside it;
3.11-3.13 (the runtimes Hermes ships) do not, so a pool worker
starts with an empty Context and loses the multiplexed profile
secret scope / HERMES_HOME override. Do it here unconditionally so
the daemon pool behaves identically on every runtime; on 3.14+ the
inner ``ctx.run`` re-applies the same immutable context and is a
no-op.
"""
ctx = copy_context()