fix(gateway): hygiene compaction keeps the profile secret scope under multiplexing
Session-hygiene compaction ran _compress_context on a bare loop.run_in_executor(None, ...) worker. Under gateway.multiplex_profiles the profile secret scope and HERMES_HOME override are ContextVars installed by the per-turn _profile_runtime_scope, and a bare worker starts with an empty Context — so the summary model's get_secret(<PROVIDER>_API_KEY) failed closed with UnscopedSecretError on EVERY hygiene pass and compaction silently degraded to a lossy truncation (#100849 debug bundle: 'Failed to generate context summary: get_secret(SURPLUS_API_KEY) called with no profile secret scope active'). - gateway/run.py: run both hygiene executor hops (detached-agent path and codex app-server path) inside copy_context().run, keeping the default executor so a fence-cancelled hung summary never occupies a gateway agent-work slot. - agent/context_compressor.py: UnscopedSecretError is a missing-credential class failure — abort and preserve the session instead of dropping the middle window for a placeholder summary (same carve-out as 401/402/403). - tools/daemon_pool.py: correct the salvaged docstrings — stdlib ThreadPoolExecutor only propagates contextvars from 3.14; nothing is stripped from the bundled runtime. - tests: hygiene worker inherits caller ContextVars (fails on bare run_in_executor); UnscopedSecretError classified as access failure. Live A/B (real get_secret in a run_in_executor worker, multiplex on, profile .env scope installed): main -> UnscopedSecretError; fixed -> scoped value.
This commit is contained in:
@@ -210,6 +210,20 @@ _TRUNCATED_SUMMARY_MARKER = "finish_reason=length"
|
||||
def _is_summary_access_or_quota_error(exc: Exception) -> bool:
|
||||
"""Return True for non-retryable summary auth, permission, or quota errors."""
|
||||
|
||||
# A credential read that failed closed because no profile secret scope
|
||||
# was active (multiplexed gateway, worker thread without the caller's
|
||||
# ContextVars) is a missing-credential failure of our own making: the
|
||||
# summary model cannot be reached until the spawn site is fixed, and a
|
||||
# placeholder summary would only destroy the middle window for nothing.
|
||||
# Classify it with the credential class so compress() preserves the
|
||||
# session unchanged (#100849 bundle: every hygiene pass truncated).
|
||||
try:
|
||||
from agent.secret_scope import UnscopedSecretError
|
||||
except Exception: # pragma: no cover - import guard
|
||||
UnscopedSecretError = () # type: ignore[assignment]
|
||||
if UnscopedSecretError and isinstance(exc, UnscopedSecretError):
|
||||
return True
|
||||
|
||||
classified = classify_api_error(exc)
|
||||
if classified.reason is FailoverReason.rate_limit:
|
||||
return False
|
||||
|
||||
@@ -392,8 +392,12 @@ async def run_codex_hygiene_compaction(
|
||||
count_before = getattr(compressor, "compression_count", 0)
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
# copy_context().run: keep the caller's profile secret scope /
|
||||
# HERMES_HOME override in the worker (multiplex_profiles) — same
|
||||
# class as the detached-agent hygiene path below.
|
||||
loop.run_in_executor(
|
||||
None,
|
||||
copy_context().run,
|
||||
lambda: agent._compress_context(
|
||||
history,
|
||||
"",
|
||||
@@ -21305,8 +21309,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
_hyg_commit_fence = CompressionCommitFence(
|
||||
total_ceiling_seconds=_hyg_total_ceiling_seconds
|
||||
)
|
||||
# Default executor (NOT self._get_executor):
|
||||
# a fence-cancelled hung summary must never
|
||||
# occupy one of the gateway's agent-work
|
||||
# slots. But it MUST run inside the caller's
|
||||
# contextvars: under multiplex_profiles the
|
||||
# profile secret scope / HERMES_HOME override
|
||||
# live in ContextVars, and a bare
|
||||
# run_in_executor worker starts with an empty
|
||||
# Context — the summary model's
|
||||
# get_secret(<PROVIDER>_API_KEY) then fails
|
||||
# closed (UnscopedSecretError) and every
|
||||
# hygiene compaction silently degrades to a
|
||||
# lossy truncation (#100849 bundle).
|
||||
_hyg_future = loop.run_in_executor(
|
||||
None,
|
||||
copy_context().run,
|
||||
lambda: _hyg_agent._compress_context(
|
||||
_hyg_msgs, "",
|
||||
approx_tokens=_approx_tokens,
|
||||
|
||||
@@ -855,6 +855,19 @@ class TestAuthFailureAborts:
|
||||
)
|
||||
assert _is_summary_access_or_quota_error(err) is True
|
||||
|
||||
def test_unscoped_secret_read_is_terminal_access_failure(self):
|
||||
# Multiplexed gateway: a credential read reached get_secret() from a
|
||||
# worker thread without the profile scope. The summary model is
|
||||
# unreachable until the spawn site is fixed — abort and preserve the
|
||||
# session rather than truncating the middle window (#100849 bundle).
|
||||
from agent.secret_scope import UnscopedSecretError
|
||||
|
||||
err = UnscopedSecretError(
|
||||
"get_secret('SURPLUS_API_KEY') called with no profile secret scope "
|
||||
"active while multiplexing is on."
|
||||
)
|
||||
assert _is_summary_access_or_quota_error(err) is True
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -336,3 +336,46 @@ def test_manual_compress_without_live_thread_reports_honestly():
|
||||
host._compress_codex_app_server_session("tg:123", "sess-1")
|
||||
)
|
||||
assert "Nothing to compact" in reply
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multiplexed gateway: the hygiene worker must see the caller's ContextVars
|
||||
# (profile secret scope / HERMES_HOME override). A bare run_in_executor worker
|
||||
# starts with an EMPTY Context, so get_secret(<PROVIDER>_API_KEY) inside the
|
||||
# summary path fails closed and every hygiene compaction degrades to a lossy
|
||||
# truncation (#100849 bundle).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_hygiene_worker_inherits_caller_contextvars(tmp_path):
|
||||
import contextvars
|
||||
import threading
|
||||
|
||||
marker = contextvars.ContextVar("hygiene_scope_marker", default=None)
|
||||
seen = {}
|
||||
|
||||
class ScopeProbeAgent(LiveCodexAgent):
|
||||
def _compress_context(self, messages, system_message, **kwargs):
|
||||
seen["value"] = marker.get()
|
||||
seen["thread"] = threading.current_thread().name
|
||||
return super()._compress_context(messages, system_message, **kwargs)
|
||||
|
||||
agent = ScopeProbeAgent(mode="hermes")
|
||||
key = "tg:ctx"
|
||||
gw, _db = _gateway(tmp_path, key, agent)
|
||||
|
||||
async def _scoped():
|
||||
token = marker.set("profile-scope")
|
||||
try:
|
||||
return await run_codex_hygiene_compaction(
|
||||
gw, key, agent.session_id, auto_mode="hermes",
|
||||
history=_history(), approx_tokens=345_000, timeout_seconds=30.0,
|
||||
)
|
||||
finally:
|
||||
marker.reset(token)
|
||||
|
||||
assert asyncio.run(_scoped()) == "compacted"
|
||||
assert seen["thread"] != "MainThread", "compaction must still run off-loop"
|
||||
assert seen["value"] == "profile-scope", (
|
||||
"hygiene worker lost the caller's ContextVars — under multiplex_profiles "
|
||||
"this is the UnscopedSecretError / lossy-truncation regression"
|
||||
)
|
||||
|
||||
+18
-21
@@ -16,16 +16,16 @@ exit hook insists on joining.
|
||||
- the interpreter's non-daemon thread join at shutdown skips them.
|
||||
|
||||
Semantics are otherwise identical (initializer/initargs, work queue,
|
||||
idle-thread reuse) and, since #95119, so is context propagation:
|
||||
``submit`` snapshots the submitting context with ``copy_context()`` and
|
||||
runs each work item inside it, matching stdlib ``ThreadPoolExecutor``.
|
||||
That matters because some bundled CPython runtime builds omit stdlib's
|
||||
context propagation entirely, which silently drops contextvar-based state
|
||||
(profile secret scope, HERMES_HOME override) in pool workers — e.g. the
|
||||
context-compression timeout fence resolved auxiliary provider keys with
|
||||
``UnscopedSecretError`` under the multiplexed gateway. Use it for any
|
||||
pool whose work is best-effort or
|
||||
independently interruptible and must never hold the process open:
|
||||
idle-thread reuse), plus context propagation: ``submit`` snapshots the
|
||||
submitting context with ``copy_context()`` and runs each work item inside
|
||||
it. Stdlib ``ThreadPoolExecutor`` only does this from Python 3.14; on the
|
||||
3.11-3.13 runtimes Hermes ships, a bare pool worker starts with an EMPTY
|
||||
Context and silently drops contextvar-based state (profile secret scope,
|
||||
HERMES_HOME override) — under the multiplexed gateway a credential read in
|
||||
such a worker fails closed with ``UnscopedSecretError``. Propagating by
|
||||
default makes every consumer safe even when it forgets
|
||||
``propagate_context_to_thread``. Use it for any pool whose work is
|
||||
best-effort or independently interruptible and must never hold the process open:
|
||||
concurrent tool execution, background memory sync, catalog fan-out,
|
||||
subagent timeout wrappers. Do NOT use it for work that must complete
|
||||
before exit (durable writes) — those belong on foreground threads with
|
||||
@@ -49,17 +49,14 @@ class DaemonThreadPoolExecutor(ThreadPoolExecutor):
|
||||
def submit(self, fn, /, *args, **kwargs):
|
||||
"""Submit a callable, propagating the caller's contextvars.
|
||||
|
||||
Stdlib ``ThreadPoolExecutor`` snapshots the submitting context with
|
||||
``copy_context()`` and runs each work item inside it, so pool
|
||||
workers inherit contextvar state such as the multiplexed profile
|
||||
secret scope. Some bundled CPython runtime builds strip that
|
||||
propagation from the stdlib executor (their ``_WorkItem.run`` calls
|
||||
the callable directly), which broke auxiliary LLM key resolution
|
||||
from the context-compression timeout fence with
|
||||
``UnscopedSecretError``. Restore the stdlib behavior explicitly so
|
||||
the daemon pool behaves identically on every runtime; on runtimes
|
||||
that already propagate, the inner ``ctx.run`` re-applies the same
|
||||
immutable context and is a no-op.
|
||||
Python 3.14's ``ThreadPoolExecutor`` snapshots the submitting
|
||||
context with ``copy_context()`` and runs each work item inside it;
|
||||
3.11-3.13 (the runtimes Hermes ships) do not, so a pool worker
|
||||
starts with an empty Context and loses the multiplexed profile
|
||||
secret scope / HERMES_HOME override. Do it here unconditionally so
|
||||
the daemon pool behaves identically on every runtime; on 3.14+ the
|
||||
inner ``ctx.run`` re-applies the same immutable context and is a
|
||||
no-op.
|
||||
"""
|
||||
ctx = copy_context()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user