From c5c9aa8d44e03f4e8b5fe7f230cfd97ab2dde0bf Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:02:29 -0700 Subject: [PATCH] fix(gateway): hygiene compaction keeps the profile secret scope under multiplexing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Session-hygiene compaction ran _compress_context on a bare loop.run_in_executor(None, ...) worker. Under gateway.multiplex_profiles the profile secret scope and HERMES_HOME override are ContextVars installed by the per-turn _profile_runtime_scope, and a bare worker starts with an empty Context — so the summary model's get_secret(_API_KEY) failed closed with UnscopedSecretError on EVERY hygiene pass and compaction silently degraded to a lossy truncation (#100849 debug bundle: 'Failed to generate context summary: get_secret(SURPLUS_API_KEY) called with no profile secret scope active'). - gateway/run.py: run both hygiene executor hops (detached-agent path and codex app-server path) inside copy_context().run, keeping the default executor so a fence-cancelled hung summary never occupies a gateway agent-work slot. - agent/context_compressor.py: UnscopedSecretError is a missing-credential class failure — abort and preserve the session instead of dropping the middle window for a placeholder summary (same carve-out as 401/402/403). - tools/daemon_pool.py: correct the salvaged docstrings — stdlib ThreadPoolExecutor only propagates contextvars from 3.14; nothing is stripped from the bundled runtime. - tests: hygiene worker inherits caller ContextVars (fails on bare run_in_executor); UnscopedSecretError classified as access failure. Live A/B (real get_secret in a run_in_executor worker, multiplex on, profile .env scope installed): main -> UnscopedSecretError; fixed -> scoped value. --- agent/context_compressor.py | 14 ++++++ gateway/run.py | 18 ++++++++ tests/agent/test_context_compressor.py | 13 ++++++ .../gateway/test_codex_hygiene_compaction.py | 43 +++++++++++++++++++ tools/daemon_pool.py | 39 ++++++++--------- 5 files changed, 106 insertions(+), 21 deletions(-) diff --git a/agent/context_compressor.py b/agent/context_compressor.py index 45ba95200f..53cfad134c 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -210,6 +210,20 @@ _TRUNCATED_SUMMARY_MARKER = "finish_reason=length" def _is_summary_access_or_quota_error(exc: Exception) -> bool: """Return True for non-retryable summary auth, permission, or quota errors.""" + # A credential read that failed closed because no profile secret scope + # was active (multiplexed gateway, worker thread without the caller's + # ContextVars) is a missing-credential failure of our own making: the + # summary model cannot be reached until the spawn site is fixed, and a + # placeholder summary would only destroy the middle window for nothing. + # Classify it with the credential class so compress() preserves the + # session unchanged (#100849 bundle: every hygiene pass truncated). + try: + from agent.secret_scope import UnscopedSecretError + except Exception: # pragma: no cover - import guard + UnscopedSecretError = () # type: ignore[assignment] + if UnscopedSecretError and isinstance(exc, UnscopedSecretError): + return True + classified = classify_api_error(exc) if classified.reason is FailoverReason.rate_limit: return False diff --git a/gateway/run.py b/gateway/run.py index 405217d4cf..11d43932b2 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -392,8 +392,12 @@ async def run_codex_hygiene_compaction( count_before = getattr(compressor, "compression_count", 0) try: await asyncio.wait_for( + # copy_context().run: keep the caller's profile secret scope / + # HERMES_HOME override in the worker (multiplex_profiles) — same + # class as the detached-agent hygiene path below. loop.run_in_executor( None, + copy_context().run, lambda: agent._compress_context( history, "", @@ -21305,8 +21309,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew _hyg_commit_fence = CompressionCommitFence( total_ceiling_seconds=_hyg_total_ceiling_seconds ) + # Default executor (NOT self._get_executor): + # a fence-cancelled hung summary must never + # occupy one of the gateway's agent-work + # slots. But it MUST run inside the caller's + # contextvars: under multiplex_profiles the + # profile secret scope / HERMES_HOME override + # live in ContextVars, and a bare + # run_in_executor worker starts with an empty + # Context — the summary model's + # get_secret(_API_KEY) then fails + # closed (UnscopedSecretError) and every + # hygiene compaction silently degrades to a + # lossy truncation (#100849 bundle). _hyg_future = loop.run_in_executor( None, + copy_context().run, lambda: _hyg_agent._compress_context( _hyg_msgs, "", approx_tokens=_approx_tokens, diff --git a/tests/agent/test_context_compressor.py b/tests/agent/test_context_compressor.py index 7e373f209e..10997fe94a 100644 --- a/tests/agent/test_context_compressor.py +++ b/tests/agent/test_context_compressor.py @@ -855,6 +855,19 @@ class TestAuthFailureAborts: ) assert _is_summary_access_or_quota_error(err) is True + def test_unscoped_secret_read_is_terminal_access_failure(self): + # Multiplexed gateway: a credential read reached get_secret() from a + # worker thread without the profile scope. The summary model is + # unreachable until the spawn site is fixed — abort and preserve the + # session rather than truncating the middle window (#100849 bundle). + from agent.secret_scope import UnscopedSecretError + + err = UnscopedSecretError( + "get_secret('SURPLUS_API_KEY') called with no profile secret scope " + "active while multiplexing is on." + ) + assert _is_summary_access_or_quota_error(err) is True + diff --git a/tests/gateway/test_codex_hygiene_compaction.py b/tests/gateway/test_codex_hygiene_compaction.py index 71dd907f3b..7fa795d694 100644 --- a/tests/gateway/test_codex_hygiene_compaction.py +++ b/tests/gateway/test_codex_hygiene_compaction.py @@ -336,3 +336,46 @@ def test_manual_compress_without_live_thread_reports_honestly(): host._compress_codex_app_server_session("tg:123", "sess-1") ) assert "Nothing to compact" in reply + + +# --------------------------------------------------------------------------- +# Multiplexed gateway: the hygiene worker must see the caller's ContextVars +# (profile secret scope / HERMES_HOME override). A bare run_in_executor worker +# starts with an EMPTY Context, so get_secret(_API_KEY) inside the +# summary path fails closed and every hygiene compaction degrades to a lossy +# truncation (#100849 bundle). +# --------------------------------------------------------------------------- + +def test_hygiene_worker_inherits_caller_contextvars(tmp_path): + import contextvars + import threading + + marker = contextvars.ContextVar("hygiene_scope_marker", default=None) + seen = {} + + class ScopeProbeAgent(LiveCodexAgent): + def _compress_context(self, messages, system_message, **kwargs): + seen["value"] = marker.get() + seen["thread"] = threading.current_thread().name + return super()._compress_context(messages, system_message, **kwargs) + + agent = ScopeProbeAgent(mode="hermes") + key = "tg:ctx" + gw, _db = _gateway(tmp_path, key, agent) + + async def _scoped(): + token = marker.set("profile-scope") + try: + return await run_codex_hygiene_compaction( + gw, key, agent.session_id, auto_mode="hermes", + history=_history(), approx_tokens=345_000, timeout_seconds=30.0, + ) + finally: + marker.reset(token) + + assert asyncio.run(_scoped()) == "compacted" + assert seen["thread"] != "MainThread", "compaction must still run off-loop" + assert seen["value"] == "profile-scope", ( + "hygiene worker lost the caller's ContextVars — under multiplex_profiles " + "this is the UnscopedSecretError / lossy-truncation regression" + ) diff --git a/tools/daemon_pool.py b/tools/daemon_pool.py index 368a9c614d..33e99c5143 100644 --- a/tools/daemon_pool.py +++ b/tools/daemon_pool.py @@ -16,16 +16,16 @@ exit hook insists on joining. - the interpreter's non-daemon thread join at shutdown skips them. Semantics are otherwise identical (initializer/initargs, work queue, -idle-thread reuse) and, since #95119, so is context propagation: -``submit`` snapshots the submitting context with ``copy_context()`` and -runs each work item inside it, matching stdlib ``ThreadPoolExecutor``. -That matters because some bundled CPython runtime builds omit stdlib's -context propagation entirely, which silently drops contextvar-based state -(profile secret scope, HERMES_HOME override) in pool workers — e.g. the -context-compression timeout fence resolved auxiliary provider keys with -``UnscopedSecretError`` under the multiplexed gateway. Use it for any -pool whose work is best-effort or -independently interruptible and must never hold the process open: +idle-thread reuse), plus context propagation: ``submit`` snapshots the +submitting context with ``copy_context()`` and runs each work item inside +it. Stdlib ``ThreadPoolExecutor`` only does this from Python 3.14; on the +3.11-3.13 runtimes Hermes ships, a bare pool worker starts with an EMPTY +Context and silently drops contextvar-based state (profile secret scope, +HERMES_HOME override) — under the multiplexed gateway a credential read in +such a worker fails closed with ``UnscopedSecretError``. Propagating by +default makes every consumer safe even when it forgets +``propagate_context_to_thread``. Use it for any pool whose work is +best-effort or independently interruptible and must never hold the process open: concurrent tool execution, background memory sync, catalog fan-out, subagent timeout wrappers. Do NOT use it for work that must complete before exit (durable writes) — those belong on foreground threads with @@ -49,17 +49,14 @@ class DaemonThreadPoolExecutor(ThreadPoolExecutor): def submit(self, fn, /, *args, **kwargs): """Submit a callable, propagating the caller's contextvars. - Stdlib ``ThreadPoolExecutor`` snapshots the submitting context with - ``copy_context()`` and runs each work item inside it, so pool - workers inherit contextvar state such as the multiplexed profile - secret scope. Some bundled CPython runtime builds strip that - propagation from the stdlib executor (their ``_WorkItem.run`` calls - the callable directly), which broke auxiliary LLM key resolution - from the context-compression timeout fence with - ``UnscopedSecretError``. Restore the stdlib behavior explicitly so - the daemon pool behaves identically on every runtime; on runtimes - that already propagate, the inner ``ctx.run`` re-applies the same - immutable context and is a no-op. + Python 3.14's ``ThreadPoolExecutor`` snapshots the submitting + context with ``copy_context()`` and runs each work item inside it; + 3.11-3.13 (the runtimes Hermes ships) do not, so a pool worker + starts with an empty Context and loses the multiplexed profile + secret scope / HERMES_HOME override. Do it here unconditionally so + the daemon pool behaves identically on every runtime; on 3.14+ the + inner ``ctx.run`` re-applies the same immutable context and is a + no-op. """ ctx = copy_context()