fix(tui-gateway): adopt late compute-host compress acks instead of a false 120s timeout (#97948)

Manual /compress on a compute-host (turn_isolation) session blocked its RPC
waiter for a hard-coded 120s, answered error 5019, and then DROPPED the
host's late `control.ack`: HostSupervisor.control() popped the pending
queue in `finally`, so `_handle_host_frame` had nothing to deliver to. The
host kept compressing, succeeded minutes later, rotated the session — and
the gateway session never mirrored the new session_key/history_version and
the desktop never refreshed its transcript.

- host_supervisor: `control(..., on_late_ack=)` leaves a one-shot handler
  registered when the waiter times out; control.ack/control.error/error
  frames for that request_id fire it (bounded: 30min TTL, cap 64). A host
  crash fails outstanding handlers with a synthetic control.error.
- server: `_compute_host_compress_wait_seconds()` derives the wait from
  `compression.context_total_ceiling_seconds` (+30s slack, floor 120s,
  cap 630s) instead of the literal 120. `_adopt_late_compute_host_compress_ack`
  applies the metadata mirror and emits the same `session.info` a normal
  compress does plus the existing `status.update kind=compacted` edge; a
  late error goes out through the existing `error` event.
- session.compress / slash.compress (methods_tools + _mirror_slash_side_effects):
  on waiter timeout answer `status: pending` (not 5019) and register the
  late-ack handler.
- desktop: SESSION_COMPRESS_TIMEOUT_MS 120s -> 660s (above the gateway cap);
  `status: 'pending'` renders as an info notice, not `error:`; the
  `compacted` status edge rehydrates an idle active session's transcript
  (mid-turn compaction still defers to the turn settle path).

Minimal extraction of the design in #99630 by @vsd2807 (design trace by
@andrexibiza and @JoaoMarcos44 in the #97948 thread); no new DB tables,
modules, or polling protocol.

Refs #97948

Co-authored-by: VVV <vaibhavdahiya28@gmail.com>
This commit is contained in:
Teknium
2026-09-01 21:44:50 -07:00
parent 9bc249c7e5
commit aa80626764
11 changed files with 519 additions and 39 deletions
@@ -1,6 +1,7 @@
import { act, cleanup } from '@testing-library/react'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { createClientSessionState } from '@/lib/chat-runtime'
import { $compactingSessions, setSessionCompacting } from '@/store/compaction'
import type { RpcEvent } from '@/types/hermes'
@@ -56,6 +57,31 @@ describe('useMessageStream compaction lifecycle', () => {
expect($compactingSessions.get()).toEqual({ [OTHER_SID]: true })
})
// #97948: a manual /compress whose RPC answered `pending` (the compute host
// outlived the gateway's wait) has no turn-end hydrate — the `compacted`
// edge is the only signal the transcript changed.
it('rehydrates the idle active session on the compacted edge', () => {
const hydrateFromStoredSession = vi.fn(async () => undefined)
const states = new Map([[SID, { ...createClientSessionState(), storedSessionId: 'stored-1' }]])
stream = renderMessageStream(SID, { hydrateFromStoredSession, states })
emit('status.update', { kind: 'compacted' })
expect(hydrateFromStoredSession).toHaveBeenCalledWith(3, 'stored-1', SID)
})
it('leaves the transcript to the turn settle path when compaction ends mid-turn', () => {
const hydrateFromStoredSession = vi.fn(async () => undefined)
const states = new Map([[SID, { ...createClientSessionState(), busy: true, storedSessionId: 'stored-1' }]])
stream = renderMessageStream(SID, { hydrateFromStoredSession, states })
emit('status.update', { kind: 'compacted' })
expect(hydrateFromStoredSession).not.toHaveBeenCalled()
})
it('reconciles a reconnecting compaction only from trusted terminal server state', () => {
mountStream()
emit('status.update', { kind: 'compacting' })
@@ -21,7 +21,16 @@ import type { GatewayEventContext } from './types'
* error — the status-and-notice tail of the dispatcher. */
export function handleStatusEvent(ctx: GatewayEventContext): boolean {
const { deps, event, payload, sessionId, isActiveEvent, occurredAt } = ctx
const { compactedTurnRef, failAssistantMessage, flushQueuedDeltas, queryClient, updateSessionState } = deps
const {
compactedTurnRef,
failAssistantMessage,
flushQueuedDeltas,
hydrateFromStoredSession,
queryClient,
sessionStateByRuntimeIdRef,
updateSessionState
} = deps
if (event.type === 'status.update') {
if (sessionId && payload?.kind === 'compacting') {
@@ -30,6 +39,17 @@ export function handleStatusEvent(ctx: GatewayEventContext): boolean {
} else if (sessionId && payload?.kind === 'compacted') {
reconcileSessionCompacting(sessionId, 'terminal')
compactedTurnRef.current.delete(sessionId)
// A compress that finished with no live turn (manual /compress whose
// RPC answered `pending` because the compute host outlived the wait,
// #97948) has no turn-end hydrate to refresh the transcript — the
// summarized bubbles would stay on screen forever. Mid-turn compaction
// still defers to the turn's own settle path.
const state = sessionStateByRuntimeIdRef.current.get(sessionId)
if (isActiveEvent && state && !state.busy && !state.awaitingResponse && !state.streamId) {
void hydrateFromStoredSession(3, state.storedSessionId, sessionId)
}
} else if (sessionId && payload?.kind === 'process') {
// The gateway's notification poller announces background process
// completions / watch matches here — re-sync the status stack.
@@ -31,6 +31,7 @@ import { $wakeWord, resetWakeWordState } from '@/store/wake-word'
import type { SessionInfo } from '@/types/hermes'
import { clearSingleFlightSessionResumeState } from './single-flight-resume'
import { SESSION_COMPRESS_TIMEOUT_MS } from './slash'
import type { SubmitTextOptions } from './utils'
import { uploadComposerAttachment, usePromptActions } from '.'
@@ -692,7 +693,7 @@ describe('usePromptActions /compress', () => {
vi.restoreAllMocks()
})
it('routes through session.compress (not slash.exec) with a 120s timeout and renders the summary', async () => {
it('routes through session.compress (not slash.exec) with the compute-host ceiling timeout and renders the summary', async () => {
const seeds: Record<string, unknown>[] = []
const requestGateway = vi.fn(async (method: string, _params?: Record<string, unknown>, _timeoutMs?: number) => {
@@ -728,7 +729,7 @@ describe('usePromptActions /compress', () => {
expect(requestGateway).toHaveBeenCalledWith(
'session.compress',
expect.objectContaining({ session_id: RUNTIME_SESSION_ID }),
120_000
SESSION_COMPRESS_TIMEOUT_MS
)
expect(requestGateway).not.toHaveBeenCalledWith('slash.exec', expect.anything())
expect(requestGateway).not.toHaveBeenCalledWith('command.dispatch', expect.anything())
@@ -862,7 +863,7 @@ describe('usePromptActions /compress', () => {
expect(requestGateway).toHaveBeenCalledWith(
'session.compress',
expect.objectContaining({ focus_topic: 'the auth refactor' }),
120_000
SESSION_COMPRESS_TIMEOUT_MS
)
})
@@ -969,7 +970,7 @@ describe('usePromptActions /compress', () => {
act(() => {
submitted = handle!.submitTextRaw('/compress')
})
await waitFor(() => expect(requestGateway).toHaveBeenCalledWith('session.compress', expect.anything(), 120_000))
await waitFor(() => expect(requestGateway).toHaveBeenCalledWith('session.compress', expect.anything(), SESSION_COMPRESS_TIMEOUT_MS))
// Switch to session B before compression resolves.
activeSessionIdRef.current = RUNTIME_SESSION_B
@@ -1028,7 +1029,7 @@ describe('usePromptActions /compress', () => {
act(() => {
submitted = handle!.submitTextRaw('/compress')
})
await waitFor(() => expect(requestGateway).toHaveBeenCalledWith('session.compress', expect.anything(), 120_000))
await waitFor(() => expect(requestGateway).toHaveBeenCalledWith('session.compress', expect.anything(), SESSION_COMPRESS_TIMEOUT_MS))
activeSessionIdRef.current = RUNTIME_SESSION_B
storedSessionIdRef.current = 'stored-b'
rejectCompress(new Error('compression failed'))
@@ -74,9 +74,13 @@ import {
} from './utils'
// Manual compression is LLM-bound and routinely outlives the desktop's 30s
// default WS request timeout on large sessions — give it the TUI client's
// 120s RPC budget (HERMES_TUI_RPC_TIMEOUT_MS default) instead.
const SESSION_COMPRESS_TIMEOUT_MS = 120_000
// default WS request timeout on large sessions. The gateway blocks its own
// compute-host wait for up to compression.context_total_ceiling_seconds + 30s
// (capped at 630s, tui_gateway/server.py _COMPUTE_HOST_COMPRESS_WAIT_CAP_SECS)
// and then answers `status: 'pending'` rather than an error, so this budget
// must sit above that cap or the desktop reports a false timeout while the
// host is still compressing (#97948).
export const SESSION_COMPRESS_TIMEOUT_MS = 660_000
const WAKE_START_TIMEOUT_MS = 180_000
const wakeDeviceLabel = (device?: WakeInputDeviceStatus): string => {
@@ -666,6 +670,16 @@ export function useSlashCommand(deps: SlashCommandDeps) {
sessionId = liveSessionId
// The gateway's compute-host wait expired but compression is still
// running there; it pushes session.info + a `compacted` status edge
// when the host finishes. Not an error (#97948).
if (result?.status === 'pending') {
const pendingMessage = result.message || 'compression still running in the background'
notify({ durationMs: 8_000, id: noticeId, kind: 'info', message: pendingMessage })
return
}
// Replace the transcript with the post-compress history so the
// summarized bubbles actually disappear. `messages` is the same
// shape session.resume returns (_history_to_messages), so
+4
View File
@@ -66,6 +66,10 @@ export interface SessionCompressResponse {
usage?: Partial<UsageStats>
}
messages?: SessionMessage[]
/** Set with `status: 'pending'` when the gateway's compute-host wait expired
* while compression is still running; the transcript refreshes from the
* pushed session.info / `compacted` status edge (#97948). */
message?: string
removed?: number
status?: string
summary?: {
+16 -13
View File
@@ -633,7 +633,7 @@ def test_slash_exec_compress_flag_on_applies_host_control_mirror(monkeypatch):
def __init__(self):
self.controls = []
def control(self, sid, *, route_name, payload=None, wait=True, timeout=30.0):
def control(self, sid, *, route_name, payload=None, wait=True, timeout=30.0, on_late_ack=None):
self.controls.append((sid, route_name, dict(payload or {}), wait))
return {
"type": "control.ack",
@@ -10500,7 +10500,7 @@ def test_session_compress_returns_compute_host_history(monkeypatch):
}
def test_session_compress_forwards_120_second_budget_to_compute_host(monkeypatch):
def test_session_compress_forwards_config_ceiling_budget_to_compute_host(monkeypatch):
session = _session(agent=None, _compute_host_active=True)
server._sessions["sid"] = session
calls = []
@@ -10519,6 +10519,9 @@ def test_session_compress_forwards_120_second_budget_to_compute_host(monkeypatch
monkeypatch.setattr(server, "_session_uses_compute_host", lambda _session: True)
monkeypatch.setattr(server, "_send_compute_host_control", send_control)
monkeypatch.setattr(
server, "_load_cfg", lambda: {"compression": {"context_total_ceiling_seconds": 300}}
)
try:
resp = server.handle_request(
@@ -10528,17 +10531,17 @@ def test_session_compress_forwards_120_second_budget_to_compute_host(monkeypatch
server._sessions.pop("sid", None)
assert resp["result"]["status"] == "compressed"
assert calls == [
(
("sid",),
{
"route_name": "session.compress",
"command": "/compress",
"wait": True,
"timeout": 120.0,
},
)
]
assert len(calls) == 1
(sid_arg,), kwargs = calls[0]
assert sid_arg == "sid"
assert kwargs["route_name"] == "session.compress"
assert kwargs["command"] == "/compress"
assert kwargs["wait"] is True
# #97948: the waiter follows compression.context_total_ceiling_seconds
# (+30s slack) instead of a hard-coded 120s, and registers a late-ack
# handler so a compress that outlives it is still adopted.
assert kwargs["timeout"] == 330.0
assert callable(kwargs["on_late_ack"])
def test_session_compress_preserves_compute_host_aborted_summary(monkeypatch):
@@ -0,0 +1,235 @@
"""Regression tests for #97948 symptom A (salvaged from #99630).
A manual /compress on a compute-host session used to block its RPC waiter for
a hard-coded 120s, return a 5019 timeout error, and then DROP the host's late
``control.ack`` — so the rotated session_key / history_version / session_info
never reached the gateway session and the desktop never refreshed.
"""
import queue
import sys
import threading
import time
import types
import pytest
from tui_gateway import server
from tui_gateway.host_supervisor import HostSupervisor
def _supervisor() -> tuple[HostSupervisor, list]:
sup = HostSupervisor(argv=[sys.executable, "-c", ""], autostart=False)
sent: list = []
sup._send_frame = lambda frame: sent.append(frame)
sup.start = lambda: None # never spawn a child
return sup, sent
def _session(**extra) -> dict:
return {
"agent": types.SimpleNamespace(),
"session_key": "old-session-key",
"history": [],
"history_lock": threading.Lock(),
"history_version": 3,
"running": False,
"attached_images": [],
"image_counter": 0,
"cols": 80,
"slash_worker": None,
"show_reasoning": False,
"tool_progress_mode": "all",
"_compute_host_active": True,
**extra,
}
# ── HostSupervisor: late-ack registration ───────────────────────────────────
def test_control_timeout_registers_one_shot_late_ack_handler():
sup, sent = _supervisor()
fired: list = []
with pytest.raises(queue.Empty):
sup.control("sid", route_name="session.compress", payload={"command": "/compress"},
wait=True, timeout=0.05, on_late_ack=fired.append)
request_id = sent[0]["request_id"]
assert request_id not in sup._pending_controls
assert request_id in sup._late_control_handlers
late = {"type": "control.ack", "request_id": request_id, "result": {"status": "compressed"}}
sup._handle_host_frame(late)
assert fired == [late]
# One-shot: a duplicate ack for the same request is ignored.
sup._handle_host_frame(late)
assert fired == [late]
assert request_id not in sup._late_control_handlers
def test_control_timeout_without_handler_still_drops_late_ack():
sup, sent = _supervisor()
with pytest.raises(queue.Empty):
sup.control("sid", route_name="session.compress", wait=True, timeout=0.05)
assert sup._late_control_handlers == {}
sup._handle_host_frame({"type": "control.ack", "request_id": sent[0]["request_id"]})
def test_late_control_error_and_bare_error_frames_fire_handler():
sup, sent = _supervisor()
fired: list = []
for _ in range(2):
with pytest.raises(queue.Empty):
sup.control("sid", route_name="session.compress", wait=True, timeout=0.01,
on_late_ack=fired.append)
rid_a, rid_b = sent[0]["request_id"], sent[1]["request_id"]
sup._handle_host_frame({"type": "control.error", "request_id": rid_a, "message": "boom"})
sup._handle_host_frame({"type": "error", "request_id": rid_b, "message": "bad frame"})
assert [f["request_id"] for f in fired] == [rid_a, rid_b]
def test_late_ack_handlers_are_bounded_by_ttl_and_cap(monkeypatch):
from tui_gateway import host_supervisor as hs
monkeypatch.setattr(hs, "_LATE_CONTROL_MAX", 3)
sup, _sent = _supervisor()
for i in range(5):
sup._register_late_control_handler(f"r{i}", lambda _f: None)
assert len(sup._late_control_handlers) == 3
assert set(sup._late_control_handlers) == {"r2", "r3", "r4"}
# TTL: an old registration is dropped on the next registration.
monkeypatch.setattr(hs, "_LATE_CONTROL_TTL_SECS", 0.0)
time.sleep(0.01)
sup._register_late_control_handler("fresh", lambda _f: None)
assert set(sup._late_control_handlers) == {"fresh"}
def test_host_crash_fails_outstanding_late_ack_handlers():
sup, sent = _supervisor()
fired: list = []
with pytest.raises(queue.Empty):
sup.control("sid", route_name="session.compress", wait=True, timeout=0.01,
on_late_ack=fired.append)
sup._fail_pending_turns(reason="crash", message="compute host exited with code 1")
assert len(fired) == 1
assert fired[0]["type"] == "control.error"
assert fired[0]["request_id"] == sent[0]["request_id"]
assert sup._late_control_handlers == {}
# ── session.compress RPC: pending answer + late adoption ────────────────────
@pytest.fixture
def compute_host_gateway(monkeypatch):
sup, sent = _supervisor()
emitted: list = []
monkeypatch.setattr(server, "_compute_host_supervisor", sup)
monkeypatch.setattr(server, "_emit", lambda event, sid, payload=None: emitted.append((event, sid, payload)))
monkeypatch.setattr(server, "_session_uses_compute_host", lambda _s, cfg=None: True)
monkeypatch.setattr(server, "_compute_host_compress_wait_seconds", lambda cfg=None: 0.05)
monkeypatch.setattr(server, "_session_info", lambda _agent, _session=None: {"model": "mirrored"})
session = _session()
server._sessions["sid"] = session
try:
yield sup, sent, emitted, session
finally:
server._sessions.pop("sid", None)
def _late_ack(request_id: str) -> dict:
return {
"type": "control.ack",
"sid": "sid",
"request_id": request_id,
"route_name": "session.compress",
"result": {"status": "compressed", "removed": 12, "summary": {"headline": "Compressed 14 → 2"}},
"session_key": "rotated-session-key",
"history_version": 9,
"message_count": 2,
"session_info": {"model": "host-model", "usage": {"total": 111}},
}
def test_session_compress_reports_pending_and_adopts_late_ack(compute_host_gateway):
sup, sent, emitted, session = compute_host_gateway
resp = server.handle_request({"id": "1", "method": "session.compress", "params": {"session_id": "sid"}})
assert "error" not in resp, resp
assert resp["result"]["status"] == "pending"
assert resp["result"]["turn_isolation"] is True
assert "background" in resp["result"]["message"]
assert sent[0]["route_name"] == "session.compress"
# Nothing adopted yet, the host is still working.
assert session["session_key"] == "old-session-key"
assert emitted == []
sup._handle_host_frame(_late_ack(sent[0]["request_id"]))
assert session["session_key"] == "rotated-session-key"
assert session["history_version"] == 9
assert session["_metadata_message_count"] == 2
assert session["_metadata_mirror"]["model"] == "host-model"
events = [(event, payload) for event, _sid, payload in emitted]
assert ("session.info", {"model": "mirrored"}) in events
assert ("status.update", {"kind": "compacted", "text": "✓ Context compression complete"}) in events
def test_session_compress_late_control_error_surfaces_as_error_event(compute_host_gateway):
sup, sent, emitted, session = compute_host_gateway
resp = server.handle_request({"id": "1", "method": "session.compress", "params": {"session_id": "sid"}})
assert resp["result"]["status"] == "pending"
sup._handle_host_frame({"type": "control.error", "request_id": sent[0]["request_id"], "message": "provider down"})
assert session["session_key"] == "old-session-key"
assert ("error", "sid", {"message": "compression failed: provider down"}) in emitted
def test_session_compress_late_ack_ignored_after_session_closed(compute_host_gateway):
sup, sent, emitted, session = compute_host_gateway
server.handle_request({"id": "1", "method": "session.compress", "params": {"session_id": "sid"}})
server._sessions.pop("sid")
sup._handle_host_frame(_late_ack(sent[0]["request_id"]))
assert session["session_key"] == "old-session-key"
assert emitted == []
def test_slash_compress_route_reports_pending_and_adopts_late_ack(compute_host_gateway):
sup, sent, emitted, session = compute_host_gateway
resp = server.handle_request(
{"id": "1", "method": "slash.exec", "params": {"session_id": "sid", "command": "/compress"}}
)
assert "error" not in resp, resp
assert "compression still running in the background" in resp["result"]["output"]
assert sent[0]["route_name"] == "slash.compress"
sup._handle_host_frame({**_late_ack(sent[0]["request_id"]), "route_name": "slash.compress"})
assert session["session_key"] == "rotated-session-key"
assert any(event == "session.info" for event, _sid, _p in emitted)
# ── wait budget follows compression.context_total_ceiling_seconds ───────────
def test_compress_wait_budget_follows_config_ceiling():
assert server._compute_host_compress_wait_seconds({"compression": {}}) == 630.0
assert server._compute_host_compress_wait_seconds(
{"compression": {"context_total_ceiling_seconds": 200}}
) == 230.0
# Never below the historical 120s floor, never above the RPC-safe cap.
assert server._compute_host_compress_wait_seconds(
{"compression": {"context_total_ceiling_seconds": 10, "context_timeout_seconds": 0}}
) == 120.0
assert server._compute_host_compress_wait_seconds(
{"compression": {"context_total_ceiling_seconds": 99999}}
) == server._COMPUTE_HOST_COMPRESS_WAIT_CAP_SECS
+70 -16
View File
@@ -47,6 +47,11 @@ MUTATOR_ROUTE_TABLE: dict[str, str] = {
_REGISTRY_NAME = "dashboard-compute-host.json"
_RESPAWN_WINDOW_SECS = 300.0
_SHUTDOWN_TIMEOUT_SECS = 10.0
# Late control-ack handlers (#97948): a compress that outlives its RPC waiter
# can legitimately run for the full compression ceiling plus a stall-fallback
# retry, so keep registrations around well past that — but bounded.
_LATE_CONTROL_TTL_SECS = 1800.0
_LATE_CONTROL_MAX = 64
def append_log_record(path: str | Path, record: str) -> None:
@@ -167,6 +172,11 @@ class HostSupervisor:
self._restart_times: list[float] = []
self._pending_turns: dict[str, tuple[str, Callable[[dict], None] | None]] = {}
self._pending_controls: dict[str, queue.Queue[dict]] = {}
# request_id -> (registered_at, handler) for control waiters that timed
# out but whose host work is still running (#97948). The host emits
# its control.ack whenever it finishes; without this the ack matched
# no queue and was silently dropped.
self._late_control_handlers: dict[str, tuple[float, Callable[[dict], None]]] = {}
self._stderr_tail: list[str] = []
self._last_progress_counter = 0
@@ -307,7 +317,17 @@ class HostSupervisor:
payload: dict[str, Any] | None = None,
wait: bool = True,
timeout: float = 30.0,
on_late_ack: Callable[[dict], None] | None = None,
) -> dict:
"""Send a control frame; with ``wait`` block up to ``timeout`` for its ack.
``on_late_ack`` (only meaningful with ``wait``) keeps the request
adoptable after the waiter gives up: when the host's ``control.ack`` /
``control.error`` / ``error`` for this ``request_id`` eventually
arrives, the handler fires once instead of the frame being dropped.
Registrations are bounded by ``_LATE_CONTROL_TTL_SECS`` /
``_LATE_CONTROL_MAX``.
"""
if route_name not in MUTATOR_ROUTE_TABLE:
raise ValueError(f"unclassified host mutator route: {route_name}")
self.start()
@@ -327,10 +347,47 @@ class HostSupervisor:
return {"status": "sent", "request_id": request_id}
try:
return q.get(timeout=timeout)
except queue.Empty:
if on_late_ack is not None:
self._register_late_control_handler(request_id, on_late_ack)
raise
finally:
with self._lock:
self._pending_controls.pop(request_id, None)
def _register_late_control_handler(self, request_id: str, handler: Callable[[dict], None]) -> None:
now = time.monotonic()
with self._lock:
expired = [
rid
for rid, (registered_at, _cb) in self._late_control_handlers.items()
if now - registered_at > _LATE_CONTROL_TTL_SECS
]
for rid in expired:
self._late_control_handlers.pop(rid, None)
while len(self._late_control_handlers) >= _LATE_CONTROL_MAX:
oldest = min(self._late_control_handlers, key=lambda rid: self._late_control_handlers[rid][0])
self._late_control_handlers.pop(oldest, None)
self._late_control_handlers[request_id] = (now, handler)
def _deliver_control_frame(self, request_id: str, frame: dict[str, Any]) -> None:
with self._lock:
q = self._pending_controls.get(request_id)
late = None if q is not None else self._late_control_handlers.pop(request_id, None)
if q is not None:
try:
q.put_nowait(frame)
except queue.Full:
pass
return
if late is None:
return
_registered_at, handler = late
try:
handler(frame)
except Exception:
logger.exception("compute host late control ack handler failed (request_id=%s)", request_id)
def _spawn_locked(self, *, reason: str) -> None:
if self._stopped_respawning:
raise RuntimeError("compute host respawn disabled after crash loop")
@@ -452,24 +509,10 @@ class HostSupervisor:
self._complete_turn(frame)
return
if ftype in {"control.ack", "control.error", "respond.ack", "respond.error", "interrupt.ack", "reload_mcp.ack", "shutdown.ack"}:
request_id = str(frame.get("request_id") or "")
with self._lock:
q = self._pending_controls.get(request_id)
if q is not None:
try:
q.put_nowait(frame)
except queue.Full:
pass
self._deliver_control_frame(str(frame.get("request_id") or ""), frame)
return
if ftype == "error" and frame.get("request_id"):
request_id = str(frame.get("request_id") or "")
with self._lock:
q = self._pending_controls.get(request_id)
if q is not None:
try:
q.put_nowait(frame)
except queue.Full:
pass
self._deliver_control_frame(str(frame.get("request_id") or ""), frame)
def _complete_turn(self, frame: dict[str, Any]) -> None:
request_id = str(frame.get("request_id") or "")
@@ -524,6 +567,17 @@ class HostSupervisor:
cb(frame)
except Exception:
logger.exception("compute host error callback failed")
# A crashed host will never emit the late acks the timed-out control
# waiters are still expecting; fail them the same way so the client's
# "still running in the background" notice does not hang forever.
with self._lock:
late = self._late_control_handlers
self._late_control_handlers = {}
for request_id, (_registered_at, handler) in late.items():
try:
handler({"type": "control.error", "request_id": request_id, "reason": reason, "message": message})
except Exception:
logger.exception("compute host late control error handler failed")
def _maybe_respawn_after_crash(self) -> None:
now = time.monotonic()
+25 -1
View File
@@ -3006,13 +3006,37 @@ def _(rid, params: dict) -> dict:
sid = str(params.get("session_id") or "")
focus_topic = str(params.get("focus_topic", "") or "").strip()
command = "/compress" + (f" {focus_topic}" if focus_topic else "")
_late_session = session
def _on_late_ack(late: dict, _sid=sid) -> None:
_adopt_late_compute_host_compress_ack(_sid, _late_session, late, route_name="session.compress")
try:
ack = _send_compute_host_control(
sid,
route_name="session.compress",
command=command,
wait=True,
timeout=120.0,
# Follows compression.context_total_ceiling_seconds instead of
# a fixed 120s: the host legitimately runs that long (#97948).
timeout=_compute_host_compress_wait_seconds(),
on_late_ack=_on_late_ack,
)
except queue.Empty:
# The waiter gave up but the host is still compressing; the late
# ack handler adopts the rotated session and pushes session.info
# when it lands. Not an error — the old 5019 made Desktop/TUI
# report a timeout while compression later succeeded silently.
return _ok(
rid,
{
"status": "pending",
"turn_isolation": True,
"message": (
"compression still running in the background; "
"the transcript will refresh when it finishes"
),
},
)
except Exception as exc:
return _err(rid, 5019, f"compute-host compress failed: {exc}")
+19
View File
@@ -1057,12 +1057,31 @@ def _(rid, params: dict) -> dict:
sid = params.get("session_id", "")
if _session_uses_compute_host(session):
command = f"/{name}" + (f" {arg}" if arg else "")
_late_session = session
def _on_late_ack(late: dict, _sid=sid) -> None:
_adopt_late_compute_host_compress_ack(_sid, _late_session, late, route_name="slash.compress")
try:
ack = _send_compute_host_control(
sid,
route_name="slash.compress",
command=command,
wait=True,
timeout=_compute_host_compress_wait_seconds(),
on_late_ack=_on_late_ack,
)
except queue.Empty:
return _ok(
rid,
{
"type": "exec",
"status": "pending",
"output": (
"compression still running in the background; "
"the transcript will refresh when it finishes"
),
},
)
except Exception as exc:
return _err(rid, 5019, f"compute-host slash.compress failed: {exc}")
+80
View File
@@ -2656,6 +2656,11 @@ def _broadcast_global_event(event: str, payload: dict | None = None) -> None:
_compute_host_supervisor = None
_compute_host_supervisor_lock = threading.Lock()
# Hard cap on how long session.compress blocks its RPC waiting for the compute
# host (#97948). Must stay below the desktop's SESSION_COMPRESS_TIMEOUT_MS
# (660s) so the client receives the `pending` answer instead of its own
# timeout error; the late-ack path covers anything slower.
_COMPUTE_HOST_COMPRESS_WAIT_CAP_SECS = 630.0
def _inside_compute_host_child() -> bool:
@@ -2931,6 +2936,7 @@ def _send_compute_host_control(
payload: dict | None = None,
wait: bool = True,
timeout: float = 30.0,
on_late_ack=None,
) -> dict:
frame = dict(payload or {})
frame.setdefault("type", "control")
@@ -2941,9 +2947,68 @@ def _send_compute_host_control(
payload=frame,
wait=wait,
timeout=timeout,
on_late_ack=on_late_ack,
)
def _compute_host_compress_wait_seconds(cfg: dict | None = None) -> float:
"""RPC wait budget for a compute-host compress control (#97948).
Manual compression legitimately runs up to the configured
``compression.context_total_ceiling_seconds`` (default 600s), so a fixed
120s waiter reported a false timeout while the host kept working. Follow
the ceiling with a little slack, but cap the blocking wait so it stays
below the desktop's own RPC timeout; anything longer is adopted through
the late-ack path instead of failing.
"""
from agent.conversation_compression import resolve_context_compression_timeouts
try:
compression_cfg = (cfg if cfg is not None else _load_cfg()).get("compression", {})
except Exception:
compression_cfg = {}
_idle, ceiling = resolve_context_compression_timeouts(
compression_cfg if isinstance(compression_cfg, dict) else {}
)
return float(min(max(ceiling + 30.0, 120.0), _COMPUTE_HOST_COMPRESS_WAIT_CAP_SECS))
def _announce_compute_host_compress_done(sid: str, session: dict, ack: dict) -> None:
"""Mirror a compute-host compress ack and push the client-visible edges.
Emits the same ``session.info`` the in-process /compress path does plus
the ``compacted`` status edge, so a client whose own RPC wait already
expired still learns the transcript changed.
"""
_apply_compute_host_metadata_mirror(session, ack)
try:
info = _session_info(session.get("agent"), session)
except TypeError:
info = _session_info(session.get("agent"))
_emit("session.info", sid, info)
_status_update(sid, "compacted", "✓ Context compression complete")
def _adopt_late_compute_host_compress_ack(sid: str, session: dict, ack: dict, *, route_name: str) -> None:
"""Adopt a compute-host compress ack that arrived after its RPC waiter gave up.
The RPC already answered ``status: pending``; this is the only place the
rotated session_key / history_version / session_info mirror can land, and
the only signal the client gets that the transcript changed. A late
``control.error`` surfaces through the existing ``error`` event path.
"""
with _sessions_lock:
live = _sessions.get(sid)
if live is not session:
return
if not isinstance(ack, dict) or ack.get("type") in {"control.error", "error"}:
message = str((ack or {}).get("message") or f"compute-host {route_name} failed")
_emit("error", sid, {"message": f"compression failed: {message}"})
_status_update(sid, "ready")
return
_announce_compute_host_compress_done(sid, session, ack)
def _approval_request_payload(data: dict | None) -> dict:
"""Build the client-safe representation of a pending approval."""
payload = dict(data or {})
@@ -16708,13 +16773,28 @@ def _mirror_slash_side_effects(sid: str, session: dict, command: str) -> str:
_MUTATES_WHILE_RUNNING = {"model", "personality", "prompt", "compress"}
if _session_uses_compute_host(session) and name in _MUTATES_WHILE_RUNNING:
route_name = f"slash.{name}"
is_compress = name == "compress"
_late_session = session
def _on_late_ack(late: dict, _sid=sid) -> None:
_adopt_late_compute_host_compress_ack(_sid, _late_session, late, route_name=route_name)
try:
ack = _send_compute_host_control(
sid,
route_name=route_name,
command=command,
wait=True,
**(
{"timeout": _compute_host_compress_wait_seconds(), "on_late_ack": _on_late_ack}
if is_compress
else {}
),
)
except queue.Empty:
if is_compress:
return "compression still running in the background; the transcript will refresh when it finishes"
return f"compute-host {route_name} failed: timed out"
except Exception as exc:
return f"compute-host {route_name} failed: {exc}"
if ack.get("type") in {"control.error", "error"}: