From de25545dce1d5e77c0da0b222bec5476a802373d Mon Sep 17 00:00:00 2001 From: Ryan Tucker Date: Fri, 14 Aug 2026 23:18:44 -0500 Subject: [PATCH] fix(tui_gateway): fan session events out instead of rebinding the transport slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A session held exactly one transport, and prompt.submit, session.resume, session.activate, and the queued-prompt drain all rebound that slot. A second client therefore took the stream away from the first: the earlier client stopped receiving the turn it was already rendering, and either client disconnecting parked the whole session on the drop sentinel. FanoutTransport goes in the same slot and satisfies the same Transport protocol, so write_json and every other reader of the slot are unchanged. It delivers each frame to a snapshot of its peers, concurrently when more than one peer is attached and the caller is not on an event loop, and prunes any peer that returns False or raises. A dead client is dropped; a slow one costs the emitter at most one write timeout per frame rather than one per peer. Request/response RPCs are unaffected: they still answer on the request's context-bound transport, so a client only ever sees replies to its own calls. The rebind sites become attach sites through _attach_session_transport, whose ladder keeps the single-client shape identical. The same object already in the slot is a no-op; an empty, stdio, or parked slot is taken outright; only the arrival of a second live client wraps both. The queued-prompt drain is included because it pinned the drained turn to the queuer and silenced everyone else. A non-peer newcomer such as stdio or the drop sentinel never displaces a live client, so an activate dispatched without a bound websocket cannot silence the socket that owns the session. Disconnect detaches first. A session that retains another client keeps streaming and is neither parked nor reaped, and only the clientless ones follow the existing close_on_disconnect and park-sentinel path, so a single-client disconnect, the orphan reaper, and its grace window behave as before. _ws_session_is_orphaned is unchanged: it still asks whether the drop sentinel is in the slot, and a fan-out is never the sentinel, so a session that still has a peer is never reported as orphaned. Attach performs no entitlement check: any authenticated peer may mirror any session. The fan-out architecture follows the approach in #40822 by @OmarB97. What the slot's later history forces. _close_sessions_for_transport drops the #83716 rebind-to-the-most-recent-surviving-viewer, which fan-out membership subsumes — a pop-out window is a peer, so a session that still shows in one is never returned as clientless — and keeps the #77129 revalidation before parking, now expressed as a liveness check under _session_transport_lock so it is race-free against attach and detach. _transport_is_live_peer defers its last answer to _transport_is_dead: a socket that already latched _closed is a departed client, and admitting it would keep a session out of both the park and the reap. _transport_is_dead also learns the fan-out: a FanoutTransport with no live peer is dead, so a session whose peers were all pruned by failed writes cannot outlive the TTL and LRU reapers. The upstream test that pinned the #83716 rebind, test_close_transport_rebinds_session_to_remaining_viewer, is re-expressed in fan-out terms: both windows attached, the pop-out closes, the session stays with the main window unparked and still receiving frames. --- tests/test_tui_gateway_server.py | 83 +- tests/tui_gateway/test_multi_client_fanout.py | 813 ++++++++++++++++++ tui_gateway/methods_prompt.py | 2 +- tui_gateway/methods_session.py | 5 +- tui_gateway/server.py | 177 +++- tui_gateway/session_auto_continue.py | 5 +- tui_gateway/session_lifecycle.py | 36 +- tui_gateway/session_reaper.py | 13 +- tui_gateway/transport.py | 263 ++++++ 9 files changed, 1362 insertions(+), 35 deletions(-) create mode 100644 tests/tui_gateway/test_multi_client_fanout.py diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index 3cf2e25bd6..ca030284bf 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -5294,27 +5294,60 @@ def test_finalize_session_closes_slash_worker(monkeypatch): def test_close_transport_rebinds_session_to_remaining_viewer(monkeypatch): - """Closing a pop-out window's transport must re-bind the session to a - still-open window instead of stranding it on the drop sentinel (#83716).""" + """Closing a pop-out window's transport must leave the session with the + still-open window instead of stranding it on the drop sentinel (#83716). + + The rebind #83716 added is gone; multi-client fan-out subsumes it. Both + windows are attached to the slot at once, so the pop-out is a fan-out peer + rather than a viewer waiting to be promoted, and closing it detaches that + peer and collapses the slot back onto the main window. This pins the same + guarantee through the mechanism that replaced the rebind: the session is + not parked, not reaped, not handed to the orphan reaper, and the surviving + window keeps receiving frames. + """ reap_calls = [] monkeypatch.setattr(server, "_schedule_ws_orphan_reap", lambda sid: reap_calls.append(sid)) class _LiveTransport: - def write(self, *a, **k): + def __init__(self): + self.frames = [] + + def write(self, obj=None, *a, **k): + self.frames.append(obj) return True main = _LiveTransport() popout = _LiveTransport() - session = _session(transport=popout, running=False) + session = _session(transport=None, running=False) + # Build the state the way production does: every window that resumes goes + # through _live_session_payload, which attaches it into the slot and then + # stamps it into the viewers registry. + server._attach_session_transport(session, main) + server._attach_session_transport(session, popout) session["viewers"] = {main: 100.0, popout: 200.0} server._sessions["multi-sid"] = session + assert isinstance(session["transport"], server.FanoutTransport) - reaped, detached = server._close_sessions_for_transport(popout) + try: + reaped, detached = server._close_sessions_for_transport(popout) - assert reaped == 0 and detached == 0 - assert session["transport"] is main - assert "multi-sid" not in reap_calls - assert server._ws_session_is_orphaned(session) is False + assert reaped == 0 and detached == 0 + # One peer left, so the fan-out collapses back to the bare transport — + # the slot is indistinguishable from a session that never fanned out. + assert session["transport"] is main + assert "multi-sid" not in reap_calls + assert server._ws_session_is_orphaned(session) is False + + # And it is still a working stream, not just a surviving reference. + server._emit("message.delta", "multi-sid", {"text": "still here"}) + assert [(f.get("params") or {}).get("type") for f in main.frames] == [ + "message.delta" + ] + assert popout.frames == [] + finally: + # The fake slot must not outlive the test: _sessions is module state and + # later sweeps would walk it. + server._sessions.pop("multi-sid", None) def test_close_transport_detaches_when_no_viewers_remain(monkeypatch): @@ -5340,7 +5373,15 @@ def test_close_transport_detaches_when_no_viewers_remain(monkeypatch): def test_close_transport_skips_dead_remaining_viewers(monkeypatch): - """A viewer whose socket is already dead must not win the re-bind.""" + """A viewer whose socket is already dead must not hold the session open. + + #83716's rebind refused to hand the session to a dead viewer; fan-out + membership keeps that filter through _transport_is_live_peer, which is what + decides whether anything survives the departing client. Both windows are + ATTACHED here, which is the state production builds — a viewer that was + never attached leaves the slot single-client and exercises the ordinary park + path instead of this one. + """ reap_calls = [] monkeypatch.setattr(server, "_schedule_ws_orphan_reap", lambda sid: reap_calls.append(sid)) @@ -5349,17 +5390,25 @@ def test_close_transport_skips_dead_remaining_viewers(monkeypatch): return True dead = _LiveTransport() + popout = _LiveTransport() + session = _session(transport=None, running=False) + server._attach_session_transport(session, dead) + server._attach_session_transport(session, popout) + session["viewers"] = {dead: 100.0, popout: 200.0} + assert isinstance(session["transport"], server.FanoutTransport) + # The socket goes away without a disconnect reaching the gateway; the latch + # _transport_is_dead reads is the only trace it leaves behind. dead._closed = True - owner = _LiveTransport() - session = _session(transport=owner, running=False) - session["viewers"] = {dead: 100.0, owner: 200.0} server._sessions["dead-viewer-sid"] = session - reaped, detached = server._close_sessions_for_transport(owner) + try: + reaped, detached = server._close_sessions_for_transport(popout) - assert detached == 1 - assert session["transport"] is server._detached_ws_transport - assert reap_calls == ["dead-viewer-sid"] + assert reaped == 0 and detached == 1 + assert session["transport"] is server._detached_ws_transport + assert reap_calls == ["dead-viewer-sid"] + finally: + server._sessions.pop("dead-viewer-sid", None) def test_live_session_payload_registers_transport_as_viewer(): diff --git a/tests/tui_gateway/test_multi_client_fanout.py b/tests/tui_gateway/test_multi_client_fanout.py new file mode 100644 index 0000000000..1f01f78310 --- /dev/null +++ b/tests/tui_gateway/test_multi_client_fanout.py @@ -0,0 +1,813 @@ +"""A session streams to EVERY attached client, not just the newest one. + +Before this, ``session["transport"]`` held exactly one client and every +``prompt.submit`` / ``session.resume`` / ``session.activate`` / queued-prompt +drain rebound it — so a second client either saw nothing, stole the stream from +the first, or silenced the turn the first was reading. The slot now holds a +``FanoutTransport`` as soon as a second client attaches, and the disconnect path +detaches instead of parking whenever another client is still there. + +Single-client behaviour is the control condition throughout: with one client +attached the slot holds the bare transport and every path behaves exactly as it +did before fan-out existed. +""" + +import threading +import types + +from tui_gateway import server +from tui_gateway.transport import FanoutTransport + + +class _FakeClient: + """A connected client: records the frames it receives. + + ``ok`` False models a peer that has gone away (``write`` returns False, the + gateway's peer-gone signal); ``boom`` models a wedged peer whose write + raises. Both must be pruned without disturbing the healthy clients. + """ + + def __init__(self, name: str, *, ok: bool = True, boom: bool = False) -> None: + self.name = name + self.frames: list[dict] = [] + self.closed = False + self._ok = ok + self._boom = boom + + def write(self, obj: dict) -> bool: + if self._boom: + raise RuntimeError(f"{self.name} is wedged") + self.frames.append(obj) + return self._ok + + def close(self) -> None: + self.closed = True + + def types(self) -> list[str]: + return [(f.get("params") or {}).get("type") for f in self.frames] + + +def _session(**extra) -> dict: + return { + "agent": None, + "session_key": "session-key", + "history": [], + "history_lock": threading.Lock(), + "history_version": 0, + "running": False, + "attached_images": [], + "transport": None, + **extra, + } + + +# ── FanoutTransport ──────────────────────────────────────────────────────── + + +def test_fanout_delivers_to_every_attached_transport(): + a, b = _FakeClient("a"), _FakeClient("b") + fan = FanoutTransport(a, b) + + assert fan.write({"frame": 1}) is True + assert a.frames == b.frames == [{"frame": 1}] + assert fan.transports() == [a, b] + + +def test_fanout_attach_is_idempotent_and_detach_is_by_identity(): + a, b = _FakeClient("a"), _FakeClient("b") + fan = FanoutTransport(a) + + assert fan.attach(a) is False # already attached + assert fan.attach(b) is True + assert fan.contains(a) and fan.contains(b) + assert fan.has_transports(excluding=a) is True + + assert fan.detach(a) is True + assert fan.detach(a) is False + assert fan.transports() == [b] + assert fan.has_transports(excluding=b) is False + + +def test_fanout_prunes_a_wedged_peer_without_disturbing_the_rest(): + """(g) A raising peer is dropped; the healthy client keeps its stream.""" + healthy, wedged = _FakeClient("healthy"), _FakeClient("wedged", boom=True) + fan = FanoutTransport(wedged, healthy) + + assert fan.write({"frame": 1}) is True + assert healthy.frames == [{"frame": 1}] + assert fan.transports() == [healthy] + + # And the pruning is permanent — no retry storm against the wedged peer. + assert fan.write({"frame": 2}) is True + assert len(healthy.frames) == 2 + + +def test_fanout_prunes_a_peer_that_reports_gone_and_reports_all_dead(): + gone = _FakeClient("gone", ok=False) + fan = FanoutTransport(gone) + + # write() returned False -> peer gone -> pruned, and an empty fan-out + # reports peer-gone exactly like a single dead transport would. + assert fan.write({"frame": 1}) is False + assert fan.transports() == [] + assert fan.write({"frame": 2}) is False + + +def test_fanout_close_releases_peers_without_closing_their_sockets(): + a = _FakeClient("a") + fan = FanoutTransport(a) + + fan.close() + + assert fan.transports() == [] + # Each client owns its own socket; the WS handler closes it on disconnect. + assert a.closed is False + + +# ── attach ladder ────────────────────────────────────────────────────────── + + +def test_attach_replaces_an_empty_or_stdio_slot_without_wrapping(): + """The single-client shape is unchanged: no FanoutTransport in sight.""" + a = _FakeClient("a") + + empty = _session(transport=None) + assert server._attach_session_transport(empty, a) is True + assert empty["transport"] is a + + stdio = _session(transport=server._stdio_transport) + assert server._attach_session_transport(stdio, a) is True + assert stdio["transport"] is a + + parked = _session(transport=server._detached_ws_transport) + assert server._attach_session_transport(parked, a) is True + assert parked["transport"] is a + + +def test_attach_of_the_same_transport_is_a_noop(): + a = _FakeClient("a") + session = _session(transport=a) + + assert server._attach_session_transport(session, a) is True + assert session["transport"] is a + + +def test_attach_of_a_second_client_wraps_both_and_a_third_joins_the_fanout(): + a, b, c = _FakeClient("a"), _FakeClient("b"), _FakeClient("c") + session = _session(transport=a) + + server._attach_session_transport(session, b) + assert isinstance(session["transport"], FanoutTransport) + assert session["transport"].transports() == [a, b] + + server._attach_session_transport(session, c) + assert session["transport"].transports() == [a, b, c] + + +def test_attach_never_lets_stdio_displace_a_live_client(): + """An unbound-context activate must not silence the websocket that owns it.""" + a = _FakeClient("a") + session = _session(transport=a) + + assert server._attach_session_transport(session, server._stdio_transport) is False + assert session["transport"] is a + + +def test_attach_flattens_a_fanout_argument_instead_of_nesting_it(): + a, b, c = _FakeClient("a"), _FakeClient("b"), _FakeClient("c") + session = _session(transport=a) + server._attach_session_transport(session, b) + + server._attach_session_transport(session, FanoutTransport(b, c)) + + assert session["transport"].transports() == [a, b, c] + + +def test_attach_flattens_a_fanout_argument_onto_a_single_client_slot(monkeypatch): + """The nesting the queued-prompt paths can actually reach. + + A busy submit pins ``t or session["transport"]`` to the queued prompt, so + the envelope can hold the FAN-OUT that was in the slot, and the drain hands + that straight back to attach. Flattening only when the slot ALREADY fans out + leaves the leaf-slot case nesting one fan-out inside another, and every + reader of the slot scans a single level — ``FanoutTransport.contains``, the + steer-authority check, detach — so a peer in the inner fan-out is invisible + to all of them, and a peer in BOTH levels is written to twice per frame. + """ + a, b = _FakeClient("a"), _FakeClient("b") + + # (1) A fan-out arriving at a leaf slot is flattened into it, not wrapped. + session = _session(transport=a) + assert server._attach_session_transport(session, FanoutTransport(a, b)) is True + slot = session["transport"] + assert isinstance(slot, FanoutTransport) + assert slot.transports() == [a, b] + assert not any(isinstance(t, FanoutTransport) for t in slot.transports()) + assert server._session_transport_contains(session, b) is True + + # (2) The sequence that produces it. Two clients attach, so the slot holds a + # fan-out; a busy submit captures that slot in the queued envelope; the + # second client disconnects and the slot collapses back to the first; the + # drain then attaches the captured fan-out to a leaf slot. + monkeypatch.setattr(server, "_run_prompt_submit", lambda *a_, **k: None) + + drained = _session(transport=a) + server._attach_session_transport(drained, b) + captured = drained["transport"] + assert isinstance(captured, FanoutTransport) + server._enqueue_prompt(drained, "queued while busy", captured) + + assert server._detach_session_transport(drained, b) is True + assert drained["transport"] is a + + server._sessions["flatten-sid"] = drained + try: + assert server._drain_queued_prompt("drain", "flatten-sid", drained) is True + # Flat: the captured fan-out lost b to the same detach that collapsed + # the slot, so flattening it re-attaches only a, which is already there. + assert drained["transport"] is a + server._emit("message.delta", "flatten-sid", {"text": "once"}) + finally: + server._sessions.pop("flatten-sid", None) + + # Nesting would have put a inside the slot AND under it: one frame, two + # writes to the same client. + assert a.types() == ["message.delta"] + + +def test_detach_collapses_back_to_the_single_remaining_client(): + a, b = _FakeClient("a"), _FakeClient("b") + session = _session(transport=a) + server._attach_session_transport(session, b) + + assert server._detach_session_transport(session, a) is True + assert session["transport"] is b + assert server._detach_session_transport(session, b) is False + + +def test_detach_of_a_single_client_leaves_the_slot_for_the_caller_to_park(): + a = _FakeClient("a") + session = _session(transport=a) + + # False == "no live client remains" — the disconnect path parks the sentinel. + assert server._detach_session_transport(session, a) is False + assert session["transport"] is a + + +def test_live_transport_predicates_ignore_stdio_and_the_drop_sentinel(): + a = _FakeClient("a") + + assert server._session_has_live_transport(_session(transport=a)) is True + assert server._session_has_live_transport(_session(transport=a), excluding=a) is False + assert server._session_has_live_transport(_session(transport=None)) is False + assert ( + server._session_has_live_transport(_session(transport=server._stdio_transport)) + is False + ) + assert ( + server._session_has_live_transport( + _session(transport=server._detached_ws_transport) + ) + is False + ) + + +def test_a_closed_socket_is_not_a_live_peer(): + """A transport that already latched ``_closed`` is a departed client. + + ``_transport_is_live_peer`` ended on a bare ``return True``, so a WSTransport + whose socket had gone away still counted as a live peer: a session whose only + remaining peer was that dead socket answered "another client is still here" + and escaped both the park and the reap. ``_transport_is_dead`` is the + module's deadness predicate; the ladder now defers to it. + """ + dead = _FakeClient("dead") + dead._closed = True # the flag WSTransport latches when its socket goes away + + assert server._transport_is_live_peer(dead) is False + assert server._session_has_live_transport(_session(transport=dead)) is False + + # And a fan-out that collapses onto a dead peer is parked, not skipped: the + # live client leaving was the last real client. + live_client = _FakeClient("live") + session = _session(transport=FanoutTransport(live_client, dead)) + server._sessions["dead-peer-sid"] = session + try: + assert server._close_sessions_for_transport(live_client) == (0, 1) + assert session["transport"] is server._detached_ws_transport + finally: + server._sessions.pop("dead-peer-sid", None) + + +def test_a_fanout_with_no_live_peer_is_dead(): + """``_transport_is_dead`` is the reapers' gate, and a fan-out could not fail it. + + A ``FanoutTransport`` is never the drop sentinel and its ``__slots__`` give + it no ``_closed``, so a session that fanned out once read as alive forever: + the TTL reaper, the LRU cap and the disconnect revalidation all ask this one + predicate. A fan-out reaches the no-live-peer state without any disconnect + passing through ``_close_sessions_for_transport`` — a write that returns + False or raises prunes that peer — so the empty case is reachable, and it is + dead. + """ + a, b = _FakeClient("a"), _FakeClient("b") + session = _session(transport=a) + server._attach_session_transport(session, b) + assert isinstance(session["transport"], FanoutTransport) + assert server._transport_is_dead(session["transport"]) is False + + # One peer gone, one still reading: the session keeps its client. + a._closed = True # the flag WSTransport latches when its socket goes away + assert server._transport_is_dead(session["transport"]) is False + + b._closed = True + assert server._transport_is_dead(session["transport"]) is True + + # Pruning every peer leaves the fan-out empty, which is nobody attached. + assert server._transport_is_dead(FanoutTransport()) is True + + +# ── event delivery ───────────────────────────────────────────────────────── + + +def test_two_attached_clients_both_receive_a_session_event(monkeypatch): + """(a) The headline behaviour: one emit, two clients.""" + a, b = _FakeClient("a"), _FakeClient("b") + session = _session(transport=a) + server._attach_session_transport(session, b) + server._sessions["sid"] = session + try: + server._emit("message.delta", "sid", {"text": "hi"}) + finally: + server._sessions.pop("sid", None) + + assert a.types() == ["message.delta"] + assert b.types() == ["message.delta"] + assert a.frames[0] == b.frames[0] + + +def test_a_single_client_session_writes_through_the_bare_transport(monkeypatch): + """(j) Control: nothing about the one-client path changed.""" + a = _FakeClient("a") + server._sessions["sid"] = _session(transport=a) + try: + assert server._sessions["sid"]["transport"] is a + server._emit("message.delta", "sid", {"text": "hi"}) + finally: + server._sessions.pop("sid", None) + + # The replay contract stamps a monotonic ``seq`` on every WS event frame. + # Strip it: this control test is about ROUTING, not numbering. + frames = [ + {**f, "params": {k: v for k, v in f["params"].items() if k != "seq"}} + for f in a.frames + ] + assert frames == [ + { + "jsonrpc": "2.0", + "method": "event", + "params": { + "type": "message.delta", + "session_id": "sid", + "payload": {"text": "hi"}, + }, + } + ] + + +def test_both_attached_clients_receive_the_terminal_message_complete(): + """The frame that ENDS a turn fans out too, not just the streaming deltas. + + ``message.complete`` is how a client knows the turn is over. A mirrored + session that delivered deltas but dropped the terminal frame would leave the + second client rendering a turn that never finishes. + """ + a, b = _FakeClient("a"), _FakeClient("b") + session = _session(transport=a) + server._attach_session_transport(session, b) + server._sessions["sid"] = session + try: + server._emit("message.delta", "sid", {"text": "par"}) + server._emit("message.complete", "sid", {"text": "part", "status": "ok"}) + finally: + server._sessions.pop("sid", None) + + assert a.types() == ["message.delta", "message.complete"] + assert b.types() == ["message.delta", "message.complete"] + assert a.frames[-1] == b.frames[-1] + assert a.frames[-1]["params"]["payload"] == {"text": "part", "status": "ok"} + + # Control: the one-client path delivers the same terminal frame through the + # bare transport, with no fan-out in the slot. + solo = _FakeClient("solo") + server._sessions["solo-sid"] = _session(transport=solo) + try: + assert server._sessions["solo-sid"]["transport"] is solo + server._emit("message.complete", "solo-sid", {"text": "part", "status": "ok"}) + finally: + server._sessions.pop("solo-sid", None) + + assert solo.types() == ["message.complete"] + assert solo.frames[-1]["params"]["payload"] == {"text": "part", "status": "ok"} + + +# ── prompt.submit / queued drain ─────────────────────────────────────────── + + +def test_second_clients_submit_attaches_and_the_first_keeps_streaming(monkeypatch): + """(c) A second client's prompt.submit must not cut the first one out.""" + monkeypatch.setattr(server, "_run_prompt_submit", lambda *a, **k: None) + monkeypatch.setattr(server, "_ensure_active_session_slot", lambda *a, **k: None) + + watcher, submitter = _FakeClient("watcher"), _FakeClient("submitter") + session = _session(transport=watcher, agent=types.SimpleNamespace()) + server._sessions["sid"] = session + token = server.bind_transport(submitter) + try: + server._methods["prompt.submit"]("r1", {"session_id": "sid", "text": "hello"}) + server._emit("message.delta", "sid", {"text": "answer"}) + finally: + server.reset_transport(token) + server._sessions.pop("sid", None) + + assert isinstance(session["transport"], FanoutTransport) + assert watcher.types() == ["message.delta"] + assert submitter.types() == ["message.delta"] + + +def test_queued_prompt_drain_keeps_both_clients_attached(monkeypatch): + """(d) The hole every competing patch missed: the drain used to rebind. + + Client A is streaming; client B submits mid-turn, so B's prompt is queued + with B's transport pinned to it. When the drain fires it must ATTACH B — + rebinding pinned the whole drained turn to B and silenced A. + """ + dispatched = [] + monkeypatch.setattr( + server, + "_run_prompt_submit", + lambda rid, sid, _session, text, **kw: dispatched.append((rid, text)), + ) + + a, b = _FakeClient("a"), _FakeClient("b") + session = _session(transport=a) + server._enqueue_prompt(session, "from B", b) + server._sessions["sid"] = session + try: + assert server._drain_queued_prompt("drain", "sid", session) is True + server._emit("message.delta", "sid", {"text": "drained answer"}) + finally: + server._sessions.pop("sid", None) + + assert dispatched == [("drain", "from B")] + assert isinstance(session["transport"], FanoutTransport) + assert a.types() == ["message.delta"] + assert b.types() == ["message.delta"] + + +def test_queued_prompt_drain_still_rebinds_a_single_client_session(monkeypatch): + """(j) Control: with one client the drain lands on the queuer's transport.""" + monkeypatch.setattr(server, "_run_prompt_submit", lambda *a, **k: None) + + b = _FakeClient("b") + session = _session(transport=server._detached_ws_transport) + server._enqueue_prompt(session, "from B", b) + + assert server._drain_queued_prompt("drain", "sid", session) is True + assert session["transport"] is b + + +# ── disconnect ───────────────────────────────────────────────────────────── + + +def test_watcher_disconnect_leaves_the_other_client_streaming(): + """(e) One client leaving must not park or reap a session someone is reading.""" + a, watcher = _FakeClient("a"), _FakeClient("watcher") + session = _session(transport=a) + server._attach_session_transport(session, watcher) + server._sessions["sid"] = session + try: + assert server._close_sessions_for_transport(watcher) == (0, 0) + assert session["transport"] is a + assert server._ws_session_is_orphaned(session) is False + server._emit("message.delta", "sid", {"text": "still here"}) + finally: + server._sessions.pop("sid", None) + + assert a.types() == ["message.delta"] + assert watcher.types() == [] + + +def test_last_client_disconnect_parks_exactly_as_before(monkeypatch): + """(f) Once the last client goes, today's park + grace-reap path runs.""" + monkeypatch.setattr(server, "_WS_ORPHAN_REAP_GRACE_S", 0) + + a, watcher = _FakeClient("a"), _FakeClient("watcher") + session = _session(transport=a) + server._attach_session_transport(session, watcher) + server._sessions["sid"] = session + try: + assert server._close_sessions_for_transport(watcher) == (0, 0) + assert server._close_sessions_for_transport(a) == (0, 1) + assert session["transport"] is server._detached_ws_transport + assert server._ws_session_is_orphaned(session) is True + finally: + server._sessions.pop("sid", None) + + +def test_last_client_disconnect_reaps_a_close_on_disconnect_session(monkeypatch): + """(f) close_on_disconnect still fires — but only for the LAST client.""" + # The disconnect path claims the session with _pop_session_by_id and finishes + # the close through _teardown_popped_session, so that is what this stubs; the + # sid is asserted through the pop rather than through the call arguments. + closed = [] + + def _fake_teardown(session, *, end_reason: str = "tui_close") -> bool: + closed.append((session, end_reason)) + return True + + monkeypatch.setattr(server, "_teardown_popped_session", _fake_teardown) + + a, watcher = _FakeClient("a"), _FakeClient("watcher") + session = _session(transport=a, close_on_disconnect=True) + server._attach_session_transport(session, watcher) + server._sessions["sid"] = session + try: + assert server._close_sessions_for_transport(watcher) == (0, 0) + assert closed == [] + assert server._close_sessions_for_transport( + a, end_reason="ws_disconnect" + ) == (1, 0) + assert "sid" not in server._sessions + finally: + server._sessions.pop("sid", None) + + assert closed == [(session, "ws_disconnect")] + + +def test_orphan_check_spares_a_session_that_still_has_a_fanout_peer(): + # _ws_session_is_orphaned stayed slot-identity based on main: a fan-out is + # never the drop sentinel, and the disconnect path only parks the sentinel + # once the last peer is gone, so these hold without a liveness rewrite. + a, watcher = _FakeClient("a"), _FakeClient("watcher") + session = _session(transport=a) + server._attach_session_transport(session, watcher) + + assert server._ws_session_is_orphaned(session) is False + + # Losing one of two clients collapses the slot but keeps the session live. + assert server._detach_session_transport(session, a) is True + assert server._ws_session_is_orphaned(session) is False + + # Losing the last one reports "no client left"; the caller then parks the + # drop sentinel, which is what makes the session orphaned. + assert server._detach_session_transport(session, watcher) is False + session["transport"] = server._detached_ws_transport + assert server._ws_session_is_orphaned(session) is True + + +# --------------------------------------------------------------------------- +# Concurrent fan-out delivery +# +# A SERIAL ``FanoutTransport.write`` — each peer written in turn — would let a +# client whose write parks for the full ``tui_gateway.ws._WS_WRITE_TIMEOUT_S`` +# (10s, the non-streaming path's ``fut.result`` wait for a stalled event loop) +# hold the same frame back from every healthy client behind it in the list. +# That is the property these tests pin, so the writes run concurrently, with two +# cases deliberately kept inline: a lone peer (single-client sessions must not +# change at all) and a caller that is already on an event loop (handing that +# write to a pool thread would make the pool thread block on the loop it just +# left, freezing both). +# +# Extra imports for this block: ``asyncio`` below, plus a function-local +# ``tui_gateway.transport`` in the deadline test (it monkeypatches a module +# constant). Everything else — ``threading``, ``FanoutTransport``, +# ``_FakeClient`` — comes from the top of this file. +# --------------------------------------------------------------------------- + +import asyncio + + +class _SlowPeer: + """A peer whose write parks until the test releases it. + + Models the real stall: a non-streaming ``WSTransport.write`` blocks on + ``fut.result(timeout=_WS_WRITE_TIMEOUT_S)`` while the owning event loop is + busy, so the emitting thread sits inside that one peer's write for seconds. + """ + + def __init__(self, name: str = "slow") -> None: + self.name = name + self.frames: list[dict] = [] + self.entered = threading.Event() + self.release = threading.Event() + self.closed = False + + def write(self, obj: dict) -> bool: + self.entered.set() + # Bounded: a regression must fail the assertion below, never hang the + # suite waiting for a release that the failing path never reaches. + self.release.wait(timeout=5.0) + self.frames.append(obj) + return True + + def close(self) -> None: + self.closed = True + + +class _SignallingPeer: + """A healthy peer that announces each frame the moment it lands.""" + + def __init__(self, name: str = "healthy") -> None: + self.name = name + self.frames: list[dict] = [] + self.got_frame = threading.Event() + self.closed = False + + def write(self, obj: dict) -> bool: + self.frames.append(obj) + self.got_frame.set() + return True + + def close(self) -> None: + self.closed = True + + +class _ThreadRecordingPeer: + """A healthy peer that records which thread each of its writes ran on.""" + + def __init__(self, name: str = "peer") -> None: + self.name = name + self.frames: list[dict] = [] + self.threads: list[threading.Thread] = [] + self.closed = False + + def write(self, obj: dict) -> bool: + self.threads.append(threading.current_thread()) + self.frames.append(obj) + return True + + def close(self) -> None: + self.closed = True + + +def _healthy_peer_is_not_held_behind(slow_first: bool) -> None: + """One wedged peer, one healthy peer, one ``write`` from a worker thread. + + The healthy peer must hold the frame while the wedged peer is still parked + inside its own write. Asserted for both list orders so the fix cannot be a + special case that only ever hurries the first (or the last) peer along. + """ + slow = _SlowPeer() + healthy = _SignallingPeer() + peers = (slow, healthy) if slow_first else (healthy, slow) + fanout = FanoutTransport(*peers) + frame = {"jsonrpc": "2.0", "method": "event", "params": {"type": "message.complete"}} + + returned = threading.Event() + result: dict = {} + + def emit() -> None: + try: + result["ok"] = fanout.write(frame) + finally: + returned.set() + + # Emit from a worker thread, which is where the gateway's event writes come + # from: ``handle_ws`` runs ``server.dispatch`` on ``asyncio.to_thread``. + worker = threading.Thread(target=emit, name="fanout-emitter", daemon=True) + worker.start() + try: + assert slow.entered.wait(timeout=2.0), "the wedged peer never got the frame" + assert healthy.got_frame.wait(timeout=2.0), ( + "the healthy peer did not get the frame while a wedged peer held its " + "write open — the fan-out is still serial" + ) + assert healthy.frames == [frame] + assert slow.frames == [] + finally: + slow.release.set() + assert returned.wait(timeout=5.0), "the fan-out write never returned" + worker.join(timeout=5.0) + + # The fan-out still collects: the wedged peer's frame landed before the + # call returned, and both peers stay attached. + assert slow.frames == [frame] + assert result["ok"] is True + assert fanout.transports() == [*peers] + + +def test_a_wedged_peer_does_not_hold_the_frame_from_a_healthy_peer_behind_it(): + _healthy_peer_is_not_held_behind(slow_first=True) + + +def test_a_wedged_peer_does_not_hold_the_frame_from_a_healthy_peer_ahead_of_it(): + _healthy_peer_is_not_held_behind(slow_first=False) + + +def test_a_write_from_inside_an_event_loop_stays_on_the_loop_thread(): + """The deadlock the concurrent path must never introduce. + + ``WSTransport.write`` fires and forgets when it can see it is running on + its own loop, and BLOCKS on ``fut.result`` when it cannot. Hand a + loop-thread write to a pool thread and that pool thread takes the blocking + path, waiting on a loop that is itself waiting on the pool — the whole + gateway stalls for the write timeout. So an on-loop caller stays inline. + """ + a, b = _ThreadRecordingPeer("a"), _ThreadRecordingPeer("b") + fanout = FanoutTransport(a, b) + frame = {"params": {"type": "message.complete"}} + outcome: dict = {} + + async def emit() -> None: + outcome["loop_thread"] = threading.current_thread() + outcome["ok"] = fanout.write(frame) + + # Driven from a side thread with a bounded join so a real deadlock fails + # this test instead of hanging the run. + runner = threading.Thread(target=lambda: asyncio.run(emit()), daemon=True) + runner.start() + runner.join(timeout=5.0) + assert not runner.is_alive(), "fan-out write from the event loop did not return" + + assert outcome["ok"] is True + assert a.frames == [frame] and b.frames == [frame] + assert a.threads == [outcome["loop_thread"]] + assert b.threads == [outcome["loop_thread"]] + + +def test_a_lone_peer_is_written_on_the_calling_thread(): + """Single-client sessions keep the exact behaviour they had before.""" + only = _ThreadRecordingPeer("only") + fanout = FanoutTransport(only) + + frame = {"params": {"type": "message.complete"}} + assert fanout.write(frame) is True + + assert only.frames == [frame] + assert only.threads == [threading.current_thread()] + + +def test_the_concurrent_path_prunes_dead_peers_and_still_delivers(): + """Pruning is unchanged: ``False`` and a raise both detach the peer.""" + gone = _FakeClient("gone", ok=False) + wedged = _FakeClient("wedged", boom=True) + healthy = _FakeClient("healthy") + fanout = FanoutTransport(gone, wedged, healthy) + + frame = {"params": {"type": "message.complete"}} + assert fanout.write(frame) is True + + assert healthy.frames == [frame] + assert fanout.transports() == [healthy] + + +def test_the_concurrent_path_reports_peer_gone_when_every_peer_is_dead(): + gone = _FakeClient("gone", ok=False) + wedged = _FakeClient("wedged", boom=True) + fanout = FanoutTransport(gone, wedged) + + assert fanout.write({"params": {"type": "message.complete"}}) is False + assert fanout.transports() == [] + + +def test_consecutive_frames_reach_every_peer_in_order(): + """The fan-out collects before returning, so frame A is on every peer + before frame B is dispatched to any of them.""" + a, b = _ThreadRecordingPeer("a"), _ThreadRecordingPeer("b") + fanout = FanoutTransport(a, b) + first = {"params": {"type": "message.delta", "text": "1"}} + second = {"params": {"type": "message.complete"}} + + assert fanout.write(first) is True + # Collect-before-return: nothing is still in flight when write() answers. + assert a.frames == [first] and b.frames == [first] + + assert fanout.write(second) is True + assert a.frames == [first, second] + assert b.frames == [first, second] + + +def test_a_peer_that_misses_the_deadline_is_kept_and_counts_as_delivered(monkeypatch): + """Slow is not dead. + + A peer still writing when the fan-out deadline expires stays attached and + the frame counts as in flight — which is what ``WSTransport.write`` itself + reports when its own write times out. Here it is the ONLY live peer, so the + ``True`` return rests entirely on it. + """ + from tui_gateway import transport as transport_module + + monkeypatch.setattr(transport_module, "_FANOUT_WRITE_DEADLINE_S", 0.05) + + slow = _SlowPeer() + gone = _FakeClient("gone", ok=False) + fanout = FanoutTransport(slow, gone) + frame = {"params": {"type": "message.complete"}} + + assert fanout.write(frame) is True + # The dead peer is pruned; the slow one is not. + assert fanout.transports() == [slow] + assert slow.frames == [] + + slow.release.set() diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index 4455fa22c1..fdc53a2530 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -578,7 +578,7 @@ def _(rid, params: dict) -> dict: if (refusal := _reattach_refusal(rid, sid, session)) is not None: return refusal if (t := current_transport()) is not None: - session["transport"] = t + _attach_session_transport(session, t) _cancel_ws_orphan_reap(sid) # Claim the turn against a possibly-running session (busy/queued reply, else fall # through once ``running`` is observed False). The provider interrupt happens after diff --git a/tui_gateway/methods_session.py b/tui_gateway/methods_session.py index 167e573a49..a1eaff2bf4 100644 --- a/tui_gateway/methods_session.py +++ b/tui_gateway/methods_session.py @@ -519,7 +519,7 @@ def _find_live_unpersisted(needle: str, home) -> str: def _resume_live_unpersisted(ctx: _Resume, live_sid: str, live: dict) -> dict: """Reattach a LIVE lazy session with no state.db row yet (every fresh Bot Chat; a 404 here killed messaging - for never-spoken bots). Rebind the transport and cancel the armed orphan-reap Timer (a WS drop may have + for never-spoken bots). Attach the transport and cancel the armed orphan-reap Timer (a WS drop may have sentinel-parked the record) or it fires against this client.""" if ctx.owns_db: _release_db(ctx.db) @@ -628,7 +628,8 @@ def _resume_guard(ctx: _Resume) -> dict | None: def _resume_reuse_live(ctx: _Resume, sid: str, session: dict) -> dict: """Reattach an already-live session under the resume lock (held across the client-gone check, - transport rebind and reap cancel so grace expiry is atomic).""" + transport attach and reap cancel so grace expiry is atomic). _live_session_payload ATTACHES this + caller alongside the client(s) already streaming instead of taking the slot from them.""" with _session_resume_lock: return _resume_reuse_live_locked(ctx, sid, session) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 6e050072aa..5b3a853562 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -34,7 +34,8 @@ from agent.conversation_loop import INTERRUPT_WAITING_FOR_MODEL_PREFIX # noqa: from tui_gateway import git_probe from tui_gateway._env import env_float, env_int from tui_gateway.turn_marker import clear_turn_marker, read_turn_marker, record_turn_start # noqa: F401 -from tui_gateway.transport import (StdioTransport, Transport, bind_transport, current_transport, reset_transport) +from tui_gateway.transport import (FanoutTransport, StdioTransport, Transport, bind_transport, + current_transport, reset_transport) logger = logging.getLogger(__name__) @@ -211,6 +212,180 @@ _stdio_transport = StdioTransport(lambda: _real_stdout, _stdout_lock) _detached_ws_transport = _DropTransport() +# ── multi-client fan-out ───────────────────────────────────────────────────── +# +# A session's ``transport`` slot holds either ONE transport — the historical, +# single-client shape, byte-identical to pre-fan-out behaviour — or a +# ``FanoutTransport`` wrapping several. Because the fan-out satisfies the same +# Transport protocol, ``write_json`` and every other reader of the slot are +# untouched; only the attach/detach ladder below knows the difference. +# +# ``_session_transport_lock`` serializes the read-modify-write of that one slot. +# It is a LEAF lock: nothing under it acquires ``_sessions_lock`` or a session's +# ``history_lock`` (the fan-out's own lock is likewise a leaf), so it is safe to +# take while holding either of those. +_session_transport_lock = threading.Lock() + + +def _transport_is_live_peer(transport) -> bool: + """True when *transport* is a real, currently attached client. + + Excluded: the parked drop sentinel (by definition clientless) and stdio, + which is the process-wide fallback sink — a standalone ``hermes --tui`` + writes there, but nothing "attaches" to it and a second client can never + share it. + """ + if transport is None: + return False + if transport is _detached_ws_transport or transport is _stdio_transport: + return False + if isinstance(transport, (_DropTransport, StdioTransport)): + return False + # A socket that already latched ``_closed`` is a departed client, not a live + # peer: without this, a stale WSTransport keeps a session out of the park / + # reap path and keeps answering "another client is still here". + # ``_transport_is_dead`` is the deadness predicate this module shares with + # the reaper (defined in ``session_reaper``, published onto this namespace + # by ``method_ctx.bind_module``; module globals resolve at call time). + return not _transport_is_dead(transport) + + +def _session_transport_contains(session: dict | None, transport) -> bool: + """True when *transport* is attached to *session*, directly or via fan-out.""" + if not session or transport is None: + return False + existing = session.get("transport") + if existing is transport: + return True + if isinstance(existing, FanoutTransport): + return existing.contains(transport) + return False + + +def _session_live_transports(session: dict | None) -> list: + """Every live client attached to *session* (empty for a parked/stdio slot).""" + existing = (session or {}).get("transport") + if isinstance(existing, FanoutTransport): + return [t for t in existing.transports() if _transport_is_live_peer(t)] + return [existing] if _transport_is_live_peer(existing) else [] + + +def _session_has_live_transport(session: dict | None, *, excluding=None) -> bool: + """True when *session* still has a live client attached, ignoring *excluding*. + + ``excluding`` answers whether another live client remains once one peer is + ignored, which is what the detach and orphan paths need: whether anything + survives the departing transport. + """ + return any(t is not excluding for t in _session_live_transports(session)) + + +def _attach_session_transport(session: dict | None, transport) -> bool: + """Attach *transport* to *session* ADDITIVELY — never steal the slot. + + A ``FanoutTransport`` argument is FLATTENED before the ladder runs, whatever + the slot holds: each of its peers is attached as a leaf, so the slot never + nests one fan-out inside another. The queued-prompt paths make that + reachable — a busy submit records ``session["transport"]`` as the prompt's + transport, which is the fan-out itself when two clients are attached, and + the drain hands it back here after a disconnect has collapsed the slot to a + single client. Nesting would blind every single-level reader of the slot + (``FanoutTransport.contains``, the steer-authority scan, detach) to the + inner peers, so a client inside the inner fan-out could never be found, + parked, reaped, or granted authority over its own turn. + + The ladder, for the leaf newcomer that always reaches it: + + * same object already in the slot → no-op; + * the slot already fans out → attach into it; + * the slot is empty / stdio / the parked drop sentinel → take the slot, + which is exactly what the old rebind did; + * otherwise → wrap the incumbent and the newcomer in a ``FanoutTransport`` + so both clients keep streaming. + + A non-peer newcomer (stdio, drop sentinel) never displaces a live client: + an activate/resume dispatched without a bound websocket must not silence the + socket that owns the session. + + Returns ``True`` when *transport* is attached afterwards. + """ + if not session or transport is None: + return False + if isinstance(transport, FanoutTransport): + # Flatten OUTSIDE the lock — _session_transport_lock is not reentrant. + # Each peer then walks the ladder on its own, so a peer whose socket + # died while the prompt sat in the queue is dropped by the liveness rung + # instead of being pinned back into the slot. + attached = False + for peer in transport.transports(): + if _attach_session_transport(session, peer): + attached = True + return attached + with _session_transport_lock: + existing = session.get("transport") + if existing is transport: + return True + if isinstance(existing, FanoutTransport): + if isinstance(transport, FanoutTransport): + for member in transport.transports(): + existing.attach(member) + else: + existing.attach(transport) + return True + if not _transport_is_live_peer(transport): + if _transport_is_live_peer(existing): + return False + session["transport"] = transport + return True + if not _transport_is_live_peer(existing): + session["transport"] = transport + return True + session["transport"] = FanoutTransport(existing, transport) + return True + + +def _detach_session_transport(session: dict | None, transport) -> bool: + """Detach *transport* from *session*'s slot. + + Returns ``True`` when a live client OTHER than *transport* remains — i.e. + the session must keep streaming and must NOT be parked or reaped. A + single-client session leaves the departing transport in the slot, exactly as + before fan-out existed; its caller parks the drop sentinel over it. + """ + if not session: + return False + with _session_transport_lock: + existing = session.get("transport") + if isinstance(existing, FanoutTransport): + existing.detach(transport) + remaining = existing.transports() + if len(remaining) == 1: + # Collapse: a session back down to one client is indistinguishable + # from one that never fanned out. + session["transport"] = remaining[0] + return _session_has_live_transport(session, excluding=transport) + + +def _detach_transport_from_sessions(transport) -> list[tuple[str, dict]]: + """Detach *transport* from every session holding it. + + Returns the ``(sid, session)`` pairs left with NO live client — the ones the + disconnect path must park or reap. Sessions that retain another attached + client keep streaming and are not returned. + """ + with _sessions_lock: + attached = [ + (sid, s) + for sid, s in _sessions.items() + if _session_transport_contains(s, transport) + ] + return [ + (sid, session) + for sid, session in attached + if not _detach_session_transport(session, transport) + ] + + def _prepend_tool_paths(env: dict[str, str]) -> dict[str, str]: """Prepend managed bin (first: managed-first policy for the Browser Use CLI), venv bin and ~/.local/bin to PATH so slash_worker children resolve Hermes-managed CLIs under the Desktop's minimal PATH.""" diff --git a/tui_gateway/session_auto_continue.py b/tui_gateway/session_auto_continue.py index f31bdf0feb..858a1f736f 100644 --- a/tui_gateway/session_auto_continue.py +++ b/tui_gateway/session_auto_continue.py @@ -288,7 +288,10 @@ def _drain_queued_prompt(rid, sid: str, session: dict) -> bool: _ac_set_queue(session, session.get("queued_prompts") or []) session["running"] = True if queued.get("transport") is not None: - session["transport"] = queued["transport"] + # The queuer's transport is pinned so the drained turn reaches the client that sent it — but + # ATTACHED, not rebound: a mid-turn prompt from a second client used to silence the first for the + # whole drained turn. + _attach_session_transport(session, queued["transport"]) use_compute_host = _session_uses_compute_host(session) with session["history_lock"]: if int(session.get("_queued_prompt_generation", 0)) != queue_generation: diff --git a/tui_gateway/session_lifecycle.py b/tui_gateway/session_lifecycle.py index 90bbf31783..8792565c85 100644 --- a/tui_gateway/session_lifecycle.py +++ b/tui_gateway/session_lifecycle.py @@ -465,8 +465,8 @@ def _reattach_refusal(rid, sid: str, session: dict) -> dict | None: def _rebind_live_transport(sid: str, session: dict, transport: Transport) -> None: - """Point a live session at ``transport`` (caller holds ``history_lock``).""" - session["transport"] = transport + """Attach a live peer without displacing existing subscribers (caller holds ``history_lock``).""" + _attach_session_transport(session, transport) # Every transport that showed this session (pop-outs resume the same sid); on disconnect the last # viewer becomes the transport instead of the drop sentinel. session.setdefault("viewers", {})[transport] = time.time() @@ -592,23 +592,35 @@ def _schedule_ws_orphan_reap( def _close_sessions_for_transport(transport, *, end_reason: str = "ws_disconnect") -> tuple[int, int]: """Single WS-disconnect teardown entry point: reap close_on_disconnect sessions (sidecar/dashboard) immediately; re-point the rest at the detached transport (later emits miss the dead socket) for the grace-windowed WS-orphan - reaper. Returns ``(reaped, detached)`` counts.""" - with _sessions_lock: - owned = [(sid, s) for sid, s in _sessions.items() if s.get("transport") is transport] + reaper. Returns ``(reaped, detached)`` counts. + + Multi-client fan-out: the departing transport is DETACHED from every session first. A session that still has + another client attached keeps streaming and is neither parked nor reaped — a watcher leaving must not end the + turn the remaining client is reading. Only the sessions left clientless take the historical + close_on_disconnect / park-sentinel path, so a single-client disconnect behaves exactly as it always has.""" + clientless = _detach_transport_from_sessions(transport) reaped = detached = 0 - for sid, session in owned: + for sid, session in clientless: claimed_for_teardown = None should_schedule_reap = False - # session.resume fast-path rebinds under _session_resume_lock: take it so a reconnect can't move the transport - # between check and claim. + # session.resume fast-path attaches under _session_resume_lock: take it so a reconnect can't attach + # between the detach above and the claim. with _session_resume_lock, _sessions_lock: current = _sessions.get(sid) if current is not session: continue - if current.get("transport") is not transport: - # The reconnect owns this session now; drop only the old viewer registration. - (current.get("viewers") or {}).pop(transport, None) - continue + # Prune the departing viewer registration in every branch; it must not affect the new owner. + (current.get("viewers") or {}).pop(transport, None) + # Revalidate before claiming (#77129, kept under fan-out): between _detach_transport_from_sessions + # returning this session as clientless and this claim, a concurrent session.resume can attach a NEW + # live transport. Tearing the session down, or parking the sentinel over it, would knock an attached + # client into detached state and arm an orphan reap against a session that has a live owner. Attach + # and detach both serialize on _session_transport_lock, so this check is race-free against them; that + # lock is a leaf, so taking it under _sessions_lock is safe and _session_has_live_transport does not + # re-acquire it. + with _session_transport_lock: + if _session_has_live_transport(current, excluding=transport): + continue if current.get("close_on_disconnect"): claimed_for_teardown = _pop_session_by_id(sid) else: diff --git a/tui_gateway/session_reaper.py b/tui_gateway/session_reaper.py index 9412ddcd70..1611ab3cb1 100644 --- a/tui_gateway/session_reaper.py +++ b/tui_gateway/session_reaper.py @@ -133,7 +133,18 @@ def install_exit_flush_signal_handlers() -> bool: def _transport_is_dead(transport) -> bool: # _detached_ws_transport is the post-disconnect drop sentinel. _stdio_transport is the REAL transport for # standalone `hermes --tui` and must NOT count as dead. - return transport is _detached_ws_transport or getattr(transport, "_closed", None) is True + if transport is _detached_ws_transport: + return True + if isinstance(transport, FanoutTransport): + # A fan-out is never the sentinel and has no ``_closed`` of its own, so without this arm every + # multi-client session reads as alive forever — the TTL reaper, the LRU cap and the #77129 disconnect + # revalidation all gate on this predicate. A fan-out can legitimately end up empty, or holding nothing + # but closed sockets, with no disconnect passing through _close_sessions_for_transport: a failed write + # prunes the peer that failed. It is dead exactly when no peer of its own is alive, and an empty one is + # dead. Peers are always leaf transports (attach flattens a fan-out argument instead of nesting it), so + # this recurses one level at most. + return all(_transport_is_dead(peer) for peer in transport.transports()) + return getattr(transport, "_closed", None) is True def _session_is_lru_evictable(sid: str, session: dict) -> bool: diff --git a/tui_gateway/transport.py b/tui_gateway/transport.py index 7b67e937ff..26ba3fd12b 100644 --- a/tui_gateway/transport.py +++ b/tui_gateway/transport.py @@ -9,6 +9,8 @@ A :class:`Transport` forwards a JSON-serialisable dict to its peer, so one dispa from __future__ import annotations +import asyncio +import concurrent.futures import contextlib import contextvars import errno @@ -16,6 +18,7 @@ import json import logging import os import threading +import time from typing import Any, Callable, Optional, Protocol, runtime_checkable # Errno values that mean "the peer is gone" rather than "the host has a real I/O problem". Anything @@ -32,6 +35,64 @@ logger = logging.getLogger(__name__) # fully buffered on a pipe, so this ONLY makes sense with ``-u``/``PYTHONUNBUFFERED=1``; otherwise the TUI hangs. _DISABLE_FLUSH = (os.environ.get("HERMES_TUI_GATEWAY_NO_FLUSH", "") or "").strip().lower() in {"1", "true", "yes", "on"} +# Worker pool behind FanoutTransport.write. A non-streaming WebSocket write +# blocks the calling thread while the owning event loop flushes the frame — up +# to ``tui_gateway.ws._WS_WRITE_TIMEOUT_S`` (10s) when that loop is stalled — +# so walking a session's peers one at a time lets one wedged client hold the +# frame back from every healthy client behind it. The pool is created lazily +# and only ever used when a session has more than one peer, so the ordinary +# single-client gateway never starts a thread. +# +# Sizing: these workers are idle except while a peer is actually wedged, so the +# cap only has to cover the wedged peers of all sessions at once. If it were +# ever saturated the excess writes queue and run in submission order, which is +# the pre-pool behaviour — degraded, not broken. +_FANOUT_POOL_MAX_WORKERS = 32 + +# Wall-clock bound on ONE fan-out write, measured from before the first peer is +# dispatched. Deliberately larger than ``tui_gateway.ws._WS_WRITE_TIMEOUT_S`` +# (10.0) so a merely-slow peer reaches its own timeout and reports for itself; +# this deadline only exists so a peer that never returns at all cannot pin the +# emitting thread forever. Not imported from ``tui_gateway.ws``: that module +# imports ``tui_gateway.server``, which imports this one, so the import would +# be circular. Drift is harmless — a peer that misses this deadline is treated +# as in-flight, not as dead (see ``FanoutTransport.write``). +_FANOUT_WRITE_DEADLINE_S = 12.0 + +_fanout_pool: "concurrent.futures.ThreadPoolExecutor | None" = None +_fanout_pool_lock = threading.Lock() + + +def _fanout_write_pool() -> "concurrent.futures.ThreadPoolExecutor": + """The shared fan-out pool, created on first multi-peer write.""" + global _fanout_pool + pool = _fanout_pool + if pool is not None: + return pool + with _fanout_pool_lock: + if _fanout_pool is None: + _fanout_pool = concurrent.futures.ThreadPoolExecutor( + max_workers=_FANOUT_POOL_MAX_WORKERS, + thread_name_prefix="tui-fanout", + ) + return _fanout_pool + + +def _caller_is_on_event_loop() -> bool: + """True when the CALLING thread is running an asyncio event loop. + + Mirrors the probe in ``tui_gateway.ws.WSTransport.write``, which takes a + fire-and-forget path when it can see it is on its own loop and a BLOCKING + ``fut.result`` path when it cannot. ``FanoutTransport`` owns no loop and + its peers may sit on different ones, so the test here is the conservative + one: if this thread is running any loop at all, keep the peer writes on it. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + @runtime_checkable class Transport(Protocol): @@ -115,6 +176,208 @@ class StdioTransport: return None +class FanoutTransport: + """Delivers one JSON frame to every client attached to a session. + + A session's ``transport`` slot used to hold exactly one client, so every + ``prompt.submit`` / ``session.resume`` / ``session.activate`` / queued-prompt + drain REBOUND it and whichever client was there before went silent. This + object goes in the same slot and satisfies the same :class:`Transport` + protocol, so ``server.write_json`` — and every other reader of that slot — + is unchanged; the only difference is that N clients receive the frame + instead of the most recent one. + + Scope: this carries a session's ASYNC EVENT stream only. Request/response + RPCs still answer on the request's context-bound transport + (``dispatch`` → ``current_transport()``), so a client only ever sees replies + to its own calls. + + Peers are pruned on failure: a transport that returns ``False`` (peer gone) + or raises is dropped from the fan-out instead of being written to forever. + Pruning is about dead peers, not slow ones — a client that is merely wedged + is kept, and :meth:`write` runs the peer writes concurrently so its stall + stays its own. + """ + + __slots__ = ("_lock", "_transports") + + def __init__(self, *transports: "Transport") -> None: + self._lock = threading.Lock() + self._transports: list["Transport"] = [] + for transport in transports: + self.attach(transport) + + def attach(self, transport: "Transport") -> bool: + """Add *transport*. Returns ``True`` when it was not already attached.""" + if transport is None or transport is self: + return False + with self._lock: + for existing in self._transports: + if existing is transport: + return False + self._transports.append(transport) + return True + + def detach(self, transport: "Transport") -> bool: + """Remove *transport*. Returns ``True`` when it was attached.""" + with self._lock: + for idx, existing in enumerate(self._transports): + if existing is transport: + del self._transports[idx] + return True + return False + + def contains(self, transport: "Transport") -> bool: + with self._lock: + return any(existing is transport for existing in self._transports) + + def transports(self) -> list["Transport"]: + """A snapshot of the attached transports (safe to iterate).""" + with self._lock: + return list(self._transports) + + def has_transports(self, *, excluding: "Transport | None" = None) -> bool: + """True when any transport other than *excluding* is still attached.""" + with self._lock: + return any(existing is not excluding for existing in self._transports) + + @staticmethod + def _deliver(transport: "Transport", obj: dict, dead: list) -> bool: + """Write *obj* to one peer inline. Records a dead peer in *dead*. + + Returns ``True`` when the peer took the frame. + """ + try: + ok = transport.write(obj) + except Exception: + logger.debug("fanout write failed; pruning peer", exc_info=True) + dead.append(transport) + return False + if ok: + return True + dead.append(transport) + return False + + def write(self, obj: dict) -> bool: + """Deliver *obj* to every attached transport. + + Iterates a SNAPSHOT and never holds the lock across a peer write, so a + peer write can never block an attach or a detach. + + With more than one peer the writes run CONCURRENTLY: every peer but one + is handed to a shared worker pool and the last runs on the calling + thread, which is going to block here anyway. Serial delivery would let + a client wedged for the full WS write timeout hold the frame back from + every healthy client behind it in the list; this way each peer's stall + is its own. Two cases stay inline: + + * exactly one peer, so a single-client session is untouched; + * a caller already running on an event loop. ``WSTransport.write`` + fires and forgets when it sees its own loop and BLOCKS on + ``fut.result`` when it does not, so moving a loop-thread write onto a + pool thread would make that thread wait on the loop it just left + while the loop waits on it — a full write-timeout freeze. + + Pruning is unchanged: a peer that returns ``False`` or raises is + detached, whichever path it took. A peer that has not answered by + ``_FANOUT_WRITE_DEADLINE_S`` is NOT pruned — slow is not dead — and + counts as delivered, which is what ``WSTransport.write`` itself reports + when its own write times out: the frame is queued on that peer's loop + and flushes when the loop breathes. If that peer really is gone, its + next write returns ``False`` promptly and prunes it then. + + The collect happens before the return, so frame A is on every peer that + answered within the deadline before frame B is dispatched to any of + them. A peer whose write was still queued on the pool when the deadline + passed is the exception: the next frame can reach that peer's own + ``_token_lock`` while the previous one is still waiting for it, and the + lock — not this method — decides the order they land in. Within a peer, + ordering is that peer's own business (``WSTransport`` queues under its + token lock, ``StdioTransport`` writes under its stream lock); + ``server.write_json`` takes no lock, so concurrent emitters interleave + here exactly as they already did. + + Returns ``True`` when at least one client accepted the frame or still + has it in flight — an all-dead fan-out reports peer-gone exactly like a + single dead transport would. + """ + targets = self.transports() + if not targets: + return False + + dead: list["Transport"] = [] + delivered = False + + if len(targets) == 1 or _caller_is_on_event_loop(): + for transport in targets: + if self._deliver(transport, obj, dead): + delivered = True + for transport in dead: + self.detach(transport) + return delivered + + deadline = time.monotonic() + _FANOUT_WRITE_DEADLINE_S + pool = _fanout_write_pool() + dispatched: list[tuple["Transport", "concurrent.futures.Future | None"]] = [] + for transport in targets[:-1]: + try: + dispatched.append((transport, pool.submit(transport.write, obj))) + except RuntimeError: + # Pool refused the work (interpreter shutting down). Deliver + # inline below rather than dropping the frame. + dispatched.append((transport, None)) + + # The last peer runs here: one fewer worker per write, and a two-client + # session costs a single pool thread. + if self._deliver(targets[-1], obj, dead): + delivered = True + + # One deadline for the whole batch, then read only the futures that + # finished. Waiting first and calling ``result()`` on a settled future + # keeps our deadline distinguishable from a peer that raised: on 3.11+ + # ``concurrent.futures.TimeoutError`` IS the builtin ``TimeoutError``, + # so a ``result(timeout=...)`` cannot tell the two apart. + futures = [fut for _, fut in dispatched if fut is not None] + if futures: + concurrent.futures.wait( + futures, timeout=max(0.0, deadline - time.monotonic()) + ) + + for transport, fut in dispatched: + if fut is None: + if self._deliver(transport, obj, dead): + delivered = True + continue + if not fut.done(): + # Slow, not dead: leave it attached and count the frame as in + # flight. Cancelling would not stop a write already running. + logger.warning( + "fanout write still pending after %ss; peer left attached", + _FANOUT_WRITE_DEADLINE_S, + ) + delivered = True + continue + try: + ok = fut.result() + except Exception: + logger.debug("fanout write failed; pruning peer", exc_info=True) + dead.append(transport) + continue + if ok: + delivered = True + else: + dead.append(transport) + + for transport in dead: + self.detach(transport) + return delivered + + def close(self) -> None: + """Detach every peer. Does NOT close them — each owns its own socket.""" + with self._lock: + self._transports = [] + + class TeeTransport: """Mirrors writes to one primary plus N best-effort secondaries. The primary's return value (and exceptions) determine the result; secondaries swallow failures so a wedged sidecar never stalls the