diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index 3cf2e25bd6..ca030284bf 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -5294,27 +5294,60 @@ def test_finalize_session_closes_slash_worker(monkeypatch): def test_close_transport_rebinds_session_to_remaining_viewer(monkeypatch): - """Closing a pop-out window's transport must re-bind the session to a - still-open window instead of stranding it on the drop sentinel (#83716).""" + """Closing a pop-out window's transport must leave the session with the + still-open window instead of stranding it on the drop sentinel (#83716). + + The rebind #83716 added is gone; multi-client fan-out subsumes it. Both + windows are attached to the slot at once, so the pop-out is a fan-out peer + rather than a viewer waiting to be promoted, and closing it detaches that + peer and collapses the slot back onto the main window. This pins the same + guarantee through the mechanism that replaced the rebind: the session is + not parked, not reaped, not handed to the orphan reaper, and the surviving + window keeps receiving frames. + """ reap_calls = [] monkeypatch.setattr(server, "_schedule_ws_orphan_reap", lambda sid: reap_calls.append(sid)) class _LiveTransport: - def write(self, *a, **k): + def __init__(self): + self.frames = [] + + def write(self, obj=None, *a, **k): + self.frames.append(obj) return True main = _LiveTransport() popout = _LiveTransport() - session = _session(transport=popout, running=False) + session = _session(transport=None, running=False) + # Build the state the way production does: every window that resumes goes + # through _live_session_payload, which attaches it into the slot and then + # stamps it into the viewers registry. + server._attach_session_transport(session, main) + server._attach_session_transport(session, popout) session["viewers"] = {main: 100.0, popout: 200.0} server._sessions["multi-sid"] = session + assert isinstance(session["transport"], server.FanoutTransport) - reaped, detached = server._close_sessions_for_transport(popout) + try: + reaped, detached = server._close_sessions_for_transport(popout) - assert reaped == 0 and detached == 0 - assert session["transport"] is main - assert "multi-sid" not in reap_calls - assert server._ws_session_is_orphaned(session) is False + assert reaped == 0 and detached == 0 + # One peer left, so the fan-out collapses back to the bare transport — + # the slot is indistinguishable from a session that never fanned out. + assert session["transport"] is main + assert "multi-sid" not in reap_calls + assert server._ws_session_is_orphaned(session) is False + + # And it is still a working stream, not just a surviving reference. + server._emit("message.delta", "multi-sid", {"text": "still here"}) + assert [(f.get("params") or {}).get("type") for f in main.frames] == [ + "message.delta" + ] + assert popout.frames == [] + finally: + # The fake slot must not outlive the test: _sessions is module state and + # later sweeps would walk it. + server._sessions.pop("multi-sid", None) def test_close_transport_detaches_when_no_viewers_remain(monkeypatch): @@ -5340,7 +5373,15 @@ def test_close_transport_detaches_when_no_viewers_remain(monkeypatch): def test_close_transport_skips_dead_remaining_viewers(monkeypatch): - """A viewer whose socket is already dead must not win the re-bind.""" + """A viewer whose socket is already dead must not hold the session open. + + #83716's rebind refused to hand the session to a dead viewer; fan-out + membership keeps that filter through _transport_is_live_peer, which is what + decides whether anything survives the departing client. Both windows are + ATTACHED here, which is the state production builds — a viewer that was + never attached leaves the slot single-client and exercises the ordinary park + path instead of this one. + """ reap_calls = [] monkeypatch.setattr(server, "_schedule_ws_orphan_reap", lambda sid: reap_calls.append(sid)) @@ -5349,17 +5390,25 @@ def test_close_transport_skips_dead_remaining_viewers(monkeypatch): return True dead = _LiveTransport() + popout = _LiveTransport() + session = _session(transport=None, running=False) + server._attach_session_transport(session, dead) + server._attach_session_transport(session, popout) + session["viewers"] = {dead: 100.0, popout: 200.0} + assert isinstance(session["transport"], server.FanoutTransport) + # The socket goes away without a disconnect reaching the gateway; the latch + # _transport_is_dead reads is the only trace it leaves behind. dead._closed = True - owner = _LiveTransport() - session = _session(transport=owner, running=False) - session["viewers"] = {dead: 100.0, owner: 200.0} server._sessions["dead-viewer-sid"] = session - reaped, detached = server._close_sessions_for_transport(owner) + try: + reaped, detached = server._close_sessions_for_transport(popout) - assert detached == 1 - assert session["transport"] is server._detached_ws_transport - assert reap_calls == ["dead-viewer-sid"] + assert reaped == 0 and detached == 1 + assert session["transport"] is server._detached_ws_transport + assert reap_calls == ["dead-viewer-sid"] + finally: + server._sessions.pop("dead-viewer-sid", None) def test_live_session_payload_registers_transport_as_viewer(): diff --git a/tests/tui_gateway/test_multi_client_fanout.py b/tests/tui_gateway/test_multi_client_fanout.py new file mode 100644 index 0000000000..1f01f78310 --- /dev/null +++ b/tests/tui_gateway/test_multi_client_fanout.py @@ -0,0 +1,813 @@ +"""A session streams to EVERY attached client, not just the newest one. + +Before this, ``session["transport"]`` held exactly one client and every +``prompt.submit`` / ``session.resume`` / ``session.activate`` / queued-prompt +drain rebound it — so a second client either saw nothing, stole the stream from +the first, or silenced the turn the first was reading. The slot now holds a +``FanoutTransport`` as soon as a second client attaches, and the disconnect path +detaches instead of parking whenever another client is still there. + +Single-client behaviour is the control condition throughout: with one client +attached the slot holds the bare transport and every path behaves exactly as it +did before fan-out existed. +""" + +import threading +import types + +from tui_gateway import server +from tui_gateway.transport import FanoutTransport + + +class _FakeClient: + """A connected client: records the frames it receives. + + ``ok`` False models a peer that has gone away (``write`` returns False, the + gateway's peer-gone signal); ``boom`` models a wedged peer whose write + raises. Both must be pruned without disturbing the healthy clients. + """ + + def __init__(self, name: str, *, ok: bool = True, boom: bool = False) -> None: + self.name = name + self.frames: list[dict] = [] + self.closed = False + self._ok = ok + self._boom = boom + + def write(self, obj: dict) -> bool: + if self._boom: + raise RuntimeError(f"{self.name} is wedged") + self.frames.append(obj) + return self._ok + + def close(self) -> None: + self.closed = True + + def types(self) -> list[str]: + return [(f.get("params") or {}).get("type") for f in self.frames] + + +def _session(**extra) -> dict: + return { + "agent": None, + "session_key": "session-key", + "history": [], + "history_lock": threading.Lock(), + "history_version": 0, + "running": False, + "attached_images": [], + "transport": None, + **extra, + } + + +# ── FanoutTransport ──────────────────────────────────────────────────────── + + +def test_fanout_delivers_to_every_attached_transport(): + a, b = _FakeClient("a"), _FakeClient("b") + fan = FanoutTransport(a, b) + + assert fan.write({"frame": 1}) is True + assert a.frames == b.frames == [{"frame": 1}] + assert fan.transports() == [a, b] + + +def test_fanout_attach_is_idempotent_and_detach_is_by_identity(): + a, b = _FakeClient("a"), _FakeClient("b") + fan = FanoutTransport(a) + + assert fan.attach(a) is False # already attached + assert fan.attach(b) is True + assert fan.contains(a) and fan.contains(b) + assert fan.has_transports(excluding=a) is True + + assert fan.detach(a) is True + assert fan.detach(a) is False + assert fan.transports() == [b] + assert fan.has_transports(excluding=b) is False + + +def test_fanout_prunes_a_wedged_peer_without_disturbing_the_rest(): + """(g) A raising peer is dropped; the healthy client keeps its stream.""" + healthy, wedged = _FakeClient("healthy"), _FakeClient("wedged", boom=True) + fan = FanoutTransport(wedged, healthy) + + assert fan.write({"frame": 1}) is True + assert healthy.frames == [{"frame": 1}] + assert fan.transports() == [healthy] + + # And the pruning is permanent — no retry storm against the wedged peer. + assert fan.write({"frame": 2}) is True + assert len(healthy.frames) == 2 + + +def test_fanout_prunes_a_peer_that_reports_gone_and_reports_all_dead(): + gone = _FakeClient("gone", ok=False) + fan = FanoutTransport(gone) + + # write() returned False -> peer gone -> pruned, and an empty fan-out + # reports peer-gone exactly like a single dead transport would. + assert fan.write({"frame": 1}) is False + assert fan.transports() == [] + assert fan.write({"frame": 2}) is False + + +def test_fanout_close_releases_peers_without_closing_their_sockets(): + a = _FakeClient("a") + fan = FanoutTransport(a) + + fan.close() + + assert fan.transports() == [] + # Each client owns its own socket; the WS handler closes it on disconnect. + assert a.closed is False + + +# ── attach ladder ────────────────────────────────────────────────────────── + + +def test_attach_replaces_an_empty_or_stdio_slot_without_wrapping(): + """The single-client shape is unchanged: no FanoutTransport in sight.""" + a = _FakeClient("a") + + empty = _session(transport=None) + assert server._attach_session_transport(empty, a) is True + assert empty["transport"] is a + + stdio = _session(transport=server._stdio_transport) + assert server._attach_session_transport(stdio, a) is True + assert stdio["transport"] is a + + parked = _session(transport=server._detached_ws_transport) + assert server._attach_session_transport(parked, a) is True + assert parked["transport"] is a + + +def test_attach_of_the_same_transport_is_a_noop(): + a = _FakeClient("a") + session = _session(transport=a) + + assert server._attach_session_transport(session, a) is True + assert session["transport"] is a + + +def test_attach_of_a_second_client_wraps_both_and_a_third_joins_the_fanout(): + a, b, c = _FakeClient("a"), _FakeClient("b"), _FakeClient("c") + session = _session(transport=a) + + server._attach_session_transport(session, b) + assert isinstance(session["transport"], FanoutTransport) + assert session["transport"].transports() == [a, b] + + server._attach_session_transport(session, c) + assert session["transport"].transports() == [a, b, c] + + +def test_attach_never_lets_stdio_displace_a_live_client(): + """An unbound-context activate must not silence the websocket that owns it.""" + a = _FakeClient("a") + session = _session(transport=a) + + assert server._attach_session_transport(session, server._stdio_transport) is False + assert session["transport"] is a + + +def test_attach_flattens_a_fanout_argument_instead_of_nesting_it(): + a, b, c = _FakeClient("a"), _FakeClient("b"), _FakeClient("c") + session = _session(transport=a) + server._attach_session_transport(session, b) + + server._attach_session_transport(session, FanoutTransport(b, c)) + + assert session["transport"].transports() == [a, b, c] + + +def test_attach_flattens_a_fanout_argument_onto_a_single_client_slot(monkeypatch): + """The nesting the queued-prompt paths can actually reach. + + A busy submit pins ``t or session["transport"]`` to the queued prompt, so + the envelope can hold the FAN-OUT that was in the slot, and the drain hands + that straight back to attach. Flattening only when the slot ALREADY fans out + leaves the leaf-slot case nesting one fan-out inside another, and every + reader of the slot scans a single level — ``FanoutTransport.contains``, the + steer-authority check, detach — so a peer in the inner fan-out is invisible + to all of them, and a peer in BOTH levels is written to twice per frame. + """ + a, b = _FakeClient("a"), _FakeClient("b") + + # (1) A fan-out arriving at a leaf slot is flattened into it, not wrapped. + session = _session(transport=a) + assert server._attach_session_transport(session, FanoutTransport(a, b)) is True + slot = session["transport"] + assert isinstance(slot, FanoutTransport) + assert slot.transports() == [a, b] + assert not any(isinstance(t, FanoutTransport) for t in slot.transports()) + assert server._session_transport_contains(session, b) is True + + # (2) The sequence that produces it. Two clients attach, so the slot holds a + # fan-out; a busy submit captures that slot in the queued envelope; the + # second client disconnects and the slot collapses back to the first; the + # drain then attaches the captured fan-out to a leaf slot. + monkeypatch.setattr(server, "_run_prompt_submit", lambda *a_, **k: None) + + drained = _session(transport=a) + server._attach_session_transport(drained, b) + captured = drained["transport"] + assert isinstance(captured, FanoutTransport) + server._enqueue_prompt(drained, "queued while busy", captured) + + assert server._detach_session_transport(drained, b) is True + assert drained["transport"] is a + + server._sessions["flatten-sid"] = drained + try: + assert server._drain_queued_prompt("drain", "flatten-sid", drained) is True + # Flat: the captured fan-out lost b to the same detach that collapsed + # the slot, so flattening it re-attaches only a, which is already there. + assert drained["transport"] is a + server._emit("message.delta", "flatten-sid", {"text": "once"}) + finally: + server._sessions.pop("flatten-sid", None) + + # Nesting would have put a inside the slot AND under it: one frame, two + # writes to the same client. + assert a.types() == ["message.delta"] + + +def test_detach_collapses_back_to_the_single_remaining_client(): + a, b = _FakeClient("a"), _FakeClient("b") + session = _session(transport=a) + server._attach_session_transport(session, b) + + assert server._detach_session_transport(session, a) is True + assert session["transport"] is b + assert server._detach_session_transport(session, b) is False + + +def test_detach_of_a_single_client_leaves_the_slot_for_the_caller_to_park(): + a = _FakeClient("a") + session = _session(transport=a) + + # False == "no live client remains" — the disconnect path parks the sentinel. + assert server._detach_session_transport(session, a) is False + assert session["transport"] is a + + +def test_live_transport_predicates_ignore_stdio_and_the_drop_sentinel(): + a = _FakeClient("a") + + assert server._session_has_live_transport(_session(transport=a)) is True + assert server._session_has_live_transport(_session(transport=a), excluding=a) is False + assert server._session_has_live_transport(_session(transport=None)) is False + assert ( + server._session_has_live_transport(_session(transport=server._stdio_transport)) + is False + ) + assert ( + server._session_has_live_transport( + _session(transport=server._detached_ws_transport) + ) + is False + ) + + +def test_a_closed_socket_is_not_a_live_peer(): + """A transport that already latched ``_closed`` is a departed client. + + ``_transport_is_live_peer`` ended on a bare ``return True``, so a WSTransport + whose socket had gone away still counted as a live peer: a session whose only + remaining peer was that dead socket answered "another client is still here" + and escaped both the park and the reap. ``_transport_is_dead`` is the + module's deadness predicate; the ladder now defers to it. + """ + dead = _FakeClient("dead") + dead._closed = True # the flag WSTransport latches when its socket goes away + + assert server._transport_is_live_peer(dead) is False + assert server._session_has_live_transport(_session(transport=dead)) is False + + # And a fan-out that collapses onto a dead peer is parked, not skipped: the + # live client leaving was the last real client. + live_client = _FakeClient("live") + session = _session(transport=FanoutTransport(live_client, dead)) + server._sessions["dead-peer-sid"] = session + try: + assert server._close_sessions_for_transport(live_client) == (0, 1) + assert session["transport"] is server._detached_ws_transport + finally: + server._sessions.pop("dead-peer-sid", None) + + +def test_a_fanout_with_no_live_peer_is_dead(): + """``_transport_is_dead`` is the reapers' gate, and a fan-out could not fail it. + + A ``FanoutTransport`` is never the drop sentinel and its ``__slots__`` give + it no ``_closed``, so a session that fanned out once read as alive forever: + the TTL reaper, the LRU cap and the disconnect revalidation all ask this one + predicate. A fan-out reaches the no-live-peer state without any disconnect + passing through ``_close_sessions_for_transport`` — a write that returns + False or raises prunes that peer — so the empty case is reachable, and it is + dead. + """ + a, b = _FakeClient("a"), _FakeClient("b") + session = _session(transport=a) + server._attach_session_transport(session, b) + assert isinstance(session["transport"], FanoutTransport) + assert server._transport_is_dead(session["transport"]) is False + + # One peer gone, one still reading: the session keeps its client. + a._closed = True # the flag WSTransport latches when its socket goes away + assert server._transport_is_dead(session["transport"]) is False + + b._closed = True + assert server._transport_is_dead(session["transport"]) is True + + # Pruning every peer leaves the fan-out empty, which is nobody attached. + assert server._transport_is_dead(FanoutTransport()) is True + + +# ── event delivery ───────────────────────────────────────────────────────── + + +def test_two_attached_clients_both_receive_a_session_event(monkeypatch): + """(a) The headline behaviour: one emit, two clients.""" + a, b = _FakeClient("a"), _FakeClient("b") + session = _session(transport=a) + server._attach_session_transport(session, b) + server._sessions["sid"] = session + try: + server._emit("message.delta", "sid", {"text": "hi"}) + finally: + server._sessions.pop("sid", None) + + assert a.types() == ["message.delta"] + assert b.types() == ["message.delta"] + assert a.frames[0] == b.frames[0] + + +def test_a_single_client_session_writes_through_the_bare_transport(monkeypatch): + """(j) Control: nothing about the one-client path changed.""" + a = _FakeClient("a") + server._sessions["sid"] = _session(transport=a) + try: + assert server._sessions["sid"]["transport"] is a + server._emit("message.delta", "sid", {"text": "hi"}) + finally: + server._sessions.pop("sid", None) + + # The replay contract stamps a monotonic ``seq`` on every WS event frame. + # Strip it: this control test is about ROUTING, not numbering. + frames = [ + {**f, "params": {k: v for k, v in f["params"].items() if k != "seq"}} + for f in a.frames + ] + assert frames == [ + { + "jsonrpc": "2.0", + "method": "event", + "params": { + "type": "message.delta", + "session_id": "sid", + "payload": {"text": "hi"}, + }, + } + ] + + +def test_both_attached_clients_receive_the_terminal_message_complete(): + """The frame that ENDS a turn fans out too, not just the streaming deltas. + + ``message.complete`` is how a client knows the turn is over. A mirrored + session that delivered deltas but dropped the terminal frame would leave the + second client rendering a turn that never finishes. + """ + a, b = _FakeClient("a"), _FakeClient("b") + session = _session(transport=a) + server._attach_session_transport(session, b) + server._sessions["sid"] = session + try: + server._emit("message.delta", "sid", {"text": "par"}) + server._emit("message.complete", "sid", {"text": "part", "status": "ok"}) + finally: + server._sessions.pop("sid", None) + + assert a.types() == ["message.delta", "message.complete"] + assert b.types() == ["message.delta", "message.complete"] + assert a.frames[-1] == b.frames[-1] + assert a.frames[-1]["params"]["payload"] == {"text": "part", "status": "ok"} + + # Control: the one-client path delivers the same terminal frame through the + # bare transport, with no fan-out in the slot. + solo = _FakeClient("solo") + server._sessions["solo-sid"] = _session(transport=solo) + try: + assert server._sessions["solo-sid"]["transport"] is solo + server._emit("message.complete", "solo-sid", {"text": "part", "status": "ok"}) + finally: + server._sessions.pop("solo-sid", None) + + assert solo.types() == ["message.complete"] + assert solo.frames[-1]["params"]["payload"] == {"text": "part", "status": "ok"} + + +# ── prompt.submit / queued drain ─────────────────────────────────────────── + + +def test_second_clients_submit_attaches_and_the_first_keeps_streaming(monkeypatch): + """(c) A second client's prompt.submit must not cut the first one out.""" + monkeypatch.setattr(server, "_run_prompt_submit", lambda *a, **k: None) + monkeypatch.setattr(server, "_ensure_active_session_slot", lambda *a, **k: None) + + watcher, submitter = _FakeClient("watcher"), _FakeClient("submitter") + session = _session(transport=watcher, agent=types.SimpleNamespace()) + server._sessions["sid"] = session + token = server.bind_transport(submitter) + try: + server._methods["prompt.submit"]("r1", {"session_id": "sid", "text": "hello"}) + server._emit("message.delta", "sid", {"text": "answer"}) + finally: + server.reset_transport(token) + server._sessions.pop("sid", None) + + assert isinstance(session["transport"], FanoutTransport) + assert watcher.types() == ["message.delta"] + assert submitter.types() == ["message.delta"] + + +def test_queued_prompt_drain_keeps_both_clients_attached(monkeypatch): + """(d) The hole every competing patch missed: the drain used to rebind. + + Client A is streaming; client B submits mid-turn, so B's prompt is queued + with B's transport pinned to it. When the drain fires it must ATTACH B — + rebinding pinned the whole drained turn to B and silenced A. + """ + dispatched = [] + monkeypatch.setattr( + server, + "_run_prompt_submit", + lambda rid, sid, _session, text, **kw: dispatched.append((rid, text)), + ) + + a, b = _FakeClient("a"), _FakeClient("b") + session = _session(transport=a) + server._enqueue_prompt(session, "from B", b) + server._sessions["sid"] = session + try: + assert server._drain_queued_prompt("drain", "sid", session) is True + server._emit("message.delta", "sid", {"text": "drained answer"}) + finally: + server._sessions.pop("sid", None) + + assert dispatched == [("drain", "from B")] + assert isinstance(session["transport"], FanoutTransport) + assert a.types() == ["message.delta"] + assert b.types() == ["message.delta"] + + +def test_queued_prompt_drain_still_rebinds_a_single_client_session(monkeypatch): + """(j) Control: with one client the drain lands on the queuer's transport.""" + monkeypatch.setattr(server, "_run_prompt_submit", lambda *a, **k: None) + + b = _FakeClient("b") + session = _session(transport=server._detached_ws_transport) + server._enqueue_prompt(session, "from B", b) + + assert server._drain_queued_prompt("drain", "sid", session) is True + assert session["transport"] is b + + +# ── disconnect ───────────────────────────────────────────────────────────── + + +def test_watcher_disconnect_leaves_the_other_client_streaming(): + """(e) One client leaving must not park or reap a session someone is reading.""" + a, watcher = _FakeClient("a"), _FakeClient("watcher") + session = _session(transport=a) + server._attach_session_transport(session, watcher) + server._sessions["sid"] = session + try: + assert server._close_sessions_for_transport(watcher) == (0, 0) + assert session["transport"] is a + assert server._ws_session_is_orphaned(session) is False + server._emit("message.delta", "sid", {"text": "still here"}) + finally: + server._sessions.pop("sid", None) + + assert a.types() == ["message.delta"] + assert watcher.types() == [] + + +def test_last_client_disconnect_parks_exactly_as_before(monkeypatch): + """(f) Once the last client goes, today's park + grace-reap path runs.""" + monkeypatch.setattr(server, "_WS_ORPHAN_REAP_GRACE_S", 0) + + a, watcher = _FakeClient("a"), _FakeClient("watcher") + session = _session(transport=a) + server._attach_session_transport(session, watcher) + server._sessions["sid"] = session + try: + assert server._close_sessions_for_transport(watcher) == (0, 0) + assert server._close_sessions_for_transport(a) == (0, 1) + assert session["transport"] is server._detached_ws_transport + assert server._ws_session_is_orphaned(session) is True + finally: + server._sessions.pop("sid", None) + + +def test_last_client_disconnect_reaps_a_close_on_disconnect_session(monkeypatch): + """(f) close_on_disconnect still fires — but only for the LAST client.""" + # The disconnect path claims the session with _pop_session_by_id and finishes + # the close through _teardown_popped_session, so that is what this stubs; the + # sid is asserted through the pop rather than through the call arguments. + closed = [] + + def _fake_teardown(session, *, end_reason: str = "tui_close") -> bool: + closed.append((session, end_reason)) + return True + + monkeypatch.setattr(server, "_teardown_popped_session", _fake_teardown) + + a, watcher = _FakeClient("a"), _FakeClient("watcher") + session = _session(transport=a, close_on_disconnect=True) + server._attach_session_transport(session, watcher) + server._sessions["sid"] = session + try: + assert server._close_sessions_for_transport(watcher) == (0, 0) + assert closed == [] + assert server._close_sessions_for_transport( + a, end_reason="ws_disconnect" + ) == (1, 0) + assert "sid" not in server._sessions + finally: + server._sessions.pop("sid", None) + + assert closed == [(session, "ws_disconnect")] + + +def test_orphan_check_spares_a_session_that_still_has_a_fanout_peer(): + # _ws_session_is_orphaned stayed slot-identity based on main: a fan-out is + # never the drop sentinel, and the disconnect path only parks the sentinel + # once the last peer is gone, so these hold without a liveness rewrite. + a, watcher = _FakeClient("a"), _FakeClient("watcher") + session = _session(transport=a) + server._attach_session_transport(session, watcher) + + assert server._ws_session_is_orphaned(session) is False + + # Losing one of two clients collapses the slot but keeps the session live. + assert server._detach_session_transport(session, a) is True + assert server._ws_session_is_orphaned(session) is False + + # Losing the last one reports "no client left"; the caller then parks the + # drop sentinel, which is what makes the session orphaned. + assert server._detach_session_transport(session, watcher) is False + session["transport"] = server._detached_ws_transport + assert server._ws_session_is_orphaned(session) is True + + +# --------------------------------------------------------------------------- +# Concurrent fan-out delivery +# +# A SERIAL ``FanoutTransport.write`` — each peer written in turn — would let a +# client whose write parks for the full ``tui_gateway.ws._WS_WRITE_TIMEOUT_S`` +# (10s, the non-streaming path's ``fut.result`` wait for a stalled event loop) +# hold the same frame back from every healthy client behind it in the list. +# That is the property these tests pin, so the writes run concurrently, with two +# cases deliberately kept inline: a lone peer (single-client sessions must not +# change at all) and a caller that is already on an event loop (handing that +# write to a pool thread would make the pool thread block on the loop it just +# left, freezing both). +# +# Extra imports for this block: ``asyncio`` below, plus a function-local +# ``tui_gateway.transport`` in the deadline test (it monkeypatches a module +# constant). Everything else — ``threading``, ``FanoutTransport``, +# ``_FakeClient`` — comes from the top of this file. +# --------------------------------------------------------------------------- + +import asyncio + + +class _SlowPeer: + """A peer whose write parks until the test releases it. + + Models the real stall: a non-streaming ``WSTransport.write`` blocks on + ``fut.result(timeout=_WS_WRITE_TIMEOUT_S)`` while the owning event loop is + busy, so the emitting thread sits inside that one peer's write for seconds. + """ + + def __init__(self, name: str = "slow") -> None: + self.name = name + self.frames: list[dict] = [] + self.entered = threading.Event() + self.release = threading.Event() + self.closed = False + + def write(self, obj: dict) -> bool: + self.entered.set() + # Bounded: a regression must fail the assertion below, never hang the + # suite waiting for a release that the failing path never reaches. + self.release.wait(timeout=5.0) + self.frames.append(obj) + return True + + def close(self) -> None: + self.closed = True + + +class _SignallingPeer: + """A healthy peer that announces each frame the moment it lands.""" + + def __init__(self, name: str = "healthy") -> None: + self.name = name + self.frames: list[dict] = [] + self.got_frame = threading.Event() + self.closed = False + + def write(self, obj: dict) -> bool: + self.frames.append(obj) + self.got_frame.set() + return True + + def close(self) -> None: + self.closed = True + + +class _ThreadRecordingPeer: + """A healthy peer that records which thread each of its writes ran on.""" + + def __init__(self, name: str = "peer") -> None: + self.name = name + self.frames: list[dict] = [] + self.threads: list[threading.Thread] = [] + self.closed = False + + def write(self, obj: dict) -> bool: + self.threads.append(threading.current_thread()) + self.frames.append(obj) + return True + + def close(self) -> None: + self.closed = True + + +def _healthy_peer_is_not_held_behind(slow_first: bool) -> None: + """One wedged peer, one healthy peer, one ``write`` from a worker thread. + + The healthy peer must hold the frame while the wedged peer is still parked + inside its own write. Asserted for both list orders so the fix cannot be a + special case that only ever hurries the first (or the last) peer along. + """ + slow = _SlowPeer() + healthy = _SignallingPeer() + peers = (slow, healthy) if slow_first else (healthy, slow) + fanout = FanoutTransport(*peers) + frame = {"jsonrpc": "2.0", "method": "event", "params": {"type": "message.complete"}} + + returned = threading.Event() + result: dict = {} + + def emit() -> None: + try: + result["ok"] = fanout.write(frame) + finally: + returned.set() + + # Emit from a worker thread, which is where the gateway's event writes come + # from: ``handle_ws`` runs ``server.dispatch`` on ``asyncio.to_thread``. + worker = threading.Thread(target=emit, name="fanout-emitter", daemon=True) + worker.start() + try: + assert slow.entered.wait(timeout=2.0), "the wedged peer never got the frame" + assert healthy.got_frame.wait(timeout=2.0), ( + "the healthy peer did not get the frame while a wedged peer held its " + "write open — the fan-out is still serial" + ) + assert healthy.frames == [frame] + assert slow.frames == [] + finally: + slow.release.set() + assert returned.wait(timeout=5.0), "the fan-out write never returned" + worker.join(timeout=5.0) + + # The fan-out still collects: the wedged peer's frame landed before the + # call returned, and both peers stay attached. + assert slow.frames == [frame] + assert result["ok"] is True + assert fanout.transports() == [*peers] + + +def test_a_wedged_peer_does_not_hold_the_frame_from_a_healthy_peer_behind_it(): + _healthy_peer_is_not_held_behind(slow_first=True) + + +def test_a_wedged_peer_does_not_hold_the_frame_from_a_healthy_peer_ahead_of_it(): + _healthy_peer_is_not_held_behind(slow_first=False) + + +def test_a_write_from_inside_an_event_loop_stays_on_the_loop_thread(): + """The deadlock the concurrent path must never introduce. + + ``WSTransport.write`` fires and forgets when it can see it is running on + its own loop, and BLOCKS on ``fut.result`` when it cannot. Hand a + loop-thread write to a pool thread and that pool thread takes the blocking + path, waiting on a loop that is itself waiting on the pool — the whole + gateway stalls for the write timeout. So an on-loop caller stays inline. + """ + a, b = _ThreadRecordingPeer("a"), _ThreadRecordingPeer("b") + fanout = FanoutTransport(a, b) + frame = {"params": {"type": "message.complete"}} + outcome: dict = {} + + async def emit() -> None: + outcome["loop_thread"] = threading.current_thread() + outcome["ok"] = fanout.write(frame) + + # Driven from a side thread with a bounded join so a real deadlock fails + # this test instead of hanging the run. + runner = threading.Thread(target=lambda: asyncio.run(emit()), daemon=True) + runner.start() + runner.join(timeout=5.0) + assert not runner.is_alive(), "fan-out write from the event loop did not return" + + assert outcome["ok"] is True + assert a.frames == [frame] and b.frames == [frame] + assert a.threads == [outcome["loop_thread"]] + assert b.threads == [outcome["loop_thread"]] + + +def test_a_lone_peer_is_written_on_the_calling_thread(): + """Single-client sessions keep the exact behaviour they had before.""" + only = _ThreadRecordingPeer("only") + fanout = FanoutTransport(only) + + frame = {"params": {"type": "message.complete"}} + assert fanout.write(frame) is True + + assert only.frames == [frame] + assert only.threads == [threading.current_thread()] + + +def test_the_concurrent_path_prunes_dead_peers_and_still_delivers(): + """Pruning is unchanged: ``False`` and a raise both detach the peer.""" + gone = _FakeClient("gone", ok=False) + wedged = _FakeClient("wedged", boom=True) + healthy = _FakeClient("healthy") + fanout = FanoutTransport(gone, wedged, healthy) + + frame = {"params": {"type": "message.complete"}} + assert fanout.write(frame) is True + + assert healthy.frames == [frame] + assert fanout.transports() == [healthy] + + +def test_the_concurrent_path_reports_peer_gone_when_every_peer_is_dead(): + gone = _FakeClient("gone", ok=False) + wedged = _FakeClient("wedged", boom=True) + fanout = FanoutTransport(gone, wedged) + + assert fanout.write({"params": {"type": "message.complete"}}) is False + assert fanout.transports() == [] + + +def test_consecutive_frames_reach_every_peer_in_order(): + """The fan-out collects before returning, so frame A is on every peer + before frame B is dispatched to any of them.""" + a, b = _ThreadRecordingPeer("a"), _ThreadRecordingPeer("b") + fanout = FanoutTransport(a, b) + first = {"params": {"type": "message.delta", "text": "1"}} + second = {"params": {"type": "message.complete"}} + + assert fanout.write(first) is True + # Collect-before-return: nothing is still in flight when write() answers. + assert a.frames == [first] and b.frames == [first] + + assert fanout.write(second) is True + assert a.frames == [first, second] + assert b.frames == [first, second] + + +def test_a_peer_that_misses_the_deadline_is_kept_and_counts_as_delivered(monkeypatch): + """Slow is not dead. + + A peer still writing when the fan-out deadline expires stays attached and + the frame counts as in flight — which is what ``WSTransport.write`` itself + reports when its own write times out. Here it is the ONLY live peer, so the + ``True`` return rests entirely on it. + """ + from tui_gateway import transport as transport_module + + monkeypatch.setattr(transport_module, "_FANOUT_WRITE_DEADLINE_S", 0.05) + + slow = _SlowPeer() + gone = _FakeClient("gone", ok=False) + fanout = FanoutTransport(slow, gone) + frame = {"params": {"type": "message.complete"}} + + assert fanout.write(frame) is True + # The dead peer is pruned; the slow one is not. + assert fanout.transports() == [slow] + assert slow.frames == [] + + slow.release.set() diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index 4455fa22c1..fdc53a2530 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -578,7 +578,7 @@ def _(rid, params: dict) -> dict: if (refusal := _reattach_refusal(rid, sid, session)) is not None: return refusal if (t := current_transport()) is not None: - session["transport"] = t + _attach_session_transport(session, t) _cancel_ws_orphan_reap(sid) # Claim the turn against a possibly-running session (busy/queued reply, else fall # through once ``running`` is observed False). The provider interrupt happens after diff --git a/tui_gateway/methods_session.py b/tui_gateway/methods_session.py index 167e573a49..a1eaff2bf4 100644 --- a/tui_gateway/methods_session.py +++ b/tui_gateway/methods_session.py @@ -519,7 +519,7 @@ def _find_live_unpersisted(needle: str, home) -> str: def _resume_live_unpersisted(ctx: _Resume, live_sid: str, live: dict) -> dict: """Reattach a LIVE lazy session with no state.db row yet (every fresh Bot Chat; a 404 here killed messaging - for never-spoken bots). Rebind the transport and cancel the armed orphan-reap Timer (a WS drop may have + for never-spoken bots). Attach the transport and cancel the armed orphan-reap Timer (a WS drop may have sentinel-parked the record) or it fires against this client.""" if ctx.owns_db: _release_db(ctx.db) @@ -628,7 +628,8 @@ def _resume_guard(ctx: _Resume) -> dict | None: def _resume_reuse_live(ctx: _Resume, sid: str, session: dict) -> dict: """Reattach an already-live session under the resume lock (held across the client-gone check, - transport rebind and reap cancel so grace expiry is atomic).""" + transport attach and reap cancel so grace expiry is atomic). _live_session_payload ATTACHES this + caller alongside the client(s) already streaming instead of taking the slot from them.""" with _session_resume_lock: return _resume_reuse_live_locked(ctx, sid, session) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 6e050072aa..5b3a853562 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -34,7 +34,8 @@ from agent.conversation_loop import INTERRUPT_WAITING_FOR_MODEL_PREFIX # noqa: from tui_gateway import git_probe from tui_gateway._env import env_float, env_int from tui_gateway.turn_marker import clear_turn_marker, read_turn_marker, record_turn_start # noqa: F401 -from tui_gateway.transport import (StdioTransport, Transport, bind_transport, current_transport, reset_transport) +from tui_gateway.transport import (FanoutTransport, StdioTransport, Transport, bind_transport, + current_transport, reset_transport) logger = logging.getLogger(__name__) @@ -211,6 +212,180 @@ _stdio_transport = StdioTransport(lambda: _real_stdout, _stdout_lock) _detached_ws_transport = _DropTransport() +# ── multi-client fan-out ───────────────────────────────────────────────────── +# +# A session's ``transport`` slot holds either ONE transport — the historical, +# single-client shape, byte-identical to pre-fan-out behaviour — or a +# ``FanoutTransport`` wrapping several. Because the fan-out satisfies the same +# Transport protocol, ``write_json`` and every other reader of the slot are +# untouched; only the attach/detach ladder below knows the difference. +# +# ``_session_transport_lock`` serializes the read-modify-write of that one slot. +# It is a LEAF lock: nothing under it acquires ``_sessions_lock`` or a session's +# ``history_lock`` (the fan-out's own lock is likewise a leaf), so it is safe to +# take while holding either of those. +_session_transport_lock = threading.Lock() + + +def _transport_is_live_peer(transport) -> bool: + """True when *transport* is a real, currently attached client. + + Excluded: the parked drop sentinel (by definition clientless) and stdio, + which is the process-wide fallback sink — a standalone ``hermes --tui`` + writes there, but nothing "attaches" to it and a second client can never + share it. + """ + if transport is None: + return False + if transport is _detached_ws_transport or transport is _stdio_transport: + return False + if isinstance(transport, (_DropTransport, StdioTransport)): + return False + # A socket that already latched ``_closed`` is a departed client, not a live + # peer: without this, a stale WSTransport keeps a session out of the park / + # reap path and keeps answering "another client is still here". + # ``_transport_is_dead`` is the deadness predicate this module shares with + # the reaper (defined in ``session_reaper``, published onto this namespace + # by ``method_ctx.bind_module``; module globals resolve at call time). + return not _transport_is_dead(transport) + + +def _session_transport_contains(session: dict | None, transport) -> bool: + """True when *transport* is attached to *session*, directly or via fan-out.""" + if not session or transport is None: + return False + existing = session.get("transport") + if existing is transport: + return True + if isinstance(existing, FanoutTransport): + return existing.contains(transport) + return False + + +def _session_live_transports(session: dict | None) -> list: + """Every live client attached to *session* (empty for a parked/stdio slot).""" + existing = (session or {}).get("transport") + if isinstance(existing, FanoutTransport): + return [t for t in existing.transports() if _transport_is_live_peer(t)] + return [existing] if _transport_is_live_peer(existing) else [] + + +def _session_has_live_transport(session: dict | None, *, excluding=None) -> bool: + """True when *session* still has a live client attached, ignoring *excluding*. + + ``excluding`` answers whether another live client remains once one peer is + ignored, which is what the detach and orphan paths need: whether anything + survives the departing transport. + """ + return any(t is not excluding for t in _session_live_transports(session)) + + +def _attach_session_transport(session: dict | None, transport) -> bool: + """Attach *transport* to *session* ADDITIVELY — never steal the slot. + + A ``FanoutTransport`` argument is FLATTENED before the ladder runs, whatever + the slot holds: each of its peers is attached as a leaf, so the slot never + nests one fan-out inside another. The queued-prompt paths make that + reachable — a busy submit records ``session["transport"]`` as the prompt's + transport, which is the fan-out itself when two clients are attached, and + the drain hands it back here after a disconnect has collapsed the slot to a + single client. Nesting would blind every single-level reader of the slot + (``FanoutTransport.contains``, the steer-authority scan, detach) to the + inner peers, so a client inside the inner fan-out could never be found, + parked, reaped, or granted authority over its own turn. + + The ladder, for the leaf newcomer that always reaches it: + + * same object already in the slot → no-op; + * the slot already fans out → attach into it; + * the slot is empty / stdio / the parked drop sentinel → take the slot, + which is exactly what the old rebind did; + * otherwise → wrap the incumbent and the newcomer in a ``FanoutTransport`` + so both clients keep streaming. + + A non-peer newcomer (stdio, drop sentinel) never displaces a live client: + an activate/resume dispatched without a bound websocket must not silence the + socket that owns the session. + + Returns ``True`` when *transport* is attached afterwards. + """ + if not session or transport is None: + return False + if isinstance(transport, FanoutTransport): + # Flatten OUTSIDE the lock — _session_transport_lock is not reentrant. + # Each peer then walks the ladder on its own, so a peer whose socket + # died while the prompt sat in the queue is dropped by the liveness rung + # instead of being pinned back into the slot. + attached = False + for peer in transport.transports(): + if _attach_session_transport(session, peer): + attached = True + return attached + with _session_transport_lock: + existing = session.get("transport") + if existing is transport: + return True + if isinstance(existing, FanoutTransport): + if isinstance(transport, FanoutTransport): + for member in transport.transports(): + existing.attach(member) + else: + existing.attach(transport) + return True + if not _transport_is_live_peer(transport): + if _transport_is_live_peer(existing): + return False + session["transport"] = transport + return True + if not _transport_is_live_peer(existing): + session["transport"] = transport + return True + session["transport"] = FanoutTransport(existing, transport) + return True + + +def _detach_session_transport(session: dict | None, transport) -> bool: + """Detach *transport* from *session*'s slot. + + Returns ``True`` when a live client OTHER than *transport* remains — i.e. + the session must keep streaming and must NOT be parked or reaped. A + single-client session leaves the departing transport in the slot, exactly as + before fan-out existed; its caller parks the drop sentinel over it. + """ + if not session: + return False + with _session_transport_lock: + existing = session.get("transport") + if isinstance(existing, FanoutTransport): + existing.detach(transport) + remaining = existing.transports() + if len(remaining) == 1: + # Collapse: a session back down to one client is indistinguishable + # from one that never fanned out. + session["transport"] = remaining[0] + return _session_has_live_transport(session, excluding=transport) + + +def _detach_transport_from_sessions(transport) -> list[tuple[str, dict]]: + """Detach *transport* from every session holding it. + + Returns the ``(sid, session)`` pairs left with NO live client — the ones the + disconnect path must park or reap. Sessions that retain another attached + client keep streaming and are not returned. + """ + with _sessions_lock: + attached = [ + (sid, s) + for sid, s in _sessions.items() + if _session_transport_contains(s, transport) + ] + return [ + (sid, session) + for sid, session in attached + if not _detach_session_transport(session, transport) + ] + + def _prepend_tool_paths(env: dict[str, str]) -> dict[str, str]: """Prepend managed bin (first: managed-first policy for the Browser Use CLI), venv bin and ~/.local/bin to PATH so slash_worker children resolve Hermes-managed CLIs under the Desktop's minimal PATH.""" diff --git a/tui_gateway/session_auto_continue.py b/tui_gateway/session_auto_continue.py index f31bdf0feb..858a1f736f 100644 --- a/tui_gateway/session_auto_continue.py +++ b/tui_gateway/session_auto_continue.py @@ -288,7 +288,10 @@ def _drain_queued_prompt(rid, sid: str, session: dict) -> bool: _ac_set_queue(session, session.get("queued_prompts") or []) session["running"] = True if queued.get("transport") is not None: - session["transport"] = queued["transport"] + # The queuer's transport is pinned so the drained turn reaches the client that sent it — but + # ATTACHED, not rebound: a mid-turn prompt from a second client used to silence the first for the + # whole drained turn. + _attach_session_transport(session, queued["transport"]) use_compute_host = _session_uses_compute_host(session) with session["history_lock"]: if int(session.get("_queued_prompt_generation", 0)) != queue_generation: diff --git a/tui_gateway/session_lifecycle.py b/tui_gateway/session_lifecycle.py index 90bbf31783..8792565c85 100644 --- a/tui_gateway/session_lifecycle.py +++ b/tui_gateway/session_lifecycle.py @@ -465,8 +465,8 @@ def _reattach_refusal(rid, sid: str, session: dict) -> dict | None: def _rebind_live_transport(sid: str, session: dict, transport: Transport) -> None: - """Point a live session at ``transport`` (caller holds ``history_lock``).""" - session["transport"] = transport + """Attach a live peer without displacing existing subscribers (caller holds ``history_lock``).""" + _attach_session_transport(session, transport) # Every transport that showed this session (pop-outs resume the same sid); on disconnect the last # viewer becomes the transport instead of the drop sentinel. session.setdefault("viewers", {})[transport] = time.time() @@ -592,23 +592,35 @@ def _schedule_ws_orphan_reap( def _close_sessions_for_transport(transport, *, end_reason: str = "ws_disconnect") -> tuple[int, int]: """Single WS-disconnect teardown entry point: reap close_on_disconnect sessions (sidecar/dashboard) immediately; re-point the rest at the detached transport (later emits miss the dead socket) for the grace-windowed WS-orphan - reaper. Returns ``(reaped, detached)`` counts.""" - with _sessions_lock: - owned = [(sid, s) for sid, s in _sessions.items() if s.get("transport") is transport] + reaper. Returns ``(reaped, detached)`` counts. + + Multi-client fan-out: the departing transport is DETACHED from every session first. A session that still has + another client attached keeps streaming and is neither parked nor reaped — a watcher leaving must not end the + turn the remaining client is reading. Only the sessions left clientless take the historical + close_on_disconnect / park-sentinel path, so a single-client disconnect behaves exactly as it always has.""" + clientless = _detach_transport_from_sessions(transport) reaped = detached = 0 - for sid, session in owned: + for sid, session in clientless: claimed_for_teardown = None should_schedule_reap = False - # session.resume fast-path rebinds under _session_resume_lock: take it so a reconnect can't move the transport - # between check and claim. + # session.resume fast-path attaches under _session_resume_lock: take it so a reconnect can't attach + # between the detach above and the claim. with _session_resume_lock, _sessions_lock: current = _sessions.get(sid) if current is not session: continue - if current.get("transport") is not transport: - # The reconnect owns this session now; drop only the old viewer registration. - (current.get("viewers") or {}).pop(transport, None) - continue + # Prune the departing viewer registration in every branch; it must not affect the new owner. + (current.get("viewers") or {}).pop(transport, None) + # Revalidate before claiming (#77129, kept under fan-out): between _detach_transport_from_sessions + # returning this session as clientless and this claim, a concurrent session.resume can attach a NEW + # live transport. Tearing the session down, or parking the sentinel over it, would knock an attached + # client into detached state and arm an orphan reap against a session that has a live owner. Attach + # and detach both serialize on _session_transport_lock, so this check is race-free against them; that + # lock is a leaf, so taking it under _sessions_lock is safe and _session_has_live_transport does not + # re-acquire it. + with _session_transport_lock: + if _session_has_live_transport(current, excluding=transport): + continue if current.get("close_on_disconnect"): claimed_for_teardown = _pop_session_by_id(sid) else: diff --git a/tui_gateway/session_reaper.py b/tui_gateway/session_reaper.py index 9412ddcd70..1611ab3cb1 100644 --- a/tui_gateway/session_reaper.py +++ b/tui_gateway/session_reaper.py @@ -133,7 +133,18 @@ def install_exit_flush_signal_handlers() -> bool: def _transport_is_dead(transport) -> bool: # _detached_ws_transport is the post-disconnect drop sentinel. _stdio_transport is the REAL transport for # standalone `hermes --tui` and must NOT count as dead. - return transport is _detached_ws_transport or getattr(transport, "_closed", None) is True + if transport is _detached_ws_transport: + return True + if isinstance(transport, FanoutTransport): + # A fan-out is never the sentinel and has no ``_closed`` of its own, so without this arm every + # multi-client session reads as alive forever — the TTL reaper, the LRU cap and the #77129 disconnect + # revalidation all gate on this predicate. A fan-out can legitimately end up empty, or holding nothing + # but closed sockets, with no disconnect passing through _close_sessions_for_transport: a failed write + # prunes the peer that failed. It is dead exactly when no peer of its own is alive, and an empty one is + # dead. Peers are always leaf transports (attach flattens a fan-out argument instead of nesting it), so + # this recurses one level at most. + return all(_transport_is_dead(peer) for peer in transport.transports()) + return getattr(transport, "_closed", None) is True def _session_is_lru_evictable(sid: str, session: dict) -> bool: diff --git a/tui_gateway/transport.py b/tui_gateway/transport.py index 7b67e937ff..26ba3fd12b 100644 --- a/tui_gateway/transport.py +++ b/tui_gateway/transport.py @@ -9,6 +9,8 @@ A :class:`Transport` forwards a JSON-serialisable dict to its peer, so one dispa from __future__ import annotations +import asyncio +import concurrent.futures import contextlib import contextvars import errno @@ -16,6 +18,7 @@ import json import logging import os import threading +import time from typing import Any, Callable, Optional, Protocol, runtime_checkable # Errno values that mean "the peer is gone" rather than "the host has a real I/O problem". Anything @@ -32,6 +35,64 @@ logger = logging.getLogger(__name__) # fully buffered on a pipe, so this ONLY makes sense with ``-u``/``PYTHONUNBUFFERED=1``; otherwise the TUI hangs. _DISABLE_FLUSH = (os.environ.get("HERMES_TUI_GATEWAY_NO_FLUSH", "") or "").strip().lower() in {"1", "true", "yes", "on"} +# Worker pool behind FanoutTransport.write. A non-streaming WebSocket write +# blocks the calling thread while the owning event loop flushes the frame — up +# to ``tui_gateway.ws._WS_WRITE_TIMEOUT_S`` (10s) when that loop is stalled — +# so walking a session's peers one at a time lets one wedged client hold the +# frame back from every healthy client behind it. The pool is created lazily +# and only ever used when a session has more than one peer, so the ordinary +# single-client gateway never starts a thread. +# +# Sizing: these workers are idle except while a peer is actually wedged, so the +# cap only has to cover the wedged peers of all sessions at once. If it were +# ever saturated the excess writes queue and run in submission order, which is +# the pre-pool behaviour — degraded, not broken. +_FANOUT_POOL_MAX_WORKERS = 32 + +# Wall-clock bound on ONE fan-out write, measured from before the first peer is +# dispatched. Deliberately larger than ``tui_gateway.ws._WS_WRITE_TIMEOUT_S`` +# (10.0) so a merely-slow peer reaches its own timeout and reports for itself; +# this deadline only exists so a peer that never returns at all cannot pin the +# emitting thread forever. Not imported from ``tui_gateway.ws``: that module +# imports ``tui_gateway.server``, which imports this one, so the import would +# be circular. Drift is harmless — a peer that misses this deadline is treated +# as in-flight, not as dead (see ``FanoutTransport.write``). +_FANOUT_WRITE_DEADLINE_S = 12.0 + +_fanout_pool: "concurrent.futures.ThreadPoolExecutor | None" = None +_fanout_pool_lock = threading.Lock() + + +def _fanout_write_pool() -> "concurrent.futures.ThreadPoolExecutor": + """The shared fan-out pool, created on first multi-peer write.""" + global _fanout_pool + pool = _fanout_pool + if pool is not None: + return pool + with _fanout_pool_lock: + if _fanout_pool is None: + _fanout_pool = concurrent.futures.ThreadPoolExecutor( + max_workers=_FANOUT_POOL_MAX_WORKERS, + thread_name_prefix="tui-fanout", + ) + return _fanout_pool + + +def _caller_is_on_event_loop() -> bool: + """True when the CALLING thread is running an asyncio event loop. + + Mirrors the probe in ``tui_gateway.ws.WSTransport.write``, which takes a + fire-and-forget path when it can see it is on its own loop and a BLOCKING + ``fut.result`` path when it cannot. ``FanoutTransport`` owns no loop and + its peers may sit on different ones, so the test here is the conservative + one: if this thread is running any loop at all, keep the peer writes on it. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + @runtime_checkable class Transport(Protocol): @@ -115,6 +176,208 @@ class StdioTransport: return None +class FanoutTransport: + """Delivers one JSON frame to every client attached to a session. + + A session's ``transport`` slot used to hold exactly one client, so every + ``prompt.submit`` / ``session.resume`` / ``session.activate`` / queued-prompt + drain REBOUND it and whichever client was there before went silent. This + object goes in the same slot and satisfies the same :class:`Transport` + protocol, so ``server.write_json`` — and every other reader of that slot — + is unchanged; the only difference is that N clients receive the frame + instead of the most recent one. + + Scope: this carries a session's ASYNC EVENT stream only. Request/response + RPCs still answer on the request's context-bound transport + (``dispatch`` → ``current_transport()``), so a client only ever sees replies + to its own calls. + + Peers are pruned on failure: a transport that returns ``False`` (peer gone) + or raises is dropped from the fan-out instead of being written to forever. + Pruning is about dead peers, not slow ones — a client that is merely wedged + is kept, and :meth:`write` runs the peer writes concurrently so its stall + stays its own. + """ + + __slots__ = ("_lock", "_transports") + + def __init__(self, *transports: "Transport") -> None: + self._lock = threading.Lock() + self._transports: list["Transport"] = [] + for transport in transports: + self.attach(transport) + + def attach(self, transport: "Transport") -> bool: + """Add *transport*. Returns ``True`` when it was not already attached.""" + if transport is None or transport is self: + return False + with self._lock: + for existing in self._transports: + if existing is transport: + return False + self._transports.append(transport) + return True + + def detach(self, transport: "Transport") -> bool: + """Remove *transport*. Returns ``True`` when it was attached.""" + with self._lock: + for idx, existing in enumerate(self._transports): + if existing is transport: + del self._transports[idx] + return True + return False + + def contains(self, transport: "Transport") -> bool: + with self._lock: + return any(existing is transport for existing in self._transports) + + def transports(self) -> list["Transport"]: + """A snapshot of the attached transports (safe to iterate).""" + with self._lock: + return list(self._transports) + + def has_transports(self, *, excluding: "Transport | None" = None) -> bool: + """True when any transport other than *excluding* is still attached.""" + with self._lock: + return any(existing is not excluding for existing in self._transports) + + @staticmethod + def _deliver(transport: "Transport", obj: dict, dead: list) -> bool: + """Write *obj* to one peer inline. Records a dead peer in *dead*. + + Returns ``True`` when the peer took the frame. + """ + try: + ok = transport.write(obj) + except Exception: + logger.debug("fanout write failed; pruning peer", exc_info=True) + dead.append(transport) + return False + if ok: + return True + dead.append(transport) + return False + + def write(self, obj: dict) -> bool: + """Deliver *obj* to every attached transport. + + Iterates a SNAPSHOT and never holds the lock across a peer write, so a + peer write can never block an attach or a detach. + + With more than one peer the writes run CONCURRENTLY: every peer but one + is handed to a shared worker pool and the last runs on the calling + thread, which is going to block here anyway. Serial delivery would let + a client wedged for the full WS write timeout hold the frame back from + every healthy client behind it in the list; this way each peer's stall + is its own. Two cases stay inline: + + * exactly one peer, so a single-client session is untouched; + * a caller already running on an event loop. ``WSTransport.write`` + fires and forgets when it sees its own loop and BLOCKS on + ``fut.result`` when it does not, so moving a loop-thread write onto a + pool thread would make that thread wait on the loop it just left + while the loop waits on it — a full write-timeout freeze. + + Pruning is unchanged: a peer that returns ``False`` or raises is + detached, whichever path it took. A peer that has not answered by + ``_FANOUT_WRITE_DEADLINE_S`` is NOT pruned — slow is not dead — and + counts as delivered, which is what ``WSTransport.write`` itself reports + when its own write times out: the frame is queued on that peer's loop + and flushes when the loop breathes. If that peer really is gone, its + next write returns ``False`` promptly and prunes it then. + + The collect happens before the return, so frame A is on every peer that + answered within the deadline before frame B is dispatched to any of + them. A peer whose write was still queued on the pool when the deadline + passed is the exception: the next frame can reach that peer's own + ``_token_lock`` while the previous one is still waiting for it, and the + lock — not this method — decides the order they land in. Within a peer, + ordering is that peer's own business (``WSTransport`` queues under its + token lock, ``StdioTransport`` writes under its stream lock); + ``server.write_json`` takes no lock, so concurrent emitters interleave + here exactly as they already did. + + Returns ``True`` when at least one client accepted the frame or still + has it in flight — an all-dead fan-out reports peer-gone exactly like a + single dead transport would. + """ + targets = self.transports() + if not targets: + return False + + dead: list["Transport"] = [] + delivered = False + + if len(targets) == 1 or _caller_is_on_event_loop(): + for transport in targets: + if self._deliver(transport, obj, dead): + delivered = True + for transport in dead: + self.detach(transport) + return delivered + + deadline = time.monotonic() + _FANOUT_WRITE_DEADLINE_S + pool = _fanout_write_pool() + dispatched: list[tuple["Transport", "concurrent.futures.Future | None"]] = [] + for transport in targets[:-1]: + try: + dispatched.append((transport, pool.submit(transport.write, obj))) + except RuntimeError: + # Pool refused the work (interpreter shutting down). Deliver + # inline below rather than dropping the frame. + dispatched.append((transport, None)) + + # The last peer runs here: one fewer worker per write, and a two-client + # session costs a single pool thread. + if self._deliver(targets[-1], obj, dead): + delivered = True + + # One deadline for the whole batch, then read only the futures that + # finished. Waiting first and calling ``result()`` on a settled future + # keeps our deadline distinguishable from a peer that raised: on 3.11+ + # ``concurrent.futures.TimeoutError`` IS the builtin ``TimeoutError``, + # so a ``result(timeout=...)`` cannot tell the two apart. + futures = [fut for _, fut in dispatched if fut is not None] + if futures: + concurrent.futures.wait( + futures, timeout=max(0.0, deadline - time.monotonic()) + ) + + for transport, fut in dispatched: + if fut is None: + if self._deliver(transport, obj, dead): + delivered = True + continue + if not fut.done(): + # Slow, not dead: leave it attached and count the frame as in + # flight. Cancelling would not stop a write already running. + logger.warning( + "fanout write still pending after %ss; peer left attached", + _FANOUT_WRITE_DEADLINE_S, + ) + delivered = True + continue + try: + ok = fut.result() + except Exception: + logger.debug("fanout write failed; pruning peer", exc_info=True) + dead.append(transport) + continue + if ok: + delivered = True + else: + dead.append(transport) + + for transport in dead: + self.detach(transport) + return delivered + + def close(self) -> None: + """Detach every peer. Does NOT close them — each owns its own socket.""" + with self._lock: + self._transports = [] + + class TeeTransport: """Mirrors writes to one primary plus N best-effort secondaries. The primary's return value (and exceptions) determine the result; secondaries swallow failures so a wedged sidecar never stalls the