fix(relay): gate sends on the socket handle, not supervisor state

The mid-redial fail-fast guard used 'supervisor task not done' as the
definition of the redial window, and that signal is wrong from both
directions:

- Too narrow: the wedge it fixed also occurs on reader exits that arm
  NO supervisor (terminal 4401 revocation, reconnect=False) — those
  stayed wedged.
- Too broad: _reconnect_loop -> _dial_and_start installs the fresh
  socket and starts its reader, THEN awaits one hello send per fronted
  identity before the supervisor unwinds. Through those awaits the
  transport is fully live, yet the guard rejected every send as
  'reconnecting' — refusing real traffic on a healthy socket.

With the previous commit the reader clears _ws on unexpected exit, so
the existing 'is None' check now covers the entire outage window
honestly: _ws is the single liveness signal. Drop the supervisor-state
guard.

The redial-window test now drives the real sequence (reader exit arms
the supervisor and clears _ws) instead of hand-crafting a stale-_ws
state the transport can no longer reach, and a new test pins the
post-dial window: a send issued while the supervisor is still
unwinding past a live socket must reach that socket (fails with the
guard reinstated).
This commit is contained in:
Ben Barclay
2026-08-20 14:12:33 +10:00
parent 52f9979a6c
commit ca3438d395
2 changed files with 106 additions and 21 deletions
-10
View File
@@ -791,16 +791,6 @@ class WebSocketRelayTransport:
return {"success": False, "error": "relay transport closed"}
if self._ws is None:
return {"success": False, "error": "relay transport not connected"}
if self._supervisor is not None and not self._supervisor.done():
# The reconnect supervisor is mid-redial (backing off after an
# unexpected close). `self._ws` still points at the DEAD socket
# until _dial_and_start() replaces it, so the check above does not
# cover this window — a send here would register a future no
# reader can resolve and block the caller for _outbound_timeout_s
# (Coatue incident 2026-08-18). Fail fast with the dict shape
# callers expect; a live supervisor's dial success ends the task,
# so `not done()` is exactly the redial window.
return {"success": False, "error": "relay transport reconnecting"}
request_id = uuid.uuid4().hex
loop = asyncio.get_running_loop()
fut: asyncio.Future[Dict[str, Any]] = loop.create_future()
@@ -90,26 +90,121 @@ async def test_read_loop_exit_fails_pending_futures_promptly():
@pytest.mark.asyncio
async def test_send_during_redial_window_fails_fast():
"""While the reconnect supervisor is backing off, _ws still points at the
dead socket — a send must return an error dict immediately (no
RuntimeError, no 30s timeout on an unresolvable future)."""
t = WebSocketRelayTransport("ws://unused", "discord", "bot1", outbound_timeout_s=30.0)
t._ws = _DroppingWS() # stale/dead socket left over from before the drop
t._supervisor = asyncio.create_task(asyncio.sleep(60)) # mid-redial backoff
"""While the reconnect supervisor is backing off after a drop, a send must
return an error dict immediately (no RuntimeError, no 30s timeout on an
unresolvable future). Drives the REAL sequence — reader exit arms the
supervisor and clears _ws — rather than hand-crafting a stale-_ws state
the transport can no longer reach."""
t = WebSocketRelayTransport(
"ws://unused",
"discord",
"bot1",
reconnect=True,
reconnect_backoff_s=60.0, # park the supervisor in backoff
outbound_timeout_s=30.0,
)
fake = _DroppingWS()
t._ws = fake
await _run_reader_to_exit(t, fake)
supervisor = t._supervisor
try:
assert supervisor is not None and not supervisor.done(), (
"reader exit must arm the reconnect supervisor"
)
result = await asyncio.wait_for(
t.send_outbound({"op": "send_message", "text": "hi"}), timeout=1.0
)
assert result["success"] is False
assert "reconnect" in result["error"]
assert t._pending == {}
finally:
t._supervisor.cancel()
try:
await t._supervisor
except asyncio.CancelledError:
if supervisor is not None:
supervisor.cancel()
try:
await supervisor
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_send_allowed_once_redial_installs_fresh_socket(monkeypatch):
"""The moment _dial_and_start() installs a fresh socket and its reader,
the transport is genuinely usable — even though the supervisor task has
not finished unwinding (it is still awaiting the hello sends). A send in
that window must be ACCEPTED, not rejected as 'reconnecting': gating
sends on supervisor state rejected real traffic on a live socket."""
class _LiveWS:
def __init__(self):
self.sent: list[str] = []
self.hello_seen = asyncio.Event()
self.release = asyncio.Event()
async def send(self, data):
self.sent.append(data)
if '"hello"' in data:
# Inside _dial_and_start, AFTER _ws and the reader are
# installed. Park here to hold the window open.
self.hello_seen.set()
await self.release.wait()
def __aiter__(self):
return self
async def __anext__(self):
await asyncio.sleep(3600)
async def close(self):
pass
live = _LiveWS()
async def _fake_connect(url, **kwargs):
return live
monkeypatch.setattr(ws_transport_mod.websockets, "connect", _fake_connect)
t = WebSocketRelayTransport(
"ws://unused",
"discord",
"bot1",
reconnect=True,
reconnect_backoff_s=0.01,
outbound_timeout_s=5.0,
)
# Arm the supervisor exactly as the reader's fall-through does.
t._supervisor = asyncio.create_task(t._reconnect_loop())
await asyncio.wait_for(live.hello_seen.wait(), timeout=2.0)
try:
assert t._ws is live and not t._supervisor.done()
send_task = asyncio.create_task(
t.send_outbound({"op": "send_message", "text": "hi"})
)
# The send must reach the live socket (registered + frame written),
# not fail fast: wait for the outbound frame to land.
for _ in range(100):
if any('"outbound"' in s for s in live.sent):
break
await asyncio.sleep(0.01)
assert any('"outbound"' in s for s in live.sent), (
"send was rejected during the post-dial window despite a live "
"socket and running reader"
)
# Resolve it via the reader path shape: answer directly.
rid = next(iter(t._pending))
t._pending[rid].set_result({"success": True})
assert (await asyncio.wait_for(send_task, timeout=2.0)) == {"success": True}
finally:
live.release.set()
await asyncio.wait_for(t._supervisor, timeout=2.0)
if t._reader is not None:
t._reader.cancel()
try:
await t._reader
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_connect_passes_wan_keepalive_tuning(monkeypatch):