fix(relay): gate sends on the socket handle, not supervisor state
The mid-redial fail-fast guard used 'supervisor task not done' as the definition of the redial window, and that signal is wrong from both directions: - Too narrow: the wedge it fixed also occurs on reader exits that arm NO supervisor (terminal 4401 revocation, reconnect=False) — those stayed wedged. - Too broad: _reconnect_loop -> _dial_and_start installs the fresh socket and starts its reader, THEN awaits one hello send per fronted identity before the supervisor unwinds. Through those awaits the transport is fully live, yet the guard rejected every send as 'reconnecting' — refusing real traffic on a healthy socket. With the previous commit the reader clears _ws on unexpected exit, so the existing 'is None' check now covers the entire outage window honestly: _ws is the single liveness signal. Drop the supervisor-state guard. The redial-window test now drives the real sequence (reader exit arms the supervisor and clears _ws) instead of hand-crafting a stale-_ws state the transport can no longer reach, and a new test pins the post-dial window: a send issued while the supervisor is still unwinding past a live socket must reach that socket (fails with the guard reinstated).
This commit is contained in:
@@ -791,16 +791,6 @@ class WebSocketRelayTransport:
|
||||
return {"success": False, "error": "relay transport closed"}
|
||||
if self._ws is None:
|
||||
return {"success": False, "error": "relay transport not connected"}
|
||||
if self._supervisor is not None and not self._supervisor.done():
|
||||
# The reconnect supervisor is mid-redial (backing off after an
|
||||
# unexpected close). `self._ws` still points at the DEAD socket
|
||||
# until _dial_and_start() replaces it, so the check above does not
|
||||
# cover this window — a send here would register a future no
|
||||
# reader can resolve and block the caller for _outbound_timeout_s
|
||||
# (Coatue incident 2026-08-18). Fail fast with the dict shape
|
||||
# callers expect; a live supervisor's dial success ends the task,
|
||||
# so `not done()` is exactly the redial window.
|
||||
return {"success": False, "error": "relay transport reconnecting"}
|
||||
request_id = uuid.uuid4().hex
|
||||
loop = asyncio.get_running_loop()
|
||||
fut: asyncio.Future[Dict[str, Any]] = loop.create_future()
|
||||
|
||||
@@ -90,26 +90,121 @@ async def test_read_loop_exit_fails_pending_futures_promptly():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_during_redial_window_fails_fast():
|
||||
"""While the reconnect supervisor is backing off, _ws still points at the
|
||||
dead socket — a send must return an error dict immediately (no
|
||||
RuntimeError, no 30s timeout on an unresolvable future)."""
|
||||
t = WebSocketRelayTransport("ws://unused", "discord", "bot1", outbound_timeout_s=30.0)
|
||||
t._ws = _DroppingWS() # stale/dead socket left over from before the drop
|
||||
t._supervisor = asyncio.create_task(asyncio.sleep(60)) # mid-redial backoff
|
||||
"""While the reconnect supervisor is backing off after a drop, a send must
|
||||
return an error dict immediately (no RuntimeError, no 30s timeout on an
|
||||
unresolvable future). Drives the REAL sequence — reader exit arms the
|
||||
supervisor and clears _ws — rather than hand-crafting a stale-_ws state
|
||||
the transport can no longer reach."""
|
||||
t = WebSocketRelayTransport(
|
||||
"ws://unused",
|
||||
"discord",
|
||||
"bot1",
|
||||
reconnect=True,
|
||||
reconnect_backoff_s=60.0, # park the supervisor in backoff
|
||||
outbound_timeout_s=30.0,
|
||||
)
|
||||
fake = _DroppingWS()
|
||||
t._ws = fake
|
||||
await _run_reader_to_exit(t, fake)
|
||||
supervisor = t._supervisor
|
||||
try:
|
||||
assert supervisor is not None and not supervisor.done(), (
|
||||
"reader exit must arm the reconnect supervisor"
|
||||
)
|
||||
result = await asyncio.wait_for(
|
||||
t.send_outbound({"op": "send_message", "text": "hi"}), timeout=1.0
|
||||
)
|
||||
assert result["success"] is False
|
||||
assert "reconnect" in result["error"]
|
||||
assert t._pending == {}
|
||||
finally:
|
||||
t._supervisor.cancel()
|
||||
try:
|
||||
await t._supervisor
|
||||
except asyncio.CancelledError:
|
||||
if supervisor is not None:
|
||||
supervisor.cancel()
|
||||
try:
|
||||
await supervisor
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_allowed_once_redial_installs_fresh_socket(monkeypatch):
|
||||
"""The moment _dial_and_start() installs a fresh socket and its reader,
|
||||
the transport is genuinely usable — even though the supervisor task has
|
||||
not finished unwinding (it is still awaiting the hello sends). A send in
|
||||
that window must be ACCEPTED, not rejected as 'reconnecting': gating
|
||||
sends on supervisor state rejected real traffic on a live socket."""
|
||||
|
||||
class _LiveWS:
|
||||
def __init__(self):
|
||||
self.sent: list[str] = []
|
||||
self.hello_seen = asyncio.Event()
|
||||
self.release = asyncio.Event()
|
||||
|
||||
async def send(self, data):
|
||||
self.sent.append(data)
|
||||
if '"hello"' in data:
|
||||
# Inside _dial_and_start, AFTER _ws and the reader are
|
||||
# installed. Park here to hold the window open.
|
||||
self.hello_seen.set()
|
||||
await self.release.wait()
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
live = _LiveWS()
|
||||
|
||||
async def _fake_connect(url, **kwargs):
|
||||
return live
|
||||
|
||||
monkeypatch.setattr(ws_transport_mod.websockets, "connect", _fake_connect)
|
||||
|
||||
t = WebSocketRelayTransport(
|
||||
"ws://unused",
|
||||
"discord",
|
||||
"bot1",
|
||||
reconnect=True,
|
||||
reconnect_backoff_s=0.01,
|
||||
outbound_timeout_s=5.0,
|
||||
)
|
||||
# Arm the supervisor exactly as the reader's fall-through does.
|
||||
t._supervisor = asyncio.create_task(t._reconnect_loop())
|
||||
await asyncio.wait_for(live.hello_seen.wait(), timeout=2.0)
|
||||
try:
|
||||
assert t._ws is live and not t._supervisor.done()
|
||||
|
||||
send_task = asyncio.create_task(
|
||||
t.send_outbound({"op": "send_message", "text": "hi"})
|
||||
)
|
||||
# The send must reach the live socket (registered + frame written),
|
||||
# not fail fast: wait for the outbound frame to land.
|
||||
for _ in range(100):
|
||||
if any('"outbound"' in s for s in live.sent):
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
assert any('"outbound"' in s for s in live.sent), (
|
||||
"send was rejected during the post-dial window despite a live "
|
||||
"socket and running reader"
|
||||
)
|
||||
|
||||
# Resolve it via the reader path shape: answer directly.
|
||||
rid = next(iter(t._pending))
|
||||
t._pending[rid].set_result({"success": True})
|
||||
assert (await asyncio.wait_for(send_task, timeout=2.0)) == {"success": True}
|
||||
finally:
|
||||
live.release.set()
|
||||
await asyncio.wait_for(t._supervisor, timeout=2.0)
|
||||
if t._reader is not None:
|
||||
t._reader.cancel()
|
||||
try:
|
||||
await t._reader
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_passes_wan_keepalive_tuning(monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user