fix(buzz): bound WebSocket read idle time to force reconnect on silent relays
A relay-side close the transport never surfaces (observed as a CLOSE_WAIT socket behind Cloudflare, #98097) parks the read loop forever while the gateway keeps reporting connected: inbound stops, gateway_state.json stays healthy, and only a restart recovers. The library keepalive should catch this first, but as a last resort the read side now waits at most _WS_READ_IDLE_TIMEOUT (300s) for a frame before raising into the existing reconnect path, which re-authenticates and re-subscribes with per-channel since filters intact. Fixes #98097
This commit is contained in:
@@ -241,6 +241,13 @@ _PROFILE_NAME_TTL = 300.0
|
||||
# WebSocket transport (NIP-42 authenticated Nostr subscription).
|
||||
# kind 44100 is Buzz's channel-membership event — used for live DM discovery.
|
||||
_WS_AUTH_TIMEOUT = 20.0
|
||||
# Last-resort bound on how long the read loop may wait for a frame. The
|
||||
# library keepalive (ping_interval/ping_timeout below) should catch a dead
|
||||
# relay first, but a relay-side close the transport never surfaces (observed
|
||||
# as a CLOSE_WAIT socket with the loop parked on recv, #98097) leaves the
|
||||
# gateway "connected" while inbound stops; this timeout forces the normal
|
||||
# reconnect path instead.
|
||||
_WS_READ_IDLE_TIMEOUT = 300.0
|
||||
_WS_MAX_MESSAGE_BYTES = 2_000_000
|
||||
_WS_MEMBERSHIP_KIND = 44100
|
||||
_WS_MEMBERSHIP_SUB_ID = "hermes-buzz-membership"
|
||||
@@ -1483,7 +1490,20 @@ class BuzzAdapter(BasePlatformAdapter):
|
||||
if self._ws_ready is not None:
|
||||
self._ws_ready.set()
|
||||
backoff = 1.0
|
||||
async for raw in websocket:
|
||||
frame_iter = websocket.__aiter__()
|
||||
while True:
|
||||
try:
|
||||
raw = await asyncio.wait_for(
|
||||
frame_iter.__anext__(),
|
||||
timeout=_WS_READ_IDLE_TIMEOUT,
|
||||
)
|
||||
except StopAsyncIteration:
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
raise ConnectionError(
|
||||
f"no WebSocket frame for {_WS_READ_IDLE_TIMEOUT:.0f}s; "
|
||||
"assuming the connection went silent"
|
||||
) from None
|
||||
try:
|
||||
message = json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
|
||||
@@ -104,6 +104,134 @@ class _FakeWebSocket:
|
||||
self.sent.append(json.loads(raw))
|
||||
|
||||
|
||||
# ── _websocket_loop: read-idle watchdog (#98097) ──────────────────────────
|
||||
|
||||
|
||||
class _ScriptedWebSocket(_FakeWebSocket):
|
||||
"""A connect() target whose event frames come from a scripted behavior.
|
||||
|
||||
The auth handshake is the relay's (inherited from _FakeWebSocket); after
|
||||
it, each ``__anext__`` delegates to ``anext_behavior`` — a coroutine
|
||||
function returning the next raw frame or raising StopAsyncIteration for
|
||||
a clean close.
|
||||
"""
|
||||
|
||||
def __init__(self, anext_behavior):
|
||||
super().__init__()
|
||||
self._anext_behavior = anext_behavior
|
||||
self.exited = False
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc_info):
|
||||
self.exited = True
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
return await self._anext_behavior()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_loop_reconnects_when_read_goes_silent(monkeypatch, caplog):
|
||||
"""A relay close the transport never surfaces must not park the loop.
|
||||
|
||||
Reproduces the #98097 shape: a socket stuck in CLOSE_WAIT yields no
|
||||
frame and no error, so without a read-side bound the loop would wait
|
||||
forever while the gateway keeps reporting "connected".
|
||||
"""
|
||||
import logging
|
||||
|
||||
adapter = _make_adapter()
|
||||
monkeypatch.setattr(_buzz_mod, "_WS_READ_IDLE_TIMEOUT", 0.05)
|
||||
caplog.set_level(logging.WARNING)
|
||||
|
||||
sockets = []
|
||||
|
||||
async def dead_anext():
|
||||
await asyncio.Event().wait() # never yields, never raises
|
||||
|
||||
def fake_connect(*args, **kwargs):
|
||||
ws = _ScriptedWebSocket(dead_anext)
|
||||
sockets.append(ws)
|
||||
return ws
|
||||
|
||||
import websockets as _ws_mod
|
||||
|
||||
monkeypatch.setattr(_ws_mod, "connect", fake_connect)
|
||||
|
||||
task = asyncio.create_task(adapter._websocket_loop())
|
||||
try:
|
||||
deadline = time.monotonic() + 5.0
|
||||
while len(sockets) < 2 and time.monotonic() < deadline:
|
||||
await asyncio.sleep(0.02)
|
||||
finally:
|
||||
task.cancel()
|
||||
try:
|
||||
await asyncio.wait_for(task, 5.0)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
|
||||
assert len(sockets) >= 2, "idle read watchdog did not force a reconnect"
|
||||
assert sockets[0].exited, "the silent connection was not closed before reconnecting"
|
||||
assert any("went silent" in record.message for record in caplog.records)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_loop_dispatches_frames_and_closes_cleanly(monkeypatch):
|
||||
"""The watchdog refactor preserves the healthy path: frames dispatch to
|
||||
_handle_event and a server-side close (StopAsyncIteration) exits the
|
||||
connection cleanly before the loop reconnects."""
|
||||
adapter = _make_adapter()
|
||||
adapter._channel_state = {CHANNEL: {"last_ts": 1, "seen": {}}}
|
||||
|
||||
handled = []
|
||||
|
||||
async def record_handle_event(channel_id, state, event):
|
||||
handled.append((channel_id, event))
|
||||
|
||||
monkeypatch.setattr(adapter, "_handle_event", record_handle_event)
|
||||
|
||||
frames = iter(
|
||||
[
|
||||
json.dumps(
|
||||
["EVENT", "hermes-buzz-0", {"id": "e1", "kind": 9, "created_at": 2, "content": "hi"}]
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
async def scripted_anext():
|
||||
try:
|
||||
return next(frames)
|
||||
except StopIteration:
|
||||
raise StopAsyncIteration from None
|
||||
|
||||
sockets = []
|
||||
|
||||
def fake_connect(*args, **kwargs):
|
||||
# Second connect ends the loop: CancelledError re-raises out of the
|
||||
# loop's except-order, unlike a regular Exception which would retry.
|
||||
if len(sockets) == 1:
|
||||
raise asyncio.CancelledError()
|
||||
ws = _ScriptedWebSocket(scripted_anext)
|
||||
sockets.append(ws)
|
||||
return ws
|
||||
|
||||
import websockets as _ws_mod
|
||||
|
||||
monkeypatch.setattr(_ws_mod, "connect", fake_connect)
|
||||
|
||||
task = asyncio.create_task(adapter._websocket_loop())
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await asyncio.wait_for(task, 10.0)
|
||||
|
||||
assert sockets[0].exited, "clean close did not exit the async-with block"
|
||||
assert handled and handled[0][0] == CHANNEL
|
||||
assert handled[0][1]["id"] == "e1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_auth_raises_on_rejection():
|
||||
adapter = _make_adapter()
|
||||
|
||||
Reference in New Issue
Block a user