fix(relay): WAN-friendly keepalive tuning (ping_interval=30, ping_timeout=60)
Customer gateways cross WAN paths to the connector; the websockets library defaults (ping 20s, pong deadline 20s) close the socket with 1011 keepalive ping timeout under transient latency or event-loop stalls — the trigger of the Coatue 2026-08-18 incident. 60s tolerates stalls while still detecting a genuinely dead link within ~90s worst case. Set explicitly at both connect() call sites (with and without auth headers) so the tuning is visible and pinned by test rather than inherited from library defaults.
This commit is contained in:
@@ -497,10 +497,26 @@ class WebSocketRelayTransport:
|
||||
# normal fast backoff, not the dormant cadence.
|
||||
self._dormant = False
|
||||
headers = self._upgrade_headers()
|
||||
# WAN-friendly keepalive: customer gateways cross WAN paths to the
|
||||
# connector; the websockets library default (ping_interval=20,
|
||||
# ping_timeout=20) gives the peer only a 20s pong deadline, which
|
||||
# produces spurious `1011 keepalive ping timeout` closes under
|
||||
# transient latency / event-loop stalls (Coatue incident 2026-08-18).
|
||||
# ping_timeout=60 tolerates such stalls while still detecting a dead
|
||||
# link within ~90s worst case (30s interval + 60s pong deadline).
|
||||
if headers:
|
||||
self._ws = await websockets.connect(self._url, additional_headers=headers) # type: ignore[union-attr]
|
||||
self._ws = await websockets.connect( # type: ignore[union-attr]
|
||||
self._url,
|
||||
additional_headers=headers,
|
||||
ping_interval=30,
|
||||
ping_timeout=60,
|
||||
)
|
||||
else:
|
||||
self._ws = await websockets.connect(self._url) # type: ignore[union-attr]
|
||||
self._ws = await websockets.connect( # type: ignore[union-attr]
|
||||
self._url,
|
||||
ping_interval=30,
|
||||
ping_timeout=60,
|
||||
)
|
||||
self._reader = asyncio.create_task(self._read_loop(), name="relay-ws-reader")
|
||||
# Send one hello PER fronted identity (Phase 1.5 Shape A). The connector
|
||||
# accumulates them into its advertised set (the first sets the session
|
||||
|
||||
@@ -104,3 +104,51 @@ async def test_send_during_redial_window_fails_fast():
|
||||
await t._supervisor
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_passes_wan_keepalive_tuning(monkeypatch):
|
||||
"""connect() must pass ping_interval=30 / ping_timeout=60 explicitly —
|
||||
the library defaults (20/20) caused spurious 1011 keepalive closes over
|
||||
WAN paths (Coatue 2026-08-18). Both call sites (with/without auth
|
||||
headers) are exercised."""
|
||||
captured: list[dict] = []
|
||||
|
||||
class _IdleWS:
|
||||
async def send(self, data):
|
||||
pass
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
async def _fake_connect(url, **kwargs):
|
||||
captured.append(kwargs)
|
||||
return _IdleWS()
|
||||
|
||||
monkeypatch.setattr(ws_transport_mod.websockets, "connect", _fake_connect)
|
||||
|
||||
# Site 1: no upgrade secret -> the headerless connect() call.
|
||||
t = WebSocketRelayTransport("ws://unused", "discord", "bot1")
|
||||
await t.connect()
|
||||
await t.disconnect(budget_s=0)
|
||||
|
||||
# Site 2: secret + gateway_id -> the additional_headers connect() call.
|
||||
t2 = WebSocketRelayTransport(
|
||||
"ws://unused", "discord", "bot1", gateway_id="gw-1", upgrade_secret="s3cret"
|
||||
)
|
||||
await t2.connect()
|
||||
await t2.disconnect(budget_s=0)
|
||||
|
||||
assert len(captured) == 2
|
||||
no_header_kwargs, header_kwargs = captured
|
||||
assert "additional_headers" not in no_header_kwargs
|
||||
assert "additional_headers" in header_kwargs
|
||||
for kwargs in captured:
|
||||
assert kwargs.get("ping_interval") == 30
|
||||
assert kwargs.get("ping_timeout") == 60
|
||||
|
||||
Reference in New Issue
Block a user