fix(relay): WAN-friendly keepalive tuning (ping_interval=30, ping_timeout=60)

Customer gateways cross WAN paths to the connector; the websockets
library defaults (ping 20s, pong deadline 20s) close the socket with
1011 keepalive ping timeout under transient latency or event-loop
stalls — the trigger of the Coatue 2026-08-18 incident. 60s tolerates
stalls while still detecting a genuinely dead link within ~90s worst
case. Set explicitly at both connect() call sites (with and without
auth headers) so the tuning is visible and pinned by test rather than
inherited from library defaults.
This commit is contained in:
Victor Kyriazakos
2026-08-19 00:22:01 +00:00
parent a7946f6502
commit 534f8d4fb0
2 changed files with 66 additions and 2 deletions
+18 -2
View File
@@ -497,10 +497,26 @@ class WebSocketRelayTransport:
# normal fast backoff, not the dormant cadence.
self._dormant = False
headers = self._upgrade_headers()
# WAN-friendly keepalive: customer gateways cross WAN paths to the
# connector; the websockets library default (ping_interval=20,
# ping_timeout=20) gives the peer only a 20s pong deadline, which
# produces spurious `1011 keepalive ping timeout` closes under
# transient latency / event-loop stalls (Coatue incident 2026-08-18).
# ping_timeout=60 tolerates such stalls while still detecting a dead
# link within ~90s worst case (30s interval + 60s pong deadline).
if headers:
self._ws = await websockets.connect(self._url, additional_headers=headers) # type: ignore[union-attr]
self._ws = await websockets.connect( # type: ignore[union-attr]
self._url,
additional_headers=headers,
ping_interval=30,
ping_timeout=60,
)
else:
self._ws = await websockets.connect(self._url) # type: ignore[union-attr]
self._ws = await websockets.connect( # type: ignore[union-attr]
self._url,
ping_interval=30,
ping_timeout=60,
)
self._reader = asyncio.create_task(self._read_loop(), name="relay-ws-reader")
# Send one hello PER fronted identity (Phase 1.5 Shape A). The connector
# accumulates them into its advertised set (the first sets the session
@@ -104,3 +104,51 @@ async def test_send_during_redial_window_fails_fast():
await t._supervisor
except asyncio.CancelledError:
pass
@pytest.mark.asyncio
async def test_connect_passes_wan_keepalive_tuning(monkeypatch):
"""connect() must pass ping_interval=30 / ping_timeout=60 explicitly —
the library defaults (20/20) caused spurious 1011 keepalive closes over
WAN paths (Coatue 2026-08-18). Both call sites (with/without auth
headers) are exercised."""
captured: list[dict] = []
class _IdleWS:
async def send(self, data):
pass
def __aiter__(self):
return self
async def __anext__(self):
await asyncio.sleep(3600)
async def close(self):
pass
async def _fake_connect(url, **kwargs):
captured.append(kwargs)
return _IdleWS()
monkeypatch.setattr(ws_transport_mod.websockets, "connect", _fake_connect)
# Site 1: no upgrade secret -> the headerless connect() call.
t = WebSocketRelayTransport("ws://unused", "discord", "bot1")
await t.connect()
await t.disconnect(budget_s=0)
# Site 2: secret + gateway_id -> the additional_headers connect() call.
t2 = WebSocketRelayTransport(
"ws://unused", "discord", "bot1", gateway_id="gw-1", upgrade_secret="s3cret"
)
await t2.connect()
await t2.disconnect(budget_s=0)
assert len(captured) == 2
no_header_kwargs, header_kwargs = captured
assert "additional_headers" not in no_header_kwargs
assert "additional_headers" in header_kwargs
for kwargs in captured:
assert kwargs.get("ping_interval") == 30
assert kwargs.get("ping_timeout") == 60