fix(server): move blocking credential-pool calls off the event loop

Network off (unplugged) froze the backend 17 minutes: the async
/api/credentials/pool endpoints (GET/POST/DELETE) called load_pool()
synchronously on the event-loop thread -> Copilot token exchange ->
blocking urlopen -> getaddrinfo stuck in C for 1016s, immune to
urlopen(timeout=10). WS dropped (1006), sessions detached, even log
writes stalled.

Fix 1 (web_server.py): move the three endpoint bodies into
asyncio.to_thread, matching the file's existing _run pattern.

Fix 2 (copilot_auth.py): _urlopen_bounded() runs the request in a
daemon thread with a wall-clock hard cap (timeout+5s) so DNS hangs
can no longer block any caller indefinitely.

Measured: simulated DNS hang now raises TimeoutError after 6.0s
instead of freezing the loop.
This commit is contained in:
peetteerr
2026-08-22 07:42:45 +08:00
committed by kshitij
parent 2b4e70ec07
commit ff0afff0e4
2 changed files with 154 additions and 90 deletions
+44 -1
View File
@@ -23,6 +23,7 @@ import logging
import os
import shutil
import subprocess
import threading
import time
from pathlib import Path
from typing import Optional
@@ -520,6 +521,48 @@ def _save_jwt_to_disk(
logger.debug("Failed to persist Copilot JWT: %s", exc)
# Hard wall-clock cap for the token-exchange HTTP call. urllib's ``timeout``
# only bounds socket operations AFTER DNS resolution succeeds; getaddrinfo
# blocks in C and ignores it entirely, so on a networkless Windows host the
# resolver can hang for many minutes (observed: a 17-minute event-loop stall
# on 2026-08-22 that took the whole backend down with it).
_DNS_GRACE_SECONDS = 5.0
def _urlopen_bounded(req, timeout: float):
"""urlopen() with a hard wall-clock cap of timeout + _DNS_GRACE_SECONDS.
Runs the call on a daemon thread and abandons it if the cap fires, so a
DNS/getaddrinfo hang cannot block the caller indefinitely. Raises the
worker's exception, or TimeoutError when the cap fires.
"""
import urllib.request
box: dict = {}
def _worker() -> None:
try:
box["resp"] = urllib.request.urlopen(req, timeout=timeout)
except BaseException as exc: # re-raised on the caller's thread
box["exc"] = exc
t = threading.Thread(
target=_worker, name="copilot-token-exchange", daemon=True
)
t.start()
t.join(timeout + _DNS_GRACE_SECONDS)
if t.is_alive():
raise TimeoutError(
"copilot token exchange exceeded hard cap of "
f"{timeout + _DNS_GRACE_SECONDS:.0f}s (DNS/getaddrinfo hang?)"
)
if "exc" in box:
raise box["exc"]
if "resp" not in box:
raise TimeoutError("copilot token exchange worker died without result")
return box["resp"]
def exchange_copilot_token(raw_token: str, *, timeout: float = 10.0) -> tuple[str, float, Optional[str]]:
"""Exchange a raw GitHub token for a short-lived Copilot API token.
@@ -593,7 +636,7 @@ def exchange_copilot_token(raw_token: str, *, timeout: float = 10.0) -> tuple[st
permanent_failure = False
for attempt in range(_EXCHANGE_MAX_ATTEMPTS):
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
with _urlopen_bounded(req, timeout) as resp:
data = json.loads(resp.read().decode())
break
except Exception as exc: # noqa: BLE001 — retry all, re-raise below