fix(agent): race Codex IPv6 and IPv4 connections
- Add RFC 8305-style staggered address attempts for synchronous ChatGPT Codex requests. - Share the keepalive client builder across primary and auxiliary model paths. - Cover blackholed IPv6 fallback, provider scoping, and existing proxy and TLS behavior.
This commit is contained in:
+233
-4
@@ -15,6 +15,9 @@ Three concerns, all tied to ``AIAgent`` boot-time / runtime IO setup:
|
||||
3. **HTTP proxy resolution** — ``_get_proxy_from_env`` reads
|
||||
``HTTPS_PROXY`` / ``HTTP_PROXY`` / ``ALL_PROXY``;
|
||||
``_get_proxy_for_base_url`` respects ``NO_PROXY`` for the given base URL.
|
||||
4. **Codex dual-stack resilience** — the synchronous ChatGPT/Codex transport
|
||||
races resolved IPv6/IPv4 addresses so a blackholed family cannot exhaust
|
||||
the request watchdog before a working address is attempted.
|
||||
|
||||
``run_agent`` re-exports every name so existing
|
||||
``from run_agent import _get_proxy_from_env`` imports keep working
|
||||
@@ -23,8 +26,12 @@ unchanged.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import os
|
||||
import selectors
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any, Optional
|
||||
|
||||
@@ -34,6 +41,226 @@ from utils import base_url_hostname, normalize_proxy_url
|
||||
# Cached at module level so we only pay the OpenAI SDK import cost once
|
||||
# per process (after the first lazy load).
|
||||
_OPENAI_CLS_CACHE = None
|
||||
_HAPPY_EYEBALLS_DELAY_SECONDS = 0.25
|
||||
|
||||
|
||||
def _interleave_addrinfos(addrinfos: list[tuple]) -> list[tuple]:
|
||||
"""Interleave resolved address families while preserving resolver order."""
|
||||
queues: dict[int, list[tuple]] = {}
|
||||
family_order: list[int] = []
|
||||
seen: set[tuple] = set()
|
||||
for addrinfo in addrinfos:
|
||||
family, socktype, proto, _canonname, sockaddr = addrinfo
|
||||
marker = (family, socktype, proto, sockaddr)
|
||||
if marker in seen:
|
||||
continue
|
||||
seen.add(marker)
|
||||
if family not in queues:
|
||||
queues[family] = []
|
||||
family_order.append(family)
|
||||
queues[family].append(addrinfo)
|
||||
|
||||
interleaved: list[tuple] = []
|
||||
while any(queues.values()):
|
||||
for family in family_order:
|
||||
if queues[family]:
|
||||
interleaved.append(queues[family].pop(0))
|
||||
return interleaved
|
||||
|
||||
|
||||
def _happy_eyeballs_create_connection(
|
||||
address: tuple[str, int],
|
||||
timeout: Optional[float],
|
||||
source_address: Optional[tuple[str, int]] = None,
|
||||
socket_options=(),
|
||||
):
|
||||
"""Connect using staggered non-blocking attempts across resolved families.
|
||||
|
||||
``socket.create_connection`` tries every address serially. A host with
|
||||
broken-but-advertised IPv6 can therefore consume the full connect timeout
|
||||
for each AAAA record before trying a working IPv4 address. This follows the
|
||||
Happy Eyeballs shape from RFC 8305: retain resolver preference, interleave
|
||||
families, and start the next candidate after a short delay.
|
||||
"""
|
||||
host, port = address
|
||||
addrinfos = _interleave_addrinfos(
|
||||
socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||
)
|
||||
if not addrinfos:
|
||||
raise OSError(f"getaddrinfo returned no addresses for {host}")
|
||||
|
||||
selector = selectors.DefaultSelector()
|
||||
active: set[socket.socket] = set()
|
||||
winner = None
|
||||
last_error: Optional[OSError] = None
|
||||
deadline = None if timeout is None else time.monotonic() + max(timeout, 0.0)
|
||||
next_launch = time.monotonic()
|
||||
pending = list(addrinfos)
|
||||
in_progress = {
|
||||
0,
|
||||
errno.EINPROGRESS,
|
||||
errno.EWOULDBLOCK,
|
||||
errno.EALREADY,
|
||||
errno.EINTR,
|
||||
getattr(errno, "WSAEWOULDBLOCK", 10035),
|
||||
}
|
||||
|
||||
def start_attempt(addrinfo):
|
||||
family, socktype, proto, _canonname, sockaddr = addrinfo
|
||||
candidate = socket.socket(family, socktype, proto)
|
||||
try:
|
||||
if source_address is not None:
|
||||
local_infos = socket.getaddrinfo(
|
||||
source_address[0],
|
||||
source_address[1],
|
||||
family=family,
|
||||
type=socktype,
|
||||
)
|
||||
if not local_infos:
|
||||
raise OSError(
|
||||
f"getaddrinfo returned no local {family} address for "
|
||||
f"{source_address[0]}"
|
||||
)
|
||||
candidate.bind(local_infos[0][4])
|
||||
candidate.setblocking(False)
|
||||
result = candidate.connect_ex(sockaddr)
|
||||
if result == 0 or result == errno.EISCONN:
|
||||
return candidate
|
||||
if result not in in_progress:
|
||||
raise OSError(result, os.strerror(result))
|
||||
selector.register(candidate, selectors.EVENT_WRITE)
|
||||
active.add(candidate)
|
||||
return None
|
||||
except Exception:
|
||||
candidate.close()
|
||||
raise
|
||||
|
||||
try:
|
||||
while pending or active:
|
||||
now = time.monotonic()
|
||||
if deadline is not None and now >= deadline:
|
||||
raise socket.timeout("timed out")
|
||||
|
||||
if pending and now >= next_launch:
|
||||
addrinfo = pending.pop(0)
|
||||
try:
|
||||
winner = start_attempt(addrinfo)
|
||||
except OSError as exc:
|
||||
last_error = exc
|
||||
if not active:
|
||||
next_launch = now
|
||||
continue
|
||||
if winner is not None:
|
||||
break
|
||||
next_launch = now + _HAPPY_EYEBALLS_DELAY_SECONDS
|
||||
|
||||
wait_timeout = None if deadline is None else max(0.0, deadline - now)
|
||||
if pending:
|
||||
until_launch = max(0.0, next_launch - now)
|
||||
wait_timeout = (
|
||||
until_launch
|
||||
if wait_timeout is None
|
||||
else min(wait_timeout, until_launch)
|
||||
)
|
||||
|
||||
events = selector.select(wait_timeout)
|
||||
for key, _mask in events:
|
||||
candidate = key.fileobj
|
||||
error_code = candidate.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
|
||||
selector.unregister(candidate)
|
||||
active.discard(candidate)
|
||||
if error_code == 0:
|
||||
winner = candidate
|
||||
break
|
||||
candidate.close()
|
||||
last_error = OSError(error_code, os.strerror(error_code))
|
||||
if winner is not None:
|
||||
break
|
||||
if not active and pending:
|
||||
next_launch = time.monotonic()
|
||||
|
||||
if winner is None:
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
raise OSError(f"Could not connect to {host}:{port}")
|
||||
|
||||
try:
|
||||
selector.unregister(winner)
|
||||
except Exception:
|
||||
pass
|
||||
active.discard(winner)
|
||||
winner.settimeout(timeout)
|
||||
for option in socket_options or ():
|
||||
winner.setsockopt(*option)
|
||||
winner.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
return winner
|
||||
finally:
|
||||
for candidate in active:
|
||||
try:
|
||||
selector.unregister(candidate)
|
||||
except Exception:
|
||||
pass
|
||||
candidate.close()
|
||||
selector.close()
|
||||
|
||||
|
||||
class _HappyEyeballsSyncBackend:
|
||||
"""httpcore sync backend with concurrent IPv6/IPv4 connection fallback."""
|
||||
|
||||
def __init__(self):
|
||||
self._fallback = None
|
||||
|
||||
def _default_backend(self):
|
||||
if self._fallback is None:
|
||||
from httpcore import SyncBackend
|
||||
|
||||
self._fallback = SyncBackend()
|
||||
return self._fallback
|
||||
|
||||
def connect_tcp(
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
timeout: Optional[float] = None,
|
||||
local_address: Optional[str] = None,
|
||||
socket_options=None,
|
||||
):
|
||||
from httpcore import ConnectError, ConnectTimeout
|
||||
from httpcore._backends.sync import SyncStream
|
||||
|
||||
source_address = None if local_address is None else (local_address, 0)
|
||||
try:
|
||||
sock = _happy_eyeballs_create_connection(
|
||||
(host, port),
|
||||
timeout,
|
||||
source_address=source_address,
|
||||
socket_options=socket_options or (),
|
||||
)
|
||||
except socket.timeout as exc:
|
||||
raise ConnectTimeout(str(exc)) from exc
|
||||
except OSError as exc:
|
||||
raise ConnectError(str(exc)) from exc
|
||||
return SyncStream(sock)
|
||||
|
||||
def connect_unix_socket(self, *args, **kwargs):
|
||||
return self._default_backend().connect_unix_socket(*args, **kwargs)
|
||||
|
||||
def sleep(self, seconds: float) -> None:
|
||||
self._default_backend().sleep(seconds)
|
||||
|
||||
|
||||
def _uses_codex_cloud_transport(base_url: str) -> bool:
|
||||
return (
|
||||
base_url_hostname(base_url).lower() == "chatgpt.com"
|
||||
and "/backend-api/codex" in str(base_url).lower()
|
||||
)
|
||||
|
||||
|
||||
def _enable_happy_eyeballs(transport) -> None:
|
||||
"""Install the sync racing backend on one httpx transport, if compatible."""
|
||||
pool = getattr(transport, "_pool", None)
|
||||
if pool is not None and hasattr(pool, "_network_backend"):
|
||||
pool._network_backend = _HappyEyeballsSyncBackend()
|
||||
|
||||
|
||||
def _load_openai_cls() -> type:
|
||||
@@ -186,10 +413,12 @@ def build_keepalive_http_client(
|
||||
client_cls = httpx.AsyncClient if async_mode else httpx.Client
|
||||
mounts = {}
|
||||
if proxy is None:
|
||||
mounts = {
|
||||
"http://": transport_cls(verify=verify),
|
||||
"https://": transport_cls(verify=verify),
|
||||
}
|
||||
http_transport = transport_cls(verify=verify)
|
||||
https_transport = transport_cls(verify=verify)
|
||||
if not async_mode and _uses_codex_cloud_transport(base_url):
|
||||
_enable_happy_eyeballs(http_transport)
|
||||
_enable_happy_eyeballs(https_transport)
|
||||
mounts = {"http://": http_transport, "https://": https_transport}
|
||||
return client_cls(
|
||||
limits=limits,
|
||||
timeout=timeout,
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
nateEc
|
||||
+3
-66
@@ -5170,73 +5170,10 @@ class AIAgent:
|
||||
|
||||
@staticmethod
|
||||
def _build_keepalive_http_client(base_url: str = "", *, verify: Any = True) -> Any:
|
||||
"""Build an httpx.Client with proactive idle-connection reaping.
|
||||
"""Build the shared OpenAI httpx client used by main and aux paths."""
|
||||
from agent.process_bootstrap import build_keepalive_http_client
|
||||
|
||||
Previously this method injected a custom ``httpx.HTTPTransport``
|
||||
with ``socket_options`` (``SO_KEEPALIVE``, ``TCP_KEEPIDLE``, …) to
|
||||
prevent CLOSE-WAIT accumulation on long-lived connections (#10324).
|
||||
|
||||
That approach broke streaming for providers behind reverse proxies
|
||||
(OpenResty, Cloudflare, etc.) because the custom socket options
|
||||
conflict with the proxy's chunked-transfer handling (#54049,
|
||||
#12952). It also stripped ``TCP_NODELAY``, stalling TLS handshakes
|
||||
and SSE encoding.
|
||||
|
||||
The fix moves connection lifecycle management from the socket layer
|
||||
to the HTTP pool layer: ``keepalive_expiry=20.0`` tells httpx to
|
||||
close idle pooled connections *before* a reverse proxy's typical
|
||||
30–60 s timeout drops them, preventing CLOSE-WAIT accumulation
|
||||
without modifying socket options. The default httpx transport
|
||||
preserves OS TCP defaults (including ``TCP_NODELAY``).
|
||||
|
||||
``verify`` carries per-provider ``ssl_ca_cert`` / ``ssl_verify`` and
|
||||
``HERMES_CA_BUNDLE`` settings. It is passed on the client AND on
|
||||
the plain no-proxy mounts (a mounted transport owns the SSL context
|
||||
for its scheme).
|
||||
"""
|
||||
try:
|
||||
import httpx as _httpx
|
||||
|
||||
# Explicitly read proxy settings so requests route through
|
||||
# HTTP_PROXY / HTTPS_PROXY / NO_PROXY correctly.
|
||||
_proxy = _get_proxy_for_base_url(base_url)
|
||||
|
||||
# Proactive pool reaping: close idle connections at 20 s,
|
||||
# before reverse proxies (30–60 s typical) send FIN and
|
||||
# cause CLOSE-WAIT accumulation.
|
||||
_limits = _httpx.Limits(
|
||||
max_keepalive_connections=20,
|
||||
max_connections=100,
|
||||
keepalive_expiry=20.0,
|
||||
)
|
||||
|
||||
# Timeouts: generous read=None for SSE streaming endpoints.
|
||||
_timeout = _httpx.Timeout(
|
||||
connect=15.0,
|
||||
read=None,
|
||||
write=15.0,
|
||||
pool=10.0,
|
||||
)
|
||||
|
||||
# When _proxy is None (NO_PROXY bypass or no proxy configured),
|
||||
# mount plain transports to prevent httpx from reading env proxy
|
||||
# vars and creating an HTTPProxy mount that would bypass our
|
||||
# NO_PROXY resolution.
|
||||
_mounts = {}
|
||||
if _proxy is None:
|
||||
_mounts = {
|
||||
"http://": _httpx.HTTPTransport(verify=verify),
|
||||
"https://": _httpx.HTTPTransport(verify=verify),
|
||||
}
|
||||
return _httpx.Client(
|
||||
limits=_limits,
|
||||
timeout=_timeout,
|
||||
proxy=_proxy,
|
||||
mounts=_mounts or None,
|
||||
verify=verify,
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
return build_keepalive_http_client(base_url, verify=verify)
|
||||
|
||||
def _create_openai_client(self, client_kwargs: dict, *, reason: str, shared: bool) -> Any:
|
||||
"""Forwarder — see ``agent.agent_runtime_helpers.create_openai_client``."""
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import errno
|
||||
import selectors
|
||||
import socket
|
||||
|
||||
import httpcore
|
||||
import pytest
|
||||
|
||||
from agent import process_bootstrap
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_proxy_env(monkeypatch):
|
||||
for name in (
|
||||
"HTTPS_PROXY",
|
||||
"HTTP_PROXY",
|
||||
"ALL_PROXY",
|
||||
"https_proxy",
|
||||
"http_proxy",
|
||||
"all_proxy",
|
||||
"NO_PROXY",
|
||||
"no_proxy",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
|
||||
def _client_backends(client):
|
||||
transports = [client._transport, *client._mounts.values()]
|
||||
return [
|
||||
transport._pool._network_backend
|
||||
for transport in transports
|
||||
if transport is not None and hasattr(transport, "_pool")
|
||||
]
|
||||
|
||||
|
||||
def test_codex_sync_client_uses_happy_eyeballs_backend(no_proxy_env):
|
||||
from run_agent import AIAgent
|
||||
|
||||
client = AIAgent._build_keepalive_http_client(
|
||||
"https://chatgpt.com/backend-api/codex"
|
||||
)
|
||||
try:
|
||||
assert any(
|
||||
isinstance(backend, process_bootstrap._HappyEyeballsSyncBackend)
|
||||
for backend in _client_backends(client)
|
||||
)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
def test_other_sync_clients_keep_httpcore_default_backend(no_proxy_env):
|
||||
client = process_bootstrap.build_keepalive_http_client(
|
||||
"https://api.openai.com/v1"
|
||||
)
|
||||
try:
|
||||
assert all(
|
||||
isinstance(backend, httpcore.SyncBackend)
|
||||
for backend in _client_backends(client)
|
||||
)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
def test_connection_staggers_past_blackholed_ipv6(monkeypatch):
|
||||
clock = [0.0]
|
||||
sockets = []
|
||||
|
||||
class FakeSocket:
|
||||
def __init__(self, family, socktype, proto):
|
||||
self.family = family
|
||||
self.closed = False
|
||||
self.timeout = None
|
||||
sockets.append(self)
|
||||
|
||||
def setsockopt(self, *_args):
|
||||
pass
|
||||
|
||||
def setblocking(self, _blocking):
|
||||
pass
|
||||
|
||||
def settimeout(self, timeout):
|
||||
self.timeout = timeout
|
||||
|
||||
def bind(self, _address):
|
||||
pass
|
||||
|
||||
def connect_ex(self, _address):
|
||||
if self.family == socket.AF_INET6:
|
||||
return errno.EINPROGRESS
|
||||
return 0
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
class FakeSelector:
|
||||
def __init__(self):
|
||||
self.registered = set()
|
||||
|
||||
def register(self, fileobj, _events):
|
||||
self.registered.add(fileobj)
|
||||
|
||||
def unregister(self, fileobj):
|
||||
self.registered.discard(fileobj)
|
||||
|
||||
def select(self, timeout):
|
||||
clock[0] += timeout or 0.0
|
||||
return []
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(
|
||||
process_bootstrap.socket,
|
||||
"getaddrinfo",
|
||||
lambda *_args, **_kwargs: [
|
||||
(
|
||||
socket.AF_INET6,
|
||||
socket.SOCK_STREAM,
|
||||
socket.IPPROTO_TCP,
|
||||
"",
|
||||
("2001:db8::1", 443, 0, 0),
|
||||
),
|
||||
(
|
||||
socket.AF_INET,
|
||||
socket.SOCK_STREAM,
|
||||
socket.IPPROTO_TCP,
|
||||
"",
|
||||
("192.0.2.1", 443),
|
||||
),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(process_bootstrap.socket, "socket", FakeSocket)
|
||||
monkeypatch.setattr(
|
||||
process_bootstrap.selectors, "DefaultSelector", FakeSelector
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
process_bootstrap.time, "monotonic", lambda: clock[0]
|
||||
)
|
||||
|
||||
winner = process_bootstrap._happy_eyeballs_create_connection(
|
||||
("chatgpt.com", 443),
|
||||
timeout=10.0,
|
||||
)
|
||||
|
||||
assert winner.family == socket.AF_INET
|
||||
assert winner.timeout == 10.0
|
||||
assert clock[0] == process_bootstrap._HAPPY_EYEBALLS_DELAY_SECONDS
|
||||
assert sockets[0].closed is True
|
||||
assert sockets[1].closed is False
|
||||
Reference in New Issue
Block a user