fix(agent): race Codex IPv6 and IPv4 connections

- Add RFC 8305-style staggered address attempts for synchronous ChatGPT Codex requests.
- Share the keepalive client builder across primary and auxiliary model paths.
- Cover blackholed IPv6 fallback, provider scoping, and existing proxy and TLS behavior.
This commit is contained in:
Nathan Shan
2026-07-23 17:17:44 +08:00
committed by Teknium
parent bf8b28f27a
commit d934bbd4d5
4 changed files with 385 additions and 70 deletions
+233 -4
View File
@@ -15,6 +15,9 @@ Three concerns, all tied to ``AIAgent`` boot-time / runtime IO setup:
3. **HTTP proxy resolution** — ``_get_proxy_from_env`` reads
``HTTPS_PROXY`` / ``HTTP_PROXY`` / ``ALL_PROXY``;
``_get_proxy_for_base_url`` respects ``NO_PROXY`` for the given base URL.
4. **Codex dual-stack resilience** — the synchronous ChatGPT/Codex transport
races resolved IPv6/IPv4 addresses so a blackholed family cannot exhaust
the request watchdog before a working address is attempted.
``run_agent`` re-exports every name so existing
``from run_agent import _get_proxy_from_env`` imports keep working
@@ -23,8 +26,12 @@ unchanged.
from __future__ import annotations
import errno
import os
import selectors
import socket
import sys
import time
import urllib.request
from typing import Any, Optional
@@ -34,6 +41,226 @@ from utils import base_url_hostname, normalize_proxy_url
# Cached at module level so we only pay the OpenAI SDK import cost once
# per process (after the first lazy load).
_OPENAI_CLS_CACHE = None
_HAPPY_EYEBALLS_DELAY_SECONDS = 0.25
def _interleave_addrinfos(addrinfos: list[tuple]) -> list[tuple]:
"""Interleave resolved address families while preserving resolver order."""
queues: dict[int, list[tuple]] = {}
family_order: list[int] = []
seen: set[tuple] = set()
for addrinfo in addrinfos:
family, socktype, proto, _canonname, sockaddr = addrinfo
marker = (family, socktype, proto, sockaddr)
if marker in seen:
continue
seen.add(marker)
if family not in queues:
queues[family] = []
family_order.append(family)
queues[family].append(addrinfo)
interleaved: list[tuple] = []
while any(queues.values()):
for family in family_order:
if queues[family]:
interleaved.append(queues[family].pop(0))
return interleaved
def _happy_eyeballs_create_connection(
address: tuple[str, int],
timeout: Optional[float],
source_address: Optional[tuple[str, int]] = None,
socket_options=(),
):
"""Connect using staggered non-blocking attempts across resolved families.
``socket.create_connection`` tries every address serially. A host with
broken-but-advertised IPv6 can therefore consume the full connect timeout
for each AAAA record before trying a working IPv4 address. This follows the
Happy Eyeballs shape from RFC 8305: retain resolver preference, interleave
families, and start the next candidate after a short delay.
"""
host, port = address
addrinfos = _interleave_addrinfos(
socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
)
if not addrinfos:
raise OSError(f"getaddrinfo returned no addresses for {host}")
selector = selectors.DefaultSelector()
active: set[socket.socket] = set()
winner = None
last_error: Optional[OSError] = None
deadline = None if timeout is None else time.monotonic() + max(timeout, 0.0)
next_launch = time.monotonic()
pending = list(addrinfos)
in_progress = {
0,
errno.EINPROGRESS,
errno.EWOULDBLOCK,
errno.EALREADY,
errno.EINTR,
getattr(errno, "WSAEWOULDBLOCK", 10035),
}
def start_attempt(addrinfo):
family, socktype, proto, _canonname, sockaddr = addrinfo
candidate = socket.socket(family, socktype, proto)
try:
if source_address is not None:
local_infos = socket.getaddrinfo(
source_address[0],
source_address[1],
family=family,
type=socktype,
)
if not local_infos:
raise OSError(
f"getaddrinfo returned no local {family} address for "
f"{source_address[0]}"
)
candidate.bind(local_infos[0][4])
candidate.setblocking(False)
result = candidate.connect_ex(sockaddr)
if result == 0 or result == errno.EISCONN:
return candidate
if result not in in_progress:
raise OSError(result, os.strerror(result))
selector.register(candidate, selectors.EVENT_WRITE)
active.add(candidate)
return None
except Exception:
candidate.close()
raise
try:
while pending or active:
now = time.monotonic()
if deadline is not None and now >= deadline:
raise socket.timeout("timed out")
if pending and now >= next_launch:
addrinfo = pending.pop(0)
try:
winner = start_attempt(addrinfo)
except OSError as exc:
last_error = exc
if not active:
next_launch = now
continue
if winner is not None:
break
next_launch = now + _HAPPY_EYEBALLS_DELAY_SECONDS
wait_timeout = None if deadline is None else max(0.0, deadline - now)
if pending:
until_launch = max(0.0, next_launch - now)
wait_timeout = (
until_launch
if wait_timeout is None
else min(wait_timeout, until_launch)
)
events = selector.select(wait_timeout)
for key, _mask in events:
candidate = key.fileobj
error_code = candidate.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
selector.unregister(candidate)
active.discard(candidate)
if error_code == 0:
winner = candidate
break
candidate.close()
last_error = OSError(error_code, os.strerror(error_code))
if winner is not None:
break
if not active and pending:
next_launch = time.monotonic()
if winner is None:
if last_error is not None:
raise last_error
raise OSError(f"Could not connect to {host}:{port}")
try:
selector.unregister(winner)
except Exception:
pass
active.discard(winner)
winner.settimeout(timeout)
for option in socket_options or ():
winner.setsockopt(*option)
winner.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
return winner
finally:
for candidate in active:
try:
selector.unregister(candidate)
except Exception:
pass
candidate.close()
selector.close()
class _HappyEyeballsSyncBackend:
"""httpcore sync backend with concurrent IPv6/IPv4 connection fallback."""
def __init__(self):
self._fallback = None
def _default_backend(self):
if self._fallback is None:
from httpcore import SyncBackend
self._fallback = SyncBackend()
return self._fallback
def connect_tcp(
self,
host: str,
port: int,
timeout: Optional[float] = None,
local_address: Optional[str] = None,
socket_options=None,
):
from httpcore import ConnectError, ConnectTimeout
from httpcore._backends.sync import SyncStream
source_address = None if local_address is None else (local_address, 0)
try:
sock = _happy_eyeballs_create_connection(
(host, port),
timeout,
source_address=source_address,
socket_options=socket_options or (),
)
except socket.timeout as exc:
raise ConnectTimeout(str(exc)) from exc
except OSError as exc:
raise ConnectError(str(exc)) from exc
return SyncStream(sock)
def connect_unix_socket(self, *args, **kwargs):
return self._default_backend().connect_unix_socket(*args, **kwargs)
def sleep(self, seconds: float) -> None:
self._default_backend().sleep(seconds)
def _uses_codex_cloud_transport(base_url: str) -> bool:
return (
base_url_hostname(base_url).lower() == "chatgpt.com"
and "/backend-api/codex" in str(base_url).lower()
)
def _enable_happy_eyeballs(transport) -> None:
"""Install the sync racing backend on one httpx transport, if compatible."""
pool = getattr(transport, "_pool", None)
if pool is not None and hasattr(pool, "_network_backend"):
pool._network_backend = _HappyEyeballsSyncBackend()
def _load_openai_cls() -> type:
@@ -186,10 +413,12 @@ def build_keepalive_http_client(
client_cls = httpx.AsyncClient if async_mode else httpx.Client
mounts = {}
if proxy is None:
mounts = {
"http://": transport_cls(verify=verify),
"https://": transport_cls(verify=verify),
}
http_transport = transport_cls(verify=verify)
https_transport = transport_cls(verify=verify)
if not async_mode and _uses_codex_cloud_transport(base_url):
_enable_happy_eyeballs(http_transport)
_enable_happy_eyeballs(https_transport)
mounts = {"http://": http_transport, "https://": https_transport}
return client_cls(
limits=limits,
timeout=timeout,
@@ -0,0 +1 @@
nateEc
+3 -66
View File
@@ -5170,73 +5170,10 @@ class AIAgent:
@staticmethod
def _build_keepalive_http_client(base_url: str = "", *, verify: Any = True) -> Any:
"""Build an httpx.Client with proactive idle-connection reaping.
"""Build the shared OpenAI httpx client used by main and aux paths."""
from agent.process_bootstrap import build_keepalive_http_client
Previously this method injected a custom ``httpx.HTTPTransport``
with ``socket_options`` (``SO_KEEPALIVE``, ``TCP_KEEPIDLE``, …) to
prevent CLOSE-WAIT accumulation on long-lived connections (#10324).
That approach broke streaming for providers behind reverse proxies
(OpenResty, Cloudflare, etc.) because the custom socket options
conflict with the proxy's chunked-transfer handling (#54049,
#12952). It also stripped ``TCP_NODELAY``, stalling TLS handshakes
and SSE encoding.
The fix moves connection lifecycle management from the socket layer
to the HTTP pool layer: ``keepalive_expiry=20.0`` tells httpx to
close idle pooled connections *before* a reverse proxy's typical
30–60 s timeout drops them, preventing CLOSE-WAIT accumulation
without modifying socket options. The default httpx transport
preserves OS TCP defaults (including ``TCP_NODELAY``).
``verify`` carries per-provider ``ssl_ca_cert`` / ``ssl_verify`` and
``HERMES_CA_BUNDLE`` settings. It is passed on the client AND on
the plain no-proxy mounts (a mounted transport owns the SSL context
for its scheme).
"""
try:
import httpx as _httpx
# Explicitly read proxy settings so requests route through
# HTTP_PROXY / HTTPS_PROXY / NO_PROXY correctly.
_proxy = _get_proxy_for_base_url(base_url)
# Proactive pool reaping: close idle connections at 20 s,
# before reverse proxies (30–60 s typical) send FIN and
# cause CLOSE-WAIT accumulation.
_limits = _httpx.Limits(
max_keepalive_connections=20,
max_connections=100,
keepalive_expiry=20.0,
)
# Timeouts: generous read=None for SSE streaming endpoints.
_timeout = _httpx.Timeout(
connect=15.0,
read=None,
write=15.0,
pool=10.0,
)
# When _proxy is None (NO_PROXY bypass or no proxy configured),
# mount plain transports to prevent httpx from reading env proxy
# vars and creating an HTTPProxy mount that would bypass our
# NO_PROXY resolution.
_mounts = {}
if _proxy is None:
_mounts = {
"http://": _httpx.HTTPTransport(verify=verify),
"https://": _httpx.HTTPTransport(verify=verify),
}
return _httpx.Client(
limits=_limits,
timeout=_timeout,
proxy=_proxy,
mounts=_mounts or None,
verify=verify,
)
except Exception:
return None
return build_keepalive_http_client(base_url, verify=verify)
def _create_openai_client(self, client_kwargs: dict, *, reason: str, shared: bool) -> Any:
"""Forwarder — see ``agent.agent_runtime_helpers.create_openai_client``."""
+148
View File
@@ -0,0 +1,148 @@
import errno
import selectors
import socket
import httpcore
import pytest
from agent import process_bootstrap
@pytest.fixture
def no_proxy_env(monkeypatch):
for name in (
"HTTPS_PROXY",
"HTTP_PROXY",
"ALL_PROXY",
"https_proxy",
"http_proxy",
"all_proxy",
"NO_PROXY",
"no_proxy",
):
monkeypatch.delenv(name, raising=False)
def _client_backends(client):
transports = [client._transport, *client._mounts.values()]
return [
transport._pool._network_backend
for transport in transports
if transport is not None and hasattr(transport, "_pool")
]
def test_codex_sync_client_uses_happy_eyeballs_backend(no_proxy_env):
from run_agent import AIAgent
client = AIAgent._build_keepalive_http_client(
"https://chatgpt.com/backend-api/codex"
)
try:
assert any(
isinstance(backend, process_bootstrap._HappyEyeballsSyncBackend)
for backend in _client_backends(client)
)
finally:
client.close()
def test_other_sync_clients_keep_httpcore_default_backend(no_proxy_env):
client = process_bootstrap.build_keepalive_http_client(
"https://api.openai.com/v1"
)
try:
assert all(
isinstance(backend, httpcore.SyncBackend)
for backend in _client_backends(client)
)
finally:
client.close()
def test_connection_staggers_past_blackholed_ipv6(monkeypatch):
clock = [0.0]
sockets = []
class FakeSocket:
def __init__(self, family, socktype, proto):
self.family = family
self.closed = False
self.timeout = None
sockets.append(self)
def setsockopt(self, *_args):
pass
def setblocking(self, _blocking):
pass
def settimeout(self, timeout):
self.timeout = timeout
def bind(self, _address):
pass
def connect_ex(self, _address):
if self.family == socket.AF_INET6:
return errno.EINPROGRESS
return 0
def close(self):
self.closed = True
class FakeSelector:
def __init__(self):
self.registered = set()
def register(self, fileobj, _events):
self.registered.add(fileobj)
def unregister(self, fileobj):
self.registered.discard(fileobj)
def select(self, timeout):
clock[0] += timeout or 0.0
return []
def close(self):
pass
monkeypatch.setattr(
process_bootstrap.socket,
"getaddrinfo",
lambda *_args, **_kwargs: [
(
socket.AF_INET6,
socket.SOCK_STREAM,
socket.IPPROTO_TCP,
"",
("2001:db8::1", 443, 0, 0),
),
(
socket.AF_INET,
socket.SOCK_STREAM,
socket.IPPROTO_TCP,
"",
("192.0.2.1", 443),
),
],
)
monkeypatch.setattr(process_bootstrap.socket, "socket", FakeSocket)
monkeypatch.setattr(
process_bootstrap.selectors, "DefaultSelector", FakeSelector
)
monkeypatch.setattr(
process_bootstrap.time, "monotonic", lambda: clock[0]
)
winner = process_bootstrap._happy_eyeballs_create_connection(
("chatgpt.com", 443),
timeout=10.0,
)
assert winner.family == socket.AF_INET
assert winner.timeout == 10.0
assert clock[0] == process_bootstrap._HAPPY_EYEBALLS_DELAY_SECONDS
assert sockets[0].closed is True
assert sockets[1].closed is False