From d934bbd4d5145ebf46b1d433c1a6066cc3b2a729 Mon Sep 17 00:00:00 2001 From: Nathan Shan Date: Thu, 23 Jul 2026 17:17:44 +0800 Subject: [PATCH] fix(agent): race Codex IPv6 and IPv4 connections - Add RFC 8305-style staggered address attempts for synchronous ChatGPT Codex requests. - Share the keepalive client builder across primary and auxiliary model paths. - Cover blackholed IPv6 fallback, provider scoping, and existing proxy and TLS behavior. --- agent/process_bootstrap.py | 237 +++++++++++++++++- .../emails/nathanielcrush51@gmail.com | 1 + run_agent.py | 69 +---- tests/agent/test_codex_happy_eyeballs.py | 148 +++++++++++ 4 files changed, 385 insertions(+), 70 deletions(-) create mode 100644 contributors/emails/nathanielcrush51@gmail.com create mode 100644 tests/agent/test_codex_happy_eyeballs.py diff --git a/agent/process_bootstrap.py b/agent/process_bootstrap.py index 89b6278a8c..4b0d225889 100644 --- a/agent/process_bootstrap.py +++ b/agent/process_bootstrap.py @@ -15,6 +15,9 @@ Three concerns, all tied to ``AIAgent`` boot-time / runtime IO setup: 3. **HTTP proxy resolution** — ``_get_proxy_from_env`` reads ``HTTPS_PROXY`` / ``HTTP_PROXY`` / ``ALL_PROXY``; ``_get_proxy_for_base_url`` respects ``NO_PROXY`` for the given base URL. +4. **Codex dual-stack resilience** — the synchronous ChatGPT/Codex transport + races resolved IPv6/IPv4 addresses so a blackholed family cannot exhaust + the request watchdog before a working address is attempted. ``run_agent`` re-exports every name so existing ``from run_agent import _get_proxy_from_env`` imports keep working @@ -23,8 +26,12 @@ unchanged. from __future__ import annotations +import errno import os +import selectors +import socket import sys +import time import urllib.request from typing import Any, Optional @@ -34,6 +41,226 @@ from utils import base_url_hostname, normalize_proxy_url # Cached at module level so we only pay the OpenAI SDK import cost once # per process (after the first lazy load). _OPENAI_CLS_CACHE = None +_HAPPY_EYEBALLS_DELAY_SECONDS = 0.25 + + +def _interleave_addrinfos(addrinfos: list[tuple]) -> list[tuple]: + """Interleave resolved address families while preserving resolver order.""" + queues: dict[int, list[tuple]] = {} + family_order: list[int] = [] + seen: set[tuple] = set() + for addrinfo in addrinfos: + family, socktype, proto, _canonname, sockaddr = addrinfo + marker = (family, socktype, proto, sockaddr) + if marker in seen: + continue + seen.add(marker) + if family not in queues: + queues[family] = [] + family_order.append(family) + queues[family].append(addrinfo) + + interleaved: list[tuple] = [] + while any(queues.values()): + for family in family_order: + if queues[family]: + interleaved.append(queues[family].pop(0)) + return interleaved + + +def _happy_eyeballs_create_connection( + address: tuple[str, int], + timeout: Optional[float], + source_address: Optional[tuple[str, int]] = None, + socket_options=(), +): + """Connect using staggered non-blocking attempts across resolved families. + + ``socket.create_connection`` tries every address serially. A host with + broken-but-advertised IPv6 can therefore consume the full connect timeout + for each AAAA record before trying a working IPv4 address. This follows the + Happy Eyeballs shape from RFC 8305: retain resolver preference, interleave + families, and start the next candidate after a short delay. + """ + host, port = address + addrinfos = _interleave_addrinfos( + socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + ) + if not addrinfos: + raise OSError(f"getaddrinfo returned no addresses for {host}") + + selector = selectors.DefaultSelector() + active: set[socket.socket] = set() + winner = None + last_error: Optional[OSError] = None + deadline = None if timeout is None else time.monotonic() + max(timeout, 0.0) + next_launch = time.monotonic() + pending = list(addrinfos) + in_progress = { + 0, + errno.EINPROGRESS, + errno.EWOULDBLOCK, + errno.EALREADY, + errno.EINTR, + getattr(errno, "WSAEWOULDBLOCK", 10035), + } + + def start_attempt(addrinfo): + family, socktype, proto, _canonname, sockaddr = addrinfo + candidate = socket.socket(family, socktype, proto) + try: + if source_address is not None: + local_infos = socket.getaddrinfo( + source_address[0], + source_address[1], + family=family, + type=socktype, + ) + if not local_infos: + raise OSError( + f"getaddrinfo returned no local {family} address for " + f"{source_address[0]}" + ) + candidate.bind(local_infos[0][4]) + candidate.setblocking(False) + result = candidate.connect_ex(sockaddr) + if result == 0 or result == errno.EISCONN: + return candidate + if result not in in_progress: + raise OSError(result, os.strerror(result)) + selector.register(candidate, selectors.EVENT_WRITE) + active.add(candidate) + return None + except Exception: + candidate.close() + raise + + try: + while pending or active: + now = time.monotonic() + if deadline is not None and now >= deadline: + raise socket.timeout("timed out") + + if pending and now >= next_launch: + addrinfo = pending.pop(0) + try: + winner = start_attempt(addrinfo) + except OSError as exc: + last_error = exc + if not active: + next_launch = now + continue + if winner is not None: + break + next_launch = now + _HAPPY_EYEBALLS_DELAY_SECONDS + + wait_timeout = None if deadline is None else max(0.0, deadline - now) + if pending: + until_launch = max(0.0, next_launch - now) + wait_timeout = ( + until_launch + if wait_timeout is None + else min(wait_timeout, until_launch) + ) + + events = selector.select(wait_timeout) + for key, _mask in events: + candidate = key.fileobj + error_code = candidate.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + selector.unregister(candidate) + active.discard(candidate) + if error_code == 0: + winner = candidate + break + candidate.close() + last_error = OSError(error_code, os.strerror(error_code)) + if winner is not None: + break + if not active and pending: + next_launch = time.monotonic() + + if winner is None: + if last_error is not None: + raise last_error + raise OSError(f"Could not connect to {host}:{port}") + + try: + selector.unregister(winner) + except Exception: + pass + active.discard(winner) + winner.settimeout(timeout) + for option in socket_options or (): + winner.setsockopt(*option) + winner.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + return winner + finally: + for candidate in active: + try: + selector.unregister(candidate) + except Exception: + pass + candidate.close() + selector.close() + + +class _HappyEyeballsSyncBackend: + """httpcore sync backend with concurrent IPv6/IPv4 connection fallback.""" + + def __init__(self): + self._fallback = None + + def _default_backend(self): + if self._fallback is None: + from httpcore import SyncBackend + + self._fallback = SyncBackend() + return self._fallback + + def connect_tcp( + self, + host: str, + port: int, + timeout: Optional[float] = None, + local_address: Optional[str] = None, + socket_options=None, + ): + from httpcore import ConnectError, ConnectTimeout + from httpcore._backends.sync import SyncStream + + source_address = None if local_address is None else (local_address, 0) + try: + sock = _happy_eyeballs_create_connection( + (host, port), + timeout, + source_address=source_address, + socket_options=socket_options or (), + ) + except socket.timeout as exc: + raise ConnectTimeout(str(exc)) from exc + except OSError as exc: + raise ConnectError(str(exc)) from exc + return SyncStream(sock) + + def connect_unix_socket(self, *args, **kwargs): + return self._default_backend().connect_unix_socket(*args, **kwargs) + + def sleep(self, seconds: float) -> None: + self._default_backend().sleep(seconds) + + +def _uses_codex_cloud_transport(base_url: str) -> bool: + return ( + base_url_hostname(base_url).lower() == "chatgpt.com" + and "/backend-api/codex" in str(base_url).lower() + ) + + +def _enable_happy_eyeballs(transport) -> None: + """Install the sync racing backend on one httpx transport, if compatible.""" + pool = getattr(transport, "_pool", None) + if pool is not None and hasattr(pool, "_network_backend"): + pool._network_backend = _HappyEyeballsSyncBackend() def _load_openai_cls() -> type: @@ -186,10 +413,12 @@ def build_keepalive_http_client( client_cls = httpx.AsyncClient if async_mode else httpx.Client mounts = {} if proxy is None: - mounts = { - "http://": transport_cls(verify=verify), - "https://": transport_cls(verify=verify), - } + http_transport = transport_cls(verify=verify) + https_transport = transport_cls(verify=verify) + if not async_mode and _uses_codex_cloud_transport(base_url): + _enable_happy_eyeballs(http_transport) + _enable_happy_eyeballs(https_transport) + mounts = {"http://": http_transport, "https://": https_transport} return client_cls( limits=limits, timeout=timeout, diff --git a/contributors/emails/nathanielcrush51@gmail.com b/contributors/emails/nathanielcrush51@gmail.com new file mode 100644 index 0000000000..0bf1e71db8 --- /dev/null +++ b/contributors/emails/nathanielcrush51@gmail.com @@ -0,0 +1 @@ +nateEc diff --git a/run_agent.py b/run_agent.py index 3436bcf992..d55b473e18 100644 --- a/run_agent.py +++ b/run_agent.py @@ -5170,73 +5170,10 @@ class AIAgent: @staticmethod def _build_keepalive_http_client(base_url: str = "", *, verify: Any = True) -> Any: - """Build an httpx.Client with proactive idle-connection reaping. + """Build the shared OpenAI httpx client used by main and aux paths.""" + from agent.process_bootstrap import build_keepalive_http_client - Previously this method injected a custom ``httpx.HTTPTransport`` - with ``socket_options`` (``SO_KEEPALIVE``, ``TCP_KEEPIDLE``, …) to - prevent CLOSE-WAIT accumulation on long-lived connections (#10324). - - That approach broke streaming for providers behind reverse proxies - (OpenResty, Cloudflare, etc.) because the custom socket options - conflict with the proxy's chunked-transfer handling (#54049, - #12952). It also stripped ``TCP_NODELAY``, stalling TLS handshakes - and SSE encoding. - - The fix moves connection lifecycle management from the socket layer - to the HTTP pool layer: ``keepalive_expiry=20.0`` tells httpx to - close idle pooled connections *before* a reverse proxy's typical - 30–60 s timeout drops them, preventing CLOSE-WAIT accumulation - without modifying socket options. The default httpx transport - preserves OS TCP defaults (including ``TCP_NODELAY``). - - ``verify`` carries per-provider ``ssl_ca_cert`` / ``ssl_verify`` and - ``HERMES_CA_BUNDLE`` settings. It is passed on the client AND on - the plain no-proxy mounts (a mounted transport owns the SSL context - for its scheme). - """ - try: - import httpx as _httpx - - # Explicitly read proxy settings so requests route through - # HTTP_PROXY / HTTPS_PROXY / NO_PROXY correctly. - _proxy = _get_proxy_for_base_url(base_url) - - # Proactive pool reaping: close idle connections at 20 s, - # before reverse proxies (30–60 s typical) send FIN and - # cause CLOSE-WAIT accumulation. - _limits = _httpx.Limits( - max_keepalive_connections=20, - max_connections=100, - keepalive_expiry=20.0, - ) - - # Timeouts: generous read=None for SSE streaming endpoints. - _timeout = _httpx.Timeout( - connect=15.0, - read=None, - write=15.0, - pool=10.0, - ) - - # When _proxy is None (NO_PROXY bypass or no proxy configured), - # mount plain transports to prevent httpx from reading env proxy - # vars and creating an HTTPProxy mount that would bypass our - # NO_PROXY resolution. - _mounts = {} - if _proxy is None: - _mounts = { - "http://": _httpx.HTTPTransport(verify=verify), - "https://": _httpx.HTTPTransport(verify=verify), - } - return _httpx.Client( - limits=_limits, - timeout=_timeout, - proxy=_proxy, - mounts=_mounts or None, - verify=verify, - ) - except Exception: - return None + return build_keepalive_http_client(base_url, verify=verify) def _create_openai_client(self, client_kwargs: dict, *, reason: str, shared: bool) -> Any: """Forwarder — see ``agent.agent_runtime_helpers.create_openai_client``.""" diff --git a/tests/agent/test_codex_happy_eyeballs.py b/tests/agent/test_codex_happy_eyeballs.py new file mode 100644 index 0000000000..48e0bbe38f --- /dev/null +++ b/tests/agent/test_codex_happy_eyeballs.py @@ -0,0 +1,148 @@ +import errno +import selectors +import socket + +import httpcore +import pytest + +from agent import process_bootstrap + + +@pytest.fixture +def no_proxy_env(monkeypatch): + for name in ( + "HTTPS_PROXY", + "HTTP_PROXY", + "ALL_PROXY", + "https_proxy", + "http_proxy", + "all_proxy", + "NO_PROXY", + "no_proxy", + ): + monkeypatch.delenv(name, raising=False) + + +def _client_backends(client): + transports = [client._transport, *client._mounts.values()] + return [ + transport._pool._network_backend + for transport in transports + if transport is not None and hasattr(transport, "_pool") + ] + + +def test_codex_sync_client_uses_happy_eyeballs_backend(no_proxy_env): + from run_agent import AIAgent + + client = AIAgent._build_keepalive_http_client( + "https://chatgpt.com/backend-api/codex" + ) + try: + assert any( + isinstance(backend, process_bootstrap._HappyEyeballsSyncBackend) + for backend in _client_backends(client) + ) + finally: + client.close() + + +def test_other_sync_clients_keep_httpcore_default_backend(no_proxy_env): + client = process_bootstrap.build_keepalive_http_client( + "https://api.openai.com/v1" + ) + try: + assert all( + isinstance(backend, httpcore.SyncBackend) + for backend in _client_backends(client) + ) + finally: + client.close() + + +def test_connection_staggers_past_blackholed_ipv6(monkeypatch): + clock = [0.0] + sockets = [] + + class FakeSocket: + def __init__(self, family, socktype, proto): + self.family = family + self.closed = False + self.timeout = None + sockets.append(self) + + def setsockopt(self, *_args): + pass + + def setblocking(self, _blocking): + pass + + def settimeout(self, timeout): + self.timeout = timeout + + def bind(self, _address): + pass + + def connect_ex(self, _address): + if self.family == socket.AF_INET6: + return errno.EINPROGRESS + return 0 + + def close(self): + self.closed = True + + class FakeSelector: + def __init__(self): + self.registered = set() + + def register(self, fileobj, _events): + self.registered.add(fileobj) + + def unregister(self, fileobj): + self.registered.discard(fileobj) + + def select(self, timeout): + clock[0] += timeout or 0.0 + return [] + + def close(self): + pass + + monkeypatch.setattr( + process_bootstrap.socket, + "getaddrinfo", + lambda *_args, **_kwargs: [ + ( + socket.AF_INET6, + socket.SOCK_STREAM, + socket.IPPROTO_TCP, + "", + ("2001:db8::1", 443, 0, 0), + ), + ( + socket.AF_INET, + socket.SOCK_STREAM, + socket.IPPROTO_TCP, + "", + ("192.0.2.1", 443), + ), + ], + ) + monkeypatch.setattr(process_bootstrap.socket, "socket", FakeSocket) + monkeypatch.setattr( + process_bootstrap.selectors, "DefaultSelector", FakeSelector + ) + monkeypatch.setattr( + process_bootstrap.time, "monotonic", lambda: clock[0] + ) + + winner = process_bootstrap._happy_eyeballs_create_connection( + ("chatgpt.com", 443), + timeout=10.0, + ) + + assert winner.family == socket.AF_INET + assert winner.timeout == 10.0 + assert clock[0] == process_bootstrap._HAPPY_EYEBALLS_DELAY_SECONDS + assert sockets[0].closed is True + assert sockets[1].closed is False