refactor(agent/chat_completion_helpers): collapse defensive layers and compact call sites across the non-streaming region

This commit is contained in:
Teknium
2026-09-02 19:17:28 -07:00
parent 032ac76293
commit de1567095a
+54 -165
View File
@@ -605,12 +605,7 @@ def _reset_stale_streak(agent) -> None:
_INTERRUPTED_WAIT_STALE_SECONDS = 30.0
def _record_interrupted_provider_wait(
agent,
elapsed: float,
*,
response_started: bool,
) -> bool:
def _record_interrupted_provider_wait(agent, elapsed: float, *, response_started: bool) -> bool:
"""Count a user-aborted pre-response stall toward the stale breaker.
Past the no-output interval that earns a wait notice, an interrupt is
@@ -623,20 +618,13 @@ def _record_interrupted_provider_wait(
logger.warning(
"Interrupted provider wait counted as stale after %.0fs with no output; "
"consecutive stale attempts=%d.",
elapsed,
_stale_streak(agent),
elapsed, _stale_streak(agent),
)
return True
def _report_stale_nonstream_kill(
agent,
api_kwargs: dict,
elapsed: float,
stale_timeout: float,
*,
inline: bool = False,
hint: Optional[str] = None,
agent, api_kwargs: dict, elapsed: float, stale_timeout: float, *, inline: bool = False, hint: Optional[str] = None,
) -> None:
"""Log + status message for a stale non-streaming kill.
@@ -648,10 +636,7 @@ def _report_stale_nonstream_kill(
logger.warning(
"%son-streaming API call stale for %.0fs (threshold %.0fs). "
"model=%s context=~%s tokens. Killing connection.",
"Inline n" if inline else "N",
elapsed,
stale_timeout,
model,
"Inline n" if inline else "N", elapsed, stale_timeout, model,
f"{estimate_request_context_tokens(api_kwargs):,}",
)
try:
@@ -665,9 +650,7 @@ def _report_stale_nonstream_kill(
def _touch_stale_kill_activity(agent, elapsed: float) -> None:
try:
agent._touch_activity(
f"stale non-streaming call killed after {int(elapsed)}s"
)
agent._touch_activity(f"stale non-streaming call killed after {int(elapsed)}s")
except Exception:
logger.debug("stale activity touch failed", exc_info=True)
@@ -737,10 +720,7 @@ def _bedrock_reasoning_stale_floor(model_id: object) -> "float | None":
if not model_id or not isinstance(model_id, str):
return None
name = model_id.strip().lower()
for prefix in (
"global.", "us.", "eu.", "apac.", "ap.", "au.", "jp.",
"ca.", "sa.", "me.", "af.",
):
for prefix in ("global.", "us.", "eu.", "apac.", "ap.", "au.", "jp.", "ca.", "sa.", "me.", "af."):
if name.startswith(prefix):
name = name[len(prefix):]
break
@@ -750,11 +730,7 @@ def _bedrock_reasoning_stale_floor(model_id: object) -> "float | None":
base_candidates.append(name.replace(".", "-", 1)) # deepseek-r1-v1:0
candidates: list[str] = []
for cand in base_candidates:
for form in (
cand,
re.sub(r"(?<=\d)-(?=\d)", ".", cand),
re.sub(r"(?<=\d)\.(?=\d)", "-", cand),
):
for form in (cand, re.sub(r"(?<=\d)-(?=\d)", ".", cand), re.sub(r"(?<=\d)\.(?=\d)", "-", cand)):
if form not in candidates:
candidates.append(form)
for cand in candidates:
@@ -852,7 +828,6 @@ def should_use_direct_api_call(agent) -> bool:
# with the agent's platform stamp as a fallback for callers that bypass it.
try:
from agent.delegation_context import is_delegated_child_context
if is_delegated_child_context():
return True
except Exception:
@@ -877,11 +852,7 @@ def _managed_local_load_notice(agent, api_kwargs: dict) -> "Optional[str]":
if not base:
return None
from urllib.parse import urlparse
from hermes_cli.local_runtime.load_progress import (
get_loading_progress,
get_prefill_progress,
)
from hermes_cli.local_runtime.load_progress import get_loading_progress, get_prefill_progress
from hermes_cli.local_runtime.supervisor import state_path
state = json.loads(state_path().read_text(encoding="utf-8"))
@@ -919,9 +890,7 @@ def _resolve_direct_stale_timeout(agent, api_kwargs: dict) -> float:
would silently reinstate the unbounded hang this exists to fix.
"""
resolver = getattr(agent, "_compute_non_stream_stale_timeout", None)
if not callable(resolver):
return float("inf")
value = resolver(api_kwargs)
value = resolver(api_kwargs) if callable(resolver) else None
if isinstance(value, bool) or not isinstance(value, (int, float)):
return float("inf")
return float(value)
@@ -941,13 +910,7 @@ def _inline_nonstream_hard_timeout(stale_timeout: float):
conn_cap = min(stale_timeout, 60.0)
try:
import httpx as _httpx
return _httpx.Timeout(
connect=conn_cap,
read=stale_timeout,
write=conn_cap,
pool=conn_cap,
)
return _httpx.Timeout(connect=conn_cap, read=stale_timeout, write=conn_cap, pool=conn_cap)
except Exception:
return stale_timeout
@@ -1071,11 +1034,7 @@ def direct_api_call(agent, api_kwargs: dict):
except Exception:
pass
activity_hb = threading.Thread(
target=_activity_heartbeat,
name="direct-api-activity-hb",
daemon=True,
)
activity_hb = threading.Thread(target=_activity_heartbeat, name="direct-api-activity-hb", daemon=True)
# Resolve the budget BEFORE start(): the resolver may raise (fail-closed),
# and a leaked heartbeat thread would mask real stalls forever.
call_start = time.time()
@@ -1096,9 +1055,7 @@ def direct_api_call(agent, api_kwargs: dict):
if not request.abort("stale_call_kill"):
return
elapsed = time.time() - call_start
_report_stale_nonstream_kill(
agent, api_kwargs, elapsed, stale_timeout, inline=True
)
_report_stale_nonstream_kill(agent, api_kwargs, elapsed, stale_timeout, inline=True)
_touch_stale_kill_activity(agent, elapsed)
stale_watchdog = None
@@ -1112,9 +1069,7 @@ def direct_api_call(agent, api_kwargs: dict):
# close the client so the retry builds a fresh pool.
succeeded = False
try:
response = _dispatch_nonstreaming_api_request(
agent, api_kwargs, make_client=request.make_client
)
response = _dispatch_nonstreaming_api_request(agent, api_kwargs, make_client=request.make_client)
except Exception:
if getattr(agent, "_interrupt_requested", False):
raise InterruptedError("Agent interrupted during API call") from None
@@ -1187,9 +1142,7 @@ class _RequestClientRegistry:
def set_client(self, client, *, kind: str = "openai"):
with self.lock:
self.client = client
self.kind = kind
self.owner_tid = threading.get_ident()
self.client, self.kind, self.owner_tid = client, kind, threading.get_ident()
return client
@staticmethod
@@ -1203,11 +1156,7 @@ class _RequestClientRegistry:
def set_stream_handle(self, stream):
if self._stream_close_callable(stream) is None:
return stream
with self.lock:
self.client = stream
self.kind = "stream"
self.owner_tid = threading.get_ident()
return stream
return self.set_client(stream, kind="stream")
def _close_stream_handle(self, stream, reason: str) -> None:
close = self._stream_close_callable(stream)
@@ -1221,9 +1170,7 @@ class _RequestClientRegistry:
def close_once(self, reason: str) -> None:
with self.lock:
request_client = self.client
request_kind = self.kind
owner_tid = self.owner_tid
request_client, request_kind, owner_tid = self.client, self.kind, self.owner_tid
stranger_thread = (
request_kind != "stream"
and request_client is not None
@@ -1232,9 +1179,7 @@ class _RequestClientRegistry:
)
if stranger_thread:
if request_kind == "anthropic_messages":
self.agent._abort_request_anthropic_client(
request_client, reason=reason
)
self.agent._abort_request_anthropic_client(request_client, reason=reason)
else:
self.agent._abort_request_openai_client(request_client, reason=reason)
return
@@ -1310,23 +1255,13 @@ def _resolve_nonstream_watchdogs(agent, api_kwargs: dict) -> _NonStreamWatchdogs
# prefill before the first SSE event: scale the cutoff up to the idle
# default unless HERMES_CODEX_TTFB_STRICT keeps the smaller one.
disable_above = _env_float("HERMES_CODEX_TTFB_DISABLE_ABOVE_TOKENS", 10_000.0)
strict = os.environ.get("HERMES_CODEX_TTFB_STRICT", "").strip().lower() in {
"1", "true", "yes", "on"
}
if (
not strict
and disable_above > 0
and est_tokens >= disable_above
and ttfb_timeout < idle_default
):
strict = os.environ.get("HERMES_CODEX_TTFB_STRICT", "").strip().lower() in {"1", "true", "yes", "on"}
if not strict and disable_above > 0 and est_tokens >= disable_above and ttfb_timeout < idle_default:
logger.info(
"Scaling openai-codex no-byte TTFB watchdog from %.0fs to %.0fs "
"for large request (context=~%s tokens >= %.0f). "
"Set HERMES_CODEX_TTFB_STRICT=1 to keep the smaller cutoff.",
ttfb_timeout,
idle_default,
f"{est_tokens:,}",
disable_above,
ttfb_timeout, idle_default, f"{est_tokens:,}", disable_above,
)
ttfb_timeout = idle_default
ttfb_cap = _env_float("HERMES_CODEX_TTFB_MAX_SECONDS", 120.0)
@@ -1334,21 +1269,15 @@ def _resolve_nonstream_watchdogs(agent, api_kwargs: dict) -> _NonStreamWatchdogs
logger.info(
"Capping openai-codex no-byte TTFB timeout from %.0fs to %.0fs "
"(context=~%s tokens). Set HERMES_CODEX_TTFB_MAX_SECONDS to tune.",
ttfb_timeout,
ttfb_cap,
f"{est_tokens:,}",
ttfb_timeout, ttfb_cap, f"{est_tokens:,}",
)
ttfb_timeout = ttfb_cap
idle_timeout = _env_float("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", idle_default)
return _NonStreamWatchdogs(
stale_timeout=stale_timeout,
codex=codex,
est_tokens=est_tokens,
ttfb_enabled=ttfb_enabled,
ttfb_timeout=ttfb_timeout,
idle_enabled=codex and idle_timeout > 0,
idle_timeout=idle_timeout,
stale_timeout=stale_timeout, codex=codex, est_tokens=est_tokens,
ttfb_enabled=ttfb_enabled, ttfb_timeout=ttfb_timeout,
idle_enabled=codex and idle_timeout > 0, idle_timeout=idle_timeout,
)
@@ -1410,17 +1339,14 @@ class _NonStreamRequest:
if kind == "anthropic_messages":
client = self.agent._create_request_anthropic_client(reason=reason)
else:
client = self.agent._create_request_openai_client(
reason=reason, api_kwargs=self.api_kwargs
)
client = self.agent._create_request_openai_client(reason=reason, api_kwargs=self.api_kwargs)
return self.clients.set_client(client, kind=kind)
def _call(self):
try:
self._install_codex_request_token()
self.result["response"] = _dispatch_nonstreaming_api_request(
self.agent, self.api_kwargs, make_client=self._make_client
)
self.agent, self.api_kwargs, make_client=self._make_client)
except Exception as e:
# Our own force-close caused this error: swallow it, the main
# thread raises InterruptedError (#6600). Retirement logs at info
@@ -1431,16 +1357,14 @@ class _NonStreamRequest:
"Codex worker caught %s after request retirement — "
"discarding the stale partial instead of surfacing it "
"as a completed response. %s",
type(e).__name__,
self.agent._client_log_context(),
type(e).__name__, self.agent._client_log_context(),
)
return
if self.cancelled:
logger.debug(
"Non-streaming worker caught %s after request "
"cancellation — exiting without surfacing a network "
"error.",
type(e).__name__,
"error.", type(e).__name__,
)
return
self.result["error"] = e
@@ -1451,10 +1375,7 @@ class _NonStreamRequest:
# Reuse reason only on a clean response; error or cancel-swallow
# really closes so the next attempt builds a fresh pool.
self.clients.close_once(
"request_complete"
if self.result["response"] is not None
else "request_error_cleanup"
)
"request_complete" if self.result["response"] is not None else "request_error_cleanup")
def _abort_request(self, reason: str) -> None:
"""Watchdog/interrupt kill: abort the request client (kind-aware, #67142)
@@ -1479,13 +1400,9 @@ class _NonStreamRequest:
wd = self.wd
try:
recovery = _codex_wait_notice_recovery(
stale_timeout=wd.stale_timeout,
ttfb_enabled=wd.ttfb_enabled,
ttfb_timeout=wd.ttfb_timeout,
stale_timeout=wd.stale_timeout, ttfb_enabled=wd.ttfb_enabled, ttfb_timeout=wd.ttfb_timeout,
last_event_ts=getattr(self.agent, "_codex_stream_last_event_ts", None),
call_start=self.call_start,
idle_enabled=wd.idle_enabled,
idle_timeout=wd.idle_timeout,
call_start=self.call_start, idle_enabled=wd.idle_enabled, idle_timeout=wd.idle_timeout,
elapsed=elapsed,
)
self.agent._emit_wait_notice(
@@ -2743,36 +2660,26 @@ def cleanup_task_resources(agent, task_id: str) -> None:
headed mode (window stays visible; the browser inactivity reaper still
handles idle sessions).
"""
try:
if is_persistent_env(task_id):
if agent.verbose_logging:
logging.debug(
f"Skipping per-turn cleanup_vm for persistent env {task_id}; "
f"idle reaper will handle it."
)
else:
_ra().cleanup_vm(task_id)
except Exception as e:
if agent.verbose_logging:
logger.warning("Failed to cleanup VM for task %s: %s", task_id, e)
try:
headed = False
def _headed() -> bool:
try:
from tools.browser_tool import _is_headed_mode
headed = _is_headed_mode()
return _is_headed_mode()
except Exception:
headed = bool(os.environ.get("AGENT_BROWSER_HEADED"))
if headed:
return bool(os.environ.get("AGENT_BROWSER_HEADED"))
for label, skip, skip_what, cleanup in (
("VM", is_persistent_env, "cleanup_vm for persistent env", lambda: _ra().cleanup_vm(task_id)),
("browser", lambda _tid: _headed(), "cleanup_browser for headed session", lambda: _ra().cleanup_browser(task_id)),
):
try:
if skip(task_id):
if agent.verbose_logging:
logging.debug(f"Skipping per-turn {skip_what} {task_id}; idle reaper will handle it.")
else:
cleanup()
except Exception as e:
if agent.verbose_logging:
logging.debug(
f"Skipping per-turn cleanup_browser for headed session {task_id}; "
f"idle reaper will handle it."
)
else:
_ra().cleanup_browser(task_id)
except Exception as e:
if agent.verbose_logging:
logger.warning("Failed to cleanup browser for task %s: %s", task_id, e)
logger.warning("Failed to cleanup %s for task %s: %s", label, task_id, e)
def _build_partial_stream_stub(
@@ -3193,12 +3100,7 @@ class _ToolCallAccumulator:
if extra is None and hasattr(tc_delta, "model_extra"):
extra = (tc_delta.model_extra if isinstance(tc_delta.model_extra, dict) else {}).get("extra_content")
if extra is not None:
if hasattr(extra, "model_dump"):
try:
extra = extra.model_dump(warnings=False)
except TypeError:
extra = extra.model_dump()
entry["extra_content"] = extra
entry["extra_content"] = _model_dump_safe(extra) if hasattr(extra, "model_dump") else extra
name = entry["function"]["name"]
if name and idx not in self._notified:
self._notified.add(idx)
@@ -3251,8 +3153,7 @@ class _StreamingCall:
return stream
def _close_managed_stream(self) -> None:
stream = self.managed_stream_holder.pop("stream", None)
close = getattr(stream, "close", None) if stream is not None else None
close = getattr(self.managed_stream_holder.pop("stream", None), "close", None)
if callable(close):
try:
close()
@@ -3326,11 +3227,7 @@ class _StreamingCall:
the delta callback for tag extraction (the CLI drops non-reasoning text
once the stream box is closed)."""
if self.agent.stream_delta_callback:
try:
self.agent.stream_delta_callback(text)
self.agent._record_streamed_assistant_text(text)
except Exception:
pass
self._quiet(lambda: (self.agent.stream_delta_callback(text), self.agent._record_streamed_assistant_text(text)))
def _new_diag(self) -> dict:
diag = self.agent._stream_diag_init()
@@ -4101,11 +3998,7 @@ class _StreamingCall:
self.agent.base_url, self._stream_stale_timeout,
)
return
_est_tokens = estimate_request_context_tokens(self.api_kwargs)
if _est_tokens > 100_000:
base = max(base, 300.0)
elif _est_tokens > 50_000:
base = max(base, 240.0)
base = _scale_stale_timeout_for_context(base, estimate_request_context_tokens(self.api_kwargs))
from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
_reasoning_floor = get_reasoning_stale_timeout_floor(self.api_kwargs.get("model"))
self._stream_stale_timeout = base if _reasoning_floor is None else max(base, _reasoning_floor)
@@ -4152,13 +4045,9 @@ class _StreamingCall:
_content_filter_terminated = _cls.reason == FailoverReason.content_policy_blocked
except Exception:
_content_filter_terminated = False
_stub_msg = SimpleNamespace(role="assistant", content=_partial_text, tool_calls=None, reasoning_content=None)
_stub = SimpleNamespace(
id=PARTIAL_STREAM_STUB_ID,
model=getattr(self.agent, "model", "unknown"),
choices=[SimpleNamespace(index=0, message=_stub_msg, finish_reason=FINISH_REASON_LENGTH)],
usage=None,
_dropped_tool_names=_partial_names or None,
_stub = _build_partial_stream_stub(
"assistant", _partial_text, None, getattr(self.agent, "model", "unknown"), None,
dropped_tool_names=_partial_names,
)
if _content_filter_terminated:
_stub._content_filter_terminated = True