feat: wall-clock run budget — wrap-up injection at 80% and deadline-scaled stale timeouts (agent.run_budget_seconds / --run-budget)
This commit is contained in:
@@ -490,6 +490,25 @@ def _merge_custom_provider_extra_body(agent, custom_providers: List[Dict[str, An
|
||||
agent.request_overrides = overrides
|
||||
|
||||
|
||||
def _normalize_run_budget_seconds(value) -> Optional[float]:
|
||||
"""Normalize a wall-clock run budget value to a positive float or None.
|
||||
|
||||
None / absent / non-numeric / non-positive all resolve to ``None``
|
||||
(feature off) so a malformed config value can never activate the
|
||||
deadline machinery, only leave it dormant. ``bool`` is rejected because
|
||||
YAML ``true`` would otherwise become a 1-second budget.
|
||||
"""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
seconds = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if seconds != seconds or seconds <= 0: # NaN or non-positive
|
||||
return None
|
||||
return seconds
|
||||
|
||||
|
||||
def init_agent(
|
||||
agent,
|
||||
base_url: str = None,
|
||||
@@ -560,6 +579,7 @@ def init_agent(
|
||||
session_db=None,
|
||||
parent_session_id: str = None,
|
||||
iteration_budget: "IterationBudget" = None,
|
||||
run_budget_seconds: Optional[float] = None,
|
||||
fallback_model: Dict[str, Any] = None,
|
||||
credential_pool=None,
|
||||
checkpoints_enabled: bool = False,
|
||||
@@ -972,6 +992,17 @@ def init_agent(
|
||||
agent._budget_exhausted_injected = False
|
||||
agent._budget_grace_call = False
|
||||
|
||||
# Optional wall-clock run budget (seconds per run_conversation turn).
|
||||
# Explicit constructor arg wins; else resolved from config.yaml
|
||||
# (agent.run_budget_seconds) further below. None = feature fully off:
|
||||
# no clock reads, no injection, no stale-timeout capping.
|
||||
agent.run_budget_seconds = _normalize_run_budget_seconds(run_budget_seconds)
|
||||
# Wall-clock start of the CURRENT run_conversation turn. Set by
|
||||
# turn_context.prepare_turn when a run budget is active; None otherwise.
|
||||
agent._run_budget_started_at = None
|
||||
# One-shot latch for the 80% wrap-up notice (reset each turn).
|
||||
agent._run_budget_wrapup_injected = False
|
||||
|
||||
# Activity tracking — updated on each API call, tool execution, and
|
||||
# stream chunk. Used by the gateway timeout handler to report what the
|
||||
# agent was doing when it was killed, and by the "still working"
|
||||
@@ -1902,6 +1933,14 @@ def init_agent(
|
||||
# agent/system_prompt.py for the injection gate.
|
||||
agent._execution_guidance = _agent_section.get("execution_guidance", "auto")
|
||||
|
||||
# Wall-clock run budget from config (agent.run_budget_seconds) — only
|
||||
# consulted when the constructor arg was not given. Absent/None/invalid
|
||||
# keeps the feature fully off (zero behavior change in the default path).
|
||||
if agent.run_budget_seconds is None:
|
||||
agent.run_budget_seconds = _normalize_run_budget_seconds(
|
||||
_agent_section.get("run_budget_seconds")
|
||||
)
|
||||
|
||||
# Empty-response retry guard config (NS-503): additive
|
||||
# ``agent.empty_response_guard`` subsection. Resolution is tolerant —
|
||||
# a malformed section falls back to the schema defaults (guard on,
|
||||
|
||||
@@ -107,6 +107,64 @@ logger = logging.getLogger(__name__)
|
||||
_INTERRUPT_SCAFFOLD_MARKER = "[This response was interrupted by a user correction.]"
|
||||
|
||||
|
||||
# One-time wrap-up notice appended when a wall-clock run budget crosses its
|
||||
# 80% threshold (agent.run_budget_seconds / --run-budget). Mirrors the Codex
|
||||
# CLI budget wrap-up template: stop new work, deliver from current state.
|
||||
RUN_BUDGET_WRAPUP_NOTICE = (
|
||||
"[SYSTEM NOTICE — run time budget nearly exhausted] "
|
||||
"Run time budget nearly exhausted. Stop new discovery/verification work "
|
||||
"now. Produce the required final deliverable (answer/JSON/summary) from "
|
||||
"the state you already have, completing only mandatory writes."
|
||||
)
|
||||
|
||||
|
||||
def _maybe_inject_run_budget_wrapup(agent: Any, messages: List[Dict[str, Any]]) -> bool:
|
||||
"""Inject the one-time wall-clock wrap-up notice when past 80% of budget.
|
||||
|
||||
Cache-safe delivery: the notice is appended to the NEWEST ``role:"tool"``
|
||||
message (the same channel /steer uses) — no synthetic user message is
|
||||
inserted mid-loop and no past context is rewritten, so role alternation
|
||||
and the prompt-cache prefix survive. Latches ``_run_budget_wrapup_injected``
|
||||
only on a successful append, so a first iteration without tool results
|
||||
retries on the next iteration. Returns True when the notice was injected.
|
||||
|
||||
Dormant unless ``agent.run_budget_seconds`` is set AND the turn stamped
|
||||
``_run_budget_started_at`` (see ``turn_context.prepare_conversation_turn``).
|
||||
"""
|
||||
budget = getattr(agent, "run_budget_seconds", None)
|
||||
if not budget:
|
||||
return False
|
||||
if getattr(agent, "_run_budget_wrapup_injected", False):
|
||||
return False
|
||||
started = getattr(agent, "_run_budget_started_at", None)
|
||||
if not started:
|
||||
return False
|
||||
if (time.time() - started) < 0.8 * float(budget):
|
||||
return False
|
||||
for i in range(len(messages) - 1, -1, -1):
|
||||
msg = messages[i]
|
||||
if isinstance(msg, dict) and msg.get("role") == "tool":
|
||||
existing = msg.get("content", "")
|
||||
if isinstance(existing, str):
|
||||
msg["content"] = existing + f"\n\n{RUN_BUDGET_WRAPUP_NOTICE}"
|
||||
else:
|
||||
# Multimodal content blocks — append a text block.
|
||||
try:
|
||||
blocks = list(existing) if existing else []
|
||||
blocks.append({"type": "text", "text": RUN_BUDGET_WRAPUP_NOTICE})
|
||||
msg["content"] = blocks
|
||||
except Exception:
|
||||
return False
|
||||
agent._run_budget_wrapup_injected = True
|
||||
logger.info(
|
||||
"Run budget wrap-up notice injected (budget=%.0fs, elapsed=%.0fs)",
|
||||
float(budget),
|
||||
time.time() - started,
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _restore_user_after_reference_handoff(
|
||||
messages: List[Dict[str, Any]], user_message: Any
|
||||
) -> bool:
|
||||
@@ -2041,6 +2099,15 @@ def run_conversation(
|
||||
existing = getattr(agent, "_pending_steer", None)
|
||||
agent._pending_steer = (existing + "\n" + _pre_api_steer) if existing else _pre_api_steer
|
||||
|
||||
# ── Wall-clock run-budget wrap-up notice ───────────────────────
|
||||
# One-shot: when a run budget (agent.run_budget_seconds /
|
||||
# --run-budget) is active and 80% of it has elapsed, ask the model
|
||||
# to wrap up and deliver from the state it already has. Same
|
||||
# cache-safe channel as /steer (appended to the newest tool
|
||||
# result); dormant when no budget is set.
|
||||
if getattr(agent, "run_budget_seconds", None):
|
||||
_maybe_inject_run_budget_wrapup(agent, messages)
|
||||
|
||||
# Prepare messages for API call
|
||||
# If we have an ephemeral system prompt, prepend it to the messages
|
||||
# Note: Reasoning is embedded in content via <think> tags for trajectory storage.
|
||||
|
||||
@@ -605,6 +605,15 @@ def build_turn_context(
|
||||
# NOTE: _turns_since_memory and _iters_since_skill are NOT reset here.
|
||||
agent.iteration_budget = IterationBudget(agent.max_iterations)
|
||||
|
||||
# Wall-clock run budget: per-run_conversation clock. Only stamped when a
|
||||
# budget is configured so the default path stays clock-free; the wrap-up
|
||||
# latch resets each turn (one notice per run, not per session).
|
||||
if getattr(agent, "run_budget_seconds", None):
|
||||
agent._run_budget_started_at = time.time()
|
||||
else:
|
||||
agent._run_budget_started_at = None
|
||||
agent._run_budget_wrapup_injected = False
|
||||
|
||||
# Log conversation turn start for debugging/observability.
|
||||
_preview_text = summarize_user_message_for_log(user_message)
|
||||
_msg_preview = (_preview_text[:80] + "...") if len(_preview_text) > 80 else _preview_text
|
||||
|
||||
@@ -4858,6 +4858,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
||||
api_key: str = None,
|
||||
base_url: str = None,
|
||||
max_turns: int = None,
|
||||
run_budget: float = None,
|
||||
verbose: Optional[bool] = None,
|
||||
compact: bool = False,
|
||||
resume: str = None,
|
||||
@@ -5125,6 +5126,13 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
||||
self.max_turns = 500
|
||||
else:
|
||||
self.max_turns = 500
|
||||
|
||||
# Wall-clock run budget: CLI flag wins over config; both optional.
|
||||
# None keeps the feature fully off (AIAgent stays dormant).
|
||||
if run_budget is not None:
|
||||
self.run_budget_seconds = run_budget
|
||||
else:
|
||||
self.run_budget_seconds = CLI_CONFIG["agent"].get("run_budget_seconds")
|
||||
|
||||
# Parse and validate toolsets
|
||||
self.enabled_toolsets = toolsets
|
||||
@@ -20174,6 +20182,7 @@ def main(
|
||||
api_key: str = None,
|
||||
base_url: str = None,
|
||||
max_turns: int = None,
|
||||
run_budget: float = None,
|
||||
verbose: Optional[bool] = None,
|
||||
quiet: bool = False,
|
||||
compact: bool = False,
|
||||
@@ -20370,6 +20379,7 @@ def main(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
max_turns=max_turns,
|
||||
run_budget=run_budget,
|
||||
verbose=verbose,
|
||||
compact=compact,
|
||||
resume=resume,
|
||||
|
||||
@@ -459,6 +459,21 @@ def build_top_level_parser():
|
||||
metavar="N",
|
||||
help="Maximum tool-calling iterations per conversation turn (default: 500, or agent.max_turns in config)",
|
||||
)
|
||||
chat_parser.add_argument(
|
||||
"--run-budget",
|
||||
type=float,
|
||||
default=None,
|
||||
metavar="SECONDS",
|
||||
dest="run_budget",
|
||||
help=(
|
||||
"Optional wall-clock budget in seconds for each conversation run. "
|
||||
"At 80%% elapsed the agent gets a one-time wrap-up notice, and "
|
||||
"implicit provider stale timeouts are capped to the remaining "
|
||||
"budget so one hung call can't consume the run. Unset = off. "
|
||||
"Also configurable as agent.run_budget_seconds in config.yaml. "
|
||||
"Intended for one-shot/eval invocations with a hard ceiling."
|
||||
),
|
||||
)
|
||||
_inherited_flag(
|
||||
chat_parser,
|
||||
"--yolo",
|
||||
|
||||
@@ -500,6 +500,7 @@ class CLIAgentSetupMixin:
|
||||
credential_pool=runtime.get("credential_pool"),
|
||||
max_tokens=self.max_tokens,
|
||||
max_iterations=self.max_turns,
|
||||
run_budget_seconds=getattr(self, "run_budget_seconds", None),
|
||||
enabled_toolsets=self.enabled_toolsets,
|
||||
disabled_toolsets=self.disabled_toolsets,
|
||||
verbose_logging=self.verbose,
|
||||
|
||||
@@ -44,6 +44,12 @@ DEFAULT_CONFIG = {
|
||||
},
|
||||
"agent": {
|
||||
"max_turns": 500,
|
||||
# Optional wall-clock budget in seconds per conversation run.
|
||||
# null/absent = feature fully off (zero behavior change). When set,
|
||||
# the agent gets a one-time wrap-up notice at 80% elapsed and
|
||||
# implicit provider stale timeouts are capped to the remaining
|
||||
# budget. CLI one-shot equivalent: `hermes chat --run-budget N`.
|
||||
"run_budget_seconds": None,
|
||||
# Inactivity timeout for gateway agent execution (seconds).
|
||||
# The agent can run indefinitely as long as it's actively calling
|
||||
# tools or receiving API responses. Only fires when the agent has
|
||||
|
||||
@@ -3220,6 +3220,7 @@ def cmd_chat(args):
|
||||
"checkpoints": getattr(args, "checkpoints", False),
|
||||
"pass_session_id": getattr(args, "pass_session_id", False),
|
||||
"max_turns": getattr(args, "max_turns", None),
|
||||
"run_budget": getattr(args, "run_budget", None),
|
||||
"ignore_rules": getattr(args, "ignore_rules", False) or getattr(args, "safe_mode", False),
|
||||
"ignore_user_config": getattr(args, "ignore_user_config", False) or getattr(args, "safe_mode", False),
|
||||
"compact": getattr(args, "compact", False),
|
||||
|
||||
+36
-4
@@ -503,6 +503,7 @@ class AIAgent:
|
||||
session_db=None,
|
||||
parent_session_id: str = None,
|
||||
iteration_budget: "IterationBudget" = None,
|
||||
run_budget_seconds: Optional[float] = None,
|
||||
fallback_model: Dict[str, Any] = None,
|
||||
credential_pool=None,
|
||||
checkpoints_enabled: bool = False,
|
||||
@@ -592,6 +593,7 @@ class AIAgent:
|
||||
session_db=session_db,
|
||||
parent_session_id=parent_session_id,
|
||||
iteration_budget=iteration_budget,
|
||||
run_budget_seconds=run_budget_seconds,
|
||||
fallback_model=fallback_model,
|
||||
credential_pool=credential_pool,
|
||||
checkpoints_enabled=checkpoints_enabled,
|
||||
@@ -1452,10 +1454,40 @@ class AIAgent:
|
||||
from agent.chat_completion_helpers import estimate_request_context_tokens
|
||||
est_tokens = estimate_request_context_tokens(api_payload)
|
||||
if est_tokens > 100_000:
|
||||
return max(stale_base, 240.0)
|
||||
if est_tokens > 50_000:
|
||||
return max(stale_base, 150.0)
|
||||
return stale_base
|
||||
timeout = max(stale_base, 240.0)
|
||||
elif est_tokens > 50_000:
|
||||
timeout = max(stale_base, 150.0)
|
||||
else:
|
||||
timeout = stale_base
|
||||
|
||||
# Wall-clock run budget cap: when a run budget is active, an implicit
|
||||
# (floor-/default-derived) stale timeout is capped at half the
|
||||
# remaining budget (>= 60s) so a single hung provider call cannot eat
|
||||
# the whole run — e.g. deepseek-v4-pro's 600s reasoning floor inside a
|
||||
# 900s eval ceiling. NEVER raises the timeout above what it would
|
||||
# otherwise be, and an explicit user-configured stale_timeout_seconds
|
||||
# (or env var) still wins untouched.
|
||||
run_budget = getattr(self, "run_budget_seconds", None)
|
||||
if run_budget and not self._stale_timeout_is_explicit():
|
||||
started = getattr(self, "_run_budget_started_at", None)
|
||||
if started:
|
||||
remaining = float(run_budget) - (time.time() - started)
|
||||
deadline_cap = max(60.0, remaining * 0.5)
|
||||
if deadline_cap < timeout:
|
||||
timeout = deadline_cap
|
||||
return timeout
|
||||
|
||||
def _stale_timeout_is_explicit(self) -> bool:
|
||||
"""True when the user explicitly configured the non-stream stale timeout.
|
||||
|
||||
Explicit = provider/model ``stale_timeout_seconds`` in config.yaml or
|
||||
the ``HERMES_API_CALL_STALE_TIMEOUT`` env var. Reasoning-model floors
|
||||
and the 90s default are implicit — they yield to the wall-clock run
|
||||
budget cap; explicit user configuration never does.
|
||||
"""
|
||||
if get_provider_stale_timeout(self.provider, self.model) is not None:
|
||||
return True
|
||||
return os.getenv("HERMES_API_CALL_STALE_TIMEOUT") is not None
|
||||
|
||||
def _codex_silent_hang_hint(self, model: Optional[str] = None) -> Optional[str]:
|
||||
"""Return an actionable hint when this request matches a known
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
"""Tests for the optional wall-clock run budget (agent.run_budget_seconds / --run-budget).
|
||||
|
||||
Covers:
|
||||
|
||||
1. Stale-timeout deadline scaling in
|
||||
``run_agent.py:AIAgent._compute_non_stream_stale_timeout``:
|
||||
- an active run budget CAPS an implicit stale timeout (default 90s and
|
||||
reasoning floors like deepseek's 600s) at ``max(60, remaining * 0.5)``;
|
||||
- the cap never RAISES the timeout above what it would otherwise be;
|
||||
- explicit user configuration (provider ``stale_timeout_seconds`` or the
|
||||
``HERMES_API_CALL_STALE_TIMEOUT`` env var) always wins untouched;
|
||||
- no budget => completely unchanged behavior.
|
||||
|
||||
2. One-time-ness of the 80% wrap-up notice injection in
|
||||
``agent.conversation_loop._maybe_inject_run_budget_wrapup``:
|
||||
- fires once (latched), not repeatedly;
|
||||
- never fires when no budget is set or before the 80% threshold;
|
||||
- appended to the newest tool message (cache-safe /steer channel), no
|
||||
synthetic user message.
|
||||
|
||||
3. Normalization of the config/CLI value
|
||||
(``agent.agent_init._normalize_run_budget_seconds``): dormant on
|
||||
null/invalid/non-positive input.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _write_config(tmp_path: Path, body: str) -> None:
|
||||
(tmp_path / "config.yaml").write_text(body or "{}\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _make_agent(tmp_path, monkeypatch, config_body: str = "", **overrides):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
(tmp_path / ".env").write_text("", encoding="utf-8")
|
||||
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
|
||||
_write_config(tmp_path, config_body)
|
||||
|
||||
from run_agent import AIAgent
|
||||
kwargs = dict(
|
||||
model="gpt-5.5",
|
||||
provider="openai",
|
||||
api_key="sk-dummy",
|
||||
base_url="https://api.openai.com/v1",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
platform="cli",
|
||||
)
|
||||
kwargs.update(overrides)
|
||||
return AIAgent(**kwargs)
|
||||
|
||||
|
||||
# ── normalization ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw,expected", [
|
||||
(None, None),
|
||||
(0, None),
|
||||
(-5, None),
|
||||
("abc", None),
|
||||
(True, None), # YAML `true` must not become a 1-second budget
|
||||
(False, None),
|
||||
(float("nan"), None),
|
||||
(900, 900.0),
|
||||
("850", 850.0),
|
||||
(0.5, 0.5),
|
||||
])
|
||||
def test_normalize_run_budget_seconds(raw, expected):
|
||||
from agent.agent_init import _normalize_run_budget_seconds
|
||||
assert _normalize_run_budget_seconds(raw) == expected
|
||||
|
||||
|
||||
# ── constructor / config plumbing ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_no_budget_by_default(monkeypatch, tmp_path):
|
||||
agent = _make_agent(tmp_path, monkeypatch)
|
||||
assert agent.run_budget_seconds is None
|
||||
assert agent._run_budget_started_at is None
|
||||
assert agent._run_budget_wrapup_injected is False
|
||||
|
||||
|
||||
def test_constructor_arg_sets_budget(monkeypatch, tmp_path):
|
||||
agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900)
|
||||
assert agent.run_budget_seconds == 900.0
|
||||
|
||||
|
||||
def test_config_key_sets_budget(monkeypatch, tmp_path):
|
||||
agent = _make_agent(
|
||||
tmp_path, monkeypatch,
|
||||
config_body="agent:\n run_budget_seconds: 750\n",
|
||||
)
|
||||
assert agent.run_budget_seconds == 750.0
|
||||
|
||||
|
||||
def test_constructor_arg_wins_over_config(monkeypatch, tmp_path):
|
||||
agent = _make_agent(
|
||||
tmp_path, monkeypatch,
|
||||
config_body="agent:\n run_budget_seconds: 750\n",
|
||||
run_budget_seconds=900,
|
||||
)
|
||||
assert agent.run_budget_seconds == 900.0
|
||||
|
||||
|
||||
# ── stale-timeout deadline scaling ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_no_budget_stale_timeout_unchanged(monkeypatch, tmp_path):
|
||||
"""Without a run budget the implicit 90s default is untouched."""
|
||||
import run_agent
|
||||
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
|
||||
agent = _make_agent(tmp_path, monkeypatch)
|
||||
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 90.0
|
||||
|
||||
|
||||
def test_active_budget_caps_implicit_reasoning_floor(monkeypatch, tmp_path):
|
||||
"""deepseek-v4-pro's 600s implicit floor yields to a tighter deadline cap.
|
||||
|
||||
900s budget, 800s elapsed -> remaining 100s -> cap = max(60, 50) = 60s.
|
||||
"""
|
||||
import run_agent
|
||||
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
|
||||
agent = _make_agent(
|
||||
tmp_path, monkeypatch,
|
||||
model="deepseek/deepseek-v4-pro",
|
||||
run_budget_seconds=900,
|
||||
)
|
||||
# Sanity: implicit reasoning floor is 600s without a running clock.
|
||||
base, implicit = agent._resolved_api_call_stale_timeout_base()
|
||||
assert base == 600.0 and implicit is False
|
||||
|
||||
agent._run_budget_started_at = time.time() - 800
|
||||
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 60.0
|
||||
|
||||
|
||||
def test_active_budget_cap_half_remaining(monkeypatch, tmp_path):
|
||||
"""Cap is remaining * 0.5 when that exceeds the 60s floor."""
|
||||
import run_agent
|
||||
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
|
||||
agent = _make_agent(
|
||||
tmp_path, monkeypatch,
|
||||
model="deepseek/deepseek-v4-pro",
|
||||
run_budget_seconds=900,
|
||||
)
|
||||
agent._run_budget_started_at = time.time() - 100 # remaining ~800 -> cap ~400
|
||||
timeout = agent._compute_non_stream_stale_timeout({"input": "hi"})
|
||||
assert 395.0 <= timeout <= 400.0
|
||||
|
||||
|
||||
def test_active_budget_never_raises_timeout(monkeypatch, tmp_path):
|
||||
"""The deadline cap NEVER loosens an already-tighter implicit timeout.
|
||||
|
||||
90s default with lots of remaining budget (cap would be ~445s) -> stays 90s.
|
||||
"""
|
||||
import run_agent
|
||||
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
|
||||
agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900)
|
||||
agent._run_budget_started_at = time.time() - 10
|
||||
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 90.0
|
||||
|
||||
|
||||
def test_explicit_provider_config_wins_over_budget_cap(monkeypatch, tmp_path):
|
||||
"""Explicit stale_timeout_seconds is never capped by the run budget."""
|
||||
import run_agent
|
||||
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: 1800.0)
|
||||
agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900)
|
||||
agent._run_budget_started_at = time.time() - 800
|
||||
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 1800.0
|
||||
|
||||
|
||||
def test_explicit_env_var_wins_over_budget_cap(monkeypatch, tmp_path):
|
||||
"""HERMES_API_CALL_STALE_TIMEOUT is explicit config — never capped."""
|
||||
import run_agent
|
||||
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
|
||||
agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900)
|
||||
monkeypatch.setenv("HERMES_API_CALL_STALE_TIMEOUT", "1200")
|
||||
agent._run_budget_started_at = time.time() - 800
|
||||
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 1200.0
|
||||
|
||||
|
||||
def test_budget_without_started_clock_is_inert(monkeypatch, tmp_path):
|
||||
"""A configured budget with no running turn clock changes nothing."""
|
||||
import run_agent
|
||||
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
|
||||
agent = _make_agent(
|
||||
tmp_path, monkeypatch,
|
||||
model="deepseek/deepseek-v4-pro",
|
||||
run_budget_seconds=900,
|
||||
)
|
||||
agent._run_budget_started_at = None
|
||||
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 600.0
|
||||
|
||||
|
||||
# ── wrap-up injection one-time-ness ────────────────────────────────────────
|
||||
|
||||
|
||||
class _StubAgent:
|
||||
def __init__(self, budget=None, started=None):
|
||||
self.run_budget_seconds = budget
|
||||
self._run_budget_started_at = started
|
||||
self._run_budget_wrapup_injected = False
|
||||
|
||||
|
||||
def _tool_messages():
|
||||
return [
|
||||
{"role": "user", "content": "do the task"},
|
||||
{"role": "assistant", "content": "", "tool_calls": [{"id": "t1"}]},
|
||||
{"role": "tool", "tool_call_id": "t1", "content": "result one"},
|
||||
{"role": "assistant", "content": "", "tool_calls": [{"id": "t2"}]},
|
||||
{"role": "tool", "tool_call_id": "t2", "content": "result two"},
|
||||
]
|
||||
|
||||
|
||||
def test_wrapup_not_injected_when_unset():
|
||||
from agent.conversation_loop import _maybe_inject_run_budget_wrapup
|
||||
agent = _StubAgent(budget=None, started=time.time() - 10_000)
|
||||
messages = _tool_messages()
|
||||
assert _maybe_inject_run_budget_wrapup(agent, messages) is False
|
||||
assert messages == _tool_messages()
|
||||
|
||||
|
||||
def test_wrapup_not_injected_before_threshold():
|
||||
from agent.conversation_loop import _maybe_inject_run_budget_wrapup
|
||||
agent = _StubAgent(budget=900, started=time.time() - 100) # 11% elapsed
|
||||
messages = _tool_messages()
|
||||
assert _maybe_inject_run_budget_wrapup(agent, messages) is False
|
||||
assert agent._run_budget_wrapup_injected is False
|
||||
|
||||
|
||||
def test_wrapup_injected_once_after_threshold():
|
||||
from agent.conversation_loop import (
|
||||
RUN_BUDGET_WRAPUP_NOTICE,
|
||||
_maybe_inject_run_budget_wrapup,
|
||||
)
|
||||
agent = _StubAgent(budget=900, started=time.time() - 800) # 89% elapsed
|
||||
messages = _tool_messages()
|
||||
assert _maybe_inject_run_budget_wrapup(agent, messages) is True
|
||||
assert agent._run_budget_wrapup_injected is True
|
||||
# Appended to the NEWEST tool message; earlier messages untouched.
|
||||
assert RUN_BUDGET_WRAPUP_NOTICE in messages[-1]["content"]
|
||||
assert messages[-1]["content"].startswith("result two")
|
||||
assert messages[2]["content"] == "result one"
|
||||
# No synthetic user message was inserted.
|
||||
assert [m["role"] for m in messages] == [
|
||||
"user", "assistant", "tool", "assistant", "tool",
|
||||
]
|
||||
|
||||
# Second call: latched, no re-injection.
|
||||
snapshot = [dict(m) for m in messages]
|
||||
assert _maybe_inject_run_budget_wrapup(agent, messages) is False
|
||||
assert messages == snapshot
|
||||
assert messages[-1]["content"].count(RUN_BUDGET_WRAPUP_NOTICE) == 1
|
||||
|
||||
|
||||
def test_wrapup_retries_when_no_tool_message_yet():
|
||||
"""First iteration (no tool results) can't inject; the latch stays open
|
||||
so the next iteration with a tool result delivers the notice."""
|
||||
from agent.conversation_loop import (
|
||||
RUN_BUDGET_WRAPUP_NOTICE,
|
||||
_maybe_inject_run_budget_wrapup,
|
||||
)
|
||||
agent = _StubAgent(budget=900, started=time.time() - 800)
|
||||
messages = [{"role": "user", "content": "do the task"}]
|
||||
assert _maybe_inject_run_budget_wrapup(agent, messages) is False
|
||||
assert agent._run_budget_wrapup_injected is False
|
||||
|
||||
messages += [
|
||||
{"role": "assistant", "content": "", "tool_calls": [{"id": "t1"}]},
|
||||
{"role": "tool", "tool_call_id": "t1", "content": "result"},
|
||||
]
|
||||
assert _maybe_inject_run_budget_wrapup(agent, messages) is True
|
||||
assert RUN_BUDGET_WRAPUP_NOTICE in messages[-1]["content"]
|
||||
|
||||
|
||||
def test_wrapup_not_injected_without_turn_clock():
|
||||
from agent.conversation_loop import _maybe_inject_run_budget_wrapup
|
||||
agent = _StubAgent(budget=900, started=None)
|
||||
messages = _tool_messages()
|
||||
assert _maybe_inject_run_budget_wrapup(agent, messages) is False
|
||||
|
||||
|
||||
def test_wrapup_multimodal_tool_content():
|
||||
"""Content-blocks tool results get a text block appended, not clobbered."""
|
||||
from agent.conversation_loop import (
|
||||
RUN_BUDGET_WRAPUP_NOTICE,
|
||||
_maybe_inject_run_budget_wrapup,
|
||||
)
|
||||
agent = _StubAgent(budget=900, started=time.time() - 800)
|
||||
messages = [
|
||||
{"role": "user", "content": "task"},
|
||||
{"role": "assistant", "content": "", "tool_calls": [{"id": "t1"}]},
|
||||
{"role": "tool", "tool_call_id": "t1",
|
||||
"content": [{"type": "text", "text": "block"}]},
|
||||
]
|
||||
assert _maybe_inject_run_budget_wrapup(agent, messages) is True
|
||||
blocks = messages[-1]["content"]
|
||||
assert blocks[0] == {"type": "text", "text": "block"}
|
||||
assert blocks[-1] == {"type": "text", "text": RUN_BUDGET_WRAPUP_NOTICE}
|
||||
|
||||
|
||||
# ── turn clock stamping ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_turn_clock_stamped_only_with_budget(monkeypatch, tmp_path):
|
||||
"""prepare-turn stamps the clock iff a budget is set, and resets the latch."""
|
||||
agent_with = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900)
|
||||
agent_with._run_budget_wrapup_injected = True
|
||||
|
||||
# Mirror the turn_context.prepare block (unit-level: run the same logic).
|
||||
for agent in (agent_with,):
|
||||
if getattr(agent, "run_budget_seconds", None):
|
||||
agent._run_budget_started_at = time.time()
|
||||
else:
|
||||
agent._run_budget_started_at = None
|
||||
agent._run_budget_wrapup_injected = False
|
||||
|
||||
assert agent_with._run_budget_started_at is not None
|
||||
assert agent_with._run_budget_wrapup_injected is False
|
||||
@@ -1031,6 +1031,28 @@ When the iteration budget is fully exhausted, the CLI shows a notification to th
|
||||
|
||||
`agent.api_max_retries` controls how many times Hermes retries a provider API call on transient errors (rate limits, connection drops, 5xx) **before** fallback-provider switching engages. The default is `3` — four attempts total. If you have [fallback providers](/user-guide/features/fallback-providers) configured and want to fail over faster, drop this to `0` so the first transient error on your primary immediately hands off to the fallback instead of churning retries against the flaky endpoint.
|
||||
|
||||
## Wall-Clock Run Budget
|
||||
|
||||
Separate from the iteration budget, you can give each conversation run an optional **wall-clock** budget. This is designed for one-shot and eval-harness invocations that run under a hard external ceiling (e.g. a 900-second per-task limit): without it, a run can time out with the work essentially done — one generation short of emitting the final answer, or stuck in a single hung provider call.
|
||||
|
||||
```yaml
|
||||
agent:
|
||||
run_budget_seconds: null # Optional; unset/null = feature fully off (default)
|
||||
```
|
||||
|
||||
Or per-invocation via the CLI:
|
||||
|
||||
```bash
|
||||
hermes chat --run-budget 850 -q "..."
|
||||
```
|
||||
|
||||
When a budget is set, two things happen:
|
||||
|
||||
1. **Wrap-up notice at 80%.** When 80% of the budget has elapsed, Hermes injects a **one-time** notice (delivered cache-safely, appended to the newest tool result like `/steer` messages) telling the model to stop new discovery/verification work and produce the final deliverable from the state it already has. It fires at most once per run and mirrors the existing iteration-budget wrap-up mechanism — there are no repeated pressure warnings.
|
||||
2. **Deadline-scaled stale timeouts.** Implicit non-streaming stale timeouts (the 90s default and the reasoning-model floors, e.g. 600s for DeepSeek reasoning models) are capped at `max(60, remaining_budget × 0.5)` so a single silently-hung provider call can never consume the rest of the run. The cap only ever *tightens* the timeout — it never raises it — and an explicitly configured `stale_timeout_seconds` (provider/model config or `HERMES_API_CALL_STALE_TIMEOUT`) always wins untouched.
|
||||
|
||||
The budget is per `run_conversation` turn (it resets on each user message) and the feature is completely dormant when unset — no clock reads, no injection, no timeout changes.
|
||||
|
||||
## Verify-on-Stop (coding verification)
|
||||
|
||||
When enabled, Hermes refuses to accept a final answer on a turn where the agent edited code in a workspace but produced no fresh verification evidence (a passing test run, build, lint, etc.) — it injects a synthetic follow-up asking the agent to verify or explain why it can't. Doc/markdown/skill-only edits never trigger it, and the loop is bounded so it can never trap the agent.
|
||||
|
||||
Reference in New Issue
Block a user