From 803397ecc3d8fd9eb3d128b867db52a8a2d6db08 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:49:46 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20wall-clock=20run=20budget=20=E2=80=94?= =?UTF-8?q?=20wrap-up=20injection=20at=2080%=20and=20deadline-scaled=20sta?= =?UTF-8?q?le=20timeouts=20(agent.run=5Fbudget=5Fseconds=20/=20--run-budge?= =?UTF-8?q?t)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/agent_init.py | 39 +++ agent/conversation_loop.py | 67 +++++ agent/turn_context.py | 9 + cli.py | 10 + hermes_cli/_parser.py | 15 ++ hermes_cli/cli_agent_setup_mixin.py | 1 + hermes_cli/config_defaults.py | 6 + hermes_cli/main.py | 1 + run_agent.py | 40 ++- tests/agent/test_run_budget.py | 324 +++++++++++++++++++++++ website/docs/user-guide/configuration.md | 22 ++ 11 files changed, 530 insertions(+), 4 deletions(-) create mode 100644 tests/agent/test_run_budget.py diff --git a/agent/agent_init.py b/agent/agent_init.py index 60683fcafc..25cef45e0b 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -490,6 +490,25 @@ def _merge_custom_provider_extra_body(agent, custom_providers: List[Dict[str, An agent.request_overrides = overrides +def _normalize_run_budget_seconds(value) -> Optional[float]: + """Normalize a wall-clock run budget value to a positive float or None. + + None / absent / non-numeric / non-positive all resolve to ``None`` + (feature off) so a malformed config value can never activate the + deadline machinery, only leave it dormant. ``bool`` is rejected because + YAML ``true`` would otherwise become a 1-second budget. + """ + if value is None or isinstance(value, bool): + return None + try: + seconds = float(value) + except (TypeError, ValueError): + return None + if seconds != seconds or seconds <= 0: # NaN or non-positive + return None + return seconds + + def init_agent( agent, base_url: str = None, @@ -560,6 +579,7 @@ def init_agent( session_db=None, parent_session_id: str = None, iteration_budget: "IterationBudget" = None, + run_budget_seconds: Optional[float] = None, fallback_model: Dict[str, Any] = None, credential_pool=None, checkpoints_enabled: bool = False, @@ -972,6 +992,17 @@ def init_agent( agent._budget_exhausted_injected = False agent._budget_grace_call = False + # Optional wall-clock run budget (seconds per run_conversation turn). + # Explicit constructor arg wins; else resolved from config.yaml + # (agent.run_budget_seconds) further below. None = feature fully off: + # no clock reads, no injection, no stale-timeout capping. + agent.run_budget_seconds = _normalize_run_budget_seconds(run_budget_seconds) + # Wall-clock start of the CURRENT run_conversation turn. Set by + # turn_context.prepare_turn when a run budget is active; None otherwise. + agent._run_budget_started_at = None + # One-shot latch for the 80% wrap-up notice (reset each turn). + agent._run_budget_wrapup_injected = False + # Activity tracking — updated on each API call, tool execution, and # stream chunk. Used by the gateway timeout handler to report what the # agent was doing when it was killed, and by the "still working" @@ -1902,6 +1933,14 @@ def init_agent( # agent/system_prompt.py for the injection gate. agent._execution_guidance = _agent_section.get("execution_guidance", "auto") + # Wall-clock run budget from config (agent.run_budget_seconds) — only + # consulted when the constructor arg was not given. Absent/None/invalid + # keeps the feature fully off (zero behavior change in the default path). + if agent.run_budget_seconds is None: + agent.run_budget_seconds = _normalize_run_budget_seconds( + _agent_section.get("run_budget_seconds") + ) + # Empty-response retry guard config (NS-503): additive # ``agent.empty_response_guard`` subsection. Resolution is tolerant — # a malformed section falls back to the schema defaults (guard on, diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index ab8e60b6d0..e15a7d3c90 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -107,6 +107,64 @@ logger = logging.getLogger(__name__) _INTERRUPT_SCAFFOLD_MARKER = "[This response was interrupted by a user correction.]" +# One-time wrap-up notice appended when a wall-clock run budget crosses its +# 80% threshold (agent.run_budget_seconds / --run-budget). Mirrors the Codex +# CLI budget wrap-up template: stop new work, deliver from current state. +RUN_BUDGET_WRAPUP_NOTICE = ( + "[SYSTEM NOTICE — run time budget nearly exhausted] " + "Run time budget nearly exhausted. Stop new discovery/verification work " + "now. Produce the required final deliverable (answer/JSON/summary) from " + "the state you already have, completing only mandatory writes." +) + + +def _maybe_inject_run_budget_wrapup(agent: Any, messages: List[Dict[str, Any]]) -> bool: + """Inject the one-time wall-clock wrap-up notice when past 80% of budget. + + Cache-safe delivery: the notice is appended to the NEWEST ``role:"tool"`` + message (the same channel /steer uses) — no synthetic user message is + inserted mid-loop and no past context is rewritten, so role alternation + and the prompt-cache prefix survive. Latches ``_run_budget_wrapup_injected`` + only on a successful append, so a first iteration without tool results + retries on the next iteration. Returns True when the notice was injected. + + Dormant unless ``agent.run_budget_seconds`` is set AND the turn stamped + ``_run_budget_started_at`` (see ``turn_context.prepare_conversation_turn``). + """ + budget = getattr(agent, "run_budget_seconds", None) + if not budget: + return False + if getattr(agent, "_run_budget_wrapup_injected", False): + return False + started = getattr(agent, "_run_budget_started_at", None) + if not started: + return False + if (time.time() - started) < 0.8 * float(budget): + return False + for i in range(len(messages) - 1, -1, -1): + msg = messages[i] + if isinstance(msg, dict) and msg.get("role") == "tool": + existing = msg.get("content", "") + if isinstance(existing, str): + msg["content"] = existing + f"\n\n{RUN_BUDGET_WRAPUP_NOTICE}" + else: + # Multimodal content blocks — append a text block. + try: + blocks = list(existing) if existing else [] + blocks.append({"type": "text", "text": RUN_BUDGET_WRAPUP_NOTICE}) + msg["content"] = blocks + except Exception: + return False + agent._run_budget_wrapup_injected = True + logger.info( + "Run budget wrap-up notice injected (budget=%.0fs, elapsed=%.0fs)", + float(budget), + time.time() - started, + ) + return True + return False + + def _restore_user_after_reference_handoff( messages: List[Dict[str, Any]], user_message: Any ) -> bool: @@ -2041,6 +2099,15 @@ def run_conversation( existing = getattr(agent, "_pending_steer", None) agent._pending_steer = (existing + "\n" + _pre_api_steer) if existing else _pre_api_steer + # ── Wall-clock run-budget wrap-up notice ─────────────────────── + # One-shot: when a run budget (agent.run_budget_seconds / + # --run-budget) is active and 80% of it has elapsed, ask the model + # to wrap up and deliver from the state it already has. Same + # cache-safe channel as /steer (appended to the newest tool + # result); dormant when no budget is set. + if getattr(agent, "run_budget_seconds", None): + _maybe_inject_run_budget_wrapup(agent, messages) + # Prepare messages for API call # If we have an ephemeral system prompt, prepend it to the messages # Note: Reasoning is embedded in content via tags for trajectory storage. diff --git a/agent/turn_context.py b/agent/turn_context.py index a90bee9a1a..650cfe7679 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -605,6 +605,15 @@ def build_turn_context( # NOTE: _turns_since_memory and _iters_since_skill are NOT reset here. agent.iteration_budget = IterationBudget(agent.max_iterations) + # Wall-clock run budget: per-run_conversation clock. Only stamped when a + # budget is configured so the default path stays clock-free; the wrap-up + # latch resets each turn (one notice per run, not per session). + if getattr(agent, "run_budget_seconds", None): + agent._run_budget_started_at = time.time() + else: + agent._run_budget_started_at = None + agent._run_budget_wrapup_injected = False + # Log conversation turn start for debugging/observability. _preview_text = summarize_user_message_for_log(user_message) _msg_preview = (_preview_text[:80] + "...") if len(_preview_text) > 80 else _preview_text diff --git a/cli.py b/cli.py index 09ea295745..a0e0e4ff79 100644 --- a/cli.py +++ b/cli.py @@ -4858,6 +4858,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): api_key: str = None, base_url: str = None, max_turns: int = None, + run_budget: float = None, verbose: Optional[bool] = None, compact: bool = False, resume: str = None, @@ -5125,6 +5126,13 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): self.max_turns = 500 else: self.max_turns = 500 + + # Wall-clock run budget: CLI flag wins over config; both optional. + # None keeps the feature fully off (AIAgent stays dormant). + if run_budget is not None: + self.run_budget_seconds = run_budget + else: + self.run_budget_seconds = CLI_CONFIG["agent"].get("run_budget_seconds") # Parse and validate toolsets self.enabled_toolsets = toolsets @@ -20174,6 +20182,7 @@ def main( api_key: str = None, base_url: str = None, max_turns: int = None, + run_budget: float = None, verbose: Optional[bool] = None, quiet: bool = False, compact: bool = False, @@ -20370,6 +20379,7 @@ def main( api_key=api_key, base_url=base_url, max_turns=max_turns, + run_budget=run_budget, verbose=verbose, compact=compact, resume=resume, diff --git a/hermes_cli/_parser.py b/hermes_cli/_parser.py index c6b4e7cd55..13bcaa4703 100644 --- a/hermes_cli/_parser.py +++ b/hermes_cli/_parser.py @@ -459,6 +459,21 @@ def build_top_level_parser(): metavar="N", help="Maximum tool-calling iterations per conversation turn (default: 500, or agent.max_turns in config)", ) + chat_parser.add_argument( + "--run-budget", + type=float, + default=None, + metavar="SECONDS", + dest="run_budget", + help=( + "Optional wall-clock budget in seconds for each conversation run. " + "At 80%% elapsed the agent gets a one-time wrap-up notice, and " + "implicit provider stale timeouts are capped to the remaining " + "budget so one hung call can't consume the run. Unset = off. " + "Also configurable as agent.run_budget_seconds in config.yaml. " + "Intended for one-shot/eval invocations with a hard ceiling." + ), + ) _inherited_flag( chat_parser, "--yolo", diff --git a/hermes_cli/cli_agent_setup_mixin.py b/hermes_cli/cli_agent_setup_mixin.py index 8fea239d01..7a1082431c 100644 --- a/hermes_cli/cli_agent_setup_mixin.py +++ b/hermes_cli/cli_agent_setup_mixin.py @@ -500,6 +500,7 @@ class CLIAgentSetupMixin: credential_pool=runtime.get("credential_pool"), max_tokens=self.max_tokens, max_iterations=self.max_turns, + run_budget_seconds=getattr(self, "run_budget_seconds", None), enabled_toolsets=self.enabled_toolsets, disabled_toolsets=self.disabled_toolsets, verbose_logging=self.verbose, diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 5a6173d6d2..6c287725b8 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -44,6 +44,12 @@ DEFAULT_CONFIG = { }, "agent": { "max_turns": 500, + # Optional wall-clock budget in seconds per conversation run. + # null/absent = feature fully off (zero behavior change). When set, + # the agent gets a one-time wrap-up notice at 80% elapsed and + # implicit provider stale timeouts are capped to the remaining + # budget. CLI one-shot equivalent: `hermes chat --run-budget N`. + "run_budget_seconds": None, # Inactivity timeout for gateway agent execution (seconds). # The agent can run indefinitely as long as it's actively calling # tools or receiving API responses. Only fires when the agent has diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 255a729803..b061c4e352 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -3220,6 +3220,7 @@ def cmd_chat(args): "checkpoints": getattr(args, "checkpoints", False), "pass_session_id": getattr(args, "pass_session_id", False), "max_turns": getattr(args, "max_turns", None), + "run_budget": getattr(args, "run_budget", None), "ignore_rules": getattr(args, "ignore_rules", False) or getattr(args, "safe_mode", False), "ignore_user_config": getattr(args, "ignore_user_config", False) or getattr(args, "safe_mode", False), "compact": getattr(args, "compact", False), diff --git a/run_agent.py b/run_agent.py index ad8dc8d65d..e8161e5f21 100644 --- a/run_agent.py +++ b/run_agent.py @@ -503,6 +503,7 @@ class AIAgent: session_db=None, parent_session_id: str = None, iteration_budget: "IterationBudget" = None, + run_budget_seconds: Optional[float] = None, fallback_model: Dict[str, Any] = None, credential_pool=None, checkpoints_enabled: bool = False, @@ -592,6 +593,7 @@ class AIAgent: session_db=session_db, parent_session_id=parent_session_id, iteration_budget=iteration_budget, + run_budget_seconds=run_budget_seconds, fallback_model=fallback_model, credential_pool=credential_pool, checkpoints_enabled=checkpoints_enabled, @@ -1452,10 +1454,40 @@ class AIAgent: from agent.chat_completion_helpers import estimate_request_context_tokens est_tokens = estimate_request_context_tokens(api_payload) if est_tokens > 100_000: - return max(stale_base, 240.0) - if est_tokens > 50_000: - return max(stale_base, 150.0) - return stale_base + timeout = max(stale_base, 240.0) + elif est_tokens > 50_000: + timeout = max(stale_base, 150.0) + else: + timeout = stale_base + + # Wall-clock run budget cap: when a run budget is active, an implicit + # (floor-/default-derived) stale timeout is capped at half the + # remaining budget (>= 60s) so a single hung provider call cannot eat + # the whole run — e.g. deepseek-v4-pro's 600s reasoning floor inside a + # 900s eval ceiling. NEVER raises the timeout above what it would + # otherwise be, and an explicit user-configured stale_timeout_seconds + # (or env var) still wins untouched. + run_budget = getattr(self, "run_budget_seconds", None) + if run_budget and not self._stale_timeout_is_explicit(): + started = getattr(self, "_run_budget_started_at", None) + if started: + remaining = float(run_budget) - (time.time() - started) + deadline_cap = max(60.0, remaining * 0.5) + if deadline_cap < timeout: + timeout = deadline_cap + return timeout + + def _stale_timeout_is_explicit(self) -> bool: + """True when the user explicitly configured the non-stream stale timeout. + + Explicit = provider/model ``stale_timeout_seconds`` in config.yaml or + the ``HERMES_API_CALL_STALE_TIMEOUT`` env var. Reasoning-model floors + and the 90s default are implicit — they yield to the wall-clock run + budget cap; explicit user configuration never does. + """ + if get_provider_stale_timeout(self.provider, self.model) is not None: + return True + return os.getenv("HERMES_API_CALL_STALE_TIMEOUT") is not None def _codex_silent_hang_hint(self, model: Optional[str] = None) -> Optional[str]: """Return an actionable hint when this request matches a known diff --git a/tests/agent/test_run_budget.py b/tests/agent/test_run_budget.py new file mode 100644 index 0000000000..d602bb3d0d --- /dev/null +++ b/tests/agent/test_run_budget.py @@ -0,0 +1,324 @@ +"""Tests for the optional wall-clock run budget (agent.run_budget_seconds / --run-budget). + +Covers: + +1. Stale-timeout deadline scaling in + ``run_agent.py:AIAgent._compute_non_stream_stale_timeout``: + - an active run budget CAPS an implicit stale timeout (default 90s and + reasoning floors like deepseek's 600s) at ``max(60, remaining * 0.5)``; + - the cap never RAISES the timeout above what it would otherwise be; + - explicit user configuration (provider ``stale_timeout_seconds`` or the + ``HERMES_API_CALL_STALE_TIMEOUT`` env var) always wins untouched; + - no budget => completely unchanged behavior. + +2. One-time-ness of the 80% wrap-up notice injection in + ``agent.conversation_loop._maybe_inject_run_budget_wrapup``: + - fires once (latched), not repeatedly; + - never fires when no budget is set or before the 80% threshold; + - appended to the newest tool message (cache-safe /steer channel), no + synthetic user message. + +3. Normalization of the config/CLI value + (``agent.agent_init._normalize_run_budget_seconds``): dormant on + null/invalid/non-positive input. +""" + +from __future__ import annotations + +import time +from pathlib import Path + +import pytest + + +def _write_config(tmp_path: Path, body: str) -> None: + (tmp_path / "config.yaml").write_text(body or "{}\n", encoding="utf-8") + + +def _make_agent(tmp_path, monkeypatch, config_body: str = "", **overrides): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + (tmp_path / ".env").write_text("", encoding="utf-8") + monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False) + _write_config(tmp_path, config_body) + + from run_agent import AIAgent + kwargs = dict( + model="gpt-5.5", + provider="openai", + api_key="sk-dummy", + base_url="https://api.openai.com/v1", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + platform="cli", + ) + kwargs.update(overrides) + return AIAgent(**kwargs) + + +# ── normalization ────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("raw,expected", [ + (None, None), + (0, None), + (-5, None), + ("abc", None), + (True, None), # YAML `true` must not become a 1-second budget + (False, None), + (float("nan"), None), + (900, 900.0), + ("850", 850.0), + (0.5, 0.5), +]) +def test_normalize_run_budget_seconds(raw, expected): + from agent.agent_init import _normalize_run_budget_seconds + assert _normalize_run_budget_seconds(raw) == expected + + +# ── constructor / config plumbing ───────────────────────────────────────── + + +def test_no_budget_by_default(monkeypatch, tmp_path): + agent = _make_agent(tmp_path, monkeypatch) + assert agent.run_budget_seconds is None + assert agent._run_budget_started_at is None + assert agent._run_budget_wrapup_injected is False + + +def test_constructor_arg_sets_budget(monkeypatch, tmp_path): + agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900) + assert agent.run_budget_seconds == 900.0 + + +def test_config_key_sets_budget(monkeypatch, tmp_path): + agent = _make_agent( + tmp_path, monkeypatch, + config_body="agent:\n run_budget_seconds: 750\n", + ) + assert agent.run_budget_seconds == 750.0 + + +def test_constructor_arg_wins_over_config(monkeypatch, tmp_path): + agent = _make_agent( + tmp_path, monkeypatch, + config_body="agent:\n run_budget_seconds: 750\n", + run_budget_seconds=900, + ) + assert agent.run_budget_seconds == 900.0 + + +# ── stale-timeout deadline scaling ───────────────────────────────────────── + + +def test_no_budget_stale_timeout_unchanged(monkeypatch, tmp_path): + """Without a run budget the implicit 90s default is untouched.""" + import run_agent + monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None) + agent = _make_agent(tmp_path, monkeypatch) + assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 90.0 + + +def test_active_budget_caps_implicit_reasoning_floor(monkeypatch, tmp_path): + """deepseek-v4-pro's 600s implicit floor yields to a tighter deadline cap. + + 900s budget, 800s elapsed -> remaining 100s -> cap = max(60, 50) = 60s. + """ + import run_agent + monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None) + agent = _make_agent( + tmp_path, monkeypatch, + model="deepseek/deepseek-v4-pro", + run_budget_seconds=900, + ) + # Sanity: implicit reasoning floor is 600s without a running clock. + base, implicit = agent._resolved_api_call_stale_timeout_base() + assert base == 600.0 and implicit is False + + agent._run_budget_started_at = time.time() - 800 + assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 60.0 + + +def test_active_budget_cap_half_remaining(monkeypatch, tmp_path): + """Cap is remaining * 0.5 when that exceeds the 60s floor.""" + import run_agent + monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None) + agent = _make_agent( + tmp_path, monkeypatch, + model="deepseek/deepseek-v4-pro", + run_budget_seconds=900, + ) + agent._run_budget_started_at = time.time() - 100 # remaining ~800 -> cap ~400 + timeout = agent._compute_non_stream_stale_timeout({"input": "hi"}) + assert 395.0 <= timeout <= 400.0 + + +def test_active_budget_never_raises_timeout(monkeypatch, tmp_path): + """The deadline cap NEVER loosens an already-tighter implicit timeout. + + 90s default with lots of remaining budget (cap would be ~445s) -> stays 90s. + """ + import run_agent + monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None) + agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900) + agent._run_budget_started_at = time.time() - 10 + assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 90.0 + + +def test_explicit_provider_config_wins_over_budget_cap(monkeypatch, tmp_path): + """Explicit stale_timeout_seconds is never capped by the run budget.""" + import run_agent + monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: 1800.0) + agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900) + agent._run_budget_started_at = time.time() - 800 + assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 1800.0 + + +def test_explicit_env_var_wins_over_budget_cap(monkeypatch, tmp_path): + """HERMES_API_CALL_STALE_TIMEOUT is explicit config — never capped.""" + import run_agent + monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None) + agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900) + monkeypatch.setenv("HERMES_API_CALL_STALE_TIMEOUT", "1200") + agent._run_budget_started_at = time.time() - 800 + assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 1200.0 + + +def test_budget_without_started_clock_is_inert(monkeypatch, tmp_path): + """A configured budget with no running turn clock changes nothing.""" + import run_agent + monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None) + agent = _make_agent( + tmp_path, monkeypatch, + model="deepseek/deepseek-v4-pro", + run_budget_seconds=900, + ) + agent._run_budget_started_at = None + assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 600.0 + + +# ── wrap-up injection one-time-ness ──────────────────────────────────────── + + +class _StubAgent: + def __init__(self, budget=None, started=None): + self.run_budget_seconds = budget + self._run_budget_started_at = started + self._run_budget_wrapup_injected = False + + +def _tool_messages(): + return [ + {"role": "user", "content": "do the task"}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "t1"}]}, + {"role": "tool", "tool_call_id": "t1", "content": "result one"}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "t2"}]}, + {"role": "tool", "tool_call_id": "t2", "content": "result two"}, + ] + + +def test_wrapup_not_injected_when_unset(): + from agent.conversation_loop import _maybe_inject_run_budget_wrapup + agent = _StubAgent(budget=None, started=time.time() - 10_000) + messages = _tool_messages() + assert _maybe_inject_run_budget_wrapup(agent, messages) is False + assert messages == _tool_messages() + + +def test_wrapup_not_injected_before_threshold(): + from agent.conversation_loop import _maybe_inject_run_budget_wrapup + agent = _StubAgent(budget=900, started=time.time() - 100) # 11% elapsed + messages = _tool_messages() + assert _maybe_inject_run_budget_wrapup(agent, messages) is False + assert agent._run_budget_wrapup_injected is False + + +def test_wrapup_injected_once_after_threshold(): + from agent.conversation_loop import ( + RUN_BUDGET_WRAPUP_NOTICE, + _maybe_inject_run_budget_wrapup, + ) + agent = _StubAgent(budget=900, started=time.time() - 800) # 89% elapsed + messages = _tool_messages() + assert _maybe_inject_run_budget_wrapup(agent, messages) is True + assert agent._run_budget_wrapup_injected is True + # Appended to the NEWEST tool message; earlier messages untouched. + assert RUN_BUDGET_WRAPUP_NOTICE in messages[-1]["content"] + assert messages[-1]["content"].startswith("result two") + assert messages[2]["content"] == "result one" + # No synthetic user message was inserted. + assert [m["role"] for m in messages] == [ + "user", "assistant", "tool", "assistant", "tool", + ] + + # Second call: latched, no re-injection. + snapshot = [dict(m) for m in messages] + assert _maybe_inject_run_budget_wrapup(agent, messages) is False + assert messages == snapshot + assert messages[-1]["content"].count(RUN_BUDGET_WRAPUP_NOTICE) == 1 + + +def test_wrapup_retries_when_no_tool_message_yet(): + """First iteration (no tool results) can't inject; the latch stays open + so the next iteration with a tool result delivers the notice.""" + from agent.conversation_loop import ( + RUN_BUDGET_WRAPUP_NOTICE, + _maybe_inject_run_budget_wrapup, + ) + agent = _StubAgent(budget=900, started=time.time() - 800) + messages = [{"role": "user", "content": "do the task"}] + assert _maybe_inject_run_budget_wrapup(agent, messages) is False + assert agent._run_budget_wrapup_injected is False + + messages += [ + {"role": "assistant", "content": "", "tool_calls": [{"id": "t1"}]}, + {"role": "tool", "tool_call_id": "t1", "content": "result"}, + ] + assert _maybe_inject_run_budget_wrapup(agent, messages) is True + assert RUN_BUDGET_WRAPUP_NOTICE in messages[-1]["content"] + + +def test_wrapup_not_injected_without_turn_clock(): + from agent.conversation_loop import _maybe_inject_run_budget_wrapup + agent = _StubAgent(budget=900, started=None) + messages = _tool_messages() + assert _maybe_inject_run_budget_wrapup(agent, messages) is False + + +def test_wrapup_multimodal_tool_content(): + """Content-blocks tool results get a text block appended, not clobbered.""" + from agent.conversation_loop import ( + RUN_BUDGET_WRAPUP_NOTICE, + _maybe_inject_run_budget_wrapup, + ) + agent = _StubAgent(budget=900, started=time.time() - 800) + messages = [ + {"role": "user", "content": "task"}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "t1"}]}, + {"role": "tool", "tool_call_id": "t1", + "content": [{"type": "text", "text": "block"}]}, + ] + assert _maybe_inject_run_budget_wrapup(agent, messages) is True + blocks = messages[-1]["content"] + assert blocks[0] == {"type": "text", "text": "block"} + assert blocks[-1] == {"type": "text", "text": RUN_BUDGET_WRAPUP_NOTICE} + + +# ── turn clock stamping ──────────────────────────────────────────────────── + + +def test_turn_clock_stamped_only_with_budget(monkeypatch, tmp_path): + """prepare-turn stamps the clock iff a budget is set, and resets the latch.""" + agent_with = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900) + agent_with._run_budget_wrapup_injected = True + + # Mirror the turn_context.prepare block (unit-level: run the same logic). + for agent in (agent_with,): + if getattr(agent, "run_budget_seconds", None): + agent._run_budget_started_at = time.time() + else: + agent._run_budget_started_at = None + agent._run_budget_wrapup_injected = False + + assert agent_with._run_budget_started_at is not None + assert agent_with._run_budget_wrapup_injected is False diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 7b9b43e455..4417f12df4 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1031,6 +1031,28 @@ When the iteration budget is fully exhausted, the CLI shows a notification to th `agent.api_max_retries` controls how many times Hermes retries a provider API call on transient errors (rate limits, connection drops, 5xx) **before** fallback-provider switching engages. The default is `3` — four attempts total. If you have [fallback providers](/user-guide/features/fallback-providers) configured and want to fail over faster, drop this to `0` so the first transient error on your primary immediately hands off to the fallback instead of churning retries against the flaky endpoint. +## Wall-Clock Run Budget + +Separate from the iteration budget, you can give each conversation run an optional **wall-clock** budget. This is designed for one-shot and eval-harness invocations that run under a hard external ceiling (e.g. a 900-second per-task limit): without it, a run can time out with the work essentially done — one generation short of emitting the final answer, or stuck in a single hung provider call. + +```yaml +agent: + run_budget_seconds: null # Optional; unset/null = feature fully off (default) +``` + +Or per-invocation via the CLI: + +```bash +hermes chat --run-budget 850 -q "..." +``` + +When a budget is set, two things happen: + +1. **Wrap-up notice at 80%.** When 80% of the budget has elapsed, Hermes injects a **one-time** notice (delivered cache-safely, appended to the newest tool result like `/steer` messages) telling the model to stop new discovery/verification work and produce the final deliverable from the state it already has. It fires at most once per run and mirrors the existing iteration-budget wrap-up mechanism — there are no repeated pressure warnings. +2. **Deadline-scaled stale timeouts.** Implicit non-streaming stale timeouts (the 90s default and the reasoning-model floors, e.g. 600s for DeepSeek reasoning models) are capped at `max(60, remaining_budget × 0.5)` so a single silently-hung provider call can never consume the rest of the run. The cap only ever *tightens* the timeout — it never raises it — and an explicitly configured `stale_timeout_seconds` (provider/model config or `HERMES_API_CALL_STALE_TIMEOUT`) always wins untouched. + +The budget is per `run_conversation` turn (it resets on each user message) and the feature is completely dormant when unset — no clock reads, no injection, no timeout changes. + ## Verify-on-Stop (coding verification) When enabled, Hermes refuses to accept a final answer on a turn where the agent edited code in a workspace but produced no fresh verification evidence (a passing test run, build, lint, etc.) — it injects a synthetic follow-up asking the agent to verify or explain why it can't. Doc/markdown/skill-only edits never trigger it, and the loop is bounded so it can never trap the agent.