feat: wall-clock run budget — wrap-up injection at 80% and deadline-scaled stale timeouts (agent.run_budget_seconds / --run-budget)

This commit is contained in:
Teknium
2026-08-19 15:49:46 -07:00
parent 4a83b03c3a
commit 803397ecc3
11 changed files with 530 additions and 4 deletions
+39
View File
@@ -490,6 +490,25 @@ def _merge_custom_provider_extra_body(agent, custom_providers: List[Dict[str, An
agent.request_overrides = overrides
def _normalize_run_budget_seconds(value) -> Optional[float]:
"""Normalize a wall-clock run budget value to a positive float or None.
None / absent / non-numeric / non-positive all resolve to ``None``
(feature off) so a malformed config value can never activate the
deadline machinery, only leave it dormant. ``bool`` is rejected because
YAML ``true`` would otherwise become a 1-second budget.
"""
if value is None or isinstance(value, bool):
return None
try:
seconds = float(value)
except (TypeError, ValueError):
return None
if seconds != seconds or seconds <= 0: # NaN or non-positive
return None
return seconds
def init_agent(
agent,
base_url: str = None,
@@ -560,6 +579,7 @@ def init_agent(
session_db=None,
parent_session_id: str = None,
iteration_budget: "IterationBudget" = None,
run_budget_seconds: Optional[float] = None,
fallback_model: Dict[str, Any] = None,
credential_pool=None,
checkpoints_enabled: bool = False,
@@ -972,6 +992,17 @@ def init_agent(
agent._budget_exhausted_injected = False
agent._budget_grace_call = False
# Optional wall-clock run budget (seconds per run_conversation turn).
# Explicit constructor arg wins; else resolved from config.yaml
# (agent.run_budget_seconds) further below. None = feature fully off:
# no clock reads, no injection, no stale-timeout capping.
agent.run_budget_seconds = _normalize_run_budget_seconds(run_budget_seconds)
# Wall-clock start of the CURRENT run_conversation turn. Set by
# turn_context.prepare_turn when a run budget is active; None otherwise.
agent._run_budget_started_at = None
# One-shot latch for the 80% wrap-up notice (reset each turn).
agent._run_budget_wrapup_injected = False
# Activity tracking — updated on each API call, tool execution, and
# stream chunk. Used by the gateway timeout handler to report what the
# agent was doing when it was killed, and by the "still working"
@@ -1902,6 +1933,14 @@ def init_agent(
# agent/system_prompt.py for the injection gate.
agent._execution_guidance = _agent_section.get("execution_guidance", "auto")
# Wall-clock run budget from config (agent.run_budget_seconds) — only
# consulted when the constructor arg was not given. Absent/None/invalid
# keeps the feature fully off (zero behavior change in the default path).
if agent.run_budget_seconds is None:
agent.run_budget_seconds = _normalize_run_budget_seconds(
_agent_section.get("run_budget_seconds")
)
# Empty-response retry guard config (NS-503): additive
# ``agent.empty_response_guard`` subsection. Resolution is tolerant —
# a malformed section falls back to the schema defaults (guard on,
+67
View File
@@ -107,6 +107,64 @@ logger = logging.getLogger(__name__)
_INTERRUPT_SCAFFOLD_MARKER = "[This response was interrupted by a user correction.]"
# One-time wrap-up notice appended when a wall-clock run budget crosses its
# 80% threshold (agent.run_budget_seconds / --run-budget). Mirrors the Codex
# CLI budget wrap-up template: stop new work, deliver from current state.
RUN_BUDGET_WRAPUP_NOTICE = (
"[SYSTEM NOTICE — run time budget nearly exhausted] "
"Run time budget nearly exhausted. Stop new discovery/verification work "
"now. Produce the required final deliverable (answer/JSON/summary) from "
"the state you already have, completing only mandatory writes."
)
def _maybe_inject_run_budget_wrapup(agent: Any, messages: List[Dict[str, Any]]) -> bool:
"""Inject the one-time wall-clock wrap-up notice when past 80% of budget.
Cache-safe delivery: the notice is appended to the NEWEST ``role:"tool"``
message (the same channel /steer uses) — no synthetic user message is
inserted mid-loop and no past context is rewritten, so role alternation
and the prompt-cache prefix survive. Latches ``_run_budget_wrapup_injected``
only on a successful append, so a first iteration without tool results
retries on the next iteration. Returns True when the notice was injected.
Dormant unless ``agent.run_budget_seconds`` is set AND the turn stamped
``_run_budget_started_at`` (see ``turn_context.prepare_conversation_turn``).
"""
budget = getattr(agent, "run_budget_seconds", None)
if not budget:
return False
if getattr(agent, "_run_budget_wrapup_injected", False):
return False
started = getattr(agent, "_run_budget_started_at", None)
if not started:
return False
if (time.time() - started) < 0.8 * float(budget):
return False
for i in range(len(messages) - 1, -1, -1):
msg = messages[i]
if isinstance(msg, dict) and msg.get("role") == "tool":
existing = msg.get("content", "")
if isinstance(existing, str):
msg["content"] = existing + f"\n\n{RUN_BUDGET_WRAPUP_NOTICE}"
else:
# Multimodal content blocks — append a text block.
try:
blocks = list(existing) if existing else []
blocks.append({"type": "text", "text": RUN_BUDGET_WRAPUP_NOTICE})
msg["content"] = blocks
except Exception:
return False
agent._run_budget_wrapup_injected = True
logger.info(
"Run budget wrap-up notice injected (budget=%.0fs, elapsed=%.0fs)",
float(budget),
time.time() - started,
)
return True
return False
def _restore_user_after_reference_handoff(
messages: List[Dict[str, Any]], user_message: Any
) -> bool:
@@ -2041,6 +2099,15 @@ def run_conversation(
existing = getattr(agent, "_pending_steer", None)
agent._pending_steer = (existing + "\n" + _pre_api_steer) if existing else _pre_api_steer
# ── Wall-clock run-budget wrap-up notice ───────────────────────
# One-shot: when a run budget (agent.run_budget_seconds /
# --run-budget) is active and 80% of it has elapsed, ask the model
# to wrap up and deliver from the state it already has. Same
# cache-safe channel as /steer (appended to the newest tool
# result); dormant when no budget is set.
if getattr(agent, "run_budget_seconds", None):
_maybe_inject_run_budget_wrapup(agent, messages)
# Prepare messages for API call
# If we have an ephemeral system prompt, prepend it to the messages
# Note: Reasoning is embedded in content via <think> tags for trajectory storage.
+9
View File
@@ -605,6 +605,15 @@ def build_turn_context(
# NOTE: _turns_since_memory and _iters_since_skill are NOT reset here.
agent.iteration_budget = IterationBudget(agent.max_iterations)
# Wall-clock run budget: per-run_conversation clock. Only stamped when a
# budget is configured so the default path stays clock-free; the wrap-up
# latch resets each turn (one notice per run, not per session).
if getattr(agent, "run_budget_seconds", None):
agent._run_budget_started_at = time.time()
else:
agent._run_budget_started_at = None
agent._run_budget_wrapup_injected = False
# Log conversation turn start for debugging/observability.
_preview_text = summarize_user_message_for_log(user_message)
_msg_preview = (_preview_text[:80] + "...") if len(_preview_text) > 80 else _preview_text
+10
View File
@@ -4858,6 +4858,7 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
api_key: str = None,
base_url: str = None,
max_turns: int = None,
run_budget: float = None,
verbose: Optional[bool] = None,
compact: bool = False,
resume: str = None,
@@ -5125,6 +5126,13 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
self.max_turns = 500
else:
self.max_turns = 500
# Wall-clock run budget: CLI flag wins over config; both optional.
# None keeps the feature fully off (AIAgent stays dormant).
if run_budget is not None:
self.run_budget_seconds = run_budget
else:
self.run_budget_seconds = CLI_CONFIG["agent"].get("run_budget_seconds")
# Parse and validate toolsets
self.enabled_toolsets = toolsets
@@ -20174,6 +20182,7 @@ def main(
api_key: str = None,
base_url: str = None,
max_turns: int = None,
run_budget: float = None,
verbose: Optional[bool] = None,
quiet: bool = False,
compact: bool = False,
@@ -20370,6 +20379,7 @@ def main(
api_key=api_key,
base_url=base_url,
max_turns=max_turns,
run_budget=run_budget,
verbose=verbose,
compact=compact,
resume=resume,
+15
View File
@@ -459,6 +459,21 @@ def build_top_level_parser():
metavar="N",
help="Maximum tool-calling iterations per conversation turn (default: 500, or agent.max_turns in config)",
)
chat_parser.add_argument(
"--run-budget",
type=float,
default=None,
metavar="SECONDS",
dest="run_budget",
help=(
"Optional wall-clock budget in seconds for each conversation run. "
"At 80%% elapsed the agent gets a one-time wrap-up notice, and "
"implicit provider stale timeouts are capped to the remaining "
"budget so one hung call can't consume the run. Unset = off. "
"Also configurable as agent.run_budget_seconds in config.yaml. "
"Intended for one-shot/eval invocations with a hard ceiling."
),
)
_inherited_flag(
chat_parser,
"--yolo",
+1
View File
@@ -500,6 +500,7 @@ class CLIAgentSetupMixin:
credential_pool=runtime.get("credential_pool"),
max_tokens=self.max_tokens,
max_iterations=self.max_turns,
run_budget_seconds=getattr(self, "run_budget_seconds", None),
enabled_toolsets=self.enabled_toolsets,
disabled_toolsets=self.disabled_toolsets,
verbose_logging=self.verbose,
+6
View File
@@ -44,6 +44,12 @@ DEFAULT_CONFIG = {
},
"agent": {
"max_turns": 500,
# Optional wall-clock budget in seconds per conversation run.
# null/absent = feature fully off (zero behavior change). When set,
# the agent gets a one-time wrap-up notice at 80% elapsed and
# implicit provider stale timeouts are capped to the remaining
# budget. CLI one-shot equivalent: `hermes chat --run-budget N`.
"run_budget_seconds": None,
# Inactivity timeout for gateway agent execution (seconds).
# The agent can run indefinitely as long as it's actively calling
# tools or receiving API responses. Only fires when the agent has
+1
View File
@@ -3220,6 +3220,7 @@ def cmd_chat(args):
"checkpoints": getattr(args, "checkpoints", False),
"pass_session_id": getattr(args, "pass_session_id", False),
"max_turns": getattr(args, "max_turns", None),
"run_budget": getattr(args, "run_budget", None),
"ignore_rules": getattr(args, "ignore_rules", False) or getattr(args, "safe_mode", False),
"ignore_user_config": getattr(args, "ignore_user_config", False) or getattr(args, "safe_mode", False),
"compact": getattr(args, "compact", False),
+36 -4
View File
@@ -503,6 +503,7 @@ class AIAgent:
session_db=None,
parent_session_id: str = None,
iteration_budget: "IterationBudget" = None,
run_budget_seconds: Optional[float] = None,
fallback_model: Dict[str, Any] = None,
credential_pool=None,
checkpoints_enabled: bool = False,
@@ -592,6 +593,7 @@ class AIAgent:
session_db=session_db,
parent_session_id=parent_session_id,
iteration_budget=iteration_budget,
run_budget_seconds=run_budget_seconds,
fallback_model=fallback_model,
credential_pool=credential_pool,
checkpoints_enabled=checkpoints_enabled,
@@ -1452,10 +1454,40 @@ class AIAgent:
from agent.chat_completion_helpers import estimate_request_context_tokens
est_tokens = estimate_request_context_tokens(api_payload)
if est_tokens > 100_000:
return max(stale_base, 240.0)
if est_tokens > 50_000:
return max(stale_base, 150.0)
return stale_base
timeout = max(stale_base, 240.0)
elif est_tokens > 50_000:
timeout = max(stale_base, 150.0)
else:
timeout = stale_base
# Wall-clock run budget cap: when a run budget is active, an implicit
# (floor-/default-derived) stale timeout is capped at half the
# remaining budget (>= 60s) so a single hung provider call cannot eat
# the whole run — e.g. deepseek-v4-pro's 600s reasoning floor inside a
# 900s eval ceiling. NEVER raises the timeout above what it would
# otherwise be, and an explicit user-configured stale_timeout_seconds
# (or env var) still wins untouched.
run_budget = getattr(self, "run_budget_seconds", None)
if run_budget and not self._stale_timeout_is_explicit():
started = getattr(self, "_run_budget_started_at", None)
if started:
remaining = float(run_budget) - (time.time() - started)
deadline_cap = max(60.0, remaining * 0.5)
if deadline_cap < timeout:
timeout = deadline_cap
return timeout
def _stale_timeout_is_explicit(self) -> bool:
"""True when the user explicitly configured the non-stream stale timeout.
Explicit = provider/model ``stale_timeout_seconds`` in config.yaml or
the ``HERMES_API_CALL_STALE_TIMEOUT`` env var. Reasoning-model floors
and the 90s default are implicit — they yield to the wall-clock run
budget cap; explicit user configuration never does.
"""
if get_provider_stale_timeout(self.provider, self.model) is not None:
return True
return os.getenv("HERMES_API_CALL_STALE_TIMEOUT") is not None
def _codex_silent_hang_hint(self, model: Optional[str] = None) -> Optional[str]:
"""Return an actionable hint when this request matches a known
+324
View File
@@ -0,0 +1,324 @@
"""Tests for the optional wall-clock run budget (agent.run_budget_seconds / --run-budget).
Covers:
1. Stale-timeout deadline scaling in
``run_agent.py:AIAgent._compute_non_stream_stale_timeout``:
- an active run budget CAPS an implicit stale timeout (default 90s and
reasoning floors like deepseek's 600s) at ``max(60, remaining * 0.5)``;
- the cap never RAISES the timeout above what it would otherwise be;
- explicit user configuration (provider ``stale_timeout_seconds`` or the
``HERMES_API_CALL_STALE_TIMEOUT`` env var) always wins untouched;
- no budget => completely unchanged behavior.
2. One-time-ness of the 80% wrap-up notice injection in
``agent.conversation_loop._maybe_inject_run_budget_wrapup``:
- fires once (latched), not repeatedly;
- never fires when no budget is set or before the 80% threshold;
- appended to the newest tool message (cache-safe /steer channel), no
synthetic user message.
3. Normalization of the config/CLI value
(``agent.agent_init._normalize_run_budget_seconds``): dormant on
null/invalid/non-positive input.
"""
from __future__ import annotations
import time
from pathlib import Path
import pytest
def _write_config(tmp_path: Path, body: str) -> None:
(tmp_path / "config.yaml").write_text(body or "{}\n", encoding="utf-8")
def _make_agent(tmp_path, monkeypatch, config_body: str = "", **overrides):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / ".env").write_text("", encoding="utf-8")
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
_write_config(tmp_path, config_body)
from run_agent import AIAgent
kwargs = dict(
model="gpt-5.5",
provider="openai",
api_key="sk-dummy",
base_url="https://api.openai.com/v1",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
platform="cli",
)
kwargs.update(overrides)
return AIAgent(**kwargs)
# ── normalization ──────────────────────────────────────────────────────────
@pytest.mark.parametrize("raw,expected", [
(None, None),
(0, None),
(-5, None),
("abc", None),
(True, None), # YAML `true` must not become a 1-second budget
(False, None),
(float("nan"), None),
(900, 900.0),
("850", 850.0),
(0.5, 0.5),
])
def test_normalize_run_budget_seconds(raw, expected):
from agent.agent_init import _normalize_run_budget_seconds
assert _normalize_run_budget_seconds(raw) == expected
# ── constructor / config plumbing ─────────────────────────────────────────
def test_no_budget_by_default(monkeypatch, tmp_path):
agent = _make_agent(tmp_path, monkeypatch)
assert agent.run_budget_seconds is None
assert agent._run_budget_started_at is None
assert agent._run_budget_wrapup_injected is False
def test_constructor_arg_sets_budget(monkeypatch, tmp_path):
agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900)
assert agent.run_budget_seconds == 900.0
def test_config_key_sets_budget(monkeypatch, tmp_path):
agent = _make_agent(
tmp_path, monkeypatch,
config_body="agent:\n run_budget_seconds: 750\n",
)
assert agent.run_budget_seconds == 750.0
def test_constructor_arg_wins_over_config(monkeypatch, tmp_path):
agent = _make_agent(
tmp_path, monkeypatch,
config_body="agent:\n run_budget_seconds: 750\n",
run_budget_seconds=900,
)
assert agent.run_budget_seconds == 900.0
# ── stale-timeout deadline scaling ─────────────────────────────────────────
def test_no_budget_stale_timeout_unchanged(monkeypatch, tmp_path):
"""Without a run budget the implicit 90s default is untouched."""
import run_agent
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
agent = _make_agent(tmp_path, monkeypatch)
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 90.0
def test_active_budget_caps_implicit_reasoning_floor(monkeypatch, tmp_path):
"""deepseek-v4-pro's 600s implicit floor yields to a tighter deadline cap.
900s budget, 800s elapsed -> remaining 100s -> cap = max(60, 50) = 60s.
"""
import run_agent
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
agent = _make_agent(
tmp_path, monkeypatch,
model="deepseek/deepseek-v4-pro",
run_budget_seconds=900,
)
# Sanity: implicit reasoning floor is 600s without a running clock.
base, implicit = agent._resolved_api_call_stale_timeout_base()
assert base == 600.0 and implicit is False
agent._run_budget_started_at = time.time() - 800
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 60.0
def test_active_budget_cap_half_remaining(monkeypatch, tmp_path):
"""Cap is remaining * 0.5 when that exceeds the 60s floor."""
import run_agent
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
agent = _make_agent(
tmp_path, monkeypatch,
model="deepseek/deepseek-v4-pro",
run_budget_seconds=900,
)
agent._run_budget_started_at = time.time() - 100 # remaining ~800 -> cap ~400
timeout = agent._compute_non_stream_stale_timeout({"input": "hi"})
assert 395.0 <= timeout <= 400.0
def test_active_budget_never_raises_timeout(monkeypatch, tmp_path):
"""The deadline cap NEVER loosens an already-tighter implicit timeout.
90s default with lots of remaining budget (cap would be ~445s) -> stays 90s.
"""
import run_agent
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900)
agent._run_budget_started_at = time.time() - 10
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 90.0
def test_explicit_provider_config_wins_over_budget_cap(monkeypatch, tmp_path):
"""Explicit stale_timeout_seconds is never capped by the run budget."""
import run_agent
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: 1800.0)
agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900)
agent._run_budget_started_at = time.time() - 800
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 1800.0
def test_explicit_env_var_wins_over_budget_cap(monkeypatch, tmp_path):
"""HERMES_API_CALL_STALE_TIMEOUT is explicit config — never capped."""
import run_agent
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
agent = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900)
monkeypatch.setenv("HERMES_API_CALL_STALE_TIMEOUT", "1200")
agent._run_budget_started_at = time.time() - 800
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 1200.0
def test_budget_without_started_clock_is_inert(monkeypatch, tmp_path):
"""A configured budget with no running turn clock changes nothing."""
import run_agent
monkeypatch.setattr(run_agent, "get_provider_stale_timeout", lambda *a, **k: None)
agent = _make_agent(
tmp_path, monkeypatch,
model="deepseek/deepseek-v4-pro",
run_budget_seconds=900,
)
agent._run_budget_started_at = None
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 600.0
# ── wrap-up injection one-time-ness ────────────────────────────────────────
class _StubAgent:
def __init__(self, budget=None, started=None):
self.run_budget_seconds = budget
self._run_budget_started_at = started
self._run_budget_wrapup_injected = False
def _tool_messages():
return [
{"role": "user", "content": "do the task"},
{"role": "assistant", "content": "", "tool_calls": [{"id": "t1"}]},
{"role": "tool", "tool_call_id": "t1", "content": "result one"},
{"role": "assistant", "content": "", "tool_calls": [{"id": "t2"}]},
{"role": "tool", "tool_call_id": "t2", "content": "result two"},
]
def test_wrapup_not_injected_when_unset():
from agent.conversation_loop import _maybe_inject_run_budget_wrapup
agent = _StubAgent(budget=None, started=time.time() - 10_000)
messages = _tool_messages()
assert _maybe_inject_run_budget_wrapup(agent, messages) is False
assert messages == _tool_messages()
def test_wrapup_not_injected_before_threshold():
from agent.conversation_loop import _maybe_inject_run_budget_wrapup
agent = _StubAgent(budget=900, started=time.time() - 100) # 11% elapsed
messages = _tool_messages()
assert _maybe_inject_run_budget_wrapup(agent, messages) is False
assert agent._run_budget_wrapup_injected is False
def test_wrapup_injected_once_after_threshold():
from agent.conversation_loop import (
RUN_BUDGET_WRAPUP_NOTICE,
_maybe_inject_run_budget_wrapup,
)
agent = _StubAgent(budget=900, started=time.time() - 800) # 89% elapsed
messages = _tool_messages()
assert _maybe_inject_run_budget_wrapup(agent, messages) is True
assert agent._run_budget_wrapup_injected is True
# Appended to the NEWEST tool message; earlier messages untouched.
assert RUN_BUDGET_WRAPUP_NOTICE in messages[-1]["content"]
assert messages[-1]["content"].startswith("result two")
assert messages[2]["content"] == "result one"
# No synthetic user message was inserted.
assert [m["role"] for m in messages] == [
"user", "assistant", "tool", "assistant", "tool",
]
# Second call: latched, no re-injection.
snapshot = [dict(m) for m in messages]
assert _maybe_inject_run_budget_wrapup(agent, messages) is False
assert messages == snapshot
assert messages[-1]["content"].count(RUN_BUDGET_WRAPUP_NOTICE) == 1
def test_wrapup_retries_when_no_tool_message_yet():
"""First iteration (no tool results) can't inject; the latch stays open
so the next iteration with a tool result delivers the notice."""
from agent.conversation_loop import (
RUN_BUDGET_WRAPUP_NOTICE,
_maybe_inject_run_budget_wrapup,
)
agent = _StubAgent(budget=900, started=time.time() - 800)
messages = [{"role": "user", "content": "do the task"}]
assert _maybe_inject_run_budget_wrapup(agent, messages) is False
assert agent._run_budget_wrapup_injected is False
messages += [
{"role": "assistant", "content": "", "tool_calls": [{"id": "t1"}]},
{"role": "tool", "tool_call_id": "t1", "content": "result"},
]
assert _maybe_inject_run_budget_wrapup(agent, messages) is True
assert RUN_BUDGET_WRAPUP_NOTICE in messages[-1]["content"]
def test_wrapup_not_injected_without_turn_clock():
from agent.conversation_loop import _maybe_inject_run_budget_wrapup
agent = _StubAgent(budget=900, started=None)
messages = _tool_messages()
assert _maybe_inject_run_budget_wrapup(agent, messages) is False
def test_wrapup_multimodal_tool_content():
"""Content-blocks tool results get a text block appended, not clobbered."""
from agent.conversation_loop import (
RUN_BUDGET_WRAPUP_NOTICE,
_maybe_inject_run_budget_wrapup,
)
agent = _StubAgent(budget=900, started=time.time() - 800)
messages = [
{"role": "user", "content": "task"},
{"role": "assistant", "content": "", "tool_calls": [{"id": "t1"}]},
{"role": "tool", "tool_call_id": "t1",
"content": [{"type": "text", "text": "block"}]},
]
assert _maybe_inject_run_budget_wrapup(agent, messages) is True
blocks = messages[-1]["content"]
assert blocks[0] == {"type": "text", "text": "block"}
assert blocks[-1] == {"type": "text", "text": RUN_BUDGET_WRAPUP_NOTICE}
# ── turn clock stamping ────────────────────────────────────────────────────
def test_turn_clock_stamped_only_with_budget(monkeypatch, tmp_path):
"""prepare-turn stamps the clock iff a budget is set, and resets the latch."""
agent_with = _make_agent(tmp_path, monkeypatch, run_budget_seconds=900)
agent_with._run_budget_wrapup_injected = True
# Mirror the turn_context.prepare block (unit-level: run the same logic).
for agent in (agent_with,):
if getattr(agent, "run_budget_seconds", None):
agent._run_budget_started_at = time.time()
else:
agent._run_budget_started_at = None
agent._run_budget_wrapup_injected = False
assert agent_with._run_budget_started_at is not None
assert agent_with._run_budget_wrapup_injected is False
+22
View File
@@ -1031,6 +1031,28 @@ When the iteration budget is fully exhausted, the CLI shows a notification to th
`agent.api_max_retries` controls how many times Hermes retries a provider API call on transient errors (rate limits, connection drops, 5xx) **before** fallback-provider switching engages. The default is `3` — four attempts total. If you have [fallback providers](/user-guide/features/fallback-providers) configured and want to fail over faster, drop this to `0` so the first transient error on your primary immediately hands off to the fallback instead of churning retries against the flaky endpoint.
## Wall-Clock Run Budget
Separate from the iteration budget, you can give each conversation run an optional **wall-clock** budget. This is designed for one-shot and eval-harness invocations that run under a hard external ceiling (e.g. a 900-second per-task limit): without it, a run can time out with the work essentially done — one generation short of emitting the final answer, or stuck in a single hung provider call.
```yaml
agent:
run_budget_seconds: null # Optional; unset/null = feature fully off (default)
```
Or per-invocation via the CLI:
```bash
hermes chat --run-budget 850 -q "..."
```
When a budget is set, two things happen:
1. **Wrap-up notice at 80%.** When 80% of the budget has elapsed, Hermes injects a **one-time** notice (delivered cache-safely, appended to the newest tool result like `/steer` messages) telling the model to stop new discovery/verification work and produce the final deliverable from the state it already has. It fires at most once per run and mirrors the existing iteration-budget wrap-up mechanism — there are no repeated pressure warnings.
2. **Deadline-scaled stale timeouts.** Implicit non-streaming stale timeouts (the 90s default and the reasoning-model floors, e.g. 600s for DeepSeek reasoning models) are capped at `max(60, remaining_budget × 0.5)` so a single silently-hung provider call can never consume the rest of the run. The cap only ever *tightens* the timeout — it never raises it — and an explicitly configured `stale_timeout_seconds` (provider/model config or `HERMES_API_CALL_STALE_TIMEOUT`) always wins untouched.
The budget is per `run_conversation` turn (it resets on each user message) and the feature is completely dormant when unset — no clock reads, no injection, no timeout changes.
## Verify-on-Stop (coding verification)
When enabled, Hermes refuses to accept a final answer on a turn where the agent edited code in a workspace but produced no fresh verification evidence (a passing test run, build, lint, etc.) — it injects a synthetic follow-up asking the agent to verify or explain why it can't. Doc/markdown/skill-only edits never trigger it, and the loop is bounded so it can never trap the agent.