"""Tests for agent/deadline.py — the unified deadline layer (#85125). Covers: * clamp_timeout normalization (None / non-positive / oversized / NaN / junk) * resolve_timeout precedence: config.yaml ``timeouts:`` > legacy env var > default * run_bounded_sync: completion, exception propagation, timeout + on_timeout * run_bounded_async: completion, exception propagation, timeout + abandonment of cancellation-shielded tasks, on_abandon cleanup * kill_process_tree: descendants of a session-leader child die with it (POSIX) * backward-compat contract of tool_executor._resolve_concurrent_tool_timeout after its migration onto resolve_timeout """ from __future__ import annotations import asyncio import os import signal import subprocess import sys import threading import time import pytest from agent.deadline import ( MAX_SAFE_TIMEOUT_S, BoundedResult, DeadlineExpired, clamp_timeout, kill_process_tree, resolve_timeout, run_bounded_async, run_bounded_sync, ) # --------------------------------------------------------------------------- # clamp_timeout # --------------------------------------------------------------------------- class TestClampTimeout: def test_none_stays_none(self): assert clamp_timeout(None) is None def test_zero_and_negative_mean_unbounded(self): assert clamp_timeout(0) is None assert clamp_timeout(-5) is None def test_normal_value_passes_through(self): assert clamp_timeout(420.0) == 420.0 def test_oversized_value_clamped_to_platform_safe_max(self): # The #83220 class: >time_t deadlines crash Lock.acquire on macOS. assert clamp_timeout(10**18) == MAX_SAFE_TIMEOUT_S def test_clamped_value_safe_for_threading_primitives(self): # Regression proof for #83220: the clamped value itself must be # accepted by the exact primitive that used to overflow. Acquiring an # uncontended lock returns immediately regardless of timeout, so # passing the full clamped value is safe and actually exercises the # time_t conversion. big = clamp_timeout(float(10**15)) assert big is not None lock = threading.Lock() assert lock.acquire(timeout=big) lock.release() def test_nan_and_junk_treated_as_unbounded(self): assert clamp_timeout(float("nan")) is None assert clamp_timeout("not-a-number") is None # type: ignore[arg-type] # --------------------------------------------------------------------------- # resolve_timeout # --------------------------------------------------------------------------- class TestResolveTimeout: def test_default_wins_when_nothing_configured(self, monkeypatch): monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {}) monkeypatch.delenv("HERMES_TEST_DEADLINE_X", raising=False) assert resolve_timeout("a.b", default=42.0, env_var="HERMES_TEST_DEADLINE_X") == 42.0 def test_env_var_beats_default(self, monkeypatch): monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {}) monkeypatch.setenv("HERMES_TEST_DEADLINE_X", "17.5") assert resolve_timeout("a.b", default=42.0, env_var="HERMES_TEST_DEADLINE_X") == 17.5 def test_config_beats_env_var(self, monkeypatch): monkeypatch.setattr( "agent.deadline._timeouts_section", lambda: {"a": {"b": 99}} ) monkeypatch.setenv("HERMES_TEST_DEADLINE_X", "17.5") assert resolve_timeout("a.b", default=42.0, env_var="HERMES_TEST_DEADLINE_X") == 99.0 def test_dotted_key_walks_nested_maps(self, monkeypatch): monkeypatch.setattr( "agent.deadline._timeouts_section", lambda: {"tools": {"concurrent_batch": 300}}, ) assert resolve_timeout("tools.concurrent_batch", default=420.0) == 300.0 def test_zero_config_value_means_unbounded(self, monkeypatch): monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {"a": {"b": 0}}) assert resolve_timeout("a.b", default=42.0) is None def test_invalid_config_value_falls_through_to_env(self, monkeypatch): monkeypatch.setattr( "agent.deadline._timeouts_section", lambda: {"a": {"b": "soon"}} ) monkeypatch.setenv("HERMES_TEST_DEADLINE_X", "17.5") assert resolve_timeout("a.b", default=42.0, env_var="HERMES_TEST_DEADLINE_X") == 17.5 def test_invalid_env_value_falls_through_to_default(self, monkeypatch): monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {}) monkeypatch.setenv("HERMES_TEST_DEADLINE_X", "banana") assert resolve_timeout("a.b", default=42.0, env_var="HERMES_TEST_DEADLINE_X") == 42.0 def test_bool_config_value_rejected(self, monkeypatch): # YAML `true` must not silently become a 1-second deadline. monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {"a": {"b": True}}) assert resolve_timeout("a.b", default=42.0) == 42.0 def test_nan_config_value_falls_through(self, monkeypatch): # NaN must fall through to the next source, not resolve as unbounded. monkeypatch.setattr( "agent.deadline._timeouts_section", lambda: {"a": {"b": float("nan")}} ) assert resolve_timeout("a.b", default=42.0) == 42.0 def test_broken_config_read_never_breaks_the_protected_path(self, monkeypatch): # _timeouts_section swallows config-load failures internally; prove # the public contract by making the underlying loader raise. import agent.deadline as dl def _boom(): raise RuntimeError("config unreadable") monkeypatch.setattr("hermes_cli.config.load_config_readonly", _boom) assert dl._timeouts_section() == {} assert resolve_timeout("a.b", default=5.0) == 5.0 # --------------------------------------------------------------------------- # run_bounded_sync # --------------------------------------------------------------------------- class TestRunBoundedSync: def test_completion_returns_value(self): result = run_bounded_sync(lambda: "ok", 5.0, label="t") assert result.timed_out is False assert result.value == "ok" assert result.raise_if_timed_out() == "ok" def test_unbounded_when_timeout_none(self): result = run_bounded_sync(lambda: 7, None, label="t") assert result.timed_out is False and result.value == 7 def test_exception_propagates_unchanged(self): class Boom(RuntimeError): pass with pytest.raises(Boom): run_bounded_sync(lambda: (_ for _ in ()).throw(Boom("x")), 5.0, label="t") def test_timeout_abandons_worker_and_reports(self): release = threading.Event() def _wedged(): release.wait(30) return "late" start = time.monotonic() result = run_bounded_sync(_wedged, 0.2, label="wedged") elapsed = time.monotonic() - start assert result.timed_out is True assert result.value is None assert elapsed < 5.0 # returned near the deadline, not after 30s with pytest.raises(DeadlineExpired) as exc_info: result.raise_if_timed_out() assert "wedged" in str(exc_info.value) release.set() def test_on_timeout_callback_runs(self): release = threading.Event() fired = [] result = run_bounded_sync( lambda: release.wait(30), 0.1, label="t", on_timeout=lambda: fired.append(True), ) assert result.timed_out and fired == [True] release.set() def test_on_timeout_callback_failure_is_swallowed(self): release = threading.Event() result = run_bounded_sync( lambda: release.wait(30), 0.1, label="t", on_timeout=lambda: (_ for _ in ()).throw(RuntimeError("cleanup boom")), ) assert result.timed_out is True release.set() def test_deadline_expired_is_a_timeout_error(self): # Error-classification contract: our deadline must be catchable as # TimeoutError but distinguishable by type from transport timeouts. assert issubclass(DeadlineExpired, TimeoutError) # --------------------------------------------------------------------------- # run_bounded_async # --------------------------------------------------------------------------- class TestRunBoundedAsync: def test_completion_returns_value(self): async def scenario(): async def op(): return "ok" return await run_bounded_async(op(), 5.0, label="t") result = asyncio.run(scenario()) assert result.timed_out is False and result.value == "ok" def test_unbounded_when_timeout_none(self): async def scenario(): async def op(): return 7 return await run_bounded_async(op(), None, label="t") result = asyncio.run(scenario()) assert result.timed_out is False and result.value == 7 def test_exception_propagates_unchanged(self): class Boom(RuntimeError): pass async def scenario(): async def op(): raise Boom("x") await run_bounded_async(op(), 5.0, label="t") with pytest.raises(Boom): asyncio.run(scenario()) def test_timeout_returns_promptly(self): async def scenario(): async def op(): await asyncio.sleep(30) start = time.monotonic() result = await run_bounded_async(op(), 0.2, label="slow") return result, time.monotonic() - start result, elapsed = asyncio.run(scenario()) assert result.timed_out is True assert elapsed < 5.0 def test_timeout_abandons_cancellation_shielded_task(self): """The family-A killer case: asyncio.wait_for cannot expire a shielded scope; the thread-timer deadline must return anyway.""" async def scenario(): hung = asyncio.Event() async def inner(): await hung.wait() async def shielded(): # Shield swallows the cancellation run_bounded_async issues. await asyncio.shield(asyncio.ensure_future(inner())) start = time.monotonic() result = await run_bounded_async(shielded(), 0.2, label="shielded") elapsed = time.monotonic() - start hung.set() # release the orphan so the loop can drain await asyncio.sleep(0) return result, elapsed result, elapsed = asyncio.run(scenario()) assert result.timed_out is True assert elapsed < 5.0 def test_on_abandon_cleanup_runs_detached(self): async def scenario(): cleaned = asyncio.Event() async def _cleanup(): cleaned.set() async def op(): await asyncio.sleep(30) result = await run_bounded_async( op(), 0.1, label="t", on_abandon=_cleanup ) await asyncio.wait_for(cleaned.wait(), timeout=5.0) return result result = asyncio.run(scenario()) assert result.timed_out is True def test_completed_op_never_reports_timeout(self): # Race guard: completion just under the deadline must report success. async def scenario(): async def op(): await asyncio.sleep(0.01) return "made it" return await run_bounded_async(op(), 5.0, label="t") result = asyncio.run(scenario()) assert result.timed_out is False and result.value == "made it" def test_external_cancellation_cancels_inner_task(self): # If the CALLER cancels run_bounded_async, the inner task must not be # leaked running unobserved. async def scenario(): started = asyncio.Event() inner_cancelled = asyncio.Event() async def op(): started.set() try: await asyncio.sleep(30) except asyncio.CancelledError: inner_cancelled.set() raise outer = asyncio.ensure_future( run_bounded_async(op(), 25.0, label="t") ) await started.wait() outer.cancel() with pytest.raises(asyncio.CancelledError): await outer await asyncio.wait_for(inner_cancelled.wait(), timeout=5.0) return True assert asyncio.run(scenario()) is True # --------------------------------------------------------------------------- # kill_process_tree # --------------------------------------------------------------------------- @pytest.mark.skipif(sys.platform == "win32", reason="POSIX process-group semantics") class TestKillProcessTree: def test_kills_descendants_of_session_leader(self, tmp_path): """A child spawned with start_new_session must die with its own child. This is the orphan-tree class (#71148): killing only the direct child leaves grandchildren running. """ started = tmp_path / "grandchild_started" marker = tmp_path / "grandchild_alive" grandchild_py = tmp_path / "grandchild.py" grandchild_py.write_text( "import pathlib, time\n" f"pathlib.Path({str(started)!r}).write_text('x')\n" "time.sleep(10)\n" f"pathlib.Path({str(marker)!r}).write_text('x')\n" ) parent_py = tmp_path / "parent.py" parent_py.write_text( "import subprocess, sys, time\n" f"subprocess.Popen([sys.executable, {str(grandchild_py)!r}])\n" "time.sleep(10)\n" ) proc = subprocess.Popen( [sys.executable, str(parent_py)], start_new_session=True ) deadline = time.monotonic() + 10 while not started.exists() and time.monotonic() < deadline: time.sleep(0.05) assert started.exists(), "grandchild never spawned — test harness broken" assert kill_process_tree(proc.pid) is True proc.wait(timeout=5) # Grandchild must be dead too: marker never appears. time.sleep(1.5) assert not marker.exists() def test_kills_descendant_in_its_own_session(self, tmp_path): """A descendant that setsid'd out of the parent's group must die too. killpg on the parent's group cannot reach it; the psutil descendant sweep must (tools/environments/base.py documents user commands doing exactly this). """ started = tmp_path / "setsid_grandchild_started" marker = tmp_path / "setsid_grandchild_alive" grandchild_py = tmp_path / "grandchild.py" grandchild_py.write_text( "import pathlib, time\n" f"pathlib.Path({str(started)!r}).write_text('x')\n" "time.sleep(10)\n" f"pathlib.Path({str(marker)!r}).write_text('x')\n" ) parent_py = tmp_path / "parent.py" parent_py.write_text( "import subprocess, sys, time\n" # grandchild leaves the parent's session/group entirely f"subprocess.Popen([sys.executable, {str(grandchild_py)!r}], start_new_session=True)\n" "time.sleep(10)\n" ) proc = subprocess.Popen( [sys.executable, str(parent_py)], start_new_session=True ) deadline = time.monotonic() + 10 while not started.exists() and time.monotonic() < deadline: time.sleep(0.05) assert started.exists(), "grandchild never spawned — test harness broken" assert kill_process_tree(proc.pid) is True proc.wait(timeout=5) time.sleep(1.5) assert not marker.exists() def test_already_dead_pid_returns_false(self): proc = subprocess.Popen([sys.executable, "-c", "pass"], start_new_session=True) proc.wait(timeout=10) # reaped: PID is gone from the process table assert kill_process_tree(proc.pid) is False def test_non_group_leader_falls_back_to_single_kill(self): # Child in OUR process group: killpg would signal the test runner. proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) try: assert os.getpgid(proc.pid) != proc.pid # not a leader assert kill_process_tree(proc.pid, sig=signal.SIGTERM) is True proc.wait(timeout=5) finally: if proc.poll() is None: proc.kill() # --------------------------------------------------------------------------- # tool_executor migration contract # --------------------------------------------------------------------------- class TestConcurrentToolTimeoutMigration: """_resolve_concurrent_tool_timeout keeps its exact legacy contract.""" def _resolver(self): from agent import tool_executor return tool_executor._resolve_concurrent_tool_timeout def test_default_unchanged(self, monkeypatch): monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {}) monkeypatch.delenv("HERMES_CONCURRENT_TOOL_TIMEOUT_S", raising=False) assert self._resolver()() == 420.0 def test_env_var_still_works(self, monkeypatch): monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {}) monkeypatch.setenv("HERMES_CONCURRENT_TOOL_TIMEOUT_S", "60") assert self._resolver()() == 60.0 def test_env_zero_still_disables(self, monkeypatch): monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {}) monkeypatch.setenv("HERMES_CONCURRENT_TOOL_TIMEOUT_S", "0") assert self._resolver()() is None def test_env_invalid_still_falls_back_to_default(self, monkeypatch): monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {}) monkeypatch.setenv("HERMES_CONCURRENT_TOOL_TIMEOUT_S", "junk") assert self._resolver()() == 420.0 def test_new_config_key_wins(self, monkeypatch): monkeypatch.setattr( "agent.deadline._timeouts_section", lambda: {"tools": {"concurrent_batch": 300}}, ) monkeypatch.setenv("HERMES_CONCURRENT_TOOL_TIMEOUT_S", "60") assert self._resolver()() == 300.0 class TestSequentialToolTimeoutResolver: """_resolve_sequential_tool_timeout: own key, inherits concurrent default.""" def _resolver(self): from agent import tool_executor return tool_executor._resolve_sequential_tool_timeout def test_inherits_concurrent_default(self, monkeypatch): monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {}) monkeypatch.delenv("HERMES_CONCURRENT_TOOL_TIMEOUT_S", raising=False) assert self._resolver()() == 420.0 def test_inherits_concurrent_env_bridge(self, monkeypatch): # No sequential-specific setting -> concurrent env var flows through. monkeypatch.setattr("agent.deadline._timeouts_section", lambda: {}) monkeypatch.setenv("HERMES_CONCURRENT_TOOL_TIMEOUT_S", "60") assert self._resolver()() == 60.0 def test_own_config_key_wins_over_concurrent(self, monkeypatch): monkeypatch.setattr( "agent.deadline._timeouts_section", lambda: {"tools": {"concurrent_batch": 300, "sequential_call": 90}}, ) assert self._resolver()() == 90.0 def test_zero_disables_independently(self, monkeypatch): # Sequential bound can be disabled while the concurrent one stays on. monkeypatch.setattr( "agent.deadline._timeouts_section", lambda: {"tools": {"concurrent_batch": 300, "sequential_call": 0}}, ) assert self._resolver()() is None def test_concurrent_disabled_flows_through(self, monkeypatch): # concurrent disabled (None default) + no sequential key -> unbounded. monkeypatch.setattr( "agent.deadline._timeouts_section", lambda: {"tools": {"concurrent_batch": 0}}, ) assert self._resolver()() is None