teknium1
d10bb2ab6f
`scripts/run_tests.sh tests/<dir>/` is how a change gets its regression
coverage run, so a test filed under the wrong directory is a test nobody
runs when that code changes. Two kinds of drift had accumulated.
Parallel directories for one source package, folded into the mirror:
tests/acp -> tests/acp_adapter (its __init__/conftest move with it)
tests/cli -> tests/hermes_cli (prompt_toolkit fixture merged into
hermes_cli/conftest.py)
tests/run_agent -> tests/agent (backoff fixture becomes
agent/conftest.py)
tests/relay -> tests/gateway/relay
tests/state -> tests/hermes_state
246 loose files at tests/ root, routed by the package they import/patch:
hermes_cli, hermes_state, agent, gateway, tools, plugins, tui_gateway, cron.
Installer and desktop-update script tests go to tests/scripts/{install,
desktop_update}/. 43 tests of root-level modules (batch_runner, utils,
hermes_constants, packaging) stay at the root.
Filenames drop their issue numbers (95 files: test_89315_x.py -> test_x.py);
the number stays in the module docstring where it has context.
Collisions: test_cli_skin_integration.py existed in both tests/ and tests/cli
with different subsets — merged into one (10 tests, all kept);
run_agent/test_pre_compress_memory_context.py -> agent/..._handoff.py;
tests/test_account_usage.py -> agent/test_account_usage_fetch.py;
tests/test_web_server.py -> hermes_cli/test_web_server_ws_ping.py.
Deleted: test_minisweagent_path.py (empty since PR #2804),
test_model_picker_scroll.py (tested a private copy of the logic, imported
nothing), test_process_loop_event_loop_warning.py (asserted asyncio behaviour,
imported nothing from Hermes).
Repo-root path arithmetic (Path(__file__).parents[N], dirname chains) is
bumped for the 202 files that changed depth and verified by evaluating every
such expression against the new location. classify_changes' desktop-updater
lane prefix, tests-os.yml's ignore glob and every in-tree path comment follow
the moves. tests/test_tests_tree_layout.py keeps the tree from drifting back.
307 lines
11 KiB
Python
307 lines
11 KiB
Python
"""Test interrupt propagation from parent to child agents.
|
|
|
|
Reproduces the CLI scenario: user sends a message while delegate_task is
|
|
running, main thread calls parent.interrupt(), child should stop.
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import MagicMock
|
|
|
|
from tools.interrupt import get_interrupt_reason, set_interrupt, is_interrupted
|
|
|
|
|
|
class TestInterruptPropagationToChild(unittest.TestCase):
|
|
"""Verify interrupt propagates from parent to child agent."""
|
|
|
|
def setUp(self):
|
|
set_interrupt(False)
|
|
|
|
def tearDown(self):
|
|
set_interrupt(False)
|
|
|
|
def _make_bare_agent(self):
|
|
"""Create a bare AIAgent via __new__ with all interrupt-related attrs."""
|
|
from run_agent import AIAgent
|
|
agent = AIAgent.__new__(AIAgent)
|
|
agent._interrupt_requested = False
|
|
agent._interrupt_message = None
|
|
agent._hard_interrupt_requested = threading.Event()
|
|
agent._execution_thread_id = None
|
|
agent._interrupt_thread_signal_pending = False
|
|
agent._active_children = []
|
|
agent._active_children_lock = threading.Lock()
|
|
agent.quiet_mode = True
|
|
# Provider/model/base_url are read by stale-timeout resolution paths;
|
|
# the specific values don't matter for interrupt tests.
|
|
agent.provider = "openrouter"
|
|
agent.model = "test/model"
|
|
agent._base_url = "http://localhost:1234"
|
|
return agent
|
|
|
|
def test_parent_interrupt_sets_child_flag(self):
|
|
"""When parent.interrupt() is called, child._interrupt_requested should be set."""
|
|
parent = self._make_bare_agent()
|
|
child = self._make_bare_agent()
|
|
|
|
parent._active_children.append(child)
|
|
|
|
parent.interrupt("new user message")
|
|
|
|
assert parent._interrupt_requested is True
|
|
assert child._interrupt_requested is True
|
|
assert child._interrupt_message == "new user message"
|
|
assert is_interrupted() is False
|
|
assert parent._interrupt_thread_signal_pending is True
|
|
|
|
def test_hard_cancel_is_explicit_atomic_and_propagated(self):
|
|
parent = self._make_bare_agent()
|
|
child = self._make_bare_agent()
|
|
parent._active_children.append(child)
|
|
|
|
parent.interrupt("Stop requested", hard_cancel=True)
|
|
|
|
assert parent._hard_interrupt_requested.is_set()
|
|
assert child._hard_interrupt_requested.is_set()
|
|
parent.clear_interrupt()
|
|
assert not parent._hard_interrupt_requested.is_set()
|
|
|
|
def test_message_interrupt_does_not_set_hard_cancel(self):
|
|
agent = self._make_bare_agent()
|
|
|
|
agent.interrupt("new user message")
|
|
|
|
assert agent._interrupt_requested is True
|
|
assert not agent._hard_interrupt_requested.is_set()
|
|
|
|
def test_message_interrupt_records_source_without_user_text(self):
|
|
agent = self._make_bare_agent()
|
|
agent._execution_thread_id = threading.current_thread().ident
|
|
|
|
agent.interrupt("private follow-up text")
|
|
|
|
assert get_interrupt_reason() == "user sent a new message"
|
|
assert "private follow-up text" not in get_interrupt_reason()
|
|
|
|
def test_hard_interrupt_does_not_expose_diagnostic_message(self):
|
|
agent = self._make_bare_agent()
|
|
agent._execution_thread_id = threading.current_thread().ident
|
|
|
|
agent.hard_interrupt("PRIVATE_CALLER_DETAIL")
|
|
|
|
assert get_interrupt_reason() == "explicit stop requested"
|
|
assert "PRIVATE_CALLER_DETAIL" not in get_interrupt_reason()
|
|
|
|
def test_hard_interrupt_records_explicit_safe_tool_reason(self):
|
|
agent = self._make_bare_agent()
|
|
agent._execution_thread_id = threading.current_thread().ident
|
|
|
|
agent.hard_interrupt(
|
|
"PRIVATE_CALLER_DETAIL",
|
|
tool_reason="background review superseded",
|
|
)
|
|
|
|
assert get_interrupt_reason() == "background review superseded"
|
|
|
|
def test_active_turn_redirect_does_not_set_hard_cancel(self):
|
|
agent = self._make_bare_agent()
|
|
agent._model_request_active = threading.Event()
|
|
agent._model_request_active.set()
|
|
agent._pending_redirect = None
|
|
|
|
assert agent.redirect("new correction") is True
|
|
|
|
assert agent._interrupt_requested is True
|
|
assert agent._interrupt_message is None
|
|
assert not agent._hard_interrupt_requested.is_set()
|
|
|
|
def test_child_clear_interrupt_at_start_clears_thread(self):
|
|
"""child.clear_interrupt() at start of run_conversation clears the
|
|
bound execution thread's interrupt flag.
|
|
"""
|
|
child = self._make_bare_agent()
|
|
child._interrupt_requested = True
|
|
child._interrupt_message = "msg"
|
|
child._execution_thread_id = threading.current_thread().ident
|
|
|
|
# Interrupt for current thread is set
|
|
set_interrupt(True)
|
|
assert is_interrupted() is True
|
|
|
|
# child.clear_interrupt() clears both instance flag and thread flag
|
|
child.clear_interrupt()
|
|
assert child._interrupt_requested is False
|
|
assert is_interrupted() is False
|
|
|
|
def test_interrupt_during_child_api_call_detected(self):
|
|
"""Interrupt set during _interruptible_api_call is detected within 0.5s."""
|
|
child = self._make_bare_agent()
|
|
child.api_mode = "chat_completions"
|
|
child.log_prefix = ""
|
|
child._client_kwargs = {"api_key": "test", "base_url": "http://localhost:1234"}
|
|
|
|
# Mock a slow API call
|
|
mock_client = MagicMock()
|
|
def slow_api_call(**kwargs):
|
|
time.sleep(5) # Would take 5s normally
|
|
return MagicMock()
|
|
mock_client.chat.completions.create = slow_api_call
|
|
mock_client.close = MagicMock()
|
|
child.client = mock_client
|
|
|
|
# Set interrupt after 0.2s from another thread
|
|
def set_interrupt_later():
|
|
time.sleep(0.2)
|
|
child.interrupt("stop!")
|
|
t = threading.Thread(target=set_interrupt_later, daemon=True)
|
|
t.start()
|
|
|
|
start = time.monotonic()
|
|
try:
|
|
child._interruptible_api_call({"model": "test", "messages": []})
|
|
self.fail("Should have raised InterruptedError")
|
|
except InterruptedError:
|
|
elapsed = time.monotonic() - start
|
|
# Should detect within ~0.5s (0.2s delay + 0.3s poll interval)
|
|
assert elapsed < 1.0, f"Took {elapsed:.2f}s to detect interrupt (expected < 1.0s)"
|
|
finally:
|
|
t.join(timeout=2)
|
|
set_interrupt(False)
|
|
|
|
def test_concurrent_interrupt_propagation(self):
|
|
"""Simulates exact CLI flow: parent runs delegate in thread, main thread interrupts."""
|
|
parent = self._make_bare_agent()
|
|
child = self._make_bare_agent()
|
|
|
|
# Register child (simulating what _run_single_child does)
|
|
parent._active_children.append(child)
|
|
|
|
# Simulate child running (checking flag in a loop)
|
|
child_detected = threading.Event()
|
|
def simulate_child_loop():
|
|
while not child._interrupt_requested:
|
|
time.sleep(0.05)
|
|
child_detected.set()
|
|
|
|
child_thread = threading.Thread(target=simulate_child_loop, daemon=True)
|
|
child_thread.start()
|
|
|
|
# Small delay, then interrupt from "main thread"
|
|
time.sleep(0.1)
|
|
parent.interrupt("user typed something new")
|
|
|
|
# Child should detect within 200ms
|
|
detected = child_detected.wait(timeout=1.0)
|
|
assert detected, "Child never detected the interrupt!"
|
|
child_thread.join(timeout=1)
|
|
set_interrupt(False)
|
|
|
|
def test_prestart_interrupt_binds_to_execution_thread(self):
|
|
"""An interrupt that arrives before startup should bind to the agent thread."""
|
|
agent = self._make_bare_agent()
|
|
barrier = threading.Barrier(2)
|
|
result = {}
|
|
|
|
agent.interrupt("stop before start")
|
|
assert agent._interrupt_requested is True
|
|
assert agent._interrupt_thread_signal_pending is True
|
|
assert is_interrupted() is False
|
|
|
|
def run_thread():
|
|
from tools.interrupt import set_interrupt as _set_interrupt_for_test
|
|
|
|
agent._execution_thread_id = threading.current_thread().ident
|
|
_set_interrupt_for_test(False, agent._execution_thread_id)
|
|
if agent._interrupt_requested:
|
|
_set_interrupt_for_test(True, agent._execution_thread_id)
|
|
agent._interrupt_thread_signal_pending = False
|
|
barrier.wait(timeout=5)
|
|
result["thread_interrupted"] = is_interrupted()
|
|
|
|
t = threading.Thread(target=run_thread)
|
|
t.start()
|
|
barrier.wait(timeout=5)
|
|
t.join(timeout=2)
|
|
|
|
assert result["thread_interrupted"] is True
|
|
assert agent._interrupt_thread_signal_pending is False
|
|
|
|
|
|
class TestPerThreadInterruptIsolation(unittest.TestCase):
|
|
"""Verify that interrupting one agent does NOT affect another agent's thread.
|
|
|
|
This is the core fix for the gateway cross-session interrupt leak:
|
|
multiple agents run in separate threads within the same process, and
|
|
interrupting agent A must not kill agent B's running tools.
|
|
"""
|
|
|
|
def setUp(self):
|
|
set_interrupt(False)
|
|
|
|
def tearDown(self):
|
|
set_interrupt(False)
|
|
|
|
def test_interrupt_only_affects_target_thread(self):
|
|
"""set_interrupt(True, tid) only makes is_interrupted() True on that thread."""
|
|
results = {}
|
|
barrier = threading.Barrier(2)
|
|
|
|
def thread_a():
|
|
"""Agent A's execution thread — will be interrupted."""
|
|
tid = threading.current_thread().ident
|
|
results["a_tid"] = tid
|
|
barrier.wait(timeout=5) # sync with thread B
|
|
time.sleep(0.2) # let the interrupt arrive
|
|
results["a_interrupted"] = is_interrupted()
|
|
|
|
def thread_b():
|
|
"""Agent B's execution thread — should NOT be affected."""
|
|
tid = threading.current_thread().ident
|
|
results["b_tid"] = tid
|
|
barrier.wait(timeout=5) # sync with thread A
|
|
time.sleep(0.2)
|
|
results["b_interrupted"] = is_interrupted()
|
|
|
|
ta = threading.Thread(target=thread_a)
|
|
tb = threading.Thread(target=thread_b)
|
|
ta.start()
|
|
tb.start()
|
|
|
|
# Wait for both threads to register their TIDs
|
|
time.sleep(0.05)
|
|
while "a_tid" not in results or "b_tid" not in results:
|
|
time.sleep(0.01)
|
|
|
|
# Interrupt ONLY thread A (simulates gateway interrupting agent A)
|
|
set_interrupt(True, results["a_tid"])
|
|
|
|
ta.join(timeout=3)
|
|
tb.join(timeout=3)
|
|
|
|
assert results["a_interrupted"] is True, "Thread A should see the interrupt"
|
|
assert results["b_interrupted"] is False, "Thread B must NOT see thread A's interrupt"
|
|
|
|
def test_clear_interrupt_only_clears_target_thread(self):
|
|
"""Clearing one thread's interrupt doesn't clear another's."""
|
|
tid_a = 99990001
|
|
tid_b = 99990002
|
|
set_interrupt(True, tid_a)
|
|
set_interrupt(True, tid_b)
|
|
|
|
# Clear only A
|
|
set_interrupt(False, tid_a)
|
|
|
|
# Simulate checking from thread B's perspective
|
|
from tools.interrupt import _interrupted_threads, _lock
|
|
with _lock:
|
|
assert tid_a not in _interrupted_threads
|
|
assert tid_b in _interrupted_threads
|
|
|
|
# Cleanup
|
|
set_interrupt(False, tid_b)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|