test(terminal): harden watch_patterns lifetime cap — delivered-only counting, Nth-delivery promotion, docstring
Follow-ups on top of the cherry-picked #93532 cap: - Regression tests: suppressed (in-cooldown) matches must NOT consume the lifetime budget; the cap trips exactly at the Nth DELIVERED match and promotes to notify_on_complete with the watch_disabled summary queued right after the final match. - Extract _emit_lifetime_watch_disabled() and emit the summary even when the global breaker drops the final match, so the user always learns why watching went quiet (parity with the strike-limit path). - Mention the lifetime cap in the terminal tool docstring (the schema text was already updated by #93532). Refs #93513
This commit is contained in:
@@ -174,6 +174,55 @@ class TestLifetimeCap:
|
||||
assert registry.completion_queue.empty()
|
||||
assert session._watch_hits == WATCH_LIFETIME_MAX_HITS
|
||||
|
||||
def test_suppressed_matches_do_not_count_toward_lifetime_cap(self, registry):
|
||||
"""Only DELIVERED notifications consume the lifetime budget — matches
|
||||
suppressed inside the cooldown window must not increment the counter
|
||||
(they never forced an agent turn)."""
|
||||
from tools.process_registry import WATCH_LIFETIME_MAX_HITS
|
||||
|
||||
session = _make_session(watch_patterns=["E"])
|
||||
registry._check_watch_patterns(session, "E emit\n")
|
||||
assert session._watch_hits == 1
|
||||
|
||||
# Flood inside the cooldown: all suppressed, none delivered.
|
||||
for _ in range(WATCH_LIFETIME_MAX_HITS * 3):
|
||||
registry._check_watch_patterns(session, "E drop\n")
|
||||
assert session._watch_hits == 1 # unchanged
|
||||
# Strike-limit disable may or may not have tripped depending on
|
||||
# WATCH_STRIKE_LIMIT vs the single window here; assert it did NOT,
|
||||
# since all drops land in ONE window (one strike max).
|
||||
assert session._watch_consecutive_strikes == 1
|
||||
assert session._watch_disabled is False
|
||||
|
||||
def test_cap_trips_exactly_at_nth_delivery_and_promotes(self, registry):
|
||||
"""The Nth delivered match is still delivered, then the session is
|
||||
promoted to notify_on_complete in the same call, with the
|
||||
watch_disabled summary queued right after the final match."""
|
||||
from tools.process_registry import WATCH_LIFETIME_MAX_HITS
|
||||
|
||||
session = _make_session(watch_patterns=["ready"])
|
||||
for i in range(WATCH_LIFETIME_MAX_HITS - 1):
|
||||
session._watch_cooldown_until = 0.0
|
||||
registry._check_watch_patterns(session, "ready\n")
|
||||
assert session._watch_disabled is False
|
||||
assert session.notify_on_complete is False
|
||||
|
||||
# Nth delivery trips the cap.
|
||||
session._watch_cooldown_until = 0.0
|
||||
registry._check_watch_patterns(session, "ready\n")
|
||||
assert session._watch_disabled is True
|
||||
assert session.notify_on_complete is True
|
||||
|
||||
events = []
|
||||
while not registry.completion_queue.empty():
|
||||
events.append(registry.completion_queue.get_nowait())
|
||||
# Last two events: the Nth delivered match, then the summary.
|
||||
assert events[-2]["type"] == "watch_match"
|
||||
assert events[-1]["type"] == "watch_disabled"
|
||||
assert "lifetime cap" in events[-1]["message"]
|
||||
assert str(WATCH_LIFETIME_MAX_HITS) in events[-1]["message"]
|
||||
assert "notify_on_complete" in events[-1]["message"]
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Checkpoint persistence
|
||||
|
||||
+29
-19
@@ -648,6 +648,12 @@ class ProcessRegistry:
|
||||
|
||||
# Global circuit breaker — across all sessions (secondary safety net).
|
||||
if not self._global_watch_admit(now):
|
||||
if lifetime_exhausted:
|
||||
# The final match was dropped by the global breaker, but the
|
||||
# session is already disabled — still tell the user why things
|
||||
# went quiet (the strike path emits its summary unconditionally
|
||||
# too).
|
||||
self._emit_lifetime_watch_disabled(session)
|
||||
return
|
||||
|
||||
notification = {
|
||||
@@ -672,25 +678,29 @@ class ProcessRegistry:
|
||||
if lifetime_exhausted:
|
||||
# Same "why things went quiet" summary as the strike-limit path,
|
||||
# queued right after the final delivered match.
|
||||
self.completion_queue.put({
|
||||
"session_id": session.id,
|
||||
"session_key": session.session_key,
|
||||
"command": session.command,
|
||||
"type": "watch_disabled",
|
||||
"suppressed": 0,
|
||||
"platform": session.watcher_platform,
|
||||
"chat_id": session.watcher_chat_id,
|
||||
"user_id": session.watcher_user_id,
|
||||
"user_name": session.watcher_user_name,
|
||||
"thread_id": session.watcher_thread_id,
|
||||
"message_id": session.watcher_message_id,
|
||||
"message": (
|
||||
f"Watch patterns disabled for process {session.id} — "
|
||||
f"reached the lifetime cap of {WATCH_LIFETIME_MAX_HITS} delivered "
|
||||
f"matches. Falling back to notify_on_complete semantics; you'll get "
|
||||
f"exactly one notification when the process exits."
|
||||
),
|
||||
})
|
||||
self._emit_lifetime_watch_disabled(session)
|
||||
|
||||
def _emit_lifetime_watch_disabled(self, session: ProcessSession) -> None:
|
||||
"""Queue the watch_disabled summary for the lifetime-cap path (#93513)."""
|
||||
self.completion_queue.put({
|
||||
"session_id": session.id,
|
||||
"session_key": session.session_key,
|
||||
"command": session.command,
|
||||
"type": "watch_disabled",
|
||||
"suppressed": 0,
|
||||
"platform": session.watcher_platform,
|
||||
"chat_id": session.watcher_chat_id,
|
||||
"user_id": session.watcher_user_id,
|
||||
"user_name": session.watcher_user_name,
|
||||
"thread_id": session.watcher_thread_id,
|
||||
"message_id": session.watcher_message_id,
|
||||
"message": (
|
||||
f"Watch patterns disabled for process {session.id} — "
|
||||
f"reached the lifetime cap of {WATCH_LIFETIME_MAX_HITS} delivered "
|
||||
f"matches. Falling back to notify_on_complete semantics; you'll get "
|
||||
f"exactly one notification when the process exits."
|
||||
),
|
||||
})
|
||||
|
||||
def _global_watch_admit(self, now: float) -> bool:
|
||||
"""Return True if this watch_match event is allowed through the global breaker.
|
||||
|
||||
@@ -2686,7 +2686,7 @@ def terminal_tool(
|
||||
workdir: Working directory for this command (optional, uses session cwd if not set)
|
||||
pty: If True, use pseudo-terminal for interactive CLI tools (local backend only)
|
||||
notify_on_complete: If True and background=True, you'll be notified exactly once when the process exits. The right choice for almost every long task. MUTUALLY EXCLUSIVE with watch_patterns.
|
||||
watch_patterns: List of strings to watch for in background output. HARD rate limit: 1 notification per 15s per process. After 3 strike windows in a row, watch_patterns is disabled and the session is auto-promoted to notify_on_complete. Use ONLY for rare, one-shot mid-process signals on long-lived processes (server readiness, migration-done markers). NEVER use in loops/batch jobs — error patterns there will hit the strike limit and get disabled. MUTUALLY EXCLUSIVE with notify_on_complete — set one, not both.
|
||||
watch_patterns: List of strings to watch for in background output. HARD rate limit: 1 notification per 15s per process. After 3 strike windows in a row — or after a small lifetime cap of delivered matches, however cleanly spaced — watch_patterns is disabled and the session is auto-promoted to notify_on_complete. Use ONLY for rare, one-shot mid-process signals on long-lived processes (server readiness, migration-done markers). NEVER use in loops/batch jobs — error patterns there will hit the strike limit and get disabled. MUTUALLY EXCLUSIVE with notify_on_complete — set one, not both.
|
||||
|
||||
Returns:
|
||||
str: JSON string with output, exit_code, and error fields
|
||||
|
||||
Reference in New Issue
Block a user