test(terminal): harden watch_patterns lifetime cap — delivered-only counting, Nth-delivery promotion, docstring

Follow-ups on top of the cherry-picked #93532 cap:
- Regression tests: suppressed (in-cooldown) matches must NOT consume the
  lifetime budget; the cap trips exactly at the Nth DELIVERED match and
  promotes to notify_on_complete with the watch_disabled summary queued
  right after the final match.
- Extract _emit_lifetime_watch_disabled() and emit the summary even when
  the global breaker drops the final match, so the user always learns why
  watching went quiet (parity with the strike-limit path).
- Mention the lifetime cap in the terminal tool docstring (the schema text
  was already updated by #93532).

Refs #93513
This commit is contained in:
Teknium
2026-08-24 00:11:34 -07:00
committed by Teknium
parent b3730153c3
commit d8d1e18ab9
3 changed files with 79 additions and 20 deletions
+49
View File
@@ -174,6 +174,55 @@ class TestLifetimeCap:
assert registry.completion_queue.empty()
assert session._watch_hits == WATCH_LIFETIME_MAX_HITS
def test_suppressed_matches_do_not_count_toward_lifetime_cap(self, registry):
"""Only DELIVERED notifications consume the lifetime budget — matches
suppressed inside the cooldown window must not increment the counter
(they never forced an agent turn)."""
from tools.process_registry import WATCH_LIFETIME_MAX_HITS
session = _make_session(watch_patterns=["E"])
registry._check_watch_patterns(session, "E emit\n")
assert session._watch_hits == 1
# Flood inside the cooldown: all suppressed, none delivered.
for _ in range(WATCH_LIFETIME_MAX_HITS * 3):
registry._check_watch_patterns(session, "E drop\n")
assert session._watch_hits == 1 # unchanged
# Strike-limit disable may or may not have tripped depending on
# WATCH_STRIKE_LIMIT vs the single window here; assert it did NOT,
# since all drops land in ONE window (one strike max).
assert session._watch_consecutive_strikes == 1
assert session._watch_disabled is False
def test_cap_trips_exactly_at_nth_delivery_and_promotes(self, registry):
"""The Nth delivered match is still delivered, then the session is
promoted to notify_on_complete in the same call, with the
watch_disabled summary queued right after the final match."""
from tools.process_registry import WATCH_LIFETIME_MAX_HITS
session = _make_session(watch_patterns=["ready"])
for i in range(WATCH_LIFETIME_MAX_HITS - 1):
session._watch_cooldown_until = 0.0
registry._check_watch_patterns(session, "ready\n")
assert session._watch_disabled is False
assert session.notify_on_complete is False
# Nth delivery trips the cap.
session._watch_cooldown_until = 0.0
registry._check_watch_patterns(session, "ready\n")
assert session._watch_disabled is True
assert session.notify_on_complete is True
events = []
while not registry.completion_queue.empty():
events.append(registry.completion_queue.get_nowait())
# Last two events: the Nth delivered match, then the summary.
assert events[-2]["type"] == "watch_match"
assert events[-1]["type"] == "watch_disabled"
assert "lifetime cap" in events[-1]["message"]
assert str(WATCH_LIFETIME_MAX_HITS) in events[-1]["message"]
assert "notify_on_complete" in events[-1]["message"]
# =========================================================================
# Checkpoint persistence
+29 -19
View File
@@ -648,6 +648,12 @@ class ProcessRegistry:
# Global circuit breaker — across all sessions (secondary safety net).
if not self._global_watch_admit(now):
if lifetime_exhausted:
# The final match was dropped by the global breaker, but the
# session is already disabled — still tell the user why things
# went quiet (the strike path emits its summary unconditionally
# too).
self._emit_lifetime_watch_disabled(session)
return
notification = {
@@ -672,25 +678,29 @@ class ProcessRegistry:
if lifetime_exhausted:
# Same "why things went quiet" summary as the strike-limit path,
# queued right after the final delivered match.
self.completion_queue.put({
"session_id": session.id,
"session_key": session.session_key,
"command": session.command,
"type": "watch_disabled",
"suppressed": 0,
"platform": session.watcher_platform,
"chat_id": session.watcher_chat_id,
"user_id": session.watcher_user_id,
"user_name": session.watcher_user_name,
"thread_id": session.watcher_thread_id,
"message_id": session.watcher_message_id,
"message": (
f"Watch patterns disabled for process {session.id} — "
f"reached the lifetime cap of {WATCH_LIFETIME_MAX_HITS} delivered "
f"matches. Falling back to notify_on_complete semantics; you'll get "
f"exactly one notification when the process exits."
),
})
self._emit_lifetime_watch_disabled(session)
def _emit_lifetime_watch_disabled(self, session: ProcessSession) -> None:
"""Queue the watch_disabled summary for the lifetime-cap path (#93513)."""
self.completion_queue.put({
"session_id": session.id,
"session_key": session.session_key,
"command": session.command,
"type": "watch_disabled",
"suppressed": 0,
"platform": session.watcher_platform,
"chat_id": session.watcher_chat_id,
"user_id": session.watcher_user_id,
"user_name": session.watcher_user_name,
"thread_id": session.watcher_thread_id,
"message_id": session.watcher_message_id,
"message": (
f"Watch patterns disabled for process {session.id} — "
f"reached the lifetime cap of {WATCH_LIFETIME_MAX_HITS} delivered "
f"matches. Falling back to notify_on_complete semantics; you'll get "
f"exactly one notification when the process exits."
),
})
def _global_watch_admit(self, now: float) -> bool:
"""Return True if this watch_match event is allowed through the global breaker.
+1 -1
View File
@@ -2686,7 +2686,7 @@ def terminal_tool(
workdir: Working directory for this command (optional, uses session cwd if not set)
pty: If True, use pseudo-terminal for interactive CLI tools (local backend only)
notify_on_complete: If True and background=True, you'll be notified exactly once when the process exits. The right choice for almost every long task. MUTUALLY EXCLUSIVE with watch_patterns.
watch_patterns: List of strings to watch for in background output. HARD rate limit: 1 notification per 15s per process. After 3 strike windows in a row, watch_patterns is disabled and the session is auto-promoted to notify_on_complete. Use ONLY for rare, one-shot mid-process signals on long-lived processes (server readiness, migration-done markers). NEVER use in loops/batch jobs — error patterns there will hit the strike limit and get disabled. MUTUALLY EXCLUSIVE with notify_on_complete — set one, not both.
watch_patterns: List of strings to watch for in background output. HARD rate limit: 1 notification per 15s per process. After 3 strike windows in a row — or after a small lifetime cap of delivered matches, however cleanly spaced — watch_patterns is disabled and the session is auto-promoted to notify_on_complete. Use ONLY for rare, one-shot mid-process signals on long-lived processes (server readiness, migration-done markers). NEVER use in loops/batch jobs — error patterns there will hit the strike limit and get disabled. MUTUALLY EXCLUSIVE with notify_on_complete — set one, not both.
Returns:
str: JSON string with output, exit_code, and error fields