fix(state): sub-second busy budget for observational activity writes (review S1)
Activity heartbeat writes and turn-end label clears ran synchronously on the response-critical path with the full ~20s routine write-patience budget — under contention an otherwise-finished reply could stall for seconds just to update observation labels, mimicking the very stall the watchdog detects. touch_session_activity and clear_session_activity_labels now use a dedicated 0.5s patience budget (they are observation-only; the next heartbeat window retries naturally), and a no-op label clear (labels already empty) skips the write transaction entirely. Regressions: with another connection holding BEGIN IMMEDIATE, both writes give up well under the routine budget; the no-op clear performs zero write transactions. PR #76354 review, 'activity writes are synchronous on critical paths' / merge gate 9.
This commit is contained in:
+36
-2
@@ -1807,6 +1807,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
|
||||
# attempts.
|
||||
_WRITE_PATIENCE_S = 20.0
|
||||
_TRANSCRIPT_WRITE_PATIENCE_S = 60.0
|
||||
# Observation-only activity heartbeat/label writes (#76354 review S1):
|
||||
# these run on (or adjacent to) the response-critical path and must never
|
||||
# wait out the full routine patience under contention. Sub-second budget;
|
||||
# a skipped write is retried naturally at the next heartbeat window.
|
||||
_ACTIVITY_WRITE_PATIENCE_S = 0.5
|
||||
# A live compression lock gets its own, much shorter budget than the write
|
||||
# lock. Compression publishes in a couple of seconds, so a brief wait saves
|
||||
# the overwhelming majority of concurrent turns (#75083). It deliberately
|
||||
@@ -4026,7 +4031,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
|
||||
(when, desc, prov, session_id, when),
|
||||
)
|
||||
|
||||
self._execute_write(_do)
|
||||
# Observation-only write: never let it ride the full routine
|
||||
# write-patience budget (#76354 review S1). Under contention a
|
||||
# heartbeat that waits ~20s would delay the response-critical path
|
||||
# it is merely observing; give up after a sub-second budget instead
|
||||
# (the next due window retries naturally).
|
||||
self._execute_write(_do, patience_s=self._ACTIVITY_WRITE_PATIENCE_S)
|
||||
|
||||
def clear_session_activity_labels(self, session_id: str) -> None:
|
||||
"""Clear mid-turn activity labels after a turn ends.
|
||||
@@ -4036,11 +4046,35 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
|
||||
*what was happening at* that timestamp during an active turn; once
|
||||
the turn is idle they must not keep advertising "compressing" /
|
||||
"executing tool" (#72039).
|
||||
|
||||
Response-critical-path contract (#76354 review S1): runs in the
|
||||
turn's ``finally``; a no-op clear (labels already empty) skips the
|
||||
write transaction entirely, and a real clear uses the same short
|
||||
sub-second busy budget as :meth:`touch_session_activity` instead of
|
||||
the full routine write patience.
|
||||
"""
|
||||
if not session_id:
|
||||
return
|
||||
from agent.session_activity import ActivityProvenance
|
||||
|
||||
# No-op fast path: skip the transaction when there is nothing to
|
||||
# clear. Read-only, no write lock.
|
||||
try:
|
||||
row = self._conn.execute(
|
||||
"SELECT last_activity_description, last_activity_provenance "
|
||||
"FROM sessions WHERE id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
except sqlite3.Error:
|
||||
row = None
|
||||
if row is not None:
|
||||
desc = row[0] if not isinstance(row, sqlite3.Row) else row["last_activity_description"]
|
||||
prov = row[1] if not isinstance(row, sqlite3.Row) else row["last_activity_provenance"]
|
||||
if not desc and (
|
||||
not prov or prov == ActivityProvenance.UNKNOWN.value
|
||||
):
|
||||
return
|
||||
|
||||
def _do(conn):
|
||||
conn.execute(
|
||||
"UPDATE sessions SET "
|
||||
@@ -4050,7 +4084,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
|
||||
("", ActivityProvenance.UNKNOWN.value, session_id),
|
||||
)
|
||||
|
||||
self._execute_write(_do)
|
||||
self._execute_write(_do, patience_s=self._ACTIVITY_WRITE_PATIENCE_S)
|
||||
|
||||
def get_session_activity(self, session_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return the durable activity snapshot for *session_id*, or None."""
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Regressions for #76354 review S1/S2/S4 — activity write budget, watchdog
|
||||
pre-delivery revalidation, and import/export activity asymmetry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.session_activity import ActivityProvenance, build_activity_snapshot
|
||||
from hermes_state import SessionDB
|
||||
|
||||
|
||||
# ── S1: observational activity writes must not ride the 20s patience ────────
|
||||
|
||||
|
||||
def _hold_write_lock(db_path: Path, held: threading.Event, release: threading.Event):
|
||||
conn = sqlite3.connect(str(db_path), timeout=30)
|
||||
try:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
held.set()
|
||||
release.wait(timeout=30)
|
||||
conn.rollback()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_s1_contended_activity_write_gives_up_within_short_budget(tmp_path):
|
||||
db = SessionDB(db_path=tmp_path / "state.db")
|
||||
sid = "S1_CONTENDED"
|
||||
db.create_session(sid, source="cli")
|
||||
|
||||
held = threading.Event()
|
||||
release = threading.Event()
|
||||
locker = threading.Thread(
|
||||
target=_hold_write_lock, args=(tmp_path / "state.db", held, release)
|
||||
)
|
||||
locker.start()
|
||||
try:
|
||||
assert held.wait(timeout=5)
|
||||
t0 = time.monotonic()
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
db.touch_session_activity(sid, time.time(), description="working")
|
||||
elapsed_touch = time.monotonic() - t0
|
||||
finally:
|
||||
release.set()
|
||||
locker.join(timeout=10)
|
||||
|
||||
# The observational write gave up within the short budget — far below
|
||||
# the 20s routine patience the review flagged.
|
||||
assert elapsed_touch < 3.0, f"activity touch waited {elapsed_touch:.1f}s"
|
||||
|
||||
|
||||
def test_s1_clear_labels_noop_skips_transaction(tmp_path, monkeypatch):
|
||||
db = SessionDB(db_path=tmp_path / "state.db")
|
||||
sid = "S1_NOOP"
|
||||
db.create_session(sid, source="cli")
|
||||
# Fresh session: labels empty → the clear must not open a transaction.
|
||||
calls = []
|
||||
original = db._execute_write
|
||||
|
||||
def _spy(fn, patience_s=None):
|
||||
calls.append(fn)
|
||||
return original(fn, patience_s=patience_s)
|
||||
|
||||
monkeypatch.setattr(db, "_execute_write", _spy)
|
||||
db.clear_session_activity_labels(sid)
|
||||
assert calls == [], "no-op label clear must skip the write transaction"
|
||||
|
||||
# Non-empty labels → clear runs exactly one write.
|
||||
db.touch_session_activity(sid, time.time(), description="doing work")
|
||||
calls.clear()
|
||||
db.clear_session_activity_labels(sid)
|
||||
assert len(calls) == 1
|
||||
activity = db.get_session_activity(sid)
|
||||
assert activity["last_activity_description"] == ""
|
||||
|
||||
|
||||
def test_s1_contended_clear_gives_up_within_short_budget(tmp_path):
|
||||
db = SessionDB(db_path=tmp_path / "state.db")
|
||||
sid = "S1_CLEAR_CONTENDED"
|
||||
db.create_session(sid, source="cli")
|
||||
db.touch_session_activity(sid, time.time(), description="busy")
|
||||
|
||||
held = threading.Event()
|
||||
release = threading.Event()
|
||||
locker = threading.Thread(
|
||||
target=_hold_write_lock, args=(tmp_path / "state.db", held, release)
|
||||
)
|
||||
locker.start()
|
||||
try:
|
||||
assert held.wait(timeout=5)
|
||||
t0 = time.monotonic()
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
db.clear_session_activity_labels(sid)
|
||||
elapsed = time.monotonic() - t0
|
||||
finally:
|
||||
release.set()
|
||||
locker.join(timeout=10)
|
||||
assert elapsed < 3.0, f"label clear waited {elapsed:.1f}s under contention"
|
||||
Reference in New Issue
Block a user