fix(state): sub-second busy budget for observational activity writes (review S1)

Activity heartbeat writes and turn-end label clears ran synchronously on
the response-critical path with the full ~20s routine write-patience
budget — under contention an otherwise-finished reply could stall for
seconds just to update observation labels, mimicking the very stall the
watchdog detects.

touch_session_activity and clear_session_activity_labels now use a
dedicated 0.5s patience budget (they are observation-only; the next
heartbeat window retries naturally), and a no-op label clear (labels
already empty) skips the write transaction entirely.

Regressions: with another connection holding BEGIN IMMEDIATE, both writes
give up well under the routine budget; the no-op clear performs zero
write transactions.

PR #76354 review, 'activity writes are synchronous on critical paths' /
merge gate 9.
This commit is contained in:
Teknium
2026-08-01 15:21:04 -07:00
parent 99100843c6
commit 92c736919d
2 changed files with 141 additions and 2 deletions
+36 -2
View File
@@ -1807,6 +1807,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
# attempts.
_WRITE_PATIENCE_S = 20.0
_TRANSCRIPT_WRITE_PATIENCE_S = 60.0
# Observation-only activity heartbeat/label writes (#76354 review S1):
# these run on (or adjacent to) the response-critical path and must never
# wait out the full routine patience under contention. Sub-second budget;
# a skipped write is retried naturally at the next heartbeat window.
_ACTIVITY_WRITE_PATIENCE_S = 0.5
# A live compression lock gets its own, much shorter budget than the write
# lock. Compression publishes in a couple of seconds, so a brief wait saves
# the overwhelming majority of concurrent turns (#75083). It deliberately
@@ -4026,7 +4031,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
(when, desc, prov, session_id, when),
)
self._execute_write(_do)
# Observation-only write: never let it ride the full routine
# write-patience budget (#76354 review S1). Under contention a
# heartbeat that waits ~20s would delay the response-critical path
# it is merely observing; give up after a sub-second budget instead
# (the next due window retries naturally).
self._execute_write(_do, patience_s=self._ACTIVITY_WRITE_PATIENCE_S)
def clear_session_activity_labels(self, session_id: str) -> None:
"""Clear mid-turn activity labels after a turn ends.
@@ -4036,11 +4046,35 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
*what was happening at* that timestamp during an active turn; once
the turn is idle they must not keep advertising "compressing" /
"executing tool" (#72039).
Response-critical-path contract (#76354 review S1): runs in the
turn's ``finally``; a no-op clear (labels already empty) skips the
write transaction entirely, and a real clear uses the same short
sub-second busy budget as :meth:`touch_session_activity` instead of
the full routine write patience.
"""
if not session_id:
return
from agent.session_activity import ActivityProvenance
# No-op fast path: skip the transaction when there is nothing to
# clear. Read-only, no write lock.
try:
row = self._conn.execute(
"SELECT last_activity_description, last_activity_provenance "
"FROM sessions WHERE id = ?",
(session_id,),
).fetchone()
except sqlite3.Error:
row = None
if row is not None:
desc = row[0] if not isinstance(row, sqlite3.Row) else row["last_activity_description"]
prov = row[1] if not isinstance(row, sqlite3.Row) else row["last_activity_provenance"]
if not desc and (
not prov or prov == ActivityProvenance.UNKNOWN.value
):
return
def _do(conn):
conn.execute(
"UPDATE sessions SET "
@@ -4050,7 +4084,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
("", ActivityProvenance.UNKNOWN.value, session_id),
)
self._execute_write(_do)
self._execute_write(_do, patience_s=self._ACTIVITY_WRITE_PATIENCE_S)
def get_session_activity(self, session_id: str) -> Optional[Dict[str, Any]]:
"""Return the durable activity snapshot for *session_id*, or None."""
+105
View File
@@ -0,0 +1,105 @@
"""Regressions for #76354 review S1/S2/S4 — activity write budget, watchdog
pre-delivery revalidation, and import/export activity asymmetry.
"""
from __future__ import annotations
import sqlite3
import threading
import time
from pathlib import Path
from types import SimpleNamespace
import pytest
from agent.session_activity import ActivityProvenance, build_activity_snapshot
from hermes_state import SessionDB
# ── S1: observational activity writes must not ride the 20s patience ────────
def _hold_write_lock(db_path: Path, held: threading.Event, release: threading.Event):
conn = sqlite3.connect(str(db_path), timeout=30)
try:
conn.execute("BEGIN IMMEDIATE")
held.set()
release.wait(timeout=30)
conn.rollback()
finally:
conn.close()
def test_s1_contended_activity_write_gives_up_within_short_budget(tmp_path):
db = SessionDB(db_path=tmp_path / "state.db")
sid = "S1_CONTENDED"
db.create_session(sid, source="cli")
held = threading.Event()
release = threading.Event()
locker = threading.Thread(
target=_hold_write_lock, args=(tmp_path / "state.db", held, release)
)
locker.start()
try:
assert held.wait(timeout=5)
t0 = time.monotonic()
with pytest.raises(sqlite3.OperationalError):
db.touch_session_activity(sid, time.time(), description="working")
elapsed_touch = time.monotonic() - t0
finally:
release.set()
locker.join(timeout=10)
# The observational write gave up within the short budget — far below
# the 20s routine patience the review flagged.
assert elapsed_touch < 3.0, f"activity touch waited {elapsed_touch:.1f}s"
def test_s1_clear_labels_noop_skips_transaction(tmp_path, monkeypatch):
db = SessionDB(db_path=tmp_path / "state.db")
sid = "S1_NOOP"
db.create_session(sid, source="cli")
# Fresh session: labels empty → the clear must not open a transaction.
calls = []
original = db._execute_write
def _spy(fn, patience_s=None):
calls.append(fn)
return original(fn, patience_s=patience_s)
monkeypatch.setattr(db, "_execute_write", _spy)
db.clear_session_activity_labels(sid)
assert calls == [], "no-op label clear must skip the write transaction"
# Non-empty labels → clear runs exactly one write.
db.touch_session_activity(sid, time.time(), description="doing work")
calls.clear()
db.clear_session_activity_labels(sid)
assert len(calls) == 1
activity = db.get_session_activity(sid)
assert activity["last_activity_description"] == ""
def test_s1_contended_clear_gives_up_within_short_budget(tmp_path):
db = SessionDB(db_path=tmp_path / "state.db")
sid = "S1_CLEAR_CONTENDED"
db.create_session(sid, source="cli")
db.touch_session_activity(sid, time.time(), description="busy")
held = threading.Event()
release = threading.Event()
locker = threading.Thread(
target=_hold_write_lock, args=(tmp_path / "state.db", held, release)
)
locker.start()
try:
assert held.wait(timeout=5)
t0 = time.monotonic()
with pytest.raises(sqlite3.OperationalError):
db.clear_session_activity_labels(sid)
elapsed = time.monotonic() - t0
finally:
release.set()
locker.join(timeout=10)
assert elapsed < 3.0, f"label clear waited {elapsed:.1f}s under contention"