diff --git a/hermes_state.py b/hermes_state.py index c5f5aad888..28946a6b06 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -13578,7 +13578,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ).fetchone() if not exists: return [] - delegate_ids = _collect_delegate_child_ids(self._conn, [session_id]) + # Use the borrowed read connection, never self._conn: handing the + # shared writer connection to a helper here executes on it without + # self._lock — the same unsynchronized-read class as #99349/#90734. + delegate_ids = _collect_delegate_child_ids(conn, [session_id]) return [session_id, *sorted(delegate_ids)] def delete_session( diff --git a/hermes_state_search.py b/hermes_state_search.py index 738dbbe6b2..221e9747f8 100644 --- a/hermes_state_search.py +++ b/hermes_state_search.py @@ -88,11 +88,13 @@ class SessionSearchMixin: self._merge_fts_incrementally( max_pages=self._FTS_MERGE_MAX_PAGES_PER_INDEX ) - except sqlite3.Error as exc: - # Routine maintenance is best effort, but unexpected SQLite errors - # must remain visible instead of being silently mistaken for an - # optional missing index. - logger.warning("FTS incremental merge failed: %s", exc) + except Exception as exc: # noqa: BLE001 - post-commit maintenance + # The canonical write is already committed before this cadence + # runs. No maintenance failure — including the bare SystemError + # the CPython sqlite3 layer can raise under cross-thread errmsg + # scrambling — may escape and make the caller replay an + # ambiguous, possibly-durable write (#90734, #85079). + logger.warning("FTS incremental merge failed after commit: %s", exc) def fts_rebuild_status(self) -> Optional[Dict[str, Any]]: """Return deferred-rebuild progress, or None when no rebuild pending. diff --git a/tests/state/test_writer_conn_thread_safety.py b/tests/state/test_writer_conn_thread_safety.py new file mode 100644 index 0000000000..6e460eb8f5 --- /dev/null +++ b/tests/state/test_writer_conn_thread_safety.py @@ -0,0 +1,258 @@ +"""Cross-thread races on the shared writer connection (2026-08-20 incident). + +``SessionDB`` shares ONE writer connection (``check_same_thread=False``) +guarded by ``self._lock``. A handful of read-only methods executed +statements on that same connection object WITHOUT taking the lock +(``get_compression_lock_holder``, ``list_pending_handoffs``, +``get_handoff_state``, and the no-op fast path of +``clear_session_activity_labels``). When one of those SELECTs raced a +turn-boundary ``append_messages_batch`` on the same connection, CPython's +sqlite3 layer raised a bare ``SystemError`` (" returned NULL +without setting an exception") — which is NOT a ``sqlite3.Error``, so it +escaped ``_execute_write``'s entire retry net and destroyed the user's +turn as ``session_persistence_failed``. + +Two independent layers are asserted here: + +1. The unlocked readers now route through ``_read_ctx()`` (per-thread + read-only connections under WAL), so hammering them concurrently with + turn-shaped batch writes must produce ZERO errors on either side. +2. Exactly-once containment: a bare ``SystemError`` inside the transaction + callback is never replayed, while post-commit maintenance failures are + logged without invalidating the already-durable write. +""" + +import sqlite3 +import threading +import time + +import pytest + +from hermes_state import SessionDB + + +@pytest.fixture +def db(tmp_path, monkeypatch): + monkeypatch.setattr(SessionDB, "_WRITE_PATIENCE_S", 2.0) + monkeypatch.setattr(SessionDB, "_WRITE_RETRY_MIN_S", 0.001) + monkeypatch.setattr(SessionDB, "_WRITE_RETRY_MAX_S", 0.005) + d = SessionDB(db_path=tmp_path / "state.db") + yield d + d.close() + + +class TestConcurrentReadersDoNotRaceTheWriter: + """Layer 1: the formerly-unlocked readers, hammered against batch writes. + + Before the fix this reproduced the production ``SystemError`` within a + few seconds on every run (each reader independently); after routing the + readers through ``_read_ctx()`` the writer and readers touch different + connection objects and the race is structurally gone. + """ + + READER_METHODS = ( + "get_compression_lock_holder", + "list_pending_handoffs", + "get_handoff_state", + "clear_session_activity_labels", + ) + + def _run_race(self, db, reader_fn, duration_s=3.0): + sid = db.create_session("race-sess", "test") + errors = [] + stop = threading.Event() + + def writer(): + n = 0 + while not stop.is_set(): + try: + db.append_messages_batch(sid, [ + {"role": "user", "content": "u%d" % n}, + {"role": "assistant", "content": "a%d" % n}, + {"role": "tool", "content": "t%d" % n, + "tool_name": "x", "tool_call_id": "c%d" % n}, + ]) + n += 1 + except Exception as exc: # noqa: BLE001 — the assertion IS the catch + errors.append(("writer", type(exc).__name__, str(exc))) + stop.set() + return + + def reader(): + while not stop.is_set(): + try: + reader_fn(db, sid) + except Exception as exc: # noqa: BLE001 + errors.append(("reader", type(exc).__name__, str(exc))) + stop.set() + return + + threads = [threading.Thread(target=writer)] + [ + threading.Thread(target=reader) for _ in range(3) + ] + for t in threads: + t.start() + deadline = time.monotonic() + duration_s + while time.monotonic() < deadline and not stop.is_set(): + time.sleep(0.05) + stop.set() + for t in threads: + t.join(timeout=10) + return errors + + def test_compression_lock_holder_reads_race_free(self, db): + errors = self._run_race( + db, lambda d, sid: d.get_compression_lock_holder(sid) + ) + assert errors == [] + + def test_pending_handoff_reads_race_free(self, db): + errors = self._run_race( + db, lambda d, sid: (d.list_pending_handoffs(), + d.get_handoff_state(sid)) + ) + assert errors == [] + + def test_activity_label_noop_fast_path_race_free(self, db): + errors = self._run_race( + db, lambda d, sid: d.clear_session_activity_labels(sid) + ) + assert errors == [] + + def test_unlocked_writer_conn_use_is_enumerated(self): + """Sibling-sweep tripwire: no method may touch ``self._conn`` + outside ``with self._lock`` (or the __init__/close lifecycle, which + runs before/after any concurrent access is possible). + + This is the AST sweep that found the four incident sites, frozen as + a test so a future convenience read can't quietly reintroduce the + class. + """ + import ast + import inspect + + import hermes_state as hs + + src = inspect.getsource(hs) + tree = ast.parse(src) + + ALLOWED_FUNCS = { + # Lifecycle: run before the instance is shared / after readers + # are drained. Not reachable concurrently with writers. + "__init__", "_connect_and_init", + "_connect_and_init_with_lock_patience", "close", + } + + def is_lock_with(node): + if isinstance(node, ast.With): + for item in node.items: + ctx = item.context_expr + if (isinstance(ctx, ast.Attribute) + and ctx.attr == "_lock" + and isinstance(ctx.value, ast.Name) + and ctx.value.id == "self"): + return True + return False + + violations = [] + + class Sweep(ast.NodeVisitor): + def __init__(self): + self.lock_depth = 0 + self.func_stack = [] + + @staticmethod + def _is_conn_attr(node): + return (isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "self" + and node.attr == "_conn") + + def _flag(self, node): + fn = self.func_stack[-1] if self.func_stack else "" + if fn not in ALLOWED_FUNCS: + violations.append((node.lineno, fn)) + + def generic_visit(self, node): + locked = is_lock_with(node) + is_func = isinstance( + node, (ast.FunctionDef, ast.AsyncFunctionDef) + ) + if locked: + self.lock_depth += 1 + if is_func: + self.func_stack.append(node.name) + if isinstance(node, ast.Call) and self.lock_depth == 0: + # A method call ON the connection (self._conn.execute(...)) + func = node.func + if (isinstance(func, ast.Attribute) + and self._is_conn_attr(func.value)): + self._flag(func.value) + # ...or the connection handed to a helper that will + # execute on it (e.g. _collect_delegate_child_ids(self._conn, ...)). + for arg in list(node.args) + [k.value for k in node.keywords]: + if self._is_conn_attr(arg): + self._flag(arg) + super().generic_visit(node) + if locked: + self.lock_depth -= 1 + if is_func: + self.func_stack.pop() + + Sweep().visit(tree) + assert violations == [], ( + "self._conn used outside 'with self._lock' in: %r — route reads " + "through _read_ctx() (or take the lock); an unlocked statement " + "on the shared writer connection races concurrent writers and " + "raises SystemError, destroying the turn as " + "session_persistence_failed" % (violations,) + ) + + +class TestSystemErrorTransactionBoundary: + """A bare SystemError must never replay an ambiguous write.""" + + def test_matching_error_inside_callback_is_not_replayed(self, db): + calls = {"n": 0} + + def broken(_conn): + calls["n"] += 1 + raise SystemError( + " returned NULL " + "without setting an exception" + ) + + with pytest.raises(SystemError, match="returned NULL"): + db._execute_write(broken) + assert calls["n"] == 1 + + def test_post_commit_maintenance_error_does_not_replay_message( + self, db, monkeypatch + ): + db.create_session("s1", "cli") + before = db._write_count + maintenance_calls = {"n": 0} + + def fail_once(*, max_pages): + maintenance_calls["n"] += 1 + if maintenance_calls["n"] == 1: + raise SystemError( + " returned NULL " + "without setting an exception" + ) + return 0 + + monkeypatch.setattr(db, "_FTS_MERGE_EVERY_N_WRITES", 1) + monkeypatch.setattr(db, "_merge_fts_incrementally", fail_once) + + message_id = db.append_message("s1", "user", "exactly-once") + matching = [ + row for row in db.get_messages("s1") + if row["content"] == "exactly-once" + ] + + assert isinstance(message_id, int) + assert len(matching) == 1 + assert db.get_session("s1")["message_count"] == 1 + assert db._write_count == before + 1 + assert maintenance_calls["n"] == 1