fix(state): quarantine SessionDB handle after structural corruption

A bare SQLITE_CORRUPT/NOTADB on a live write (not FTS-scoped, not a
replaced file) now sets a sticky per-instance flag: later writes fail
fast with StateDbCorruptError, the handle never reopens after close(),
and close() skips its explicit PASSIVE WAL checkpoint. Gateway and agent
flush paths divert pending transcripts to JSONL/spool like the replaced
case instead of retrying forever.

Field evidence: a handle that kept writing for ~50 minutes after the
first structural error checkpointed 15 pages under the wrong page
numbers on shutdown (page 1 <- messages_fts_trigram_data leaf), turning
"malformed" into "file is not a database".

Refs #90837, #90950, #97940, #89332, #45383

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CNX8rNYHqA5pT4tAGSzXtb
This commit is contained in:
leomcamilo
2026-09-02 05:22:41 -03:00
committed by kshitij
parent d8616f1c88
commit bcc2e65818
10 changed files with 565 additions and 11 deletions
@@ -0,0 +1,70 @@
"""Gateway SessionStore must divert, not retry forever, after structural corruption.
Mirrors ``test_session_db_replaced_fallback.py``: once the SessionDB handle
is quarantined (``StateDbCorruptError``) the pending transcript goes to the
JSONL/spool fallback and no FTS surgery runs on the damaged file.
"""
import json
import sqlite3
from gateway.config import GatewayConfig
from gateway.session import SessionStore
class _MalformedConn:
def __init__(self, real_conn):
self._real = real_conn
def execute(self, *args, **kwargs):
raise sqlite3.DatabaseError("database disk image is malformed")
def __getattr__(self, name):
return getattr(self._real, name)
def _assert_diverted(tmp_path, sid, needle):
pending = list((tmp_path / "pending_messages").glob("pending-*.json"))
assert pending, "expected pending_messages/pending-*.json spool"
spooled = False
for path in pending:
payload = json.loads(path.read_text(encoding="utf-8"))
message = (payload.get("data") or {}).get("message") or {}
if needle in str(message.get("content", "")):
spooled = True
break
assert spooled, f"{needle!r} missing from pending spool"
jsonl = tmp_path / "sessions" / f"{sid}.jsonl"
assert jsonl.is_file()
assert needle in jsonl.read_text(encoding="utf-8")
def test_corrupt_state_db_diverts_pending_without_fts_rebuild(tmp_path, monkeypatch):
import hermes_state
live = tmp_path / "state.db"
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", live)
store = SessionStore(sessions_dir=tmp_path, config=GatewayConfig())
sid = "gw-corrupt"
store._db.create_session(session_id=sid, source="cli")
store.append_to_transcript(
sid, {"role": "user", "content": "before", "timestamp": 1.0}
)
real_conn = store._db._conn
store._db._conn = _MalformedConn(real_conn)
try:
store.append_to_transcript(
sid, {"role": "user", "content": "after-corrupt", "timestamp": 2.0}
)
assert store._db._db_corrupt is True
# No FTS surgery ran on either layer.
assert store._db._fts_enabled is True
assert store._db._fts_stale is False
assert store._fts_rebuild_attempted is False
assert sid not in store._dirty_transcripts
_assert_diverted(tmp_path, sid, "after-corrupt")
finally:
store._db._conn = real_conn
store.close_all_db_handles()
@@ -0,0 +1,212 @@
"""Quarantine of a live SessionDB handle after structural (non-FTS) corruption.
Field evidence (the #90837 lost/reordered-page-write class): a gateway kept
retrying writes for ~50 minutes after ``gateway_routing`` reported
``database disk image is malformed``; on SIGTERM the close-time
``PRAGMA wal_checkpoint(PASSIVE)`` then wrote 15 pages to the wrong page
numbers (page 1 received a ``messages_fts_trigram_data`` leaf) and the file
stopped opening at all. Once structural corruption is observed on a handle
the only safe policy is to stop touching the file.
"""
import sqlite3
import pytest
from hermes_state import SessionDB, StateDbCorruptError
class _MalformedConn:
"""Connection proxy whose every execute reports bare SQLITE_CORRUPT."""
def __init__(self, real_conn):
self._real = real_conn
def execute(self, *args, **kwargs):
raise sqlite3.DatabaseError("database disk image is malformed")
def __getattr__(self, name):
return getattr(self._real, name)
class TestQuarantineAfterStructuralCorruption:
def test_structural_corruption_sets_sticky_flag_and_raises_typed(self, tmp_path):
db = SessionDB(db_path=tmp_path / "state.db")
real_conn = db._conn
try:
db.create_session(session_id="s1", source="cli", model="test")
db._conn = _MalformedConn(real_conn)
with pytest.raises(StateDbCorruptError, match="malformed") as excinfo:
db.create_session(session_id="s2", source="cli", model="test")
assert isinstance(excinfo.value.__cause__, sqlite3.DatabaseError)
assert db._db_corrupt is True
# Structural damage must never be mistaken for FTS-scoped damage.
assert db._fts_stale is False
finally:
db._conn = real_conn
db.close()
class _RecordingConn:
"""Connection proxy that records every SQL text and delegates."""
def __init__(self, real_conn):
self._real = real_conn
self.recorded = []
def execute(self, sql, *args, **kwargs):
self.recorded.append(str(sql))
return self._real.execute(sql, *args, **kwargs)
def __getattr__(self, name):
return getattr(self._real, name)
def _quarantined_db(tmp_path):
"""A SessionDB whose first corrupt write already tripped the quarantine."""
db = SessionDB(db_path=tmp_path / "state.db")
real_conn = db._conn
db.create_session(session_id="s1", source="cli", model="test")
db._conn = _MalformedConn(real_conn)
with pytest.raises(StateDbCorruptError):
db.create_session(session_id="s2", source="cli", model="test")
db._conn = real_conn
assert db._db_corrupt is True
return db, real_conn
class TestQuarantinedHandleStopsTouchingTheFile:
def test_subsequent_writes_fail_fast_without_touching_connection(self, tmp_path):
db, real_conn = _quarantined_db(tmp_path)
recorder = _RecordingConn(real_conn)
db._conn = recorder
try:
with pytest.raises(StateDbCorruptError):
db.create_session(session_id="s3", source="cli", model="test")
assert recorder.recorded == []
finally:
db._conn = real_conn
db.close()
def test_close_skips_wal_checkpoint_when_quarantined(self, tmp_path, caplog):
db, real_conn = _quarantined_db(tmp_path)
recorder = _RecordingConn(real_conn)
db._conn = recorder
with caplog.at_level("WARNING", logger="hermes_state"):
db.close()
assert not any("wal_checkpoint" in sql for sql in recorder.recorded)
assert db._conn is None
assert any(
"Skipping the close-time WAL checkpoint" in rec.getMessage()
and "hermes sessions recover" in rec.getMessage()
for rec in caplog.records
)
def test_reopen_after_close_refused_when_quarantined(self, tmp_path, monkeypatch):
from unittest.mock import MagicMock
db, real_conn = _quarantined_db(tmp_path)
db.close()
reopen = MagicMock()
monkeypatch.setattr("hermes_state._connect_tracked_db", reopen)
with pytest.raises(StateDbCorruptError, match="structural corruption"):
db.create_session(session_id="s4", source="cli", model="test")
reopen.assert_not_called()
# The read fallback after close() goes through the same reopen path.
with pytest.raises(StateDbCorruptError, match="refusing to reopen"):
db.get_session("s1")
reopen.assert_not_called()
class TestQuarantineScope:
def test_fts_scoped_corruption_does_not_trip_flag(self, tmp_path):
"""Corrupt FTS shadow tables keep the existing fail-open detach path."""
path = tmp_path / "state.db"
db = SessionDB(db_path=path)
db.create_session(session_id="s1", source="cli", model="test")
db.append_message("s1", role="user", content="hello world")
raw = sqlite3.connect(str(path))
raw.execute(
"UPDATE messages_fts_data SET block = X'DEADBEEFDEADBEEFDEADBEEFDEADBEEF'"
)
raw.commit()
raw.close()
try:
db.append_message("s1", role="user", content="healed append")
assert db._db_corrupt is False
assert db._fts_stale is True
assert db._fts_enabled is False
finally:
db.close()
def test_replaced_file_takes_precedence_over_corrupt(self, tmp_path):
import os
from hermes_state import StateDbReplacedError
live = tmp_path / "state.db"
other = tmp_path / "other.db"
db = SessionDB(db_path=live)
real_conn = db._conn
try:
db.create_session(session_id="s1", source="cli", model="test")
if db._db_file_identity is None:
pytest.skip("filesystem does not expose st_dev/st_ino")
alt = SessionDB(db_path=other)
alt.create_session("other", "cli")
alt.close()
os.replace(other, live)
db._conn = _MalformedConn(real_conn)
with pytest.raises(StateDbReplacedError):
db.create_session(session_id="s2", source="cli", model="test")
assert db._db_replaced is True
assert db._db_corrupt is False
finally:
db._conn = real_conn
db.close()
def test_classify_persistence_error_maps_quarantine_to_corrupt(self):
from hermes_state import _STATE_DB_CORRUPT_MSG, classify_persistence_error
assert classify_persistence_error(StateDbCorruptError("x")) == "corrupt"
# The stringified form (RPC boundaries) must classify the same way.
assert classify_persistence_error(_STATE_DB_CORRUPT_MSG) == "corrupt"
@pytest.fixture
def _clean_registry():
import hermes_state_registry as registry
registry.close_all()
registry._generations.clear()
registry._retired.clear()
yield registry
registry.close_all()
registry._generations.clear()
registry._retired.clear()
class TestSharedRegistry:
def test_holders_share_quarantine_and_close_all_skips_checkpoint(
self, tmp_path, _clean_registry
):
registry = _clean_registry
path = tmp_path / "state.db"
holder_a = registry.acquire(path)
holder_b = registry.acquire(path)
assert holder_a is holder_b
real_conn = holder_a._conn
holder_a.create_session(session_id="s1", source="cli", model="test")
holder_a._conn = _MalformedConn(real_conn)
with pytest.raises(StateDbCorruptError):
holder_a.create_session(session_id="s2", source="cli", model="test")
recorder = _RecordingConn(real_conn)
holder_b._conn = recorder
with pytest.raises(StateDbCorruptError):
holder_b.create_session(session_id="s3", source="cli", model="test")
registry.close_all()
assert not any("wal_checkpoint" in sql for sql in recorder.recorded)
assert holder_a._conn is None
@@ -0,0 +1,70 @@
"""Agent flush path: a quarantined (structurally corrupt) SessionDB diverts to JSONL.
Mirrors the replaced-file contract: the batch that SQLite will never take
again is kept on disk under ``sessions/<id>.jsonl`` instead of only in RAM,
the flush fails closed (no retry loop), and the turn-end explanation gets the
``corrupt`` cause.
"""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from hermes_state import SessionDB, StateDbCorruptError
from run_agent import AIAgent
def _flush_agent(db, session_id):
agent = SimpleNamespace(
_session_db=db,
_session_db_created=True,
_persist_disabled=False,
session_id=session_id,
_session_persist_lock=None,
_flushed_db_message_ids=set(),
_flushed_db_message_session_id=None,
_last_flushed_db_idx=0,
_db_flush_scan_prefix=None,
_persist_user_message_idx=None,
_persist_user_message_override=None,
_persist_user_message_timestamp=None,
_pending_cli_user_message=None,
_active_session_turn_lease_holder=None,
_last_persistence_error_cause=None,
_compression_adoption_failed=False,
)
agent._ensure_db_session = lambda: None
agent._flush_messages_to_session_db = (
AIAgent._flush_messages_to_session_db.__get__(agent, AIAgent)
)
agent._flush_messages_to_session_db_unlocked = (
AIAgent._flush_messages_to_session_db_unlocked.__get__(agent, AIAgent)
)
return agent
def test_flush_diverts_batch_to_jsonl_when_handle_is_quarantined(
tmp_path: Path, monkeypatch
) -> None:
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
db = SessionDB(db_path=tmp_path / "state.db")
try:
db.create_session("live", source="cli")
agent = _flush_agent(db, "live")
def _quarantined(self, *, session_id, messages, **kwargs):
raise StateDbCorruptError("database disk image is malformed (quarantined)")
monkeypatch.setattr(SessionDB, "append_messages_batch", _quarantined)
messages = [{"role": "user", "content": "kept-on-disk-after-corruption"}]
result = agent._flush_messages_to_session_db(messages, [])
assert result is False
assert agent._last_persistence_error_cause == "corrupt"
jsonl = tmp_path / "sessions" / "live.jsonl"
assert jsonl.is_file()
assert "kept-on-disk-after-corruption" in jsonl.read_text(encoding="utf-8")
finally:
db.close()
@@ -438,3 +438,8 @@ def test_run_conversation_partial_stream_recovery_surfaces_explanation():
assert result["response_previewed"] is False
def test_classify_persistence_error_quarantined_handle_is_corrupt() -> None:
"""A quarantined SessionDB raises the typed error; it stays in the corrupt bucket."""
from hermes_state import StateDbCorruptError, classify_persistence_error
assert classify_persistence_error(StateDbCorruptError("quarantined")) == "corrupt"
+22 -1
View File
@@ -332,7 +332,15 @@ class TestRuntimeFtsRebuild:
with pytest.raises(sqlite3.DatabaseError) as caught:
db._execute_write(lambda _conn: (_ for _ in ()).throw(structural))
assert caught.value is structural
# Structural corruption quarantines the handle: the typed error wraps
# the original (cause preserved, SQLite result code copied) and the
# sticky flag is set, so later writes fail fast.
from hermes_state import StateDbCorruptError
assert isinstance(caught.value, StateDbCorruptError)
assert caught.value.__cause__ is structural
assert caught.value.sqlite_errorcode == sqlite3.SQLITE_CORRUPT
assert db._db_corrupt is True
assert rebuild_called is False
assert db._fts_stale is False
assert _meta_value(tmp_path / "state.db", FTS_STALE_KEY) is None
@@ -831,6 +839,19 @@ class TestPhysicalCorruptionAcceptance:
# The misdiagnosis message from the field incident must be gone.
assert "canonical message rows are preserved" not in caplog.text
assert "attempting one-shot in-place FTS rebuild" not in caplog.text
# Structural damage quarantines the handle: typed error, sticky
# flag, later writes fail fast, and close() must not checkpoint
# the WAL over a damaged page image (the #90950 page-1 clobber).
from hermes_state import StateDbCorruptError
assert isinstance(caught.value, StateDbCorruptError)
assert db._db_corrupt is True
with pytest.raises(StateDbCorruptError):
db.append_message("s1", "user", "second write after corruption")
caplog.clear()
with caplog.at_level("WARNING", logger="hermes_state"):
db.close()
assert "Skipping the close-time WAL checkpoint" in caplog.text
finally:
db.close()
+5 -2
View File
@@ -14,7 +14,7 @@ from unittest.mock import MagicMock
import pytest
from hermes_state import SessionDB, _on_disk_journal_mode
from hermes_state import SessionDB, StateDbCorruptError, _on_disk_journal_mode
class _NotADbOnce:
@@ -42,9 +42,12 @@ class TestFailClosedAfterNotADb:
reopen = MagicMock()
monkeypatch.setattr("hermes_state._connect_tracked_db", reopen)
db._conn = _NotADbOnce(real_conn)
with pytest.raises(sqlite3.DatabaseError, match="not a database"):
with pytest.raises(sqlite3.DatabaseError, match="not a database") as excinfo:
db.create_session(session_id="s2", source="cli", model="test")
reopen.assert_not_called()
# NOTADB on a live write is structural: the handle is quarantined.
assert isinstance(excinfo.value, StateDbCorruptError)
assert db._db_corrupt is True
finally:
db._conn = real_conn
db.close()