fix(state): quarantine SessionDB handle after structural corruption
A bare SQLITE_CORRUPT/NOTADB on a live write (not FTS-scoped, not a replaced file) now sets a sticky per-instance flag: later writes fail fast with StateDbCorruptError, the handle never reopens after close(), and close() skips its explicit PASSIVE WAL checkpoint. Gateway and agent flush paths divert pending transcripts to JSONL/spool like the replaced case instead of retrying forever. Field evidence: a handle that kept writing for ~50 minutes after the first structural error checkpointed 15 pages under the wrong page numbers on shutdown (page 1 <- messages_fts_trigram_data leaf), turning "malformed" into "file is not a database". Refs #90837, #90950, #97940, #89332, #45383 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CNX8rNYHqA5pT4tAGSzXtb
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
"""Gateway SessionStore must divert, not retry forever, after structural corruption.
|
||||
|
||||
Mirrors ``test_session_db_replaced_fallback.py``: once the SessionDB handle
|
||||
is quarantined (``StateDbCorruptError``) the pending transcript goes to the
|
||||
JSONL/spool fallback and no FTS surgery runs on the damaged file.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
from gateway.config import GatewayConfig
|
||||
from gateway.session import SessionStore
|
||||
|
||||
|
||||
class _MalformedConn:
|
||||
def __init__(self, real_conn):
|
||||
self._real = real_conn
|
||||
|
||||
def execute(self, *args, **kwargs):
|
||||
raise sqlite3.DatabaseError("database disk image is malformed")
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._real, name)
|
||||
|
||||
|
||||
def _assert_diverted(tmp_path, sid, needle):
|
||||
pending = list((tmp_path / "pending_messages").glob("pending-*.json"))
|
||||
assert pending, "expected pending_messages/pending-*.json spool"
|
||||
spooled = False
|
||||
for path in pending:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
message = (payload.get("data") or {}).get("message") or {}
|
||||
if needle in str(message.get("content", "")):
|
||||
spooled = True
|
||||
break
|
||||
assert spooled, f"{needle!r} missing from pending spool"
|
||||
jsonl = tmp_path / "sessions" / f"{sid}.jsonl"
|
||||
assert jsonl.is_file()
|
||||
assert needle in jsonl.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_corrupt_state_db_diverts_pending_without_fts_rebuild(tmp_path, monkeypatch):
|
||||
import hermes_state
|
||||
|
||||
live = tmp_path / "state.db"
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", live)
|
||||
|
||||
store = SessionStore(sessions_dir=tmp_path, config=GatewayConfig())
|
||||
sid = "gw-corrupt"
|
||||
store._db.create_session(session_id=sid, source="cli")
|
||||
store.append_to_transcript(
|
||||
sid, {"role": "user", "content": "before", "timestamp": 1.0}
|
||||
)
|
||||
real_conn = store._db._conn
|
||||
store._db._conn = _MalformedConn(real_conn)
|
||||
try:
|
||||
store.append_to_transcript(
|
||||
sid, {"role": "user", "content": "after-corrupt", "timestamp": 2.0}
|
||||
)
|
||||
assert store._db._db_corrupt is True
|
||||
# No FTS surgery ran on either layer.
|
||||
assert store._db._fts_enabled is True
|
||||
assert store._db._fts_stale is False
|
||||
assert store._fts_rebuild_attempted is False
|
||||
assert sid not in store._dirty_transcripts
|
||||
_assert_diverted(tmp_path, sid, "after-corrupt")
|
||||
finally:
|
||||
store._db._conn = real_conn
|
||||
store.close_all_db_handles()
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Quarantine of a live SessionDB handle after structural (non-FTS) corruption.
|
||||
|
||||
Field evidence (the #90837 lost/reordered-page-write class): a gateway kept
|
||||
retrying writes for ~50 minutes after ``gateway_routing`` reported
|
||||
``database disk image is malformed``; on SIGTERM the close-time
|
||||
``PRAGMA wal_checkpoint(PASSIVE)`` then wrote 15 pages to the wrong page
|
||||
numbers (page 1 received a ``messages_fts_trigram_data`` leaf) and the file
|
||||
stopped opening at all. Once structural corruption is observed on a handle
|
||||
the only safe policy is to stop touching the file.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_state import SessionDB, StateDbCorruptError
|
||||
|
||||
|
||||
class _MalformedConn:
|
||||
"""Connection proxy whose every execute reports bare SQLITE_CORRUPT."""
|
||||
|
||||
def __init__(self, real_conn):
|
||||
self._real = real_conn
|
||||
|
||||
def execute(self, *args, **kwargs):
|
||||
raise sqlite3.DatabaseError("database disk image is malformed")
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._real, name)
|
||||
|
||||
|
||||
class TestQuarantineAfterStructuralCorruption:
|
||||
def test_structural_corruption_sets_sticky_flag_and_raises_typed(self, tmp_path):
|
||||
db = SessionDB(db_path=tmp_path / "state.db")
|
||||
real_conn = db._conn
|
||||
try:
|
||||
db.create_session(session_id="s1", source="cli", model="test")
|
||||
db._conn = _MalformedConn(real_conn)
|
||||
with pytest.raises(StateDbCorruptError, match="malformed") as excinfo:
|
||||
db.create_session(session_id="s2", source="cli", model="test")
|
||||
assert isinstance(excinfo.value.__cause__, sqlite3.DatabaseError)
|
||||
assert db._db_corrupt is True
|
||||
# Structural damage must never be mistaken for FTS-scoped damage.
|
||||
assert db._fts_stale is False
|
||||
finally:
|
||||
db._conn = real_conn
|
||||
db.close()
|
||||
|
||||
|
||||
class _RecordingConn:
|
||||
"""Connection proxy that records every SQL text and delegates."""
|
||||
|
||||
def __init__(self, real_conn):
|
||||
self._real = real_conn
|
||||
self.recorded = []
|
||||
|
||||
def execute(self, sql, *args, **kwargs):
|
||||
self.recorded.append(str(sql))
|
||||
return self._real.execute(sql, *args, **kwargs)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._real, name)
|
||||
|
||||
|
||||
def _quarantined_db(tmp_path):
|
||||
"""A SessionDB whose first corrupt write already tripped the quarantine."""
|
||||
db = SessionDB(db_path=tmp_path / "state.db")
|
||||
real_conn = db._conn
|
||||
db.create_session(session_id="s1", source="cli", model="test")
|
||||
db._conn = _MalformedConn(real_conn)
|
||||
with pytest.raises(StateDbCorruptError):
|
||||
db.create_session(session_id="s2", source="cli", model="test")
|
||||
db._conn = real_conn
|
||||
assert db._db_corrupt is True
|
||||
return db, real_conn
|
||||
|
||||
|
||||
class TestQuarantinedHandleStopsTouchingTheFile:
|
||||
def test_subsequent_writes_fail_fast_without_touching_connection(self, tmp_path):
|
||||
db, real_conn = _quarantined_db(tmp_path)
|
||||
recorder = _RecordingConn(real_conn)
|
||||
db._conn = recorder
|
||||
try:
|
||||
with pytest.raises(StateDbCorruptError):
|
||||
db.create_session(session_id="s3", source="cli", model="test")
|
||||
assert recorder.recorded == []
|
||||
finally:
|
||||
db._conn = real_conn
|
||||
db.close()
|
||||
|
||||
def test_close_skips_wal_checkpoint_when_quarantined(self, tmp_path, caplog):
|
||||
db, real_conn = _quarantined_db(tmp_path)
|
||||
recorder = _RecordingConn(real_conn)
|
||||
db._conn = recorder
|
||||
with caplog.at_level("WARNING", logger="hermes_state"):
|
||||
db.close()
|
||||
assert not any("wal_checkpoint" in sql for sql in recorder.recorded)
|
||||
assert db._conn is None
|
||||
assert any(
|
||||
"Skipping the close-time WAL checkpoint" in rec.getMessage()
|
||||
and "hermes sessions recover" in rec.getMessage()
|
||||
for rec in caplog.records
|
||||
)
|
||||
|
||||
def test_reopen_after_close_refused_when_quarantined(self, tmp_path, monkeypatch):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
db, real_conn = _quarantined_db(tmp_path)
|
||||
db.close()
|
||||
reopen = MagicMock()
|
||||
monkeypatch.setattr("hermes_state._connect_tracked_db", reopen)
|
||||
with pytest.raises(StateDbCorruptError, match="structural corruption"):
|
||||
db.create_session(session_id="s4", source="cli", model="test")
|
||||
reopen.assert_not_called()
|
||||
# The read fallback after close() goes through the same reopen path.
|
||||
with pytest.raises(StateDbCorruptError, match="refusing to reopen"):
|
||||
db.get_session("s1")
|
||||
reopen.assert_not_called()
|
||||
|
||||
|
||||
class TestQuarantineScope:
|
||||
def test_fts_scoped_corruption_does_not_trip_flag(self, tmp_path):
|
||||
"""Corrupt FTS shadow tables keep the existing fail-open detach path."""
|
||||
path = tmp_path / "state.db"
|
||||
db = SessionDB(db_path=path)
|
||||
db.create_session(session_id="s1", source="cli", model="test")
|
||||
db.append_message("s1", role="user", content="hello world")
|
||||
raw = sqlite3.connect(str(path))
|
||||
raw.execute(
|
||||
"UPDATE messages_fts_data SET block = X'DEADBEEFDEADBEEFDEADBEEFDEADBEEF'"
|
||||
)
|
||||
raw.commit()
|
||||
raw.close()
|
||||
try:
|
||||
db.append_message("s1", role="user", content="healed append")
|
||||
assert db._db_corrupt is False
|
||||
assert db._fts_stale is True
|
||||
assert db._fts_enabled is False
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def test_replaced_file_takes_precedence_over_corrupt(self, tmp_path):
|
||||
import os
|
||||
|
||||
from hermes_state import StateDbReplacedError
|
||||
|
||||
live = tmp_path / "state.db"
|
||||
other = tmp_path / "other.db"
|
||||
db = SessionDB(db_path=live)
|
||||
real_conn = db._conn
|
||||
try:
|
||||
db.create_session(session_id="s1", source="cli", model="test")
|
||||
if db._db_file_identity is None:
|
||||
pytest.skip("filesystem does not expose st_dev/st_ino")
|
||||
alt = SessionDB(db_path=other)
|
||||
alt.create_session("other", "cli")
|
||||
alt.close()
|
||||
os.replace(other, live)
|
||||
db._conn = _MalformedConn(real_conn)
|
||||
with pytest.raises(StateDbReplacedError):
|
||||
db.create_session(session_id="s2", source="cli", model="test")
|
||||
assert db._db_replaced is True
|
||||
assert db._db_corrupt is False
|
||||
finally:
|
||||
db._conn = real_conn
|
||||
db.close()
|
||||
|
||||
def test_classify_persistence_error_maps_quarantine_to_corrupt(self):
|
||||
from hermes_state import _STATE_DB_CORRUPT_MSG, classify_persistence_error
|
||||
|
||||
assert classify_persistence_error(StateDbCorruptError("x")) == "corrupt"
|
||||
# The stringified form (RPC boundaries) must classify the same way.
|
||||
assert classify_persistence_error(_STATE_DB_CORRUPT_MSG) == "corrupt"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _clean_registry():
|
||||
import hermes_state_registry as registry
|
||||
|
||||
registry.close_all()
|
||||
registry._generations.clear()
|
||||
registry._retired.clear()
|
||||
yield registry
|
||||
registry.close_all()
|
||||
registry._generations.clear()
|
||||
registry._retired.clear()
|
||||
|
||||
|
||||
class TestSharedRegistry:
|
||||
def test_holders_share_quarantine_and_close_all_skips_checkpoint(
|
||||
self, tmp_path, _clean_registry
|
||||
):
|
||||
registry = _clean_registry
|
||||
path = tmp_path / "state.db"
|
||||
holder_a = registry.acquire(path)
|
||||
holder_b = registry.acquire(path)
|
||||
assert holder_a is holder_b
|
||||
real_conn = holder_a._conn
|
||||
holder_a.create_session(session_id="s1", source="cli", model="test")
|
||||
|
||||
holder_a._conn = _MalformedConn(real_conn)
|
||||
with pytest.raises(StateDbCorruptError):
|
||||
holder_a.create_session(session_id="s2", source="cli", model="test")
|
||||
recorder = _RecordingConn(real_conn)
|
||||
holder_b._conn = recorder
|
||||
|
||||
with pytest.raises(StateDbCorruptError):
|
||||
holder_b.create_session(session_id="s3", source="cli", model="test")
|
||||
|
||||
registry.close_all()
|
||||
assert not any("wal_checkpoint" in sql for sql in recorder.recorded)
|
||||
assert holder_a._conn is None
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Agent flush path: a quarantined (structurally corrupt) SessionDB diverts to JSONL.
|
||||
|
||||
Mirrors the replaced-file contract: the batch that SQLite will never take
|
||||
again is kept on disk under ``sessions/<id>.jsonl`` instead of only in RAM,
|
||||
the flush fails closed (no retry loop), and the turn-end explanation gets the
|
||||
``corrupt`` cause.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from hermes_state import SessionDB, StateDbCorruptError
|
||||
from run_agent import AIAgent
|
||||
|
||||
|
||||
def _flush_agent(db, session_id):
|
||||
agent = SimpleNamespace(
|
||||
_session_db=db,
|
||||
_session_db_created=True,
|
||||
_persist_disabled=False,
|
||||
session_id=session_id,
|
||||
_session_persist_lock=None,
|
||||
_flushed_db_message_ids=set(),
|
||||
_flushed_db_message_session_id=None,
|
||||
_last_flushed_db_idx=0,
|
||||
_db_flush_scan_prefix=None,
|
||||
_persist_user_message_idx=None,
|
||||
_persist_user_message_override=None,
|
||||
_persist_user_message_timestamp=None,
|
||||
_pending_cli_user_message=None,
|
||||
_active_session_turn_lease_holder=None,
|
||||
_last_persistence_error_cause=None,
|
||||
_compression_adoption_failed=False,
|
||||
)
|
||||
agent._ensure_db_session = lambda: None
|
||||
agent._flush_messages_to_session_db = (
|
||||
AIAgent._flush_messages_to_session_db.__get__(agent, AIAgent)
|
||||
)
|
||||
agent._flush_messages_to_session_db_unlocked = (
|
||||
AIAgent._flush_messages_to_session_db_unlocked.__get__(agent, AIAgent)
|
||||
)
|
||||
return agent
|
||||
|
||||
|
||||
def test_flush_diverts_batch_to_jsonl_when_handle_is_quarantined(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
db = SessionDB(db_path=tmp_path / "state.db")
|
||||
try:
|
||||
db.create_session("live", source="cli")
|
||||
agent = _flush_agent(db, "live")
|
||||
|
||||
def _quarantined(self, *, session_id, messages, **kwargs):
|
||||
raise StateDbCorruptError("database disk image is malformed (quarantined)")
|
||||
|
||||
monkeypatch.setattr(SessionDB, "append_messages_batch", _quarantined)
|
||||
|
||||
messages = [{"role": "user", "content": "kept-on-disk-after-corruption"}]
|
||||
result = agent._flush_messages_to_session_db(messages, [])
|
||||
|
||||
assert result is False
|
||||
assert agent._last_persistence_error_cause == "corrupt"
|
||||
jsonl = tmp_path / "sessions" / "live.jsonl"
|
||||
assert jsonl.is_file()
|
||||
assert "kept-on-disk-after-corruption" in jsonl.read_text(encoding="utf-8")
|
||||
finally:
|
||||
db.close()
|
||||
@@ -438,3 +438,8 @@ def test_run_conversation_partial_stream_recovery_surfaces_explanation():
|
||||
assert result["response_previewed"] is False
|
||||
|
||||
|
||||
def test_classify_persistence_error_quarantined_handle_is_corrupt() -> None:
|
||||
"""A quarantined SessionDB raises the typed error; it stays in the corrupt bucket."""
|
||||
from hermes_state import StateDbCorruptError, classify_persistence_error
|
||||
|
||||
assert classify_persistence_error(StateDbCorruptError("quarantined")) == "corrupt"
|
||||
|
||||
@@ -332,7 +332,15 @@ class TestRuntimeFtsRebuild:
|
||||
with pytest.raises(sqlite3.DatabaseError) as caught:
|
||||
db._execute_write(lambda _conn: (_ for _ in ()).throw(structural))
|
||||
|
||||
assert caught.value is structural
|
||||
# Structural corruption quarantines the handle: the typed error wraps
|
||||
# the original (cause preserved, SQLite result code copied) and the
|
||||
# sticky flag is set, so later writes fail fast.
|
||||
from hermes_state import StateDbCorruptError
|
||||
|
||||
assert isinstance(caught.value, StateDbCorruptError)
|
||||
assert caught.value.__cause__ is structural
|
||||
assert caught.value.sqlite_errorcode == sqlite3.SQLITE_CORRUPT
|
||||
assert db._db_corrupt is True
|
||||
assert rebuild_called is False
|
||||
assert db._fts_stale is False
|
||||
assert _meta_value(tmp_path / "state.db", FTS_STALE_KEY) is None
|
||||
@@ -831,6 +839,19 @@ class TestPhysicalCorruptionAcceptance:
|
||||
# The misdiagnosis message from the field incident must be gone.
|
||||
assert "canonical message rows are preserved" not in caplog.text
|
||||
assert "attempting one-shot in-place FTS rebuild" not in caplog.text
|
||||
# Structural damage quarantines the handle: typed error, sticky
|
||||
# flag, later writes fail fast, and close() must not checkpoint
|
||||
# the WAL over a damaged page image (the #90950 page-1 clobber).
|
||||
from hermes_state import StateDbCorruptError
|
||||
|
||||
assert isinstance(caught.value, StateDbCorruptError)
|
||||
assert db._db_corrupt is True
|
||||
with pytest.raises(StateDbCorruptError):
|
||||
db.append_message("s1", "user", "second write after corruption")
|
||||
caplog.clear()
|
||||
with caplog.at_level("WARNING", logger="hermes_state"):
|
||||
db.close()
|
||||
assert "Skipping the close-time WAL checkpoint" in caplog.text
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_state import SessionDB, _on_disk_journal_mode
|
||||
from hermes_state import SessionDB, StateDbCorruptError, _on_disk_journal_mode
|
||||
|
||||
|
||||
class _NotADbOnce:
|
||||
@@ -42,9 +42,12 @@ class TestFailClosedAfterNotADb:
|
||||
reopen = MagicMock()
|
||||
monkeypatch.setattr("hermes_state._connect_tracked_db", reopen)
|
||||
db._conn = _NotADbOnce(real_conn)
|
||||
with pytest.raises(sqlite3.DatabaseError, match="not a database"):
|
||||
with pytest.raises(sqlite3.DatabaseError, match="not a database") as excinfo:
|
||||
db.create_session(session_id="s2", source="cli", model="test")
|
||||
reopen.assert_not_called()
|
||||
# NOTADB on a live write is structural: the handle is quarantined.
|
||||
assert isinstance(excinfo.value, StateDbCorruptError)
|
||||
assert db._db_corrupt is True
|
||||
finally:
|
||||
db._conn = real_conn
|
||||
db.close()
|
||||
|
||||
Reference in New Issue
Block a user