ba7743b076
Re-applied onto 3aee29089 after `hermes update` reset main to origin/main.
1. web_routers/sessions.py: _resolve_session_id() classifies malformed-DB
errors via the existing is_malformed_db_error() and raises 503 at all five
call sites. delete_session_endpoint was the worst — an unresolvable id
counted as idempotent success, so DELETE reported it had removed a session
that was still on disk.
2. gateway/lifecycle_ledger.py: check_state_db_integrity() runs PRAGMA
quick_check(1) on the unclean-exit path only (~2s on 500MB) and records the
verdict into gateway-exit-diag.log. The 2026-08-31 corruption sat undetected
for 3.5 days because nothing ever looked.
3. hermes_cli/gateway.py: `gateway run --replace` gave the outgoing gateway 5s
before SIGKILL; SessionDB.close() runs a PASSIVE WAL checkpoint that does
not finish in 5s on a WAL 4x past the autocheckpoint threshold, and a kill
mid-checkpoint tears b-tree pages. Grace raised to 30s via a testable
_await_gateway_exit() that also re-checks after the final sleep (a PID
exiting in the last interval must not be SIGKILLed — PID-reuse hazard).
NOT added: wal_checkpoint(TRUNCATE) at shutdown — removed upstream in #45383
because a TRUNCATE reset races the live writer and tears b-tree pages.
Adversarial review: Codex gpt-5.6-sol, 9.0/10 across three groups, no must-fix.
164 lines
5.3 KiB
Python
164 lines
5.3 KiB
Python
"""A corrupt state.db must not be reported as a missing session.
|
|
|
|
Regression for the 2026-08-31 incident: ``state.db`` corruption (rowid out of
|
|
order in ``sessions``, wrong entry count in ``idx_messages_session_id``) made
|
|
``resolve_session_id`` raise ``sqlite3.DatabaseError: database disk image is
|
|
malformed``. The Desktop showed "Couldn't open this session - session
|
|
unavailable" and the main process logged ``404: {"detail":"Session not
|
|
found"}`` — a session that existed was reported as absent, which sent the
|
|
diagnosis in entirely the wrong direction for a day.
|
|
|
|
A corrupt store is an unavailable store, not an empty one.
|
|
"""
|
|
|
|
import sqlite3
|
|
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
|
|
from hermes_cli import web_server
|
|
from hermes_cli.web_routers import sessions as sessions_router
|
|
|
|
|
|
class _MalformedDB:
|
|
"""SessionDB stand-in that fails the way a corrupt file really fails."""
|
|
|
|
def __init__(self):
|
|
self.closed = False
|
|
|
|
def resolve_session_id(self, session_id):
|
|
raise sqlite3.DatabaseError("database disk image is malformed")
|
|
|
|
def get_session(self, sid): # pragma: no cover - never reached
|
|
raise AssertionError("resolve_session_id should have raised first")
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
@pytest.fixture
|
|
def malformed_db(monkeypatch):
|
|
db = _MalformedDB()
|
|
monkeypatch.setattr(
|
|
web_server, "_open_session_db_for_profile", lambda profile, *, read_only: db
|
|
)
|
|
return db
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_corrupt_db_is_not_reported_as_missing_session(malformed_db):
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
await sessions_router.get_session_detail("20260830_180820_744f05")
|
|
|
|
assert excinfo.value.status_code != 404, (
|
|
"corruption reported as 'Session not found' - this is the bug"
|
|
)
|
|
assert excinfo.value.status_code == 503
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_corrupt_db_detail_names_the_real_cause(malformed_db):
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
await sessions_router.get_session_detail("20260830_180820_744f05")
|
|
|
|
detail = str(excinfo.value.detail).lower()
|
|
assert "corrupt" in detail or "malformed" in detail
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_db_is_closed_even_when_corruption_raises(malformed_db):
|
|
with pytest.raises(HTTPException):
|
|
await sessions_router.get_session_detail("20260830_180820_744f05")
|
|
|
|
assert malformed_db.closed, "connection leaked on the corruption path"
|
|
|
|
|
|
# ── The same lookup runs in four more handlers ──────────────────────────────
|
|
#
|
|
# delete is the dangerous one: an unresolvable id is treated as idempotent
|
|
# success ({"ok": True, "already_absent": True}), so a corrupt store made
|
|
# DELETE report that it had removed a session that is still on disk — and the
|
|
# Desktop drops the sidebar row on that answer.
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_messages_endpoint_reports_corruption(malformed_db):
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
await sessions_router.get_session_messages(
|
|
"20260830_180820_744f05", None, None, 0, None, False
|
|
)
|
|
assert excinfo.value.status_code == 503
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_does_not_claim_success_on_a_corrupt_store(malformed_db):
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
await sessions_router.delete_session_endpoint("20260830_180820_744f05")
|
|
assert excinfo.value.status_code == 503
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rename_endpoint_reports_corruption(malformed_db):
|
|
from hermes_cli.web_models import SessionRename
|
|
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
await sessions_router.rename_session_endpoint(
|
|
"20260830_180820_744f05", SessionRename(title="neu")
|
|
)
|
|
assert excinfo.value.status_code == 503
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_export_endpoint_reports_corruption(malformed_db):
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
await sessions_router.export_session_endpoint("20260830_180820_744f05")
|
|
assert excinfo.value.status_code == 503
|
|
|
|
|
|
# ── A genuinely absent session must still be a 404, not a 503 ──────────────
|
|
|
|
|
|
class _EmptyDB:
|
|
def __init__(self):
|
|
self.closed = False
|
|
|
|
def resolve_session_id(self, session_id):
|
|
return None
|
|
|
|
def get_session(self, sid):
|
|
return None
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_absent_session_is_still_404(monkeypatch):
|
|
monkeypatch.setattr(
|
|
web_server,
|
|
"_open_session_db_for_profile",
|
|
lambda profile, *, read_only: _EmptyDB(),
|
|
)
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
await sessions_router.get_session_detail("does_not_exist")
|
|
assert excinfo.value.status_code == 404
|
|
|
|
|
|
# ── Unrelated DatabaseErrors must not be relabelled as corruption ──────────
|
|
|
|
|
|
class _OtherErrorDB(_EmptyDB):
|
|
def resolve_session_id(self, session_id):
|
|
raise sqlite3.DatabaseError("some unrelated database failure")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_corruption_database_error_is_not_swallowed(monkeypatch):
|
|
monkeypatch.setattr(
|
|
web_server,
|
|
"_open_session_db_for_profile",
|
|
lambda profile, *, read_only: _OtherErrorDB(),
|
|
)
|
|
with pytest.raises(sqlite3.DatabaseError):
|
|
await sessions_router.get_session_detail("20260830_180820_744f05")
|