fix(sessions): run state.db schema migration eagerly at backend startup and stop swallowing locked ALTERs

After `hermes update`, an existing state.db on an old schema made every
GET /api/sessions poll fail with sqlite3.OperationalError "no such
column: s.last_read_at" (or s.last_activity_at) until something
unrelated forced a writable open — the desktop sidebar showed "No
sessions yet" while every row sat intact on disk (#79531, #80037).

Two remaining root causes (the stale hand-written read probe was
already replaced by the SCHEMA_SQL-derived probe on main, prototyped in
draft PR #80030 by @Tilly-YL):

1. Migrations ran lazily: _init_schema/_reconcile_columns only ran on a
   writable open, typically the user's first NEW session. The dashboard
   backend now schedules one writable open of its own state.db from the
   lifespan (daemon thread, never blocks the ready-probe socket, never
   raises), so the store is brought current before the first session-
   list poll on every `hermes serve` / `hermes dashboard` / Desktop
   headless entrypoint.

2. _reconcile_columns caught sqlite3.OperationalError around every
   ALTER TABLE ADD COLUMN and logged at DEBUG. Lock contention from
   orphaned sibling backends made the ALTER fail silently — startup
   "succeeded" with a half-reconciled schema, and the open-time lock
   patience (#74478) never saw the error because it was swallowed
   inside first. Now: "duplicate column" races stay at DEBUG,
   locked/busy re-raises so _connect_and_init_with_lock_patience
   retries the whole idempotent init with jittered backoff, and any
   other failure (e.g. un-ADDable NOT NULL) logs at WARNING.

Regression tests: a store missing sessions.last_read_at is healed by
the eager startup reconcile and serves list_sessions_rich; a locked
ALTER propagates and is retried to success by the open lock patience;
duplicate-column races stay quiet; other ALTER failures warn.

Fixes #79531
Fixes #80037

Reported-by: @yenhunghuang (#79531) and @FLOW3R0111 (#80037)
Root-cause analysis: @wangyi0177-eng (stale read probe) and
@www654cc-pixel (_reconcile_columns DEBUG-swallow under lock
contention); draft PR #80030 by @Tilly-YL prototyped the probe fix.
This commit is contained in:
Teknium
2026-08-14 20:59:00 -07:00
parent 542f180e7a
commit f45813ea77
4 changed files with 278 additions and 6 deletions
+38
View File
@@ -304,6 +304,27 @@ def _resolve_restart_drain_timeout() -> float:
return DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT
def _eager_reconcile_own_session_db() -> None:
"""One writable open of this process's own state.db at startup.
``SessionDB.__init__`` runs ``_init_schema`` → ``_reconcile_columns``,
bringing a store left behind by `hermes update` current before the
dashboard's first session-list poll, with the open-time lock patience
(jittered retries) absorbing transient contention. Never raises: a
store this cannot fix is still served through the read-probe heal in
:func:`_open_session_db_at_path`, which retries on every poll.
"""
try:
from hermes_state import SessionDB, _default_db_path
SessionDB(db_path=Path(_default_db_path()), read_only=False).close()
except Exception as exc:
_log.warning(
"startup schema reconcile of state.db failed (%s); session "
"reads will retry the heal per poll", exc,
)
@asynccontextmanager
async def _lifespan(app: "FastAPI"):
app.state.event_channels = {} # dict[str, set]
@@ -315,6 +336,23 @@ async def _lifespan(app: "FastAPI"):
# event loop during lifespan startup — see _get_event_state's docstring.
app.state.chat_argv_lock = asyncio.Lock()
# Bring this profile's state.db schema current BEFORE the first
# session-list poll (#79531/#80037). Migrations used to run lazily on
# the first writable open — typically the user's first new session —
# so a store left behind by `hermes update` kept 500ing every
# /api/sessions poll (and the read-probe heal, while it retries per
# poll, can lose repeatedly to lock contention from orphaned sibling
# backends). One writable open here runs _init_schema →
# _reconcile_columns with the full open-time lock patience. Runs in a
# daemon thread so a locked store never delays the server socket (the
# Desktop ready-probe times out at 10s, GH-73083); reads that land
# before it finishes are still covered by the read-probe heal.
threading.Thread(
target=_eager_reconcile_own_session_db,
daemon=True,
name="statedb-eager-reconcile",
).start()
# Import hermes_cli.gateway eagerly *before* the lifespan yield so the
# GIL-heavy .pyc compilation and Defender scan cost is absorbed during
# backend initialisation — before the server socket accepts probes.
+29 -6
View File
@@ -566,12 +566,35 @@ class SessionSchemaMixin:
f'ALTER TABLE "{table_name}" ADD COLUMN "{safe_name}" {col_type}'
)
except sqlite3.OperationalError as exc:
# Expected: "duplicate column name" from a race or
# re-run. Unexpected: "Cannot add a NOT NULL column
# with default value NULL" from a schema mistake.
# Log at DEBUG so it's visible in agent.log.
logger.debug(
"reconcile %s.%s: %s", table_name, col_name, exc,
message = str(exc).lower()
if "duplicate column" in message:
# Expected: a sibling process won the race to ADD
# this column between our PRAGMA diff and the
# ALTER. The store ends up correct either way.
logger.debug(
"reconcile %s.%s: %s", table_name, col_name, exc,
)
continue
if "locked" in message or "busy" in message:
# Lock contention (e.g. an orphaned sibling
# backend holding the write lock, #79531). This
# used to be swallowed at DEBUG, leaving the
# store half-reconciled: startup "succeeded" and
# every session-list read then failed with
# "no such column" until an unrelated writable
# open. Re-raise instead so the open-time lock
# patience in _connect_and_init_with_lock_patience
# retries the WHOLE init (executescript is
# idempotent CREATE IF NOT EXISTS) with jittered
# backoff rather than serving a stale schema.
raise
# Anything else ("Cannot add a NOT NULL column with
# default value NULL", ...) is a schema mistake that
# permanently strands the store behind SCHEMA_SQL —
# be loud, don't bury it at DEBUG.
logger.warning(
"reconcile %s.%s failed; store remains behind "
"SCHEMA_SQL: %s", table_name, col_name, exc,
)
def _heal_gateway_routing_pk(self, cursor: sqlite3.Cursor) -> None:
+64
View File
@@ -476,6 +476,70 @@ class TestWebServerEndpoints:
"sidebar-stale"
]
def test_startup_eager_reconcile_heals_stale_store(self):
"""The lifespan's eager reconcile brings a stale store current.
#79531/#80037: after `hermes update` an old-schema state.db used to
stay stale until the first NEW session forced a writable open —
every /api/sessions poll 500ed with "no such column" in between.
The lifespan now schedules one writable open at startup; this
exercises that worker directly against a store missing
sessions.last_read_at and asserts the schema is brought current.
"""
import sqlite3
from hermes_cli import web_server
from hermes_constants import get_hermes_home
from hermes_state import SessionDB
db_path = get_hermes_home() / "state.db"
seed = SessionDB(db_path=db_path)
try:
seed.create_session("eager-stale", source="cli")
finally:
seed.close()
legacy = sqlite3.connect(str(db_path))
try:
legacy.execute("ALTER TABLE sessions DROP COLUMN last_read_at")
legacy.commit()
finally:
legacy.close()
web_server._eager_reconcile_own_session_db()
healed = sqlite3.connect(str(db_path))
try:
columns = {
row[1] for row in healed.execute("PRAGMA table_info(sessions)")
}
finally:
healed.close()
assert "last_read_at" in columns
# The healed store serves the full rich listing.
db = SessionDB(db_path=db_path, read_only=True)
try:
rows = db.list_sessions_rich(limit=10, compact_rows=True)
finally:
db.close()
assert [r["id"] for r in rows] == ["eager-stale"]
def test_startup_eager_reconcile_never_raises(self, monkeypatch):
"""A store the eager reconcile cannot open must not break startup."""
import sqlite3 as sqlite3_module
import hermes_state
from hermes_cli import web_server
def boom(*args, **kwargs):
raise sqlite3_module.OperationalError("database is locked")
monkeypatch.setattr(hermes_state, "SessionDB", boom)
# Must swallow — reads fall back to the per-poll probe heal.
web_server._eager_reconcile_own_session_db()
def test_heal_gives_up_when_reconcile_cannot_fix_the_store(self, monkeypatch):
"""A probe failure reconciliation can't cure must not retry forever.
+147
View File
@@ -1598,6 +1598,153 @@ class TestSchemaInit:
)
class TestReconcileColumnsErrorHandling:
"""_reconcile_columns must not bury migration failures (#79531/#80037).
A locked ALTER used to be swallowed at DEBUG: startup "succeeded" with a
half-reconciled schema and every session-list read then 500ed with
"no such column" until an unrelated writable open. The contract now:
duplicate-column races stay quiet, lock/busy propagates (so the open-time
lock patience retries the whole init), everything else warns.
"""
class _FailingAlterCursor:
"""Pass through to a real cursor, failing ALTER TABLE with ``exc``."""
def __init__(self, real_cursor, exc):
self._real = real_cursor
self._exc = exc
def execute(self, sql, *args, **kwargs):
if sql.lstrip().upper().startswith("ALTER TABLE"):
raise self._exc
return self._real.execute(sql, *args, **kwargs)
def __getattr__(self, name):
return getattr(self._real, name)
def _db_missing_column(self, tmp_path):
"""A store whose sessions table lacks last_read_at."""
db_path = tmp_path / "state.db"
seed = SessionDB(db_path=db_path)
seed.close()
conn = sqlite3.connect(str(db_path))
try:
conn.execute("ALTER TABLE sessions DROP COLUMN last_read_at")
conn.commit()
finally:
conn.close()
return db_path
def test_locked_alter_propagates(self, tmp_path):
"""database-is-locked must escape _reconcile_columns, not vanish.
Propagation is what lets _connect_and_init_with_lock_patience retry
the whole init with jittered backoff instead of serving a store
that is silently behind SCHEMA_SQL.
"""
db_path = self._db_missing_column(tmp_path)
conn = sqlite3.connect(str(db_path))
try:
stale = SessionDB.__new__(SessionDB)
stale._conn = conn
cursor = self._FailingAlterCursor(
conn.cursor(),
sqlite3.OperationalError("database is locked"),
)
with pytest.raises(sqlite3.OperationalError, match="locked"):
stale._reconcile_columns(cursor)
finally:
conn.close()
def test_duplicate_column_race_stays_quiet(self, tmp_path, caplog):
"""A duplicate-column race is expected and must not warn or raise."""
import logging
db_path = self._db_missing_column(tmp_path)
conn = sqlite3.connect(str(db_path))
try:
stale = SessionDB.__new__(SessionDB)
stale._conn = conn
cursor = self._FailingAlterCursor(
conn.cursor(),
sqlite3.OperationalError(
"duplicate column name: last_read_at"
),
)
with caplog.at_level(logging.WARNING, logger="hermes_state"):
stale._reconcile_columns(cursor)
finally:
conn.close()
assert not [
r for r in caplog.records if "reconcile" in r.getMessage()
]
def test_other_alter_failures_warn(self, tmp_path, caplog):
"""Schema mistakes (e.g. un-ADDable NOT NULL) log at WARNING."""
import logging
db_path = self._db_missing_column(tmp_path)
conn = sqlite3.connect(str(db_path))
try:
stale = SessionDB.__new__(SessionDB)
stale._conn = conn
cursor = self._FailingAlterCursor(
conn.cursor(),
sqlite3.OperationalError(
"Cannot add a NOT NULL column with default value NULL"
),
)
with caplog.at_level(logging.WARNING, logger="hermes_state"):
stale._reconcile_columns(cursor)
finally:
conn.close()
warnings = [
r
for r in caplog.records
if r.levelno >= logging.WARNING
and "reconcile" in r.getMessage()
]
assert warnings, "un-ADDable column failure must be logged at WARNING+"
def test_locked_alter_is_retried_by_open_lock_patience(self, tmp_path, monkeypatch):
"""End-to-end: a transiently locked ALTER heals on open retry.
The lock-patience wrapper retries on OperationalError raised out of
_connect_and_init; before this fix _reconcile_columns caught the
error internally so the retry never saw it and the store stayed
stale forever.
"""
db_path = self._db_missing_column(tmp_path)
original = SessionDB._reconcile_columns
calls = {"n": 0}
def flaky_reconcile(self, cursor):
calls["n"] += 1
if calls["n"] == 1:
raise sqlite3.OperationalError("database is locked")
return original(self, cursor)
monkeypatch.setattr(SessionDB, "_reconcile_columns", flaky_reconcile)
# Keep the retry fast — patience budget is 20s by default.
monkeypatch.setattr(SessionDB, "_WRITE_RETRY_SLOW_MIN_S", 0.001)
monkeypatch.setattr(SessionDB, "_WRITE_RETRY_SLOW_MAX_S", 0.005)
healed = SessionDB(db_path=db_path)
try:
cols = {
r[1]
for r in healed._conn.execute(
'PRAGMA table_info("sessions")'
).fetchall()
}
finally:
healed.close()
assert calls["n"] >= 2, "lock patience must retry the init"
assert "last_read_at" in cols
class TestTitleUniqueness:
"""Tests for unique title enforcement and title-based lookups."""