fix(state): fail closed when a live process still holds state.db during destructive restore (#90950)

The #91080 harness narrowed the #90950 corruption class to 'something
unlinks/replaces state.db or its -wal/-shm while another process holds
them'. The restore paths are exactly that actor:

- restore_quick_snapshot's unlink+move fallback replaced the inode and
  deleted sidecars under a live holder -> deleted-fd split brain.
- update auto-restore copy2'd a snapshot over a possibly-live DB ->
  the live writer's next checkpoint writes wrong-offset pages (the
  page-1 compression_locks clobber from the report).

Add _foreign_db_holder_pids(), a /proc fd scan that counts holders of
the DB and its sidecars (including already-deleted generations), and
refuse the destructive replacement while any holder exists. The
backup-API path (safe under live connections) remains the primary
route for /snapshot restore.
This commit is contained in:
Teknium
2026-08-26 05:16:11 -07:00
parent 33dce0eb7e
commit 7125e839d3
2 changed files with 77 additions and 1 deletions
+60
View File
@@ -635,6 +635,52 @@ def copy_db_and_verify(src: Path, dst: Path) -> bool:
return True
def _foreign_db_holder_pids(db_path: Path) -> Optional[List[int]]:
"""PIDs of OTHER processes holding *db_path* or its WAL/SHM open.
Linux-only ``/proc/<pid>/fd`` scan (no psutil dependency), preserving the
kernel's ``(deleted)`` suffix so an already-unlinked sidecar generation —
the #90950 split-brain fingerprint — still counts as held. Returns
``None`` when the scan is unavailable (non-Linux, or /proc unreadable);
callers must treat ``None`` as "unknown", not as "no holders".
"""
if not sys.platform.startswith("linux"):
return None
def _canonical(path: str) -> str:
return os.path.normcase(
os.path.abspath(path.removesuffix(" (deleted)"))
)
canonical_db = _canonical(os.fspath(db_path))
watched = {canonical_db, canonical_db + "-wal", canonical_db + "-shm"}
pids: List[int] = []
try:
own_pid = os.getpid()
for pid_str in os.listdir("/proc"):
if not pid_str.isdigit():
continue
pid = int(pid_str)
if pid == own_pid:
continue
fd_dir = f"/proc/{pid}/fd"
try:
fds = os.listdir(fd_dir)
except OSError:
continue
for fd in fds:
try:
target = os.readlink(f"{fd_dir}/{fd}")
except OSError:
continue
if _canonical(target) in watched:
pids.append(pid)
break
except OSError:
return None
return pids
def _safe_restore_db(src: Path, dst: Path) -> bool:
"""Restore a SQLite database from snapshot *src* into live *dst*.
@@ -686,6 +732,20 @@ def _safe_restore_db(src: Path, dst: Path) -> bool:
# Fallback: unlink+move (the old approach). This still works for
# the common case where no other process holds the DB open.
try:
holders = _foreign_db_holder_pids(dst)
if holders:
# Replacing the inode under a live holder is the #90950
# corruption class: the holder keeps writing through a
# deleted-inode fd (split brain), and removing its sidecars
# detaches the WAL index it is checkpointing through. The
# backup-API path above is the live-safe route; if it failed,
# fail closed rather than corrupt.
logger.error(
"Refusing unlink+move restore of %s: process(es) %s still "
"hold the database or its WAL open. Stop them and retry.",
dst, holders,
)
return False
tmp = dst.parent / f".{dst.name}.snap_restore"
shutil.copy2(src, tmp)
dst.unlink(missing_ok=True)
+17 -1
View File
@@ -1524,11 +1524,27 @@ def _restore_state_db_from_snapshot(state_path: Path, snap_state: Path) -> bool:
restored image cannot be silently overwritten by the corrupt database's WAL
replay — see :func:`_clear_stale_sqlite_sidecars`.
Refuses (returns ``False``) while another process still holds the database
or its sidecars open: copying a snapshot over a live writer's inode makes
the writer's page cache and WAL index disagree with the file bytes, and
its next checkpoint writes pages at offsets that no longer mean what it
thinks — the #90950 page-1 clobber. ``None`` (scan unavailable) proceeds:
the updater has already drained gateways, and refusing on "unknown" would
disable auto-restore on every non-Linux host.
Returns ``True`` when the restored file passes an integrity check. Raises
``OSError`` if the copy itself fails, which callers already report.
"""
from hermes_cli.backup import verify_sqlite_integrity
from hermes_cli.backup import _foreign_db_holder_pids, verify_sqlite_integrity
holders = _foreign_db_holder_pids(state_path)
if holders:
print(
f" ✗ Auto-restore refused: process(es) {holders} still hold "
"state.db or its WAL open. Stop them (hermes gateway stop), "
"then restore manually with /snapshot restore."
)
return False
_clear_stale_sqlite_sidecars(state_path)
shutil.copy2(snap_state, state_path)
restored = verify_sqlite_integrity(