fix(state): fail closed when a live process still holds state.db during destructive restore (#90950)
The #91080 harness narrowed the #90950 corruption class to 'something unlinks/replaces state.db or its -wal/-shm while another process holds them'. The restore paths are exactly that actor: - restore_quick_snapshot's unlink+move fallback replaced the inode and deleted sidecars under a live holder -> deleted-fd split brain. - update auto-restore copy2'd a snapshot over a possibly-live DB -> the live writer's next checkpoint writes wrong-offset pages (the page-1 compression_locks clobber from the report). Add _foreign_db_holder_pids(), a /proc fd scan that counts holders of the DB and its sidecars (including already-deleted generations), and refuse the destructive replacement while any holder exists. The backup-API path (safe under live connections) remains the primary route for /snapshot restore.
This commit is contained in:
@@ -635,6 +635,52 @@ def copy_db_and_verify(src: Path, dst: Path) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _foreign_db_holder_pids(db_path: Path) -> Optional[List[int]]:
|
||||
"""PIDs of OTHER processes holding *db_path* or its WAL/SHM open.
|
||||
|
||||
Linux-only ``/proc/<pid>/fd`` scan (no psutil dependency), preserving the
|
||||
kernel's ``(deleted)`` suffix so an already-unlinked sidecar generation —
|
||||
the #90950 split-brain fingerprint — still counts as held. Returns
|
||||
``None`` when the scan is unavailable (non-Linux, or /proc unreadable);
|
||||
callers must treat ``None`` as "unknown", not as "no holders".
|
||||
"""
|
||||
if not sys.platform.startswith("linux"):
|
||||
return None
|
||||
|
||||
def _canonical(path: str) -> str:
|
||||
return os.path.normcase(
|
||||
os.path.abspath(path.removesuffix(" (deleted)"))
|
||||
)
|
||||
|
||||
canonical_db = _canonical(os.fspath(db_path))
|
||||
watched = {canonical_db, canonical_db + "-wal", canonical_db + "-shm"}
|
||||
pids: List[int] = []
|
||||
try:
|
||||
own_pid = os.getpid()
|
||||
for pid_str in os.listdir("/proc"):
|
||||
if not pid_str.isdigit():
|
||||
continue
|
||||
pid = int(pid_str)
|
||||
if pid == own_pid:
|
||||
continue
|
||||
fd_dir = f"/proc/{pid}/fd"
|
||||
try:
|
||||
fds = os.listdir(fd_dir)
|
||||
except OSError:
|
||||
continue
|
||||
for fd in fds:
|
||||
try:
|
||||
target = os.readlink(f"{fd_dir}/{fd}")
|
||||
except OSError:
|
||||
continue
|
||||
if _canonical(target) in watched:
|
||||
pids.append(pid)
|
||||
break
|
||||
except OSError:
|
||||
return None
|
||||
return pids
|
||||
|
||||
|
||||
def _safe_restore_db(src: Path, dst: Path) -> bool:
|
||||
"""Restore a SQLite database from snapshot *src* into live *dst*.
|
||||
|
||||
@@ -686,6 +732,20 @@ def _safe_restore_db(src: Path, dst: Path) -> bool:
|
||||
# Fallback: unlink+move (the old approach). This still works for
|
||||
# the common case where no other process holds the DB open.
|
||||
try:
|
||||
holders = _foreign_db_holder_pids(dst)
|
||||
if holders:
|
||||
# Replacing the inode under a live holder is the #90950
|
||||
# corruption class: the holder keeps writing through a
|
||||
# deleted-inode fd (split brain), and removing its sidecars
|
||||
# detaches the WAL index it is checkpointing through. The
|
||||
# backup-API path above is the live-safe route; if it failed,
|
||||
# fail closed rather than corrupt.
|
||||
logger.error(
|
||||
"Refusing unlink+move restore of %s: process(es) %s still "
|
||||
"hold the database or its WAL open. Stop them and retry.",
|
||||
dst, holders,
|
||||
)
|
||||
return False
|
||||
tmp = dst.parent / f".{dst.name}.snap_restore"
|
||||
shutil.copy2(src, tmp)
|
||||
dst.unlink(missing_ok=True)
|
||||
|
||||
@@ -1524,11 +1524,27 @@ def _restore_state_db_from_snapshot(state_path: Path, snap_state: Path) -> bool:
|
||||
restored image cannot be silently overwritten by the corrupt database's WAL
|
||||
replay — see :func:`_clear_stale_sqlite_sidecars`.
|
||||
|
||||
Refuses (returns ``False``) while another process still holds the database
|
||||
or its sidecars open: copying a snapshot over a live writer's inode makes
|
||||
the writer's page cache and WAL index disagree with the file bytes, and
|
||||
its next checkpoint writes pages at offsets that no longer mean what it
|
||||
thinks — the #90950 page-1 clobber. ``None`` (scan unavailable) proceeds:
|
||||
the updater has already drained gateways, and refusing on "unknown" would
|
||||
disable auto-restore on every non-Linux host.
|
||||
|
||||
Returns ``True`` when the restored file passes an integrity check. Raises
|
||||
``OSError`` if the copy itself fails, which callers already report.
|
||||
"""
|
||||
from hermes_cli.backup import verify_sqlite_integrity
|
||||
from hermes_cli.backup import _foreign_db_holder_pids, verify_sqlite_integrity
|
||||
|
||||
holders = _foreign_db_holder_pids(state_path)
|
||||
if holders:
|
||||
print(
|
||||
f" ✗ Auto-restore refused: process(es) {holders} still hold "
|
||||
"state.db or its WAL open. Stop them (hermes gateway stop), "
|
||||
"then restore manually with /snapshot restore."
|
||||
)
|
||||
return False
|
||||
_clear_stale_sqlite_sidecars(state_path)
|
||||
shutil.copy2(snap_state, state_path)
|
||||
restored = verify_sqlite_integrity(
|
||||
|
||||
Reference in New Issue
Block a user