diff --git a/hermes_cli/backup.py b/hermes_cli/backup.py index e50a374c47..4a86594197 100644 --- a/hermes_cli/backup.py +++ b/hermes_cli/backup.py @@ -635,6 +635,52 @@ def copy_db_and_verify(src: Path, dst: Path) -> bool: return True +def _foreign_db_holder_pids(db_path: Path) -> Optional[List[int]]: + """PIDs of OTHER processes holding *db_path* or its WAL/SHM open. + + Linux-only ``/proc//fd`` scan (no psutil dependency), preserving the + kernel's ``(deleted)`` suffix so an already-unlinked sidecar generation — + the #90950 split-brain fingerprint — still counts as held. Returns + ``None`` when the scan is unavailable (non-Linux, or /proc unreadable); + callers must treat ``None`` as "unknown", not as "no holders". + """ + if not sys.platform.startswith("linux"): + return None + + def _canonical(path: str) -> str: + return os.path.normcase( + os.path.abspath(path.removesuffix(" (deleted)")) + ) + + canonical_db = _canonical(os.fspath(db_path)) + watched = {canonical_db, canonical_db + "-wal", canonical_db + "-shm"} + pids: List[int] = [] + try: + own_pid = os.getpid() + for pid_str in os.listdir("/proc"): + if not pid_str.isdigit(): + continue + pid = int(pid_str) + if pid == own_pid: + continue + fd_dir = f"/proc/{pid}/fd" + try: + fds = os.listdir(fd_dir) + except OSError: + continue + for fd in fds: + try: + target = os.readlink(f"{fd_dir}/{fd}") + except OSError: + continue + if _canonical(target) in watched: + pids.append(pid) + break + except OSError: + return None + return pids + + def _safe_restore_db(src: Path, dst: Path) -> bool: """Restore a SQLite database from snapshot *src* into live *dst*. @@ -686,6 +732,20 @@ def _safe_restore_db(src: Path, dst: Path) -> bool: # Fallback: unlink+move (the old approach). This still works for # the common case where no other process holds the DB open. try: + holders = _foreign_db_holder_pids(dst) + if holders: + # Replacing the inode under a live holder is the #90950 + # corruption class: the holder keeps writing through a + # deleted-inode fd (split brain), and removing its sidecars + # detaches the WAL index it is checkpointing through. The + # backup-API path above is the live-safe route; if it failed, + # fail closed rather than corrupt. + logger.error( + "Refusing unlink+move restore of %s: process(es) %s still " + "hold the database or its WAL open. Stop them and retry.", + dst, holders, + ) + return False tmp = dst.parent / f".{dst.name}.snap_restore" shutil.copy2(src, tmp) dst.unlink(missing_ok=True) diff --git a/hermes_cli/update_cmd.py b/hermes_cli/update_cmd.py index 230b0698bf..87df675839 100644 --- a/hermes_cli/update_cmd.py +++ b/hermes_cli/update_cmd.py @@ -1524,11 +1524,27 @@ def _restore_state_db_from_snapshot(state_path: Path, snap_state: Path) -> bool: restored image cannot be silently overwritten by the corrupt database's WAL replay — see :func:`_clear_stale_sqlite_sidecars`. + Refuses (returns ``False``) while another process still holds the database + or its sidecars open: copying a snapshot over a live writer's inode makes + the writer's page cache and WAL index disagree with the file bytes, and + its next checkpoint writes pages at offsets that no longer mean what it + thinks — the #90950 page-1 clobber. ``None`` (scan unavailable) proceeds: + the updater has already drained gateways, and refusing on "unknown" would + disable auto-restore on every non-Linux host. + Returns ``True`` when the restored file passes an integrity check. Raises ``OSError`` if the copy itself fails, which callers already report. """ - from hermes_cli.backup import verify_sqlite_integrity + from hermes_cli.backup import _foreign_db_holder_pids, verify_sqlite_integrity + holders = _foreign_db_holder_pids(state_path) + if holders: + print( + f" ✗ Auto-restore refused: process(es) {holders} still hold " + "state.db or its WAL open. Stop them (hermes gateway stop), " + "then restore manually with /snapshot restore." + ) + return False _clear_stale_sqlite_sidecars(state_path) shutil.copy2(snap_state, state_path) restored = verify_sqlite_integrity(