refactor(state): compact SessionMaintenanceMixin and state.db file helpers (-280 LOC, SQL-parity neutral)
hermes_state_maintenance.py 557->418, hermes_state_dbfile.py 545->404.
- _placeholders() replaces 4 inline ','.join('?'...) builders (identical output)
- _write_guards_reject() unifies the lease/lock probe in sweep_orphaned_sessions
and prune_sessions (same kwargs, same exception set; prune keeps set -= order)
- _page_pragmas() absorbs the try/except-debug shape of logical_size_bytes and
_freelist_ratio (log texts unchanged); _try_checkpoint() for the two WAL
checkpoints in vacuum(); _seconds_since() for the two state_meta float parses
- archived tri-state -> f'string' clause (byte-identical SQL)
- dbfile: contextlib.suppress for pass-only excepts, lock closures collapsed,
unreachable size<0 branch dropped, is_zeroed tail folded to one predicate
- docstrings/comments compacted by hand; every WHY/lock-safety invariant kept
SQL PARITY OK (1120 stmts), MSG PARITY OK, import smoke OK.
This commit is contained in:
Teknium
+123
-264
@@ -1,12 +1,11 @@
|
||||
"""state.db file-level health helpers.
|
||||
"""state.db file-level health helpers, split out of ``hermes_state.py``.
|
||||
|
||||
Split out of ``hermes_state.py``: header probes (application_id / zeroed-file
|
||||
detection), deleted-WAL-sidecar holder scans, quarantine of zeroed or
|
||||
lock-poisoned databases, ``collect_state_db_stats`` and holder-process
|
||||
classification. Every name is re-imported into ``hermes_state`` so
|
||||
``hermes_state.<name>`` keeps resolving — and tests that monkeypatch it keep
|
||||
intercepting, because intra-module calls to patched helpers go through a
|
||||
lazy ``from hermes_state import ...`` at call time.
|
||||
Header probes (application_id / zeroed-file detection), deleted-WAL-sidecar
|
||||
holder scans, quarantine of zeroed databases, ``collect_state_db_stats`` and
|
||||
holder-process classification. Every name is re-imported into ``hermes_state``
|
||||
so ``hermes_state.<name>`` keeps resolving — and tests that monkeypatch it keep
|
||||
intercepting, because intra-module calls to patched helpers go through a lazy
|
||||
``from hermes_state import ...`` at call time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,49 +23,35 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from hermes_state_common import (
|
||||
FTS_REBUILD_DEFERRAL_KEY,
|
||||
stat_db_file_identity as _stat_db_file_identity,
|
||||
FTS_REBUILD_DEFERRAL_KEY, stat_db_file_identity as _stat_db_file_identity
|
||||
)
|
||||
|
||||
# Log-record parity with the origin module (caplog tests pin "hermes_state").
|
||||
logger = logging.getLogger("hermes_state")
|
||||
|
||||
# _read_sqlite_application_id runs on EVERY write via _raise_if_db_replaced,
|
||||
# against the LIVE state.db. A bare open()/read()/close() there is the
|
||||
# howtocorrupt §2.2 bug: close() cancels every POSIX advisory lock this
|
||||
# process holds on the file — one probe call drops the WAL-mode DMS shared
|
||||
# lock the writer connection holds (see hermes_cli/sqlite_safe_read.py). With
|
||||
# the DMS lock gone, a fresh opener in another process can treat this writer
|
||||
# as dead and rerun WAL-index recovery underneath it.
|
||||
#
|
||||
# The probe therefore reads through a per-path fd cached for the life of the
|
||||
# process: opening an fd never cancels locks (only close() does), and
|
||||
# os.pread takes no shared file position. When the path is re-pointed at a
|
||||
# new inode (the very replacement this probe exists to detect), the stale fd
|
||||
# is RETIRED, never closed — closing it would cancel the live connection's
|
||||
# locks on the old file. Replacement events are rare and halt writes anyway,
|
||||
# so the leak is bounded.
|
||||
# _read_sqlite_application_id runs on EVERY write (_raise_if_db_replaced) against the LIVE
|
||||
# state.db. A bare open()/read()/close() there is the howtocorrupt §2.2 bug: close() cancels
|
||||
# every POSIX advisory lock this process holds on the file, dropping the writer's WAL-mode DMS
|
||||
# shared lock (see hermes_cli/sqlite_safe_read.py) so another process can treat this writer as
|
||||
# dead and rerun WAL-index recovery underneath it. So the probe preads through a per-path fd
|
||||
# cached for the life of the process (opening never cancels locks). When the path is re-pointed
|
||||
# at a new inode (the very replacement this probe detects) the stale fd is RETIRED, never closed
|
||||
# — closing it would cancel the live connection's locks. Replacements are rare and halt writes.
|
||||
_HEADER_PROBE_LOCK = threading.Lock()
|
||||
_HEADER_PROBE_FDS: "dict[str, tuple[int, int, int]]" = {} # key -> (fd, dev, ino)
|
||||
_RETIRED_HEADER_PROBE_FDS: "list[int]" = [] # intentionally never closed
|
||||
|
||||
_FTS_TABLE_NAMES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
|
||||
|
||||
|
||||
def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]":
|
||||
"""Lock-safe raw header read of a possibly-live SQLite database.
|
||||
|
||||
POSIX: pread from a cached, never-closed fd (rebound when the path names
|
||||
a new inode). Windows: plain read — advisory-lock cancellation is a
|
||||
POSIX-only hazard and msvcrt locks do not share the failure mode.
|
||||
"""
|
||||
"""Lock-safe raw header read of a possibly-live SQLite database: POSIX preads from a cached,
|
||||
never-closed fd (rebound when the path names a new inode); Windows reads plainly, since
|
||||
advisory-lock cancellation is a POSIX-only hazard."""
|
||||
from hermes_state import _IS_WINDOWS
|
||||
if _IS_WINDOWS:
|
||||
try:
|
||||
with db_path.open("rb") as handle:
|
||||
return handle.read(length)
|
||||
except OSError:
|
||||
return None
|
||||
with contextlib.suppress(OSError), db_path.open("rb") as handle:
|
||||
return handle.read(length)
|
||||
return None
|
||||
key = str(db_path)
|
||||
try:
|
||||
st = os.stat(db_path)
|
||||
@@ -76,9 +61,8 @@ def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]":
|
||||
cached = _HEADER_PROBE_FDS.get(key)
|
||||
if cached is not None and (cached[1], cached[2]) != (st.st_dev, st.st_ino):
|
||||
# Path re-pointed at a new file. Retire (never close) the old fd.
|
||||
_RETIRED_HEADER_PROBE_FDS.append(cached[0])
|
||||
_RETIRED_HEADER_PROBE_FDS.append(_HEADER_PROBE_FDS.pop(key)[0])
|
||||
cached = None
|
||||
del _HEADER_PROBE_FDS[key]
|
||||
if cached is None:
|
||||
try:
|
||||
fd = os.open(db_path, os.O_RDONLY)
|
||||
@@ -90,18 +74,13 @@ def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]":
|
||||
_RETIRED_HEADER_PROBE_FDS.append(fd)
|
||||
return None
|
||||
cached = _HEADER_PROBE_FDS[key] = (fd, fst.st_dev, fst.st_ino)
|
||||
try:
|
||||
with contextlib.suppress(OSError):
|
||||
return os.pread(cached[0], length, 0)
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _read_sqlite_application_id(db_path: Path) -> "Optional[int]":
|
||||
"""Read application_id from the SQLite header without opening a connection.
|
||||
|
||||
Routed through :func:`_pread_db_header`, which never issues a ``close()``
|
||||
that would cancel this process's POSIX locks on the file.
|
||||
"""
|
||||
"""application_id from the SQLite header, via the lock-safe :func:`_pread_db_header`."""
|
||||
from hermes_state import _STATE_DB_APPLICATION_ID_OFFSET
|
||||
end = _STATE_DB_APPLICATION_ID_OFFSET + 4
|
||||
header = _pread_db_header(db_path, end)
|
||||
@@ -112,13 +91,9 @@ def _read_sqlite_application_id(db_path: Path) -> "Optional[int]":
|
||||
|
||||
def _stat_sqlite_sidecar_identity(db_path: Path) -> Dict[str, tuple]:
|
||||
"""Snapshot ``(st_dev, st_ino)`` for existing WAL/SHM sidecars."""
|
||||
identities: Dict[str, tuple] = {}
|
||||
base = os.fspath(db_path)
|
||||
for suffix in ("-wal", "-shm"):
|
||||
ident = _stat_db_file_identity(Path(base + suffix))
|
||||
if ident is not None:
|
||||
identities[suffix] = ident
|
||||
return identities
|
||||
idents = {suffix: _stat_db_file_identity(Path(base + suffix)) for suffix in ("-wal", "-shm")}
|
||||
return {suffix: ident for suffix, ident in idents.items() if ident is not None}
|
||||
|
||||
|
||||
def _canonical_sqlite_path(path: str) -> str:
|
||||
@@ -142,25 +117,15 @@ def _iter_proc_fd_targets():
|
||||
except OSError:
|
||||
continue # process gone or not ours
|
||||
for fd in fds:
|
||||
try:
|
||||
with contextlib.suppress(OSError):
|
||||
yield int(pid_str), os.readlink(f"{fd_dir}/{fd}")
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
|
||||
def iter_deleted_sqlite_sidecar_holders(db_path) -> List[Tuple[int, str]]:
|
||||
"""Return processes holding an unlinked ``state.db-wal`` / ``-shm``.
|
||||
|
||||
Linux-only (``/proc/<pid>/fd`` readlink). Windows and other hosts
|
||||
return ``[]`` — Windows cannot unlink a sidecar another process still
|
||||
holds, and macOS does not use the `` (deleted)`` suffix.
|
||||
|
||||
The scan includes this process: on the SessionDB open/write refuse
|
||||
path, the in-process writer that still holds the orphan inode is the
|
||||
one that must not mint a replacement WAL (and must stop committing).
|
||||
``_foreign_state_db_holders`` keeps skipping this PID for FTS
|
||||
maintenance so a process does not block its own optional repair.
|
||||
"""
|
||||
"""Return processes holding an unlinked ``state.db-wal`` / ``-shm``. Linux-only; ``[]``
|
||||
elsewhere (Windows cannot unlink a held sidecar, macOS has no `` (deleted)`` suffix).
|
||||
Includes this process: on the open/write refuse path the in-process writer holding the orphan
|
||||
inode must not mint a replacement WAL (``_foreign_state_db_holders`` skips this PID)."""
|
||||
if not sys.platform.startswith("linux"):
|
||||
return []
|
||||
holders: List[Tuple[int, str]] = []
|
||||
@@ -175,11 +140,8 @@ def iter_deleted_sqlite_sidecar_holders(db_path) -> List[Tuple[int, str]]:
|
||||
|
||||
|
||||
def refuse_deleted_wal_generation(db_path) -> None:
|
||||
"""Raise if any process holds a deleted WAL/SHM generation for *db_path*.
|
||||
|
||||
Called *before* ``sqlite3.connect`` so a second opener cannot mint a
|
||||
replacement WAL inode while a live writer still holds the orphan.
|
||||
"""
|
||||
"""Raise if any process holds a deleted WAL/SHM generation for *db_path*; called
|
||||
*before* ``sqlite3.connect`` so a second opener cannot mint a replacement WAL inode."""
|
||||
from hermes_state import DeletedWalGenerationError, _DELETED_WAL_GENERATION_MSG
|
||||
if not iter_deleted_sqlite_sidecar_holders(db_path):
|
||||
return
|
||||
@@ -188,83 +150,51 @@ def refuse_deleted_wal_generation(db_path) -> None:
|
||||
|
||||
|
||||
def _connect_tracked_db(path, tracking_path=None, **kwargs):
|
||||
"""``sqlite3.connect`` that registers the open fd for lock-safety.
|
||||
|
||||
While a connection is live, byte-level probes of the same file are
|
||||
refused: an ``open()``/``close()`` cancels every POSIX advisory lock this
|
||||
process holds on it -- including a running VACUUM's EXCLUSIVE lock.
|
||||
Released automatically on ``close()``.
|
||||
|
||||
The ONLY tolerated fallback is the helper being absent entirely
|
||||
(scaffold/embed installs that ship hermes_state without hermes_cli). A
|
||||
real connection failure must propagate: silently retrying an *untracked*
|
||||
connect would disable the guard for the lifetime of that connection.
|
||||
"""
|
||||
"""``sqlite3.connect`` that registers the open fd so byte-level probes of a live file are
|
||||
refused (an ``open()``/``close()`` would cancel every POSIX lock, even a running VACUUM's
|
||||
EXCLUSIVE). The ONLY tolerated fallback is the helper being absent (scaffold/embed installs
|
||||
without hermes_cli); a real connection failure must propagate — a silent untracked retry
|
||||
would disable the guard for that connection."""
|
||||
try:
|
||||
from hermes_cli.sqlite_safe_read import connect_tracked
|
||||
except ImportError:
|
||||
logger.debug(
|
||||
"hermes_cli.sqlite_safe_read unavailable; opening %s untracked "
|
||||
"(byte-probe guard inactive in this install)",
|
||||
path,
|
||||
)
|
||||
logger.debug("hermes_cli.sqlite_safe_read unavailable; opening %s untracked "
|
||||
"(byte-probe guard inactive in this install)", path)
|
||||
return sqlite3.connect(str(path), **kwargs)
|
||||
# Open through THIS module's sqlite3.connect so callers (and tests) that
|
||||
# patch hermes_state.sqlite3.connect keep control of connection creation;
|
||||
# the helper still owns tracking.
|
||||
# Open through THIS module's sqlite3.connect so tests patching hermes_state.sqlite3.connect keep control.
|
||||
return connect_tracked(path, tracking_path=tracking_path, connect_fn=sqlite3.connect, **kwargs)
|
||||
|
||||
|
||||
def is_zeroed_state_db(path: Path, *, probe_bytes: int = 100, force: bool = False) -> bool:
|
||||
"""Detect the zeroed state.db signature (0-byte or NUL header).
|
||||
|
||||
Byte-level probe, so it is only safe BEFORE any connection to *path*
|
||||
exists in this process: ``close()`` cancels every POSIX advisory lock the
|
||||
process holds on the file, which can pull the EXCLUSIVE lock out from
|
||||
under a running VACUUM and corrupt the database. The read is routed
|
||||
through ``read_header_bytes_preopen``, which refuses (returning False
|
||||
here) once a connection is live. Pass ``force=True`` only for offline
|
||||
files -- quarantined copies, snapshots, archives.
|
||||
|
||||
Prefer ``hermes_cli.backup.is_zeroed_sqlite_file`` when available; this
|
||||
local copy keeps SessionDB openable without importing the CLI package
|
||||
in constrained embed paths.
|
||||
"""
|
||||
try:
|
||||
"""Detect the zeroed state.db signature (0-byte or NUL header). Byte-level probe, so only
|
||||
safe BEFORE any connection to *path* exists in this process (``close()`` cancels every POSIX
|
||||
lock, even a running VACUUM's EXCLUSIVE); ``read_header_bytes_preopen`` refuses (-> False)
|
||||
once a connection is live. Pass ``force=True`` only for offline files (quarantined copies,
|
||||
snapshots). Prefers ``hermes_cli.backup.is_zeroed_sqlite_file``; this copy keeps SessionDB
|
||||
openable without the CLI package in constrained embed paths."""
|
||||
with contextlib.suppress(Exception):
|
||||
from hermes_cli.backup import is_zeroed_sqlite_file
|
||||
|
||||
return is_zeroed_sqlite_file(path, probe_bytes=probe_bytes, force=force)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
# Special files (FIFO, device, socket) are never "zeroed", and probing
|
||||
# a FIFO would block until a writer appears.
|
||||
if not path.is_file():
|
||||
# Special files (FIFO, device, socket) are never "zeroed", and
|
||||
# probing a FIFO would block until a writer appears.
|
||||
return False
|
||||
size = path.stat().st_size
|
||||
path.stat()
|
||||
except OSError:
|
||||
return False
|
||||
if size < 0:
|
||||
return False
|
||||
from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen
|
||||
|
||||
if not force and has_live_connection(path):
|
||||
return False
|
||||
head = read_header_bytes_preopen(path, length=max(16, probe_bytes), force=force)
|
||||
if head is None:
|
||||
return False
|
||||
if len(head) == 0:
|
||||
return True
|
||||
if head.startswith(b"SQLite format 3"):
|
||||
return False
|
||||
return all(byte == 0 for byte in head)
|
||||
# b"" (0-byte file) is zeroed; all() over an empty header is True.
|
||||
return head is not None and not head.startswith(b"SQLite format 3") and all(b == 0 for b in head)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def quarantine_cross_process_lock(path: Path, timeout: float = 5.0):
|
||||
"""Acquire the cross-process lock for path.quarantine.lock."""
|
||||
import platform
|
||||
|
||||
lock_path = path.with_name(path.name + ".quarantine.lock")
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = lock_path.open("a+b")
|
||||
@@ -273,27 +203,21 @@ def quarantine_cross_process_lock(path: Path, timeout: float = 5.0):
|
||||
if platform.system() == "Windows":
|
||||
import msvcrt
|
||||
|
||||
def _try_lock():
|
||||
def _lock(mode): # msvcrt locks a byte range from the current position
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
msvcrt.locking(handle.fileno(), mode, 1)
|
||||
|
||||
def _unlock():
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
_try_lock = lambda: _lock(msvcrt.LK_NBLCK) # noqa: E731
|
||||
_unlock = lambda: _lock(msvcrt.LK_UNLCK) # noqa: E731
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
def _try_lock():
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
|
||||
def _unlock():
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
_try_lock = lambda: fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) # noqa: E731
|
||||
_unlock = lambda: fcntl.flock(handle.fileno(), fcntl.LOCK_UN) # noqa: E731
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
while not acquired:
|
||||
try:
|
||||
_try_lock()
|
||||
acquired = True
|
||||
break
|
||||
except OSError:
|
||||
if time.monotonic() >= deadline:
|
||||
break
|
||||
@@ -310,23 +234,16 @@ def quarantine_cross_process_lock(path: Path, timeout: float = 5.0):
|
||||
|
||||
|
||||
def quarantine_zeroed_state_db(path: Path, *, already_locked: bool = False) -> Optional[Path]:
|
||||
"""Move a zeroed state.db aside (preserve bytes) and return quarantine path.
|
||||
|
||||
Uses a cross-process lock so two concurrent startups cannot race: the first
|
||||
process moves the zeroed file and the second re-checks under the lock,
|
||||
finding the file already gone (or a fresh DB in its place) instead of
|
||||
clobbering the quarantine.
|
||||
"""
|
||||
"""Move a zeroed state.db aside (preserve bytes) and return quarantine path. A cross-process
|
||||
lock stops two concurrent startups racing: the second re-checks under the lock and finds the
|
||||
file gone (or fresh) instead of clobbering the quarantine."""
|
||||
def _do_quarantine():
|
||||
if not path.exists():
|
||||
logger.info("quarantine_zeroed_state_db: %s already moved by another process", path)
|
||||
return None
|
||||
if not is_zeroed_state_db(path):
|
||||
logger.info(
|
||||
"quarantine_zeroed_state_db: %s is no longer zeroed (another "
|
||||
"process quarantined it and a fresh DB was created)",
|
||||
path,
|
||||
)
|
||||
logger.info("quarantine_zeroed_state_db: %s is no longer zeroed (another "
|
||||
"process quarantined it and a fresh DB was created)", path)
|
||||
return None
|
||||
try:
|
||||
ts = time.strftime("%Y%m%d-%H%M%S")
|
||||
@@ -346,68 +263,44 @@ def quarantine_zeroed_state_db(path: Path, *, already_locked: bool = False) -> O
|
||||
for suffix in ("-wal", "-shm"):
|
||||
side = Path(str(path) + suffix)
|
||||
if side.exists():
|
||||
try:
|
||||
with contextlib.suppress(OSError):
|
||||
side.rename(Path(str(dest) + suffix))
|
||||
except OSError:
|
||||
pass
|
||||
return dest
|
||||
|
||||
if already_locked:
|
||||
return _do_quarantine()
|
||||
with quarantine_cross_process_lock(path) as acquired:
|
||||
if not acquired:
|
||||
logger.error(
|
||||
"quarantine lock for %s not acquired within 5s — refusing to "
|
||||
"quarantine without the cross-process lock. The zeroed file "
|
||||
"is left in place. If sessions fail to load, restore from "
|
||||
"state-snapshots via `hermes snapshot list` / "
|
||||
"`hermes snapshot restore <id>`.",
|
||||
path,
|
||||
)
|
||||
logger.error("quarantine lock for %s not acquired within 5s — refusing to "
|
||||
"quarantine without the cross-process lock. The zeroed file "
|
||||
"is left in place. If sessions fail to load, restore from "
|
||||
"state-snapshots via `hermes snapshot list` / `hermes snapshot restore <id>`.",
|
||||
path)
|
||||
return None
|
||||
return _do_quarantine()
|
||||
|
||||
|
||||
def collect_state_db_stats(db_path: Path) -> Dict[str, Any]:
|
||||
"""Best-effort, strictly read-only stats snapshot of a state.db file.
|
||||
|
||||
Opens the database with ``mode=ro`` (URI) and a short timeout so it can
|
||||
run against a *live* database held by a gateway without ever taking a
|
||||
write lock or mutating the file. Every field is collected independently:
|
||||
a failed pragma/SELECT yields ``None`` for that field, and the helper
|
||||
itself never raises. Deliberately does NOT instantiate :class:`SessionDB`
|
||||
— its constructor runs schema DDL, which a diagnostics probe must never do.
|
||||
|
||||
Returned keys (all present, any may be None on failure): ``page_count``,
|
||||
``page_size``, ``freelist_count``, ``logical_size_bytes`` (page_count *
|
||||
page_size), ``wal_size_bytes`` (stat of ``<db>-wal``, 0 when absent),
|
||||
``journal_mode``, ``messages`` / ``sessions`` row counts, ``fts_tables``
|
||||
({name: present}), ``fts_storage_version`` (None = legacy inline layout),
|
||||
``fts_rebuild_pending`` (deferred backfill unfinished),
|
||||
``fts_rebuild_high_water`` / ``fts_rebuild_progress`` raw ints, and
|
||||
``fts_rebuild_deferral`` (durable blocked-repair diagnostic).
|
||||
"""
|
||||
"""Best-effort, strictly read-only stats snapshot of a state.db file: ``mode=ro`` with a short
|
||||
timeout so it can run against a *live* database without taking a write lock. Every field is
|
||||
collected independently (a failed pragma/SELECT yields ``None`` for it); never raises.
|
||||
Deliberately does NOT instantiate :class:`SessionDB` — its constructor runs DDL.
|
||||
``wal_size_bytes`` is 0 when the sidecar is absent; ``fts_storage_version`` None means the
|
||||
legacy inline layout; ``fts_rebuild_deferral`` is the durable blocked-repair diagnostic."""
|
||||
from hermes_state import _connect_tracked_db
|
||||
stats: Dict[str, Any] = dict.fromkeys((
|
||||
"page_count", "page_size", "freelist_count", "logical_size_bytes", "wal_size_bytes",
|
||||
"journal_mode", "messages", "sessions", "fts_tables", "fts_storage_version",
|
||||
"fts_rebuild_pending", "fts_rebuild_high_water", "fts_rebuild_progress",
|
||||
"fts_rebuild_deferral",
|
||||
))
|
||||
"page_count", "page_size", "freelist_count", "logical_size_bytes", "wal_size_bytes", "journal_mode",
|
||||
"messages", "sessions", "fts_tables", "fts_storage_version", "fts_rebuild_pending",
|
||||
"fts_rebuild_high_water", "fts_rebuild_progress", "fts_rebuild_deferral"))
|
||||
# WAL sidecar size needs no connection at all.
|
||||
try:
|
||||
with contextlib.suppress(OSError):
|
||||
wal_path = Path(str(db_path) + "-wal")
|
||||
stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
# mode=ro refuses to create the file and refuses every write; a short
|
||||
# timeout keeps doctor snappy when a writer holds the lock. The tracked
|
||||
# connect lets byte-probe helpers see this connection and refuse raw
|
||||
# opens that could cancel our POSIX locks mid-read.
|
||||
conn = _connect_tracked_db(
|
||||
f"file:{Path(db_path)}?mode=ro", tracking_path=Path(db_path), uri=True, timeout=2.0
|
||||
)
|
||||
# A short timeout keeps doctor snappy when a writer holds the lock. The tracked connect
|
||||
# lets byte-probe helpers see this connection and refuse raw opens that would cancel locks.
|
||||
conn = _connect_tracked_db(f"file:{Path(db_path)}?mode=ro", tracking_path=Path(db_path),
|
||||
uri=True, timeout=2.0)
|
||||
except Exception as exc:
|
||||
logger.debug("collect_state_db_stats: cannot open %s read-only: %s", db_path, exc)
|
||||
return stats
|
||||
@@ -419,73 +312,54 @@ def collect_state_db_stats(db_path: Path) -> Dict[str, Any]:
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _int(value) -> Optional[int]:
|
||||
def _int(sql: str, params=()) -> Optional[int]:
|
||||
value = _scalar(sql, params)
|
||||
return int(value) if value is not None else None
|
||||
|
||||
def _meta_int(key: str) -> Optional[int]:
|
||||
try:
|
||||
return _int(_scalar("SELECT value FROM state_meta WHERE key = ?", (key,)))
|
||||
try: # a non-numeric meta value must yield None, not fail the snapshot
|
||||
return _int("SELECT value FROM state_meta WHERE key = ?", (key,))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
try:
|
||||
stats["page_count"] = _int(_scalar("PRAGMA page_count"))
|
||||
stats["page_size"] = _int(_scalar("PRAGMA page_size"))
|
||||
stats["page_count"] = _int("PRAGMA page_count")
|
||||
stats["page_size"] = _int("PRAGMA page_size")
|
||||
if stats["page_count"] is not None and stats["page_size"] is not None:
|
||||
stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"]
|
||||
stats["freelist_count"] = _int(_scalar("PRAGMA freelist_count"))
|
||||
stats["freelist_count"] = _int("PRAGMA freelist_count")
|
||||
jm = _scalar("PRAGMA journal_mode")
|
||||
stats["journal_mode"] = str(jm) if jm is not None else None
|
||||
stats["messages"] = _int(_scalar("SELECT COUNT(*) FROM messages"))
|
||||
stats["sessions"] = _int(_scalar("SELECT COUNT(*) FROM sessions"))
|
||||
stats["messages"] = _int("SELECT COUNT(*) FROM messages")
|
||||
stats["sessions"] = _int("SELECT COUNT(*) FROM sessions")
|
||||
# FTS table presence via sqlite_master (never SELECTs from the
|
||||
# virtual tables themselves — a corrupt index must not fail stats).
|
||||
try:
|
||||
names = {
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' "
|
||||
"AND name IN (?, ?, ?)",
|
||||
_FTS_TABLE_NAMES,
|
||||
).fetchall()
|
||||
}
|
||||
with contextlib.suppress(Exception):
|
||||
names = {row[0] for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (?, ?, ?)",
|
||||
_FTS_TABLE_NAMES).fetchall()}
|
||||
stats["fts_tables"] = {t: (t in names) for t in _FTS_TABLE_NAMES}
|
||||
except Exception:
|
||||
pass
|
||||
# Raw state_meta reads — cheap, and independent of SessionDB.
|
||||
stats["fts_storage_version"] = _meta_int("fts_storage_version")
|
||||
high_water = _meta_int("fts_rebuild_high_water")
|
||||
progress = _meta_int("fts_rebuild_progress")
|
||||
stats["fts_rebuild_high_water"] = high_water
|
||||
stats["fts_rebuild_progress"] = progress
|
||||
stats["fts_rebuild_high_water"] = high_water = _meta_int("fts_rebuild_high_water")
|
||||
stats["fts_rebuild_progress"] = progress = _meta_int("fts_rebuild_progress")
|
||||
stats["fts_rebuild_pending"] = False if high_water is None else (progress or 0) < high_water
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT value FROM state_meta WHERE key = ? LIMIT 1", (FTS_REBUILD_DEFERRAL_KEY,)
|
||||
).fetchone()
|
||||
if row:
|
||||
parsed = json.loads(row[0])
|
||||
if isinstance(parsed, dict):
|
||||
stats["fts_rebuild_deferral"] = parsed
|
||||
except Exception:
|
||||
pass
|
||||
with contextlib.suppress(Exception):
|
||||
row = conn.execute("SELECT value FROM state_meta WHERE key = ? LIMIT 1",
|
||||
(FTS_REBUILD_DEFERRAL_KEY,)).fetchone()
|
||||
parsed = json.loads(row[0]) if row else None
|
||||
if isinstance(parsed, dict):
|
||||
stats["fts_rebuild_deferral"] = parsed
|
||||
finally:
|
||||
try:
|
||||
with contextlib.suppress(Exception):
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
return stats
|
||||
|
||||
|
||||
def count_db_holders(db_path: Path) -> Optional[int]:
|
||||
"""Best-effort count of processes holding ``db_path`` open (Linux only).
|
||||
|
||||
Scans ``/proc/*/fd`` symlinks for the resolved database path. Returns
|
||||
the number of distinct PIDs with the file open, or ``None`` on any
|
||||
error or on non-Linux platforms. Never raises; no lsof dependency.
|
||||
Unreadable per-process fd dirs (other users' processes without root)
|
||||
are silently skipped, so the count is a lower bound.
|
||||
"""
|
||||
"""Best-effort count of distinct PIDs holding ``db_path`` open (``/proc/*/fd`` scan); ``None``
|
||||
on any error or non-Linux host, never raises. Unreadable fd dirs (other users' processes
|
||||
without root) are skipped, so this is a lower bound."""
|
||||
try:
|
||||
if not sys.platform.startswith("linux"):
|
||||
return None
|
||||
@@ -495,40 +369,27 @@ def count_db_holders(db_path: Path) -> Optional[int]:
|
||||
return None
|
||||
|
||||
|
||||
def _is_inactive_orphan_desktop_holder(
|
||||
*, ppid: int, age_seconds: float, min_age_seconds: float, ephemeral_backend: bool,
|
||||
connection_statuses: List[str],
|
||||
) -> bool:
|
||||
def _is_inactive_orphan_desktop_holder(*, ppid: int, age_seconds: float, min_age_seconds: float,
|
||||
ephemeral_backend: bool, connection_statuses: List[str]) -> bool:
|
||||
"""Pure safety predicate for the narrow Desktop holder reap."""
|
||||
return (
|
||||
ppid in (0, 1)
|
||||
and age_seconds >= min_age_seconds
|
||||
and ephemeral_backend
|
||||
and "ESTABLISHED" not in connection_statuses
|
||||
)
|
||||
return (ppid in (0, 1) and age_seconds >= min_age_seconds and ephemeral_backend
|
||||
and "ESTABLISHED" not in connection_statuses)
|
||||
|
||||
|
||||
def _concrete_state_db_holder_pids(db_path: Path, holders: List[Tuple[int, str]]) -> List[int]:
|
||||
"""Return unique PIDs proven to hold this DB or one of its sidecars."""
|
||||
canonical_db = os.path.normcase(os.path.abspath(os.fspath(db_path)))
|
||||
watched = {canonical_db, canonical_db + "-wal", canonical_db + "-shm"}
|
||||
pids: List[int] = []
|
||||
for pid, path in holders:
|
||||
if pid <= 0 or pid in pids or _canonical_sqlite_path(path) not in watched:
|
||||
continue
|
||||
pids.append(pid)
|
||||
return pids
|
||||
return list(dict.fromkeys(
|
||||
pid for pid, path in holders if pid > 0 and _canonical_sqlite_path(path) in watched))
|
||||
|
||||
|
||||
def _read_proc_cmdline(pid: int) -> Optional[str]:
|
||||
"""Read /proc/<pid>/cmdline (world-readable even when the fd table is not)
|
||||
as a space-joined string; None when unreadable (exited, hidepid mount)."""
|
||||
"""Space-joined /proc/<pid>/cmdline (readable even when the fd table is not); None if unreadable."""
|
||||
try:
|
||||
with open(f"/proc/{pid}/cmdline", "rb") as f:
|
||||
raw = f.read()
|
||||
if not raw:
|
||||
return None
|
||||
return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip()
|
||||
return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip() if raw else None
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
@@ -538,8 +399,6 @@ _HERMES_CMDLINE_MARKERS = ("hermes_cli.main", "hermes_cli/main", "hermes serve",
|
||||
|
||||
|
||||
def _looks_like_hermes(cmdline: str) -> bool:
|
||||
"""Heuristic: does this cmdline look like a Hermes process? Decides whether
|
||||
an uninspectable process (fd table unreadable, different user) is treated
|
||||
as a potential state.db holder; system daemons are not flagged."""
|
||||
lower = cmdline.lower()
|
||||
return any(marker in lower for marker in _HERMES_CMDLINE_MARKERS)
|
||||
"""Heuristic: is this a Hermes process? Decides whether an uninspectable process (fd
|
||||
table unreadable, other user) counts as a potential state.db holder; daemons are not."""
|
||||
return any(marker in cmdline.lower() for marker in _HERMES_CMDLINE_MARKERS)
|
||||
|
||||
Reference in New Issue
Block a user