refactor(hermes_cli): sqlite_safe_read — compact docstrings, fold duplicate try in page_count_bytes

This commit is contained in:
Teknium
2026-09-02 20:36:56 -07:00
parent c227ea7e1a
commit d844ae7610
+48 -97
View File
@@ -1,11 +1,10 @@
"""Lock-safe inspection of SQLite database files.
Why this module exists ---------------------- POSIX advisory locks are cancelled **process-wide** by
``close()`` on *any* file descriptor for that file::
So a bare ``open(db_path, "rb") ... close()`` on a **live** database silently drops every lock
SQLite holds on it from this process -- including the EXCLUSIVE lock a ``VACUUM`` is holding while
it rewrites the whole file, and the RESERVED lock an in-flight ``BEGIN IMMEDIATE`` is holding.
POSIX advisory locks are cancelled **process-wide** by ``close()`` on *any* file descriptor for
that file, so a bare ``open(db_path, "rb") ... close()`` on a **live** database silently drops
every lock SQLite holds on it from this process -- including the EXCLUSIVE lock a ``VACUUM`` is
holding while it rewrites the whole file, and the RESERVED lock of an in-flight ``BEGIN IMMEDIATE``.
This module tracks live connections so raw reads only happen when none exist.
"""
from __future__ import annotations
@@ -35,6 +34,10 @@ class UntrackableConnectionError(RuntimeError):
"""
class LiveConnectionError(RuntimeError):
"""A raw file operation was attempted on a database with live connections."""
def _key(path: Path | str) -> str:
"""Canonicalise a *filesystem* path for use as a registry key."""
try:
@@ -44,11 +47,8 @@ def _key(path: Path | str) -> str:
def _canonical_db_path(conn: sqlite3.Connection) -> Optional[str]:
"""The on-disk path of ``main``, as SQLite itself reports it.
Immune to the caller's spelling (``file:`` URIs, relative paths, symlinks). Returns ``None`` for
in-memory or unnamed databases, which cannot be byte-probed and therefore need no tracking.
"""
"""The on-disk path of ``main`` as SQLite reports it (immune to ``file:`` URIs, relative paths,
symlinks). ``None`` for in-memory/unnamed databases, which cannot be byte-probed."""
try:
row = conn.execute("PRAGMA database_list").fetchone()
except sqlite3.Error:
@@ -91,17 +91,19 @@ def has_live_connection(path: Path | str) -> bool:
class _TrackingMixin:
"""Untrack-on-close behaviour, mixable into any Connection subclass."""
"""Untrack-on-close behaviour, mixable into any Connection subclass.
The real ``close()`` and the unregister happen together under ``_live_lock`` so a concurrent
probe can never observe "no live connection" while this descriptor is still open. Unregister
runs only after ``close()`` succeeds: untracking before a failing close (e.g. cross-thread
ProgrammingError) would leave the FD open while the byte-probe guard thinks nothing is live.
"""
_hermes_tracked_path: str | None = None
def close(self) -> None: # type: ignore[misc]
with _live_lock:
path = getattr(self, "_hermes_tracked_path", None)
# Close first; untrack only once the descriptor is actually gone.
# Untracking before a failing close (e.g. cross-thread
# ProgrammingError) leaves the FD open while the byte-probe
# guard thinks nothing is live — see #75629.
super().close() # type: ignore[misc]
if path is not None:
self._hermes_tracked_path = None
@@ -109,16 +111,8 @@ class _TrackingMixin:
class TrackedConnection(_TrackingMixin, sqlite3.Connection):
"""A ``sqlite3.Connection`` that untracks its path exactly once on close.
Counting opens is easy; counting closes reliably is not, because callers close connections in
many places (and some hand them to ``contextlib.closing``).
The real ``close()`` and the unregister happen together under ``_live_lock`` so a concurrent
probe can never observe "no live connection" while this descriptor is still open. Unregister
runs only after ``close()`` succeeds; a raising close leaves the connection tracked so the byte-
probe guard keeps refusing.
"""
"""A ``sqlite3.Connection`` that untracks its path exactly once on close (callers close in many
places, some via ``contextlib.closing``, so counting closes by hand is unreliable)."""
_tracked_factory_cache: dict[type, type] = {}
@@ -143,11 +137,7 @@ def _tracking_factory(factory: type) -> type:
def connect_tracked(
path: Path | str,
*,
tracking_path: Path | str | None = None,
connect_fn=None,
**kwargs,
path: Path | str, *, tracking_path: Path | str | None = None, connect_fn=None, **kwargs,
) -> sqlite3.Connection:
"""``sqlite3.connect`` that registers the connection for the lifetime of the fd.
@@ -163,28 +153,21 @@ def connect_tracked(
with _live_lock:
conn = opener(str(path), **kwargs)
try:
resolved = (
_key(tracking_path)
if tracking_path is not None
else _canonical_db_path(conn)
)
resolved = _key(tracking_path) if tracking_path is not None else _canonical_db_path(conn)
if resolved is None:
# In-memory / unnamed: nothing on disk to byte-probe.
return conn
return conn # in-memory / unnamed: nothing on disk to byte-probe
if not isinstance(conn, _TrackingMixin):
# The opener substituted its own factory and discarded ours
# (test doubles simulating FTS5-less runtimes do this). Retag
# the instance's class with the tracking mixin so close() still
# releases the registry entry, rather than handing back a
# connection whose database has silently lost probe safety.
# The opener substituted its own factory and discarded ours (test doubles
# simulating FTS5-less runtimes do this). Retag the instance's class so close()
# still releases the registry entry rather than silently losing probe safety.
conn = _retrofit_tracking(conn, resolved)
conn._hermes_tracked_path = resolved
_track_key(resolved)
return conn
except Exception:
try:
# Close via sqlite3 directly: the tracking entry was either
# never made or is being unwound here.
# Close via sqlite3 directly: the tracking entry was either never made or is
# being unwound here.
sqlite3.Connection.close(conn)
except Exception:
pass
@@ -192,11 +175,8 @@ def connect_tracked(
def _retrofit_tracking(conn: sqlite3.Connection, resolved: str) -> sqlite3.Connection:
"""Give an already-open connection untrack-on-close semantics.
``sqlite3.Connection`` subclasses are ordinary classes, so ``__class__`` can be swapped for
one mixing in the tracking ``close()``. Used when an opener ignored the factory we asked for.
"""
"""Give an already-open connection untrack-on-close semantics by swapping ``__class__`` for
one mixing in the tracking ``close()`` (used when an opener ignored the factory we asked for)."""
cls = type(conn)
try:
conn.__class__ = _tracking_factory(cls) # type: ignore[assignment]
@@ -210,66 +190,44 @@ def _retrofit_tracking(conn: sqlite3.Connection, resolved: str) -> sqlite3.Conne
def page_count_bytes(conn: sqlite3.Connection) -> Optional[int]:
"""Logical database size in bytes, read through *conn*.
``page_count * page_size`` is the same quantity the 4-byte header field at offset 28 carries,
but reading it via ``PRAGMA`` opens no new file descriptor and therefore cannot cancel this
process's POSIX locks.
Returns ``None`` when the pragmas cannot be read.
"""
"""Logical database size in bytes (``page_count * page_size``, the header field at offset 28)
read via PRAGMA over *conn* so no new fd is opened. ``None`` when the pragmas cannot be read."""
try:
page_count = conn.execute("PRAGMA page_count").fetchone()[0]
page_size = conn.execute("PRAGMA page_size").fetchone()[0]
except (sqlite3.Error, TypeError, IndexError) as exc:
logger.debug("page_count/page_size unavailable: %s", exc)
return None
try:
return int(page_count) * int(page_size)
except (TypeError, ValueError):
except (sqlite3.Error, TypeError, IndexError, ValueError) as exc:
logger.debug("page_count/page_size unavailable: %s", exc)
return None
def file_length_matches_header(conn: sqlite3.Connection) -> Optional[bool]:
"""Whether the file on disk is at least as long as the header claims.
"""Whether the file on disk is at least as long as the header claims ("torn extend" check).
Detects the "torn extend" shape (file shorter than its own page count) without ever opening the
database file: the header side comes from ``PRAGMA page_count`` over *conn*, and the on-disk
side from ``stat()``, which takes no descriptor and cannot break locks.
Note: in WAL mode a freshly committed page may still live in the ``-wal`` file, so the main file
legitimately lags. Callers must treat this as advisory unless the database is in a rollback
journal mode.
Never opens the database file: the header side comes from ``PRAGMA page_count`` over *conn*,
the on-disk side from ``stat()``. In WAL mode a freshly committed page may still live in the
``-wal`` file so the main file legitimately lags; callers must treat this as advisory unless
the database is in a rollback journal mode.
"""
path_str = _canonical_db_path(conn)
if path_str is None:
return None
logical = page_count_bytes(conn)
if not logical:
return None
try:
actual = os.path.getsize(path_str)
return os.path.getsize(path_str) >= logical
except OSError:
return None
return actual >= logical
def read_header_bytes_preopen(
path: Path | str,
*,
length: int = 100,
force: bool = False,
) -> Optional[bytes]:
def read_header_bytes_preopen(path: Path | str, *, length: int = 100, force: bool = False) -> Optional[bytes]:
"""Read the first *length* bytes of *path* -- only when no connection is live.
This is the ONLY sanctioned byte-level read of a database file, and it is restricted to first-
open validation (is this file a real SQLite database, is it zeroed, has it been overwritten by
something else).
The registry check and the ``open``/``read``/``close`` are performed together under
``_live_lock``, so a connection cannot be opened in the window between deciding "nothing is
live" and closing this descriptor.
This is the ONLY sanctioned byte-level read of a database file, restricted to first-open
validation (real SQLite database? zeroed? overwritten?). The registry check and the
``open``/``read``/``close`` run together under ``_live_lock`` so a connection cannot be opened
between deciding "nothing is live" and closing this descriptor.
"""
with _live_lock:
if not force and _key(path) in _live_connections:
@@ -286,20 +244,13 @@ def read_header_bytes_preopen(
return None
class LiveConnectionError(RuntimeError):
"""A raw file operation was attempted on a database with live connections."""
@contextlib.contextmanager
def offline_file_access(path: Path | str, *, what: str = "read"):
"""Hold the connection-lifecycle lock across a raw read of a database file.
Checking :func:`has_live_connection` and *then* doing the raw I/O is a check/use race: a
connection can be opened in the window between the two, and the raw ``close()`` will cancel its
POSIX advisory locks — the exact failure class the registry exists to prevent.
The lock is only held for the duration of the raw I/O; it never spans caller work on an open
connection, so it does not serialise database use.
Checking :func:`has_live_connection` and *then* doing raw I/O is a check/use race: a connection
opened in between would have its POSIX locks cancelled by the raw ``close()``. The lock is
held only for the raw I/O, never across caller work on an open connection.
"""
with _live_lock:
if _key(path) in _live_connections: