refactor(hermes_cli): sqlite_safe_read — compact docstrings, fold duplicate try in page_count_bytes
This commit is contained in:
@@ -1,11 +1,10 @@
|
||||
"""Lock-safe inspection of SQLite database files.
|
||||
|
||||
Why this module exists ---------------------- POSIX advisory locks are cancelled **process-wide** by
|
||||
``close()`` on *any* file descriptor for that file::
|
||||
|
||||
So a bare ``open(db_path, "rb") ... close()`` on a **live** database silently drops every lock
|
||||
SQLite holds on it from this process -- including the EXCLUSIVE lock a ``VACUUM`` is holding while
|
||||
it rewrites the whole file, and the RESERVED lock an in-flight ``BEGIN IMMEDIATE`` is holding.
|
||||
POSIX advisory locks are cancelled **process-wide** by ``close()`` on *any* file descriptor for
|
||||
that file, so a bare ``open(db_path, "rb") ... close()`` on a **live** database silently drops
|
||||
every lock SQLite holds on it from this process -- including the EXCLUSIVE lock a ``VACUUM`` is
|
||||
holding while it rewrites the whole file, and the RESERVED lock of an in-flight ``BEGIN IMMEDIATE``.
|
||||
This module tracks live connections so raw reads only happen when none exist.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -35,6 +34,10 @@ class UntrackableConnectionError(RuntimeError):
|
||||
"""
|
||||
|
||||
|
||||
class LiveConnectionError(RuntimeError):
|
||||
"""A raw file operation was attempted on a database with live connections."""
|
||||
|
||||
|
||||
def _key(path: Path | str) -> str:
|
||||
"""Canonicalise a *filesystem* path for use as a registry key."""
|
||||
try:
|
||||
@@ -44,11 +47,8 @@ def _key(path: Path | str) -> str:
|
||||
|
||||
|
||||
def _canonical_db_path(conn: sqlite3.Connection) -> Optional[str]:
|
||||
"""The on-disk path of ``main``, as SQLite itself reports it.
|
||||
|
||||
Immune to the caller's spelling (``file:`` URIs, relative paths, symlinks). Returns ``None`` for
|
||||
in-memory or unnamed databases, which cannot be byte-probed and therefore need no tracking.
|
||||
"""
|
||||
"""The on-disk path of ``main`` as SQLite reports it (immune to ``file:`` URIs, relative paths,
|
||||
symlinks). ``None`` for in-memory/unnamed databases, which cannot be byte-probed."""
|
||||
try:
|
||||
row = conn.execute("PRAGMA database_list").fetchone()
|
||||
except sqlite3.Error:
|
||||
@@ -91,17 +91,19 @@ def has_live_connection(path: Path | str) -> bool:
|
||||
|
||||
|
||||
class _TrackingMixin:
|
||||
"""Untrack-on-close behaviour, mixable into any Connection subclass."""
|
||||
"""Untrack-on-close behaviour, mixable into any Connection subclass.
|
||||
|
||||
The real ``close()`` and the unregister happen together under ``_live_lock`` so a concurrent
|
||||
probe can never observe "no live connection" while this descriptor is still open. Unregister
|
||||
runs only after ``close()`` succeeds: untracking before a failing close (e.g. cross-thread
|
||||
ProgrammingError) would leave the FD open while the byte-probe guard thinks nothing is live.
|
||||
"""
|
||||
|
||||
_hermes_tracked_path: str | None = None
|
||||
|
||||
def close(self) -> None: # type: ignore[misc]
|
||||
with _live_lock:
|
||||
path = getattr(self, "_hermes_tracked_path", None)
|
||||
# Close first; untrack only once the descriptor is actually gone.
|
||||
# Untracking before a failing close (e.g. cross-thread
|
||||
# ProgrammingError) leaves the FD open while the byte-probe
|
||||
# guard thinks nothing is live — see #75629.
|
||||
super().close() # type: ignore[misc]
|
||||
if path is not None:
|
||||
self._hermes_tracked_path = None
|
||||
@@ -109,16 +111,8 @@ class _TrackingMixin:
|
||||
|
||||
|
||||
class TrackedConnection(_TrackingMixin, sqlite3.Connection):
|
||||
"""A ``sqlite3.Connection`` that untracks its path exactly once on close.
|
||||
|
||||
Counting opens is easy; counting closes reliably is not, because callers close connections in
|
||||
many places (and some hand them to ``contextlib.closing``).
|
||||
|
||||
The real ``close()`` and the unregister happen together under ``_live_lock`` so a concurrent
|
||||
probe can never observe "no live connection" while this descriptor is still open. Unregister
|
||||
runs only after ``close()`` succeeds; a raising close leaves the connection tracked so the byte-
|
||||
probe guard keeps refusing.
|
||||
"""
|
||||
"""A ``sqlite3.Connection`` that untracks its path exactly once on close (callers close in many
|
||||
places, some via ``contextlib.closing``, so counting closes by hand is unreliable)."""
|
||||
|
||||
|
||||
_tracked_factory_cache: dict[type, type] = {}
|
||||
@@ -143,11 +137,7 @@ def _tracking_factory(factory: type) -> type:
|
||||
|
||||
|
||||
def connect_tracked(
|
||||
path: Path | str,
|
||||
*,
|
||||
tracking_path: Path | str | None = None,
|
||||
connect_fn=None,
|
||||
**kwargs,
|
||||
path: Path | str, *, tracking_path: Path | str | None = None, connect_fn=None, **kwargs,
|
||||
) -> sqlite3.Connection:
|
||||
"""``sqlite3.connect`` that registers the connection for the lifetime of the fd.
|
||||
|
||||
@@ -163,28 +153,21 @@ def connect_tracked(
|
||||
with _live_lock:
|
||||
conn = opener(str(path), **kwargs)
|
||||
try:
|
||||
resolved = (
|
||||
_key(tracking_path)
|
||||
if tracking_path is not None
|
||||
else _canonical_db_path(conn)
|
||||
)
|
||||
resolved = _key(tracking_path) if tracking_path is not None else _canonical_db_path(conn)
|
||||
if resolved is None:
|
||||
# In-memory / unnamed: nothing on disk to byte-probe.
|
||||
return conn
|
||||
return conn # in-memory / unnamed: nothing on disk to byte-probe
|
||||
if not isinstance(conn, _TrackingMixin):
|
||||
# The opener substituted its own factory and discarded ours
|
||||
# (test doubles simulating FTS5-less runtimes do this). Retag
|
||||
# the instance's class with the tracking mixin so close() still
|
||||
# releases the registry entry, rather than handing back a
|
||||
# connection whose database has silently lost probe safety.
|
||||
# The opener substituted its own factory and discarded ours (test doubles
|
||||
# simulating FTS5-less runtimes do this). Retag the instance's class so close()
|
||||
# still releases the registry entry rather than silently losing probe safety.
|
||||
conn = _retrofit_tracking(conn, resolved)
|
||||
conn._hermes_tracked_path = resolved
|
||||
_track_key(resolved)
|
||||
return conn
|
||||
except Exception:
|
||||
try:
|
||||
# Close via sqlite3 directly: the tracking entry was either
|
||||
# never made or is being unwound here.
|
||||
# Close via sqlite3 directly: the tracking entry was either never made or is
|
||||
# being unwound here.
|
||||
sqlite3.Connection.close(conn)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -192,11 +175,8 @@ def connect_tracked(
|
||||
|
||||
|
||||
def _retrofit_tracking(conn: sqlite3.Connection, resolved: str) -> sqlite3.Connection:
|
||||
"""Give an already-open connection untrack-on-close semantics.
|
||||
|
||||
``sqlite3.Connection`` subclasses are ordinary classes, so ``__class__`` can be swapped for
|
||||
one mixing in the tracking ``close()``. Used when an opener ignored the factory we asked for.
|
||||
"""
|
||||
"""Give an already-open connection untrack-on-close semantics by swapping ``__class__`` for
|
||||
one mixing in the tracking ``close()`` (used when an opener ignored the factory we asked for)."""
|
||||
cls = type(conn)
|
||||
try:
|
||||
conn.__class__ = _tracking_factory(cls) # type: ignore[assignment]
|
||||
@@ -210,66 +190,44 @@ def _retrofit_tracking(conn: sqlite3.Connection, resolved: str) -> sqlite3.Conne
|
||||
|
||||
|
||||
def page_count_bytes(conn: sqlite3.Connection) -> Optional[int]:
|
||||
"""Logical database size in bytes, read through *conn*.
|
||||
|
||||
``page_count * page_size`` is the same quantity the 4-byte header field at offset 28 carries,
|
||||
but reading it via ``PRAGMA`` opens no new file descriptor and therefore cannot cancel this
|
||||
process's POSIX locks.
|
||||
|
||||
Returns ``None`` when the pragmas cannot be read.
|
||||
"""
|
||||
"""Logical database size in bytes (``page_count * page_size``, the header field at offset 28)
|
||||
read via PRAGMA over *conn* so no new fd is opened. ``None`` when the pragmas cannot be read."""
|
||||
try:
|
||||
page_count = conn.execute("PRAGMA page_count").fetchone()[0]
|
||||
page_size = conn.execute("PRAGMA page_size").fetchone()[0]
|
||||
except (sqlite3.Error, TypeError, IndexError) as exc:
|
||||
logger.debug("page_count/page_size unavailable: %s", exc)
|
||||
return None
|
||||
try:
|
||||
return int(page_count) * int(page_size)
|
||||
except (TypeError, ValueError):
|
||||
except (sqlite3.Error, TypeError, IndexError, ValueError) as exc:
|
||||
logger.debug("page_count/page_size unavailable: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def file_length_matches_header(conn: sqlite3.Connection) -> Optional[bool]:
|
||||
"""Whether the file on disk is at least as long as the header claims.
|
||||
"""Whether the file on disk is at least as long as the header claims ("torn extend" check).
|
||||
|
||||
Detects the "torn extend" shape (file shorter than its own page count) without ever opening the
|
||||
database file: the header side comes from ``PRAGMA page_count`` over *conn*, and the on-disk
|
||||
side from ``stat()``, which takes no descriptor and cannot break locks.
|
||||
|
||||
Note: in WAL mode a freshly committed page may still live in the ``-wal`` file, so the main file
|
||||
legitimately lags. Callers must treat this as advisory unless the database is in a rollback
|
||||
journal mode.
|
||||
Never opens the database file: the header side comes from ``PRAGMA page_count`` over *conn*,
|
||||
the on-disk side from ``stat()``. In WAL mode a freshly committed page may still live in the
|
||||
``-wal`` file so the main file legitimately lags; callers must treat this as advisory unless
|
||||
the database is in a rollback journal mode.
|
||||
"""
|
||||
path_str = _canonical_db_path(conn)
|
||||
if path_str is None:
|
||||
return None
|
||||
|
||||
logical = page_count_bytes(conn)
|
||||
if not logical:
|
||||
return None
|
||||
try:
|
||||
actual = os.path.getsize(path_str)
|
||||
return os.path.getsize(path_str) >= logical
|
||||
except OSError:
|
||||
return None
|
||||
return actual >= logical
|
||||
|
||||
|
||||
def read_header_bytes_preopen(
|
||||
path: Path | str,
|
||||
*,
|
||||
length: int = 100,
|
||||
force: bool = False,
|
||||
) -> Optional[bytes]:
|
||||
def read_header_bytes_preopen(path: Path | str, *, length: int = 100, force: bool = False) -> Optional[bytes]:
|
||||
"""Read the first *length* bytes of *path* -- only when no connection is live.
|
||||
|
||||
This is the ONLY sanctioned byte-level read of a database file, and it is restricted to first-
|
||||
open validation (is this file a real SQLite database, is it zeroed, has it been overwritten by
|
||||
something else).
|
||||
|
||||
The registry check and the ``open``/``read``/``close`` are performed together under
|
||||
``_live_lock``, so a connection cannot be opened in the window between deciding "nothing is
|
||||
live" and closing this descriptor.
|
||||
This is the ONLY sanctioned byte-level read of a database file, restricted to first-open
|
||||
validation (real SQLite database? zeroed? overwritten?). The registry check and the
|
||||
``open``/``read``/``close`` run together under ``_live_lock`` so a connection cannot be opened
|
||||
between deciding "nothing is live" and closing this descriptor.
|
||||
"""
|
||||
with _live_lock:
|
||||
if not force and _key(path) in _live_connections:
|
||||
@@ -286,20 +244,13 @@ def read_header_bytes_preopen(
|
||||
return None
|
||||
|
||||
|
||||
class LiveConnectionError(RuntimeError):
|
||||
"""A raw file operation was attempted on a database with live connections."""
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def offline_file_access(path: Path | str, *, what: str = "read"):
|
||||
"""Hold the connection-lifecycle lock across a raw read of a database file.
|
||||
|
||||
Checking :func:`has_live_connection` and *then* doing the raw I/O is a check/use race: a
|
||||
connection can be opened in the window between the two, and the raw ``close()`` will cancel its
|
||||
POSIX advisory locks — the exact failure class the registry exists to prevent.
|
||||
|
||||
The lock is only held for the duration of the raw I/O; it never spans caller work on an open
|
||||
connection, so it does not serialise database use.
|
||||
Checking :func:`has_live_connection` and *then* doing raw I/O is a check/use race: a connection
|
||||
opened in between would have its POSIX locks cancelled by the raw ``close()``. The lock is
|
||||
held only for the raw I/O, never across caller work on an open connection.
|
||||
"""
|
||||
with _live_lock:
|
||||
if _key(path) in _live_connections:
|
||||
|
||||
Reference in New Issue
Block a user