From d844ae7610635a511ef82ddbc11ac32fb1c4051b Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:36:56 -0700 Subject: [PATCH] =?UTF-8?q?refactor(hermes=5Fcli):=20sqlite=5Fsafe=5Fread?= =?UTF-8?q?=20=E2=80=94=20compact=20docstrings,=20fold=20duplicate=20try?= =?UTF-8?q?=20in=20page=5Fcount=5Fbytes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hermes_cli/sqlite_safe_read.py | 145 +++++++++++---------------------- 1 file changed, 48 insertions(+), 97 deletions(-) diff --git a/hermes_cli/sqlite_safe_read.py b/hermes_cli/sqlite_safe_read.py index e759f0437c..5171fb7008 100644 --- a/hermes_cli/sqlite_safe_read.py +++ b/hermes_cli/sqlite_safe_read.py @@ -1,11 +1,10 @@ """Lock-safe inspection of SQLite database files. -Why this module exists ---------------------- POSIX advisory locks are cancelled **process-wide** by -``close()`` on *any* file descriptor for that file:: - -So a bare ``open(db_path, "rb") ... close()`` on a **live** database silently drops every lock -SQLite holds on it from this process -- including the EXCLUSIVE lock a ``VACUUM`` is holding while -it rewrites the whole file, and the RESERVED lock an in-flight ``BEGIN IMMEDIATE`` is holding. +POSIX advisory locks are cancelled **process-wide** by ``close()`` on *any* file descriptor for +that file, so a bare ``open(db_path, "rb") ... close()`` on a **live** database silently drops +every lock SQLite holds on it from this process -- including the EXCLUSIVE lock a ``VACUUM`` is +holding while it rewrites the whole file, and the RESERVED lock of an in-flight ``BEGIN IMMEDIATE``. +This module tracks live connections so raw reads only happen when none exist. """ from __future__ import annotations @@ -35,6 +34,10 @@ class UntrackableConnectionError(RuntimeError): """ +class LiveConnectionError(RuntimeError): + """A raw file operation was attempted on a database with live connections.""" + + def _key(path: Path | str) -> str: """Canonicalise a *filesystem* path for use as a registry key.""" try: @@ -44,11 +47,8 @@ def _key(path: Path | str) -> str: def _canonical_db_path(conn: sqlite3.Connection) -> Optional[str]: - """The on-disk path of ``main``, as SQLite itself reports it. - - Immune to the caller's spelling (``file:`` URIs, relative paths, symlinks). Returns ``None`` for - in-memory or unnamed databases, which cannot be byte-probed and therefore need no tracking. - """ + """The on-disk path of ``main`` as SQLite reports it (immune to ``file:`` URIs, relative paths, + symlinks). ``None`` for in-memory/unnamed databases, which cannot be byte-probed.""" try: row = conn.execute("PRAGMA database_list").fetchone() except sqlite3.Error: @@ -91,17 +91,19 @@ def has_live_connection(path: Path | str) -> bool: class _TrackingMixin: - """Untrack-on-close behaviour, mixable into any Connection subclass.""" + """Untrack-on-close behaviour, mixable into any Connection subclass. + + The real ``close()`` and the unregister happen together under ``_live_lock`` so a concurrent + probe can never observe "no live connection" while this descriptor is still open. Unregister + runs only after ``close()`` succeeds: untracking before a failing close (e.g. cross-thread + ProgrammingError) would leave the FD open while the byte-probe guard thinks nothing is live. + """ _hermes_tracked_path: str | None = None def close(self) -> None: # type: ignore[misc] with _live_lock: path = getattr(self, "_hermes_tracked_path", None) - # Close first; untrack only once the descriptor is actually gone. - # Untracking before a failing close (e.g. cross-thread - # ProgrammingError) leaves the FD open while the byte-probe - # guard thinks nothing is live — see #75629. super().close() # type: ignore[misc] if path is not None: self._hermes_tracked_path = None @@ -109,16 +111,8 @@ class _TrackingMixin: class TrackedConnection(_TrackingMixin, sqlite3.Connection): - """A ``sqlite3.Connection`` that untracks its path exactly once on close. - - Counting opens is easy; counting closes reliably is not, because callers close connections in - many places (and some hand them to ``contextlib.closing``). - - The real ``close()`` and the unregister happen together under ``_live_lock`` so a concurrent - probe can never observe "no live connection" while this descriptor is still open. Unregister - runs only after ``close()`` succeeds; a raising close leaves the connection tracked so the byte- - probe guard keeps refusing. - """ + """A ``sqlite3.Connection`` that untracks its path exactly once on close (callers close in many + places, some via ``contextlib.closing``, so counting closes by hand is unreliable).""" _tracked_factory_cache: dict[type, type] = {} @@ -143,11 +137,7 @@ def _tracking_factory(factory: type) -> type: def connect_tracked( - path: Path | str, - *, - tracking_path: Path | str | None = None, - connect_fn=None, - **kwargs, + path: Path | str, *, tracking_path: Path | str | None = None, connect_fn=None, **kwargs, ) -> sqlite3.Connection: """``sqlite3.connect`` that registers the connection for the lifetime of the fd. @@ -163,28 +153,21 @@ def connect_tracked( with _live_lock: conn = opener(str(path), **kwargs) try: - resolved = ( - _key(tracking_path) - if tracking_path is not None - else _canonical_db_path(conn) - ) + resolved = _key(tracking_path) if tracking_path is not None else _canonical_db_path(conn) if resolved is None: - # In-memory / unnamed: nothing on disk to byte-probe. - return conn + return conn # in-memory / unnamed: nothing on disk to byte-probe if not isinstance(conn, _TrackingMixin): - # The opener substituted its own factory and discarded ours - # (test doubles simulating FTS5-less runtimes do this). Retag - # the instance's class with the tracking mixin so close() still - # releases the registry entry, rather than handing back a - # connection whose database has silently lost probe safety. + # The opener substituted its own factory and discarded ours (test doubles + # simulating FTS5-less runtimes do this). Retag the instance's class so close() + # still releases the registry entry rather than silently losing probe safety. conn = _retrofit_tracking(conn, resolved) conn._hermes_tracked_path = resolved _track_key(resolved) return conn except Exception: try: - # Close via sqlite3 directly: the tracking entry was either - # never made or is being unwound here. + # Close via sqlite3 directly: the tracking entry was either never made or is + # being unwound here. sqlite3.Connection.close(conn) except Exception: pass @@ -192,11 +175,8 @@ def connect_tracked( def _retrofit_tracking(conn: sqlite3.Connection, resolved: str) -> sqlite3.Connection: - """Give an already-open connection untrack-on-close semantics. - - ``sqlite3.Connection`` subclasses are ordinary classes, so ``__class__`` can be swapped for - one mixing in the tracking ``close()``. Used when an opener ignored the factory we asked for. - """ + """Give an already-open connection untrack-on-close semantics by swapping ``__class__`` for + one mixing in the tracking ``close()`` (used when an opener ignored the factory we asked for).""" cls = type(conn) try: conn.__class__ = _tracking_factory(cls) # type: ignore[assignment] @@ -210,66 +190,44 @@ def _retrofit_tracking(conn: sqlite3.Connection, resolved: str) -> sqlite3.Conne def page_count_bytes(conn: sqlite3.Connection) -> Optional[int]: - """Logical database size in bytes, read through *conn*. - - ``page_count * page_size`` is the same quantity the 4-byte header field at offset 28 carries, - but reading it via ``PRAGMA`` opens no new file descriptor and therefore cannot cancel this - process's POSIX locks. - - Returns ``None`` when the pragmas cannot be read. - """ + """Logical database size in bytes (``page_count * page_size``, the header field at offset 28) + read via PRAGMA over *conn* so no new fd is opened. ``None`` when the pragmas cannot be read.""" try: page_count = conn.execute("PRAGMA page_count").fetchone()[0] page_size = conn.execute("PRAGMA page_size").fetchone()[0] - except (sqlite3.Error, TypeError, IndexError) as exc: - logger.debug("page_count/page_size unavailable: %s", exc) - return None - try: return int(page_count) * int(page_size) - except (TypeError, ValueError): + except (sqlite3.Error, TypeError, IndexError, ValueError) as exc: + logger.debug("page_count/page_size unavailable: %s", exc) return None def file_length_matches_header(conn: sqlite3.Connection) -> Optional[bool]: - """Whether the file on disk is at least as long as the header claims. + """Whether the file on disk is at least as long as the header claims ("torn extend" check). - Detects the "torn extend" shape (file shorter than its own page count) without ever opening the - database file: the header side comes from ``PRAGMA page_count`` over *conn*, and the on-disk - side from ``stat()``, which takes no descriptor and cannot break locks. - - Note: in WAL mode a freshly committed page may still live in the ``-wal`` file, so the main file - legitimately lags. Callers must treat this as advisory unless the database is in a rollback - journal mode. + Never opens the database file: the header side comes from ``PRAGMA page_count`` over *conn*, + the on-disk side from ``stat()``. In WAL mode a freshly committed page may still live in the + ``-wal`` file so the main file legitimately lags; callers must treat this as advisory unless + the database is in a rollback journal mode. """ path_str = _canonical_db_path(conn) if path_str is None: return None - logical = page_count_bytes(conn) if not logical: return None try: - actual = os.path.getsize(path_str) + return os.path.getsize(path_str) >= logical except OSError: return None - return actual >= logical -def read_header_bytes_preopen( - path: Path | str, - *, - length: int = 100, - force: bool = False, -) -> Optional[bytes]: +def read_header_bytes_preopen(path: Path | str, *, length: int = 100, force: bool = False) -> Optional[bytes]: """Read the first *length* bytes of *path* -- only when no connection is live. - This is the ONLY sanctioned byte-level read of a database file, and it is restricted to first- - open validation (is this file a real SQLite database, is it zeroed, has it been overwritten by - something else). - - The registry check and the ``open``/``read``/``close`` are performed together under - ``_live_lock``, so a connection cannot be opened in the window between deciding "nothing is - live" and closing this descriptor. + This is the ONLY sanctioned byte-level read of a database file, restricted to first-open + validation (real SQLite database? zeroed? overwritten?). The registry check and the + ``open``/``read``/``close`` run together under ``_live_lock`` so a connection cannot be opened + between deciding "nothing is live" and closing this descriptor. """ with _live_lock: if not force and _key(path) in _live_connections: @@ -286,20 +244,13 @@ def read_header_bytes_preopen( return None -class LiveConnectionError(RuntimeError): - """A raw file operation was attempted on a database with live connections.""" - - @contextlib.contextmanager def offline_file_access(path: Path | str, *, what: str = "read"): """Hold the connection-lifecycle lock across a raw read of a database file. - Checking :func:`has_live_connection` and *then* doing the raw I/O is a check/use race: a - connection can be opened in the window between the two, and the raw ``close()`` will cancel its - POSIX advisory locks — the exact failure class the registry exists to prevent. - - The lock is only held for the duration of the raw I/O; it never spans caller work on an open - connection, so it does not serialise database use. + Checking :func:`has_live_connection` and *then* doing raw I/O is a check/use race: a connection + opened in between would have its POSIX locks cancelled by the raw ``close()``. The lock is + held only for the raw I/O, never across caller work on an open connection. """ with _live_lock: if _key(path) in _live_connections: