fix(sessions): capture a lost WAL generation durably before shutdown settles
After DeletedWalGenerationError the retired frames exist only in an unlinked -wal inode that this process keeps open. #106315 stops them from being checkpointed under wrong page numbers, but the canonical remediation ("stop the gateway, dashboard and cron writers, then reopen") lets the kernel drop the inode with them: committed transactions whose disposition is unknown were destroyed by the prescribed recovery path itself. Capture the exact retired generation next to the database at the first halt, or at close() if that sees the loss first. The WAL is located by the (st_dev, st_ino) recorded at open among this process's own descriptors, never by pathname, so a sibling's deleted WAL or a newer sidecar minted at the same path cannot be mistaken for it; it is read with pread and no descriptor is closed, moved or truncated. The artifact holds the WAL, the -shm when still ours, the main image (or its header past a size cap) and a manifest with identities, digests and the sidecar generation found at the path at capture time. Nothing is merged: whether the frames belong on top of the file now at the path stays an operator decision. close() refuses to settle without the capture: it raises and leaves the handle open. Regressions: capture at halt and at close, refusal to settle on capture failure, inode-not-pathname selection with a second deleted WAL under the same name, refusal to guess without a recorded identity, and a subprocess control where rows committed only in the retired WAL are recovered from the capture alone after the writer process has exited. Refs #105670
This commit is contained in:
Totoro-qaq
+218
-3
@@ -10,6 +10,7 @@ call time, so tests that monkeypatch ``hermes_state.<name>`` keep intercepting.
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -42,13 +43,14 @@ _RETIRED_HEADER_PROBE_FDS: "list[int]" = [] # intentionally never closed
|
||||
_FTS_TABLE_NAMES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
|
||||
|
||||
|
||||
def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]":
|
||||
"""Lock-safe raw header read of a possibly-live SQLite database: POSIX preads from a cached,
|
||||
def _pread_db_range(db_path: Path, offset: int, length: int) -> "Optional[bytes]":
|
||||
"""Lock-safe raw read of a possibly-live SQLite database: POSIX preads from a cached,
|
||||
never-closed fd (rebound when the path names a new inode); Windows reads plainly, since
|
||||
advisory-lock cancellation is a POSIX-only hazard."""
|
||||
from hermes_state import _IS_WINDOWS
|
||||
if _IS_WINDOWS:
|
||||
with contextlib.suppress(OSError), db_path.open("rb") as handle:
|
||||
handle.seek(offset)
|
||||
return handle.read(length)
|
||||
return None
|
||||
key = str(db_path)
|
||||
@@ -74,10 +76,15 @@ def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]":
|
||||
return None
|
||||
cached = _HEADER_PROBE_FDS[key] = (fd, fst.st_dev, fst.st_ino)
|
||||
with contextlib.suppress(OSError):
|
||||
return os.pread(cached[0], length, 0)
|
||||
return os.pread(cached[0], length, offset)
|
||||
return None
|
||||
|
||||
|
||||
def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]":
|
||||
"""Lock-safe raw header read of a possibly-live SQLite database (see :func:`_pread_db_range`)."""
|
||||
return _pread_db_range(db_path, 0, length)
|
||||
|
||||
|
||||
def _read_sqlite_application_id(db_path: Path) -> "Optional[int]":
|
||||
"""application_id from the SQLite header, via the lock-safe :func:`_pread_db_header`."""
|
||||
from hermes_state_errors import _STATE_DB_APPLICATION_ID_OFFSET
|
||||
@@ -149,6 +156,214 @@ def refuse_deleted_wal_generation(db_path) -> None:
|
||||
raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG)
|
||||
|
||||
|
||||
# ── Retired WAL generation capture ──────────────────────────────────────────────────────────────
|
||||
#
|
||||
# When a writer's -wal/-shm generation is deleted or replaced underneath it, the frames committed only in
|
||||
# that WAL survive exactly as long as this process keeps the unlinked inode open. The quarantine (#105670)
|
||||
# stops them from being checkpointed under wrong page numbers, but the canonical remediation -- "stop the
|
||||
# writers, then reopen" -- lets the kernel drop the inode with them. So the exact generation is captured
|
||||
# durably first: located by the recorded (st_dev, st_ino) among this process's OWN descriptors (never by
|
||||
# pathname, so a sibling's deleted WAL or a newer sidecar minted at the same path cannot be mistaken for
|
||||
# it), read with pread (no descriptor is closed, moved or truncated) and written with fsync.
|
||||
|
||||
RETIRED_GENERATION_DIR_SUFFIX = ".retired-wal-"
|
||||
RETIRED_GENERATION_MANIFEST = "manifest.json"
|
||||
RETIRED_GENERATION_MANIFEST_VERSION = 1
|
||||
# Up to this size the main image is copied whole, so the artifact is a self-contained state.db + -wal
|
||||
# pair that `hermes sessions recover --source <dir>/state.db` can open. Above it only the 100-byte
|
||||
# header is kept (the manifest says so): unlike the WAL inode, the main file survives process exit at
|
||||
# its path, and a multi-GB copy inside a shutdown path is a worse failure than a header-only artifact.
|
||||
RETIRED_GENERATION_MAIN_IMAGE_MAX_BYTES = 512 * 1024 * 1024
|
||||
_CAPTURE_CHUNK_BYTES = 8 * 1024 * 1024
|
||||
_SQLITE_HEADER_BYTES = 100
|
||||
|
||||
|
||||
class RetiredGenerationCaptureError(RuntimeError):
|
||||
"""The retired WAL generation could not be captured durably; nothing was mutated."""
|
||||
|
||||
|
||||
def _fsync_path(path: Path) -> None:
|
||||
"""fsync a file or directory we own. Never used on the live database: opening and closing a
|
||||
descriptor on a file SQLite has locked would cancel this process's POSIX advisory locks."""
|
||||
if os.name == "nt":
|
||||
return # directories cannot be opened; file writes fsync their own handle
|
||||
fd = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _own_descriptor_for_identity(identity) -> "Optional[int]":
|
||||
"""This process's open descriptor for the inode ``(st_dev, st_ino)``, or None.
|
||||
|
||||
Exact-generation qualified: pathnames are never consulted. The descriptor stays owned by
|
||||
SQLite; callers only pread from it."""
|
||||
if not identity:
|
||||
return None
|
||||
wanted = tuple(identity)
|
||||
for fd_dir in ("/proc/self/fd", "/dev/fd"):
|
||||
try:
|
||||
names = os.listdir(fd_dir)
|
||||
except OSError:
|
||||
continue
|
||||
for name in names:
|
||||
if not name.isdigit():
|
||||
continue
|
||||
try:
|
||||
st = os.fstat(int(name))
|
||||
except OSError:
|
||||
continue
|
||||
if (st.st_dev, st.st_ino) == wanted:
|
||||
return int(name)
|
||||
return None # the table was readable: a definitive miss
|
||||
return None
|
||||
|
||||
|
||||
def _copy_descriptor(fd: int, dest: Path, *, size: int) -> Dict[str, Any]:
|
||||
"""pread ``size`` bytes of ``fd`` from offset 0 into ``dest`` (temp file, fsync, rename)."""
|
||||
digest = hashlib.sha256()
|
||||
part = dest.with_name(dest.name + ".part")
|
||||
offset = 0
|
||||
with open(part, "wb") as out:
|
||||
while offset < size:
|
||||
chunk = os.pread(fd, min(_CAPTURE_CHUNK_BYTES, size - offset), offset)
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
digest.update(chunk)
|
||||
offset += len(chunk)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
os.replace(part, dest)
|
||||
return {"file": dest.name, "bytes": offset, "sha256": digest.hexdigest()}
|
||||
|
||||
|
||||
def _copy_main_image(db_path: Path, dest: Path, *, size: int) -> Dict[str, Any]:
|
||||
"""Copy the live main file through the lock-safe cached descriptor (see ``_pread_db_range``)."""
|
||||
digest = hashlib.sha256()
|
||||
part = dest.with_name(dest.name + ".part")
|
||||
offset = 0
|
||||
with open(part, "wb") as out:
|
||||
while offset < size:
|
||||
chunk = _pread_db_range(db_path, offset, min(_CAPTURE_CHUNK_BYTES, size - offset))
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
digest.update(chunk)
|
||||
offset += len(chunk)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
os.replace(part, dest)
|
||||
return {"file": dest.name, "bytes": offset, "sha256": digest.hexdigest()}
|
||||
|
||||
|
||||
def _parse_sqlite_header(header: bytes) -> Dict[str, Any]:
|
||||
if len(header) < _SQLITE_HEADER_BYTES or header[:16] != b"SQLite format 3\x00":
|
||||
return {"valid": False}
|
||||
raw_page_size = struct.unpack(">H", header[16:18])[0]
|
||||
fields = {"change_counter": 24, "page_count": 28, "user_version": 60, "application_id": 68,
|
||||
"version_valid_for": 92}
|
||||
parsed = {name: struct.unpack(">I", header[off:off + 4])[0] for name, off in fields.items()}
|
||||
return {"valid": True, "page_size": 65536 if raw_page_size == 1 else raw_page_size, **parsed}
|
||||
|
||||
|
||||
def _write_json_durably(path: Path, payload: Dict[str, Any]) -> None:
|
||||
part = path.with_name(path.name + ".part")
|
||||
with open(part, "w", encoding="utf-8") as out:
|
||||
json.dump(payload, out, indent=2, sort_keys=True)
|
||||
out.write("\n")
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
os.replace(part, path)
|
||||
|
||||
|
||||
def capture_retired_wal_generation(
|
||||
db_path, *, sidecar_identity: Dict[str, tuple], trigger: str,
|
||||
main_image_max_bytes: int = RETIRED_GENERATION_MAIN_IMAGE_MAX_BYTES,
|
||||
) -> Path:
|
||||
"""Durably capture the lost WAL generation this process still holds open; return the artifact dir.
|
||||
|
||||
The artifact sits next to the database as ``<name>.retired-wal-<utc ts>-<pid>/`` and holds the
|
||||
retired ``<name>-wal`` (and ``-shm`` when still ours), the main image (or its header past the size
|
||||
cap) and ``manifest.json`` with identities, sizes, digests and the sidecar generation found at the
|
||||
path at capture time. Nothing is merged: whether those frames belong on top of the main file is
|
||||
the operator's decision. Raises :class:`RetiredGenerationCaptureError` when the exact generation
|
||||
cannot be located or written; no descriptor is ever closed, moved or truncated.
|
||||
"""
|
||||
db_path = Path(db_path)
|
||||
wal_identity = tuple(sidecar_identity.get("-wal") or ())
|
||||
if not wal_identity:
|
||||
raise RetiredGenerationCaptureError(
|
||||
f"no recorded WAL generation identity for {db_path}; refusing to guess it by pathname")
|
||||
wal_fd = _own_descriptor_for_identity(wal_identity)
|
||||
if wal_fd is None:
|
||||
raise RetiredGenerationCaptureError(
|
||||
f"this process no longer holds the retired WAL inode {wal_identity} of {db_path}")
|
||||
try:
|
||||
wal_size = os.fstat(wal_fd).st_size
|
||||
except OSError as exc:
|
||||
raise RetiredGenerationCaptureError(f"cannot stat the retired WAL of {db_path}: {exc}") from exc
|
||||
|
||||
stem = f"{db_path.name}{RETIRED_GENERATION_DIR_SUFFIX}{time.strftime('%Y%m%d-%H%M%S', time.gmtime())}-{os.getpid()}"
|
||||
final = db_path.with_name(stem)
|
||||
n = 0
|
||||
while final.exists() or final.with_name(final.name + ".partial").exists():
|
||||
n += 1
|
||||
final = db_path.with_name(f"{stem}-{n}")
|
||||
staging = final.with_name(final.name + ".partial")
|
||||
try:
|
||||
staging.mkdir(parents=True, exist_ok=False)
|
||||
manifest: Dict[str, Any] = {
|
||||
"version": RETIRED_GENERATION_MANIFEST_VERSION,
|
||||
"database": str(db_path),
|
||||
"trigger": trigger,
|
||||
"pid": os.getpid(),
|
||||
"captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"python": sys.version.split()[0],
|
||||
"sqlite": sqlite3.sqlite_version,
|
||||
"wal": {"identity": list(wal_identity),
|
||||
**_copy_descriptor(wal_fd, staging / (db_path.name + "-wal"), size=wal_size)},
|
||||
"shm": None,
|
||||
# The generation living at the path when we captured: a newer writer's, if one was minted.
|
||||
"path_generation_at_capture": {
|
||||
suffix: list(ident) for suffix, ident in _stat_sqlite_sidecar_identity(db_path).items()},
|
||||
"note": ("Frames in the captured WAL were committed by the retired generation. Whether they "
|
||||
"belong on top of the main file now at the path is an operator decision; inspect "
|
||||
"the copied image with `hermes sessions recover --inspect-only` first."),
|
||||
}
|
||||
shm_identity = tuple(sidecar_identity.get("-shm") or ())
|
||||
shm_fd = _own_descriptor_for_identity(shm_identity) if shm_identity else None
|
||||
if shm_fd is not None:
|
||||
with contextlib.suppress(OSError): # SQLite rebuilds the index; the WAL is what matters
|
||||
manifest["shm"] = {"identity": list(shm_identity), **_copy_descriptor(
|
||||
shm_fd, staging / (db_path.name + "-shm"), size=os.fstat(shm_fd).st_size)}
|
||||
header = _pread_db_range(db_path, 0, _SQLITE_HEADER_BYTES)
|
||||
if header is None:
|
||||
raise RetiredGenerationCaptureError(f"cannot read the main image header of {db_path}")
|
||||
main_size = os.stat(db_path).st_size
|
||||
main: Dict[str, Any] = {"identity": list(_stat_db_file_identity(db_path) or ()) or None,
|
||||
"size": main_size, "header": _parse_sqlite_header(header)}
|
||||
if main_size <= main_image_max_bytes:
|
||||
main.update(mode="copied", **_copy_main_image(db_path, staging / db_path.name, size=main_size))
|
||||
else:
|
||||
header_file = staging / (db_path.name + ".header")
|
||||
header_file.write_bytes(header)
|
||||
_fsync_path(header_file)
|
||||
main.update(mode="header_only", file=header_file.name, bytes=len(header))
|
||||
manifest["main"] = main
|
||||
_write_json_durably(staging / RETIRED_GENERATION_MANIFEST, manifest)
|
||||
_fsync_path(staging)
|
||||
os.replace(staging, final)
|
||||
_fsync_path(final.parent)
|
||||
except RetiredGenerationCaptureError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise RetiredGenerationCaptureError(
|
||||
f"could not write the retired WAL generation of {db_path} under {staging}: {exc}") from exc
|
||||
return final
|
||||
|
||||
|
||||
def _connect_tracked_db(path, tracking_path=None, **kwargs):
|
||||
"""``sqlite3.connect`` that registers the open fd so byte-level probes of a live file are
|
||||
refused (an ``open()``/``close()`` would cancel every POSIX lock, even a running VACUUM's
|
||||
|
||||
Reference in New Issue
Block a user