fix(state): bound the state.db repair loop and stop dead-backup accumulation (#86747)

A corruption class the repair strategies cannot heal (b-tree page
damage) failed repair_state_db_schema on every process start, forever:
_claim_repair_attempt's in-memory set only bounds one process, so each
restart re-ran the full surgery AND took a fresh ~900MB forensic backup
of the same damaged bytes — 105 attempts / 89GB of dead
state.db.malformed-backup-* files over 11 days in the reporting install.

Three bounded behaviors, all sidecar-file based (no schema changes):

1. Persistent attempt ledger (<db>.repair-attempts.json): after 3 failed
   repair passes against the same file fingerprint (size + mtime_ns),
   repair_state_db_schema refuses with a terminal, actionable error
   (restore a backup / `sqlite3 state.db ".recover"` / delete the ledger
   to force a retry) instead of re-running surgery. Success clears the
   ledger; a replaced or restored file re-keys it and gets fresh
   attempts. Missing/corrupt ledger fails open (never blocks a first
   repair).

2. Backup dedupe: _backup_db_file reuses the newest existing forensic
   backup when it is byte-identical to the damaged file (size+mtime
   match, preserved by copy2) instead of copying another ~900MB.

3. Retention cap: only the 3 newest malformed-backup copies (plus
   sidecars) are kept; older ones are pruned after each new backup.
   Also fixes a same-second timestamp collision that silently
   overwrote an earlier forensic copy.

Tests cover ledger accumulation, terminal refusal (surgery not called,
no new backup), budget reset on file change, success-clears-ledger,
corrupt-ledger tolerance, dedupe, distinct-state backups, retention
prune incl. sidecars, and the end-to-end one-backup invariant.

Fixes #86747
This commit is contained in:
Teknium
2026-08-15 02:14:57 -07:00
parent ca8a47e276
commit 0d00ebef79
2 changed files with 357 additions and 2 deletions
+194 -2
View File
@@ -1807,9 +1807,144 @@ def _bump_schema_cookie(conn: sqlite3.Connection) -> None:
logger.warning("Could not bump state.db schema cookie: %s", exc)
# ── Repair-loop bounding + dead-backup hygiene (#86747) ─────────────────────
#
# ``_claim_repair_attempt`` above is an in-memory set: it bounds the loop
# only WITHIN one process. A corruption class the strategies cannot heal
# (b-tree page damage) failed repair on EVERY process start, and each pass
# took a fresh ~900MB forensic backup — 105 attempts / 89GB of identical
# dead copies in the reporting install. Two persistent bounds fix the class:
#
# * a sidecar attempt ledger (``<db>.repair-attempts.json``) that refuses
# further surgery after ``_MAX_PERSISTENT_REPAIR_ATTEMPTS`` failures on
# the SAME damaged file (fingerprint = size + mtime; any successful repair
# or replacement changes it and resets the count);
# * backup dedupe + a retention cap in ``_backup_db_file`` — an identical
# damaged file is never copied twice, and only the newest
# ``_MAX_MALFORMED_BACKUPS`` forensic copies are kept.
_MAX_PERSISTENT_REPAIR_ATTEMPTS = 3
_MAX_MALFORMED_BACKUPS = 3
def _repair_ledger_path(db_path: Path) -> Path:
return db_path.with_name(db_path.name + ".repair-attempts.json")
def _db_fingerprint(db_path: Path) -> "Optional[str]":
"""Cheap identity for a damaged DB file: size + mtime_ns.
Hashing a multi-GB corrupt file on every open is exactly the kind of
repeated cost this ledger exists to avoid; size+mtime is stable for a
file nothing can successfully write to, and any successful repair,
truncation or manual restore changes it (resetting the attempt count).
"""
try:
st = db_path.stat()
return f"{st.st_size}:{st.st_mtime_ns}"
except OSError:
return None
def _read_repair_ledger(db_path: Path) -> "Dict[str, Any]":
try:
raw = json.loads(_repair_ledger_path(db_path).read_text(encoding="utf-8"))
if isinstance(raw, dict):
return raw
except (OSError, ValueError):
pass
return {}
def _persistent_repair_attempts_exhausted(db_path: Path) -> bool:
"""Whether *db_path* has already burned its cross-restart repair budget.
True only when the ledger records ``_MAX_PERSISTENT_REPAIR_ATTEMPTS``
failed attempts against the CURRENT file fingerprint. Never raises; a
missing/corrupt ledger or unstatable DB reads as "not exhausted" (the
in-process claim and cross-process lock still bound a single run).
"""
fp = _db_fingerprint(db_path)
if fp is None:
return False
ledger = _read_repair_ledger(db_path)
return (
ledger.get("fingerprint") == fp
and int(ledger.get("failed_attempts", 0)) >= _MAX_PERSISTENT_REPAIR_ATTEMPTS
)
def _record_repair_outcome(
db_path: Path, *, repaired: bool, fingerprint: "Optional[str]" = None
) -> None:
"""Update the persistent attempt ledger after a repair pass. Never raises.
Defaults to the post-attempt fingerprint — the file state the NEXT
attempt's exhaustion probe will observe.
"""
ledger_path = _repair_ledger_path(db_path)
try:
if repaired:
ledger_path.unlink(missing_ok=True)
return
fp = fingerprint if fingerprint is not None else _db_fingerprint(db_path)
if fp is None:
return
ledger = _read_repair_ledger(db_path)
attempts = (
int(ledger.get("failed_attempts", 0)) + 1
if ledger.get("fingerprint") == fp
else 1
)
import datetime
ledger_path.write_text(
json.dumps(
{
"fingerprint": fp,
"failed_attempts": attempts,
"last_attempt": datetime.datetime.now().isoformat(
timespec="seconds"
),
}
),
encoding="utf-8",
)
except Exception as exc: # pragma: no cover - best effort
logger.warning("Could not update state.db repair ledger: %s", exc)
def _existing_malformed_backups(db_path: Path) -> "List[Path]":
"""Timestamped forensic backups of *db_path*, newest first."""
prefix = f"{db_path.name}.malformed-backup-"
try:
found = [
p
for p in db_path.parent.iterdir()
if p.name.startswith(prefix)
and not p.name.endswith(("-wal", "-shm"))
]
except OSError:
return []
return sorted(found, key=lambda p: p.name, reverse=True)
def _prune_malformed_backups(db_path: Path, keep: int = _MAX_MALFORMED_BACKUPS) -> None:
"""Delete all but the *keep* newest forensic backups (and sidecars)."""
for stale in _existing_malformed_backups(db_path)[keep:]:
for victim in (
stale,
stale.with_name(stale.name + "-wal"),
stale.with_name(stale.name + "-shm"),
):
try:
victim.unlink(missing_ok=True)
except OSError as exc: # pragma: no cover - best effort
logger.warning("Could not prune stale DB backup %s: %s", victim, exc)
def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]":
"""Copy a (possibly malformed) DB file to a timestamped backup beside it.
Raw file copy on purpose: the DB won't open cleanly, so we preserve the
bytes exactly for forensics / manual restore. WAL and SHM sidecars are
copied too when present. Returns ``(backup_path, None)`` on success or
@@ -1843,12 +1978,41 @@ def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]":
stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
backup_path = db_path.with_name(f"{db_path.name}.malformed-backup-{stamp}")
# Same-second collision (two distinct damaged states within one second)
# must not silently overwrite the earlier forensic copy.
seq = 1
while backup_path.exists():
backup_path = db_path.with_name(
f"{db_path.name}.malformed-backup-{stamp}_{seq}"
)
seq += 1
try:
# Dedupe (#86747): a repair loop used to copy the SAME damaged bytes
# on every restart — ~900MB a pass, 89GB over 11 days in the
# reporting install. If the newest existing backup already matches
# this file (size + mtime preserved by copy2), reuse it.
try:
src_stat = db_path.stat()
for existing in _existing_malformed_backups(db_path)[:1]:
est = existing.stat()
if (
est.st_size == src_stat.st_size
and est.st_mtime_ns == src_stat.st_mtime_ns
):
logger.info(
"Reusing existing forensic backup %s (identical to the "
"damaged DB).", existing,
)
return existing, None
except OSError:
pass
shutil.copy2(db_path, backup_path)
for suffix in ("-wal", "-shm"):
sidecar = db_path.with_name(db_path.name + suffix)
if sidecar.exists():
shutil.copy2(sidecar, backup_path.with_name(backup_path.name + suffix))
# Retention cap (#86747): keep only the newest few forensic copies.
_prune_malformed_backups(db_path)
return backup_path, None
except Exception as exc: # pragma: no cover - best effort
logger.warning("Could not back up malformed DB %s: %s", db_path, exc)
@@ -2116,6 +2280,25 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A
report["error"] = f"{db_path} does not exist"
return report
# Cross-restart attempt cap (#86747): the in-memory claim bounds one
# process, but a corruption class the strategies below cannot heal
# (b-tree page damage) previously re-ran the whole surgery — and took a
# fresh multi-hundred-MB forensic backup — on EVERY restart, forever.
# After _MAX_PERSISTENT_REPAIR_ATTEMPTS failures against the same
# damaged file, stop retrying and surface a terminal, actionable error.
if _persistent_repair_attempts_exhausted(db_path):
report["error"] = (
f"automatic repair has already failed "
f"{_MAX_PERSISTENT_REPAIR_ATTEMPTS} times on this exact file — "
"the corruption is beyond the schema/FTS repair strategies "
"(likely b-tree page damage). Manual recovery required: restore "
f"a backup, or salvage with `sqlite3 {db_path} \".recover\"`. "
f"Delete {_repair_ledger_path(db_path).name} to force another "
"automatic attempt."
)
logger.error("state.db repair skipped: %s", report["error"])
return report
with _cross_process_repair_lock(db_path) as holding_lock:
if not holding_lock:
# Another process is still inside its critical section. It may
@@ -2130,7 +2313,16 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A
"schema surgery to avoid racing it"
)
return report
return _repair_state_db_schema_locked(db_path, backup=backup, report=report)
result = _repair_state_db_schema_locked(db_path, backup=backup, report=report)
# Persist the outcome AFTER surgery, keyed on the post-attempt
# fingerprint — that is the file state the NEXT attempt's exhaustion
# probe will observe. Failures count toward the cross-restart cap;
# success clears the ledger. (A failing strategy that mutates the
# file re-keys the ledger and restarts the count: that keeps a
# genuinely NEW corruption event from inheriting a stale budget,
# while the backup dedupe/cap above bounds the disk cost either way.)
_record_repair_outcome(db_path, repaired=bool(result.get("repaired")))
return result
def _repair_state_db_schema_locked(
+163
View File
@@ -0,0 +1,163 @@
"""#86747 regression: the state.db repair loop must be bounded across restarts
and must not accumulate identical multi-hundred-MB forensic backups.
Reported incident: b-tree page corruption (a class none of the repair
strategies can heal) failed `repair_state_db_schema` on every process start
for 11 days — 105 attempts, each taking a fresh ~900MB
`state.db.malformed-backup-*` copy of the SAME damaged bytes, 89GB total —
while `_claim_repair_attempt`'s in-memory set only bounded a single process.
Fix under test:
- persistent sidecar attempt ledger (`<db>.repair-attempts.json`): after
`_MAX_PERSISTENT_REPAIR_ATTEMPTS` failed passes on the same file
fingerprint, `repair_state_db_schema` refuses with a terminal, actionable
error instead of re-running surgery;
- `_backup_db_file` dedupes against the newest existing backup (same
size+mtime → reuse) and prunes to `_MAX_MALFORMED_BACKUPS` copies.
"""
from __future__ import annotations
import json
import os
import sqlite3
from pathlib import Path
from unittest.mock import patch
import hermes_state
from hermes_state import (
_MAX_MALFORMED_BACKUPS,
_MAX_PERSISTENT_REPAIR_ATTEMPTS,
_backup_db_file,
_existing_malformed_backups,
_persistent_repair_attempts_exhausted,
_prune_malformed_backups,
_record_repair_outcome,
_repair_ledger_path,
repair_state_db_schema,
)
def _make_unrepairable_db(tmp_path: Path) -> Path:
"""A file sqlite3 opens as garbage — no strategy can heal it."""
db = tmp_path / "state.db"
db.write_bytes(b"SQLite format 3\x00" + os.urandom(4096))
return db
def _make_healthy_db(tmp_path: Path) -> Path:
db = tmp_path / "state.db"
conn = sqlite3.connect(str(db))
conn.execute("CREATE TABLE t (x)")
conn.execute("INSERT INTO t VALUES (1)")
conn.commit()
conn.close()
return db
# ---------------------------------------------------------------------------
# Persistent attempt ledger
# ---------------------------------------------------------------------------
class TestPersistentAttemptCap:
def test_failed_repairs_accumulate_in_ledger(self, tmp_path):
db = _make_unrepairable_db(tmp_path)
report = repair_state_db_schema(db)
assert report["repaired"] is False
ledger = json.loads(_repair_ledger_path(db).read_text())
assert ledger["failed_attempts"] == 1
def test_repair_refuses_after_cap_with_terminal_error(self, tmp_path):
db = _make_unrepairable_db(tmp_path)
for _ in range(_MAX_PERSISTENT_REPAIR_ATTEMPTS):
report = repair_state_db_schema(db)
assert report["repaired"] is False
# Budget burned: the next call must refuse WITHOUT running surgery
# (and without taking another backup).
backups_before = len(_existing_malformed_backups(db))
with patch.object(hermes_state, "_repair_state_db_schema_locked") as surgery:
report = repair_state_db_schema(db)
surgery.assert_not_called()
assert report["repaired"] is False
assert "Manual recovery required" in report["error"]
assert ".recover" in report["error"]
assert len(_existing_malformed_backups(db)) == backups_before
def test_changed_file_resets_the_budget(self, tmp_path):
db = _make_unrepairable_db(tmp_path)
for _ in range(_MAX_PERSISTENT_REPAIR_ATTEMPTS):
repair_state_db_schema(db)
assert _persistent_repair_attempts_exhausted(db)
# A restored/replaced file (different size+mtime) gets fresh attempts.
db.write_bytes(b"SQLite format 3\x00" + os.urandom(8192))
assert not _persistent_repair_attempts_exhausted(db)
def test_successful_repair_clears_the_ledger(self, tmp_path):
db = _make_healthy_db(tmp_path)
_record_repair_outcome(db, repaired=False)
assert _repair_ledger_path(db).exists()
_record_repair_outcome(db, repaired=True)
assert not _repair_ledger_path(db).exists()
def test_corrupt_ledger_is_ignored_not_fatal(self, tmp_path):
db = _make_unrepairable_db(tmp_path)
_repair_ledger_path(db).write_text("{not json")
assert not _persistent_repair_attempts_exhausted(db)
# And a repair pass overwrites it cleanly.
repair_state_db_schema(db)
assert json.loads(_repair_ledger_path(db).read_text())["failed_attempts"] == 1
# ---------------------------------------------------------------------------
# Backup dedupe + retention cap
# ---------------------------------------------------------------------------
class TestBackupDedupeAndCap:
def test_identical_file_is_not_backed_up_twice(self, tmp_path):
db = _make_unrepairable_db(tmp_path)
first, err = _backup_db_file(db)
assert err is None and first is not None
second, err = _backup_db_file(db)
assert err is None
# Same damaged bytes (size+mtime unchanged) → the existing backup is
# reused instead of copying another ~900MB.
assert second == first
assert len(_existing_malformed_backups(db)) == 1
def test_changed_file_gets_a_new_backup(self, tmp_path):
db = _make_unrepairable_db(tmp_path)
first, _ = _backup_db_file(db)
db.write_bytes(b"SQLite format 3\x00" + os.urandom(2048))
second, err = _backup_db_file(db)
assert err is None
assert second != first
def test_retention_cap_prunes_oldest(self, tmp_path):
db = _make_unrepairable_db(tmp_path)
# Seed more than the cap with distinct fake timestamped backups.
for i in range(_MAX_MALFORMED_BACKUPS + 3):
fake = db.with_name(f"{db.name}.malformed-backup-20260801_00000{i}")
fake.write_bytes(b"x")
fake.with_name(fake.name + "-wal").write_bytes(b"w")
_prune_malformed_backups(db)
remaining = _existing_malformed_backups(db)
assert len(remaining) == _MAX_MALFORMED_BACKUPS
# Newest kept (sorted by timestamp suffix, descending).
names = [p.name for p in remaining]
assert names == sorted(names, reverse=True)
# Sidecars of pruned backups are gone too.
leftover_sidecars = [
p for p in tmp_path.iterdir()
if p.name.endswith("-wal")
and p.with_name(p.name[:-4]) not in remaining
]
assert leftover_sidecars == []
def test_backup_via_repair_path_does_not_accumulate(self, tmp_path):
"""End-to-end: repeated failed repairs on the same file keep ONE backup."""
db = _make_unrepairable_db(tmp_path)
for _ in range(_MAX_PERSISTENT_REPAIR_ATTEMPTS):
repair_state_db_schema(db)
assert len(_existing_malformed_backups(db)) == 1