fix(backup): don't hang forever on locked SQLite sources
hermes backup freezes mid-archive when a .db file under HERMES_HOME is locked by another process — e.g. a live Chromium profile database held with an exclusive lock by a running browser. sqlite3.Connection.backup() retries SQLITE_BUSY indefinitely and never honors the connection's busy timeout, while a plain statement on the same source fails cleanly after ~5s with "database is locked". Fixes: - Probe the source with a cheap read before snapshotting, so a locked database fails fast instead of hanging the whole backup. - Add a watchdog that interrupts the source connection after 15 minutes as a last resort for pathological cases. - Exclude browser-profiles/ from full backups: the CDP browser profile is live, regenerable (cache + re-login), and unsafe to snapshot while running. On a real install this cut the backup from 28,396 files / 1.1 GB to ~4,000 files / 548 MB, completing in ~33s instead of hanging. The pre-update automatic backup shares this code path and was equally at risk. Adds a regression test that holds an EXCLUSIVE transaction in a separate process and asserts _safe_copy_db returns False in bounded time.
This commit is contained in:
@@ -76,6 +76,12 @@ _EXCLUDED_DIRS = {
|
||||
# zipping them re-ships the DB once per snapshot
|
||||
"checkpoints", # session-local trajectory caches — regenerated per-session,
|
||||
# session-hash-keyed so they don't port to another machine anyway
|
||||
# Live browser profiles (e.g. the CDP Brave profile under browser-profiles/).
|
||||
# Chromium holds its SQLite DBs with exclusive locks while running, and
|
||||
# sqlite3.Connection.backup() retries SQLITE_BUSY forever instead of honoring
|
||||
# the busy timeout — a full backup hangs mid-archive on the first locked DB.
|
||||
# Profiles are regenerable (cache + re-login) and unsafe to snapshot live.
|
||||
"browser-profiles",
|
||||
# Python dependency trees (plugin / MCP-server venvs under HERMES_HOME) —
|
||||
# regenerated by reinstalling; never irreplaceable state.
|
||||
".venv",
|
||||
|
||||
@@ -1135,6 +1135,51 @@ class TestSafeCopyDb:
|
||||
assert not dst.exists()
|
||||
|
||||
|
||||
def test_locked_source_fails_fast_not_hang(self, tmp_path):
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from hermes_cli.backup import _safe_copy_db
|
||||
src = tmp_path / "locked.db"
|
||||
dst = tmp_path / "copy.db"
|
||||
|
||||
conn = sqlite3.connect(str(src))
|
||||
conn.execute("CREATE TABLE t (x INTEGER)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# Hold an EXCLUSIVE transaction in a separate process. POSIX file
|
||||
# locks only conflict across processes, so an in-process connection
|
||||
# cannot reproduce the "database is locked" condition.
|
||||
holder = (
|
||||
"import sqlite3, time\n"
|
||||
f"c = sqlite3.connect({str(src)!r})\n"
|
||||
"c.execute('BEGIN EXCLUSIVE')\n"
|
||||
"print('LOCKED', flush=True)\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-c", holder],
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
try:
|
||||
assert proc.stdout is not None
|
||||
assert proc.stdout.readline().strip() == "LOCKED"
|
||||
started = time.monotonic()
|
||||
result = _safe_copy_db(src, dst)
|
||||
elapsed = time.monotonic() - started
|
||||
assert result is False
|
||||
# The busy timeout is 5s, so a fast failure lands around there.
|
||||
# The regression this guards against is backup() retrying
|
||||
# SQLITE_BUSY forever, which would never return at all.
|
||||
assert elapsed < 30
|
||||
finally:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
|
||||
|
||||
def test_is_zeroed_sqlite_file_detects_nul_header(self, tmp_path):
|
||||
from hermes_cli.backup import is_zeroed_sqlite_file
|
||||
p = tmp_path / "state.db"
|
||||
|
||||
Reference in New Issue
Block a user