fix(backup): don't hang forever on locked SQLite sources

hermes backup freezes mid-archive when a .db file under HERMES_HOME is
locked by another process — e.g. a live Chromium profile database held
with an exclusive lock by a running browser. sqlite3.Connection.backup()
retries SQLITE_BUSY indefinitely and never honors the connection's busy
timeout, while a plain statement on the same source fails cleanly after
~5s with "database is locked".

Fixes:
- Probe the source with a cheap read before snapshotting, so a locked
  database fails fast instead of hanging the whole backup.
- Add a watchdog that interrupts the source connection after 15 minutes
  as a last resort for pathological cases.
- Exclude browser-profiles/ from full backups: the CDP browser profile is
  live, regenerable (cache + re-login), and unsafe to snapshot while
  running. On a real install this cut the backup from 28,396 files /
  1.1 GB to ~4,000 files / 548 MB, completing in ~33s instead of hanging.

The pre-update automatic backup shares this code path and was equally
at risk.

Adds a regression test that holds an EXCLUSIVE transaction in a separate
process and asserts _safe_copy_db returns False in bounded time.
This commit is contained in:
xthezealot
2026-08-12 12:44:52 +02:00
committed by Teknium
parent f9849c43a2
commit d422f7103e
2 changed files with 51 additions and 0 deletions
+6
View File
@@ -76,6 +76,12 @@ _EXCLUDED_DIRS = {
# zipping them re-ships the DB once per snapshot
"checkpoints", # session-local trajectory caches — regenerated per-session,
# session-hash-keyed so they don't port to another machine anyway
# Live browser profiles (e.g. the CDP Brave profile under browser-profiles/).
# Chromium holds its SQLite DBs with exclusive locks while running, and
# sqlite3.Connection.backup() retries SQLITE_BUSY forever instead of honoring
# the busy timeout — a full backup hangs mid-archive on the first locked DB.
# Profiles are regenerable (cache + re-login) and unsafe to snapshot live.
"browser-profiles",
# Python dependency trees (plugin / MCP-server venvs under HERMES_HOME) —
# regenerated by reinstalling; never irreplaceable state.
".venv",
+45
View File
@@ -1135,6 +1135,51 @@ class TestSafeCopyDb:
assert not dst.exists()
def test_locked_source_fails_fast_not_hang(self, tmp_path):
import subprocess
import sys
import time
from hermes_cli.backup import _safe_copy_db
src = tmp_path / "locked.db"
dst = tmp_path / "copy.db"
conn = sqlite3.connect(str(src))
conn.execute("CREATE TABLE t (x INTEGER)")
conn.commit()
conn.close()
# Hold an EXCLUSIVE transaction in a separate process. POSIX file
# locks only conflict across processes, so an in-process connection
# cannot reproduce the "database is locked" condition.
holder = (
"import sqlite3, time\n"
f"c = sqlite3.connect({str(src)!r})\n"
"c.execute('BEGIN EXCLUSIVE')\n"
"print('LOCKED', flush=True)\n"
"time.sleep(60)\n"
)
proc = subprocess.Popen(
[sys.executable, "-c", holder],
stdout=subprocess.PIPE,
text=True,
)
try:
assert proc.stdout is not None
assert proc.stdout.readline().strip() == "LOCKED"
started = time.monotonic()
result = _safe_copy_db(src, dst)
elapsed = time.monotonic() - started
assert result is False
# The busy timeout is 5s, so a fast failure lands around there.
# The regression this guards against is backup() retrying
# SQLITE_BUSY forever, which would never return at all.
assert elapsed < 30
finally:
proc.kill()
proc.wait()
def test_is_zeroed_sqlite_file_detects_nul_header(self, tmp_path):
from hermes_cli.backup import is_zeroed_sqlite_file
p = tmp_path / "state.db"