diff --git a/hermes_cli/doctor_platform.py b/hermes_cli/doctor_platform.py index 9b1f7edf2d..59baa04bf7 100644 --- a/hermes_cli/doctor_platform.py +++ b/hermes_cli/doctor_platform.py @@ -92,7 +92,7 @@ def _format_db_size(db_path: Path) -> str: def _report_database_journal_modes(hermes_home: Path | None = None, version_info: tuple[int, ...] | None = None) -> None: """List each database's journal mode; warn on WAL under a vulnerable SQLite.""" from hermes_cli.doctor import HERMES_HOME - from hermes_state_wal import _wal_reset_repair_hint, is_sqlite_wal_reset_vulnerable + from hermes_state_wal import _path_on_cross_vm_fs, _wal_reset_repair_hint, is_sqlite_wal_reset_vulnerable vulnerable = is_sqlite_wal_reset_vulnerable(version_info) try: databases = _hermes_database_paths(hermes_home if hermes_home is not None else HERMES_HOME) @@ -110,6 +110,16 @@ def _report_database_journal_modes(hermes_home: Path | None = None, version_info check_warn(f"{name}: journal mode could not be read", f"({error}; cannot rule out WAL exposure)") else: check_info(f"{name}: journal mode could not be read ({error})") + elif mode == "wal" and _path_on_cross_vm_fs(str(path)): + # #110848: WAL shared-memory is not coherent across a virtiofs/9p bind mount; startup only refuses WAL + # for FRESH databases, so an existing WAL file here keeps corrupting until the operator converts it. + # Checked before the WAL-reset exposure: active cross-VM corruption outranks a latent bug class. + if vulnerable: + exposed.append(name) + check_warn(f"{name} is in WAL mode on a cross-VM filesystem (virtiofs/9p, {size})", + "(WAL can silently corrupt across the VM boundary; stop every Hermes process and run a one-time " + "offline 'PRAGMA journal_mode=DELETE' on the file, then set `database.journal_mode: delete` — " + "or move the database onto a native/named volume)") elif mode == "wal" and vulnerable: exposed.append(name) check_warn(f"{name} is in WAL mode ({size})", "(exposed to the WAL-reset bug until SQLite is upgraded)") diff --git a/hermes_state_wal.py b/hermes_state_wal.py index 7b228302d9..72eb500ebf 100644 --- a/hermes_state_wal.py +++ b/hermes_state_wal.py @@ -185,6 +185,8 @@ _cross_vm_fs_cache: Dict[str, bool] = {} # per DB directory; kanban_db.connect( _cross_vm_fs_cache_lock = threading.Lock() _cross_vm_warned_paths: set[str] = set() _cross_vm_warned_lock = threading.Lock() +_cross_vm_existing_wal_warned_paths: set[str] = set() +_cross_vm_existing_wal_warned_lock = threading.Lock() def _mountinfo_fstype(directory: str, mountinfo_path: str = "/proc/self/mountinfo") -> str: @@ -283,6 +285,11 @@ def apply_wal_with_fallback(conn: sqlite3.Connection, *, db_label: str = "state. if configured == "delete": # Never-live-downgrade keeps WAL; tell the operator their delete did not apply. _log_configured_delete_overridden_once(db_label) + # #110848: the fresh-DB refusal below never sees a DB that was already WAL before the cross-VM check + # existed (or was created on a native volume and then moved). Keeping WAL is right; staying silent is not. + db_file = _connection_db_file(conn) + if db_file and _path_on_cross_vm_fs(db_file): + _log_once("cross_vm_fs_existing_wal", db_label) _apply_wal_companions(conn) return "wal" @@ -520,6 +527,16 @@ _ONCE_LOGS = { "corrupt the database, so journal_mode=DELETE is used instead. To restore WAL concurrency, move the database " "onto a native volume (e.g. a named Docker volume) instead of a host bind mount. This message fires once per " "process per database."), + "cross_vm_fs_existing_wal": (_cross_vm_existing_wal_warned_lock, "_cross_vm_existing_wal_warned_paths", logging.ERROR, + # ERROR, unlike the fresh-DB refusal: this database IS running WAL on the corrupting filesystem and only the + # operator can fix it (a live downgrade under other openers would destroy their uncheckpointed commits). + "%s: existing WAL-mode database is on a cross-VM filesystem (virtiofs/9p — typical for Docker Desktop / " + "OrbStack / Podman host bind mounts). SQLite WAL shared-memory is not coherent across the VM boundary and " + "concurrent writers can silently corrupt the database. Hermes does not live-downgrade an on-disk WAL database. " + "Fix one of two ways: stop every Hermes process using this database and run a one-time offline " + "'PRAGMA journal_mode=DELETE' on the file (set `database.journal_mode: delete` in config.yaml to keep it), " + "or move the database onto a native volume (e.g. a named Docker volume). This message fires once per process " + "per database."), } diff --git a/tests/hermes_cli/test_doctor_journal_modes.py b/tests/hermes_cli/test_doctor_journal_modes.py index 703d583b32..401fecb1bf 100644 --- a/tests/hermes_cli/test_doctor_journal_modes.py +++ b/tests/hermes_cli/test_doctor_journal_modes.py @@ -321,6 +321,18 @@ class TestUnreadableReason: class TestReportDatabaseJournalModes: + def test_wal_db_on_cross_vm_fs_is_flagged_with_offline_remedy(self, tmp_path, capsys, monkeypatch): + # #110848: startup only refuses WAL for fresh databases on virtiofs/9p; doctor must surface an existing WAL + # file there (with a non-vulnerable SQLite, where it used to print a plain info line). + _make_db(tmp_path / "state.db", journal_mode="WAL") + monkeypatch.setattr("hermes_state_wal._path_on_cross_vm_fs", lambda p: True) + + doctor_platform._report_database_journal_modes(tmp_path, (3, 51, 3)) + + out = capsys.readouterr().out + assert "state.db is in WAL mode on a cross-VM filesystem" in out + assert "PRAGMA journal_mode=DELETE" in out + def test_vulnerable_runtime_wal_db_is_exposed(self, tmp_path, capsys): _make_db(tmp_path / "state.db", journal_mode="WAL") diff --git a/tests/hermes_state/test_cross_vm_fs_wal_refusal.py b/tests/hermes_state/test_cross_vm_fs_wal_refusal.py index 53a1f95e32..9427912554 100644 --- a/tests/hermes_state/test_cross_vm_fs_wal_refusal.py +++ b/tests/hermes_state/test_cross_vm_fs_wal_refusal.py @@ -5,6 +5,7 @@ WAL over a VM-boundary filesystem (Docker Desktop / OrbStack / Podman host bind while never live-downgrading an on-disk WAL database and never flagging an ordinary filesystem. """ +import logging import sqlite3 import pytest @@ -60,6 +61,7 @@ class TestWalRefusalOnCrossVmFs: monkeypatch.setattr(hermes_state_wal, "is_sqlite_wal_reset_vulnerable", lambda *a, **k: False) monkeypatch.setattr(hermes_state_wal, "resolve_journal_mode", lambda: "wal") hermes_state_wal._cross_vm_warned_paths.clear() + hermes_state_wal._cross_vm_existing_wal_warned_paths.clear() def test_fresh_db_on_cross_vm_fs_gets_delete_and_without_detection_gets_wal(self, tmp_path, monkeypatch): monkeypatch.setattr(hermes_state_wal, "_path_on_cross_vm_fs", lambda p: True) @@ -94,3 +96,25 @@ class TestWalRefusalOnCrossVmFs: conn = sqlite3.connect(str(db)) assert apply_wal_with_fallback(conn, db_label=str(db)) == "wal" conn.close() + + def test_existing_wal_db_on_cross_vm_fs_warns_operator_once(self, tmp_path, monkeypatch, caplog): + # #110848: the fresh-DB refusal cannot help a database that is already WAL, and staying silent left the + # reporter with a corrupting state.db and no signal. Keep WAL (never live-downgrade) but say so, once. + db = tmp_path / "already-wal.db" + seed = sqlite3.connect(str(db)) + if str(seed.execute("PRAGMA journal_mode=WAL").fetchone()[0]).lower() != "wal": + seed.close() + pytest.skip("environment refuses WAL") + seed.execute("CREATE TABLE t (x)") + seed.commit() + seed.close() + monkeypatch.setattr(hermes_state_wal, "_path_on_cross_vm_fs", lambda p: True) + with caplog.at_level(logging.ERROR, logger=hermes_state_wal.logger.name): + for _ in range(2): + conn = sqlite3.connect(str(db)) + assert apply_wal_with_fallback(conn, db_label="state.db") == "wal" + conn.close() + errors = [r for r in caplog.records if r.levelno == logging.ERROR and "cross-VM" in r.getMessage()] + assert len(errors) == 1 + assert "PRAGMA journal_mode=DELETE" in errors[0].getMessage() + assert "native volume" in errors[0].getMessage() diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 62fe7d0bbf..6bf891fbc2 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -95,10 +95,13 @@ The `database:` section controls how Hermes opens its SQLite state database ```yaml database: # Journal mode for state.db: wal (default) or delete. - # Use delete on filesystems where WAL is unsafe (network mounts, some - # virtiofs setups). Note: an existing on-disk WAL database is never + # Use delete on filesystems where WAL is unsafe (network mounts). On + # virtiofs/9p bind mounts (Docker Desktop, Podman on macOS, OrbStack) + # Hermes detects the mount and creates fresh databases in delete mode + # automatically. Note: an existing on-disk WAL database is never # live-downgraded — Hermes keeps WAL and logs an error telling you the - # configured delete did not apply. To convert an existing database, stop + # configured delete did not apply (or that the WAL database sits on a + # cross-VM mount). To convert an existing database, stop # every process using it and run a one-time offline # `PRAGMA journal_mode=DELETE` on the file. journal_mode: wal diff --git a/website/docs/user-guide/docker.md b/website/docs/user-guide/docker.md index 11ad58a237..e8fe60a261 100644 --- a/website/docs/user-guide/docker.md +++ b/website/docs/user-guide/docker.md @@ -195,6 +195,19 @@ The `/opt/data` volume is the single source of truth for all Hermes state. It ma | `logs/` | Runtime logs | | `skins/` | Custom CLI skins | +### Filesystem requirements for `state.db` in containers + +Hermes keeps sessions in a SQLite database (`/opt/data/state.db`) that is opened in WAL journal mode by default. WAL relies on shared memory (`state.db-shm`) being coherent between every process that has the file open. Bind mounts that cross a VM boundary do not provide that: **virtiofs** (Docker Desktop and Podman on macOS, OrbStack) and **9p / drvfs** (Docker Desktop on Windows) both let concurrent writers silently corrupt a WAL database while the main file still passes `PRAGMA integrity_check`. + +What Hermes does about it (since v2026.9.14): + +- A **fresh** database whose directory is on a virtiofs/9p mount is created in rollback (`DELETE`) journal mode and a one-time warning is logged. Nothing to do. +- An **existing** WAL database on such a mount is never live-downgraded — other Hermes processes may hold it open, and a live switch destroys their uncheckpointed commits. Instead, every process logs a one-time error at startup and `hermes doctor` flags the database. Fix it one of two ways: + 1. Stop every Hermes process that uses the database, then run a one-time offline conversion: `sqlite3 /opt/data/state.db 'PRAGMA journal_mode=DELETE'`. Set `database.journal_mode: delete` in `config.yaml` so a later open does not switch it back to WAL. + 2. Move the data directory onto a native volume — a named Docker volume (`-v hermes-data:/opt/data`) lives on the VM's own ext4 filesystem and supports WAL normally. + +Detection reads `/proc/self/mountinfo` inside the container, so it works regardless of the host operating system. It does not classify NFS, SMB, or generic FUSE mounts; on those, set `database.journal_mode: delete` explicitly. Hermes does not offer SQLite's `locking_mode=EXCLUSIVE` as an alternative because the gateway, cron, and worker processes open the database concurrently. + ### Immutable install tree In hosted and published Docker images, `/opt/hermes` is the installed application tree. It is root-owned and read-only to the runtime `hermes` user, so agent turns, gateway sessions, dashboard actions, and normal `docker exec hermes hermes ...` commands cannot edit the core source, bundled `.venv`, `node_modules`, or TUI bundle in place.