diff --git a/agent/turn_explainers.py b/agent/turn_explainers.py index f564572443..3453902597 100644 --- a/agent/turn_explainers.py +++ b/agent/turn_explainers.py @@ -119,6 +119,21 @@ _PERSISTENCE_CAUSE_EXPLANATIONS: Dict[str, str] = { "sessions/.jsonl and, on the gateway, " "pending_messages/pending-*.json." ), + "deleted_wal": ( + "the turn was stopped because a live Hermes process held a retired " + "state.db-wal generation after its pathname was deleted or " + "replaced. Stop the gateway, dashboard, and cron writers; " + "do not overwrite the current state.db or delete its sidecars. " + "Check the logs for whether Hermes captured the retired generation, " + "then read the adjacent state.db.retired-wal-*/manifest.json. If " + "manifest.main.mode is `copied`, inspect that artifact with `hermes " + "sessions recover --source " + "--inspect-only` before deciding whether its committed frames belong " + "on the current database. A `header_only` artifact is forensic and " + "does not contain a copied state.db to inspect. Unwritten messages " + "were diverted to sessions/.jsonl and, on the gateway, " + "pending_messages/pending-*.json." + ), "corrupt": ( "the turn was stopped because the state database " "reported structural corruption (the transcript would " diff --git a/hermes_state_errors.py b/hermes_state_errors.py index 17dd4b53fb..c1523c377e 100644 --- a/hermes_state_errors.py +++ b/hermes_state_errors.py @@ -74,8 +74,8 @@ def is_disk_full_error(exc: BaseException | str | None) -> bool: # Every classify_persistence_error bucket; consumers enumerate this tuple. PERSISTENCE_ERROR_CAUSES = ( - "locked", "compression", "compression_closed", "turn_lease", "corrupt", "replaced", "disk", - "unknown", + "locked", "compression", "compression_closed", "turn_lease", "corrupt", "replaced", + "deleted_wal", "disk", "unknown", ) @@ -182,6 +182,9 @@ _PERSISTENCE_CAUSE_BY_TYPE = ( (SessionTurnLeaseLostError, "turn_lease"), (CompressionSessionClosedError, "compression_closed"), (CompressionSessionBusyError, "compression"), + # The WAL-generation error subclasses StateDbReplacedError so existing write diversion keeps + # working; classify it first because its recovery artifact and operator action are different. + (DeletedWalGenerationError, "deleted_wal"), (StateDbReplacedError, "replaced"), (StateDbCorruptError, "corrupt"), ) @@ -189,7 +192,9 @@ _PERSISTENCE_CAUSE_BY_PHRASE = ( (("turn lease",), "turn_lease"), (("closed by compression",), "compression_closed"), (("being compressed", "compression lease"), "compression"), - (("was replaced underneath", "deleted state.db-wal", "deleted state.db-shm"), "replaced"), + # RPC-wrapped errors lose their exception type; retain the same sidecar/main-file split. + (("deleted state.db-wal", "deleted state.db-shm"), "deleted_wal"), + (("was replaced underneath",), "replaced"), (_DB_CORRUPTION_MARKERS, "corrupt"), (("locked", "busy"), "locked"), ) @@ -200,7 +205,8 @@ def classify_persistence_error(exc_or_str) -> str: matches: "locked" = busy, retry; "disk" = full/read-only/permissions; "compression" = a live lease refused the write; "compression_closed" = adopt the rotated session id; "turn_lease" = fencing, not storage; "corrupt" = - file damage (repair path, not disk space); "replaced" = stop writing.""" + file damage (repair path, not disk space); "replaced" = main-file replacement; + "deleted_wal" = a retired sidecar generation requiring capture inspection.""" if exc_or_str is None: return "unknown" # Lease refusals contain neither "locked" nor "busy": match by type first, diff --git a/tests/hermes_state/test_deleted_wal_generation_guard.py b/tests/hermes_state/test_deleted_wal_generation_guard.py index 8a936b55c6..d7e710de87 100644 --- a/tests/hermes_state/test_deleted_wal_generation_guard.py +++ b/tests/hermes_state/test_deleted_wal_generation_guard.py @@ -20,8 +20,8 @@ import hermes_state_dbfile import hermes_state_readpool import hermes_state_wal from hermes_state import ( - DeletedWalGenerationError, SessionDB, _close_time_checkpoint_configurable, classify_persistence_error, - refuse_deleted_wal_generation, + DeletedWalGenerationError, SessionDB, StateDbReplacedError, _close_time_checkpoint_configurable, + classify_persistence_error, refuse_deleted_wal_generation, ) from hermes_state_dbfile import _pread_db_header, iter_deleted_sqlite_sidecar_holders from tests.hermes_state._wal_generation_harness import ( @@ -35,13 +35,17 @@ def force_wal(monkeypatch): pin_wal(monkeypatch) -def test_classify_deleted_wal_is_replaced_not_disk(): - err = DeletedWalGenerationError( +def test_classify_deleted_wal_separately_from_main_file_replacement(): + message = ( "FATAL: a live process holds a deleted state.db-wal or state.db-shm " "inode while the path names a different (or missing) generation." ) - assert classify_persistence_error(err) == "replaced" - assert classify_persistence_error(str(err)) == "replaced" + assert classify_persistence_error(DeletedWalGenerationError(message)) == "deleted_wal" + assert classify_persistence_error(message) == "deleted_wal" + + replaced = StateDbReplacedError("state.db was replaced underneath this process") + assert classify_persistence_error(replaced) == "replaced" + assert classify_persistence_error(str(replaced)) == "replaced" def test_iter_holders_empty_on_non_linux(monkeypatch, tmp_path): diff --git a/tests/run_agent/test_turn_completion_explainer.py b/tests/run_agent/test_turn_completion_explainer.py index f4cd12aa20..a4b624e401 100644 --- a/tests/run_agent/test_turn_completion_explainer.py +++ b/tests/run_agent/test_turn_completion_explainer.py @@ -184,6 +184,21 @@ def test_explanation_persistence_replaced_cause_forbids_inplace_repair(): assert "full disk" not in lower +def test_deleted_wal_cause_is_enumerated_and_points_to_retired_capture(): + from hermes_state_errors import PERSISTENCE_ERROR_CAUSES + + out = AIAgent._format_turn_completion_explanation( + "session_persistence_failed", "deleted_wal" + ).lower() + assert "deleted_wal" in PERSISTENCE_ERROR_CAUSES + assert "retired-wal-*/manifest.json" in out + assert "manifest.main.mode" in out + assert "sessions recover" in out and "--inspect-only" in out + assert "header_only" in out and "does not contain a copied state.db" in out + assert "check the logs for whether" in out + assert "restore the intended state.db" not in out + + def test_explanation_persistence_unknown_cause_is_neutral(): """None/'unknown' cause must not claim disk-full — point at diagnostics.""" for cause in (None, "unknown"): @@ -336,6 +351,7 @@ def test_persistence_error_causes_tuple_matches_classifier(): "Session turn lease lost; refusing transcript write for 'abc'", "database disk image is malformed", "FATAL: state.db was replaced underneath the gateway", + "FATAL: a live process holds a deleted state.db-wal or state.db-shm inode.", "database or disk is full", "something else entirely", None,