fix(state): distinguish retired WAL recovery guidance

Classify deleted WAL generations separately from main-file replacement and point operators at the captured-generation manifest and mode-aware recovery path.

Co-authored-by: crazyief <8566250+crazyief@users.noreply.github.com>

agent/turn_explainers.py

@@ -119,6 +119,21 @@ _PERSISTENCE_CAUSE_EXPLANATIONS: Dict[str, str] = {
         "sessions/<session_id>.jsonl and, on the gateway, "
         "pending_messages/pending-*.json."
     ),
+    "deleted_wal": (
+        "the turn was stopped because a live Hermes process held a retired "
+        "state.db-wal generation after its pathname was deleted or "
+        "replaced. Stop the gateway, dashboard, and cron writers; "
+        "do not overwrite the current state.db or delete its sidecars. "
+        "Check the logs for whether Hermes captured the retired generation, "
+        "then read the adjacent state.db.retired-wal-*/manifest.json. If "
+        "manifest.main.mode is `copied`, inspect that artifact with `hermes "
+        "sessions recover --source <state.db.retired-wal-*/state.db> "
+        "--inspect-only` before deciding whether its committed frames belong "
+        "on the current database. A `header_only` artifact is forensic and "
+        "does not contain a copied state.db to inspect. Unwritten messages "
+        "were diverted to sessions/<session_id>.jsonl and, on the gateway, "
+        "pending_messages/pending-*.json."
+    ),
     "corrupt": (
         "the turn was stopped because the state database "
         "reported structural corruption (the transcript would "

hermes_state_errors.py

@@ -74,8 +74,8 @@ def is_disk_full_error(exc: BaseException | str | None) -> bool:
 
 # Every classify_persistence_error bucket; consumers enumerate this tuple.
 PERSISTENCE_ERROR_CAUSES = (
-    "locked", "compression", "compression_closed", "turn_lease", "corrupt", "replaced", "disk",
-    "unknown",
+    "locked", "compression", "compression_closed", "turn_lease", "corrupt", "replaced",
+    "deleted_wal", "disk", "unknown",
 )
 
 
@@ -182,6 +182,9 @@ _PERSISTENCE_CAUSE_BY_TYPE = (
     (SessionTurnLeaseLostError, "turn_lease"),
     (CompressionSessionClosedError, "compression_closed"),
     (CompressionSessionBusyError, "compression"),
+    # The WAL-generation error subclasses StateDbReplacedError so existing write diversion keeps
+    # working; classify it first because its recovery artifact and operator action are different.
+    (DeletedWalGenerationError, "deleted_wal"),
     (StateDbReplacedError, "replaced"),
     (StateDbCorruptError, "corrupt"),
 )
@@ -189,7 +192,9 @@ _PERSISTENCE_CAUSE_BY_PHRASE = (
     (("turn lease",), "turn_lease"),
     (("closed by compression",), "compression_closed"),
     (("being compressed", "compression lease"), "compression"),
-    (("was replaced underneath", "deleted state.db-wal", "deleted state.db-shm"), "replaced"),
+    # RPC-wrapped errors lose their exception type; retain the same sidecar/main-file split.
+    (("deleted state.db-wal", "deleted state.db-shm"), "deleted_wal"),
+    (("was replaced underneath",), "replaced"),
     (_DB_CORRUPTION_MARKERS, "corrupt"),
     (("locked", "busy"), "locked"),
 )
@@ -200,7 +205,8 @@ def classify_persistence_error(exc_or_str) -> str:
     matches: "locked" = busy, retry; "disk" = full/read-only/permissions;
     "compression" = a live lease refused the write; "compression_closed" = adopt
     the rotated session id; "turn_lease" = fencing, not storage; "corrupt" =
-    file damage (repair path, not disk space); "replaced" = stop writing."""
+    file damage (repair path, not disk space); "replaced" = main-file replacement;
+    "deleted_wal" = a retired sidecar generation requiring capture inspection."""
     if exc_or_str is None:
         return "unknown"
     # Lease refusals contain neither "locked" nor "busy": match by type first,

tests/hermes_state/test_deleted_wal_generation_guard.py

@@ -20,8 +20,8 @@ import hermes_state_dbfile
 import hermes_state_readpool
 import hermes_state_wal
 from hermes_state import (
-    DeletedWalGenerationError, SessionDB, _close_time_checkpoint_configurable, classify_persistence_error,
-    refuse_deleted_wal_generation,
+    DeletedWalGenerationError, SessionDB, StateDbReplacedError, _close_time_checkpoint_configurable,
+    classify_persistence_error, refuse_deleted_wal_generation,
 )
 from hermes_state_dbfile import _pread_db_header, iter_deleted_sqlite_sidecar_holders
 from tests.hermes_state._wal_generation_harness import (
@@ -35,13 +35,17 @@ def force_wal(monkeypatch):
     pin_wal(monkeypatch)
 
 
-def test_classify_deleted_wal_is_replaced_not_disk():
-    err = DeletedWalGenerationError(
+def test_classify_deleted_wal_separately_from_main_file_replacement():
+    message = (
         "FATAL: a live process holds a deleted state.db-wal or state.db-shm "
         "inode while the path names a different (or missing) generation."
     )
-    assert classify_persistence_error(err) == "replaced"
-    assert classify_persistence_error(str(err)) == "replaced"
+    assert classify_persistence_error(DeletedWalGenerationError(message)) == "deleted_wal"
+    assert classify_persistence_error(message) == "deleted_wal"
+
+    replaced = StateDbReplacedError("state.db was replaced underneath this process")
+    assert classify_persistence_error(replaced) == "replaced"
+    assert classify_persistence_error(str(replaced)) == "replaced"
 
 
 def test_iter_holders_empty_on_non_linux(monkeypatch, tmp_path):

tests/run_agent/test_turn_completion_explainer.py

@@ -184,6 +184,21 @@ def test_explanation_persistence_replaced_cause_forbids_inplace_repair():
     assert "full disk" not in lower
 
 
+def test_deleted_wal_cause_is_enumerated_and_points_to_retired_capture():
+    from hermes_state_errors import PERSISTENCE_ERROR_CAUSES
+
+    out = AIAgent._format_turn_completion_explanation(
+        "session_persistence_failed", "deleted_wal"
+    ).lower()
+    assert "deleted_wal" in PERSISTENCE_ERROR_CAUSES
+    assert "retired-wal-*/manifest.json" in out
+    assert "manifest.main.mode" in out
+    assert "sessions recover" in out and "--inspect-only" in out
+    assert "header_only" in out and "does not contain a copied state.db" in out
+    assert "check the logs for whether" in out
+    assert "restore the intended state.db" not in out
+
+
 def test_explanation_persistence_unknown_cause_is_neutral():
     """None/'unknown' cause must not claim disk-full — point at diagnostics."""
     for cause in (None, "unknown"):
@@ -336,6 +351,7 @@ def test_persistence_error_causes_tuple_matches_classifier():
         "Session turn lease lost; refusing transcript write for 'abc'",
         "database disk image is malformed",
         "FATAL: state.db was replaced underneath the gateway",
+        "FATAL: a live process holds a deleted state.db-wal or state.db-shm inode.",
         "database or disk is full",
         "something else entirely",
         None,