fix(state): classify structural DB corruption as its own persistence cause

'database disk image is malformed' contains the word 'disk', so
classify_persistence_error bucketed SQLITE_CORRUPT / SQLITE_NOTADB
failures as 'disk' and the turn-completion explainer told users to
free disk space for a structurally damaged state.db (the #77386-family
misdiagnosis, reproduced in the v0.20.0 malformed-DB incident report).

- hermes_state: new 'corrupt' bucket in PERSISTENCE_ERROR_CAUSES,
  matched via _DB_CORRUPTION_MARKERS BEFORE the locked/disk buckets
- run_agent: explainer text for 'corrupt' points at hermes doctor and
  explicitly says freeing space will not help
- cron explainer-variant suppression picks the new variant up
  automatically (it iterates PERSISTENCE_ERROR_CAUSES)
This commit is contained in:
Teknium
2026-08-16 10:24:45 -07:00
parent 21c1fe6686
commit 06b9141109
4 changed files with 75 additions and 1 deletions
+1 -1
View File
@@ -737,7 +737,7 @@ def finalize_turn(
"health (`hermes doctor`), then send your message again"
)
# Machine-readable cause for the gateway/desktop: exactly
# 'session_persistence_failed:<locked|compression|turn_lease|disk|unknown>'.
# 'session_persistence_failed:<locked|compression|turn_lease|corrupt|disk|unknown>'.
# Never clobber a failure_reason another path already stamped.
if "failure_reason" not in result:
_cause = getattr(agent, "_last_persistence_error_cause", None)
+26
View File
@@ -1605,11 +1605,27 @@ PERSISTENCE_ERROR_CAUSES = (
"compression",
"compression_closed",
"turn_lease",
"corrupt",
"disk",
"unknown",
)
# Markers that mean the database FILE itself is structurally damaged. Kept
# as plain substrings so sqlite3.DatabaseError, wrapped RPC strings, and
# logged message text all match the same helper. NOTE: "database disk image
# is malformed" contains the word "disk", so this check MUST run before the
# disk-full/readonly bucket in classify_persistence_error — otherwise real
# B-tree corruption gets reported to the user as "free some disk space"
# (the misdiagnosis documented on #77386).
_DB_CORRUPTION_MARKERS = (
"malformed", # "database disk image is malformed" (SQLITE_CORRUPT)
"file is not a database", # SQLITE_NOTADB (also connection-level poisoning)
"not a database",
"database corruption",
)
def classify_persistence_error(exc_or_str) -> str:
"""Classify a session-persistence failure into a coarse cause bucket.
@@ -1631,6 +1647,10 @@ def classify_persistence_error(exc_or_str) -> str:
* ``"turn_lease"`` — a presented session-turn-lease holder no longer
owns the conversation (expired, released, or reclaimed); fail-fast
fencing, not a storage fault.
* ``"corrupt"`` — the database file itself is structurally damaged
(``database disk image is malformed`` / SQLITE_NOTADB). Distinct from
``"disk"``: freeing space cannot help, the user needs the repair path
(``hermes doctor`` / automatic schema surgery).
* ``"disk"`` — disk full / read-only / permission-shaped failures
(delegates the disk-full patterns to :func:`is_disk_full_error` so the
two classifiers can never drift apart — e.g. ENOSPC).
@@ -1656,6 +1676,12 @@ def classify_persistence_error(exc_or_str) -> str:
return "compression_closed"
if "being compressed" in text or "compression lease" in text:
return "compression"
# Structural corruption BEFORE the lock and disk buckets: "database disk
# image is malformed" contains "disk" (and some wrapped corruption
# strings mention "locked" recovery attempts), so later buckets would
# steal it and misdiagnose damage as space/contention.
if any(marker in text for marker in _DB_CORRUPTION_MARKERS):
return "corrupt"
if (
"locked" in text
or "busy" in text
+9
View File
@@ -3865,6 +3865,15 @@ class AIAgent:
"database). Your message should already be saved — "
"please send it again in a moment."
)
if cause == "corrupt":
return (
prefix
+ "the turn was stopped because the state database "
"reported structural corruption (the transcript would "
"have been lost on restart). Freeing disk space will "
"not help — run `hermes doctor` to repair the state "
"database, then send your message again."
)
if cause == "disk":
return (
prefix
@@ -140,6 +140,20 @@ def test_explanation_persistence_disk_cause_keeps_disk_wording():
assert "free some space" in lower or "disk space" in lower
def test_explanation_persistence_corrupt_cause_never_says_free_space():
"""Structural corruption must point at the repair path, not disk space
(the #77386-family misdiagnosis: 'database disk image is malformed'
rendered as 'this is often a full disk')."""
out = AIAgent._format_turn_completion_explanation(
"session_persistence_failed", "corrupt"
)
lower = out.lower()
assert "corrupt" in lower
assert "hermes doctor" in lower
assert "free some space" not in lower
assert "full disk" not in lower
def test_explanation_persistence_unknown_cause_is_neutral():
"""None/'unknown' cause must not claim disk-full — point at diagnostics."""
for cause in (None, "unknown"):
@@ -199,6 +213,30 @@ def test_classify_persistence_error_categories():
assert classify_persistence_error("") == "unknown"
def test_classify_persistence_error_corruption_beats_disk_bucket():
"""'database disk image is malformed' contains the word 'disk', so
without an explicit corruption bucket it classified as 'disk' and the
user was told to free space for a structurally damaged file (#77386
comment thread, v0.20.0 malformed-DB incident)."""
import sqlite3
from hermes_state import classify_persistence_error
assert classify_persistence_error(
sqlite3.DatabaseError("database disk image is malformed")
) == "corrupt"
assert classify_persistence_error(
"database disk image is malformed"
) == "corrupt"
assert classify_persistence_error(
sqlite3.DatabaseError("file is not a database")
) == "corrupt"
assert classify_persistence_error("malformed database schema") == "corrupt"
# Genuine disk-space failures must keep classifying as 'disk'.
assert classify_persistence_error("database or disk is full") == "disk"
assert classify_persistence_error("disk I/O error") == "disk"
def test_classify_persistence_error_reuses_disk_full_markers():
"""The disk bucket delegates to hermes_state.is_disk_full_error, so
every marker that helper recognizes (ENOSPC, 'not enough space', ...)
@@ -266,6 +304,7 @@ def test_persistence_error_causes_tuple_matches_classifier():
"database is locked",
"Session 'abc' is being compressed by another writer",
"Session turn lease lost; refusing transcript write for 'abc'",
"database disk image is malformed",
"database or disk is full",
"something else entirely",
None,