fix(state): classify structural DB corruption as its own persistence cause
'database disk image is malformed' contains the word 'disk', so classify_persistence_error bucketed SQLITE_CORRUPT / SQLITE_NOTADB failures as 'disk' and the turn-completion explainer told users to free disk space for a structurally damaged state.db (the #77386-family misdiagnosis, reproduced in the v0.20.0 malformed-DB incident report). - hermes_state: new 'corrupt' bucket in PERSISTENCE_ERROR_CAUSES, matched via _DB_CORRUPTION_MARKERS BEFORE the locked/disk buckets - run_agent: explainer text for 'corrupt' points at hermes doctor and explicitly says freeing space will not help - cron explainer-variant suppression picks the new variant up automatically (it iterates PERSISTENCE_ERROR_CAUSES)
This commit is contained in:
@@ -737,7 +737,7 @@ def finalize_turn(
|
||||
"health (`hermes doctor`), then send your message again"
|
||||
)
|
||||
# Machine-readable cause for the gateway/desktop: exactly
|
||||
# 'session_persistence_failed:<locked|compression|turn_lease|disk|unknown>'.
|
||||
# 'session_persistence_failed:<locked|compression|turn_lease|corrupt|disk|unknown>'.
|
||||
# Never clobber a failure_reason another path already stamped.
|
||||
if "failure_reason" not in result:
|
||||
_cause = getattr(agent, "_last_persistence_error_cause", None)
|
||||
|
||||
@@ -1605,11 +1605,27 @@ PERSISTENCE_ERROR_CAUSES = (
|
||||
"compression",
|
||||
"compression_closed",
|
||||
"turn_lease",
|
||||
"corrupt",
|
||||
"disk",
|
||||
"unknown",
|
||||
)
|
||||
|
||||
|
||||
# Markers that mean the database FILE itself is structurally damaged. Kept
|
||||
# as plain substrings so sqlite3.DatabaseError, wrapped RPC strings, and
|
||||
# logged message text all match the same helper. NOTE: "database disk image
|
||||
# is malformed" contains the word "disk", so this check MUST run before the
|
||||
# disk-full/readonly bucket in classify_persistence_error — otherwise real
|
||||
# B-tree corruption gets reported to the user as "free some disk space"
|
||||
# (the misdiagnosis documented on #77386).
|
||||
_DB_CORRUPTION_MARKERS = (
|
||||
"malformed", # "database disk image is malformed" (SQLITE_CORRUPT)
|
||||
"file is not a database", # SQLITE_NOTADB (also connection-level poisoning)
|
||||
"not a database",
|
||||
"database corruption",
|
||||
)
|
||||
|
||||
|
||||
def classify_persistence_error(exc_or_str) -> str:
|
||||
"""Classify a session-persistence failure into a coarse cause bucket.
|
||||
|
||||
@@ -1631,6 +1647,10 @@ def classify_persistence_error(exc_or_str) -> str:
|
||||
* ``"turn_lease"`` — a presented session-turn-lease holder no longer
|
||||
owns the conversation (expired, released, or reclaimed); fail-fast
|
||||
fencing, not a storage fault.
|
||||
* ``"corrupt"`` — the database file itself is structurally damaged
|
||||
(``database disk image is malformed`` / SQLITE_NOTADB). Distinct from
|
||||
``"disk"``: freeing space cannot help, the user needs the repair path
|
||||
(``hermes doctor`` / automatic schema surgery).
|
||||
* ``"disk"`` — disk full / read-only / permission-shaped failures
|
||||
(delegates the disk-full patterns to :func:`is_disk_full_error` so the
|
||||
two classifiers can never drift apart — e.g. ENOSPC).
|
||||
@@ -1656,6 +1676,12 @@ def classify_persistence_error(exc_or_str) -> str:
|
||||
return "compression_closed"
|
||||
if "being compressed" in text or "compression lease" in text:
|
||||
return "compression"
|
||||
# Structural corruption BEFORE the lock and disk buckets: "database disk
|
||||
# image is malformed" contains "disk" (and some wrapped corruption
|
||||
# strings mention "locked" recovery attempts), so later buckets would
|
||||
# steal it and misdiagnose damage as space/contention.
|
||||
if any(marker in text for marker in _DB_CORRUPTION_MARKERS):
|
||||
return "corrupt"
|
||||
if (
|
||||
"locked" in text
|
||||
or "busy" in text
|
||||
|
||||
@@ -3865,6 +3865,15 @@ class AIAgent:
|
||||
"database). Your message should already be saved — "
|
||||
"please send it again in a moment."
|
||||
)
|
||||
if cause == "corrupt":
|
||||
return (
|
||||
prefix
|
||||
+ "the turn was stopped because the state database "
|
||||
"reported structural corruption (the transcript would "
|
||||
"have been lost on restart). Freeing disk space will "
|
||||
"not help — run `hermes doctor` to repair the state "
|
||||
"database, then send your message again."
|
||||
)
|
||||
if cause == "disk":
|
||||
return (
|
||||
prefix
|
||||
|
||||
@@ -140,6 +140,20 @@ def test_explanation_persistence_disk_cause_keeps_disk_wording():
|
||||
assert "free some space" in lower or "disk space" in lower
|
||||
|
||||
|
||||
def test_explanation_persistence_corrupt_cause_never_says_free_space():
|
||||
"""Structural corruption must point at the repair path, not disk space
|
||||
(the #77386-family misdiagnosis: 'database disk image is malformed'
|
||||
rendered as 'this is often a full disk')."""
|
||||
out = AIAgent._format_turn_completion_explanation(
|
||||
"session_persistence_failed", "corrupt"
|
||||
)
|
||||
lower = out.lower()
|
||||
assert "corrupt" in lower
|
||||
assert "hermes doctor" in lower
|
||||
assert "free some space" not in lower
|
||||
assert "full disk" not in lower
|
||||
|
||||
|
||||
def test_explanation_persistence_unknown_cause_is_neutral():
|
||||
"""None/'unknown' cause must not claim disk-full — point at diagnostics."""
|
||||
for cause in (None, "unknown"):
|
||||
@@ -199,6 +213,30 @@ def test_classify_persistence_error_categories():
|
||||
assert classify_persistence_error("") == "unknown"
|
||||
|
||||
|
||||
def test_classify_persistence_error_corruption_beats_disk_bucket():
|
||||
"""'database disk image is malformed' contains the word 'disk', so
|
||||
without an explicit corruption bucket it classified as 'disk' and the
|
||||
user was told to free space for a structurally damaged file (#77386
|
||||
comment thread, v0.20.0 malformed-DB incident)."""
|
||||
import sqlite3
|
||||
|
||||
from hermes_state import classify_persistence_error
|
||||
|
||||
assert classify_persistence_error(
|
||||
sqlite3.DatabaseError("database disk image is malformed")
|
||||
) == "corrupt"
|
||||
assert classify_persistence_error(
|
||||
"database disk image is malformed"
|
||||
) == "corrupt"
|
||||
assert classify_persistence_error(
|
||||
sqlite3.DatabaseError("file is not a database")
|
||||
) == "corrupt"
|
||||
assert classify_persistence_error("malformed database schema") == "corrupt"
|
||||
# Genuine disk-space failures must keep classifying as 'disk'.
|
||||
assert classify_persistence_error("database or disk is full") == "disk"
|
||||
assert classify_persistence_error("disk I/O error") == "disk"
|
||||
|
||||
|
||||
def test_classify_persistence_error_reuses_disk_full_markers():
|
||||
"""The disk bucket delegates to hermes_state.is_disk_full_error, so
|
||||
every marker that helper recognizes (ENOSPC, 'not enough space', ...)
|
||||
@@ -266,6 +304,7 @@ def test_persistence_error_causes_tuple_matches_classifier():
|
||||
"database is locked",
|
||||
"Session 'abc' is being compressed by another writer",
|
||||
"Session turn lease lost; refusing transcript write for 'abc'",
|
||||
"database disk image is malformed",
|
||||
"database or disk is full",
|
||||
"something else entirely",
|
||||
None,
|
||||
|
||||
Reference in New Issue
Block a user