fix(gateway): fail prompt.submit when session storage hits a full disk

Disk-full / ENOSPC / SQLITE_FULL on first-message session persist used to be
swallowed as a debug log while prompt.submit still returned streaming, so the
send vanished with no error. Re-raise those failures, return a real RPC error,
and stamp session_persistence_failed turns with error so clients get a terminal
error frame.
This commit is contained in:
Brooklyn Nicholson
2026-08-01 00:27:17 -05:00
parent 0454b370d1
commit 24f346ee77
6 changed files with 109 additions and 8 deletions
+7
View File
@@ -647,6 +647,13 @@ def finalize_turn(
}
if agent._tool_guardrail_halt_decision is not None:
result["guardrail"] = agent._tool_guardrail_halt_decision.to_metadata()
# Persistence failures already set failed=True + an explanation in
# final_response; also stamp `error` so gateway surfaces status="error"
# (and desktop can toast disk-full) instead of a quiet complete frame.
if failed and str(_turn_exit_reason) == "session_persistence_failed":
result["error"] = final_response or (
"session storage could not be written — free disk space and try again"
)
# Surface any post-loop cleanup failures so the caller can distinguish a
# clean turn from one whose trajectory/session/resource teardown raised
# (the response is still returned either way — #8049).
+31
View File
@@ -16,6 +16,7 @@ Key design decisions:
import asyncio
import atexit
import errno
import json
import logging
import os
@@ -939,6 +940,36 @@ def is_malformed_db_error(exc: BaseException) -> bool:
return any(marker in str(exc).lower() for marker in _MALFORMED_SCHEMA_MARKERS)
# Markers that mean the host filesystem cannot accept another write. Kept as
# plain substrings so OSError, sqlite3.OperationalError, and wrapped RPC
# error strings all match the same helper.
_DISK_FULL_MARKERS = (
"no space left on device",
"not enough space",
"database or disk is full", # SQLITE_FULL
"disk full",
"full disk",
"enospc",
)
def is_disk_full_error(exc: BaseException | str | None) -> bool:
"""True when *exc* (or a stringified error) is a disk-full / ENOSPC failure.
Covers:
* ``OSError`` with ``errno.ENOSPC``
* SQLite ``OperationalError: database or disk is full`` (SQLITE_FULL)
* Plain English / errno strings that survive RPC wrapping
"""
if exc is None:
return False
if isinstance(exc, OSError) and getattr(exc, "errno", None) == errno.ENOSPC:
return True
text = exc if isinstance(exc, str) else str(exc)
lowered = text.lower()
return any(marker in lowered for marker in _DISK_FULL_MARKERS)
def _claim_repair_attempt(db_path: Path) -> bool:
"""Claim the one-shot repair attempt for *db_path* in this process.
+2 -2
View File
@@ -3515,8 +3515,8 @@ class AIAgent:
prefix
+ "the turn was stopped because session storage could not be "
"written (the transcript would have been lost on restart). "
"Check disk space / permissions for the state DB, then send "
"your message again."
"This is often a full disk — free some space (or fix state.db "
"permissions), then send your message again."
)
# Unknown/diagnostic-only reasons (e.g. "unknown", guardrail_halt
# which already surfaces its own message) — don't second-guess.
+30
View File
@@ -0,0 +1,30 @@
"""is_disk_full_error classifies ENOSPC / SQLITE_FULL failures."""
from __future__ import annotations
import errno
import sqlite3
from hermes_state import is_disk_full_error
def test_enospc_oserror():
assert is_disk_full_error(OSError(errno.ENOSPC, "No space left on device")) is True
def test_sqlite_full_operational_error():
assert is_disk_full_error(sqlite3.OperationalError("database or disk is full")) is True
def test_string_markers():
assert is_disk_full_error("disk full: session storage could not be written") is True
assert is_disk_full_error("ENOSPC writing state.db") is True
assert is_disk_full_error("This is often a full disk — free some space") is True
def test_unrelated_errors():
assert is_disk_full_error(None) is False
assert is_disk_full_error(OSError(errno.EACCES, "Permission denied")) is False
assert is_disk_full_error(RuntimeError("network timeout")) is False
assert is_disk_full_error("session not found") is False
assert is_disk_full_error("session storage could not be written: permission denied") is False
+26 -4
View File
@@ -233,10 +233,32 @@ def _(rid, params: dict) -> dict:
)
# Persist the DB row lazily, now that the user has actually sent a message.
_ensure_session_db_row(session)
# A branch becomes real here: copy its parent's transcript into the row so it
# resumes with full context (the agent won't persist the seed itself).
_persist_branch_seed(session)
# Disk-full must fail the RPC (not stream silently): desktop maps the error
# string to a "disk full" toast so the user knows why the send vanished.
try:
_ensure_session_db_row(session)
# A branch becomes real here: copy its parent's transcript into the row so it
# resumes with full context (the agent won't persist the seed itself).
_persist_branch_seed(session)
except Exception as exc:
from hermes_state import is_disk_full_error
with session["history_lock"]:
session["running"] = False
session["last_active"] = time.time()
_clear_inflight_turn(session)
if is_disk_full_error(exc):
return _err(
rid,
5070,
"disk full: session storage could not be written — free some disk space and try again",
)
logger.warning("prompt.submit: session persist failed: %s", exc, exc_info=True)
return _err(
rid,
5071,
f"session storage could not be written: {exc}",
)
_start_agent_build(sid, session)
def run_after_agent_ready() -> None:
+13 -2
View File
@@ -2516,7 +2516,14 @@ def _ensure_session_db_row(session: dict) -> None:
# means the launch/default profile (matches run_agent's convention).
profile_name=Path(profile_home).name if profile_home else None,
)
except Exception:
except Exception as exc:
# Disk-full is not a soft failure: if we swallow it here, prompt.submit
# returns {"status":"streaming"} and the user's message vanishes with
# no toast. Re-raise so the submit handler can return a real RPC error.
from hermes_state import is_disk_full_error
if is_disk_full_error(exc):
raise
logger.debug("failed to persist desktop session row", exc_info=True)
finally:
if close_db:
@@ -2559,7 +2566,11 @@ def _persist_branch_seed(session: dict) -> None:
timestamp=msg.get("timestamp"),
)
session["_branch_seed_persisted"] = True
except Exception:
except Exception as exc:
from hermes_state import is_disk_full_error
if is_disk_full_error(exc):
raise
logger.debug("branch seed persist failed", exc_info=True)