From 24f346ee7770c8d3a7ffded11a4c500fb953f4b4 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sat, 1 Aug 2026 00:27:17 -0500 Subject: [PATCH] fix(gateway): fail prompt.submit when session storage hits a full disk Disk-full / ENOSPC / SQLITE_FULL on first-message session persist used to be swallowed as a debug log while prompt.submit still returned streaming, so the send vanished with no error. Re-raise those failures, return a real RPC error, and stamp session_persistence_failed turns with error so clients get a terminal error frame. --- agent/turn_finalizer.py | 7 +++++++ hermes_state.py | 31 +++++++++++++++++++++++++++++ run_agent.py | 4 ++-- tests/state/test_disk_full_error.py | 30 ++++++++++++++++++++++++++++ tui_gateway/methods_prompt.py | 30 ++++++++++++++++++++++++---- tui_gateway/server.py | 15 ++++++++++++-- 6 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 tests/state/test_disk_full_error.py diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index 86f3f50992..f064d9451a 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -647,6 +647,13 @@ def finalize_turn( } if agent._tool_guardrail_halt_decision is not None: result["guardrail"] = agent._tool_guardrail_halt_decision.to_metadata() + # Persistence failures already set failed=True + an explanation in + # final_response; also stamp `error` so gateway surfaces status="error" + # (and desktop can toast disk-full) instead of a quiet complete frame. + if failed and str(_turn_exit_reason) == "session_persistence_failed": + result["error"] = final_response or ( + "session storage could not be written — free disk space and try again" + ) # Surface any post-loop cleanup failures so the caller can distinguish a # clean turn from one whose trajectory/session/resource teardown raised # (the response is still returned either way — #8049). diff --git a/hermes_state.py b/hermes_state.py index 16473d9e3b..cfadc7c099 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -16,6 +16,7 @@ Key design decisions: import asyncio import atexit +import errno import json import logging import os @@ -939,6 +940,36 @@ def is_malformed_db_error(exc: BaseException) -> bool: return any(marker in str(exc).lower() for marker in _MALFORMED_SCHEMA_MARKERS) +# Markers that mean the host filesystem cannot accept another write. Kept as +# plain substrings so OSError, sqlite3.OperationalError, and wrapped RPC +# error strings all match the same helper. +_DISK_FULL_MARKERS = ( + "no space left on device", + "not enough space", + "database or disk is full", # SQLITE_FULL + "disk full", + "full disk", + "enospc", +) + + +def is_disk_full_error(exc: BaseException | str | None) -> bool: + """True when *exc* (or a stringified error) is a disk-full / ENOSPC failure. + + Covers: + * ``OSError`` with ``errno.ENOSPC`` + * SQLite ``OperationalError: database or disk is full`` (SQLITE_FULL) + * Plain English / errno strings that survive RPC wrapping + """ + if exc is None: + return False + if isinstance(exc, OSError) and getattr(exc, "errno", None) == errno.ENOSPC: + return True + text = exc if isinstance(exc, str) else str(exc) + lowered = text.lower() + return any(marker in lowered for marker in _DISK_FULL_MARKERS) + + def _claim_repair_attempt(db_path: Path) -> bool: """Claim the one-shot repair attempt for *db_path* in this process. diff --git a/run_agent.py b/run_agent.py index a240bb2925..48afb17620 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3515,8 +3515,8 @@ class AIAgent: prefix + "the turn was stopped because session storage could not be " "written (the transcript would have been lost on restart). " - "Check disk space / permissions for the state DB, then send " - "your message again." + "This is often a full disk — free some space (or fix state.db " + "permissions), then send your message again." ) # Unknown/diagnostic-only reasons (e.g. "unknown", guardrail_halt # which already surfaces its own message) — don't second-guess. diff --git a/tests/state/test_disk_full_error.py b/tests/state/test_disk_full_error.py new file mode 100644 index 0000000000..ac4e11738c --- /dev/null +++ b/tests/state/test_disk_full_error.py @@ -0,0 +1,30 @@ +"""is_disk_full_error classifies ENOSPC / SQLITE_FULL failures.""" + +from __future__ import annotations + +import errno +import sqlite3 + +from hermes_state import is_disk_full_error + + +def test_enospc_oserror(): + assert is_disk_full_error(OSError(errno.ENOSPC, "No space left on device")) is True + + +def test_sqlite_full_operational_error(): + assert is_disk_full_error(sqlite3.OperationalError("database or disk is full")) is True + + +def test_string_markers(): + assert is_disk_full_error("disk full: session storage could not be written") is True + assert is_disk_full_error("ENOSPC writing state.db") is True + assert is_disk_full_error("This is often a full disk — free some space") is True + + +def test_unrelated_errors(): + assert is_disk_full_error(None) is False + assert is_disk_full_error(OSError(errno.EACCES, "Permission denied")) is False + assert is_disk_full_error(RuntimeError("network timeout")) is False + assert is_disk_full_error("session not found") is False + assert is_disk_full_error("session storage could not be written: permission denied") is False diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index 098b6aa4c2..8983070bbd 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -233,10 +233,32 @@ def _(rid, params: dict) -> dict: ) # Persist the DB row lazily, now that the user has actually sent a message. - _ensure_session_db_row(session) - # A branch becomes real here: copy its parent's transcript into the row so it - # resumes with full context (the agent won't persist the seed itself). - _persist_branch_seed(session) + # Disk-full must fail the RPC (not stream silently): desktop maps the error + # string to a "disk full" toast so the user knows why the send vanished. + try: + _ensure_session_db_row(session) + # A branch becomes real here: copy its parent's transcript into the row so it + # resumes with full context (the agent won't persist the seed itself). + _persist_branch_seed(session) + except Exception as exc: + from hermes_state import is_disk_full_error + + with session["history_lock"]: + session["running"] = False + session["last_active"] = time.time() + _clear_inflight_turn(session) + if is_disk_full_error(exc): + return _err( + rid, + 5070, + "disk full: session storage could not be written — free some disk space and try again", + ) + logger.warning("prompt.submit: session persist failed: %s", exc, exc_info=True) + return _err( + rid, + 5071, + f"session storage could not be written: {exc}", + ) _start_agent_build(sid, session) def run_after_agent_ready() -> None: diff --git a/tui_gateway/server.py b/tui_gateway/server.py index b2bf7e0a5e..3ca6ed4616 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -2516,7 +2516,14 @@ def _ensure_session_db_row(session: dict) -> None: # means the launch/default profile (matches run_agent's convention). profile_name=Path(profile_home).name if profile_home else None, ) - except Exception: + except Exception as exc: + # Disk-full is not a soft failure: if we swallow it here, prompt.submit + # returns {"status":"streaming"} and the user's message vanishes with + # no toast. Re-raise so the submit handler can return a real RPC error. + from hermes_state import is_disk_full_error + + if is_disk_full_error(exc): + raise logger.debug("failed to persist desktop session row", exc_info=True) finally: if close_db: @@ -2559,7 +2566,11 @@ def _persist_branch_seed(session: dict) -> None: timestamp=msg.get("timestamp"), ) session["_branch_seed_persisted"] = True - except Exception: + except Exception as exc: + from hermes_state import is_disk_full_error + + if is_disk_full_error(exc): + raise logger.debug("branch seed persist failed", exc_info=True)