Files
hermes-agent/tests/gateway/test_shutdown_flush.py
T
joaomarcos 39e480c051 fix(state): close leaked SessionDB connections on exception paths (#83226)
SessionDB could leave native SQLite handles open when construction failed
partway through schema/pragma/FTS/repair/lock/interrupt handling. Other
short-lived callers (MCP reads/polling, session search, reactions, trace
upload, insights, shutdown recovery) opened temporary SessionDB handles
without a complete ownership boundary. API-server profile caches and
RetainDB shutdown had similar late-close races. Under sustained load this
exhausted file descriptors (EMFILE).

- Close partially initialized SessionDB connections on every constructor
  exception path via a finally block guarded by an initialization-complete
  flag.
- Close temporary/cross-profile SessionDB handles in finally blocks across
  CLI, MCP, search, trace, reactions, insights, and recovery paths.
- Add API-server per-profile cache ownership and disconnect cleanup.
- Make RetainDB writer-queue shutdown exception-safe: track connections per
  thread, close on worker exit, reject new enqueues after shutdown starts,
  and sweep any connections left by short-lived threads.
- Add regression coverage for constructor failures, worker-thread readers,
  API disconnect failures, shutdown recovery, RetainDB late enqueue, and
  foreign-loop async clients.

Salvage notes: the original PR's per-thread WAL-reader ownership changes
were superseded by main's read-connection pool (permits + checkout/return);
its cron timeout-abandon fix is credited separately to #72822's earlier
identical fix.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-14 21:41:26 -07:00

171 lines
4.9 KiB
Python

"""Tests for gateway/shutdown_flush.py — pending message durability (#72680)."""
import json
import os
import stat
import time
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from gateway.shutdown_flush import (
_serialise_value,
flush_pending_to_file,
recover_pending_to_db,
)
def _make_flush_dir(tmp_path: Path) -> Path:
"""Create a temp flush dir and monkeypatch _get_flush_dir to use it."""
flush_dir = tmp_path / "pending_messages"
flush_dir.mkdir(parents=True, exist_ok=True)
return flush_dir
def test_flush_writes_string_pending_to_file(tmp_path, monkeypatch):
flush_dir = _make_flush_dir(tmp_path)
monkeypatch.setattr(
"gateway.shutdown_flush._get_flush_dir", lambda: flush_dir
)
pending = {"agent:main:telegram:supergroup:123": "hello world"}
count = flush_pending_to_file(pending, reason="shutdown")
assert count == 1
files = list(flush_dir.glob("*.json"))
assert len(files) == 1
payload = json.loads(files[0].read_text(encoding="utf-8"))
assert payload["session_key"] == "agent:main:telegram:supergroup:123"
assert payload["reason"] == "shutdown"
assert payload["data"]["text"] == "hello world"
assert ":" not in files[0].name
assert "telegram" not in files[0].name
def test_flush_writes_message_event_to_file(tmp_path, monkeypatch):
flush_dir = _make_flush_dir(tmp_path)
monkeypatch.setattr(
"gateway.shutdown_flush._get_flush_dir", lambda: flush_dir
)
event = MagicMock()
event.text = "user message"
event.session_id = "20260728_120000_abc"
event.platform = "telegram"
event.sender_id = "456"
event.sender_name = "Alice"
event.reply_to = None
event.media = None
event.raw_event = None
count = flush_pending_to_file({"session_key_1": event}, reason="adapter_shutdown")
assert count == 1
files = list(flush_dir.glob("*.json"))
assert len(files) == 1
payload = json.loads(files[0].read_text(encoding="utf-8"))
assert payload["data"]["text"] == "user message"
assert payload["data"]["session_id"] == "20260728_120000_abc"
def test_recover_inserts_via_append_message_and_deletes_file(tmp_path, monkeypatch):
flush_dir = _make_flush_dir(tmp_path)
monkeypatch.setattr(
"gateway.shutdown_flush._get_flush_dir", lambda: flush_dir
)
ts = int(time.time())
# Write a flush file with session_id
payload = {
"session_key": "agent:main:telegram:supergroup:123",
"reason": "shutdown",
"ts": ts,
"data": {
"text": "lost message",
"session_id": "20260728_120000_abc",
},
}
flush_file = flush_dir / "test_session_123.json"
flush_file.write_text(json.dumps(payload), encoding="utf-8")
mock_db = MagicMock()
count = recover_pending_to_db(mock_db)
assert count == 1
mock_db.append_message.assert_called_once_with(
session_id="20260728_120000_abc",
role="user",
content="lost message",
timestamp=ts,
)
assert not flush_file.exists()
def test_recover_closes_owned_db_when_unexpected_exception_escapes(
tmp_path, monkeypatch
):
"""Owned SessionDB must close even when recovery is interrupted."""
flush_dir = _make_flush_dir(tmp_path)
monkeypatch.setattr(
"gateway.shutdown_flush._get_flush_dir", lambda: flush_dir
)
(flush_dir / "pending.json").write_text(
json.dumps(
{
"session_key": "agent:main:telegram:123",
"data": {"text": "message", "session_id": "sid"},
}
),
encoding="utf-8",
)
class InterruptingDB:
closed = False
def append_message(self, **_kwargs):
raise KeyboardInterrupt
def close(self):
self.closed = True
db = InterruptingDB()
monkeypatch.setattr("hermes_state.SessionDB", lambda: db)
with pytest.raises(KeyboardInterrupt):
recover_pending_to_db()
assert db.closed is True
def test_serialise_object_with_text():
obj = MagicMock()
obj.text = "msg"
obj.session_id = "sid"
obj.platform = None
obj.sender_id = None
obj.sender_name = None
obj.reply_to = None
obj.media = None
obj.raw_event = None
result = _serialise_value(obj)
assert result is not None
assert result["text"] == "msg"
assert result["session_id"] == "sid"
def test_get_flush_dir_uses_get_hermes_home(tmp_path, monkeypatch):
"""Flush dir must use get_hermes_home(), not hardcoded Path.home()."""
import gateway.shutdown_flush as mod
captured = {}
def fake_get_hermes_home():
from pathlib import Path
captured["called"] = True
return tmp_path
monkeypatch.setattr(
"hermes_constants.get_hermes_home", fake_get_hermes_home
)
result = mod._get_flush_dir()
assert captured.get("called") is True
assert result == tmp_path / "pending_messages"