db339f0051
A gateway process opened state.db from ~12 call sites, each minting its own writer connection, self._lock, close-time WAL checkpoint, and token-writer thread. With N independent writers on one WAL file, one connection's close-time checkpoint could race another's growth — the lost/reordered-page-write signature across 11+ incidents (#90837). Adds hermes_state_registry.py: a process-wide, per-path, refcounted shared registry owning the writer boundary. - acquire(path): same resolved path returns the same instance (one writer connection, one lock, one token-writer thread) for every long-lived in-process caller (gateway runner, SessionStore, per-agent lazy recall, cron per-job, mirror, channel_directory, slash_commands, shutdown_flush, session_search, react_to_message, delegate, mcp_serve, auto_archive, tui_gateway). - close() on a shared instance is a NO-OP — the registry owns the lifecycle, so one caller's close can never tear down a writer other callers still hold. - Generation-aware retirement on inode change: a replaced state.db RETIRES the live generation (never lent again) but keeps it alive for existing holders; release is object-keyed so holders of the old generation drain it independently of the new one. The old generation's own write path still fails with the typed StateDbReplacedError (existing protection, unchanged). - Replacement-open failure leaves NO registry entry for the path — the next acquire retries fresh, never hands out a closed stale object. - All teardown runs OUTSIDE the registry lock: a final release's WAL checkpoint can never stall acquisition for every state.db. - close_shared_session_dbs() at gateway shutdown drains every generation (live + retired) as the final safety net. CLI one-shots, recovery flows, and read-only cross-profile opens keep using SessionDB() directly with their own close() — only long-lived in-process sites route through the registry. References #90837 (root-cause tracker stays open: the #10 EOF signature and the WAL-lifecycle A/B verdict remain under investigation there).
171 lines
5.0 KiB
Python
171 lines
5.0 KiB
Python
"""Tests for gateway/shutdown_flush.py — pending message durability (#72680)."""
|
|
|
|
import json
|
|
import os
|
|
import stat
|
|
import time
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from gateway.shutdown_flush import (
|
|
_serialise_value,
|
|
flush_pending_to_file,
|
|
recover_pending_to_db,
|
|
)
|
|
|
|
|
|
def _make_flush_dir(tmp_path: Path) -> Path:
|
|
"""Create a temp flush dir and monkeypatch _get_flush_dir to use it."""
|
|
flush_dir = tmp_path / "pending_messages"
|
|
flush_dir.mkdir(parents=True, exist_ok=True)
|
|
return flush_dir
|
|
|
|
|
|
def test_flush_writes_string_pending_to_file(tmp_path, monkeypatch):
|
|
flush_dir = _make_flush_dir(tmp_path)
|
|
monkeypatch.setattr(
|
|
"gateway.shutdown_flush._get_flush_dir", lambda: flush_dir
|
|
)
|
|
pending = {"agent:main:telegram:supergroup:123": "hello world"}
|
|
count = flush_pending_to_file(pending, reason="shutdown")
|
|
assert count == 1
|
|
files = list(flush_dir.glob("*.json"))
|
|
assert len(files) == 1
|
|
payload = json.loads(files[0].read_text(encoding="utf-8"))
|
|
assert payload["session_key"] == "agent:main:telegram:supergroup:123"
|
|
assert payload["reason"] == "shutdown"
|
|
assert payload["data"]["text"] == "hello world"
|
|
assert ":" not in files[0].name
|
|
assert "telegram" not in files[0].name
|
|
|
|
|
|
def test_flush_writes_message_event_to_file(tmp_path, monkeypatch):
|
|
flush_dir = _make_flush_dir(tmp_path)
|
|
monkeypatch.setattr(
|
|
"gateway.shutdown_flush._get_flush_dir", lambda: flush_dir
|
|
)
|
|
event = MagicMock()
|
|
event.text = "user message"
|
|
event.session_id = "20260728_120000_abc"
|
|
event.platform = "telegram"
|
|
event.sender_id = "456"
|
|
event.sender_name = "Alice"
|
|
event.reply_to = None
|
|
event.media = None
|
|
event.raw_event = None
|
|
|
|
count = flush_pending_to_file({"session_key_1": event}, reason="adapter_shutdown")
|
|
assert count == 1
|
|
files = list(flush_dir.glob("*.json"))
|
|
assert len(files) == 1
|
|
payload = json.loads(files[0].read_text(encoding="utf-8"))
|
|
assert payload["data"]["text"] == "user message"
|
|
assert payload["data"]["session_id"] == "20260728_120000_abc"
|
|
|
|
|
|
def test_recover_inserts_via_append_message_and_deletes_file(tmp_path, monkeypatch):
|
|
flush_dir = _make_flush_dir(tmp_path)
|
|
monkeypatch.setattr(
|
|
"gateway.shutdown_flush._get_flush_dir", lambda: flush_dir
|
|
)
|
|
ts = int(time.time())
|
|
# Write a flush file with session_id
|
|
payload = {
|
|
"session_key": "agent:main:telegram:supergroup:123",
|
|
"reason": "shutdown",
|
|
"ts": ts,
|
|
"data": {
|
|
"text": "lost message",
|
|
"session_id": "20260728_120000_abc",
|
|
},
|
|
}
|
|
flush_file = flush_dir / "test_session_123.json"
|
|
flush_file.write_text(json.dumps(payload), encoding="utf-8")
|
|
|
|
mock_db = MagicMock()
|
|
count = recover_pending_to_db(mock_db)
|
|
|
|
assert count == 1
|
|
mock_db.append_message.assert_called_once_with(
|
|
session_id="20260728_120000_abc",
|
|
role="user",
|
|
content="lost message",
|
|
timestamp=ts,
|
|
)
|
|
assert not flush_file.exists()
|
|
|
|
|
|
def test_recover_closes_owned_db_when_unexpected_exception_escapes(
|
|
tmp_path, monkeypatch
|
|
):
|
|
"""Owned SessionDB must close even when recovery is interrupted."""
|
|
flush_dir = _make_flush_dir(tmp_path)
|
|
monkeypatch.setattr(
|
|
"gateway.shutdown_flush._get_flush_dir", lambda: flush_dir
|
|
)
|
|
(flush_dir / "pending.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"session_key": "agent:main:telegram:123",
|
|
"data": {"text": "message", "session_id": "sid"},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
class InterruptingDB:
|
|
released = False
|
|
|
|
def append_message(self, **_kwargs):
|
|
raise KeyboardInterrupt
|
|
|
|
db = InterruptingDB()
|
|
monkeypatch.setattr("hermes_state.get_shared_session_db", lambda: db)
|
|
monkeypatch.setattr(
|
|
"hermes_state.release_or_close", lambda _: setattr(db, "released", True)
|
|
)
|
|
|
|
with pytest.raises(KeyboardInterrupt):
|
|
recover_pending_to_db()
|
|
|
|
assert db.released is True
|
|
|
|
|
|
def test_serialise_object_with_text():
|
|
obj = MagicMock()
|
|
obj.text = "msg"
|
|
obj.session_id = "sid"
|
|
obj.platform = None
|
|
obj.sender_id = None
|
|
obj.sender_name = None
|
|
obj.reply_to = None
|
|
obj.media = None
|
|
obj.raw_event = None
|
|
result = _serialise_value(obj)
|
|
assert result is not None
|
|
assert result["text"] == "msg"
|
|
assert result["session_id"] == "sid"
|
|
|
|
|
|
def test_get_flush_dir_uses_get_hermes_home(tmp_path, monkeypatch):
|
|
"""Flush dir must use get_hermes_home(), not hardcoded Path.home()."""
|
|
import gateway.shutdown_flush as mod
|
|
|
|
captured = {}
|
|
|
|
def fake_get_hermes_home():
|
|
from pathlib import Path
|
|
captured["called"] = True
|
|
return tmp_path
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_constants.get_hermes_home", fake_get_hermes_home
|
|
)
|
|
result = mod._get_flush_dir()
|
|
assert captured.get("called") is True
|
|
assert result == tmp_path / "pending_messages"
|
|
|
|
|