From 94b10eccf5d444a272bbd52d8e662d8f8aec843c Mon Sep 17 00:00:00 2001 From: And Date: Sat, 11 Jul 2026 21:04:05 +0300 Subject: [PATCH] feat(memory/hindsight): order background prefetch after pending retains Async retain already keeps the memory WRITE off the reply path (writes drain on the single writer thread while the user gets their response immediately). This closes the remaining retain/prefetch race: the next turn's warm prefetch runs on its own thread and could recall BEFORE the just-enqueued retain write lands, silently dropping the latest turn from recall. - The background prefetch now waits (bounded) for pending retains to drain before recalling, so warmed context includes the just-completed turn. - The wait runs only on the background prefetch thread, never the reply path, so it adds zero latency to the user's response and loses no writes. - Bounded by prefetch_retain_drain_timeout (default 10s) and polls unfinished_tasks so a wedged write can't hang the prefetch. - New config keys: prefetch_waits_for_retain (default true), prefetch_retain_drain_timeout (default 10.0). --- plugins/memory/hindsight/__init__.py | 47 +++++++++++++++++++ .../plugins/memory/test_hindsight_provider.py | 41 ++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/plugins/memory/hindsight/__init__.py b/plugins/memory/hindsight/__init__.py index 03666f2c41..b64992a425 100644 --- a/plugins/memory/hindsight/__init__.py +++ b/plugins/memory/hindsight/__init__.py @@ -39,6 +39,7 @@ import os import queue import sys import threading +import time from datetime import datetime, timezone from typing import Any, Dict, List @@ -742,6 +743,14 @@ class HindsightMemoryProvider(MemoryProvider): self._auto_retain = True self._retain_every_n_turns = 1 self._retain_async = True + # Async retain never blocks the reply (writes drain on the single + # writer thread). But the next turn's warm prefetch runs on its own + # thread and could read BEFORE the just-enqueued retain write lands, + # dropping the latest turn from recall. When True, the background + # prefetch waits (bounded) for pending retains to drain first, closing + # that race without putting any write on the reply path. + self._prefetch_waits_for_retain = True + self._prefetch_retain_drain_timeout = 10.0 self._retain_context = "conversation between Hermes Agent and the User" self._turn_counter = 0 self._session_turns: list[str] = [] # accumulates ALL turns for the session @@ -1058,6 +1067,8 @@ class HindsightMemoryProvider(MemoryProvider): {"key": "auto_retain", "description": "Automatically retain conversation turns", "default": True}, {"key": "retain_every_n_turns", "description": "Retain every N turns (1 = every turn)", "default": 1}, {"key": "retain_async","description": "Process retain asynchronously on the Hindsight server", "default": True}, + {"key": "prefetch_waits_for_retain", "description": "Have the background next-turn prefetch wait for pending retain writes to drain first, so recall includes the just-completed turn (runs off the reply path, adds no response latency)", "default": True}, + {"key": "prefetch_retain_drain_timeout", "description": "Max seconds the background prefetch waits for pending retains to drain before recalling anyway", "default": 10.0}, {"key": "retain_context", "description": "Context label for retained memories", "default": "conversation between Hermes Agent and the User"}, {"key": "recall_max_tokens", "description": "Maximum tokens for recall results", "default": 4096}, {"key": "recall_max_input_chars", "description": "Maximum input query length for auto-recall", "default": 800}, @@ -1162,6 +1173,32 @@ class HindsightMemoryProvider(MemoryProvider): self._sync_thread = thread thread.start() + def _wait_for_retains_drained(self, timeout: float) -> bool: + """Block up to *timeout* seconds for queued retains to finish. + + Used by the background prefetch so the next turn's recall observes + the just-completed turn's write instead of racing ahead of it. Runs + only on the background prefetch thread — never on the reply path. + + Returns True if the queue drained within the budget, False on timeout. + Polls ``unfinished_tasks`` rather than ``queue.join()`` so the wait is + bounded even if a write is wedged (the writer is best-effort). + """ + if timeout <= 0: + return self._retain_queue.unfinished_tasks == 0 + deadline = time.monotonic() + timeout + while self._retain_queue.unfinished_tasks > 0: + if self._shutting_down.is_set(): + return False + if time.monotonic() >= deadline: + logger.debug( + "Prefetch: retain drain timed out after %.1fs (%d pending)", + timeout, self._retain_queue.unfinished_tasks, + ) + return False + time.sleep(0.05) + return True + def _writer_loop(self) -> None: """Drain the retain queue serially. Exits on sentinel. @@ -1404,6 +1441,10 @@ class HindsightMemoryProvider(MemoryProvider): self._recall_prompt_preamble = self._config.get("recall_prompt_preamble", "") self._recall_max_input_chars = int(self._config.get("recall_max_input_chars", 800)) self._retain_async = self._config.get("retain_async", True) + self._prefetch_waits_for_retain = self._config.get("prefetch_waits_for_retain", True) + self._prefetch_retain_drain_timeout = float( + self._config.get("prefetch_retain_drain_timeout", 10.0) + ) _client_version = "unknown" try: @@ -1548,6 +1589,12 @@ class HindsightMemoryProvider(MemoryProvider): query = query[:self._recall_max_input_chars] def _run(): + # Ensure the just-completed turn's retain has landed before we + # recall, so the warmed context for the next turn includes it. + # This runs on the background prefetch thread, never the reply + # path, so it adds no latency to the user's response. + if self._prefetch_waits_for_retain: + self._wait_for_retains_drained(self._prefetch_retain_drain_timeout) try: if self._prefetch_method == "reflect": logger.debug("Prefetch: calling reflect (bank=%s, query_len=%d)", self._bank_id, len(query)) diff --git a/tests/plugins/memory/test_hindsight_provider.py b/tests/plugins/memory/test_hindsight_provider.py index 0c72bd6203..092061830e 100644 --- a/tests/plugins/memory/test_hindsight_provider.py +++ b/tests/plugins/memory/test_hindsight_provider.py @@ -10,6 +10,7 @@ import os import re import stat import sys +import time from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -501,6 +502,46 @@ class TestPrefetch: # Should not start a thread assert p._prefetch_thread is None + def test_prefetch_waits_for_pending_retain_before_recall(self, provider): + """The background prefetch must wait for queued retains to drain so the + next turn's recall observes the just-completed turn (no retain race).""" + import threading + + order = [] + release = threading.Event() + + async def _slow_retain(*args, **kwargs): + release.wait(timeout=5.0) + order.append("retain") + + async def _recall(**kwargs): + order.append("recall") + return SimpleNamespace(results=[SimpleNamespace(text="m")]) + + provider._client.aretain_batch = AsyncMock(side_effect=_slow_retain) + provider._client.arecall = AsyncMock(side_effect=_recall) + + # Enqueue a slow retain, then immediately queue the next-turn prefetch. + provider.sync_turn("hello", "world") + provider.queue_prefetch("next turn query") + + # Let the prefetch thread start and reach the drain barrier. + time.sleep(0.2) + assert order == [], "recall ran before the pending retain drained" + + # Release the retain; the prefetch should now proceed AFTER it. + release.set() + if provider._prefetch_thread: + provider._prefetch_thread.join(timeout=5.0) + provider._retain_queue.join() + assert order and order[0] == "retain" + assert "recall" in order + + def test_prefetch_wait_for_retain_can_be_disabled(self, provider_with_config): + p = provider_with_config(prefetch_waits_for_retain=False) + p._client = _make_mock_client() + assert p._prefetch_waits_for_retain is False + # --------------------------------------------------------------------------- # sync_turn tests