fix(memory/hindsight): evict timed-out retain ops + coarser status polls

Review follow-up on the #62871 salvage (simplify pass, HIGH):

1. Ops unresolved at the wait deadline were RETAINED in the pending set.
   A permanently failing status endpoint (auth error, endless 500s, or a
   server that loses ops without 404) would grow the set forever and make
   EVERY later prefetch burn the full 10s budget re-polling it — and
   prefetch()'s bounded 3s join sits on the reply path, so that money-quote
   'adds no response latency' claim breaks. Timed-out ops are now dropped
   (identical degradation to prefetch_waits_for_retain=False: possibly
   stale recall) with a WARNING so persistent server trouble is visible.
   Guard test mutation-checked (fails with eviction disabled).

2. Status polls now spaced 0.5s (was 0.05s shared with the local drain
   poll): a wedged op cost up to ~200 get_operation_status round trips
   per prefetch; now ~20 max over the default 10s budget.
This commit is contained in:
kshitij
2026-08-02 21:54:40 +05:30
parent 1be353bf9c
commit 9bbd956b73
2 changed files with 78 additions and 9 deletions
@@ -649,6 +649,38 @@ class TestPrefetchServerRetainVisibility:
assert order == ["recall"], "prefetch should recall after the timeout"
assert elapsed < 3.0, "prefetch must not block well past the drain budget"
def test_timed_out_ops_are_dropped_not_repolled(self, provider_with_config):
"""Ops unresolved at deadline must be EVICTED so a permanently failing
status endpoint can't make every later prefetch re-burn the full
timeout on a growing pending set (unbounded session-wide degradation
+ reply-path join penalty)."""
p = provider_with_config(prefetch_retain_drain_timeout=0.3)
p._client = self._client_with_ops(["pending"]) # never completes
p._client.arecall = AsyncMock(
return_value=SimpleNamespace(results=[SimpleNamespace(text="m")])
)
p.sync_turn("hello", "world")
p._retain_queue.join()
assert p._pending_retain_ops, "op should be tracked before the wait"
# First prefetch burns the budget and must DROP the wedged op.
p.queue_prefetch("q1")
if p._prefetch_thread:
p._prefetch_thread.join(timeout=5.0)
assert p._pending_retain_ops == set(), (
"unresolved ops must be evicted at deadline, not retained"
)
# A later prefetch with nothing pending must be near-instant.
start = time.monotonic()
p.queue_prefetch("q2")
if p._prefetch_thread:
p._prefetch_thread.join(timeout=5.0)
assert time.monotonic() - start < 0.25, (
"second prefetch re-polled dropped ops — eviction regressed"
)
def test_operation_notfound_treated_as_complete(self, provider):
"""A NotFound (completed+evicted) op is treated as done, not pending."""
from hindsight_client_api.exceptions import NotFoundException