From 56104af1965fcd4368651c8b2b92aaf73ea27c3a Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 13 Aug 2026 12:26:35 -0700 Subject: [PATCH] fix(relay): unwrap lazy completed streams Signed-off-by: Bryan Bednarski --- agent/relay_llm.py | 21 +++++++++++++++++---- tests/agent/test_relay_llm.py | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/agent/relay_llm.py b/agent/relay_llm.py index 58a5c1bd5e..610deff98f 100644 --- a/agent/relay_llm.py +++ b/agent/relay_llm.py @@ -312,11 +312,11 @@ def stream_current( defer_logical_completion=defer_logical_completion, completed_response_predicate=completed_response_predicate, ) - # In the non-managed path the factory already ran eagerly during __init__, - # so a completed response is visible immediately and must surface raw. - # In the managed path the factory runs lazily on first pull, so - # final_response is still None here and the managed stream is returned. if completed_response_predicate is not None: + # Relay may defer the provider callback until the first stream pull. + # Prime once so adapters that ignore stream=True can still return their + # completed response directly. A real first chunk is buffered. + managed._prime_completed_response() completed = getattr(managed, "final_response", None) if completed is not None: return completed @@ -400,6 +400,7 @@ class ManagedLlmStream(Iterator[Any]): self._relay_observes_chunks = False self._provider_completed = False self._raw_chunks: list[tuple[Any, Any]] = [] + self._prefetched_chunks: list[Any] = [] self.output_modified = False callback_context = contextvars.copy_context() @@ -569,9 +570,20 @@ class ManagedLlmStream(Iterator[Any]): def __iter__(self) -> "ManagedLlmStream": return self + def _prime_completed_response(self) -> None: + """Advance once while preserving a genuine first chunk.""" + if self._closed or self._prefetched_chunks: + return + try: + self._prefetched_chunks.append(next(self)) + except StopIteration: + pass + def __next__(self) -> Any: if self._closed: raise StopIteration + if self._prefetched_chunks: + return self._prefetched_chunks.pop() if self._loop is None: try: chunk = next(self._stream) @@ -684,6 +696,7 @@ class ManagedLlmStream(Iterator[Any]): if self._closed: return self._closed = True + self._prefetched_chunks.clear() loop = self._loop self._loop = None if loop is None: diff --git a/tests/agent/test_relay_llm.py b/tests/agent/test_relay_llm.py index e24897412a..045c580710 100644 --- a/tests/agent/test_relay_llm.py +++ b/tests/agent/test_relay_llm.py @@ -796,6 +796,31 @@ def test_stream_current_streams_iterators_with_predicate(tmp_path, monkeypatch): relay_runtime._reset_for_tests() +def test_stream_current_primes_lazy_completed_response(relay_turn, monkeypatch): + """A lazy Relay stream must run once before Hermes decides its shape.""" + _relay, _turn = relay_turn + completed = _completed_response() + + class LazyCompletedStream: + final_response = None + + def _prime_completed_response(self): + self.final_response = completed + + lazy_stream = LazyCompletedStream() + monkeypatch.setattr(relay_llm, "stream", lambda *args, **kwargs: lazy_stream) + + result = relay_llm.stream_current( + {"model": "test-model", "messages": [], "stream": True}, + lambda request: completed, + name="test-provider", + model_name="test-model", + finalizer=dict, + completed_response_predicate=_choices_predicate, + ) + + assert result is completed + def _completed_response(content: str = "done") -> SimpleNamespace: return SimpleNamespace( @@ -833,6 +858,8 @@ def test_stream_managed_traps_direct_completed_response(relay_turn): finalizer=lambda: {}, completed_response_predicate=_choices_predicate, ) + stream._prime_completed_response() + assert stream._closed assert list(stream) == [] assert stream.final_response is not None assert stream.final_response.choices[0].message.content == "done"