fix(relay): unwrap lazy completed streams

Signed-off-by: Bryan Bednarski <bbednarski@nvidia.com>
This commit is contained in:
Bryan Bednarski
2026-08-13 12:26:35 -07:00
parent dcbfc79f8d
commit 56104af196
2 changed files with 44 additions and 4 deletions
+17 -4
View File
@@ -312,11 +312,11 @@ def stream_current(
defer_logical_completion=defer_logical_completion,
completed_response_predicate=completed_response_predicate,
)
# In the non-managed path the factory already ran eagerly during __init__,
# so a completed response is visible immediately and must surface raw.
# In the managed path the factory runs lazily on first pull, so
# final_response is still None here and the managed stream is returned.
if completed_response_predicate is not None:
# Relay may defer the provider callback until the first stream pull.
# Prime once so adapters that ignore stream=True can still return their
# completed response directly. A real first chunk is buffered.
managed._prime_completed_response()
completed = getattr(managed, "final_response", None)
if completed is not None:
return completed
@@ -400,6 +400,7 @@ class ManagedLlmStream(Iterator[Any]):
self._relay_observes_chunks = False
self._provider_completed = False
self._raw_chunks: list[tuple[Any, Any]] = []
self._prefetched_chunks: list[Any] = []
self.output_modified = False
callback_context = contextvars.copy_context()
@@ -569,9 +570,20 @@ class ManagedLlmStream(Iterator[Any]):
def __iter__(self) -> "ManagedLlmStream":
return self
def _prime_completed_response(self) -> None:
"""Advance once while preserving a genuine first chunk."""
if self._closed or self._prefetched_chunks:
return
try:
self._prefetched_chunks.append(next(self))
except StopIteration:
pass
def __next__(self) -> Any:
if self._closed:
raise StopIteration
if self._prefetched_chunks:
return self._prefetched_chunks.pop()
if self._loop is None:
try:
chunk = next(self._stream)
@@ -684,6 +696,7 @@ class ManagedLlmStream(Iterator[Any]):
if self._closed:
return
self._closed = True
self._prefetched_chunks.clear()
loop = self._loop
self._loop = None
if loop is None:
+27
View File
@@ -796,6 +796,31 @@ def test_stream_current_streams_iterators_with_predicate(tmp_path, monkeypatch):
relay_runtime._reset_for_tests()
def test_stream_current_primes_lazy_completed_response(relay_turn, monkeypatch):
"""A lazy Relay stream must run once before Hermes decides its shape."""
_relay, _turn = relay_turn
completed = _completed_response()
class LazyCompletedStream:
final_response = None
def _prime_completed_response(self):
self.final_response = completed
lazy_stream = LazyCompletedStream()
monkeypatch.setattr(relay_llm, "stream", lambda *args, **kwargs: lazy_stream)
result = relay_llm.stream_current(
{"model": "test-model", "messages": [], "stream": True},
lambda request: completed,
name="test-provider",
model_name="test-model",
finalizer=dict,
completed_response_predicate=_choices_predicate,
)
assert result is completed
def _completed_response(content: str = "done") -> SimpleNamespace:
return SimpleNamespace(
@@ -833,6 +858,8 @@ def test_stream_managed_traps_direct_completed_response(relay_turn):
finalizer=lambda: {},
completed_response_predicate=_choices_predicate,
)
stream._prime_completed_response()
assert stream._closed
assert list(stream) == []
assert stream.final_response is not None
assert stream.final_response.choices[0].message.content == "done"