fix(relay): unwrap lazy completed streams
Signed-off-by: Bryan Bednarski <bbednarski@nvidia.com>
This commit is contained in:
+17
-4
@@ -312,11 +312,11 @@ def stream_current(
|
||||
defer_logical_completion=defer_logical_completion,
|
||||
completed_response_predicate=completed_response_predicate,
|
||||
)
|
||||
# In the non-managed path the factory already ran eagerly during __init__,
|
||||
# so a completed response is visible immediately and must surface raw.
|
||||
# In the managed path the factory runs lazily on first pull, so
|
||||
# final_response is still None here and the managed stream is returned.
|
||||
if completed_response_predicate is not None:
|
||||
# Relay may defer the provider callback until the first stream pull.
|
||||
# Prime once so adapters that ignore stream=True can still return their
|
||||
# completed response directly. A real first chunk is buffered.
|
||||
managed._prime_completed_response()
|
||||
completed = getattr(managed, "final_response", None)
|
||||
if completed is not None:
|
||||
return completed
|
||||
@@ -400,6 +400,7 @@ class ManagedLlmStream(Iterator[Any]):
|
||||
self._relay_observes_chunks = False
|
||||
self._provider_completed = False
|
||||
self._raw_chunks: list[tuple[Any, Any]] = []
|
||||
self._prefetched_chunks: list[Any] = []
|
||||
self.output_modified = False
|
||||
callback_context = contextvars.copy_context()
|
||||
|
||||
@@ -569,9 +570,20 @@ class ManagedLlmStream(Iterator[Any]):
|
||||
def __iter__(self) -> "ManagedLlmStream":
|
||||
return self
|
||||
|
||||
def _prime_completed_response(self) -> None:
|
||||
"""Advance once while preserving a genuine first chunk."""
|
||||
if self._closed or self._prefetched_chunks:
|
||||
return
|
||||
try:
|
||||
self._prefetched_chunks.append(next(self))
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
def __next__(self) -> Any:
|
||||
if self._closed:
|
||||
raise StopIteration
|
||||
if self._prefetched_chunks:
|
||||
return self._prefetched_chunks.pop()
|
||||
if self._loop is None:
|
||||
try:
|
||||
chunk = next(self._stream)
|
||||
@@ -684,6 +696,7 @@ class ManagedLlmStream(Iterator[Any]):
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
self._prefetched_chunks.clear()
|
||||
loop = self._loop
|
||||
self._loop = None
|
||||
if loop is None:
|
||||
|
||||
@@ -796,6 +796,31 @@ def test_stream_current_streams_iterators_with_predicate(tmp_path, monkeypatch):
|
||||
relay_runtime._reset_for_tests()
|
||||
|
||||
|
||||
def test_stream_current_primes_lazy_completed_response(relay_turn, monkeypatch):
|
||||
"""A lazy Relay stream must run once before Hermes decides its shape."""
|
||||
_relay, _turn = relay_turn
|
||||
completed = _completed_response()
|
||||
|
||||
class LazyCompletedStream:
|
||||
final_response = None
|
||||
|
||||
def _prime_completed_response(self):
|
||||
self.final_response = completed
|
||||
|
||||
lazy_stream = LazyCompletedStream()
|
||||
monkeypatch.setattr(relay_llm, "stream", lambda *args, **kwargs: lazy_stream)
|
||||
|
||||
result = relay_llm.stream_current(
|
||||
{"model": "test-model", "messages": [], "stream": True},
|
||||
lambda request: completed,
|
||||
name="test-provider",
|
||||
model_name="test-model",
|
||||
finalizer=dict,
|
||||
completed_response_predicate=_choices_predicate,
|
||||
)
|
||||
|
||||
assert result is completed
|
||||
|
||||
|
||||
def _completed_response(content: str = "done") -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
@@ -833,6 +858,8 @@ def test_stream_managed_traps_direct_completed_response(relay_turn):
|
||||
finalizer=lambda: {},
|
||||
completed_response_predicate=_choices_predicate,
|
||||
)
|
||||
stream._prime_completed_response()
|
||||
assert stream._closed
|
||||
assert list(stream) == []
|
||||
assert stream.final_response is not None
|
||||
assert stream.final_response.choices[0].message.content == "done"
|
||||
|
||||
Reference in New Issue
Block a user