fix(stream): do not misclassify stream with final usage chunk as mid-stream drop (#91373)
When stream_options={'include_usage': True} is requested, OpenAI-compliant
providers (e.g. vLLM, OpenAI, DeepSeek) emit a final usage-only chunk with
empty choices (choices=[]) and no finish_reason.
If the preceding text chunks did not explicitly set finish_reason, the
check in _call_chat_completions evaluated _text_only_dropped_no_finish to
True and returned a partial-stream stub with finish_reason='length'. The
conversation loop then assumed the connection was cut off and injected a
spurious continuation nudge, causing the model to rewrite the full answer.
Require usage_obj is None in _text_only_dropped_no_finish so streams that
delivered valid usage metadata complete cleanly with finish_reason='stop'.
This commit is contained in:
@@ -4609,10 +4609,16 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
|
||||
# text content but no tool calls. Without this guard the partial
|
||||
# text is silently stamped finish_reason="stop" and the turn ends as
|
||||
# if complete — the model's intended next step is lost (#32086).
|
||||
# When `include_usage` is requested, OpenAI-compliant providers (e.g.
|
||||
# vLLM, OpenAI, DeepSeek) emit a final usage-only chunk with empty
|
||||
# choices and no finish_reason. Receiving a valid usage object proves
|
||||
# the provider completed generation and closed the stream cleanly
|
||||
# (#91373), so it is not a mid-stream drop.
|
||||
_text_only_dropped_no_finish = (
|
||||
finish_reason is None
|
||||
and content_parts
|
||||
and not tool_calls_acc
|
||||
and usage_obj is None
|
||||
)
|
||||
if _text_only_dropped_no_finish:
|
||||
logger.warning(
|
||||
|
||||
@@ -820,3 +820,68 @@ class TestPortalLastOneWithoutDone:
|
||||
assert getattr(response, "id", None) != PARTIAL_STREAM_STUB_ID
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
assert response.choices[0].message.content == "LONGCAT_OK"
|
||||
|
||||
|
||||
# ── Trailing usage chunk with no finish_reason (#91373) ───────────────────
|
||||
|
||||
class TestStreamIncludeUsageFinalChunk:
|
||||
"""OpenAI / vLLM streaming with stream_options={'include_usage': True} emits
|
||||
a final usage-only chunk with empty choices (choices=[]) and no finish_reason.
|
||||
Hermes must recognize that the presence of usage proves the stream completed
|
||||
cleanly and must NOT misclassify it as a mid-stream network drop (#91373).
|
||||
"""
|
||||
|
||||
@patch("run_agent.AIAgent._create_request_openai_client")
|
||||
@patch("run_agent.AIAgent._close_request_openai_client")
|
||||
def test_text_stream_with_final_usage_chunk_completes_as_stop(
|
||||
self, _mock_close, mock_create, monkeypatch,
|
||||
):
|
||||
"""When text is streamed and the final chunk is a usage-only chunk
|
||||
without finish_reason, the completion must complete with finish_reason='stop'
|
||||
instead of returning PARTIAL_STREAM_STUB_ID."""
|
||||
def _vllm_stream():
|
||||
# Chunks 1 and 2: text deltas with finish_reason=None
|
||||
yield _make_stream_chunk(content="The final answer is 42.")
|
||||
# Chunk 3: trailing usage-only chunk (choices=[], usage present)
|
||||
usage = SimpleNamespace(prompt_tokens=100, completion_tokens=10, total_tokens=110)
|
||||
yield SimpleNamespace(choices=[], model="vllm/test-model", usage=usage)
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.chat.completions.create.side_effect = lambda *a, **kw: _vllm_stream()
|
||||
mock_create.return_value = mock_client
|
||||
|
||||
agent = _make_agent()
|
||||
monkeypatch.setenv("HERMES_STREAM_RETRIES", "0")
|
||||
response = agent._interruptible_streaming_api_call({})
|
||||
|
||||
assert response.id != PARTIAL_STREAM_STUB_ID
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
assert response.choices[0].message.content == "The final answer is 42."
|
||||
assert response.usage is not None
|
||||
assert response.usage.prompt_tokens == 100
|
||||
assert response.usage.completion_tokens == 10
|
||||
|
||||
@patch("run_agent.AIAgent._create_request_openai_client")
|
||||
@patch("run_agent.AIAgent._close_request_openai_client")
|
||||
def test_text_stream_abrupt_drop_without_usage_still_returns_stub(
|
||||
self, _mock_close, mock_create, monkeypatch,
|
||||
):
|
||||
"""When text is streamed but the connection is severed before any usage
|
||||
chunk arrives (usage is None and finish_reason is None), it remains
|
||||
correctly classified as a partial stream stub."""
|
||||
def _dropped_stream():
|
||||
yield _make_stream_chunk(content="Partial text before drop")
|
||||
# Stream ends abruptly without finish_reason and without usage
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.chat.completions.create.side_effect = lambda *a, **kw: _dropped_stream()
|
||||
mock_create.return_value = mock_client
|
||||
|
||||
agent = _make_agent()
|
||||
agent._current_streamed_assistant_text = "Partial text before drop"
|
||||
monkeypatch.setenv("HERMES_STREAM_RETRIES", "0")
|
||||
response = agent._interruptible_streaming_api_call({})
|
||||
|
||||
assert response.id == PARTIAL_STREAM_STUB_ID
|
||||
assert response.choices[0].finish_reason == FINISH_REASON_LENGTH
|
||||
assert response.choices[0].message.content == "Partial text before drop"
|
||||
|
||||
Reference in New Issue
Block a user