fix(stream): do not misclassify stream with final usage chunk as mid-stream drop (#91373)

When stream_options={'include_usage': True} is requested, OpenAI-compliant
providers (e.g. vLLM, OpenAI, DeepSeek) emit a final usage-only chunk with
empty choices (choices=[]) and no finish_reason.

If the preceding text chunks did not explicitly set finish_reason, the
check in _call_chat_completions evaluated _text_only_dropped_no_finish to
True and returned a partial-stream stub with finish_reason='length'. The
conversation loop then assumed the connection was cut off and injected a
spurious continuation nudge, causing the model to rewrite the full answer.

Require usage_obj is None in _text_only_dropped_no_finish so streams that
delivered valid usage metadata complete cleanly with finish_reason='stop'.
This commit is contained in:
loulanyue
2026-08-21 15:35:24 +08:00
committed by Teknium
parent ce7f805869
commit 66d42e0dba
2 changed files with 71 additions and 0 deletions
+6
View File
@@ -4609,10 +4609,16 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=
# text content but no tool calls. Without this guard the partial
# text is silently stamped finish_reason="stop" and the turn ends as
# if complete — the model's intended next step is lost (#32086).
# When `include_usage` is requested, OpenAI-compliant providers (e.g.
# vLLM, OpenAI, DeepSeek) emit a final usage-only chunk with empty
# choices and no finish_reason. Receiving a valid usage object proves
# the provider completed generation and closed the stream cleanly
# (#91373), so it is not a mid-stream drop.
_text_only_dropped_no_finish = (
finish_reason is None
and content_parts
and not tool_calls_acc
and usage_obj is None
)
if _text_only_dropped_no_finish:
logger.warning(
@@ -820,3 +820,68 @@ class TestPortalLastOneWithoutDone:
assert getattr(response, "id", None) != PARTIAL_STREAM_STUB_ID
assert response.choices[0].finish_reason == "stop"
assert response.choices[0].message.content == "LONGCAT_OK"
# ── Trailing usage chunk with no finish_reason (#91373) ───────────────────
class TestStreamIncludeUsageFinalChunk:
"""OpenAI / vLLM streaming with stream_options={'include_usage': True} emits
a final usage-only chunk with empty choices (choices=[]) and no finish_reason.
Hermes must recognize that the presence of usage proves the stream completed
cleanly and must NOT misclassify it as a mid-stream network drop (#91373).
"""
@patch("run_agent.AIAgent._create_request_openai_client")
@patch("run_agent.AIAgent._close_request_openai_client")
def test_text_stream_with_final_usage_chunk_completes_as_stop(
self, _mock_close, mock_create, monkeypatch,
):
"""When text is streamed and the final chunk is a usage-only chunk
without finish_reason, the completion must complete with finish_reason='stop'
instead of returning PARTIAL_STREAM_STUB_ID."""
def _vllm_stream():
# Chunks 1 and 2: text deltas with finish_reason=None
yield _make_stream_chunk(content="The final answer is 42.")
# Chunk 3: trailing usage-only chunk (choices=[], usage present)
usage = SimpleNamespace(prompt_tokens=100, completion_tokens=10, total_tokens=110)
yield SimpleNamespace(choices=[], model="vllm/test-model", usage=usage)
mock_client = MagicMock()
mock_client.chat.completions.create.side_effect = lambda *a, **kw: _vllm_stream()
mock_create.return_value = mock_client
agent = _make_agent()
monkeypatch.setenv("HERMES_STREAM_RETRIES", "0")
response = agent._interruptible_streaming_api_call({})
assert response.id != PARTIAL_STREAM_STUB_ID
assert response.choices[0].finish_reason == "stop"
assert response.choices[0].message.content == "The final answer is 42."
assert response.usage is not None
assert response.usage.prompt_tokens == 100
assert response.usage.completion_tokens == 10
@patch("run_agent.AIAgent._create_request_openai_client")
@patch("run_agent.AIAgent._close_request_openai_client")
def test_text_stream_abrupt_drop_without_usage_still_returns_stub(
self, _mock_close, mock_create, monkeypatch,
):
"""When text is streamed but the connection is severed before any usage
chunk arrives (usage is None and finish_reason is None), it remains
correctly classified as a partial stream stub."""
def _dropped_stream():
yield _make_stream_chunk(content="Partial text before drop")
# Stream ends abruptly without finish_reason and without usage
mock_client = MagicMock()
mock_client.chat.completions.create.side_effect = lambda *a, **kw: _dropped_stream()
mock_create.return_value = mock_client
agent = _make_agent()
agent._current_streamed_assistant_text = "Partial text before drop"
monkeypatch.setenv("HERMES_STREAM_RETRIES", "0")
response = agent._interruptible_streaming_api_call({})
assert response.id == PARTIAL_STREAM_STUB_ID
assert response.choices[0].finish_reason == FINISH_REASON_LENGTH
assert response.choices[0].message.content == "Partial text before drop"