Teknium
1548 lines
73 KiB
Python
1548 lines
73 KiB
Python
"""OpenAI-compatible routes for the API server adapter.
|
|
|
|
``OpenAICompatRoutesMixin`` carries ``/v1/chat/completions``, ``/v1/responses``
|
|
(+ GET/DELETE), their two SSE writers, and the Responses-transcript helpers.
|
|
``APIServerAdapter`` inherits it; every ``self.*`` call resolves via the MRO.
|
|
|
|
api_server-internal helpers are imported lazily inside each method (the origin
|
|
imports this module, so a top-level import would be a cycle), which also keeps
|
|
``patch("gateway.platforms.api_server.X")`` effective for the moved bodies.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import re
|
|
import time
|
|
import uuid
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
try:
|
|
from aiohttp import web
|
|
except ImportError: # pragma: no cover - mirrors api_server's optional import
|
|
web = None # type: ignore[assignment]
|
|
|
|
# Logger parity with the origin module (moved log records keep their name).
|
|
logger = logging.getLogger("gateway.platforms.api_server")
|
|
|
|
|
|
async def _iter_stream_items(stream_q, agent_task, response):
|
|
"""Yield agent stream items until end-of-stream, writing SSE keepalives while idle.
|
|
|
|
Woken directly by ``put_threadsafe`` (no executor hop / poll latency). Yields
|
|
the ``None`` sentinel once so callers can run EOS-only work; when
|
|
``agent_task`` is already done the remaining queue is drained and the
|
|
sentinel swallowed (matching the historical inline loops).
|
|
"""
|
|
from gateway.platforms.api_server import CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS
|
|
|
|
last_activity = time.monotonic()
|
|
while True:
|
|
try:
|
|
item = await asyncio.wait_for(stream_q.get(), timeout=0.5)
|
|
except asyncio.TimeoutError:
|
|
if agent_task.done():
|
|
while True:
|
|
try:
|
|
item = stream_q.get_nowait()
|
|
except asyncio.QueueEmpty:
|
|
return
|
|
if item is None:
|
|
return
|
|
yield item
|
|
last_activity = time.monotonic()
|
|
if time.monotonic() - last_activity >= CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS:
|
|
await response.write(b": keepalive\n\n")
|
|
last_activity = time.monotonic()
|
|
continue
|
|
if item is None:
|
|
yield None
|
|
return
|
|
yield item
|
|
last_activity = time.monotonic()
|
|
|
|
|
|
class OpenAICompatRoutesMixin:
|
|
"""/v1/chat/completions and /v1/responses handlers + SSE writers."""
|
|
|
|
async def _handle_chat_completions(self, request: "web.Request") -> "web.Response":
|
|
"""POST /v1/chat/completions — OpenAI Chat Completions format."""
|
|
from gateway.platforms.api_server import (
|
|
ThreadSafeAsyncQueue,
|
|
_chat_usage_payload,
|
|
_coerce_request_bool,
|
|
_content_has_visible_payload,
|
|
_derive_chat_session_id,
|
|
_error_response,
|
|
_multimodal_validation_error,
|
|
_normalize_chat_content,
|
|
_normalize_multimodal_content,
|
|
_openai_error,
|
|
_redact_api_error_text,
|
|
_request_agent_overrides,
|
|
_resolve_media_to_data_urls,
|
|
)
|
|
# Bound total in-flight agent runs (configurable; #7483).
|
|
limited = self._concurrency_limited_response()
|
|
if limited is not None:
|
|
return limited
|
|
|
|
# Parse request body
|
|
try:
|
|
body = await request.json()
|
|
except (json.JSONDecodeError, Exception):
|
|
return _error_response("Invalid JSON in request body", 400)
|
|
messages = body.get("messages")
|
|
if not messages or not isinstance(messages, list):
|
|
return web.json_response(
|
|
{"error": {"message": "Missing or invalid 'messages' field", "type": "invalid_request_error"}},
|
|
status=400,
|
|
)
|
|
stream = _coerce_request_bool(body.get("stream"), default=False)
|
|
|
|
# Extract system message (becomes ephemeral system prompt layered ON TOP of core)
|
|
system_prompt = None
|
|
conversation_messages: List[Dict[str, str]] = []
|
|
for idx, msg in enumerate(messages):
|
|
role = msg.get("role", "")
|
|
raw_content = msg.get("content", "")
|
|
if role == "system":
|
|
# System messages don't support images (Anthropic rejects, OpenAI
|
|
# text-model systems don't render them). Flatten to text.
|
|
content = _normalize_chat_content(raw_content)
|
|
if system_prompt is None:
|
|
system_prompt = content
|
|
else:
|
|
system_prompt = system_prompt + "\n" + content
|
|
elif role in {"user", "assistant"}:
|
|
try:
|
|
content = _normalize_multimodal_content(raw_content)
|
|
except ValueError as exc:
|
|
return _multimodal_validation_error(exc, param=f"messages[{idx}].content")
|
|
conversation_messages.append({"role": role, "content": content})
|
|
|
|
# Extract the last user message as the primary input
|
|
user_message: Any = ""
|
|
history = []
|
|
if conversation_messages:
|
|
user_message = conversation_messages[-1].get("content", "")
|
|
history = conversation_messages[:-1]
|
|
if not _content_has_visible_payload(user_message):
|
|
return web.json_response(
|
|
{"error": {"message": "No user message found in messages", "type": "invalid_request_error"}},
|
|
status=400,
|
|
)
|
|
|
|
# Allow caller to scope long-term memory (e.g. Honcho) with a
|
|
# stable per-channel identifier via X-Hermes-Session-Key. This
|
|
# is independent of X-Hermes-Session-Id: the key persists across
|
|
# transcripts while the id rotates when the caller starts a new
|
|
# transcript (i.e. /new semantics). See _parse_session_key_header.
|
|
gateway_session_key, key_err = self._parse_session_key_header(request)
|
|
if key_err is not None:
|
|
return key_err
|
|
|
|
# X-Hermes-Session-Id continues an existing session (history from state.db, not
|
|
# the body). Requires a configured API key: otherwise any client could read
|
|
# arbitrary history by guessing session ids.
|
|
provided_session_id = request.headers.get("X-Hermes-Session-Id", "").strip()
|
|
if provided_session_id:
|
|
if not self._api_key:
|
|
logger.warning(
|
|
"Session continuation via X-Hermes-Session-Id rejected: "
|
|
"no API key configured. Set API_SERVER_KEY to enable "
|
|
"session continuity."
|
|
)
|
|
return _error_response("Session continuation requires API key authentication. "
|
|
"Configure API_SERVER_KEY to enable this feature.", 403)
|
|
# Sanitize: reject control characters that could enable header
|
|
# injection, and path-traversal-shaped IDs that would escape the
|
|
# sessions directory when interpolated into on-disk artifact
|
|
# filenames (session snapshots, request dumps). Mirrors the native
|
|
# gateway's entry-boundary guard (gateway.session._is_path_unsafe).
|
|
from gateway.session import _is_path_unsafe
|
|
if re.search(r'[\r\n\x00]', provided_session_id) or _is_path_unsafe(provided_session_id):
|
|
return web.json_response(
|
|
{"error": {"message": "Invalid session ID", "type": "invalid_request_error"}},
|
|
status=400,
|
|
)
|
|
if len(provided_session_id) > self._MAX_SESSION_HEADER_LEN:
|
|
return web.json_response(
|
|
{"error": {"message": "Session ID too long", "type": "invalid_request_error"}},
|
|
status=400,
|
|
)
|
|
session_id = provided_session_id
|
|
try:
|
|
db = await self._ensure_session_db_async()
|
|
if db is not None:
|
|
history = await asyncio.to_thread(db.get_messages_as_conversation, session_id)
|
|
except Exception as e:
|
|
logger.warning("Failed to load session history for %s: %s", session_id, e)
|
|
history = []
|
|
else:
|
|
# Derive a stable session ID from the conversation fingerprint so
|
|
# that consecutive messages from the same Open WebUI (or similar)
|
|
# conversation map to the same Hermes session. The first user
|
|
# message + system prompt are constant across all turns.
|
|
first_user = ""
|
|
for cm in conversation_messages:
|
|
if cm.get("role") == "user":
|
|
first_user = cm.get("content", "")
|
|
break
|
|
session_id = _derive_chat_session_id(system_prompt, first_user)
|
|
# history already set from request body above
|
|
completion_id = f"chatcmpl-{uuid.uuid4().hex[:29]}"
|
|
model_name = body.get("model", self._model_name)
|
|
created = int(time.time())
|
|
|
|
# Per-client model routing: if the requested model matches a
|
|
# configured model_routes alias, this request's agent is created
|
|
# with that route's model/provider instead of the global default.
|
|
route = self._resolve_route(model_name)
|
|
agent_overrides = _request_agent_overrides(
|
|
body,
|
|
virtual_model=self._model_name,
|
|
allow_bare_model=self._direct_model_requests,
|
|
)
|
|
selection_error = self._request_route_conflict_error(
|
|
session_id=session_id,
|
|
gateway_session_key=gateway_session_key,
|
|
requested_model=agent_overrides.get("requested_model"),
|
|
requested_provider=agent_overrides.get("requested_provider"),
|
|
route=route,
|
|
)
|
|
if selection_error:
|
|
return _error_response(selection_error, 400)
|
|
if stream:
|
|
_stream_q = ThreadSafeAsyncQueue()
|
|
|
|
def _on_delta(delta):
|
|
# None from the agent is a CLI box-close signal, not EOS — forwarding it
|
|
# would end the stream before the post-tool answer. Called from the
|
|
# run_conversation worker thread, so put_threadsafe is required.
|
|
if delta is not None:
|
|
_stream_q.put_threadsafe(delta)
|
|
|
|
# Track which tool_call_ids we've emitted a "running" lifecycle
|
|
# event for, so a "completed" event without a matching "running"
|
|
# (e.g. internal/filtered tools) is silently dropped instead of
|
|
# producing an orphaned event clients can't correlate.
|
|
_started_tool_call_ids: set[str] = set()
|
|
|
|
def _on_tool_start(tool_call_id, function_name, function_args):
|
|
"""Emit ``hermes.tool.progress`` with ``status: running``.
|
|
|
|
Replaces the old ``tool_progress_callback("tool.started",
|
|
...)`` emit so SSE consumers receive a single event per
|
|
tool start, carrying both the legacy ``tool``/``emoji``/
|
|
``label`` payload (for #6972 frontends) and the new
|
|
``toolCallId``/``status`` correlation fields (#16588).
|
|
|
|
Skips tools whose names start with ``_`` so internal
|
|
events (``_thinking``, …) stay off the wire — matching
|
|
the prior ``_on_tool_progress`` filter exactly.
|
|
"""
|
|
if not tool_call_id or function_name.startswith("_"):
|
|
return
|
|
_started_tool_call_ids.add(tool_call_id)
|
|
from agent.display import build_tool_preview, get_tool_emoji
|
|
label = build_tool_preview(function_name, function_args) or function_name
|
|
_stream_q.put_threadsafe(("__tool_progress__", {
|
|
"tool": function_name,
|
|
"emoji": get_tool_emoji(function_name),
|
|
"label": label,
|
|
"toolCallId": tool_call_id,
|
|
"status": "running",
|
|
}))
|
|
|
|
def _on_tool_complete(tool_call_id, function_name, function_args, function_result):
|
|
"""Emit the matching ``status: completed`` event.
|
|
|
|
Dropped if the start was filtered (internal tool, missing
|
|
id, or never seen) so clients never get an orphaned
|
|
``completed`` they can't correlate to a prior ``running``.
|
|
"""
|
|
if not tool_call_id or tool_call_id not in _started_tool_call_ids:
|
|
return
|
|
_started_tool_call_ids.discard(tool_call_id)
|
|
_stream_q.put_threadsafe(("__tool_progress__", {
|
|
"tool": function_name,
|
|
"toolCallId": tool_call_id,
|
|
"status": "completed",
|
|
}))
|
|
|
|
# agent_ref lets the SSE writer interrupt on disconnect. tool_progress_callback
|
|
# is deliberately NOT wired: it fires alongside the structured start/complete
|
|
# callbacks (which carry the tool_call id) and would duplicate every emit.
|
|
agent_ref = [None]
|
|
agent_task = asyncio.ensure_future(self._run_agent(
|
|
user_message=user_message,
|
|
conversation_history=history,
|
|
ephemeral_system_prompt=system_prompt,
|
|
session_id=session_id,
|
|
stream_delta_callback=_on_delta,
|
|
tool_start_callback=_on_tool_start,
|
|
tool_complete_callback=_on_tool_complete,
|
|
agent_ref=agent_ref,
|
|
gateway_session_key=gateway_session_key,
|
|
**agent_overrides,
|
|
route=route,
|
|
))
|
|
# Ensure SSE drain loops can terminate without relying on polling
|
|
# agent_task.done(), which can race with queue timeout checks.
|
|
agent_task.add_done_callback(lambda _fut: _stream_q.put_nowait(None))
|
|
return await self._write_sse_chat_completion(
|
|
request, completion_id, model_name, created, _stream_q,
|
|
agent_task, agent_ref, session_id=session_id,
|
|
gateway_session_key=gateway_session_key,
|
|
)
|
|
|
|
# Non-streaming: run the agent (with optional Idempotency-Key)
|
|
async def _compute_completion():
|
|
return await self._run_agent(
|
|
user_message=user_message,
|
|
conversation_history=history,
|
|
ephemeral_system_prompt=system_prompt,
|
|
session_id=session_id,
|
|
gateway_session_key=gateway_session_key,
|
|
**agent_overrides,
|
|
route=route,
|
|
)
|
|
outcome, err = await self._run_idempotent(
|
|
request, body, _compute_completion, log_label="chat completions",
|
|
fingerprint_keys=["model", "provider", "model_options", "messages", "tools", "tool_choice", "stream"],
|
|
)
|
|
if err is not None:
|
|
return err
|
|
result, usage = outcome
|
|
final_response = _resolve_media_to_data_urls(result.get("final_response") or "")
|
|
is_partial = bool(result.get("partial"))
|
|
is_failed = bool(result.get("failed"))
|
|
completed = bool(result.get("completed", True))
|
|
raw_err_msg = result.get("error")
|
|
err_msg = _redact_api_error_text(raw_err_msg) if raw_err_msg else raw_err_msg
|
|
|
|
# Decide finish_reason. OpenAI uses "length" for truncation, "stop"
|
|
# for normal completion, and downstream SDKs accept "error" / custom
|
|
# codes. See issue #22496.
|
|
if is_partial and err_msg and "truncat" in err_msg.lower():
|
|
finish_reason = "length"
|
|
elif is_failed or (not completed and err_msg):
|
|
finish_reason = "error"
|
|
else:
|
|
finish_reason = "stop"
|
|
response_headers = {"X-Hermes-Session-Id": result.get("session_id", session_id)}
|
|
if gateway_session_key:
|
|
response_headers["X-Hermes-Session-Key"] = gateway_session_key
|
|
|
|
# Hard-fail path: no usable assistant text AND a real failure → 5xx
|
|
# with OpenAI-style error envelope so SDK clients raise instead of
|
|
# silently rendering the internal failure string as message.content.
|
|
if not final_response and (is_failed or is_partial):
|
|
err_body = _openai_error(
|
|
err_msg or "Agent run did not produce a response.",
|
|
err_type="server_error",
|
|
code="agent_incomplete",
|
|
)
|
|
err_body["error"]["hermes"] = {
|
|
"completed": completed,
|
|
"partial": is_partial,
|
|
"failed": is_failed,
|
|
}
|
|
response_headers["X-Hermes-Completed"] = "false"
|
|
response_headers["X-Hermes-Partial"] = "true" if is_partial else "false"
|
|
return web.json_response(err_body, status=502, headers=response_headers)
|
|
|
|
# Soft-partial path: we have *some* text but the run did not complete
|
|
# (e.g. truncation with partial buffered output). Still 200 but signal
|
|
# truncation via finish_reason="length" + Hermes-specific extras.
|
|
response_data = {
|
|
"id": completion_id,
|
|
"object": "chat.completion",
|
|
"created": created,
|
|
"model": model_name,
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"message": {"role": "assistant", "content": final_response},
|
|
"finish_reason": finish_reason,
|
|
}
|
|
],
|
|
"usage": _chat_usage_payload(usage),
|
|
}
|
|
if is_partial or is_failed or not completed:
|
|
response_data["hermes"] = {
|
|
"completed": completed,
|
|
"partial": is_partial,
|
|
"failed": is_failed,
|
|
"error": err_msg,
|
|
"error_code": "output_truncated" if finish_reason == "length" else "agent_error",
|
|
}
|
|
response_headers["X-Hermes-Completed"] = "false"
|
|
response_headers["X-Hermes-Partial"] = "true" if is_partial else "false"
|
|
if err_msg:
|
|
response_headers["X-Hermes-Error"] = _redact_api_error_text(err_msg, limit=200)
|
|
return web.json_response(response_data, headers=response_headers)
|
|
|
|
async def _run_idempotent(
|
|
self, request: "web.Request", body: Dict[str, Any], compute, *,
|
|
log_label: str, fingerprint_keys: List[str],
|
|
) -> tuple:
|
|
"""Run ``compute()`` once per Idempotency-Key + body fingerprint.
|
|
|
|
Returns ``((result, usage), None)`` or ``(None, 500 response)``.
|
|
"""
|
|
from gateway.platforms.api_server import (
|
|
_error_response,
|
|
_idem_cache,
|
|
_make_request_fingerprint,
|
|
)
|
|
idempotency_key = request.headers.get("Idempotency-Key")
|
|
try:
|
|
if idempotency_key:
|
|
fp = _make_request_fingerprint(body, keys=fingerprint_keys)
|
|
result, usage = await _idem_cache.get_or_set(idempotency_key, fp, compute)
|
|
else:
|
|
result, usage = await compute()
|
|
return (result, usage), None
|
|
except Exception as e:
|
|
logger.error("Error running agent for %s: %s", log_label, e, exc_info=True)
|
|
return None, _error_response(f"Internal server error: {e}", 500, err_type="server_error")
|
|
|
|
async def _prepare_sse_response(
|
|
self, request: "web.Request", session_id: Optional[str], gateway_session_key: Optional[str],
|
|
) -> "web.StreamResponse":
|
|
"""Open a prepared SSE StreamResponse with CORS + session headers.
|
|
|
|
CORS middleware can't inject headers after ``prepare()`` flushes them,
|
|
so they are resolved up front here.
|
|
"""
|
|
sse_headers = {
|
|
"Content-Type": "text/event-stream",
|
|
"Cache-Control": "no-cache",
|
|
"X-Accel-Buffering": "no",
|
|
}
|
|
origin = request.headers.get("Origin", "")
|
|
cors = self._cors_headers_for_origin(origin) if origin else None
|
|
if cors:
|
|
sse_headers.update(cors)
|
|
if session_id:
|
|
sse_headers["X-Hermes-Session-Id"] = session_id
|
|
if gateway_session_key:
|
|
sse_headers["X-Hermes-Session-Key"] = gateway_session_key
|
|
response = web.StreamResponse(status=200, headers=sse_headers)
|
|
await response.prepare(request)
|
|
return response
|
|
|
|
async def _write_sse_chat_completion(
|
|
self, request: "web.Request", completion_id: str, model: str,
|
|
created: int, stream_q, agent_task, agent_ref=None, session_id: str = None,
|
|
gateway_session_key: str = None,
|
|
) -> "web.StreamResponse":
|
|
"""Write real streaming SSE from agent's stream_delta_callback queue.
|
|
|
|
If the client disconnects mid-stream (network drop, browser tab close),
|
|
the agent is interrupted via ``agent.interrupt()`` so it stops making
|
|
LLM API calls, and the asyncio task wrapper is cancelled.
|
|
"""
|
|
from gateway.platforms.api_server import (
|
|
_abandon_agent_task,
|
|
_chat_usage_payload,
|
|
_sse_frame,
|
|
)
|
|
response = await self._prepare_sse_response(request, session_id, gateway_session_key)
|
|
try:
|
|
# Role chunk
|
|
role_chunk = {
|
|
"id": completion_id, "object": "chat.completion.chunk",
|
|
"created": created, "model": model,
|
|
"choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}],
|
|
}
|
|
await response.write(_sse_frame(role_chunk))
|
|
|
|
# Helper — route a queue item to the correct SSE event.
|
|
async def _emit(item):
|
|
"""Write a single queue item to the SSE stream.
|
|
|
|
Plain strings are sent as normal ``delta.content`` chunks.
|
|
Tagged tuples ``("__tool_progress__", payload)`` are sent
|
|
as a custom ``event: hermes.tool.progress`` SSE event so
|
|
frontends can display them without storing the markers in
|
|
conversation history. See #6972 for the original event,
|
|
#16588 for the ``toolCallId``/``status`` lifecycle fields.
|
|
"""
|
|
if isinstance(item, tuple) and len(item) == 2 and item[0] == "__tool_progress__":
|
|
await response.write(_sse_frame(item[1], event="hermes.tool.progress"))
|
|
else:
|
|
content_chunk = {
|
|
"id": completion_id, "object": "chat.completion.chunk",
|
|
"created": created, "model": model,
|
|
"choices": [{"index": 0, "delta": {"content": item}, "finish_reason": None}],
|
|
}
|
|
await response.write(_sse_frame(content_chunk))
|
|
|
|
async for delta in _iter_stream_items(stream_q, agent_task, response):
|
|
if delta is None: # End of stream sentinel
|
|
break
|
|
await _emit(delta)
|
|
|
|
# The agent can fail after the queue drains cleanly: agent_task raises, or
|
|
# result is flagged failed/partial. Either must surface as a non-"stop"
|
|
# finish_reason (mirrors the non-streaming path) instead of a fake success.
|
|
usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
|
|
result = None
|
|
agent_error = None
|
|
try:
|
|
result, agent_usage = await agent_task
|
|
usage = agent_usage or usage
|
|
except Exception as exc:
|
|
agent_error = exc
|
|
logger.error("Agent task %s failed during SSE streaming: %s", completion_id, exc)
|
|
|
|
# Inspect the result dict for a flagged (non-exception) failure.
|
|
is_partial = bool(result.get("partial")) if isinstance(result, dict) else False
|
|
is_failed = bool(result.get("failed")) if isinstance(result, dict) else False
|
|
completed = bool(result.get("completed", True)) if isinstance(result, dict) else True
|
|
err_msg = result.get("error") if isinstance(result, dict) else None
|
|
if agent_error is not None:
|
|
is_failed = True
|
|
err_msg = err_msg or str(agent_error)
|
|
|
|
# Decide finish_reason, matching the non-streaming logic: "length"
|
|
# for truncation, "error" for failure, "stop" for normal completion.
|
|
if is_partial and err_msg and "truncat" in err_msg.lower():
|
|
finish_reason = "length"
|
|
elif agent_error is not None or is_failed or (not completed and err_msg):
|
|
finish_reason = "error"
|
|
else:
|
|
finish_reason = "stop"
|
|
|
|
# Finish chunk
|
|
finish_chunk = {
|
|
"id": completion_id, "object": "chat.completion.chunk",
|
|
"created": created, "model": model,
|
|
"choices": [{"index": 0, "delta": {}, "finish_reason": finish_reason}],
|
|
"usage": _chat_usage_payload(usage),
|
|
}
|
|
if finish_reason != "stop":
|
|
finish_chunk["choices"][0]["delta"] = {}
|
|
if err_msg:
|
|
finish_chunk["error"] = {
|
|
"message": err_msg,
|
|
"type": type(agent_error).__name__ if agent_error else "agent_error",
|
|
}
|
|
finish_chunk["hermes"] = {
|
|
"completed": completed,
|
|
"partial": is_partial,
|
|
"failed": is_failed,
|
|
"error": err_msg,
|
|
"error_code": "output_truncated" if finish_reason == "length" else "agent_error",
|
|
}
|
|
await response.write(_sse_frame(finish_chunk))
|
|
await response.write(b"data: [DONE]\n\n")
|
|
except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError, OSError):
|
|
# Client disconnected mid-stream: interrupt the agent so it stops
|
|
# making LLM calls, then cancel the task wrapper.
|
|
await _abandon_agent_task(agent_ref, agent_task, "SSE client disconnected")
|
|
logger.info("SSE client disconnected; interrupted agent task %s", completion_id)
|
|
except Exception as _exc:
|
|
# Agent crashed mid-stream. Try to emit an error chunk
|
|
# so the client gets a proper response instead of a
|
|
# TransferEncodingError from incomplete chunked encoding.
|
|
import traceback as _tb
|
|
logger.error("Agent crashed mid-stream for %s: %s", completion_id, _tb.format_exc()[:300])
|
|
try:
|
|
error_chunk = {
|
|
"id": completion_id, "object": "chat.completion.chunk",
|
|
"created": created, "model": model,
|
|
"choices": [{"index": 0, "delta": {}, "finish_reason": "error"}],
|
|
}
|
|
await response.write(_sse_frame(error_chunk))
|
|
await response.write(b"data: [DONE]\n\n")
|
|
except Exception:
|
|
pass
|
|
return response
|
|
|
|
async def _write_sse_responses(
|
|
self,
|
|
request: "web.Request",
|
|
response_id: str,
|
|
model: str,
|
|
created_at: int,
|
|
stream_q,
|
|
agent_task,
|
|
agent_ref,
|
|
conversation_history: List[Dict[str, str]],
|
|
user_message: str,
|
|
instructions: Optional[str],
|
|
conversation: Optional[str],
|
|
store: bool,
|
|
session_id: str,
|
|
gateway_session_key: Optional[str] = None,
|
|
) -> "web.StreamResponse":
|
|
"""Write the SSE stream for POST /v1/responses (OpenAI Responses API).
|
|
|
|
Events: ``response.created`` → ``response.output_text.delta/done`` and
|
|
``response.output_item.added/done`` (function_call / function_call_output)
|
|
→ ``response.completed`` (full envelope, same shape as non-streaming) or
|
|
``response.failed``. On disconnect the agent is interrupted and, when
|
|
``store=True``, an ``incomplete`` snapshot replaces the ``in_progress``
|
|
one so GET / ``previous_response_id`` chaining still work.
|
|
"""
|
|
from gateway.platforms.api_server import (
|
|
_abandon_agent_task,
|
|
_redact_api_error_text,
|
|
_responses_usage_payload,
|
|
_sse_frame,
|
|
)
|
|
response = await self._prepare_sse_response(request, session_id, gateway_session_key)
|
|
|
|
# State accumulated during the stream
|
|
final_text_parts: List[str] = []
|
|
# Track open function_call items by name so we can emit a matching
|
|
# ``done`` event when the tool completes. Order preserved.
|
|
pending_tool_calls: List[Dict[str, Any]] = []
|
|
# Output items we've emitted so far (used to build the terminal
|
|
# response.completed payload). Kept in the order they appeared.
|
|
emitted_items: List[Dict[str, Any]] = []
|
|
# Monotonic counter for output_index (spec requires it).
|
|
output_index = 0
|
|
# Monotonic counter for call_id generation if the agent doesn't
|
|
# provide one (it doesn't, from tool_progress_callback).
|
|
call_counter = 0
|
|
# Canonical Responses SSE events include a monotonically increasing
|
|
# sequence_number. Add it server-side for every emitted event so
|
|
# clients that validate the OpenAI event schema can parse our stream.
|
|
sequence_number = 0
|
|
# Track the assistant message item id + content index for text
|
|
# delta events — the spec ties deltas to a specific item.
|
|
message_item_id = f"msg_{uuid.uuid4().hex[:24]}"
|
|
message_output_index: Optional[int] = None
|
|
message_opened = False
|
|
|
|
async def _write_event(event_type: str, data: Dict[str, Any]) -> None:
|
|
nonlocal sequence_number
|
|
if "sequence_number" not in data:
|
|
data["sequence_number"] = sequence_number
|
|
sequence_number += 1
|
|
await response.write(_sse_frame(data, event=event_type))
|
|
|
|
def _envelope(status: str) -> Dict[str, Any]:
|
|
env: Dict[str, Any] = {
|
|
"id": response_id,
|
|
"object": "response",
|
|
"status": status,
|
|
"created_at": created_at,
|
|
"model": model,
|
|
}
|
|
return env
|
|
final_response_text = ""
|
|
agent_error: Optional[str] = None
|
|
usage: Dict[str, int] = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
|
|
terminal_snapshot_persisted = False
|
|
|
|
def _persist_response_snapshot(
|
|
response_env: Dict[str, Any],
|
|
*,
|
|
conversation_history_snapshot: Optional[List[Dict[str, Any]]] = None,
|
|
session_id_snapshot: Optional[str] = None,
|
|
) -> None:
|
|
if not store:
|
|
return
|
|
if conversation_history_snapshot is None:
|
|
conversation_history_snapshot = list(conversation_history)
|
|
conversation_history_snapshot.append({"role": "user", "content": user_message})
|
|
self._response_store.put(response_id, {
|
|
"response": response_env,
|
|
"conversation_history": conversation_history_snapshot,
|
|
"instructions": instructions,
|
|
"session_id": session_id_snapshot or session_id,
|
|
})
|
|
if conversation:
|
|
self._response_store.set_conversation(conversation, response_id)
|
|
|
|
def _persist_incomplete_if_needed() -> None:
|
|
"""Persist an ``incomplete`` snapshot if no terminal one was written.
|
|
|
|
Called from both the client-disconnect (``ConnectionResetError``)
|
|
and server-cancellation (``asyncio.CancelledError``) paths so
|
|
GET /v1/responses/{id} and ``previous_response_id`` chaining keep
|
|
working after abrupt stream termination.
|
|
"""
|
|
if not store or terminal_snapshot_persisted:
|
|
return
|
|
incomplete_text = "".join(final_text_parts) or final_response_text
|
|
incomplete_items: List[Dict[str, Any]] = list(emitted_items)
|
|
if incomplete_text:
|
|
incomplete_items.append({
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": [{"type": "output_text", "text": incomplete_text}],
|
|
})
|
|
incomplete_env = _envelope("incomplete")
|
|
incomplete_env["output"] = incomplete_items
|
|
incomplete_env["usage"] = _responses_usage_payload(usage)
|
|
incomplete_history = list(conversation_history)
|
|
incomplete_history.append({"role": "user", "content": user_message})
|
|
if incomplete_text:
|
|
incomplete_history.append({"role": "assistant", "content": incomplete_text})
|
|
_persist_response_snapshot(
|
|
incomplete_env,
|
|
conversation_history_snapshot=incomplete_history,
|
|
)
|
|
try:
|
|
# response.created — initial envelope, status=in_progress
|
|
created_env = _envelope("in_progress")
|
|
created_env["output"] = []
|
|
await _write_event("response.created", {
|
|
"type": "response.created",
|
|
"response": created_env,
|
|
})
|
|
_persist_response_snapshot(created_env)
|
|
|
|
async def _open_message_item() -> None:
|
|
"""Emit response.output_item.added for the assistant message
|
|
the first time any text delta arrives."""
|
|
nonlocal message_opened, message_output_index, output_index
|
|
if message_opened:
|
|
return
|
|
message_opened = True
|
|
message_output_index = output_index
|
|
output_index += 1
|
|
item = {
|
|
"id": message_item_id,
|
|
"type": "message",
|
|
"status": "in_progress",
|
|
"role": "assistant",
|
|
"content": [],
|
|
}
|
|
await _write_event("response.output_item.added", {
|
|
"type": "response.output_item.added",
|
|
"output_index": message_output_index,
|
|
"item": item,
|
|
})
|
|
|
|
async def _emit_text_delta(delta_text: str) -> None:
|
|
await _open_message_item()
|
|
final_text_parts.append(delta_text)
|
|
await _write_event("response.output_text.delta", {
|
|
"type": "response.output_text.delta",
|
|
"item_id": message_item_id,
|
|
"output_index": message_output_index,
|
|
"content_index": 0,
|
|
"delta": delta_text,
|
|
"logprobs": [],
|
|
})
|
|
|
|
async def _emit_tool_started(payload: Dict[str, Any]) -> str:
|
|
"""Emit response.output_item.added for a function_call.
|
|
|
|
Returns the call_id so the matching completion event can
|
|
reference it. Prefer the real ``tool_call_id`` from the
|
|
agent when available; fall back to a generated call id for
|
|
safety in tests or older code paths.
|
|
"""
|
|
nonlocal output_index, call_counter
|
|
call_counter += 1
|
|
call_id = payload.get("tool_call_id") or f"call_{response_id[5:]}_{call_counter}"
|
|
args = payload.get("arguments", {})
|
|
if isinstance(args, dict):
|
|
arguments_str = json.dumps(args)
|
|
else:
|
|
arguments_str = str(args)
|
|
item = {
|
|
"id": f"fc_{uuid.uuid4().hex[:24]}",
|
|
"type": "function_call",
|
|
"status": "in_progress",
|
|
"name": payload.get("name", ""),
|
|
"call_id": call_id,
|
|
"arguments": arguments_str,
|
|
}
|
|
idx = output_index
|
|
output_index += 1
|
|
pending_tool_calls.append({
|
|
"call_id": call_id,
|
|
"name": payload.get("name", ""),
|
|
"arguments": arguments_str,
|
|
"item_id": item["id"],
|
|
"output_index": idx,
|
|
})
|
|
emitted_items.append({
|
|
"type": "function_call",
|
|
"name": payload.get("name", ""),
|
|
"arguments": arguments_str,
|
|
"call_id": call_id,
|
|
})
|
|
await _write_event("response.output_item.added", {
|
|
"type": "response.output_item.added",
|
|
"output_index": idx,
|
|
"item": item,
|
|
})
|
|
return call_id
|
|
|
|
async def _emit_tool_completed(payload: Dict[str, Any]) -> None:
|
|
"""Emit response.output_item.done (function_call) followed
|
|
by response.output_item.added (function_call_output)."""
|
|
nonlocal output_index
|
|
call_id = payload.get("tool_call_id")
|
|
result = payload.get("result", "")
|
|
pending = None
|
|
if call_id:
|
|
for i, p in enumerate(pending_tool_calls):
|
|
if p["call_id"] == call_id:
|
|
pending = pending_tool_calls.pop(i)
|
|
break
|
|
if pending is None:
|
|
# Completion without a matching start — skip to avoid
|
|
# emitting orphaned done events.
|
|
return
|
|
|
|
# function_call done
|
|
done_item = {
|
|
"id": pending["item_id"],
|
|
"type": "function_call",
|
|
"status": "completed",
|
|
"name": pending["name"],
|
|
"call_id": pending["call_id"],
|
|
"arguments": pending["arguments"],
|
|
}
|
|
await _write_event("response.output_item.done", {
|
|
"type": "response.output_item.done",
|
|
"output_index": pending["output_index"],
|
|
"item": done_item,
|
|
})
|
|
|
|
# function_call_output added (result)
|
|
result_str = result if isinstance(result, str) else json.dumps(result)
|
|
output_parts = [{"type": "input_text", "text": result_str}]
|
|
output_item = {
|
|
"id": f"fco_{uuid.uuid4().hex[:24]}",
|
|
"type": "function_call_output",
|
|
"call_id": pending["call_id"],
|
|
"output": output_parts,
|
|
"status": "completed",
|
|
}
|
|
idx = output_index
|
|
output_index += 1
|
|
emitted_items.append({
|
|
"type": "function_call_output",
|
|
"call_id": pending["call_id"],
|
|
"output": output_parts,
|
|
})
|
|
await _write_event("response.output_item.added", {
|
|
"type": "response.output_item.added",
|
|
"output_index": idx,
|
|
"item": output_item,
|
|
})
|
|
await _write_event("response.output_item.done", {
|
|
"type": "response.output_item.done",
|
|
"output_index": idx,
|
|
"item": output_item,
|
|
})
|
|
|
|
# Main drain loop — thread-safe queue fed by agent callbacks.
|
|
async def _dispatch(it) -> None:
|
|
"""Route a queue item to the correct SSE emitter.
|
|
|
|
Plain strings are text deltas — they are batched (50ms)
|
|
to reduce Open WebUI re-render storms. Tagged tuples
|
|
with ``__tool_started__`` / ``__tool_completed__``
|
|
prefixes are tool lifecycle events and flush the buffer
|
|
before emitting.
|
|
"""
|
|
nonlocal _batch_timer
|
|
if isinstance(it, tuple) and len(it) == 2 and isinstance(it[0], str):
|
|
tag, payload = it
|
|
# Flush batched text before tool events
|
|
if _batch_buf:
|
|
await _flush_batch()
|
|
if tag == "__tool_started__":
|
|
await _emit_tool_started(payload)
|
|
elif tag == "__tool_completed__":
|
|
await _emit_tool_completed(payload)
|
|
elif isinstance(it, str):
|
|
# Batch text deltas — append to buffer, flush on timer
|
|
_batch_buf.append(it)
|
|
if _batch_timer is None:
|
|
_batch_timer = asyncio.create_task(_batch_flush_after(0.05))
|
|
# Other types are silently dropped.
|
|
|
|
# ── Batching state ──
|
|
_batch_buf: List[str] = []
|
|
_batch_timer: Optional[asyncio.Task] = None
|
|
_batch_lock = asyncio.Lock()
|
|
|
|
async def _batch_flush_after(delay: float) -> None:
|
|
"""Wait delay seconds, then flush accumulated text deltas."""
|
|
try:
|
|
await asyncio.sleep(delay)
|
|
except asyncio.CancelledError:
|
|
return
|
|
# Clear timer reference BEFORE flush so new deltas
|
|
# can start a fresh timer while we emit
|
|
nonlocal _batch_buf, _batch_timer
|
|
_batch_timer = None
|
|
await _flush_batch()
|
|
|
|
async def _flush_batch() -> None:
|
|
"""Emit a single SSE delta for all accumulated text."""
|
|
nonlocal _batch_buf
|
|
async with _batch_lock:
|
|
if _batch_buf:
|
|
combined = "".join(_batch_buf)
|
|
_batch_buf = []
|
|
await _emit_text_delta(combined)
|
|
async for item in _iter_stream_items(stream_q, agent_task, response):
|
|
if item is None: # EOS sentinel
|
|
# Cancel pending timer and flush remaining batched text
|
|
if _batch_timer and not _batch_timer.done():
|
|
_batch_timer.cancel()
|
|
_batch_timer = None
|
|
if _batch_buf:
|
|
await _flush_batch()
|
|
break
|
|
await _dispatch(item)
|
|
|
|
# Flush any final batched text before processing result
|
|
if _batch_buf:
|
|
await _flush_batch()
|
|
|
|
# Pick up agent result + usage from the completed task
|
|
try:
|
|
result, agent_usage = await agent_task
|
|
usage = agent_usage or usage
|
|
# If the agent produced a final_response but no text
|
|
# deltas were streamed (e.g. some providers only emit
|
|
# the full response at the end), emit a single fallback
|
|
# delta so Responses clients still receive a live text part.
|
|
agent_final = result.get("final_response", "") if isinstance(result, dict) else ""
|
|
if agent_final and not final_text_parts:
|
|
await _emit_text_delta(agent_final)
|
|
if agent_final and not final_response_text:
|
|
final_response_text = agent_final
|
|
if isinstance(result, dict) and result.get("error") and not final_response_text:
|
|
agent_error = _redact_api_error_text(result["error"])
|
|
except Exception as e: # noqa: BLE001
|
|
logger.error("Error running agent for streaming responses: %s", e, exc_info=True)
|
|
agent_error = _redact_api_error_text(e)
|
|
|
|
# Close the message item if it was opened
|
|
final_response_text = "".join(final_text_parts) or final_response_text
|
|
if message_opened:
|
|
await _write_event("response.output_text.done", {
|
|
"type": "response.output_text.done",
|
|
"item_id": message_item_id,
|
|
"output_index": message_output_index,
|
|
"content_index": 0,
|
|
"text": final_response_text,
|
|
"logprobs": [],
|
|
})
|
|
msg_done_item = {
|
|
"id": message_item_id,
|
|
"type": "message",
|
|
"status": "completed",
|
|
"role": "assistant",
|
|
"content": [{"type": "output_text", "text": final_response_text}],
|
|
}
|
|
await _write_event("response.output_item.done", {
|
|
"type": "response.output_item.done",
|
|
"output_index": message_output_index,
|
|
"item": msg_done_item,
|
|
})
|
|
|
|
# Always append a final message item in the completed
|
|
# response envelope so clients that only parse the terminal
|
|
# payload still see the assistant text. This mirrors the
|
|
# shape produced by _extract_output_items in the batch path.
|
|
final_items: List[Dict[str, Any]] = list(emitted_items)
|
|
|
|
# Trim large content from tool call arguments to keep the
|
|
# response.completed event under ~100KB. Clients already
|
|
# received full details via incremental events.
|
|
for _item in final_items:
|
|
if _item.get("type") == "function_call":
|
|
try:
|
|
_args = json.loads(_item.get("arguments", "{}")) if isinstance(_item.get("arguments"), str) else _item.get("arguments", {})
|
|
if isinstance(_args, dict):
|
|
for _k in ("content", "query", "pattern", "old_string", "new_string"):
|
|
if isinstance(_args.get(_k), str) and len(_args[_k]) > 500:
|
|
_args[_k] = "[" + str(len(_args[_k])) + " chars — truncated for response.completed]"
|
|
_item["arguments"] = json.dumps(_args)
|
|
except Exception:
|
|
pass
|
|
elif _item.get("type") == "function_call_output":
|
|
_output = _item.get("output", [])
|
|
if isinstance(_output, list) and _output:
|
|
_first = _output[0]
|
|
if isinstance(_first, dict) and _first.get("type") == "input_text":
|
|
_text = _first.get("text", "")
|
|
if len(_text) > 1000:
|
|
_first["text"] = _text[:500] + "...[" + str(len(_text) - 500) + " more chars]"
|
|
_item["output"] = [_first]
|
|
final_items.append({
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": [
|
|
{"type": "output_text", "text": final_response_text or (_redact_api_error_text(agent_error) if agent_error else "")}
|
|
],
|
|
})
|
|
if agent_error:
|
|
failed_env = _envelope("failed")
|
|
failed_env["output"] = final_items
|
|
failed_env["error"] = {"message": _redact_api_error_text(agent_error), "type": "server_error"}
|
|
failed_env["usage"] = _responses_usage_payload(usage)
|
|
_failed_history = list(conversation_history)
|
|
_failed_history.append({"role": "user", "content": user_message})
|
|
if final_response_text or agent_error:
|
|
_failed_history.append({
|
|
"role": "assistant",
|
|
"content": final_response_text or _redact_api_error_text(agent_error),
|
|
})
|
|
_persist_response_snapshot(
|
|
failed_env,
|
|
conversation_history_snapshot=_failed_history,
|
|
)
|
|
terminal_snapshot_persisted = True
|
|
await _write_event("response.failed", {
|
|
"type": "response.failed",
|
|
"response": failed_env,
|
|
})
|
|
else:
|
|
completed_env = _envelope("completed")
|
|
completed_env["output"] = final_items
|
|
completed_env["usage"] = _responses_usage_payload(usage)
|
|
full_history = self._build_response_conversation_history(
|
|
conversation_history,
|
|
user_message,
|
|
result,
|
|
final_response_text,
|
|
)
|
|
# Compression-aware transcript substitution happens inside
|
|
# _build_response_conversation_history (result["_compressed"]);
|
|
# here we only propagate a compression-rotated session_id so
|
|
# previous_response_id chaining resumes the child session.
|
|
_result_sid = result.get("session_id") if isinstance(result, dict) else None
|
|
_persist_response_snapshot(
|
|
completed_env,
|
|
conversation_history_snapshot=full_history,
|
|
session_id_snapshot=_result_sid if isinstance(_result_sid, str) and _result_sid else None,
|
|
)
|
|
terminal_snapshot_persisted = True
|
|
await _write_event("response.completed", {
|
|
"type": "response.completed",
|
|
"response": completed_env,
|
|
})
|
|
except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError, OSError):
|
|
_persist_incomplete_if_needed()
|
|
await _abandon_agent_task(agent_ref, agent_task, "SSE client disconnected")
|
|
logger.info("SSE client disconnected; interrupted agent task %s", response_id)
|
|
except asyncio.CancelledError:
|
|
# Server-side cancellation (e.g. shutdown, request timeout) —
|
|
# persist an incomplete snapshot so GET /v1/responses/{id} and
|
|
# previous_response_id chaining still work, then re-raise so the
|
|
# runtime's cancellation semantics are respected.
|
|
_persist_incomplete_if_needed()
|
|
await _abandon_agent_task(
|
|
agent_ref, agent_task, "SSE task cancelled",
|
|
reap_source="api_server_sse_cancelled", await_cancel=False,
|
|
)
|
|
logger.info("SSE task cancelled; persisted incomplete snapshot for %s", response_id)
|
|
raise
|
|
except Exception as _exc:
|
|
# Agent crashed with an unhandled error (e.g. model API error like
|
|
# BadRequestError, AuthenticationError). Emit a response.failed
|
|
# event and properly terminate the SSE stream so the client doesn't
|
|
# get a TransferEncodingError from incomplete chunked encoding.
|
|
import traceback as _tb
|
|
_persist_incomplete_if_needed()
|
|
agent_error = _redact_api_error_text(_tb.format_exc())
|
|
try:
|
|
failed_env = _envelope("failed")
|
|
failed_env["output"] = list(emitted_items)
|
|
failed_env["error"] = {"message": _redact_api_error_text(_exc, limit=500), "type": "server_error"}
|
|
failed_env["usage"] = _responses_usage_payload(usage)
|
|
await _write_event("response.failed", {
|
|
"type": "response.failed",
|
|
"response": failed_env,
|
|
})
|
|
except Exception:
|
|
pass
|
|
logger.error("Agent crashed mid-stream for %s: %s", response_id, str(agent_error)[:300])
|
|
return response
|
|
|
|
async def _handle_responses(self, request: "web.Request") -> "web.Response":
|
|
"""POST /v1/responses — OpenAI Responses API format."""
|
|
from gateway.platforms.api_server import (
|
|
ThreadSafeAsyncQueue,
|
|
_auto_truncate_response_history,
|
|
_coerce_request_bool,
|
|
_content_has_visible_payload,
|
|
_error_response,
|
|
_multimodal_validation_error,
|
|
_normalize_multimodal_content,
|
|
_redact_api_error_text,
|
|
_request_agent_overrides,
|
|
_resolve_media_to_data_urls,
|
|
_responses_usage_payload,
|
|
)
|
|
# Bound total in-flight agent runs (configurable; #7483).
|
|
limited = self._concurrency_limited_response()
|
|
if limited is not None:
|
|
return limited
|
|
|
|
# Long-term memory scope header (see chat_completions for details).
|
|
gateway_session_key, key_err = self._parse_session_key_header(request)
|
|
if key_err is not None:
|
|
return key_err
|
|
|
|
# Parse request body
|
|
try:
|
|
body = await request.json()
|
|
except (json.JSONDecodeError, Exception):
|
|
return web.json_response(
|
|
{"error": {"message": "Invalid JSON in request body", "type": "invalid_request_error"}},
|
|
status=400,
|
|
)
|
|
raw_input = body.get("input")
|
|
if raw_input is None:
|
|
return _error_response("Missing 'input' field", 400)
|
|
instructions = body.get("instructions")
|
|
previous_response_id = body.get("previous_response_id")
|
|
conversation = body.get("conversation")
|
|
store = _coerce_request_bool(body.get("store"), default=True)
|
|
|
|
# conversation and previous_response_id are mutually exclusive
|
|
if conversation and previous_response_id:
|
|
return _error_response("Cannot use both 'conversation' and 'previous_response_id'", 400)
|
|
|
|
# Resolve conversation name to latest response_id
|
|
if conversation:
|
|
previous_response_id = self._response_store.get_conversation(conversation)
|
|
# No error if conversation doesn't exist yet — it's a new conversation
|
|
|
|
# Normalize input to message list
|
|
input_messages: List[Dict[str, Any]] = []
|
|
if isinstance(raw_input, str):
|
|
input_messages = [{"role": "user", "content": raw_input}]
|
|
elif isinstance(raw_input, list):
|
|
for idx, item in enumerate(raw_input):
|
|
if isinstance(item, str):
|
|
input_messages.append({"role": "user", "content": item})
|
|
elif isinstance(item, dict):
|
|
role = item.get("role", "user")
|
|
try:
|
|
content = _normalize_multimodal_content(item.get("content", ""))
|
|
except ValueError as exc:
|
|
return _multimodal_validation_error(exc, param=f"input[{idx}].content")
|
|
input_messages.append({"role": role, "content": content})
|
|
else:
|
|
return _error_response("'input' must be a string or array", 400)
|
|
|
|
# Accept explicit conversation_history from the request body.
|
|
# This lets stateless clients supply their own history instead of
|
|
# relying on server-side response chaining via previous_response_id.
|
|
# Precedence: explicit conversation_history > previous_response_id.
|
|
conversation_history: List[Dict[str, Any]] = []
|
|
raw_history = body.get("conversation_history")
|
|
if raw_history:
|
|
if not isinstance(raw_history, list):
|
|
return _error_response("'conversation_history' must be an array of message objects", 400)
|
|
for i, entry in enumerate(raw_history):
|
|
if not isinstance(entry, dict) or "role" not in entry or "content" not in entry:
|
|
return _error_response(f"conversation_history[{i}] must have 'role' and 'content' fields", 400)
|
|
try:
|
|
entry_content = _normalize_multimodal_content(entry["content"])
|
|
except ValueError as exc:
|
|
return _multimodal_validation_error(exc, param=f"conversation_history[{i}].content")
|
|
conversation_history.append({"role": str(entry["role"]), "content": entry_content})
|
|
if previous_response_id:
|
|
logger.debug("Both conversation_history and previous_response_id provided; using conversation_history")
|
|
stored_session_id = None
|
|
if not conversation_history and previous_response_id:
|
|
stored = self._response_store.get(previous_response_id)
|
|
if stored is None:
|
|
return _error_response(f"Previous response not found: {previous_response_id}", 404)
|
|
conversation_history = list(stored.get("conversation_history", []))
|
|
stored_session_id = stored.get("session_id")
|
|
# If no instructions provided, carry forward from previous
|
|
if instructions is None:
|
|
instructions = stored.get("instructions")
|
|
|
|
# Append new input messages to history (all but the last become history)
|
|
for msg in input_messages[:-1]:
|
|
conversation_history.append(msg)
|
|
|
|
# Last input message is the user_message
|
|
user_message: Any = input_messages[-1].get("content", "") if input_messages else ""
|
|
if not _content_has_visible_payload(user_message):
|
|
return _error_response("No user message found in input", 400)
|
|
|
|
# Truncation support
|
|
if body.get("truncation") == "auto":
|
|
conversation_history = _auto_truncate_response_history(conversation_history)
|
|
|
|
# Session precedence: previous_response_id chain > declared X-Hermes-Session-Key
|
|
# > fresh id. Binding the declared key is gated on that same precedence — a
|
|
# chain-selected session must not have its routing key rewritten to this header.
|
|
_declared_selected = not stored_session_id and bool(gateway_session_key)
|
|
session_id = (
|
|
stored_session_id
|
|
or self._declared_conversation_session(gateway_session_key)
|
|
or str(uuid.uuid4())
|
|
)
|
|
stream = _coerce_request_bool(body.get("stream"), default=False)
|
|
route = self._resolve_route(body.get("model"))
|
|
agent_overrides = _request_agent_overrides(
|
|
body,
|
|
virtual_model=self._model_name,
|
|
allow_bare_model=self._direct_model_requests,
|
|
)
|
|
selection_error = self._request_route_conflict_error(
|
|
session_id=session_id,
|
|
gateway_session_key=gateway_session_key,
|
|
requested_model=agent_overrides.get("requested_model"),
|
|
requested_provider=agent_overrides.get("requested_provider"),
|
|
route=route,
|
|
)
|
|
if selection_error:
|
|
return _error_response(selection_error, 400)
|
|
if stream:
|
|
# Streaming branch — emit OpenAI Responses SSE events as the
|
|
# agent runs so frontends can render text deltas and tool
|
|
# calls in real time. See _write_sse_responses for details.
|
|
_stream_q = ThreadSafeAsyncQueue()
|
|
|
|
def _on_delta(delta):
|
|
# None from the agent is a CLI box-close signal, not EOS.
|
|
# Forwarding would kill the SSE stream prematurely; the
|
|
# SSE writer detects completion via agent_task.done().
|
|
# Called from the worker thread running run_conversation —
|
|
# put_threadsafe (not put_nowait) is required here.
|
|
if delta is not None:
|
|
_stream_q.put_threadsafe(delta)
|
|
|
|
def _on_tool_progress(event_type, name, preview, args, **kwargs):
|
|
"""Queue non-start tool progress events if needed in future.
|
|
|
|
The structured Responses stream uses ``tool_start_callback``
|
|
and ``tool_complete_callback`` for exact call-id correlation,
|
|
so progress events are currently ignored here.
|
|
"""
|
|
return
|
|
|
|
def _on_tool_start(tool_call_id, function_name, function_args):
|
|
"""Queue a started tool for live function_call streaming."""
|
|
_stream_q.put_threadsafe(("__tool_started__", {
|
|
"tool_call_id": tool_call_id,
|
|
"name": function_name,
|
|
"arguments": function_args or {},
|
|
}))
|
|
|
|
def _on_tool_complete(tool_call_id, function_name, function_args, function_result):
|
|
"""Queue a completed tool result for live function_call_output streaming."""
|
|
_stream_q.put_threadsafe(("__tool_completed__", {
|
|
"tool_call_id": tool_call_id,
|
|
"name": function_name,
|
|
"arguments": function_args or {},
|
|
"result": function_result,
|
|
}))
|
|
agent_ref = [None]
|
|
agent_task = asyncio.ensure_future(self._run_agent(
|
|
user_message=user_message,
|
|
conversation_history=conversation_history,
|
|
ephemeral_system_prompt=instructions,
|
|
session_id=session_id,
|
|
stream_delta_callback=_on_delta,
|
|
tool_progress_callback=_on_tool_progress,
|
|
tool_start_callback=_on_tool_start,
|
|
tool_complete_callback=_on_tool_complete,
|
|
agent_ref=agent_ref,
|
|
gateway_session_key=gateway_session_key,
|
|
bind_declared_conversation=_declared_selected,
|
|
**agent_overrides,
|
|
route=route,
|
|
))
|
|
# Ensure SSE drain loops can terminate without relying on polling
|
|
# agent_task.done(), which can race with queue timeout checks.
|
|
agent_task.add_done_callback(lambda _fut: _stream_q.put_nowait(None))
|
|
response_id = f"resp_{uuid.uuid4().hex[:28]}"
|
|
model_name = body.get("model", self._model_name)
|
|
created_at = int(time.time())
|
|
return await self._write_sse_responses(
|
|
request=request,
|
|
response_id=response_id,
|
|
model=model_name,
|
|
created_at=created_at,
|
|
stream_q=_stream_q,
|
|
agent_task=agent_task,
|
|
agent_ref=agent_ref,
|
|
conversation_history=conversation_history,
|
|
user_message=user_message,
|
|
instructions=instructions,
|
|
conversation=conversation,
|
|
store=store,
|
|
session_id=session_id,
|
|
gateway_session_key=gateway_session_key,
|
|
)
|
|
|
|
async def _compute_response():
|
|
return await self._run_agent(
|
|
user_message=user_message,
|
|
conversation_history=conversation_history,
|
|
ephemeral_system_prompt=instructions,
|
|
session_id=session_id,
|
|
gateway_session_key=gateway_session_key,
|
|
bind_declared_conversation=_declared_selected,
|
|
**agent_overrides,
|
|
route=route,
|
|
)
|
|
outcome, err = await self._run_idempotent(
|
|
request, body, _compute_response, log_label="responses",
|
|
fingerprint_keys=["input", "instructions", "previous_response_id", "conversation", "model", "provider", "model_options", "tools"],
|
|
)
|
|
if err is not None:
|
|
return err
|
|
result, usage = outcome
|
|
final_response = _resolve_media_to_data_urls(result.get("final_response", ""))
|
|
if not final_response:
|
|
final_response = _redact_api_error_text(result.get("error", "(No response generated)"))
|
|
response_id = f"resp_{uuid.uuid4().hex[:28]}"
|
|
created_at = int(time.time())
|
|
|
|
# Build the full conversation history for storage
|
|
# (includes tool calls from the agent run)
|
|
full_history = self._build_response_conversation_history(
|
|
conversation_history,
|
|
user_message,
|
|
result,
|
|
final_response,
|
|
)
|
|
|
|
# Persist the effective session ID surfaced by _run_agent so that
|
|
# compression-triggered session rotations propagate to the stored
|
|
# response and the X-Hermes-Session-Id header. Without this,
|
|
# previous_response_id chaining keeps resuming the pre-rotation
|
|
# session and re-triggers compression on every subsequent request.
|
|
_effective_session_id = session_id
|
|
_result_sid = result.get("session_id") if isinstance(result, dict) else None
|
|
if isinstance(_result_sid, str) and _result_sid:
|
|
_effective_session_id = _result_sid
|
|
|
|
# Build output items from the current turn only. AIAgent returns a
|
|
# full transcript in result["messages"], while older/mocked paths may
|
|
# return only the current turn suffix.
|
|
output_start_index = self._response_messages_turn_start_index(
|
|
conversation_history,
|
|
user_message,
|
|
result,
|
|
)
|
|
output_items = self._extract_output_items(result, start_index=output_start_index)
|
|
response_data = {
|
|
"id": response_id,
|
|
"object": "response",
|
|
"status": "completed",
|
|
"created_at": created_at,
|
|
"model": body.get("model", self._model_name),
|
|
"output": output_items,
|
|
"usage": _responses_usage_payload(usage),
|
|
}
|
|
|
|
# Store the complete response object for future chaining / GET retrieval
|
|
if store:
|
|
self._response_store.put(response_id, {
|
|
"response": response_data,
|
|
"conversation_history": full_history,
|
|
"instructions": instructions,
|
|
"session_id": _effective_session_id,
|
|
})
|
|
# Update conversation mapping so the next request with the same
|
|
# conversation name automatically chains to this response
|
|
if conversation:
|
|
self._response_store.set_conversation(conversation, response_id)
|
|
response_headers = {"X-Hermes-Session-Id": _effective_session_id}
|
|
if gateway_session_key:
|
|
response_headers["X-Hermes-Session-Key"] = gateway_session_key
|
|
return web.json_response(response_data, headers=response_headers)
|
|
|
|
async def _handle_get_response(self, request: "web.Request") -> "web.Response":
|
|
"""GET /v1/responses/{response_id} — retrieve a stored response."""
|
|
from gateway.platforms.api_server import _error_response
|
|
auth_err = self._check_auth(request)
|
|
if auth_err:
|
|
return auth_err
|
|
response_id = request.match_info["response_id"]
|
|
stored = self._response_store.get(response_id)
|
|
if stored is None:
|
|
return _error_response(f"Response not found: {response_id}", 404)
|
|
return web.json_response(stored["response"])
|
|
|
|
async def _handle_delete_response(self, request: "web.Request") -> "web.Response":
|
|
"""DELETE /v1/responses/{response_id} — delete a stored response."""
|
|
from gateway.platforms.api_server import _error_response
|
|
auth_err = self._check_auth(request)
|
|
if auth_err:
|
|
return auth_err
|
|
response_id = request.match_info["response_id"]
|
|
deleted = self._response_store.delete(response_id)
|
|
if not deleted:
|
|
return _error_response(f"Response not found: {response_id}", 404)
|
|
return web.json_response({"id": response_id, "object": "response", "deleted": True})
|
|
|
|
@staticmethod
|
|
def _build_response_conversation_history(
|
|
conversation_history: List[Dict[str, Any]],
|
|
user_message: Any,
|
|
result: Dict[str, Any],
|
|
final_response: Any,
|
|
) -> List[Dict[str, Any]]:
|
|
"""Build the stored Responses transcript without duplicating history.
|
|
|
|
When context compression occurs during a turn the agent returns a
|
|
compressed full transcript in ``result["messages"]`` (starting with a
|
|
summary) and sets ``result["_compressed"] = True``. Because the
|
|
compressed transcript does not share the input ``conversation_history``
|
|
prefix, the normal turn-start detection fails and old code would
|
|
concatenate the uncompressed history on front, bloating the stored
|
|
context and re-triggering compression on every subsequent request.
|
|
"""
|
|
from gateway.platforms.api_server import APIServerAdapter
|
|
prior = list(conversation_history)
|
|
current_user = {"role": "user", "content": user_message}
|
|
agent_messages = result.get("messages") if isinstance(result, dict) else None
|
|
if isinstance(agent_messages, list) and agent_messages:
|
|
turn_start = APIServerAdapter._response_messages_turn_start_index(
|
|
conversation_history,
|
|
user_message,
|
|
result,
|
|
)
|
|
if turn_start:
|
|
return list(agent_messages)
|
|
|
|
# turn_start == 0: either compression rewrote the transcript (use it as-is —
|
|
# the _compressed flag says so) or agent_messages is only the current turn.
|
|
if result.get("_compressed"):
|
|
return list(agent_messages)
|
|
full_history = prior
|
|
full_history.append(current_user)
|
|
full_history.extend(agent_messages)
|
|
return full_history
|
|
full_history = prior
|
|
full_history.append(current_user)
|
|
full_history.append({"role": "assistant", "content": final_response})
|
|
return full_history
|
|
|
|
@staticmethod
|
|
def _response_messages_turn_start_index(
|
|
conversation_history: List[Dict[str, Any]],
|
|
user_message: Any,
|
|
result: Dict[str, Any],
|
|
) -> int:
|
|
"""Detect transcript-shaped result["messages"] and return turn start."""
|
|
agent_messages = result.get("messages") if isinstance(result, dict) else None
|
|
if not isinstance(agent_messages, list) or not agent_messages:
|
|
return 0
|
|
prior = list(conversation_history)
|
|
current_user = {"role": "user", "content": user_message}
|
|
expected_prefix = prior + [current_user]
|
|
if agent_messages[:len(expected_prefix)] == expected_prefix:
|
|
return len(expected_prefix)
|
|
if prior and agent_messages[:len(prior)] == prior:
|
|
return len(prior)
|
|
return 0
|
|
|
|
@classmethod
|
|
def _turn_transcript_messages(
|
|
cls,
|
|
conversation_history: List[Dict[str, Any]],
|
|
user_message: Any,
|
|
result: Dict[str, Any],
|
|
) -> List[Dict[str, Any]]:
|
|
"""Return this turn's assistant/tool messages in client-safe shape.
|
|
|
|
The streaming SSE contract delivers all assistant text as
|
|
``assistant.delta`` events under one ``message_id`` interleaved with
|
|
``tool.*`` events, and a single ``assistant.completed`` carrying only
|
|
the final reply. A client that accumulates deltas into one buffer
|
|
cannot reconstruct *intermediate* assistant text segments that preceded
|
|
tool calls — so when the page is re-opened mid/post-stream those
|
|
segments appear lost, even though state.db persisted them correctly.
|
|
|
|
Emitting the authoritative per-turn transcript on ``run.completed`` lets
|
|
any SSE consumer reconcile its live view against ground truth without a
|
|
separate ``GET /messages`` round-trip. Purely additive: clients that
|
|
ignore the field are unaffected. Refs #34703.
|
|
"""
|
|
agent_messages = result.get("messages") if isinstance(result, dict) else None
|
|
if not isinstance(agent_messages, list) or not agent_messages:
|
|
return []
|
|
start = cls._response_messages_turn_start_index(conversation_history, user_message, result)
|
|
turn = agent_messages[start:]
|
|
out: List[Dict[str, Any]] = []
|
|
for msg in turn:
|
|
if not isinstance(msg, dict):
|
|
continue
|
|
if msg.get("role") not in {"assistant", "tool"}:
|
|
continue
|
|
# _message_response projects compaction scaffolding itself and
|
|
# marks pure handoffs display_kind == "hidden"; classifying here
|
|
# first would re-run the content classifier (a full content
|
|
# flatten + prefix scan) a second time per message.
|
|
projected = cls._message_response(msg)
|
|
if projected.get("display_kind") == "hidden":
|
|
continue
|
|
out.append(projected)
|
|
return out
|
|
|
|
@staticmethod
|
|
def _extract_output_items(result: Dict[str, Any], start_index: int = 0) -> List[Dict[str, Any]]:
|
|
"""
|
|
Build the output item array from the agent's messages.
|
|
|
|
Walks *result["messages"]* starting at *start_index* and emits:
|
|
- ``function_call`` items for each tool_call on assistant messages
|
|
- ``function_call_output`` items for each tool-role message
|
|
- a final ``message`` item with the assistant's text reply
|
|
"""
|
|
from gateway.platforms.api_server import _redact_api_error_text
|
|
items: List[Dict[str, Any]] = []
|
|
messages = result.get("messages", [])
|
|
if start_index > 0:
|
|
messages = messages[start_index:]
|
|
for msg in messages:
|
|
role = msg.get("role")
|
|
if role == "assistant" and msg.get("tool_calls"):
|
|
for tc in msg["tool_calls"]:
|
|
func = tc.get("function", {})
|
|
items.append({
|
|
"id": f"fc_{uuid.uuid4().hex[:24]}",
|
|
"type": "function_call",
|
|
# These calls were already executed server-side by the
|
|
# Hermes agent; they are replayed for structured tool
|
|
# UI only. Mark them completed (matching the SSE
|
|
# streaming path) so OpenAI clients don't interpret
|
|
# them as pending calls the client must execute.
|
|
"status": "completed",
|
|
"name": func.get("name", ""),
|
|
"arguments": func.get("arguments", ""),
|
|
"call_id": tc.get("id", ""),
|
|
})
|
|
elif role == "tool":
|
|
items.append({
|
|
"id": f"fco_{uuid.uuid4().hex[:24]}",
|
|
"type": "function_call_output",
|
|
"status": "completed",
|
|
"call_id": msg.get("tool_call_id", ""),
|
|
"output": msg.get("content", ""),
|
|
})
|
|
|
|
# Final assistant message
|
|
final = result.get("final_response", "")
|
|
if not final:
|
|
final = _redact_api_error_text(result.get("error", "(No response generated)"))
|
|
items.append({
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": [{"type": "output_text", "text": final}],
|
|
})
|
|
return items
|