"""OpenAI-compatible routes for the API server adapter. ``OpenAICompatRoutesMixin`` carries ``/v1/chat/completions``, ``/v1/responses`` (+ GET/DELETE), their two SSE writers, and the Responses-transcript helpers. ``APIServerAdapter`` inherits it; every ``self.*`` call resolves via the MRO. api_server-internal helpers are imported lazily inside each method (the origin imports this module, so a top-level import would be a cycle), which also keeps ``patch("gateway.platforms.api_server.X")`` effective for the moved bodies. """ import asyncio import json import logging import re import time import uuid from typing import Any, Dict, List, Optional try: from aiohttp import web except ImportError: # pragma: no cover - mirrors api_server's optional import web = None # type: ignore[assignment] # Logger parity with the origin module (moved log records keep their name). logger = logging.getLogger("gateway.platforms.api_server") async def _iter_stream_items(stream_q, agent_task, response): """Yield agent stream items until end-of-stream, writing SSE keepalives while idle. Woken directly by ``put_threadsafe`` (no executor hop / poll latency). Yields the ``None`` sentinel once so callers can run EOS-only work; when ``agent_task`` is already done the remaining queue is drained and the sentinel swallowed (matching the historical inline loops). """ from gateway.platforms.api_server import CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS last_activity = time.monotonic() while True: try: item = await asyncio.wait_for(stream_q.get(), timeout=0.5) except asyncio.TimeoutError: if agent_task.done(): while True: try: item = stream_q.get_nowait() except asyncio.QueueEmpty: return if item is None: return yield item last_activity = time.monotonic() if time.monotonic() - last_activity >= CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS: await response.write(b": keepalive\n\n") last_activity = time.monotonic() continue if item is None: yield None return yield item last_activity = time.monotonic() class OpenAICompatRoutesMixin: """/v1/chat/completions and /v1/responses handlers + SSE writers.""" async def _handle_chat_completions(self, request: "web.Request") -> "web.Response": """POST /v1/chat/completions — OpenAI Chat Completions format.""" from gateway.platforms.api_server import ( ThreadSafeAsyncQueue, _chat_usage_payload, _coerce_request_bool, _content_has_visible_payload, _derive_chat_session_id, _error_response, _multimodal_validation_error, _normalize_chat_content, _normalize_multimodal_content, _openai_error, _redact_api_error_text, _request_agent_overrides, _resolve_media_to_data_urls, ) # Bound total in-flight agent runs (configurable; #7483). limited = self._concurrency_limited_response() if limited is not None: return limited # Parse request body try: body = await request.json() except (json.JSONDecodeError, Exception): return _error_response("Invalid JSON in request body", 400) messages = body.get("messages") if not messages or not isinstance(messages, list): return web.json_response( {"error": {"message": "Missing or invalid 'messages' field", "type": "invalid_request_error"}}, status=400, ) stream = _coerce_request_bool(body.get("stream"), default=False) # Extract system message (becomes ephemeral system prompt layered ON TOP of core) system_prompt = None conversation_messages: List[Dict[str, str]] = [] for idx, msg in enumerate(messages): role = msg.get("role", "") raw_content = msg.get("content", "") if role == "system": # System messages don't support images (Anthropic rejects, OpenAI # text-model systems don't render them). Flatten to text. content = _normalize_chat_content(raw_content) if system_prompt is None: system_prompt = content else: system_prompt = system_prompt + "\n" + content elif role in {"user", "assistant"}: try: content = _normalize_multimodal_content(raw_content) except ValueError as exc: return _multimodal_validation_error(exc, param=f"messages[{idx}].content") conversation_messages.append({"role": role, "content": content}) # Extract the last user message as the primary input user_message: Any = "" history = [] if conversation_messages: user_message = conversation_messages[-1].get("content", "") history = conversation_messages[:-1] if not _content_has_visible_payload(user_message): return web.json_response( {"error": {"message": "No user message found in messages", "type": "invalid_request_error"}}, status=400, ) # Allow caller to scope long-term memory (e.g. Honcho) with a # stable per-channel identifier via X-Hermes-Session-Key. This # is independent of X-Hermes-Session-Id: the key persists across # transcripts while the id rotates when the caller starts a new # transcript (i.e. /new semantics). See _parse_session_key_header. gateway_session_key, key_err = self._parse_session_key_header(request) if key_err is not None: return key_err # X-Hermes-Session-Id continues an existing session (history from state.db, not # the body). Requires a configured API key: otherwise any client could read # arbitrary history by guessing session ids. provided_session_id = request.headers.get("X-Hermes-Session-Id", "").strip() if provided_session_id: if not self._api_key: logger.warning( "Session continuation via X-Hermes-Session-Id rejected: " "no API key configured. Set API_SERVER_KEY to enable " "session continuity." ) return _error_response("Session continuation requires API key authentication. " "Configure API_SERVER_KEY to enable this feature.", 403) # Sanitize: reject control characters that could enable header # injection, and path-traversal-shaped IDs that would escape the # sessions directory when interpolated into on-disk artifact # filenames (session snapshots, request dumps). Mirrors the native # gateway's entry-boundary guard (gateway.session._is_path_unsafe). from gateway.session import _is_path_unsafe if re.search(r'[\r\n\x00]', provided_session_id) or _is_path_unsafe(provided_session_id): return web.json_response( {"error": {"message": "Invalid session ID", "type": "invalid_request_error"}}, status=400, ) if len(provided_session_id) > self._MAX_SESSION_HEADER_LEN: return web.json_response( {"error": {"message": "Session ID too long", "type": "invalid_request_error"}}, status=400, ) session_id = provided_session_id try: db = await self._ensure_session_db_async() if db is not None: history = await asyncio.to_thread(db.get_messages_as_conversation, session_id) except Exception as e: logger.warning("Failed to load session history for %s: %s", session_id, e) history = [] else: # Derive a stable session ID from the conversation fingerprint so # that consecutive messages from the same Open WebUI (or similar) # conversation map to the same Hermes session. The first user # message + system prompt are constant across all turns. first_user = "" for cm in conversation_messages: if cm.get("role") == "user": first_user = cm.get("content", "") break session_id = _derive_chat_session_id(system_prompt, first_user) # history already set from request body above completion_id = f"chatcmpl-{uuid.uuid4().hex[:29]}" model_name = body.get("model", self._model_name) created = int(time.time()) # Per-client model routing: if the requested model matches a # configured model_routes alias, this request's agent is created # with that route's model/provider instead of the global default. route = self._resolve_route(model_name) agent_overrides = _request_agent_overrides( body, virtual_model=self._model_name, allow_bare_model=self._direct_model_requests, ) selection_error = self._request_route_conflict_error( session_id=session_id, gateway_session_key=gateway_session_key, requested_model=agent_overrides.get("requested_model"), requested_provider=agent_overrides.get("requested_provider"), route=route, ) if selection_error: return _error_response(selection_error, 400) if stream: _stream_q = ThreadSafeAsyncQueue() def _on_delta(delta): # None from the agent is a CLI box-close signal, not EOS — forwarding it # would end the stream before the post-tool answer. Called from the # run_conversation worker thread, so put_threadsafe is required. if delta is not None: _stream_q.put_threadsafe(delta) # Track which tool_call_ids we've emitted a "running" lifecycle # event for, so a "completed" event without a matching "running" # (e.g. internal/filtered tools) is silently dropped instead of # producing an orphaned event clients can't correlate. _started_tool_call_ids: set[str] = set() def _on_tool_start(tool_call_id, function_name, function_args): """Emit ``hermes.tool.progress`` with ``status: running``. Replaces the old ``tool_progress_callback("tool.started", ...)`` emit so SSE consumers receive a single event per tool start, carrying both the legacy ``tool``/``emoji``/ ``label`` payload (for #6972 frontends) and the new ``toolCallId``/``status`` correlation fields (#16588). Skips tools whose names start with ``_`` so internal events (``_thinking``, …) stay off the wire — matching the prior ``_on_tool_progress`` filter exactly. """ if not tool_call_id or function_name.startswith("_"): return _started_tool_call_ids.add(tool_call_id) from agent.display import build_tool_preview, get_tool_emoji label = build_tool_preview(function_name, function_args) or function_name _stream_q.put_threadsafe(("__tool_progress__", { "tool": function_name, "emoji": get_tool_emoji(function_name), "label": label, "toolCallId": tool_call_id, "status": "running", })) def _on_tool_complete(tool_call_id, function_name, function_args, function_result): """Emit the matching ``status: completed`` event. Dropped if the start was filtered (internal tool, missing id, or never seen) so clients never get an orphaned ``completed`` they can't correlate to a prior ``running``. """ if not tool_call_id or tool_call_id not in _started_tool_call_ids: return _started_tool_call_ids.discard(tool_call_id) _stream_q.put_threadsafe(("__tool_progress__", { "tool": function_name, "toolCallId": tool_call_id, "status": "completed", })) # agent_ref lets the SSE writer interrupt on disconnect. tool_progress_callback # is deliberately NOT wired: it fires alongside the structured start/complete # callbacks (which carry the tool_call id) and would duplicate every emit. agent_ref = [None] agent_task = asyncio.ensure_future(self._run_agent( user_message=user_message, conversation_history=history, ephemeral_system_prompt=system_prompt, session_id=session_id, stream_delta_callback=_on_delta, tool_start_callback=_on_tool_start, tool_complete_callback=_on_tool_complete, agent_ref=agent_ref, gateway_session_key=gateway_session_key, **agent_overrides, route=route, )) # Ensure SSE drain loops can terminate without relying on polling # agent_task.done(), which can race with queue timeout checks. agent_task.add_done_callback(lambda _fut: _stream_q.put_nowait(None)) return await self._write_sse_chat_completion( request, completion_id, model_name, created, _stream_q, agent_task, agent_ref, session_id=session_id, gateway_session_key=gateway_session_key, ) # Non-streaming: run the agent (with optional Idempotency-Key) async def _compute_completion(): return await self._run_agent( user_message=user_message, conversation_history=history, ephemeral_system_prompt=system_prompt, session_id=session_id, gateway_session_key=gateway_session_key, **agent_overrides, route=route, ) outcome, err = await self._run_idempotent( request, body, _compute_completion, log_label="chat completions", fingerprint_keys=["model", "provider", "model_options", "messages", "tools", "tool_choice", "stream"], ) if err is not None: return err result, usage = outcome final_response = _resolve_media_to_data_urls(result.get("final_response") or "") is_partial = bool(result.get("partial")) is_failed = bool(result.get("failed")) completed = bool(result.get("completed", True)) raw_err_msg = result.get("error") err_msg = _redact_api_error_text(raw_err_msg) if raw_err_msg else raw_err_msg # Decide finish_reason. OpenAI uses "length" for truncation, "stop" # for normal completion, and downstream SDKs accept "error" / custom # codes. See issue #22496. if is_partial and err_msg and "truncat" in err_msg.lower(): finish_reason = "length" elif is_failed or (not completed and err_msg): finish_reason = "error" else: finish_reason = "stop" response_headers = {"X-Hermes-Session-Id": result.get("session_id", session_id)} if gateway_session_key: response_headers["X-Hermes-Session-Key"] = gateway_session_key # Hard-fail path: no usable assistant text AND a real failure → 5xx # with OpenAI-style error envelope so SDK clients raise instead of # silently rendering the internal failure string as message.content. if not final_response and (is_failed or is_partial): err_body = _openai_error( err_msg or "Agent run did not produce a response.", err_type="server_error", code="agent_incomplete", ) err_body["error"]["hermes"] = { "completed": completed, "partial": is_partial, "failed": is_failed, } response_headers["X-Hermes-Completed"] = "false" response_headers["X-Hermes-Partial"] = "true" if is_partial else "false" return web.json_response(err_body, status=502, headers=response_headers) # Soft-partial path: we have *some* text but the run did not complete # (e.g. truncation with partial buffered output). Still 200 but signal # truncation via finish_reason="length" + Hermes-specific extras. response_data = { "id": completion_id, "object": "chat.completion", "created": created, "model": model_name, "choices": [ { "index": 0, "message": {"role": "assistant", "content": final_response}, "finish_reason": finish_reason, } ], "usage": _chat_usage_payload(usage), } if is_partial or is_failed or not completed: response_data["hermes"] = { "completed": completed, "partial": is_partial, "failed": is_failed, "error": err_msg, "error_code": "output_truncated" if finish_reason == "length" else "agent_error", } response_headers["X-Hermes-Completed"] = "false" response_headers["X-Hermes-Partial"] = "true" if is_partial else "false" if err_msg: response_headers["X-Hermes-Error"] = _redact_api_error_text(err_msg, limit=200) return web.json_response(response_data, headers=response_headers) async def _run_idempotent( self, request: "web.Request", body: Dict[str, Any], compute, *, log_label: str, fingerprint_keys: List[str], ) -> tuple: """Run ``compute()`` once per Idempotency-Key + body fingerprint. Returns ``((result, usage), None)`` or ``(None, 500 response)``. """ from gateway.platforms.api_server import ( _error_response, _idem_cache, _make_request_fingerprint, ) idempotency_key = request.headers.get("Idempotency-Key") try: if idempotency_key: fp = _make_request_fingerprint(body, keys=fingerprint_keys) result, usage = await _idem_cache.get_or_set(idempotency_key, fp, compute) else: result, usage = await compute() return (result, usage), None except Exception as e: logger.error("Error running agent for %s: %s", log_label, e, exc_info=True) return None, _error_response(f"Internal server error: {e}", 500, err_type="server_error") async def _prepare_sse_response( self, request: "web.Request", session_id: Optional[str], gateway_session_key: Optional[str], ) -> "web.StreamResponse": """Open a prepared SSE StreamResponse with CORS + session headers. CORS middleware can't inject headers after ``prepare()`` flushes them, so they are resolved up front here. """ sse_headers = { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", "X-Accel-Buffering": "no", } origin = request.headers.get("Origin", "") cors = self._cors_headers_for_origin(origin) if origin else None if cors: sse_headers.update(cors) if session_id: sse_headers["X-Hermes-Session-Id"] = session_id if gateway_session_key: sse_headers["X-Hermes-Session-Key"] = gateway_session_key response = web.StreamResponse(status=200, headers=sse_headers) await response.prepare(request) return response async def _write_sse_chat_completion( self, request: "web.Request", completion_id: str, model: str, created: int, stream_q, agent_task, agent_ref=None, session_id: str = None, gateway_session_key: str = None, ) -> "web.StreamResponse": """Write real streaming SSE from agent's stream_delta_callback queue. If the client disconnects mid-stream (network drop, browser tab close), the agent is interrupted via ``agent.interrupt()`` so it stops making LLM API calls, and the asyncio task wrapper is cancelled. """ from gateway.platforms.api_server import ( _abandon_agent_task, _chat_usage_payload, _sse_frame, ) response = await self._prepare_sse_response(request, session_id, gateway_session_key) try: # Role chunk role_chunk = { "id": completion_id, "object": "chat.completion.chunk", "created": created, "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], } await response.write(_sse_frame(role_chunk)) # Helper — route a queue item to the correct SSE event. async def _emit(item): """Write a single queue item to the SSE stream. Plain strings are sent as normal ``delta.content`` chunks. Tagged tuples ``("__tool_progress__", payload)`` are sent as a custom ``event: hermes.tool.progress`` SSE event so frontends can display them without storing the markers in conversation history. See #6972 for the original event, #16588 for the ``toolCallId``/``status`` lifecycle fields. """ if isinstance(item, tuple) and len(item) == 2 and item[0] == "__tool_progress__": await response.write(_sse_frame(item[1], event="hermes.tool.progress")) else: content_chunk = { "id": completion_id, "object": "chat.completion.chunk", "created": created, "model": model, "choices": [{"index": 0, "delta": {"content": item}, "finish_reason": None}], } await response.write(_sse_frame(content_chunk)) async for delta in _iter_stream_items(stream_q, agent_task, response): if delta is None: # End of stream sentinel break await _emit(delta) # The agent can fail after the queue drains cleanly: agent_task raises, or # result is flagged failed/partial. Either must surface as a non-"stop" # finish_reason (mirrors the non-streaming path) instead of a fake success. usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0} result = None agent_error = None try: result, agent_usage = await agent_task usage = agent_usage or usage except Exception as exc: agent_error = exc logger.error("Agent task %s failed during SSE streaming: %s", completion_id, exc) # Inspect the result dict for a flagged (non-exception) failure. is_partial = bool(result.get("partial")) if isinstance(result, dict) else False is_failed = bool(result.get("failed")) if isinstance(result, dict) else False completed = bool(result.get("completed", True)) if isinstance(result, dict) else True err_msg = result.get("error") if isinstance(result, dict) else None if agent_error is not None: is_failed = True err_msg = err_msg or str(agent_error) # Decide finish_reason, matching the non-streaming logic: "length" # for truncation, "error" for failure, "stop" for normal completion. if is_partial and err_msg and "truncat" in err_msg.lower(): finish_reason = "length" elif agent_error is not None or is_failed or (not completed and err_msg): finish_reason = "error" else: finish_reason = "stop" # Finish chunk finish_chunk = { "id": completion_id, "object": "chat.completion.chunk", "created": created, "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": finish_reason}], "usage": _chat_usage_payload(usage), } if finish_reason != "stop": finish_chunk["choices"][0]["delta"] = {} if err_msg: finish_chunk["error"] = { "message": err_msg, "type": type(agent_error).__name__ if agent_error else "agent_error", } finish_chunk["hermes"] = { "completed": completed, "partial": is_partial, "failed": is_failed, "error": err_msg, "error_code": "output_truncated" if finish_reason == "length" else "agent_error", } await response.write(_sse_frame(finish_chunk)) await response.write(b"data: [DONE]\n\n") except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError, OSError): # Client disconnected mid-stream: interrupt the agent so it stops # making LLM calls, then cancel the task wrapper. await _abandon_agent_task(agent_ref, agent_task, "SSE client disconnected") logger.info("SSE client disconnected; interrupted agent task %s", completion_id) except Exception as _exc: # Agent crashed mid-stream. Try to emit an error chunk # so the client gets a proper response instead of a # TransferEncodingError from incomplete chunked encoding. import traceback as _tb logger.error("Agent crashed mid-stream for %s: %s", completion_id, _tb.format_exc()[:300]) try: error_chunk = { "id": completion_id, "object": "chat.completion.chunk", "created": created, "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "error"}], } await response.write(_sse_frame(error_chunk)) await response.write(b"data: [DONE]\n\n") except Exception: pass return response async def _write_sse_responses( self, request: "web.Request", response_id: str, model: str, created_at: int, stream_q, agent_task, agent_ref, conversation_history: List[Dict[str, str]], user_message: str, instructions: Optional[str], conversation: Optional[str], store: bool, session_id: str, gateway_session_key: Optional[str] = None, ) -> "web.StreamResponse": """Write the SSE stream for POST /v1/responses (OpenAI Responses API). Events: ``response.created`` → ``response.output_text.delta/done`` and ``response.output_item.added/done`` (function_call / function_call_output) → ``response.completed`` (full envelope, same shape as non-streaming) or ``response.failed``. On disconnect the agent is interrupted and, when ``store=True``, an ``incomplete`` snapshot replaces the ``in_progress`` one so GET / ``previous_response_id`` chaining still work. """ from gateway.platforms.api_server import ( _abandon_agent_task, _redact_api_error_text, _responses_usage_payload, _sse_frame, ) response = await self._prepare_sse_response(request, session_id, gateway_session_key) # State accumulated during the stream final_text_parts: List[str] = [] # Track open function_call items by name so we can emit a matching # ``done`` event when the tool completes. Order preserved. pending_tool_calls: List[Dict[str, Any]] = [] # Output items we've emitted so far (used to build the terminal # response.completed payload). Kept in the order they appeared. emitted_items: List[Dict[str, Any]] = [] # Monotonic counter for output_index (spec requires it). output_index = 0 # Monotonic counter for call_id generation if the agent doesn't # provide one (it doesn't, from tool_progress_callback). call_counter = 0 # Canonical Responses SSE events include a monotonically increasing # sequence_number. Add it server-side for every emitted event so # clients that validate the OpenAI event schema can parse our stream. sequence_number = 0 # Track the assistant message item id + content index for text # delta events — the spec ties deltas to a specific item. message_item_id = f"msg_{uuid.uuid4().hex[:24]}" message_output_index: Optional[int] = None message_opened = False async def _write_event(event_type: str, data: Dict[str, Any]) -> None: nonlocal sequence_number if "sequence_number" not in data: data["sequence_number"] = sequence_number sequence_number += 1 await response.write(_sse_frame(data, event=event_type)) def _envelope(status: str) -> Dict[str, Any]: env: Dict[str, Any] = { "id": response_id, "object": "response", "status": status, "created_at": created_at, "model": model, } return env final_response_text = "" agent_error: Optional[str] = None usage: Dict[str, int] = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0} terminal_snapshot_persisted = False def _persist_response_snapshot( response_env: Dict[str, Any], *, conversation_history_snapshot: Optional[List[Dict[str, Any]]] = None, session_id_snapshot: Optional[str] = None, ) -> None: if not store: return if conversation_history_snapshot is None: conversation_history_snapshot = list(conversation_history) conversation_history_snapshot.append({"role": "user", "content": user_message}) self._response_store.put(response_id, { "response": response_env, "conversation_history": conversation_history_snapshot, "instructions": instructions, "session_id": session_id_snapshot or session_id, }) if conversation: self._response_store.set_conversation(conversation, response_id) def _persist_incomplete_if_needed() -> None: """Persist an ``incomplete`` snapshot if no terminal one was written. Called from both the client-disconnect (``ConnectionResetError``) and server-cancellation (``asyncio.CancelledError``) paths so GET /v1/responses/{id} and ``previous_response_id`` chaining keep working after abrupt stream termination. """ if not store or terminal_snapshot_persisted: return incomplete_text = "".join(final_text_parts) or final_response_text incomplete_items: List[Dict[str, Any]] = list(emitted_items) if incomplete_text: incomplete_items.append({ "type": "message", "role": "assistant", "content": [{"type": "output_text", "text": incomplete_text}], }) incomplete_env = _envelope("incomplete") incomplete_env["output"] = incomplete_items incomplete_env["usage"] = _responses_usage_payload(usage) incomplete_history = list(conversation_history) incomplete_history.append({"role": "user", "content": user_message}) if incomplete_text: incomplete_history.append({"role": "assistant", "content": incomplete_text}) _persist_response_snapshot( incomplete_env, conversation_history_snapshot=incomplete_history, ) try: # response.created — initial envelope, status=in_progress created_env = _envelope("in_progress") created_env["output"] = [] await _write_event("response.created", { "type": "response.created", "response": created_env, }) _persist_response_snapshot(created_env) async def _open_message_item() -> None: """Emit response.output_item.added for the assistant message the first time any text delta arrives.""" nonlocal message_opened, message_output_index, output_index if message_opened: return message_opened = True message_output_index = output_index output_index += 1 item = { "id": message_item_id, "type": "message", "status": "in_progress", "role": "assistant", "content": [], } await _write_event("response.output_item.added", { "type": "response.output_item.added", "output_index": message_output_index, "item": item, }) async def _emit_text_delta(delta_text: str) -> None: await _open_message_item() final_text_parts.append(delta_text) await _write_event("response.output_text.delta", { "type": "response.output_text.delta", "item_id": message_item_id, "output_index": message_output_index, "content_index": 0, "delta": delta_text, "logprobs": [], }) async def _emit_tool_started(payload: Dict[str, Any]) -> str: """Emit response.output_item.added for a function_call. Returns the call_id so the matching completion event can reference it. Prefer the real ``tool_call_id`` from the agent when available; fall back to a generated call id for safety in tests or older code paths. """ nonlocal output_index, call_counter call_counter += 1 call_id = payload.get("tool_call_id") or f"call_{response_id[5:]}_{call_counter}" args = payload.get("arguments", {}) if isinstance(args, dict): arguments_str = json.dumps(args) else: arguments_str = str(args) item = { "id": f"fc_{uuid.uuid4().hex[:24]}", "type": "function_call", "status": "in_progress", "name": payload.get("name", ""), "call_id": call_id, "arguments": arguments_str, } idx = output_index output_index += 1 pending_tool_calls.append({ "call_id": call_id, "name": payload.get("name", ""), "arguments": arguments_str, "item_id": item["id"], "output_index": idx, }) emitted_items.append({ "type": "function_call", "name": payload.get("name", ""), "arguments": arguments_str, "call_id": call_id, }) await _write_event("response.output_item.added", { "type": "response.output_item.added", "output_index": idx, "item": item, }) return call_id async def _emit_tool_completed(payload: Dict[str, Any]) -> None: """Emit response.output_item.done (function_call) followed by response.output_item.added (function_call_output).""" nonlocal output_index call_id = payload.get("tool_call_id") result = payload.get("result", "") pending = None if call_id: for i, p in enumerate(pending_tool_calls): if p["call_id"] == call_id: pending = pending_tool_calls.pop(i) break if pending is None: # Completion without a matching start — skip to avoid # emitting orphaned done events. return # function_call done done_item = { "id": pending["item_id"], "type": "function_call", "status": "completed", "name": pending["name"], "call_id": pending["call_id"], "arguments": pending["arguments"], } await _write_event("response.output_item.done", { "type": "response.output_item.done", "output_index": pending["output_index"], "item": done_item, }) # function_call_output added (result) result_str = result if isinstance(result, str) else json.dumps(result) output_parts = [{"type": "input_text", "text": result_str}] output_item = { "id": f"fco_{uuid.uuid4().hex[:24]}", "type": "function_call_output", "call_id": pending["call_id"], "output": output_parts, "status": "completed", } idx = output_index output_index += 1 emitted_items.append({ "type": "function_call_output", "call_id": pending["call_id"], "output": output_parts, }) await _write_event("response.output_item.added", { "type": "response.output_item.added", "output_index": idx, "item": output_item, }) await _write_event("response.output_item.done", { "type": "response.output_item.done", "output_index": idx, "item": output_item, }) # Main drain loop — thread-safe queue fed by agent callbacks. async def _dispatch(it) -> None: """Route a queue item to the correct SSE emitter. Plain strings are text deltas — they are batched (50ms) to reduce Open WebUI re-render storms. Tagged tuples with ``__tool_started__`` / ``__tool_completed__`` prefixes are tool lifecycle events and flush the buffer before emitting. """ nonlocal _batch_timer if isinstance(it, tuple) and len(it) == 2 and isinstance(it[0], str): tag, payload = it # Flush batched text before tool events if _batch_buf: await _flush_batch() if tag == "__tool_started__": await _emit_tool_started(payload) elif tag == "__tool_completed__": await _emit_tool_completed(payload) elif isinstance(it, str): # Batch text deltas — append to buffer, flush on timer _batch_buf.append(it) if _batch_timer is None: _batch_timer = asyncio.create_task(_batch_flush_after(0.05)) # Other types are silently dropped. # ── Batching state ── _batch_buf: List[str] = [] _batch_timer: Optional[asyncio.Task] = None _batch_lock = asyncio.Lock() async def _batch_flush_after(delay: float) -> None: """Wait delay seconds, then flush accumulated text deltas.""" try: await asyncio.sleep(delay) except asyncio.CancelledError: return # Clear timer reference BEFORE flush so new deltas # can start a fresh timer while we emit nonlocal _batch_buf, _batch_timer _batch_timer = None await _flush_batch() async def _flush_batch() -> None: """Emit a single SSE delta for all accumulated text.""" nonlocal _batch_buf async with _batch_lock: if _batch_buf: combined = "".join(_batch_buf) _batch_buf = [] await _emit_text_delta(combined) async for item in _iter_stream_items(stream_q, agent_task, response): if item is None: # EOS sentinel # Cancel pending timer and flush remaining batched text if _batch_timer and not _batch_timer.done(): _batch_timer.cancel() _batch_timer = None if _batch_buf: await _flush_batch() break await _dispatch(item) # Flush any final batched text before processing result if _batch_buf: await _flush_batch() # Pick up agent result + usage from the completed task try: result, agent_usage = await agent_task usage = agent_usage or usage # If the agent produced a final_response but no text # deltas were streamed (e.g. some providers only emit # the full response at the end), emit a single fallback # delta so Responses clients still receive a live text part. agent_final = result.get("final_response", "") if isinstance(result, dict) else "" if agent_final and not final_text_parts: await _emit_text_delta(agent_final) if agent_final and not final_response_text: final_response_text = agent_final if isinstance(result, dict) and result.get("error") and not final_response_text: agent_error = _redact_api_error_text(result["error"]) except Exception as e: # noqa: BLE001 logger.error("Error running agent for streaming responses: %s", e, exc_info=True) agent_error = _redact_api_error_text(e) # Close the message item if it was opened final_response_text = "".join(final_text_parts) or final_response_text if message_opened: await _write_event("response.output_text.done", { "type": "response.output_text.done", "item_id": message_item_id, "output_index": message_output_index, "content_index": 0, "text": final_response_text, "logprobs": [], }) msg_done_item = { "id": message_item_id, "type": "message", "status": "completed", "role": "assistant", "content": [{"type": "output_text", "text": final_response_text}], } await _write_event("response.output_item.done", { "type": "response.output_item.done", "output_index": message_output_index, "item": msg_done_item, }) # Always append a final message item in the completed # response envelope so clients that only parse the terminal # payload still see the assistant text. This mirrors the # shape produced by _extract_output_items in the batch path. final_items: List[Dict[str, Any]] = list(emitted_items) # Trim large content from tool call arguments to keep the # response.completed event under ~100KB. Clients already # received full details via incremental events. for _item in final_items: if _item.get("type") == "function_call": try: _args = json.loads(_item.get("arguments", "{}")) if isinstance(_item.get("arguments"), str) else _item.get("arguments", {}) if isinstance(_args, dict): for _k in ("content", "query", "pattern", "old_string", "new_string"): if isinstance(_args.get(_k), str) and len(_args[_k]) > 500: _args[_k] = "[" + str(len(_args[_k])) + " chars — truncated for response.completed]" _item["arguments"] = json.dumps(_args) except Exception: pass elif _item.get("type") == "function_call_output": _output = _item.get("output", []) if isinstance(_output, list) and _output: _first = _output[0] if isinstance(_first, dict) and _first.get("type") == "input_text": _text = _first.get("text", "") if len(_text) > 1000: _first["text"] = _text[:500] + "...[" + str(len(_text) - 500) + " more chars]" _item["output"] = [_first] final_items.append({ "type": "message", "role": "assistant", "content": [ {"type": "output_text", "text": final_response_text or (_redact_api_error_text(agent_error) if agent_error else "")} ], }) if agent_error: failed_env = _envelope("failed") failed_env["output"] = final_items failed_env["error"] = {"message": _redact_api_error_text(agent_error), "type": "server_error"} failed_env["usage"] = _responses_usage_payload(usage) _failed_history = list(conversation_history) _failed_history.append({"role": "user", "content": user_message}) if final_response_text or agent_error: _failed_history.append({ "role": "assistant", "content": final_response_text or _redact_api_error_text(agent_error), }) _persist_response_snapshot( failed_env, conversation_history_snapshot=_failed_history, ) terminal_snapshot_persisted = True await _write_event("response.failed", { "type": "response.failed", "response": failed_env, }) else: completed_env = _envelope("completed") completed_env["output"] = final_items completed_env["usage"] = _responses_usage_payload(usage) full_history = self._build_response_conversation_history( conversation_history, user_message, result, final_response_text, ) # Compression-aware transcript substitution happens inside # _build_response_conversation_history (result["_compressed"]); # here we only propagate a compression-rotated session_id so # previous_response_id chaining resumes the child session. _result_sid = result.get("session_id") if isinstance(result, dict) else None _persist_response_snapshot( completed_env, conversation_history_snapshot=full_history, session_id_snapshot=_result_sid if isinstance(_result_sid, str) and _result_sid else None, ) terminal_snapshot_persisted = True await _write_event("response.completed", { "type": "response.completed", "response": completed_env, }) except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError, OSError): _persist_incomplete_if_needed() await _abandon_agent_task(agent_ref, agent_task, "SSE client disconnected") logger.info("SSE client disconnected; interrupted agent task %s", response_id) except asyncio.CancelledError: # Server-side cancellation (e.g. shutdown, request timeout) — # persist an incomplete snapshot so GET /v1/responses/{id} and # previous_response_id chaining still work, then re-raise so the # runtime's cancellation semantics are respected. _persist_incomplete_if_needed() await _abandon_agent_task( agent_ref, agent_task, "SSE task cancelled", reap_source="api_server_sse_cancelled", await_cancel=False, ) logger.info("SSE task cancelled; persisted incomplete snapshot for %s", response_id) raise except Exception as _exc: # Agent crashed with an unhandled error (e.g. model API error like # BadRequestError, AuthenticationError). Emit a response.failed # event and properly terminate the SSE stream so the client doesn't # get a TransferEncodingError from incomplete chunked encoding. import traceback as _tb _persist_incomplete_if_needed() agent_error = _redact_api_error_text(_tb.format_exc()) try: failed_env = _envelope("failed") failed_env["output"] = list(emitted_items) failed_env["error"] = {"message": _redact_api_error_text(_exc, limit=500), "type": "server_error"} failed_env["usage"] = _responses_usage_payload(usage) await _write_event("response.failed", { "type": "response.failed", "response": failed_env, }) except Exception: pass logger.error("Agent crashed mid-stream for %s: %s", response_id, str(agent_error)[:300]) return response async def _handle_responses(self, request: "web.Request") -> "web.Response": """POST /v1/responses — OpenAI Responses API format.""" from gateway.platforms.api_server import ( ThreadSafeAsyncQueue, _auto_truncate_response_history, _coerce_request_bool, _content_has_visible_payload, _error_response, _multimodal_validation_error, _normalize_multimodal_content, _redact_api_error_text, _request_agent_overrides, _resolve_media_to_data_urls, _responses_usage_payload, ) # Bound total in-flight agent runs (configurable; #7483). limited = self._concurrency_limited_response() if limited is not None: return limited # Long-term memory scope header (see chat_completions for details). gateway_session_key, key_err = self._parse_session_key_header(request) if key_err is not None: return key_err # Parse request body try: body = await request.json() except (json.JSONDecodeError, Exception): return web.json_response( {"error": {"message": "Invalid JSON in request body", "type": "invalid_request_error"}}, status=400, ) raw_input = body.get("input") if raw_input is None: return _error_response("Missing 'input' field", 400) instructions = body.get("instructions") previous_response_id = body.get("previous_response_id") conversation = body.get("conversation") store = _coerce_request_bool(body.get("store"), default=True) # conversation and previous_response_id are mutually exclusive if conversation and previous_response_id: return _error_response("Cannot use both 'conversation' and 'previous_response_id'", 400) # Resolve conversation name to latest response_id if conversation: previous_response_id = self._response_store.get_conversation(conversation) # No error if conversation doesn't exist yet — it's a new conversation # Normalize input to message list input_messages: List[Dict[str, Any]] = [] if isinstance(raw_input, str): input_messages = [{"role": "user", "content": raw_input}] elif isinstance(raw_input, list): for idx, item in enumerate(raw_input): if isinstance(item, str): input_messages.append({"role": "user", "content": item}) elif isinstance(item, dict): role = item.get("role", "user") try: content = _normalize_multimodal_content(item.get("content", "")) except ValueError as exc: return _multimodal_validation_error(exc, param=f"input[{idx}].content") input_messages.append({"role": role, "content": content}) else: return _error_response("'input' must be a string or array", 400) # Accept explicit conversation_history from the request body. # This lets stateless clients supply their own history instead of # relying on server-side response chaining via previous_response_id. # Precedence: explicit conversation_history > previous_response_id. conversation_history: List[Dict[str, Any]] = [] raw_history = body.get("conversation_history") if raw_history: if not isinstance(raw_history, list): return _error_response("'conversation_history' must be an array of message objects", 400) for i, entry in enumerate(raw_history): if not isinstance(entry, dict) or "role" not in entry or "content" not in entry: return _error_response(f"conversation_history[{i}] must have 'role' and 'content' fields", 400) try: entry_content = _normalize_multimodal_content(entry["content"]) except ValueError as exc: return _multimodal_validation_error(exc, param=f"conversation_history[{i}].content") conversation_history.append({"role": str(entry["role"]), "content": entry_content}) if previous_response_id: logger.debug("Both conversation_history and previous_response_id provided; using conversation_history") stored_session_id = None if not conversation_history and previous_response_id: stored = self._response_store.get(previous_response_id) if stored is None: return _error_response(f"Previous response not found: {previous_response_id}", 404) conversation_history = list(stored.get("conversation_history", [])) stored_session_id = stored.get("session_id") # If no instructions provided, carry forward from previous if instructions is None: instructions = stored.get("instructions") # Append new input messages to history (all but the last become history) for msg in input_messages[:-1]: conversation_history.append(msg) # Last input message is the user_message user_message: Any = input_messages[-1].get("content", "") if input_messages else "" if not _content_has_visible_payload(user_message): return _error_response("No user message found in input", 400) # Truncation support if body.get("truncation") == "auto": conversation_history = _auto_truncate_response_history(conversation_history) # Session precedence: previous_response_id chain > declared X-Hermes-Session-Key # > fresh id. Binding the declared key is gated on that same precedence — a # chain-selected session must not have its routing key rewritten to this header. _declared_selected = not stored_session_id and bool(gateway_session_key) session_id = ( stored_session_id or self._declared_conversation_session(gateway_session_key) or str(uuid.uuid4()) ) stream = _coerce_request_bool(body.get("stream"), default=False) route = self._resolve_route(body.get("model")) agent_overrides = _request_agent_overrides( body, virtual_model=self._model_name, allow_bare_model=self._direct_model_requests, ) selection_error = self._request_route_conflict_error( session_id=session_id, gateway_session_key=gateway_session_key, requested_model=agent_overrides.get("requested_model"), requested_provider=agent_overrides.get("requested_provider"), route=route, ) if selection_error: return _error_response(selection_error, 400) if stream: # Streaming branch — emit OpenAI Responses SSE events as the # agent runs so frontends can render text deltas and tool # calls in real time. See _write_sse_responses for details. _stream_q = ThreadSafeAsyncQueue() def _on_delta(delta): # None from the agent is a CLI box-close signal, not EOS. # Forwarding would kill the SSE stream prematurely; the # SSE writer detects completion via agent_task.done(). # Called from the worker thread running run_conversation — # put_threadsafe (not put_nowait) is required here. if delta is not None: _stream_q.put_threadsafe(delta) def _on_tool_progress(event_type, name, preview, args, **kwargs): """Queue non-start tool progress events if needed in future. The structured Responses stream uses ``tool_start_callback`` and ``tool_complete_callback`` for exact call-id correlation, so progress events are currently ignored here. """ return def _on_tool_start(tool_call_id, function_name, function_args): """Queue a started tool for live function_call streaming.""" _stream_q.put_threadsafe(("__tool_started__", { "tool_call_id": tool_call_id, "name": function_name, "arguments": function_args or {}, })) def _on_tool_complete(tool_call_id, function_name, function_args, function_result): """Queue a completed tool result for live function_call_output streaming.""" _stream_q.put_threadsafe(("__tool_completed__", { "tool_call_id": tool_call_id, "name": function_name, "arguments": function_args or {}, "result": function_result, })) agent_ref = [None] agent_task = asyncio.ensure_future(self._run_agent( user_message=user_message, conversation_history=conversation_history, ephemeral_system_prompt=instructions, session_id=session_id, stream_delta_callback=_on_delta, tool_progress_callback=_on_tool_progress, tool_start_callback=_on_tool_start, tool_complete_callback=_on_tool_complete, agent_ref=agent_ref, gateway_session_key=gateway_session_key, bind_declared_conversation=_declared_selected, **agent_overrides, route=route, )) # Ensure SSE drain loops can terminate without relying on polling # agent_task.done(), which can race with queue timeout checks. agent_task.add_done_callback(lambda _fut: _stream_q.put_nowait(None)) response_id = f"resp_{uuid.uuid4().hex[:28]}" model_name = body.get("model", self._model_name) created_at = int(time.time()) return await self._write_sse_responses( request=request, response_id=response_id, model=model_name, created_at=created_at, stream_q=_stream_q, agent_task=agent_task, agent_ref=agent_ref, conversation_history=conversation_history, user_message=user_message, instructions=instructions, conversation=conversation, store=store, session_id=session_id, gateway_session_key=gateway_session_key, ) async def _compute_response(): return await self._run_agent( user_message=user_message, conversation_history=conversation_history, ephemeral_system_prompt=instructions, session_id=session_id, gateway_session_key=gateway_session_key, bind_declared_conversation=_declared_selected, **agent_overrides, route=route, ) outcome, err = await self._run_idempotent( request, body, _compute_response, log_label="responses", fingerprint_keys=["input", "instructions", "previous_response_id", "conversation", "model", "provider", "model_options", "tools"], ) if err is not None: return err result, usage = outcome final_response = _resolve_media_to_data_urls(result.get("final_response", "")) if not final_response: final_response = _redact_api_error_text(result.get("error", "(No response generated)")) response_id = f"resp_{uuid.uuid4().hex[:28]}" created_at = int(time.time()) # Build the full conversation history for storage # (includes tool calls from the agent run) full_history = self._build_response_conversation_history( conversation_history, user_message, result, final_response, ) # Persist the effective session ID surfaced by _run_agent so that # compression-triggered session rotations propagate to the stored # response and the X-Hermes-Session-Id header. Without this, # previous_response_id chaining keeps resuming the pre-rotation # session and re-triggers compression on every subsequent request. _effective_session_id = session_id _result_sid = result.get("session_id") if isinstance(result, dict) else None if isinstance(_result_sid, str) and _result_sid: _effective_session_id = _result_sid # Build output items from the current turn only. AIAgent returns a # full transcript in result["messages"], while older/mocked paths may # return only the current turn suffix. output_start_index = self._response_messages_turn_start_index( conversation_history, user_message, result, ) output_items = self._extract_output_items(result, start_index=output_start_index) response_data = { "id": response_id, "object": "response", "status": "completed", "created_at": created_at, "model": body.get("model", self._model_name), "output": output_items, "usage": _responses_usage_payload(usage), } # Store the complete response object for future chaining / GET retrieval if store: self._response_store.put(response_id, { "response": response_data, "conversation_history": full_history, "instructions": instructions, "session_id": _effective_session_id, }) # Update conversation mapping so the next request with the same # conversation name automatically chains to this response if conversation: self._response_store.set_conversation(conversation, response_id) response_headers = {"X-Hermes-Session-Id": _effective_session_id} if gateway_session_key: response_headers["X-Hermes-Session-Key"] = gateway_session_key return web.json_response(response_data, headers=response_headers) async def _handle_get_response(self, request: "web.Request") -> "web.Response": """GET /v1/responses/{response_id} — retrieve a stored response.""" from gateway.platforms.api_server import _error_response auth_err = self._check_auth(request) if auth_err: return auth_err response_id = request.match_info["response_id"] stored = self._response_store.get(response_id) if stored is None: return _error_response(f"Response not found: {response_id}", 404) return web.json_response(stored["response"]) async def _handle_delete_response(self, request: "web.Request") -> "web.Response": """DELETE /v1/responses/{response_id} — delete a stored response.""" from gateway.platforms.api_server import _error_response auth_err = self._check_auth(request) if auth_err: return auth_err response_id = request.match_info["response_id"] deleted = self._response_store.delete(response_id) if not deleted: return _error_response(f"Response not found: {response_id}", 404) return web.json_response({"id": response_id, "object": "response", "deleted": True}) @staticmethod def _build_response_conversation_history( conversation_history: List[Dict[str, Any]], user_message: Any, result: Dict[str, Any], final_response: Any, ) -> List[Dict[str, Any]]: """Build the stored Responses transcript without duplicating history. When context compression occurs during a turn the agent returns a compressed full transcript in ``result["messages"]`` (starting with a summary) and sets ``result["_compressed"] = True``. Because the compressed transcript does not share the input ``conversation_history`` prefix, the normal turn-start detection fails and old code would concatenate the uncompressed history on front, bloating the stored context and re-triggering compression on every subsequent request. """ from gateway.platforms.api_server import APIServerAdapter prior = list(conversation_history) current_user = {"role": "user", "content": user_message} agent_messages = result.get("messages") if isinstance(result, dict) else None if isinstance(agent_messages, list) and agent_messages: turn_start = APIServerAdapter._response_messages_turn_start_index( conversation_history, user_message, result, ) if turn_start: return list(agent_messages) # turn_start == 0: either compression rewrote the transcript (use it as-is — # the _compressed flag says so) or agent_messages is only the current turn. if result.get("_compressed"): return list(agent_messages) full_history = prior full_history.append(current_user) full_history.extend(agent_messages) return full_history full_history = prior full_history.append(current_user) full_history.append({"role": "assistant", "content": final_response}) return full_history @staticmethod def _response_messages_turn_start_index( conversation_history: List[Dict[str, Any]], user_message: Any, result: Dict[str, Any], ) -> int: """Detect transcript-shaped result["messages"] and return turn start.""" agent_messages = result.get("messages") if isinstance(result, dict) else None if not isinstance(agent_messages, list) or not agent_messages: return 0 prior = list(conversation_history) current_user = {"role": "user", "content": user_message} expected_prefix = prior + [current_user] if agent_messages[:len(expected_prefix)] == expected_prefix: return len(expected_prefix) if prior and agent_messages[:len(prior)] == prior: return len(prior) return 0 @classmethod def _turn_transcript_messages( cls, conversation_history: List[Dict[str, Any]], user_message: Any, result: Dict[str, Any], ) -> List[Dict[str, Any]]: """Return this turn's assistant/tool messages in client-safe shape. The streaming SSE contract delivers all assistant text as ``assistant.delta`` events under one ``message_id`` interleaved with ``tool.*`` events, and a single ``assistant.completed`` carrying only the final reply. A client that accumulates deltas into one buffer cannot reconstruct *intermediate* assistant text segments that preceded tool calls — so when the page is re-opened mid/post-stream those segments appear lost, even though state.db persisted them correctly. Emitting the authoritative per-turn transcript on ``run.completed`` lets any SSE consumer reconcile its live view against ground truth without a separate ``GET /messages`` round-trip. Purely additive: clients that ignore the field are unaffected. Refs #34703. """ agent_messages = result.get("messages") if isinstance(result, dict) else None if not isinstance(agent_messages, list) or not agent_messages: return [] start = cls._response_messages_turn_start_index(conversation_history, user_message, result) turn = agent_messages[start:] out: List[Dict[str, Any]] = [] for msg in turn: if not isinstance(msg, dict): continue if msg.get("role") not in {"assistant", "tool"}: continue # _message_response projects compaction scaffolding itself and # marks pure handoffs display_kind == "hidden"; classifying here # first would re-run the content classifier (a full content # flatten + prefix scan) a second time per message. projected = cls._message_response(msg) if projected.get("display_kind") == "hidden": continue out.append(projected) return out @staticmethod def _extract_output_items(result: Dict[str, Any], start_index: int = 0) -> List[Dict[str, Any]]: """ Build the output item array from the agent's messages. Walks *result["messages"]* starting at *start_index* and emits: - ``function_call`` items for each tool_call on assistant messages - ``function_call_output`` items for each tool-role message - a final ``message`` item with the assistant's text reply """ from gateway.platforms.api_server import _redact_api_error_text items: List[Dict[str, Any]] = [] messages = result.get("messages", []) if start_index > 0: messages = messages[start_index:] for msg in messages: role = msg.get("role") if role == "assistant" and msg.get("tool_calls"): for tc in msg["tool_calls"]: func = tc.get("function", {}) items.append({ "id": f"fc_{uuid.uuid4().hex[:24]}", "type": "function_call", # These calls were already executed server-side by the # Hermes agent; they are replayed for structured tool # UI only. Mark them completed (matching the SSE # streaming path) so OpenAI clients don't interpret # them as pending calls the client must execute. "status": "completed", "name": func.get("name", ""), "arguments": func.get("arguments", ""), "call_id": tc.get("id", ""), }) elif role == "tool": items.append({ "id": f"fco_{uuid.uuid4().hex[:24]}", "type": "function_call_output", "status": "completed", "call_id": msg.get("tool_call_id", ""), "output": msg.get("content", ""), }) # Final assistant message final = result.get("final_response", "") if not final: final = _redact_api_error_text(result.get("error", "(No response generated)")) items.append({ "type": "message", "role": "assistant", "content": [{"type": "output_text", "text": final}], }) return items