Teknium
1117 lines
47 KiB
Python
1117 lines
47 KiB
Python
"""Durable ``/v1/runs`` admission, status, events, and control handlers."""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
import uuid
|
|
from contextlib import suppress
|
|
from dataclasses import dataclass
|
|
from typing import Any, Callable, Dict, List, Optional
|
|
|
|
try:
|
|
from aiohttp import web
|
|
from aiohttp.web_request import RequestKey
|
|
except ImportError:
|
|
web = None # type: ignore[assignment]
|
|
RequestKey = None # type: ignore[assignment,misc]
|
|
|
|
from gateway.platforms.api_server_room_grants import _json_error, _room_grant_error_response
|
|
from gateway.platforms.api_server_run_idempotency import TERMINAL_STATUSES
|
|
|
|
|
|
logger = logging.getLogger("gateway.platforms.api_server")
|
|
_ROOM_RETENTION_REQUEST_KEY = (
|
|
RequestKey("hermes.room_run_retention_until", float)
|
|
if RequestKey is not None
|
|
else "hermes.room_run_retention_until"
|
|
)
|
|
# Forwarded subagent lifecycle fields; free-text ones are secret-redacted.
|
|
_SUBAGENT_EVENT_KEYS = (
|
|
"goal", "task_count", "task_index", "subagent_id", "child_session_id",
|
|
"delegation_id", "parent_id", "depth", "model", "tool_count", "status",
|
|
"summary", "duration_seconds", "input_tokens", "output_tokens",
|
|
"reasoning_tokens", "api_calls", "cost_usd", "files_read", "files_written",
|
|
"output_tail",
|
|
)
|
|
_SUBAGENT_TEXT_KEYS = ("goal", "summary", "output_tail")
|
|
|
|
|
|
def _remember_room_retention(request: "web.Request", claims: dict[str, Any]) -> None:
|
|
value = float(claims.get("status_expires_at") or claims.get("expires_at") or 0)
|
|
try:
|
|
request[_ROOM_RETENTION_REQUEST_KEY] = value
|
|
except (AttributeError, TypeError):
|
|
setattr(request, "_hermes_room_run_retention_until", value)
|
|
|
|
|
|
def _room_retention_until(request: "web.Request") -> float:
|
|
try:
|
|
value = request.get(_ROOM_RETENTION_REQUEST_KEY, 0)
|
|
except AttributeError:
|
|
value = getattr(request, "_hermes_room_run_retention_until", 0)
|
|
return max(0.0, float(value or 0))
|
|
|
|
|
|
def _run_event(run_id: str, name: str, **fields: Any) -> Dict[str, Any]:
|
|
"""Build one SSE event payload (key order is part of the wire format)."""
|
|
return {"event": name, "run_id": run_id, "timestamp": time.time(), **fields}
|
|
|
|
|
|
def _run_not_found(_openai_error, run_id: str) -> "web.Response":
|
|
return _json_error(_openai_error, f"Run not found: {run_id}", code="run_not_found", status=404)
|
|
|
|
|
|
def _idempotency_conflict(_openai_error) -> "web.Response":
|
|
return _json_error(
|
|
_openai_error, "Idempotency-Key was already used with a different request payload",
|
|
code="idempotency_key_conflict", status=409,
|
|
)
|
|
|
|
|
|
def _uses_room_run_auth(self, request: "web.Request") -> bool:
|
|
return request.path.endswith("/v1/runs") and bool(self._room_grant_token(request))
|
|
|
|
|
|
def _initialize_run_state(self, *, store_factory) -> None:
|
|
"""Initialize adapter-owned durable and live ``/v1/runs`` state."""
|
|
self._run_idempotency_store = store_factory()
|
|
self._run_idempotency_ids: set[str] = set()
|
|
self._run_owners: Dict[str, str] = {}
|
|
self._run_owner_pid = os.getpid()
|
|
try:
|
|
from gateway.status import get_process_start_time
|
|
|
|
self._run_owner_started = int(get_process_start_time(self._run_owner_pid) or 0)
|
|
except Exception:
|
|
self._run_owner_started = 0
|
|
# run_id -> asyncio.Queue of SSE event dicts, plus creation time for the TTL sweep
|
|
self._run_streams: Dict[str, "asyncio.Queue[Optional[Dict]]"] = {}
|
|
self._run_streams_created: Dict[str, float] = {}
|
|
# Runs with a connected SSE consumer; their queue is actively draining.
|
|
self._run_stream_subscribers: set[str] = set()
|
|
# Live agent/task refs for stop support. Stop is cooperative: the executor
|
|
# thread may outlive the HTTP request, hence the separate stopping set.
|
|
self._active_run_agents: Dict[str, Any] = {}
|
|
self._active_run_tasks: Dict[str, "asyncio.Task"] = {}
|
|
self._stopping_run_ids: set[str] = set()
|
|
# Pollable run status for dashboards and external control-plane UIs.
|
|
self._run_statuses: Dict[str, Dict[str, Any]] = {}
|
|
# run_id -> approval session key: the approval core resolves requests by
|
|
# session key, while API clients address them by run_id.
|
|
self._run_approval_sessions: Dict[str, str] = {}
|
|
|
|
|
|
def _http_routes(self) -> list[tuple[str, str, Any]]:
|
|
return [
|
|
("POST", "/v1/runs", self._handle_runs),
|
|
("GET", "/v1/runs/{run_id}", self._handle_get_run),
|
|
("GET", "/v1/runs/{run_id}/events", self._handle_run_events),
|
|
("POST", "/v1/runs/{run_id}/approval", self._handle_run_approval),
|
|
("POST", "/v1/runs/{run_id}/steer", self._handle_steer_run),
|
|
("POST", "/v1/runs/{run_id}/stop", self._handle_stop_run),
|
|
]
|
|
|
|
|
|
def _idempotency_capabilities(self, *, store_type) -> dict[str, Any]:
|
|
return {
|
|
"supported": True,
|
|
"durable": self._run_idempotency_store.durable,
|
|
"retention_seconds": store_type.RETENTION_SECONDS,
|
|
}
|
|
|
|
|
|
def _close_run_state(self) -> None:
|
|
store = getattr(self, "_run_idempotency_store", None)
|
|
if store is None:
|
|
return
|
|
try:
|
|
store.close()
|
|
except Exception:
|
|
logger.debug("Failed to close run idempotency store for %s", self.name, exc_info=True)
|
|
|
|
|
|
def _set_run_status(self, run_id: str, status: str, **fields: Any) -> Dict[str, Any]:
|
|
"""Update pollable run status without exposing private agent objects."""
|
|
now = time.time()
|
|
current = self._run_statuses.get(run_id, {})
|
|
previous_status = str(current.get("status") or "")
|
|
field_names = set(fields)
|
|
current.update({"object": "hermes.run", "run_id": run_id, "status": status, "updated_at": now})
|
|
current.setdefault("created_at", fields.pop("created_at", now))
|
|
current.update(fields)
|
|
if status != "waiting_for_approval":
|
|
current.pop("approval", None)
|
|
self._run_statuses[run_id] = current
|
|
should_persist = (
|
|
status != previous_status
|
|
or status in TERMINAL_STATUSES
|
|
or bool(field_names & {"output", "error", "usage", "pending_steer", "session_id"})
|
|
)
|
|
if run_id in self._run_idempotency_ids and should_persist:
|
|
try:
|
|
self._run_idempotency_store.update_status(run_id, current)
|
|
except Exception:
|
|
logger.exception("[api_server] failed to persist idempotent run status %s", run_id)
|
|
return current
|
|
|
|
|
|
def _make_run_event_callback(self, run_id: str, loop: "asyncio.AbstractEventLoop", *, _api_server):
|
|
"""Return a callback that pushes structured events to the run SSE queue."""
|
|
redact_sensitive_text = _api_server.redact_sensitive_text
|
|
|
|
def _push(event: Dict[str, Any]) -> None:
|
|
self._set_run_status(
|
|
run_id,
|
|
self._run_statuses.get(run_id, {}).get("status", "running"),
|
|
last_event=event.get("event"),
|
|
)
|
|
q = self._run_streams.get(run_id)
|
|
if q is None:
|
|
return
|
|
with suppress(Exception):
|
|
loop.call_soon_threadsafe(q.put_nowait, event)
|
|
|
|
def _callback(event_type: str, tool_name: str = None, preview: str = None, args=None, **kwargs):
|
|
if event_type == "tool.started":
|
|
_push(_run_event(run_id, event_type, tool=tool_name, preview=preview))
|
|
elif event_type == "tool.completed":
|
|
_push(_run_event(
|
|
run_id, event_type, tool=tool_name,
|
|
duration=round(kwargs.get("duration", 0), 3),
|
|
error=kwargs.get("is_error", False),
|
|
))
|
|
elif event_type == "reasoning.available":
|
|
_push(_run_event(run_id, event_type, text=preview or ""))
|
|
elif event_type in {"subagent.start", "subagent.complete"}:
|
|
event = _run_event(run_id, event_type)
|
|
if preview is not None:
|
|
event["preview"] = redact_sensitive_text(str(preview), force=True)
|
|
for key in _SUBAGENT_EVENT_KEYS:
|
|
value = kwargs.get(key)
|
|
if value is None:
|
|
continue
|
|
# Free text can carry child terminal/tool output: force the same
|
|
# secret redaction the API applies to error text on this public stream.
|
|
if key in _SUBAGENT_TEXT_KEYS and isinstance(value, str):
|
|
value = redact_sensitive_text(value, force=True)
|
|
event[key] = value
|
|
_push(event)
|
|
# _thinking, subagent.tool, and subagent_progress are deliberately not
|
|
# forwarded (high-volume UI noise); lifecycle boundaries must land so
|
|
# clients can observe delegate_task timeouts and failures.
|
|
|
|
return _callback
|
|
|
|
|
|
def _room_permission_for(request: "web.Request") -> str:
|
|
if request.path.endswith("/stop"):
|
|
return "stop"
|
|
if request.path.endswith("/approval"):
|
|
return "approve"
|
|
return "status" if request.method == "GET" else "dispatch"
|
|
|
|
|
|
def _run_idempotency_scope(self, request: "web.Request", *, _api_server) -> str:
|
|
"""Opaque auth/profile namespace; never persist bearer credentials."""
|
|
if self._room_grant_token(request):
|
|
claims = self._room_grant_claims(request, permission=_room_permission_for(request))
|
|
_remember_room_retention(request, claims)
|
|
identity = (
|
|
f"{claims['room_id']}\0{claims['home_install_id']}\0"
|
|
f"{claims['authority_gateway_id']}\0{claims['authority_epoch']}\0"
|
|
f"{claims['member_id']}\0{claims['target_install_id']}\0"
|
|
f"{claims['target_profile']}"
|
|
)
|
|
return hashlib.sha256(identity.encode()).hexdigest()
|
|
profile = _api_server._api_request_profile.get() or "default"
|
|
identity = self._expected_api_key() or "unauthenticated-test-listener"
|
|
return hashlib.sha256(f"{profile}\0{identity}".encode()).hexdigest()
|
|
|
|
|
|
def _check_run_auth(self, request: "web.Request", *, permission: str, _api_server) -> "web.Response | None":
|
|
if not self._room_grant_token(request):
|
|
return self._check_auth(request)
|
|
try:
|
|
self._room_grant_claims(request, permission=permission)
|
|
except Exception as exc:
|
|
return _room_grant_error_response(exc, _openai_error=_api_server._openai_error)
|
|
return None
|
|
|
|
|
|
def _durable_run_status(self, request: "web.Request", run_id: str) -> Dict[str, Any] | None:
|
|
"""Hydrate a scoped run status and fail stale owners closed."""
|
|
status = self._run_statuses.get(run_id)
|
|
if status is not None:
|
|
if run_id in self._run_idempotency_ids:
|
|
scope = self._run_idempotency_scope(request)
|
|
self._run_idempotency_store.extend_retention(scope, run_id, _room_retention_until(request))
|
|
return status
|
|
|
|
scope = self._run_idempotency_scope(request)
|
|
record = self._run_idempotency_store.status_for_run(
|
|
scope, run_id, retention_until=_room_retention_until(request)
|
|
)
|
|
if record is None:
|
|
return None
|
|
|
|
status = dict(record["status"])
|
|
owner_pid = int(record.get("owner_pid") or 0)
|
|
owner_started = int(record.get("owner_started") or 0)
|
|
owner_alive = False
|
|
if owner_pid > 0:
|
|
try:
|
|
from gateway.status import _pid_exists, get_process_start_time
|
|
|
|
owner_alive = bool(_pid_exists(owner_pid))
|
|
if owner_alive and owner_started:
|
|
owner_alive = int(get_process_start_time(owner_pid) or 0) == owner_started
|
|
except Exception:
|
|
owner_alive = False
|
|
|
|
if status.get("status") not in TERMINAL_STATUSES and not owner_alive:
|
|
status.update({
|
|
"status": "interrupted",
|
|
"error": "The gateway restarted before this run settled.",
|
|
"last_event": "run.interrupted",
|
|
"updated_at": time.time(),
|
|
})
|
|
self._run_idempotency_store.update_status(run_id, status)
|
|
|
|
self._run_statuses[run_id] = status
|
|
self._run_idempotency_ids.add(run_id)
|
|
self._run_owners[run_id] = scope
|
|
return status
|
|
|
|
|
|
def _resolve_conversation_history(
|
|
self, body: dict, raw_input: Any, *, _openai_error
|
|
) -> "tuple[List[Dict[str, str]], Any, Any, web.Response | None]":
|
|
"""Return ``(history, instructions, stored_session_id, error)``.
|
|
|
|
Precedence: explicit ``conversation_history`` > ``previous_response_id``
|
|
chain > all-but-last messages of a multi-message ``input`` array.
|
|
"""
|
|
instructions = body.get("instructions")
|
|
previous_response_id = body.get("previous_response_id")
|
|
conversation_history: List[Dict[str, str]] = []
|
|
raw_history = body.get("conversation_history")
|
|
if raw_history:
|
|
if not isinstance(raw_history, list):
|
|
return [], instructions, None, _json_error(
|
|
_openai_error, "'conversation_history' must be an array of message objects",
|
|
status=400,
|
|
)
|
|
for i, entry in enumerate(raw_history):
|
|
if not isinstance(entry, dict) or "role" not in entry or "content" not in entry:
|
|
return [], instructions, None, _json_error(
|
|
_openai_error, f"conversation_history[{i}] must have 'role' and 'content' fields",
|
|
status=400,
|
|
)
|
|
conversation_history.append({"role": str(entry["role"]), "content": str(entry["content"])})
|
|
if previous_response_id:
|
|
logger.debug("Both conversation_history and previous_response_id provided; using conversation_history")
|
|
|
|
stored_session_id = None
|
|
if not conversation_history and previous_response_id:
|
|
stored = self._response_store.get(previous_response_id)
|
|
if stored:
|
|
conversation_history = list(stored.get("conversation_history", []))
|
|
stored_session_id = stored.get("session_id")
|
|
if instructions is None:
|
|
instructions = stored.get("instructions")
|
|
|
|
if not conversation_history and isinstance(raw_input, list) and len(raw_input) > 1:
|
|
for msg in raw_input[:-1]:
|
|
if isinstance(msg, dict) and msg.get("role") and msg.get("content"):
|
|
content = msg["content"]
|
|
if isinstance(content, list): # flatten multi-part content blocks to text
|
|
content = " ".join(
|
|
part.get("text", "") for part in content
|
|
if isinstance(part, dict) and part.get("type") == "text"
|
|
)
|
|
conversation_history.append({"role": msg["role"], "content": str(content)})
|
|
return conversation_history, instructions, stored_session_id, None
|
|
|
|
|
|
def _replay_response(self, request: "web.Request", record: dict, gateway_session_key) -> "web.Response":
|
|
"""202 replay of an already-admitted idempotent run."""
|
|
original_id = str(record["run_id"])
|
|
status = self._durable_run_status(request, original_id) or record["status"]
|
|
headers = {"Idempotency-Replayed": "true"}
|
|
if gateway_session_key:
|
|
headers["X-Hermes-Session-Key"] = gateway_session_key
|
|
return web.json_response(
|
|
{"run_id": original_id, "status": status.get("status", "queued"), "replayed": True},
|
|
status=202,
|
|
headers=headers,
|
|
)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _RunLaunch:
|
|
"""Everything an admitted run needs once the HTTP request has returned.
|
|
|
|
The background task outlives the request (and thus the middleware profile
|
|
scope), so contextvar values are captured here and re-entered later.
|
|
"""
|
|
|
|
run_id: str
|
|
queue: "asyncio.Queue[Optional[Dict]]"
|
|
loop: "asyncio.AbstractEventLoop"
|
|
put_event: Callable[[Optional[Dict]], None]
|
|
session_id: str
|
|
gateway_session_key: Optional[str]
|
|
declared_selected: bool
|
|
approval_session_key: str
|
|
user_message: str
|
|
conversation_history: List[Dict[str, str]]
|
|
ephemeral_system_prompt: Any
|
|
agent_overrides: dict
|
|
route: Any
|
|
room_dispatch: Optional[dict]
|
|
room_execution_policy: Optional[dict]
|
|
request_profile: Any
|
|
browser_control_principal: Any
|
|
browser_control_transport_family: Any
|
|
text_cb: Callable
|
|
event_cb: Callable
|
|
|
|
|
|
async def _handle_runs(self, request: "web.Request", *, _api_server) -> "web.Response":
|
|
"""POST /v1/runs — start an agent run, return run_id immediately."""
|
|
_openai_error = _api_server._openai_error
|
|
|
|
# Long-term memory scope header (see chat_completions for details).
|
|
gateway_session_key, key_err = self._parse_session_key_header(request)
|
|
if key_err is not None:
|
|
return key_err
|
|
|
|
try:
|
|
body = await request.json()
|
|
except Exception:
|
|
return _json_error(_openai_error, "Invalid JSON", status=400)
|
|
|
|
body, room_error = await self._normalize_room_dispatch(request, body)
|
|
if room_error is not None:
|
|
return room_error
|
|
room_dispatch = body.get("hosted_room_dispatch") if isinstance(body, dict) else None
|
|
room_dispatch = room_dispatch if isinstance(room_dispatch, dict) else None
|
|
room_execution_policy = body.get("_room_execution_policy") if isinstance(body, dict) else None
|
|
room_execution_policy = room_execution_policy if isinstance(room_execution_policy, dict) else None
|
|
|
|
idempotency_key = request.headers.get("Idempotency-Key", "").strip()
|
|
if idempotency_key and (
|
|
len(idempotency_key) > 255 or any(ord(ch) < 33 or ord(ch) > 126 for ch in idempotency_key)
|
|
):
|
|
return _json_error(
|
|
_openai_error, "Idempotency-Key must be 1-255 visible ASCII characters",
|
|
code="invalid_idempotency_key", status=400,
|
|
)
|
|
idempotency_scope = idempotency_fingerprint = ""
|
|
if idempotency_key:
|
|
idempotency_scope = self._run_idempotency_scope(request)
|
|
idempotency_fingerprint = hashlib.sha256(
|
|
json.dumps(
|
|
{"body": body, "gateway_session_key": gateway_session_key or ""},
|
|
sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
|
).encode()
|
|
).hexdigest()
|
|
|
|
raw_input = body.get("input")
|
|
if not raw_input:
|
|
return _json_error(_openai_error, "Missing 'input' field", status=400)
|
|
if isinstance(raw_input, str):
|
|
user_message = raw_input
|
|
else:
|
|
user_message = raw_input[-1].get("content", "") if isinstance(raw_input, list) else ""
|
|
if not user_message:
|
|
return _json_error(_openai_error, "No user message found in input", status=400)
|
|
|
|
conversation_history, instructions, stored_session_id, history_err = (
|
|
_resolve_conversation_history(self, body, raw_input, _openai_error=_openai_error)
|
|
)
|
|
if history_err is not None:
|
|
return history_err
|
|
previous_response_id = body.get("previous_response_id")
|
|
|
|
session_id = body.get("session_id") or stored_session_id
|
|
route = self._resolve_route(body.get("model"))
|
|
agent_overrides = _api_server._request_agent_overrides(body, virtual_model=self._model_name)
|
|
selection_error = self._request_route_conflict_error(
|
|
session_id=session_id,
|
|
gateway_session_key=gateway_session_key,
|
|
requested_model=agent_overrides.get("requested_model"),
|
|
requested_provider=agent_overrides.get("requested_provider"),
|
|
route=route,
|
|
)
|
|
if selection_error:
|
|
return _json_error(_openai_error, selection_error, status=400)
|
|
|
|
# A lost-acceptance replay must resolve even while the original run
|
|
# consumes the final concurrency slot. This read does not reserve a
|
|
# missing key; the atomic reserve below closes the concurrent-miss race.
|
|
if idempotency_key:
|
|
outcome, record = self._run_idempotency_store.lookup(
|
|
idempotency_scope, idempotency_key, idempotency_fingerprint,
|
|
retention_until=_room_retention_until(request),
|
|
)
|
|
if outcome == "conflict":
|
|
return _idempotency_conflict(_openai_error)
|
|
if outcome == "reused" and record is not None:
|
|
return _replay_response(self, request, record, gateway_session_key)
|
|
|
|
# Enforce concurrency only for a genuinely new run.
|
|
limited = self._concurrency_limited_response()
|
|
if limited is not None:
|
|
return limited
|
|
|
|
if not conversation_history and session_id and not previous_response_id:
|
|
conversation_history = await self._conversation_history_for_session(str(session_id))
|
|
|
|
run_id = f"run_{uuid.uuid4().hex}"
|
|
self._run_owners[run_id] = self._run_idempotency_scope(request)
|
|
# Same precedence as /v1/responses: explicit body session_id wins, then the
|
|
# response chain, then the conversation declared via X-Hermes-Session-Key.
|
|
# Falling straight through to run_id would make the run id the conversation
|
|
# identity and re-key every affinity surface once per run. An explicit or
|
|
# chained session owns its routing key and must not be rebound to the header.
|
|
_declared_selected = not session_id and bool(gateway_session_key)
|
|
session_id = session_id or self._declared_conversation_session(gateway_session_key) or run_id
|
|
# Approval queues gate host-side tool execution and must be isolated per
|
|
# run: session ids / memory keys are conversation scopes, not authorization
|
|
# namespaces, and resolving one run's approval must not unblock another's.
|
|
approval_session_key = run_id
|
|
loop = asyncio.get_running_loop()
|
|
q: "asyncio.Queue[Optional[Dict]]" = asyncio.Queue()
|
|
created_at = time.time()
|
|
self._run_streams[run_id] = q
|
|
self._run_streams_created[run_id] = created_at
|
|
self._run_approval_sessions[run_id] = approval_session_key
|
|
|
|
event_cb = self._make_run_event_callback(run_id, loop)
|
|
|
|
def _put_event_if_active(event: Optional[Dict]) -> None:
|
|
"""Enqueue only while this run still owns live transport state."""
|
|
if self._run_streams.get(run_id) is q:
|
|
q.put_nowait(event)
|
|
|
|
def _text_cb(delta: Optional[str]) -> None:
|
|
if delta is None or run_id not in self._run_streams:
|
|
return
|
|
with suppress(Exception):
|
|
loop.call_soon_threadsafe(
|
|
_put_event_if_active, _run_event(run_id, "message.delta", delta=delta)
|
|
)
|
|
|
|
initial_status = self._set_run_status(
|
|
run_id, "queued", created_at=created_at, session_id=session_id,
|
|
model=body.get("model", self._model_name),
|
|
)
|
|
if idempotency_key:
|
|
outcome, record = self._run_idempotency_store.reserve(
|
|
idempotency_scope, idempotency_key, idempotency_fingerprint, run_id, initial_status,
|
|
owner_pid=self._run_owner_pid,
|
|
owner_started=self._run_owner_started,
|
|
retention_until=_room_retention_until(request),
|
|
)
|
|
if outcome != "created":
|
|
for table in (
|
|
self._run_streams, self._run_streams_created, self._run_approval_sessions,
|
|
self._run_statuses, self._run_owners,
|
|
):
|
|
table.pop(run_id, None)
|
|
if outcome == "conflict":
|
|
return _idempotency_conflict(_openai_error)
|
|
return _replay_response(self, request, record, gateway_session_key)
|
|
self._run_idempotency_ids.add(run_id)
|
|
|
|
launch = _RunLaunch(
|
|
run_id=run_id,
|
|
queue=q,
|
|
loop=loop,
|
|
put_event=_put_event_if_active,
|
|
session_id=session_id,
|
|
gateway_session_key=gateway_session_key,
|
|
declared_selected=_declared_selected,
|
|
approval_session_key=approval_session_key,
|
|
user_message=user_message,
|
|
conversation_history=conversation_history,
|
|
ephemeral_system_prompt=instructions,
|
|
agent_overrides=agent_overrides,
|
|
route=route,
|
|
room_dispatch=room_dispatch,
|
|
room_execution_policy=room_execution_policy,
|
|
request_profile=_api_server._api_request_profile.get(),
|
|
browser_control_principal=_api_server._api_request_browser_control_principal.get(),
|
|
browser_control_transport_family=(
|
|
_api_server._api_request_browser_control_transport_family.get()
|
|
),
|
|
text_cb=_text_cb,
|
|
event_cb=event_cb,
|
|
)
|
|
|
|
self._activate_admitted_request()
|
|
task = asyncio.create_task(_execute_run(self, launch, _api_server=_api_server))
|
|
self._active_run_tasks[run_id] = task
|
|
try:
|
|
self._background_tasks.add(task)
|
|
except TypeError:
|
|
pass
|
|
if hasattr(task, "add_done_callback"):
|
|
task.add_done_callback(self._background_tasks.discard)
|
|
|
|
response_headers = {"X-Hermes-Session-Key": gateway_session_key} if gateway_session_key else {}
|
|
return web.json_response(
|
|
{"run_id": run_id, "status": "started", "replayed": False},
|
|
status=202,
|
|
headers=response_headers,
|
|
)
|
|
|
|
|
|
def _run_agent_sync(self, run: _RunLaunch, agent, approval_notify, *, _api_server):
|
|
"""Executor-thread body of one run; returns ``(result, usage)``."""
|
|
from gateway.session_context import clear_session_vars
|
|
from tools.approval import (
|
|
register_gateway_notify,
|
|
reset_current_session_key,
|
|
set_current_session_key,
|
|
unregister_gateway_notify,
|
|
)
|
|
|
|
session_id = run.session_id
|
|
effective_task_id = session_id or run.run_id
|
|
approval_token = None
|
|
session_tokens = []
|
|
room_policy_token = None
|
|
with self._profile_scope(run.request_profile):
|
|
try:
|
|
# Bind approval/session identity via contextvars so concurrent
|
|
# runs do not share process environment state.
|
|
approval_token = set_current_session_key(run.approval_session_key)
|
|
# chat_id carries the raw session id exactly like the other
|
|
# agent-entry routes bind it via _run_agent(); without it
|
|
# tools.async_delegation sees an empty HERMES_SESSION_CHAT_ID and
|
|
# background delegations stay forced-sync (no wake target).
|
|
session_tokens = self._bind_api_server_session(
|
|
chat_id=session_id or "",
|
|
session_key=run.approval_session_key,
|
|
session_id=session_id or "",
|
|
browser_control_principal=run.browser_control_principal,
|
|
browser_control_transport_family=run.browser_control_transport_family,
|
|
)
|
|
if run.room_dispatch is not None:
|
|
from gateway.hosted_room_execution_policy import (
|
|
RoomExecutionPolicy,
|
|
bind_room_execution_policy,
|
|
)
|
|
|
|
policy = RoomExecutionPolicy.from_mapping(run.room_execution_policy or {})
|
|
room_policy_token = bind_room_execution_policy(policy)
|
|
register_gateway_notify(run.approval_session_key, approval_notify)
|
|
# /v1/runs owns its agent lifecycle (no TurnRunner/_run_agent):
|
|
# record turn process ownership so stop/cancel reaps only the
|
|
# background processes this run created.
|
|
_api_server._publish_turn_process_ownership(agent, effective_task_id)
|
|
r = agent.run_conversation(
|
|
user_message=run.user_message,
|
|
conversation_history=run.conversation_history,
|
|
task_id=effective_task_id,
|
|
)
|
|
finally:
|
|
# Clear ownership immediately so a later stop/cancel can't reap
|
|
# background work this run deliberately left running.
|
|
_api_server._clear_turn_process_ownership(agent)
|
|
# Record the declared conversation ourselves (not via _run_agent's
|
|
# bind_declared_conversation), with the same precedence gate.
|
|
if run.declared_selected:
|
|
self._bind_declared_conversation(
|
|
getattr(agent, "session_id", None) or session_id, run.gateway_session_key
|
|
)
|
|
try:
|
|
unregister_gateway_notify(run.approval_session_key)
|
|
finally:
|
|
if approval_token is not None:
|
|
with suppress(Exception):
|
|
reset_current_session_key(approval_token)
|
|
if session_tokens:
|
|
with suppress(Exception):
|
|
clear_session_vars(session_tokens)
|
|
if room_policy_token is not None:
|
|
with suppress(Exception):
|
|
from gateway.hosted_room_execution_policy import reset_room_execution_policy
|
|
|
|
reset_room_execution_policy(room_policy_token)
|
|
u = {
|
|
"input_tokens": getattr(agent, "session_prompt_tokens", 0) or 0,
|
|
"output_tokens": getattr(agent, "session_completion_tokens", 0) or 0,
|
|
"total_tokens": getattr(agent, "session_total_tokens", 0) or 0,
|
|
}
|
|
return r, u
|
|
|
|
|
|
async def _execute_run(self, run: _RunLaunch, *, _api_server) -> None:
|
|
"""Background task for one admitted run: drives the agent, then publishes
|
|
the terminal event/status and releases live state."""
|
|
_redact_api_error_text = _api_server._redact_api_error_text
|
|
run_id, q, loop = run.run_id, run.queue, run.loop
|
|
|
|
def _emit(name: str, **fields: Any) -> None:
|
|
run.put_event(_run_event(run_id, name, **fields))
|
|
|
|
def _fail(error_msg: str) -> None: # status first, then best-effort event
|
|
self._set_run_status(run_id, "failed", error=error_msg, last_event="run.failed")
|
|
with suppress(Exception):
|
|
_emit("run.failed", error=error_msg)
|
|
|
|
try:
|
|
self._set_run_status(run_id, "running")
|
|
if run_id in self._stopping_run_ids:
|
|
_emit("run.cancelled")
|
|
self._set_run_status(run_id, "cancelled", last_event="run.cancelled")
|
|
return
|
|
with self._profile_scope(run.request_profile):
|
|
agent = self._create_agent(
|
|
ephemeral_system_prompt=run.ephemeral_system_prompt,
|
|
session_id=run.session_id,
|
|
stream_delta_callback=run.text_cb,
|
|
tool_progress_callback=run.event_cb,
|
|
gateway_session_key=run.gateway_session_key,
|
|
requested_model=run.agent_overrides.get("requested_model"),
|
|
requested_provider=run.agent_overrides.get("requested_provider"),
|
|
model_options=run.agent_overrides.get("model_options"),
|
|
route=run.route,
|
|
room_dispatch=run.room_dispatch,
|
|
room_execution_policy=run.room_execution_policy,
|
|
)
|
|
self._active_run_agents[run_id] = agent
|
|
|
|
def _approval_notify(approval_data: Dict[str, Any]) -> None:
|
|
event = dict(approval_data or {})
|
|
# Redact credentials before the command enters the SSE/API stream;
|
|
# API/desktop clients must never receive the raw flagged command.
|
|
if "command" in event:
|
|
from gateway.run import _redact_approval_command
|
|
|
|
event["command"] = _redact_approval_command(event.get("command"))
|
|
event.update({
|
|
"event": "approval.request",
|
|
"run_id": run_id,
|
|
"timestamp": time.time(),
|
|
"choices": _api_server._approval_event_choices(
|
|
smart_denied=bool(event.get("smart_denied")),
|
|
allow_session=event.get("allow_session") is not False,
|
|
allow_permanent=event.get("allow_permanent") is not False,
|
|
),
|
|
})
|
|
self._set_run_status(
|
|
run_id, "waiting_for_approval", last_event="approval.request", approval=event
|
|
)
|
|
with suppress(Exception):
|
|
loop.call_soon_threadsafe(q.put_nowait, event)
|
|
|
|
result, usage = await asyncio.get_running_loop().run_in_executor(
|
|
None, lambda: _run_agent_sync(self, run, agent, _approval_notify, _api_server=_api_server)
|
|
)
|
|
if (
|
|
run_id in self._stopping_run_ids
|
|
and isinstance(result, dict)
|
|
and result.get("interrupted") is True
|
|
):
|
|
_emit("run.cancelled")
|
|
self._set_run_status(run_id, "cancelled", last_event="run.cancelled")
|
|
elif isinstance(result, dict) and result.get("failed"):
|
|
# Non-retryable client errors (401/400) return failed=True instead
|
|
# of raising, so the except branches below never fire for them.
|
|
error_msg = _redact_api_error_text(result.get("error") or "agent run failed")
|
|
_emit("run.failed", error=error_msg)
|
|
self._set_run_status(run_id, "failed", error=error_msg, last_event="run.failed")
|
|
else:
|
|
final_response = result.get("final_response", "") if isinstance(result, dict) else ""
|
|
# Undelivered steer text (accepted after the final response) rides on
|
|
# the terminal event/status so the client can replay it as the next turn.
|
|
pending_steer = result.get("pending_steer") if isinstance(result, dict) else None
|
|
extra = {"pending_steer": pending_steer} if pending_steer else {}
|
|
_emit("run.completed", output=final_response, usage=usage, **extra)
|
|
self._set_run_status(
|
|
run_id, "completed", output=final_response, usage=usage,
|
|
last_event="run.completed", **extra,
|
|
)
|
|
except asyncio.CancelledError:
|
|
self._set_run_status(run_id, "cancelled", last_event="run.cancelled")
|
|
with suppress(Exception):
|
|
_emit("run.cancelled")
|
|
raise
|
|
except _api_server._ProviderAuthResolutionError as exc:
|
|
# /v1/runs bypasses _run_agent(), so it needs its own branch to surface
|
|
# the same controlled provider-auth message the other endpoints give.
|
|
logger.warning("Provider authentication failed for run=%s: %s", run_id, exc)
|
|
_fail(f"⚠️ Provider authentication failed: {exc}")
|
|
except Exception as exc:
|
|
logger.exception("[api_server] run %s failed", run_id)
|
|
_fail(_redact_api_error_text(exc))
|
|
finally:
|
|
# If the asyncio wrapper is cancelled (e.g. via /stop) the executor
|
|
# thread may still block on an approval Event; unregistering here
|
|
# releases it. Harmlessly idempotent on normal completion.
|
|
with suppress(Exception):
|
|
from tools.approval import unregister_gateway_notify
|
|
|
|
unregister_gateway_notify(run.approval_session_key)
|
|
with suppress(Exception):
|
|
run.put_event(None) # sentinel: close the SSE stream
|
|
self._active_run_agents.pop(run_id, None)
|
|
self._active_run_tasks.pop(run_id, None)
|
|
self._run_approval_sessions.pop(run_id, None)
|
|
self._stopping_run_ids.discard(run_id)
|
|
self._release_run_owner_if_forgotten(run_id)
|
|
|
|
|
|
def _release_run_owner_if_forgotten(self, run_id: str) -> None:
|
|
"""Drop the owner stamp only once nothing keyed by *run_id* survives.
|
|
|
|
Ownership must outlive every surface it protects (statuses, live
|
|
agent/task refs, SSE transport, approval sessions), which are retired on
|
|
different clocks; ``_request_owns_run`` treats ownerless state as fail-closed.
|
|
"""
|
|
if any(
|
|
run_id in table
|
|
for table in (
|
|
self._run_statuses, self._active_run_agents, self._active_run_tasks,
|
|
self._run_streams, self._run_approval_sessions,
|
|
)
|
|
):
|
|
return
|
|
self._run_owners.pop(run_id, None)
|
|
|
|
|
|
def _request_owns_run(self, request: "web.Request", run_id: str) -> bool:
|
|
scope = self._run_idempotency_scope(request)
|
|
owner = self._run_owners.get(run_id)
|
|
if owner is not None:
|
|
return owner == scope
|
|
# No in-memory owner: only a durable record under the caller's own scope
|
|
# admits it. Ownerless run state is an unanswered authorization question;
|
|
# under multiplex_profiles every served profile holds a valid key, so
|
|
# admitting it would make the boundary allow-all.
|
|
return self._run_idempotency_store.owns_run(scope, run_id)
|
|
|
|
|
|
def _load_owned_run(self, request: "web.Request", *, _openai_error, active_fallback: bool):
|
|
"""Resolve ``(run_id, status, agent, task, error)`` for a control endpoint.
|
|
|
|
With *active_fallback*, an in-process run registered before pollable
|
|
status existed is reported as ``running`` instead of 404.
|
|
"""
|
|
run_id = request.match_info["run_id"]
|
|
if not self._request_owns_run(request, run_id):
|
|
return run_id, None, None, None, _run_not_found(_openai_error, run_id)
|
|
agent = self._active_run_agents.get(run_id)
|
|
task = self._active_run_tasks.get(run_id)
|
|
status = self._durable_run_status(request, run_id)
|
|
if status is None and active_fallback and (agent is not None or task is not None):
|
|
status = self._set_run_status(run_id, "running")
|
|
if status is None:
|
|
return run_id, None, agent, task, _run_not_found(_openai_error, run_id)
|
|
return run_id, status, agent, task, None
|
|
|
|
|
|
async def _handle_get_run(self, request: "web.Request", *, _api_server) -> "web.Response":
|
|
"""GET /v1/runs/{run_id} — return pollable run status for external UIs."""
|
|
auth_err = self._check_run_auth(request, permission="status")
|
|
if auth_err:
|
|
return auth_err
|
|
_, status, _, _, err = _load_owned_run(
|
|
self, request, _openai_error=_api_server._openai_error, active_fallback=True
|
|
)
|
|
return err or web.json_response(status)
|
|
|
|
|
|
async def _handle_run_events(self, request: "web.Request", *, _api_server) -> "web.StreamResponse":
|
|
"""GET /v1/runs/{run_id}/events — stream structured agent lifecycle events."""
|
|
_openai_error = _api_server._openai_error
|
|
|
|
auth_err = self._check_auth(request)
|
|
if auth_err:
|
|
return auth_err
|
|
|
|
run_id = request.match_info["run_id"]
|
|
if not self._request_owns_run(request, run_id):
|
|
return _run_not_found(_openai_error, run_id)
|
|
|
|
# Allow subscribing slightly before the run is registered (race window).
|
|
for _ in range(20):
|
|
if run_id in self._run_streams:
|
|
break
|
|
await asyncio.sleep(0.05)
|
|
else:
|
|
return _run_not_found(_openai_error, run_id)
|
|
|
|
q = self._run_streams[run_id]
|
|
self._run_stream_subscribers.add(run_id)
|
|
|
|
response = web.StreamResponse(
|
|
status=200,
|
|
headers={
|
|
"Content-Type": "text/event-stream",
|
|
"Cache-Control": "no-cache",
|
|
"X-Accel-Buffering": "no",
|
|
},
|
|
)
|
|
await response.prepare(request)
|
|
|
|
try:
|
|
while True:
|
|
try:
|
|
event = await asyncio.wait_for(q.get(), timeout=30.0)
|
|
except asyncio.TimeoutError:
|
|
await response.write(b": keepalive\n\n")
|
|
continue
|
|
if event is None: # run finished
|
|
await response.write(b": stream closed\n\n")
|
|
break
|
|
await response.write(_api_server._sse_frame(event))
|
|
except Exception as exc:
|
|
logger.debug("[api_server] SSE stream error for run %s: %s", run_id, exc)
|
|
finally:
|
|
self._run_stream_subscribers.discard(run_id)
|
|
self._run_streams.pop(run_id, None)
|
|
self._run_streams_created.pop(run_id, None)
|
|
self._release_run_owner_if_forgotten(run_id)
|
|
|
|
return response
|
|
|
|
|
|
def _emit_to_stream(self, run_id: str, event: Dict[str, Any]) -> None:
|
|
q = self._run_streams.get(run_id)
|
|
if q is not None:
|
|
with suppress(Exception):
|
|
q.put_nowait(event)
|
|
|
|
|
|
async def _handle_run_approval(self, request: "web.Request", *, _api_server) -> "web.Response":
|
|
"""POST /v1/runs/{run_id}/approval — resolve a pending run approval."""
|
|
_coerce_request_bool = _api_server._coerce_request_bool
|
|
_openai_error = _api_server._openai_error
|
|
|
|
auth_err = self._check_run_auth(request, permission="approve")
|
|
if auth_err:
|
|
return auth_err
|
|
run_id, _, _, _, err = _load_owned_run(
|
|
self, request, _openai_error=_openai_error, active_fallback=False
|
|
)
|
|
if err is not None:
|
|
return err
|
|
|
|
try:
|
|
body = await request.json()
|
|
except Exception:
|
|
return _json_error(_openai_error, "Invalid JSON", status=400)
|
|
|
|
raw_choice = str(body.get("choice", "")).strip().lower()
|
|
aliases = {"approve": "once", "approved": "once", "allow": "once"}
|
|
choice = aliases.get(raw_choice, raw_choice)
|
|
room_scoped = bool(self._room_grant_token(request))
|
|
raw_request_id = body.get("request_id")
|
|
request_id = raw_request_id.strip() if isinstance(raw_request_id, str) else ""
|
|
if raw_request_id is not None and (not request_id or len(request_id) > 256):
|
|
return _json_error(
|
|
_openai_error, "Approval request_id is invalid.",
|
|
code="invalid_approval_request", status=400,
|
|
)
|
|
allowed = {"once", "deny"} if room_scoped else {"once", "session", "always", "deny"}
|
|
if choice not in allowed:
|
|
return _json_error(
|
|
_openai_error, "Invalid approval choice; expected one of: " + ", ".join(sorted(allowed)),
|
|
code="invalid_approval_choice", status=400,
|
|
)
|
|
|
|
resolve_all = (
|
|
_coerce_request_bool(body.get("all"), default=False)
|
|
or _coerce_request_bool(body.get("resolve_all"), default=False)
|
|
)
|
|
if room_scoped and resolve_all:
|
|
return _json_error(
|
|
_openai_error, "Room approvals can resolve only one exact request",
|
|
code="invalid_approval_scope", status=400,
|
|
)
|
|
if room_scoped and not request_id:
|
|
return _json_error(
|
|
_openai_error, "Room approvals require the exact request_id.",
|
|
code="approval_request_required", status=400,
|
|
)
|
|
|
|
approval_session_key = self._run_approval_sessions.get(run_id)
|
|
if not approval_session_key:
|
|
return _json_error(
|
|
_openai_error, f"Run has no active approval session: {run_id}",
|
|
code="approval_not_active", status=409,
|
|
)
|
|
try:
|
|
from tools.approval import resolve_gateway_approval
|
|
|
|
resolved = resolve_gateway_approval(
|
|
approval_session_key, choice, resolve_all=resolve_all, request_id=request_id or None
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("[api_server] approval resolution failed for run %s", run_id)
|
|
return _json_error(_openai_error, str(exc), status=500)
|
|
|
|
if resolved <= 0:
|
|
return _json_error(
|
|
_openai_error, f"Run has no pending approval: {run_id}",
|
|
code="approval_not_pending", status=409,
|
|
)
|
|
|
|
self._set_run_status(run_id, "running", last_event="approval.responded")
|
|
request_id_field = {"request_id": request_id} if request_id else {}
|
|
_emit_to_stream(
|
|
self, run_id,
|
|
_run_event(run_id, "approval.responded", choice=choice, **request_id_field, resolved=resolved),
|
|
)
|
|
return web.json_response({
|
|
"object": "hermes.run.approval_response",
|
|
"run_id": run_id,
|
|
"choice": choice,
|
|
**request_id_field,
|
|
"resolved": resolved,
|
|
})
|
|
|
|
|
|
async def _handle_steer_run(self, request: "web.Request", *, _api_server) -> "web.Response":
|
|
"""POST /v1/runs/{run_id}/steer — inject guidance into a running agent."""
|
|
_openai_error = _api_server._openai_error
|
|
_redact_api_error_text = _api_server._redact_api_error_text
|
|
|
|
auth_err = self._check_auth(request)
|
|
if auth_err:
|
|
return auth_err
|
|
run_id, status, agent, _, err = _load_owned_run(
|
|
self, request, _openai_error=_openai_error, active_fallback=False
|
|
)
|
|
if err is not None:
|
|
return err
|
|
# Only genuinely running runs are steerable. /stop retains agent/task refs
|
|
# during cooperative shutdown, so the status gate (not the mere presence
|
|
# of an agent ref) is what rejects stop-then-steer.
|
|
if status.get("status") != "running" or not hasattr(agent, "steer"):
|
|
return _json_error(
|
|
_openai_error, f"Run is not currently accepting steer input: {run_id}",
|
|
code="run_not_accepting_steer", status=409,
|
|
)
|
|
|
|
body, err = await self._read_json_body(request)
|
|
if err:
|
|
return err
|
|
raw_text = body.get("input") or body.get("message") or body.get("text") or ""
|
|
steer_text = _api_server._normalize_chat_content(raw_text).strip()
|
|
if not steer_text:
|
|
return _json_error(
|
|
_openai_error, "Missing non-empty steer text; expected 'input', 'message', or 'text'.",
|
|
code="invalid_steer_input", status=400,
|
|
)
|
|
|
|
try:
|
|
accepted = bool(agent.steer(steer_text))
|
|
except Exception as exc:
|
|
logger.exception("[api_server] steer failed for run %s", run_id)
|
|
return _json_error(
|
|
_openai_error, _redact_api_error_text(exc), code="steer_failed", status=500
|
|
)
|
|
if not accepted:
|
|
return _json_error(
|
|
_openai_error, f"Run did not accept steer text: {run_id}",
|
|
code="steer_not_accepted", status=409,
|
|
)
|
|
|
|
self._set_run_status(run_id, "running", last_event="run.steered")
|
|
_emit_to_stream(self, run_id, _run_event(run_id, "run.steered", accepted=True))
|
|
return web.json_response({"object": "hermes.run.steer", "run_id": run_id, "accepted": True})
|
|
|
|
|
|
async def _handle_stop_run(self, request: "web.Request", *, _api_server) -> "web.Response":
|
|
"""POST /v1/runs/{run_id}/stop — interrupt a running agent."""
|
|
_openai_error = _api_server._openai_error
|
|
|
|
auth_err = self._check_run_auth(request, permission="stop")
|
|
if auth_err:
|
|
return auth_err
|
|
run_id, status, agent, task, err = _load_owned_run(
|
|
self, request, _openai_error=_openai_error, active_fallback=True
|
|
)
|
|
if err is not None:
|
|
return err
|
|
if status.get("status") in TERMINAL_STATUSES:
|
|
return web.json_response(status)
|
|
|
|
if agent is None and task is None:
|
|
return _json_error(
|
|
_openai_error, f"Run is not active in this gateway process: {run_id}",
|
|
code="run_not_active", status=409,
|
|
)
|
|
|
|
self._set_run_status(run_id, "stopping", last_event="run.stopping")
|
|
self._stopping_run_ids.add(run_id)
|
|
|
|
if agent is not None:
|
|
with suppress(Exception):
|
|
_api_server.request_hard_interrupt(agent, "Stop requested via API")
|
|
# The stopped run is abandoned — reap only the background processes it
|
|
# created. Epoch-gated inside, so a concurrent run sharing the same
|
|
# session_id keeps its own processes; no-op if the run already finished.
|
|
_api_server._reap_disconnected_agent_processes(agent, source="api_server_run_stop")
|
|
|
|
return web.json_response({"run_id": run_id, "status": "stopping"})
|
|
|
|
|
|
async def _sweep_orphaned_runs(self) -> None:
|
|
"""Periodically expire transport buffers and terminal status records."""
|
|
while True:
|
|
await asyncio.sleep(60)
|
|
self._sweep_orphaned_runs_once(time.time())
|
|
|
|
|
|
def _sweep_orphaned_runs_once(self, now: Optional[float] = None) -> None:
|
|
"""Expire old SSE buffers without treating transport age as run age."""
|
|
if now is None:
|
|
now = time.time()
|
|
stale = [
|
|
run_id
|
|
for run_id, created_at in list(self._run_streams_created.items())
|
|
if now - created_at > self._RUN_STREAM_TTL and run_id not in self._run_stream_subscribers
|
|
]
|
|
for run_id in stale:
|
|
logger.debug("[api_server] sweeping expired run transport %s", run_id)
|
|
task = self._active_run_tasks.get(run_id)
|
|
task_done = task is None or task.done()
|
|
if task_done:
|
|
with suppress(Exception):
|
|
from tools.approval import unregister_gateway_notify
|
|
|
|
approval_session_key = self._run_approval_sessions.get(run_id)
|
|
if approval_session_key:
|
|
unregister_gateway_notify(approval_session_key)
|
|
# The transport TTL always bounds buffering. Live control state is
|
|
# independent and survives until the executor-backed task returns.
|
|
self._run_streams.pop(run_id, None)
|
|
self._run_streams_created.pop(run_id, None)
|
|
if task_done:
|
|
self._active_run_agents.pop(run_id, None)
|
|
self._active_run_tasks.pop(run_id, None)
|
|
self._run_approval_sessions.pop(run_id, None)
|
|
self._stopping_run_ids.discard(run_id)
|
|
self._release_run_owner_if_forgotten(run_id)
|
|
|
|
stale_statuses = [
|
|
run_id
|
|
for run_id, status in list(self._run_statuses.items())
|
|
if status.get("status") in {"completed", "failed", "cancelled"}
|
|
and now - float(status.get("updated_at", 0) or 0) > self._RUN_STATUS_TTL
|
|
]
|
|
for run_id in stale_statuses:
|
|
self._run_statuses.pop(run_id, None)
|
|
self._run_idempotency_ids.discard(run_id)
|
|
self._release_run_owner_if_forgotten(run_id)
|