"""Durable execution state for a same-gateway hosted room driver. Owns only the driver lease and task state machine: no model calls, no sessions, no dependency on the hosted-room event log. Callers supply the database path and clock so recovery and fencing are testable without process-global state. """ from __future__ import annotations import dataclasses import hashlib import json import math import sqlite3 from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, Literal, get_args from gateway.hosted_rooms_common import ( canonical_json, identifier, non_negative_int, positive_int, table_columns, transaction, ) Clock = Callable[[], float] TaskStatus = Literal["queued", "running", "settled", "failed", "cancelled", "indeterminate", "deferred", "stopping"] TerminalStatus = Literal["settled", "failed"] MAX_IDENTIFIER_CHARS = 128 MAX_PROMPT_BYTES = 128 * 1024 MAX_RESULT_JSON_BYTES = 256 * 1024 TERMINAL_TASK_RETENTION_SECONDS = 30 * 24 * 60 * 60 MAX_RETAINED_TERMINAL_TASKS = 2048 MAX_TASK_PRUNE_BATCH = 1000 TASK_STATUSES = frozenset(get_args(TaskStatus)) TERMINAL_STATUSES = frozenset({"settled", "failed", "cancelled"}) _TASK_PAYLOAD_REQUIRED_FIELDS = frozenset({"target_profile", "prompt", "source_event_seq"}) _TASK_PAYLOAD_OPTIONAL_FIELDS = frozenset({"target_member_id"}) _LEASE_COLUMNS = frozenset({ "room_id", "gateway_id", "authority_epoch", "process_generation", "lease_generation", "expires_at", "acquired_at", "updated_at", "released_at", }) _TASK_COLUMN_ORDER = ( "room_id", "task_id", "thread_id", "turn_id", "source_event_seq", "payload_json", "payload_digest", "status", "execution_generation", "cancel_generation", "run_gateway_id", "run_process_generation", "run_lease_generation", "cancel_id", "settlement_id", "settlement_status", "result_json", "created_at", "updated_at", "started_at", "terminal_at", "indeterminate_at", ) _TASK_COLUMNS = frozenset(_TASK_COLUMN_ORDER) _TASK_ORDER = "ORDER BY source_event_seq, created_at, task_id" _SELECT_LEASE = "SELECT * FROM hosted_room_driver_leases WHERE room_id=?" _SELECT_TASK = "SELECT * FROM hosted_room_driver_tasks WHERE room_id=? AND task_id=?" _TASK_INDEX_SQL = """CREATE INDEX {if_not_exists}idx_hosted_room_driver_tasks_status ON hosted_room_driver_tasks(room_id, status, source_event_seq, created_at, task_id)""" # --- Fenced task UPDATE statements (one per state-machine transition) --------- # Every transition is "UPDATE ... SET WHERE room_id=? AND task_id=? AND "; # the fence names the expected status plus the generations that must not have moved. _GENERATION_FENCE = "execution_generation=? AND cancel_generation=?" _RUN_FENCE = "run_gateway_id=? AND run_process_generation=? AND run_lease_generation=?" _SETTLE_SET = "status=?, settlement_id=?, settlement_status=?, result_json=?, terminal_at=?, updated_at=?" _REQUEUE_SET = "status='queued', run_gateway_id=NULL, run_process_generation=NULL, run_lease_generation=NULL" _CANCEL_SET = "status='cancelled', cancel_generation=?, cancel_id=?, terminal_at=?, updated_at=?" def _task_update(set_clause: str, fence: str) -> str: return f"UPDATE hosted_room_driver_tasks SET {set_clause} WHERE room_id=? AND task_id=? AND {fence}" def _generation_update(set_clause: str, status: str) -> str: """Transition fenced on ``status`` + both generations (terminal settlements and the recovery family).""" return _task_update(set_clause, f"status='{status}' AND {_GENERATION_FENCE}") _SETTLE_RUNNING_SQL = _generation_update(_SETTLE_SET, "running") + f" AND {_RUN_FENCE}" _SETTLE_STOPPING_SQL = _generation_update(_SETTLE_SET, "stopping") _REQUEUE_RUNNING_SQL = _task_update( f"{_REQUEUE_SET}, started_at=NULL, updated_at=?", f"status='running' AND {_GENERATION_FENCE} AND {_RUN_FENCE}", ) _CANCEL_QUEUED_SQL = _task_update(_CANCEL_SET, "status IN ('queued', 'deferred') AND cancel_generation=?") _BEGIN_STOP_SQL = _task_update( "status='stopping', cancel_generation=?, cancel_id=?, updated_at=?", "status IN ('running', 'indeterminate') AND cancel_generation=?", ) _COMPLETE_STOP_SQL = _task_update( "status='cancelled', terminal_at=?, updated_at=?", "status='stopping' AND cancel_id=? AND cancel_generation=?", ) # Lease-first recovery transitions: name -> (fenced status, SET clause, generation-guard stale message, # row stale message); the UPDATE is _generation_update(set_clause, status). _INDETERMINATE_STALE = "indeterminate task generation changed" _GENERATION_TRANSITIONS = { "resolve": ( "indeterminate", _SETTLE_SET, _INDETERMINATE_STALE, "indeterminate task changed during reconciliation", ), "resolve_cancel": ( "indeterminate", _CANCEL_SET, "indeterminate cancellation proof is stale", "indeterminate cancellation proof lost its fence", ), "requeue": ( "indeterminate", f"{_REQUEUE_SET}, started_at=NULL, indeterminate_at=NULL, updated_at=?", _INDETERMINATE_STALE, "indeterminate task changed during requeue", ), "defer": ( "indeterminate", "status='deferred', result_json=?, terminal_at=?, updated_at=?", _INDETERMINATE_STALE, "indeterminate task changed during deferral", ), "requeue_deferred": ( "deferred", f"{_REQUEUE_SET}, result_json=NULL, started_at=NULL, terminal_at=NULL, indeterminate_at=NULL, updated_at=?", "deferred task generation changed", "deferred task changed during requeue", ), } class DriverStateError(ValueError): """Base class for invalid or conflicting driver-state operations.""" class DriverValidationError(DriverStateError): """Raised when an identifier, clock, TTL, or payload is invalid.""" class RoomUnavailableError(DriverStateError): """Raised when the hosted room does not exist or was disbanded.""" class LeaseHeldError(DriverStateError): """Raised when another unexpired driver generation owns the room.""" class StaleLeaseError(DriverStateError): """Raised when a lease generation can no longer mutate room state.""" class TaskConflictError(DriverStateError): """Raised when an idempotency key is reused for different task state.""" class StaleTaskError(DriverStateError): """Raised when an obsolete task attempt or cancellation tries to commit.""" class InvalidTaskTransitionError(DriverStateError): """Raised when a requested task transition is not allowed.""" def _identifier(value: Any, *, label: str) -> str: return identifier(value, label=label, error=DriverValidationError, max_chars=MAX_IDENTIFIER_CHARS) def _finite(compute: Callable[[], Any], message: str, *, positive: bool = False) -> float: try: value = float(compute()) except (TypeError, ValueError, OverflowError) as exc: raise DriverValidationError(message) from exc if not math.isfinite(value) or (positive and value <= 0): raise DriverValidationError(message) return value def _timestamp(clock: Clock) -> float: if not callable(clock): raise DriverValidationError("clock must be callable") return _finite(clock, "clock must return a finite number") def _ttl(value: Any) -> float: return _finite(lambda: value, "ttl_seconds must be a finite positive number", positive=True) def _expiry(now: float, ttl: float) -> float: expires_at = now + ttl if not math.isfinite(expires_at): raise DriverValidationError("lease expiry must be finite") return expires_at def _canonical_json(value: Any) -> str: return canonical_json( value, error=DriverValidationError, label="result", max_bytes=MAX_RESULT_JSON_BYTES, ensure_ascii=True ) def _task_payload(value: Any) -> tuple[dict[str, Any], str, str]: if not isinstance(value, dict): raise DriverValidationError("payload must be an object") unknown = set(value) - _TASK_PAYLOAD_REQUIRED_FIELDS - _TASK_PAYLOAD_OPTIONAL_FIELDS missing = _TASK_PAYLOAD_REQUIRED_FIELDS - set(value) if unknown: raise DriverValidationError(f"unknown payload fields: {', '.join(sorted(unknown))}") if missing: raise DriverValidationError(f"missing payload fields: {', '.join(sorted(missing))}") target_profile = _identifier(value["target_profile"], label="target_profile") prompt = value["prompt"] if not isinstance(prompt, str): raise DriverValidationError("prompt must be a string") if not prompt.strip(): raise DriverValidationError("prompt must not be empty") if len(prompt.encode("utf-8")) > MAX_PROMPT_BYTES: raise DriverValidationError("prompt is too large") source_event_seq = positive_int( value["source_event_seq"], error=DriverValidationError, message="source_event_seq must be a positive integer", ) normalized = {"target_profile": target_profile, "prompt": prompt, "source_event_seq": source_event_seq} if "target_member_id" in value: normalized["target_member_id"] = _identifier(value["target_member_id"], label="target_member_id") encoded = json.dumps(normalized, ensure_ascii=True, sort_keys=True, separators=(",", ":")) return normalized, encoded, hashlib.sha256(encoded.encode("utf-8")).hexdigest() @dataclass(frozen=True) class TaskIdentity: """Stable identity for one admitted room turn.""" room_id: str task_id: str thread_id: str turn_id: str def __post_init__(self) -> None: for field in ("room_id", "task_id", "thread_id", "turn_id"): object.__setattr__(self, field, _identifier(getattr(self, field), label=field)) @dataclass(frozen=True) class DriverLease: """A fenced lease held by one gateway process incarnation.""" room_id: str gateway_id: str authority_epoch: int process_generation: str lease_generation: int expires_at: float reclaimed: bool = False @dataclass(frozen=True) class TaskAttempt: """The exact running generation authorized to settle one task.""" identity: TaskIdentity lease: DriverLease execution_generation: int cancel_generation: int def _create_task_table(conn: sqlite3.Connection, table: str = "hosted_room_driver_tasks") -> None: if table not in {"hosted_room_driver_tasks", "hosted_room_driver_tasks_next"}: raise DriverStateError("invalid hosted-room task table name") conn.execute( f"""CREATE TABLE IF NOT EXISTS {table} ( room_id TEXT NOT NULL, task_id TEXT NOT NULL, thread_id TEXT NOT NULL, turn_id TEXT NOT NULL, source_event_seq INTEGER NOT NULL CHECK (source_event_seq >= 1), payload_json TEXT NOT NULL, payload_digest TEXT NOT NULL, status TEXT NOT NULL CHECK (status IN ( 'queued', 'running', 'settled', 'failed', 'cancelled', 'indeterminate', 'deferred', 'stopping')), execution_generation INTEGER NOT NULL DEFAULT 0 CHECK (execution_generation >= 0), cancel_generation INTEGER NOT NULL DEFAULT 0 CHECK (cancel_generation >= 0), run_gateway_id TEXT, run_process_generation TEXT, run_lease_generation INTEGER, cancel_id TEXT, settlement_id TEXT, settlement_status TEXT, result_json TEXT, created_at REAL NOT NULL, updated_at REAL NOT NULL, started_at REAL, terminal_at REAL, indeterminate_at REAL, PRIMARY KEY (room_id, task_id), UNIQUE (room_id, thread_id, turn_id), FOREIGN KEY (room_id) REFERENCES hosted_rooms(room_id))""" ) def _initialize_schema(conn: sqlite3.Connection) -> None: conn.execute( """CREATE TABLE IF NOT EXISTS hosted_room_driver_leases ( room_id TEXT PRIMARY KEY, gateway_id TEXT NOT NULL, authority_epoch INTEGER NOT NULL CHECK (authority_epoch >= 1), process_generation TEXT NOT NULL, lease_generation INTEGER NOT NULL CHECK (lease_generation >= 1), expires_at REAL NOT NULL, acquired_at REAL NOT NULL, updated_at REAL NOT NULL, released_at REAL, FOREIGN KEY (room_id) REFERENCES hosted_rooms(room_id))""" ) _create_task_table(conn) _validate_schema(conn) conn.execute(_TASK_INDEX_SQL.format(if_not_exists="IF NOT EXISTS ")) def _validate_schema(conn: sqlite3.Connection) -> None: lease_columns = table_columns(conn, "hosted_room_driver_leases") task_columns = table_columns(conn, "hosted_room_driver_tasks") if lease_columns != _LEASE_COLUMNS or task_columns != _TASK_COLUMNS: raise DriverStateError( "unsupported unpublished hosted-room driver schema; " "recreate the driver tables before starting the driver" ) for table in ("hosted_room_driver_leases", "hosted_room_driver_tasks"): foreign_keys = conn.execute(f"PRAGMA foreign_key_list({table})").fetchall() if not any(row[2] == "hosted_rooms" and row[3] == "room_id" and row[4] == "room_id" for row in foreign_keys): raise DriverStateError(f"{table} is missing its hosted_rooms foreign key") def _schema_objects_exist(conn: sqlite3.Connection) -> bool: rows = conn.execute( """SELECT name FROM sqlite_master WHERE type='table' AND name IN ('hosted_room_driver_leases', 'hosted_room_driver_tasks')""" ).fetchall() if {row[0] for row in rows} != {"hosted_room_driver_leases", "hosted_room_driver_tasks"}: return False index = conn.execute( "SELECT 1 FROM sqlite_master WHERE type='index' AND name='idx_hosted_room_driver_tasks_status'" ).fetchone() return index is not None def _task_schema_supports_current_statuses(conn: sqlite3.Connection) -> bool: row = conn.execute( "SELECT sql FROM sqlite_master WHERE type='table' AND name='hosted_room_driver_tasks'" ).fetchone() sql = str(row[0] or "").lower() if row else "" return "'stopping'" in sql and "'deferred'" in sql def _migrate_task_status_constraint(conn: sqlite3.Connection) -> None: """Expand the unpublished task-state CHECK without losing durable work.""" conn.execute("DROP INDEX IF EXISTS idx_hosted_room_driver_tasks_status") _create_task_table(conn, "hosted_room_driver_tasks_next") columns = ", ".join(_TASK_COLUMN_ORDER) conn.execute( f"INSERT INTO hosted_room_driver_tasks_next ({columns}) SELECT {columns} FROM hosted_room_driver_tasks" ) conn.execute("DROP TABLE hosted_room_driver_tasks") conn.execute("ALTER TABLE hosted_room_driver_tasks_next RENAME TO hosted_room_driver_tasks") conn.execute(_TASK_INDEX_SQL.format(if_not_exists="")) def _connect(db_path: Path | str) -> sqlite3.Connection: from hermes_state import apply_wal_with_fallback path = Path(db_path) path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(path, timeout=10) conn.row_factory = sqlite3.Row try: apply_wal_with_fallback(conn, db_label="state.db (hosted_room_driver)") conn.execute("PRAGMA foreign_keys=ON") if _schema_objects_exist(conn): if not _task_schema_supports_current_statuses(conn): conn.execute("BEGIN IMMEDIATE") _migrate_task_status_constraint(conn) conn.commit() _validate_schema(conn) return conn # Schema creation is one database-wide transaction. The driver schema # has never shipped, so an incompatible draft schema fails closed # instead of attempting a partial in-place migration. conn.execute("BEGIN IMMEDIATE") _initialize_schema(conn) conn.commit() except Exception: conn.rollback() conn.close() raise return conn def _transaction(db_path: Path | str): return transaction(_connect, db_path, immediate=True) def _lease_from_row(row: sqlite3.Row | dict[str, Any], *, reclaimed: bool = False) -> DriverLease: return DriverLease( room_id=row["room_id"], gateway_id=row["gateway_id"], authority_epoch=int(row["authority_epoch"]), process_generation=row["process_generation"], lease_generation=int(row["lease_generation"]), expires_at=float(row["expires_at"]), reclaimed=reclaimed, ) def _task_identity_from_row(row: sqlite3.Row) -> TaskIdentity: return TaskIdentity( room_id=row["room_id"], task_id=row["task_id"], thread_id=row["thread_id"], turn_id=row["turn_id"] ) def _optional(cast: Callable[[Any], Any]) -> Callable[[Any], Any]: return lambda value: cast(value) if value is not None else None # Task-view casts per row column (columns after payload_digest in _TASK_COLUMN_ORDER, same key order; # result_json is exposed as "result"). Columns not listed are passed through untouched. _TASK_VIEW_CASTS: dict[str, Callable[[Any], Any]] = { "execution_generation": int, "cancel_generation": int, "run_lease_generation": _optional(int), "result_json": _optional(json.loads), "created_at": float, "updated_at": float, "started_at": _optional(float), "terminal_at": _optional(float), "indeterminate_at": _optional(float), } def _task_from_row(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, Any]: try: payload, encoded_payload, payload_digest = _task_payload(json.loads(row["payload_json"])) except (TypeError, json.JSONDecodeError, DriverValidationError) as exc: raise TaskConflictError("stored task payload is invalid") from exc if (encoded_payload, payload_digest, payload["source_event_seq"]) != ( row["payload_json"], row["payload_digest"], int(row["source_event_seq"]) ): raise TaskConflictError("stored task payload failed its integrity check") task: dict[str, Any] = {"identity": _task_identity_from_row(row), "payload": payload} for column in _TASK_COLUMN_ORDER[_TASK_COLUMN_ORDER.index("payload_digest"):]: task["result" if column == "result_json" else column] = _TASK_VIEW_CASTS.get(column, lambda v: v)(row[column]) task["idempotent"] = idempotent return task def _load_task(conn: sqlite3.Connection, identity: TaskIdentity) -> sqlite3.Row: row = conn.execute(_SELECT_TASK, (identity.room_id, identity.task_id)).fetchone() if row is None: raise TaskConflictError("task does not exist") if _task_identity_from_row(row) != identity: raise TaskConflictError("task_id is already bound to a different turn") return row def _tasks_in_order(conn: sqlite3.Connection, room_id: str, status: str | None = None) -> list[sqlite3.Row]: where, params = ("", (room_id,)) if status is None else (" AND status=?", (room_id, status)) sql = f"SELECT * FROM hosted_room_driver_tasks WHERE room_id=?{where} {_TASK_ORDER}" return conn.execute(sql, params).fetchall() def _load_active_room(conn: sqlite3.Connection, room_id: str) -> sqlite3.Row: try: row = conn.execute( "SELECT room_id, authority_gateway_id, authority_epoch, disbanded_at FROM hosted_rooms WHERE room_id=?", (room_id,), ).fetchone() except sqlite3.OperationalError as exc: if "no such table" in str(exc).lower(): raise RoomUnavailableError("hosted room does not exist") from exc raise if row is None: raise RoomUnavailableError("hosted room does not exist") if row["disbanded_at"] is not None: raise RoomUnavailableError("hosted room is disbanded") return row def _require_room_authority( conn: sqlite3.Connection, *, room_id: str, gateway_id: str, authority_epoch: int ) -> sqlite3.Row: room = _load_active_room(conn, room_id) if room["authority_gateway_id"] != gateway_id or int(room["authority_epoch"]) != authority_epoch: raise StaleLeaseError("hosted room authority changed") return room def _require_lease_authority(conn: sqlite3.Connection, lease: DriverLease) -> sqlite3.Row: return _require_room_authority( conn, room_id=lease.room_id, gateway_id=lease.gateway_id, authority_epoch=lease.authority_epoch ) def _lease_row_matches(row: sqlite3.Row | None, lease: DriverLease) -> bool: return row is not None and ( row["gateway_id"], int(row["authority_epoch"]), row["process_generation"], int(row["lease_generation"]) ) == (lease.gateway_id, lease.authority_epoch, lease.process_generation, lease.lease_generation) def _require_active_lease(conn: sqlite3.Connection, lease: DriverLease, *, now: float) -> sqlite3.Row: _require_lease_authority(conn, lease) row = conn.execute(_SELECT_LEASE, (lease.room_id,)).fetchone() if not _lease_row_matches(row, lease) or row["released_at"] is not None or float(row["expires_at"]) <= now: raise StaleLeaseError("driver lease is stale or expired") return row def _check_same_room(lease: DriverLease, identity: TaskIdentity) -> None: if lease.room_id != identity.room_id: raise DriverValidationError("lease and task belong to different rooms") def _cancel_generation(value: int) -> int: # Deliberately accepts bool (a bool is an int); do not swap for non_negative_int. if not isinstance(value, int) or value < 0: raise DriverValidationError("expected_cancel_generation must be non-negative") return value def _expected_generations( lease: DriverLease, identity: TaskIdentity, execution_generation: int, cancel_generation: int ) -> None: _check_same_room(lease, identity) if not isinstance(execution_generation, int) or execution_generation < 1: raise DriverValidationError("expected_execution_generation must be a positive integer") _cancel_generation(cancel_generation) def _terminal_settlement_id(settlement_id: Any, status: Any) -> str: settlement_id = _identifier(settlement_id, label="settlement_id") if status not in {"settled", "failed"}: raise DriverValidationError("status must be 'settled' or 'failed'") return settlement_id def _settlement_replay(settlement_id: str, status: str, result_json: str) -> Callable[[sqlite3.Row], Any]: """Replay predicate: identical settlement already committed; a different one is a conflict.""" def replay(row: sqlite3.Row) -> dict[str, Any] | None: if row["settlement_id"] is None: return None if (row["settlement_id"], row["settlement_status"], row["result_json"]) == (settlement_id, status, result_json): return _task_from_row(row, idempotent=True) raise TaskConflictError("task already has a different terminal settlement") return replay def _cancel_replay(cancel_id: str, status: str = "cancelled") -> Callable[[sqlite3.Row], Any]: """Replay predicate: same cancel_id already committed in ``status``.""" return lambda row: ( _task_from_row(row, idempotent=True) if row["status"] == status and row["cancel_id"] == cancel_id else None ) def _generations_match(row: sqlite3.Row, status: str, execution_generation: int, cancel_generation: int) -> bool: return (row["status"], int(row["execution_generation"]), int(row["cancel_generation"])) == ( status, execution_generation, cancel_generation ) def _require_cancel_generation(row: sqlite3.Row, expected_cancel_generation: int) -> None: if int(row["cancel_generation"]) != expected_cancel_generation: raise StaleTaskError("task cancellation generation changed") def _transition( db_path: Path | str, identity: TaskIdentity, *, sql: str, set_params: tuple[Any, ...], fence_params: tuple[Any, ...], stale: str, now: float, lease: DriverLease | None = None, lease_first: bool = True, replay: Callable[[sqlite3.Row], dict[str, Any] | None] | None = None, guard: Callable[[sqlite3.Row], None] | None = None, ) -> dict[str, Any]: """Run one fenced task transition: load -> idempotent replay -> lease/fence guard -> UPDATE. ``sql`` binds ``(*set_params, room_id, task_id, *fence_params)`` and must hit exactly one row or ``stale`` is raised. ``lease_first`` puts the active-lease check before the row load (recovery paths) instead of after the replay (settlement paths: an identical replay still succeeds after the lease moved on). """ params = (*set_params, identity.room_id, identity.task_id, *fence_params) with _transaction(db_path) as conn: if lease is not None and lease_first: _require_active_lease(conn, lease, now=now) row = _load_task(conn, identity) if replay is not None: replayed = replay(row) if replayed is not None: return replayed if lease is not None and not lease_first: _require_active_lease(conn, lease, now=now) if guard is not None: guard(row) if conn.execute(sql, params).rowcount != 1: raise StaleTaskError(stale) return _task_from_row(_load_task(conn, identity)) def _generation_transition( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, name: str, execution_generation: int, cancel_generation: int, *, now: float, set_params: tuple[Any, ...], replay: Callable[[sqlite3.Row], Any] | None = None, ) -> dict[str, Any]: """Lease-first transition from ``_GENERATION_TRANSITIONS`` fenced on status + both generations.""" status, set_clause, generation_stale, stale = _GENERATION_TRANSITIONS[name] def guard(row: sqlite3.Row) -> None: if not _generations_match(row, status, execution_generation, cancel_generation): raise StaleTaskError(generation_stale) return _transition( db_path, identity, lease=lease, now=now, replay=replay, guard=guard, sql=_generation_update(set_clause, status), set_params=set_params, fence_params=(execution_generation, cancel_generation), stale=stale, ) def _run_fence_transition( db_path: Path | str, attempt: TaskAttempt, *, guard_stale: str, lease_generation: Callable[[Any], int] = int, **transition: Any, ) -> dict[str, Any]: """Transition fenced on this attempt's running generation under its exact lease (row guard + SQL fence). ``lease_generation`` casts the stored run_lease_generation: ``int`` raises on NULL, ``int(v or 0)`` reads it as generation 0. """ lease = attempt.lease def guard(row: sqlite3.Row) -> None: if not ( _generations_match(row, "running", attempt.execution_generation, attempt.cancel_generation) and row["run_gateway_id"] == lease.gateway_id and row["run_process_generation"] == lease.process_generation and lease_generation(row["run_lease_generation"]) == lease.lease_generation ): raise StaleTaskError(guard_stale) return _transition( db_path, attempt.identity, lease=lease, guard=guard, fence_params=( attempt.execution_generation, attempt.cancel_generation, lease.gateway_id, lease.process_generation, lease.lease_generation, ), **transition, ) def acquire_lease( db_path: Path | str, *, room_id: Any, gateway_id: Any, authority_epoch: Any, process_generation: Any, ttl_seconds: Any, clock: Clock, ) -> DriverLease: """Acquire an empty or expired room lease with a monotonic generation.""" room_id = _identifier(room_id, label="room_id") gateway_id = _identifier(gateway_id, label="gateway_id") authority_epoch = positive_int( authority_epoch, error=DriverValidationError, message="authority_epoch must be a positive integer" ) process_generation = _identifier(process_generation, label="process_generation") ttl_seconds = _ttl(ttl_seconds) now = _timestamp(clock) expires_at = _expiry(now, ttl_seconds) with _transaction(db_path) as conn: _require_room_authority(conn, room_id=room_id, gateway_id=gateway_id, authority_epoch=authority_epoch) row = conn.execute(_SELECT_LEASE, (room_id,)).fetchone() if row is None: conn.execute( """INSERT INTO hosted_room_driver_leases ( room_id, gateway_id, authority_epoch, process_generation, lease_generation, expires_at, acquired_at, updated_at, released_at ) VALUES (?, ?, ?, ?, 1, ?, ?, ?, NULL)""", (room_id, gateway_id, authority_epoch, process_generation, expires_at, now, now), ) return _lease_from_row(conn.execute(_SELECT_LEASE, (room_id,)).fetchone()) same_authority = row["gateway_id"] == gateway_id and int(row["authority_epoch"]) == authority_epoch live = row["released_at"] is None and float(row["expires_at"]) > now if same_authority and row["process_generation"] == process_generation and live: renewed_expiry = max(float(row["expires_at"]), expires_at) conn.execute( "UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=? WHERE room_id=? AND lease_generation=?", (renewed_expiry, now, room_id, int(row["lease_generation"])), ) return _lease_from_row({**dict(row), "expires_at": renewed_expiry}) if same_authority and live: raise LeaseHeldError("room driver lease is held by another generation") updated = conn.execute( """UPDATE hosted_room_driver_leases SET gateway_id=?, authority_epoch=?, process_generation=?, lease_generation=lease_generation + 1, expires_at=?, acquired_at=?, updated_at=?, released_at=NULL WHERE room_id=? AND lease_generation=? AND ( gateway_id != ? OR authority_epoch != ? OR released_at IS NOT NULL OR expires_at <= ?)""", ( gateway_id, authority_epoch, process_generation, expires_at, now, now, room_id, int(row["lease_generation"]), gateway_id, authority_epoch, now, ), ) if updated.rowcount != 1: raise LeaseHeldError("room driver lease changed during acquisition") return _lease_from_row(conn.execute(_SELECT_LEASE, (room_id,)).fetchone(), reclaimed=True) def renew_lease(db_path: Path | str, lease: DriverLease, *, ttl_seconds: Any, clock: Clock) -> DriverLease: """Renew the exact active lease generation or fail closed.""" ttl_seconds = _ttl(ttl_seconds) now = _timestamp(clock) requested_expiry = _expiry(now, ttl_seconds) with _transaction(db_path) as conn: current = _require_active_lease(conn, lease, now=now) expires_at = max(float(current["expires_at"]), requested_expiry) updated = conn.execute( """UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=? WHERE room_id=? AND gateway_id=? AND process_generation=? AND lease_generation=? AND released_at IS NULL AND expires_at > ?""", (expires_at, now, lease.room_id, lease.gateway_id, lease.process_generation, lease.lease_generation, now), ) if updated.rowcount != 1: raise StaleLeaseError("driver lease changed during renewal") return dataclasses.replace(lease, expires_at=expires_at, reclaimed=False) def release_lease(db_path: Path | str, lease: DriverLease, *, clock: Clock) -> dict[str, Any]: """Release the exact active lease generation idempotently.""" now = _timestamp(clock) with _transaction(db_path) as conn: _require_lease_authority(conn, lease) row = conn.execute(_SELECT_LEASE, (lease.room_id,)).fetchone() if not _lease_row_matches(row, lease): raise StaleLeaseError("driver lease is stale") if row["released_at"] is not None: return {"lease": _lease_from_row(row), "idempotent": True} if float(row["expires_at"]) <= now: raise StaleLeaseError("driver lease expired before release") running = conn.execute( "SELECT 1 FROM hosted_room_driver_tasks WHERE room_id=? AND status='running' LIMIT 1", (lease.room_id,) ).fetchone() if running is not None: raise InvalidTaskTransitionError("cannot release a room lease while tasks are running") conn.execute( """UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=?, released_at=? WHERE room_id=? AND lease_generation=?""", (now, now, now, lease.room_id, lease.lease_generation), ) current = {**dict(row), "expires_at": now, "updated_at": now, "released_at": now} return {"lease": _lease_from_row(current), "idempotent": False} def admit_task(db_path: Path | str, identity: TaskIdentity, *, payload: Any, clock: Clock) -> dict[str, Any]: """Persist a queued task, or return the identical admission.""" normalized_payload, payload_json, payload_digest = _task_payload(payload) now = _timestamp(clock) with _transaction(db_path) as conn: _load_active_room(conn, identity.room_id) existing = conn.execute(_SELECT_TASK, (identity.room_id, identity.task_id)).fetchone() if existing is not None: if _task_identity_from_row(existing) != identity: raise TaskConflictError("task_id is already bound to a different turn") if existing["payload_digest"] != payload_digest or existing["payload_json"] != payload_json: raise TaskConflictError("task_id is already bound to a different payload") return _task_from_row(existing, idempotent=True) turn = conn.execute( "SELECT * FROM hosted_room_driver_tasks WHERE room_id=? AND thread_id=? AND turn_id=?", (identity.room_id, identity.thread_id, identity.turn_id), ).fetchone() if turn is not None: raise TaskConflictError("thread_id and turn_id are already bound to a task") conn.execute( """INSERT INTO hosted_room_driver_tasks ( room_id, task_id, thread_id, turn_id, source_event_seq, payload_json, payload_digest, status, execution_generation, cancel_generation, created_at, updated_at ) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, 0, ?, ?)""", ( identity.room_id, identity.task_id, identity.thread_id, identity.turn_id, normalized_payload["source_event_seq"], payload_json, payload_digest, now, now, ), ) return _task_from_row(_load_task(conn, identity)) def start_task( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_cancel_generation: int, clock: Clock ) -> TaskAttempt: """Move one queued task to running under the current driver lease.""" _check_same_room(lease, identity) _cancel_generation(expected_cancel_generation) now = _timestamp(clock) with _transaction(db_path) as conn: _require_active_lease(conn, lease, now=now) row = _load_task(conn, identity) _require_cancel_generation(row, expected_cancel_generation) if row["status"] != "queued": raise InvalidTaskTransitionError(f"cannot start task in state '{row['status']}'") unresolved = conn.execute( f"""SELECT task_id, status FROM hosted_room_driver_tasks WHERE room_id=? AND status IN ('running', 'indeterminate', 'stopping') {_TASK_ORDER} LIMIT 1""", (identity.room_id,), ).fetchone() if unresolved is not None: raise InvalidTaskTransitionError("room recovery must resolve the prior task before starting new work") next_queued = conn.execute( f"SELECT task_id FROM hosted_room_driver_tasks WHERE room_id=? AND status='queued' {_TASK_ORDER} LIMIT 1", (identity.room_id,), ).fetchone() if next_queued is None or next_queued["task_id"] != identity.task_id: raise InvalidTaskTransitionError("task is not next in the hosted room event order") execution_generation = int(row["execution_generation"]) + 1 updated = conn.execute( """UPDATE hosted_room_driver_tasks SET status='running', execution_generation=?, run_gateway_id=?, run_process_generation=?, run_lease_generation=?, started_at=?, updated_at=? WHERE room_id=? AND task_id=? AND status='queued' AND cancel_generation=?""", ( execution_generation, lease.gateway_id, lease.process_generation, lease.lease_generation, now, now, identity.room_id, identity.task_id, expected_cancel_generation, ), ) if updated.rowcount != 1: raise StaleTaskError("task changed during start") return TaskAttempt( identity=identity, lease=lease, execution_generation=execution_generation, cancel_generation=expected_cancel_generation, ) def settle_task( db_path: Path | str, attempt: TaskAttempt, *, settlement_id: Any, status: TerminalStatus, result: Any, clock: Clock ) -> dict[str, Any]: """Commit one terminal result if every lease and task fence still matches.""" settlement_id = _terminal_settlement_id(settlement_id, status) result_json = _canonical_json(result) now = _timestamp(clock) return _run_fence_transition( db_path, attempt, guard_stale="task attempt is stale or cancelled", lease_first=False, now=now, replay=_settlement_replay(settlement_id, status, result_json), sql=_SETTLE_RUNNING_SQL, set_params=(status, settlement_id, status, result_json, now, now), stale="task changed during settlement", ) def settle_stopping_task( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int, expected_cancel_generation: int, settlement_id: Any, status: TerminalStatus, result: Any, clock: Clock, ) -> dict[str, Any]: """Commit a completion that won the race with an unacknowledged Stop.""" settlement_id = _terminal_settlement_id(settlement_id, status) if expected_execution_generation < 1 or expected_cancel_generation < 1: raise DriverValidationError("stopping settlement generations are invalid") result_json = _canonical_json(result) now = _timestamp(clock) return _transition( db_path, identity, lease=lease, lease_first=False, now=now, replay=_settlement_replay(settlement_id, status, result_json), sql=_SETTLE_STOPPING_SQL, set_params=(status, settlement_id, status, result_json, now, now), fence_params=(expected_execution_generation, expected_cancel_generation), stale="task completion lost the stop race", ) def resolve_indeterminate_task( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int, expected_cancel_generation: int, settlement_id: Any, status: TerminalStatus, result: Any, clock: Clock, ) -> dict[str, Any]: """Commit a verified historical receipt under the current room lease.""" _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation) settlement_id = _terminal_settlement_id(settlement_id, status) result_json = _canonical_json(result) now = _timestamp(clock) return _generation_transition( db_path, identity, lease, "resolve", expected_execution_generation, expected_cancel_generation, now=now, replay=_settlement_replay(settlement_id, status, result_json), set_params=(status, settlement_id, status, result_json, now, now), ) def resolve_indeterminate_cancellation( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int, expected_cancel_generation: int, cancel_id: Any, clock: Clock, ) -> dict[str, Any]: """Commit a verified terminal cancellation for an uncertain attempt.""" _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation) cancel_id = _identifier(cancel_id, label="cancel_id") now = _timestamp(clock) return _generation_transition( db_path, identity, lease, "resolve_cancel", expected_execution_generation, expected_cancel_generation, now=now, replay=_cancel_replay(cancel_id), set_params=(expected_cancel_generation + 1, cancel_id, now, now), ) def requeue_indeterminate_task( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int, expected_cancel_generation: int, clock: Clock, ) -> dict[str, Any]: """Explicitly retry uncertain work after an operator accepts at-least-once risk.""" _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation) now = _timestamp(clock) return _generation_transition( db_path, identity, lease, "requeue", expected_execution_generation, expected_cancel_generation, now=now, set_params=(now,), ) def defer_indeterminate_task( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int, expected_cancel_generation: int, reason: Any, clock: Clock, ) -> dict[str, Any]: """Fence one uncertain attempt and release later room work.""" _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation) reason = _identifier(reason, label="defer_reason") result_json = _canonical_json({"reason": reason, "retryable": True}) now = _timestamp(clock) def replay(row: sqlite3.Row) -> dict[str, Any] | None: deferred = _generations_match(row, "deferred", expected_execution_generation, expected_cancel_generation) return _task_from_row(row, idempotent=True) if deferred and row["result_json"] == result_json else None return _generation_transition( db_path, identity, lease, "defer", expected_execution_generation, expected_cancel_generation, now=now, replay=replay, set_params=(result_json, now, now), ) def requeue_deferred_task( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int, expected_cancel_generation: int, clock: Clock, ) -> dict[str, Any]: """Explicitly retry a fenced deferred turn under a new generation.""" _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation) now = _timestamp(clock) return _generation_transition( db_path, identity, lease, "requeue_deferred", expected_execution_generation, expected_cancel_generation, now=now, set_params=(now,), ) def requeue_not_admitted_task(db_path: Path | str, attempt: TaskAttempt, *, clock: Clock) -> dict[str, Any]: """Return a running task to its durable queue after proven non-admission.""" now = _timestamp(clock) _check_same_room(attempt.lease, attempt.identity) def replay(row: sqlite3.Row) -> dict[str, Any] | None: requeued = ( _generations_match(row, "queued", attempt.execution_generation, attempt.cancel_generation) and row["run_gateway_id"] is None and row["run_process_generation"] is None and row["run_lease_generation"] is None ) return _task_from_row(row, idempotent=True) if requeued else None return _run_fence_transition( db_path, attempt, guard_stale="not-admitted task attempt lost its fence", lease_generation=lambda value: int(value or 0), now=now, replay=replay, sql=_REQUEUE_RUNNING_SQL, set_params=(now,), stale="not-admitted task changed during requeue", ) def cancel_task( db_path: Path | str, identity: TaskIdentity, *, cancel_id: Any, expected_cancel_generation: int, clock: Clock ) -> dict[str, Any]: """Cancel a queued task before any external work was admitted.""" cancel_id = _identifier(cancel_id, label="cancel_id") _cancel_generation(expected_cancel_generation) now = _timestamp(clock) def guard(row: sqlite3.Row) -> None: if row["status"] in TERMINAL_STATUSES: raise InvalidTaskTransitionError(f"cannot cancel task in state '{row['status']}'") if row["status"] not in {"queued", "deferred"}: raise InvalidTaskTransitionError("running work requires acknowledged two-phase cancellation") _require_cancel_generation(row, expected_cancel_generation) return _transition( db_path, identity, now=now, replay=_cancel_replay(cancel_id), guard=guard, sql=_CANCEL_QUEUED_SQL, set_params=(expected_cancel_generation + 1, cancel_id, now, now), fence_params=(expected_cancel_generation,), stale="task changed during cancellation", ) def begin_task_cancel( db_path: Path | str, identity: TaskIdentity, *, cancel_id: Any, expected_cancel_generation: int, clock: Clock ) -> dict[str, Any]: """Persist a stop intent without claiming the remote run has stopped.""" cancel_id = _identifier(cancel_id, label="cancel_id") _cancel_generation(expected_cancel_generation) now = _timestamp(clock) def guard(row: sqlite3.Row) -> None: if row["status"] in TERMINAL_STATUSES or row["status"] == "queued": raise InvalidTaskTransitionError(f"cannot request remote stop in state '{row['status']}'") _require_cancel_generation(row, expected_cancel_generation) return _transition( db_path, identity, now=now, replay=_cancel_replay(cancel_id, "stopping"), guard=guard, sql=_BEGIN_STOP_SQL, set_params=(expected_cancel_generation + 1, cancel_id, now), fence_params=(expected_cancel_generation,), stale="task changed during stop request", ) def complete_task_cancel( db_path: Path | str, identity: TaskIdentity, *, cancel_id: Any, expected_cancel_generation: int, clock: Clock ) -> dict[str, Any]: """Commit cancellation only after the transport acknowledges exact Stop.""" cancel_id = _identifier(cancel_id, label="cancel_id") now = _timestamp(clock) def guard(row: sqlite3.Row) -> None: if (row["status"], row["cancel_id"], int(row["cancel_generation"])) != ( "stopping", cancel_id, expected_cancel_generation ): raise StaleTaskError("task stop acknowledgement is stale") return _transition( db_path, identity, now=now, replay=_cancel_replay(cancel_id), guard=guard, sql=_COMPLETE_STOP_SQL, set_params=(now, now), fence_params=(cancel_id, expected_cancel_generation), stale="task changed during stop acknowledgement", ) def recover_room(db_path: Path | str, lease: DriverLease, *, clock: Clock) -> dict[str, list[TaskIdentity]]: """Fence abandoned running attempts without requeueing uncertain work.""" now = _timestamp(clock) foreign_running = f"room_id=? AND status='running' AND NOT ({_RUN_FENCE})" fence = (lease.room_id, lease.gateway_id, lease.process_generation, lease.lease_generation) with _transaction(db_path) as conn: _require_active_lease(conn, lease, now=now) stale_rows = conn.execute( f"SELECT * FROM hosted_room_driver_tasks WHERE {foreign_running} {_TASK_ORDER}", fence ).fetchall() if stale_rows: conn.execute( f"""UPDATE hosted_room_driver_tasks SET status='indeterminate', indeterminate_at=?, updated_at=? WHERE {foreign_running}""", (now, now, *fence), ) return { status: [_task_identity_from_row(row) for row in _tasks_in_order(conn, lease.room_id, status)] for status in ("queued", "indeterminate") } def _read(db_path: Path | str, query: Callable[[sqlite3.Connection], Any]) -> Any: conn = _connect(db_path) try: return query(conn) finally: conn.close() def get_task(db_path: Path | str, identity: TaskIdentity) -> dict[str, Any]: """Read one task without mutating its state.""" return _read(db_path, lambda conn: _task_from_row(_load_task(conn, identity))) def list_tasks(db_path: Path | str, *, room_id: Any, status: TaskStatus | None = None) -> list[dict[str, Any]]: """Return room tasks in deterministic admission order.""" room_id = _identifier(room_id, label="room_id") if status is not None and status not in TASK_STATUSES: raise DriverValidationError("invalid task status") return _read(db_path, lambda conn: [_task_from_row(row) for row in _tasks_in_order(conn, room_id, status)]) def prune_published_terminal_tasks( db_path: Path | str, *, room_id: Any, clock: Clock, retention_seconds: float = TERMINAL_TASK_RETENTION_SECONDS, retain: int = MAX_RETAINED_TERMINAL_TASKS, ) -> int: """Bound execution rows after outcomes are durable in the room log.""" room_id = _identifier(room_id, label="room_id") now = _timestamp(clock) if retention_seconds <= 0: raise DriverValidationError("retention_seconds must be positive") non_negative_int(retain, error=DriverValidationError, message="retain must be a non-negative integer") with _transaction(db_path) as conn: publications = conn.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name='hosted_room_policy_publications'" ).fetchone() if publications is None: return 0 rows = conn.execute( """SELECT t.task_id, t.terminal_at FROM hosted_room_driver_tasks t WHERE t.room_id=? AND t.status IN ('settled', 'failed', 'cancelled') AND EXISTS (SELECT 1 FROM hosted_room_policy_publications p WHERE p.room_id=t.room_id AND p.task_id=t.task_id AND p.kind IN ('turn.settled', 'turn.failed', 'turn.cancelled')) ORDER BY t.terminal_at DESC, t.task_id ASC""", (room_id,), ).fetchall() cutoff = now - float(retention_seconds) candidates = [ str(row["task_id"]) for index, row in enumerate(rows) if index >= retain or (row["terminal_at"] is not None and float(row["terminal_at"]) <= cutoff) ][:MAX_TASK_PRUNE_BATCH] if not candidates: return 0 placeholders = ",".join("?" for _ in candidates) deleted = conn.execute( f"DELETE FROM hosted_room_driver_tasks WHERE room_id=? AND task_id IN ({placeholders})", (room_id, *candidates), ) return max(0, int(deleted.rowcount))