Files
EvoScientist-Multi/EvoScientist/llm/host_execution_registry.py
T

370 lines
23 KiB
Python

"""Opt-in host identity prototype. Records never prove resource quiescence.
Trusted local host API, not an authenticated remote control endpoint. A future
PG host store can implement this protocol without introducing a dispatcher.
Only the runtime resource owner may attest cleanup and release its claim.
"""
from contextlib import closing, contextmanager
from pathlib import Path
import sqlite3
import json
import hashlib
import os
from typing import Protocol
from .contracts import EvoRuntimeError, canonical_json_v1
class HostExecutionRegistry(Protocol):
def claim_checkpoint(self, execution_id: str, *, store_id: str, checkpoint_thread_id: str, subject_id: str) -> None: ...
def release_unbound_checkpoint(self, execution_id: str) -> None: ...
def continuation(self, execution_id: str) -> dict: ...
def prepare_terminal(self, execution_id: str, *, event: dict) -> None: ...
def terminal_intent(self, execution_id: str) -> dict | None: ...
def confirm_terminal(self, execution_id: str, *, digest: str) -> None: ...
def finish(self, execution_id: str, *, outcome: str, checkpoint_id: str = "") -> None: ...
def bind(self, *, execution_id: str, grant_id: str, digest: str,
thread_id: str, turn_id: str, predecessor_execution_id: str = "",
predecessor_checkpoint_id: str = "", predecessor_owner_epoch: int = 0,
continuation_pending_hash: str = "", continuation_decision_hash: str = "") -> None: ...
def lookup_grant(self, grant_id: str, digest: str) -> dict | None: ...
def inspect(self, execution_id: str) -> dict: ...
def transfer_control(self, execution_id: str, *, expected_epoch: int, new_epoch: int) -> int: ...
def require_control(self, execution_id: str, *, owner_epoch: int) -> None: ...
def inspect_control(self, execution_id: str, *, owner_epoch: int | None, boot_id: str | None) -> dict: ...
def accept_cancel(self, execution_id: str, *, owner_epoch: int | None, boot_id: str | None, reason: str) -> int: ...
class SQLiteHostRegistry:
def __init__(self, path: str | Path, *, host_id: str, boot_id: str):
self.path, self.host_id, self.boot_id = str(path), host_id, boot_id
directory = Path(path).parent
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
# Exclusive creation gives new user-content stores safe defaults without
# changing permissions on an existing host's directory or database.
try:
fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
except FileExistsError:
pass
else:
os.close(fd)
with self._transaction() as db:
db.execute("BEGIN IMMEDIATE")
db.execute("""CREATE TABLE IF NOT EXISTS executions (
execution_id TEXT PRIMARY KEY, grant_id TEXT UNIQUE NOT NULL,
digest TEXT NOT NULL, thread_id TEXT NOT NULL,
turn_id TEXT NOT NULL, host_id TEXT NOT NULL, boot_id TEXT NOT NULL,
owner_epoch INTEGER NOT NULL DEFAULT 1)""")
schema = db.execute("SELECT sql FROM sqlite_master WHERE name='executions'").fetchone()[0]
if "UNIQUE(thread_id, turn_id)" in schema:
db.execute("ALTER TABLE executions RENAME TO legacy_executions")
db.execute("""CREATE TABLE executions (
execution_id TEXT PRIMARY KEY, grant_id TEXT UNIQUE NOT NULL,
digest TEXT NOT NULL, thread_id TEXT NOT NULL, turn_id TEXT NOT NULL,
host_id TEXT NOT NULL, boot_id TEXT NOT NULL,
owner_epoch INTEGER NOT NULL DEFAULT 1)""")
db.execute("INSERT INTO executions SELECT * FROM legacy_executions")
db.execute("DROP TABLE legacy_executions")
db.execute("""CREATE TABLE IF NOT EXISTS cancel_intents (
intent_id INTEGER PRIMARY KEY AUTOINCREMENT,
execution_id TEXT NOT NULL, owner_epoch INTEGER NOT NULL,
boot_id TEXT NOT NULL, reason TEXT NOT NULL)""")
db.execute("""CREATE TABLE IF NOT EXISTS terminal_evidence (
execution_id TEXT PRIMARY KEY, outcome TEXT NOT NULL)""")
db.execute("""CREATE TABLE IF NOT EXISTS terminal_intents (
execution_id TEXT PRIMARY KEY, event_json TEXT NOT NULL,
digest TEXT NOT NULL, phase TEXT NOT NULL,
cleanup_boot_id TEXT NOT NULL)""")
db.execute("""CREATE TABLE IF NOT EXISTS pending_continuations (
execution_id TEXT PRIMARY KEY, checkpoint_id TEXT NOT NULL,
consumed_by TEXT UNIQUE)""")
columns = {row[1] for row in db.execute("PRAGMA table_info(pending_continuations)")}
for name in ("pending_hash", "decision_hash"):
if name not in columns:
db.execute(f"ALTER TABLE pending_continuations ADD COLUMN {name} TEXT NOT NULL DEFAULT ''")
db.execute("""CREATE TABLE IF NOT EXISTS active_claims (
execution_id TEXT PRIMARY KEY, thread_id TEXT NOT NULL,
turn_id TEXT NOT NULL, UNIQUE(thread_id, turn_id))""")
db.execute("""INSERT OR IGNORE INTO active_claims
SELECT execution_id, thread_id, turn_id FROM executions
WHERE execution_id NOT IN (SELECT execution_id FROM terminal_evidence)""")
db.execute("""CREATE TABLE IF NOT EXISTS checkpoint_scopes (
store_id TEXT NOT NULL, checkpoint_thread_id TEXT NOT NULL,
checkpoint_ns TEXT NOT NULL, subject_id TEXT NOT NULL,
PRIMARY KEY(store_id, checkpoint_thread_id, checkpoint_ns))""")
db.execute("""CREATE TABLE IF NOT EXISTS checkpoint_writers (
execution_id TEXT PRIMARY KEY, store_id TEXT NOT NULL,
checkpoint_thread_id TEXT NOT NULL, checkpoint_ns TEXT NOT NULL,
host_id TEXT NOT NULL, boot_id TEXT NOT NULL,
UNIQUE(store_id, checkpoint_thread_id, checkpoint_ns))""")
db.execute("""CREATE TABLE IF NOT EXISTS execution_checkpoint_scopes (
execution_id TEXT PRIMARY KEY, store_id TEXT NOT NULL,
checkpoint_thread_id TEXT NOT NULL, checkpoint_ns TEXT NOT NULL)""")
def claim_checkpoint(self, execution_id: str, *, store_id: str,
checkpoint_thread_id: str, subject_id: str) -> None:
"""Root Graph writer, including child namespaces; no expiry/takeover."""
scope = (store_id, checkpoint_thread_id, "")
if not all((execution_id, store_id, checkpoint_thread_id, subject_id)):
raise EvoRuntimeError("CHECKPOINT_SCOPE_INVALID")
with self._transaction() as db:
db.execute("BEGIN IMMEDIATE")
owner = db.execute("SELECT subject_id FROM checkpoint_scopes WHERE store_id=? AND checkpoint_thread_id=? AND checkpoint_ns=?", scope).fetchone()
if owner and owner[0] != subject_id:
raise EvoRuntimeError("CHECKPOINT_SUBJECT_MISMATCH")
if db.execute("SELECT 1 FROM checkpoint_writers WHERE store_id=? AND checkpoint_thread_id=? AND checkpoint_ns=?", scope).fetchone():
raise EvoRuntimeError("CHECKPOINT_WRITER_BUSY")
if db.execute("SELECT 1 FROM active_claims WHERE execution_id NOT IN (SELECT execution_id FROM execution_checkpoint_scopes)").fetchone():
raise EvoRuntimeError("CHECKPOINT_LEGACY_WRITER_UNKNOWN")
db.execute("INSERT OR IGNORE INTO checkpoint_scopes VALUES (?, ?, ?, ?)", (*scope, subject_id))
db.execute("INSERT INTO checkpoint_writers VALUES (?, ?, ?, ?, ?, ?)",
(execution_id, *scope, self.host_id, self.boot_id))
def release_unbound_checkpoint(self, execution_id: str) -> None:
"""Only preparation failure, before bind/construction/writes began."""
with self._transaction() as db:
db.execute("BEGIN IMMEDIATE")
if db.execute("SELECT 1 FROM executions WHERE execution_id=?", (execution_id,)).fetchone():
raise EvoRuntimeError("CHECKPOINT_WRITER_ALREADY_BOUND")
db.execute("DELETE FROM checkpoint_writers WHERE execution_id=? AND host_id=? AND boot_id=?",
(execution_id, self.host_id, self.boot_id))
def _connect(self):
db = sqlite3.connect(self.path, timeout=2)
db.row_factory = sqlite3.Row
db.execute("PRAGMA synchronous=FULL")
return db
@contextmanager
def _transaction(self):
try:
with closing(self._connect()) as db, db:
yield db
except sqlite3.OperationalError as exc:
code = getattr(exc, "sqlite_errorcode", 0) & 255
if code in {sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED}:
raise EvoRuntimeError("HOST_REGISTRY_BUSY") from exc
raise
def bind(self, *, execution_id: str, grant_id: str, digest: str,
thread_id: str, turn_id: str, predecessor_execution_id: str = "",
predecessor_checkpoint_id: str = "", predecessor_owner_epoch: int = 0,
continuation_pending_hash: str = "", continuation_decision_hash: str = "") -> None:
with self._transaction() as db:
db.execute("BEGIN IMMEDIATE")
if db.execute("SELECT 1 FROM executions WHERE execution_id=? OR grant_id=?",
(execution_id, grant_id)).fetchone():
raise EvoRuntimeError("EXECUTION_IDENTITY_CONFLICT")
writer = db.execute("SELECT * FROM checkpoint_writers WHERE execution_id=?", (execution_id,)).fetchone()
if writer and (writer["host_id"] != self.host_id or writer["boot_id"] != self.boot_id):
raise EvoRuntimeError("EXECUTION_BOOT_MISMATCH")
if writer and predecessor_execution_id:
previous = db.execute("SELECT * FROM execution_checkpoint_scopes WHERE execution_id=?", (predecessor_execution_id,)).fetchone()
if previous is None or any(previous[k] != writer[k] for k in ("store_id", "checkpoint_thread_id", "checkpoint_ns")):
raise EvoRuntimeError("CONTINUATION_CHECKPOINT_SCOPE_MISMATCH")
if db.execute("SELECT 1 FROM active_claims WHERE thread_id=? AND turn_id=?",
(thread_id, turn_id)).fetchone():
raise EvoRuntimeError("TURN_EXECUTION_UNKNOWN")
prior = db.execute("SELECT 1 FROM executions WHERE thread_id=? AND turn_id=?",
(thread_id, turn_id)).fetchone()
if prior or predecessor_execution_id or predecessor_checkpoint_id:
if not predecessor_execution_id:
raise EvoRuntimeError("CONTINUATION_REQUIRED")
pending = db.execute("""SELECT p.*, e.owner_epoch, e.host_id FROM pending_continuations p
JOIN executions e USING(execution_id)
WHERE p.execution_id=? AND e.thread_id=? AND e.turn_id=?""",
(predecessor_execution_id, thread_id, turn_id)).fetchone()
if (pending is not None and pending["consumed_by"] is not None
and pending["checkpoint_id"] == predecessor_checkpoint_id):
failed = db.execute(
"SELECT 1 FROM terminal_evidence WHERE execution_id=? AND outcome='failed'",
(pending["consumed_by"],),
).fetchone()
if failed:
# Never unconsume a decision on failure. A new grant alone
# is insufficient; recovery needs fresh pending authority.
raise EvoRuntimeError("CONTINUATION_CONSUMED_FAILURE_REQUIRES_REAUTHORIZATION")
if (pending is None or not predecessor_checkpoint_id
or pending["checkpoint_id"] != predecessor_checkpoint_id
or pending["consumed_by"] is not None):
raise EvoRuntimeError("CONTINUATION_INVALID")
if (pending["host_id"] != self.host_id or not pending["pending_hash"]
or pending["owner_epoch"] != predecessor_owner_epoch
or isinstance(predecessor_owner_epoch, bool)
or pending["pending_hash"] != continuation_pending_hash
or len(continuation_decision_hash) != 64):
raise EvoRuntimeError("CONTINUATION_AUTHORIZATION_INVALID")
if db.execute("SELECT 1 FROM cancel_intents WHERE execution_id=?",
(predecessor_execution_id,)).fetchone():
raise EvoRuntimeError("CONTINUATION_CANCELLED")
db.execute("UPDATE pending_continuations SET consumed_by=?, decision_hash=? WHERE execution_id=?",
(execution_id, continuation_decision_hash, predecessor_execution_id))
db.execute("INSERT INTO executions VALUES (?, ?, ?, ?, ?, ?, ?, 1)",
(execution_id, grant_id, digest, thread_id, turn_id,
self.host_id, self.boot_id))
db.execute("INSERT INTO active_claims VALUES (?, ?, ?)",
(execution_id, thread_id, turn_id))
if writer:
db.execute("INSERT INTO execution_checkpoint_scopes VALUES (?, ?, ?, ?)",
(execution_id, writer["store_id"], writer["checkpoint_thread_id"], writer["checkpoint_ns"]))
def finish(self, execution_id: str, *, outcome: str, checkpoint_id: str = "") -> None:
"""Trusted resource-owner attestation, not transferable control authority."""
self._finish(execution_id, outcome=outcome, checkpoint_id=checkpoint_id)
def prepare_terminal(self, execution_id: str, *, event: dict) -> None:
"""Original resource owner only, AFTER all owned cleanup returns."""
body = canonical_json_v1(event).decode()
digest = hashlib.sha256(body.encode()).hexdigest()
if (event.get("run_id") != execution_id or event.get("kind") != "run"
or event.get("payload", {}).get("kind") != "run_terminal"):
raise EvoRuntimeError("EXECUTION_TERMINAL_CONFLICT")
with self._transaction() as db:
db.execute("BEGIN IMMEDIATE")
row = db.execute("SELECT * FROM executions WHERE execution_id=?", (execution_id,)).fetchone()
if row is None or row['host_id'] != self.host_id or row['boot_id'] != self.boot_id:
raise EvoRuntimeError("EXECUTION_BOOT_MISMATCH")
prior = db.execute("SELECT digest FROM terminal_intents WHERE execution_id=?", (execution_id,)).fetchone()
if prior and prior['digest'] != digest:
raise EvoRuntimeError("EXECUTION_TERMINAL_CONFLICT")
db.execute("INSERT OR IGNORE INTO terminal_intents VALUES (?, ?, ?, 'prepared', ?)",
(execution_id, body, digest, self.boot_id))
def terminal_intent(self, execution_id: str) -> dict | None:
with closing(self._connect()) as db:
row = db.execute("""SELECT i.* FROM terminal_intents i JOIN executions e USING(execution_id)
WHERE execution_id=? AND e.host_id=?""", (execution_id, self.host_id)).fetchone()
if row is None:
return None
return {**dict(row), 'event': json.loads(row['event_json']), 'cleanup_confirmed': True}
def confirm_terminal(self, execution_id: str, *, digest: str) -> None:
with self._transaction() as db:
row = db.execute("""SELECT i.* FROM terminal_intents i JOIN executions e USING(execution_id)
WHERE execution_id=? AND e.host_id=?""", (execution_id, self.host_id)).fetchone()
if row is None or row['digest'] != digest:
raise EvoRuntimeError("EXECUTION_TERMINAL_CONFLICT")
db.execute("UPDATE terminal_intents SET phase='sink_confirmed' WHERE execution_id=? AND phase='prepared'",
(execution_id,))
def _finish(self, execution_id: str, *, outcome: str, checkpoint_id: str) -> None:
if outcome not in {"completed", "cancelled", "failed", "awaiting_input"}:
raise EvoRuntimeError("EXECUTION_OUTCOME_INVALID")
with self._transaction() as db:
db.execute("BEGIN IMMEDIATE")
row = db.execute("SELECT * FROM executions WHERE execution_id=?",
(execution_id,)).fetchone()
intent = db.execute("SELECT * FROM terminal_intents WHERE execution_id=?", (execution_id,)).fetchone()
if row is None or row["host_id"] != self.host_id or (row["boot_id"] != self.boot_id and
(intent is None or intent['phase'] not in {'sink_confirmed', 'registry_finished'})):
raise EvoRuntimeError("EXECUTION_BOOT_MISMATCH")
if intent is not None:
payload = json.loads(intent['event_json'])['payload']
if (intent['phase'] not in {'sink_confirmed', 'registry_finished'}
or payload['outcome'] != outcome
or str(payload.get('checkpoint_id') or '') != checkpoint_id):
raise EvoRuntimeError("EXECUTION_TERMINAL_CONFLICT")
prior = db.execute("SELECT outcome FROM terminal_evidence WHERE execution_id=?",
(execution_id,)).fetchone()
if prior is not None and prior["outcome"] != outcome:
raise EvoRuntimeError("EXECUTION_TERMINAL_CONFLICT")
db.execute("INSERT OR IGNORE INTO terminal_evidence VALUES (?, ?)",
(execution_id, outcome))
if outcome == "awaiting_input" and checkpoint_id:
pending = db.execute("SELECT checkpoint_id FROM pending_continuations WHERE execution_id=?",
(execution_id,)).fetchone()
if pending is not None and pending["checkpoint_id"] != checkpoint_id:
raise EvoRuntimeError("EXECUTION_TERMINAL_CONFLICT")
identity = {}
if intent is not None:
payload = json.loads(intent['event_json'])['payload']
identity = {k: payload.get(k) for k in (
"checkpoint_thread_id", "checkpoint_id", "checkpoint_ns", "pending_interrupts")}
pending_hash = hashlib.sha256(canonical_json_v1(identity)).hexdigest() if identity else ""
db.execute("INSERT OR IGNORE INTO pending_continuations "
"(execution_id, checkpoint_id, consumed_by, pending_hash, decision_hash) VALUES (?, ?, NULL, ?, '')",
(execution_id, checkpoint_id, pending_hash))
db.execute("DELETE FROM active_claims WHERE execution_id=?", (execution_id,))
db.execute("DELETE FROM checkpoint_writers WHERE execution_id=?", (execution_id,))
db.execute("UPDATE terminal_intents SET phase='registry_finished' WHERE execution_id=?", (execution_id,))
def continuation(self, execution_id: str) -> dict:
with closing(self._connect()) as db:
row = db.execute("""SELECT p.*, e.owner_epoch FROM pending_continuations p
JOIN executions e USING(execution_id) WHERE execution_id=? AND e.host_id=?""",
(execution_id, self.host_id)).fetchone()
if row is None:
raise EvoRuntimeError("CONTINUATION_INVALID")
return dict(row)
def lookup_grant(self, grant_id: str, digest: str) -> dict | None:
with closing(self._connect()) as db:
row = db.execute("SELECT * FROM executions WHERE grant_id=?", (grant_id,)).fetchone()
if row is None:
return None
if row["digest"] != digest:
raise EvoRuntimeError("CONTRACT_REPLAYED")
return dict(row)
def transfer_control(self, execution_id: str, *, expected_epoch: int, new_epoch: int) -> int:
if new_epoch <= expected_epoch:
raise EvoRuntimeError("OWNER_EPOCH_STALE")
with self._transaction() as db:
changed = db.execute(
"UPDATE executions SET owner_epoch=? WHERE execution_id=? AND owner_epoch=? AND host_id=? AND boot_id=?",
(new_epoch, execution_id, expected_epoch, self.host_id, self.boot_id),
).rowcount
if changed != 1:
raise EvoRuntimeError("OWNER_EPOCH_STALE")
return new_epoch
def require_control(self, execution_id: str, *, owner_epoch: int) -> None:
with closing(self._connect()) as db:
row = db.execute("SELECT owner_epoch FROM executions WHERE execution_id=?",
(execution_id,)).fetchone()
if row is None or row["owner_epoch"] != owner_epoch:
raise EvoRuntimeError("OWNER_EPOCH_STALE")
def inspect(self, execution_id: str) -> dict:
with closing(self._connect()) as db:
row = db.execute("SELECT * FROM executions WHERE execution_id=?", (execution_id,)).fetchone()
terminal = db.execute("SELECT outcome FROM terminal_evidence WHERE execution_id=?",
(execution_id,)).fetchone()
cancel = db.execute("SELECT 1 FROM cancel_intents WHERE execution_id=? LIMIT 1",
(execution_id,)).fetchone()
return {**(dict(row) if row else {"execution_id": execution_id}),
"cancel_requested": cancel is not None,
"recovery_action": "inspect_only",
"status": terminal["outcome"] if terminal else "unknown",
"resources_confirmed_exited": terminal is not None,
"source": "host_binding_only" if row else "no_host_binding"}
def _control_row(self, db, execution_id, owner_epoch, boot_id):
if owner_epoch is None:
raise EvoRuntimeError("OWNER_EPOCH_REQUIRED")
row = db.execute("SELECT * FROM executions WHERE execution_id=?", (execution_id,)).fetchone()
if row is None:
raise EvoRuntimeError("EXECUTION_UNKNOWN")
if row["host_id"] != self.host_id or row["boot_id"] != self.boot_id or boot_id != self.boot_id:
raise EvoRuntimeError("EXECUTION_BOOT_MISMATCH")
if isinstance(owner_epoch, bool) or row["owner_epoch"] != owner_epoch:
raise EvoRuntimeError("OWNER_EPOCH_STALE")
return row
def inspect_control(self, execution_id: str, *, owner_epoch: int | None, boot_id: str | None) -> dict:
with closing(self._connect()) as db:
row = self._control_row(db, execution_id, owner_epoch, boot_id)
return self.inspect(execution_id)
def accept_cancel(self, execution_id: str, *, owner_epoch: int | None, boot_id: str | None, reason: str) -> int:
# COMMIT is the linearization point shared with transfer's conditional UPDATE.
# Accepted commands survive a later transfer; no write lock crosses an await.
with self._transaction() as db:
db.execute("BEGIN IMMEDIATE")
self._control_row(db, execution_id, owner_epoch, boot_id)
cursor = db.execute(
"INSERT INTO cancel_intents (execution_id, owner_epoch, boot_id, reason) VALUES (?, ?, ?, ?)",
(execution_id, owner_epoch, boot_id, reason),
)
assert cursor.lastrowid is not None
return cursor.lastrowid