"""Durable delivery-obligation ledger for gateway final responses (rows in the shared ``state.db``; WAL, owner pid + process-start liveness, bounded retention) so a crash between finalize and platform ACK cannot lose a response silently. Checkpoints: record_obligation() 'pending' before any send | mark_attempting() 'attempting' right before the await | mark_delivered() 'delivered' only on SendResult.success | mark_failed() 'failed' on a definitive rejection. Crash semantics (never silently resend an ambiguous send): pending = never started, redeliver plainly; attempting = crashed mid-await, platform MAY have it, redeliver WITH a visible recovered marker; failed = rejected once, restart is a retry boundary, also marked; delivered = prune. Attempts are capped and stale rows expire, both -> 'abandoned' (kept briefly, then pruned). Everything is best-effort: ledger failures must never block a send; callers wrap every call in try/except. """ from __future__ import annotations import hashlib import logging import os import sqlite3 import threading import time from contextlib import closing, contextmanager from typing import Any, Dict, Iterator, List, Optional from hermes_constants import get_hermes_home logger = logging.getLogger(__name__) _DB_LOCK = threading.Lock() # Redelivery policy knobs (deliberately not config — the ledger is gated by # ``gateway.delivery_ledger`` and these only matter in the rare recovery path). MAX_ATTEMPTS = 3 STALE_AFTER_SECONDS = 24 * 60 * 60 _RETENTION_SECONDS = 7 * 24 * 60 * 60 _MAX_ROWS = 500 # Visible prefixes for redeliveries that might duplicate an already-received message (crash mid-send / # post-rejection retry) — honest at-least-once. Runtime recovery uses a distinct marker: no restart # occurred, but a network rejection's acknowledgement can still have been lost independently. RECOVERED_MARKER = "♻️ Recovered reply — the gateway restarted during delivery, so this may be a duplicate:\n\n" RECONNECTED_MARKER = ("♻️ Recovered reply — the messaging platform reconnected after the original " "delivery failed, so this may be a duplicate:\n\n") # Runtime replay is fail-closed: only errors whose send contract proves they are transient reconnect # failures. Permanent rejects (blocked bot, bad auth, missing chat) must not be retried on reconnect. _RUNTIME_RETRYABLE_ERRORS = frozenset({"send_path_degraded"}) def _db_path(): return get_hermes_home() / "state.db" def _connect() -> sqlite3.Connection: path = _db_path() path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(path, timeout=10) try: _initialize_schema(conn) except Exception: conn.close() # a PRAGMA/DDL failure after connect() must not leak the connection raise return conn def _initialize_schema(conn: sqlite3.Connection) -> None: from hermes_state import apply_wal_with_fallback apply_wal_with_fallback(conn, db_label="state.db (delivery_ledger)") conn.execute( """CREATE TABLE IF NOT EXISTS delivery_obligations ( obligation_id TEXT PRIMARY KEY, session_key TEXT NOT NULL, platform TEXT NOT NULL, chat_id TEXT NOT NULL, thread_id TEXT, content TEXT NOT NULL, state TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL, owner_pid INTEGER, owner_started_at INTEGER, last_error TEXT, adapter_profile TEXT )""" ) if "adapter_profile" not in {row[1] for row in conn.execute("PRAGMA table_info(delivery_obligations)")}: try: conn.execute("ALTER TABLE delivery_obligations ADD COLUMN adapter_profile TEXT") except sqlite3.OperationalError as exc: # Concurrent first-use connections can both observe the old schema. if "duplicate column" not in str(exc).lower(): raise @contextmanager def _transaction() -> Iterator[sqlite3.Connection]: """Open a connection, commit/rollback on exit, and ALWAYS close it: ``sqlite3.Connection`` as a context manager only commits/rolls back, so ``with _connect()`` alone leaks a connection (and its WAL/SHM fds) per call — ``record_obligation`` runs on every final response; exhausts RLIMIT_NOFILE.""" conn = _connect() with closing(conn), conn: yield conn def _start_time(pid: int) -> Optional[int]: try: from gateway.status import get_process_start_time # lazy: tests monkeypatch gateway.status return get_process_start_time(pid) except Exception: return None def _owner_stamp() -> tuple[int, Optional[int]]: pid = os.getpid() return pid, _start_time(pid) def _owner_alive(pid: Any, started_at: Any) -> bool: """True when the recorded owning process still exists (pid + start time).""" if not pid: return False try: pid = int(pid) except (TypeError, ValueError): return False current_start = _start_time(pid) if current_start is None: # Start time unreadable: alive iff the pid exists. Route through the cross-platform probe — on Windows # ``os.kill(pid, 0)`` is NOT a no-op (bpo-14484: it maps to ``GenerateConsoleCtrlEvent(0, pid)`` and could # Ctrl+C the gateway's own console group). ``_pid_exists`` keeps EPERM-means-alive (pid owned by another user). try: from gateway.status import _pid_exists except Exception: if os.name == "nt": return False # never fall back to a raw sig-0 probe on Windows try: os.kill(pid, 0) # windows-footgun: ok — POSIX-only fallback branch return True except OSError as exc: # incl. ProcessLookupError; EPERM means the pid exists return isinstance(exc, PermissionError) try: return bool(_pid_exists(pid)) except Exception: return False try: return started_at is None or int(current_start) == int(started_at) except (TypeError, ValueError): return True def compute_obligation_id(session_key: str, message_ref: str, content: str) -> str: """Stable id: same turn + same content re-records idempotently, while distinct threads/topics on one chat never collide (session_key carries platform/chat/thread; ``message_ref`` = inbound message id).""" return hashlib.sha256(f"{session_key}|{message_ref}|{content}".encode("utf-8", "replace")).hexdigest()[:24] def record_obligation(*, obligation_id: str, session_key: str, platform: str, chat_id: str, thread_id: Optional[str], content: str, adapter_profile: Optional[str] = None) -> None: """Record a final response as owed to the platform (state='pending').""" now, (pid, started) = time.time(), _owner_stamp() with _DB_LOCK, _transaction() as conn: conn.execute( """INSERT OR REPLACE INTO delivery_obligations (obligation_id, session_key, platform, chat_id, thread_id, content, state, attempts, created_at, updated_at, owner_pid, owner_started_at, adapter_profile) VALUES (?, ?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?, ?, ?)""", (obligation_id, session_key, platform, str(chat_id), str(thread_id) if thread_id else None, content, now, now, pid, started, str(adapter_profile).strip() if adapter_profile else "default")) _prune() def mark_attempting(obligation_id: str) -> None: _update_state(obligation_id, "attempting") def mark_delivered(obligation_id: str) -> None: _update_state(obligation_id, "delivered") def mark_failed(obligation_id: str, error: str = "") -> None: _update_state(obligation_id, "failed", error=error) def release_runtime_claim(obligation_id: str, error: str = "") -> bool: """Return an unsent runtime claim to ``failed`` without spending an attempt. Runtime recovery claims before clearing ``resume_pending`` so two reconnect paths cannot send the same row; if the flag cannot be cleared no send was attempted and the claim must not consume the redelivery budget. Fail-closed to the exact current process instance and ``attempting`` state.""" pid, started = _owner_stamp() if started is None: return False with _DB_LOCK, _transaction() as conn: cursor = conn.execute( """UPDATE delivery_obligations SET state='failed', attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END, updated_at=?, last_error=? WHERE obligation_id=? AND state='attempting' AND owner_pid IS ? AND owner_started_at IS ?""", (time.time(), error[:500] if error else None, obligation_id, pid, started)) return bool(cursor.rowcount) def _update_state(obligation_id: str, state: str, error: str = "") -> None: with _DB_LOCK, _transaction() as conn: conn.execute( """UPDATE delivery_obligations SET state=?, updated_at=?, last_error=? WHERE obligation_id=?""", (state, time.time(), error[:500] if error else None, obligation_id)) def _claimed_row(oid, session_key, platform, chat_id, thread_id, content, attempts, profile, *, needs_marker: bool, runtime: bool = False) -> Dict[str, Any]: """Claimed-row dict handed back for redelivery; ``runtime`` adds the reconnect-marker fields.""" return {"obligation_id": oid, "session_key": session_key, "platform": platform, "chat_id": chat_id, "thread_id": thread_id, "content": content, "needs_marker": needs_marker, **({"marker": RECONNECTED_MARKER} if runtime else {}), "profile": profile, **({"runtime_recovery": True} if runtime else {}), "attempts": attempts + 1} def sweep_recoverable(now: Optional[float] = None, *, deliverable_platforms: Optional[set] = None, deliverable_targets: Optional[set] = None) -> List[Dict[str, Any]]: """Claim undelivered rows owned by dead processes; return them for redelivery. Claiming atomically re-stamps the owner to THIS process and increments ``attempts`` (the UPDATE is guarded on the previous owner stamp, so a second gateway racing the same sweep cannot double-claim). Rows over the attempts cap or stale cutoff become 'abandoned'. ``deliverable_platforms`` restricts claiming to platforms the caller can send on this boot: ``attempts`` is the redelivery budget and must only be spent on a real send, else a platform that failed to connect burns one attempt per boot and hits the cap having never been sent once (the stale cutoff still bounds untouched rows). ``deliverable_targets`` further scopes multiplexed gateways by exact ``(platform, adapter_profile)`` so one connected bot cannot spend another disconnected bot's retry budget.""" now, (pid, started) = now if now is not None else time.time(), _owner_stamp() claimed: List[Dict[str, Any]] = [] with _DB_LOCK, _transaction() as conn: rows = conn.execute( """SELECT obligation_id, session_key, platform, chat_id, thread_id, content, state, attempts, created_at, owner_pid, owner_started_at, adapter_profile FROM delivery_obligations WHERE state IN ('pending', 'attempting', 'failed')""" ).fetchall() for (oid, session_key, platform, chat_id, thread_id, content, state, attempts, created_at, owner_pid, owner_started_at, adapter_profile) in rows: if _owner_alive(owner_pid, owner_started_at): continue # a live gateway still owns this row if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: # exhausted -> abandoned conn.execute( """UPDATE delivery_obligations SET state='abandoned', updated_at=? WHERE obligation_id=?""", (now, oid)) continue if ((deliverable_platforms is not None and platform not in deliverable_platforms) or (deliverable_targets is not None and (platform, adapter_profile) not in deliverable_targets)): continue # no adapter this boot — claiming would spend an attempt on a no-op cursor = conn.execute( """UPDATE delivery_obligations SET owner_pid=?, owner_started_at=?, attempts=attempts+1, updated_at=? WHERE obligation_id=? AND (owner_pid IS ? OR owner_pid=?)""", (pid, started, now, oid, owner_pid, owner_pid)) if cursor.rowcount: # pending = never started, redeliver plainly; else carry marker claimed.append(_claimed_row(oid, session_key, platform, chat_id, thread_id, content, attempts, adapter_profile, needs_marker=state != "pending")) return claimed def sweep_failed_for_runtime(platform: str, now: Optional[float] = None, *, profile: Optional[str] = None) -> List[Dict[str, Any]]: """Claim this process's reconnect-retryable failed rows for one adapter. ``profile`` scopes multiplexed gateways to the bot identity that owned the failed send (``None`` = primary/default adapter); unowned rows and rows owned by another process are left for the startup/dead-owner sweep. Startup recovery ignores rows owned by a live gateway, so a response rejected with ``send_path_degraded`` would stay stranded when only the adapter reconnects; this closes that gap without weakening ownership: only rows stamped to this exact process instance, only allowlisted transient errors, same attempts/staleness bounds, every update guarded by the prior owner stamp and ``failed`` state. Claimed rows always carry the reconnect marker (the failed send's ack is not safe to infer).""" now, (pid, started) = now if now is not None else time.time(), _owner_stamp() if started is None: # PID alone cannot distinguish this process from a stale row left after PID return [] # reuse; runtime replay is optional, so fail closed (startup recovery remains). expected_profile = "default" if not profile or profile == "default" else str(profile) claimed: List[Dict[str, Any]] = [] with _DB_LOCK, _transaction() as conn: rows = conn.execute( """SELECT obligation_id, session_key, platform, chat_id, thread_id, content, attempts, created_at, owner_pid, owner_started_at, last_error, adapter_profile FROM delivery_obligations WHERE state='failed' AND platform=?""", (platform,)).fetchall() for (oid, session_key, row_platform, chat_id, thread_id, content, attempts, created_at, owner_pid, owner_started_at, last_error, adapter_profile) in rows: # Exact process-start matching prevents PID reuse from stealing work. if (adapter_profile != expected_profile or owner_pid != pid or owner_started_at != started or str(last_error or "").strip().lower() not in _RUNTIME_RETRYABLE_ERRORS): continue owner_guard = (now, oid, owner_pid, owner_started_at) if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: # exhausted -> abandoned conn.execute( """UPDATE delivery_obligations SET state='abandoned', updated_at=? WHERE obligation_id=? AND state='failed' AND owner_pid IS ? AND owner_started_at IS ?""", owner_guard) continue cursor = conn.execute( """UPDATE delivery_obligations SET state='attempting', attempts=attempts+1, updated_at=? WHERE obligation_id=? AND state='failed' AND owner_pid IS ? AND owner_started_at IS ?""", owner_guard) if cursor.rowcount: claimed.append(_claimed_row(oid, session_key, row_platform, chat_id, thread_id, content, attempts, adapter_profile, needs_marker=True, runtime=True)) return claimed def _prune(now: Optional[float] = None) -> None: now = now if now is not None else time.time() try: with _transaction() as conn: conn.execute( """DELETE FROM delivery_obligations WHERE state IN ('delivered', 'abandoned') AND updated_at < ?""", (now - _RETENTION_SECONDS,)) total = conn.execute("SELECT COUNT(*) FROM delivery_obligations").fetchone()[0] if total > _MAX_ROWS: conn.execute( """DELETE FROM delivery_obligations WHERE obligation_id IN ( SELECT obligation_id FROM delivery_obligations ORDER BY CASE state WHEN 'delivered' THEN 0 WHEN 'abandoned' THEN 1 ELSE 2 END, updated_at ASC LIMIT ?)""", (total - _MAX_ROWS,)) except Exception: logger.debug("delivery ledger prune failed", exc_info=True) def ledger_enabled(config: Optional[Dict[str, Any]] = None) -> bool: """Read the ``gateway.delivery_ledger`` config gate (default on).""" try: if config is None: from hermes_cli.config import load_config config = load_config() value = (config.get("gateway") or {}).get("delivery_ledger", True) return value.strip().lower() not in {"false", "0", "no", "off"} if isinstance(value, str) else bool(value) except Exception: return True