"""Offline, non-destructive recovery for a damaged Hermes session database. * the supplied source database is never opened by SQLite; * the source file and any WAL/SHM/rollback-journal sidecars are copied into a disposable working directory first; * canonical rows are copied into a newly initialized current-schema database; * derived FTS tables and migration bookkeeping are rebuilt, not copied; and * the recovered database is never installed over the active database. """ from __future__ import annotations import json import os import shutil import sqlite3 import tempfile from contextlib import contextmanager from pathlib import Path from typing import Any, Callable, Iterator, Optional from hermes_state import ( FTS_STORAGE_VERSION, SCHEMA_VERSION, SessionDB, _db_opens_cleanly, ) ProgressCallback = Callable[[dict[str, Any]], None] _CANONICAL_TABLES = ( "system_prompts", "sessions", "messages", "session_model_usage", "compression_locks", "gateway_routing", "async_delegations", ) _TOPIC_TABLES = ( "telegram_dm_topic_mode", "telegram_dm_topic_bindings", ) def _init_delivery_ledger_schema(conn: sqlite3.Connection) -> None: from gateway.delivery_ledger import _initialize_schema _initialize_schema(conn) # Tables that live in state.db but are created lazily by a gateway module on # first use, so base ``SessionDB`` never creates them on a fresh destination. # Every entry maps the table to the initializer that owns its DDL; recovery # creates the table on the destination before copying, so owed rows survive # instead of silently vanishing from a "complete" salvage (#100313, #86236). # Add new lazily-created state.db tables HERE, never as one-off ``if table ==`` # branches. _AUXILIARY_TABLE_SCHEMAS: dict[str, Callable[[sqlite3.Connection], None]] = { "delivery_obligations": _init_delivery_ledger_schema, } _AUXILIARY_TABLES = tuple(_AUXILIARY_TABLE_SCHEMAS) _INVENTORY_TABLES = ( *_CANONICAL_TABLES, "state_meta", *_TOPIC_TABLES, *_AUXILIARY_TABLES, ) # These values describe derived indexes or the schema that owns an optional # table. A fresh destination must generate them from its own current schema. _GENERATED_META_KEYS = frozenset({ "fts_storage_version", "fts_optimize_available", "fts_rebuild_high_water", "fts_rebuild_progress", "fts_cjk_stale", "fts_cjk_rebuild_high_water", "fts_cjk_rebuild_progress", "telegram_dm_topic_schema_version", }) _SIDECAR_SUFFIXES = ("", "-wal", "-shm", "-journal") _MINIMUM_SPACE_HEADROOM = 256 * 1024 * 1024 _MAX_SALVAGE_RANGE_QUERIES = 10_000 _MIN_SQLITE_ROWID = -(2**63) _MAX_SQLITE_ROWID = 2**63 - 1 class SessionRecoveryError(RuntimeError): """Base error for offline session recovery.""" class SessionRecoverySafetyError(SessionRecoveryError): """Raised before recovery when a path or overwrite guard fails.""" class SessionRecoverySourceError(SessionRecoveryError): """Raised when the source cannot provide the required canonical tables.""" def _sidecar_path(db_path: Path, suffix: str) -> Path: return db_path if not suffix else db_path.with_name(db_path.name + suffix) def _resolved_output_path(path: Path) -> Path: """Resolve a not-yet-created output path without requiring it to exist.""" parent = path.expanduser().parent.resolve(strict=True) return parent / path.name def _validate_paths( source_path: Path, output_path: Optional[Path] = None, work_dir: Optional[Path] = None, ) -> tuple[Path, Optional[Path], Path]: source = source_path.expanduser().resolve(strict=True) if not source.is_file(): raise SessionRecoverySafetyError(f"Source is not a file: {source}") output: Optional[Path] = None if output_path is not None: output = _resolved_output_path(output_path) protected = {_sidecar_path(source, suffix).resolve(strict=False) for suffix in _SIDECAR_SUFFIXES} if output.resolve(strict=False) in protected: raise SessionRecoverySafetyError( "The recovery output must not be the source database or one of " "its journal sidecars." ) for suffix in _SIDECAR_SUFFIXES: candidate = _sidecar_path(output, suffix) if os.path.lexists(candidate): raise SessionRecoverySafetyError( f"Refusing to overwrite existing recovery output: {candidate}" ) work_root = ( work_dir.expanduser().resolve(strict=True) if work_dir is not None else (output.parent if output is not None else source.parent) ) if not work_root.is_dir(): raise SessionRecoverySafetyError( f"Recovery work directory is not a directory: {work_root}" ) return source, output, work_root def _source_fingerprint(source: Path) -> dict[str, dict[str, int]]: fingerprint: dict[str, dict[str, int]] = {} for suffix in _SIDECAR_SUFFIXES: path = _sidecar_path(source, suffix) if not path.exists(): continue stat = path.stat() fingerprint[suffix or "main"] = {"size": stat.st_size, "mtime_ns": stat.st_mtime_ns} return fingerprint def _format_bytes(value: int) -> str: units = ("B", "KiB", "MiB", "GiB", "TiB") amount = float(value) for unit in units: if amount < 1024 or unit == units[-1]: return f"{amount:.1f} {unit}" amount /= 1024 return f"{value} B" def _same_filesystem(left: Path, right: Path) -> bool: try: return os.stat(left).st_dev == os.stat(right).st_dev except OSError: # Existing directories were already required by _validate_paths. This # fallback is defensive for platforms with incomplete st_dev support. return left.anchor.casefold() == right.anchor.casefold() def _disk_space_preflight( source: Path, work_root: Path, output_parent: Optional[Path], ) -> dict[str, Any]: """Require space for the disposable bundle, output, and safety headroom.""" bundle_bytes = sum( _sidecar_path(source, suffix).stat().st_size for suffix in _SIDECAR_SUFFIXES if _sidecar_path(source, suffix).exists() ) # The v23 external-content rebuild is normally substantially smaller than # a legacy database, but using the complete source bundle as the estimate # avoids betting the user's disk on that expectation. output_allowance = bundle_bytes if output_parent is not None else 0 headroom = max(_MINIMUM_SPACE_HEADROOM, int((bundle_bytes + output_allowance) * 0.05)) work_free = int(shutil.disk_usage(work_root).free) report: dict[str, Any] = { "source_bundle_bytes": bundle_bytes, "estimated_output_bytes": output_allowance, "headroom_bytes": headroom, "work_dir": str(work_root), "work_dir_free_bytes": work_free, } if output_parent is None or _same_filesystem(work_root, output_parent): required = bundle_bytes + output_allowance + headroom report["shared_filesystem"] = True report["work_dir_required_bytes"] = required if work_free < required: raise SessionRecoverySafetyError( "Not enough free disk space for a safe recovery copy: " f"{_format_bytes(work_free)} available at {work_root}, " f"{_format_bytes(required)} required " f"({_format_bytes(bundle_bytes)} source bundle + " f"{_format_bytes(output_allowance)} output allowance + " f"{_format_bytes(headroom)} headroom). Use --work-dir or " "--output on a filesystem with more free space." ) return report output_free = int(shutil.disk_usage(output_parent).free) work_required = bundle_bytes + headroom output_required = output_allowance + headroom report.update({ "shared_filesystem": False, "work_dir_required_bytes": work_required, "output_dir": str(output_parent), "output_dir_free_bytes": output_free, "output_dir_required_bytes": output_required, }) shortages: list[str] = [] if work_free < work_required: shortages.append( f"{work_root}: {_format_bytes(work_free)} available, " f"{_format_bytes(work_required)} required" ) if output_free < output_required: shortages.append( f"{output_parent}: {_format_bytes(output_free)} available, " f"{_format_bytes(output_required)} required" ) if shortages: raise SessionRecoverySafetyError( "Not enough free disk space for safe recovery: " + "; ".join(shortages) + ". Choose work/output filesystems with more free space." ) return report def _copy_source_bundle(source: Path, snapshot_dir: Path) -> tuple[Path, list[str]]: """Copy the source DB bundle aside so SQLite never opens the original. The whole copy runs inside ``offline_file_access``, which holds the connection-lifecycle lock for its duration. Recovery normally runs as its own short-lived CLI process against an offline/quarantined file, so the refusal should never fire; the guard keeps this path consistent with ``hermes_state._backup_db_file``. """ from hermes_cli.sqlite_safe_read import LiveConnectionError, offline_file_access snapshot_source = snapshot_dir / source.name copied: list[str] = [] try: with offline_file_access(source, what="snapshot"): for suffix in _SIDECAR_SUFFIXES: source_part = _sidecar_path(source, suffix) if not source_part.exists(): continue destination_part = _sidecar_path(snapshot_source, suffix) shutil.copy2(source_part, destination_part) copied.append(destination_part.name) except LiveConnectionError as exc: raise SessionRecoverySafetyError(str(exc)) from exc return snapshot_source, copied def _table_columns(conn: sqlite3.Connection, table: str) -> list[str]: return [str(row[1]) for row in conn.execute(f'PRAGMA table_info("{table}")')] def _count_rows(conn: sqlite3.Connection, table: str) -> int: return int(conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]) def _connect(path: Path) -> sqlite3.Connection: """Autocommit connection with a short busy timeout (source snapshot or fresh output).""" return sqlite3.connect(str(path), isolation_level=None, timeout=1.0) @contextmanager def _immediate_transaction(conn: sqlite3.Connection) -> Iterator[None]: """``BEGIN IMMEDIATE`` ... ``COMMIT``, rolling back on any exception.""" conn.execute("BEGIN IMMEDIATE") try: yield conn.execute("COMMIT") except BaseException: conn.execute("ROLLBACK") raise def _compatible_columns( source: sqlite3.Connection, destination: sqlite3.Connection, table: str, result: dict[str, Any], ) -> Optional[list[str]]: """Columns shared by source and destination; sets a terminal status and returns None otherwise.""" source_columns = _table_columns(source, table) columns = [column for column in _table_columns(destination, table) if column in source_columns] result["columns"] = columns if not source_columns: result["status"] = "missing" return None if not columns: result["status"] = "failed" result["error"] = "source and destination have no compatible columns" return None return columns def _quoted_columns(columns: list[str]) -> tuple[str, str]: """``("a", "b")`` column list and matching ``?, ?`` placeholders.""" return ", ".join(f'"{column}"' for column in columns), ", ".join("?" for _ in columns) def _copy_rows( source: sqlite3.Connection, destination: sqlite3.Connection, select_sql: str, params: tuple[Any, ...], insert_sql: str, *, table: str, chunk_size: int, progress_cb: Optional[ProgressCallback], expected_rows: Optional[int], result: dict[str, Any], ) -> dict[str, Any]: """Chunked straight copy; fills ``status``/``error`` on ``result``.""" try: cursor = source.execute(select_sql, params) while True: rows = cursor.fetchmany(chunk_size) if not rows: break with _immediate_transaction(destination): destination.executemany(insert_sql, rows) result["copied_rows"] += len(rows) if progress_cb is not None: progress_cb( {"table": table, "copied_rows": result["copied_rows"], "source_rows": expected_rows} ) except sqlite3.DatabaseError as exc: result["status"] = "partial" if result["copied_rows"] else "failed" result["error"] = str(exc) return result complete = expected_rows is None or result["copied_rows"] == expected_rows result["status"] = "complete" if complete else "partial" if result["status"] == "partial": result["error"] = f"copied {result['copied_rows']} of {expected_rows} readable rows" return result def _table_inventory( conn: sqlite3.Connection, table: str, ) -> dict[str, Any]: result: dict[str, Any] = {"available": False, "columns": [], "rows": None} try: columns = _table_columns(conn, table) if not columns: return result result["available"] = True result["columns"] = columns result["rows"] = _count_rows(conn, table) except sqlite3.DatabaseError as exc: result["error"] = str(exc) return result def _journal_mode(conn: sqlite3.Connection) -> Optional[str]: row = conn.execute("PRAGMA journal_mode").fetchone() return str(row[0]).lower() if row else None def _inspect_connection(conn: sqlite3.Connection) -> dict[str, Any]: conn.execute("PRAGMA writable_schema=ON") report: dict[str, Any] = {"tables": {}, "errors": [], "warnings": []} try: report["journal_mode"] = _journal_mode(conn) except sqlite3.DatabaseError as exc: report["journal_mode"] = None # Journal metadata is useful context but not canonical session data. # A damaged journal pragma must not block rows that are still readable. report["warnings"].append(f"journal mode: {exc}") for table in _INVENTORY_TABLES: report["tables"][table] = _table_inventory(conn, table) for required in ("sessions", "messages"): table_report = report["tables"][required] if not table_report.get("available") or table_report.get("rows") is None: report["errors"].append(f"required table {required} is not completely readable") report["recoverable"] = not report["errors"] return report def _snapshot_and_inspect( source: Path, work_root: Path, ) -> tuple[tempfile.TemporaryDirectory[str], Path, dict[str, Any]]: before = _source_fingerprint(source) temp_dir = tempfile.TemporaryDirectory( prefix="hermes-session-recovery-", dir=str(work_root), ) snapshot_dir = Path(temp_dir.name) try: snapshot_source, copied = _copy_source_bundle(source, snapshot_dir) after = _source_fingerprint(source) if before != after: raise SessionRecoverySafetyError( "The source database bundle changed while it was being copied. " "Stop every Hermes process using this profile and retry. " "This includes the interactive `hermes` CLI session this " "command may have been launched from: a running parent CLI " "writes session bookkeeping (compression ticks, context " "tracking) to state.db in the background and counts as a " "Hermes process even after the gateway is stopped. Run the " "recovery from a fresh shell with no `hermes` session open, " "or point --source at an immutable snapshot copy of the " "database." ) conn = _connect(snapshot_source) try: inspection = _inspect_connection(conn) finally: conn.close() inspection["source_bundle"] = copied inspection["source_fingerprint"] = before return temp_dir, snapshot_source, inspection except BaseException: temp_dir.cleanup() raise def inspect_session_database( source_path: Path, *, work_dir: Optional[Path] = None, ) -> dict[str, Any]: """Inspect canonical table readability without opening the source itself.""" source, _, work_root = _validate_paths(source_path, work_dir=work_dir) disk_space = _disk_space_preflight(source, work_root, None) temp_dir, _, inspection = _snapshot_and_inspect(source, work_root) try: return { "operation": "inspect", "source": str(source), "disk_space": disk_space, **inspection, "source_unchanged": _source_fingerprint(source) == inspection["source_fingerprint"], } finally: temp_dir.cleanup() def _ensure_auxiliary_destination_schema( destination: sqlite3.Connection, table: str, ) -> None: """Create a lazy auxiliary table on the recovered destination. Recovery initializes the destination via base ``SessionDB``, which does not create gateway-owned tables; copying into a missing table would report ``missing``/``no compatible columns`` and drop the rows. """ initialize = _AUXILIARY_TABLE_SCHEMAS.get(table) if initialize is None: raise SessionRecoverySafetyError(f"no destination schema initializer registered for table {table!r}") initialize(destination) def _fresh_destination(output: Path, *, topic_tables: bool = False) -> sqlite3.Connection: """Initialize a current-schema database at ``output`` and open it with foreign keys off.""" destination_db = SessionDB(db_path=output) try: if topic_tables: destination_db.apply_telegram_topic_migration() finally: destination_db.close() conn = _connect(output) conn.execute("PRAGMA foreign_keys=OFF") return conn def _copy_table( source: sqlite3.Connection, destination: sqlite3.Connection, table: str, *, chunk_size: int, progress_cb: Optional[ProgressCallback], source_rows: Optional[int], ) -> dict[str, Any]: result: dict[str, Any] = { "source_rows": source_rows, "copied_rows": 0, } columns = _compatible_columns(source, destination, table, result) if columns is None: return result quoted, placeholders = _quoted_columns(columns) insert_prefix = "INSERT OR REPLACE" if table == "state_meta" else "INSERT" return _copy_rows( source, destination, f'SELECT {quoted} FROM "{table}"', (), f'{insert_prefix} INTO "{table}" ({quoted}) VALUES ({placeholders})', table=table, chunk_size=chunk_size, progress_cb=progress_cb, expected_rows=source_rows, result=result, ) def _append_skipped_range( ranges: list[dict[str, Any]], low: int, high: int, error: str, ) -> None: """Record skipped rowid ranges without producing one entry per row.""" if ranges and ranges[-1]["high"] + 1 == low and ranges[-1]["error"] == error: ranges[-1]["high"] = high return ranges.append({"low": low, "high": high, "error": error}) def _salvage_rowid_bounds( source: sqlite3.Connection, table: str, ) -> dict[str, Any]: """Find the readable rowid edges without scanning the complete table.""" result: dict[str, Any] = {"errors": [], "fallback_edges": []} rows: dict[str, Optional[int]] = {"low": None, "high": None} directions = (("low", "ASC"), ("high", "DESC")) for edge, direction in directions: try: row = source.execute(f'SELECT rowid FROM "{table}" ORDER BY rowid {direction} LIMIT 1').fetchone() if row is not None: rows[edge] = int(row[0]) except sqlite3.DatabaseError as exc: result["errors"].append(f"{edge} rowid: {exc}") if rows["low"] is None and rows["high"] is None: result["empty" if not result["errors"] else "unavailable"] = True return result # A damaged edge can prevent one of the ordered probes from completing. # Keep the other readable edge and bound the missing side by SQLite's # rowid domain. Range bisection can then approach the surviving data # without assuming that user-created databases contain only positive IDs. if rows["low"] is None: rows["low"] = _MIN_SQLITE_ROWID result["fallback_edges"].append("low") if rows["high"] is None: rows["high"] = _MAX_SQLITE_ROWID result["fallback_edges"].append("high") result.update(rows) # Issue #80205: a damaged ordered edge probe used to substitute the whole # SQLite rowid domain, and bisecting that synthetic tail exhausted the # range-query budget while readable tail rows were still waiting to be # copied. Gallop outward from the surviving edge for a finite bound first. if result["fallback_edges"]: result["edge_probes"] = [] for edge, anchor_edge in (("high", "low"), ("low", "high")): if edge in result["fallback_edges"]: probe = _probe_populated_edge(source, table, edge=edge, anchor=int(result[anchor_edge])) result["edge_probes"].append(probe) if probe["capped"]: result[edge] = int(probe["bound"]) return result def _probe_populated_edge( source: sqlite3.Connection, table: str, *, edge: str, anchor: int, ) -> dict[str, Any]: """Find a finite bound for a damaged rowid edge (issue #80205). Substituting the whole rowid domain for a failed edge probe made range bisection burn the entire salvage budget on an empty synthetic tail, after which readable rows were silently skipped. Instead gallop outward from the readable ``anchor`` with exponential offsets: a clean "no rows beyond X" caps the domain, an error or hit keeps growing. At most ~64 probes per edge. """ ascending = edge == "high" comparison = ">" if ascending else "<" probe_sql = ( f'SELECT rowid FROM "{table}" WHERE rowid {comparison} ? ' f'ORDER BY rowid {"ASC" if ascending else "DESC"} LIMIT 1' ) domain_limit = _MAX_SQLITE_ROWID if ascending else _MIN_SQLITE_ROWID result: dict[str, Any] = {"edge": edge, "probes": 0, "capped": False} position = anchor span = 1 while True: candidate = position + span if ascending else position - span if (ascending and candidate >= domain_limit) or ( not ascending and candidate <= domain_limit ): # No clean empty-tail answer before the domain edge; keep the # domain fallback rather than inventing a bound. result["bound"] = domain_limit return result result["probes"] += 1 try: row = source.execute(probe_sql, (candidate,)).fetchone() except sqlite3.DatabaseError: # Damage on the probe path — inconclusive, widen further. span *= 2 continue if row is None: # Clean answer: nothing beyond ``candidate``. The salvageable # data ends at or before it, so the synthetic domain tail is # provably empty and need not be bisected at all. result["bound"] = candidate result["capped"] = True return result # Rows exist beyond the candidate; advance the anchor. The span keeps # doubling (never resets) so the whole gallop stays O(log range). position = int(row[0]) span *= 2 def _copy_table_salvage( source: sqlite3.Connection, destination: sqlite3.Connection, table: str, *, chunk_size: int, progress_cb: Optional[ProgressCallback], source_rows: Optional[int], insert_prefix: str = "INSERT", row_filter: Optional[ Callable[[tuple[Any, ...], tuple[str, ...]], bool] ] = None, ) -> dict[str, Any]: """Best-effort rowid-range copy that continues past damaged source pages.""" result: dict[str, Any] = { "mode": "rowid_range_salvage", "source_rows": source_rows, "copied_rows": 0, "excluded_rows": 0, "columns": [], "range_queries": 0, "exact_lookup_recovered": 0, "skipped_rowid_ranges": [], } columns = _compatible_columns(source, destination, table, result) if columns is None: return result bounds = _salvage_rowid_bounds(source, table) result["rowid_bounds"] = bounds if bounds.get("empty"): result["status"] = "complete" return result if bounds.get("low") is None or bounds.get("high") is None: result["status"] = "failed" details = "; ".join(bounds.get("errors") or []) result["error"] = "could not determine a rowid range for salvage" if details: result["error"] += f": {details}" return result quoted, placeholders = _quoted_columns(columns) select_sql = ( f'SELECT rowid, {quoted} FROM "{table}" ' "WHERE rowid BETWEEN ? AND ? ORDER BY rowid" ) insert_sql = ( f'{insert_prefix} INTO "{table}" ({quoted}) VALUES ({placeholders})' ) column_names = tuple(columns) stopped_at_query_limit = False exact_sql = f'SELECT {quoted} FROM "{table}" WHERE rowid = ?' def recover_exact_rowid(rowid: int) -> bool: """Issue #80205: salvage one row by exact-key lookup. A singleton range scan (``rowid BETWEEN x AND x ORDER BY rowid``) must advance the cursor past ``x`` to prove the range is exhausted; when the *next* cell or page is damaged that advance raises AFTER the row was produced, and the driver discards the already-fetched row. """ result["range_queries"] += 1 try: row = source.execute(exact_sql, (rowid,)).fetchone() except sqlite3.DatabaseError: return False if row is None: return True # genuinely absent: nothing to skip value = tuple(row) if row_filter is not None and not row_filter(value, column_names): result["excluded_rows"] += 1 return True with _immediate_transaction(destination): destination.execute(insert_sql, value) result["copied_rows"] += 1 result["exact_lookup_recovered"] += 1 return True def copy_range(low: int, high: int) -> None: nonlocal stopped_at_query_limit if low > high: return if result["range_queries"] >= _MAX_SALVAGE_RANGE_QUERIES: stopped_at_query_limit = True _append_skipped_range( result["skipped_rowid_ranges"], low, high, "salvage range query limit reached" ) return result["range_queries"] += 1 last_committed_rowid: Optional[int] = None try: cursor = source.execute(select_sql, (low, high)) while True: fetched = cursor.fetchmany(chunk_size) if not fetched: return values = [tuple(row[1:]) for row in fetched] if row_filter is not None: included = [ row for row in values if row_filter(row, column_names) ] excluded_count = len(values) - len(included) else: included = values excluded_count = 0 if included: with _immediate_transaction(destination): destination.executemany(insert_sql, included) result["copied_rows"] += len(included) result["excluded_rows"] += excluded_count last_committed_rowid = int(fetched[-1][0]) if progress_cb is not None: progress_cb({ "table": table, "copied_rows": result["copied_rows"], "source_rows": source_rows, "skipped_ranges": len(result["skipped_rowid_ranges"]), }) except sqlite3.DatabaseError as exc: retry_low = last_committed_rowid + 1 if last_committed_rowid is not None else low if retry_low > high: return if retry_low == high: if not recover_exact_rowid(retry_low): _append_skipped_range(result["skipped_rowid_ranges"], retry_low, high, str(exc)) return midpoint = retry_low + (high - retry_low) // 2 copy_range(retry_low, midpoint) copy_range(midpoint + 1, high) copy_range(int(bounds["low"]), int(bounds["high"])) skipped_ranges = result["skipped_rowid_ranges"] result["skipped_rowid_span"] = sum(item["high"] - item["low"] + 1 for item in skipped_ranges) result["query_limit_reached"] = stopped_at_query_limit if skipped_ranges: result["status"] = "partial" if result["copied_rows"] else "failed" result["error"] = f"{len(skipped_ranges)} rowid range(s) skipped" elif ( source_rows is not None and result["copied_rows"] + result["excluded_rows"] != source_rows ): result["status"] = "partial" result["error"] = ( f"copied {result['copied_rows']} and excluded " f"{result['excluded_rows']} of {source_rows} source rows" ) else: result["status"] = "complete" return result def _state_meta_result(source_rows: Optional[int], **extra: Any) -> dict[str, Any]: return { "source_meta_rows": source_rows, "copied_rows": 0, "columns": ["key", "value"], "excluded_keys": sorted(_GENERATED_META_KEYS), **extra, } def _state_meta_precheck( source: sqlite3.Connection, destination: sqlite3.Connection, source_rows: Optional[int], *, salvage: bool, ) -> Optional[dict[str, Any]]: """Terminal ``state_meta`` result when the key/value schema is unusable, else ``None``. Status matters here. In salvage mode an unusable-but-PRESENT table reports ``failed``, not ``missing``: verification only escalates ``failed``/``partial`` into a warning + ``loss_detected``, so reporting ``missing`` would silently drop real metadata and still claim ``complete=True``. """ extra = {"mode": "rowid_range_salvage"} if salvage else {} source_columns = _table_columns(source, "state_meta") if not {"key", "value"}.issubset(source_columns): if salvage and source_columns: # Present but unusable: this IS data loss and must be reported. return _state_meta_result( source_rows, **extra, status="failed", error=( "source state_meta exists but is missing the key/value " f"columns (found: {', '.join(source_columns) or 'none'})" ), ) # Genuinely absent from the source — nothing was lost. return _state_meta_result(source_rows, **extra, status="missing") if not {"key", "value"}.issubset(_table_columns(destination, "state_meta")): return _state_meta_result( source_rows, **extra, status="failed", error="destination state_meta schema is incomplete" ) return None def _copy_state_meta( source: sqlite3.Connection, destination: sqlite3.Connection, *, chunk_size: int, progress_cb: Optional[ProgressCallback], source_rows: Optional[int], ) -> dict[str, Any]: problem = _state_meta_precheck(source, destination, source_rows, salvage=False) if problem is not None: return problem placeholders = ", ".join("?" for _ in _GENERATED_META_KEYS) params = tuple(_GENERATED_META_KEYS) filtered_source_rows: Optional[int] = None try: filtered_source_rows = int( source.execute( f"SELECT COUNT(*) FROM state_meta WHERE key NOT IN ({placeholders})", params, ).fetchone()[0] ) except sqlite3.DatabaseError: # The copy loop below will return the concrete read error. pass return _copy_rows( source, destination, f"SELECT key, value FROM state_meta WHERE key NOT IN ({placeholders})", params, "INSERT OR REPLACE INTO state_meta(key, value) VALUES (?, ?)", table="state_meta", chunk_size=chunk_size, progress_cb=progress_cb, expected_rows=filtered_source_rows, result=_state_meta_result(source_rows), ) def _copy_state_meta_salvage( source: sqlite3.Connection, destination: sqlite3.Connection, *, chunk_size: int, progress_cb: Optional[ProgressCallback], source_rows: Optional[int], ) -> dict[str, Any]: """Salvage readable user metadata while regenerating derived FTS state.""" problem = _state_meta_precheck(source, destination, source_rows, salvage=True) if problem is not None: return problem def keep_user_meta(row: tuple[Any, ...], columns: tuple[str, ...]) -> bool: return str(row[columns.index("key")]) not in _GENERATED_META_KEYS result = _copy_table_salvage( source, destination, "state_meta", chunk_size=chunk_size, progress_cb=progress_cb, source_rows=source_rows, insert_prefix="INSERT OR REPLACE", row_filter=keep_user_meta, ) result["source_meta_rows"] = result.pop("source_rows") result["excluded_keys"] = sorted(_GENERATED_META_KEYS) return result def _reconstruct_missing_sessions( destination: sqlite3.Connection, ) -> dict[str, Any]: """Recreate placeholder session rows for salvaged orphaned messages. When the ``sessions`` b-tree is damaged worse than ``messages``, salvage can recover the conversation text while recovering few or none of the session rows that own it. Deleting those messages as "orphans" throws away the only readable copy of the user's data — the exact opposite of what ``--allow-partial`` is for. Instead, synthesize a minimal session row per orphaned ``session_id`` (only ``id``/``source``/``started_at`` are NOT NULL) so the messages stay reachable and foreign keys hold. ``started_at`` is taken from the earliest surviving message so ordering stays sane. """ result: dict[str, Any] = {"sessions_reconstructed": 0, "messages_retained": 0} if not (_table_columns(destination, "sessions") and _table_columns(destination, "messages")): return result orphaned = destination.execute( "SELECT m.session_id, MIN(m.timestamp), COUNT(*) " "FROM messages AS m " "WHERE m.session_id IS NOT NULL AND NOT EXISTS (" "SELECT 1 FROM sessions WHERE sessions.id = m.session_id) " "GROUP BY m.session_id" ).fetchall() if not orphaned: return result title_sequence = 1 for session_id, first_timestamp, message_count in orphaned: started_at = float(first_timestamp) if first_timestamp is not None else 0.0 while True: title = f"[recovered {title_sequence}] session metadata was unreadable" title_sequence += 1 taken = destination.execute("SELECT 1 FROM sessions WHERE title = ? LIMIT 1", (title,)).fetchone() if taken is None: break cursor = destination.execute( "INSERT INTO sessions " "(id, source, started_at, title, message_count) " "VALUES (?, 'recovered', ?, ?, ?)", (session_id, started_at, title, int(message_count)), ) if cursor.rowcount != 1: raise sqlite3.IntegrityError(f"failed to reconstruct missing session {session_id!r}") result["sessions_reconstructed"] += 1 result["messages_retained"] += int(message_count) return result def _reconcile(destination: sqlite3.Connection, table: str, where: str, mutation: str) -> int: """Count rows of ``table`` matching ``where``; run ``mutation WHERE where`` only when there are any.""" count = int(destination.execute(f'SELECT COUNT(*) FROM "{table}" WHERE {where}').fetchone()[0]) if count: destination.execute(f"{mutation} WHERE {where}") return count _DEPENDENT_TABLES = ( "messages", "session_model_usage", "compression_locks", "telegram_dm_topic_bindings", ) _RELINK_COUNTERS = ("session_prompt_refs_cleared", "sessions_parent_cleared") def _cleanup_partial_orphans( destination: sqlite3.Connection, ) -> dict[str, Any]: """Reconcile references to sessions that could not be salvaged. Messages are never discarded for lack of a session row: their owning session is reconstructed as a placeholder first (see :func:`_reconstruct_missing_sessions`). Only rows that remain orphaned after that — and rows in tables carrying no recoverable user content — are removed. """ result: dict[str, Any] = { "session_prompt_refs_cleared": 0, "system_prompts_removed": 0, "sessions_parent_cleared": 0, "sessions_reconstructed": 0, "messages_retained": 0, **{f"{table}_removed": 0 for table in _DEPENDENT_TABLES}, } with _immediate_transaction(destination): # Rebuild owners BEFORE any orphan deletion so salvaged conversation # text is never dropped for want of a session row. result.update(_reconstruct_missing_sessions(destination)) result["sessions_parent_cleared"] = _reconcile( destination, "sessions", "parent_session_id IS NOT NULL AND NOT EXISTS (" "SELECT 1 FROM sessions AS parent WHERE parent.id = sessions.parent_session_id)", "UPDATE sessions SET parent_session_id = NULL", ) result["session_prompt_refs_cleared"] = _reconcile( destination, "sessions", "system_prompt_hash IS NOT NULL AND NOT EXISTS (" "SELECT 1 FROM system_prompts WHERE system_prompts.hash = sessions.system_prompt_hash)", "UPDATE sessions SET system_prompt_hash = NULL", ) result["system_prompts_removed"] = _reconcile( destination, "system_prompts", "NOT EXISTS (SELECT 1 FROM sessions WHERE sessions.system_prompt_hash = system_prompts.hash)", "DELETE FROM system_prompts", ) for table in _DEPENDENT_TABLES: if not _table_columns(destination, table): continue result[f"{table}_removed"] = _reconcile( destination, table, f'NOT EXISTS (SELECT 1 FROM sessions WHERE sessions.id = "{table}".session_id)', f'DELETE FROM "{table}"', ) # Only destructive/relinking actions belong in this total. The # reconstruction counters describe data RETAINED, so summing them here # would report saving the user's messages as if it were losing them. result["total_removed_or_relinked"] = sum( int(result[key]) for key in (*_RELINK_COUNTERS, *(f"{table}_removed" for table in _DEPENDENT_TABLES)) ) return result def _verify_recovered_database( output: Path, *, expected_counts: dict[str, Optional[int]], copy_report: dict[str, dict[str, Any]], allow_partial: bool = False, orphan_cleanup: Optional[dict[str, Any]] = None, ) -> dict[str, Any]: verification: dict[str, Any] = { "errors": [], "warnings": [], "loss_detected": False, } def flag(message: str, *, soft: bool) -> None: """Record data loss as a warning (``soft``) or as a verification error.""" if soft: verification["warnings"].append(message) verification["loss_detected"] = True else: verification["errors"].append(message) open_error = _db_opens_cleanly(output) verification["opens_cleanly"] = open_error is None if open_error is not None: verification["errors"].append(f"database health probe: {open_error}") conn = sqlite3.connect(str(output), isolation_level=None) try: integrity_rows = [str(row[0]) for row in conn.execute("PRAGMA integrity_check").fetchall()] verification["integrity_check"] = integrity_rows if integrity_rows != ["ok"]: verification["errors"].append("PRAGMA integrity_check did not return exactly 'ok'") foreign_key_rows = [list(row) for row in conn.execute("PRAGMA foreign_key_check").fetchall()] verification["foreign_key_check"] = foreign_key_rows if foreign_key_rows: verification["errors"].append("foreign key violations remain") verification["journal_mode"] = _journal_mode(conn) schema_row = conn.execute("SELECT version FROM schema_version LIMIT 1").fetchone() verification["schema_version"] = int(schema_row[0]) if schema_row else None if verification["schema_version"] != SCHEMA_VERSION: verification["errors"].append( f"schema version is {verification['schema_version']}, " f"expected {SCHEMA_VERSION}" ) meta = { str(row[0]): row[1] for row in conn.execute("SELECT key, value FROM state_meta WHERE key LIKE 'fts_%'").fetchall() } verification["fts_meta"] = meta if meta.get("fts_storage_version") != str(FTS_STORAGE_VERSION): verification["errors"].append("fresh FTS storage version was not established") pending_keys = sorted( key for key in _GENERATED_META_KEYS if key.startswith("fts_") and key != "fts_storage_version" and key in meta ) verification["pending_fts_keys"] = pending_keys if pending_keys: verification["errors"].append("derived FTS transition markers remain in the recovered database") counts: dict[str, int] = {} for table in _INVENTORY_TABLES: if _table_columns(conn, table): counts[table] = _count_rows(conn, table) verification["table_counts"] = counts for table in ("sessions", "messages", *_AUXILIARY_TABLES): expected = expected_counts.get(table) if expected is not None and counts.get(table) != expected: flag(f"{table} count is {counts.get(table)}, expected {expected}", soft=allow_partial) cleanup = orphan_cleanup or {} rebuilt_sessions = int(cleanup.get("sessions_reconstructed") or 0) retained_messages = int(cleanup.get("messages_retained") or 0) removed_messages = int(cleanup.get("messages_removed") or 0) # A wholly unreadable sessions b-tree is recoverable when every output # parent was rebuilt from the surviving messages and none were dropped. # This is still data loss, but it is not structural verification failure. sessions_fully_reconstructed = bool( rebuilt_sessions > 0 and counts.get("sessions") == rebuilt_sessions and counts.get("messages") == retained_messages and removed_messages == 0 ) for table, table_report in copy_report.items(): status = table_report.get("status") if status not in {"failed", "partial"}: continue flag( f"{table} copy status is {status}", soft=allow_partial and ( status == "partial" or table not in {"sessions", "messages"} or (table == "sessions" and sessions_fully_reconstructed) ), ) if orphan_cleanup: orphan_count = int(orphan_cleanup.get("total_removed_or_relinked") or 0) if orphan_count: flag(f"{orphan_count} orphaned reference(s) were removed or relinked", soft=True) if rebuilt_sessions: # Not a clean recovery: the conversation text survived but its # session metadata did not, so these rows are placeholders. flag( f"{rebuilt_sessions} session(s) could not be salvaged and " f"were reconstructed as placeholders to retain " f"{retained_messages} message(s); their metadata (title, model, " "timestamps, cost) is lost", soft=True, ) fts_checks: dict[str, str] = {} for table in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"): if not _table_columns(conn, table): continue try: conn.execute( f'INSERT INTO "{table}" ("{table}") VALUES (\'integrity-check\')' ) conn.execute( f'SELECT 1 FROM "{table}" WHERE "{table}" MATCH \'""\' LIMIT 1' ).fetchone() fts_checks[table] = "ok" except sqlite3.DatabaseError as exc: fts_checks[table] = str(exc) verification["errors"].append(f"{table} integrity check failed: {exc}") verification["fts_checks"] = fts_checks except sqlite3.DatabaseError as exc: verification["errors"].append(f"verification query failed: {exc}") finally: conn.close() verification["healthy"] = not verification["errors"] verification["complete"] = bool(verification["healthy"] and not verification["loss_detected"]) return verification def _finalize_derived_metadata(destination: sqlite3.Connection) -> dict[str, Any]: """Stamp only metadata that the newly created destination actually owns.""" fts_tables = { str(row[0]) for row in destination.execute( "SELECT name FROM sqlite_master " "WHERE type='table' AND name IN ('messages_fts', 'messages_fts_trigram')" ).fetchall() } result: dict[str, Any] = {"fts_tables": sorted(fts_tables), "finalized": False} if fts_tables != {"messages_fts", "messages_fts_trigram"}: result["error"] = "fresh destination is missing required FTS tables" return result fts_keys = tuple(key for key in _GENERATED_META_KEYS if key.startswith("fts_")) placeholders = ", ".join("?" for _ in fts_keys) with _immediate_transaction(destination): destination.execute( f"DELETE FROM state_meta WHERE key IN ({placeholders})", fts_keys, ) destination.execute( "INSERT INTO state_meta(key, value) VALUES (?, ?) " "ON CONFLICT(key) DO UPDATE SET value = excluded.value", ("fts_storage_version", str(FTS_STORAGE_VERSION)), ) result["finalized"] = True return result def _recover_via_lost_and_found( *, source: Path, snapshot_source: Path, snapshot_dir: Path, output: Path, inspection: dict[str, Any], disk_space: dict[str, Any], missing_required: list[str], ) -> dict[str, Any]: """Best-effort page-level salvage when table schemas are unreadable. Shells out to the sqlite3 CLI's ``.recover`` (a shell-only feature, not part of Python's ``sqlite3`` module) to rebuild rows into a scratch lost_and_found database, then heuristically maps them into a fresh current-schema database. The result is explicitly labeled best-effort. """ from hermes_cli.session_lost_and_found import ( SQLITE3_CLI_GUIDANCE, LostAndFoundError, find_sqlite3_cli, map_lost_and_found_rows, rebuild_fts_indexes, run_cli_lost_and_found_recover, stub_missing_parent_sessions, ) sqlite3_bin = find_sqlite3_cli() if sqlite3_bin is None: raise SessionRecoverySourceError( "Partial recovery still requires readable table schemas for: " + ", ".join(missing_required) + ". " + SQLITE3_CLI_GUIDANCE ) lf_path = snapshot_dir / "lost_and_found.db" try: cli_report = run_cli_lost_and_found_recover(snapshot_source, lf_path, sqlite3_bin) except (LostAndFoundError, OSError) as exc: raise SessionRecoverySourceError( "Partial recovery could not read the table schemas for: " + ", ".join(missing_required) + f", and page-level .recover salvage failed: {exc}" ) from exc lf_conn = sqlite3.connect(str(lf_path), isolation_level=None) destination_conn = _fresh_destination(output) try: mapping = map_lost_and_found_rows(lf_conn, destination_conn) stubbing = stub_missing_parent_sessions(destination_conn) fts = rebuild_fts_indexes(destination_conn) derived_metadata = _finalize_derived_metadata(destination_conn) finally: lf_conn.close() destination_conn.close() copy_report: dict[str, dict[str, Any]] = { table: { "mode": "lost_and_found_salvage", "status": "partial", "copied_rows": ( int(mapping["direct_table_rows"].get(table) or 0) + int(mapping["mapped"].get(table) or 0) ), "error": "recovered via page-level lost_and_found salvage; " "row completeness cannot be verified against the source", } for table in ("sessions", "messages", "session_model_usage") } orphan_cleanup = { "sessions_reconstructed": stubbing["sessions_stubbed"], "messages_retained": stubbing["messages_retained"], "messages_removed": 0, "total_removed_or_relinked": 0, } verification = _verify_recovered_database( output, expected_counts={"sessions": None, "messages": None}, copy_report=copy_report, allow_partial=True, orphan_cleanup=orphan_cleanup, ) verification["loss_detected"] = True verification["warnings"].append( "BEST-EFFORT page-level salvage: the source table schemas were " "unreadable, so rows were rebuilt from raw pages and mapped " "heuristically. Review every count before trusting this output." ) verification["complete"] = False return _recovery_report( source, output, inspection, disk_space, verification, on_source_change="healthy", allow_partial=True, mode="lost_and_found_salvage", best_effort=True, unreadable_schemas=missing_required, sqlite3_cli=cli_report, lost_and_found=mapping, session_stubs=stubbing, fts_rebuild=fts, copy=copy_report, orphan_cleanup=orphan_cleanup, derived_metadata=derived_metadata, ) def _recovery_report( source: Path, output: Path, inspection: dict[str, Any], disk_space: dict[str, Any], verification: dict[str, Any], *, on_source_change: str, **fields: Any, ) -> dict[str, Any]: """The ``recover`` report: shared header, mode-specific ``fields``, then the verdict. A source bundle that changed during recovery is a verification error and additionally clears ``verification[on_source_change]``. """ source_unchanged = _source_fingerprint(source) == inspection["source_fingerprint"] if not source_unchanged: verification["errors"].append("the source database bundle changed during recovery") verification[on_source_change] = False return { "operation": "recover", "source": str(source), "output": str(output), "source_bundle": inspection["source_bundle"], "source_fingerprint": inspection["source_fingerprint"], "source_unchanged": source_unchanged, "disk_space": disk_space, "inspection": { "journal_mode": inspection.get("journal_mode"), "tables": inspection["tables"], "errors": inspection["errors"], "warnings": inspection["warnings"], }, **fields, "verification": verification, "complete": bool(verification.get("complete") and source_unchanged), "partial": bool(verification.get("loss_detected")), "verified": bool(verification.get("healthy") and source_unchanged), "installed": False, } def recover_session_database( source_path: Path, output_path: Path, *, work_dir: Optional[Path] = None, chunk_size: int = 1_000, progress_cb: Optional[ProgressCallback] = None, allow_partial: bool = False, ) -> dict[str, Any]: """Recover canonical rows into a separate current-schema database. The source path and its sidecars are copied before SQLite opens anything. ``output_path`` must not exist and is never swapped into place. """ if chunk_size <= 0: raise SessionRecoverySafetyError("chunk_size must be greater than zero") source, output, work_root = _validate_paths( source_path, output_path=output_path, work_dir=work_dir, ) assert output is not None disk_space = _disk_space_preflight(source, work_root, output.parent) temp_dir, snapshot_source, inspection = _snapshot_and_inspect(source, work_root) try: if not inspection.get("recoverable") and not allow_partial: reasons = "; ".join(inspection.get("errors") or ["unknown source error"]) raise SessionRecoverySourceError( f"Required canonical tables are not readable: {reasons}. " "Re-run with --allow-partial to salvage every readable row " "into a new database (the source is never modified)." ) if allow_partial: missing_required = [ table for table in ("sessions", "messages") if not inspection["tables"][table].get("available") ] if missing_required: # SQL-level salvage is impossible without readable table # schemas. Fall back to page-level lost_and_found salvage via # the sqlite3 CLI's .recover (a shell-only feature). return _recover_via_lost_and_found( source=source, snapshot_source=snapshot_source, snapshot_dir=Path(temp_dir.name), output=output, inspection=inspection, disk_space=disk_space, missing_required=missing_required, ) source_conn = _connect(snapshot_source) source_conn.execute("PRAGMA writable_schema=ON") destination_conn: Optional[sqlite3.Connection] = None try: destination_conn = _fresh_destination( output, topic_tables=any(inspection["tables"][table].get("available") for table in _TOPIC_TABLES), ) copy_meta, copy_rows = ( (_copy_state_meta_salvage, _copy_table_salvage) if allow_partial else (_copy_state_meta, _copy_table) ) copy_report: dict[str, dict[str, Any]] = {} for table in (*_CANONICAL_TABLES, "state_meta", *_TOPIC_TABLES, *_AUXILIARY_TABLES): if table not in _CANONICAL_TABLES and not inspection["tables"][table].get("available"): copy_report[table] = {"status": "missing", "copied_rows": 0} continue if table in _AUXILIARY_TABLES: _ensure_auxiliary_destination_schema(destination_conn, table) copy_function, args = (copy_meta, ()) if table == "state_meta" else (copy_rows, (table,)) copy_report[table] = copy_function( source_conn, destination_conn, *args, chunk_size=chunk_size, progress_cb=progress_cb, source_rows=inspection["tables"][table].get("rows"), ) orphan_cleanup = _cleanup_partial_orphans(destination_conn) if allow_partial else None derived_metadata = _finalize_derived_metadata(destination_conn) finally: source_conn.close() if destination_conn is not None: destination_conn.close() verification = _verify_recovered_database( output, expected_counts={ table: inspection["tables"][table].get("rows") for table in (*_CANONICAL_TABLES, *_AUXILIARY_TABLES) if table in _CANONICAL_TABLES or inspection["tables"].get(table, {}).get("available") }, copy_report=copy_report, allow_partial=allow_partial, orphan_cleanup=orphan_cleanup, ) return _recovery_report( source, output, inspection, disk_space, verification, on_source_change="complete", allow_partial=allow_partial, copy=copy_report, orphan_cleanup=orphan_cleanup, derived_metadata=derived_metadata, ) finally: temp_dir.cleanup() def write_recovery_report(path: Path, report: dict[str, Any]) -> Path: """Write a JSON report without overwriting an existing file.""" destination = _resolved_output_path(path) with destination.open("x", encoding="utf-8") as handle: json.dump(report, handle, indent=2, sort_keys=True) handle.write("\n") return destination