Teknium
1611 lines
73 KiB
Python
1611 lines
73 KiB
Python
"""Backup and import commands for hermes CLI."""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import sqlite3
|
|
import stat
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import zipfile
|
|
from contextlib import contextmanager, suppress
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from hermes_constants import (
|
|
_get_platform_default_hermes_home, get_default_hermes_root, get_hermes_home, display_hermes_home,
|
|
)
|
|
from utils import (
|
|
_preserve_file_mode, _preserve_file_owner, _restore_file_mode, _restore_file_owner, atomic_replace,
|
|
)
|
|
|
|
# Private alias kept: claw.py and the backup tests import ``_format_size`` from here.
|
|
from hermes_cli.sizefmt import format_bytes as _format_size
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# --- Exclusion rules ---
|
|
|
|
# Where ``hermes backup --quick`` / ``/snapshot`` / the pre-update safety net write state
|
|
# snapshots (see ``create_quick_snapshot``); defined here because the exclusion set needs it.
|
|
_QUICK_SNAPSHOTS_DIR = "state-snapshots"
|
|
|
|
# Directory names to skip (matched against each path component). ``hermes-agent`` is
|
|
# special-cased to the root level in ``_should_exclude`` so skill dirs like
|
|
# ``skills/autonomous-ai-agents/hermes-agent/`` survive. The dependency/cache entries matter:
|
|
# one plugin venv or pip/uv cache under HERMES_HOME walked file-by-file balloons a backup to
|
|
# hundreds of thousands of entries ("backup stuck for days"). They mostly mirror
|
|
# ``agent.skill_utils.EXCLUDED_SKILL_DIRS``; ``.cache`` is backup-only. ``.archive`` is
|
|
# deliberately NOT excluded: the curator's ``skills/.archive/`` holds restorable user skills.
|
|
_EXCLUDED_DIRS = {
|
|
"hermes-agent", # the codebase repo — re-clone instead
|
|
"__pycache__", # bytecode caches — regenerated on import
|
|
".git", # nested git dirs (profiles shouldn't have these, but safety)
|
|
"node_modules", # js deps — reinstalled on demand
|
|
"backups", # prior auto-backups — don't nest backups exponentially
|
|
_QUICK_SNAPSHOTS_DIR, # each holds a full state.db copy — same reason as ``backups``
|
|
"checkpoints", # session-hash-keyed trajectory caches — regenerated, don't port
|
|
# Live CDP browser profiles: Chromium holds their SQLite DBs with exclusive locks while
|
|
# running and sqlite3.backup() retries SQLITE_BUSY forever, hanging the backup mid-archive.
|
|
# Regenerable (cache + re-login) and unsafe to snapshot live.
|
|
"browser-profiles",
|
|
# Real-profile browsing snapshot (browser.use_real_profile): copies of the user's Cookies /
|
|
# Login Data — a credential store that must NOT enter an archive. Regenerated on next launch.
|
|
"browser-profile",
|
|
# Python dependency trees (plugin / MCP-server venvs) — regenerated by reinstalling.
|
|
".venv", "venv", "site-packages",
|
|
# Tool / build caches — all regeneratable.
|
|
".cache", ".tox", ".nox", ".pytest_cache", ".mypy_cache", ".ruff_cache",
|
|
}
|
|
|
|
# Hermes-managed runtime downloads (GGUF models, llama.cpp runtimes, managed Node): all
|
|
# re-downloaded on demand and routinely tens to hundreds of GB of incompressible weights.
|
|
# Matched ONLY at the root of HERMES_HOME and at ``profiles/<name>/`` — a deeper directory
|
|
# sharing one of these names (a skill's ``models/``) is user data and stays in the backup.
|
|
_EXCLUDED_ROOT_DIRS = {"models", "runtimes", "node"}
|
|
|
|
|
|
def _in_excluded_root_dir(rel_path: Path) -> bool:
|
|
"""True when *rel_path* is, or sits inside, a managed runtime tree at a profile-home root."""
|
|
parts = rel_path.parts
|
|
return bool(parts) and (
|
|
parts[0] in _EXCLUDED_ROOT_DIRS
|
|
or (len(parts) >= 3 and parts[0] == "profiles" and parts[2] in _EXCLUDED_ROOT_DIRS))
|
|
|
|
|
|
# SQLite sidecars are excluded because ``*.db`` is snapshotted via ``sqlite3.backup()``:
|
|
# shipping the live WAL/SHM/journal alongside would pair a fresh snapshot with stale sidecar
|
|
# state and produce a torn restore on next open. They are regenerated on first connection.
|
|
_EXCLUDED_SUFFIXES = (".pyc", ".pyo", ".db-wal", ".db-shm", ".db-journal")
|
|
|
|
# File names to skip (runtime state that's meaningless on another machine)
|
|
_EXCLUDED_NAMES = {".backup.lock", "gateway.pid", "cron.pid"}
|
|
|
|
# The desktop updater's pre-flight drops ``state.db.pre-update-emergency-<ts>.bak`` at the root
|
|
# — a backup artifact like ``backups/``. Prefix-matched because the name carries a timestamp;
|
|
# a plain ``.bak`` suffix rule would drop user files.
|
|
_EXCLUDED_PREFIXES = ("state.db.pre-update-emergency-",)
|
|
|
|
# Files ``hermes import`` must never overwrite, matched by basename so root and named profiles
|
|
# (``profiles/<name>/gateway_state.json``) are both covered. They hold gateway/process runtime
|
|
# state namespaced to the source machine: ``gateway_state.json`` drives the container-boot
|
|
# reconciler (a stale/foreign value leaves the gateway stuck "starting" and disconnected from
|
|
# the Nous portal); the PID/lock/registry files reference the SOURCE process namespace. Mirrors
|
|
# ``container_boot._STALE_RUNTIME_FILES``. Older backups predate the backup-side exclusions,
|
|
# so import filters too rather than trusting the archive.
|
|
_IMPORT_SKIP_NAMES = {"gateway_state.json", "gateway.pid", "cron.pid", "gateway.lock", "processes.json"}
|
|
|
|
# zipfile.open() drops Unix mode bits on extract; restore tightens these to 0600.
|
|
_SECRET_FILE_NAMES = {".env", "auth.json", "state.db"}
|
|
|
|
# Reserved archive subtree for memory-provider state OUTSIDE HERMES_HOME (e.g. ~/.honcho),
|
|
# declared via MemoryProvider.backup_paths(). Stored relative to the user's home and restored
|
|
# to the same home-relative location; anything not under home is skipped.
|
|
_EXTERNAL_PREFIX = "_external/"
|
|
|
|
|
|
class BackupInProgressError(RuntimeError):
|
|
"""Raised when another process already owns the Hermes backup slot."""
|
|
|
|
|
|
class _SQLiteSnapshotError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class _SQLiteBackupTimeout(RuntimeError):
|
|
"""Raised when a SQLite snapshot remains busy past its deadline."""
|
|
|
|
|
|
@contextmanager
|
|
def _backup_operation_lock(hermes_home: Path, timeout_seconds: float = 0.25):
|
|
"""Acquire one cross-process backup slot for full and quick snapshots."""
|
|
lock_path = hermes_home / ".backup.lock"
|
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
handle = lock_path.open("a+b")
|
|
acquired = False
|
|
deadline = time.monotonic() + max(0.0, timeout_seconds)
|
|
try:
|
|
if os.name == "nt":
|
|
import msvcrt
|
|
if lock_path.stat().st_size == 0:
|
|
handle.write(b" ")
|
|
handle.flush()
|
|
def _lock_op(flag: int) -> None:
|
|
handle.seek(0)
|
|
msvcrt.locking(handle.fileno(), flag, 1)
|
|
lock_flag, unlock_flag = msvcrt.LK_NBLCK, msvcrt.LK_UNLCK
|
|
else:
|
|
import fcntl
|
|
def _lock_op(flag: int) -> None:
|
|
fcntl.flock(handle.fileno(), flag)
|
|
lock_flag, unlock_flag = fcntl.LOCK_EX | fcntl.LOCK_NB, fcntl.LOCK_UN
|
|
while not acquired:
|
|
try:
|
|
_lock_op(lock_flag)
|
|
acquired = True
|
|
except OSError:
|
|
if time.monotonic() >= deadline:
|
|
raise BackupInProgressError("another Hermes backup is already running")
|
|
time.sleep(0.05)
|
|
yield
|
|
finally:
|
|
if acquired:
|
|
with suppress(OSError):
|
|
_lock_op(unlock_flag)
|
|
handle.close()
|
|
|
|
|
|
@contextmanager
|
|
def _atomic_output_path(final_path: Path):
|
|
"""Yield a hidden sibling path and publish it only after a clean close."""
|
|
partial_path = final_path.with_name(f".{final_path.name}.{os.getpid()}-{threading.get_ident()}.partial")
|
|
partial_path.unlink(missing_ok=True)
|
|
try:
|
|
yield partial_path
|
|
os.replace(partial_path, final_path)
|
|
except BaseException:
|
|
partial_path.unlink(missing_ok=True)
|
|
raise
|
|
|
|
|
|
def _is_within(path: Path, root: Path) -> bool:
|
|
"""True when *path* resolves inside the already-resolved *root* (traversal / symlink guard)."""
|
|
try:
|
|
path.resolve().relative_to(root)
|
|
except ValueError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _collect_memory_provider_external_paths() -> List[Path]:
|
|
"""Existing absolute paths the active memory provider declares via ``backup_paths()``.
|
|
|
|
``[]`` when no external provider is active or it can't be loaded: backup must never fail
|
|
because of a flaky plugin.
|
|
"""
|
|
try:
|
|
from plugins.memory import _get_active_memory_provider, load_memory_provider
|
|
active = _get_active_memory_provider()
|
|
provider = load_memory_provider(active) if active else None
|
|
except Exception:
|
|
return []
|
|
if provider is None:
|
|
return []
|
|
try:
|
|
declared = provider.backup_paths() or []
|
|
except Exception as exc:
|
|
logger.warning("backup_paths() failed for memory provider %r: %s", active, exc)
|
|
return []
|
|
out: List[Path] = []
|
|
seen: set = set()
|
|
for raw in declared:
|
|
try:
|
|
p = Path(raw).expanduser()
|
|
resolved = p.resolve() if p.exists() else None
|
|
except Exception:
|
|
continue
|
|
if resolved is not None and resolved not in seen:
|
|
seen.add(resolved)
|
|
out.append(p)
|
|
return out
|
|
|
|
|
|
def _iter_external_files(base: Path) -> List[Path]:
|
|
"""Regular files under *base* (a file or a directory), skipping symlinks, caches, and pyc."""
|
|
if base.is_file() and not base.is_symlink():
|
|
return [base]
|
|
if not base.is_dir():
|
|
return []
|
|
files: List[Path] = []
|
|
for dirpath, dirnames, filenames in os.walk(base, followlinks=False):
|
|
dp = Path(dirpath)
|
|
dirnames[:] = [d for d in dirnames if d not in _EXCLUDED_DIRS]
|
|
for fname in filenames:
|
|
fpath = dp / fname
|
|
if fpath.is_symlink() or fname in _EXCLUDED_NAMES or fname.endswith(_EXCLUDED_SUFFIXES):
|
|
continue
|
|
files.append(fpath)
|
|
return files
|
|
|
|
|
|
def _should_exclude(rel_path: Path) -> bool:
|
|
"""Return True if *rel_path* (relative to hermes root) should be skipped."""
|
|
parts = rel_path.parts
|
|
if _in_excluded_root_dir(rel_path):
|
|
return True
|
|
# ``hermes-agent`` only matches at the root level; nested same-named dirs are preserved.
|
|
if any(p in _EXCLUDED_DIRS and (p != "hermes-agent" or p == parts[0]) for p in parts):
|
|
return True
|
|
name = rel_path.name
|
|
return name in _EXCLUDED_NAMES or name.startswith(_EXCLUDED_PREFIXES) or name.endswith(_EXCLUDED_SUFFIXES)
|
|
|
|
|
|
def _iter_backup_files(hermes_root: Path, out_path: Path, skipped_dirs: Optional[set] = None):
|
|
"""Yield ``(abs_path, rel_path)`` for every file a full backup should hold.
|
|
|
|
The one owner of the backup walk policy — directory pruning (so os.walk never descends a
|
|
multi-GB excluded tree), the root-only ``hermes-agent`` carve-out, profile-home-root runtime
|
|
trees, and the per-file rules — shared by ``hermes backup`` and the automatic pre-update /
|
|
pre-migration path so the two can never drift.
|
|
"""
|
|
for dirpath, dirnames, filenames in os.walk(hermes_root, followlinks=False):
|
|
rel_dir = Path(dirpath).relative_to(hermes_root)
|
|
is_root = rel_dir == Path(".")
|
|
kept = [
|
|
d for d in dirnames
|
|
if (d not in _EXCLUDED_DIRS or (d == "hermes-agent" and not is_root))
|
|
and not _in_excluded_root_dir(rel_dir / d)]
|
|
if skipped_dirs is not None:
|
|
skipped_dirs.update(str(rel_dir / d) for d in set(dirnames) - set(kept))
|
|
dirnames[:] = kept
|
|
for fname in filenames:
|
|
rel = rel_dir / fname
|
|
fpath = hermes_root / rel
|
|
# zipfile.write() follows file symlinks, so skip links before any archive write can
|
|
# copy data from outside HERMES_HOME; never archive the output zip into itself.
|
|
if _should_exclude(rel) or fpath.is_symlink():
|
|
continue
|
|
with suppress(OSError, ValueError):
|
|
if fpath.resolve() == out_path.resolve():
|
|
continue
|
|
yield fpath, rel
|
|
|
|
|
|
# --- SQLite safe copy ---
|
|
|
|
def _close_quietly(conn: Optional[sqlite3.Connection]) -> None:
|
|
if conn is not None:
|
|
with suppress(Exception):
|
|
conn.close()
|
|
|
|
|
|
def _query_ro_sqlite(path: Path, fn):
|
|
"""Run ``fn(conn)`` on a read-only connection to *path*; return ``(value, None)`` or ``(None, exc)``."""
|
|
conn = None
|
|
try:
|
|
conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=1.0)
|
|
return fn(conn), None
|
|
except Exception as exc:
|
|
return None, exc
|
|
finally:
|
|
_close_quietly(conn)
|
|
|
|
|
|
def _safe_copy_db(src: Path, dst: Path, *, timeout_seconds: float = 10.0) -> bool:
|
|
"""Copy a SQLite database with the backup() API (WAL-safe consistent snapshot).
|
|
|
|
Fails closed when no consistent snapshot can be made: copying only the live main file can
|
|
omit committed WAL data.
|
|
"""
|
|
conn = backup_conn = None
|
|
try:
|
|
# timeout=0.0 disables sqlite3's implicit busy wait so the progress callback owns the
|
|
# full locked-source deadline instead of adding the default timeout before each callback.
|
|
conn = sqlite3.connect(f"file:{src}?mode=ro", uri=True, timeout=0.0)
|
|
backup_conn = sqlite3.connect(str(dst))
|
|
busy_deadline = time.monotonic() + max(0.0, timeout_seconds)
|
|
|
|
def _check_backup_progress(status: int, _remaining: int, _total: int) -> None:
|
|
nonlocal busy_deadline
|
|
now = time.monotonic()
|
|
if status in (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED):
|
|
if now >= busy_deadline:
|
|
raise _SQLiteBackupTimeout(f"database remained locked for {timeout_seconds:g} seconds")
|
|
else:
|
|
busy_deadline = now + max(0.0, timeout_seconds)
|
|
|
|
conn.backup(backup_conn, pages=256, progress=_check_backup_progress, sleep=0.1)
|
|
return True
|
|
except Exception as exc:
|
|
logger.warning("SQLite safe copy failed for %s: %s", src, exc)
|
|
# Windows won't remove the partial destination while SQLite still has it open.
|
|
_close_quietly(backup_conn)
|
|
backup_conn = None
|
|
with suppress(OSError):
|
|
dst.unlink(missing_ok=True)
|
|
return False
|
|
finally:
|
|
_close_quietly(backup_conn)
|
|
_close_quietly(conn)
|
|
|
|
|
|
def is_zeroed_sqlite_file(path: Path, *, probe_bytes: int = 100, force: bool = False) -> bool:
|
|
"""True when *path* looks like the #68474 zeroed-state.db signature.
|
|
|
|
Only regular files qualify: probing a FIFO/device/socket could block indefinitely, so refuse
|
|
before any I/O.
|
|
"""
|
|
try:
|
|
if not path.is_file():
|
|
return False
|
|
except OSError:
|
|
return False
|
|
from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen
|
|
if not force and has_live_connection(path):
|
|
return False
|
|
|
|
head = read_header_bytes_preopen(path, length=max(16, probe_bytes), force=force)
|
|
# Empty or all-NUL header => zeroed; a real header (or unreadable) => not.
|
|
return head is not None and not head.startswith(b"SQLite format 3") and not any(head)
|
|
|
|
|
|
# --- SQLite integrity verification ---
|
|
|
|
_SQLITE_HEADER = b"SQLite format 3\0"
|
|
|
|
# Above this size ``PRAGMA integrity_check`` (which walks every b-tree page — many minutes of
|
|
# pegged CPU on a 30 GB state.db, reading as a hung ``hermes update``) is replaced by the O(1)
|
|
# header + schema probe. Tens-of-GB session databases are normal for heavy users.
|
|
DEFAULT_INTEGRITY_CHECK_MAX_BYTES = 2 << 30 # 2 GiB
|
|
|
|
|
|
def verify_sqlite_integrity(
|
|
path: Path, *, check_header: bool = True, run_pragma: bool = True,
|
|
max_bytes: int = DEFAULT_INTEGRITY_CHECK_MAX_BYTES) -> dict:
|
|
"""Verify that a SQLite database at *path* is intact.
|
|
|
|
Checks, in order: file exists with a minimum size; SQLite header magic; then for files at or
|
|
under ``max_bytes`` a read-only ``PRAGMA integrity_check``, else a cheap structural probe.
|
|
"""
|
|
def _done(message: str, valid: bool = False, size: Optional[int] = None) -> dict:
|
|
return {"valid": valid, "message": message, "size": size}
|
|
try:
|
|
st = path.stat()
|
|
except FileNotFoundError:
|
|
return _done(f"not found: {path}")
|
|
except OSError as exc:
|
|
return _done(f"cannot stat: {exc}")
|
|
size = st.st_size
|
|
if size < 100: # SQLite minimum viable size (header + 1 page)
|
|
return _done(f"too small ({size} bytes) to be a valid SQLite database", size=size)
|
|
if check_header:
|
|
# Byte-level read is refused when a live connection exists (close() would cancel this
|
|
# process's POSIX locks — see hermes_cli.sqlite_safe_read); verification targets
|
|
# snapshots and backup artifacts, which are offline by construction.
|
|
from hermes_cli.sqlite_safe_read import read_header_bytes_preopen
|
|
head = read_header_bytes_preopen(path, length=len(_SQLITE_HEADER))
|
|
if head is None:
|
|
return _done("cannot read header", size=size)
|
|
if head != _SQLITE_HEADER:
|
|
return _done(f"missing SQLite header magic (got {head[:16].hex()!r})", size=size)
|
|
if max_bytes > 0 and size > max_bytes:
|
|
# O(1) structural probe: the header check catches the zeroed signature; opening
|
|
# read-only plus reading sqlite_master + page geometry catches malformed-schema and
|
|
# truncated-header-page classes without walking the data.
|
|
_, exc = _query_ro_sqlite(path, lambda c: (
|
|
c.execute("PRAGMA schema_version").fetchone(),
|
|
c.execute("SELECT count(*) FROM sqlite_master").fetchone()))
|
|
if exc is not None:
|
|
kind = "failed" if isinstance(exc, sqlite3.DatabaseError) else "error"
|
|
return _done(f"schema probe {kind}: {exc}", size=size)
|
|
return _done(
|
|
f"size {size:,} bytes exceeds max_bytes {max_bytes:,}; "
|
|
"skipped PRAGMA integrity_check (header + schema probe passed)",
|
|
valid=True, size=size)
|
|
if run_pragma:
|
|
rows, exc = _query_ro_sqlite(path, lambda c: [str(r[0]) for r in c.execute("PRAGMA integrity_check")])
|
|
if exc is not None:
|
|
kind = "cannot open database" if isinstance(exc, sqlite3.DatabaseError) else "integrity check error"
|
|
return _done(f"{kind}: {exc}", size=size)
|
|
if rows == ["ok"]:
|
|
return _done("integrity check passed", valid=True, size=size)
|
|
return _done(f"integrity check failed: {'; '.join(rows[:5])}", size=size)
|
|
return _done("header check passed", valid=True, size=size)
|
|
|
|
|
|
def _foreign_db_holder_pids(db_path: Path) -> Optional[List[int]]:
|
|
"""PIDs of OTHER processes holding *db_path* or its WAL/SHM open (Linux ``/proc`` scan).
|
|
|
|
Preserves the kernel's ``(deleted)`` suffix so an already-unlinked sidecar generation — the
|
|
#90950 split-brain fingerprint — still counts as held. None off-Linux or when /proc fails.
|
|
"""
|
|
if not sys.platform.startswith("linux"):
|
|
return None
|
|
|
|
def _canonical(path: str) -> str:
|
|
return os.path.normcase(os.path.abspath(path.removesuffix(" (deleted)")))
|
|
|
|
def _holds_watched(fd_dir: str) -> bool:
|
|
for fd in os.listdir(fd_dir):
|
|
with suppress(OSError):
|
|
if _canonical(os.readlink(f"{fd_dir}/{fd}")) in watched:
|
|
return True
|
|
return False
|
|
|
|
canonical_db = _canonical(os.fspath(db_path))
|
|
watched = {canonical_db, canonical_db + "-wal", canonical_db + "-shm"}
|
|
pids: List[int] = []
|
|
try:
|
|
own_pid = os.getpid()
|
|
for pid_str in os.listdir("/proc"):
|
|
if not pid_str.isdigit() or int(pid_str) == own_pid:
|
|
continue
|
|
with suppress(OSError):
|
|
if _holds_watched(f"/proc/{pid_str}/fd"):
|
|
pids.append(int(pid_str))
|
|
except OSError:
|
|
return None
|
|
return pids
|
|
|
|
|
|
def _safe_restore_db(src: Path, dst: Path) -> bool:
|
|
"""Restore snapshot *src* into live *dst* through the backup() API.
|
|
|
|
Writing pages into the live file preserves its inode and WAL state, so any other process
|
|
still holding the DB open (gateway, dashboard, another CLI) sees the restored data instead of
|
|
serving stale pages from a replaced inode. Falls back to unlink+move on failure.
|
|
"""
|
|
try:
|
|
dst_conn = sqlite3.connect(str(dst))
|
|
# Checkpoint first so the backup starts clean rather than writing on top of a deep WAL.
|
|
with suppress(Exception):
|
|
dst_conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
src_conn = sqlite3.connect(f"file:{src}?mode=ro", uri=True)
|
|
try:
|
|
src_conn.backup(dst_conn)
|
|
finally:
|
|
src_conn.close()
|
|
dst_conn.close()
|
|
with suppress(Exception):
|
|
dst.chmod(src.stat().st_mode)
|
|
return True
|
|
except Exception as exc:
|
|
logger.warning("SQLite safe restore failed for %s -> %s: %s", src, dst, exc)
|
|
return _unlink_move_restore_db(src, dst)
|
|
|
|
|
|
def _unlink_move_restore_db(src: Path, dst: Path) -> bool:
|
|
"""Fallback restore: unlink+move. Only safe when no process holds the DB open.
|
|
|
|
Replacing the inode under a live holder is the #90950 corruption class (the holder keeps
|
|
writing through a deleted-inode fd and loses its WAL index), so fail closed rather than
|
|
corrupt. The foreign-pid scan excludes THIS process, but an in-process SessionDB is exactly
|
|
as much of a live holder, so ``offline_file_access`` fails CLOSED when any tracked
|
|
connection to *dst* is live and holds the connection-lifecycle lock across the whole swap.
|
|
"""
|
|
from hermes_cli.sqlite_safe_read import LiveConnectionError, offline_file_access
|
|
try:
|
|
holders = _foreign_db_holder_pids(dst)
|
|
if holders:
|
|
logger.error(
|
|
"Refusing unlink+move restore of %s: process(es) %s still "
|
|
"hold the database or its WAL open. Stop them and retry.",
|
|
dst, holders)
|
|
return False
|
|
with offline_file_access(dst, what="unlink+move restore of"):
|
|
tmp = dst.parent / f".{dst.name}.snap_restore"
|
|
shutil.copy2(src, tmp)
|
|
dst.unlink(missing_ok=True)
|
|
# The snapshot is a checkpointed backup() image that owns no WAL, so any -wal/-shm
|
|
# left here belongs to the database just unlinked (an ungracefully killed gateway
|
|
# leaves them behind — exactly when a restore runs). SQLite would replay that foreign
|
|
# WAL over the restored file and come up "malformed" or resurrect post-snapshot rows.
|
|
for _sidecar_suffix in ("-wal", "-shm", "-journal"):
|
|
dst.with_name(dst.name + _sidecar_suffix).unlink(missing_ok=True)
|
|
shutil.move(str(tmp), str(dst))
|
|
return True
|
|
except LiveConnectionError as exc2:
|
|
logger.error(
|
|
"Refusing unlink+move restore of %s: %s Close the in-process "
|
|
"database handles (or restart Hermes) and retry.",
|
|
dst, exc2)
|
|
return False
|
|
except Exception as exc2:
|
|
logger.error("Fallback restore also failed for %s -> %s: %s", src, dst, exc2)
|
|
return False
|
|
|
|
|
|
def _zip_sqlite_snapshot(zf: zipfile.ZipFile, abs_path: Path, rel_path: Path, out_path: Path) -> Optional[int]:
|
|
"""Add a WAL-safe snapshot of *abs_path* to *zf*; return its byte size, or None on failure.
|
|
|
|
Staged beside the output zip (same filesystem): the system /tmp may be a small tmpfs that
|
|
cannot hold large databases, causing silent backup incompleteness.
|
|
"""
|
|
with tempfile.NamedTemporaryFile(suffix=".db", delete=False, dir=str(out_path.parent)) as tmp:
|
|
tmp_db = Path(tmp.name)
|
|
try:
|
|
if not _safe_copy_db(abs_path, tmp_db):
|
|
return None
|
|
zf.write(tmp_db, arcname=str(rel_path))
|
|
return tmp_db.stat().st_size
|
|
finally:
|
|
tmp_db.unlink(missing_ok=True)
|
|
|
|
|
|
def _write_zip_entries(
|
|
zf: zipfile.ZipFile, files_to_add: List[Tuple[Path, Path]], out_path: Path,
|
|
*, on_db_failure, on_error, on_progress, track_bytes: bool) -> int:
|
|
"""Add every ``(abs_path, rel_path)`` to *zf*, WAL-safe for ``*.db``; return bytes archived.
|
|
|
|
``on_db_failure(rel_path)`` runs when a SQLite snapshot fails (it may raise to abort);
|
|
``on_error(rel_path, exc)`` records a per-file read failure; ``on_progress(index)`` fires
|
|
every 500 files. ``track_bytes`` stats each archived plain file for the size total.
|
|
"""
|
|
total_bytes = 0
|
|
for i, (abs_path, rel_path) in enumerate(files_to_add, 1):
|
|
try:
|
|
if abs_path.suffix == ".db":
|
|
size = _zip_sqlite_snapshot(zf, abs_path, rel_path, out_path)
|
|
if size is None:
|
|
on_db_failure(rel_path)
|
|
continue
|
|
total_bytes += size
|
|
else:
|
|
zf.write(abs_path, arcname=str(rel_path))
|
|
if track_bytes:
|
|
total_bytes += abs_path.stat().st_size
|
|
except (PermissionError, OSError, ValueError) as exc:
|
|
on_error(rel_path, exc)
|
|
continue
|
|
if i % 500 == 0:
|
|
on_progress(i)
|
|
return total_bytes
|
|
|
|
|
|
def _print_capped(header: str, lines: List[str], indent: str) -> None:
|
|
"""Print *header*, then at most 10 of *lines* (each prefixed by *indent*) and a "... and N more" tail."""
|
|
print(header)
|
|
for line in lines[:10]:
|
|
print(f"{indent}{line}")
|
|
if len(lines) > 10:
|
|
print(f"{indent}... and {len(lines) - 10} more")
|
|
|
|
|
|
# --- Backup ---
|
|
|
|
def _resolve_backup_output_path(output: Optional[str]) -> Path:
|
|
"""Turn ``--output`` (file, directory, or None) into a ``.zip`` path whose parent exists.
|
|
|
|
An unwritable output path gives a clean one-line error, not a raw traceback.
|
|
"""
|
|
out_path = None
|
|
default_name = f"hermes-backup-{datetime.now().strftime('%Y-%m-%d-%H%M%S')}.zip"
|
|
try:
|
|
if output:
|
|
out_path = Path(output).expanduser().resolve()
|
|
if out_path.is_dir():
|
|
out_path = out_path / default_name
|
|
else:
|
|
out_path = Path.home() / default_name
|
|
if out_path.suffix.lower() != ".zip":
|
|
out_path = out_path.with_suffix(out_path.suffix + ".zip")
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
except OSError as exc:
|
|
print(f"Error: cannot write backup to {output or out_path}: {exc}")
|
|
raise SystemExit(1) from exc
|
|
return out_path
|
|
|
|
|
|
def _collect_external_entries() -> tuple[list[tuple[Path, str]], list[str]]:
|
|
"""``([(abs_path, arcname)], [skipped])`` for the active memory provider's external state.
|
|
|
|
Staged under the reserved ``_external/`` arc prefix, encoded relative to the user's home.
|
|
Only paths under home are captured (security + portability); others are returned as skipped.
|
|
"""
|
|
home_dir = Path.home().resolve()
|
|
external_to_add: list[tuple[Path, str]] = []
|
|
skipped_external: list[str] = []
|
|
for base in _collect_memory_provider_external_paths():
|
|
try:
|
|
base.resolve().relative_to(home_dir)
|
|
except (ValueError, OSError):
|
|
skipped_external.append(str(base))
|
|
continue
|
|
for fpath in _iter_external_files(base):
|
|
try:
|
|
rel_to_home = fpath.resolve().relative_to(home_dir)
|
|
except (ValueError, OSError):
|
|
continue
|
|
external_to_add.append((fpath, _EXTERNAL_PREFIX + rel_to_home.as_posix()))
|
|
return external_to_add, skipped_external
|
|
|
|
|
|
def run_backup(args) -> None:
|
|
"""Create a zip backup of the Hermes home directory."""
|
|
hermes_root = get_default_hermes_root()
|
|
|
|
if not hermes_root.is_dir():
|
|
print(f"Error: Hermes home directory not found at {hermes_root}")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
with _backup_operation_lock(hermes_root):
|
|
_run_backup_locked(args, hermes_root)
|
|
except BackupInProgressError as exc:
|
|
print(f"Error: {exc}")
|
|
raise SystemExit(2) from exc
|
|
|
|
|
|
def _run_backup_locked(args, hermes_root: Path) -> None:
|
|
"""Write a full backup while the cross-process backup slot is held."""
|
|
out_path = _resolve_backup_output_path(args.output)
|
|
scan_started = time.monotonic()
|
|
logger.info("backup phase=scan status=started")
|
|
print(f"Scanning {display_hermes_home()} ...")
|
|
skipped_dirs: set = set()
|
|
files_to_add: list[tuple[Path, Path]] = list(_iter_backup_files(hermes_root, out_path, skipped_dirs))
|
|
external_to_add, skipped_external = _collect_external_entries()
|
|
if not files_to_add and not external_to_add:
|
|
logger.info("backup phase=scan status=empty duration_ms=%.1f", (time.monotonic() - scan_started) * 1000)
|
|
print("No files to back up.")
|
|
return
|
|
|
|
file_count = len(files_to_add) + len(external_to_add)
|
|
logger.info("backup phase=scan status=complete duration_ms=%.1f files=%d",
|
|
(time.monotonic() - scan_started) * 1000, file_count)
|
|
logger.info("backup phase=archive status=started files=%d", file_count)
|
|
print(f"Backing up {file_count} files ...")
|
|
errors = []
|
|
t0 = time.monotonic()
|
|
|
|
def _progress(i: int) -> None:
|
|
print(f" {i}/{file_count} files ...")
|
|
logger.info("backup phase=archive status=progress completed=%d total=%d", i, file_count)
|
|
|
|
with _atomic_output_path(out_path) as archive_path, zipfile.ZipFile(
|
|
archive_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zf:
|
|
total_bytes = _write_zip_entries(
|
|
zf, files_to_add, out_path, on_progress=_progress, track_bytes=True,
|
|
on_db_failure=lambda rel: errors.append(f"{rel}: SQLite safe copy failed"),
|
|
on_error=lambda rel, exc: errors.append(f"{rel}: {exc}"))
|
|
# External memory-provider state never includes ``.db`` files in practice, so a
|
|
# straight zf.write is fine.
|
|
for abs_path, arcname in external_to_add:
|
|
try:
|
|
zf.write(abs_path, arcname=arcname)
|
|
total_bytes += abs_path.stat().st_size
|
|
except (PermissionError, OSError, ValueError) as exc:
|
|
errors.append(f"{arcname}: {exc}")
|
|
elapsed = time.monotonic() - t0
|
|
zip_size = out_path.stat().st_size
|
|
logger.info("backup phase=archive status=complete duration_ms=%.1f files=%d errors=%d bytes=%d",
|
|
elapsed * 1000, file_count, len(errors), zip_size)
|
|
print(f"\nBackup {'incomplete' if errors else 'complete'}: {out_path}\n"
|
|
f" Files: {file_count}\n"
|
|
f" Original: {_format_size(total_bytes)}\n"
|
|
f" Compressed: {_format_size(zip_size)}\n"
|
|
f" Time: {elapsed:.1f}s")
|
|
if external_to_add:
|
|
print(f"\n Included {len(external_to_add)} memory-provider file(s) stored outside {display_hermes_home()}.")
|
|
if skipped_external:
|
|
print(f"\n Skipped {len(skipped_external)} memory-provider path(s) outside your home directory "
|
|
"(not portable):\n" + "\n".join(f" {p}" for p in sorted(skipped_external)[:10]))
|
|
if skipped_dirs:
|
|
print("\n Excluded directories:\n" + "\n".join(f" {d}/" for d in sorted(skipped_dirs)))
|
|
if errors:
|
|
_print_capped(f"\n Warnings ({len(errors)} files skipped):", errors, " ")
|
|
else:
|
|
print(f"\nRestore with: hermes import {out_path.name}")
|
|
|
|
|
|
# --- Import ---
|
|
|
|
def _validate_backup_zip(zf: zipfile.ZipFile) -> tuple[bool, str]:
|
|
"""Check that a zip looks like a Hermes backup."""
|
|
names = zf.namelist()
|
|
if not names:
|
|
return False, "zip archive is empty"
|
|
# Telltale files a hermes home has — at the root or one level deep (zipped directory).
|
|
if not any(Path(n).name in {"config.yaml", ".env", "state.db"} for n in names):
|
|
return False, "zip does not appear to be a Hermes backup (no config.yaml, .env, or state databases found)"
|
|
return True, ""
|
|
|
|
|
|
def _detect_prefix(zf: zipfile.ZipFile) -> str:
|
|
"""Detect if the zip has a common directory prefix wrapping all entries."""
|
|
names = [n for n in zf.namelist() if not n.endswith("/")]
|
|
if not names:
|
|
return ""
|
|
first_parts = {Path(n).parts[0] for n in names if len(Path(n).parts) > 1}
|
|
if len(first_parts) == 1 and first_parts <= {".hermes", "hermes"}:
|
|
return first_parts.pop() + "/"
|
|
return ""
|
|
|
|
|
|
def _default_new_file_mode() -> Optional[int]:
|
|
"""The mode ``open(path, "wb")`` gives a file it has to create.
|
|
|
|
``tempfile.mkstemp`` always creates at 0600, so staging an import through a temp file would
|
|
tighten every *newly created* file to owner-only — the hazard ``utils._restore_file_mode``
|
|
documents for Docker/NAS volume mounts that rely on broader permissions.
|
|
"""
|
|
try:
|
|
current = os.umask(0o077)
|
|
os.umask(current)
|
|
except OSError:
|
|
return None
|
|
return 0o666 & ~current
|
|
|
|
|
|
def _extract_member_atomically(
|
|
zf: zipfile.ZipFile, member: str, target: Path, new_file_mode: Optional[int] = None) -> None:
|
|
"""Restore one zip member onto *target* with no truncation window.
|
|
|
|
``open(target, "wb")`` would truncate the user's file before any replacement bytes exist.
|
|
``atomic_replace`` (not bare ``os.replace``) resolves a symlinked target first, so a
|
|
dotfiles-linked ``config.yaml`` keeps the link (GitHub #16743), and falls back to
|
|
copy/fsync/unlink on ``EXDEV``/``EBUSY`` for cross-device and bind-mount installs.
|
|
"""
|
|
# ``_preserve_file_mode`` is None when the target does not exist, in which case the
|
|
# umask-derived create-mode applies (same shape as ``atomic_yaml_write``'s ``create_mode``).
|
|
mode = _preserve_file_mode(target)
|
|
owner = _preserve_file_owner(target)
|
|
if mode is None:
|
|
mode = new_file_mode
|
|
else:
|
|
# Deliberately NOT a faithful copy: setuid/setgid are dropped. The bytes replacing this
|
|
# file come from the archive, so carrying elevated bits across would hand whoever produced
|
|
# the zip the identity an existing setuid file runs as — and the ``_external/`` branch
|
|
# publishes members anywhere under ``$HOME``. The sticky bit is inert on a regular file.
|
|
mode &= ~(stat.S_ISUID | stat.S_ISGID)
|
|
|
|
# Truncate the stem: mkstemp adds ~16 chars and a member near NAME_MAX would otherwise fail.
|
|
fd, tmp_name = tempfile.mkstemp(dir=str(target.parent), prefix=f".{target.name[:80]}.", suffix=".partial")
|
|
try:
|
|
with os.fdopen(fd, "wb") as dst:
|
|
if mode is not None:
|
|
# Apply the mode BEFORE the replace so the target never transits through
|
|
# mkstemp's 0600 and the EXDEV/EBUSY ``copystat`` fallback copies the intended
|
|
# bits. fchmod is Unix-only; Windows takes the path-based chmod.
|
|
if hasattr(os, "fchmod"):
|
|
os.fchmod(dst.fileno(), mode)
|
|
else:
|
|
os.chmod(tmp_name, mode)
|
|
# Stream: a multi-gigabyte state.db member must not be held in memory in one piece.
|
|
with zf.open(member) as src:
|
|
shutil.copyfileobj(src, dst)
|
|
dst.flush()
|
|
os.fsync(dst.fileno())
|
|
real_path = Path(atomic_replace(tmp_name, target))
|
|
# Owner first, mode second (as ``atomic_yaml_write``): chown drops setuid/setgid, and
|
|
# ``mode`` no longer carries them, so neither step can re-elevate the restored file.
|
|
_restore_file_owner(real_path, owner)
|
|
_restore_file_mode(real_path, mode)
|
|
except BaseException:
|
|
with suppress(OSError):
|
|
os.unlink(tmp_name)
|
|
raise
|
|
|
|
|
|
def _confirm_import_overwrite(hermes_root: Path) -> bool:
|
|
"""Prompt before importing over an existing installation; True when import may proceed."""
|
|
if not any((hermes_root / m).exists() for m in ("config.yaml", ".env")):
|
|
return True
|
|
print("\nWarning: Target directory already has Hermes configuration.\n"
|
|
"Importing will overwrite existing files with backup contents.\n")
|
|
try:
|
|
answer = input("Continue? [y/N] ").strip().lower()
|
|
except (EOFError, KeyboardInterrupt):
|
|
print("\nAborted.")
|
|
sys.exit(1)
|
|
if answer not in {"y", "yes"}:
|
|
print("Aborted.")
|
|
return False
|
|
return True
|
|
|
|
|
|
def _import_members(
|
|
zf: zipfile.ZipFile, members: List[str], prefix: str, hermes_root: Path, file_count: int
|
|
) -> tuple[int, int, list[str], list[str]]:
|
|
"""Publish every member; return ``(restored, restored_external, errors, skipped_runtime)``."""
|
|
errors: list[str] = []
|
|
skipped_runtime: list[str] = []
|
|
restored = restored_external = 0
|
|
home_dir = Path.home().resolve()
|
|
# Resolved once: every member is published via a temp file, and mkstemp would otherwise
|
|
# create newly restored files as 0600.
|
|
new_file_mode = _default_new_file_mode()
|
|
|
|
for member in members:
|
|
# ``_external/`` members restore to their original home-relative location (e.g.
|
|
# ~/.honcho/config.json), NOT under HERMES_HOME. Provider configs commonly hold
|
|
# credentials, so they are tightened to 0600 best-effort.
|
|
external = member.startswith(_EXTERNAL_PREFIX)
|
|
if external:
|
|
rel = member[len(_EXTERNAL_PREFIX):]
|
|
target = home_dir / rel
|
|
root = home_dir
|
|
tighten = target.suffix in {".json", ".env", ".conf"} or target.name in _SECRET_FILE_NAMES
|
|
else:
|
|
rel = member[len(prefix):] if prefix and member.startswith(prefix) else member
|
|
# Never overwrite volatile runtime state namespaced to the source machine — see
|
|
# ``_IMPORT_SKIP_NAMES``. Basename match covers root and named profiles.
|
|
if rel and Path(rel).name in _IMPORT_SKIP_NAMES:
|
|
skipped_runtime.append(rel)
|
|
continue
|
|
target = hermes_root / rel
|
|
root = hermes_root.resolve()
|
|
tighten = target.name in _SECRET_FILE_NAMES
|
|
if not rel:
|
|
continue
|
|
|
|
label = member if external else rel
|
|
if not _is_within(target, root):
|
|
errors.append(f"{label}: path traversal blocked")
|
|
else:
|
|
try:
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
_extract_member_atomically(zf, member, target, new_file_mode)
|
|
if tighten:
|
|
try:
|
|
os.chmod(target, 0o600)
|
|
except OSError:
|
|
if not external: # external provider configs are tightened best-effort only
|
|
raise
|
|
restored += 1
|
|
restored_external += external
|
|
except (PermissionError, OSError) as exc:
|
|
errors.append(f"{label}: {exc}")
|
|
|
|
if restored % 500 == 0:
|
|
print(f" {restored}/{file_count} files ...")
|
|
|
|
return restored, restored_external, errors, skipped_runtime
|
|
|
|
|
|
def run_import(args) -> None:
|
|
"""Restore a Hermes backup from a zip file."""
|
|
zip_path = Path(args.zipfile).expanduser().resolve()
|
|
if not zip_path.is_file():
|
|
print(f"Error: File not found: {zip_path}")
|
|
sys.exit(1)
|
|
if not zipfile.is_zipfile(zip_path):
|
|
print(f"Error: Not a valid zip file: {zip_path}")
|
|
sys.exit(1)
|
|
# The restore target must be the home the command operates under — the same path printed as
|
|
# "Target:". ``get_default_hermes_root()`` would map a profile home back to <root> and
|
|
# silently retarget the restore at the live root while the profile directory stays empty.
|
|
hermes_root = get_hermes_home()
|
|
with zipfile.ZipFile(zip_path, "r") as zf:
|
|
ok, reason = _validate_backup_zip(zf)
|
|
if not ok:
|
|
print(f"Error: {reason}")
|
|
sys.exit(1)
|
|
prefix = _detect_prefix(zf)
|
|
members = [n for n in zf.namelist() if not n.endswith("/")]
|
|
file_count = len(members)
|
|
print(f"Backup contains {file_count} files\nTarget: {display_hermes_home()}")
|
|
if prefix:
|
|
print(f"Detected archive prefix: {prefix!r} (will be stripped)")
|
|
if not args.force and not _confirm_import_overwrite(hermes_root):
|
|
return
|
|
print(f"\nImporting {file_count} files ...")
|
|
hermes_root.mkdir(parents=True, exist_ok=True)
|
|
t0 = time.monotonic()
|
|
restored, restored_external, errors, skipped_runtime = _import_members(
|
|
zf, members, prefix, hermes_root, file_count)
|
|
elapsed = time.monotonic() - t0
|
|
print(f"\nImport complete: {restored} files restored in {elapsed:.1f}s\n Target: {display_hermes_home()}")
|
|
if restored_external:
|
|
print(f"\n Restored {restored_external} memory-provider file(s) to "
|
|
f"their original location(s) outside {display_hermes_home()}.")
|
|
if errors:
|
|
_print_capped(f"\n Warnings ({len(errors)} files skipped):", errors, " ")
|
|
if skipped_runtime:
|
|
_print_capped(f"\n Preserved {len(skipped_runtime)} runtime state "
|
|
f"file(s) (kept this machine's, not the backup's):", sorted(skipped_runtime), " ")
|
|
restored_profiles = _restore_profile_wrappers(hermes_root)
|
|
print()
|
|
if not (hermes_root / "hermes-agent").is_dir():
|
|
print("Note: The hermes-agent codebase was not included in the backup.\n"
|
|
" If this is a fresh install, run: hermes update")
|
|
if restored_profiles:
|
|
print("\nTo re-enable gateway services for profiles:")
|
|
for pname in restored_profiles:
|
|
print(f" hermes -p {pname} gateway install")
|
|
_revive_gateway_after_import(hermes_root)
|
|
print("Done. Your Hermes configuration has been restored.")
|
|
|
|
|
|
def _restore_profile_wrappers(hermes_root: Path) -> List[str]:
|
|
"""Re-create shell wrapper scripts for restored named profiles; return the profile names seen."""
|
|
profiles_dir = hermes_root / "profiles"
|
|
restored_profiles: list[tuple[str, bool]] = []
|
|
if not profiles_dir.is_dir():
|
|
return []
|
|
try:
|
|
from hermes_cli.profiles import (
|
|
create_wrapper_script, check_alias_collision, _is_wrapper_dir_in_path, _get_wrapper_dir)
|
|
for entry in sorted(profiles_dir.iterdir()):
|
|
# Only create wrappers for directories with config
|
|
if not entry.is_dir() or not any((entry / m).exists() for m in ("config.yaml", ".env")):
|
|
continue
|
|
profile_name = entry.name
|
|
collision = check_alias_collision(profile_name)
|
|
if collision:
|
|
print(f" Skipped alias '{profile_name}': {collision}")
|
|
restored_profiles.append(
|
|
(profile_name, not collision and create_wrapper_script(profile_name) is not None))
|
|
if restored_profiles:
|
|
created = [n for n, ok in restored_profiles if ok]
|
|
skipped = [n for n, ok in restored_profiles if not ok]
|
|
if created:
|
|
print(f"\n Profile aliases restored: {', '.join(created)}")
|
|
if skipped:
|
|
print(f" Profile aliases skipped: {', '.join(skipped)}")
|
|
if not _is_wrapper_dir_in_path():
|
|
print(f"\n Note: {_get_wrapper_dir()} is not in your PATH.\n"
|
|
" Add to your shell config (~/.bashrc or ~/.zshrc):\n"
|
|
' export PATH="$HOME/.local/bin:$PATH"')
|
|
except ImportError:
|
|
# hermes_cli.profiles might not be available (fresh install)
|
|
if any(profiles_dir.iterdir()):
|
|
print("\n Profiles detected but aliases could not be created.\n"
|
|
" Run: hermes profile list (after installing hermes)")
|
|
return [n for n, _ in restored_profiles]
|
|
|
|
|
|
def _revive_gateway_after_import(hermes_root: Path) -> None:
|
|
"""Install/start the gateway service after a restore, best-effort and prompt-free.
|
|
|
|
Bot tokens and cron jobs in the backup are inert without a gateway; a platform-less gateway
|
|
is a supported mode, so this is safe for any backup. Failures print a manual fallback, never
|
|
fail the import. A restore into a sandbox or profile home must not silently install a second
|
|
gateway on the default service name (it would shadow the machine's primary install), so the
|
|
service is only revived when the restore landed in the default home or no other install
|
|
exists.
|
|
"""
|
|
native_default = _get_platform_default_hermes_home()
|
|
if hermes_root != native_default and any(
|
|
(native_default / marker).exists() for marker in ("config.yaml", ".env", "state.db")):
|
|
print("\nRestored into a non-default home; leaving the gateway service alone to avoid clashing "
|
|
f"with the install at {native_default}.\n"
|
|
"To start a gateway for this home, run: hermes gateway install")
|
|
return
|
|
try:
|
|
from hermes_cli.gateway import ensure_gateway_service, _is_service_running
|
|
if not _is_service_running():
|
|
print()
|
|
ensure_gateway_service(context="import")
|
|
except Exception:
|
|
print("\nStart the gateway to activate cron jobs and messaging:\n hermes gateway install")
|
|
|
|
|
|
# --- Quick state snapshots (used by /snapshot slash command and hermes backup --quick) ---
|
|
|
|
# Critical state files (relative to HERMES_HOME) for quick snapshots; everything else is
|
|
# regeneratable or managed separately (skills, repo, sessions/). Entries may be files OR
|
|
# directories (captured recursively); missing entries are silently skipped. Pairing data lives
|
|
# in platform JSON blobs outside state.db, so it is listed explicitly — ``hermes update``
|
|
# snapshots this set before pulling so approved-user lists are recoverable (#15733).
|
|
_QUICK_STATE_FILES = (
|
|
"state.db",
|
|
"config.yaml",
|
|
".env",
|
|
"auth.json",
|
|
"cron/jobs.json",
|
|
"cron/executions.db",
|
|
"gateway_state.json",
|
|
"channel_directory.json",
|
|
"channel_aliases.json",
|
|
"processes.json",
|
|
"gateway/discord_message_recovery.db", # Discord reconnect replay ledger
|
|
# Per-profile user-created stores outside the git checkout, destroyed if the update flow
|
|
# replaces the file and the post-update schema-init re-creates an empty one (#52889). On
|
|
# non-root profiles the real path is outside HERMES_HOME and the entry is silently skipped.
|
|
"projects.db", # per-profile project store
|
|
"response_store.db", # gateway conversation history / tool payloads
|
|
"memory_store.db", # holographic memory facts/entities
|
|
"verification_evidence.db", # agent verification audit trail
|
|
"kanban.db", # default board (back-compat <root>/kanban.db)
|
|
"kanban/boards", # non-default boards (workspaces/ + attachments/ skipped as regenerable)
|
|
# Pairing stores (generic + per-platform JSONs outside state.db)
|
|
"pairing", # legacy location (gateway/pairing.py)
|
|
"platforms/pairing", # new location (gateway/pairing.py)
|
|
"feishu_comment_pairing.json", # Feishu comment subscription pairings
|
|
)
|
|
|
|
_QUICK_DEFAULT_KEEP = 20
|
|
|
|
|
|
def _quick_snapshot_root(hermes_home: Optional[Path] = None) -> Path:
|
|
home = hermes_home or get_hermes_home()
|
|
return home / _QUICK_SNAPSHOTS_DIR
|
|
|
|
|
|
def create_quick_snapshot(
|
|
label: Optional[str] = None, hermes_home: Optional[Path] = None, keep: Optional[int] = None,
|
|
max_file_size: Optional[int] = None) -> Optional[str]:
|
|
"""Create one atomic quick snapshot while holding the shared backup slot."""
|
|
home = hermes_home or get_hermes_home()
|
|
with _backup_operation_lock(home):
|
|
return _create_quick_snapshot_locked(label, home, keep, max_file_size)
|
|
|
|
|
|
def _quick_snapshot_candidates(home: Path):
|
|
"""Yield ``(src, rel_posix, in_dir)`` for every regular file a quick snapshot captures.
|
|
|
|
Directory entries are walked so restore treats every file uniformly; heavy, regenerable
|
|
per-board subtrees (scratch workspaces and task attachments) are skipped.
|
|
"""
|
|
for rel in _QUICK_STATE_FILES:
|
|
src = home / rel
|
|
if src.is_dir():
|
|
for sub in filter(Path.is_file, src.rglob("*")):
|
|
sub_rel = sub.relative_to(home).as_posix()
|
|
if "/workspaces/" in f"/{sub_rel}/" or "/attachments/" in f"/{sub_rel}/":
|
|
continue
|
|
yield sub, sub_rel, True
|
|
elif src.is_file():
|
|
yield src, rel, False
|
|
|
|
|
|
def _copy_quick_snapshot_files(
|
|
home: Path, staging_dir: Path, max_file_size: Optional[int]
|
|
) -> tuple[Dict[str, int], list[str], list[str]]:
|
|
"""Copy every quick-snapshot candidate into *staging_dir*.
|
|
|
|
Returns ``(manifest, failed_dbs, oversized_skipped)``: ``manifest`` maps rel_path -> size;
|
|
``failed_dbs`` lists present ``*.db`` that could not be snapshotted; ``oversized_skipped``
|
|
lists DB files skipped for size (#68805) — both are snapshot incompleteness, so the caller
|
|
must suppress pruning to preserve the older snapshot that may hold the only recoverable DB.
|
|
"""
|
|
manifest: Dict[str, int] = {}
|
|
failed_dbs: list[str] = []
|
|
oversized_skipped: list[str] = []
|
|
for src, rel, in_dir in _quick_snapshot_candidates(home):
|
|
if max_file_size is not None:
|
|
try:
|
|
size = src.stat().st_size
|
|
except OSError:
|
|
size = None
|
|
if size is not None and size > max_file_size:
|
|
print(f" ⚠ Snapshot: skipping {rel} "
|
|
f"({_format_size(size)} exceeds {_format_size(max_file_size)} limit)")
|
|
logger.warning("Quick snapshot skipped %s: %d bytes exceeds %d byte limit", rel, size, max_file_size)
|
|
if src.suffix == ".db":
|
|
oversized_skipped.append(rel)
|
|
continue
|
|
dst = staging_dir / rel
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
# SQLite DBs go through the WAL-safe backup() path (the gateway may hold the WAL open).
|
|
if src.suffix == ".db":
|
|
if not _safe_copy_db(src, dst):
|
|
failed_dbs.append(rel)
|
|
print(f" ⚠ Snapshot: SQLite safe copy FAILED for {rel} — file may be locked or corrupted")
|
|
if is_zeroed_sqlite_file(src):
|
|
nuls = " of NULs?" if in_dir else ""
|
|
print(f" ⚠ Snapshot: {rel} looks ZEROED "
|
|
f"(no SQLite header; {src.stat().st_size} bytes{nuls})")
|
|
continue
|
|
else:
|
|
shutil.copy2(src, dst)
|
|
manifest[rel] = dst.stat().st_size
|
|
except (OSError, PermissionError) as exc:
|
|
logger.warning("Could not snapshot %s: %s", rel, exc)
|
|
return manifest, failed_dbs, oversized_skipped
|
|
|
|
|
|
def _create_quick_snapshot_locked(
|
|
label: Optional[str], hermes_home: Optional[Path], keep: Optional[int], max_file_size: Optional[int]
|
|
) -> Optional[str]:
|
|
"""Copy the quick-snapshot set to a timestamped dir under state-snapshots/ and prune old ones.
|
|
|
|
``max_file_size`` skips (with a warning) files above that many bytes; the pre-update snapshot
|
|
uses it so a multi-GB ``state.db`` can never stall ``hermes update`` while the small
|
|
pairing/cron/config files are always captured. ``None`` copies everything.
|
|
"""
|
|
home = hermes_home or get_hermes_home()
|
|
root = _quick_snapshot_root(home)
|
|
ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
|
|
base_snap_id = f"{ts}-{label}" if label else ts
|
|
snap_id, suffix = base_snap_id, 2
|
|
while (root / snap_id).exists():
|
|
snap_id = f"{base_snap_id}-{suffix}"
|
|
suffix += 1
|
|
snap_dir = root / snap_id
|
|
staging_dir = root / f".{snap_id}.{os.getpid()}.partial"
|
|
shutil.rmtree(staging_dir, ignore_errors=True)
|
|
staging_dir.mkdir(parents=True, exist_ok=False)
|
|
logger.info("quick snapshot phase=copy status=started id=%s", snap_id)
|
|
manifest, failed_dbs, oversized_skipped = _copy_quick_snapshot_files(home, staging_dir, max_file_size)
|
|
if failed_dbs:
|
|
# The update path used to log-and-continue with exit 0, so a missing state.db backup
|
|
# looked like a successful pre-update snapshot (#68474). Surface it on stdout.
|
|
print(f" ⚠ CRITICAL: could not snapshot DB file(s): {', '.join(failed_dbs)}\n"
|
|
f" ⚠ If sessions disappear after update, check {root} and run: hermes snapshot list")
|
|
logger.error("Quick snapshot failed to capture DB file(s): %s", ", ".join(failed_dbs))
|
|
if not manifest:
|
|
shutil.rmtree(staging_dir, ignore_errors=True)
|
|
if failed_dbs:
|
|
# Distinguish "nothing to snapshot" from "state.db present but unreadable"
|
|
print(f" ⚠ Snapshot aborted: no files captured (failed DBs: {', '.join(failed_dbs)})")
|
|
return None
|
|
meta = {
|
|
"id": snap_id, "timestamp": ts, "label": label, "file_count": len(manifest),
|
|
"total_size": sum(manifest.values()), "files": manifest,
|
|
"failed_dbs": failed_dbs, "oversized_skipped": oversized_skipped,
|
|
}
|
|
with open(staging_dir / "manifest.json", "w", encoding="utf-8") as f:
|
|
json.dump(meta, f, indent=2)
|
|
os.replace(staging_dir, snap_dir)
|
|
# Auto-prune; callers with high-churn safety snapshots (pre-update) pass a smaller keep so
|
|
# large state.db copies don't accumulate. Skip pruning when a present DB failed to capture
|
|
# OR was skipped for size (#68805): the snapshot is incomplete and the older one may hold
|
|
# the only recoverable database.
|
|
if not (failed_dbs or oversized_skipped):
|
|
_prune_oldest(_snapshot_dirs(root), _QUICK_DEFAULT_KEEP if keep is None else keep, shutil.rmtree, "snapshot")
|
|
else:
|
|
if oversized_skipped:
|
|
print(" ⚠ Skipping snapshot prune: DB file(s) skipped for size: " + ", ".join(oversized_skipped))
|
|
logger.warning("Quick snapshot skipped oversized DB file(s): %s", ", ".join(oversized_skipped))
|
|
logger.warning(
|
|
"Skipping snapshot prune because %d DB(s) failed to capture and/or %d were oversized "
|
|
"— preserving older snapshots as recovery source", len(failed_dbs), len(oversized_skipped))
|
|
logger.info("quick snapshot phase=copy status=complete id=%s files=%d bytes=%d",
|
|
snap_id, len(manifest), sum(manifest.values()))
|
|
return snap_id
|
|
|
|
|
|
def _newest_first(root: Path, keep_entry) -> List[Path]:
|
|
"""Entries of *root* passing ``keep_entry``, newest (by name) first; ``[]`` when *root* is missing."""
|
|
if not root.exists():
|
|
return []
|
|
return sorted(filter(keep_entry, root.iterdir()), key=lambda p: p.name, reverse=True)
|
|
|
|
|
|
def _snapshot_dirs(root: Path) -> List[Path]:
|
|
"""Published snapshot directories under *root*, newest first."""
|
|
return _newest_first(
|
|
root, lambda d: d.is_dir() and not d.name.startswith(".") and not d.name.endswith(".partial"))
|
|
|
|
|
|
def list_quick_snapshots(limit: int = 20, hermes_home: Optional[Path] = None) -> List[Dict[str, Any]]:
|
|
"""List existing quick state snapshots, most recent first."""
|
|
results = []
|
|
for d in _snapshot_dirs(_quick_snapshot_root(hermes_home)):
|
|
manifest_path = d / "manifest.json"
|
|
if manifest_path.exists():
|
|
try:
|
|
with open(manifest_path, encoding="utf-8") as f:
|
|
results.append(json.load(f))
|
|
except (json.JSONDecodeError, OSError):
|
|
results.append({"id": d.name, "file_count": 0, "total_size": 0})
|
|
if len(results) >= limit:
|
|
break
|
|
return results
|
|
|
|
|
|
def restore_quick_snapshot(snapshot_id: str, hermes_home: Optional[Path] = None) -> bool:
|
|
"""Restore state from a quick snapshot."""
|
|
home = hermes_home or get_hermes_home()
|
|
root = _quick_snapshot_root(home)
|
|
# Reject ids with separators or traversal so ``root / snapshot_id`` stays inside root.
|
|
if not snapshot_id or "/" in snapshot_id or "\\" in snapshot_id or snapshot_id in (".", ".."):
|
|
logger.error("Invalid snapshot_id: %s", snapshot_id)
|
|
return False
|
|
snap_dir = root / snapshot_id
|
|
if not _is_within(snap_dir, root.resolve()): # handles symlinks etc.
|
|
logger.error("Snapshot path traversal blocked for id: %s", snapshot_id)
|
|
return False
|
|
manifest_path = snap_dir / "manifest.json"
|
|
if not snap_dir.is_dir() or not manifest_path.exists():
|
|
return False
|
|
with open(manifest_path, encoding="utf-8") as f:
|
|
meta = json.load(f)
|
|
snap_res, home_res = snap_dir.resolve(), home.resolve()
|
|
restored = 0
|
|
for rel in meta.get("files", {}):
|
|
src = snap_dir / rel
|
|
dst = home / rel
|
|
if not (_is_within(src, snap_res) and _is_within(dst, home_res)):
|
|
logger.error("Manifest path traversal blocked: %s", rel)
|
|
continue
|
|
if not src.exists():
|
|
continue
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
if dst.suffix == ".db":
|
|
# Through the backup API so live connections see the restored data instead of
|
|
# stale pages from a replaced inode (#65942).
|
|
_safe_restore_db(src, dst)
|
|
else:
|
|
shutil.copy2(src, dst)
|
|
restored += 1
|
|
except (OSError, PermissionError) as exc:
|
|
logger.error("Failed to restore %s: %s", rel, exc)
|
|
logger.info("Restored %d files from snapshot %s", restored, snapshot_id)
|
|
return restored > 0
|
|
|
|
|
|
# Relative path of the cron job database inside HERMES_HOME. Kept in sync with the entry in
|
|
# ``_QUICK_STATE_FILES`` and with ``cron/jobs.py``'s ``JOBS_FILE``.
|
|
_CRON_JOBS_REL = "cron/jobs.json"
|
|
|
|
|
|
def _count_cron_jobs(path: Path) -> Optional[int]:
|
|
"""Number of cron jobs in ``path`` (canonical ``{"jobs": [...]}`` or legacy bare list).
|
|
|
|
``None`` if missing or unparseable; callers must treat that as "unknown", not zero, since
|
|
acting on an unreadable file could mask a real corruption the user needs to see.
|
|
"""
|
|
if not path.is_file():
|
|
return None
|
|
try:
|
|
# utf-8-sig: same dialect as cron/jobs.load_jobs — a Windows-editor BOM would otherwise
|
|
# count as "unreadable" and silently disable the post-update auto-restore safety net.
|
|
with open(path, "r", encoding="utf-8-sig") as f:
|
|
data = json.load(f)
|
|
except (OSError, json.JSONDecodeError):
|
|
return None
|
|
if isinstance(data, dict):
|
|
data = data.get("jobs", [])
|
|
return len(data) if isinstance(data, list) else None
|
|
|
|
|
|
def restore_cron_jobs_if_emptied(snapshot_id: str, hermes_home: Optional[Path] = None) -> Optional[Dict[str, Any]]:
|
|
"""Safety net for silent cron-job loss across ``hermes update``.
|
|
|
|
Deliberately conservative: restores only on unambiguous evidence of loss (snapshot had more
|
|
jobs than the live file), so a user who genuinely deleted jobs is never second-guessed, and
|
|
an unreadable live file (``None``) is left untouched so real corruption still surfaces.
|
|
"""
|
|
if not snapshot_id:
|
|
return None
|
|
home = hermes_home or get_hermes_home()
|
|
live_path = home / _CRON_JOBS_REL
|
|
live_count = _count_cron_jobs(live_path)
|
|
if live_count is None:
|
|
return None
|
|
snap_path = _quick_snapshot_root(home) / snapshot_id / _CRON_JOBS_REL
|
|
snap_count = _count_cron_jobs(snap_path)
|
|
# Fewer live jobs than the snapshot catches both total loss (0 vs N) and partial loss
|
|
# (1 vs 19) — the desktop scheduler can overwrite jobs.json with its own small set.
|
|
if not snap_count or live_count >= snap_count: # None or 0 — nothing worth restoring
|
|
return None
|
|
try:
|
|
live_path.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(snap_path, live_path)
|
|
except (OSError, PermissionError) as exc:
|
|
logger.error("Cron jobs were emptied during update but auto-restore failed: %s", exc)
|
|
return None
|
|
logger.warning(
|
|
"Restored %d cron job(s) from pre-update snapshot %s "
|
|
"(live file had %d job(s), snapshot had %d — jobs were lost during migration)",
|
|
snap_count, snapshot_id, live_count, snap_count)
|
|
return {"restored": True, "job_count": snap_count, "snapshot_id": snapshot_id}
|
|
|
|
|
|
def _sibling_profile_homes(invoking_home: Path) -> list[tuple[str, Path]]:
|
|
"""(name, home) for every OTHER profile on this install. Never raises.
|
|
|
|
The update's code swap and fleet restart touch every profile, so the pre-update snapshot
|
|
must too (#66140). The invoking profile is excluded — its snapshot is taken separately.
|
|
"""
|
|
homes: list[tuple[str, Path]] = []
|
|
try:
|
|
from hermes_cli.profiles import _get_default_hermes_home, _get_profiles_root, _PROFILE_ID_RE
|
|
invoking = invoking_home.resolve()
|
|
default_home = _get_default_hermes_home()
|
|
if default_home.is_dir() and default_home.resolve() != invoking:
|
|
homes.append(("default", default_home))
|
|
root = _get_profiles_root()
|
|
if root.is_dir():
|
|
for entry in sorted(root.iterdir()):
|
|
if (entry.is_dir() and entry.name != "default" and _PROFILE_ID_RE.match(entry.name)
|
|
and entry.resolve() != invoking):
|
|
homes.append((entry.name, entry))
|
|
except Exception as exc:
|
|
logger.debug("Sibling profile enumeration failed: %s", exc)
|
|
return homes
|
|
|
|
|
|
def create_pre_update_snapshots_all_profiles(
|
|
invoking_home: Optional[Path] = None, keep: Optional[int] = None, max_file_size: Optional[int] = None
|
|
) -> Dict[str, str]:
|
|
"""Pre-update quick snapshots for every SIBLING profile (#66140).
|
|
|
|
Same snapshot set, size cap, and keep policy as the invoking profile's snapshot; each lands
|
|
under its OWN ``<home>/state-snapshots/`` so per-profile restore tooling finds it.
|
|
"""
|
|
results: Dict[str, str] = {}
|
|
home = invoking_home or get_hermes_home()
|
|
for name, profile_home in _sibling_profile_homes(home):
|
|
try:
|
|
snap_id = create_quick_snapshot(
|
|
label="pre-update", hermes_home=profile_home, keep=keep, max_file_size=max_file_size)
|
|
if snap_id:
|
|
results[name] = snap_id
|
|
except Exception as exc:
|
|
logger.debug("Pre-update snapshot for profile %s failed: %s", name, exc)
|
|
return results
|
|
|
|
|
|
# Config paths the update flow must never change (#64160): model routing keys and the MoA
|
|
# section are consumed machine-wide (gateway, cron, desktop), so an update/repair cycle that
|
|
# rewrites them silently redirects paid inference. Dotted paths into the raw config.yaml
|
|
# document; a single-element tuple protects the whole section.
|
|
_PROTECTED_CONFIG_PATHS: Tuple[Tuple[str, ...], ...] = (
|
|
("model", "provider"), ("model", "default"), ("model", "base_url"), ("model", "api_key"), ("moa",))
|
|
|
|
|
|
def _read_raw_yaml_dict(path: Path) -> Optional[Dict[str, Any]]:
|
|
"""Parse ``path`` as a YAML mapping. ``None`` = missing/unreadable/non-dict."""
|
|
if not path.is_file():
|
|
return None
|
|
try:
|
|
import yaml
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
data = yaml.safe_load(f)
|
|
except Exception:
|
|
return None
|
|
return data if isinstance(data, dict) else None
|
|
|
|
|
|
def _get_config_path_value(data: Dict[str, Any], dotted: Tuple[str, ...]) -> Any:
|
|
node: Any = data
|
|
for key in dotted:
|
|
if not isinstance(node, dict):
|
|
return None
|
|
node = node.get(key)
|
|
return node
|
|
|
|
|
|
def _set_config_path_value(data: Dict[str, Any], dotted: Tuple[str, ...], value: Any) -> None:
|
|
node = data
|
|
for key in dotted[:-1]:
|
|
if not isinstance(node.get(key), dict):
|
|
node[key] = {}
|
|
node = node[key]
|
|
node[dotted[-1]] = value
|
|
|
|
|
|
def restore_config_model_settings_if_rewritten(
|
|
snapshot_id: str, hermes_home: Optional[Path] = None) -> Optional[Dict[str, Any]]:
|
|
"""Safety net for silent config.yaml model/MoA loss across ``hermes update``.
|
|
|
|
Mirrors :func:`restore_cron_jobs_if_emptied`: compare the current config against the
|
|
pre-update snapshot taken by this same update run and restore only the protected keys —
|
|
never the whole file — when a value the user had set was changed or dropped.
|
|
"""
|
|
if not snapshot_id:
|
|
return None
|
|
home = hermes_home or get_hermes_home()
|
|
live_path = home / "config.yaml"
|
|
snap = _read_raw_yaml_dict(_quick_snapshot_root(home) / snapshot_id / "config.yaml")
|
|
if not snap:
|
|
return None # no snapshot copy — nothing to compare against
|
|
live = _read_raw_yaml_dict(live_path)
|
|
if live is None:
|
|
# Missing/unparseable live config is a failure the user should see (matches the cron net).
|
|
return None
|
|
restored_keys: list[str] = []
|
|
for dotted in _PROTECTED_CONFIG_PATHS:
|
|
snap_val = _get_config_path_value(snap, dotted)
|
|
if snap_val in (None, "", {}, []):
|
|
continue # user never set it — nothing to protect
|
|
if _get_config_path_value(live, dotted) != snap_val:
|
|
_set_config_path_value(live, dotted, snap_val)
|
|
restored_keys.append(".".join(dotted))
|
|
if not restored_keys:
|
|
return None
|
|
try:
|
|
from utils import atomic_yaml_write
|
|
atomic_yaml_write(live_path, live)
|
|
except (OSError, PermissionError) as exc:
|
|
logger.error("config.yaml model settings were rewritten during update but auto-restore failed: %s", exc)
|
|
return None
|
|
logger.warning(
|
|
"Restored user config value(s) %s from pre-update snapshot %s — the update flow rewrote them (#64160)",
|
|
", ".join(restored_keys), snapshot_id)
|
|
return {"restored": True, "keys": restored_keys, "snapshot_id": snapshot_id}
|
|
|
|
|
|
def _restore_all_sibling_profiles(
|
|
profile_snapshots: Dict[str, str], invoking_home: Optional[Path], restore_fn, failure_log: str
|
|
) -> list[Dict[str, Any]]:
|
|
"""Run a per-profile safety net (``restore_fn(snap_id, hermes_home=...)``) for every sibling.
|
|
|
|
Each profile's live file is compared against ITS OWN same-generation pre-update snapshot.
|
|
Returns one result dict per restored profile, each with a ``profile`` key added. Never raises.
|
|
"""
|
|
restored: list[Dict[str, Any]] = []
|
|
if not profile_snapshots:
|
|
return restored
|
|
home = invoking_home or get_hermes_home()
|
|
by_name = dict(_sibling_profile_homes(home))
|
|
for name, snap_id in profile_snapshots.items():
|
|
profile_home = by_name.get(name)
|
|
if profile_home is None:
|
|
continue
|
|
try:
|
|
result = restore_fn(snap_id, hermes_home=profile_home)
|
|
except Exception as exc:
|
|
logger.debug(failure_log, name, exc)
|
|
continue
|
|
if result:
|
|
result["profile"] = name
|
|
restored.append(result)
|
|
return restored
|
|
|
|
|
|
def restore_config_model_settings_all_profiles(
|
|
profile_snapshots: Dict[str, str], invoking_home: Optional[Path] = None) -> list[Dict[str, Any]]:
|
|
"""Run the config model-settings safety net for every sibling profile (see ``_restore_all_sibling_profiles``)."""
|
|
return _restore_all_sibling_profiles(
|
|
profile_snapshots, invoking_home, restore_config_model_settings_if_rewritten,
|
|
"Config model-settings restore check for profile %s failed: %s")
|
|
|
|
|
|
def restore_cron_jobs_all_profiles(
|
|
profile_snapshots: Dict[str, str], invoking_home: Optional[Path] = None) -> list[Dict[str, Any]]:
|
|
"""Run the cron-jobs safety net for every sibling profile (#66140).
|
|
|
|
``profile_snapshots`` comes from :func:`create_pre_update_snapshots_all_profiles`, so
|
|
restores are same-generation by construction.
|
|
"""
|
|
return _restore_all_sibling_profiles(
|
|
profile_snapshots, invoking_home, restore_cron_jobs_if_emptied,
|
|
"Cron restore check for profile %s failed: %s")
|
|
|
|
|
|
def _prune_oldest(newest_first: List[Path], keep: int, remove, what: str) -> int:
|
|
"""``remove(path)`` every entry past the first *keep*; return how many succeeded."""
|
|
deleted = 0
|
|
for p in newest_first[keep:]:
|
|
try:
|
|
remove(p)
|
|
deleted += 1
|
|
except OSError as exc:
|
|
logger.warning("Failed to prune %s %s: %s", what, p.name, exc)
|
|
return deleted
|
|
|
|
|
|
def prune_quick_snapshots(keep: int = _QUICK_DEFAULT_KEEP, hermes_home: Optional[Path] = None) -> int:
|
|
"""Remove oldest quick snapshots beyond the keep limit. Returns count deleted."""
|
|
return _prune_oldest(_snapshot_dirs(_quick_snapshot_root(hermes_home)), keep, shutil.rmtree, "snapshot")
|
|
|
|
|
|
def run_quick_backup(args) -> None:
|
|
"""CLI entry point for hermes backup --quick."""
|
|
snap_id = create_quick_snapshot(label=getattr(args, "label", None))
|
|
if snap_id:
|
|
print(f"State snapshot created: {snap_id}\n"
|
|
f" {len(list_quick_snapshots())} snapshot(s) stored in {display_hermes_home()}/state-snapshots/\n"
|
|
f" Restore with: /snapshot restore {snap_id}")
|
|
else:
|
|
print("No state files found to snapshot.")
|
|
|
|
|
|
# --- Shared full-zip backup helper ---
|
|
|
|
def _write_full_zip_backup(out_path: Path, hermes_root: Path) -> Optional[Path]:
|
|
"""Write a full zip snapshot of ``hermes_root`` to ``out_path`` while holding the backup slot.
|
|
|
|
Same exclusion rules and SQLite safe-copy as :func:`run_backup`. Returns the output path on
|
|
success, None on failure (nothing to back up, another backup running, or write error).
|
|
"""
|
|
try:
|
|
with _backup_operation_lock(hermes_root):
|
|
return _write_full_zip_backup_locked(out_path, hermes_root)
|
|
except BackupInProgressError as exc:
|
|
logger.warning("Full-zip backup skipped: %s", exc)
|
|
return None
|
|
|
|
|
|
def _write_full_zip_backup_locked(out_path: Path, hermes_root: Path) -> Optional[Path]:
|
|
scan_started = time.monotonic()
|
|
logger.info("automatic backup phase=scan status=started")
|
|
try:
|
|
files_to_add = list(_iter_backup_files(hermes_root, out_path))
|
|
except OSError as exc:
|
|
logger.warning("Full-zip backup: walk failed: %s", exc)
|
|
return None
|
|
if not files_to_add:
|
|
return None
|
|
logger.info("automatic backup phase=scan status=complete duration_ms=%.1f files=%d",
|
|
(time.monotonic() - scan_started) * 1000, len(files_to_add))
|
|
|
|
def _db_failure(rel_path: Path) -> None:
|
|
logger.warning("Full-zip backup aborted: SQLite snapshot failed for %s", rel_path)
|
|
raise _SQLiteSnapshotError(str(rel_path))
|
|
|
|
archive_started = time.monotonic()
|
|
try:
|
|
with _atomic_output_path(out_path) as archive_path, zipfile.ZipFile(
|
|
archive_path, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zf:
|
|
_write_zip_entries(
|
|
zf, files_to_add, out_path, on_db_failure=_db_failure, track_bytes=False,
|
|
on_error=lambda rel, exc: logger.debug("Skipping %s in zip backup: %s", rel, exc),
|
|
on_progress=lambda i: logger.info(
|
|
"automatic backup phase=archive status=progress completed=%d total=%d", i, len(files_to_add)))
|
|
except (OSError, _SQLiteSnapshotError) as exc:
|
|
# ``_atomic_output_path`` already removed the hidden partial. Do not unlink ``out_path``:
|
|
# it may be a previous valid backup that the atomic publisher deliberately preserved.
|
|
logger.warning("Full-zip backup: zip write failed: %s", exc)
|
|
return None
|
|
logger.info("automatic backup phase=archive status=complete duration_ms=%.1f files=%d bytes=%d",
|
|
(time.monotonic() - archive_started) * 1000, len(files_to_add), out_path.stat().st_size)
|
|
return out_path
|
|
|
|
|
|
# --- Pre-update / pre-migration auto-backups ---
|
|
|
|
_PRE_UPDATE_BACKUPS_DIR = "backups"
|
|
_PRE_UPDATE_PREFIX = "pre-update-"
|
|
_PRE_UPDATE_DEFAULT_KEEP = 5
|
|
_PRE_MIGRATION_PREFIX = "pre-migration-"
|
|
_PRE_MIGRATION_DEFAULT_KEEP = 5
|
|
|
|
|
|
def _prune_prefixed_zips(backup_dir: Path, prefix: str, keep: int, what: str) -> int:
|
|
"""Remove oldest ``<prefix>*.zip`` files in *backup_dir* beyond the keep limit; return count deleted.
|
|
|
|
Only prefix-matched files are touched, so hand-made zips or other backup kinds in the same
|
|
directory are never removed. Operators who don't want a backup set
|
|
``updates.pre_update_backup: off`` — that gates creation.
|
|
"""
|
|
backups = _newest_first(
|
|
backup_dir, lambda p: p.is_file() and p.name.startswith(prefix) and p.suffix.lower() == ".zip")
|
|
return _prune_oldest(backups, keep, Path.unlink, what)
|
|
|
|
|
|
def _create_prefixed_full_backup(
|
|
hermes_home: Optional[Path], prefix: str, keep: int, what: str, prune_what: str) -> Optional[Path]:
|
|
"""Write ``<HERMES_HOME>/backups/<prefix><timestamp>.zip`` and prune older same-prefix zips.
|
|
|
|
Returns the created path, or ``None`` if nothing was found to back up or the write failed.
|
|
Never raises.
|
|
"""
|
|
hermes_root = hermes_home or get_default_hermes_root()
|
|
if not hermes_root.is_dir():
|
|
return None
|
|
backup_dir = hermes_root / _PRE_UPDATE_BACKUPS_DIR
|
|
try:
|
|
backup_dir.mkdir(parents=True, exist_ok=True)
|
|
except OSError as exc:
|
|
logger.warning("Could not create %s backup dir %s: %s", what, backup_dir, exc)
|
|
return None
|
|
out_path = backup_dir / f"{prefix}{datetime.now().strftime('%Y-%m-%d-%H%M%S')}.zip"
|
|
if _write_full_zip_backup(out_path, hermes_root) is None:
|
|
return None
|
|
_prune_prefixed_zips(backup_dir, prefix, keep, prune_what)
|
|
return out_path
|
|
|
|
|
|
def create_pre_update_backup(
|
|
hermes_home: Optional[Path] = None, keep: int = _PRE_UPDATE_DEFAULT_KEEP) -> Optional[Path]:
|
|
"""Full zip backup to ``<HERMES_HOME>/backups/pre-update-<timestamp>.zip``, auto-pruned.
|
|
|
|
Same exclusions and SQLite safe-copy as :func:`run_backup`. Returns the zip path, or ``None``
|
|
if nothing was found or the backup failed. Never raises — ``hermes update`` continues anyway.
|
|
"""
|
|
return _create_prefixed_full_backup(hermes_home, _PRE_UPDATE_PREFIX, max(keep, 1), "pre-update", "backup")
|
|
|
|
|
|
def create_pre_migration_backup(
|
|
hermes_home: Optional[Path] = None, keep: int = _PRE_MIGRATION_DEFAULT_KEEP) -> Optional[Path]:
|
|
"""Full zip backup to ``<HERMES_HOME>/backups/pre-migration-<timestamp>.zip`` before ``hermes claw migrate``.
|
|
|
|
Shares the shared ``backups/`` dir (so ``hermes import`` and the update-backup listing pick
|
|
it up), restorable with ``hermes import <archive>``. Returns the zip path, or ``None`` if
|
|
nothing was found (fresh install) or the write failed. Never raises.
|
|
"""
|
|
return _create_prefixed_full_backup(
|
|
hermes_home, _PRE_MIGRATION_PREFIX, max(keep, 0), "pre-migration", "pre-migration backup")
|