hermes-agent/tests/conformance/persistence/_harness.py

"""Shared harness for the crash/resume persistence conformance cells (#80921).

Method (per the spot-probes in the tracking issue): real ``SessionDB``
against an isolated temp database, real ``SIGKILL`` delivered to a separate
OS process mid-write, deterministic and LLM-free. Every wait has a hard
deadline so a wedged child can never hang the suite; coordination uses file
barriers, never bare sleeps.

Journal-mode policy mirrors the issue's caveat: cells run on the mode the
repo's own ``resolve_journal_mode()`` selects for this interpreter/filesystem
(recorded per cell), plus an explicit ``DELETE`` run; an explicit ``WAL`` run
is attempted and skipped when the resolver's downgrade gates trip (e.g. the
WAL-reset interpreter bug), so WAL semantics are probed exactly where they
are actually deployable.
"""

from __future__ import annotations

import os
import signal
import sqlite3
import subprocess
import sys
import time
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parents[3]

# Generous deadlines: xdist-loaded CI boxes stall; correctness never depends
# on these being tight, they only bound a hung child.
CHILD_DEADLINE = 60.0
POLL_INTERVAL = 0.02


def spawn_child(script_body: str, *, cwd: Path | None = None, env: dict | None = None) -> subprocess.Popen:
    """Run ``script_body`` in a fresh interpreter with the repo importable."""
    child_env = dict(os.environ)
    # Prepend, don't clobber: CI images may rely on an inherited PYTHONPATH
    # for dependencies — losing it would fail child imports while the parent
    # collects fine, surfacing only as an opaque wait_for deadline.
    inherited = child_env.get("PYTHONPATH")
    child_env["PYTHONPATH"] = (
        f"{REPO_ROOT}{os.pathsep}{inherited}" if inherited else str(REPO_ROOT)
    )
    child_env["PYTHONUNBUFFERED"] = "1"
    if env:
        child_env.update(env)
    return subprocess.Popen(
        [sys.executable, "-c", script_body],
        cwd=str(cwd or REPO_ROOT),
        env=child_env,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )


def wait_for(
    predicate,
    *,
    deadline: float = CHILD_DEADLINE,
    what: str = "condition",
    child: "subprocess.Popen | None" = None,
) -> None:
    """Poll ``predicate`` until true or fail loudly at the deadline.

    When ``child`` is given and it exits before the predicate turns true,
    fail IMMEDIATELY with its captured stderr — a crashed writer must be an
    instant diagnostic, not a 60s opaque deadline.
    """
    end = time.monotonic() + deadline
    while time.monotonic() < end:
        if predicate():
            return
        if child is not None and child.poll() is not None:
            err = b""
            if child.stderr is not None:
                err = child.stderr.read() or b""
            raise AssertionError(
                f"child exited early (rc={child.returncode}) while waiting "
                f"for {what}; stderr:\n{err.decode(errors='replace')[-2000:]}"
            )
        time.sleep(POLL_INTERVAL)
    raise AssertionError(f"deadline ({deadline}s) waiting for {what}")


def kill9_and_reap(proc: subprocess.Popen, *, deadline: float = CHILD_DEADLINE) -> None:
    """SIGKILL ``proc`` and reap it within ``deadline``."""
    try:
        proc.kill()  # SIGKILL on POSIX
    except ProcessLookupError:
        pass
    proc.wait(timeout=deadline)


def reap(proc: subprocess.Popen, *, deadline: float = CHILD_DEADLINE) -> tuple[int, str, str]:
    """Wait for a child to exit on its own; kill + fail if it doesn't."""
    try:
        out, err = proc.communicate(timeout=deadline)
    except subprocess.TimeoutExpired:
        proc.kill()
        out, err = proc.communicate(timeout=10)
        raise AssertionError(
            f"child did not exit within {deadline}s; stderr:\n"
            f"{err.decode(errors='replace')[-2000:]}"
        )
    return proc.returncode, out.decode(errors="replace"), err.decode(errors="replace")


def on_disk_journal_mode(db_path: Path) -> str:
    """Record the journal mode actually in effect for a cell result."""
    conn = sqlite3.connect(str(db_path))
    try:
        row = conn.execute("PRAGMA journal_mode").fetchone()
        return str(row[0]) if row else "unknown"
    finally:
        conn.close()


def integrity_ok(db_path: Path) -> bool:
    conn = sqlite3.connect(str(db_path))
    try:
        row = conn.execute("PRAGMA integrity_check").fetchone()
        return bool(row) and str(row[0]).lower() == "ok"
    finally:
        conn.close()


def make_hermes_home(base: Path, journal_mode: str) -> Path:
    """Create an isolated HERMES_HOME whose config pins ``database.journal_mode``.

    The journal-mode matrix legs must steer the CHILD's own resolver:
    pre-seeding the DB file alone is not enough, because ``SessionDB.__init__``
    runs ``apply_wal_with_fallback()`` which upgrades a non-WAL file to WAL
    whenever the configured mode (default ``wal``) says so — on healthy
    SQLite the DELETE leg would silently run in WAL.
    """
    home = base / f"hermes-home-{journal_mode}"
    home.mkdir(parents=True, exist_ok=True)
    (home / "config.yaml").write_text(
        f"database:\n  journal_mode: {journal_mode}\n", encoding="utf-8"
    )
    return home


def effective_mode_or_skip(db_path: Path, requested_mode: str | None) -> str:
    """Record the on-disk journal mode; skip if a requested leg wasn't honored.

    A leg that ran in a different mode than advertised must never count as
    green evidence for the advertised mode.
    """
    import pytest

    mode = on_disk_journal_mode(db_path)
    if requested_mode is not None and mode.upper() != requested_mode.upper():
        pytest.skip(
            f"journal_mode={requested_mode} not honored by this environment "
            f"(effective {mode}) — matrix leg not probeable here"
        )
    return mode