Files
hermes-agent/tools/process_registry_results.py
T
teknium1 939a2f64b4 fix: long-lived processes stop minting duplicate state.db writer handles
Gateway, dashboard, ACP server and the CLI already hold one registry-shared
SessionDB per state.db path, yet several in-process call paths still opened
a bare SessionDB() beside it. Each one is a full writer: schema init, write
lock, token-writer thread and a close-time WAL checkpoint. On a dashboard
serving overlapping requests that stacked up to the "5 live SessionDB
handles" precursor within seconds; per #110544 they are now harmless to each
other's WAL generation, but the leak itself remained.

Pure readers attach read_only=True (no writer connection, no write lock):
  - plugins/hermes-achievements/dashboard/plugin_api.py::scan_sessions
    (dashboard, per background scan and per /rescan; highest-frequency site)
  - hermes_cli/console_engine.py::_session_db (dashboard console; list,
    stats and export are reads; rename/optimize opt in to a writer)
  - tools/process_registry_results.py::_owns_result (gateway, per retained
    result load)
  - hermes_cli/main.py::_session_db (last-session / title / cwd lookups)
  - hermes_cli/terminal_breadcrumbs.py, hermes_cli/status.py,
    hermes_cli/main_tui_launch.py (lookup one-shots)

Writers share the process's registry handle (hermes_state_registry.acquire;
close()/release_or_close release one refcount):
  - acp_adapter/session.py::SessionManager._get_db — the AIAgent it builds
    acquires the same path, so the ACP server held two writers per process
  - hermes_cli/kanban_db_dispatch.py::_retag_legacy_worker_sessions
    (gateway dispatcher tick)
  - hermes_cli/main.py::_create_titled_session, hermes_cli/oneshot.py,
    hermes_cli/foreign_sessions.py — the CLI acquires the same handle a
    moment later

The "N live SessionDB handles" warning now counts only writable members:
read-only attaches are the sanctioned per-request shape for dashboard
routers and CLI lookups, and counting them turned a healthy topology into
an operator alarm (#100896 field reports of restart loops keyed on it).

Live repro (one registry writer + 4 overlapping dashboard/gateway paths in
one process): before 5 writable opens, 5 live handles, warning fired;
after 1 writable open (the registry handle), 0 from the request paths,
no warning.

Refs #100896 #103339
2026-09-14 08:10:36 -07:00

117 lines
4.4 KiB
Python

"""Bounded, profile-local receipts for completed terminal processes.
Receipts are read through process_manage, never replayed as notifications or
adopted as live PIDs. Each producer writes its own file so independent one-shot
parents cannot overwrite each other's results in the running-PID checkpoint.
"""
import json
import logging
import re
import sqlite3
import time
from hermes_constants import get_hermes_home
from utils import atomic_json_write
logger = logging.getLogger("tools.process_registry")
RESULT_RETENTION_SECONDS = 7 * 24 * 60 * 60
MAX_RETAINED_RESULTS = 64
_RESULT_FIELDS = (
"id", "command", "cwd", "task_id", "owner_task_id", "session_key",
"parent_session_id", "started_at", "exit_code", "completion_reason",
"termination_source", "notify_on_complete",
)
def _result_paths():
"""Prune by completion time, not start time (jobs can take days)."""
directory = get_hermes_home() / "logs" / "process-results"
cutoff = time.time() - RESULT_RETENTION_SECONDS
retained = []
for path in directory.glob("proc_*.json"):
try:
modified = path.stat().st_mtime
if modified < cutoff:
path.unlink(missing_ok=True)
else:
retained.append((modified, path))
except FileNotFoundError:
continue # Another producer pruned it.
retained.sort(key=lambda item: (item[0], item[1].name), reverse=True)
for _, path in retained[MAX_RETAINED_RESULTS:]:
path.unlink(missing_ok=True)
return [path for _, path in retained[:MAX_RETAINED_RESULTS]]
def save_completed_result(session) -> None:
from agent.redact import redact_sensitive_text, redact_terminal_output
from tools.process_registry import MAX_OUTPUT_CHARS
with session._lock:
record = {key: getattr(session, key) for key in _RESULT_FIELDS}
record["output"] = session.output_buffer[-MAX_OUTPUT_CHARS:]
# Live-output opt-out must not persist raw credentials in durable receipts.
record["output"] = redact_terminal_output(record["output"], record["command"], force=True)
record["command"] = redact_sensitive_text(record["command"], code_file=True, force=True)
directory = get_hermes_home() / "logs" / "process-results"
try:
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
atomic_json_write(directory / f"{session.id}.json", record, mode=0o600)
_result_paths()
except OSError:
# Preserve live delivery on disk failure, but never silently claim durability.
logger.warning("Could not retain completed process result %s", session.id, exc_info=True)
def _owns_result(owner: str, parent: str | None) -> bool:
if not parent:
return False
if owner == parent:
return True
from hermes_state import SessionDB
# Pure lineage read on the hot path of every retained-result load; a writable open here
# was one more writer handle per call inside the gateway (#100896).
db = SessionDB(read_only=True)
try:
return db.get_compression_tip(parent) == owner
finally:
db.close()
def load_completed_results(prefix: str = "") -> dict:
"""Restore read-only snapshots; no process handles, watchers, or queue events."""
from tools.process_registry import ProcessSession
from gateway.session_context import get_session_env
owner = get_session_env("HERMES_SESSION_ID", "")
if not owner:
return {}
results = {}
try:
paths = _result_paths()
except OSError:
logger.warning("Could not read retained process results", exc_info=True)
return results
for path in paths:
if not path.stem.startswith(prefix):
continue
try:
record = json.loads(path.read_text(encoding="utf-8"))
if record["id"] != path.stem or not re.fullmatch(r"proc_[\w]+", record["id"]):
continue
if not _owns_result(owner, record.get("parent_session_id")):
continue
session = ProcessSession(
**{key: record[key] for key in _RESULT_FIELDS},
exited=True, output_buffer=record["output"],
)
session._completion_event.set()
results[session.id] = session
except (OSError, ValueError, KeyError, TypeError, sqlite3.Error):
logger.debug("Skipping unreadable process result %s", path.name, exc_info=True)
return results