939a2f64b4
Gateway, dashboard, ACP server and the CLI already hold one registry-shared SessionDB per state.db path, yet several in-process call paths still opened a bare SessionDB() beside it. Each one is a full writer: schema init, write lock, token-writer thread and a close-time WAL checkpoint. On a dashboard serving overlapping requests that stacked up to the "5 live SessionDB handles" precursor within seconds; per #110544 they are now harmless to each other's WAL generation, but the leak itself remained. Pure readers attach read_only=True (no writer connection, no write lock): - plugins/hermes-achievements/dashboard/plugin_api.py::scan_sessions (dashboard, per background scan and per /rescan; highest-frequency site) - hermes_cli/console_engine.py::_session_db (dashboard console; list, stats and export are reads; rename/optimize opt in to a writer) - tools/process_registry_results.py::_owns_result (gateway, per retained result load) - hermes_cli/main.py::_session_db (last-session / title / cwd lookups) - hermes_cli/terminal_breadcrumbs.py, hermes_cli/status.py, hermes_cli/main_tui_launch.py (lookup one-shots) Writers share the process's registry handle (hermes_state_registry.acquire; close()/release_or_close release one refcount): - acp_adapter/session.py::SessionManager._get_db — the AIAgent it builds acquires the same path, so the ACP server held two writers per process - hermes_cli/kanban_db_dispatch.py::_retag_legacy_worker_sessions (gateway dispatcher tick) - hermes_cli/main.py::_create_titled_session, hermes_cli/oneshot.py, hermes_cli/foreign_sessions.py — the CLI acquires the same handle a moment later The "N live SessionDB handles" warning now counts only writable members: read-only attaches are the sanctioned per-request shape for dashboard routers and CLI lookups, and counting them turned a healthy topology into an operator alarm (#100896 field reports of restart loops keyed on it). Live repro (one registry writer + 4 overlapping dashboard/gateway paths in one process): before 5 writable opens, 5 live handles, warning fired; after 1 writable open (the registry handle), 0 from the request paths, no warning. Refs #100896 #103339
117 lines
4.4 KiB
Python
117 lines
4.4 KiB
Python
"""Bounded, profile-local receipts for completed terminal processes.
|
|
|
|
Receipts are read through process_manage, never replayed as notifications or
|
|
adopted as live PIDs. Each producer writes its own file so independent one-shot
|
|
parents cannot overwrite each other's results in the running-PID checkpoint.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
|
|
from hermes_constants import get_hermes_home
|
|
from utils import atomic_json_write
|
|
|
|
logger = logging.getLogger("tools.process_registry")
|
|
|
|
RESULT_RETENTION_SECONDS = 7 * 24 * 60 * 60
|
|
MAX_RETAINED_RESULTS = 64
|
|
_RESULT_FIELDS = (
|
|
"id", "command", "cwd", "task_id", "owner_task_id", "session_key",
|
|
"parent_session_id", "started_at", "exit_code", "completion_reason",
|
|
"termination_source", "notify_on_complete",
|
|
)
|
|
|
|
|
|
def _result_paths():
|
|
"""Prune by completion time, not start time (jobs can take days)."""
|
|
directory = get_hermes_home() / "logs" / "process-results"
|
|
cutoff = time.time() - RESULT_RETENTION_SECONDS
|
|
retained = []
|
|
for path in directory.glob("proc_*.json"):
|
|
try:
|
|
modified = path.stat().st_mtime
|
|
if modified < cutoff:
|
|
path.unlink(missing_ok=True)
|
|
else:
|
|
retained.append((modified, path))
|
|
except FileNotFoundError:
|
|
continue # Another producer pruned it.
|
|
retained.sort(key=lambda item: (item[0], item[1].name), reverse=True)
|
|
for _, path in retained[MAX_RETAINED_RESULTS:]:
|
|
path.unlink(missing_ok=True)
|
|
return [path for _, path in retained[:MAX_RETAINED_RESULTS]]
|
|
|
|
|
|
def save_completed_result(session) -> None:
|
|
from agent.redact import redact_sensitive_text, redact_terminal_output
|
|
from tools.process_registry import MAX_OUTPUT_CHARS
|
|
|
|
with session._lock:
|
|
record = {key: getattr(session, key) for key in _RESULT_FIELDS}
|
|
record["output"] = session.output_buffer[-MAX_OUTPUT_CHARS:]
|
|
# Live-output opt-out must not persist raw credentials in durable receipts.
|
|
record["output"] = redact_terminal_output(record["output"], record["command"], force=True)
|
|
record["command"] = redact_sensitive_text(record["command"], code_file=True, force=True)
|
|
directory = get_hermes_home() / "logs" / "process-results"
|
|
try:
|
|
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
atomic_json_write(directory / f"{session.id}.json", record, mode=0o600)
|
|
_result_paths()
|
|
except OSError:
|
|
# Preserve live delivery on disk failure, but never silently claim durability.
|
|
logger.warning("Could not retain completed process result %s", session.id, exc_info=True)
|
|
|
|
|
|
def _owns_result(owner: str, parent: str | None) -> bool:
|
|
if not parent:
|
|
return False
|
|
if owner == parent:
|
|
return True
|
|
from hermes_state import SessionDB
|
|
|
|
# Pure lineage read on the hot path of every retained-result load; a writable open here
|
|
# was one more writer handle per call inside the gateway (#100896).
|
|
db = SessionDB(read_only=True)
|
|
try:
|
|
return db.get_compression_tip(parent) == owner
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def load_completed_results(prefix: str = "") -> dict:
|
|
"""Restore read-only snapshots; no process handles, watchers, or queue events."""
|
|
from tools.process_registry import ProcessSession
|
|
|
|
from gateway.session_context import get_session_env
|
|
|
|
owner = get_session_env("HERMES_SESSION_ID", "")
|
|
if not owner:
|
|
return {}
|
|
results = {}
|
|
try:
|
|
paths = _result_paths()
|
|
except OSError:
|
|
logger.warning("Could not read retained process results", exc_info=True)
|
|
return results
|
|
for path in paths:
|
|
if not path.stem.startswith(prefix):
|
|
continue
|
|
try:
|
|
record = json.loads(path.read_text(encoding="utf-8"))
|
|
if record["id"] != path.stem or not re.fullmatch(r"proc_[\w]+", record["id"]):
|
|
continue
|
|
if not _owns_result(owner, record.get("parent_session_id")):
|
|
continue
|
|
session = ProcessSession(
|
|
**{key: record[key] for key in _RESULT_FIELDS},
|
|
exited=True, output_buffer=record["output"],
|
|
)
|
|
session._completion_event.set()
|
|
results[session.id] = session
|
|
except (OSError, ValueError, KeyError, TypeError, sqlite3.Error):
|
|
logger.debug("Skipping unreadable process result %s", path.name, exc_info=True)
|
|
return results
|