301 lines
17 KiB
Python
301 lines
17 KiB
Python
"""Source-backed project recall; no external recall plugin or cached source text."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import base64
|
|
import binascii
|
|
import re
|
|
|
|
from agent.redact import redact_sensitive_text
|
|
|
|
_SCAN_BATCH = 100
|
|
_SCAN_BUDGET = 1000
|
|
|
|
|
|
class ProjectRecall:
|
|
def __init__(self, db, current_session_id):
|
|
from hermes_project_scope import resolve_project_scope
|
|
|
|
self.db = db
|
|
self.current_session_id = current_session_id
|
|
self.scope = resolve_project_scope(db, current_session_id)
|
|
self.allowed = frozenset(self.scope.get('allowed_session_ids', ()))
|
|
self.members_json = json.dumps(sorted(self.allowed))
|
|
self.self_only = False
|
|
self.read_has_more = False
|
|
self.read_cursor = None
|
|
self.before_cursor = None
|
|
|
|
def _rows(self, sql, params=()):
|
|
with self.db._read_ctx() as conn:
|
|
return [dict(row) for row in conn.execute(sql, params).fetchall()]
|
|
|
|
def _eligible_sql(self):
|
|
where = """m.session_id IN (SELECT value FROM json_each(?))
|
|
AND (m.active=1 OR m.compacted=1)
|
|
AND m.role IN ('user','assistant')
|
|
AND COALESCE(m._compressed_summary,0)=0
|
|
AND COALESCE(m.display_kind,'') NOT IN ('hidden','internal','progress','auto_continue')
|
|
AND COALESCE(s.hidden,0)=0
|
|
AND s.source NOT IN ('subagent','tool','kanban','cron')"""
|
|
return where + (' AND m.compacted=1' if self.self_only else '')
|
|
|
|
def _safe(self, row, *, content_offset=0, content_length: int | None = 4000):
|
|
stored = row.get('content') or ''
|
|
decoded = self.db._decode_content(stored)
|
|
def text_parts(value) -> str:
|
|
if isinstance(value, str):
|
|
return value if not value.startswith(self.db._CONTENT_JSON_PREFIX) else ''
|
|
if isinstance(value, list):
|
|
return '\n'.join(text_parts(part) for part in value)
|
|
if isinstance(value, dict) and value.get('type') in (None, 'text', 'input_text', 'output_text'):
|
|
text = value.get('text')
|
|
return text if isinstance(text, str) else ''
|
|
return ''
|
|
content = text_parts(decoded)
|
|
if not content.strip() or redact_sensitive_text(content, force=True) != content:
|
|
return None
|
|
from tui_gateway.session_history import _is_display_hidden_marker, _legacy_display_kind
|
|
if _is_display_hidden_marker(row['role'], content) or _legacy_display_kind(row['role'], content):
|
|
return None
|
|
from agent.context_compressor import is_compaction_summary_message
|
|
|
|
if is_compaction_summary_message({'role': row['role'], 'content': content}):
|
|
return None
|
|
end = len(content) if content_length is None else min(len(content), content_offset + content_length)
|
|
return {'id': row['id'], 'session_id': row['session_id'], 'role': row['role'],
|
|
'content': content[content_offset:end],
|
|
'content_offset': content_offset, 'content_total_chars': len(content),
|
|
'next_content_offset': end if end < len(content) else None,
|
|
'content_truncated': content_offset > 0 or end < len(content),
|
|
'timestamp': row['timestamp'],
|
|
'content_hash': hashlib.sha256(content.encode()).hexdigest(),
|
|
'assertion': 'historical_user' if row['role'] == 'user' else 'historical_assistant_not_confirmed'}
|
|
|
|
def _scan(self, sid, boundary, *, backward=False, count=30):
|
|
"""Seek eligible rows, then fill by safe rows; retain the scanned frontier."""
|
|
comparison, order = ('<', 'DESC') if backward else ('>', 'ASC')
|
|
base = ('SELECT m.* FROM messages m JOIN sessions s ON s.id=m.session_id WHERE '
|
|
+ self._eligible_sql() + f' AND m.session_id=? AND m.id{comparison}?')
|
|
rows, cursor, scanned = [], boundary, 0
|
|
while scanned < _SCAN_BUDGET:
|
|
batch = self._rows(base + f' ORDER BY m.id {order} LIMIT ?',
|
|
(self.members_json, sid, cursor, min(_SCAN_BATCH, _SCAN_BUDGET - scanned)))
|
|
if not batch:
|
|
return rows, None
|
|
for row in batch:
|
|
cursor = row['id']
|
|
scanned += 1
|
|
if (safe := self._safe(row)) is not None:
|
|
rows.append(safe)
|
|
if len(rows) == count:
|
|
more = self._rows(base + ' LIMIT 1', (self.members_json, sid, cursor))
|
|
return rows, cursor if more else None
|
|
if len(batch) < _SCAN_BATCH:
|
|
return rows, None
|
|
more = self._rows(base + ' LIMIT 1', (self.members_json, sid, cursor))
|
|
return rows, cursor if more else None
|
|
|
|
def messages(self, sid, anchor=None, window=5, after_id=0, *, before_id=None,
|
|
content_offset=0, content_length=4000):
|
|
if sid not in self.allowed:
|
|
return []
|
|
self.read_has_more, self.read_cursor, self.before_cursor = False, None, None
|
|
base = 'SELECT m.* FROM messages m JOIN sessions s ON s.id=m.session_id WHERE '
|
|
where = self._eligible_sql() + ' AND m.session_id=?'
|
|
args = (self.members_json, sid)
|
|
if anchor is None:
|
|
rows, self.read_cursor = self._scan(sid, after_id)
|
|
else:
|
|
hit = self._rows(base + where + ' AND m.id=?', (*args, anchor))
|
|
safe = self._safe(hit[0], content_offset=content_offset,
|
|
content_length=content_length) if hit else None
|
|
if safe is None:
|
|
return []
|
|
before, self.before_cursor = self._scan(
|
|
sid, anchor if before_id is None else before_id, backward=True, count=window)
|
|
after, self.read_cursor = self._scan(sid, after_id or anchor, count=window)
|
|
rows = list(reversed(before)) + [safe] + after
|
|
self.read_has_more = self.read_cursor is not None or self.before_cursor is not None
|
|
return rows
|
|
|
|
def meta(self, sid):
|
|
rows = self._rows('''SELECT id,title,source,started_at,archived FROM sessions
|
|
WHERE id=? AND id IN (SELECT value FROM json_each(?)) AND COALESCE(hidden,0)=0''',
|
|
(sid, self.members_json))
|
|
if not rows:
|
|
return {}
|
|
meta = rows[0]
|
|
meta['title'] = redact_sensitive_text(meta.get('title') or '', force=True)
|
|
return meta
|
|
|
|
def response(self, **payload):
|
|
from hermes_project_scope import resolve_project_scope
|
|
|
|
now = resolve_project_scope(self.db, self.current_session_id)
|
|
if now.get('revision') != self.scope.get('revision') or now.get('status') != self.scope.get('status'):
|
|
return {'success': False, 'status': 'scope_changed', 'results': []}
|
|
def sources(value):
|
|
if isinstance(value, dict):
|
|
if 'content_hash' in value and 'session_id' in value:
|
|
yield value
|
|
for child in value.values():
|
|
yield from sources(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
yield from sources(child)
|
|
|
|
for source in sources(payload):
|
|
rows = self._rows('SELECT m.* FROM messages m JOIN sessions s ON s.id=m.session_id WHERE '
|
|
+ self._eligible_sql() + ' AND m.id=? AND m.session_id=?',
|
|
(self.members_json, source['id'], source['session_id']))
|
|
current = self._safe(rows[0]) if rows else None
|
|
if not current or current['content_hash'] != source['content_hash']:
|
|
return {'success': False, 'status': 'source_revoked', 'results': []}
|
|
public_scope = {k: v for k, v in self.scope.items() if k not in ('allowed_session_ids', 'sessions')}
|
|
public_scope['coverage'] = {k:v for k,v in public_scope.get('coverage', {}).items()
|
|
if k != 'scanned_sessions'}
|
|
return {'success': True, 'status': 'ok', 'scope': public_scope,
|
|
'retrieval': {'keyword': 'ready', 'holographic': 'not_connected', 'semantic': 'disabled'},
|
|
**payload}
|
|
|
|
def _discovery_entry(self, sid, mid, query, window):
|
|
from tools.session_search_tool import _session_link
|
|
|
|
if sid == self.current_session_id:
|
|
state = self._rows('SELECT compacted FROM messages WHERE id=? AND session_id=?', (mid, sid))
|
|
if not state or not state[0]['compacted']:
|
|
return None
|
|
source = self._rows('SELECT m.* FROM messages m JOIN sessions s ON s.id=m.session_id WHERE '
|
|
+ self._eligible_sql() + ' AND m.id=? AND m.session_id=?',
|
|
(self.members_json, mid, sid))
|
|
full = self._safe(source[0], content_length=None) if source else None
|
|
if full is None:
|
|
return None
|
|
rows = self.messages(sid, mid, window)
|
|
anchor = next((r for r in rows if r['id'] == mid), None)
|
|
if anchor is None or anchor['content_hash'] != full['content_hash']:
|
|
return None
|
|
terms = [t.strip('"\'*') for t in re.findall(r'"[^"]+"|\S+', query)
|
|
if t.upper() not in ('AND', 'OR', 'NOT', 'NEAR')]
|
|
match = re.search('|'.join(re.escape(t) for t in terms if t), full['content'], re.IGNORECASE) if any(terms) else None
|
|
start = max(0, match.start() - 200) if match else 0
|
|
return {'session_id': sid, 'match_message_id': mid, 'link': _session_link(sid),
|
|
'snippet': full['content'][start:start + 600], 'snippet_content_offset': start,
|
|
'messages': rows, 'next_before_message_id': self.before_cursor,
|
|
'next_after_message_id': self.read_cursor,
|
|
'bookend_start': [], 'bookend_end': [], **self.meta(sid)}
|
|
|
|
def _discover(self, query, window, limit, sort, search_cursor):
|
|
binding = hashlib.sha256(json.dumps([query, sort, self.current_session_id,
|
|
self.scope.get('revision')]).encode()).hexdigest()
|
|
offset = 0
|
|
if search_cursor is not None:
|
|
try:
|
|
if not isinstance(search_cursor, str) or len(search_cursor) > 2048:
|
|
raise ValueError('invalid cursor')
|
|
decoded = json.loads(base64.b64decode(search_cursor, altchars=b'-_', validate=True))
|
|
offset = decoded['offset']
|
|
if (decoded['binding'] != binding or decoded['v'] != 1
|
|
or type(offset) is not int or not 0 <= offset <= 2**63 - 1):
|
|
raise ValueError('invalid cursor')
|
|
except (ValueError, KeyError, TypeError, binascii.Error):
|
|
return {'success': False, 'status': 'invalid_cursor', 'results': []}
|
|
results, seen = [], set()
|
|
if search_cursor is None:
|
|
title_rows = self._rows('''SELECT id FROM sessions WHERE id IN (SELECT value FROM json_each(?))
|
|
AND lower(title)=lower(?) AND id!=? AND COALESCE(hidden,0)=0 ORDER BY started_at DESC LIMIT ?''',
|
|
(self.members_json, query.strip('`\"\''), self.current_session_id, limit))
|
|
for row in title_rows:
|
|
first, _ = self._scan(row['id'], 0, count=1)
|
|
if first and (entry := self._discovery_entry(row['id'], first[0]['id'], query, window)):
|
|
results.append(entry)
|
|
seen.add(row['id'])
|
|
scanned, complete = 0, False
|
|
while scanned < _SCAN_BUDGET and len(results) < limit:
|
|
batch_size = min(_SCAN_BATCH, _SCAN_BUDGET - scanned)
|
|
hits = self.db.search_messages(
|
|
query, allowed_session_ids=sorted(self.allowed), role_filter=['user', 'assistant'],
|
|
exclude_sources=['subagent', 'tool', 'kanban', 'cron'], include_inactive=False,
|
|
limit=batch_size, offset=offset, sort=sort, fields=['id', 'session_id'])
|
|
if not hits:
|
|
complete = True
|
|
break
|
|
consumed = 0
|
|
for hit in hits:
|
|
offset += 1
|
|
scanned += 1
|
|
consumed += 1
|
|
sid = hit['session_id']
|
|
if sid not in seen and (entry := self._discovery_entry(sid, hit['id'], query, window)):
|
|
results.append(entry)
|
|
seen.add(sid)
|
|
if len(results) >= limit:
|
|
break
|
|
if consumed == len(hits) and len(hits) < batch_size:
|
|
complete = True
|
|
break
|
|
cursor = None if complete else base64.urlsafe_b64encode(json.dumps(
|
|
{'v': 1, 'binding': binding, 'offset': offset}, separators=(',', ':')).encode()).decode()
|
|
return self.response(mode='discover', results=results, count=len(results),
|
|
truncated=not complete, next_search_cursor=cursor,
|
|
coverage={'eligible_sessions': len(self.allowed), 'candidate_limit': _SCAN_BUDGET,
|
|
'candidates_scanned': scanned, 'complete': complete,
|
|
'pagination': 'live_offset_not_snapshot; sessions may repeat across pages'})
|
|
|
|
def run(self, query='', session_id=None, around_message_id=None, window=5, limit=3, sort=None,
|
|
after_message_id=0, *, before_message_id=None, content_offset=0, content_length=4000,
|
|
search_cursor=None):
|
|
from tools.session_search_tool import _session_link
|
|
|
|
if self.scope.get('status') != 'ready':
|
|
resolution = self.scope.get('coverage', {}).get('resolution')
|
|
if self.scope.get('status') == 'scope_unresolved' and resolution == 'no_project' and (query or session_id == self.current_session_id):
|
|
self.self_only = True
|
|
self.allowed = frozenset([self.current_session_id])
|
|
self.members_json = json.dumps([self.current_session_id])
|
|
else:
|
|
return {'success': False, 'status': self.scope.get('status', 'scope_unresolved'), 'results': []}
|
|
if session_id:
|
|
if session_id not in self.allowed:
|
|
return {'success': False, 'status': 'access_denied', 'results': []}
|
|
if around_message_id is not None and (
|
|
(before_message_id is not None and before_message_id > around_message_id)
|
|
or (after_message_id and after_message_id < around_message_id)):
|
|
return {'success': False, 'status': 'invalid_cursor', 'results': []}
|
|
rows = self.messages(session_id, around_message_id, window, after_id=after_message_id,
|
|
before_id=before_message_id, content_offset=content_offset,
|
|
content_length=content_length)
|
|
if around_message_id is not None and not rows:
|
|
return {'success': False, 'status': 'source_unavailable', 'results': []}
|
|
return self.response(mode='scroll' if around_message_id is not None else 'read',
|
|
session_id=session_id, link=_session_link(session_id),
|
|
session_meta=self.meta(session_id), messages=rows,
|
|
truncated=self.read_has_more,
|
|
next_after_message_id=self.read_cursor,
|
|
next_before_message_id=self.before_cursor)
|
|
if not query:
|
|
rows = self._rows('''SELECT id FROM sessions WHERE id IN (SELECT value FROM json_each(?))
|
|
AND id!=? AND COALESCE(hidden,0)=0 ORDER BY started_at DESC,id DESC LIMIT ?''',
|
|
(self.members_json, self.current_session_id, limit))
|
|
results = [{'session_id': r['id'], 'link': _session_link(r['id']), **self.meta(r['id'])} for r in rows]
|
|
return self.response(mode='browse', results=results, count=len(results))
|
|
return self._discover(query, window, limit, sort, search_cursor)
|
|
|
|
|
|
def project_session_search(db, current_session_id, *, profile=None, session_id=None, **kwargs):
|
|
from hermes_cli.profiles import get_active_profile_name
|
|
|
|
active_profile = get_active_profile_name()
|
|
if session_id and '/' in session_id:
|
|
embedded_profile, session_id = session_id.split('/',1)
|
|
if embedded_profile != active_profile:
|
|
return json.dumps({'success':False,'status':'access_denied','results':[]})
|
|
if profile and profile != active_profile:
|
|
return json.dumps({'success':False,'status':'access_denied','results':[]})
|
|
if not current_session_id:
|
|
return json.dumps({'success':False,'status':'scope_unresolved','results':[]})
|
|
result = ProjectRecall(db, current_session_id).run(session_id=session_id, **kwargs)
|
|
return json.dumps(result, ensure_ascii=False)
|