Merge branch 'simp/r3-27-D' into simp/integration3
This commit is contained in:
@@ -25,26 +25,18 @@ logger = logging.getLogger(__name__)
|
||||
FACT_STORE_SCHEMA = {
|
||||
"name": "fact_store",
|
||||
"description": (
|
||||
"Deep structured memory with algebraic reasoning. "
|
||||
"Use alongside the memory tool — memory for always-on context, "
|
||||
"fact_store for deep recall and compositional queries.\n\n"
|
||||
"ACTIONS (simple → powerful):\n"
|
||||
"• add — Store a fact the user would expect you to remember.\n"
|
||||
"• search — Keyword lookup ('editor config', 'deploy process').\n"
|
||||
"• probe — Entity recall: ALL facts about a person/thing.\n"
|
||||
"• related — What connects to an entity? Structural adjacency.\n"
|
||||
"Deep structured memory with algebraic reasoning. Use alongside the memory tool — memory for always-on "
|
||||
"context, fact_store for deep recall and compositional queries.\n\nACTIONS (simple → powerful):\n"
|
||||
"• add — Store a fact the user would expect you to remember.\n• search — Keyword lookup ('editor config', 'deploy process').\n"
|
||||
"• probe — Entity recall: ALL facts about a person/thing.\n• related — What connects to an entity? Structural adjacency.\n"
|
||||
"• reason — Compositional: facts connected to MULTIPLE entities simultaneously.\n"
|
||||
"• contradict — Memory hygiene: find facts making conflicting claims.\n"
|
||||
"• update/remove/list — CRUD operations.\n\n"
|
||||
"• contradict — Memory hygiene: find facts making conflicting claims.\n• update/remove/list — CRUD operations.\n\n"
|
||||
"IMPORTANT: Before answering questions about the user, ALWAYS probe or reason first."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["add", "search", "probe", "related", "reason", "contradict", "update", "remove", "list"],
|
||||
},
|
||||
"action": {"type": "string", "enum": ["add", "search", "probe", "related", "reason", "contradict", "update", "remove", "list"]},
|
||||
"content": {"type": "string", "description": "Fact content (required for 'add')."},
|
||||
"query": {"type": "string", "description": "Search query (required for 'search')."},
|
||||
"entity": {"type": "string", "description": "Entity name for 'probe'/'related'."},
|
||||
@@ -62,31 +54,24 @@ FACT_STORE_SCHEMA = {
|
||||
|
||||
FACT_FEEDBACK_SCHEMA = {
|
||||
"name": "fact_feedback",
|
||||
"description": (
|
||||
"Rate a fact after using it. Mark 'helpful' if accurate, 'unhelpful' if outdated. "
|
||||
"This trains the memory — good facts rise, bad facts sink."
|
||||
),
|
||||
"description": ("Rate a fact after using it. Mark 'helpful' if accurate, 'unhelpful' if outdated. "
|
||||
"This trains the memory — good facts rise, bad facts sink."),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {"type": "string", "enum": ["helpful", "unhelpful"]},
|
||||
"fact_id": {"type": "integer", "description": "The fact ID to rate."},
|
||||
},
|
||||
"properties": {"action": {"type": "string", "enum": ["helpful", "unhelpful"]},
|
||||
"fact_id": {"type": "integer", "description": "The fact ID to rate."}},
|
||||
"required": ["action", "fact_id"],
|
||||
},
|
||||
}
|
||||
|
||||
# Auto-extraction patterns (on_session_end): user preferences -> user_pref, decisions -> project.
|
||||
_PREF_PATTERNS = [
|
||||
re.compile(r'\bI\s+(?:prefer|like|love|use|want|need)\s+(.+)', re.IGNORECASE),
|
||||
re.compile(r'\bmy\s+(?:favorite|preferred|default)\s+\w+\s+is\s+(.+)', re.IGNORECASE),
|
||||
re.compile(r'\bI\s+(?:always|never|usually)\s+(.+)', re.IGNORECASE),
|
||||
]
|
||||
_DECISION_PATTERNS = [
|
||||
re.compile(r'\bwe\s+(?:decided|agreed|chose)\s+(?:to\s+)?(.+)', re.IGNORECASE),
|
||||
re.compile(r'\bthe\s+project\s+(?:uses|needs|requires)\s+(.+)', re.IGNORECASE),
|
||||
]
|
||||
_EXTRACT_CATEGORIES = ((_PREF_PATTERNS, "user_pref"), (_DECISION_PATTERNS, "project"))
|
||||
# Auto-extraction (on_session_end): (patterns, category) — user preferences -> user_pref, decisions -> project.
|
||||
_EXTRACT_CATEGORIES = (
|
||||
([re.compile(r'\bI\s+(?:prefer|like|love|use|want|need)\s+(.+)', re.IGNORECASE),
|
||||
re.compile(r'\bmy\s+(?:favorite|preferred|default)\s+\w+\s+is\s+(.+)', re.IGNORECASE),
|
||||
re.compile(r'\bI\s+(?:always|never|usually)\s+(.+)', re.IGNORECASE)], "user_pref"),
|
||||
([re.compile(r'\bwe\s+(?:decided|agreed|chose)\s+(?:to\s+)?(.+)', re.IGNORECASE),
|
||||
re.compile(r'\bthe\s+project\s+(?:uses|needs|requires)\s+(.+)', re.IGNORECASE)], "project"),
|
||||
)
|
||||
|
||||
|
||||
def _load_plugin_config() -> dict:
|
||||
@@ -106,12 +91,9 @@ def _limit(args: dict) -> int:
|
||||
|
||||
|
||||
def _tool_handler(actions: dict):
|
||||
"""Return a bound-style handler dispatching on args["action"] over ``actions`` (unknown -> tool_error)."""
|
||||
def handle(self, args: dict) -> str:
|
||||
action = args["action"]
|
||||
handler = actions.get(action)
|
||||
return handler(self, args) if handler is not None else tool_error(f"Unknown action: {action}")
|
||||
return handle
|
||||
"""(self, args) handler dispatching on args["action"] over ``actions``; unknown action -> tool_error."""
|
||||
return lambda self, args: (actions[args["action"]](self, args) if args["action"] in actions
|
||||
else tool_error(f"Unknown action: {args['action']}"))
|
||||
|
||||
|
||||
class HolographicMemoryProvider(MemoryProvider):
|
||||
@@ -144,9 +126,8 @@ class HolographicMemoryProvider(MemoryProvider):
|
||||
|
||||
def get_config_schema(self):
|
||||
from hermes_constants import display_hermes_home
|
||||
_default_db = f"{display_hermes_home()}/memory_store.db"
|
||||
return [
|
||||
{"key": "db_path", "description": "SQLite database path", "default": _default_db},
|
||||
{"key": "db_path", "description": "SQLite database path", "default": f"{display_hermes_home()}/memory_store.db"},
|
||||
{"key": "auto_extract", "description": "Auto-extract facts at session end", "default": "false", "choices": ["true", "false"]},
|
||||
{"key": "default_trust", "description": "Default trust score for new facts", "default": "0.5"},
|
||||
{"key": "hrr_dim", "description": "HRR vector dimensions", "default": "1024"},
|
||||
@@ -160,10 +141,8 @@ class HolographicMemoryProvider(MemoryProvider):
|
||||
db_path = db_path.replace("$HERMES_HOME", _hermes_home).replace("${HERMES_HOME}", _hermes_home)
|
||||
hrr_dim = int(self._config.get("hrr_dim", 1024))
|
||||
self._store = MemoryStore(db_path=db_path, default_trust=float(self._config.get("default_trust", 0.5)), hrr_dim=hrr_dim)
|
||||
self._retriever = FactRetriever(
|
||||
store=self._store, temporal_decay_half_life=int(self._config.get("temporal_decay_half_life", 0)),
|
||||
hrr_weight=float(self._config.get("hrr_weight", 0.3)), hrr_dim=hrr_dim,
|
||||
)
|
||||
self._retriever = FactRetriever(store=self._store, hrr_dim=hrr_dim, hrr_weight=float(self._config.get("hrr_weight", 0.3)),
|
||||
temporal_decay_half_life=int(self._config.get("temporal_decay_half_life", 0)))
|
||||
self._session_id = session_id
|
||||
|
||||
def system_prompt_block(self) -> str:
|
||||
@@ -173,13 +152,11 @@ class HolographicMemoryProvider(MemoryProvider):
|
||||
total = self._store._conn.execute("SELECT COUNT(*) FROM facts").fetchone()[0]
|
||||
except Exception:
|
||||
total = 0
|
||||
body = (
|
||||
"Active. Empty fact store — proactively add facts the user would expect you to remember.\n"
|
||||
"Use fact_store(action='add') to store durable structured facts about people, projects, preferences, decisions.\n"
|
||||
) if total == 0 else (
|
||||
f"Active. {total} facts stored with entity resolution and trust scoring.\n"
|
||||
"Use fact_store to search, probe entities, reason across entities, or add facts.\n"
|
||||
)
|
||||
body = ("Active. Empty fact store — proactively add facts the user would expect you to remember.\n"
|
||||
"Use fact_store(action='add') to store durable structured facts about people, projects, preferences, decisions.\n"
|
||||
if total == 0 else
|
||||
f"Active. {total} facts stored with entity resolution and trust scoring.\n"
|
||||
"Use fact_store to search, probe entities, reason across entities, or add facts.\n")
|
||||
return "# Holographic Memory\n" + body + "Use fact_feedback to rate facts after using them (trains trust scores)."
|
||||
|
||||
def prefetch(self, query: str, *, session_id: str = "") -> str:
|
||||
@@ -197,11 +174,10 @@ class HolographicMemoryProvider(MemoryProvider):
|
||||
return [FACT_STORE_SCHEMA, FACT_FEEDBACK_SCHEMA]
|
||||
|
||||
def handle_tool_call(self, tool_name: str, args: Dict[str, Any], **kwargs) -> str:
|
||||
handler = self._TOOL_HANDLERS.get(tool_name)
|
||||
if handler is None:
|
||||
if tool_name not in self._TOOL_HANDLERS:
|
||||
return tool_error(f"Unknown tool: {tool_name}")
|
||||
try:
|
||||
return handler(self, args)
|
||||
return self._TOOL_HANDLERS[tool_name](self, args)
|
||||
except KeyError as exc:
|
||||
return tool_error(f"Missing required argument: {exc}")
|
||||
except Exception as exc:
|
||||
@@ -221,15 +197,13 @@ class HolographicMemoryProvider(MemoryProvider):
|
||||
logger.debug("Holographic memory_write mirror failed: %s", e)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# Close on the caller's thread: leaving the shared connection (and its write lock) to GC keeps it
|
||||
# alive on a long-running gateway. close() is idempotent and refcount-guarded.
|
||||
# Close on the caller's thread: leaving the shared connection (+ write lock) to GC keeps it alive on a gateway.
|
||||
if self._store is not None:
|
||||
try:
|
||||
self._store.close()
|
||||
except Exception as e:
|
||||
logger.debug("Holographic shutdown close() failed: %s", e)
|
||||
self._store = None
|
||||
self._retriever = None
|
||||
self._store = self._retriever = None
|
||||
|
||||
# Tool handlers (self, args) -> str. KeyError from args[...] / Exception -> tool_error in handle_tool_call;
|
||||
# argument coercion order (and therefore which error surfaces first) mirrors the underlying call order.
|
||||
@@ -259,28 +233,22 @@ class HolographicMemoryProvider(MemoryProvider):
|
||||
"fact_feedback": lambda self, a: json.dumps(self._store.record_feedback(int(a["fact_id"]), helpful=a["action"] == "helpful")),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _harvestable_text(msg: dict):
|
||||
"""User text eligible for extraction, or None. Compaction handoff summaries arrive as role="user" and
|
||||
reliably match the decision patterns; never store the compactor's own output as a durable fact. A
|
||||
merge-into-tail row holds genuine prior user text BEFORE _MERGED_SUMMARY_DELIMITER (prefixed with the
|
||||
header) and the summary AFTER it — harvest only the pre-delimiter segment."""
|
||||
# Local import: the compressor module is heavier than this plugin and only needed here.
|
||||
from agent.context_compressor import _MERGED_PRIOR_CONTEXT_HEADER, _MERGED_SUMMARY_DELIMITER, is_compaction_summary_message
|
||||
|
||||
content, pre = msg.get("content", ""), ""
|
||||
if isinstance(content, str) and _MERGED_SUMMARY_DELIMITER in content:
|
||||
pre = content.split(_MERGED_SUMMARY_DELIMITER, 1)[0].removeprefix(_MERGED_PRIOR_CONTEXT_HEADER).strip()
|
||||
if pre:
|
||||
content = pre
|
||||
elif is_compaction_summary_message(msg):
|
||||
return None
|
||||
return content if isinstance(content, str) and len(content) >= 10 else None
|
||||
|
||||
def _auto_extract_facts(self, messages: list) -> None:
|
||||
# Compaction handoff summaries arrive as role="user" and match the decision patterns; never store the
|
||||
# compactor's own output as a fact. A merge-into-tail row holds genuine prior user text BEFORE
|
||||
# _MERGED_SUMMARY_DELIMITER (after the header) and the summary AFTER it — harvest only that segment.
|
||||
from agent.context_compressor import _MERGED_PRIOR_CONTEXT_HEADER, _MERGED_SUMMARY_DELIMITER, is_compaction_summary_message # heavy; lazy
|
||||
extracted = 0
|
||||
texts = filter(None, (self._harvestable_text(m) for m in messages if m.get("role") == "user"))
|
||||
for content in texts:
|
||||
for msg in messages:
|
||||
content = msg.get("content", "") if msg.get("role") == "user" else None
|
||||
pre = content.split(_MERGED_SUMMARY_DELIMITER, 1)[0].removeprefix(_MERGED_PRIOR_CONTEXT_HEADER).strip() \
|
||||
if isinstance(content, str) and _MERGED_SUMMARY_DELIMITER in content else ""
|
||||
if pre:
|
||||
content = pre
|
||||
elif content is None or is_compaction_summary_message(msg):
|
||||
continue
|
||||
if not isinstance(content, str) or len(content) < 10:
|
||||
continue
|
||||
for patterns, category in _EXTRACT_CATEGORIES:
|
||||
if any(p.search(content) for p in patterns):
|
||||
try:
|
||||
@@ -291,7 +259,6 @@ class HolographicMemoryProvider(MemoryProvider):
|
||||
if extracted:
|
||||
logger.info("Auto-extracted %d facts from conversation", extracted)
|
||||
|
||||
|
||||
def register(ctx) -> None:
|
||||
"""Register the holographic memory provider with the plugin system."""
|
||||
ctx.register_memory_provider(HolographicMemoryProvider(config=_load_plugin_config()))
|
||||
|
||||
@@ -1,17 +1,13 @@
|
||||
"""Holographic Reduced Representations (HRR) with phase encoding.
|
||||
|
||||
Each concept is a vector of angles in [0, 2π): bind = circular convolution (phase addition),
|
||||
unbind = circular correlation (phase subtraction), bundle = superposition (circular mean).
|
||||
Phase encoding avoids the magnitude collapse of complex-number HRRs and maps cleanly to
|
||||
cosine similarity. Atoms derive deterministically from SHA-256 so representations are
|
||||
identical across processes, machines, and Python versions.
|
||||
References: Plate (1995) HRRs; Gayler (2004) Vector Symbolic Architectures.
|
||||
"""
|
||||
"""Holographic Reduced Representations (HRR) with phase encoding. Each concept is a vector of angles in [0, 2π):
|
||||
bind = circular convolution (phase addition), unbind = circular correlation (phase subtraction), bundle =
|
||||
superposition (circular mean). Phase encoding avoids the magnitude collapse of complex-number HRRs and maps cleanly
|
||||
to cosine similarity; atoms derive deterministically from SHA-256 so representations are identical across processes,
|
||||
machines, and Python versions. References: Plate (1995) HRRs; Gayler (2004) Vector Symbolic Architectures."""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import struct
|
||||
import math
|
||||
import struct
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
@@ -37,9 +33,8 @@ def encode_atom(word: str, dim: int = 1024) -> "np.ndarray":
|
||||
"""Deterministic phase vector: SHA-256 counter blocks of f"{word}:{i}" -> uint16 -> [0, 2π).
|
||||
hashlib rather than numpy RNG so atoms are reproducible across platforms."""
|
||||
_require_numpy()
|
||||
uint16_values: list[int] = []
|
||||
for i in range(math.ceil(dim / 16)): # 32-byte digest = 16 uint16 values
|
||||
uint16_values.extend(struct.unpack("<16H", hashlib.sha256(f"{word}:{i}".encode()).digest()))
|
||||
uint16_values = [v for i in range(math.ceil(dim / 16)) # 32-byte digest = 16 uint16 values
|
||||
for v in struct.unpack("<16H", hashlib.sha256(f"{word}:{i}".encode()).digest())]
|
||||
return np.array(uint16_values[:dim], dtype=np.float64) * (_TWO_PI / 65536.0)
|
||||
|
||||
|
||||
@@ -84,11 +79,8 @@ def encode_fact(content: str, entities: list[str], dim: int = 1024) -> "np.ndarr
|
||||
|
||||
|
||||
def phases_to_bytes(phases: "np.ndarray", dim: int | None = None) -> bytes:
|
||||
"""Serialize as prefixed float32 (half the size of legacy float64 blobs).
|
||||
|
||||
At dim=1 the prefixed float32 blob and a raw float64 blob are both 8 bytes,
|
||||
so we write legacy float64 there to keep ``bytes_to_phases`` unambiguous.
|
||||
"""
|
||||
"""Serialize as prefixed float32 (half the size of legacy float64 blobs). At dim=1 the prefixed float32 blob
|
||||
and a raw float64 blob are both 8 bytes, so write legacy float64 there to keep ``bytes_to_phases`` unambiguous."""
|
||||
_require_numpy()
|
||||
dim = int(phases.shape[0]) if dim is None else dim
|
||||
if len(_FLOAT32_BLOB_PREFIX) + dim * _F32 == dim * _F64:
|
||||
@@ -97,11 +89,9 @@ def phases_to_bytes(phases: "np.ndarray", dim: int | None = None) -> bytes:
|
||||
|
||||
|
||||
def bytes_to_phases(data: bytes, dim: int | None = None) -> "np.ndarray":
|
||||
"""Deserialize prefixed float32 or legacy raw float64 blobs (always returns float64).
|
||||
|
||||
With ``dim`` given, a prefixed blob whose size equals the float64 size (dim=1) is read
|
||||
as legacy float64: ``phases_to_bytes`` never writes a prefixed blob at that size.
|
||||
"""
|
||||
"""Deserialize prefixed float32 or legacy raw float64 blobs (always returns float64). With ``dim`` given, a
|
||||
prefixed blob whose size equals the float64 size (dim=1) is read as legacy float64: ``phases_to_bytes`` never
|
||||
writes a prefixed blob at that size."""
|
||||
_require_numpy()
|
||||
plen = len(_FLOAT32_BLOB_PREFIX)
|
||||
prefixed = data.startswith(_FLOAT32_BLOB_PREFIX)
|
||||
@@ -120,10 +110,9 @@ def bytes_to_phases(data: bytes, dim: int | None = None) -> "np.ndarray":
|
||||
return f64(data)
|
||||
if not prefixed:
|
||||
raise ValueError(f"HRR legacy vector blob has {len(data)} bytes; expected {float64_bytes} (float64) for dim={dim}")
|
||||
expected = (f"{float64_bytes} (legacy float64)" if collides
|
||||
else f"{float32_blob_bytes} (prefixed float32) or {float64_bytes} (legacy float64)")
|
||||
raise ValueError(f"HRR vector blob has {len(data)} bytes ({len(data) - plen} payload bytes after "
|
||||
f"the float32 prefix); expected {expected} for dim={dim}")
|
||||
expected = f"{float64_bytes} (legacy float64)" if collides else f"{float32_blob_bytes} (prefixed float32) or {float64_bytes} (legacy float64)"
|
||||
raise ValueError(f"HRR vector blob has {len(data)} bytes ({len(data) - plen} payload bytes after the float32 prefix); "
|
||||
f"expected {expected} for dim={dim}")
|
||||
|
||||
|
||||
def snr_estimate(dim: int, n_items: int) -> float:
|
||||
|
||||
@@ -11,28 +11,21 @@ from typing import TYPE_CHECKING
|
||||
if TYPE_CHECKING:
|
||||
from .store import MemoryStore
|
||||
|
||||
try:
|
||||
from . import holographic as hrr
|
||||
except ImportError:
|
||||
import holographic as hrr # type: ignore[no-redef]
|
||||
from . import holographic as hrr
|
||||
|
||||
_FACT_COLUMNS = (
|
||||
"fact_id, content, category, tags, trust_score, "
|
||||
"retrieval_count, helpful_count, created_at, updated_at"
|
||||
)
|
||||
_FACT_COLUMNS = "fact_id, content, category, tags, trust_score, retrieval_count, helpful_count, created_at, updated_at"
|
||||
_ROLE_ENTITY, _ROLE_CONTENT = hrr.ROLE_ENTITY, hrr.ROLE_CONTENT
|
||||
_PUNCT = ".,;:!?\"'()[]{}#@<>"
|
||||
_FTS_OPERATORS = str.maketrans("", "", '"()*^:-+')
|
||||
# Stopwords dropped before FTS5 OR-expansion: short English function words that
|
||||
# carry no retrieval signal and force false-negative AND matches.
|
||||
_FTS_STOPWORDS = frozenset("""
|
||||
a about above after again all am an and any are as at be because been before being between both but by
|
||||
can could did do does doing don down during each few for from further had has have having he her here hers
|
||||
herself him himself his how i if in into is it its itself just me more most my myself no nor not now of off
|
||||
on once only or other our ours ourselves out over own same she should so some such than that the their theirs
|
||||
them themselves then there these they this those through to too under until up very was we were what when
|
||||
where which while who whom why will with would you your yours yourself yourselves
|
||||
""".split())
|
||||
a about above after again all am an and any are as at be because been before being between both but by can could
|
||||
did do does doing don down during each few for from further had has have having he her here hers herself him himself
|
||||
his how i if in into is it its itself just me more most my myself no nor not now of off on once only or other our
|
||||
ours ourselves out over own same she should so some such than that the their theirs them themselves then there these
|
||||
they this those through to too under until up very was we were what when where which while who whom why will with
|
||||
would you your yours yourself yourselves""".split())
|
||||
|
||||
|
||||
def _shift(sim: float) -> float:
|
||||
@@ -57,14 +50,12 @@ class FactRetriever:
|
||||
return hrr.bytes_to_phases(blob, dim=self.hrr_dim)
|
||||
|
||||
def search(self, query: str, category: str | None = None, min_trust: float = 0.3, limit: int = 10) -> list[dict]:
|
||||
"""Hybrid search: FTS5 candidates (limit*3) → Jaccard + HRR rerank → trust weighting →
|
||||
optional temporal decay 0.5^(age_days / half_life). Returns fact dicts with 'score', sorted desc."""
|
||||
"""FTS5 candidates (limit*3) → Jaccard + HRR rerank → trust weighting → optional temporal decay
|
||||
0.5^(age_days / half_life). Returns fact dicts with 'score', sorted desc."""
|
||||
candidates = self._fts_candidates(query, category, min_trust, limit * 3)
|
||||
if not candidates:
|
||||
return []
|
||||
query_tokens = self._tokenize(query)
|
||||
# Query vector is loop-invariant; encode lazily on the first candidate that carries
|
||||
# an HRR vector so stores whose hrr_vector was never backfilled don't pay for it.
|
||||
# Query vector is loop-invariant; encode lazily on the first candidate that carries an HRR vector
|
||||
# so stores whose hrr_vector was never backfilled don't pay for it.
|
||||
query_vec = None
|
||||
for fact in candidates:
|
||||
jaccard = self._jaccard_similarity(query_tokens, self._tokenize(fact["content"]) | self._tokenize(fact.get("tags", "")))
|
||||
@@ -83,18 +74,14 @@ class FactRetriever:
|
||||
fact.pop("hrr_vector", None) # callers expect JSON-serializable dicts
|
||||
return results
|
||||
|
||||
def _vector_query(self, fallback: str, category: str | None, limit: int, make_sim: Callable[[], Callable]) -> list[dict]:
|
||||
"""Rank every fact vector (optionally per category) with the sim fn built by make_sim(); FTS5 fallback
|
||||
when no vectors exist. make_sim runs after the rows check so role atoms are encoded once, never for fallback."""
|
||||
def _vector_query(self, fallback: str, category: str | None, limit: int, sim_fn: Callable) -> list[dict]:
|
||||
"""Rank every fact vector (optionally per category) by sim_fn; FTS5 fallback when no vectors exist."""
|
||||
rows = self._vector_rows(category)
|
||||
if not rows:
|
||||
return self.search(fallback, category=category, limit=limit)
|
||||
return self._rank_by_vector(rows, make_sim(), limit)
|
||||
return self._rank_by_vector(rows, sim_fn, limit) if rows else self.search(fallback, category=category, limit=limit)
|
||||
|
||||
def probe(self, entity: str, category: str | None = None, limit: int = 10) -> list[dict]:
|
||||
"""Compositional entity query: unbind bind(entity, ROLE_ENTITY) from the category bank
|
||||
(or each fact vector) to find facts where the entity plays a structural role.
|
||||
Not keyword search. Falls back to FTS5 without numpy."""
|
||||
"""Compositional entity query: unbind bind(entity, ROLE_ENTITY) from the category bank (or each fact vector)
|
||||
to find facts where the entity plays a structural role. Not keyword search. Falls back to FTS5 without numpy."""
|
||||
if not hrr._HAS_NUMPY:
|
||||
return self.search(entity, category=category, limit=limit)
|
||||
probe_key = hrr.bind(self._atom(entity.lower()), self._atom(_ROLE_ENTITY))
|
||||
@@ -103,47 +90,35 @@ class FactRetriever:
|
||||
if bank_row:
|
||||
extracted = hrr.unbind(self._phases(bank_row["vector"]), probe_key)
|
||||
return self._rank_by_vector(self._vector_rows(category), lambda _f, fact_vec: hrr.similarity(extracted, fact_vec), limit)
|
||||
|
||||
def make_sim():
|
||||
role_content = self._atom(_ROLE_CONTENT)
|
||||
# Does unbinding the probe key leave the fact's content signal?
|
||||
return lambda fact, fact_vec: hrr.similarity(
|
||||
hrr.unbind(fact_vec, probe_key), hrr.bind(hrr.encode_text(fact["content"], self.hrr_dim), role_content))
|
||||
|
||||
return self._vector_query(entity, category, limit, make_sim)
|
||||
role_content = self._atom(_ROLE_CONTENT) # loop-invariant: encode once, not per row
|
||||
# Does unbinding the probe key leave the fact's content signal?
|
||||
return self._vector_query(entity, category, limit, lambda fact, fact_vec: hrr.similarity(
|
||||
hrr.unbind(fact_vec, probe_key), hrr.bind(hrr.encode_text(fact["content"], self.hrr_dim), role_content)))
|
||||
|
||||
def related(self, entity: str, category: str | None = None, limit: int = 10) -> list[dict]:
|
||||
"""Facts structurally connected to an entity (shared context), not just facts
|
||||
*about* it as in probe. Falls back to FTS5 without numpy."""
|
||||
"""Facts structurally connected to an entity (shared context), not just facts *about* it as in probe.
|
||||
Falls back to FTS5 without numpy."""
|
||||
if not hrr._HAS_NUMPY:
|
||||
return self.search(entity, category=category, limit=limit)
|
||||
entity_vec = self._atom(entity.lower()) # bare atom, not role-bound: ANY structural match
|
||||
|
||||
def make_sim():
|
||||
roles = (self._atom(_ROLE_ENTITY), self._atom(_ROLE_CONTENT))
|
||||
# A residual similar to ANY role vector means the entity plays a structural role.
|
||||
return lambda _f, fact_vec: max(hrr.similarity(hrr.unbind(fact_vec, entity_vec), role) for role in roles)
|
||||
|
||||
return self._vector_query(entity, category, limit, make_sim)
|
||||
roles = (self._atom(_ROLE_ENTITY), self._atom(_ROLE_CONTENT)) # loop-invariant: encode once
|
||||
# A residual similar to ANY role vector means the entity plays a structural role in the fact.
|
||||
return self._vector_query(entity, category, limit, lambda _f, fact_vec: max(
|
||||
hrr.similarity(hrr.unbind(fact_vec, entity_vec), role) for role in roles))
|
||||
|
||||
def reason(self, entities: list[str], category: str | None = None, limit: int = 10) -> list[dict]:
|
||||
"""Multi-entity compositional query (vector-space JOIN): facts where ALL entities
|
||||
play structural roles. Falls back to FTS5 without numpy."""
|
||||
"""Multi-entity compositional query (vector-space JOIN): facts where ALL entities play structural roles.
|
||||
Falls back to FTS5 without numpy."""
|
||||
if not hrr._HAS_NUMPY or not entities:
|
||||
return self.search(" ".join(entities), category=category, limit=limit)
|
||||
role_entity = self._atom(_ROLE_ENTITY)
|
||||
role_entity, role_content = self._atom(_ROLE_ENTITY), self._atom(_ROLE_CONTENT)
|
||||
probe_keys = [hrr.bind(self._atom(entity.lower()), role_entity) for entity in entities]
|
||||
|
||||
def make_sim():
|
||||
role_content = self._atom(_ROLE_CONTENT)
|
||||
# AND semantics via min: high only if EVERY entity is structurally present.
|
||||
return lambda _f, fact_vec: min(hrr.similarity(hrr.unbind(fact_vec, key), role_content) for key in probe_keys)
|
||||
|
||||
return self._vector_query(" ".join(entities), category, limit, make_sim)
|
||||
# AND semantics via min: high only if EVERY entity is structurally present.
|
||||
return self._vector_query(" ".join(entities), category, limit, lambda _f, fact_vec: min(
|
||||
hrr.similarity(hrr.unbind(fact_vec, key), role_content) for key in probe_keys))
|
||||
|
||||
def contradict(self, category: str | None = None, threshold: float = 0.3, limit: int = 10) -> list[dict]:
|
||||
"""Memory hygiene: pairs of facts that share entities (same subject) but have low
|
||||
content-vector similarity (different claims). Empty without numpy."""
|
||||
"""Pairs of facts sharing entities (same subject) with low content-vector similarity (different claims). Empty without numpy."""
|
||||
if not hrr._HAS_NUMPY:
|
||||
return []
|
||||
rows = self._vector_rows(category, columns="fact_id, content, category, tags, trust_score, created_at, updated_at, hrr_vector")
|
||||
@@ -234,9 +209,7 @@ class FactRetriever:
|
||||
return 1.0
|
||||
try:
|
||||
ts = datetime.fromisoformat(timestamp_str.replace("Z", "+00:00")) if isinstance(timestamp_str, str) else timestamp_str
|
||||
if ts.tzinfo is None:
|
||||
ts = ts.replace(tzinfo=timezone.utc)
|
||||
age_days = (datetime.now(timezone.utc) - ts).total_seconds() / 86400
|
||||
age_days = (datetime.now(timezone.utc) - (ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc))).total_seconds() / 86400
|
||||
return 1.0 if age_days < 0 else math.pow(0.5, age_days / self.half_life)
|
||||
except (ValueError, TypeError):
|
||||
return 1.0
|
||||
|
||||
@@ -6,10 +6,7 @@ import sqlite3
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from . import holographic as hrr
|
||||
except ImportError:
|
||||
import holographic as hrr # type: ignore[no-redef]
|
||||
from . import holographic as hrr
|
||||
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS facts (
|
||||
@@ -73,23 +70,16 @@ CREATE TABLE IF NOT EXISTS memory_banks (
|
||||
);
|
||||
"""
|
||||
|
||||
_HELPFUL_DELTA = 0.05
|
||||
_UNHELPFUL_DELTA = -0.10
|
||||
_HELPFUL_DELTA, _UNHELPFUL_DELTA = 0.05, -0.10
|
||||
|
||||
# Entity extraction patterns, applied in order: capitalized multi-word phrases
|
||||
# ("John Doe"), double-quoted terms, single-quoted terms, then "X aka Y" (both sides).
|
||||
_RE_SINGLE_ENTITY = (
|
||||
re.compile(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b'),
|
||||
re.compile(r'"([^"]+)"'),
|
||||
re.compile(r"'([^']+)'"),
|
||||
)
|
||||
# Entity extraction patterns, applied in order: capitalized multi-word phrases ("John Doe"), double-quoted terms,
|
||||
# single-quoted terms, then "X aka Y" (both sides).
|
||||
_RE_SINGLE_ENTITY = (re.compile(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b'), re.compile(r'"([^"]+)"'), re.compile(r"'([^']+)'"))
|
||||
_RE_AKA = re.compile(r'(\w+(?:\s+\w+)*)\s+(?:aka|also known as)\s+(\w+(?:\s+\w+)*)', re.IGNORECASE)
|
||||
_ENTITY_NAMES_SQL = "SELECT e.name FROM entities e JOIN fact_entities fe ON fe.entity_id = e.entity_id WHERE fe.fact_id = ?"
|
||||
# Entity lookup order: exact name, then aliases (comma-separated; wrapped in commas for whole-alias matching).
|
||||
_ENTITY_LOOKUPS = (
|
||||
"SELECT entity_id FROM entities WHERE name LIKE ?",
|
||||
"SELECT entity_id FROM entities WHERE ',' || aliases || ',' LIKE '%,' || ? || ',%'",
|
||||
)
|
||||
_ENTITY_LOOKUPS = ("SELECT entity_id FROM entities WHERE name LIKE ?",
|
||||
"SELECT entity_id FROM entities WHERE ',' || aliases || ',' LIKE '%,' || ? || ',%'")
|
||||
|
||||
|
||||
def _clamp_trust(value: float) -> float:
|
||||
@@ -121,21 +111,20 @@ class MemoryStore:
|
||||
with MemoryStore._shared_guard:
|
||||
entry = MemoryStore._shared.get(self._key)
|
||||
if entry is None:
|
||||
# Autocommit: a write that raises mid-method can never leave a dangling
|
||||
# transaction (and its write lock) open; explicit commit() calls are no-ops.
|
||||
# Autocommit: a write that raises mid-method can't leave a dangling transaction (and its
|
||||
# write lock) open; the explicit commit() calls in _write are then harmless no-ops.
|
||||
conn = sqlite3.connect(self._key, check_same_thread=False, timeout=10.0, isolation_level=None)
|
||||
conn.row_factory = sqlite3.Row
|
||||
entry = MemoryStore._shared[self._key] = {"conn": conn, "lock": threading.RLock(), "refs": 0, "ready": False}
|
||||
entry["refs"] += 1
|
||||
self._entry, self._conn, self._lock = entry, entry["conn"], entry["lock"]
|
||||
with self._lock: # schema initialised once per shared connection
|
||||
if not self._entry["ready"]:
|
||||
if not entry["ready"]:
|
||||
self._init_db()
|
||||
self._entry["ready"] = True
|
||||
entry["ready"] = True
|
||||
|
||||
def _init_db(self) -> None:
|
||||
"""Create schema, enable WAL via the shared fallback helper (NFS/SMB/FUSE HERMES_HOME
|
||||
degrades gracefully), and add hrr_vector to pre-HRR databases."""
|
||||
"""Create schema, enable WAL via the shared fallback helper (NFS/SMB/FUSE degrade gracefully), add hrr_vector to pre-HRR DBs."""
|
||||
from hermes_state import apply_wal_with_fallback
|
||||
apply_wal_with_fallback(self._conn, db_label="memory_store.db (holographic)")
|
||||
self._conn.executescript(_SCHEMA)
|
||||
@@ -152,8 +141,8 @@ class MemoryStore:
|
||||
return cur
|
||||
|
||||
def add_fact(self, content: str, category: str = "general", tags: str = "") -> int:
|
||||
"""Insert a fact and return its fact_id; on duplicate content (UNIQUE) return the
|
||||
existing fact_id untouched. Links extracted entities and rebuilds the category bank."""
|
||||
"""Insert a fact and return its fact_id; on duplicate content (UNIQUE) return the existing fact_id untouched.
|
||||
Links extracted entities and rebuilds the category bank."""
|
||||
with self._lock:
|
||||
content = content.strip()
|
||||
if not content:
|
||||
@@ -229,9 +218,8 @@ class MemoryStore:
|
||||
for m in _RE_AKA.finditer(text):
|
||||
raw += [m.group(1), m.group(2)]
|
||||
uniq: dict[str, str] = {} # lower-cased key -> first-seen spelling, insertion-ordered
|
||||
for name in (n.strip() for n in raw):
|
||||
if name:
|
||||
uniq.setdefault(name.lower(), name)
|
||||
for name in filter(None, (n.strip() for n in raw)):
|
||||
uniq.setdefault(name.lower(), name)
|
||||
return list(uniq.values())
|
||||
|
||||
def _link_entities(self, fact_id: int, content: str) -> None:
|
||||
@@ -267,13 +255,10 @@ class MemoryStore:
|
||||
return
|
||||
bank_vector = hrr.bundle(*[hrr.bytes_to_phases(row["hrr_vector"], dim=self.hrr_dim) for row in rows])
|
||||
hrr.snr_estimate(self.hrr_dim, len(rows)) # warns when near capacity
|
||||
self._write(
|
||||
"INSERT INTO memory_banks (bank_name, vector, dim, fact_count, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) ON CONFLICT(bank_name) DO UPDATE SET "
|
||||
"vector = excluded.vector, dim = excluded.dim, fact_count = excluded.fact_count, "
|
||||
"updated_at = excluded.updated_at",
|
||||
(bank_name, hrr.phases_to_bytes(bank_vector), self.hrr_dim, len(rows)),
|
||||
)
|
||||
self._write("INSERT INTO memory_banks (bank_name, vector, dim, fact_count, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) ON CONFLICT(bank_name) DO UPDATE SET "
|
||||
"vector = excluded.vector, dim = excluded.dim, fact_count = excluded.fact_count, "
|
||||
"updated_at = excluded.updated_at", (bank_name, hrr.phases_to_bytes(bank_vector), self.hrr_dim, len(rows)))
|
||||
|
||||
@classmethod
|
||||
def release_all_under(cls, directory: "str | Path") -> int:
|
||||
@@ -283,9 +268,8 @@ class MemoryStore:
|
||||
stale holder is expected to fail."""
|
||||
root = os.path.normcase(str(Path(directory).expanduser().resolve())) + os.sep
|
||||
with cls._shared_guard:
|
||||
doomed = [key for key in cls._shared if os.path.normcase(key).startswith(root)]
|
||||
for key in doomed:
|
||||
entry = cls._shared.pop(key)
|
||||
doomed = [cls._shared.pop(key) for key in list(cls._shared) if os.path.normcase(key).startswith(root)]
|
||||
for entry in doomed:
|
||||
try:
|
||||
with entry["lock"]:
|
||||
entry["conn"].close()
|
||||
|
||||
@@ -40,13 +40,6 @@ def _is_client_error(exc: Exception) -> bool:
|
||||
return type(exc).__name__ in _CLIENT_ERROR_TYPES or any(s in err_str for s in ("404", "not found", "valid uuid"))
|
||||
|
||||
|
||||
def _truthy(value: Any, falsy_strings: tuple[str, ...] | None = None) -> bool:
|
||||
"""Coerce a flag; strings match ``falsy_strings`` (deny-list) when given, else a truthy allow-list."""
|
||||
if not isinstance(value, str):
|
||||
return bool(value)
|
||||
return value.lower() not in falsy_strings if falsy_strings else value.lower() in ("true", "1", "yes")
|
||||
|
||||
|
||||
def _read_mem0_json(config_path: Path) -> dict:
|
||||
"""Best-effort read of mem0.json; missing/corrupt file -> {}."""
|
||||
if config_path.exists():
|
||||
@@ -117,9 +110,7 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
"""Merge-write config to $HERMES_HOME/mem0.json."""
|
||||
from utils import atomic_json_write
|
||||
config_path = Path(hermes_home) / "mem0.json"
|
||||
existing = _read_mem0_json(config_path)
|
||||
existing.update(values)
|
||||
atomic_json_write(config_path, existing, mode=0o600)
|
||||
atomic_json_write(config_path, {**_read_mem0_json(config_path), **values}, mode=0o600)
|
||||
|
||||
def get_config_schema(self):
|
||||
api_key_required = _load_config().get("mode", "platform") != "oss"
|
||||
@@ -137,9 +128,7 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
|
||||
def _oss_hint(self, template: str, default: str = "vector store") -> str:
|
||||
"""OSS-only hint; ``{vs}`` is the configured vector-store provider. "" in other modes."""
|
||||
if self._mode != "oss":
|
||||
return ""
|
||||
return template.format(vs=self._config.get("oss", {}).get("vector_store", {}).get("provider", default))
|
||||
return template.format(vs=self._config.get("oss", {}).get("vector_store", {}).get("provider", default)) if self._mode == "oss" else ""
|
||||
|
||||
def _create_backend(self):
|
||||
# Lazy-install the mem0 SDK before the backend imports it (honors security.allow_lazy_installs);
|
||||
@@ -151,9 +140,7 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
from . import _backend
|
||||
if self._mode == "oss":
|
||||
return _backend.OSSBackend(self._config.get("oss", {}))
|
||||
if self._host:
|
||||
return _backend.SelfHostedBackend(self._api_key, self._host)
|
||||
return _backend.PlatformBackend(self._api_key)
|
||||
return _backend.SelfHostedBackend(self._api_key, self._host) if self._host else _backend.PlatformBackend(self._api_key)
|
||||
except Exception as e:
|
||||
logger.error("Mem0 backend failed to initialize (%s mode): %s", self._mode, e)
|
||||
self._init_error = str(e)
|
||||
@@ -191,12 +178,12 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
"""Background-path wrapper: run ``call`` under the breaker; on error log ``msg`` and return None."""
|
||||
try:
|
||||
result = call()
|
||||
self._record_success()
|
||||
return result
|
||||
except Exception as e:
|
||||
self._record_failure()
|
||||
log(msg, e)
|
||||
return None
|
||||
self._record_success()
|
||||
return result
|
||||
|
||||
def initialize(self, session_id: str, **kwargs) -> None:
|
||||
self._config = cfg = _load_config()
|
||||
@@ -206,7 +193,8 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
configured = cfg.get("user_id")
|
||||
self._user_id = (None if configured == _DEFAULT_USER_ID else configured) or kwargs.get("user_id") or _DEFAULT_USER_ID
|
||||
# Persisted rerank preference: default for mem0_search when the model omits ``rerank``. Platform-only.
|
||||
self._rerank_default = _truthy(cfg.get("rerank", False))
|
||||
_rr = cfg.get("rerank", False)
|
||||
self._rerank_default = _rr.lower() in ("true", "1", "yes") if isinstance(_rr, str) else bool(_rr)
|
||||
self._channel = kwargs.get("platform") or "cli"
|
||||
self._backend = self._create_backend()
|
||||
if self._backend and not self._atexit_registered:
|
||||
@@ -243,11 +231,6 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
backend = self._backend
|
||||
if not query or backend is None or self._is_breaker_open():
|
||||
return
|
||||
with self._prefetch_lock:
|
||||
# Same query already answered or still in flight: don't restart it.
|
||||
if self._prefetch_query == query and (self._prefetch_done or (self._prefetch_thread and self._prefetch_thread.is_alive())):
|
||||
return
|
||||
self._prefetch_query, self._prefetch_result, self._prefetch_done = query, "", False
|
||||
|
||||
def _run():
|
||||
results = self._try(lambda: self._search(query, backend=backend), logger.debug, "Mem0 prefetch failed: %s")
|
||||
@@ -257,9 +240,12 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
if self._prefetch_query == query:
|
||||
self._prefetch_result, self._prefetch_done = body, True
|
||||
|
||||
t = threading.Thread(target=_run, daemon=True, name="mem0-prefetch")
|
||||
with self._prefetch_lock:
|
||||
self._prefetch_thread = t
|
||||
# Same query already answered or still in flight: don't restart it.
|
||||
if self._prefetch_query == query and (self._prefetch_done or (self._prefetch_thread and self._prefetch_thread.is_alive())):
|
||||
return
|
||||
self._prefetch_query, self._prefetch_result, self._prefetch_done = query, "", False
|
||||
self._prefetch_thread = t = threading.Thread(target=_run, daemon=True, name="mem0-prefetch")
|
||||
t.start()
|
||||
|
||||
def prefetch(self, query: str, *, session_id: str = "") -> str:
|
||||
@@ -271,8 +257,7 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
thread = self._prefetch_thread if self._prefetch_query == query else None
|
||||
if thread:
|
||||
thread.join(timeout=_PREFETCH_WAIT_SECS)
|
||||
# Slow backend: skip injection; mem0_search tool remains the backstop.
|
||||
return self._consume_prefetch_result(query) or ""
|
||||
return self._consume_prefetch_result(query) or "" # slow backend: skip injection; mem0_search remains the backstop
|
||||
|
||||
def sync_turn(self, user_content: str, assistant_content: str, *, session_id: str = "") -> None:
|
||||
"""Send the turn to Mem0 for server-side fact extraction (non-blocking)."""
|
||||
@@ -302,7 +287,8 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
|
||||
def _tool_search(self, args: dict) -> str:
|
||||
top_k = max(1, min(int(args.get("top_k", 10)), 50))
|
||||
rerank = _truthy(args.get("rerank", self._rerank_default), falsy_strings=("false", "0", "no"))
|
||||
rerank_raw = args.get("rerank", self._rerank_default)
|
||||
rerank = rerank_raw.lower() not in ("false", "0", "no") if isinstance(rerank_raw, str) else bool(rerank_raw)
|
||||
results = self._search(args["query"], top_k, rerank)
|
||||
if not results:
|
||||
return json.dumps({"result": "No relevant memories found."})
|
||||
@@ -336,8 +322,6 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
return tool_error(f"Missing required parameter: {missing}")
|
||||
try:
|
||||
result = body(self, args)
|
||||
self._record_success()
|
||||
return result
|
||||
except Exception as e:
|
||||
client = _is_client_error(e)
|
||||
if client and on_client_error == "not_found":
|
||||
@@ -345,6 +329,8 @@ class Mem0MemoryProvider(MemoryProvider):
|
||||
if not client or on_client_error == "count":
|
||||
self._record_failure()
|
||||
return tool_error(self._format_error(label, e))
|
||||
self._record_success()
|
||||
return result
|
||||
|
||||
def _shutdown_backend(self):
|
||||
with suppress(Exception):
|
||||
|
||||
@@ -13,9 +13,7 @@ def _add_kwargs(user_id: str, agent_id: str, infer: bool, metadata: dict | None)
|
||||
|
||||
def _unwrap_results(response: Any) -> list:
|
||||
"""Normalize API response — extract results list from dict or pass through."""
|
||||
if isinstance(response, dict):
|
||||
return response.get("results", [])
|
||||
return response if isinstance(response, list) else []
|
||||
return response.get("results", []) if isinstance(response, dict) else response if isinstance(response, list) else []
|
||||
|
||||
|
||||
class Mem0Backend(ABC):
|
||||
@@ -105,7 +103,6 @@ def _register_direct_openai_provider() -> None:
|
||||
"""Register Hermes' OpenAI-only Mem0 LLM provider once per factory."""
|
||||
from mem0.configs.llms.openai import OpenAIConfig
|
||||
from mem0.utils.factory import LlmFactory
|
||||
|
||||
provider_map = getattr(LlmFactory, "provider_to_class", None)
|
||||
register_provider = getattr(LlmFactory, "register_provider", None)
|
||||
if not isinstance(provider_map, dict) or not callable(register_provider):
|
||||
@@ -143,7 +140,6 @@ class OSSBackend(Mem0Backend):
|
||||
vs_config["embedding_model_dims"] = dims
|
||||
self._recreate_collection_if_dims_changed(vector_store.get("provider", "qdrant"), vs_config, dims)
|
||||
vector_store["config"] = vs_config
|
||||
|
||||
config = {"vector_store": vector_store, "llm": _provider_block("llm", LLM_PROVIDERS), "embedder": _provider_block("embedder", EMBEDDER_PROVIDERS), "version": "v1.1"}
|
||||
if str(config["llm"].get("provider") or "").strip().lower() == "openai":
|
||||
# mem0 validates LlmConfig.provider before its factory lookup: build the supported OpenAI config, then swap the provider.
|
||||
|
||||
@@ -54,12 +54,9 @@ def _prompt_api_key(label: str, env_var: str, hermes_home: str) -> str:
|
||||
"""Prompt for API key, showing masked existing value if found."""
|
||||
existing = os.environ.get(env_var, "")
|
||||
env_path = Path(hermes_home) / ".env"
|
||||
if not existing and env_path.exists():
|
||||
# utf-8-sig: a Notepad BOM on line 1 would otherwise defeat the key match.
|
||||
for line in env_path.read_text(encoding="utf-8-sig", errors="replace").splitlines():
|
||||
if line.startswith(f"{env_var}="):
|
||||
existing = line.split("=", 1)[1].strip()
|
||||
break
|
||||
if not existing and env_path.exists(): # utf-8-sig: a Notepad BOM on line 1 would otherwise defeat the key match
|
||||
lines = env_path.read_text(encoding="utf-8-sig", errors="replace").splitlines()
|
||||
existing = next((line.split("=", 1)[1].strip() for line in lines if line.startswith(f"{env_var}=")), "")
|
||||
hint = f" (current: {_masked(existing)}, blank to keep)" if existing else ""
|
||||
return getpass.getpass(f" {label} API key{hint}: ").strip()
|
||||
|
||||
@@ -93,8 +90,7 @@ _FLAG_DEFAULTS = {"oss_llm": "openai", "oss_embedder": "openai", "oss_vector": "
|
||||
|
||||
def parse_flags(argv: list[str] | None = None) -> dict[str, str]:
|
||||
args = argv if argv is not None else sys.argv[1:]
|
||||
flags: dict[str, Any] = {k: _FLAG_DEFAULTS.get(k, "") for k in _FLAG_KEYS}
|
||||
flags["dry_run"] = False
|
||||
flags: dict[str, Any] = {**{k: _FLAG_DEFAULTS.get(k, "") for k in _FLAG_KEYS}, "dry_run": False}
|
||||
flag_map = {"--" + k.replace("_", "-"): k for k in _FLAG_KEYS}
|
||||
i = 0
|
||||
while i < len(args):
|
||||
@@ -127,7 +123,6 @@ def build_oss_config(flags: dict[str, str]) -> tuple[dict, dict[str, str]]:
|
||||
dims = KNOWN_DIMS.get(embedder_config["model"])
|
||||
if dims:
|
||||
embedder_config["embedding_dims"] = dims
|
||||
|
||||
vector_id = flags.get("oss_vector", "qdrant")
|
||||
vector_config = dict(VECTOR_PROVIDERS[vector_id]["default_config"])
|
||||
for key in _VECTOR_FLAG_KEYS.get(vector_id, ()):
|
||||
@@ -135,7 +130,6 @@ def build_oss_config(flags: dict[str, str]) -> tuple[dict, dict[str, str]]:
|
||||
vector_config[key] = int(val) if key == "port" else val
|
||||
if "url" in vector_config:
|
||||
vector_config.pop("path", None) # a remote Qdrant URL replaces local storage
|
||||
|
||||
oss_config = {"llm": {"provider": llm_id, "config": llm_config}, "embedder": {"provider": embedder_id, "config": embedder_config}, "vector_store": {"provider": vector_id, "config": vector_config}}
|
||||
# An embedder sharing the LLM's provider reuses the LLM key when no embedder key was given.
|
||||
llm_key = flags.get("oss_llm_key") if llm_def.get("needs_key") else ""
|
||||
@@ -170,14 +164,8 @@ def _persist_provider_config(hermes_home: str, config: dict, provider_config: di
|
||||
_write_env(Path(hermes_home) / ".env", env_writes)
|
||||
if server:
|
||||
_check_selfhosted_server(server)
|
||||
print(f"\n Memory provider: {label}")
|
||||
if server:
|
||||
print(f" Server: {server}")
|
||||
print(" Activation saved to config.yaml")
|
||||
print(" Provider config saved")
|
||||
if env_writes:
|
||||
print(f" {key_line}")
|
||||
print("\n Start a new session to activate.\n")
|
||||
print("\n".join(["", f" Memory provider: {label}", *([f" Server: {server}"] if server else []), " Activation saved to config.yaml", " Provider config saved",
|
||||
*([f" {key_line}"] if env_writes else []), "", " Start a new session to activate.", ""]))
|
||||
|
||||
|
||||
def _setup_platform(hermes_home: str, config: dict, flags: dict[str, str]) -> None:
|
||||
@@ -191,14 +179,12 @@ def _setup_platform(hermes_home: str, config: dict, flags: dict[str, str]) -> No
|
||||
choices = ["true", "false"]
|
||||
current = str(provider_config.get("rerank", "false") or "").lower()
|
||||
provider_config["rerank"] = choices[_curses_select(" Enable reranking for recall", [(c, "") for c in choices], default=choices.index(current) if current in choices else 0)]
|
||||
|
||||
if flags.get("dry_run"):
|
||||
_print_dry_run(str(provider_config), env_writes)
|
||||
return
|
||||
provider_config["mode"] = "platform"
|
||||
# Routing checks ``host`` before platform, so clear a stale self-hosted host. "" rather than
|
||||
# pop(): save_config merges into the existing mem0.json, so a popped key would survive.
|
||||
provider_config["host"] = ""
|
||||
provider_config.update(mode="platform", host="")
|
||||
# _load_config() also seeds ``host`` from MEM0_HOST (.env); the file clear can't help there, so warn.
|
||||
if os.environ.get("MEM0_HOST", "").strip():
|
||||
print(f"\n ⚠ MEM0_HOST is set in your environment ({os.environ['MEM0_HOST']}). It overrides platform mode — remove it from ~/.hermes/.env (or unset it) or Hermes will keep routing to the self-hosted server.")
|
||||
@@ -229,7 +215,6 @@ def _setup_selfhosted(hermes_home: str, config: dict, flags: dict[str, str]) ->
|
||||
env_writes = _api_key_writes(flags, "Server API key", fresh_label="Server API key (blank if AUTH_DISABLED)")
|
||||
user_id = flags.get("user_id") or _prompt("User identifier", default=provider_config.get("user_id") or "hermes-user")
|
||||
agent_id = _prompt("Agent identifier", default=provider_config.get("agent_id") or "hermes")
|
||||
|
||||
if flags.get("dry_run"):
|
||||
_print_dry_run(f"host={host}, user_id={user_id}, agent_id={agent_id}", env_writes, lambda: _check_selfhosted_server(host))
|
||||
return
|
||||
@@ -240,19 +225,14 @@ def _setup_selfhosted(hermes_home: str, config: dict, flags: dict[str, str]) ->
|
||||
def _print_oss_summary(oss_config: dict, env_writes: dict, dry_run: bool = False) -> None:
|
||||
llm, emb = oss_config["llm"], oss_config["embedder"]
|
||||
w = 0 if dry_run else 9 # final summary column-aligns the labels
|
||||
print("\n [dry-run] OSS config would be:" if dry_run else "\n ✓ Mem0 configured (OSS mode)")
|
||||
print(f" {'LLM:':<{w}} {llm['provider']} ({llm['config'].get('model', '')})")
|
||||
print(f" {'Embedder:':<{w}} {emb['provider']} ({emb['config'].get('model', '')})")
|
||||
print(f" {'Vector:':<{w}} {oss_config['vector_store']['provider']}")
|
||||
lines = ["", " [dry-run] OSS config would be:" if dry_run else " ✓ Mem0 configured (OSS mode)",
|
||||
f" {'LLM:':<{w}} {llm['provider']} ({llm['config'].get('model', '')})", f" {'Embedder:':<{w}} {emb['provider']} ({emb['config'].get('model', '')})",
|
||||
f" {'Vector:':<{w}} {oss_config['vector_store']['provider']}"]
|
||||
if dry_run:
|
||||
if env_writes:
|
||||
print(f" Env vars: {', '.join(env_writes.keys())}")
|
||||
return
|
||||
if env_writes:
|
||||
print(" API keys saved to .env")
|
||||
print(" Config saved to mem0.json")
|
||||
print(" Provider set in config.yaml")
|
||||
print("\n Start a new session to activate.\n")
|
||||
lines += [f" Env vars: {', '.join(env_writes.keys())}"] if env_writes else []
|
||||
else:
|
||||
lines += [*([" API keys saved to .env"] if env_writes else []), " Config saved to mem0.json", " Provider set in config.yaml", "", " Start a new session to activate.", ""]
|
||||
print("\n".join(lines))
|
||||
|
||||
|
||||
def _finish_oss(hermes_home: str, config: dict, oss_config: dict, env_writes: dict[str, str], user_id: str, agent_id: str, pgvector_config: dict | None = None) -> None:
|
||||
@@ -313,13 +293,9 @@ def _ensure_pgvector(host: str = "localhost", port: int = 5432) -> dict | None:
|
||||
if _pg_ready(host, port, 15):
|
||||
print(" ✓ PostgreSQL container restarted")
|
||||
return None
|
||||
if input(" Start pgvector via Docker? [Y/n]: ").strip().lower() in ("", "y", "yes"):
|
||||
return _start_pgvector_docker(host, port)
|
||||
print(" Skipping Docker setup. Make sure PostgreSQL with pgvector is running.")
|
||||
return None
|
||||
|
||||
|
||||
def _start_pgvector_docker(host: str, port: int) -> dict | None:
|
||||
if input(" Start pgvector via Docker? [Y/n]: ").strip().lower() not in ("", "y", "yes"):
|
||||
print(" Skipping Docker setup. Make sure PostgreSQL with pgvector is running.")
|
||||
return None
|
||||
try:
|
||||
print(f" Pulling {_PGVECTOR_IMAGE}...")
|
||||
_docker("pull", _PGVECTOR_IMAGE, timeout=120)
|
||||
@@ -357,7 +333,11 @@ def _ensure_ollama(models: list[str]) -> bool:
|
||||
print(" Warning: Ollama not reachable. Models cannot be pulled.")
|
||||
return False
|
||||
for model in models:
|
||||
if any(model in n or model.split(":")[0] in n for n in _ollama_models(_OLLAMA_URL)):
|
||||
try:
|
||||
names = [m.get("name", "") for m in json.loads(_http_get(_OLLAMA_URL, "/api/tags", 5).read()).get("models", [])]
|
||||
except Exception:
|
||||
names = []
|
||||
if any(model in n or model.split(":")[0] in n for n in names):
|
||||
print(f" ✓ Model '{model}' available")
|
||||
continue
|
||||
print(f" Pulling '{model}'... (this may take a few minutes)")
|
||||
@@ -369,13 +349,6 @@ def _ensure_ollama(models: list[str]) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _ollama_models(url: str) -> list[str]:
|
||||
try:
|
||||
return [m.get("name", "") for m in json.loads(_http_get(url, "/api/tags", 5).read()).get("models", [])]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _ensure_pgvector_extension(pg_config: dict) -> None:
|
||||
try:
|
||||
import psycopg2
|
||||
@@ -403,18 +376,13 @@ def _wait_for_port(host: str, port: int, timeout: int = 15) -> None:
|
||||
|
||||
|
||||
# Picker descriptions: LLM/embedder show model (+ URL); vector stores by provider id (default: the id itself).
|
||||
def _provider_description(v: dict) -> str:
|
||||
model, url = v.get("default_model", ""), v.get("default_url")
|
||||
return f"{model} ({url})" if url else model
|
||||
|
||||
|
||||
_VECTOR_DESCRIPTIONS = {"qdrant": lambda cfg: cfg.get("path", "local storage"), "pgvector": lambda cfg: f"{cfg.get('host', 'localhost')}:{cfg.get('port', 5432)}"}
|
||||
|
||||
|
||||
def _configure_model_provider(kind: str, registry: dict, hermes_home: str, env_writes: dict[str, str], llm: tuple[str, dict] | None = None) -> tuple[str, dict, str, str | None]:
|
||||
"""Pick an LLM/embedder provider, collect its key, and (for Ollama) model + URL -> (id, definition, model, url).
|
||||
For the embedder (``llm`` given), a provider shared with the LLM reuses the LLM key instead of prompting again."""
|
||||
items = [(v["label"], _provider_description(v)) for v in registry.values()]
|
||||
items = [(v["label"], f"{v.get('default_model', '')} ({v['default_url']})" if v.get("default_url") else v.get("default_model", "")) for v in registry.values()]
|
||||
pid = list(registry)[_curses_select(f"{kind} Provider", items, 0)]
|
||||
pdef = registry[pid]
|
||||
model, url = pdef["default_model"], pdef.get("default_url")
|
||||
@@ -430,31 +398,23 @@ def _configure_model_provider(kind: str, registry: dict, hermes_home: str, env_w
|
||||
return pid, pdef, model, url
|
||||
|
||||
|
||||
def _prompt_pgvector_config() -> dict:
|
||||
"""Native PostgreSQL — prompt for connection details (user first, matching the historical order)."""
|
||||
pg = {k: _input(f"PostgreSQL {label}", d) for k, label, d in (
|
||||
("user", "user", os.getenv("USER", "postgres")), ("host", "host", "localhost"), ("port", "port", "5432"), ("dbname", "database", "postgres"))}
|
||||
pg_password = getpass.getpass(" PostgreSQL password (blank if none): ").strip()
|
||||
return {**pg, "port": int(pg["port"]), **({"password": pg_password} if pg_password else {})}
|
||||
|
||||
|
||||
def _setup_oss_interactive(hermes_home: str, config: dict) -> None:
|
||||
env_writes: dict[str, str] = {}
|
||||
llm_id, llm_def, llm_model, llm_url = _configure_model_provider("LLM", LLM_PROVIDERS, hermes_home, env_writes)
|
||||
embedder_id, _, embedder_model, embedder_url = _configure_model_provider("Embedder", EMBEDDER_PROVIDERS, hermes_home, env_writes, llm=(llm_id, llm_def))
|
||||
vector_items = [(v["label"], _VECTOR_DESCRIPTIONS.get(pid, lambda cfg: pid)(v.get("default_config", {}))) for pid, v in VECTOR_PROVIDERS.items()]
|
||||
vector_id = list(VECTOR_PROVIDERS)[_curses_select("Vector Store", vector_items, 0)]
|
||||
|
||||
# Auto-setup: ensure Ollama is running and models are pulled; ensure pgvector is reachable (offer Docker if not).
|
||||
ollama_models = [m for pid, m in ((llm_id, llm_model), (embedder_id, embedder_model)) if pid == "ollama"]
|
||||
if ollama_models:
|
||||
_ensure_ollama(ollama_models)
|
||||
pgvector_config = None
|
||||
if vector_id == "pgvector":
|
||||
pgvector_config = _ensure_pgvector() or _prompt_pgvector_config()
|
||||
pgvector_config = _ensure_pgvector() if vector_id == "pgvector" else None
|
||||
if vector_id == "pgvector" and not pgvector_config: # native PostgreSQL: prompt for connection details (user first, historical order)
|
||||
pg = {k: _input(f"PostgreSQL {label}", d) for k, label, d in (("user", "user", os.getenv("USER", "postgres")), ("host", "host", "localhost"), ("port", "port", "5432"), ("dbname", "database", "postgres"))}
|
||||
pg_password = getpass.getpass(" PostgreSQL password (blank if none): ").strip()
|
||||
pgvector_config = {**pg, "port": int(pg["port"]), **({"password": pg_password} if pg_password else {})}
|
||||
user_id = _input("User ID", os.getenv("USER", "hermes-user"))
|
||||
agent_id = _input("Agent ID", "hermes")
|
||||
|
||||
flags = {
|
||||
"oss_llm": llm_id, "oss_llm_model": llm_model, "oss_llm_url": llm_url or "",
|
||||
"oss_llm_key": env_writes.get(llm_def["env_var"], "") if llm_def.get("env_var") else "",
|
||||
@@ -476,12 +436,8 @@ def _install_provider_deps(llm_id: str, embedder_id: str, vector_id: str) -> Non
|
||||
outcome = install_specs([dep], timeout=60)
|
||||
except Exception:
|
||||
outcome = None
|
||||
if outcome is not None and outcome.ok:
|
||||
print(f" ✓ Installed {dep}")
|
||||
elif outcome is not None and outcome.blocked:
|
||||
print(f" Warning: cannot install {dep}: {outcome.reason}")
|
||||
else:
|
||||
print(f" Warning: Could not install {dep}. Install manually: uv pip install {dep}")
|
||||
print(f" ✓ Installed {dep}" if outcome is not None and outcome.ok else f" Warning: cannot install {dep}: {outcome.reason}" if outcome is not None and outcome.blocked
|
||||
else f" Warning: Could not install {dep}. Install manually: uv pip install {dep}")
|
||||
if deps:
|
||||
import importlib
|
||||
importlib.invalidate_caches()
|
||||
|
||||
Reference in New Issue
Block a user