Merge branch 'simp/r3-27-D' into simp/integration3

This commit is contained in:
Teknium
2026-09-02 22:40:58 -07:00
7 changed files with 168 additions and 317 deletions
+47 -80
View File
@@ -25,26 +25,18 @@ logger = logging.getLogger(__name__)
FACT_STORE_SCHEMA = {
"name": "fact_store",
"description": (
"Deep structured memory with algebraic reasoning. "
"Use alongside the memory tool — memory for always-on context, "
"fact_store for deep recall and compositional queries.\n\n"
"ACTIONS (simple → powerful):\n"
"• add — Store a fact the user would expect you to remember.\n"
"• search — Keyword lookup ('editor config', 'deploy process').\n"
"• probe — Entity recall: ALL facts about a person/thing.\n"
"• related — What connects to an entity? Structural adjacency.\n"
"Deep structured memory with algebraic reasoning. Use alongside the memory tool — memory for always-on "
"context, fact_store for deep recall and compositional queries.\n\nACTIONS (simple → powerful):\n"
"• add — Store a fact the user would expect you to remember.\n• search — Keyword lookup ('editor config', 'deploy process').\n"
"• probe — Entity recall: ALL facts about a person/thing.\n• related — What connects to an entity? Structural adjacency.\n"
"• reason — Compositional: facts connected to MULTIPLE entities simultaneously.\n"
"• contradict — Memory hygiene: find facts making conflicting claims.\n"
"• update/remove/list — CRUD operations.\n\n"
"• contradict — Memory hygiene: find facts making conflicting claims.\n• update/remove/list — CRUD operations.\n\n"
"IMPORTANT: Before answering questions about the user, ALWAYS probe or reason first."
),
"parameters": {
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": ["add", "search", "probe", "related", "reason", "contradict", "update", "remove", "list"],
},
"action": {"type": "string", "enum": ["add", "search", "probe", "related", "reason", "contradict", "update", "remove", "list"]},
"content": {"type": "string", "description": "Fact content (required for 'add')."},
"query": {"type": "string", "description": "Search query (required for 'search')."},
"entity": {"type": "string", "description": "Entity name for 'probe'/'related'."},
@@ -62,31 +54,24 @@ FACT_STORE_SCHEMA = {
FACT_FEEDBACK_SCHEMA = {
"name": "fact_feedback",
"description": (
"Rate a fact after using it. Mark 'helpful' if accurate, 'unhelpful' if outdated. "
"This trains the memory — good facts rise, bad facts sink."
),
"description": ("Rate a fact after using it. Mark 'helpful' if accurate, 'unhelpful' if outdated. "
"This trains the memory — good facts rise, bad facts sink."),
"parameters": {
"type": "object",
"properties": {
"action": {"type": "string", "enum": ["helpful", "unhelpful"]},
"fact_id": {"type": "integer", "description": "The fact ID to rate."},
},
"properties": {"action": {"type": "string", "enum": ["helpful", "unhelpful"]},
"fact_id": {"type": "integer", "description": "The fact ID to rate."}},
"required": ["action", "fact_id"],
},
}
# Auto-extraction patterns (on_session_end): user preferences -> user_pref, decisions -> project.
_PREF_PATTERNS = [
re.compile(r'\bI\s+(?:prefer|like|love|use|want|need)\s+(.+)', re.IGNORECASE),
re.compile(r'\bmy\s+(?:favorite|preferred|default)\s+\w+\s+is\s+(.+)', re.IGNORECASE),
re.compile(r'\bI\s+(?:always|never|usually)\s+(.+)', re.IGNORECASE),
]
_DECISION_PATTERNS = [
re.compile(r'\bwe\s+(?:decided|agreed|chose)\s+(?:to\s+)?(.+)', re.IGNORECASE),
re.compile(r'\bthe\s+project\s+(?:uses|needs|requires)\s+(.+)', re.IGNORECASE),
]
_EXTRACT_CATEGORIES = ((_PREF_PATTERNS, "user_pref"), (_DECISION_PATTERNS, "project"))
# Auto-extraction (on_session_end): (patterns, category) — user preferences -> user_pref, decisions -> project.
_EXTRACT_CATEGORIES = (
([re.compile(r'\bI\s+(?:prefer|like|love|use|want|need)\s+(.+)', re.IGNORECASE),
re.compile(r'\bmy\s+(?:favorite|preferred|default)\s+\w+\s+is\s+(.+)', re.IGNORECASE),
re.compile(r'\bI\s+(?:always|never|usually)\s+(.+)', re.IGNORECASE)], "user_pref"),
([re.compile(r'\bwe\s+(?:decided|agreed|chose)\s+(?:to\s+)?(.+)', re.IGNORECASE),
re.compile(r'\bthe\s+project\s+(?:uses|needs|requires)\s+(.+)', re.IGNORECASE)], "project"),
)
def _load_plugin_config() -> dict:
@@ -106,12 +91,9 @@ def _limit(args: dict) -> int:
def _tool_handler(actions: dict):
"""Return a bound-style handler dispatching on args["action"] over ``actions`` (unknown -> tool_error)."""
def handle(self, args: dict) -> str:
action = args["action"]
handler = actions.get(action)
return handler(self, args) if handler is not None else tool_error(f"Unknown action: {action}")
return handle
"""(self, args) handler dispatching on args["action"] over ``actions``; unknown action -> tool_error."""
return lambda self, args: (actions[args["action"]](self, args) if args["action"] in actions
else tool_error(f"Unknown action: {args['action']}"))
class HolographicMemoryProvider(MemoryProvider):
@@ -144,9 +126,8 @@ class HolographicMemoryProvider(MemoryProvider):
def get_config_schema(self):
from hermes_constants import display_hermes_home
_default_db = f"{display_hermes_home()}/memory_store.db"
return [
{"key": "db_path", "description": "SQLite database path", "default": _default_db},
{"key": "db_path", "description": "SQLite database path", "default": f"{display_hermes_home()}/memory_store.db"},
{"key": "auto_extract", "description": "Auto-extract facts at session end", "default": "false", "choices": ["true", "false"]},
{"key": "default_trust", "description": "Default trust score for new facts", "default": "0.5"},
{"key": "hrr_dim", "description": "HRR vector dimensions", "default": "1024"},
@@ -160,10 +141,8 @@ class HolographicMemoryProvider(MemoryProvider):
db_path = db_path.replace("$HERMES_HOME", _hermes_home).replace("${HERMES_HOME}", _hermes_home)
hrr_dim = int(self._config.get("hrr_dim", 1024))
self._store = MemoryStore(db_path=db_path, default_trust=float(self._config.get("default_trust", 0.5)), hrr_dim=hrr_dim)
self._retriever = FactRetriever(
store=self._store, temporal_decay_half_life=int(self._config.get("temporal_decay_half_life", 0)),
hrr_weight=float(self._config.get("hrr_weight", 0.3)), hrr_dim=hrr_dim,
)
self._retriever = FactRetriever(store=self._store, hrr_dim=hrr_dim, hrr_weight=float(self._config.get("hrr_weight", 0.3)),
temporal_decay_half_life=int(self._config.get("temporal_decay_half_life", 0)))
self._session_id = session_id
def system_prompt_block(self) -> str:
@@ -173,13 +152,11 @@ class HolographicMemoryProvider(MemoryProvider):
total = self._store._conn.execute("SELECT COUNT(*) FROM facts").fetchone()[0]
except Exception:
total = 0
body = (
"Active. Empty fact store — proactively add facts the user would expect you to remember.\n"
"Use fact_store(action='add') to store durable structured facts about people, projects, preferences, decisions.\n"
) if total == 0 else (
f"Active. {total} facts stored with entity resolution and trust scoring.\n"
"Use fact_store to search, probe entities, reason across entities, or add facts.\n"
)
body = ("Active. Empty fact store — proactively add facts the user would expect you to remember.\n"
"Use fact_store(action='add') to store durable structured facts about people, projects, preferences, decisions.\n"
if total == 0 else
f"Active. {total} facts stored with entity resolution and trust scoring.\n"
"Use fact_store to search, probe entities, reason across entities, or add facts.\n")
return "# Holographic Memory\n" + body + "Use fact_feedback to rate facts after using them (trains trust scores)."
def prefetch(self, query: str, *, session_id: str = "") -> str:
@@ -197,11 +174,10 @@ class HolographicMemoryProvider(MemoryProvider):
return [FACT_STORE_SCHEMA, FACT_FEEDBACK_SCHEMA]
def handle_tool_call(self, tool_name: str, args: Dict[str, Any], **kwargs) -> str:
handler = self._TOOL_HANDLERS.get(tool_name)
if handler is None:
if tool_name not in self._TOOL_HANDLERS:
return tool_error(f"Unknown tool: {tool_name}")
try:
return handler(self, args)
return self._TOOL_HANDLERS[tool_name](self, args)
except KeyError as exc:
return tool_error(f"Missing required argument: {exc}")
except Exception as exc:
@@ -221,15 +197,13 @@ class HolographicMemoryProvider(MemoryProvider):
logger.debug("Holographic memory_write mirror failed: %s", e)
def shutdown(self) -> None:
# Close on the caller's thread: leaving the shared connection (and its write lock) to GC keeps it
# alive on a long-running gateway. close() is idempotent and refcount-guarded.
# Close on the caller's thread: leaving the shared connection (+ write lock) to GC keeps it alive on a gateway.
if self._store is not None:
try:
self._store.close()
except Exception as e:
logger.debug("Holographic shutdown close() failed: %s", e)
self._store = None
self._retriever = None
self._store = self._retriever = None
# Tool handlers (self, args) -> str. KeyError from args[...] / Exception -> tool_error in handle_tool_call;
# argument coercion order (and therefore which error surfaces first) mirrors the underlying call order.
@@ -259,28 +233,22 @@ class HolographicMemoryProvider(MemoryProvider):
"fact_feedback": lambda self, a: json.dumps(self._store.record_feedback(int(a["fact_id"]), helpful=a["action"] == "helpful")),
}
@staticmethod
def _harvestable_text(msg: dict):
"""User text eligible for extraction, or None. Compaction handoff summaries arrive as role="user" and
reliably match the decision patterns; never store the compactor's own output as a durable fact. A
merge-into-tail row holds genuine prior user text BEFORE _MERGED_SUMMARY_DELIMITER (prefixed with the
header) and the summary AFTER it — harvest only the pre-delimiter segment."""
# Local import: the compressor module is heavier than this plugin and only needed here.
from agent.context_compressor import _MERGED_PRIOR_CONTEXT_HEADER, _MERGED_SUMMARY_DELIMITER, is_compaction_summary_message
content, pre = msg.get("content", ""), ""
if isinstance(content, str) and _MERGED_SUMMARY_DELIMITER in content:
pre = content.split(_MERGED_SUMMARY_DELIMITER, 1)[0].removeprefix(_MERGED_PRIOR_CONTEXT_HEADER).strip()
if pre:
content = pre
elif is_compaction_summary_message(msg):
return None
return content if isinstance(content, str) and len(content) >= 10 else None
def _auto_extract_facts(self, messages: list) -> None:
# Compaction handoff summaries arrive as role="user" and match the decision patterns; never store the
# compactor's own output as a fact. A merge-into-tail row holds genuine prior user text BEFORE
# _MERGED_SUMMARY_DELIMITER (after the header) and the summary AFTER it — harvest only that segment.
from agent.context_compressor import _MERGED_PRIOR_CONTEXT_HEADER, _MERGED_SUMMARY_DELIMITER, is_compaction_summary_message # heavy; lazy
extracted = 0
texts = filter(None, (self._harvestable_text(m) for m in messages if m.get("role") == "user"))
for content in texts:
for msg in messages:
content = msg.get("content", "") if msg.get("role") == "user" else None
pre = content.split(_MERGED_SUMMARY_DELIMITER, 1)[0].removeprefix(_MERGED_PRIOR_CONTEXT_HEADER).strip() \
if isinstance(content, str) and _MERGED_SUMMARY_DELIMITER in content else ""
if pre:
content = pre
elif content is None or is_compaction_summary_message(msg):
continue
if not isinstance(content, str) or len(content) < 10:
continue
for patterns, category in _EXTRACT_CATEGORIES:
if any(p.search(content) for p in patterns):
try:
@@ -291,7 +259,6 @@ class HolographicMemoryProvider(MemoryProvider):
if extracted:
logger.info("Auto-extracted %d facts from conversation", extracted)
def register(ctx) -> None:
"""Register the holographic memory provider with the plugin system."""
ctx.register_memory_provider(HolographicMemoryProvider(config=_load_plugin_config()))
+16 -27
View File
@@ -1,17 +1,13 @@
"""Holographic Reduced Representations (HRR) with phase encoding.
Each concept is a vector of angles in [0, 2π): bind = circular convolution (phase addition),
unbind = circular correlation (phase subtraction), bundle = superposition (circular mean).
Phase encoding avoids the magnitude collapse of complex-number HRRs and maps cleanly to
cosine similarity. Atoms derive deterministically from SHA-256 so representations are
identical across processes, machines, and Python versions.
References: Plate (1995) HRRs; Gayler (2004) Vector Symbolic Architectures.
"""
"""Holographic Reduced Representations (HRR) with phase encoding. Each concept is a vector of angles in [0, 2π):
bind = circular convolution (phase addition), unbind = circular correlation (phase subtraction), bundle =
superposition (circular mean). Phase encoding avoids the magnitude collapse of complex-number HRRs and maps cleanly
to cosine similarity; atoms derive deterministically from SHA-256 so representations are identical across processes,
machines, and Python versions. References: Plate (1995) HRRs; Gayler (2004) Vector Symbolic Architectures."""
import hashlib
import logging
import struct
import math
import struct
try:
import numpy as np
@@ -37,9 +33,8 @@ def encode_atom(word: str, dim: int = 1024) -> "np.ndarray":
"""Deterministic phase vector: SHA-256 counter blocks of f"{word}:{i}" -> uint16 -> [0, 2π).
hashlib rather than numpy RNG so atoms are reproducible across platforms."""
_require_numpy()
uint16_values: list[int] = []
for i in range(math.ceil(dim / 16)): # 32-byte digest = 16 uint16 values
uint16_values.extend(struct.unpack("<16H", hashlib.sha256(f"{word}:{i}".encode()).digest()))
uint16_values = [v for i in range(math.ceil(dim / 16)) # 32-byte digest = 16 uint16 values
for v in struct.unpack("<16H", hashlib.sha256(f"{word}:{i}".encode()).digest())]
return np.array(uint16_values[:dim], dtype=np.float64) * (_TWO_PI / 65536.0)
@@ -84,11 +79,8 @@ def encode_fact(content: str, entities: list[str], dim: int = 1024) -> "np.ndarr
def phases_to_bytes(phases: "np.ndarray", dim: int | None = None) -> bytes:
"""Serialize as prefixed float32 (half the size of legacy float64 blobs).
At dim=1 the prefixed float32 blob and a raw float64 blob are both 8 bytes,
so we write legacy float64 there to keep ``bytes_to_phases`` unambiguous.
"""
"""Serialize as prefixed float32 (half the size of legacy float64 blobs). At dim=1 the prefixed float32 blob
and a raw float64 blob are both 8 bytes, so write legacy float64 there to keep ``bytes_to_phases`` unambiguous."""
_require_numpy()
dim = int(phases.shape[0]) if dim is None else dim
if len(_FLOAT32_BLOB_PREFIX) + dim * _F32 == dim * _F64:
@@ -97,11 +89,9 @@ def phases_to_bytes(phases: "np.ndarray", dim: int | None = None) -> bytes:
def bytes_to_phases(data: bytes, dim: int | None = None) -> "np.ndarray":
"""Deserialize prefixed float32 or legacy raw float64 blobs (always returns float64).
With ``dim`` given, a prefixed blob whose size equals the float64 size (dim=1) is read
as legacy float64: ``phases_to_bytes`` never writes a prefixed blob at that size.
"""
"""Deserialize prefixed float32 or legacy raw float64 blobs (always returns float64). With ``dim`` given, a
prefixed blob whose size equals the float64 size (dim=1) is read as legacy float64: ``phases_to_bytes`` never
writes a prefixed blob at that size."""
_require_numpy()
plen = len(_FLOAT32_BLOB_PREFIX)
prefixed = data.startswith(_FLOAT32_BLOB_PREFIX)
@@ -120,10 +110,9 @@ def bytes_to_phases(data: bytes, dim: int | None = None) -> "np.ndarray":
return f64(data)
if not prefixed:
raise ValueError(f"HRR legacy vector blob has {len(data)} bytes; expected {float64_bytes} (float64) for dim={dim}")
expected = (f"{float64_bytes} (legacy float64)" if collides
else f"{float32_blob_bytes} (prefixed float32) or {float64_bytes} (legacy float64)")
raise ValueError(f"HRR vector blob has {len(data)} bytes ({len(data) - plen} payload bytes after "
f"the float32 prefix); expected {expected} for dim={dim}")
expected = f"{float64_bytes} (legacy float64)" if collides else f"{float32_blob_bytes} (prefixed float32) or {float64_bytes} (legacy float64)"
raise ValueError(f"HRR vector blob has {len(data)} bytes ({len(data) - plen} payload bytes after the float32 prefix); "
f"expected {expected} for dim={dim}")
def snr_estimate(dim: int, n_items: int) -> float:
+35 -62
View File
@@ -11,28 +11,21 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .store import MemoryStore
try:
from . import holographic as hrr
except ImportError:
import holographic as hrr # type: ignore[no-redef]
from . import holographic as hrr
_FACT_COLUMNS = (
"fact_id, content, category, tags, trust_score, "
"retrieval_count, helpful_count, created_at, updated_at"
)
_FACT_COLUMNS = "fact_id, content, category, tags, trust_score, retrieval_count, helpful_count, created_at, updated_at"
_ROLE_ENTITY, _ROLE_CONTENT = hrr.ROLE_ENTITY, hrr.ROLE_CONTENT
_PUNCT = ".,;:!?\"'()[]{}#@<>"
_FTS_OPERATORS = str.maketrans("", "", '"()*^:-+')
# Stopwords dropped before FTS5 OR-expansion: short English function words that
# carry no retrieval signal and force false-negative AND matches.
_FTS_STOPWORDS = frozenset("""
a about above after again all am an and any are as at be because been before being between both but by
can could did do does doing don down during each few for from further had has have having he her here hers
herself him himself his how i if in into is it its itself just me more most my myself no nor not now of off
on once only or other our ours ourselves out over own same she should so some such than that the their theirs
them themselves then there these they this those through to too under until up very was we were what when
where which while who whom why will with would you your yours yourself yourselves
""".split())
a about above after again all am an and any are as at be because been before being between both but by can could
did do does doing don down during each few for from further had has have having he her here hers herself him himself
his how i if in into is it its itself just me more most my myself no nor not now of off on once only or other our
ours ourselves out over own same she should so some such than that the their theirs them themselves then there these
they this those through to too under until up very was we were what when where which while who whom why will with
would you your yours yourself yourselves""".split())
def _shift(sim: float) -> float:
@@ -57,14 +50,12 @@ class FactRetriever:
return hrr.bytes_to_phases(blob, dim=self.hrr_dim)
def search(self, query: str, category: str | None = None, min_trust: float = 0.3, limit: int = 10) -> list[dict]:
"""Hybrid search: FTS5 candidates (limit*3) → Jaccard + HRR rerank → trust weighting →
optional temporal decay 0.5^(age_days / half_life). Returns fact dicts with 'score', sorted desc."""
"""FTS5 candidates (limit*3) → Jaccard + HRR rerank → trust weighting → optional temporal decay
0.5^(age_days / half_life). Returns fact dicts with 'score', sorted desc."""
candidates = self._fts_candidates(query, category, min_trust, limit * 3)
if not candidates:
return []
query_tokens = self._tokenize(query)
# Query vector is loop-invariant; encode lazily on the first candidate that carries
# an HRR vector so stores whose hrr_vector was never backfilled don't pay for it.
# Query vector is loop-invariant; encode lazily on the first candidate that carries an HRR vector
# so stores whose hrr_vector was never backfilled don't pay for it.
query_vec = None
for fact in candidates:
jaccard = self._jaccard_similarity(query_tokens, self._tokenize(fact["content"]) | self._tokenize(fact.get("tags", "")))
@@ -83,18 +74,14 @@ class FactRetriever:
fact.pop("hrr_vector", None) # callers expect JSON-serializable dicts
return results
def _vector_query(self, fallback: str, category: str | None, limit: int, make_sim: Callable[[], Callable]) -> list[dict]:
"""Rank every fact vector (optionally per category) with the sim fn built by make_sim(); FTS5 fallback
when no vectors exist. make_sim runs after the rows check so role atoms are encoded once, never for fallback."""
def _vector_query(self, fallback: str, category: str | None, limit: int, sim_fn: Callable) -> list[dict]:
"""Rank every fact vector (optionally per category) by sim_fn; FTS5 fallback when no vectors exist."""
rows = self._vector_rows(category)
if not rows:
return self.search(fallback, category=category, limit=limit)
return self._rank_by_vector(rows, make_sim(), limit)
return self._rank_by_vector(rows, sim_fn, limit) if rows else self.search(fallback, category=category, limit=limit)
def probe(self, entity: str, category: str | None = None, limit: int = 10) -> list[dict]:
"""Compositional entity query: unbind bind(entity, ROLE_ENTITY) from the category bank
(or each fact vector) to find facts where the entity plays a structural role.
Not keyword search. Falls back to FTS5 without numpy."""
"""Compositional entity query: unbind bind(entity, ROLE_ENTITY) from the category bank (or each fact vector)
to find facts where the entity plays a structural role. Not keyword search. Falls back to FTS5 without numpy."""
if not hrr._HAS_NUMPY:
return self.search(entity, category=category, limit=limit)
probe_key = hrr.bind(self._atom(entity.lower()), self._atom(_ROLE_ENTITY))
@@ -103,47 +90,35 @@ class FactRetriever:
if bank_row:
extracted = hrr.unbind(self._phases(bank_row["vector"]), probe_key)
return self._rank_by_vector(self._vector_rows(category), lambda _f, fact_vec: hrr.similarity(extracted, fact_vec), limit)
def make_sim():
role_content = self._atom(_ROLE_CONTENT)
# Does unbinding the probe key leave the fact's content signal?
return lambda fact, fact_vec: hrr.similarity(
hrr.unbind(fact_vec, probe_key), hrr.bind(hrr.encode_text(fact["content"], self.hrr_dim), role_content))
return self._vector_query(entity, category, limit, make_sim)
role_content = self._atom(_ROLE_CONTENT) # loop-invariant: encode once, not per row
# Does unbinding the probe key leave the fact's content signal?
return self._vector_query(entity, category, limit, lambda fact, fact_vec: hrr.similarity(
hrr.unbind(fact_vec, probe_key), hrr.bind(hrr.encode_text(fact["content"], self.hrr_dim), role_content)))
def related(self, entity: str, category: str | None = None, limit: int = 10) -> list[dict]:
"""Facts structurally connected to an entity (shared context), not just facts
*about* it as in probe. Falls back to FTS5 without numpy."""
"""Facts structurally connected to an entity (shared context), not just facts *about* it as in probe.
Falls back to FTS5 without numpy."""
if not hrr._HAS_NUMPY:
return self.search(entity, category=category, limit=limit)
entity_vec = self._atom(entity.lower()) # bare atom, not role-bound: ANY structural match
def make_sim():
roles = (self._atom(_ROLE_ENTITY), self._atom(_ROLE_CONTENT))
# A residual similar to ANY role vector means the entity plays a structural role.
return lambda _f, fact_vec: max(hrr.similarity(hrr.unbind(fact_vec, entity_vec), role) for role in roles)
return self._vector_query(entity, category, limit, make_sim)
roles = (self._atom(_ROLE_ENTITY), self._atom(_ROLE_CONTENT)) # loop-invariant: encode once
# A residual similar to ANY role vector means the entity plays a structural role in the fact.
return self._vector_query(entity, category, limit, lambda _f, fact_vec: max(
hrr.similarity(hrr.unbind(fact_vec, entity_vec), role) for role in roles))
def reason(self, entities: list[str], category: str | None = None, limit: int = 10) -> list[dict]:
"""Multi-entity compositional query (vector-space JOIN): facts where ALL entities
play structural roles. Falls back to FTS5 without numpy."""
"""Multi-entity compositional query (vector-space JOIN): facts where ALL entities play structural roles.
Falls back to FTS5 without numpy."""
if not hrr._HAS_NUMPY or not entities:
return self.search(" ".join(entities), category=category, limit=limit)
role_entity = self._atom(_ROLE_ENTITY)
role_entity, role_content = self._atom(_ROLE_ENTITY), self._atom(_ROLE_CONTENT)
probe_keys = [hrr.bind(self._atom(entity.lower()), role_entity) for entity in entities]
def make_sim():
role_content = self._atom(_ROLE_CONTENT)
# AND semantics via min: high only if EVERY entity is structurally present.
return lambda _f, fact_vec: min(hrr.similarity(hrr.unbind(fact_vec, key), role_content) for key in probe_keys)
return self._vector_query(" ".join(entities), category, limit, make_sim)
# AND semantics via min: high only if EVERY entity is structurally present.
return self._vector_query(" ".join(entities), category, limit, lambda _f, fact_vec: min(
hrr.similarity(hrr.unbind(fact_vec, key), role_content) for key in probe_keys))
def contradict(self, category: str | None = None, threshold: float = 0.3, limit: int = 10) -> list[dict]:
"""Memory hygiene: pairs of facts that share entities (same subject) but have low
content-vector similarity (different claims). Empty without numpy."""
"""Pairs of facts sharing entities (same subject) with low content-vector similarity (different claims). Empty without numpy."""
if not hrr._HAS_NUMPY:
return []
rows = self._vector_rows(category, columns="fact_id, content, category, tags, trust_score, created_at, updated_at, hrr_vector")
@@ -234,9 +209,7 @@ class FactRetriever:
return 1.0
try:
ts = datetime.fromisoformat(timestamp_str.replace("Z", "+00:00")) if isinstance(timestamp_str, str) else timestamp_str
if ts.tzinfo is None:
ts = ts.replace(tzinfo=timezone.utc)
age_days = (datetime.now(timezone.utc) - ts).total_seconds() / 86400
age_days = (datetime.now(timezone.utc) - (ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc))).total_seconds() / 86400
return 1.0 if age_days < 0 else math.pow(0.5, age_days / self.half_life)
except (ValueError, TypeError):
return 1.0
+22 -38
View File
@@ -6,10 +6,7 @@ import sqlite3
import threading
from pathlib import Path
try:
from . import holographic as hrr
except ImportError:
import holographic as hrr # type: ignore[no-redef]
from . import holographic as hrr
_SCHEMA = """
CREATE TABLE IF NOT EXISTS facts (
@@ -73,23 +70,16 @@ CREATE TABLE IF NOT EXISTS memory_banks (
);
"""
_HELPFUL_DELTA = 0.05
_UNHELPFUL_DELTA = -0.10
_HELPFUL_DELTA, _UNHELPFUL_DELTA = 0.05, -0.10
# Entity extraction patterns, applied in order: capitalized multi-word phrases
# ("John Doe"), double-quoted terms, single-quoted terms, then "X aka Y" (both sides).
_RE_SINGLE_ENTITY = (
re.compile(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b'),
re.compile(r'"([^"]+)"'),
re.compile(r"'([^']+)'"),
)
# Entity extraction patterns, applied in order: capitalized multi-word phrases ("John Doe"), double-quoted terms,
# single-quoted terms, then "X aka Y" (both sides).
_RE_SINGLE_ENTITY = (re.compile(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b'), re.compile(r'"([^"]+)"'), re.compile(r"'([^']+)'"))
_RE_AKA = re.compile(r'(\w+(?:\s+\w+)*)\s+(?:aka|also known as)\s+(\w+(?:\s+\w+)*)', re.IGNORECASE)
_ENTITY_NAMES_SQL = "SELECT e.name FROM entities e JOIN fact_entities fe ON fe.entity_id = e.entity_id WHERE fe.fact_id = ?"
# Entity lookup order: exact name, then aliases (comma-separated; wrapped in commas for whole-alias matching).
_ENTITY_LOOKUPS = (
"SELECT entity_id FROM entities WHERE name LIKE ?",
"SELECT entity_id FROM entities WHERE ',' || aliases || ',' LIKE '%,' || ? || ',%'",
)
_ENTITY_LOOKUPS = ("SELECT entity_id FROM entities WHERE name LIKE ?",
"SELECT entity_id FROM entities WHERE ',' || aliases || ',' LIKE '%,' || ? || ',%'")
def _clamp_trust(value: float) -> float:
@@ -121,21 +111,20 @@ class MemoryStore:
with MemoryStore._shared_guard:
entry = MemoryStore._shared.get(self._key)
if entry is None:
# Autocommit: a write that raises mid-method can never leave a dangling
# transaction (and its write lock) open; explicit commit() calls are no-ops.
# Autocommit: a write that raises mid-method can't leave a dangling transaction (and its
# write lock) open; the explicit commit() calls in _write are then harmless no-ops.
conn = sqlite3.connect(self._key, check_same_thread=False, timeout=10.0, isolation_level=None)
conn.row_factory = sqlite3.Row
entry = MemoryStore._shared[self._key] = {"conn": conn, "lock": threading.RLock(), "refs": 0, "ready": False}
entry["refs"] += 1
self._entry, self._conn, self._lock = entry, entry["conn"], entry["lock"]
with self._lock: # schema initialised once per shared connection
if not self._entry["ready"]:
if not entry["ready"]:
self._init_db()
self._entry["ready"] = True
entry["ready"] = True
def _init_db(self) -> None:
"""Create schema, enable WAL via the shared fallback helper (NFS/SMB/FUSE HERMES_HOME
degrades gracefully), and add hrr_vector to pre-HRR databases."""
"""Create schema, enable WAL via the shared fallback helper (NFS/SMB/FUSE degrade gracefully), add hrr_vector to pre-HRR DBs."""
from hermes_state import apply_wal_with_fallback
apply_wal_with_fallback(self._conn, db_label="memory_store.db (holographic)")
self._conn.executescript(_SCHEMA)
@@ -152,8 +141,8 @@ class MemoryStore:
return cur
def add_fact(self, content: str, category: str = "general", tags: str = "") -> int:
"""Insert a fact and return its fact_id; on duplicate content (UNIQUE) return the
existing fact_id untouched. Links extracted entities and rebuilds the category bank."""
"""Insert a fact and return its fact_id; on duplicate content (UNIQUE) return the existing fact_id untouched.
Links extracted entities and rebuilds the category bank."""
with self._lock:
content = content.strip()
if not content:
@@ -229,9 +218,8 @@ class MemoryStore:
for m in _RE_AKA.finditer(text):
raw += [m.group(1), m.group(2)]
uniq: dict[str, str] = {} # lower-cased key -> first-seen spelling, insertion-ordered
for name in (n.strip() for n in raw):
if name:
uniq.setdefault(name.lower(), name)
for name in filter(None, (n.strip() for n in raw)):
uniq.setdefault(name.lower(), name)
return list(uniq.values())
def _link_entities(self, fact_id: int, content: str) -> None:
@@ -267,13 +255,10 @@ class MemoryStore:
return
bank_vector = hrr.bundle(*[hrr.bytes_to_phases(row["hrr_vector"], dim=self.hrr_dim) for row in rows])
hrr.snr_estimate(self.hrr_dim, len(rows)) # warns when near capacity
self._write(
"INSERT INTO memory_banks (bank_name, vector, dim, fact_count, updated_at) "
"VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) ON CONFLICT(bank_name) DO UPDATE SET "
"vector = excluded.vector, dim = excluded.dim, fact_count = excluded.fact_count, "
"updated_at = excluded.updated_at",
(bank_name, hrr.phases_to_bytes(bank_vector), self.hrr_dim, len(rows)),
)
self._write("INSERT INTO memory_banks (bank_name, vector, dim, fact_count, updated_at) "
"VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) ON CONFLICT(bank_name) DO UPDATE SET "
"vector = excluded.vector, dim = excluded.dim, fact_count = excluded.fact_count, "
"updated_at = excluded.updated_at", (bank_name, hrr.phases_to_bytes(bank_vector), self.hrr_dim, len(rows)))
@classmethod
def release_all_under(cls, directory: "str | Path") -> int:
@@ -283,9 +268,8 @@ class MemoryStore:
stale holder is expected to fail."""
root = os.path.normcase(str(Path(directory).expanduser().resolve())) + os.sep
with cls._shared_guard:
doomed = [key for key in cls._shared if os.path.normcase(key).startswith(root)]
for key in doomed:
entry = cls._shared.pop(key)
doomed = [cls._shared.pop(key) for key in list(cls._shared) if os.path.normcase(key).startswith(root)]
for entry in doomed:
try:
with entry["lock"]:
entry["conn"].close()
+17 -31
View File
@@ -40,13 +40,6 @@ def _is_client_error(exc: Exception) -> bool:
return type(exc).__name__ in _CLIENT_ERROR_TYPES or any(s in err_str for s in ("404", "not found", "valid uuid"))
def _truthy(value: Any, falsy_strings: tuple[str, ...] | None = None) -> bool:
"""Coerce a flag; strings match ``falsy_strings`` (deny-list) when given, else a truthy allow-list."""
if not isinstance(value, str):
return bool(value)
return value.lower() not in falsy_strings if falsy_strings else value.lower() in ("true", "1", "yes")
def _read_mem0_json(config_path: Path) -> dict:
"""Best-effort read of mem0.json; missing/corrupt file -> {}."""
if config_path.exists():
@@ -117,9 +110,7 @@ class Mem0MemoryProvider(MemoryProvider):
"""Merge-write config to $HERMES_HOME/mem0.json."""
from utils import atomic_json_write
config_path = Path(hermes_home) / "mem0.json"
existing = _read_mem0_json(config_path)
existing.update(values)
atomic_json_write(config_path, existing, mode=0o600)
atomic_json_write(config_path, {**_read_mem0_json(config_path), **values}, mode=0o600)
def get_config_schema(self):
api_key_required = _load_config().get("mode", "platform") != "oss"
@@ -137,9 +128,7 @@ class Mem0MemoryProvider(MemoryProvider):
def _oss_hint(self, template: str, default: str = "vector store") -> str:
"""OSS-only hint; ``{vs}`` is the configured vector-store provider. "" in other modes."""
if self._mode != "oss":
return ""
return template.format(vs=self._config.get("oss", {}).get("vector_store", {}).get("provider", default))
return template.format(vs=self._config.get("oss", {}).get("vector_store", {}).get("provider", default)) if self._mode == "oss" else ""
def _create_backend(self):
# Lazy-install the mem0 SDK before the backend imports it (honors security.allow_lazy_installs);
@@ -151,9 +140,7 @@ class Mem0MemoryProvider(MemoryProvider):
from . import _backend
if self._mode == "oss":
return _backend.OSSBackend(self._config.get("oss", {}))
if self._host:
return _backend.SelfHostedBackend(self._api_key, self._host)
return _backend.PlatformBackend(self._api_key)
return _backend.SelfHostedBackend(self._api_key, self._host) if self._host else _backend.PlatformBackend(self._api_key)
except Exception as e:
logger.error("Mem0 backend failed to initialize (%s mode): %s", self._mode, e)
self._init_error = str(e)
@@ -191,12 +178,12 @@ class Mem0MemoryProvider(MemoryProvider):
"""Background-path wrapper: run ``call`` under the breaker; on error log ``msg`` and return None."""
try:
result = call()
self._record_success()
return result
except Exception as e:
self._record_failure()
log(msg, e)
return None
self._record_success()
return result
def initialize(self, session_id: str, **kwargs) -> None:
self._config = cfg = _load_config()
@@ -206,7 +193,8 @@ class Mem0MemoryProvider(MemoryProvider):
configured = cfg.get("user_id")
self._user_id = (None if configured == _DEFAULT_USER_ID else configured) or kwargs.get("user_id") or _DEFAULT_USER_ID
# Persisted rerank preference: default for mem0_search when the model omits ``rerank``. Platform-only.
self._rerank_default = _truthy(cfg.get("rerank", False))
_rr = cfg.get("rerank", False)
self._rerank_default = _rr.lower() in ("true", "1", "yes") if isinstance(_rr, str) else bool(_rr)
self._channel = kwargs.get("platform") or "cli"
self._backend = self._create_backend()
if self._backend and not self._atexit_registered:
@@ -243,11 +231,6 @@ class Mem0MemoryProvider(MemoryProvider):
backend = self._backend
if not query or backend is None or self._is_breaker_open():
return
with self._prefetch_lock:
# Same query already answered or still in flight: don't restart it.
if self._prefetch_query == query and (self._prefetch_done or (self._prefetch_thread and self._prefetch_thread.is_alive())):
return
self._prefetch_query, self._prefetch_result, self._prefetch_done = query, "", False
def _run():
results = self._try(lambda: self._search(query, backend=backend), logger.debug, "Mem0 prefetch failed: %s")
@@ -257,9 +240,12 @@ class Mem0MemoryProvider(MemoryProvider):
if self._prefetch_query == query:
self._prefetch_result, self._prefetch_done = body, True
t = threading.Thread(target=_run, daemon=True, name="mem0-prefetch")
with self._prefetch_lock:
self._prefetch_thread = t
# Same query already answered or still in flight: don't restart it.
if self._prefetch_query == query and (self._prefetch_done or (self._prefetch_thread and self._prefetch_thread.is_alive())):
return
self._prefetch_query, self._prefetch_result, self._prefetch_done = query, "", False
self._prefetch_thread = t = threading.Thread(target=_run, daemon=True, name="mem0-prefetch")
t.start()
def prefetch(self, query: str, *, session_id: str = "") -> str:
@@ -271,8 +257,7 @@ class Mem0MemoryProvider(MemoryProvider):
thread = self._prefetch_thread if self._prefetch_query == query else None
if thread:
thread.join(timeout=_PREFETCH_WAIT_SECS)
# Slow backend: skip injection; mem0_search tool remains the backstop.
return self._consume_prefetch_result(query) or ""
return self._consume_prefetch_result(query) or "" # slow backend: skip injection; mem0_search remains the backstop
def sync_turn(self, user_content: str, assistant_content: str, *, session_id: str = "") -> None:
"""Send the turn to Mem0 for server-side fact extraction (non-blocking)."""
@@ -302,7 +287,8 @@ class Mem0MemoryProvider(MemoryProvider):
def _tool_search(self, args: dict) -> str:
top_k = max(1, min(int(args.get("top_k", 10)), 50))
rerank = _truthy(args.get("rerank", self._rerank_default), falsy_strings=("false", "0", "no"))
rerank_raw = args.get("rerank", self._rerank_default)
rerank = rerank_raw.lower() not in ("false", "0", "no") if isinstance(rerank_raw, str) else bool(rerank_raw)
results = self._search(args["query"], top_k, rerank)
if not results:
return json.dumps({"result": "No relevant memories found."})
@@ -336,8 +322,6 @@ class Mem0MemoryProvider(MemoryProvider):
return tool_error(f"Missing required parameter: {missing}")
try:
result = body(self, args)
self._record_success()
return result
except Exception as e:
client = _is_client_error(e)
if client and on_client_error == "not_found":
@@ -345,6 +329,8 @@ class Mem0MemoryProvider(MemoryProvider):
if not client or on_client_error == "count":
self._record_failure()
return tool_error(self._format_error(label, e))
self._record_success()
return result
def _shutdown_backend(self):
with suppress(Exception):
+1 -5
View File
@@ -13,9 +13,7 @@ def _add_kwargs(user_id: str, agent_id: str, infer: bool, metadata: dict | None)
def _unwrap_results(response: Any) -> list:
"""Normalize API response — extract results list from dict or pass through."""
if isinstance(response, dict):
return response.get("results", [])
return response if isinstance(response, list) else []
return response.get("results", []) if isinstance(response, dict) else response if isinstance(response, list) else []
class Mem0Backend(ABC):
@@ -105,7 +103,6 @@ def _register_direct_openai_provider() -> None:
"""Register Hermes' OpenAI-only Mem0 LLM provider once per factory."""
from mem0.configs.llms.openai import OpenAIConfig
from mem0.utils.factory import LlmFactory
provider_map = getattr(LlmFactory, "provider_to_class", None)
register_provider = getattr(LlmFactory, "register_provider", None)
if not isinstance(provider_map, dict) or not callable(register_provider):
@@ -143,7 +140,6 @@ class OSSBackend(Mem0Backend):
vs_config["embedding_model_dims"] = dims
self._recreate_collection_if_dims_changed(vector_store.get("provider", "qdrant"), vs_config, dims)
vector_store["config"] = vs_config
config = {"vector_store": vector_store, "llm": _provider_block("llm", LLM_PROVIDERS), "embedder": _provider_block("embedder", EMBEDDER_PROVIDERS), "version": "v1.1"}
if str(config["llm"].get("provider") or "").strip().lower() == "openai":
# mem0 validates LlmConfig.provider before its factory lookup: build the supported OpenAI config, then swap the provider.
+30 -74
View File
@@ -54,12 +54,9 @@ def _prompt_api_key(label: str, env_var: str, hermes_home: str) -> str:
"""Prompt for API key, showing masked existing value if found."""
existing = os.environ.get(env_var, "")
env_path = Path(hermes_home) / ".env"
if not existing and env_path.exists():
# utf-8-sig: a Notepad BOM on line 1 would otherwise defeat the key match.
for line in env_path.read_text(encoding="utf-8-sig", errors="replace").splitlines():
if line.startswith(f"{env_var}="):
existing = line.split("=", 1)[1].strip()
break
if not existing and env_path.exists(): # utf-8-sig: a Notepad BOM on line 1 would otherwise defeat the key match
lines = env_path.read_text(encoding="utf-8-sig", errors="replace").splitlines()
existing = next((line.split("=", 1)[1].strip() for line in lines if line.startswith(f"{env_var}=")), "")
hint = f" (current: {_masked(existing)}, blank to keep)" if existing else ""
return getpass.getpass(f" {label} API key{hint}: ").strip()
@@ -93,8 +90,7 @@ _FLAG_DEFAULTS = {"oss_llm": "openai", "oss_embedder": "openai", "oss_vector": "
def parse_flags(argv: list[str] | None = None) -> dict[str, str]:
args = argv if argv is not None else sys.argv[1:]
flags: dict[str, Any] = {k: _FLAG_DEFAULTS.get(k, "") for k in _FLAG_KEYS}
flags["dry_run"] = False
flags: dict[str, Any] = {**{k: _FLAG_DEFAULTS.get(k, "") for k in _FLAG_KEYS}, "dry_run": False}
flag_map = {"--" + k.replace("_", "-"): k for k in _FLAG_KEYS}
i = 0
while i < len(args):
@@ -127,7 +123,6 @@ def build_oss_config(flags: dict[str, str]) -> tuple[dict, dict[str, str]]:
dims = KNOWN_DIMS.get(embedder_config["model"])
if dims:
embedder_config["embedding_dims"] = dims
vector_id = flags.get("oss_vector", "qdrant")
vector_config = dict(VECTOR_PROVIDERS[vector_id]["default_config"])
for key in _VECTOR_FLAG_KEYS.get(vector_id, ()):
@@ -135,7 +130,6 @@ def build_oss_config(flags: dict[str, str]) -> tuple[dict, dict[str, str]]:
vector_config[key] = int(val) if key == "port" else val
if "url" in vector_config:
vector_config.pop("path", None) # a remote Qdrant URL replaces local storage
oss_config = {"llm": {"provider": llm_id, "config": llm_config}, "embedder": {"provider": embedder_id, "config": embedder_config}, "vector_store": {"provider": vector_id, "config": vector_config}}
# An embedder sharing the LLM's provider reuses the LLM key when no embedder key was given.
llm_key = flags.get("oss_llm_key") if llm_def.get("needs_key") else ""
@@ -170,14 +164,8 @@ def _persist_provider_config(hermes_home: str, config: dict, provider_config: di
_write_env(Path(hermes_home) / ".env", env_writes)
if server:
_check_selfhosted_server(server)
print(f"\n Memory provider: {label}")
if server:
print(f" Server: {server}")
print(" Activation saved to config.yaml")
print(" Provider config saved")
if env_writes:
print(f" {key_line}")
print("\n Start a new session to activate.\n")
print("\n".join(["", f" Memory provider: {label}", *([f" Server: {server}"] if server else []), " Activation saved to config.yaml", " Provider config saved",
*([f" {key_line}"] if env_writes else []), "", " Start a new session to activate.", ""]))
def _setup_platform(hermes_home: str, config: dict, flags: dict[str, str]) -> None:
@@ -191,14 +179,12 @@ def _setup_platform(hermes_home: str, config: dict, flags: dict[str, str]) -> No
choices = ["true", "false"]
current = str(provider_config.get("rerank", "false") or "").lower()
provider_config["rerank"] = choices[_curses_select(" Enable reranking for recall", [(c, "") for c in choices], default=choices.index(current) if current in choices else 0)]
if flags.get("dry_run"):
_print_dry_run(str(provider_config), env_writes)
return
provider_config["mode"] = "platform"
# Routing checks ``host`` before platform, so clear a stale self-hosted host. "" rather than
# pop(): save_config merges into the existing mem0.json, so a popped key would survive.
provider_config["host"] = ""
provider_config.update(mode="platform", host="")
# _load_config() also seeds ``host`` from MEM0_HOST (.env); the file clear can't help there, so warn.
if os.environ.get("MEM0_HOST", "").strip():
print(f"\n ⚠ MEM0_HOST is set in your environment ({os.environ['MEM0_HOST']}). It overrides platform mode — remove it from ~/.hermes/.env (or unset it) or Hermes will keep routing to the self-hosted server.")
@@ -229,7 +215,6 @@ def _setup_selfhosted(hermes_home: str, config: dict, flags: dict[str, str]) ->
env_writes = _api_key_writes(flags, "Server API key", fresh_label="Server API key (blank if AUTH_DISABLED)")
user_id = flags.get("user_id") or _prompt("User identifier", default=provider_config.get("user_id") or "hermes-user")
agent_id = _prompt("Agent identifier", default=provider_config.get("agent_id") or "hermes")
if flags.get("dry_run"):
_print_dry_run(f"host={host}, user_id={user_id}, agent_id={agent_id}", env_writes, lambda: _check_selfhosted_server(host))
return
@@ -240,19 +225,14 @@ def _setup_selfhosted(hermes_home: str, config: dict, flags: dict[str, str]) ->
def _print_oss_summary(oss_config: dict, env_writes: dict, dry_run: bool = False) -> None:
llm, emb = oss_config["llm"], oss_config["embedder"]
w = 0 if dry_run else 9 # final summary column-aligns the labels
print("\n [dry-run] OSS config would be:" if dry_run else "\n ✓ Mem0 configured (OSS mode)")
print(f" {'LLM:':<{w}} {llm['provider']} ({llm['config'].get('model', '')})")
print(f" {'Embedder:':<{w}} {emb['provider']} ({emb['config'].get('model', '')})")
print(f" {'Vector:':<{w}} {oss_config['vector_store']['provider']}")
lines = ["", " [dry-run] OSS config would be:" if dry_run else " ✓ Mem0 configured (OSS mode)",
f" {'LLM:':<{w}} {llm['provider']} ({llm['config'].get('model', '')})", f" {'Embedder:':<{w}} {emb['provider']} ({emb['config'].get('model', '')})",
f" {'Vector:':<{w}} {oss_config['vector_store']['provider']}"]
if dry_run:
if env_writes:
print(f" Env vars: {', '.join(env_writes.keys())}")
return
if env_writes:
print(" API keys saved to .env")
print(" Config saved to mem0.json")
print(" Provider set in config.yaml")
print("\n Start a new session to activate.\n")
lines += [f" Env vars: {', '.join(env_writes.keys())}"] if env_writes else []
else:
lines += [*([" API keys saved to .env"] if env_writes else []), " Config saved to mem0.json", " Provider set in config.yaml", "", " Start a new session to activate.", ""]
print("\n".join(lines))
def _finish_oss(hermes_home: str, config: dict, oss_config: dict, env_writes: dict[str, str], user_id: str, agent_id: str, pgvector_config: dict | None = None) -> None:
@@ -313,13 +293,9 @@ def _ensure_pgvector(host: str = "localhost", port: int = 5432) -> dict | None:
if _pg_ready(host, port, 15):
print(" ✓ PostgreSQL container restarted")
return None
if input(" Start pgvector via Docker? [Y/n]: ").strip().lower() in ("", "y", "yes"):
return _start_pgvector_docker(host, port)
print(" Skipping Docker setup. Make sure PostgreSQL with pgvector is running.")
return None
def _start_pgvector_docker(host: str, port: int) -> dict | None:
if input(" Start pgvector via Docker? [Y/n]: ").strip().lower() not in ("", "y", "yes"):
print(" Skipping Docker setup. Make sure PostgreSQL with pgvector is running.")
return None
try:
print(f" Pulling {_PGVECTOR_IMAGE}...")
_docker("pull", _PGVECTOR_IMAGE, timeout=120)
@@ -357,7 +333,11 @@ def _ensure_ollama(models: list[str]) -> bool:
print(" Warning: Ollama not reachable. Models cannot be pulled.")
return False
for model in models:
if any(model in n or model.split(":")[0] in n for n in _ollama_models(_OLLAMA_URL)):
try:
names = [m.get("name", "") for m in json.loads(_http_get(_OLLAMA_URL, "/api/tags", 5).read()).get("models", [])]
except Exception:
names = []
if any(model in n or model.split(":")[0] in n for n in names):
print(f" ✓ Model '{model}' available")
continue
print(f" Pulling '{model}'... (this may take a few minutes)")
@@ -369,13 +349,6 @@ def _ensure_ollama(models: list[str]) -> bool:
return True
def _ollama_models(url: str) -> list[str]:
try:
return [m.get("name", "") for m in json.loads(_http_get(url, "/api/tags", 5).read()).get("models", [])]
except Exception:
return []
def _ensure_pgvector_extension(pg_config: dict) -> None:
try:
import psycopg2
@@ -403,18 +376,13 @@ def _wait_for_port(host: str, port: int, timeout: int = 15) -> None:
# Picker descriptions: LLM/embedder show model (+ URL); vector stores by provider id (default: the id itself).
def _provider_description(v: dict) -> str:
model, url = v.get("default_model", ""), v.get("default_url")
return f"{model} ({url})" if url else model
_VECTOR_DESCRIPTIONS = {"qdrant": lambda cfg: cfg.get("path", "local storage"), "pgvector": lambda cfg: f"{cfg.get('host', 'localhost')}:{cfg.get('port', 5432)}"}
def _configure_model_provider(kind: str, registry: dict, hermes_home: str, env_writes: dict[str, str], llm: tuple[str, dict] | None = None) -> tuple[str, dict, str, str | None]:
"""Pick an LLM/embedder provider, collect its key, and (for Ollama) model + URL -> (id, definition, model, url).
For the embedder (``llm`` given), a provider shared with the LLM reuses the LLM key instead of prompting again."""
items = [(v["label"], _provider_description(v)) for v in registry.values()]
items = [(v["label"], f"{v.get('default_model', '')} ({v['default_url']})" if v.get("default_url") else v.get("default_model", "")) for v in registry.values()]
pid = list(registry)[_curses_select(f"{kind} Provider", items, 0)]
pdef = registry[pid]
model, url = pdef["default_model"], pdef.get("default_url")
@@ -430,31 +398,23 @@ def _configure_model_provider(kind: str, registry: dict, hermes_home: str, env_w
return pid, pdef, model, url
def _prompt_pgvector_config() -> dict:
"""Native PostgreSQL — prompt for connection details (user first, matching the historical order)."""
pg = {k: _input(f"PostgreSQL {label}", d) for k, label, d in (
("user", "user", os.getenv("USER", "postgres")), ("host", "host", "localhost"), ("port", "port", "5432"), ("dbname", "database", "postgres"))}
pg_password = getpass.getpass(" PostgreSQL password (blank if none): ").strip()
return {**pg, "port": int(pg["port"]), **({"password": pg_password} if pg_password else {})}
def _setup_oss_interactive(hermes_home: str, config: dict) -> None:
env_writes: dict[str, str] = {}
llm_id, llm_def, llm_model, llm_url = _configure_model_provider("LLM", LLM_PROVIDERS, hermes_home, env_writes)
embedder_id, _, embedder_model, embedder_url = _configure_model_provider("Embedder", EMBEDDER_PROVIDERS, hermes_home, env_writes, llm=(llm_id, llm_def))
vector_items = [(v["label"], _VECTOR_DESCRIPTIONS.get(pid, lambda cfg: pid)(v.get("default_config", {}))) for pid, v in VECTOR_PROVIDERS.items()]
vector_id = list(VECTOR_PROVIDERS)[_curses_select("Vector Store", vector_items, 0)]
# Auto-setup: ensure Ollama is running and models are pulled; ensure pgvector is reachable (offer Docker if not).
ollama_models = [m for pid, m in ((llm_id, llm_model), (embedder_id, embedder_model)) if pid == "ollama"]
if ollama_models:
_ensure_ollama(ollama_models)
pgvector_config = None
if vector_id == "pgvector":
pgvector_config = _ensure_pgvector() or _prompt_pgvector_config()
pgvector_config = _ensure_pgvector() if vector_id == "pgvector" else None
if vector_id == "pgvector" and not pgvector_config: # native PostgreSQL: prompt for connection details (user first, historical order)
pg = {k: _input(f"PostgreSQL {label}", d) for k, label, d in (("user", "user", os.getenv("USER", "postgres")), ("host", "host", "localhost"), ("port", "port", "5432"), ("dbname", "database", "postgres"))}
pg_password = getpass.getpass(" PostgreSQL password (blank if none): ").strip()
pgvector_config = {**pg, "port": int(pg["port"]), **({"password": pg_password} if pg_password else {})}
user_id = _input("User ID", os.getenv("USER", "hermes-user"))
agent_id = _input("Agent ID", "hermes")
flags = {
"oss_llm": llm_id, "oss_llm_model": llm_model, "oss_llm_url": llm_url or "",
"oss_llm_key": env_writes.get(llm_def["env_var"], "") if llm_def.get("env_var") else "",
@@ -476,12 +436,8 @@ def _install_provider_deps(llm_id: str, embedder_id: str, vector_id: str) -> Non
outcome = install_specs([dep], timeout=60)
except Exception:
outcome = None
if outcome is not None and outcome.ok:
print(f" ✓ Installed {dep}")
elif outcome is not None and outcome.blocked:
print(f" Warning: cannot install {dep}: {outcome.reason}")
else:
print(f" Warning: Could not install {dep}. Install manually: uv pip install {dep}")
print(f" ✓ Installed {dep}" if outcome is not None and outcome.ok else f" Warning: cannot install {dep}: {outcome.reason}" if outcome is not None and outcome.blocked
else f" Warning: Could not install {dep}. Install manually: uv pip install {dep}")
if deps:
import importlib
importlib.invalidate_caches()