From 5705b68f703670dc32a57838f2b5834ea24bdc1c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:18:25 -0700 Subject: [PATCH] fix: memory-plugin and Qwen-CLI config JSON survives Windows BOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port from earendil-works/pi#8337 (UTF-8 BOM normalization in text inputs): sibling sites the merged #81967 BOM sweep missed. json.loads hard-fails on a leading U+FEFF and every one of these loaders swallows the exception and silently falls back to defaults — a user who edited mem0.json, honcho.json, hindsight/config.json, or supermemory.json in Notepad lost their whole config with no error, and Qwen CLI OAuth creds saved with a BOM raised qwen_auth_read_failed. - plugins/memory/{honcho,mem0,hindsight,supermemory}: 13 read sites -> utf-8-sig - hermes_cli/auth.py: _read_qwen_cli_tokens -> utf-8-sig - tests: BOM regression tests per loader (sabotage-proven) + plain-UTF-8 guard --- hermes_cli/auth_qwen.py | 2 +- plugins/memory/hindsight/setup.py | 2 +- plugins/memory/honcho/client.py | 2 +- plugins/memory/honcho/client_cache.py | 2 +- plugins/memory/openviking/__init__.py | 2 +- plugins/memory/supermemory/__init__.py | 2 +- .../memory/test_bom_tolerant_config_reads.py | 101 ++++++++++++++++++ utils.py | 2 +- 8 files changed, 108 insertions(+), 7 deletions(-) create mode 100644 tests/plugins/memory/test_bom_tolerant_config_reads.py diff --git a/hermes_cli/auth_qwen.py b/hermes_cli/auth_qwen.py index acb85a89b8..67b3d9315b 100644 --- a/hermes_cli/auth_qwen.py +++ b/hermes_cli/auth_qwen.py @@ -32,7 +32,7 @@ def _read_qwen_cli_tokens() -> Dict[str, Any]: if not auth_path.exists(): raise _qwen_err("Qwen CLI credentials not found. Run 'qwen auth qwen-oauth' first.", "qwen_auth_missing") try: - data = json.loads(auth_path.read_text(encoding="utf-8")) + data = json.loads(auth_path.read_text(encoding="utf-8-sig")) except Exception as exc: raise _qwen_err( f"Failed to read Qwen CLI credentials from {auth_path}: {exc}", "qwen_auth_read_failed", diff --git a/plugins/memory/hindsight/setup.py b/plugins/memory/hindsight/setup.py index 5f6692c64e..e6fdbac351 100644 --- a/plugins/memory/hindsight/setup.py +++ b/plugins/memory/hindsight/setup.py @@ -168,7 +168,7 @@ def run_setup(provider, hermes_home: str, config: dict) -> None: materialized_config = dict(provider_config) with contextlib.suppress(Exception): materialized_config = json.loads( - (Path(hermes_home) / "hindsight" / "config.json").read_text(encoding="utf-8") + (Path(hermes_home) / "hindsight" / "config.json").read_text(encoding="utf-8-sig") ) llm_api_key = ( env_writes.get("HINDSIGHT_LLM_API_KEY", "") diff --git a/plugins/memory/honcho/client.py b/plugins/memory/honcho/client.py index 1aa05f1c8b..5765425451 100644 --- a/plugins/memory/honcho/client.py +++ b/plugins/memory/honcho/client.py @@ -95,7 +95,7 @@ def resolve_active_host() -> str: def _read_config(path: Path) -> dict: """Parse a honcho.json; {} when absent (parse/OS errors propagate).""" - return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + return json.loads(path.read_text(encoding="utf-8-sig")) if path.exists() else {} def resolve_global_config_path() -> Path: diff --git a/plugins/memory/honcho/client_cache.py b/plugins/memory/honcho/client_cache.py index 6af5549ec9..e17dc2e87e 100644 --- a/plugins/memory/honcho/client_cache.py +++ b/plugins/memory/honcho/client_cache.py @@ -75,7 +75,7 @@ def _ambient_host_block() -> tuple[dict | None, dict]: path = resolve_config_path() if not path.exists(): return None, {} - raw = json.loads(path.read_text(encoding="utf-8")) + raw = json.loads(path.read_text(encoding="utf-8-sig")) return raw, _host_block(raw, resolve_active_host()) diff --git a/plugins/memory/openviking/__init__.py b/plugins/memory/openviking/__init__.py index 97a3c255d6..25353cd2af 100644 --- a/plugins/memory/openviking/__init__.py +++ b/plugins/memory/openviking/__init__.py @@ -560,7 +560,7 @@ def _load_ovcli_config(path: Optional[Path] = None) -> dict: config_path = path or _resolve_ovcli_config_path() if not config_path.exists(): return {} - data = json.loads(config_path.read_text(encoding="utf-8")) + data = json.loads(config_path.read_text(encoding="utf-8-sig")) if not isinstance(data, dict): raise ValueError(f"OpenViking CLI config must be a JSON object: {config_path}") return data diff --git a/plugins/memory/supermemory/__init__.py b/plugins/memory/supermemory/__init__.py index 1a3fa222c4..b69d9795f7 100644 --- a/plugins/memory/supermemory/__init__.py +++ b/plugins/memory/supermemory/__init__.py @@ -99,7 +99,7 @@ _CONFIG_SPEC: Dict[str, tuple] = { def _read_json_dict(path: Path) -> dict: - raw = _quietly(lambda: json.loads(path.read_text(encoding="utf-8")), "Failed to parse %s", path) if path.exists() else None + raw = _quietly(lambda: json.loads(path.read_text(encoding="utf-8-sig")), "Failed to parse %s", path) if path.exists() else None return raw if isinstance(raw, dict) else {} diff --git a/tests/plugins/memory/test_bom_tolerant_config_reads.py b/tests/plugins/memory/test_bom_tolerant_config_reads.py new file mode 100644 index 0000000000..13c677165c --- /dev/null +++ b/tests/plugins/memory/test_bom_tolerant_config_reads.py @@ -0,0 +1,101 @@ +"""BOM-tolerant reads of user-editable plugin/auth JSON files. + +Windows GUI editors (Notepad, PowerShell ``>``) prepend a UTF-8 BOM when +saving JSON. ``json.loads`` hard-fails on a leading BOM ("Unexpected UTF-8 +BOM (decode using utf-8-sig)"), and every loader here swallows the exception +and silently falls back to defaults — so a user who edited mem0.json / +honcho.json / hindsight config.json / supermemory.json in Notepad lost their +entire config with no error. Same class as the merged #81967 sweep (auth +store, .env, memory files); these plugin-config readers were the missed +sibling sites. Ported alongside earendil-works/pi#8337's BOM normalization. + +Each test writes the file with a real BOM (utf-8-sig encoding) and asserts +the loader still returns the configured values. +""" + +import json + +import pytest + + +def _write_bom_json(path, payload: dict) -> None: + path.write_text(json.dumps(payload), encoding="utf-8-sig") + # Sanity: the BOM must actually be on disk for the test to mean anything. + assert path.read_bytes().startswith(b"\xef\xbb\xbf") + + +def test_mem0_load_config_tolerates_bom(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr( + "hermes_constants.get_hermes_home", lambda: tmp_path + ) + _write_bom_json(tmp_path / "mem0.json", {"agent_id": "bom-agent"}) + + from plugins.memory.mem0 import _load_config + + assert _load_config()["agent_id"] == "bom-agent" + + +def test_supermemory_load_config_tolerates_bom(tmp_path): + _write_bom_json( + tmp_path / "supermemory.json", {"container_tag": "bom-tag"} + ) + + from plugins.memory.supermemory import _load_supermemory_config + + assert _load_supermemory_config(str(tmp_path))["container_tag"] == "bom-tag" + + +def test_hindsight_load_config_tolerates_bom(tmp_path, monkeypatch): + (tmp_path / "hindsight").mkdir() + _write_bom_json( + tmp_path / "hindsight" / "config.json", {"mode": "bom-mode"} + ) + import plugins.memory.hindsight as hs + + monkeypatch.setattr(hs, "get_hermes_home", lambda: tmp_path) + assert hs._load_config()["mode"] == "bom-mode" + + +def test_honcho_cli_read_config_tolerates_bom(tmp_path, monkeypatch): + import plugins.memory.honcho.cli as hcli + + cfg_path = tmp_path / "honcho.json" + monkeypatch.setattr(hcli, "_config_path", lambda: cfg_path) + _write_bom_json(cfg_path, {"workspace": "bom-ws"}) + assert hcli._read_config()["workspace"] == "bom-ws" + + +def test_honcho_client_from_global_config_tolerates_bom(tmp_path): + from plugins.memory.honcho.client import HonchoClientConfig + + cfg_path = tmp_path / "honcho.json" + _write_bom_json(cfg_path, {"workspace": "bom-ws", "enabled": True}) + cfg = HonchoClientConfig.from_global_config(config_path=cfg_path) + assert cfg.workspace_id == "bom-ws" + assert cfg.explicitly_configured is True + + +def test_qwen_cli_tokens_tolerates_bom(tmp_path, monkeypatch): + import hermes_cli.auth as auth_mod + + creds = tmp_path / "oauth_creds.json" + _write_bom_json( + creds, + {"access_token": "tok", "expiry_date": 4102444800000}, + ) + monkeypatch.setattr(auth_mod, "_qwen_cli_auth_path", lambda: creds) + data = auth_mod._read_qwen_cli_tokens() + assert data["access_token"] == "tok" + + +def test_plain_utf8_still_parses(tmp_path): + """utf-8-sig reads plain UTF-8 unchanged — no regression for normal files.""" + from plugins.memory.supermemory import _load_supermemory_config + + (tmp_path / "supermemory.json").write_text( + json.dumps({"container_tag": "plain-tag"}), encoding="utf-8" + ) + assert ( + _load_supermemory_config(str(tmp_path))["container_tag"] == "plain-tag" + ) diff --git a/utils.py b/utils.py index 91c3a8399d..6de0be3312 100644 --- a/utils.py +++ b/utils.py @@ -319,7 +319,7 @@ def read_json_or_empty(path: Union[str, Path]) -> dict: (memory-provider ``save_config``), so a corrupt sidecar degrades to defaults instead of taking the provider down.""" try: - data = json.loads(Path(path).read_text(encoding="utf-8")) + data = json.loads(Path(path).read_text(encoding="utf-8-sig")) # utf-8-sig: a Windows-editor BOM must not wipe the config except (OSError, ValueError): return {} return data if isinstance(data, dict) else {}