From 0edab1ac4d4f72f489945edbb62a34f69e6864b3 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 6 Sep 2026 02:25:27 -0700 Subject: [PATCH] test(profiles): two invariants for the truncated-JSON refusal, comment trimmed to the why Parametrized refusal test (bare/fenced/uppercase-fenced truncated objects) asserts profile.yaml is byte-identical afterwards; one prose test keeps the lenient fallback contract. Replaces five near-duplicate tests from #104075. Campaign tracker: https://github.com/NousResearch/hermes-agent/issues/104154 --- hermes_cli/profile_describer.py | 8 +- tests/hermes_cli/test_profile_describer.py | 108 +++++---------------- 2 files changed, 25 insertions(+), 91 deletions(-) diff --git a/hermes_cli/profile_describer.py b/hermes_cli/profile_describer.py index 2e17e0b19f..31c9598c2a 100644 --- a/hermes_cli/profile_describer.py +++ b/hermes_cli/profile_describer.py @@ -166,12 +166,8 @@ def describe_profile(profile_name: str, *, overwrite: bool = False, timeout: Opt raw = "" parsed = _extract_json_blob(raw) if parsed is None: - # A response that is JSON-SHAPED (starts with `{`, once code fences are stripped) but - # failed to parse is a malformed/truncated structured reply -- e.g. the aux model or - # transport cut it off mid-object -- not free-form prose. Persisting it verbatim writes - # a raw JSON fragment (literal `{`, `\n` escapes, a sentence chopped mid-word) into - # profile.yaml's description field (#104067). Only fall back to "whole reply is prose" - # when the reply never looked like JSON in the first place. + # JSON-shaped but unparseable = the requested object got cut off (#104067); the prose + # fallback below is only for models that never attempted JSON. stripped = _FENCE_RE.sub("", raw.strip()) if stripped.startswith("{"): logger.info( diff --git a/tests/hermes_cli/test_profile_describer.py b/tests/hermes_cli/test_profile_describer.py index 649c44da22..d01d9a3128 100644 --- a/tests/hermes_cli/test_profile_describer.py +++ b/tests/hermes_cli/test_profile_describer.py @@ -76,101 +76,39 @@ def test_describer_writes_description_with_auto_true(profile_env, monkeypatch): assert meta["description_auto"] is True -def test_describer_rejects_truncated_json_response(profile_env, monkeypatch): - # Real-world shape (#104067): the aux model's response is cut off mid-object -- no - # closing brace/quote -- so `_extract_json_blob` can't parse it. The old fallback - # persisted this raw fragment verbatim as the description; it must now be refused. +@pytest.fixture +def registered_profile(profile_env, monkeypatch): monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof") monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n) monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env) + return profile_env - truncated = '{\n "description": "Generalist agent that writes and debugs code, orchestrates autonomous sub-agents, and automates macOS/App' - with _patch_aux_client(truncated), patch( - "agent.auxiliary_client.get_auxiliary_extra_body", return_value={} - ): - outcome = describer.describe_profile("myprof") +@pytest.mark.parametrize("raw", [ + '{\n "description": "Generalist agent that writes and debugs code, orchestrates autonomous sub-agents, and automates macOS/App', + '{\n "desc', + '```json\n{\n "description": "Generalist agent that writes and debugs cod', + '```JSON\n{\n "description": "Generalist agent that writes and debugs cod', +]) +def test_describer_refuses_json_shaped_reply_that_does_not_parse(registered_profile, raw): + """A reply that started as the requested JSON object but was cut off (#104067) is not prose: + it must be refused and leave profile.yaml untouched -- including behind an uppercase fence.""" + profiles_mod.write_profile_meta(registered_profile, description="previous", description_auto=True) + before = (registered_profile / "profile.yaml").read_bytes() + with _patch_aux_client(raw), patch("agent.auxiliary_client.get_auxiliary_extra_body", return_value={}): + outcome = describer.describe_profile("myprof", overwrite=True) assert outcome.ok is False - assert "malformed" in outcome.reason.lower() or "truncated" in outcome.reason.lower() - # Nothing was written: no profile.yaml, or if one exists it has no description key. - meta = profiles_mod.read_profile_meta(profile_env) - assert not meta.get("description") + assert (registered_profile / "profile.yaml").read_bytes() == before -def test_describer_rejects_json_shaped_response_with_no_closing_brace_at_all(profile_env, monkeypatch): - # Even more truncated: cut off right after the opening brace, before any key. - monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof") - monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n) - monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env) - - with _patch_aux_client("{\n \"desc"), patch( - "agent.auxiliary_client.get_auxiliary_extra_body", return_value={} - ): +def test_describer_still_accepts_plain_prose_fallback(registered_profile): + """A reply that never looked like JSON keeps the lenient one-paragraph prose fallback.""" + with _patch_aux_client("Writes and debugs Python codebases.\n\nSecond paragraph is dropped."), \ + patch("agent.auxiliary_client.get_auxiliary_extra_body", return_value={}): outcome = describer.describe_profile("myprof") - - assert outcome.ok is False - meta = profiles_mod.read_profile_meta(profile_env) - assert not meta.get("description") - - -def test_describer_rejects_fenced_json_shaped_truncated_response(profile_env, monkeypatch): - # A ```json fence around a truncated object must still be recognized as JSON-shaped - # after fence stripping, not fall through to the raw-text fallback. - monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof") - monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n) - monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env) - - fenced_truncated = '```json\n{\n "description": "Generalist agent that writes and debugs cod' - with _patch_aux_client(fenced_truncated), patch( - "agent.auxiliary_client.get_auxiliary_extra_body", return_value={} - ): - outcome = describer.describe_profile("myprof") - - assert outcome.ok is False - meta = profiles_mod.read_profile_meta(profile_env) - assert not meta.get("description") - - -def test_describer_rejects_uppercase_fenced_json_shaped_truncated_response(profile_env, monkeypatch): - # Regression for review feedback on #104075: an uppercase ```JSON fence must be - # stripped the same as a lowercase one. Before, _FENCE_RE was case-sensitive, so - # stripping this left "JSON\n{...}" -- which doesn't start with "{" -- and the - # truncated fragment fell through to the raw-text-as-prose fallback and got persisted - # anyway, defeating the fix for this fence variant. - monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof") - monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n) - monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env) - - fenced_truncated = '```JSON\n{\n "description": "Generalist agent that writes and debugs cod' - with _patch_aux_client(fenced_truncated), patch( - "agent.auxiliary_client.get_auxiliary_extra_body", return_value={} - ): - outcome = describer.describe_profile("myprof") - - assert outcome.ok is False - meta = profiles_mod.read_profile_meta(profile_env) - assert not meta.get("description") - - -def test_describer_still_accepts_plain_prose_fallback(profile_env, monkeypatch): - # Regression guard: a model that ignores the JSON instruction and just replies in - # plain prose (never looked JSON-shaped) must still hit the existing lenient - # raw-text fallback -- this fix must not make the describer stricter than before - # for genuinely non-JSON replies. - monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof") - monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n) - monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env) - - with _patch_aux_client("Writes and debugs Python codebases end to end."), patch( - "agent.auxiliary_client.get_auxiliary_extra_body", return_value={} - ): - outcome = describer.describe_profile("myprof") - assert outcome.ok, outcome.reason - assert outcome.description == "Writes and debugs Python codebases end to end." - meta = profiles_mod.read_profile_meta(profile_env) - assert meta["description"] == "Writes and debugs Python codebases end to end." - assert meta["description_auto"] is True + assert outcome.description == "Writes and debugs Python codebases." + assert profiles_mod.read_profile_meta(registered_profile)["description"] == outcome.description def test_describer_refuses_to_overwrite_user_authored(profile_env, monkeypatch):