test(profiles): two invariants for the truncated-JSON refusal, comment trimmed to the why

Parametrized refusal test (bare/fenced/uppercase-fenced truncated objects) asserts
profile.yaml is byte-identical afterwards; one prose test keeps the lenient
fallback contract. Replaces five near-duplicate tests from #104075.

Campaign tracker: https://github.com/NousResearch/hermes-agent/issues/104154

hermes_cli/profile_describer.py

@@ -166,12 +166,8 @@ def describe_profile(profile_name: str, *, overwrite: bool = False, timeout: Opt
         raw = ""
     parsed = _extract_json_blob(raw)
     if parsed is None:
-        # A response that is JSON-SHAPED (starts with `{`, once code fences are stripped) but
-        # failed to parse is a malformed/truncated structured reply -- e.g. the aux model or
-        # transport cut it off mid-object -- not free-form prose. Persisting it verbatim writes
-        # a raw JSON fragment (literal `{`, `\n` escapes, a sentence chopped mid-word) into
-        # profile.yaml's description field (#104067). Only fall back to "whole reply is prose"
-        # when the reply never looked like JSON in the first place.
+        # JSON-shaped but unparseable = the requested object got cut off (#104067); the prose
+        # fallback below is only for models that never attempted JSON.
         stripped = _FENCE_RE.sub("", raw.strip())
         if stripped.startswith("{"):
             logger.info(

tests/hermes_cli/test_profile_describer.py

@@ -76,101 +76,39 @@ def test_describer_writes_description_with_auto_true(profile_env, monkeypatch):
     assert meta["description_auto"] is True
 
 
-def test_describer_rejects_truncated_json_response(profile_env, monkeypatch):
-    # Real-world shape (#104067): the aux model's response is cut off mid-object -- no
-    # closing brace/quote -- so `_extract_json_blob` can't parse it. The old fallback
-    # persisted this raw fragment verbatim as the description; it must now be refused.
+@pytest.fixture
+def registered_profile(profile_env, monkeypatch):
     monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof")
     monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n)
     monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env)
+    return profile_env
 
-    truncated = '{\n  "description": "Generalist agent that writes and debugs code, orchestrates autonomous sub-agents, and automates macOS/App'
-    with _patch_aux_client(truncated), patch(
-        "agent.auxiliary_client.get_auxiliary_extra_body", return_value={}
-    ):
-        outcome = describer.describe_profile("myprof")
 
+@pytest.mark.parametrize("raw", [
+    '{\n  "description": "Generalist agent that writes and debugs code, orchestrates autonomous sub-agents, and automates macOS/App',
+    '{\n  "desc',
+    '```json\n{\n  "description": "Generalist agent that writes and debugs cod',
+    '```JSON\n{\n  "description": "Generalist agent that writes and debugs cod',
+])
+def test_describer_refuses_json_shaped_reply_that_does_not_parse(registered_profile, raw):
+    """A reply that started as the requested JSON object but was cut off (#104067) is not prose:
+    it must be refused and leave profile.yaml untouched -- including behind an uppercase fence."""
+    profiles_mod.write_profile_meta(registered_profile, description="previous", description_auto=True)
+    before = (registered_profile / "profile.yaml").read_bytes()
+    with _patch_aux_client(raw), patch("agent.auxiliary_client.get_auxiliary_extra_body", return_value={}):
+        outcome = describer.describe_profile("myprof", overwrite=True)
     assert outcome.ok is False
-    assert "malformed" in outcome.reason.lower() or "truncated" in outcome.reason.lower()
-    # Nothing was written: no profile.yaml, or if one exists it has no description key.
-    meta = profiles_mod.read_profile_meta(profile_env)
-    assert not meta.get("description")
+    assert (registered_profile / "profile.yaml").read_bytes() == before
 
 
-def test_describer_rejects_json_shaped_response_with_no_closing_brace_at_all(profile_env, monkeypatch):
-    # Even more truncated: cut off right after the opening brace, before any key.
-    monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof")
-    monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n)
-    monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env)
-
-    with _patch_aux_client("{\n  \"desc"), patch(
-        "agent.auxiliary_client.get_auxiliary_extra_body", return_value={}
-    ):
+def test_describer_still_accepts_plain_prose_fallback(registered_profile):
+    """A reply that never looked like JSON keeps the lenient one-paragraph prose fallback."""
+    with _patch_aux_client("Writes and debugs Python codebases.\n\nSecond paragraph is dropped."), \
+         patch("agent.auxiliary_client.get_auxiliary_extra_body", return_value={}):
         outcome = describer.describe_profile("myprof")
-
-    assert outcome.ok is False
-    meta = profiles_mod.read_profile_meta(profile_env)
-    assert not meta.get("description")
-
-
-def test_describer_rejects_fenced_json_shaped_truncated_response(profile_env, monkeypatch):
-    # A ```json fence around a truncated object must still be recognized as JSON-shaped
-    # after fence stripping, not fall through to the raw-text fallback.
-    monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof")
-    monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n)
-    monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env)
-
-    fenced_truncated = '```json\n{\n  "description": "Generalist agent that writes and debugs cod'
-    with _patch_aux_client(fenced_truncated), patch(
-        "agent.auxiliary_client.get_auxiliary_extra_body", return_value={}
-    ):
-        outcome = describer.describe_profile("myprof")
-
-    assert outcome.ok is False
-    meta = profiles_mod.read_profile_meta(profile_env)
-    assert not meta.get("description")
-
-
-def test_describer_rejects_uppercase_fenced_json_shaped_truncated_response(profile_env, monkeypatch):
-    # Regression for review feedback on #104075: an uppercase ```JSON fence must be
-    # stripped the same as a lowercase one. Before, _FENCE_RE was case-sensitive, so
-    # stripping this left "JSON\n{...}" -- which doesn't start with "{" -- and the
-    # truncated fragment fell through to the raw-text-as-prose fallback and got persisted
-    # anyway, defeating the fix for this fence variant.
-    monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof")
-    monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n)
-    monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env)
-
-    fenced_truncated = '```JSON\n{\n  "description": "Generalist agent that writes and debugs cod'
-    with _patch_aux_client(fenced_truncated), patch(
-        "agent.auxiliary_client.get_auxiliary_extra_body", return_value={}
-    ):
-        outcome = describer.describe_profile("myprof")
-
-    assert outcome.ok is False
-    meta = profiles_mod.read_profile_meta(profile_env)
-    assert not meta.get("description")
-
-
-def test_describer_still_accepts_plain_prose_fallback(profile_env, monkeypatch):
-    # Regression guard: a model that ignores the JSON instruction and just replies in
-    # plain prose (never looked JSON-shaped) must still hit the existing lenient
-    # raw-text fallback -- this fix must not make the describer stricter than before
-    # for genuinely non-JSON replies.
-    monkeypatch.setattr(profiles_mod, "profile_exists", lambda n: n == "myprof")
-    monkeypatch.setattr(profiles_mod, "normalize_profile_name", lambda n: n)
-    monkeypatch.setattr(profiles_mod, "get_profile_dir", lambda n: profile_env)
-
-    with _patch_aux_client("Writes and debugs Python codebases end to end."), patch(
-        "agent.auxiliary_client.get_auxiliary_extra_body", return_value={}
-    ):
-        outcome = describer.describe_profile("myprof")
-
     assert outcome.ok, outcome.reason
-    assert outcome.description == "Writes and debugs Python codebases end to end."
-    meta = profiles_mod.read_profile_meta(profile_env)
-    assert meta["description"] == "Writes and debugs Python codebases end to end."
-    assert meta["description_auto"] is True
+    assert outcome.description == "Writes and debugs Python codebases."
+    assert profiles_mod.read_profile_meta(registered_profile)["description"] == outcome.description
 
 
 def test_describer_refuses_to_overwrite_user_authored(profile_env, monkeypatch):