fix(gateway): a leaked terminal <|eos|> sentinel no longer hides the MEDIA attachment

A provider that leaks its end-of-sequence control token glued to the last
MEDIA path (MEDIA:/x.png<|eos|>) made BasePlatformAdapter.extract_media
return no media: neither MEDIA_TAG_CLEANUP_RE nor MEDIA_EXTENSIONLESS_TAG_RE
accepts '<' as a path terminator (deliberately - widening the delimiter set
reopens the glue classes #68773/#88038), so the file was silently dropped
and the raw tag leaked into the chat.

Recognise the one exact token at the shared normalization seam instead:
_mask_media_scan_text blanks a terminal <|eos|> (offset-preserving) so the
glued tag ends on whitespace for every scan - extraction, display strip and
stream cleanup - across the known-extension, unknown-extension and
extension-less branches; _deliverable_tag_spans deletes the sentinel with
the tag it terminated. Not the exact token, not terminal, or inside a
protected code/JSON span: byte-identical to before.

Fixes #111046

Co-authored-by: Hukla <129692708+huklaa@users.noreply.github.com>
Co-authored-by: joaomarcos <joaomarcosdias444@gmail.com>

gateway/platforms/base.py

@@ -1318,11 +1318,41 @@ def _has_media_directives(text: str) -> bool:
     return "MEDIA:" in text or "[[audio_as_voice]]" in text or "[[as_document]]" in text
 
 
+# A provider can leak its exact end-of-sequence control token glued to the last MEDIA path
+# (``MEDIA:/x.png<|eos|>``). Neither tag regex accepts ``<`` as a path terminator — deliberately,
+# since widening the delimiter set re-opens the glue classes #68773/#88038 — so the attachment was
+# silently dropped (#111046). The one exact token is recognised here, at the seam every scan shares
+# (extraction, display strip, stream cleanup), and only when it terminates the whole response.
+_TERMINAL_SENTINEL = "<|eos|>"
+
+
+def _terminal_sentinel_start(text: str) -> int:
+    """Offset of an exact ``<|eos|>`` closing ``text`` (trailing whitespace ignored), else -1."""
+    end = len(text.rstrip())
+    start = end - len(_TERMINAL_SENTINEL)
+    return start if start >= 0 and text[start:end] == _TERMINAL_SENTINEL else -1
+
+
 def _mask_media_scan_text(text: str) -> str:
-    """Offset-preserving mask of protected spans (code, quotes, JSON string values).
+    """Offset-preserving mask of protected spans (code, quotes, JSON string values) and of a
+    terminal ``<|eos|>`` sentinel, so a tag glued to it ends on whitespace like any other.
     BasePlatformAdapter is defined later in this module; resolved at call time."""
     A = BasePlatformAdapter
-    return A._mask_json_string_media(A._mask_protected_spans(text))
+    masked = A._mask_json_string_media(A._mask_protected_spans(text))
+    start = _terminal_sentinel_start(text)
+    if start >= 0:
+        masked = _blank_spans(masked, [(start, start + len(_TERMINAL_SENTINEL))])
+    return masked
+
+
+def _deliverable_tag_spans(text: str) -> list:
+    """Spans to delete from ``text``: its deliverable MEDIA tags (located on the masked copy)
+    plus a terminal ``<|eos|>`` sentinel, which is a control token and never user content."""
+    spans = _real_media_tag_spans(_mask_media_scan_text(text))
+    start = _terminal_sentinel_start(text)
+    if spans and start >= 0:
+        spans.append((start, start + len(_TERMINAL_SENTINEL)))
+    return spans
 
 
 def _extensionless_media_matches(masked: str):
@@ -1389,7 +1419,7 @@ def _strip_media_tag_directives(text: str) -> str:
     if not text or not _has_media_directives(text):
         return text
     cleaned = text.replace("[[audio_as_voice]]", "").replace("[[as_document]]", "")
-    return _delete_spans(cleaned, _real_media_tag_spans(_mask_media_scan_text(cleaned)))
+    return _delete_spans(cleaned, _deliverable_tag_spans(cleaned))
 
 
 def cache_document_from_bytes(data: bytes, filename: str) -> str:
@@ -3010,7 +3040,7 @@ class BasePlatformAdapter(ABC):
         # Locate tag spans on a masked copy, delete them from the unmasked text (protected spans
         # survive).
         if media:
-            spans = _real_media_tag_spans(_mask_media_scan_text(cleaned))
+            spans = _deliverable_tag_spans(cleaned)
             if spans:
                 cleaned = re.sub(r'\n{3,}', '\n\n', _delete_spans(cleaned, spans)).strip()
         return media, cleaned

tests/gateway/test_media_terminal_sentinel.py

@@ -0,0 +1,34 @@
+"""A leaked terminal ``<|eos|>`` sentinel must not hide a MEDIA attachment (#111046)."""
+
+import pytest
+
+from gateway.platforms.base import BasePlatformAdapter
+
+
+@pytest.mark.parametrize("filename", ["chart.png", "payload.weirdext", "Caddyfile"])
+def test_terminal_eos_sentinel_leaves_extraction_and_cleanup_unchanged(tmp_path, monkeypatch, filename):
+    """Known-extension, unknown-extension and extension-less tags glued to ``<|eos|>`` extract
+    and clean exactly like the same response without the sentinel."""
+    root = tmp_path / "media-cache"
+    root.mkdir()
+    media_file = root / filename
+    media_file.write_bytes(b"media")
+    monkeypatch.setattr("gateway.platforms.base.MEDIA_DELIVERY_SAFE_ROOTS", (root,))
+    clean = f"Here is the file.\nMEDIA:{media_file}"
+
+    assert BasePlatformAdapter.extract_media(clean + "<|eos|>") == BasePlatformAdapter.extract_media(clean)
+    assert BasePlatformAdapter.strip_media_directives_for_display(clean + "<|eos|>") == "Here is the file."
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "MEDIA:/tmp/example.png<|EOS|>",  # not the exact token: still not a path delimiter
+        "MEDIA:/tmp/example.png<|eos|> trailing prose",  # not terminal
+        "```text\nMEDIA:/tmp/example.png<|eos|>\n```",  # protected code stays byte-identical
+        '{"example":"MEDIA:/tmp/example.png<|eos|>"}',  # protected JSON value
+    ],
+)
+def test_non_terminal_or_protected_sentinel_is_not_a_media_boundary(text):
+    assert BasePlatformAdapter.extract_media(text) == ([], text)
+    assert BasePlatformAdapter.strip_media_directives_for_display(text) == text