fix(whatsapp): resolve original media for quoted-media replies

Baileys' contextInfo.quotedMessage only ever carries a thumbnail-sized stub for media, or nothing at all for an uncaptioned attachment — never a way to fetch the original file. When a user replies to an earlier photo/video/document/voice note with no caption on it (e.g. "did you save this?" quoting an uncaptioned wedding invite image), the agent saw no text and no media reference at all: it looked like the message never had an attachment. Add createQuotedMediaCache, a bounded in-memory cache (keyed by chatId:messageId) of each inbound message's already-downloaded media and text, populated as extractBridgeEvent processes every message. When a later message quotes one of these, extractBridgeEvent resolves quotedMediaUrls/quotedMediaType from the cache and falls back to a human-readable quotedText ("sent an image", etc.) when the quote had no caption to extract. The adapter folds resolved quoted media into the event's own media_urls/media_types — reusing the existing vision/audio pipeline and the existing _is_allowed_bridge_path path validation — rather than adding a parallel reply-media code path. Reimplements the same feature as #52875 (credit: dhruvkej9) against current main, whose 11627fdcb refactor (native polls, locations, rich inbound metadata) moved this code into bridge_helpers.js and made that PR's diff no longer apply cleanly.
2026-08-03 13:37:45 +01:00
parent fa425c942e
commit cd89e4b4a6
5 changed files with 308 additions and 1 deletions
@@ -802,6 +802,34 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter):
            # Quoted message stays in structured fields only — GatewayRunner renders the "[Replying to: ...]" pointer.
            quoted = bool(data.get("hasQuotedMessage"))
            raw_reply_id = data.get("quotedMessageId") if quoted else None
+            if quoted:
+                # contextInfo.quotedMessage only ever carries a thumbnail-sized
+                # stub for media (or nothing for an uncaptioned attachment) —
+                # never a way to fetch the original file. The bridge resolves
+                # the quoted message's already-downloaded media via its own
+                # cache and hands back real local paths in quotedMediaUrls.
+                # Append them to this event's own media so the existing
+                # vision/audio pipeline (which reads media_urls/media_types)
+                # picks up the quoted attachment exactly like a direct one —
+                # no separate reply-media code path needed.
+                quoted_media_urls = data.get("quotedMediaUrls") or []
+                quoted_media_type = str(data.get("quotedMediaType") or "").strip()
+                _quoted_mime_map = {
+                    "image": "image/jpeg",
+                    "video": "video/mp4",
+                    "gif": "video/mp4",
+                    "audio": "audio/ogg",
+                    "ptt": "audio/ogg",
+                    "document": "application/octet-stream",
+                    "sticker": "image/webp",
+                }
+                for _qurl in quoted_media_urls:
+                    if not (isinstance(_qurl, str) and os.path.isabs(_qurl) and _is_allowed_bridge_path(_qurl)):
+                        print(f"[{self.name}] Rejected quoted-media path outside cache dir: {_qurl}", flush=True)
+                        continue
+                    cached_urls.append(_qurl)
+                    media_types.append(_quoted_mime_map.get(quoted_media_type, "application/octet-stream"))
+                    print(f"[{self.name}] Attached quoted-reply media: {_qurl}", flush=True)
            if msg_type == MessageType.DOCUMENT and cached_urls:
                body = self._inject_document_text(cached_urls, body)
            native_metadata = data.get("nativeMetadata")
@@ -40,6 +40,7 @@ import {
  buildLocationPayload,
  buildTextSendPayload,
  createBoundedMessageStore,
+  createQuotedMediaCache,
  extractBridgeEvent,
  getMessageContent,
  inboundReadReceiptKeys,
@@ -260,6 +261,10 @@ const MAX_QUEUE_SIZE = 100;
 const recentlySentIds = createOutboundIdTracker(512);
 const recentlyProcessedPollUpdates = createOutboundIdTracker(512);
 const messageStore = createBoundedMessageStore(512);
+// Bounded cache of already-downloaded inbound media, so a later reply to an
+// uncaptioned photo/video/document/voice note can still surface the original
+// file — see createQuotedMediaCache's doc comment in bridge_helpers.js.
+const quotedMediaCache = createQuotedMediaCache(512);

 function normalizePollUpdateOptions(aggregation, pollUpdateMessage, meId) {
  const selected = [];
@@ -711,6 +716,7 @@ async function startSocket() {
          document: DOCUMENT_CACHE_DIR,
          audio: AUDIO_CACHE_DIR,
        },
+        lookupQuotedMedia: (quotedChatId, quotedMessageId) => quotedMediaCache.get(quotedChatId, quotedMessageId),
      });
      event.fromOwner = fromOwner;

@@ -739,6 +745,15 @@ async function startSocket() {
      }

      messageStore.remember(msg);
+      // Remember this message's already-downloaded media/text so a later
+      // reply to it (even uncaptioned media) can resolve the original
+      // content instead of seeing only a stripped-down quoted-message stub.
+      quotedMediaCache.remember(chatId, msg.key.id, {
+        body: event.body,
+        hasMedia: event.hasMedia,
+        mediaType: event.mediaType,
+        mediaUrls: event.mediaUrls,
+      });
      messageQueue.push(event);
      emitDebugEvent({
        stage: 'queued',
@@ -16,6 +16,7 @@ import {
  buildPollPayload,
  buildTextSendPayload,
  createBoundedMessageStore,
+  createQuotedMediaCache,
  appendMediaFailureNote,
  extractBridgeEvent,
  inboundReadReceiptKeys,
@@ -129,6 +130,89 @@ import {
  console.log('  ✓ inbound quoted metadata includes quoted text');
 }

+// -- reply to uncaptioned quoted media resolves the cached original file --
+{
+  // contextInfo.quotedMessage only ever carries a thumbnail-sized stub for
+  // media (or nothing for an uncaptioned attachment) — extractBridgeEvent
+  // must fall back to lookupQuotedMedia to find the original cached file.
+  const event = await extractBridgeEvent({
+    msg: {
+      key: {
+        id: 'incoming-2',
+        remoteJid: '15551234567@s.whatsapp.net',
+        participant: '15550001111@s.whatsapp.net',
+        fromMe: false,
+      },
+      pushName: 'Tester',
+      messageTimestamp: 123,
+      message: {
+        extendedTextMessage: {
+          text: 'did you save this?',
+          contextInfo: {
+            stanzaId: 'original-image-1',
+            participant: '15550001111@s.whatsapp.net',
+            remoteJid: '15551234567@s.whatsapp.net',
+            // Real WhatsApp traffic: an uncaptioned quoted image carries no
+            // usable text, just a thumbnail-only imageMessage stub.
+            quotedMessage: { imageMessage: {} },
+          },
+        },
+      },
+    },
+    chatId: '15551234567@s.whatsapp.net',
+    senderId: '15550001111@s.whatsapp.net',
+    senderNumber: '15550001111',
+    botIds: [],
+    downloadMedia: async () => Buffer.from(''),
+    lookupQuotedMedia: (chatId, messageId) => {
+      assert.equal(chatId, '15551234567@s.whatsapp.net');
+      assert.equal(messageId, 'original-image-1');
+      return { hasMedia: true, mediaType: 'image', mediaUrls: ['/cache/image/img_original.jpg'] };
+    },
+  });
+
+  assert.deepEqual(event.quotedMediaUrls, ['/cache/image/img_original.jpg']);
+  assert.equal(event.quotedMediaType, 'image');
+  assert.equal(event.quotedText, 'sent an image');
+  console.log('  ✓ reply to uncaptioned quoted image resolves cached original file');
+}
+
+// -- quoted media lookup miss leaves quoted fields empty (no crash) -------
+{
+  const event = await extractBridgeEvent({
+    msg: {
+      key: {
+        id: 'incoming-3',
+        remoteJid: '15551234567@s.whatsapp.net',
+        participant: '15550001111@s.whatsapp.net',
+        fromMe: false,
+      },
+      messageTimestamp: 123,
+      message: {
+        extendedTextMessage: {
+          text: 'thanks',
+          contextInfo: {
+            stanzaId: 'long-gone-message',
+            participant: '15550001111@s.whatsapp.net',
+            remoteJid: '15551234567@s.whatsapp.net',
+            quotedMessage: { imageMessage: {} },
+          },
+        },
+      },
+    },
+    chatId: '15551234567@s.whatsapp.net',
+    senderId: '15550001111@s.whatsapp.net',
+    senderNumber: '15550001111',
+    botIds: [],
+    downloadMedia: async () => Buffer.from(''),
+    lookupQuotedMedia: () => null,
+  });
+
+  assert.deepEqual(event.quotedMediaUrls, []);
+  assert.equal(event.quotedMediaType, '');
+  console.log('  ✓ quoted media cache miss leaves quoted media fields empty');
+}
+
 {
  const event = await extractBridgeEvent({
    msg: {
@@ -507,4 +591,25 @@ import {
  console.log('  ✓ nested envelopes: quote text resolves through both layers');
 }

+// -- createQuotedMediaCache ------------------------------------------------
+{
+  const cache = createQuotedMediaCache(2);
+  cache.remember('chat-a', 'msg-1', { hasMedia: true, mediaType: 'image', mediaUrls: ['/cache/img1.jpg'] });
+
+  assert.deepEqual(cache.get('chat-a', 'msg-1'), { hasMedia: true, mediaType: 'image', mediaUrls: ['/cache/img1.jpg'] });
+  // Different chatId, same messageId — must not collide; message ids are
+  // only unique within their own chat/JID.
+  assert.equal(cache.get('chat-b', 'msg-1'), null);
+  assert.equal(cache.get('chat-a', 'unknown-msg'), null);
+
+  cache.remember('chat-a', 'msg-2', { hasMedia: false });
+  cache.remember('chat-a', 'msg-3', { hasMedia: false });
+  // Capacity is 2: the oldest entry (msg-1) must have been evicted.
+  assert.equal(cache.get('chat-a', 'msg-1'), null);
+  assert.notEqual(cache.get('chat-a', 'msg-2'), null);
+  assert.notEqual(cache.get('chat-a', 'msg-3'), null);
+
+  console.log('  ✓ createQuotedMediaCache resolves by chatId+messageId and evicts oldest past capacity');
+}
+
 console.log('\n✅ All WhatsApp native bridge helper tests passed.');
@@ -81,6 +81,45 @@ export function createBoundedMessageStore(limit = 512) {
  return { remember, get };
 }

+/**
+ * Bounded cache of the already-downloaded media (and plain text) for
+ * recently seen inbound messages, keyed by "chatId:messageId".
+ *
+ * When a user replies to an earlier message, Baileys' contextInfo.quotedMessage
+ * only carries a thumbnail-sized stub for media (or nothing at all for an
+ * uncaptioned attachment) — never a way to re-fetch the full original file.
+ * Without this cache, replying to a photo/video/document/voice note with no
+ * caption gives the agent no text and no media reference: it looks like the
+ * message never had an attachment. Since extractBridgeEvent already downloads
+ * and caches the media for every inbound message as it arrives, remembering
+ * that outcome here lets a later reply resolve the original file path.
+ */
+export function createQuotedMediaCache(limit = 512) {
+  const byKey = new Map();
+
+  function key(chatId, messageId) {
+    return `${chatId || ''}:${messageId || ''}`;
+  }
+
+  function remember(chatId, messageId, payload) {
+    if (!messageId) return;
+    const k = key(chatId, messageId);
+    byKey.delete(k);
+    byKey.set(k, payload);
+    while (byKey.size > limit) {
+      const oldest = byKey.keys().next().value;
+      byKey.delete(oldest);
+    }
+  }
+
+  function get(chatId, messageId) {
+    if (!messageId) return null;
+    return byKey.get(key(chatId, messageId)) || null;
+  }
+
+  return { remember, get };
+}
+
 export function pollCreationMessageSecret(pollCreation) {
  return pollCreation?.message?.messageContextInfo?.messageSecret
    || pollCreation?.messageContextInfo?.messageSecret
@@ -323,6 +362,7 @@ export async function extractBridgeEvent({
  downloadMedia,
  writeMediaFile,
  cacheDirs = {},
+  lookupQuotedMedia,
 }) {
  const messageContent = getMessageContent(msg);
  const contextInfo = getContextInfo(messageContent);
@@ -331,7 +371,38 @@ export async function extractBridgeEvent({
  const quotedParticipant = normalizeWhatsAppId(contextInfo?.participant || '') || null;
  const quotedRemoteJid = normalizeWhatsAppId(contextInfo?.remoteJid || '') || null;
  const hasQuotedMessage = !!contextInfo?.quotedMessage;
-  const quotedText = textFromQuotedMessage(contextInfo?.quotedMessage);
+  let quotedText = textFromQuotedMessage(contextInfo?.quotedMessage);
+  let quotedMediaUrls = [];
+  let quotedMediaType = '';
+
+  // contextInfo.quotedMessage only ever carries a thumbnail-sized stub for
+  // media (or nothing for an uncaptioned attachment) — never a way to
+  // re-fetch the original file. Resolve the quoted message's already-cached
+  // media (downloaded when it first arrived) via lookupQuotedMedia so a
+  // reply to an uncaptioned photo/video/document/voice note still gives the
+  // agent the original file, not just silence.
+  if (quotedMessageId && typeof lookupQuotedMedia === 'function') {
+    const original = lookupQuotedMedia(quotedRemoteJid || chatId, quotedMessageId);
+    if (original) {
+      if (original.hasMedia && original.mediaUrls?.length) {
+        quotedMediaUrls = original.mediaUrls;
+        quotedMediaType = original.mediaType || '';
+        if (!quotedText) {
+          quotedText = {
+            image: 'sent an image',
+            video: 'sent a video',
+            gif: 'sent a GIF',
+            audio: 'sent an audio message',
+            ptt: 'sent a voice message',
+            document: 'sent a document',
+            sticker: 'sent a sticker',
+          }[original.mediaType] || 'sent media';
+        }
+      } else if (original.body && !quotedText) {
+        quotedText = original.body;
+      }
+    }
+  }

  let body = '';
  let hasMedia = false;
@@ -498,6 +569,8 @@ export async function extractBridgeEvent({
    quotedParticipant,
    quotedRemoteJid,
    quotedText,
+    quotedMediaUrls,
+    quotedMediaType,
    hasQuotedMessage,
    botIds,
    readReceiptKey: {
@@ -216,6 +216,92 @@ class TestBridgeEventMetadata:
        assert event.raw_message["quotedRemoteJid"] == "15551234567@s.whatsapp.net"
        assert event.raw_message["hasQuotedMessage"] is True

+    @pytest.mark.asyncio
+    async def test_reply_to_uncaptioned_image_attaches_quoted_media(self, tmp_path, monkeypatch):
+        # contextInfo.quotedMessage only ever carries a thumbnail-sized stub
+        # for media (or nothing for an uncaptioned attachment). The bridge
+        # resolves the quoted message's already-downloaded media via its own
+        # cache (createQuotedMediaCache) and hands back the real cached path
+        # in quotedMediaUrls. The adapter must fold that into this event's
+        # own media_urls/media_types so the existing vision pipeline picks it
+        # up — otherwise a reply like "save this" to an uncaptioned photo
+        # someone else sent looks to the agent like there is no image at all.
+        adapter = _make_adapter()
+
+        cache_dir = tmp_path / "cache" / "image"
+        cache_dir.mkdir(parents=True)
+        quoted_image_path = cache_dir / "img_original.jpg"
+        quoted_image_path.write_bytes(b"fake-jpeg-bytes")
+
+        from plugins.platforms.whatsapp import adapter as adapter_module
+        monkeypatch.setattr(
+            adapter_module, "_is_allowed_bridge_path", lambda url: True,
+        )
+
+        data = {
+            "messageId": "reply-msg",
+            "chatId": "15551234567@s.whatsapp.net",
+            "senderId": "15551234567@s.whatsapp.net",
+            "senderName": "Ananya",
+            "chatName": "Family",
+            "isGroup": True,
+            "body": "did you save this wedding invite?",
+            "hasMedia": False,
+            "mediaUrls": [],
+            "mediaType": "",
+            "quotedMessageId": "original-image-msg",
+            "quotedParticipant": "99999999999@s.whatsapp.net",
+            "quotedRemoteJid": "15551234567@s.whatsapp.net",
+            "hasQuotedMessage": True,
+            "quotedText": "",
+            "quotedMediaUrls": [str(quoted_image_path)],
+            "quotedMediaType": "image",
+        }
+
+        event = await adapter._build_message_event(data)
+
+        assert event is not None
+        assert str(quoted_image_path) in event.media_urls
+        idx = event.media_urls.index(str(quoted_image_path))
+        assert event.media_types[idx] == "image/jpeg"
+
+    @pytest.mark.asyncio
+    async def test_quoted_media_path_outside_cache_dir_is_rejected(self, monkeypatch):
+        # _is_allowed_bridge_path guards against a compromised/buggy bridge
+        # handing back an arbitrary absolute path; quoted-media handling must
+        # respect the same guard as direct media, not bypass it.
+        adapter = _make_adapter()
+
+        from plugins.platforms.whatsapp import adapter as adapter_module
+        monkeypatch.setattr(
+            adapter_module, "_is_allowed_bridge_path", lambda url: False,
+        )
+
+        data = {
+            "messageId": "reply-msg-2",
+            "chatId": "15551234567@s.whatsapp.net",
+            "senderId": "15551234567@s.whatsapp.net",
+            "senderName": "Ananya",
+            "chatName": "Family",
+            "isGroup": True,
+            "body": "did you save this?",
+            "hasMedia": False,
+            "mediaUrls": [],
+            "mediaType": "",
+            "quotedMessageId": "original-image-msg",
+            "quotedParticipant": "99999999999@s.whatsapp.net",
+            "quotedRemoteJid": "15551234567@s.whatsapp.net",
+            "hasQuotedMessage": True,
+            "quotedText": "",
+            "quotedMediaUrls": ["/etc/passwd"],
+            "quotedMediaType": "image",
+        }
+
+        event = await adapter._build_message_event(data)
+
+        assert event is not None
+        assert "/etc/passwd" not in event.media_urls
+

 # ---------------------------------------------------------------------------
 # display_config tier classification