ouyangbo/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Merge branch 'simp/r3-13-G' into simp/r3-13

Browse Source
This commit is contained in:
Teknium
2026-09-02 23:26:22 -07:00
parent 58e54136a9 4009eeb59f
commit 2673b647fb
9 changed files with 588 additions and 941 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/platforms/bluebubbles.py
+96 -152
View File
@@ -1,8 +1,5 @@
"""BlueBubbles iMessage platform adapter.
Uses the local BlueBubbles macOS server for outbound REST sends and inbound
webhooks: text, media attachments, typing indicators, and read receipts.
"""
"""BlueBubbles iMessage platform adapter: local BlueBubbles macOS server for outbound REST sends and
inbound webhooks (text, media attachments, typing indicators, read receipts)."""
import asyncio
import json
@@ -28,9 +25,8 @@ from .media_cache import ext_for_mime
from gateway.platforms.helpers import compile_mention_patterns, strip_markdown
from utils import TRUTHY_STRINGS
# Historical BlueBubbles mime→ext maps, preserved verbatim as overrides for
# the shared dispatch in gateway.platforms.media_cache. Both maps are
# CLOSED: unlisted mimes fall back to .jpg / .mp3 (never mimetypes).
# Historical BlueBubbles mime→ext maps, preserved verbatim as overrides for the shared dispatch in
# gateway.platforms.media_cache. Both maps are CLOSED: unlisted mimes fall back to .jpg / .mp3.
_BLUEBUBBLES_IMAGE_EXT_OVERRIDES = {
"image/jpeg": ".jpg", "image/png": ".png", "image/gif": ".gif", "image/webp": ".webp",
"image/heic": ".jpg", "image/heif": ".jpg", "image/tiff": ".jpg", # historical mapping
@@ -44,23 +40,20 @@ _BLUEBUBBLES_AUDIO_EXT_OVERRIDES = {
logger = logging.getLogger(__name__)
DEFAULT_WEBHOOK_HOST = "127.0.0.1"
# Webhook events are small JSON/form payloads; attachments come through the
# REST API. 1 MiB keeps oversized/chunked bodies from buffering unbounded.
# Webhook events are small JSON/form payloads (attachments come through the REST API); 1 MiB keeps
# oversized/chunked bodies from buffering unbounded.
_WEBHOOK_MAX_BODY_BYTES = 1_048_576
DEFAULT_WEBHOOK_PORT = 8645
DEFAULT_WEBHOOK_PATH = "/bluebubbles-webhook"
MAX_TEXT_LENGTH = 4000
# iMessage has no stable bot mention identity (unlike <@U...>/@botname/MXID),
# so `require_mention: true` without custom aliases uses Hermes wake words.
# iMessage has no stable bot mention identity (unlike <@U...>/@botname/MXID), so
# `require_mention: true` without custom aliases uses Hermes wake words.
DEFAULT_MENTION_PATTERNS = [r"(?<![\w@])@?hermes\s+agent\b[,:\-]?", r"(?<![\w@])@?hermes\b[,:\-]?"]
# Tapback associatedMessageType codes: 2000-2005 added, 3000-3005 removed
# (love, like, dislike, laugh, emphasize, question).
# Tapback associatedMessageType codes: 2000-2005 added, 3000-3005 removed (love, like, dislike, ...).
_TAPBACK_CODES = {*range(2000, 2006), *range(3000, 3006)}
# Webhook event types that carry user messages
_MESSAGE_EVENTS = {"new-message", "message", "updated-message"}
_MESSAGE_EVENTS = {"new-message", "message", "updated-message"} # webhook event types carrying user messages
_PHONE_RE = re.compile(r"\+?\d{7,15}")
_EMAIL_RE = re.compile(r"[\w.+-]+@[\w-]+\.[\w.]+")
@@ -86,18 +79,15 @@ def check_bluebubbles_requirements() -> bool:
def _normalize_server_url(raw: str) -> str:
value = (raw or "").strip()
if not value:
return ""
if not re.match(r"^https?://", value, flags=re.I):
if value and not re.match(r"^https?://", value, flags=re.I):
value = f"http://{value}"
return value.rstrip("/")
def _closed_ext(mime: str, overrides: Dict[str, str], fallback: str) -> str:
"""Historical maps were closed: unlisted mimes fall back without consulting mimetypes."""
return ext_for_mime(
mime, overrides=overrides, use_defaults=False, use_mimetypes=False, fallback=fallback
) or fallback
return ext_for_mime(mime, overrides=overrides, use_defaults=False, use_mimetypes=False,
fallback=fallback) or fallback
def _setting(extra: Dict[str, Any], key: str, env: str, default: str = "") -> Any:
@@ -128,17 +118,15 @@ class BlueBubblesAdapter(BasePlatformAdapter):
self.password = extra.get("password") or _get_scoped_secret("BLUEBUBBLES_PASSWORD", "")
self.webhook_host = _setting(extra, "webhook_host", "BLUEBUBBLES_WEBHOOK_HOST", DEFAULT_WEBHOOK_HOST)
self.webhook_port = int(_setting(extra, "webhook_port", "BLUEBUBBLES_WEBHOOK_PORT", str(DEFAULT_WEBHOOK_PORT)))
self.webhook_path = _setting(extra, "webhook_path", "BLUEBUBBLES_WEBHOOK_PATH", DEFAULT_WEBHOOK_PATH)
if not str(self.webhook_path).startswith("/"):
self.webhook_path = f"/{self.webhook_path}"
path = str(_setting(extra, "webhook_path", "BLUEBUBBLES_WEBHOOK_PATH", DEFAULT_WEBHOOK_PATH))
self.webhook_path = path if path.startswith("/") else f"/{path}"
self.send_read_receipts = bool(extra.get("send_read_receipts", True))
_require_mention = extra.get("require_mention")
if _require_mention is None:
_require_mention = os.getenv("BLUEBUBBLES_REQUIRE_MENTION")
self.require_mention = str(_require_mention).strip().lower() in TRUTHY_STRINGS
self._mention_patterns = self._compile_mention_patterns(
extra["mention_patterns"] if "mention_patterns" in extra else os.getenv("BLUEBUBBLES_MENTION_PATTERNS")
)
extra["mention_patterns"] if "mention_patterns" in extra else os.getenv("BLUEBUBBLES_MENTION_PATTERNS"))
self.client: Optional[httpx.AsyncClient] = None
self._runner = None
self._private_api_enabled: Optional[bool] = None
@@ -148,30 +136,29 @@ class BlueBubblesAdapter(BasePlatformAdapter):
# --- API helpers ---
def _api_url(self, path: str) -> str:
sep = "&" if "?" in path else "?"
return f"{self.server_url}{path}{sep}password={quote(self.password, safe='')}"
return f"{self.server_url}{path}{'&' if '?' in path else '?'}password={quote(self.password, safe='')}"
@staticmethod
def _compile_mention_patterns(raw: Any) -> List[re.Pattern]:
"""Compile group-mention wake words; ``raw`` is a list, a raw env string
(JSON list or comma/newline-separated), or None (Hermes defaults)."""
return compile_mention_patterns(raw, log_prefix="bluebubbles", defaults=DEFAULT_MENTION_PATTERNS, logger_=logger)
"""Compile group-mention wake words; ``raw`` is a list, a raw env string (JSON list or
comma/newline-separated), or None (Hermes defaults)."""
return compile_mention_patterns(raw, log_prefix="bluebubbles", defaults=DEFAULT_MENTION_PATTERNS,
logger_=logger)
def _message_matches_mention_patterns(self, text: str) -> bool:
return bool(text) and any(pattern.search(text) for pattern in self._mention_patterns)
def _clean_mention_text(self, text: str) -> str:
"""Strip a leading wake word only — patterns are regexes, so stripping
anywhere later in the prompt could delete ordinary words."""
if not text:
return text
"""Strip a leading wake word only — patterns are regexes, so stripping anywhere later in the
prompt could delete ordinary words."""
stripped = (text or "").lstrip()
for pattern in self._mention_patterns:
match = pattern.match(text.lstrip())
if match:
return text.lstrip()[match.end():].lstrip(" ,:-") or text
if match := pattern.match(stripped):
return stripped[match.end():].lstrip(" ,:-") or text
return text
async def _api_json(self, method: str, path: str, **kwargs) -> Dict[str, Any]:
"""Authenticated request to the BlueBubbles REST API; raises on HTTP errors, returns decoded JSON."""
assert self.client is not None
res = await getattr(self.client, method)(self._api_url(path), **kwargs)
res.raise_for_status()
@@ -198,8 +185,7 @@ class BlueBubblesAdapter(BasePlatformAdapter):
if not self._private_api_enabled or not self._helper_connected or not self.client:
return False
with suppress(Exception):
guid = await self._resolve_chat_guid(chat_id)
if guid:
if guid := await self._resolve_chat_guid(chat_id):
url = self._api_url(f"/api/v1/chat/{quote(guid, safe='')}/{action}")
await getattr(self.client, method)(url, timeout=5)
return True
@@ -227,22 +213,21 @@ class BlueBubblesAdapter(BasePlatformAdapter):
logger.error("[bluebubbles] cannot reach server at %s: %s", self.server_url, exc)
await self._close_client()
return False
# client_max_size makes aiohttp enforce the cap on every read path,
# including chunked requests with no Content-Length.
# client_max_size makes aiohttp enforce the cap on every read path, incl. chunked requests
# with no Content-Length.
app = web.Application(client_max_size=_WEBHOOK_MAX_BODY_BYTES)
app.router.add_get("/health", lambda _: web.Response(text="ok"))
app.router.add_post(self.webhook_path, self._handle_webhook)
# The webhook auth value rides in the query string (BlueBubbles cannot
# send custom headers) — keep it out of aiohttp access logs.
# The webhook auth value rides in the query string (BlueBubbles cannot send custom headers)
# — keep it out of aiohttp access logs.
self._runner = web.AppRunner(app, access_log=None)
await self._runner.setup()
site = web.TCPSite(self._runner, self.webhook_host, self.webhook_port)
await site.start()
self._mark_connected()
logger.info("[bluebubbles] webhook listening on http://%s:%s%s",
self.webhook_host, self.webhook_port, self.webhook_path)
# The server only sends events to webhooks registered via its API.
await self._register_webhook()
logger.info("[bluebubbles] webhook listening on http://%s:%s%s", self.webhook_host, self.webhook_port,
self.webhook_path)
await self._register_webhook() # the server only sends events to webhooks registered via its API
# Plugin-registered native handlers (ctx.register_platform_handler).
self._wire_plugin_handlers(None)
return True
@@ -267,16 +252,13 @@ class BlueBubblesAdapter(BasePlatformAdapter):
return f"http://{host}:{self.webhook_port}{self.webhook_path}"
def _webhook_register_url_with(self, password_param: str) -> str:
base = self._webhook_url
return f"{base}?password={password_param}" if self.password else base
return f"{self._webhook_url}?password={password_param}" if self.password else self._webhook_url
@property
def _webhook_register_url(self) -> str:
"""Registered webhook URL, password embedded as a query param.
BlueBubbles posts to the exact registered URL and its registration
API cannot set custom headers, so this is the only way to
authenticate inbound webhooks without disabling auth."""
"""Registered webhook URL with the password as a query param: BlueBubbles posts to the exact
registered URL and cannot set custom headers, so this is the only way to authenticate inbound
webhooks without disabling auth."""
return self._webhook_register_url_with(quote(self.password, safe=""))
@property
@@ -292,18 +274,17 @@ class BlueBubblesAdapter(BasePlatformAdapter):
return []
async def _register_webhook(self) -> bool:
"""Register this webhook URL, reusing an existing registration if present
(crash resilience — avoids duplicates after an unclean shutdown)."""
"""Register this webhook URL, reusing an existing registration if present (crash resilience —
avoids duplicates after an unclean shutdown)."""
if not self.client:
return False
webhook_url = self._webhook_register_url
log_url = self._webhook_register_url_for_log
webhook_url, log_url = self._webhook_register_url, self._webhook_register_url_for_log
if await self._find_registered_webhooks(webhook_url):
logger.info("[bluebubbles] webhook already registered: %s", log_url)
return True
payload = {"url": webhook_url, "events": ["new-message", "updated-message"]}
try:
res = await self._api_post("/api/v1/webhook", payload)
res = await self._api_post("/api/v1/webhook",
{"url": webhook_url, "events": ["new-message", "updated-message"]})
status = res.get("status", 0)
if 200 <= status < 300:
logger.info("[bluebubbles] webhook registered with server: %s", log_url)
@@ -321,10 +302,8 @@ class BlueBubblesAdapter(BasePlatformAdapter):
removed = False
try:
for wh in await self._find_registered_webhooks(self._webhook_register_url):
wh_id = wh.get("id")
if wh_id:
res = await self.client.delete(self._api_url(f"/api/v1/webhook/{wh_id}"))
res.raise_for_status()
if wh_id := wh.get("id"):
(await self.client.delete(self._api_url(f"/api/v1/webhook/{wh_id}"))).raise_for_status()
removed = True
if removed:
logger.info("[bluebubbles] webhook unregistered: %s", self._webhook_register_url_for_log)
@@ -335,18 +314,13 @@ class BlueBubblesAdapter(BasePlatformAdapter):
# --- Chat GUID resolution ---
async def _resolve_chat_guid(self, target: str) -> Optional[str]:
"""Resolve an email/phone to a chat GUID (raw ``a;-;b`` GUIDs pass through).
Matches strictly on ``chatIdentifier`` / ``identifier``. Participant
membership is intentionally NOT a fallback: the same contact appears in
a 1:1 DM and any number of groups, so a participant match could leak a
DM reply into a group thread. Return ``None`` and let the caller create
a fresh DM via ``_create_chat_for_handle``."""
"""Resolve an email/phone to a chat GUID (raw ``a;-;b`` GUIDs pass through). Matches strictly on
``chatIdentifier`` / ``identifier``; participant membership is intentionally NOT a fallback —
the same contact appears in a 1:1 DM and any number of groups, so a participant match could
leak a DM reply into a group thread. ``None`` lets the caller create a fresh DM."""
target = (target or "").strip()
if not target:
return None
if ";" in target:
return target
if not target or ";" in target:
return target or None
if target in self._guid_cache:
self._guid_cache.move_to_end(target)
return self._guid_cache[target]
@@ -355,8 +329,7 @@ class BlueBubblesAdapter(BasePlatformAdapter):
for chat in payload.get("data", []) or []:
if (chat.get("chatIdentifier") or chat.get("identifier")) != target:
continue
guid = chat.get("guid") or chat.get("chatGuid")
if guid:
if guid := chat.get("guid") or chat.get("chatGuid"):
self._guid_cache[target] = guid
while len(self._guid_cache) > _GUID_CACHE_SIZE:
self._guid_cache.popitem(last=False)
@@ -365,50 +338,43 @@ class BlueBubblesAdapter(BasePlatformAdapter):
async def _create_chat_for_handle(self, address: str, message: str) -> SendResult:
"""Create a new chat by sending the first message to *address*."""
payload = {"addresses": [address], "message": message, "tempGuid": _temp_guid()}
return await self._post_message("/api/v1/chat/new", payload)
return await self._post_message(
"/api/v1/chat/new", {"addresses": [address], "message": message, "tempGuid": _temp_guid()})
# --- Text sending ---
@staticmethod
def truncate_message(content: str, max_length: int = MAX_TEXT_LENGTH) -> List[str]:
# Base splitter minus "(1/3)" pagination suffixes — iMessage bubbles flow naturally.
chunks = BasePlatformAdapter.truncate_message(content, max_length)
return [_PAGINATION_SUFFIX_RE.sub("", c) for c in chunks]
return [_PAGINATION_SUFFIX_RE.sub("", c) for c in BasePlatformAdapter.truncate_message(content, max_length)]
async def send(
self, chat_id: str, content: str, reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None) -> SendResult:
async def send(self, chat_id: str, content: str, reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None) -> SendResult:
text = self.format_message(content)
if not text:
return SendResult(success=False, error="BlueBubbles send requires text")
# Each paragraph becomes its own iMessage bubble; truncate any still too long.
paragraphs = [p.strip() for p in re.split(r'\n\s*\n', text) if p.strip()]
chunks: List[str] = []
for para in (paragraphs or [text]):
chunks.extend([para] if len(para) <= self.MAX_MESSAGE_LENGTH
else self.truncate_message(para, max_length=self.MAX_MESSAGE_LENGTH))
paragraphs = [p.strip() for p in re.split(r'\n\s*\n', text) if p.strip()] or [text]
chunks = [c for para in paragraphs for c in (
[para] if len(para) <= self.MAX_MESSAGE_LENGTH else self.truncate_message(para, self.MAX_MESSAGE_LENGTH))]
last = SendResult(success=True)
for chunk in chunks:
guid = await self._resolve_chat_guid(chat_id)
if not guid:
# If the target looks like an address, try creating a new chat
if self._private_api_enabled and ("@" in chat_id or _ADDRESS_RE.match(chat_id)):
if self._private_api_enabled and ("@" in chat_id or _ADDRESS_RE.match(chat_id)): # address → new chat
return await self._create_chat_for_handle(chat_id, chunk)
return SendResult(success=False, error=f"BlueBubbles chat not found for target: {chat_id}")
payload: Dict[str, Any] = {"chatGuid": guid, "tempGuid": _temp_guid(), "message": chunk}
if reply_to and self._private_api_enabled and self._helper_connected:
payload.update(method="private-api", selectedMessageGuid=reply_to, partIndex=0)
last = await self._post_message("/api/v1/message/text", payload)
if not last.success:
if not (last := await self._post_message("/api/v1/message/text", payload)).success:
return last
return last
# --- Media sending (outbound) ---
async def _send_attachment(
self, chat_id: str, file_path: str, filename: Optional[str] = None,
caption: Optional[str] = None, is_audio_message: bool = False) -> SendResult:
async def _send_attachment(self, chat_id: str, file_path: str, filename: Optional[str] = None,
caption: Optional[str] = None, is_audio_message: bool = False) -> SendResult:
"""Send a file attachment via BlueBubbles multipart upload."""
if not self.client:
return SendResult(success=False, error="Not connected")
@@ -419,34 +385,31 @@ class BlueBubblesAdapter(BasePlatformAdapter):
return SendResult(success=False, error=f"Chat not found: {chat_id}")
fname = filename or os.path.basename(file_path)
try:
# httpx's async multipart iterator reads file objects through a sync
# chunk generator — read the bytes off the event-loop thread first.
# httpx's async multipart iterator reads file objects through a sync chunk generator —
# read the bytes off the event-loop thread first.
payload = await asyncio.to_thread(Path(file_path).read_bytes)
files = {"attachment": (fname, payload, "application/octet-stream")}
data: Dict[str, str] = {"chatGuid": guid, "name": fname, "tempGuid": uuid.uuid4().hex}
if is_audio_message:
data["isAudioMessage"] = "true"
res = await self.client.post(
self._api_url("/api/v1/message/attachment"), files=files, data=data, timeout=120)
res = await self.client.post(self._api_url("/api/v1/message/attachment"), data=data, timeout=120,
files={"attachment": (fname, payload, "application/octet-stream")})
res.raise_for_status()
result = res.json()
if caption:
await self.send(chat_id, caption)
if result.get("status") == 200:
rdata = result.get("data") or {}
msg_id = rdata.get("guid") if isinstance(rdata, dict) else None
return SendResult(success=True, message_id=msg_id, raw_response=result)
return SendResult(success=True, message_id=rdata.get("guid") if isinstance(rdata, dict) else None,
raw_response=result)
return SendResult(success=False, error=result.get("message", "Attachment upload failed"))
except Exception as e:
return SendResult(success=False, error=str(e))
async def send_image(
self, chat_id: str, image_url: str, caption: Optional[str] = None,
reply_to: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None) -> SendResult:
async def send_image(self, chat_id: str, image_url: str, caption: Optional[str] = None,
reply_to: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None) -> SendResult:
try:
from gateway.platforms.base import cache_image_from_url
local_path = await cache_image_from_url(image_url)
return await self._send_attachment(chat_id, local_path, caption=caption)
return await self._send_attachment(chat_id, await cache_image_from_url(image_url), caption=caption)
except Exception:
return await super().send_image(chat_id, image_url, caption, reply_to)
@@ -482,14 +445,12 @@ class BlueBubblesAdapter(BasePlatformAdapter):
is_group = ";+;" in (chat_id or "")
info: Dict[str, Any] = {"name": chat_id, "type": "group" if is_group else "dm"}
with suppress(Exception):
guid = await self._resolve_chat_guid(chat_id)
if guid:
if guid := await self._resolve_chat_guid(chat_id):
res = await self._api_get(f"/api/v1/chat/{quote(guid, safe='')}?with=participants")
data = (res or {}).get("data", {})
info["name"] = data.get("displayName") or data.get("chatIdentifier") or chat_id
participants = [
addr for p in data.get("participants", []) or [] if (addr := (p.get("address") or "").strip())
]
participants = [addr for p in data.get("participants", []) or []
if (addr := (p.get("address") or "").strip())]
if participants:
info["participants"] = participants
return info
@@ -504,9 +465,8 @@ class BlueBubblesAdapter(BasePlatformAdapter):
if not self.client:
return None
try:
resp = await self.client.get(
self._api_url(f"/api/v1/attachment/{quote(att_guid, safe='')}/download"),
timeout=60, follow_redirects=True)
resp = await self.client.get(self._api_url(f"/api/v1/attachment/{quote(att_guid, safe='')}/download"),
timeout=60, follow_redirects=True)
resp.raise_for_status()
data = resp.content
mime = (att_meta.get("mimeType") or "").lower()
@@ -515,8 +475,7 @@ class BlueBubblesAdapter(BasePlatformAdapter):
if mime.startswith("audio/"):
return cache_audio_from_bytes(data, _closed_ext(mime, _BLUEBUBBLES_AUDIO_EXT_OVERRIDES, ".mp3"))
# Videos, documents, and everything else
filename = att_meta.get("transferName", "") or f"file_{uuid.uuid4().hex[:8]}"
return cache_document_from_bytes(data, filename)
return cache_document_from_bytes(data, att_meta.get("transferName", "") or f"file_{uuid.uuid4().hex[:8]}")
except Exception as exc:
logger.warning("[bluebubbles] failed to download attachment %s: %s", _redact(att_guid), exc)
return None
@@ -527,10 +486,8 @@ class BlueBubblesAdapter(BasePlatformAdapter):
data = payload.get("data")
if isinstance(data, dict):
return data
if isinstance(data, list):
for item in data:
if isinstance(item, dict):
return item
if isinstance(data, list) and (first := next((i for i in data if isinstance(i, dict)), None)):
return first
if isinstance(payload.get("message"), dict):
return payload.get("message")
return payload if isinstance(payload, dict) else None
@@ -557,9 +514,7 @@ class BlueBubblesAdapter(BasePlatformAdapter):
msg_type = MessageType.TEXT
for att in record.get("attachments") or []:
att_guid = att.get("guid", "")
if not att_guid:
continue
cached = await self._download_attachment(att_guid, att)
cached = await self._download_attachment(att_guid, att) if att_guid else None
if not cached:
continue
mime = (att.get("mimeType") or "").lower()
@@ -568,8 +523,7 @@ class BlueBubblesAdapter(BasePlatformAdapter):
is_voice = mime.startswith("audio/") or (att.get("uti") or "").endswith("caf")
msg_type = (MessageType.PHOTO if mime.startswith("image/") else MessageType.VOICE if is_voice
else MessageType.VIDEO if mime.startswith("video/") else MessageType.DOCUMENT)
# With multiple attachments, prefer PHOTO if any images present
if len(media_urls) > 1 and any(m.split("/")[0] == "image" for m in media_types):
if len(media_urls) > 1 and any(m.split("/")[0] == "image" for m in media_types): # any image → PHOTO
msg_type = MessageType.PHOTO
return media_urls, media_types, msg_type
@@ -579,23 +533,17 @@ class BlueBubblesAdapter(BasePlatformAdapter):
def _resolve_chat_and_sender(self, payload: Dict[str, Any], record: Dict[str, Any]):
"""Returns ``(chat_guid, chat_identifier, sender)`` from the many BlueBubbles payload shapes."""
chat_guid = self._value(
record.get("chatGuid"), payload.get("chatGuid"),
record.get("chat_guid"), payload.get("chat_guid"), payload.get("guid"))
chat_guid = self._value(record.get("chatGuid"), payload.get("chatGuid"), record.get("chat_guid"),
payload.get("chat_guid"), payload.get("guid"))
# BlueBubbles v1.9+ payloads omit top-level chatGuid; it's nested under data.chats[0].guid.
_chats = record.get("chats") or []
if not chat_guid and _chats and isinstance(_chats[0], dict):
chat_guid = _chats[0].get("guid") or _chats[0].get("chatGuid")
chat_identifier = self._value(
record.get("chatIdentifier"), record.get("identifier"),
payload.get("chatIdentifier"), payload.get("identifier"))
chat_identifier = self._value(record.get("chatIdentifier"), record.get("identifier"),
payload.get("chatIdentifier"), payload.get("identifier"))
handle = record.get("handle")
sender = (
self._value(
handle.get("address") if isinstance(handle, dict) else None,
record.get("sender"), record.get("from"), record.get("address"))
or chat_identifier
or chat_guid)
sender = (self._value(handle.get("address") if isinstance(handle, dict) else None, record.get("sender"),
record.get("from"), record.get("address")) or chat_identifier or chat_guid)
if not (chat_guid or chat_identifier) and sender:
chat_identifier = sender
return chat_guid, chat_identifier, sender
@@ -611,15 +559,13 @@ class BlueBubblesAdapter(BasePlatformAdapter):
logger.error("[bluebubbles] webhook parse error: %s", exc)
return web.json_response({"error": "invalid payload"}, status=400)
event_type = self._value(payload.get("type"), payload.get("event")) or ""
# Only process message events; silently acknowledge everything else
if event_type and event_type not in _MESSAGE_EVENTS:
if event_type and event_type not in _MESSAGE_EVENTS: # ack non-message events silently
return _ok()
record = self._extract_payload_record(payload) or {}
if record.get("isFromMe") or record.get("fromMe") or record.get("is_from_me"):
return _ok()
# Skip tapback reactions delivered as messages
assoc_type = record.get("associatedMessageType")
if isinstance(assoc_type, int) and assoc_type in _TAPBACK_CODES:
if isinstance(assoc_type, int) and assoc_type in _TAPBACK_CODES: # tapback reactions delivered as messages
return _ok()
text = self._value(record.get("text"), record.get("message"), record.get("body")) or ""
media_urls, media_types, msg_type = await self._collect_attachments(record)
@@ -635,10 +581,9 @@ class BlueBubblesAdapter(BasePlatformAdapter):
logger.debug("[bluebubbles] ignoring group message (require_mention=true, no mention pattern matched)")
return _ok()
text = self._clean_mention_text(text)
source = self.build_source(
chat_id=session_chat_id, chat_name=chat_identifier or sender,
chat_type="group" if is_group else "dm", user_id=sender, user_name=sender,
chat_id_alt=chat_identifier)
source = self.build_source(chat_id=session_chat_id, chat_name=chat_identifier or sender,
chat_type="group" if is_group else "dm", user_id=sender, user_name=sender,
chat_id_alt=chat_identifier)
event = MessageEvent(
text=text, message_type=msg_type, source=source, raw_message=payload,
message_id=self._value(record.get("guid"), record.get("messageGuid"), record.get("id")),
@@ -647,7 +592,6 @@ class BlueBubblesAdapter(BasePlatformAdapter):
task = asyncio.create_task(self.handle_message(event))
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
# Fire-and-forget read receipt
if self.send_read_receipts and session_chat_id:
if self.send_read_receipts and session_chat_id: # fire-and-forget read receipt
asyncio.create_task(self.mark_read(session_chat_id))
return _ok()
gateway/platforms/msgraph_webhook.py
+33 -55
View File
@@ -26,10 +26,9 @@ from gateway.platforms.base import (
logger = logging.getLogger(__name__)
# ``None`` → aiohttp binds one socket per address family (IPv4 + IPv6); the old
# "0.0.0.0" default was unreachable over IPv6-only private networks. Pin a host
# via extra.host. The all-interfaces default still requires
# extra.allowed_source_cidrs (see _source_allowlist_required_but_missing).
# ``None`` → aiohttp binds one socket per address family (IPv4 + IPv6); the old "0.0.0.0" default was
# unreachable over IPv6-only private networks. Pin a host via extra.host. The all-interfaces default
# still requires extra.allowed_source_cidrs (see _source_allowlist_required_but_missing).
DEFAULT_HOST = None
DEFAULT_PORT = 8646
DEFAULT_WEBHOOK_PATH = "/msgraph/webhook"
@@ -54,11 +53,9 @@ def _normalize_path(path: Any) -> str:
def _parse_allowed_source_cidrs(raw: Any) -> list[ipaddress._BaseNetwork]:
"""Parse the optional CIDR allowlist; empty/missing means "allow everything".
When populated, requests from source IPs outside every listed CIDR are
rejected with 403 before the body is parsed (restrict to Microsoft
Graph's published webhook source ranges in production)."""
"""Parse the optional CIDR allowlist; empty/missing means "allow everything". When populated, source
IPs outside every listed CIDR get 403 before the body is parsed (restrict to Microsoft Graph's
published webhook source ranges in production)."""
if isinstance(raw, str):
candidates = raw.split(",")
elif isinstance(raw, (list, tuple, set)):
@@ -66,8 +63,7 @@ def _parse_allowed_source_cidrs(raw: Any) -> list[ipaddress._BaseNetwork]:
else:
return []
networks: list[ipaddress._BaseNetwork] = []
for chunk in candidates:
chunk = chunk.strip()
for chunk in (c.strip() for c in candidates):
if not chunk:
continue
try:
@@ -111,8 +107,7 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
self._webhook_path: str = _normalize_path(extra.get("webhook_path", DEFAULT_WEBHOOK_PATH))
self._health_path: str = _normalize_path(extra.get("health_path", "/health"))
self._accepted_resources: list[str] = [
str(value).strip() for value in (extra.get("accepted_resources") or []) if str(value).strip()
]
str(value).strip() for value in (extra.get("accepted_resources") or []) if str(value).strip()]
self._client_state: Optional[str] = _string_or_none(extra.get("client_state"))
self._max_seen_receipts = max(1, int(extra.get("max_seen_receipts", DEFAULT_MAX_SEEN_RECEIPTS)))
self._max_body_bytes = max(1, int(extra.get("max_body_bytes", DEFAULT_MAX_BODY_BYTES)))
@@ -121,8 +116,7 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
self._notification_scheduler: Optional[NotificationScheduler] = None
self._seen_receipts: set[str] = set()
self._seen_receipt_order: deque[str] = deque()
self._accepted_count = 0
self._duplicate_count = 0
self._accepted_count = self._duplicate_count = 0
def set_notification_scheduler(self, scheduler: Optional[NotificationScheduler]) -> None:
self._notification_scheduler = scheduler
@@ -137,12 +131,9 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
logger.error("[msgraph_webhook] Refusing to start without extra.client_state configured")
return False
if self._source_allowlist_required_but_missing():
logger.error(
"[msgraph_webhook] Refusing to start: binding to %s requires "
"extra.allowed_source_cidrs. Configure the Microsoft Graph "
"source CIDRs or bind to loopback (127.0.0.1/::1) behind a "
"tunnel or reverse proxy.",
self._host)
logger.error("[msgraph_webhook] Refusing to start: binding to %s requires extra.allowed_source_cidrs. "
"Configure the Microsoft Graph source CIDRs or bind to loopback (127.0.0.1/::1) behind a "
"tunnel or reverse proxy.", self._host)
return False
app = web.Application(client_max_size=self._max_body_bytes)
app.router.add_get(self._health_path, self._handle_health)
@@ -164,9 +155,8 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
self._runner = None
self._mark_disconnected()
async def send(
self, chat_id: str, content: str, reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None) -> SendResult:
async def send(self, chat_id: str, content: str, reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None) -> SendResult:
logger.info("[msgraph_webhook] Response for %s: %s", chat_id, content[:200])
return SendResult(success=True)
@@ -181,12 +171,11 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
"accepted": self._accepted_count, "duplicates": self._duplicate_count})
async def _handle_validation(self, request: "web.Request") -> "web.Response":
"""Graph subscription validation handshake: echo ``validationToken`` verbatim
as text/plain. Bare GETs are rejected so the endpoint can't be enumerated."""
"""Graph subscription validation handshake: echo ``validationToken`` verbatim as text/plain. Bare GETs
are rejected so the endpoint can't be enumerated."""
if not self._source_ip_allowed(request):
return web.Response(status=403)
validation_token = request.query.get("validationToken", "")
if not validation_token:
if not (validation_token := request.query.get("validationToken", "")):
return web.Response(status=400)
return web.Response(text=validation_token, content_type="text/plain")
@@ -198,12 +187,10 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
if not self._resource_accepted(str(notification.get("resource") or "")):
return "other"
if not self._verify_client_state(notification):
# Bad clientState is an auth failure: a fully forged batch gets 403
# so the sender stops retrying; legitimate Graph retries carry a
# valid clientState and hit the accepted/duplicate paths.
# Bad clientState is an auth failure: a fully forged batch gets 403 so the sender stops
# retrying; legitimate Graph retries carry a valid clientState → accepted/duplicate paths.
return "auth"
explicit_id = str(notification.get("id") or "").strip()
receipt_key = f"id:{explicit_id}" if explicit_id else None
receipt_key = f"id:{explicit_id}" if (explicit_id := str(notification.get("id") or "").strip()) else None
if receipt_key is not None:
if receipt_key in self._seen_receipts:
return "duplicate"
@@ -216,8 +203,7 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
if not self._source_ip_allowed(request):
return web.Response(status=403)
# Graph never sends validationToken on POST, but tolerate clients replaying it in-band.
validation_token = request.query.get("validationToken", "")
if validation_token:
if validation_token := request.query.get("validationToken", ""):
return web.Response(text=validation_token, content_type="text/plain")
status, notifications = await self._read_notifications(request)
if status:
@@ -226,9 +212,8 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
for raw_notification in notifications:
counts[self._ingest_notification(raw_notification)] += 1
self._duplicate_count += counts["duplicate"]
# Anything ingested OR deduped → 202 with empty body (Graph acks; no
# counter leak). Every item failed auth → 403 so forged POSTs get a
# clear reject. Otherwise (malformed / resource not accepted) → 400.
# Anything ingested OR deduped → 202 with empty body (Graph acks; no counter leak). Every item
# failed auth → 403 so forged POSTs get a clear reject. Otherwise (malformed / not accepted) → 400.
if counts["accepted"] or counts["duplicate"]:
return web.Response(status=202)
if counts["auth"] and not counts["other"]:
@@ -254,13 +239,11 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
except (json.JSONDecodeError, UnicodeDecodeError):
return 400, []
notifications = body.get("value") if isinstance(body, dict) else None
if not isinstance(notifications, list):
return 400, []
return 0, notifications
return (0, notifications) if isinstance(notifications, list) else (400, [])
def _source_ip_allowed(self, request: "web.Request") -> bool:
"""Loopback-only binds may omit ``allowed_source_cidrs`` (local proxies,
dev tunnels); network-accessible binds fail closed without one."""
"""Loopback-only binds may omit ``allowed_source_cidrs`` (local proxies, dev tunnels);
network-accessible binds fail closed without one."""
if self._source_allowlist_required_but_missing():
return False
if not self._allowed_source_networks:
@@ -275,25 +258,21 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
if not self._accepted_resources:
return True
resource = resource.strip().strip("/")
for pattern in self._accepted_resources:
pattern = pattern.strip().strip("/")
if not pattern:
continue
for pattern in (p.strip().strip("/") for p in self._accepted_resources):
if pattern.endswith("*"):
pattern = pattern[:-1].rstrip("/")
if _prefix_match(resource, pattern):
if pattern and _prefix_match(resource, pattern):
return True
return False
def _verify_client_state(self, notification: Dict[str, Any]) -> bool:
"""Timing-safe compare of the Graph-supplied clientState against the
configured shared secret (``openssl rand -hex 32`` in the setup guide)."""
"""Timing-safe compare of the Graph-supplied clientState against the configured shared secret
(``openssl rand -hex 32`` in the setup guide)."""
expected = self._client_state
provided = _string_or_none(notification.get("clientState"))
if expected is None or provided is None:
return False
# Compare as bytes: compare_digest raises TypeError on non-ASCII str,
# and clientState comes from the request body.
# Compare as bytes: compare_digest raises TypeError on non-ASCII str (clientState is request-controlled).
return hmac.compare_digest(provided.encode(), expected.encode())
def _remember_receipt(self, receipt_key: str) -> None:
@@ -305,9 +284,8 @@ class MSGraphWebhookAdapter(BasePlatformAdapter):
def _build_message_event(self, notification: Dict[str, Any], receipt_key: Optional[str]) -> MessageEvent:
message_id = receipt_key or f"sha1:{sha1(json.dumps(notification, sort_keys=True).encode('utf-8')).hexdigest()}"
source = self.build_source(
chat_id=f"msgraph:{notification.get('subscriptionId', 'unknown')}",
chat_name="msgraph/webhook", chat_type="webhook",
user_id="msgraph", user_name="Microsoft Graph")
chat_id=f"msgraph:{notification.get('subscriptionId', 'unknown')}", chat_name="msgraph/webhook",
chat_type="webhook", user_id="msgraph", user_name="Microsoft Graph")
return MessageEvent(
text=self._render_prompt(notification), message_type=MessageType.TEXT, source=source,
raw_message=notification, message_id=message_id, internal=True)
gateway/platforms/qqbot/chunked_upload.py
+57 -105
View File
@@ -20,11 +20,11 @@ from gateway.platforms.qqbot.constants import FILE_UPLOAD_TIMEOUT
logger = logging.getLogger(__name__)
_BIZ_CODE_DAILY_LIMIT = 40093002 # upload_prepare: daily cumulative limit
_BIZ_CODE_DAILY_LIMIT = 40093002 # upload_prepare: daily cumulative limit
_BIZ_CODE_PART_RETRYABLE = 40093001 # upload_part_finish: transient
_DEFAULT_CONCURRENT_PARTS = 1
_MAX_CONCURRENT_PARTS = 10
_PART_UPLOAD_TIMEOUT = 300.0 # 5 minutes per COS PUT
_PART_UPLOAD_TIMEOUT = 300.0 # 5 minutes per COS PUT
_PART_UPLOAD_MAX_RETRIES = 2
_PART_FINISH_RETRY_INTERVAL = 1.0
_PART_FINISH_DEFAULT_TIMEOUT = 120.0
@@ -33,12 +33,9 @@ _COMPLETE_UPLOAD_MAX_RETRIES = 2
_COMPLETE_UPLOAD_BASE_DELAY = 2.0
_MD5_10M_SIZE = 10_002_432 # first N bytes used for the ``md5_10m`` hash (per QQ API spec)
# ── Exceptions ──
class _UploadError(Exception):
def __init__(self, file_name: str, file_size: int, message: str) -> None:
self.file_name = file_name
self.file_size = file_size
self.file_name, self.file_size = file_name, file_size
super().__init__(message)
@property
@@ -59,18 +56,14 @@ class UploadFileTooLargeError(_UploadError):
def __init__(self, file_name: str, file_size: int, limit_bytes: int = 0, message: str = "") -> None:
self.limit_bytes = limit_bytes
limit_str = f" ({format_size(limit_bytes)})" if limit_bytes else ""
super().__init__(
file_name, file_size,
message or f"File {file_name!r} ({format_size(file_size)}) exceeds platform limit{limit_str}",
)
super().__init__(file_name, file_size,
message or f"File {file_name!r} ({format_size(file_size)}) exceeds platform limit{limit_str}")
@property
def limit_human(self) -> str:
return format_size(self.limit_bytes) if self.limit_bytes else "unknown"
# ── Prepare-response shape ──
@dataclass
class _PreparePart:
index: int
@@ -90,35 +83,26 @@ class _PrepareResult:
def _parse_prepare_response(raw: Dict[str, Any]) -> _PrepareResult:
"""Parse upload_prepare response (either bare or wrapped in ``data``)."""
src = raw.get("data") if isinstance(raw.get("data"), dict) else raw
upload_id = str(src.get("upload_id", ""))
if not upload_id:
if not (upload_id := str(src.get("upload_id", ""))):
raise ValueError(f"upload_prepare response missing upload_id: {str(raw)[:200]}")
block_size = int(src.get("block_size", 0))
raw_parts = src.get("parts") or src.get("part_list") or []
if not isinstance(raw_parts, list) or not raw_parts:
raise ValueError(f"upload_prepare response missing parts: {str(raw)[:200]}")
parts = [
_PreparePart(
index=int(p.get("part_index") or p.get("index") or 0),
presigned_url=str(p.get("presigned_url") or p.get("url") or ""),
block_size=int(p.get("block_size", 0)))
for p in raw_parts
if isinstance(p, dict)]
parts = [_PreparePart(index=int(p.get("part_index") or p.get("index") or 0),
presigned_url=str(p.get("presigned_url") or p.get("url") or ""),
block_size=int(p.get("block_size", 0)))
for p in raw_parts if isinstance(p, dict)]
return _PrepareResult(
upload_id=upload_id,
block_size=block_size,
parts=parts,
upload_id=upload_id, block_size=block_size, parts=parts,
concurrency=int(src.get("concurrency", _DEFAULT_CONCURRENT_PARTS)) or _DEFAULT_CONCURRENT_PARTS,
retry_timeout=float(src.get("retry_timeout", 0.0) or 0.0))
def _api_path(chat_type: str, target_id: str, endpoint: str) -> str:
base = "/v2/users" if chat_type == "c2c" else "/v2/groups"
return f"{base}/{target_id}/{endpoint}"
return f"{'/v2/users' if chat_type == 'c2c' else '/v2/groups'}/{target_id}/{endpoint}"
# ── Chunked upload driver ──
@dataclass
class _Job:
"""Per-upload state shared by the part workers."""
@@ -142,9 +126,8 @@ class ChunkedUploader:
circular import; must raise RuntimeError with the biz_code in the message);
``http_put`` is ``(url, data, headers) -> httpx-like response`` for COS PUTs."""
def __init__(
self, api_request: Callable[..., Awaitable[Dict[str, Any]]], http_put: Callable[..., Awaitable[Any]],
log_tag: str = "QQBot") -> None:
def __init__(self, api_request: Callable[..., Awaitable[Dict[str, Any]]],
http_put: Callable[..., Awaitable[Any]], log_tag: str = "QQBot") -> None:
self._api_request = api_request
self._http_put = http_put
self._log_tag = log_tag
@@ -158,24 +141,18 @@ class ChunkedUploader:
Raises UploadDailyLimitExceededError (40093002), UploadFileTooLargeError, RuntimeError."""
if chat_type not in {"c2c", "group"}:
raise ValueError(f"ChunkedUploader: unsupported chat_type {chat_type!r}")
job = _Job(chat_type, target_id, file_path, Path(file_path).stat().st_size)
logger.info(
"[%s] Chunked upload start: file=%s size=%s type=%d",
self._log_tag, file_name, format_size(job.file_size), file_type)
logger.info("[%s] Chunked upload start: file=%s size=%s type=%d", self._log_tag, file_name,
format_size(job.file_size), file_type)
# Hashing is blocking I/O → executor.
hashes = await asyncio.get_running_loop().run_in_executor(None, _compute_file_hashes, file_path, job.file_size)
prepare = await self._prepare(job, file_type, file_name, hashes)
max_concurrent = min(prepare.concurrency, _MAX_CONCURRENT_PARTS)
job.upload_id, job.block_size, job.total_parts = prepare.upload_id, prepare.block_size, len(prepare.parts)
job.retry_timeout = min(
prepare.retry_timeout if prepare.retry_timeout > 0 else _PART_FINISH_DEFAULT_TIMEOUT,
_PART_FINISH_MAX_TIMEOUT)
logger.info(
"[%s] Prepared: upload_id=%s block_size=%s parts=%d concurrency=%d",
self._log_tag, job.upload_id, format_size(job.block_size), job.total_parts, max_concurrent)
job.retry_timeout = min(prepare.retry_timeout if prepare.retry_timeout > 0 else _PART_FINISH_DEFAULT_TIMEOUT,
_PART_FINISH_MAX_TIMEOUT)
logger.info("[%s] Prepared: upload_id=%s block_size=%s parts=%d concurrency=%d", self._log_tag,
job.upload_id, format_size(job.block_size), job.total_parts, max_concurrent)
sem = asyncio.Semaphore(max(max_concurrent, 1))
async def _run(part: _PreparePart) -> None:
@@ -187,42 +164,34 @@ class ChunkedUploader:
return await self._complete(job)
async def _prepare(self, job: _Job, file_type: int, file_name: str, hashes: Dict[str, str]) -> _PrepareResult:
body = {
"file_type": file_type, "file_name": file_name, "file_size": job.file_size,
"md5": hashes["md5"], "sha1": hashes["sha1"], "md5_10m": hashes["md5_10m"]}
body = {"file_type": file_type, "file_name": file_name, "file_size": job.file_size,
"md5": hashes["md5"], "sha1": hashes["sha1"], "md5_10m": hashes["md5_10m"]}
try:
raw = await self._post(job, "upload_prepare", body)
except RuntimeError as exc:
err_msg = str(exc)
if f"{_BIZ_CODE_DAILY_LIMIT}" in err_msg:
raise UploadDailyLimitExceededError(file_name, job.file_size, err_msg) from exc
if f"{_BIZ_CODE_DAILY_LIMIT}" in str(exc):
raise UploadDailyLimitExceededError(file_name, job.file_size, str(exc)) from exc
raise
return _parse_prepare_response(raw)
async def _upload_one_part(self, job: _Job, part: _PreparePart) -> None:
"""PUT one part to COS, then call ``upload_part_finish``."""
part_index, total_parts = part.index, job.total_parts
# Per-part block_size wins; fall back to the response-level value.
actual_block_size = part.block_size if part.block_size > 0 else job.block_size
offset = (part_index - 1) * job.block_size
length = min(actual_block_size, job.file_size - offset)
# Per-part block_size wins; fall back to the response-level value.
length = min(part.block_size if part.block_size > 0 else job.block_size, job.file_size - offset)
data = await asyncio.get_running_loop().run_in_executor(None, _read_file_chunk, job.file_path, offset, length)
md5_hex = hashlib.md5(data).hexdigest()
logger.debug(
"[%s] Part %d/%d: uploading %s (offset=%d md5=%s)",
self._log_tag, part_index, total_parts, format_size(length), offset, md5_hex)
logger.debug("[%s] Part %d/%d: uploading %s (offset=%d md5=%s)", self._log_tag, part_index, total_parts,
format_size(length), offset, md5_hex)
await self._put_to_presigned_url(part.presigned_url, data, part_index, total_parts)
await self._part_finish_with_retry(job, part_index, length, md5_hex)
job.completed += 1
logger.debug(
"[%s] Part %d/%d done (%d/%d total)", self._log_tag, part_index, total_parts, job.completed, total_parts)
logger.debug("[%s] Part %d/%d done (%d/%d total)", self._log_tag, part_index, total_parts, job.completed,
total_parts)
async def _with_retries(
self, attempt_fn: Callable[[], Awaitable[Any]], *, max_retries: int, base_delay: float, label: str,
failure_label: str) -> Any:
async def _with_retries(self, attempt_fn: Callable[[], Awaitable[Any]], *, max_retries: int, base_delay: float,
label: str, failure_label: str) -> Any:
"""Run *attempt_fn* up to ``max_retries + 1`` times with exponential backoff."""
last_exc: Exception | None = None
for attempt in range(max_retries + 1):
@@ -232,9 +201,8 @@ class ChunkedUploader:
last_exc = exc
if attempt < max_retries:
delay = base_delay * (2 ** attempt)
logger.warning(
"[%s] %s attempt %d failed, retry in %.1fs: %s",
self._log_tag, label, attempt + 1, delay, exc)
logger.warning("[%s] %s attempt %d failed, retry in %.1fs: %s", self._log_tag, label, attempt + 1,
delay, exc)
await asyncio.sleep(delay)
raise RuntimeError(f"{failure_label} failed after {max_retries + 1} attempts: {last_exc}")
@@ -242,9 +210,8 @@ class ChunkedUploader:
"""PUT part data to a pre-signed COS URL with retry."""
async def _attempt() -> None:
resp = await asyncio.wait_for(
self._http_put(url, data=data, headers={"Content-Length": str(len(data))}),
timeout=_PART_UPLOAD_TIMEOUT)
resp = await asyncio.wait_for(self._http_put(url, data=data, headers={"Content-Length": str(len(data))}),
timeout=_PART_UPLOAD_TIMEOUT)
status = getattr(resp, "status_code", 0)
if 200 <= status < 300:
logger.debug("[%s] PUT part %d/%d: %d OK", self._log_tag, part_index, total_parts, status)
@@ -255,16 +222,15 @@ class ChunkedUploader:
body_preview = ""
raise RuntimeError(f"COS PUT returned {status}: {body_preview}")
await self._with_retries(
_attempt, max_retries=_PART_UPLOAD_MAX_RETRIES, base_delay=1.0,
label=f"PUT part {part_index}/{total_parts}", failure_label=f"Part {part_index}/{total_parts} upload")
await self._with_retries(_attempt, max_retries=_PART_UPLOAD_MAX_RETRIES, base_delay=1.0,
label=f"PUT part {part_index}/{total_parts}",
failure_label=f"Part {part_index}/{total_parts} upload")
async def _part_finish_with_retry(self, job: _Job, part_index: int, block_size: int, md5: str) -> None:
"""Call ``upload_part_finish``, retrying on biz_code 40093001 until ``job.retry_timeout``."""
body = {"upload_id": job.upload_id, "part_index": part_index, "block_size": block_size, "md5": md5}
loop = asyncio.get_running_loop()
start = loop.time()
attempt = 0
start, attempt = loop.time(), 0
while True:
try:
await self._post(job, "upload_part_finish", body)
@@ -274,26 +240,21 @@ class ChunkedUploader:
raise
elapsed = loop.time() - start
if elapsed >= job.retry_timeout:
raise RuntimeError(
f"upload_part_finish persistent retry timed out "
f"after {job.retry_timeout:.0f}s ({attempt} retries): {exc}") from exc
raise RuntimeError(f"upload_part_finish persistent retry timed out after "
f"{job.retry_timeout:.0f}s ({attempt} retries): {exc}") from exc
attempt += 1
logger.debug(
"[%s] part_finish retryable error, attempt %d, elapsed=%.1fs: %s",
self._log_tag, attempt, elapsed, exc)
logger.debug("[%s] part_finish retryable error, attempt %d, elapsed=%.1fs: %s", self._log_tag, attempt,
elapsed, exc)
await asyncio.sleep(_PART_FINISH_RETRY_INTERVAL)
async def _complete(self, job: _Job) -> Dict[str, Any]:
"""Call ``complete_upload`` with retry — the ``/files`` endpoint (same as the simple
URL upload) selects the chunked-completion path when only ``upload_id`` is sent."""
"""Call ``complete_upload`` with retry — the ``/files`` endpoint (same as the simple URL upload)
selects the chunked-completion path when only ``upload_id`` is sent."""
return await self._with_retries(
lambda: self._post(job, "files", {"upload_id": job.upload_id}),
max_retries=_COMPLETE_UPLOAD_MAX_RETRIES, base_delay=_COMPLETE_UPLOAD_BASE_DELAY,
label="complete_upload", failure_label="complete_upload")
lambda: self._post(job, "files", {"upload_id": job.upload_id}), max_retries=_COMPLETE_UPLOAD_MAX_RETRIES,
base_delay=_COMPLETE_UPLOAD_BASE_DELAY, label="complete_upload", failure_label="complete_upload")
# ── Helpers (module-level for testability) ──
def format_size(size_bytes: int) -> str:
"""Return a human-readable file size string (e.g. ``'12.3 MB'``)."""
size = float(size_bytes)
@@ -309,32 +270,23 @@ def _read_file_chunk(file_path: str, offset: int, length: int) -> bytes:
with open(file_path, "rb") as fh:
fh.seek(offset)
data = fh.read(length)
if len(data) != length:
raise IOError(
f"Short read from {file_path}: expected {length} bytes at "
f"offset {offset}, got {len(data)} (file may be truncated)")
return data
if len(data) != length:
raise IOError(f"Short read from {file_path}: expected {length} bytes at offset {offset}, got {len(data)} "
f"(file may be truncated)")
return data
def _compute_file_hashes(file_path: str, file_size: int) -> Dict[str, str]:
"""Compute md5, sha1, and md5_10m in a single pass."""
md5 = hashlib.md5()
sha1 = hashlib.sha1()
md5_10m = hashlib.md5()
"""Compute md5, sha1, and md5_10m in a single pass (for small files md5_10m is just the full md5)."""
md5, sha1, md5_10m = hashlib.md5(), hashlib.sha1(), hashlib.md5()
need_10m = file_size > _MD5_10M_SIZE
bytes_read = 0
with open(file_path, "rb") as fh:
while chunk := fh.read(65536):
md5.update(chunk)
sha1.update(chunk)
if need_10m:
remaining = _MD5_10M_SIZE - bytes_read
if remaining > 0:
md5_10m.update(chunk[:remaining])
if need_10m and (remaining := _MD5_10M_SIZE - bytes_read) > 0:
md5_10m.update(chunk[:remaining])
bytes_read += len(chunk)
full_md5 = md5.hexdigest()
return {
"md5": full_md5,
"sha1": sha1.hexdigest(),
# For small files the "10m" hash is just the full md5.
"md5_10m": md5_10m.hexdigest() if need_10m else full_md5}
return {"md5": full_md5, "sha1": sha1.hexdigest(), "md5_10m": md5_10m.hexdigest() if need_10m else full_md5}
gateway/platforms/qqbot/keyboards.py
+22 -46
View File
@@ -1,11 +1,8 @@
"""QQ Bot inline keyboards + approval / update-prompt helpers.
A button click dispatches an ``INTERACTION_CREATE`` event carrying the button's
``data``; the bot must ACK promptly via ``PUT /interactions/{id}`` or the user
sees an error indicator. ``button_data`` formats: ``approve:<session_key>:<decision>``
(decision = allow-once|allow-always|deny) and ``update_prompt:<answer>`` (y|n).
Ported from WideLee's qqbot-agent-sdk v1.2.2 (authorship via Co-authored-by).
"""
"""QQ Bot inline keyboards + approval / update-prompt helpers. A button click dispatches an
``INTERACTION_CREATE`` event carrying the button's ``data``; the bot must ACK promptly via
``PUT /interactions/{id}`` or the user sees an error indicator. ``button_data`` formats:
``approve:<session_key>:<decision>`` (allow-once|allow-always|deny) and ``update_prompt:<answer>`` (y|n).
Ported from WideLee's qqbot-agent-sdk v1.2.2 (authorship via Co-authored-by)."""
from __future__ import annotations
@@ -16,13 +13,10 @@ from typing import Any, Dict, List, Optional
APPROVAL_BUTTON_PREFIX = "approve:"
UPDATE_PROMPT_PREFIX = "update_prompt:"
# session_key may itself contain colons (e.g. agent:main:qqbot:c2c:OPENID),
# so the session_key group is greedy but trails the decision.
# session_key may itself contain colons (agent:main:qqbot:c2c:OPENID): greedy group, decision trails.
_APPROVAL_DATA_RE = re.compile(r"^approve:(.+):(allow-once|allow-always|deny)$")
_UPDATE_PROMPT_RE = re.compile(r"^update_prompt:(y|n)$")
# ── Keyboard dataclasses ──
def _to_dict(value: Any) -> Any:
"""Serialize a dataclass tree in field-declaration order (the wire shape)."""
if is_dataclass(value):
@@ -31,8 +25,7 @@ def _to_dict(value: Any) -> Any:
class _Serializable:
def to_dict(self) -> Dict[str, Any]:
return _to_dict(self)
to_dict = _to_dict
@dataclass
@@ -84,36 +77,28 @@ class InlineKeyboard(_Serializable):
content: KeyboardContent = field(default_factory=KeyboardContent)
# ── INTERACTION_CREATE parsing ──
def parse_approval_button_data(button_data: str) -> Optional[tuple[str, str]]:
"""Parse approval ``button_data`` into ``(session_key, decision)`` or ``None``."""
m = _APPROVAL_DATA_RE.match(button_data or "")
return m.groups() if m else None
return m.groups() if (m := _APPROVAL_DATA_RE.match(button_data or "")) else None
def parse_update_prompt_button_data(button_data: str) -> Optional[str]:
"""Parse update-prompt ``button_data`` into ``'y'`` / ``'n'`` or ``None``."""
m = _UPDATE_PROMPT_RE.match(button_data or "")
return m.group(1) if m else None
return m.group(1) if (m := _UPDATE_PROMPT_RE.match(button_data or "")) else None
# ── Keyboard builders ──
def _single_row_keyboard(group_id: str, *buttons: tuple) -> InlineKeyboard:
"""One row of callback buttons from ``(id, label, visited_label, data, style)`` tuples."""
row = KeyboardRow(buttons=[
KeyboardButton(
id=btn_id, group_id=group_id, action=KeyboardButtonAction(type=1, data=data),
render_data=KeyboardButtonRenderData(label=label, visited_label=visited, style=style))
KeyboardButton(id=btn_id, group_id=group_id, action=KeyboardButtonAction(type=1, data=data),
render_data=KeyboardButtonRenderData(label=label, visited_label=visited, style=style))
for btn_id, label, visited, data, style in buttons])
return InlineKeyboard(content=KeyboardContent(rows=[row]))
def build_approval_keyboard(session_key: str, *, allow_permanent: bool = True) -> InlineKeyboard:
"""Build ``[✅ 允许一次] [⭐ 始终允许] [❌ 拒绝]`` (one group, so a click greys the rest).
⭐ is hidden when persistent scope is unavailable; *session_key* is embedded in
``button_data`` so the decision routes to the right approval."""
"""Build ``[✅ 允许一次] [⭐ 始终允许] [❌ 拒绝]`` (one group, so a click greys the rest). ⭐ is hidden when
persistent scope is unavailable; *session_key* rides in ``button_data`` so the decision routes correctly."""
prefix = f"{APPROVAL_BUTTON_PREFIX}{session_key}"
buttons = [("allow", "✅ 允许一次", "已允许", f"{prefix}:allow-once", 1)]
if allow_permanent:
@@ -124,18 +109,14 @@ def build_approval_keyboard(session_key: str, *, allow_permanent: bool = True) -
def build_update_prompt_keyboard() -> InlineKeyboard:
"""Build a Yes/No keyboard for update confirmation prompts."""
return _single_row_keyboard(
"update_prompt",
("yes", "✓ 确认", "已确认", f"{UPDATE_PROMPT_PREFIX}y", 1),
("no", "✗ 取消", "已取消", f"{UPDATE_PROMPT_PREFIX}n", 0))
return _single_row_keyboard("update_prompt", ("yes", "✓ 确认", "已确认", f"{UPDATE_PROMPT_PREFIX}y", 1),
("no", "✗ 取消", "已取消", f"{UPDATE_PROMPT_PREFIX}n", 0))
# ── ApprovalRequest + text builder ──
@dataclass
class ApprovalRequest:
"""Approval-request display data. ``command_preview`` / ``cwd`` are set for exec
approvals, ``tool_name`` for plugin approvals; ``severity`` is ``'critical' | 'info' | ''``."""
"""Approval-request display data. ``command_preview`` / ``cwd`` are set for exec approvals, ``tool_name``
for plugin approvals; ``severity`` is ``'critical' | 'info' | ''``."""
session_key: str
title: str
description: str = ""
@@ -172,8 +153,6 @@ def build_approval_text(req: ApprovalRequest) -> str:
return "\n".join(lines)
# ── INTERACTION_CREATE event shape ──
@dataclass
class InteractionEvent:
"""Parsed ``INTERACTION_CREATE`` payload (api-v2 event-emit docs)."""
@@ -202,14 +181,11 @@ _SCENE_NAMES = {0: "guild", 1: "group", 2: "c2c"}
def parse_interaction_event(raw: Dict[str, Any]) -> InteractionEvent:
"""Parse a raw ``INTERACTION_CREATE`` dispatch payload (``d``)."""
data_raw = raw.get("data") or {}
resolved = data_raw.get("resolved") or {}
scene_code = int(raw.get("chat_type", 0) or 0)
resolved, scene_code = data_raw.get("resolved") or {}, int(raw.get("chat_type", 0) or 0)
return InteractionEvent(
id=str(raw.get("id", "")), type=int(data_raw.get("type", 0) or 0), chat_type=scene_code,
scene=_SCENE_NAMES.get(scene_code, ""), group_openid=str(raw.get("group_openid", "")),
group_member_openid=str(raw.get("group_member_openid", "")),
user_openid=str(raw.get("user_openid", "")), channel_id=str(raw.get("channel_id", "")),
guild_id=str(raw.get("guild_id", "")), button_data=str(resolved.get("button_data", "")),
button_id=str(resolved.get("button_id", "")),
resolver_user_id=str(resolved.get("user_id", "")),
)
group_member_openid=str(raw.get("group_member_openid", "")), user_openid=str(raw.get("user_openid", "")),
channel_id=str(raw.get("channel_id", "")), guild_id=str(raw.get("guild_id", "")),
button_data=str(resolved.get("button_data", "")), button_id=str(resolved.get("button_id", "")),
resolver_user_id=str(resolved.get("user_id", "")))
gateway/platforms/qqbot/onboard.py
+5 -13
View File
@@ -1,6 +1,5 @@
"""QQBot scan-to-configure (QR code onboard). Mirrors the Feishu pattern: synchronous
HTTP + one entry-point ``qr_register()`` (create task → display QR → poll → decrypt
credentials) against ``q.qq.com`` ``create_bind_task`` / ``poll_bind_result``."""
"""QQBot scan-to-configure (QR code onboard). Mirrors the Feishu pattern: synchronous HTTP + one entry-point
``qr_register()`` (create task → display QR → poll → decrypt) against ``q.qq.com`` bind-task endpoints."""
from __future__ import annotations
@@ -11,8 +10,7 @@ from typing import Optional, Tuple
from urllib.parse import quote
from .constants import (
ONBOARD_API_TIMEOUT, ONBOARD_CREATE_PATH, ONBOARD_POLL_INTERVAL, ONBOARD_POLL_PATH, PORTAL_HOST, QR_URL_TEMPLATE,
)
ONBOARD_API_TIMEOUT, ONBOARD_CREATE_PATH, ONBOARD_POLL_INTERVAL, ONBOARD_POLL_PATH, PORTAL_HOST, QR_URL_TEMPLATE)
from .crypto import decrypt_secret, generate_bind_key
from .utils import get_api_headers
@@ -71,7 +69,8 @@ def _create_bind_task(timeout: float = ONBOARD_API_TIMEOUT) -> Tuple[str, str]:
def _poll_bind_result(task_id: str, timeout: float = ONBOARD_API_TIMEOUT) -> Tuple[BindStatus, str, str, str]:
"""Poll *task_id*; returns ``(status, bot_appid, bot_encrypt_secret, user_openid)``."""
d = _portal_post(ONBOARD_POLL_PATH, {"task_id": task_id}, timeout, "poll_bind_result failed").get("data", {})
return BindStatus(d.get("status", 0)), str(d.get("bot_appid", "")), d.get("bot_encrypt_secret", ""), d.get("user_openid", "")
return (BindStatus(d.get("status", 0)), str(d.get("bot_appid", "")), d.get("bot_encrypt_secret", ""),
d.get("user_openid", ""))
def build_connect_url(task_id: str) -> str:
@@ -86,14 +85,12 @@ def qr_register(timeout_seconds: int = 600) -> Optional[dict]:
"""Run the QR registration flow; returns ``{"app_id", "client_secret", "user_openid"}``
or None on failure / expiry / cancellation. Unexpected errors propagate."""
deadline = time.monotonic() + timeout_seconds
for refresh_count in range(_MAX_REFRESHES + 1):
try:
task_id, aes_key = _create_bind_task()
except Exception as exc:
logger.warning("[QQBot onboard] Failed to create bind task: %s", exc)
return None
url = build_connect_url(task_id)
print()
if _render_qr(url):
@@ -102,14 +99,12 @@ def qr_register(timeout_seconds: int = 600) -> Optional[dict]:
print(f" Open this URL in QQ on your phone:\n {url}")
print(" Tip: pip install qrcode to display a scannable QR code here")
print()
while time.monotonic() < deadline:
try:
status, app_id, encrypted_secret, user_openid = _poll_bind_result(task_id)
except Exception:
time.sleep(ONBOARD_POLL_INTERVAL)
continue
if status == BindStatus.COMPLETED:
client_secret = decrypt_secret(encrypted_secret, aes_key)
print()
@@ -117,17 +112,14 @@ def qr_register(timeout_seconds: int = 600) -> Optional[dict]:
if user_openid:
print(f" Scanner's OpenID: {user_openid}")
return {"app_id": app_id, "client_secret": client_secret, "user_openid": user_openid}
if status == BindStatus.EXPIRED:
if refresh_count >= _MAX_REFRESHES:
logger.warning("[QQBot onboard] QR code expired %d times — giving up", _MAX_REFRESHES)
return None
print(f"\n QR code expired, refreshing... ({refresh_count + 1}/{_MAX_REFRESHES})")
break # next for-loop iteration creates a new task
time.sleep(ONBOARD_POLL_INTERVAL)
else:
logger.warning("[QQBot onboard] Poll timed out after %ds", timeout_seconds)
return None
return None
gateway/platforms/signal.py
+136 -209
View File
@@ -1,9 +1,6 @@
"""Signal messenger platform adapter.
Connects to a signal-cli daemon in HTTP mode (``signal-cli daemon --http 127.0.0.1:8080``):
inbound messages arrive via SSE, outbound messages/actions use JSON-RPC 2.0 over HTTP.
Requires SIGNAL_HTTP_URL and SIGNAL_ACCOUNT.
"""
"""Signal messenger platform adapter for a signal-cli daemon in HTTP mode (``signal-cli daemon
--http 127.0.0.1:8080``): inbound via SSE, outbound via JSON-RPC 2.0. Requires SIGNAL_HTTP_URL
and SIGNAL_ACCOUNT."""
import asyncio
import base64
@@ -55,6 +52,11 @@ _MAGIC_EXTENSIONS = ((b"\x89PNG", ".png"), (b"\xff\xd8", ".jpg"), (b"GIF8", ".gi
_MEDIA_TYPE_BY_MIME_PREFIX = (
("audio/", MessageType.VOICE), ("image/", MessageType.PHOTO), ("video/", MessageType.VIDEO))
_OUTCOME_REACTION = {ProcessingOutcome.SUCCESS: "✅", ProcessingOutcome.FAILURE: "❌"}
# send_multiple_images skip reasons → logger.warning args (url, detail).
_SKIP_IMAGE_LOG = {
"download": lambda url, detail: ("Signal: failed to download image %s: %s", url, detail),
"missing": lambda url, detail: ("Signal: image file not found for %s", url),
"oversize": lambda url, detail: ("Signal: image too large (%d bytes), skipping %s", detail, url)}
_QUOTE_AUTHOR_KEYS = (
"author", "authorNumber", "authorUuid", "authorAci", "authorServiceId", "authorServiceIdString")
@@ -72,8 +74,7 @@ def _guess_extension(data: bytes) -> str:
return ext
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
return ".webp"
container = sniff_container(data)
if container is not None:
if (container := sniff_container(data)) is not None:
return CONTAINER_TO_EXT[container]
return ".zip" if data[:2] == b"PK" else ".bin"
@@ -107,10 +108,8 @@ def _remux_aac_to_m4a(aac_data: bytes) -> Optional[Tuple[bytes, str]]:
src_path = src.name
dst_path = src_path[:-4] + ".m4a"
try:
proc = subprocess.run(
[ffmpeg, "-y", "-loglevel", "error", "-i", src_path,
"-c:a", "copy", "-movflags", "+faststart", dst_path],
capture_output=True, timeout=10)
proc = subprocess.run([ffmpeg, "-y", "-loglevel", "error", "-i", src_path, "-c:a", "copy", "-movflags",
"+faststart", dst_path], capture_output=True, timeout=10)
if proc.returncode != 0:
logger.warning("Signal: AAC→M4A remux failed (ffmpeg exit %d): %s",
proc.returncode, proc.stderr.decode("utf-8", "replace")[:300])
@@ -141,24 +140,20 @@ def _render_mentions(text: str, mentions: list) -> str:
def _is_signal_service_id(value: str) -> bool:
"""Return True if *value* already looks like a Signal service identifier."""
"""Return True if *value* already looks like a Signal service identifier (PNI:/u: prefix or UUID)."""
if not value:
return False
if value.startswith(("PNI:", "u:")):
return True
try:
with suppress(ValueError, AttributeError, TypeError):
uuid.UUID(value)
return True
except (ValueError, AttributeError, TypeError):
return False
return False
def _looks_like_e164_number(value: str) -> bool:
"""Return True for a plausible E.164 phone number."""
if not value or not value.startswith("+"):
return False
digits = value[1:]
return digits.isdigit() and 7 <= len(digits) <= 15
return bool(value) and value.startswith("+") and value[1:].isdigit() and 7 <= len(value) - 1 <= 15
def check_signal_requirements() -> bool:
@@ -189,15 +184,13 @@ class SignalAdapter(BasePlatformAdapter):
self.http_url = extra.get("http_url", "http://127.0.0.1:8080").rstrip("/")
self.account = extra.get("account", "")
self.ignore_stories = extra.get("ignore_stories", True)
# Allowlists are per-profile (scoped reads); group policy derives from the group
# allowlist's presence. Mention filter: only respond in groups when @mentioned.
# Allowlists are per-profile (scoped reads); group policy derives from the group allowlist's
# presence. The DM allowlist mirrors run.py's SIGNAL_ALLOWED_USERS so reaction hooks (which
# fire before run.py's auth gate) can skip unauthorized senders; "*" = open.
self.group_allow_from = set(_parse_comma_list(_sig_secret("SIGNAL_GROUP_ALLOWED_USERS", "")))
_rm_cfg = extra.get("require_mention")
self.require_mention = (
bool(_rm_cfg) if _rm_cfg is not None
else os.getenv("SIGNAL_REQUIRE_MENTION", "false").lower() in TRUTHY_STRINGS)
# DM allowlist mirrors run.py's SIGNAL_ALLOWED_USERS so reaction hooks (which fire
# before run.py's auth gate) can skip unauthorized senders. "*" = open.
self.require_mention = (bool(_rm_cfg) if _rm_cfg is not None
else os.getenv("SIGNAL_REQUIRE_MENTION", "false").lower() in TRUTHY_STRINGS)
self.dm_allow_from = set(_parse_comma_list(_sig_secret("SIGNAL_ALLOWED_USERS", "*")))
self.client: Optional[httpx.AsyncClient] = None
self._sse_task: Optional[asyncio.Task] = None
@@ -210,30 +203,28 @@ class SignalAdapter(BasePlatformAdapter):
self._last_sse_activity = 0.0
self._sse_response: Optional[httpx.Response] = None
self._account_normalized = self.account.strip()
# Recently sent timestamps filter echo-backs (Note to Self / linked-device sync-sents).
# LRU + TTL so a pending echo in a chatty group isn't evicted by many outbounds.
# Recently sent timestamps filter echo-backs (Note to Self / linked-device sync-sents); LRU + TTL
# so a pending echo in a chatty group isn't evicted by many outbounds.
self._recent_sent_timestamps: "OrderedDict[int, float]" = OrderedDict()
self._max_recent_timestamps = 512
self._recent_sent_ttl_seconds = 300.0
# Separate FIFO of outbound timestamps: Signal quote.id is the quoted message's
# timestamp, so replies to this bot are recognised after the echo was consumed.
# Separate FIFO of outbound timestamps: Signal quote.id is the quoted message's timestamp, so
# replies to this bot are recognised after the echo was consumed.
self._sent_message_timestamps: "OrderedDict[str, None]" = OrderedDict()
self._max_sent_message_timestamps = 500
# Best-effort number↔ACI/PNI UUID mapping so sends can upgrade a number to the UUID signal-cli prefers.
self._recipient_uuid_by_number: Dict[str, str] = {}
self._recipient_number_by_uuid: Dict[str, str] = {}
self._recipient_cache_lock = asyncio.Lock()
logger.info("Signal adapter initialized: url=%s account=%s groups=%s",
self.http_url, redact_phone(self.account),
"enabled" if self.group_allow_from else "disabled")
logger.info("Signal adapter initialized: url=%s account=%s groups=%s", self.http_url,
redact_phone(self.account), "enabled" if self.group_allow_from else "disabled")
async def connect(self, *, is_reconnect: bool = False) -> bool:
"""Connect to signal-cli daemon and start SSE listener."""
if not self.http_url or not self.account:
logger.error("Signal: SIGNAL_HTTP_URL and SIGNAL_ACCOUNT are required")
return False
# Scoped lock prevents duplicate Signal listeners for the same phone.
lock_acquired = False
lock_acquired = False # scoped lock prevents duplicate Signal listeners for the same phone
try:
if not self._acquire_platform_lock('signal-phone', self.account, 'Signal account'):
return False
@@ -246,12 +237,12 @@ class SignalAdapter(BasePlatformAdapter):
try:
try:
resp = await self.client.get(f"{self.http_url}/api/v1/check", timeout=10.0)
if resp.status_code != 200:
logger.error("Signal: health check failed (status %d)", resp.status_code)
return False
except Exception as e:
logger.error("Signal: cannot reach signal-cli at %s: %s", self.http_url, e)
return False
if resp.status_code != 200:
logger.error("Signal: health check failed (status %d)", resp.status_code)
return False
self._running = True
self._last_sse_activity = time.time()
self._sse_task = asyncio.create_task(self._sse_listener())
@@ -266,11 +257,6 @@ class SignalAdapter(BasePlatformAdapter):
if lock_acquired:
self._release_platform_lock()
def _spawn_background(self, coro) -> None:
task = asyncio.create_task(coro)
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
async def _close_client(self) -> None:
if self.client:
await self.client.aclose()
@@ -299,10 +285,7 @@ class SignalAdapter(BasePlatformAdapter):
"""Dispatch one SSE line; keepalive comments (":") count as activity so the health monitor stays quiet."""
if line.startswith(":"):
self._last_sse_activity = time.time()
elif line.startswith("data:"):
data_str = line[5:].strip()
if not data_str:
return
elif line.startswith("data:") and (data_str := line[5:].strip()):
self._last_sse_activity = time.time()
try:
await self._handle_envelope(json.loads(data_str))
@@ -318,8 +301,8 @@ class SignalAdapter(BasePlatformAdapter):
while self._running:
try:
logger.debug("Signal SSE: connecting to %s", url)
headers = {"Accept": "text/event-stream"}
async with self.client.stream("GET", url, headers=headers, timeout=None) as response:
async with self.client.stream("GET", url, headers={"Accept": "text/event-stream"},
timeout=None) as response:
self._sse_response = response
backoff = SSE_RETRY_DELAY_INITIAL # Reset on successful connection
self._last_sse_activity = time.time()
@@ -342,8 +325,7 @@ class SignalAdapter(BasePlatformAdapter):
if self._running:
logger.warning("Signal SSE: error: %s (reconnecting in %.0fs)", e, backoff)
if self._running:
# 20% jitter prevents thundering herd on reconnection
await asyncio.sleep(backoff + backoff * 0.2 * random.random())
await asyncio.sleep(backoff + backoff * 0.2 * random.random()) # 20% jitter vs thundering herd
backoff = min(backoff * 2, SSE_RETRY_DELAY_MAX)
self._sse_response = None
@@ -353,8 +335,7 @@ class SignalAdapter(BasePlatformAdapter):
await asyncio.sleep(HEALTH_CHECK_INTERVAL)
if not self._running:
break
elapsed = time.time() - self._last_sse_activity
if elapsed <= HEALTH_CHECK_STALE_THRESHOLD:
if (elapsed := time.time() - self._last_sse_activity) <= HEALTH_CHECK_STALE_THRESHOLD:
continue
logger.warning("Signal: SSE idle for %.0fs, checking daemon health", elapsed)
try:
@@ -363,8 +344,7 @@ class SignalAdapter(BasePlatformAdapter):
logger.warning("Signal: health check error: %s, forcing reconnect", e)
self._force_reconnect()
continue
if resp.status_code == 200:
# Daemon alive but SSE quiet — reset activity to avoid repeated warnings
if resp.status_code == 200: # daemon alive but SSE quiet — reset activity to avoid repeated warnings
self._last_sse_activity = time.time()
logger.debug("Signal: daemon healthy, SSE idle")
else:
@@ -375,7 +355,9 @@ class SignalAdapter(BasePlatformAdapter):
"""Force SSE reconnection by closing the current response."""
if self._sse_response and not self._sse_response.is_stream_consumed:
with suppress(Exception):
self._spawn_background(self._sse_response.aclose())
task = asyncio.create_task(self._sse_response.aclose())
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
self._sse_response = None
def _unwrap_sync_message(self, envelope_data: dict) -> Optional[dict]:
@@ -386,8 +368,7 @@ class SignalAdapter(BasePlatformAdapter):
if not sent_msg or not isinstance(sent_msg, dict):
return None
dest = sent_msg.get("destinationNumber") or sent_msg.get("destination")
sent_msg_group_id = (sent_msg.get("groupInfo") or {}).get("groupId")
if dest != self._account_normalized and not sent_msg_group_id:
if dest != self._account_normalized and not (sent_msg.get("groupInfo") or {}).get("groupId"):
return None
if self._consume_sent_timestamp(sent_msg.get("timestamp")):
return None # echo of our own outbound reply
@@ -399,19 +380,16 @@ class SignalAdapter(BasePlatformAdapter):
account_norm = self._account_normalized
if self.require_mention:
mentioned_in_text = account_norm and (f"@{account_norm}" in (text or ""))
mentioned_in_metadata = any(
m.get("number") == account_norm or m.get("uuid") == account_norm
for m in (data_message.get("mentions") or []))
mentioned_in_metadata = any(account_norm in (m.get("number"), m.get("uuid"))
for m in (data_message.get("mentions") or []))
if not mentioned_in_text and not mentioned_in_metadata:
logger.debug("Signal: ignoring group message (require_mention=true, bot not mentioned)")
return False, text
if text and account_norm:
text = text.replace(f"@{account_norm}", "")
bot_uuid = self._recipient_uuid_by_number.get(account_norm)
if bot_uuid:
if bot_uuid := self._recipient_uuid_by_number.get(account_norm):
text = text.replace(f"@{bot_uuid}", "")
# Collapse only the doubled space the removal introduced; newlines are preserved.
text = text.replace(" ", " ").strip()
text = text.replace(" ", " ").strip() # collapse only the doubled space; newlines preserved
return True, text
async def _collect_attachments(self, attachments_data: list) -> Tuple[List[str], List[str]]:
@@ -442,9 +420,8 @@ class SignalAdapter(BasePlatformAdapter):
envelope_data = self._unwrap_sync_message(envelope_data)
if envelope_data is None:
return
sender = (
envelope_data.get("sourceNumber") or envelope_data.get("sourceUuid") or envelope_data.get("source")
)
sender = (envelope_data.get("sourceNumber") or envelope_data.get("sourceUuid")
or envelope_data.get("source"))
sender_name = envelope_data.get("sourceName", "")
sender_uuid = envelope_data.get("sourceUuid", "")
self._remember_recipient_identifiers(sender, sender_uuid)
@@ -457,9 +434,8 @@ class SignalAdapter(BasePlatformAdapter):
if self.ignore_stories and envelope_data.get("storyMessage"):
return
# Edited messages carry their updated dataMessage inside editMessage
data_message = (
envelope_data.get("dataMessage") or (envelope_data.get("editMessage") or {}).get("dataMessage")
)
data_message = (envelope_data.get("dataMessage")
or (envelope_data.get("editMessage") or {}).get("dataMessage"))
if not data_message:
return
group_info = data_message.get("groupInfo")
@@ -469,8 +445,7 @@ class SignalAdapter(BasePlatformAdapter):
return
chat_id = f"group:{group_id}" if is_group else sender
text = data_message.get("message", "")
mentions = data_message.get("mentions", [])
if text and mentions:
if text and (mentions := data_message.get("mentions", [])):
text = _render_mentions(text, mentions)
if is_group:
mentioned, text = self._apply_group_mention_rules(text, data_message)
@@ -485,19 +460,19 @@ class SignalAdapter(BasePlatformAdapter):
media_urls, media_types = [], []
if attachments_data and not getattr(self, "ignore_attachments", False):
media_urls, media_types = await self._collect_attachments(attachments_data)
# Skip contentless envelopes (profile key updates, empty messages) that still
# carry a dataMessage wrapper — otherwise msg='' triggers a full agent turn.
# Skip contentless envelopes (profile key updates, empty messages) that still carry a
# dataMessage wrapper — otherwise msg='' triggers a full agent turn.
if (not text or not text.strip()) and not media_urls:
logger.debug("Signal: skipping contentless envelope from %s (%d attachments)",
redact_phone(sender), len(media_urls) if media_urls else 0)
logger.debug("Signal: skipping contentless envelope from %s (%d attachments)", redact_phone(sender),
len(media_urls) if media_urls else 0)
return
source = self.build_source(
chat_id=chat_id, chat_name=group_info.get("groupName") if group_info else sender_name,
chat_type="group" if is_group else "dm", user_id=sender,
user_name=sender_name or sender, user_id_alt=sender_uuid if sender_uuid else None,
chat_id_alt=group_id if is_group else None)
# First matching MIME prefix wins; everything else (application/*, text/*, unknown)
# is a DOCUMENT so run.py's document-context injection surfaces the cached path.
# First matching MIME prefix wins; everything else (application/*, text/*, unknown) is a DOCUMENT
# so run.py's document-context injection surfaces the cached path.
msg_type = MessageType.TEXT if not media_types else next(
(mt for prefix, mt in _MEDIA_TYPE_BY_MIME_PREFIX if any(m.startswith(prefix) for m in media_types)),
MessageType.DOCUMENT)
@@ -532,17 +507,15 @@ class SignalAdapter(BasePlatformAdapter):
def _remember_recipient_identifiers(self, number: Optional[str], service_id: Optional[str]) -> None:
"""Cache any number↔UUID mapping observed from Signal envelopes."""
if not number or not service_id or not _is_signal_service_id(service_id):
return
self._recipient_uuid_by_number[number] = service_id
self._recipient_number_by_uuid[service_id] = number
if number and service_id and _is_signal_service_id(service_id):
self._recipient_uuid_by_number[number] = service_id
self._recipient_number_by_uuid[service_id] = number
@staticmethod
def _extract_quote_author(quote_data: Any) -> Optional[str]:
"""Return the best available Signal sender identifier from quote metadata."""
if not isinstance(quote_data, dict):
return None
return next((str(quote_data[k]) for k in _QUOTE_AUTHOR_KEYS if quote_data.get(k)), None)
keys = _QUOTE_AUTHOR_KEYS if isinstance(quote_data, dict) else ()
return next((str(quote_data[k]) for k in keys if quote_data.get(k)), None)
def _quote_references_own_message(self, reply_to_id: Optional[str], reply_to_author: Optional[str]) -> bool:
"""True when a Signal quote points at this adapter's outbound message."""
@@ -550,22 +523,17 @@ class SignalAdapter(BasePlatformAdapter):
return True
if not reply_to_author:
return False
author = str(reply_to_author).strip()
if self._account_normalized and author == self._account_normalized:
return True
cached_uuid = self._recipient_uuid_by_number.get(self._account_normalized)
if cached_uuid and author == cached_uuid:
return True
cached_number = self._recipient_number_by_uuid.get(author)
return bool(cached_number and cached_number == self._account_normalized)
author, acct = str(reply_to_author).strip(), self._account_normalized
# Cached number↔UUID mappings are only ever stored with truthy keys and values.
return bool(acct) and (author == acct or author == self._recipient_uuid_by_number.get(acct)
or self._recipient_number_by_uuid.get(author) == acct)
def _remember_sent_message_timestamp(self, timestamp: Any) -> None:
"""Keep a bounded cache of outbound Signal timestamps for quote matching."""
if timestamp is None:
return
key = str(timestamp)
# Re-insert to mark most-recently-used so eviction drops genuinely old entries.
self._sent_message_timestamps.pop(key, None)
self._sent_message_timestamps.pop(key, None) # re-insert as most-recently-used so eviction drops old ones
self._sent_message_timestamps[key] = None
while len(self._sent_message_timestamps) > self._max_sent_message_timestamps:
self._sent_message_timestamps.popitem(last=False)
@@ -578,8 +546,8 @@ class SignalAdapter(BasePlatformAdapter):
profile = contact.get("profile")
if not service_id and isinstance(profile, dict):
service_id = profile.get("serviceId") or profile.get("uuid")
if service_id and _is_signal_service_id(service_id) and (
contact.get("number") == phone_number or contact.get("recipient") == phone_number):
if service_id and _is_signal_service_id(service_id) and phone_number in (
contact.get("number"), contact.get("recipient")):
return service_id
return None
@@ -587,20 +555,17 @@ class SignalAdapter(BasePlatformAdapter):
"""Return the preferred Signal recipient identifier for a direct chat."""
if not chat_id or chat_id.startswith("group:") or not _looks_like_e164_number(chat_id):
return chat_id
cached = self._recipient_uuid_by_number.get(chat_id)
if cached:
if cached := self._recipient_uuid_by_number.get(chat_id):
return cached
async with self._recipient_cache_lock:
cached = self._recipient_uuid_by_number.get(chat_id)
if cached:
if cached := self._recipient_uuid_by_number.get(chat_id):
return cached
contacts = await self._rpc("listContacts", {"account": self.account, "allRecipients": True})
if isinstance(contacts, list):
for contact in contacts:
number = contact.get("number") if isinstance(contact, dict) else None
service_id = self._extract_contact_uuid(contact, chat_id)
if number and service_id:
self._remember_recipient_identifiers(number, service_id)
for contact in contacts if isinstance(contacts, list) else ():
number = contact.get("number") if isinstance(contact, dict) else None
service_id = self._extract_contact_uuid(contact, chat_id)
if number and service_id:
self._remember_recipient_identifiers(number, service_id)
return self._recipient_uuid_by_number.get(chat_id, chat_id)
async def _with_target(self, params: Dict[str, Any], chat_id: str, *, resolve: bool = True) -> Dict[str, Any]:
@@ -616,8 +581,7 @@ class SignalAdapter(BasePlatformAdapter):
result = await self._rpc("getAttachment", {"account": self.account, "id": attachment_id})
if not result:
return None, ""
# signal-cli returns {"data": "base64..."}
if isinstance(result, dict):
if isinstance(result, dict): # signal-cli returns {"data": "base64..."}
result = result.get("data")
if not result:
logger.warning("Signal: attachment response missing 'data' key")
@@ -632,18 +596,16 @@ class SignalAdapter(BasePlatformAdapter):
else cache_audio_from_bytes if _is_audio_ext(ext) else cache_document_from_bytes)
return cache(raw_data, ext), ext
async def _rpc(
self, method: str, params: dict, rpc_id: str = None, *,
log_failures: bool = True, raise_on_rate_limit: bool = False, timeout: float = 30.0) -> Any:
"""Send a JSON-RPC 2.0 request to signal-cli. ``log_failures=False`` logs failures at DEBUG
(typing path: silence NETWORK_FAILURE spam); ``raise_on_rate_limit=True`` raises
``SignalRateLimitError`` on a 429 / RateLimitException instead of swallowing it."""
async def _rpc(self, method: str, params: dict, rpc_id: str = None, *, log_failures: bool = True,
raise_on_rate_limit: bool = False, timeout: float = 30.0) -> Any:
"""Send a JSON-RPC 2.0 request to signal-cli. ``log_failures=False`` logs failures at DEBUG (typing
path: silence NETWORK_FAILURE spam); ``raise_on_rate_limit=True`` raises ``SignalRateLimitError``
on a 429 / RateLimitException instead of swallowing it."""
if not self.client:
logger.warning("Signal: RPC called but client not connected")
return None
if rpc_id is None:
rpc_id = f"{method}_{int(time.time() * 1000)}"
payload = {"jsonrpc": "2.0", "method": method, "params": params, "id": rpc_id}
payload = {"jsonrpc": "2.0", "method": method, "params": params,
"id": rpc_id if rpc_id is not None else f"{method}_{int(time.time() * 1000)}"}
fail_level = logging.WARNING if log_failures else logging.DEBUG
try:
resp = await self.client.post(f"{self.http_url}/api/v1/rpc", json=payload, timeout=timeout)
@@ -660,8 +622,8 @@ class SignalAdapter(BasePlatformAdapter):
if isinstance(result, dict) and raise_on_rate_limit:
for r in result.get("results") if isinstance(result.get("results"), list) else ():
if isinstance(r, dict) and r.get("type") == "RATE_LIMIT_FAILURE":
raise SignalRateLimitError(
"Rate limit exceeded for recipient", retry_after=r.get("retryAfterSeconds"))
raise SignalRateLimitError("Rate limit exceeded for recipient",
retry_after=r.get("retryAfterSeconds"))
return result
except SignalRateLimitError:
raise
@@ -669,10 +631,8 @@ class SignalAdapter(BasePlatformAdapter):
logger.log(fail_level, "Signal RPC %s failed: %s", method, e)
return None
@staticmethod
def _markdown_to_signal(text: str) -> tuple[str, list[str]]:
"""Backward-compatible wrapper around shared Signal formatting helper."""
return markdown_to_signal(text)
# Backward-compatible alias for the shared formatting helper.
_markdown_to_signal = staticmethod(markdown_to_signal)
def format_message(self, content: str) -> str:
"""Plain-text fallback for the base-class send path; send() applies rich styles itself."""
@@ -680,9 +640,7 @@ class SignalAdapter(BasePlatformAdapter):
def _validate_send_result(self, result: Any) -> tuple[bool, Optional[str]]:
"""Validate signal-cli send response results. Returns (success, error_message)."""
if not result or not isinstance(result, dict):
return True, None
results = result.get("results")
results = result.get("results") if isinstance(result, dict) else None
for r in results if isinstance(results, list) else ():
if not isinstance(r, dict):
continue
@@ -709,26 +667,22 @@ class SignalAdapter(BasePlatformAdapter):
except (TypeError, ValueError):
logger.debug("[Signal] Ignoring malformed textStyle range: %r", style_string)
continue
overlap_start = max(style_start, chunk_start)
overlap_end = min(style_end, chunk_end)
overlap_start, overlap_end = max(style_start, chunk_start), min(style_end, chunk_end)
if overlap_start < overlap_end:
adjusted.append(f"{overlap_start - chunk_start}:{overlap_end - overlap_start}:{style_type}")
return adjusted
@classmethod
def _split_signal_formatted_message(
cls, plain_text: str, text_styles: list[str], max_length: int,
) -> list[tuple[str, list[str]]]:
cls, plain_text: str, text_styles: list[str], max_length: int) -> list[tuple[str, list[str]]]:
"""Split converted Signal text into chunks, translating body ranges per chunk. Splitting after
conversion keeps styles that cross a chunk boundary intact instead of leaking Markdown markers."""
if utf16_len(plain_text) <= max_length:
return [(plain_text, text_styles)]
indicator_reserve = 10 # Mirrors BasePlatformAdapter.truncate_message().
body_limit = max(1, max_length - indicator_reserve)
body_limit = max(1, max_length - 10) # 10 = indicator reserve, mirrors truncate_message().
offsets = cls._utf16_offsets(plain_text)
chunks: list[tuple[str, list[str]]] = []
start_idx = 0
total_u16 = offsets[-1]
start_idx, total_u16 = 0, offsets[-1]
while offsets[start_idx] < total_u16:
end_budget = min(total_u16, offsets[start_idx] + body_limit)
end_idx = start_idx + 1
@@ -744,8 +698,7 @@ class SignalAdapter(BasePlatformAdapter):
async def _rpc_send(self, params: Dict[str, Any], fail_error: str) -> Tuple[Any, Optional[SendResult]]:
"""Run a ``send`` RPC, validate and track it; ``(result, None)`` or ``(None, failed SendResult)``."""
result = await self._rpc("send", params)
if result is None:
if (result := await self._rpc("send", params)) is None:
return None, SendResult(success=False, error=fail_error)
success, err_msg = self._validate_send_result(result)
if not success:
@@ -753,16 +706,14 @@ class SignalAdapter(BasePlatformAdapter):
self._track_sent_timestamp(result)
return result, None
async def send(
self, chat_id: str, content: str, reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None) -> SendResult:
async def send(self, chat_id: str, content: str, reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None) -> SendResult:
"""Send a text message with native Signal formatting."""
await self._stop_typing_indicator(chat_id)
if not content or not content.strip():
return SendResult(success=True, message_id=None)
base_params = await self._with_target({"account": self.account}, chat_id)
plain_message, message_styles = self._markdown_to_signal(content)
chunks = self._split_signal_formatted_message(plain_message, message_styles, self.MAX_MESSAGE_LENGTH)
chunks = self._split_signal_formatted_message(*self._markdown_to_signal(content), self.MAX_MESSAGE_LENGTH)
last_result = None
for idx, (plain_text, text_styles) in enumerate(chunks, start=1):
params: Dict[str, Any] = dict(base_params, message=plain_text)
@@ -770,8 +721,8 @@ class SignalAdapter(BasePlatformAdapter):
params["textStyle"] = text_styles[0]
elif text_styles:
params["textStyles"] = text_styles
logger.info("[Signal] Sending response chunk %d/%d (%d chars) to %s",
idx, len(chunks), len(plain_text), chat_id)
logger.info("[Signal] Sending response chunk %d/%d (%d chars) to %s", idx, len(chunks), len(plain_text),
chat_id)
last_result, err = await self._rpc_send(params, "RPC send failed")
if err:
return err
@@ -784,8 +735,7 @@ class SignalAdapter(BasePlatformAdapter):
if not ts:
return
self._remember_sent_message_timestamp(ts)
now = time.monotonic()
recent = self._recent_sent_timestamps
now, recent = time.monotonic(), self._recent_sent_timestamps
recent.pop(ts, None) # re-insert to mark as most-recently-used
recent[ts] = now
# Drop entries older than TTL first, then enforce the hard cap.
@@ -807,14 +757,12 @@ class SignalAdapter(BasePlatformAdapter):
return
params = await self._with_target({"account": self.account}, chat_id)
fails = self._typing_failures.get(chat_id, 0)
result = await self._rpc("sendTyping", params, rpc_id="typing", log_failures=(fails == 0))
if result is not None:
if await self._rpc("sendTyping", params, rpc_id="typing", log_failures=(fails == 0)) is not None:
self._typing_failures.pop(chat_id, None)
self._typing_skip_until.pop(chat_id, None)
return
fails = self._typing_failures[chat_id] = fails + 1
# After 3 consecutive failures back off exponentially (16s, 32s, 60s cap).
if fails >= 3:
if fails >= 3: # exponential backoff: 16s, 32s, 60s cap
self._typing_skip_until[chat_id] = now + min(60.0, 16.0 * (2 ** (fails - 3)))
async def _resolve_image_path(self, image_url: str) -> Tuple[Optional[str], Optional[str], Any]:
@@ -828,21 +776,19 @@ class SignalAdapter(BasePlatformAdapter):
return None, "download", e
if not file_path or not Path(file_path).exists():
return None, "missing", None
file_size = Path(file_path).stat().st_size
if file_size > SIGNAL_MAX_ATTACHMENT_SIZE:
if (file_size := Path(file_path).stat().st_size) > SIGNAL_MAX_ATTACHMENT_SIZE:
return None, "oversize", file_size
return file_path, None, None
async def send_multiple_images(
self, chat_id: str, images: List[Tuple[str, str]], metadata: Optional[Dict[str, Any]] = None,
human_delay: float = 0.0) -> None:
async def send_multiple_images(self, chat_id: str, images: List[Tuple[str, str]],
metadata: Optional[Dict[str, Any]] = None, human_delay: float = 0.0) -> None:
"""Send a batch of images via chunked Signal RPC calls. Alt texts are dropped (one shared body
per send); bad images are skipped with a warning; ``human_delay`` is ignored (scheduler paces)."""
if not images:
return
scheduler = get_scheduler()
logger.info("Signal send_multiple_images: received %d image(s) for %s — scheduler state: %s",
len(images), chat_id[:30], scheduler.state())
logger.info("Signal send_multiple_images: received %d image(s) for %s — scheduler state: %s", len(images),
chat_id[:30], scheduler.state())
await self._stop_typing_indicator(chat_id)
attachments: List[str] = []
skipped = {"download": 0, "missing": 0, "oversize": 0}
@@ -852,18 +798,12 @@ class SignalAdapter(BasePlatformAdapter):
attachments.append(file_path)
continue
skipped[reason] += 1
if reason == "download":
logger.warning("Signal: failed to download image %s: %s", image_url, detail)
elif reason == "missing":
logger.warning("Signal: image file not found for %s", image_url)
else:
logger.warning("Signal: image too large (%d bytes), skipping %s", detail, image_url)
logger.warning(*_SKIP_IMAGE_LOG[reason](image_url, detail))
if not attachments:
logger.error("Signal: no valid images in batch of %d (download=%d missing=%d oversize=%d)",
len(images), skipped["download"], skipped["missing"], skipped["oversize"])
logger.error("Signal: no valid images in batch of %d (download=%d missing=%d oversize=%d)", len(images),
skipped["download"], skipped["missing"], skipped["oversize"])
return
logger.info("Signal send_multiple_images: %d/%d images valid, sending in chunks",
len(attachments), len(images))
logger.info("Signal send_multiple_images: %d/%d images valid, sending in chunks", len(attachments), len(images))
base_params = await self._with_target({"account": self.account, "message": ""}, chat_id)
per = SIGNAL_MAX_ATTACHMENTS_PER_MSG
att_batches = [attachments[i:i + per] for i in range(0, len(attachments), per)]
@@ -874,14 +814,13 @@ class SignalAdapter(BasePlatformAdapter):
logger.debug("Signal batch %d/%d: %d attachments, estimated wait=%.1fs", idx, n_batches, n, estimated)
if estimated >= SIGNAL_BATCH_PACING_NOTICE_THRESHOLD:
await self._notify_batch_pacing(chat_id, idx, n_batches, estimated)
await self._send_attachment_batch(
scheduler, dict(base_params, attachments=att_batch), n, f"{idx}/{n_batches}")
await self._send_attachment_batch(scheduler, dict(base_params, attachments=att_batch), n,
f"{idx}/{n_batches}")
async def _send_attachment_batch(self, scheduler, params: Dict[str, Any], n: int, label: str) -> None:
"""Send one attachment batch with rate-limit pacing and a single transient retry. Tokens are
deducted only on validated success (None = server never accepted it); 429s feed the scheduler."""
send_timeout = _signal_send_timeout(n)
max_attempts = SIGNAL_RATE_LIMIT_MAX_ATTEMPTS
send_timeout, max_attempts = _signal_send_timeout(n), SIGNAL_RATE_LIMIT_MAX_ATTEMPTS
for attempt in range(1, max_attempts + 1):
await scheduler.acquire(n)
t0 = time.monotonic()
@@ -902,22 +841,21 @@ class SignalAdapter(BasePlatformAdapter):
if success:
self._track_sent_timestamp(result)
await scheduler.report_rpc_duration(duration, n)
logger.info("Signal batch %s: %d attachments sent in %.1fs (attempt %d/%d)",
label, n, duration, attempt, max_attempts)
logger.info("Signal batch %s: %d attachments sent in %.1fs (attempt %d/%d)", label, n, duration,
attempt, max_attempts)
return
logger.error("Signal: RPC send failed for batch %s (%d attachments, attempt %d/%d, rpc_duration=%.1fs)%s",
label, n, attempt, max_attempts, duration, f": {err_msg}" if result is not None else "")
if attempt >= max_attempts:
return
backoff = 2.0 ** attempt
logger.info("Signal: retrying batch %s after %.1fs backoff", label, backoff)
await asyncio.sleep(backoff)
logger.info("Signal: retrying batch %s after %.1fs backoff", label, 2.0 ** attempt)
await asyncio.sleep(2.0 ** attempt)
async def _notify_batch_pacing(self, chat_id: str, next_batch_idx: int, total_batches: int, wait_s: float) -> None:
"""Tell the user about an inter-batch pacing wait over the notice threshold (best-effort)."""
try:
await self.send(chat_id, f"(More images coming — pausing ~{_format_wait(wait_s)} "
f"for Signal rate limit, batch {next_batch_idx}/{total_batches}.)")
await self.send(chat_id, f"(More images coming — pausing ~{_format_wait(wait_s)} for Signal rate limit, "
f"batch {next_batch_idx}/{total_batches}.)")
except Exception as e:
logger.warning("Signal: failed to send pacing notice: %s", e)
@@ -928,21 +866,20 @@ class SignalAdapter(BasePlatformAdapter):
if reason == "download":
logger.warning("Signal: failed to download image: %s", detail)
if reason:
error = {"download": str(detail), "missing": "Image file not found",
"oversize": f"Image too large ({detail} bytes)"}[reason]
return SendResult(success=False, error=error)
return SendResult(success=False, error={
"download": str(detail), "missing": "Image file not found",
"oversize": f"Image too large ({detail} bytes)"}[reason])
return await self._send_file(chat_id, file_path, caption, "RPC send with attachment failed")
async def _send_file(self, chat_id: str, file_path: str, caption: Optional[str], fail_error: str) -> SendResult:
"""Send one local file as a Signal attachment via the ``send`` RPC."""
params = {"account": self.account, "message": caption or "", "attachments": [file_path]}
await self._with_target(params, chat_id)
params = await self._with_target(
{"account": self.account, "message": caption or "", "attachments": [file_path]}, chat_id)
_, err = await self._rpc_send(params, fail_error)
return err or SendResult(success=True)
async def _send_attachment(
self, chat_id: str, file_path: str, media_label: str, caption: Optional[str] = None,
) -> SendResult:
async def _send_attachment(self, chat_id: str, file_path: str, media_label: str,
caption: Optional[str] = None) -> SendResult:
"""Send any local file as a Signal attachment (shared by send_document/image_file/voice/video)."""
await self._stop_typing_indicator(chat_id)
try:
@@ -1000,47 +937,37 @@ class SignalAdapter(BasePlatformAdapter):
async def remove_reaction(self, chat_id: str, target_author: str, target_timestamp: int) -> bool:
"""Remove a reaction by sending an empty-string emoji."""
return await self._send_reaction_rpc(chat_id, {
"account": self.account, "emoji": "", "targetAuthor": target_author,
"targetTimestamp": target_timestamp, "remove": True})
"account": self.account, "emoji": "", "targetAuthor": target_author, "targetTimestamp": target_timestamp,
"remove": True})
def _extract_reaction_target(self, event: MessageEvent) -> Optional[tuple]:
"""Extract (target_author, target_timestamp) from a MessageEvent, or None."""
raw = event.raw_message
if isinstance(raw, dict) and raw.get("sender") and raw.get("timestamp_ms"):
return (raw["sender"], raw["timestamp_ms"])
return None
ok = isinstance(raw, dict) and raw.get("sender") and raw.get("timestamp_ms")
return (raw["sender"], raw["timestamp_ms"]) if ok else None
def _reactions_enabled(self, event: "MessageEvent" = None) -> bool:
"""SIGNAL_REACTIONS env gate, then the DM allowlist: reactions fire before run.py's auth gate,
so an unauthorized contact's 👀 would otherwise reveal a listening bot."""
if os.getenv("SIGNAL_REACTIONS", "true").lower() in {"false", "0", "no"}:
return False
if event is not None:
sender = getattr(getattr(event, "source", None), "user_id", None)
if sender and "*" not in self.dm_allow_from and sender not in self.dm_allow_from:
return False
return True
sender = getattr(getattr(event, "source", None), "user_id", None) if event is not None else None
return not (sender and "*" not in self.dm_allow_from and sender not in self.dm_allow_from)
async def on_processing_start(self, event: MessageEvent) -> None:
"""React with 👀 when processing begins."""
if not self._reactions_enabled(event):
return
target = self._extract_reaction_target(event)
if target:
if self._reactions_enabled(event) and (target := self._extract_reaction_target(event)):
await self.send_reaction(event.source.chat_id, "👀", *target)
async def on_processing_complete(self, event: MessageEvent, outcome: "ProcessingOutcome") -> None:
"""Swap 👀 for ✅/❌; on CANCELLED the 👀 stays to keep reflecting "in progress" (matches Telegram)."""
if outcome == ProcessingOutcome.CANCELLED or not self._reactions_enabled(event):
return
target = self._extract_reaction_target(event)
if not target:
if not (target := self._extract_reaction_target(event)):
return
chat_id = event.source.chat_id
await self.remove_reaction(chat_id, *target)
emoji = _OUTCOME_REACTION.get(outcome)
if emoji:
await self.send_reaction(chat_id, emoji, *target)
await self.remove_reaction(event.source.chat_id, *target)
if emoji := _OUTCOME_REACTION.get(outcome):
await self.send_reaction(event.source.chat_id, emoji, *target)
async def get_chat_info(self, chat_id: str) -> Dict[str, Any]:
"""Get information about a chat/contact."""
gateway/platforms/signal_format.py
+18 -34
View File
@@ -1,8 +1,5 @@
"""Shared Signal formatting helpers.
Markdown → Signal native formatting lives here so both the live adapter and the
standalone send paths emit the same bodyRanges.
"""
"""Shared Signal formatting helpers: Markdown → Signal native formatting lives here so both the live
adapter and the standalone send paths emit the same bodyRanges."""
from __future__ import annotations
@@ -25,32 +22,24 @@ def _utf16_len(s: str) -> int:
def _normalize_bullet_markers(source: str) -> str:
"""Replace Markdown bullet markers with plain Unicode bullets.
Signal renders ``- item`` / ``* item`` literally. Fenced code blocks are kept
byte-for-byte; list-looking lines inside code are code, not prose bullets."""
"""Replace Markdown bullet markers with plain Unicode bullets (Signal renders ``- item`` literally).
Fenced code blocks are kept byte-for-byte: list-looking lines inside code are code, not bullets."""
parts = re.split(r"(```.*?```)", source, flags=re.DOTALL)
for idx, part in enumerate(parts):
if idx % 2 == 0:
parts[idx] = re.sub(r"(?m)^([ \t]{0,3})[-*+]\s+", r"\1• ", part)
return "".join(parts)
return "".join(re.sub(r"(?m)^([ \t]{0,3})[-*+]\s+", r"\1• ", part) if idx % 2 == 0 else part
for idx, part in enumerate(parts))
def markdown_to_signal(text: str) -> tuple[str, list[str]]:
"""Convert markdown to plain text + Signal textStyles list.
Signal uses ``bodyRanges`` (signal-cli ``textStyle`` / ``textStyles`` params) in
the form ``start:length:STYLE``, positions in UTF-16 code units.
"""Convert markdown to plain text + Signal textStyles list. Signal uses ``bodyRanges`` (signal-cli
``textStyle`` / ``textStyles`` params) as ``start:length:STYLE`` with positions in UTF-16 code units.
Supported styles: BOLD, ITALIC, STRIKETHROUGH, MONOSPACE."""
text = re.sub(r"\n{3,}", "\n\n", text).strip()
text = _normalize_bullet_markers(text)
text = _normalize_bullet_markers(re.sub(r"\n{3,}", "\n\n", text).strip())
styles: list[tuple[int, int, str]] = []
while match := _CODE_BLOCK_RE.search(text):
inner = match.group(1).rstrip("\n")
styles.append((match.start(), len(inner), "MONOSPACE"))
text = text[: match.start()] + inner + text[match.end() :]
new_text = ""
last_end = 0
new_text, last_end = "", 0
for match in _HEADING_RE.finditer(text):
new_text += text[last_end : match.start()]
eol = text.find("\n", match.end())
@@ -71,10 +60,9 @@ def markdown_to_signal(text: str) -> tuple[str, list[str]]:
all_matches.append((ms, me, match.start(1), match.end(1), style))
occupied.append((ms, me))
all_matches.sort()
# Strip the markers, recording (pos, len) removals so earlier block/heading
# ranges can be shifted, and capturing inline ranges in the stripped text.
result = ""
last_end = 0
# Strip the markers, recording (pos, len) removals so earlier block/heading ranges can be
# shifted, and capturing inline ranges in the stripped text.
result, last_end = "", 0
removals: list[tuple[int, int]] = []
inline_styles: list[tuple[int, int, str]] = []
for ms, me, g1s, g1e, style in all_matches:
@@ -97,16 +85,12 @@ def markdown_to_signal(text: str) -> tuple[str, list[str]]:
shift += min(remove_len, pos - remove_pos)
return pos - shift
adjusted_prior = [
(_adjust(start), _adjust(start + length) - _adjust(start), style)
for start, length, style in styles
if _adjust(start + length) > _adjust(start)]
adjusted_prior = [(_adjust(start), _adjust(start + length) - _adjust(start), style)
for start, length, style in styles if _adjust(start + length) > _adjust(start)]
text = result + text[last_end:]
style_strings: list[str] = []
for cp_start, cp_len, style_type in sorted(adjusted_prior + inline_styles):
if cp_start < 0 or cp_start + cp_len > len(text):
continue
u16_start = _utf16_len(text[:cp_start])
u16_len = _utf16_len(text[cp_start : cp_start + cp_len])
style_strings.append(f"{u16_start}:{u16_len}:{style_type}")
if 0 <= cp_start and cp_start + cp_len <= len(text):
u16_start, u16_len = _utf16_len(text[:cp_start]), _utf16_len(text[cp_start : cp_start + cp_len])
style_strings.append(f"{u16_start}:{u16_len}:{style_type}")
return text, style_strings
gateway/platforms/signal_rate_limit.py
+60 -94
View File
@@ -1,12 +1,8 @@
"""Signal attachment rate-limit scheduler.
Process-wide token-bucket simulator mirroring the per-account attachment rate limit
signal-cli/Signal-Server enforce. Producers (``SignalAdapter.send_multiple_images`` and
the ``send_message`` tool's Signal path) call ``acquire(n)`` before an attachment send;
on a 429 they call ``feedback(retry_after, n)`` so the model recalibrates from the
server's authoritative hint. Concurrent calls serialize through an ``asyncio.Lock``
— FIFO fairness across sessions sharing one daemon.
"""
"""Signal attachment rate-limit scheduler: process-wide token-bucket simulator mirroring the per-account
attachment rate limit signal-cli/Signal-Server enforce. Producers (``SignalAdapter.send_multiple_images``
and the ``send_message`` tool's Signal path) call ``acquire(n)`` before an attachment send; on a 429 they
call ``feedback(retry_after, n)`` so the model recalibrates from the server's authoritative hint.
Concurrent calls serialize through an ``asyncio.Lock`` — FIFO fairness across sessions sharing one daemon."""
from __future__ import annotations
@@ -20,11 +16,11 @@ from agent.retry_utils import parse_retry_after_seconds
logger = logging.getLogger(__name__)
SIGNAL_MAX_ATTACHMENTS_PER_MSG = 32 # per-message attachment cap (source: Signal-{Android,Desktop} source code)
SIGNAL_RATE_LIMIT_BUCKET_CAPACITY = 50 # server-side token-bucket capacity for attachments rate limiting
SIGNAL_MAX_ATTACHMENTS_PER_MSG = 32 # per-message attachment cap (Signal-{Android,Desktop} source)
SIGNAL_RATE_LIMIT_BUCKET_CAPACITY = 50 # server-side token-bucket capacity for attachments
SIGNAL_RATE_LIMIT_DEFAULT_RETRY_AFTER = 4 # fallback token refill interval for signal-cli < v0.14.3
SIGNAL_RATE_LIMIT_MAX_ATTEMPTS = 2 # initial attempt + 1 retry
SIGNAL_BATCH_PACING_NOTICE_THRESHOLD = 10.0 # if estimated waiting time > 10s, notify the user about the delay
SIGNAL_BATCH_PACING_NOTICE_THRESHOLD = 10.0 # estimated wait above this → notify the user
SIGNAL_RPC_ERROR_RATELIMIT = -5 # signal-cli (v0.14.3+) JSON-RPC error code for RateLimitException
@@ -41,9 +37,8 @@ class SignalSchedulerError(Exception):
pass
# "Retry after 4 seconds" — libsignal-net's RetryLaterException string form, surfaced
# when 429s hit during attachment upload (signal-cli wraps these as
# AttachmentInvalidException rather than RateLimitException, so the typed path doesn't fire).
# "Retry after 4 seconds" — libsignal-net's RetryLaterException string form, surfaced when 429s hit during
# attachment upload (signal-cli wraps these as AttachmentInvalidException, so the typed path doesn't fire).
_RETRY_AFTER_RE = re.compile(r"Retry after (\d+(?:\.\d+)?)\s*second", re.IGNORECASE)
@@ -52,37 +47,28 @@ def _error_message(err: Any) -> str:
def _extract_retry_after_seconds(err: Any) -> Optional[float]:
"""Per-token Retry-After from a signal-cli rate-limit error, or None.
Sources, in order: ``error.data.response.results[*].retryAfterSeconds`` (signal-cli
≥ v0.14.3), then "Retry after N seconds" parsed from the message (libsignal-net
RetryLaterException wrapped as AttachmentInvalidException, structured field null)."""
"""Per-token Retry-After from a signal-cli rate-limit error, or None. Sources, in order:
``error.data.response.results[*].retryAfterSeconds`` (signal-cli ≥ v0.14.3), then "Retry after N
seconds" parsed from the message (RetryLaterException wrapped as AttachmentInvalidException)."""
if isinstance(err, dict):
results = ((err.get("data") or {}).get("response") or {}).get("results") or []
candidates = [
parse_retry_after_seconds(r.get("retryAfterSeconds")) for r in results
if isinstance(r, dict) and r.get("retryAfterSeconds")]
candidates = [c for c in candidates if c is not None]
if candidates:
candidates = [parse_retry_after_seconds(r.get("retryAfterSeconds")) for r in results
if isinstance(r, dict) and r.get("retryAfterSeconds")]
if candidates := [c for c in candidates if c is not None]:
return max(candidates)
match = _RETRY_AFTER_RE.search(_error_message(err))
return parse_retry_after_seconds(match.group(1)) if match else None
def _is_signal_rate_limit_error(err: Any) -> bool:
"""True if a signal-cli RPC error reflects a rate-limit failure: the typed
``RATELIMIT_ERROR`` code (≥ v0.14.3), legacy ``[429]`` / ``RateLimitException``
substrings, or libsignal-net's ``RetryLaterException`` / "Retry after N seconds"
leaked through AttachmentInvalidException during upload (substring is the only signal)."""
"""True if a signal-cli RPC error reflects a rate-limit failure: the typed ``RATELIMIT_ERROR`` code
(≥ v0.14.3), legacy ``[429]`` / ``RateLimitException`` substrings, or libsignal-net's
``RetryLaterException`` / "Retry after N seconds" leaked through AttachmentInvalidException."""
if isinstance(err, dict) and err.get("code") == SIGNAL_RPC_ERROR_RATELIMIT:
return True
message = _error_message(err)
msg_lower = message.lower()
return (
"[429]" in message
or "ratelimit" in msg_lower
or "retrylaterexception" in msg_lower
or "retry after" in msg_lower)
return "[429]" in message or any(s in msg_lower for s in ("ratelimit", "retrylaterexception", "retry after"))
def _format_wait(seconds: float) -> str:
@@ -92,23 +78,19 @@ def _format_wait(seconds: float) -> str:
def _signal_send_timeout(num_attachments: int) -> float:
"""HTTP timeout for a Signal ``send`` RPC: signal-cli uploads attachments serially inside
the call, so the default 30s truncates large batches mid-upload and logs a phantom
failure even though the send completes. Scale at 5s/attachment with a 60s floor."""
"""HTTP timeout for a Signal ``send`` RPC: signal-cli uploads attachments serially inside the call, so
the default 30s truncates large batches mid-upload (phantom failure). 5s/attachment, 60s floor."""
return 30.0 if num_attachments <= 0 else max(60.0, 5.0 * num_attachments)
class SignalAttachmentScheduler:
"""Process-wide token-bucket simulator for Signal attachment sends.
"""Process-wide token-bucket simulator for Signal attachment sends: up to ``capacity`` tokens (default
50 = Signal's server bucket), one per attachment, refilling at ``refill_rate``/s (calibrated from the
server's per-token Retry-After once a 429 has been observed; default 1 token / 4s). ``acquire(n)``
calls serialize through an ``asyncio.Lock`` — FIFO across sessions."""
Holds up to ``capacity`` tokens (default 50 = Signal's server bucket); each
attachment consumes one. Tokens refill at ``refill_rate``/s, calibrated from the
server's per-token Retry-After once a 429 has been observed (default 1 token / 4s).
``acquire(n)`` calls serialize through an ``asyncio.Lock`` — FIFO across sessions."""
def __init__(
self, capacity: float = float(SIGNAL_RATE_LIMIT_BUCKET_CAPACITY),
default_retry_after: float = float(SIGNAL_RATE_LIMIT_DEFAULT_RETRY_AFTER)) -> None:
def __init__(self, capacity: float = float(SIGNAL_RATE_LIMIT_BUCKET_CAPACITY),
default_retry_after: float = float(SIGNAL_RATE_LIMIT_DEFAULT_RETRY_AFTER)) -> None:
self.capacity = float(capacity)
self.tokens = float(capacity)
self.refill_rate = 1.0 / float(default_retry_after)
@@ -118,10 +100,9 @@ class SignalAttachmentScheduler:
def _projected_tokens(self, now: Optional[float] = None) -> float:
"""Tokens the bucket would hold at ``now``, without mutating state."""
elapsed = (time.monotonic() if now is None else now) - self.last_refill
projected = self.tokens
if elapsed > 0 and projected < self.capacity:
projected = min(self.capacity, projected + elapsed * self.refill_rate)
return projected
if elapsed > 0 and self.tokens < self.capacity:
return min(self.capacity, self.tokens + elapsed * self.refill_rate)
return self.tokens
def _refill(self) -> None:
now = time.monotonic()
@@ -129,27 +110,22 @@ class SignalAttachmentScheduler:
self.last_refill = now
def estimate_wait(self, n: int) -> float:
"""Seconds until ``n`` tokens would be available (lock-free, informational —
decides whether to emit a pacing notice *before* a possibly-blocking ``acquire``;
races vs. concurrent acquires are benign)."""
"""Seconds until ``n`` tokens would be available (lock-free, informational — decides whether to emit
a pacing notice *before* a possibly-blocking ``acquire``; races vs. concurrent acquires are benign)."""
deficit = n - self._projected_tokens()
return 0.0 if deficit <= 0 else deficit / self.refill_rate
async def acquire(self, n: int) -> float:
"""Block until at least ``n`` tokens are available; return the seconds slept.
Does **not** deduct tokens — the bucket is a read-only model of server-side
capacity; call ``report_rpc_duration()`` after the RPC to sync. The lock is
released during ``asyncio.sleep`` so other callers interleave, and the loop
re-checks after each sleep in case the deadline was pessimistic. Signal's
server is ground truth and will 429 (→ requeue) if the model drifts."""
"""Block until at least ``n`` tokens are available; return the seconds slept. Does **not** deduct
tokens — the bucket is a read-only model of server-side capacity; call ``report_rpc_duration()``
after the RPC to sync. The lock is released during ``asyncio.sleep`` so other callers interleave,
and the loop re-checks after each sleep in case the deadline was pessimistic. Signal's server is
ground truth and will 429 (→ requeue) if the model drifts."""
if n <= 0:
return 0.0
if n > self.capacity:
raise SignalSchedulerError(
f"Signal scheduler was called requesting {n} tokens (max is {self.capacity})")
total_slept = 0.0
first_pass = True
raise SignalSchedulerError(f"Signal scheduler was called requesting {n} tokens (max is {self.capacity})")
total_slept, first_pass = 0.0, True
while True:
async with self._lock:
self._refill()
@@ -161,53 +137,43 @@ class SignalAttachmentScheduler:
deficit = n - self.tokens
wait = deficit / self.refill_rate
if first_pass:
logger.info(
"Signal scheduler: pausing %.1fs for %d tokens "
"(available=%.1f, deficit=%.1f, refill=%.4f/s ≈ %.1fs/token)",
wait, n, self.tokens, deficit, self.refill_rate, 1.0 / self.refill_rate)
logger.info("Signal scheduler: pausing %.1fs for %d tokens (available=%.1f, deficit=%.1f, "
"refill=%.4f/s ≈ %.1fs/token)", wait, n, self.tokens, deficit, self.refill_rate,
1.0 / self.refill_rate)
first_pass = False
await asyncio.sleep(wait)
total_slept += wait
async def report_rpc_duration(self, rpc_duration: float, n_attachments: int) -> None:
"""Deduct ``n_attachments`` tokens for a completed send RPC.
No refill is credited for the upload window: Signal's server checks the bucket
at RPC start and resumes refill only after the response, so crediting it causes
cumulative drift that eventually triggers 429s. Advances ``last_refill``."""
"""Deduct ``n_attachments`` tokens for a completed send RPC. No refill is credited for the upload
window: Signal's server checks the bucket at RPC start and resumes refill only after the response,
so crediting it causes cumulative drift that eventually triggers 429s. Advances ``last_refill``."""
if n_attachments <= 0:
return
async with self._lock:
now = time.monotonic()
token_before = self.tokens
self.tokens = max(0.0, token_before - float(n_attachments))
self.last_refill = now
logger.log(
logging.INFO if rpc_duration > 10 and n_attachments > 5 else logging.DEBUG,
"Signal scheduler: RPC for %d att took %.1fs — "
"tokens %.1f → %.1f (deducted=%d, no upload refill credited, refill=%.4fs⁻¹)",
n_attachments, rpc_duration, token_before, self.tokens, n_attachments, self.refill_rate)
self.last_refill = time.monotonic()
logger.log(logging.INFO if rpc_duration > 10 and n_attachments > 5 else logging.DEBUG,
"Signal scheduler: RPC for %d att took %.1fs — tokens %.1f → %.1f (deducted=%d, no upload refill "
"credited, refill=%.4fs⁻¹)", n_attachments, rpc_duration, token_before, self.tokens, n_attachments,
self.refill_rate)
def feedback(self, retry_after: Optional[float], n_attempted: int) -> None:
"""Apply server feedback after a 429: empty the bucket and, when ``retry_after``
(per-token refill window) is present, calibrate ``refill_rate`` from it."""
if retry_after and retry_after > 0:
new_rate = 1.0 / float(retry_after)
if new_rate != self.refill_rate:
logger.info("Signal scheduler: calibrating refill_rate to %.4f tokens/sec "
"(server retry_after=%.1fs per token)", new_rate, retry_after)
self.refill_rate = new_rate
"""Apply server feedback after a 429: empty the bucket and, when ``retry_after`` (per-token refill
window) is present, calibrate ``refill_rate`` from it."""
if retry_after and retry_after > 0 and (new_rate := 1.0 / float(retry_after)) != self.refill_rate:
logger.info("Signal scheduler: calibrating refill_rate to %.4f tokens/sec (server retry_after=%.1fs "
"per token)", new_rate, retry_after)
self.refill_rate = new_rate
self.tokens = 0.0
self.last_refill = time.monotonic()
def state(self) -> dict:
"""Current scheduler state for diagnostic logging (read-only, doesn't advance ``last_refill``)."""
return {
"tokens": round(self._projected_tokens(), 1),
"capacity": int(self.capacity),
"refill_rate": round(self.refill_rate, 4),
"refill_seconds_per_token": round(1.0 / self.refill_rate, 1) if self.refill_rate > 0 else float("inf"),
}
return {"tokens": round(self._projected_tokens(), 1), "capacity": int(self.capacity),
"refill_rate": round(self.refill_rate, 4),
"refill_seconds_per_token": round(1.0 / self.refill_rate, 1) if self.refill_rate > 0 else float("inf")}
_scheduler: Optional[SignalAttachmentScheduler] = None
gateway/platforms/webhook.py
+161 -233
View File
@@ -1,10 +1,10 @@
"""Generic webhook platform adapter: aiohttp server that validates HMAC-signed POSTs (GitHub,
GitLab, Svix, Linear, generic), renders payloads into agent prompts, and routes responses back
(github_comment or any gateway platform). Routes live under platforms.webhook.extra.routes:
events (header filter), secret (REQUIRED; "INSECURE_NO_AUTH" skips validation, loopback only),
prompt template, skills, deliver/deliver_extra, deliver_only (rendered prompt IS the message).
Per-route rate limiting, idempotency cache, body-size caps checked before reading. Generic HMAC
V2 binds a timestamp for replay protection; body-only V1 is deprecated but accepted with a warning."""
"""Generic webhook platform adapter: aiohttp server that validates HMAC-signed POSTs (GitHub, GitLab,
Svix, Linear, generic), renders payloads into agent prompts, and routes responses back (github_comment
or any gateway platform). Routes live under platforms.webhook.extra.routes: events (header filter),
secret (REQUIRED; "INSECURE_NO_AUTH" skips validation, loopback only), prompt template, skills,
deliver/deliver_extra, deliver_only (rendered prompt IS the message). Per-route rate limiting,
idempotency cache, body-size caps checked before reading. Generic HMAC V2 binds a timestamp for
replay protection; body-only V1 is deprecated but accepted with a warning."""
import asyncio
import base64
@@ -36,8 +36,8 @@ from gateway.response_filters import is_autonomous_silence_response
logger = logging.getLogger(__name__)
# Sentinel from _resolve_request_profile: a /p/<profile>/ prefix names a profile
# this gateway does not serve (→ 404). Distinct from None (no prefix / default).
# _resolve_request_profile sentinel: /p/<profile>/ names a profile this gateway does not serve (→ 404);
# distinct from None (no prefix / default).
_PROFILE_REJECTED = object()
_UNPARSEABLE = object()
@@ -46,10 +46,9 @@ _BUILTIN_DELIVER_PLATFORMS = {
"homeassistant", "email", "dingtalk", "feishu", "wecom", "wecom_callback", "weixin",
"bluebubbles", "qqbot", "yuanbao"}
# ``None`` → aiohttp binds BOTH address families. "0.0.0.0" is IPv4-only and was
# unreachable on IPv6-only networks (Fly.io 6PN); "::" becomes IPv6-only where the
# kernel sets IPV6_V6ONLY=1, breaking the 127.0.0.1 health check. Users can pin a
# host via ``platforms.webhook.extra.host``.
# ``None`` → aiohttp binds BOTH address families. "0.0.0.0" is IPv4-only (unreachable on IPv6-only
# networks such as Fly.io 6PN); "::" becomes IPv6-only where the kernel sets IPV6_V6ONLY=1, breaking
# the 127.0.0.1 health check. Users can pin a host via ``platforms.webhook.extra.host``.
DEFAULT_HOST = None
DEFAULT_PORT = 8644
_INSECURE_NO_AUTH = "INSECURE_NO_AUTH"
@@ -77,12 +76,17 @@ def _hex_hmac(secret: str, data: bytes) -> str:
return hmac.new(secret.encode(), data, hashlib.sha256).hexdigest()
def _timestamp_age(raw: str) -> Optional[int]:
"""Seconds between now and an integer timestamp header; None when unparseable."""
def _timestamp_fresh(raw: str, stale_msg: str, *args) -> bool:
"""True when integer timestamp header *raw* is within the replay window; unparseable → False,
stale → warn ``stale_msg % args`` and False."""
try:
return abs(int(time.time()) - int(raw))
age = abs(int(time.time()) - int(raw))
except (TypeError, ValueError):
return None
return False
if age > _V2_REPLAY_WINDOW_SECONDS:
logger.warning(stale_msg, *args)
return False
return True
def _is_known_platform(name: str) -> bool:
@@ -100,10 +104,8 @@ def _json_error(message: str, status: int) -> "web.Response":
def _peek_session_id(store, session_key: str):
"""Prefer the store's lock-held accessor; fall back to the private path only for
older stores / test doubles that predate it."""
peek = getattr(store, "peek_session_id", None)
if callable(peek):
"""Prefer the store's lock-held accessor; the private-path fallback is for older stores / test doubles."""
if callable(peek := getattr(store, "peek_session_id", None)):
return peek(session_key)
if hasattr(store, "_ensure_loaded"):
with suppress(Exception):
@@ -121,11 +123,7 @@ def _validate_svix_signature(body: bytes, secret: str, msg_id: str, timestamp: s
"""Svix-compatible signatures (AgentMail): base64 HMAC-SHA256 of "{id}.{timestamp}.{body}"."""
if not (msg_id and timestamp and signature_header and secret):
return False
age = _timestamp_age(timestamp)
if age is None:
return False
if age > _V2_REPLAY_WINDOW_SECONDS:
logger.warning("[webhook] Svix signature timestamp outside replay window")
if not _timestamp_fresh(timestamp, "[webhook] Svix signature timestamp outside replay window"):
return False
if secret.startswith("whsec_"):
try:
@@ -150,8 +148,7 @@ def _validate_svix_signature(body: bytes, secret: str, msg_id: str, timestamp: s
class WebhookAdapter(BasePlatformAdapter):
"""Generic webhook receiver that triggers agent runs from HTTP POSTs."""
# Event-triggered, no human present: the startup auto-resume turn must FINISH
# the interrupted work instead of asking "what next?".
# Event-triggered, no human present: startup auto-resume must FINISH the interrupted work, not ask "what next?".
interactive_resume: bool = False
def __init__(self, config: PlatformConfig):
@@ -166,21 +163,18 @@ class WebhookAdapter(BasePlatformAdapter):
self._dynamic_routes_mtime: float = 0.0
self._routes: Dict[str, dict] = dict(self._static_routes)
self._runner = None
# Routes already warned about legacy V1 body-only signatures (once per route).
self._v1_signature_warned: set[str] = set()
# Keyed by session chat_id; read by EVERY send() (interim status messages AND
# the final response) so never pop on send(). TTL-pruned on each POST.
self._v1_signature_warned: set[str] = set() # routes already warned about legacy V1 (once per route)
# Keyed by session chat_id; read by EVERY send() (interim status messages AND the final
# response) so never pop on send(). TTL-pruned on each POST.
self._delivery_info: Dict[str, dict] = {}
self._delivery_info_created: Dict[str, float] = {}
self._delivery_info_order: Deque[tuple[float, str]] = deque()
# Gateway runner for cross-platform delivery (set externally).
self.gateway_runner = None
self.gateway_runner = None # set externally; needed for cross-platform delivery
# Idempotency: TTL cache of recently processed delivery IDs.
self._seen_deliveries: Dict[str, float] = {}
self._idempotency_ttl: int = 3600 # 1 hour
self._seen_deliveries_next_prune_at: float = 0.0
# Rate limiting: per-route timestamps in a fixed window.
self._rate_counts: Dict[str, Deque[float]] = {}
self._rate_counts: Dict[str, Deque[float]] = {} # per-route hit timestamps in a fixed window
self._rate_limit: int = int(extra.get("rate_limit", 30)) # per minute
self._max_body_bytes: int = int(extra.get("max_body_bytes", 1_048_576)) # 1MB
self._script_timeout_seconds: int = int(extra.get("script_timeout_seconds", DEFAULT_SCRIPT_TIMEOUT_SECONDS))
@@ -189,62 +183,50 @@ class WebhookAdapter(BasePlatformAdapter):
# --- Lifecycle ---
def _validate_route(self, name: str, route: dict) -> None:
"""Startup validation: secret is required; INSECURE_NO_AUTH only on loopback
(an unauthenticated route on a public interface is a footgun — crash early)."""
"""Startup validation: secret required; INSECURE_NO_AUTH only on loopback (crash early on a public footgun)."""
secret = route.get("secret", self._global_secret)
if not secret:
raise ValueError(
f"[webhook] Route '{name}' has no HMAC secret. "
f"Set 'secret' on the route or globally. "
f"For testing without auth, set secret to '{_INSECURE_NO_AUTH}'.")
raise ValueError(f"[webhook] Route '{name}' has no HMAC secret. Set 'secret' on the route or globally. "
f"For testing without auth, set secret to '{_INSECURE_NO_AUTH}'.")
if secret == _INSECURE_NO_AUTH and not _is_loopback_host(self._host):
raise ValueError(
f"[webhook] Route '{name}' uses INSECURE_NO_AUTH secret "
f"but is bound to non-loopback host '{self._host}'. "
f"INSECURE_NO_AUTH is for local testing only. "
f"Refusing to start to prevent accidental exposure.")
raise ValueError(f"[webhook] Route '{name}' uses INSECURE_NO_AUTH secret but is bound to non-loopback "
f"host '{self._host}'. INSECURE_NO_AUTH is for local testing only. "
f"Refusing to start to prevent accidental exposure.")
if route.get("deliver_only"):
deliver = route.get("deliver", "log")
if not deliver or deliver == "log":
raise ValueError(
f"[webhook] Route '{name}' has deliver_only=true but "
f"deliver is '{deliver}'. Direct delivery requires a "
f"real target (telegram, discord, slack, github_comment, etc.).")
raise ValueError(f"[webhook] Route '{name}' has deliver_only=true but deliver is '{deliver}'. Direct "
f"delivery requires a real target (telegram, discord, slack, github_comment, etc.).")
async def connect(self, *, is_reconnect: bool = False) -> bool:
self._reload_dynamic_routes()
for name, route in self._routes.items():
self._validate_route(name, route)
# client_max_size enforces the cap on every read path, including chunked
# bodies without Content-Length that bypass the header check.
# client_max_size enforces the cap on every read path, including chunked bodies without
# Content-Length that bypass the header check.
app = web.Application(client_max_size=self._max_body_bytes)
app.router.add_get("/health", self._handle_health)
app.router.add_post("/webhooks/{route_name}", self._handle_webhook)
# /p/<profile>/ prefix routes the event to that profile (only honored when
# gateway.multiplex_profiles is on; the handler validates).
# /p/<profile>/ routes the event to that profile (honored only under gateway.multiplex_profiles).
app.router.add_post("/p/{profile}/webhooks/{route_name}", self._handle_webhook)
self._runner = web.AppRunner(app)
await self._runner.setup()
# SO_REUSEADDR: on macOS (BSD) two wildcard/specific sockets can silently split traffic
# while both report success → disable. On Linux it only permits rebinding past TIME_WAIT
# (a quick restart would otherwise fail to bind for ~60s) → keep the default.
site = web.TCPSite(
self._runner, self._host, self._port, reuse_address=False if sys.platform == "darwin" else None,
)
# SO_REUSEADDR: on macOS (BSD) two wildcard/specific sockets can silently split traffic while
# both report success → disable. On Linux it only permits rebinding past TIME_WAIT (a quick
# restart would otherwise fail to bind for ~60s) → keep the default.
site = web.TCPSite(self._runner, self._host, self._port,
reuse_address=False if sys.platform == "darwin" else None)
try:
await site.start()
except OSError as exc:
await self._runner.cleanup()
self._runner = None
logger.error(
"[webhook] Could not bind %s:%d: %s. "
"Set a different host or port in config.yaml under platforms.webhook.extra.",
self._host or "all IPv4+IPv6 interfaces", self._port, exc)
logger.error("[webhook] Could not bind %s:%d: %s. Set a different host or port in config.yaml under "
"platforms.webhook.extra.", self._host or "all IPv4+IPv6 interfaces", self._port, exc)
return False
self._mark_connected()
route_names = ", ".join(self._routes.keys()) or "(none configured)"
logger.info("[webhook] Listening on %s:%d — routes: %s",
self._host or "* (all interfaces, IPv4+IPv6)", self._port, route_names)
logger.info("[webhook] Listening on %s:%d — routes: %s", self._host or "* (all interfaces, IPv4+IPv6)",
self._port, ", ".join(self._routes.keys()) or "(none configured)")
self._wire_plugin_handlers(None)
return True
@@ -255,14 +237,13 @@ class WebhookAdapter(BasePlatformAdapter):
self._mark_disconnected()
logger.info("[webhook] Disconnected")
async def send(
self, chat_id: str, content: str, reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None) -> SendResult:
async def send(self, chat_id: str, content: str, reply_to: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None) -> SendResult:
"""Deliver the agent's response to the destination stored for ``chat_id``
(``webhook:{route}:{delivery_id}``) — read with ``.get()``, never popped."""
# Autonomous lane (no human reader): the loose marker matcher shared with cron —
# marker on its own first/last line — because models add a sentence explaining
# why they stayed quiet, which the interactive exact-match rule would deliver.
# Autonomous lane (no human reader): the loose marker matcher shared with cron (marker on its own
# first/last line), because models add a sentence explaining why they stayed quiet, which the
# interactive exact-match rule would deliver.
if is_autonomous_silence_response(content):
logger.info("[webhook] Response for %s is a silence marker — not delivering", chat_id)
return SendResult(success=True)
@@ -279,14 +260,15 @@ class WebhookAdapter(BasePlatformAdapter):
return SendResult(success=False, error=f"Unknown deliver type: {deliver_type}")
def _prune_delivery_info(self, now: float) -> None:
"""Drop delivery_info entries older than the idempotency TTL (bounds the dict
by ``rate_limit * TTL`` even when runs never produce a final response)."""
created = self._delivery_info_created
if len(self._delivery_info_order) < len(created):
self._delivery_info_order = deque((at, key) for key, at in sorted(created.items(), key=lambda kv: kv[1]))
"""Drop delivery_info entries older than the idempotency TTL (bounds the dict by ``rate_limit * TTL``
even when runs never produce a final response)."""
created, order = self._delivery_info_created, self._delivery_info_order
if len(order) < len(created):
order = self._delivery_info_order = deque(
(at, key) for key, at in sorted(created.items(), key=lambda kv: kv[1]))
cutoff = now - self._idempotency_ttl
while self._delivery_info_order and self._delivery_info_order[0][0] < cutoff:
created_at, key = self._delivery_info_order.popleft()
while order and order[0][0] < cutoff:
created_at, key = order.popleft()
if created.get(key) == created_at:
self._delivery_info.pop(key, None)
created.pop(key, None)
@@ -302,8 +284,7 @@ class WebhookAdapter(BasePlatformAdapter):
def _record_rate_limit_hit(self, route_name: str, now: float) -> bool:
"""Return True if route is still within limit after recording this hit."""
window = self._rate_counts.get(route_name)
if not isinstance(window, deque):
if not isinstance(window := self._rate_counts.get(route_name), deque):
window = self._rate_counts[route_name] = deque(window or ())
cutoff = now - _RATE_WINDOW_SECONDS
while window and window[0] < cutoff:
@@ -315,8 +296,7 @@ class WebhookAdapter(BasePlatformAdapter):
def _record_delivery_id(self, delivery_id: str, now: float) -> bool:
"""Return True when this delivery should be processed."""
seen_at = self._seen_deliveries.get(delivery_id)
if seen_at is not None and now - seen_at < self._idempotency_ttl:
if (seen_at := self._seen_deliveries.get(delivery_id)) is not None and now - seen_at < self._idempotency_ttl:
return False
if seen_at is not None:
self._seen_deliveries.pop(delivery_id, None)
@@ -329,17 +309,15 @@ class WebhookAdapter(BasePlatformAdapter):
return {"name": chat_id, "type": "webhook"}
def toolsets_for_source(self, source) -> Optional[List[str]]:
"""Per-route ``toolsets`` override (config.yaml or a manual key in
webhook_subscriptions.json — deliberately NOT settable via `hermes webhook
subscribe`, so an agent-created subscription cannot self-grant tools)."""
"""Per-route ``toolsets`` override (config.yaml or a manual key in webhook_subscriptions.json —
deliberately NOT settable via `hermes webhook subscribe`, so an agent-created subscription
cannot self-grant tools)."""
parts = str(getattr(source, "chat_id", "") or "").split(":", 2)
if len(parts) < 2 or parts[0] != "webhook":
return None
route_config = self._routes.get(parts[1])
if not isinstance(route_config, dict):
return None
toolsets = route_config.get("toolsets")
if not isinstance(toolsets, list) or not toolsets:
toolsets = route_config.get("toolsets") if isinstance(route_config, dict) else None
if not isinstance(toolsets, list):
return None
return [str(t).strip() for t in toolsets if str(t).strip()] or None
@@ -350,21 +328,16 @@ class WebhookAdapter(BasePlatformAdapter):
return web.json_response({"status": "ok", "platform": "webhook"})
def _dynamic_route_allowed(self, name: str, route: dict) -> bool:
"""An empty effective secret would make _handle_webhook skip HMAC validation →
reject such dynamic routes; INSECURE_NO_AUTH is loopback-only."""
"""An empty effective secret would make _handle_webhook skip HMAC validation → reject such
dynamic routes; INSECURE_NO_AUTH is loopback-only."""
effective_secret = route.get("secret", self._global_secret)
if not effective_secret:
logger.warning(
"[webhook] Dynamic route '%s' skipped: 'secret' is "
"missing or empty. Set a valid HMAC secret, or use "
"'%s' to explicitly disable auth (testing only).",
name, _INSECURE_NO_AUTH)
logger.warning("[webhook] Dynamic route '%s' skipped: 'secret' is missing or empty. Set a valid HMAC "
"secret, or use '%s' to explicitly disable auth (testing only).", name, _INSECURE_NO_AUTH)
return False
if effective_secret == _INSECURE_NO_AUTH and not _is_loopback_host(self._host):
logger.warning(
"[webhook] Dynamic route '%s' skipped: INSECURE_NO_AUTH "
"is only allowed on loopback hosts. Current host: '%s'.",
name, self._host)
logger.warning("[webhook] Dynamic route '%s' skipped: INSECURE_NO_AUTH is only allowed on loopback "
"hosts. Current host: '%s'.", name, self._host)
return False
return True
@@ -374,8 +347,7 @@ class WebhookAdapter(BasePlatformAdapter):
subs_path = get_hermes_home() / _DYNAMIC_ROUTES_FILENAME
if not subs_path.exists():
if self._dynamic_routes:
self._dynamic_routes = {}
self._routes = dict(self._static_routes)
self._dynamic_routes, self._routes = {}, dict(self._static_routes)
logger.debug("[webhook] Dynamic subscriptions file removed, cleared dynamic routes")
return
try:
@@ -385,14 +357,12 @@ class WebhookAdapter(BasePlatformAdapter):
data = json.loads(subs_path.read_text(encoding="utf-8"))
if not isinstance(data, dict):
return
# Static routes take precedence.
self._dynamic_routes = {
k: v for k, v in data.items()
if k not in self._static_routes and self._dynamic_route_allowed(k, v)}
self._dynamic_routes = { # static routes take precedence
k: v for k, v in data.items() if k not in self._static_routes and self._dynamic_route_allowed(k, v)}
self._routes = {**self._dynamic_routes, **self._static_routes}
self._dynamic_routes_mtime = mtime
logger.info("[webhook] Reloaded %d dynamic route(s): %s",
len(self._dynamic_routes), ", ".join(self._dynamic_routes.keys()) or "(none)")
logger.info("[webhook] Reloaded %d dynamic route(s): %s", len(self._dynamic_routes),
", ".join(self._dynamic_routes.keys()) or "(none)")
except Exception as e:
logger.error("[webhook] Failed to reload dynamic routes: %s", e)
@@ -405,9 +375,8 @@ class WebhookAdapter(BasePlatformAdapter):
return None
cfg = getattr(self.gateway_runner, "config", None)
if not getattr(cfg, "multiplex_profiles", False):
# Only a self-referential prefix may fall through to the bare route;
# anything else fails closed (silently ignoring the prefix served the
# owner's routes under another profile's URL).
# Only a self-referential prefix may fall through to the bare route; anything else fails
# closed (silently ignoring the prefix served the owner's routes under another profile's URL).
with suppress(Exception):
from hermes_cli.profiles import profile_matches_home
if profile_matches_home(profile):
@@ -424,10 +393,10 @@ class WebhookAdapter(BasePlatformAdapter):
@staticmethod
def _route_allows_profile(route_config: dict, request_profile: Optional[str]) -> bool:
"""Omitting ``profile`` binds a route to default; an explicit null/blank/non-string fails closed."""
configured_profile = route_config.get("profile") if "profile" in route_config else "default"
if not isinstance(configured_profile, str) or not configured_profile.strip():
configured = route_config.get("profile") if "profile" in route_config else "default"
if not isinstance(configured, str) or not configured.strip():
return False
return configured_profile.strip() == (request_profile or "default")
return configured.strip() == (request_profile or "default")
@staticmethod
def _profile_scope(profile: Optional[str]):
@@ -438,25 +407,22 @@ class WebhookAdapter(BasePlatformAdapter):
from hermes_cli.profiles import get_profile_dir
return _profile_runtime_scope(get_profile_dir(profile))
async def _read_authenticated_body(
self, request: "web.Request", route_name: str, route_config: dict
) -> "tuple[Optional[bytes], Optional[web.Response]]":
async def _read_authenticated_body(self, request: "web.Request", route_name: str,
route_config: dict) -> "tuple[Optional[bytes], Optional[web.Response]]":
"""Auth-before-body: size-cap, read, then HMAC-validate. Returns ``(body, None)`` or ``(None, response)``."""
if (request.content_length or 0) > self._max_body_bytes:
return None, _json_error("Payload too large", 413)
try:
raw_body = await request.read()
except web.HTTPRequestEntityTooLarge:
# aiohttp's client_max_size tripped — chunked or lying Content-Length.
except web.HTTPRequestEntityTooLarge: # client_max_size tripped — chunked or lying Content-Length
return None, _json_error("Payload too large", 413)
except Exception as e:
logger.error("[webhook] Failed to read body: %s", e)
return None, _json_error("Bad request", 400)
if len(raw_body) > self._max_body_bytes:
# Defense in depth if the server-level limit was bypassed or misconfigured.
if len(raw_body) > self._max_body_bytes: # defense in depth if the server-level limit was bypassed
return None, _json_error("Payload too large", 413)
# Missing/empty secrets fail closed here too (not only in connect()), so
# direct handler reuse cannot become an unauthenticated dispatch surface.
# Missing/empty secrets fail closed here too (not only in connect()), so direct handler reuse
# cannot become an unauthenticated dispatch surface.
secret = route_config.get("secret", self._global_secret)
if not secret:
logger.error("[webhook] Route %s has no HMAC secret; refusing request", route_name)
@@ -478,17 +444,14 @@ class WebhookAdapter(BasePlatformAdapter):
except Exception:
return _UNPARSEABLE
async def _handle_deliver_only(
self, prompt: str, payload: Any, route_config: dict, route_name: str, event_type: str, delivery_id: str
) -> "web.Response":
"""deliver_only: the rendered prompt IS the message — skip the agent, reuse
the same auth/rate-limit/idempotency/template pipeline."""
delivery = {
"deliver": route_config.get("deliver", "log"),
"deliver_extra": self._render_delivery_extra(route_config.get("deliver_extra", {}), payload),
"payload": payload}
logger.info("[webhook] direct-deliver event=%s route=%s target=%s msg_len=%d delivery=%s",
event_type, route_name, delivery["deliver"], len(prompt), delivery_id)
async def _handle_deliver_only(self, prompt: str, payload: Any, route_config: dict, route_name: str,
event_type: str, delivery_id: str) -> "web.Response":
"""deliver_only: the rendered prompt IS the message — skip the agent, reuse the same
auth/rate-limit/idempotency/template pipeline."""
delivery = {"deliver": route_config.get("deliver", "log"), "payload": payload,
"deliver_extra": self._render_delivery_extra(route_config.get("deliver_extra", {}), payload)}
logger.info("[webhook] direct-deliver event=%s route=%s target=%s msg_len=%d delivery=%s", event_type,
route_name, delivery["deliver"], len(prompt), delivery_id)
failed = {"status": "error", "error": "Delivery failed", "delivery_id": delivery_id}
try:
result = await self._direct_deliver(prompt, delivery)
@@ -496,18 +459,16 @@ class WebhookAdapter(BasePlatformAdapter):
logger.exception("[webhook] direct-deliver failed route=%s delivery=%s", route_name, delivery_id)
return web.json_response(failed, status=502)
if result.success:
return web.json_response(
{"status": "delivered", "route": route_name, "target": delivery["deliver"], "delivery_id": delivery_id},
status=200)
return web.json_response({"status": "delivered", "route": route_name, "target": delivery["deliver"],
"delivery_id": delivery_id}, status=200)
# Target rejected it — 502 with a generic error (don't leak adapter detail).
logger.warning("[webhook] direct-deliver target rejected route=%s target=%s error=%s",
route_name, delivery["deliver"], result.error)
logger.warning("[webhook] direct-deliver target rejected route=%s target=%s error=%s", route_name,
delivery["deliver"], result.error)
return web.json_response(failed, status=502)
def _resolve_route(self, request: "web.Request") -> "tuple[str, Optional[dict], Any, Optional[web.Response]]":
"""Route + profile lookup for a POST; ``(route_name, route_config, profile, error_response)``."""
# Hot-reload dynamic subscriptions on each request (mtime-gated, cheap)
self._reload_dynamic_routes()
self._reload_dynamic_routes() # hot-reload dynamic subscriptions (mtime-gated, cheap)
route_name = request.match_info.get("route_name", "")
route_config = self._routes.get(route_name)
profile = self._resolve_request_profile(request)
@@ -519,16 +480,16 @@ class WebhookAdapter(BasePlatformAdapter):
logger.warning("[webhook] Route %s is not authorized for profile %r", route_name, profile or "default")
# Same as unknown-route so profile mismatches can't enumerate route bindings.
return route_name, None, profile, _json_error(f"Unknown route: {route_name}", 404)
# Disabled routes stay in the subscriptions file (dashboard can re-enable)
# but reject events. Only an explicit ``enabled: false`` turns a route off.
# Disabled routes stay in the subscriptions file (dashboard can re-enable) but reject events.
# Only an explicit ``enabled: false`` turns a route off.
if route_config.get("enabled", True) is False:
return route_name, None, profile, _json_error(f"Route disabled: {route_name}", 403)
return route_name, route_config, profile, None
@staticmethod
def _apply_skills(prompt: str, skills: list) -> str:
"""Inject the first matching skill via build_skill_invocation_message() directly —
/skill-name slash commands would be intercepted by the command parser."""
"""Inject the first matching skill via build_skill_invocation_message() directly — /skill-name slash
commands would be intercepted by the command parser."""
try:
from agent.skill_commands import build_skill_invocation_message, get_skill_commands
skill_cmds = get_skill_commands()
@@ -557,16 +518,13 @@ class WebhookAdapter(BasePlatformAdapter):
payload = self._parse_body(raw_body)
if payload is _UNPARSEABLE:
return _json_error("Cannot parse body", 400)
event_type = (
request.headers.get("X-GitHub-Event", "")
or request.headers.get("X-GitLab-Event", "")
or payload.get("event_type", "")
or payload.get("type", "")
or "unknown")
headers = request.headers
event_type = (headers.get("X-GitHub-Event", "") or headers.get("X-GitLab-Event", "")
or payload.get("event_type", "") or payload.get("type", "") or "unknown")
allowed_events = route_config.get("events", [])
if allowed_events and event_type not in allowed_events:
logger.debug("[webhook] Ignoring event %s for route %s (allowed: %s)",
event_type, route_name, allowed_events)
logger.debug("[webhook] Ignoring event %s for route %s (allowed: %s)", event_type, route_name,
allowed_events)
return web.json_response({"status": "ignored", "event": event_type})
if not self._route_processor.route_filters_match(route_config, payload, event_type, request.headers):
logger.info("[webhook] filtered event=%s route=%s", event_type, route_name)
@@ -585,14 +543,11 @@ class WebhookAdapter(BasePlatformAdapter):
return web.json_response({"status": "ignored", "reason": "script", "route": route_name})
payload = transformed_payload or payload
prompt = self._render_prompt(route_config.get("prompt", ""), payload, event_type, route_name)
skills = route_config.get("skills", [])
if skills:
if skills := route_config.get("skills", []):
prompt = self._apply_skills(prompt, skills)
headers = request.headers
delivery_id = headers.get("X-GitHub-Delivery", headers.get(
"svix-id", headers.get("X-Request-ID", str(int(time.time() * 1000)))))
# Idempotency: skip duplicate deliveries (webhook retries).
now = time.time()
now = time.time() # idempotency: skip duplicate deliveries (webhook retries)
if not self._record_delivery_id(delivery_id, now):
logger.info("[webhook] Skipping duplicate delivery %s", delivery_id)
return web.json_response({"status": "duplicate", "delivery_id": delivery_id}, status=200)
@@ -601,13 +556,10 @@ class WebhookAdapter(BasePlatformAdapter):
return self._dispatch_agent_run(request, route_config, route_name, profile, payload, prompt, event_type,
delivery_id, now)
def _dispatch_agent_run(
self, request, route_config: dict, route_name: str, profile, payload: Any, prompt: str,
event_type: str, delivery_id: str, now: float,
) -> "web.Response":
def _dispatch_agent_run(self, request, route_config: dict, route_name: str, profile, payload: Any, prompt: str,
event_type: str, delivery_id: str, now: float) -> "web.Response":
"""Record delivery info, spawn the agent run, and return 202 immediately."""
# delivery_id in the session key → concurrent webhooks on one route get
# independent agent runs (not queued/interrupted).
# delivery_id in the session key → concurrent webhooks on one route get independent runs.
session_chat_id = f"webhook:{route_name}:{delivery_id}"
self._delivery_info[session_chat_id] = {
"deliver": route_config.get("deliver", "log"),
@@ -615,35 +567,33 @@ class WebhookAdapter(BasePlatformAdapter):
self._delivery_info_created[session_chat_id] = now
self._delivery_info_order.append((now, session_chat_id))
self._prune_delivery_info(now)
source = self.build_source(
chat_id=session_chat_id, chat_name=f"webhook/{route_name}", chat_type="webhook",
user_id=f"webhook:{route_name}", user_name=route_name)
source = self.build_source(chat_id=session_chat_id, chat_name=f"webhook/{route_name}", chat_type="webhook",
user_id=f"webhook:{route_name}", user_name=route_name)
if profile and isinstance(profile, str):
source.profile = profile
event = MessageEvent(
text=prompt, message_type=MessageType.TEXT, source=source, raw_message=payload, message_id=delivery_id)
logger.info("[webhook] %s event=%s route=%s prompt_len=%d delivery=%s",
request.method, event_type, route_name, len(prompt), delivery_id)
# The per-delivery session is closed by ``on_processing_complete`` once the run
# finishes (``handle_message`` is fire-and-forget, so nothing can be closed here).
event = MessageEvent(text=prompt, message_type=MessageType.TEXT, source=source, raw_message=payload,
message_id=delivery_id)
logger.info("[webhook] %s event=%s route=%s prompt_len=%d delivery=%s", request.method, event_type, route_name,
len(prompt), delivery_id)
# The per-delivery session is closed by ``on_processing_complete`` once the run finishes
# (``handle_message`` is fire-and-forget, so nothing can be closed here).
task = asyncio.create_task(self.handle_message(event))
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
return web.json_response(
{"status": "accepted", "route": route_name, "event": event_type, "delivery_id": delivery_id}, status=202)
return web.json_response({"status": "accepted", "route": route_name, "event": event_type,
"delivery_id": delivery_id}, status=202)
async def on_processing_complete(self, event: "MessageEvent", outcome: Any) -> None:
"""Close the one-shot per-delivery session: ``prune_sessions`` only reaps rows with ``ended_at``
set, so unclosed webhook sessions leak unbounded. Fires at the true end of the run (success,
failure, cancellation); ``end_session()`` is first-reason-wins."""
"""Close the one-shot per-delivery session: ``prune_sessions`` only reaps rows with ``ended_at`` set, so
unclosed webhook sessions leak unbounded. Fires at the true end of the run; ``end_session()`` is
first-reason-wins."""
await self._end_webhook_session(event, event.source.chat_id)
async def _end_webhook_session(self, event: "MessageEvent", session_chat_id: str) -> None:
"""Mark the per-delivery session ended via ``SessionDB.end_session`` (never a hand-written UPDATE),
resolving session_id from the SAME source the run was keyed on."""
runner = self.gateway_runner
session_db = getattr(runner, "_session_db", None)
store = getattr(runner, "session_store", None)
session_db, store = getattr(runner, "_session_db", None), getattr(runner, "session_store", None)
key_fn = getattr(runner, "_session_key_for_source", None)
if runner is None or session_db is None or store is None or key_fn is None:
return
@@ -676,32 +626,25 @@ class WebhookAdapter(BasePlatformAdapter):
return _validate_svix_signature(body, secret, *svix)
# Linear (any header case): hex HMAC of the body. GitHub: sha256=<hex>. GitLab: plain token.
for provided, expected in (
(_header("linear-signature"), lambda: _hex_hmac(secret, body)),
(headers.get("X-Hub-Signature-256", ""), lambda: "sha256=" + _hex_hmac(secret, body)),
(headers.get("X-Gitlab-Token", ""), lambda: secret),
):
(_header("linear-signature"), lambda: _hex_hmac(secret, body)),
(headers.get("X-Hub-Signature-256", ""), lambda: "sha256=" + _hex_hmac(secret, body)),
(headers.get("X-Gitlab-Token", ""), lambda: secret)):
if provided:
return _hmac_str_equal(provided, expected())
route_name = request.match_info.get("route_name", "")
# Generic V2: X-Webhook-Signature-V2 = hex HMAC-SHA256 of "<timestamp>.<body>",
# X-Webhook-Timestamp required. Presence of the V2 header COMMITS to V2 — it
# must not fall through to V1 on a missing/bad timestamp, or an attacker could
# strip the timestamp from a captured mixed V1+V2 request and replay it
# against the still-present body-only V1 signature.
# Generic V2: X-Webhook-Signature-V2 = hex HMAC-SHA256 of "<timestamp>.<body>", X-Webhook-Timestamp
# required. Presence of the V2 header COMMITS to V2 — it must not fall through to V1 on a
# missing/bad timestamp, or an attacker could strip the timestamp from a captured mixed V1+V2
# request and replay it against the still-present body-only V1 signature.
v2_sig = headers.get("X-Webhook-Signature-V2", "")
if v2_sig:
v2_timestamp = headers.get("X-Webhook-Timestamp", "")
if not v2_timestamp:
logger.warning(
"[webhook] Route '%s' sent X-Webhook-Signature-V2 with "
"no X-Webhook-Timestamp — rejecting rather than falling back to legacy V1",
route_name)
logger.warning("[webhook] Route '%s' sent X-Webhook-Signature-V2 with no X-Webhook-Timestamp — "
"rejecting rather than falling back to legacy V1", route_name)
return False
age = _timestamp_age(v2_timestamp)
if age is None:
return False
if age > _V2_REPLAY_WINDOW_SECONDS:
logger.warning("[webhook] Route '%s' generic HMAC V2 timestamp outside replay window", route_name)
if not _timestamp_fresh(
v2_timestamp, "[webhook] Route '%s' generic HMAC V2 timestamp outside replay window", route_name):
return False
return _hmac_str_equal(v2_sig, _hex_hmac(secret, v2_timestamp.encode() + b"." + body))
# Generic V1 (legacy, deprecated): body-only HMAC → replays indefinitely.
@@ -709,12 +652,9 @@ class WebhookAdapter(BasePlatformAdapter):
if generic_sig:
if route_name not in self._v1_signature_warned:
self._v1_signature_warned.add(route_name)
logger.warning(
"[webhook] Route '%s' uses legacy body-only HMAC (no "
"timestamp), which is vulnerable to replay attacks. Add "
"an 'X-Webhook-Timestamp' header and switch to "
"'X-Webhook-Signature-V2' (HMAC-SHA256 of '<timestamp>.<body>').",
route_name)
logger.warning("[webhook] Route '%s' uses legacy body-only HMAC (no timestamp), which is vulnerable "
"to replay attacks. Add an 'X-Webhook-Timestamp' header and switch to "
"'X-Webhook-Signature-V2' (HMAC-SHA256 of '<timestamp>.<body>').", route_name)
return _hmac_str_equal(generic_sig, _hex_hmac(secret, body))
logger.debug("[webhook] Secret configured but no signature header found")
return False
@@ -739,25 +679,21 @@ class WebhookAdapter(BasePlatformAdapter):
if not isinstance(value, dict):
return f"{{{key}}}"
value = value.get(part, f"{{{key}}}")
if isinstance(value, (dict, list)):
return json.dumps(value, indent=2)[:2000]
return str(value)
return json.dumps(value, indent=2)[:2000] if isinstance(value, (dict, list)) else str(value)
return _TEMPLATE_KEY_RE.sub(_resolve, template)
def _render_delivery_extra(self, extra: dict, payload: dict) -> dict:
"""Render delivery_extra template values with payload data."""
return {
key: self._render_prompt(value, payload, "", "") if isinstance(value, str) else value
for key, value in extra.items()}
return {key: self._render_prompt(value, payload, "", "") if isinstance(value, str) else value
for key, value in extra.items()}
# --- Response delivery ---
async def _direct_deliver(self, content: str, delivery: dict) -> SendResult:
"""deliver_only: dispatch *content* to the same delivery helpers agent-mode ``send()`` uses."""
deliver_type = delivery.get("deliver", "log")
if deliver_type == "log":
# Startup validation rejects deliver_only + log, but guard defensively.
if deliver_type == "log": # startup validation rejects deliver_only + log; guard defensively
logger.info("[webhook] direct-deliver log-only: %s", content[:200])
return SendResult(success=True)
if deliver_type == "github_comment":
@@ -767,13 +703,11 @@ class WebhookAdapter(BasePlatformAdapter):
async def _deliver_github_comment(self, content: str, delivery: dict) -> SendResult:
"""Post agent response as a GitHub PR/issue comment via ``gh`` CLI."""
extra = delivery.get("deliver_extra", {})
repo = extra.get("repo", "")
pr_number = extra.get("pr_number", "")
repo, pr_number = extra.get("repo", ""), extra.get("pr_number", "")
if not repo or not pr_number:
logger.error("[webhook] github_comment delivery missing repo or pr_number")
return SendResult(success=False, error="Missing repo or pr_number")
# Input validation (prevent CLI argument injection).
try:
try: # input validation (prevent CLI argument injection)
pr_int = int(pr_number)
if pr_int <= 0:
raise ValueError("non-positive")
@@ -784,11 +718,10 @@ class WebhookAdapter(BasePlatformAdapter):
logger.error("[webhook] invalid repo format: %r", repo)
return SendResult(success=False, error="Invalid repo format")
try:
# Off-loop: `gh` does network I/O up to its 30s timeout; inline it froze
# every adapter and timer on the gateway event loop.
# Off-loop: `gh` does network I/O up to its 30s timeout; inline it froze every adapter and
# timer on the gateway event loop.
result = await asyncio.to_thread(
subprocess.run,
["gh", "pr", "comment", str(pr_int), "--repo", repo, "--body", content],
subprocess.run, ["gh", "pr", "comment", str(pr_int), "--repo", repo, "--body", content],
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=30)
if result.returncode == 0:
logger.info("[webhook] Posted comment on %s#%s", repo, pr_number)
@@ -803,10 +736,8 @@ class WebhookAdapter(BasePlatformAdapter):
return SendResult(success=False, error=str(e))
def _find_adapter(self, target_platform: Platform):
"""Default adapters first; multiplex may park a platform only on a secondary
profile (_profile_adapters), so fall back to those."""
adapter = self.gateway_runner.adapters.get(target_platform)
if adapter:
"""Default adapters first; multiplex may park a platform only on a secondary profile (_profile_adapters)."""
if adapter := self.gateway_runner.adapters.get(target_platform):
return adapter
for amap in (getattr(self.gateway_runner, "_profile_adapters", None) or {}).values():
if isinstance(amap, dict) and amap.get(target_platform) is not None:
@@ -821,8 +752,7 @@ class WebhookAdapter(BasePlatformAdapter):
target_platform = Platform(platform_name)
except ValueError:
return SendResult(success=False, error=f"Unknown platform: {platform_name}")
adapter = self._find_adapter(target_platform)
if not adapter:
if not (adapter := self._find_adapter(target_platform)):
return SendResult(success=False, error=f"Platform {platform_name} not connected")
extra = delivery.get("deliver_extra", {})
chat_id = extra.get("chat_id", "")
@@ -831,7 +761,5 @@ class WebhookAdapter(BasePlatformAdapter):
if not home:
return SendResult(success=False, error=f"No chat_id or home channel for {platform_name}")
chat_id = home.chat_id
# thread_id from deliver_extra so Telegram forum topics work.
thread_id = extra.get("message_thread_id") or extra.get("thread_id")
metadata = {"thread_id": thread_id} if thread_id else None
return await adapter.send(chat_id, content, metadata=metadata)
thread_id = extra.get("message_thread_id") or extra.get("thread_id") # Telegram forum topics
return await adapter.send(chat_id, content, metadata={"thread_id": thread_id} if thread_id else None)