refactor(plugins/telegram): split send/inbound god methods into phase helpers, dedupe media/retry paths, dispatch tables, drop dead code (8817->6578 LOC, 13073-case Bot API payload parity)

This commit is contained in:
Teknium
2026-09-03 01:29:35 -07:00
parent ce08af6139
commit ab48dccf97
5 changed files with 2406 additions and 4644 deletions
File diff suppressed because it is too large Load Diff
+23 -57
View File
@@ -1,14 +1,10 @@
#!/usr/bin/env python3
"""Telegram inline command picker — searchable access to EVERY command/skill.
The BotCommand menu is capped (100/scope, Hermes uses 60), so inline mode
(``@yourbot <query>``, live per keystroke, 50 per page) exposes the rest. Tapping a
result sends ``/cmd args`` as the user; it starts with ``/`` so it arrives even under
privacy mode and dispatches through the normal command path.
PTB-object-free on purpose: plain dicts keep catalog/filter/pagination unit-testable
without python-telegram-bot; the adapter converts to ``InlineQueryResultArticle``.
Inline mode must be enabled via BotFather ``/setinline``; until then the handler is inert.
The BotCommand menu is capped (100/scope, Hermes uses 60), so inline mode (``@yourbot <query>``, 50 per
page) exposes the rest; tapping a result sends ``/cmd args`` as the user through the normal command path.
PTB-object-free on purpose: plain dicts keep catalog/filter/pagination unit-testable; the adapter converts
to ``InlineQueryResultArticle``. Inert until inline mode is enabled via BotFather ``/setinline``.
"""
from __future__ import annotations
@@ -26,49 +22,34 @@ CACHE_TIME_SECONDS = 10
def collect_inline_catalog() -> List[Dict[str, str]]:
"""Every dispatchable command as ``{name, description}``, first occurrence wins.
Core gateway commands first (menu gating), then plugin + skill commands via the
shared collector with ``max_slots=None`` — the inline picker has no cap.
"""
"""Every dispatchable command as ``{name, description}``, first occurrence wins: core gateway commands
first (menu gating), then plugin + skill commands with ``max_slots=None`` (no cap)."""
catalog: List[Dict[str, str]] = []
seen: set[str] = set()
def _add(name, desc):
if name and name not in seen:
seen.add(name)
catalog.append({"name": name, "description": desc or ""})
try:
from hermes_cli.commands import (
_collect_gateway_skill_entries,
_sanitize_telegram_name,
telegram_bot_commands,
)
from hermes_cli.commands import _collect_gateway_skill_entries, _sanitize_telegram_name, telegram_bot_commands
except Exception: # pragma: no cover - defensive
logger.debug("inline picker: commands registry unavailable", exc_info=True)
return catalog
try:
for name, desc in telegram_bot_commands():
if name and name not in seen:
seen.add(name)
catalog.append({"name": name, "description": desc or ""})
_add(name, desc)
except Exception:
logger.debug("inline picker: core command collection failed", exc_info=True)
try:
entries, _hidden = _collect_gateway_skill_entries(
platform="telegram",
max_slots=None, # inline mode has no cap — collect everything
reserved_names=set(seen),
desc_limit=100,
sanitize_name=_sanitize_telegram_name,
platform="telegram", max_slots=None, reserved_names=set(seen), desc_limit=100, sanitize_name=_sanitize_telegram_name,
)
for entry in entries:
# Entry shape is (name, desc, cmd_key[, raw_name]) — tolerate both.
name, desc = entry[0], entry[1]
if name and name not in seen:
seen.add(name)
catalog.append({"name": name, "description": desc or ""})
for entry in entries: # shape is (name, desc, cmd_key[, raw_name])
_add(entry[0], entry[1])
except Exception:
logger.debug("inline picker: skill/plugin collection failed", exc_info=True)
return catalog
@@ -78,12 +59,10 @@ def filter_catalog(catalog: List[Dict[str, str]], term: str) -> List[Dict[str, s
term = (term or "").strip().lower().lstrip("/")
if not term:
return list(catalog)
prefix: List[Dict[str, str]] = []
name_sub: List[Dict[str, str]] = []
desc_sub: List[Dict[str, str]] = []
# Treat hyphens/underscores as equivalent, mirroring command dispatch.
norm_term = term.replace("-", "_")
norm_term = term.replace("-", "_") # hyphens/underscores equivalent, mirroring command dispatch
for item in catalog:
norm_name = item["name"].lower().replace("-", "_")
if norm_name.startswith(norm_term):
@@ -95,34 +74,21 @@ def filter_catalog(catalog: List[Dict[str, str]], term: str) -> List[Dict[str, s
return prefix + name_sub + desc_sub
def build_inline_results(
query: str, offset: str = "", page_size: int = PAGE_SIZE,
) -> Tuple[List[Dict[str, Any]], str]:
"""One page of inline results for *query*.
First token filters the catalog; the remainder becomes the command argument
(``@bot plan migrate auth`` → tapping ``/plan`` sends ``/plan migrate auth``).
Returns ``(results, next_offset)``; ``next_offset == ""`` means last page.
"""
query = (query or "").strip()
parts = query.split(None, 1)
def build_inline_results(query: str, offset: str = "", page_size: int = PAGE_SIZE) -> Tuple[List[Dict[str, Any]], str]:
"""One page of inline results for *query*: first token filters the catalog; the remainder becomes the
command argument (``@bot plan migrate auth`` → ``/plan migrate auth``). ``next_offset == ""`` = last page."""
parts = (query or "").strip().split(None, 1)
term = parts[0] if parts else ""
args = parts[1].strip() if len(parts) > 1 else ""
matches = filter_catalog(collect_inline_catalog(), term)
try:
start = int(offset) if offset else 0
except (TypeError, ValueError):
start = 0
page = matches[start:start + page_size]
next_offset = str(start + page_size) if len(matches) > start + page_size else ""
results: List[Dict[str, Any]] = []
for item in page:
message_text = f"/{item['name']}"
if args:
message_text += f" {args}"
for item in matches[start:start + page_size]:
message_text = f"/{item['name']}" + (f" {args}" if args else "")
results.append({
"id": f"{start}:{item['name']}"[:64], # offset-scoped: unique across pages
"title": f"/{item['name']}",
+5 -20
View File
@@ -1,33 +1,18 @@
"""Helpers for Telegram Bot API chat identifiers.
Telegram's Bot API accepts a ``chat_id`` in two forms: a numeric ID (an int,
e.g. ``123456789`` for a DM or ``-1001234567890`` for a channel/supergroup) or
an ``@username`` string for public channels and groups. Hermes historically
coerced every ``chat_id`` with ``int()``, which crashes on the username form
(``ValueError: invalid literal for int()``). Normalizing here lets numeric IDs
pass through as ints while usernames pass through unchanged — both are valid
values for the Bot API.
"""
"""Helpers for Telegram Bot API chat identifiers: a ``chat_id`` is a numeric ID (int) or an ``@username``
string for public channels/groups; a bare ``int(chat_id)`` crashes on the username form."""
from __future__ import annotations
import re
from typing import Any, Union
# Telegram usernames are 5-32 chars: letters, digits, underscores, with a
# leading "@". (Telegram also permits 4-char usernames for some legacy/official
# accounts, but the 5-32 public rule is the safe lower bound for routing.)
# Usernames are 5-32 chars (letters, digits, underscores) with a leading "@"; 4-char legacy handles are tolerated.
_TELEGRAM_USERNAME_RE = re.compile(r"@[A-Za-z0-9_]{4,32}")
def normalize_telegram_chat_id(chat_id: Any) -> Union[int, str]:
"""Return a Bot API-compatible chat_id.
Numeric values (incl. negative channel IDs) are returned as ``int``; any
non-numeric value (e.g. an ``@username``) is returned as a stripped string.
Telegram's Bot API accepts both, so this never raises on a username the way
a bare ``int(chat_id)`` would.
"""
"""Bot API-compatible chat_id: numeric values (incl. negative channel IDs) as ``int``, anything
else (e.g. ``@username``) as a stripped string; never raises."""
chat_id_str = str(chat_id).strip()
try:
return int(chat_id_str)
+39 -86
View File
@@ -15,39 +15,32 @@ logger = logging.getLogger(__name__)
_TELEGRAM_API_HOST = "api.telegram.org"
# TCP keepalive so a half-open/CLOSE-WAIT long-poll errors out instead of blocking
# getUpdates forever (Windows leaves SO_KEEPALIVE off by default). Idle/interval knobs
# are best-effort — not every Python/OS combo exposes TCP_KEEPIDLE / TCP_KEEPALIVE.
# TCP keepalive so a half-open/CLOSE-WAIT long-poll errors out instead of blocking getUpdates forever
# (Windows leaves SO_KEEPALIVE off). Idle/interval knobs are best-effort per Python/OS combo.
_TCP_KEEPALIVE_IDLE_S = 30
_TCP_KEEPALIVE_INTERVAL_S = 10
_TCP_KEEPALIVE_COUNT = 3
def tcp_keepalive_socket_options() -> list[tuple[int, int, int]]:
"""``setsockopt`` tuples for httpx ``socket_options``: always SO_KEEPALIVE, plus
idle/interval/count when the interpreter exposes those option names."""
"""``setsockopt`` tuples for httpx ``socket_options``: always SO_KEEPALIVE, plus idle/interval/count
when the interpreter exposes those option names."""
options: list[tuple[int, int, int]] = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
idle = getattr(socket, "TCP_KEEPIDLE", None) or getattr(socket, "TCP_KEEPALIVE", None)
if idle is not None:
options.append((socket.IPPROTO_TCP, idle, _TCP_KEEPALIVE_IDLE_S))
interval = getattr(socket, "TCP_KEEPINTVL", None)
if interval is not None:
options.append((socket.IPPROTO_TCP, interval, _TCP_KEEPALIVE_INTERVAL_S))
count = getattr(socket, "TCP_KEEPCNT", None)
if count is not None:
options.append((socket.IPPROTO_TCP, count, _TCP_KEEPALIVE_COUNT))
for opt, value in ((idle, _TCP_KEEPALIVE_IDLE_S), (getattr(socket, "TCP_KEEPINTVL", None), _TCP_KEEPALIVE_INTERVAL_S),
(getattr(socket, "TCP_KEEPCNT", None), _TCP_KEEPALIVE_COUNT)):
if opt is not None:
options.append((socket.IPPROTO_TCP, opt, value))
return options
# DNS-over-HTTPS providers: discover Telegram API IPs that may differ from the
# (possibly unreachable) one the local resolver returns. Bounded so connect() isn't delayed.
# DNS-over-HTTPS providers: discover Telegram API IPs the (possibly unreachable) local resolver may not
# return. Bounded so connect() isn't delayed.
_DOH_TIMEOUT = 4.0
_DOH_PROVIDERS: list[dict] = [
{"url": "https://dns.google/resolve", "params": {"name": _TELEGRAM_API_HOST, "type": "A"}, "headers": {}},
{
"url": "https://cloudflare-dns.com/dns-query",
"params": {"name": _TELEGRAM_API_HOST, "type": "A"},
"headers": {"Accept": "application/dns-json"},
},
"url": "https://cloudflare-dns.com/dns-query", "params": {"name": _TELEGRAM_API_HOST, "type": "A"},
"headers": {"Accept": "application/dns-json"}},
]
# Last-resort IPv4 Bot API endpoints (149.154.160.0/20). Used when DoH is blocked AND as
# first-try connect targets so a blackholed IPv6 AAAA for the hostname can't pin initialize().
@@ -61,15 +54,10 @@ def _resolve_proxy_url(target_hosts=None) -> str | None:
class TelegramFallbackTransport(httpx.AsyncBaseTransport):
"""Reach the Bot API via known IPv4 literals first, dual-stack hostname last.
"""Reach the Bot API via known IPv4 literals first, dual-stack hostname last. Host + SNI stay on
api.telegram.org (like ``curl --resolve``) so a blackholed IPv6 AAAA can't pin initialize()."""
Logically requests still target https://api.telegram.org (Host + SNI stay on the
hostname) — like ``curl --resolve api.telegram.org:443:<ip>`` — so a blackholed
IPv6 AAAA can't pin initialize(); the hostname remains for IPv6-only networks.
"""
# Bound every pool: httpx's 100-connection default × (wedged endpoint + seed IPs)
# can outgrow the process fd limit on its own.
# Bound every pool: httpx's 100-connection default × (wedged endpoint + seed IPs) can outgrow the fd limit.
_POOL_LIMITS = httpx.Limits(max_connections=8, max_keepalive_connections=4)
def __init__(self, fallback_ips: Iterable[str], **transport_kwargs):
@@ -99,8 +87,7 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport):
return transport
async def _reset_primary(self, transport: httpx.AsyncHTTPTransport) -> None:
# Retryable primary failures leave half-closed sockets in the pool; replace the
# generation and close the old one before trying fallback.
# Retryable primary failures leave half-closed sockets in the pool; replace the generation first.
async with self._primary_lock:
if self._primary_closed or transport is not self._primary:
return
@@ -111,8 +98,8 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport):
logger.debug("[Telegram] Error closing primary transport: %s", exc)
async def _reset_fallback(self, ip: str) -> None:
"""Discard a failed fallback pool: a peer-closed connect leaves a CLOSE_WAIT socket
in it, and keeping the poisoned pool leaks one fd per retry until the process limit."""
"""Discard a failed fallback pool: a peer-closed connect leaves a CLOSE_WAIT socket in it, and the
poisoned pool would leak one fd per retry."""
async with self._fallback_lock:
transport = self._fallbacks.pop(ip, None)
if transport is None:
@@ -123,18 +110,12 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport):
logger.debug("[Telegram] Error closing fallback transport %s: %s", ip, exc)
def _attempt_order(self) -> list[Optional[str]]:
"""Sticky path first, then IPv4 literals, dual-stack hostname last.
A blackholed IPv6 path never errors — Happy Eyeballs waits on AAAA until the OS
TCP timeout and can pin the loop so the thread deadline never fires.
"""
"""Sticky path first, then IPv4 literals, dual-stack hostname last (a blackholed IPv6 path never
errors — Happy Eyeballs waits on AAAA until the OS TCP timeout and can pin the loop)."""
order: list[Optional[str]] = []
if self._sticky_ip is not _UNSET:
sticky = self._sticky_ip
order.append(sticky if sticky is None else str(sticky))
for ip in self._fallback_ips:
if ip not in order:
order.append(ip)
order.append(None if self._sticky_ip is None else str(self._sticky_ip))
order.extend(ip for ip in self._fallback_ips if ip not in order)
if None not in order:
order.append(None)
return order
@@ -142,11 +123,8 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport):
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
if request.url.host != _TELEGRAM_API_HOST or not self._fallback_ips:
return await self._primary.handle_async_request(request)
attempt_order = self._attempt_order()
last_error: Exception | None = None
for ip in attempt_order:
for ip in self._attempt_order():
candidate = request if ip is None else _rewrite_request_for_ip(request, ip)
transport = self._primary if ip is None else await self._get_fallback(ip)
try:
@@ -157,10 +135,7 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport):
self._sticky_ip = ip
if ip is not None:
log = logger.warning if last_error is not None else logger.info
log(
"[Telegram] Using sticky IPv4 Telegram API path %s "
"(dual-stack hostname tried last — #87015)", ip,
)
log("[Telegram] Using sticky IPv4 Telegram API path %s (dual-stack hostname tried last — #87015)", ip)
return response
except Exception as exc:
last_error = exc
@@ -171,10 +146,8 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport):
if self._sticky_ip is not _UNSET and self._sticky_ip == ip:
self._sticky_ip = _UNSET
logger.warning(
"[Telegram] Sticky Telegram path %s failed; "
"re-walking IPv4 literals before the hostname",
ip if ip is not None else "api.telegram.org",
)
"[Telegram] Sticky Telegram path %s failed; re-walking IPv4 literals before the hostname",
ip if ip is not None else "api.telegram.org")
if ip is None:
await self._reset_primary(transport)
logger.warning("[Telegram] Dual-stack api.telegram.org path failed (%s)", exc)
@@ -182,7 +155,6 @@ class TelegramFallbackTransport(httpx.AsyncBaseTransport):
logger.warning("[Telegram] IPv4 Telegram API IP %s failed: %s", ip, exc)
await self._reset_fallback(ip)
continue
if last_error is None:
raise RuntimeError("All Telegram fallback IPs exhausted but no error was recorded")
raise last_error
@@ -212,19 +184,15 @@ def _normalize_fallback_ips(values: Iterable[str]) -> list[str]:
continue
if addr.version != 4:
logger.warning("Ignoring non-IPv4 Telegram fallback IP: %s", raw)
continue
if addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_unspecified:
elif addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_unspecified:
logger.warning("Ignoring private/internal Telegram fallback IP: %s", raw)
continue
normalized.append(str(addr))
else:
normalized.append(str(addr))
return normalized
def parse_fallback_ip_env(value: str | None) -> list[str]:
if not value:
return []
parts = [part.strip() for part in value.split(",")]
return _normalize_fallback_ips(parts)
return _normalize_fallback_ips(part.strip() for part in value.split(",")) if value else []
def _resolve_system_dns() -> set[str]:
@@ -249,9 +217,9 @@ async def _query_doh_provider(client: httpx.AsyncClient, provider: dict) -> list
raw = answer.get("data", "").strip()
try:
ipaddress.ip_address(raw)
ips.append(raw)
except ValueError:
continue
ips.append(raw)
return ips
except Exception as exc:
logger.debug("DoH query to %s failed: %s", provider["url"], exc)
@@ -259,19 +227,13 @@ async def _query_doh_provider(client: httpx.AsyncClient, provider: dict) -> list
async def discover_fallback_ips() -> list[str]:
"""Resolve api.telegram.org via Google + Cloudflare DoH; unique A records, in order.
IPs matching the system resolver are deliberately KEPT (often the most reliable
path; a transient primary failure should retry it via IP-rewrite before the seed
list). Falls back to ``SEED_FALLBACK_IPS`` only when DoH yields nothing usable.
"""
"""Resolve api.telegram.org via Google + Cloudflare DoH; unique A records, in order. IPs matching the
system resolver are deliberately KEPT (often the most reliable path). Falls back to
``SEED_FALLBACK_IPS`` only when DoH yields nothing usable."""
async with httpx.AsyncClient(timeout=httpx.Timeout(_DOH_TIMEOUT)) as client:
doh_tasks = [_query_doh_provider(client, p) for p in _DOH_PROVIDERS]
system_dns_task = asyncio.ensure_future(asyncio.to_thread(_resolve_system_dns))
results = await asyncio.gather(*doh_tasks, return_exceptions=True)
# The getaddrinfo leg has no timeout of its own (a wedged resolver can sit for
# minutes) and only feeds the log line below — bound it, never gate discovery on it.
results = await asyncio.gather(*[_query_doh_provider(client, p) for p in _DOH_PROVIDERS], return_exceptions=True)
# The getaddrinfo leg has no timeout of its own and only feeds the log line below — bound it.
system_ips: set[str] = set()
try:
system_result = await asyncio.wait_for(system_dns_task, timeout=_DOH_TIMEOUT)
@@ -279,21 +241,14 @@ async def discover_fallback_ips() -> list[str]:
system_ips = system_result
except Exception:
logger.debug("System-DNS resolution for %s did not complete in time", _TELEGRAM_API_HOST)
doh_ips: list[str] = []
for r in results:
if isinstance(r, list):
doh_ips.extend(r)
doh_ips = [ip for r in results if isinstance(r, list) for ip in r]
validated = _normalize_fallback_ips(list(dict.fromkeys(doh_ips))) # dedupe, keep order
if validated:
logger.debug("Discovered Telegram fallback IPs via DoH: %s", ", ".join(validated))
return validated
logger.info(
"DoH discovery yielded no usable IPs (system DNS: %s); using seed fallback IPs %s",
", ".join(system_ips) or "unknown",
", ".join(SEED_FALLBACK_IPS),
)
", ".join(system_ips) or "unknown", ", ".join(SEED_FALLBACK_IPS))
return list(SEED_FALLBACK_IPS)
@@ -304,9 +259,7 @@ def _rewrite_request_for_ip(request: httpx.Request, ip: str) -> httpx.Request:
headers["host"] = original_host
extensions = dict(request.extensions)
extensions["sni_hostname"] = original_host
return httpx.Request(
method=request.method, url=url, headers=headers, stream=request.stream, extensions=extensions,
)
return httpx.Request(method=request.method, url=url, headers=headers, stream=request.stream, extensions=extensions)
def _is_retryable_connect_error(exc: Exception) -> bool:
@@ -194,7 +194,8 @@ class TestThreadNotFoundFallbackSitesPruneBinding:
def test_streaming_send_fallback_calls_prune(self):
from plugins.platforms.telegram import adapter as telegram_mod
src = inspect.getsource(telegram_mod.TelegramAdapter.send)
# The per-chunk retry loop (with the fallback) lives in _send_chunk_with_retries, called by send().
src = inspect.getsource(telegram_mod.TelegramAdapter._send_chunk_with_retries)
# Locate the second-failure branch (the one that flips
# ``used_thread_fallback``). It must invoke the prune
# helper before flipping the flag.