commit c2743251e9e6ff2b989cb9c869a67bdc3dbf2f56 Author: m4 Date: Mon Jul 13 08:07:45 2026 +0800 Initial commit of EvoScientist framework Self-evolving AI scientist framework built on LangGraph/LangChain with CLI/TUI core, FastAPI gateway, and Next.js frontend. Co-Authored-By: Claude Opus 4 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..43bab4c --- /dev/null +++ b/.env.example @@ -0,0 +1,27 @@ +# EvoScientist CLI environment variables +# The preferred configuration flow is `evosci onboard`, which writes +# ~/.evoscientist/config/settings.yaml. Environment variables can override it. + +# Optional application directories +# EVOSCIENTIST_HOME=~/.evoscientist +# EVOSCIENTIST_DATA_ROOT=~/.evoscientist/data + +# Logging +EVOSCIENTIST_LOG_LEVEL=INFO +# EVOSCIENTIST_LOG_DIR=~/.evoscientist/data/logs +EVOSCIENTIST_LOG_RETENTION_DAYS=30 + +# Optional PostgreSQL checkpoint storage. Without this value the CLI uses an +# in-memory checkpointer for the current process. +# EVOSCIENTIST_SESSION_DB_URL=postgresql://user:password@localhost:5432/evoscientist + +# Model provider examples. Provider-specific settings can also be configured +# through `evosci onboard`. +# OPENAI_API_KEY=sk-... +# OPENAI_BASE_URL=https://api.openai.com/v1 +# ANTHROPIC_API_KEY=sk-ant-... +# GOOGLE_API_KEY=... +# TAVILY_API_KEY=tvly-... + +# Optional default model +# DEFAULT_MODEL=openai/gpt-5.4 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..bfca2fc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Report a bug to help us improve EvoScientist +title: '' +labels: bug +assignees: '' +type: Bug + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Run `EvoSci ...` +2. Provide input '...' +3. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Error output / logs** +If applicable, paste the relevant terminal output or traceback. + +``` + +``` + +**System information** + - OS: [e.g. Ubuntu 22.04, macOS 14.3, Windows 11] + - Python version: [e.g. 3.11.8] + - Terminal: [e.g. iTerm2, Alacritty, Windows Terminal] + - EvoScientist version: [e.g. 0.0.2 — run `EvoSci --version`] + - LLM provider: [e.g. Anthropic, OpenAI, Ollama] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..878e52b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: EvoSkills — niche features & custom workflows + url: https://github.com/EvoScientist/EvoSkills + about: Want to add a feature that serves a specific use case rather than all users? Contribute it as an EvoSkill instead. + - name: Community Discord + url: https://discord.gg/AZ9ZMXkunY + about: Ask questions, get help, and chat with the community. diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000..3feac05 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,18 @@ +--- +name: Documentation +about: Report missing, incorrect, or unclear documentation +title: '' +labels: documentation +assignees: '' +type: Documentation + +--- + +**What documentation is affected?** +Describe which part of the project needs better docs. + +**What is wrong or missing?** +Describe the issue — is something inaccurate, outdated, hard to follow, or missing entirely? + +**Suggested improvement** +If you have a specific fix or addition in mind, describe it here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..8dec7cb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,24 @@ +--- +name: Feature request +about: Suggest an idea for EvoScientist core functionality +title: '' +labels: enhancement +assignees: '' +type: Feature + +--- + +> **Note:** We only accept feature requests for core functionality that benefits the majority of users. +> If your idea is a niche workflow or personal utility, consider contributing it to [EvoSkills](https://github.com/EvoScientist/EvoSkills) instead. + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of the problem. Ex. "I'm always frustrated when [...]" + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..b6cc2c0 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ +## Description + + + +## Type of change + + + +- [ ] Bug fix +- [ ] New feature — link issue: # +- [ ] Documentation / examples +- [ ] Test improvement +- [ ] Refactor (no behavior change) + +## Checklist + +- [ ] I have read the [Contributing Guidelines](../CONTRIBUTING.md) +- [ ] This targets **core functionality** used by the majority of users (niche features belong in [EvoSkills](https://github.com/EvoScientist/EvoSkills)) +- [ ] I have added/updated tests where applicable +- [ ] `uv run ruff check .` passes +- [ ] `uv run pytest` passes diff --git a/.github/assets/EvoScientist_channels.png b/.github/assets/EvoScientist_channels.png new file mode 100644 index 0000000..76e257a Binary files /dev/null and b/.github/assets/EvoScientist_channels.png differ diff --git a/.github/assets/EvoScientist_cli.png b/.github/assets/EvoScientist_cli.png new file mode 100644 index 0000000..fdc07b9 Binary files /dev/null and b/.github/assets/EvoScientist_cli.png differ diff --git a/.github/assets/EvoScientist_cli_help.png b/.github/assets/EvoScientist_cli_help.png new file mode 100644 index 0000000..064c537 Binary files /dev/null and b/.github/assets/EvoScientist_cli_help.png differ diff --git a/.github/assets/EvoScientist_logo.png b/.github/assets/EvoScientist_logo.png new file mode 100644 index 0000000..2b6d76d Binary files /dev/null and b/.github/assets/EvoScientist_logo.png differ diff --git a/.github/assets/EvoScientist_mcp.png b/.github/assets/EvoScientist_mcp.png new file mode 100644 index 0000000..088c834 Binary files /dev/null and b/.github/assets/EvoScientist_mcp.png differ diff --git a/.github/assets/EvoScientist_onboard.png b/.github/assets/EvoScientist_onboard.png new file mode 100644 index 0000000..c3c7a63 Binary files /dev/null and b/.github/assets/EvoScientist_onboard.png differ diff --git a/.github/assets/EvoScientist_team.png b/.github/assets/EvoScientist_team.png new file mode 100644 index 0000000..cbd7ac4 Binary files /dev/null and b/.github/assets/EvoScientist_team.png differ diff --git a/.github/assets/ICAIS_awards.JPG b/.github/assets/ICAIS_awards.JPG new file mode 100644 index 0000000..e8703c0 Binary files /dev/null and b/.github/assets/ICAIS_awards.JPG differ diff --git a/.github/assets/ICAIS_best_paper.png b/.github/assets/ICAIS_best_paper.png new file mode 100644 index 0000000..7e36ebd Binary files /dev/null and b/.github/assets/ICAIS_best_paper.png differ diff --git a/.github/assets/asta_bench_code.png b/.github/assets/asta_bench_code.png new file mode 100644 index 0000000..de67c5a Binary files /dev/null and b/.github/assets/asta_bench_code.png differ diff --git a/.github/assets/asta_bench_data.png b/.github/assets/asta_bench_data.png new file mode 100644 index 0000000..06cc39b Binary files /dev/null and b/.github/assets/asta_bench_data.png differ diff --git a/.github/assets/badge-framework-dark.svg b/.github/assets/badge-framework-dark.svg new file mode 100644 index 0000000..0e877ad --- /dev/null +++ b/.github/assets/badge-framework-dark.svg @@ -0,0 +1,9 @@ + + + Built with + + DeepAgents + \ No newline at end of file diff --git a/.github/assets/badge-framework-light.svg b/.github/assets/badge-framework-light.svg new file mode 100644 index 0000000..87c6077 --- /dev/null +++ b/.github/assets/badge-framework-light.svg @@ -0,0 +1,9 @@ + + + Built with + + DeepAgents + \ No newline at end of file diff --git a/.github/assets/badge-license-dark.svg b/.github/assets/badge-license-dark.svg new file mode 100644 index 0000000..c7106df --- /dev/null +++ b/.github/assets/badge-license-dark.svg @@ -0,0 +1,9 @@ + + + License + + Apache 2.0 + \ No newline at end of file diff --git a/.github/assets/badge-license-light.svg b/.github/assets/badge-license-light.svg new file mode 100644 index 0000000..334a5e5 --- /dev/null +++ b/.github/assets/badge-license-light.svg @@ -0,0 +1,9 @@ + + + License + + Apache 2.0 + \ No newline at end of file diff --git a/.github/assets/badge-pypi-dark.svg b/.github/assets/badge-pypi-dark.svg new file mode 100644 index 0000000..31154d8 --- /dev/null +++ b/.github/assets/badge-pypi-dark.svg @@ -0,0 +1,9 @@ + + + PyPI + + v0.0.7 + \ No newline at end of file diff --git a/.github/assets/badge-pypi-light.svg b/.github/assets/badge-pypi-light.svg new file mode 100644 index 0000000..ad30b27 --- /dev/null +++ b/.github/assets/badge-pypi-light.svg @@ -0,0 +1,9 @@ + + + PyPI + + v0.0.7 + \ No newline at end of file diff --git a/.github/assets/badge-website-dark.svg b/.github/assets/badge-website-dark.svg new file mode 100644 index 0000000..a57eb45 --- /dev/null +++ b/.github/assets/badge-website-dark.svg @@ -0,0 +1,9 @@ + + + Website + + Page + \ No newline at end of file diff --git a/.github/assets/badge-website-light.svg b/.github/assets/badge-website-light.svg new file mode 100644 index 0000000..b0193b1 --- /dev/null +++ b/.github/assets/badge-website-light.svg @@ -0,0 +1,9 @@ + + + Website + + Page + \ No newline at end of file diff --git a/.github/assets/cn_info.md b/.github/assets/cn_info.md new file mode 100644 index 0000000..d27a824 --- /dev/null +++ b/.github/assets/cn_info.md @@ -0,0 +1,7 @@ +# 📱 EvoScientist 微信二维码 + +### 👥 微信交流群: + + + +微信群二维码 \ No newline at end of file diff --git a/.github/assets/deepresearch_bench_2.JPG b/.github/assets/deepresearch_bench_2.JPG new file mode 100644 index 0000000..0f7c846 Binary files /dev/null and b/.github/assets/deepresearch_bench_2.JPG differ diff --git a/.github/assets/logo-dark.svg b/.github/assets/logo-dark.svg new file mode 100644 index 0000000..0432e6d --- /dev/null +++ b/.github/assets/logo-dark.svg @@ -0,0 +1,15 @@ + + + + + + EvoScientist + + + \ No newline at end of file diff --git a/.github/assets/logo-light.svg b/.github/assets/logo-light.svg new file mode 100644 index 0000000..97b0a10 --- /dev/null +++ b/.github/assets/logo-light.svg @@ -0,0 +1,15 @@ + + + + + + EvoScientist + + + \ No newline at end of file diff --git a/.github/assets/wechat_group.jpeg b/.github/assets/wechat_group.jpeg new file mode 100644 index 0000000..6b0e42a Binary files /dev/null and b/.github/assets/wechat_group.jpeg differ diff --git a/.github/assets/xiaoyi-deepresearch.webp b/.github/assets/xiaoyi-deepresearch.webp new file mode 100644 index 0000000..710633d Binary files /dev/null and b/.github/assets/xiaoyi-deepresearch.webp differ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..1cb59a3 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,24 @@ +name: Build + +on: + push: + branches: ["main"] + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.11" + cache-dependency-glob: "**/pyproject.toml" + - name: Build package + run: uv build + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..8678300 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,23 @@ +name: Lint + +on: + push: + branches: ["main"] + pull_request: + +jobs: + ruff: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.11" + cache-dependency-glob: "**/pyproject.toml" + - name: Install dependencies + run: uv sync --dev + - name: Run ruff linter + run: uv run ruff check --output-format=github . + - name: Run ruff format + run: uv run ruff format --check . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..714940d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: Test + +on: + push: + branches: ["main"] + pull_request: + +jobs: + pytest: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + matrix: + python-version: ["3.11", "3.12"] + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v6 + with: + python-version: ${{ matrix.python-version }} + cache-dependency-glob: "**/pyproject.toml" + - name: Install dependencies + run: uv sync --dev + - name: Run pytest + run: uv run pytest -v --timeout=30 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5c90c6c --- /dev/null +++ b/.gitignore @@ -0,0 +1,67 @@ +# macOS +.DS_Store + +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +dist/ +build/ +*.egg +*.pytest_cache/ +.coverage +.ipynb_checkpoints/ + +# Environment +.env +.env.* +.env_* +!.env.example +.venv/ +venv/ +bridge/node_modules/ +bridge/package-lock.json + +# IDE / Tools +.vscode/ +.idea/ +.agents/ +.claude/ +.cursor/ +.codex/ +.cursorrules +.ruff_cache/ + +# Project specific +.langgraph_api/ +workspace/ +skills/ +memory/ +media/ +conversation_history/ +.deno_cache/ +*.ipynb +*CLAUDE.md +*AGENTS.md +*meals/ +botpy.log +large_tool_results/ + +# Docker runtime data +docker/data/ + +# Project-level config data +.data/ + +# Sensitive / credentials (never commit) +postgresql:* +_s3_backup/ + +# Local scratch / tooling data +.superpowers/ +.test-home/ +tmp/ + +# Root-level debug scratch scripts (proper tests live in tests/) +/test_*.py +/research_lookup_temp.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3f2b222 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.9.9 + hooks: + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f35f2ee --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,111 @@ +# Contributing to EvoScientist + +We appreciate your interest and the time you spend helping improve EvoScientist. Please read the following guidelines before contributing. + +## How you can contribute + +- **Report bugs and request features:** open an issue using the provided templates. Make sure to use the correct template and labels. +- **Propose design changes:** use issues or discussion threads to outline the problem, alternatives, and trade-offs before implementing. +- **Contribute code or docs:** submit PRs that address an open issue. They must have a clear rationale and tests where applicable. + +## What we are looking for in PRs + +We aim to keep EvoScientist focused on core functionality that benefits the majority of users. PRs should only include: + +- Bug fixes / improvements to existing features +- New features that were proposed in an issue and agreed upon with maintainers +- Documentation updates and examples +- Meaningful additions to the test suite + +If you want to add a niche or specialized workflow, consider contributing to the [EvoSkills repository](https://github.com/EvoScientist/EvoSkills) instead. + +## Development setup + +1. **Fork and clone** the repository: + ```bash + git clone https://github.com//EvoScientist.git + cd EvoScientist + ``` + +2. **Install dependencies** (requires [uv](https://docs.astral.sh/uv/)): + ```bash + uv sync --dev + ``` + +3. **Run the test suite** (no API keys needed): + ```bash + uv run pytest + ``` + +4. **Run the linter:** + ```bash + uv run ruff check . + ``` + +## Submitting a pull request + +1. Create a branch from `main` with a descriptive name (e.g. `fix/session-crash`, `feat/export-csv`). +2. Make your changes, keeping commits focused and well-described. +3. Ensure `uv run ruff check .` and `uv run pytest` pass locally — these also run in CI. +4. Open a PR against `main` and fill in the PR template. +5. A maintainer will review your PR. Please be responsive to feedback. + +## Code style + +- We use [Ruff](https://docs.astral.sh/ruff/) for linting. Run `uv run ruff check .` before pushing. +- Follow the existing code patterns and conventions in the area you're modifying. +- Keep changes minimal and focused on the task at hand. + +--- + +## Project overview + +EvoScientist is a multi-agent AI system for automated scientific experimentation and discovery. It orchestrates specialized sub-agents that plan experiments, search literature, write code, debug, analyze data, and draft reports. + +| Fact | Value | +|------|-------| +| Language | Python 3.11+ | +| License | Apache 2.0 | +| Framework | [DeepAgents](https://github.com/langchain-ai/deepagents) + [LangChain](https://python.langchain.com/) + [LangGraph](https://langchain-ai.github.io/langgraph/) | +| Default model | `claude-sonnet-4-6` (Anthropic) | +| Tests | ~890 across 36 files, no API keys needed | +| Config file | `.data/.config/settings.yaml` (project root) | + +### Sub-Agents (defined in `EvoScientist/subagent.yaml`) + +| Agent | Purpose | +|-------|---------| +| `planner-agent` | Creates and updates experimental plans (no web search, no implementation) | +| `research-agent` | Web research for methods, baselines, and datasets (Tavily search) | +| `code-agent` | Implements experiment code and runnable scripts | +| `debug-agent` | Reproduces failures, identifies root causes, applies minimal fixes | +| `data-analysis-agent` | Computes metrics, creates plots, summarizes insights | +| `writing-agent` | Drafts paper-ready Markdown experiment reports | + +### Data flow + +```txt +User Input (CLI / TUI / 10 Channel Integrations) + | +CLI (cli/) / TUI (cli/tui_*) / Channel Server (channels/) + | +Main Agent (EvoScientist.py) -- create_deep_agent() + +-- System Prompt (prompts.py) + +-- Chat Model (llm/ -- multi-provider) + +-- Middleware: Memory (middleware/memory.py) + +-- Backend: CompositeBackend (backends.py) + | / --> CustomSandboxBackend (workspace read/write + execute) + | /skills/ --> MergedReadOnlyBackend (user > built-in) + | /memory/ --> FilesystemBackend (persistent cross-session) + +-- MCP Tools (mcp/ -- optional, cached by config signature) + | +task tool --> Delegates to Sub-Agents + | +Stream Events --> Emitter --> Tracker --> State --> Rich Display / TUI +``` + +--- + +## Need help? + +Reach us on [Discord](https://discord.gg/AZ9ZMXkunY) or WeChat (linked in README). diff --git a/EvoScientist/EvoScientist.py b/EvoScientist/EvoScientist.py new file mode 100644 index 0000000..44afc64 --- /dev/null +++ b/EvoScientist/EvoScientist.py @@ -0,0 +1,766 @@ +"""EvoScientist Agent graph construction. + +This module defines the agent graph and its factory functions. All heavy +initialization (deepagents, backends, LLM, middleware) is deferred to first +use so that importing this module is fast and non-agent CLI commands +(``EvoSci config list``, ``EvoSci onboard``) never pay the cost. + +Usage: + from EvoScientist import EvoScientist_agent + + # Notebook / programmatic usage + for state in EvoScientist_agent.stream( + {"messages": [HumanMessage(content="your question")]}, + config={"configurable": {"thread_id": "1"}}, + stream_mode="values", + ): + ... +""" + +import json +import logging +import os +from datetime import datetime +from pathlib import Path +from typing import Any + +from langchain.agents.middleware import AgentMiddleware + +from . import paths as _paths_mod +from .config import apply_config_to_env, get_effective_config +from .paths import set_active_workspace, set_workspace_root +from .path_semantics import path_semantics_prompt +from .prompts import ( + RESEARCHER_INSTRUCTIONS, + USER_FACING_LANGUAGE_POLICY, + get_system_prompt, +) + +# Suppress noisy warnings from deepagents skill loader (non-string frontmatter fields, etc.) +logging.getLogger("deepagents.middleware.skills").setLevel(logging.ERROR) +logger = logging.getLogger(__name__) + +# ============================================================================= +# Constants +# ============================================================================= + +SUBAGENTS_CONFIG = Path(__file__).parent / "subagent.yaml" +SKILLS_DIR = str(Path(__file__).parent / "skills") + +# ============================================================================= +# Lazy state — initialized on first use, not at import time +# ============================================================================= + +_config = None +_chat_model = None + +# Cache MCP tools by the effective config signature to avoid reconnecting +# to MCP servers on every `/new` when config is unchanged. +_MCP_TOOLS_CACHE_KEY: str | None = None +_MCP_TOOLS_CACHE_VALUE: dict[str, list] | None = None + +# Default agent (no checkpointer) — used by langgraph dev / LangSmith / notebooks. +# Lazily constructed on first access so MCP tools are included without +# spawning subprocesses at import time. +_EvoScientist_agent = None + + +# ============================================================================= +# Lazy initialization helpers +# ============================================================================= + + +def reset_runtime_caches() -> None: + """Clear cached default runtime objects after configuration changes.""" + import sys + + global _config, _chat_model, _EvoScientist_agent + _config = None + _chat_model = None + _EvoScientist_agent = None + package = sys.modules.get("EvoScientist") + if package is not None and "EvoScientist_agent" in vars(package): + delattr(package, "EvoScientist_agent") + + +def _ensure_config(config=None): + """Return cached config. If *config* is passed, cache and use it.""" + global _config + if config is not None: + _config = config + apply_config_to_env(_config) + if _config is None: + _config = get_effective_config() + apply_config_to_env(_config) + return _config + + +def _ensure_chat_model(): + """Return cached chat model, creating it on first call.""" + global _chat_model + if _chat_model is None: + from .llm import get_chat_model + + cfg = _ensure_config() + _chat_model = get_chat_model(model=cfg.model, provider=cfg.provider) + return _chat_model + + +# ============================================================================= +# MCP caching +# ============================================================================= + + +def _load_mcp_config_once() -> tuple[str, dict]: + """Load MCP config and return ``(signature, config)``. + + The signature includes the active workspace directory so that MCP + subprocesses are spawned with the correct ``cwd`` and environment + for each isolated workspace (e.g. per web thread). + """ + from .mcp.client import load_mcp_config + from .paths import _active_workspace + + cfg = load_mcp_config() + if not cfg: + logger.info("MCP startup: no MCP servers configured") + return "", {} + logger.info( + "MCP startup: loaded %d server config(s): %s", + len(cfg), + ", ".join(cfg.keys()), + ) + try: + sig = json.dumps({"mcp": cfg, "workspace": str(_active_workspace)}, sort_keys=True, ensure_ascii=True) + except TypeError: + sig = repr(cfg) + str(_active_workspace) + return sig, cfg + + +def _load_mcp_tools_cached() -> dict[str, list]: + """Load MCP tools with config + workspace-aware caching. + + MCP subprocesses are spawned with ``cwd`` and ``EVOSCIENTIST_WORKSPACE_DIR`` + set to the active workspace. Including the workspace in the cache key + ensures that each isolated workspace (e.g. per web thread) gets its own + MCP subprocess with the correct working directory. + """ + global _MCP_TOOLS_CACHE_KEY, _MCP_TOOLS_CACHE_VALUE + + from .mcp import load_mcp_tools + + cfg_key, cfg = _load_mcp_config_once() + if not cfg_key: + _MCP_TOOLS_CACHE_KEY = "" + _MCP_TOOLS_CACHE_VALUE = {} + return {} + + if _MCP_TOOLS_CACHE_KEY == cfg_key and _MCP_TOOLS_CACHE_VALUE is not None: + logger.info( + "MCP startup: using cached MCP tools for %d agent route(s)", + len(_MCP_TOOLS_CACHE_VALUE), + ) + return {k: list(v) for k, v in _MCP_TOOLS_CACHE_VALUE.items()} + + loaded = load_mcp_tools(config=cfg) + if not loaded: + logger.warning("MCP startup: no tools loaded from configured MCP servers") + _MCP_TOOLS_CACHE_KEY = cfg_key + _MCP_TOOLS_CACHE_VALUE = {} + return {} + _MCP_TOOLS_CACHE_KEY = cfg_key + _MCP_TOOLS_CACHE_VALUE = {k: list(v) for k, v in loaded.items()} + return {k: list(v) for k, v in loaded.items()} + + +# ============================================================================= +# Agent construction helpers +# ============================================================================= + + +def _inject_subagent_middleware(subs: list[dict]) -> None: + """Ensure every subagent gets error handling and context management middleware. + + Without this, subagent tool errors are caught by LangGraph's default + ToolNode handler which produces terse messages without tracebacks or + retry guidance — reducing the subagent's ability to self-recover. + """ + from .middleware import ( + ContextOverflowMapperMiddleware, + ToolErrorHandlerMiddleware, + create_context_editing_middleware, + ) + + for sa in subs: + sa.setdefault("middleware", []).extend( + [ + # Uses main agent's model for trigger — subagents currently + # share the same model, so context window matches. + create_context_editing_middleware(), + ToolErrorHandlerMiddleware(), + ContextOverflowMapperMiddleware(), + ] + ) + + +def _inject_subagent_language_policy(subs: list[dict]) -> None: + """Append the shared user-facing language policy to every subagent.""" + for sa in subs: + prompt = str(sa.get("system_prompt") or "") + if "## User-Facing Language Policy" not in prompt: + sa["system_prompt"] = prompt.rstrip() + "\n\n" + USER_FACING_LANGUAGE_POLICY + + +def _inject_subagent_path_policy(subs: list[dict]) -> None: + """Append the shared path semantics policy to every subagent.""" + policy = path_semantics_prompt() + for sa in subs: + prompt = str(sa.get("system_prompt") or "") + if "## Path Semantics" not in prompt: + sa["system_prompt"] = prompt.rstrip() + "\n\n" + policy + + +def _build_prompt_refs() -> dict: + """Build prompt references with the current date (not frozen at import).""" + return { + "RESEARCHER_INSTRUCTIONS": RESEARCHER_INSTRUCTIONS.format( + date=datetime.now().strftime("%Y-%m-%d"), + ), + } + + +def _build_base_kwargs(base_backend, base_middleware, *, chat_model=None): + """Build agent kwargs *without* MCP (fast, no subprocess spawning).""" + from .tools import ( + edit_image, + generate_image, + refresh_image_tool_descriptions, + skill_manager, + think_tool, + web_crawl, + web_extract, + web_search, + ) + from .utils import load_subagents + + tool_registry = {"think_tool": think_tool} + # Register unified search tools when any search backend key is available + _has_search = any(os.environ.get(k) for k in ( + "ZHIPU_API_KEY", "TAVILY_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "EXA_API_KEY", + )) + if _has_search: + tool_registry["web_search"] = web_search + tool_registry["web_extract"] = web_extract + tool_registry["web_crawl"] = web_crawl + refresh_image_tool_descriptions() + base_tools = [think_tool, skill_manager, generate_image, edit_image] + + subs = load_subagents( + SUBAGENTS_CONFIG, + tool_registry=tool_registry, + prompt_refs=_build_prompt_refs(), + ) + _inject_subagent_path_policy(subs) + _inject_subagent_language_policy(subs) + _inject_subagent_middleware(subs) + return { + "name": "EvoScientist", + "model": chat_model if chat_model is not None else _ensure_chat_model(), + "tools": list(base_tools), + "backend": base_backend, + "subagents": subs, + "middleware": base_middleware, + "system_prompt": get_system_prompt(has_parse_documents=False), + "skills": ["/skills/"], + } + + +def _apply_conversation_profile(kwargs: dict, conversation_mode: str | None) -> dict: + """Apply per-request capability limits without mutating shared inputs.""" + from .conversation_modes import get_conversation_profile + + profile = get_conversation_profile(conversation_mode) + if profile.enable_subagents: + return kwargs + adjusted = dict(kwargs) + adjusted["subagents"] = [] + return adjusted + + +def _resolve_reasoning_effort(effort: str | None, cfg) -> str: + """Resolve reasoning effort: explicit value wins, else fall back to config. + + Decoupled from conversation mode — the profile no longer carries a + reasoning_effort default, so the fallback is the top-level config field. + """ + if effort is not None: + return effort + return getattr(cfg, "reasoning_effort", "medium") + + +def _create_tool_selector_for_profile(factory, threshold: int): + """Call tool-selector factory with threshold, tolerating legacy test doubles.""" + try: + return factory(threshold=threshold) + except TypeError as exc: + if "threshold" not in str(exc) and "unexpected keyword" not in str(exc): + raise + return factory() + + +def load_mcp_and_build_kwargs(base_backend, base_middleware, *, chat_model=None): + """Load MCP tools (cached by config) and build agent kwargs. + + Re-connects to MCP servers only when the effective MCP config changes. + Falls back to base kwargs if no MCP configured. + """ + from .tools import ( + edit_image, + generate_image, + refresh_image_tool_descriptions, + skill_manager, + think_tool, + web_crawl, + web_extract, + web_search, + ) + from .utils import load_subagents + + mcp_by_agent = _load_mcp_tools_cached() + if not mcp_by_agent: + logger.info("MCP startup: agent will start without MCP tools") + return _build_base_kwargs(base_backend, base_middleware, chat_model=chat_model) + + tool_registry = {"think_tool": think_tool} + # Register unified search tools when any search backend key is available + _has_search = any(os.environ.get(k) for k in ( + "ZHIPU_API_KEY", "TAVILY_API_KEY", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "EXA_API_KEY", + )) + if _has_search: + tool_registry["web_search"] = web_search + tool_registry["web_extract"] = web_extract + tool_registry["web_crawl"] = web_crawl + refresh_image_tool_descriptions() + base_tools = [think_tool, skill_manager, generate_image, edit_image] + + # Fresh tool registry — start from base tools + MCP tools + registry = dict(tool_registry) + for tools in mcp_by_agent.values(): + for t in tools: + registry[t.name] = t + + mcp_main = mcp_by_agent.pop("main", []) + has_parse_documents = any(t.name == "parse_documents" for t in mcp_main) + logger.info( + "MCP startup: injecting %d MCP tool(s) into main agent: %s", + len(mcp_main), + ", ".join(t.name for t in mcp_main) or "(none)", + ) + + subs = load_subagents( + SUBAGENTS_CONFIG, + tool_registry=registry, + prompt_refs=_build_prompt_refs(), + ) + + _inject_subagent_language_policy(subs) + _inject_subagent_path_policy(subs) + _inject_subagent_middleware(subs) + + # Inject MCP tools into subagents by name + for sa in subs: + if sa_tools := mcp_by_agent.get(sa["name"], []): + sa.setdefault("tools", []).extend(sa_tools) + logger.info( + "MCP startup: injecting %d MCP tool(s) into subagent %s: %s", + len(sa_tools), + sa["name"], + ", ".join(t.name for t in sa_tools), + ) + + return { + "name": "EvoScientist", + "model": chat_model if chat_model is not None else _ensure_chat_model(), + "tools": base_tools + mcp_main, + "backend": base_backend, + "subagents": subs, + "middleware": base_middleware, + "system_prompt": get_system_prompt(has_parse_documents=has_parse_documents), + "skills": ["/skills/"], + } + + +# ============================================================================= +# Default agent (langgraph dev / notebooks) +# ============================================================================= + + +def _get_default_backend(): + """Build the default composite backend from current paths.""" + from deepagents.backends import CompositeBackend, FilesystemBackend + + from .backends import CustomSandboxBackend, MergedReadOnlyBackend + + workspace_dir = str(_paths_mod.WORKSPACE_ROOT) + set_active_workspace(workspace_dir) + memory_dir = str(_paths_mod.MEMORY_DIR) + user_skills_dir = str(_paths_mod.USER_SKILLS_DIR) + global_skills_dir = str(_paths_mod.GLOBAL_SKILLS_DIR) + + ws_backend = CustomSandboxBackend( + root_dir=workspace_dir, + virtual_mode=True, + timeout=300, + ) + sk_backend = MergedReadOnlyBackend( + primary_dir=user_skills_dir, + global_dir=global_skills_dir, + secondary_dir=SKILLS_DIR, + ) + mem_backend = FilesystemBackend( + root_dir=memory_dir, + virtual_mode=True, + ) + return CompositeBackend( + default=ws_backend, + routes={ + "/skills/": sk_backend, + "/memory/": mem_backend, + }, + ) + + +def _get_default_middleware(): + """Build the default middleware list.""" + from .middleware import ( + ContextOverflowMapperMiddleware, + ToolErrorHandlerMiddleware, + create_context_editing_middleware, + create_memory_middleware, + create_tool_selector_middleware, + ) + + cfg = _ensure_config() + model = _ensure_chat_model() + memory_dir = str(_paths_mod.MEMORY_DIR) + mw: list[AgentMiddleware] = [ + create_context_editing_middleware(model), + ContextOverflowMapperMiddleware(), + ToolErrorHandlerMiddleware(), + *create_tool_selector_middleware(), + create_memory_middleware(memory_dir, extraction_model=model), + ] + + if cfg.enable_ask_user and not cfg.auto_mode: + from .middleware.ask_user import AskUserMiddleware + + mw.insert(0, AskUserMiddleware()) + return mw + + +def _get_default_agent(): + """Build the default agent (with MCP, no checkpointer) on first access.""" + global _EvoScientist_agent + if _EvoScientist_agent is None: + from deepagents import create_deep_agent + + be = _get_default_backend() + mw = _get_default_middleware() + kwargs = load_mcp_and_build_kwargs(be, mw) + _EvoScientist_agent = create_deep_agent(**kwargs).with_config( + {"recursion_limit": 1000} + ) + return _EvoScientist_agent + + +def __getattr__(name: str): + if name == "EvoScientist_agent": + return _get_default_agent() + # Backward compat for module-level names + if name == "chat_model": + return _ensure_chat_model() + if name == "SYSTEM_PROMPT": + return get_system_prompt(has_parse_documents=False) + if name == "backend": + return _get_default_backend() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +# ============================================================================= +# CLI agent factory +# ============================================================================= + + +def create_cli_agent( + workspace_dir: str | None = None, + checkpointer=None, + config=None, + memory_dir: str | None = None, + source: str = "cli", + user_id: str | None = None, + model: str | None = None, + provider: str | None = None, + reasoning_effort: str | None = None, + # ── Compute (Phase 2B v1.6) ───────────────────────────────────── + compute_client=None, # ComputeClient for remote execution + storage_service=None, # StorageService for object storage + app_state=None, # Gateway FastAPI app.state + plan: str = "starter", # User plan for quota checks + thread_id: str = "", # Thread ID for compute billing + compute_quota_ok: bool = False, + remaining_compute_minutes: int = 0, + input_objects: list | None = None, + artifact_prefix: str = "", + default_output_paths: list[str] | None = None, + max_runtime_seconds: int | None = None, + budget_snapshot: dict | None = None, + storage_backend: str = "nfs", + execution_mode: str = "standard", + resource_class: str = "small", + backend_policy: str = "auto", + compute_fallback_mode: str = "local_sandbox", + conversation_mode: str | None = None, + _backend_ref: dict | None = None, +): + """Create agent with checkpointer for CLI multi-turn support. + + A fresh backend is constructed on every call using the current + ``paths.WORKSPACE_ROOT`` (or the explicit *workspace_dir*), so + runtime ``set_workspace_root()`` changes are always respected. + + Args: + workspace_dir: Per-session workspace directory. If ``None``, + defaults to the current ``paths.WORKSPACE_ROOT``. + checkpointer: Optional LangGraph checkpointer. If ``None``, + falls back to ``InMemorySaver`` (non-persistent). + config: Optional pre-loaded ``EvoScientistConfig``. If ``None``, + loads from file/env/defaults. Passing this avoids double + loading when the CLI has already loaded config. + memory_dir: Optional memory directory override. If ``None``, + uses the default global memory path. Gateway passes per-user path. + source: Thread source identifier (\"cli\" or \"web\"). Default \"cli\". + user_id: Optional user identifier for multi-tenant isolation. + model: Optional model override (short name or full ID). Gateway + passes the user-selected model from the frontend. + provider: Optional provider override. Inferred from *model* if + not specified. + reasoning_effort: Optional reasoning effort level for OpenAI models. + One of "low", "medium", "high". Passed to the LLM as + ``reasoning={"effort": value, "summary": "auto"}``. + """ + import os as _os + + from deepagents import create_deep_agent + from deepagents.backends import CompositeBackend, FilesystemBackend + + from . import paths as _paths + from .backends import ( + CustomSandboxBackend, + MergedReadOnlyBackend, + MultiRootSandboxBackend, + NonComputingFallbackBackend, + ) + from .middleware import ( + ContextOverflowMapperMiddleware, + ToolErrorHandlerMiddleware, + create_context_editing_middleware, + create_memory_middleware, + create_tool_selector_middleware, + ToolResultLimiterMiddleware, + ) + + cfg = _ensure_config(config) + from .conversation_modes import ( + get_conversation_profile, + reset_current_conversation_mode, + set_current_conversation_mode, + ) + + mode_profile = get_conversation_profile(conversation_mode or getattr(cfg, "conversation_mode", "medium")) + + if checkpointer is None: + from langgraph.checkpoint.memory import InMemorySaver + + checkpointer = InMemorySaver() + + # When no explicit workspace_dir is provided, apply config.default_workdir + # as a fallback. This covers direct callers (notebooks, iMessage server) + # that never call set_workspace_root() themselves. CLI callers always + # pass workspace_dir explicitly, so their --workdir is never overwritten. + if workspace_dir is None: + if cfg.default_workdir: + set_workspace_root( + _os.path.abspath(_os.path.expanduser(cfg.default_workdir)) + ) + workspace_dir = str(_paths.WORKSPACE_ROOT) + + # Read paths dynamically so runtime set_workspace_root() changes are picked up + _mem_dir = memory_dir or str(_paths.MEMORY_DIR) + _usr_skills_dir = str(_paths.USER_SKILLS_DIR) + _global_skills_dir = str(_paths.GLOBAL_SKILLS_DIR) + + # Always construct fresh backends from current paths (avoids stale + # module-level backend when workspace root changed at runtime). + set_active_workspace(workspace_dir) + + if source == "web" and user_id: + # Web user: use remote compute only when the v1.6 preflight approved it. + if ( + compute_client is not None + and getattr(compute_client, "ready", False) + and compute_quota_ok + and storage_backend in ("nfs", "file_gateway") + ): + from .container_sandbox import ContainerSandboxBackend + + ws_backend = ContainerSandboxBackend( + compute_client=compute_client, + user_id=str(user_id), + thread_id=thread_id, + input_objects=input_objects or [], + artifact_prefix=artifact_prefix, + default_output_paths=default_output_paths or ["/workspace/artifacts"], + max_runtime_seconds=max_runtime_seconds, + budget_snapshot=budget_snapshot or {}, + execution_mode=execution_mode, + resource_class=resource_class, + backend_policy=backend_policy, + ) + if _backend_ref is not None: + _backend_ref["backend"] = ws_backend + else: + # Fallback: file browsing/recovery only. Web shell execution is disabled. + from .paths import global_data_dir, user_data_dir + + _uid = str(user_id) + if storage_backend == "file_gateway": + try: + from .runtime_integrations import resolve_user_storage_root + + _user_root = resolve_user_storage_root(_uid) + if _user_root is None: + raise RuntimeError("No external user storage provider is configured") + _user_global = _user_root / "__global__" + except Exception: + # The mounted thread workspace is authoritative in + # FileGateway mode, even when the host integration provider + # is unavailable (for example in isolated tests/tools). + _user_root = Path(workspace_dir).resolve().parent + _user_global = _user_root / "__global__" + else: + _user_root = user_data_dir(_uid) + _user_global = global_data_dir(_uid) + _thread_root = Path(workspace_dir).resolve() + + # Collect peer thread dirs (same user, other threads) as read-only sources + _peer_threads: list[str] = [] + if _user_root.is_dir(): + for d in sorted(_user_root.iterdir()): + if d.is_dir() and d.name != "__global__" and d.resolve() != _thread_root: + _peer_threads.append(str(d)) + + reason = "remote compute is unavailable" + if storage_backend not in ("nfs", "file_gateway"): + reason = f"storage backend {storage_backend!r} is unsupported for Web compute" + elif not compute_quota_ok: + reason = "compute quota preflight failed" + fallback_mode = ( + compute_fallback_mode + or os.getenv("COMPUTE_FALLBACK_MODE", "local_sandbox") + or "local_sandbox" + ).strip().lower() + if fallback_mode == "local_sandbox": + logger.warning( + "web_remote_compute_unavailable_using_local_sandbox thread=%s reason=%s", + thread_id, + reason, + ) + ws_backend = MultiRootSandboxBackend( + write_root=str(_thread_root), + read_roots=_peer_threads, + global_root=str(_user_global), + virtual_mode=True, + timeout=max_runtime_seconds or 300, + ) + else: + ws_backend = NonComputingFallbackBackend( + write_root=str(_thread_root), + read_roots=_peer_threads, + global_root=str(_user_global), + virtual_mode=True, + reason=reason, + ) + else: + # CLI user: isolated sandbox (unchanged behavior) + ws_backend = CustomSandboxBackend( + root_dir=workspace_dir, + virtual_mode=True, + timeout=300, + ) + sk_backend = MergedReadOnlyBackend( + primary_dir=_usr_skills_dir, + global_dir=_global_skills_dir, + secondary_dir=SKILLS_DIR, + ) + # Memory always uses SHARED directory (not per-session) for cross-session persistence + mem_backend = FilesystemBackend( + root_dir=_mem_dir, + virtual_mode=True, + ) + be = CompositeBackend( + default=ws_backend, + routes={ + "/skills/": sk_backend, + "/memory/": mem_backend, + }, + ) + + # Create chat model — use override if provided, otherwise use cached default + reasoning_effort = _resolve_reasoning_effort(reasoning_effort, cfg) + + if model: + from .llm import get_chat_model + + chat_model = get_chat_model(model=model, provider=provider) + else: + chat_model = _ensure_chat_model() + + mw: list[AgentMiddleware] = [ + create_context_editing_middleware(chat_model), + ContextOverflowMapperMiddleware(), + ToolErrorHandlerMiddleware(), + ToolResultLimiterMiddleware(mode_profile.max_tool_result_chars), + *_create_tool_selector_for_profile( + create_tool_selector_middleware, + mode_profile.tool_selector_threshold, + ), + create_memory_middleware( + _mem_dir, + extraction_model=chat_model, + inject_memory=mode_profile.memory_injection_enabled, + max_injected_chars=mode_profile.memory_injection_max_chars, + ), + ] + if cfg.enable_ask_user and not cfg.auto_mode: + from .middleware.ask_user import AskUserMiddleware + + mw.insert(0, AskUserMiddleware()) + + # Re-load MCP tools from current config (picks up /mcp add changes) + kwargs = load_mcp_and_build_kwargs(be, mw, chat_model=chat_model) + kwargs = _apply_conversation_profile(kwargs, mode_profile.mode) + + # HITL: gate shell execution for user approval + _interrupt_on: dict[str, bool] | None = None + if not cfg.auto_approve: + _interrupt_on = {"execute": True} + + mode_token = set_current_conversation_mode(mode_profile.mode) + try: + return create_deep_agent( + **kwargs, + checkpointer=checkpointer, + interrupt_on=_interrupt_on, + ).with_config({"recursion_limit": 1000}) + finally: + reset_current_conversation_mode(mode_token) diff --git a/EvoScientist/__init__.py b/EvoScientist/__init__.py new file mode 100644 index 0000000..87ae462 --- /dev/null +++ b/EvoScientist/__init__.py @@ -0,0 +1,79 @@ +"""EvoScientist Agent - AI-powered research and code execution. + +This package exposes a convenience API at the package root while keeping +imports lazy, so lightweight modules (for example config helpers) can be used +without importing heavy runtime dependencies. +""" + +from __future__ import annotations + +from importlib import import_module + +from ._version import __version__ + +_EXPORTS: dict[str, tuple[str, str]] = { + # Agent graph (lazy to avoid expensive initialization at import time) + "EvoScientist_agent": (".EvoScientist", "EvoScientist_agent"), + "create_cli_agent": (".EvoScientist", "create_cli_agent"), + # Backends + "CustomSandboxBackend": (".backends", "CustomSandboxBackend"), + "ReadOnlyFilesystemBackend": (".backends", "ReadOnlyFilesystemBackend"), + # Configuration + "EvoScientistConfig": (".config", "EvoScientistConfig"), + "load_config": (".config", "load_config"), + "save_config": (".config", "save_config"), + "get_effective_config": (".config", "get_effective_config"), + "get_config_path": (".config", "get_config_path"), + # LLM + "get_chat_model": (".llm", "get_chat_model"), + "list_models": (".llm", "list_models"), + # Prompts + "get_system_prompt": (".prompts", "get_system_prompt"), + "RESEARCHER_INSTRUCTIONS": (".prompts", "RESEARCHER_INSTRUCTIONS"), + # Tools + "tavily_search": (".tools", "tavily_search"), + "web_search": (".tools", "web_search"), + "web_extract": (".tools", "web_extract"), + "web_crawl": (".tools", "web_crawl"), + "think_tool": (".tools", "think_tool"), + # Sessions + "get_checkpointer": (".sessions", "get_checkpointer"), + "generate_thread_id": (".sessions", "generate_thread_id"), + "list_threads": (".sessions", "list_threads"), + "delete_thread": (".sessions", "delete_thread"), + "get_storage_stats": (".sessions", "get_storage_stats"), + "get_aggregated_storage_stats": (".sessions", "get_aggregated_storage_stats"), + "list_all_session_db_paths": (".sessions", "list_all_session_db_paths"), +} + + +def __getattr__(name: str): + """Lazily import and cache package-level attributes. + + Args: + name: The attribute name to look up. + + Returns: + The resolved attribute value. + + Raises: + AttributeError: If the name is not in _EXPORTS. + """ + target = _EXPORTS.get(name) + if target is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + module_name, attr_name = target + module = import_module(module_name, package=__name__) + value = getattr(module, attr_name) + # Cache after first load to avoid repeated import lookups. + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """List available public attributes including lazy exports.""" + return sorted(set(globals()) | set(_EXPORTS)) + + +__all__ = list(_EXPORTS) diff --git a/EvoScientist/__main__.py b/EvoScientist/__main__.py new file mode 100644 index 0000000..a278280 --- /dev/null +++ b/EvoScientist/__main__.py @@ -0,0 +1,5 @@ +"""Enable `python -m EvoScientist` execution.""" + +from EvoScientist.cli import main + +main() diff --git a/EvoScientist/_version.py b/EvoScientist/_version.py new file mode 100644 index 0000000..d38c350 --- /dev/null +++ b/EvoScientist/_version.py @@ -0,0 +1 @@ +__version__ = "0.1.19" diff --git a/EvoScientist/backends.py b/EvoScientist/backends.py new file mode 100644 index 0000000..fa36669 --- /dev/null +++ b/EvoScientist/backends.py @@ -0,0 +1,1101 @@ +"""Custom backends for EvoScientist agent.""" + +import os +import re +import shlex +import uuid +from pathlib import Path, PurePosixPath + +from deepagents.backends import FilesystemBackend, LocalShellBackend +from deepagents.backends.protocol import ( + BackendProtocol, + EditResult, + ExecuteResponse, + FileDownloadResponse, + FileUploadResponse, + GlobResult, + GrepResult, + LsResult, + WriteResult, +) + +# System path prefixes that should never appear in virtual paths. +# If the agent hallucinates an absolute system path, we block it. +_SYSTEM_PATH_PREFIXES = ( + "/Users/", + "/home/", + "/tmp/", + "/var/", + "/etc/", + "/opt/", + "/usr/", + "/bin/", + "/sbin/", + "/dev/", + "/proc/", + "/sys/", + "/root/", +) + +# Dangerous patterns that could escape the workspace +BLOCKED_PATTERNS = [ + r"~/", # home directory + r"\bcd\s+/", # cd to absolute path + r"\brm\s+-rf\s+/", # rm -rf with absolute path +] + +# Dangerous commands that should never be executed +BLOCKED_COMMANDS = [ + "sudo", + "chmod", + "chown", + "mkfs", + "dd", + "shutdown", + "reboot", +] + + +def _split_shell_commands(command: str) -> list[str]: + """Split a compound shell command into individual base commands. + + Handles &&, ||, ;, and | operators. Returns base command names. + """ + base_commands: list[str] = [] + # Split by sequential operators first + for segment in re.split(r"\s*(?:&&|\|\||;)\s*", command): + # Then split by pipe + for pipe_seg in segment.split("|"): + pipe_seg = pipe_seg.strip() + if not pipe_seg: + continue + try: + tokens = shlex.split(pipe_seg) + except ValueError: + tokens = pipe_seg.split() + if tokens: + base_commands.append(tokens[0]) + return base_commands + + +def _has_traversal_component(command: str) -> bool: + """Check if command contains '..' as a path component (not substring).""" + from pathlib import PurePosixPath + + for token in command.split(): + if ".." in PurePosixPath(token).parts: + return True + return False + + +def _collect_executable_positions(command: str) -> set[int]: + """Return the string offsets of executable tokens (first token per segment). + + These are command names/paths that appear in executable position (e.g. + ``/usr/bin/python`` in ``/usr/bin/python script.py``) and should not be + treated as dangerous operand paths. Also covers the argument position + right after ``pip install`` / ``pip3 install`` (package path). + """ + offsets: set[int] = set() + for segment in re.split(r"\s*(?:&&|\|\||;)\s*", command): + for pipe_seg in segment.split("|"): + pipe_seg_stripped = pipe_seg.strip() + if not pipe_seg_stripped: + continue + # Offset of this pipe segment within *command* + seg_start = command.find(pipe_seg_stripped) + try: + tokens = shlex.split(pipe_seg_stripped) + except ValueError: + tokens = pipe_seg_stripped.split() + if not tokens: + continue + # First token is the executable itself — mark its offset + offsets.add(seg_start) + # pip install — mark the install-target token + if ( + len(tokens) >= 3 + and tokens[0] in ("pip", "pip3") + and tokens[1] == "install" + ): + # Find position of the 3rd token (the package arg) onwards + rest = pipe_seg_stripped + for t in tokens[:2]: + idx = rest.find(t) + rest = rest[idx + len(t) :] + pkg_offset = seg_start + (len(pipe_seg_stripped) - len(rest.lstrip())) + offsets.add(pkg_offset) + return offsets + + +def _normalize_grep_match(match: dict) -> dict: + """Return a grep match compatible with deepagents' current protocol.""" + normalized = dict(match) + if "text" not in normalized and "content" in normalized: + normalized["text"] = normalized["content"] + if "content" not in normalized and "text" in normalized: + normalized["content"] = normalized["text"] + return normalized + + +def _extract_all_paths(command: str) -> list[str]: + """Extract potential file paths from a command, including inside quoted strings. + + Scans both shell tokens and string literals (single/double quoted) to find + paths that start with system prefixes like /Users/, /etc/, /tmp/, etc. + Skips paths in executable position (command name) and pip install targets. + Also excludes /dev/null which is a special device file for output redirection. + """ + exe_offsets = _collect_executable_positions(command) + paths: list[str] = [] + # Pattern: match absolute paths starting with / followed by word chars, dots, + # dashes, slashes. Looks inside quotes and unquoted tokens alike. + # Excludes URL-like patterns (preceded by ://) + path_re = re.compile( + r"(?)}\]]*)?)' # rest of the path + ) + for m in path_re.finditer(command): + # Skip paths that land at an executable-position offset + if m.start(1) in exe_offsets: + continue + path = m.group(1) + # Skip /dev/null - it's a special device file for output redirection, not a real file + if path == "/dev/null": + continue + paths.append(path) + return paths + + +def validate_command(command: str) -> str | None: + """ + Validate a shell command for safety. + + Returns: + None if command is safe, error message string if blocked. + """ + # Check for '..' path traversal as a path component + if _has_traversal_component(command): + return ( + "Command blocked: contains '..' path traversal. " + "All commands must operate within the workspace directory. " + "Use relative paths (e.g., './file.py') instead." + ) + + # Check for dangerous patterns + for pattern in BLOCKED_PATTERNS: + if re.search(pattern, command): + return ( + f"Command blocked: contains forbidden pattern '{pattern}'. " + f"All commands must operate within the workspace directory. " + f"Use relative paths (e.g., './file.py') instead." + ) + + # Check for dangerous commands (pipeline-aware) + for base_cmd in _split_shell_commands(command): + if base_cmd in BLOCKED_COMMANDS: + return ( + f"Command blocked: '{base_cmd}' is not allowed in sandbox mode. " + f"Only standard development commands are permitted." + ) + + # Check for absolute system paths (including inside quoted strings). + # This catches attacks like: python -c "os.remove('/Users/foo/file')" + escaped_paths = _extract_all_paths(command) + if escaped_paths: + path_sample = escaped_paths[0] + return ( + f"Command blocked: contains absolute system path '{path_sample}'. " + f"All file operations must use relative paths within the workspace. " + f"Use relative paths (e.g., './file.py') instead." + ) + + return None + + +def convert_virtual_paths_in_command( + command: str, + workspace_name: str | None = None, +) -> str: + """ + Convert virtual paths (starting with /) in commands to relative paths. + + Also auto-corrects hallucinated system absolute paths that reference the + workspace directory (e.g. ``/Users/.../myproject/file.py`` → ``./file.py``). + + Args: + command: Original command. + workspace_name: Basename of the workspace directory (e.g. ``"workspace"``, + ``"my-project"``). When provided, system paths containing + ``//`` are auto-corrected. + + Examples: + >>> convert_virtual_paths_in_command("python /main.py") + 'python ./main.py' + >>> convert_virtual_paths_in_command("ls /") + 'ls .' + >>> convert_virtual_paths_in_command( + ... "mkdir -p /Users/u/proj/dir", workspace_name="proj") + 'mkdir -p ./dir' + """ + + def replace_virtual_path(match: re.Match[str]) -> str: + path = match.group(0) + + # Skip content that looks like a URL + if "://" in command[max(0, match.start() - 10) : match.end() + 10]: + return path + + # Fix hallucinated system absolute paths that reference the workspace. + # E.g. /Users/user/.../myproject/file.py → ./file.py + # This mirrors _resolve_path() logic but for shell command strings. + if workspace_name: + for prefix in _SYSTEM_PATH_PREFIXES: + if path.startswith(prefix): + marker = f"/{workspace_name}/" + idx = path.find(marker) + if idx != -1: + relative = path[idx + len(marker) :] + return "./" + relative if relative else "." + elif path.endswith(f"/{workspace_name}"): + return "." + break # Matched system prefix but no workspace → fall through + + # Convert virtual path + if path == "/": + return "." + else: + return "." + path + + # Match pattern: paths starting with / (but not URLs) + pattern = r'(?<=\s)/[^\s;|&<>\'"`]*|^/[^\s;|&<>\'"`]*' + converted = re.sub(pattern, replace_virtual_path, command) + + return converted + + +class ReadOnlyFilesystemBackend(FilesystemBackend): + """ + Read-only filesystem backend. + + Allows read, ls, grep, glob operations but blocks write and edit. + Used for skills directory — agent can read skill definitions but cannot + modify them. + """ + + def write(self, file_path: str, content: str) -> WriteResult: + return WriteResult( + error="This directory is read-only. Write operations are not permitted here." + ) + + def edit( + self, + file_path: str, + old_string: str, + new_string: str, + replace_all: bool = False, + ) -> EditResult: + return EditResult( + error="This directory is read-only. Edit operations are not permitted here." + ) + + +class MergedReadOnlyBackend(BackendProtocol): + """Read-only backend that merges up to three skill directories. + + Priority (high → low): + 1. primary — workspace/skills/ (project-local) + 2. global — ~/.config/ai4scientist/skills/ (user global, optional) + 3. secondary — EvoScientist/skills/ (built-in, PyPI) + + Higher-priority skills override lower-priority skills with the same name. + All directories share the same virtual path namespace (/skills/). + """ + + def __init__( + self, + primary_dir: str, + secondary_dir: str, + global_dir: str | None = None, + ): + self._primary = ReadOnlyFilesystemBackend( + root_dir=primary_dir, virtual_mode=True + ) + self._global = ( + ReadOnlyFilesystemBackend(root_dir=global_dir, virtual_mode=True) + if global_dir + else None + ) + self._secondary = ReadOnlyFilesystemBackend( + root_dir=secondary_dir, virtual_mode=True + ) + + def _backends(self): + """Yield backends in priority order: primary → global → secondary.""" + yield self._primary + if self._global: + yield self._global + yield self._secondary + + # -- read: try each tier in priority order -- + + def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> str: + for backend in list(self._backends())[:-1]: + try: + result = backend.read(file_path, offset, limit) + if hasattr(result, "error"): + if result.error is None: + return result + elif not str(result).startswith("Error:"): + return result + except (ValueError, FileNotFoundError, OSError): + pass + return self._secondary.read(file_path, offset, limit) + + # -- ls: merge all tiers, higher priority wins on name conflicts -- + + def ls(self, path: str = "/") -> LsResult: + merged: dict = {} + for backend in reversed(list(self._backends())): + result = backend.ls(path) + for item in result.entries or []: + merged[item["path"]] = item + return LsResult(entries=sorted(merged.values(), key=lambda x: x["path"])) + + # -- grep: search all tiers -- + + def grep( + self, pattern: str, path: str | None = None, glob: str | None = None + ) -> GrepResult: + matches = [] + for backend in self._backends(): + try: + result = backend.grep(pattern, path, glob) + matches.extend(_normalize_grep_match(match) for match in (result.matches or [])) + except Exception: + pass + return GrepResult(matches=matches) + + # -- glob: merge all tiers, higher priority wins on name conflicts -- + + def glob(self, pattern: str, path: str = "/") -> GlobResult: + merged: dict = {} + for backend in reversed(list(self._backends())): + try: + result = backend.glob(pattern, path) + for item in result.matches or []: + merged[item["path"]] = item + except Exception: + pass + return GlobResult(matches=sorted(merged.values(), key=lambda x: x["path"])) + + # -- write / edit: blocked -- + + def write(self, file_path: str, content: str) -> WriteResult: + return WriteResult( + error="This directory is read-only. Write operations are not permitted here." + ) + + def edit( + self, + file_path: str, + old_string: str, + new_string: str, + replace_all: bool = False, + ) -> EditResult: + return EditResult( + error="This directory is read-only. Edit operations are not permitted here." + ) + + # -- download / upload -- + + def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: + """Download files, trying each tier in priority order.""" + backends = list(self._backends()) + responses: list[FileDownloadResponse] = [] + for path in paths: + resp = backends[-1].download_files([path])[0] + for backend in backends[:-1]: + candidate = backend.download_files([path])[0] + if candidate.error is None: + resp = candidate + break + responses.append(resp) + return responses + + def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: + return [ + FileUploadResponse(path=path, error="permission_denied") + for path, _ in files + ] + + +class CustomSandboxBackend(LocalShellBackend): + """ + Custom sandbox backend - inherits LocalShellBackend with added safety. + + Features: + - Inherits all file operations (ls, read, write, edit, grep, glob) + - Inherits shell command execution with output truncation and timeout + - Adds command validation to prevent directory traversal and dangerous operations + - Adds path sanitization to auto-correct common LLM path mistakes + - Compatible with LangGraph checkpointer (no thread locks) + """ + + def __init__( + self, + root_dir: str = ".", + *, + virtual_mode: bool = True, + timeout: int = 300, + max_output_bytes: int = 100_000, + env: dict[str, str] | None = None, + inherit_env: bool = True, + ): + """ + Initialize custom sandbox backend. + + Args: + root_dir: File system root directory + virtual_mode: Whether to enable virtual path mode + timeout: Command execution timeout in seconds + max_output_bytes: Max output size before truncation (default 100KB) + env: Extra environment variables for subprocess + inherit_env: Whether to inherit parent process env (default True) + """ + super().__init__( + root_dir=root_dir, + virtual_mode=virtual_mode, + timeout=timeout, + max_output_bytes=max_output_bytes, + env=env, + inherit_env=inherit_env, + ) + # Override parent's "local-" prefix with our own + self._sandbox_id = f"evosci-{uuid.uuid4().hex[:8]}" + # Ensure working directory exists + os.makedirs(str(self.cwd), exist_ok=True) + + def _resolve_path(self, key: str) -> Path: + """Resolve path with sanitization to prevent nested directories. + + Intercepts all file operations (read, write, edit, ls, grep, glob). + Auto-corrects common LLM path mistakes instead of crashing: + 1. //file.py → /file.py + 2. /Users/name/...//f → /f (strip up to ws dir) + 3. /Users/name/file.py → /file.py (keep basename) + """ + ws_name = Path(str(self.cwd)).name # e.g. "workspace", "my-project" + + # Auto-strip // prefix to prevent nesting + ws_prefix = f"/{ws_name}/" + if key.startswith(ws_prefix): + key = key[len(ws_prefix) - 1 :] # "//main.py" → "/main.py" + elif key == f"/{ws_name}": + key = "/" + + # Auto-correct system absolute paths + for prefix in _SYSTEM_PATH_PREFIXES: + if key.startswith(prefix): + # Try to extract path after "/" + idx = key.find(ws_prefix) + if idx != -1: + key = "/" + key[idx + len(ws_prefix) :] + elif key.endswith(f"/{ws_name}"): + key = "/" + else: + # Fall back to basename + key = "/" + Path(key).name + break + + return super()._resolve_path(key) + + def _default_extension_for_content(self, content: str) -> str: + stripped = (content or "").lstrip() + if stripped.startswith(" str: + if value in ("/artifacts", "/artifacts/", "artifacts", "artifacts/"): + return "/artifacts/output" + self._default_extension_for_content(content) + path = PurePosixPath(value) + name = path.name + if name in ("", ".", "/"): + return value.rstrip("/") + "/output" + self._default_extension_for_content(content) + if "." not in name: + return value.rstrip("/") + self._default_extension_for_content(content) + return value + + def _resolve_artifact_output_path(self, key: str, content: str = "") -> Path: + value = (key or "artifacts").replace("\\", "/") + # DeepAgents SummarizationMiddleware offloads old messages to + # /conversation_history/{thread_id}.md. Preserve that namespace + # instead of falling through to the basename-flattening branch below, + # which would land the dump at /artifacts/{thread_id}.md --- polluting + # user artifacts with a thread-id-named file the agent mistakes for a + # real product. + if ( + value == "/conversation_history" + or value.startswith("/conversation_history/") + or value == "conversation_history" + or value.startswith("conversation_history/") + ): + value = self._ensure_output_file_path(value, content) + return self._resolve_path(value) + if value in ("", "/", "/workspace", "workspace"): + value = "/artifacts/output" + self._default_extension_for_content(content) + elif value.startswith("/workspace/artifacts/"): + value = value[len("/workspace") :] + elif value == "/workspace/artifacts": + value = "/artifacts/output" + self._default_extension_for_content(content) + elif value.startswith("/workspace/uploads/") or value.startswith("/uploads/"): + pass + elif value.startswith("/workspace/"): + value = "/artifacts/" + value[len("/workspace/") :] + elif not value.startswith("/"): + if not ( + value == "artifacts" + or value.startswith("artifacts/") + or value.startswith("uploads/") + ): + value = "artifacts/" + value + elif not ( + value == "/artifacts" + or value.startswith("/artifacts/") + or value.startswith("/uploads/") + ): + value = "/artifacts/" + Path(value).name + value = self._ensure_output_file_path(value, content) + return self._resolve_path(value) + + def write(self, file_path: str, content: str) -> WriteResult: + target = self._resolve_artifact_output_path(file_path, content) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return WriteResult(path=str(target.relative_to(self.cwd))) + + def edit( + self, + file_path: str, + old_string: str, + new_string: str, + replace_all: bool = False, + ) -> EditResult: + target = self._resolve_artifact_output_path(file_path) + return super().edit( + str(target.relative_to(self.cwd)), + old_string, + new_string, + replace_all, + ) + + def _host_path_for_virtual_absolute(self, path: str) -> str | None: + if path == "/workspace": + return str(self.cwd) + if path.startswith("/workspace/"): + return str(self.cwd / path[len("/workspace/") :]) + if path == "/artifacts": + return str(self.cwd / "artifacts") + if path.startswith("/artifacts/"): + return str(self.cwd / "artifacts" / path[len("/artifacts/") :]) + if path == "/uploads": + return str(self.cwd / "uploads") + if path.startswith("/uploads/"): + return str(self.cwd / "uploads" / path[len("/uploads/") :]) + return None + + def _rewrite_script_virtual_absolute_paths(self, command: str) -> None: + """Materialize remote-compute virtual paths inside local Python scripts.""" + script_paths: set[Path] = set() + for segment in re.split(r"\s*(?:&&|\|\||;|\|)\s*", command): + try: + tokens = shlex.split(segment) + except ValueError: + continue + if len(tokens) < 2: + continue + exe = Path(tokens[0]).name + if exe not in {"python", "python3", "python3.11"}: + continue + for token in tokens[1:]: + if token.startswith("-"): + continue + if token.endswith(".py"): + script_paths.add(self._resolve_path(token)) + break + + if not script_paths: + return + + path_pattern = re.compile(r"""(?P['"])(?P/(?:workspace|artifacts|uploads)(?:/[^'"]*)?)(?P=quote)""") + for script in script_paths: + try: + if not script.is_file(): + continue + text = script.read_text(encoding="utf-8") + except OSError: + continue + + def _replace(match: re.Match[str]) -> str: + quote = match.group("quote") + virtual_path = match.group("path") + host_path = self._host_path_for_virtual_absolute(virtual_path) + if host_path is None: + return match.group(0) + return f"{quote}{host_path}{quote}" + + rewritten = path_pattern.sub(_replace, text) + if rewritten != text: + try: + script.write_text(rewritten, encoding="utf-8") + except OSError: + pass + + def execute(self, command: str, *, timeout: int | None = None) -> ExecuteResponse: + """ + Execute shell command in sandbox environment. + + Commands are validated before execution to prevent: + - Directory traversal (../) + - Access to paths outside workspace + - Dangerous system commands + + Then delegates to LocalShellBackend.execute() for actual execution. + """ + # Replace literal workspace-root absolute paths with ./ + # Must happen BEFORE validation so workspace paths (e.g. /tmp/...) + # are sanitized before the system-path check fires. + ws = str(self.cwd).rstrip("/") + "/" + if ws in command: + command = command.replace(ws, "./") + + # Convert virtual paths to relative paths + if self.virtual_mode: + command = convert_virtual_paths_in_command( + command=command, + workspace_name=Path(str(self.cwd)).name, + ) + + # Validate command safety (after path sanitization) + error = validate_command(command) + if error: + return ExecuteResponse( + output=error, + exit_code=1, + truncated=False, + ) + + self._rewrite_script_virtual_absolute_paths(command) + + # Delegate to parent for subprocess execution + response = super().execute(command, timeout=timeout) + + # Enhance timeout errors with actionable recovery guidance + if response.exit_code == 124: + cmd_words = command.split() + grep_hint = cmd_words[0] if cmd_words else "process" + bg_cmd = f"{command} > /output.log 2>&1 &" + response = ExecuteResponse( + output=( + f"{response.output}\n\n" + f"Recovery: re-run in background to avoid the sandbox timeout:\n" + f" {bg_cmd}\n" + f"Then check progress: ps aux | grep {grep_hint}\n" + f"Read results: cat /output.log" + ), + exit_code=response.exit_code, + truncated=response.truncated, + ) + + return response + + +class MultiRootSandboxBackend(CustomSandboxBackend): + """Multi-root sandbox backend for web users with cross-thread read access. + + Features (compared to CustomSandboxBackend): + - write_root: current thread dir --- **only** place where write/edit/delete creates files + - global_root: ``__global__/`` dir --- readable and writable by all threads + - read_roots: peer thread dirs under the same user --- readable only + + Shell commands execute from write_root (current thread cwd). + """ + + def __init__( + self, + write_root: str, + read_roots: list[str], + global_root: str, + *, + virtual_mode: bool = True, + timeout: int = 300, + ): + super().__init__(root_dir=write_root, virtual_mode=virtual_mode, timeout=timeout) + self._write_root = Path(write_root).resolve() + self._global_root = Path(global_root).resolve() + self._read_roots: list[Path] = [] + for r in read_roots: + p = Path(r).resolve() + if p != self._write_root: # skip write_root itself (already covered) + self._read_roots.append(p) + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _in_write_root(self, resolved: Path) -> bool: + return str(resolved) == str(self._write_root) or str(resolved).startswith( + str(self._write_root) + "/" + ) + + def _in_global_root(self, resolved: Path) -> bool: + return str(resolved) == str(self._global_root) or str(resolved).startswith( + str(self._global_root) + "/" + ) + + def _is_writable(self, resolved: Path) -> bool: + """Return True if *resolved* is under write_root or global_root.""" + return self._in_write_root(resolved) or self._in_global_root(resolved) + + def _resolve_single(self, key: str) -> Path: + """Resolve a path against write_root (parent's _resolve_path logic). + + Applies same hallucination correction as CustomSandboxBackend but + does NOT strip the workspace-name prefix (which would block access + to sibling thread dirs). + """ + key = key.strip() + if key == "/workspace": + key = "/" + elif key.startswith("/workspace/"): + key = "/" + key[len("/workspace/"):] + # Auto-correct system absolute paths (hallucination cleanup) + for prefix in _SYSTEM_PATH_PREFIXES: + if key.startswith(prefix): + # Try to extract a usable basename + fallback = "/" + Path(key).name + if fallback.startswith("/."): + break # don't reduce to "/." --- return as-is + key = fallback + break + # Resolve relative to write_root via parent's basic virtual-path resolution + return self._write_root / key.lstrip("/") + + def _key_matches_global(self, key: str) -> bool: + return key.startswith("/__global__/") or key == "/__global__" + + def _resolve_global(self, key: str) -> Path: + rest = key[len("/__global__"):] # "/paper.pdf" or "" + return (self._global_root / rest.lstrip("/")).resolve() if rest else self._global_root + + def _try_read_roots(self, key: str) -> Path | None: + """Try to find *key* under any read_root. Returns resolved Path or None.""" + for root in self._read_roots: + candidate = (root / key.lstrip("/")).resolve() + if candidate.exists(): + return candidate + return None + + # ------------------------------------------------------------------ + # Read operations --- try write_root -> read_roots -> global_root + # ------------------------------------------------------------------ + + def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> str: + if self._key_matches_global(file_path): + resolved = self._resolve_global(file_path) + else: + resolved = self._resolve_single(file_path) + if not resolved.exists(): + alt = self._try_read_roots(file_path) + if alt is not None: + resolved = alt + return self._filesystem_read(resolved, offset, limit) + + def _filesystem_read(self, resolved: Path, offset: int, limit: int) -> str: + """Read from an arbitrary resolved path (no root enforcement).""" + if not resolved.is_file(): + msg = f"Error: File '{resolved.name}' not found." + return msg + try: + text = resolved.read_text(encoding="utf-8", errors="replace") + except (OSError, UnicodeDecodeError) as e: + msg = f"Error: Cannot read file '{resolved.name}': {e}" + return msg + lines = text.splitlines(keepends=True) + start = max(0, offset) + end = min(len(lines), offset + limit) if limit else len(lines) + selected = lines[start:end] + return "".join(selected) + + # ------------------------------------------------------------------ + # Write operations --- locked to write_root + global_root + # ------------------------------------------------------------------ + + def _display_write_path(self, resolved: Path) -> str: + """Return a workspace-relative path string for a written file. + + Mirrors ``CustomSandboxBackend.write`` so the ``write_file`` tool reports + where the file actually landed (e.g. ``artifacts/report.md``) instead of + ``None``. Global-folder writes are shown under the ``__global__/`` prefix. + """ + try: + return str(resolved.relative_to(self._write_root)) + except ValueError: + pass + try: + return f"__global__/{resolved.relative_to(self._global_root)}" + except ValueError: + return str(resolved) + + def write(self, file_path: str, content: str) -> WriteResult: + if self._key_matches_global(file_path): + resolved = self._resolve_global(file_path) + else: + resolved = self._resolve_artifact_output_path(file_path) + if not self._is_writable(resolved): + return WriteResult( + error=( + f"Cannot write to '{file_path}': outside writable directory. " + f"Files can only be created in the current conversation folder " + f"or the global folder (/__global__/)." + ) + ) + try: + resolved.parent.mkdir(parents=True, exist_ok=True) + resolved.write_text(content, encoding="utf-8") + return WriteResult(path=self._display_write_path(resolved)) + except OSError as e: + return WriteResult(error=str(e)) + + def edit( + self, + file_path: str, + old_string: str, + new_string: str, + replace_all: bool = False, + ) -> EditResult: + if self._key_matches_global(file_path): + resolved = self._resolve_global(file_path) + else: + resolved = self._resolve_artifact_output_path(file_path) + if not self._is_writable(resolved): + return EditResult( + error=( + f"Cannot edit '{file_path}': outside writable directory. " + f"Files can only be edited in the current conversation folder " + f"or the global folder (/__global__/)." + ) + ) + if not resolved.is_file(): + return EditResult(error=f"File '{resolved.name}' not found.") + try: + text = resolved.read_text(encoding="utf-8", errors="replace") + except OSError as e: + return EditResult(error=str(e)) + if replace_all: + new_text = text.replace(old_string, new_string) + else: + idx = text.find(old_string) + if idx == -1: + return EditResult( + error=f"Could not find '{old_string[:40]}...' in file." + ) + new_text = text[:idx] + new_string + text[idx + len(old_string) :] + try: + resolved.write_text(new_text, encoding="utf-8") + except OSError as e: + return EditResult(error=str(e)) + return EditResult() + + # ------------------------------------------------------------------ + # ls --- merge write_root + read_roots + global_root + # ------------------------------------------------------------------ + + def ls(self, path: str = "/") -> LsResult: + merged: dict[str, dict] = {} + + def _collect(root: Path, prefix: str = "") -> None: + target = root / path.lstrip("/") if path != "/" else root + if not target.is_dir(): + return + try: + for child in target.iterdir(): + name = child.name + vpath = f"/{prefix}{name}" if prefix else f"/{name}" + merged[vpath] = { + "path": vpath, + "type": "dir" if child.is_dir() else "file", + "size": child.stat().st_size if child.is_file() else 0, + } + except (OSError, PermissionError): + pass + + _collect(self._write_root) + for root in self._read_roots: + _collect(root) + _collect(self._global_root) + return LsResult(entries=sorted(merged.values(), key=lambda x: x["path"])) + + # ------------------------------------------------------------------ + # grep --- search write_root + read_roots + global_root + # ------------------------------------------------------------------ + + def grep( + self, pattern: str, path: str | None = None, glob: str | None = None + ) -> GrepResult: + import fnmatch + + matches: list[dict] = [] + + def _search(root: Path) -> None: + search_root = root / (path.lstrip("/") if path else "") + if not search_root.is_dir(): + return + for fpath in search_root.rglob("*"): + if not fpath.is_file(): + continue + if glob and not fnmatch.fnmatch(fpath.name, glob): + continue + try: + text = fpath.read_text(encoding="utf-8", errors="replace") + except (OSError, UnicodeDecodeError): + continue + for lineno, line in enumerate(text.splitlines(), 1): + if re.search(pattern, line): + matches.append({ + "path": f"/{fpath.relative_to(root)}", + "line": lineno, + "text": line.strip(), + "content": line.strip(), + }) + if len(matches) >= 100: + return + + _search(self._write_root) + for root in self._read_roots: + if len(matches) >= 100: + break + _search(root) + if len(matches) < 100: + _search(self._global_root) + return GrepResult(matches=matches[:100]) + + # ------------------------------------------------------------------ + # glob --- merge write_root + read_roots + global_root + # ------------------------------------------------------------------ + + def glob(self, pattern: str, path: str = "/") -> GlobResult: + merged: dict[str, dict] = {} + + def _collect(root: Path) -> None: + base = root / (path.lstrip("/") if path != "/" else "") + if not base.is_dir(): + return + for match_path in base.rglob(pattern): + rel = match_path.relative_to(root) + vpath = f"/{rel}" + merged[vpath] = { + "path": vpath, + "type": "dir" if match_path.is_dir() else "file", + "size": match_path.stat().st_size if match_path.is_file() else 0, + } + + _collect(self._write_root) + for root in self._read_roots: + _collect(root) + _collect(self._global_root) + return GlobResult(matches=sorted(merged.values(), key=lambda x: x["path"])) + + # ------------------------------------------------------------------ + # download / upload + # ------------------------------------------------------------------ + + def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: + responses: list[FileDownloadResponse] = [] + for fp in paths: + if self._key_matches_global(fp): + resolved = self._resolve_global(fp) + else: + resolved = self._resolve_single(fp) + if not resolved.exists(): + alt = self._try_read_roots(fp) + if alt is not None: + resolved = alt + if not resolved.is_file(): + responses.append( + FileDownloadResponse(error=f"File '{fp}' not found.") + ) + continue + try: + data = resolved.read_bytes() + responses.append(FileDownloadResponse(data=data)) + except OSError as e: + responses.append(FileDownloadResponse(error=str(e))) + return responses + + def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: + responses: list[FileUploadResponse] = [] + for fpath, data in files: + if self._key_matches_global(fpath): + resolved = self._resolve_global(fpath) + else: + resolved = self._resolve_artifact_output_path(fpath) + if not self._is_writable(resolved): + responses.append( + FileUploadResponse(path=fpath, error="permission_denied") + ) + continue + try: + resolved.parent.mkdir(parents=True, exist_ok=True) + resolved.write_bytes(data) + responses.append(FileUploadResponse(path=fpath)) + except OSError as e: + responses.append( + FileUploadResponse(path=fpath, error=str(e)) + ) + return responses + + +class NonComputingFallbackBackend(MultiRootSandboxBackend): + """Read/write fallback for Web threads when remote compute is unavailable. + + The important contract is that Web users must never fall back to local shell + execution. File operations keep the UI usable for browsing and recovery, but + execute always returns a controlled failure. + """ + + def __init__( + self, + write_root: str, + read_roots: list[str], + global_root: str, + *, + reason: str = "remote compute is unavailable", + virtual_mode: bool = True, + timeout: int = 300, + ): + super().__init__( + write_root=write_root, + read_roots=read_roots, + global_root=global_root, + virtual_mode=virtual_mode, + timeout=timeout, + ) + self.reason = reason + + def execute(self, command: str, *, timeout: int | None = None) -> ExecuteResponse: + return ExecuteResponse( + output=( + "ComputeUnavailable: remote compute is unavailable; local execution " + "is disabled for Web threads." + f" Reason: {self.reason}" + ), + exit_code=1, + truncated=False, + ) diff --git a/EvoScientist/ccproxy_manager.py b/EvoScientist/ccproxy_manager.py new file mode 100644 index 0000000..1dfc4ec --- /dev/null +++ b/EvoScientist/ccproxy_manager.py @@ -0,0 +1,348 @@ +"""ccproxy lifecycle management for OAuth-based Anthropic access. + +Provides functions to start/stop/health-check ccproxy, which allows +users with a Claude Pro/Max subscription to use EvoScientist without +a separate API key by reusing Claude Code's OAuth tokens. + +ccproxy is invoked via subprocess (not Python imports) so the +``ccproxy-api`` package is truly optional at runtime. +""" + +from __future__ import annotations + +import logging +import os +import shutil +import subprocess +import time + +from EvoScientist.config import EvoScientistConfig + +logger = logging.getLogger(__name__) + + +# ============================================================================= +# Availability & auth checks +# ============================================================================= + + +def _ccproxy_exe() -> str | None: + """Return the path to the ccproxy binary, or None if not found. + + Checks PATH first, then the current Python environment's bin directory + (handles conda envs where newly installed binaries may not be visible + to shutil.which immediately after pip install). + """ + found = shutil.which("ccproxy") + if found: + return found + import sys as _sys + + candidate = os.path.join(os.path.dirname(_sys.executable), "ccproxy") + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + return None + + +def is_ccproxy_available() -> bool: + """Check whether the ``ccproxy`` CLI binary is available.""" + return _ccproxy_exe() is not None + + +def _is_editable_install() -> bool: + """Return True if EvoScientist was installed in editable/development mode. + + Checks all matching distributions because a stale ``.egg-info`` in the + project root can shadow the real ``dist-info`` in site-packages. + """ + try: + import importlib.metadata as _meta + import json + + for dist in _meta.distributions(): + name = dist.metadata.get("Name", "") + if name.lower() != "evoscientist": + continue + direct_url = dist.read_text("direct_url.json") + if direct_url is not None: + data = json.loads(direct_url) + if data.get("dir_info", {}).get("editable", False) is True: + return True + except Exception: + pass + return False + + +def _oauth_install_hint() -> str: + """Return the appropriate install command depending on install method.""" + if _is_editable_install(): + return "uv sync --extra oauth or pip install -e '.[oauth]'" + return "pip install 'evoscientist[oauth]'" + + +def _summarize_auth_output(raw: str) -> str: + """Extract key fields from ccproxy auth status output into a one-line summary. + + Parses the Rich table output for Email, Subscription, and Status fields. + Returns e.g. ``"user@example.com (plus, active)"``. + Falls back to ``"Authenticated"`` if parsing fails. + """ + import re as _re + + # Strip ANSI escape sequences + clean = _re.sub(r"\x1b\[[0-9;]*m", "", raw) + + # Parse "Key<2+ spaces>Value" table rows, match exact key names + fields: dict[str, str] = {} + for line in clean.splitlines(): + m = _re.match(r"\s*(.+?)\s{2,}(.+)", line) + if not m: + continue + key, val = m.group(1).strip(), m.group(2).strip() + if key in ("Email", "Subscription", "Subscription Status"): + fields[key.lower().replace(" ", "_")] = val + + email = fields.get("email", "") + sub = fields.get("subscription", "") + status = fields.get("subscription_status", "") + + if email: + detail = ", ".join(filter(None, [sub, status])) + return f"{email} ({detail})" if detail else email + return "Authenticated" + + +def check_ccproxy_auth(provider: str = "claude_api") -> tuple[bool, str]: + """Check if ccproxy has valid OAuth credentials. + + Args: + provider: ccproxy provider name ("claude_api" or "codex"). + + Returns: + (is_valid, message) tuple. + """ + try: + exe = _ccproxy_exe() or "ccproxy" + result = subprocess.run( + [exe, "auth", "status", provider], + capture_output=True, + text=True, + timeout=10, + ) + import re as _re + + raw = (result.stdout + result.stderr).strip() + clean = _re.sub(r"\x1b\[[0-9;]*m", "", raw) + + # Filter out structlog warning/noise lines, keep only status lines + status_lines = [ + line + for line in clean.splitlines() + if line.strip() + and not _re.match(r"\d{4}-\d{2}-\d{2}", line.strip()) + and "warning" not in line.lower() + and "plugin" not in line.lower() + ] + status_msg = " ".join(status_lines).strip() + + # ccproxy auth status may exit 0 even when not authenticated — + # detect failure by checking output content + if result.returncode != 0 or "not authenticated" in clean.lower(): + return False, status_msg or "Not authenticated" + + summary = _summarize_auth_output(result.stdout) + return True, summary or "Authenticated" + except FileNotFoundError: + return False, "ccproxy not found" + except subprocess.TimeoutExpired: + return False, "Auth check timed out" + except Exception as exc: + return False, f"Auth check failed: {exc}" + + +# ============================================================================= +# Process management +# ============================================================================= + + +def is_ccproxy_running(port: int) -> bool: + """Check if ccproxy is already serving on the given port.""" + import httpx + + try: + resp = httpx.get(f"http://127.0.0.1:{port}/health/live", timeout=2.0) + return resp.status_code == 200 + except (httpx.ConnectError, httpx.TimeoutException, OSError): + return False + + +def start_ccproxy(port: int) -> subprocess.Popen: + """Start ccproxy serve as a background process. + + Args: + port: Port number for the proxy server. + + Returns: + The Popen handle for the ccproxy process. + + Raises: + RuntimeError: If ccproxy fails to become healthy within 30 seconds. + FileNotFoundError: If ccproxy binary is not found. + """ + exe = _ccproxy_exe() or "ccproxy" + proc = subprocess.Popen( + [exe, "serve", "--port", str(port)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + # Wait for health (ccproxy can take up to ~11s on first start) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if proc.poll() is not None: + raise RuntimeError( + f"ccproxy exited immediately with code {proc.returncode}" + ) + if is_ccproxy_running(port): + return proc + time.sleep(0.3) + + # Timed out — clean up + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + raise RuntimeError("ccproxy did not become healthy within 30 seconds") + + +def stop_ccproxy(proc: subprocess.Popen | None) -> None: + """Gracefully stop a ccproxy process. + + Safe to call with None (no-op). + """ + if proc is None: + return + try: + proc.terminate() + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=2) + except Exception: + pass + + +def ensure_ccproxy(port: int) -> subprocess.Popen | None: + """Ensure ccproxy is running — reuse existing or start new. + + Returns: + Popen handle if we started a new process, None if already running. + """ + if is_ccproxy_running(port): + logger.debug("ccproxy already running on port %d", port) + return None + return start_ccproxy(port) + + +# ============================================================================= +# Environment setup +# ============================================================================= + + +def setup_ccproxy_env(port: int) -> None: + """Set environment variables for Anthropic ccproxy routing. + + Force-sets ``ANTHROPIC_BASE_URL`` and ``ANTHROPIC_API_KEY`` so that + downstream LangChain/Anthropic clients route through ccproxy. + + Always overrides existing values — when this function is called, + we've decided to use ccproxy, so env must point to it. + """ + os.environ["ANTHROPIC_BASE_URL"] = f"http://127.0.0.1:{port}/claude" + os.environ["ANTHROPIC_API_KEY"] = "ccproxy-oauth" + + +def setup_codex_env(port: int) -> None: + """Set environment variables for OpenAI/Codex ccproxy routing. + + Force-sets ``OPENAI_BASE_URL`` and ``OPENAI_API_KEY`` so that + downstream LangChain/OpenAI clients route through ccproxy's Codex + endpoint. + + Always overrides existing values — when this function is called, + we've decided to use ccproxy, so env must point to it. + """ + os.environ["OPENAI_BASE_URL"] = f"http://127.0.0.1:{port}/codex/v1" + os.environ["OPENAI_API_KEY"] = "ccproxy-oauth" + + +# ============================================================================= +# High-level orchestration +# ============================================================================= + + +def maybe_start_ccproxy(config: EvoScientistConfig) -> subprocess.Popen | None: + """High-level: conditionally start ccproxy based on config. + + Checks ``config.anthropic_auth_mode`` and ``config.openai_auth_mode``: + - ``oauth``: ccproxy must work — raises on failure. + - ``api_key``: no-op for that provider. + + When either provider uses OAuth, ccproxy is started (single process + serves both providers). Environment variables are set for each + provider that uses OAuth. + + Args: + config: An ``EvoScientistConfig`` instance. + + Returns: + Popen handle if we started ccproxy, None otherwise. + """ + anthropic_oauth = getattr(config, "anthropic_auth_mode", "api_key") == "oauth" + openai_oauth = getattr(config, "openai_auth_mode", "api_key") == "oauth" + + if not anthropic_oauth and not openai_oauth: + return None + + if not is_ccproxy_available(): + raise RuntimeError( + "ccproxy is required for OAuth mode but not found. " + f"Install it with: {_oauth_install_hint()}" + ) + + # Check auth for each provider that uses OAuth + if anthropic_oauth: + authed, msg = check_ccproxy_auth("claude_api") + if not authed: + raise RuntimeError( + f"ccproxy Anthropic OAuth not authenticated: {msg}\n" + "Run: ccproxy auth login claude_api" + ) + + if openai_oauth: + authed, msg = check_ccproxy_auth("codex") + if not authed: + raise RuntimeError( + f"ccproxy Codex OAuth not authenticated: {msg}\n" + "Run: ccproxy auth login codex" + ) + + port = config.ccproxy_port + if not (1 <= port <= 65535): + raise ValueError(f"Invalid ccproxy port: {port}. Must be between 1 and 65535.") + + # Start ccproxy (single process serves both providers) + proc = ensure_ccproxy(port) + + # Set environment for each OAuth provider + if anthropic_oauth: + setup_ccproxy_env(port) + if openai_oauth: + setup_codex_env(port) + + if proc: + logger.info("Started ccproxy on port %d", port) + else: + logger.info("Reusing existing ccproxy on port %d", port) + return proc diff --git a/EvoScientist/channels/README.md b/EvoScientist/channels/README.md new file mode 100644 index 0000000..8edf8f6 --- /dev/null +++ b/EvoScientist/channels/README.md @@ -0,0 +1,928 @@ +# Channels + +EvoScientist provides unified integration with 10 messaging platforms. This document covers the architecture overview, message processing pipeline, capability matrix, security model, deployment guides, and troubleshooting. + +Configuration file: `~/.config/ai4scientist/settings.yaml` (or use environment variables with the `EVOSCIENTIST_` prefix). + +## Table of Contents + +- [Architecture](#architecture) +- [Message Processing Pipeline](#message-processing-pipeline) +- [Middleware Pipeline](#middleware-pipeline) +- [Capability Matrix](#capability-matrix) +- [Security and Access Control](#security-and-access-control) +- [Quick Start](#quick-start) +- [Channel Deployment Guides](#channel-deployment-guides) + - [Telegram](#telegram) | [Discord](#discord) | [Slack](#slack) | [Feishu (Lark)](#feishu-lark) | [WeChat](#wechat) + - [DingTalk](#dingtalk) | [QQ](#qq) | [Signal](#signal) | [Email](#email) | [iMessage](#imessage) +- [Running Multiple Channels](#running-multiple-channels) +- [Docker Deployment](#docker-deployment) +- [Troubleshooting](#troubleshooting) + +## Architecture + +``` + ┌─────────────────────────────────────────────┐ + │ Messaging Platforms │ + │ │ + │ ┌────────┐ ┌────────┐ ┌────────┐ │ + │ │Telegram│ │Discord │ │ Slack │ ...x10 │ + │ └───┬────┘ └───┬────┘ └───┬────┘ │ + └──────┼──────────┼──────────┼───────────────┘ + │ │ │ + ┌──────┴──────────┴──────────┴───────────────┐ + │ Inbound Middleware │ + │ │ + │ Dedup → AllowList → Pairing → GroupHist │ + │ → Mention │ + └──────────────────┬─────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────┐ + │ MessageBus │ + │ │ + │ inbound queue ──► outbound queue │ + │ (asyncio.Queue, capacity 5000) │ + └──────────┬───────────────┬───────────┘ + │ │ + ▼ ▼ + ┌──────────────────┐ ┌─────────────────┐ + │ InboundConsumer │ │ Dispatcher │ + │ │ │ │ + │ Worker pool (8) │ │ Routes replies │ + │ Per-chat locks │ │ to origin │ + │ Session dedup │ │ channel │ + │ Timeout handling │ │ │ + │ │ │ └─────────────────┘ + │ ▼ │ + │ Agent Core │ + └──────────────────┘ +``` + +### Core Modules + +| Module | Responsibility | +|--------|---------------| +| `base.py` | Abstract `Channel` base class — readiness checks, retry strategy, mention stripping, format fallback, media handling, debounce, send locks | +| `capabilities.py` | `ChannelCapabilities` frozen dataclass — each channel declares features, framework adapts automatically | +| `plugin.py` | `ChannelPlugin` base with adapter slots — `ConfigAdapter`, `SecurityAdapter`, `GroupAdapter`, `MentionAdapter`, `OutboundAdapter`, `ThreadingAdapter`, etc. | +| `mixins.py` | Reusable async patterns: `WebhookMixin` (aiohttp server + httpx client), `WebSocketMixin` (connect/reconnect/heartbeat), `PollingMixin` (async polling loop), `TokenMixin` (OAuth token auto-refresh) | +| `config.py` | `BaseChannelConfig` — shared config fields (allowed_senders, proxy, text_chunk_limit, etc.) + `SingleAccountConfigAdapter` / `MultiAccountConfigAdapter` | +| `bus/` | `MessageBus` async event bus with `InboundMessage` / `OutboundMessage` dataclasses, decoupling channels from agent core | +| `channel_manager.py` | `ChannelManager` — lifecycle management (start/stop), health monitoring, channel registry, account management, outbound dispatch | +| `consumer.py` | `InboundConsumer` — worker pool, per-chat serial locks, session deduplication, timeout handling | +| `retry.py` | `RetryConfig` — exponential backoff retry with per-channel presets (attempts, min/max delay, jitter) | +| `formatter.py` | `UnifiedFormatter` — Markdown to platform-specific format conversion (HTML, Slack mrkdwn, Discord, plain text) | +| `standalone.py` | Headless channel runner (`run_standalone`) for running channels without the CLI | + +## Message Processing Pipeline + +### Inbound (User Message → Agent) + +``` +1. Platform SDK/Webhook receives raw message + │ +2. Channel._on_message() parses into RawIncoming + │ +3. Channel._enqueue_raw() runs middleware pipeline: + ├── DedupMiddleware — drop duplicates (LRU cache, 60s TTL) + ├── AllowListMiddleware — enforce sender/channel restrictions + ├── PairingMiddleware — handle DM pairing flow (if dm_policy="pairing") + ├── GroupHistoryMiddleware — buffer group context for injection + └── MentionGatingMiddleware — filter by @mention policy in groups + │ +4. InboundMessage queued on Channel._queue + │ +5. Channel.run() → receive() → queue_message() with debounce + │ + │ (500ms debounce window: rapid messages from same sender merged) + │ +6. MessageBus.publish_inbound() + │ +7. InboundConsumer acquires per-chat lock → invokes Agent + │ +8. Agent response → OutboundMessage → MessageBus.publish_outbound() +``` + +### Outbound (Agent Response → User) + +``` +1. OutboundMessage arrives on MessageBus outbound queue + │ +2. Dispatcher routes to origin channel by name + │ +3. Channel.send() processes the response: + ├── Stop typing indicator + ├── Format text (Markdown → platform format) + ├── Chunk text to platform limit (code-block-aware splitting) + ├── Send each chunk via _send_chunk() with format fallback + ├── Send media attachments via _send_media_impl() + └── Retry on transient errors (exponential backoff) +``` + +### Text Chunking + +Long responses are split intelligently with this priority: + +1. Markdown code block fence boundaries +2. Double newlines (paragraph breaks) +3. Single newlines +4. Space characters +5. Hard cut at limit (last resort) + +Code blocks are never split mid-block when possible. Each chunk is sent as a separate message. + +## Middleware Pipeline + +Middleware runs sequentially on each inbound message. Each middleware can pass, modify, or drop the message. + +### 1. DedupMiddleware + +Prevents duplicate message processing using a bounded LRU cache with TTL. + +- Cache size: 1000 entries (configurable) +- TTL: 60 seconds +- Key: `message_id` from the platform +- Messages with the same ID within the TTL window are silently dropped + +### 2. AllowListMiddleware + +Enforces sender and channel restrictions based on the `dm_policy` config. + +| Policy | Behavior | +|--------|----------| +| `"open"` | Accept messages from anyone | +| `"allowlist"` | Only accept from `allowed_senders` / `allowed_channels` | +| `"pairing"` | Require DM pairing before accepting (see PairingMiddleware) | + +When `allowed_senders` is set (non-empty), only messages from listed sender IDs pass through. Same for `allowed_channels`. + +### 3. PairingMiddleware + +Handles an interactive DM pairing flow for the `"pairing"` dm_policy. + +- First message from an unknown sender triggers a pairing request +- The sender must provide a valid pairing code +- Once paired, the sender is added to the allowlist for future messages + +### 4. GroupHistoryMiddleware + +Buffers recent group chat messages to provide conversation context. + +- Only active when `capabilities.groups = True` +- Maintains a per-chat rolling buffer (default: 50 messages, 5-minute max age) +- When the bot is mentioned in a group, recent history is injected into the message metadata so the agent can see prior context +- Non-mentioned group messages are buffered but not forwarded (see MentionGating) + +### 5. MentionGatingMiddleware + +Controls whether the bot responds in group chats. + +| `require_mention` | Behavior | +|-------------------|----------| +| `True` / `"group"` | Only respond to @mentions in groups; always respond in DMs | +| `False` / `"none"` | Respond to all messages in all contexts | +| `"always"` | Require @mention even in DMs | + +Default: `"group"` — the bot ignores group messages unless explicitly @mentioned. + +Mention detection is platform-specific: +- **Telegram**: checks for `@bot_username` in text +- **Discord**: checks `message.mentions` for bot user +- **Slack**: handled via separate `app_mention` event type +- **Feishu**: checks `mentions` array in event payload +- **DingTalk**: checks `isInAtList` flag or `atUsers` array +- **WeChat (WeCom)**: checks `AtUserList` XML field + +## Capability Matrix + +| Channel | Format | Max Len | Media | Voice | Sticker | Location | Video | Typing | Reaction | Thread | Group | @Mention | No Public IP | Token Refresh | Proxy | Allowlist | +|:--------|:------:|:-------:|:-----:|:-----:|:-------:|:--------:|:-----:|:------:|:--------:|:------:|:-----:|:--------:|:------------:|:-------------:|:-----:|:---------:| +| Telegram | HTML | 4000 | S/R | R | R | R | | 4s | emoji | | G | @ | yes | | yes | yes | +| Discord | Discord | 2000 | S/R | | | | | 8s | emoji | yes | G | @ | yes | | yes | yes | +| Slack | Mrkdwn | 4000 | S/R | | | | | post | emoji | yes | G | @ | yes | | yes | yes | +| Feishu | Post | 4096 | S/R | R | R | | | | emoji | | G | @ | ws mode | 2h | yes | yes | +| WeChat | MD | 4096 | S/R | R | | R | R | recall | | | G | @ | no | 2h | yes | yes | +| DingTalk | MD | 4096 | S/R | R | | | R | | | | G | @ | yes | 2h | yes | yes | +| QQ | MD/plain | 4096 | S/R | | | | | | | | G | @ | yes | | | yes | +| Signal | Plain | 4096 | S/R | R | | | | api | emoji | | G | UUID | yes | | | yes | +| iMessage | Plain | - | S/R | R | | | | | | | G | | yes | | | yes | +| Email | HTML | - | S/R | | | | | | | | | | yes | | | yes | + +Legend: **S** = send, **R** = receive, **G** = group chat, **@** = @mention detection, **-** = no practical limit + +### Connection Types + +| Channel | Transport | Connection Mode | Default Port | +|-------------|-----------|----------------------------------------|:------------:| +| Telegram | HTTPS | Long polling (`getUpdates`) | -- | +| Discord | WebSocket | Gateway events (`discord.py`) | -- | +| Slack | WebSocket | Socket Mode (`slack-sdk`) | -- | +| Feishu | HTTP/WS | Webhook or WebSocket long connection | 9000/-- | +| WeChat | HTTP | Webhook `POST /wechat/callback` | 9001 | +| DingTalk | WebSocket | Stream Mode (DingTalk gateway) | -- | +| QQ | WebSocket | Bot Gateway (`qq-botpy`) | -- | +| Signal | TCP | JSON-RPC (`signal-cli` daemon) | 7583 | +| iMessage | stdio | JSON-RPC (`imsg` CLI) | -- | +| Email | TCP | IMAP polling + SMTP send | 993/587 | + +> **"--"** means no listening port is required -- no public IP or port forwarding needed. + +### Format Conversion + +The `UnifiedFormatter` converts Markdown output from the agent into platform-native formats: + +| Target Format | Conversion | +|:-------------|:-----------| +| HTML (Telegram, Email) | `**bold**` → `bold`, `` `code` `` → `code`, code blocks → `
`, special chars escaped |
+| Slack mrkdwn | `**bold**` → `*bold*`, `_italic_` → `_italic_`, code blocks preserved, `<>&` escaped |
+| Discord Markdown | Mostly passthrough, minor adjustments for Discord-specific rendering |
+| Feishu Post | Markdown → Feishu rich text JSON (code blocks, bold, italic, strikethrough, links, headings, quotes, lists) |
+| Plain text | All formatting stripped, structure preserved via indentation |
+
+## Security and Access Control
+
+### Sender Allowlist
+
+Every channel supports `allowed_senders` to restrict who can interact with the bot:
+
+```yaml
+telegram_allowed_senders: "123456789,987654321"     # Telegram user IDs
+discord_allowed_senders: "111222333444555666"        # Discord user IDs
+slack_allowed_senders: "U0123ABCDEF"                # Slack Member IDs
+feishu_allowed_senders: "ou_xxxxxxxxxxxx"            # Feishu open_ids
+signal_allowed_senders: "+1234567890"                # Phone numbers
+email_allowed_senders: "alice@example.com"           # Email addresses
+imessage_allowed_senders: "+1234567890,user@icloud.com"  # Phone or email
+```
+
+When `allowed_senders` is empty, the channel accepts messages from anyone. **For production deployments, always set an allowlist.**
+
+### Channel Allowlist
+
+For platforms with multiple channels/groups (Discord, Slack), restrict which channels the bot operates in:
+
+```yaml
+discord_allowed_channels: "111222333444555666,777888999000111222"
+slack_allowed_channels: "C0123ABCDEF,C0456GHIJKL"
+```
+
+### Token and Secret Handling
+
+- API tokens are stored in the config file or environment variables, never logged at INFO level
+- Discord logs only the first 8 and last 4 characters of the bot token for debugging
+- WeChat/Feishu tokens are auto-refreshed before expiry (5-minute margin on 2-hour TTL)
+- Webhook signature verification is enforced when `token`/`encoding_aes_key` is configured (WeChat, Feishu)
+
+### Group Chat Behavior
+
+By default, the bot only responds in group chats when explicitly @mentioned. This prevents the bot from responding to every message in a busy group. Configure via:
+
+```yaml
+# Default: only respond when mentioned in groups
+channel_require_mention: "group"
+
+# Respond to all messages (including groups)
+channel_require_mention: "none"
+```
+
+## Quick Start
+
+### 1. Install channel dependencies
+
+```bash
+pip install evoscientist[telegram]
+# Available extras: telegram, discord, slack, feishu, wechat,
+#   dingtalk, qq, email, signal
+# iMessage requires no extra Python dependencies
+```
+
+### 2. Configure
+
+```bash
+# Option A: Interactive wizard
+EvoSci onboard
+
+# Option B: CLI commands
+EvoSci config set channel_enabled telegram
+EvoSci config set telegram_bot_token "123456:ABC-xxx"
+
+# Option C: Environment variables (EVOSCIENTIST_ prefix, uppercase)
+export EVOSCIENTIST_CHANNEL_ENABLED=telegram
+export EVOSCIENTIST_TELEGRAM_BOT_TOKEN="123456:ABC-xxx"
+```
+
+### 3. Start
+
+```bash
+EvoSci serve                # Start agent + all enabled channels
+# or
+EvoSci channel start        # Standalone channel mode (message loop only)
+```
+
+### 4. Health check
+
+```bash
+curl http://localhost:8080/healthz
+```
+
+```json
+{
+  "status": "healthy",
+  "channels": { "enabled": ["telegram"], "running": ["telegram"] }
+}
+```
+
+---
+
+## Channel Deployment Guides
+
+---
+
+### Telegram
+
+**Install:** `pip install evoscientist[telegram]`
+
+**Prerequisites:**
+
+1. Search for [@BotFather](https://t.me/BotFather) in Telegram, send `/newbot`, and follow the prompts to create a bot.
+2. BotFather will return a Bot Token (format: `123456789:ABCdefGHI...`) -- save it securely.
+3. Get your user ID: send any message to [@userinfobot](https://t.me/userinfobot), it will reply with your numeric ID.
+4. (Optional) For group use: add the bot to a group, then in BotFather send `/setprivacy` -> `Disable` so the bot can read group messages.
+
+**Configuration:**
+
+```yaml
+channel_enabled: "telegram"
+telegram_bot_token: "123456789:ABCdefGHIjklMNOpqrSTUvwxYZ"
+telegram_allowed_senders: ""       # Comma-separated user IDs; empty = no restriction
+telegram_proxy: ""                 # Optional HTTPS proxy (e.g. http://proxy:8080)
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `telegram_bot_token` | `str` | `""` | **Required.** Bot API Token from BotFather |
+| `telegram_allowed_senders` | `str` | `""` | Comma-separated user IDs, empty = allow all |
+| `telegram_proxy` | `str` | `""` | HTTPS proxy URL |
+
+**Env vars:** `EVOSCIENTIST_TELEGRAM_BOT_TOKEN`, `EVOSCIENTIST_TELEGRAM_ALLOWED_SENDERS`, `EVOSCIENTIST_TELEGRAM_PROXY`
+
+**Technical details:** Long polling mode, `drop_pending_updates=True` on startup to skip backlog. Markdown to Telegram HTML auto-conversion (bold, italic, strikethrough, links, code blocks, headings, lists). Falls back to plain text on HTML parse failure. Media routed by extension to `send_photo`/`send_video`/`send_audio`/`send_document`. In groups, only responds when @mentioned; auto-strips @mention. Typing indicator refreshes every 4s. ACK reaction (eyes emoji) on message receipt, removed after reply. Retry: 3 attempts, min delay 0.4s, parse errors not retried. Text chunk limit: 4000 chars.
+
+---
+
+### Discord
+
+**Install:** `pip install evoscientist[discord]`
+
+**Prerequisites:**
+
+1. Go to [Discord Developer Portal](https://discord.com/developers/applications) -> New Application -> enter a name.
+2. Left menu **Bot** -> Reset Token -> copy the Bot Token.
+3. Under **Privileged Gateway Intents**, enable **Message Content Intent** (required to read message content).
+4. Left menu **OAuth2** -> URL Generator:
+   - Scopes: check `bot`
+   - Bot Permissions: check `Send Messages`, `Read Message History`, `Attach Files`, `Add Reactions`
+   - Copy the generated URL, open in browser, select a server to invite the bot.
+5. Get user ID: Discord Settings -> Advanced -> enable Developer Mode -> right-click username -> Copy User ID.
+
+**Configuration:**
+
+```yaml
+channel_enabled: "discord"
+discord_bot_token: "MTIzNDU2Nzg5.xxxx.xxxxx"
+discord_allowed_senders: ""        # Comma-separated user IDs
+discord_allowed_channels: ""       # Comma-separated channel IDs
+discord_proxy: ""
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `discord_bot_token` | `str` | `""` | **Required.** Bot Token |
+| `discord_allowed_senders` | `str` | `""` | Comma-separated user IDs, empty = allow all |
+| `discord_allowed_channels` | `str` | `""` | Comma-separated channel IDs, empty = allow all |
+| `discord_proxy` | `str` | `""` | HTTPS proxy URL |
+
+**Env vars:** `EVOSCIENTIST_DISCORD_BOT_TOKEN`, `EVOSCIENTIST_DISCORD_ALLOWED_SENDERS`, `EVOSCIENTIST_DISCORD_ALLOWED_CHANNELS`, `EVOSCIENTIST_DISCORD_PROXY`
+
+**Technical details:** WebSocket Gateway (`discord.py`). In server channels, only responds when @mentioned; DMs respond directly. Thread-aware: messages in threads are tracked with `parent_channel_id` and `thread_id`. Replies via `MessageReference`. Message cache (200 entries) for ACK emoji reactions. Attachment download (max 20 MB) with safe filename sanitization. Media sent via `discord.File`. Typing indicator refreshes every 8s. Retry: 3 attempts, parses `Retry-After` header for 429s. Text chunk limit: 2000 chars.
+
+---
+
+### Slack
+
+**Install:** `pip install evoscientist[slack]`
+
+**Prerequisites:**
+
+1. Go to [Slack API](https://api.slack.com/apps) -> Create New App -> From scratch -> select workspace.
+2. Left menu **Socket Mode** -> enable -> Generate App-Level Token, scope `connections:write` -> copy App Token (`xapp-...`).
+3. Left menu **OAuth & Permissions** -> add Bot Token Scopes:
+   - `chat:write`, `channels:history`, `groups:history`, `im:history`, `files:read`, `files:write`, `reactions:write`
+4. Click **Install to Workspace** -> copy Bot User OAuth Token (`xoxb-...`).
+5. Left menu **Event Subscriptions** -> enable -> Subscribe to bot events: `message.channels`, `message.groups`, `message.im`, `app_mention`.
+6. Get Member ID: click user avatar -> profile -> **...** -> Copy member ID.
+
+**Configuration:**
+
+```yaml
+channel_enabled: "slack"
+slack_bot_token: "xoxb-xxxx-xxxx-xxxx"
+slack_app_token: "xapp-1-xxxx-xxxx"
+slack_allowed_senders: ""          # Member ID (U...)
+slack_allowed_channels: ""         # Channel ID (C...)
+slack_proxy: ""
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `slack_bot_token` | `str` | `""` | **Required.** Bot User OAuth Token (`xoxb-`) |
+| `slack_app_token` | `str` | `""` | **Required.** Socket Mode App Token (`xapp-`) |
+| `slack_allowed_senders` | `str` | `""` | Comma-separated Member IDs |
+| `slack_allowed_channels` | `str` | `""` | Comma-separated Channel IDs |
+| `slack_proxy` | `str` | `""` | HTTPS proxy URL |
+
+**Env vars:** `EVOSCIENTIST_SLACK_BOT_TOKEN`, `EVOSCIENTIST_SLACK_APP_TOKEN`, `EVOSCIENTIST_SLACK_ALLOWED_SENDERS`, `EVOSCIENTIST_SLACK_ALLOWED_CHANNELS`, `EVOSCIENTIST_SLACK_PROXY`
+
+**Technical details:** Socket Mode (no public URL needed). Markdown to mrkdwn conversion. DMs respond directly; channels respond to `app_mention` events. Thread replies via `thread_ts` -- all replies are threaded to the original message. Typing indicator approximated by posting/deleting a "..." message (Slack has no bot typing API). ACK reaction (eyes emoji) on message receipt. Attachments downloaded with Bearer auth. Media sent via `files_upload_v2`. Runs `auth_test()` on startup to verify credentials and cache bot user ID. Retry: 3 attempts, exponential backoff + jitter. Text chunk limit: 4000 chars.
+
+---
+
+### Feishu (Lark)
+
+**Install:** `pip install evoscientist[feishu]`
+
+**Prerequisites:**
+
+1. Go to [Feishu Open Platform](https://open.feishu.cn/app) (international: [Lark Developer](https://open.larksuite.com/app)) -> create a custom app.
+2. Copy the **App ID** and **App Secret**.
+3. Left menu **Event Subscriptions**:
+   - **Webhook mode**: set request URL to `http://your-host:9000/webhook/event` -> copy **Verification Token** and **Encrypt Key**.
+   - **WebSocket mode**: select **长连接** (Long Connection) as the subscription method. No URL needed.
+4. Add event: `im.message.receive_v1` (receive messages).
+5. Left menu **Permissions** -> enable `im:message:send_as_bot`.
+6. Create a version and publish.
+
+> **Webhook mode** requires a publicly reachable URL. For local dev, use `ngrok http 9000` or [natapp](https://natapp.cn/) (recommended for China).
+
+#### Subscription Modes
+
+Feishu supports two subscription modes:
+
+| Mode | Transport | Public IP? | Best For |
+|------|-----------|:----------:|----------|
+| `webhook` (default) | HTTP POST callback | Yes | Servers with public IP / cloud deployment |
+| `websocket` | WebSocket long connection | **No** | Local dev, behind NAT/firewall, China (no ngrok needed) |
+
+**WebSocket mode** uses the official `lark-oapi` SDK to maintain an outbound WebSocket connection to Feishu servers. No public IP, port forwarding, or tunnel is required.
+
+To use WebSocket mode:
+
+```bash
+# Install the SDK
+pip install 'evoscientist[feishu]'
+
+# Via config file
+feishu_subscription_mode: "websocket"
+
+# Via CLI (standalone)
+python -m EvoScientist.channels.feishu.serve --app-id ID --app-secret SECRET --mode websocket
+```
+
+> **Note:** In WebSocket mode, `feishu_verification_token`, `feishu_encrypt_key`, and `feishu_webhook_port` are not used — the SDK handles authentication and encryption internally.
+
+**Configuration:**
+
+```yaml
+channel_enabled: "feishu"
+feishu_app_id: "cli_xxxxxxx"
+feishu_app_secret: "xxxxxxxxxxxxxxxxxx"
+feishu_subscription_mode: "webhook"   # or "websocket"
+feishu_webhook_port: 9000
+feishu_allowed_senders: ""         # open_id
+feishu_domain: "https://open.feishu.cn"
+feishu_proxy: ""
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `feishu_app_id` | `str` | `""` | **Required.** App ID |
+| `feishu_app_secret` | `str` | `""` | **Required.** App Secret |
+| `feishu_webhook_port` | `int` | `9000` | Webhook HTTP port |
+| `feishu_allowed_senders` | `str` | `""` | Comma-separated open_ids |
+| `feishu_domain` | `str` | `"https://open.feishu.cn"` | API domain (use `https://open.larksuite.com` for Lark) |
+| `feishu_proxy` | `str` | `""` | HTTPS proxy URL |
+| `feishu_subscription_mode` | `str` | `"webhook"` | `"webhook"` or `"websocket"` (WebSocket long connection, no public IP) |
+
+**Env vars:** `EVOSCIENTIST_FEISHU_APP_ID`, `EVOSCIENTIST_FEISHU_APP_SECRET`, `EVOSCIENTIST_FEISHU_WEBHOOK_PORT`, `EVOSCIENTIST_FEISHU_DOMAIN`
+
+**Technical details:** Webhook on `POST /webhook/event` with URL verification challenge-response. Supports both v1 (legacy) and v2 event schemas. Optional AES-256-CBC event decryption (when `encrypt_key` configured). `tenant_access_token` auto-refresh (2h TTL, refreshes 5 min before expiry). Markdown to Feishu Post rich text conversion (code blocks, bold, italic, strikethrough, links, headings, quotes, ordered/unordered lists). Plain text fallback. Group @mention filtering with mention key caching. Media: images via `/im/v1/images`, files via `/im/v1/files`. Replies via `/messages/{id}/reply` API. ACK reaction via `/messages/{id}/reactions`. Retry: 3 attempts, rate limit delay 2.0s, matches `99991400`/`rate limit`. Non-retryable: permission denied (`99991401`), invalid credentials. Text chunk limit: 4096 chars.
+
+---
+
+### WeChat
+
+**Install:** `pip install evoscientist[wechat]`
+
+Two backends supported: **WeCom** (recommended, free, no certification needed) and **WeChat Official Account** (requires verified service account).
+
+#### WeCom
+
+**Prerequisites:**
+
+1. Log in to [WeCom Admin Console](https://work.weixin.qq.com) -> App Management -> create a custom app.
+2. Copy the **Corp ID**, **AgentId**, and **Secret**.
+3. In app details -> Receive Messages -> Set API Receive -> URL: `http://your-host:9001/wechat/callback` -> copy **Token** and **EncodingAESKey**.
+4. In app details -> **Trusted IP** -> add your server's public IP address. Without this, all API calls will fail with error `60020`.
+
+```yaml
+channel_enabled: "wechat"
+wechat_backend: "wecom"
+wechat_webhook_port: 9001
+wechat_wecom_corp_id: "ww..."
+wechat_wecom_agent_id: "1000002"
+wechat_wecom_secret: "xxxxxxxxxxxxxxxxxx"
+wechat_wecom_token: "xxxxxxxxxxxxxxxxxx"
+wechat_wecom_encoding_aes_key: "xxxxxxxxxxxxxxxxxx"
+wechat_allowed_senders: ""
+wechat_proxy: ""
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `wechat_backend` | `str` | `"wecom"` | `"wecom"` or `"wechatmp"` |
+| `wechat_webhook_port` | `int` | `9001` | Callback HTTP port |
+| `wechat_wecom_corp_id` | `str` | `""` | **Required (WeCom).** Corp ID |
+| `wechat_wecom_agent_id` | `str` | `""` | **Required (WeCom).** App AgentId |
+| `wechat_wecom_secret` | `str` | `""` | **Required (WeCom).** App Secret |
+| `wechat_wecom_token` | `str` | `""` | **Required (WeCom).** Callback Token |
+| `wechat_wecom_encoding_aes_key` | `str` | `""` | **Required (WeCom).** Callback EncodingAESKey |
+
+#### WeChat Official Account
+
+**Prerequisites:**
+
+1. Log in to [WeChat Official Account Platform](https://mp.weixin.qq.com) -> Settings & Development -> Basic Configuration.
+2. Copy the **AppID** and **AppSecret**.
+3. Server Configuration -> URL: `http://your-host:9001/wechat/callback` -> set **Token** and **EncodingAESKey**.
+
+```yaml
+wechat_backend: "wechatmp"
+wechat_mp_app_id: "wx..."
+wechat_mp_app_secret: "xxxxxxxxxxxxxxxxxx"
+wechat_mp_token: "xxxxxxxxxxxxxxxxxx"
+wechat_mp_encoding_aes_key: "xxxxxxxxxxxxxxxxxx"
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `wechat_mp_app_id` | `str` | `""` | **Required (MP).** AppID |
+| `wechat_mp_app_secret` | `str` | `""` | **Required (MP).** AppSecret |
+| `wechat_mp_token` | `str` | `""` | **Required (MP).** Server Token |
+| `wechat_mp_encoding_aes_key` | `str` | `""` | **Required (MP).** Server EncodingAESKey |
+
+**Technical details:** Webhook HTTP server for inbound (XML message parsing). GET callback for URL verification (SHA1 signature check). POST callback for message handling -- returns `"success"` within 5s and processes asynchronously. Optional AES encryption/decryption via `WeChatCrypto`. `access_token` auto-refresh (2h TTL, 5-min margin). Token-expired errors (40014, 42001) trigger automatic retry with refreshed token. WeCom supports Markdown message format with plain text fallback; Official Account uses plain text only (customer service API). WeCom group messages sent via `/appchat/send` endpoint (group IDs start with `wr`). Typing indicator approximated by posting/recalling a "..." message (WeCom only). Supports text, image, voice, video, location, file, and link message types. Media upload via `/media/upload`. Text chunk limit: 4096 chars.
+
+---
+
+### DingTalk
+
+**Install:** `pip install evoscientist[dingtalk]`
+
+**Prerequisites:**
+
+1. Go to [DingTalk Open Platform](https://open-dev.dingtalk.com) -> App Development -> create a bot app.
+2. Copy the **AppKey** (Client ID) and **AppSecret** (Client Secret).
+3. Enable **Stream Mode** in the app configuration -- no public IP needed.
+4. Publish the app and add the bot to a group, or test via direct message.
+
+**Configuration:**
+
+```yaml
+channel_enabled: "dingtalk"
+dingtalk_client_id: "ding..."
+dingtalk_client_secret: "xxxxxxxxxxxxxxxxxx"
+dingtalk_allowed_senders: ""
+dingtalk_proxy: ""
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `dingtalk_client_id` | `str` | `""` | **Required.** AppKey |
+| `dingtalk_client_secret` | `str` | `""` | **Required.** AppSecret |
+| `dingtalk_allowed_senders` | `str` | `""` | Comma-separated user IDs |
+| `dingtalk_proxy` | `str` | `""` | HTTPS proxy URL |
+
+**Env vars:** `EVOSCIENTIST_DINGTALK_CLIENT_ID`, `EVOSCIENTIST_DINGTALK_CLIENT_SECRET`
+
+**Technical details:** Stream Mode via WebSocket -- connects to DingTalk gateway (`/v1.0/gateway/connections/open`) with automatic ticket-based auth. Ping/pong heartbeat with system topic handling. Message ACK via JSON response. `accessToken` auto-refresh. Sends via robot `oToMessages/batchSend` API in Markdown format (`sampleMarkdown`). Image uploads via `/media/upload` API with `sampleImageMsg`. Group @mention detection via `isInAtList` flag with `atUsers` array fallback. `downloadCode` resolution via `/robot/messageFiles/download` API for file/image/video/audio attachments. Auth errors (`invalidauthentication`/`forbidden`/`40014`) not retried. Text chunk limit: 4096 chars.
+
+---
+
+### QQ
+
+**Install:** `pip install evoscientist[qq]`
+
+**Prerequisites:**
+
+1. Go to [QQ Open Platform](https://q.qq.com) -> create a bot application.
+2. Complete developer verification, create a sandbox or production bot.
+3. Copy the **AppID** and **AppSecret**.
+4. Search for and add the bot as a friend in QQ, or add it to a group.
+
+**Configuration:**
+
+```yaml
+channel_enabled: "qq"
+qq_app_id: "xxxxxxxxxx"
+qq_app_secret: "xxxxxxxxxxxxxxxxxx"
+qq_allowed_senders: ""
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `qq_app_id` | `str` | `""` | **Required.** AppID |
+| `qq_app_secret` | `str` | `""` | **Required.** AppSecret |
+| `qq_allowed_senders` | `str` | `""` | Comma-separated user IDs |
+
+**Env vars:** `EVOSCIENTIST_QQ_APP_ID`, `EVOSCIENTIST_QQ_APP_SECRET`
+
+**Technical details:** Uses `qq-botpy` SDK via WebSocket to connect to QQ Bot Gateway. Supports C2C (direct) and group messages. Outbound replies prefer native QQ markdown messages (`msg_type=2`) so headings/lists/code fences keep their structure; when the SDK/API rejects markdown, EvoScientist falls back to plain text with Markdown stripped but line structure preserved. Message deduplication (1000-entry LRU cache). Group @mention filtering (strips first `@bot`). Intents: `public_messages=True`, `direct_message=True`. Text chunk limit: 4096 chars.
+
+---
+
+### Signal
+
+**Install:** `pip install evoscientist[signal]` (also requires [signal-cli](https://github.com/AsamK/signal-cli) installed separately)
+
+**Prerequisites:**
+
+1. Install signal-cli: see [signal-cli installation guide](https://github.com/AsamK/signal-cli#installation).
+2. Register or link a phone number:
+   - Register: `signal-cli -u +1234567890 register`, then `signal-cli -u +1234567890 verify CODE`
+   - Link existing device: `signal-cli link -n "EvoScientist"`
+3. EvoScientist will auto-start the signal-cli daemon if it's not already running.
+
+**Configuration:**
+
+```yaml
+channel_enabled: "signal"
+signal_phone_number: "+1234567890"
+signal_cli_path: "signal-cli"
+signal_config_dir: ""
+signal_allowed_senders: ""
+signal_rpc_port: 7583
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `signal_phone_number` | `str` | `""` | **Required.** Signal phone number (E.164 format) |
+| `signal_cli_path` | `str` | `"signal-cli"` | Path to signal-cli binary |
+| `signal_config_dir` | `str` | `""` | signal-cli config directory (optional) |
+| `signal_allowed_senders` | `str` | `""` | Comma-separated phone numbers |
+| `signal_rpc_port` | `int` | `7583` | JSON RPC socket port |
+
+**Env vars:** `EVOSCIENTIST_SIGNAL_PHONE_NUMBER`, `EVOSCIENTIST_SIGNAL_CLI_PATH`, `EVOSCIENTIST_SIGNAL_RPC_PORT`
+
+**Technical details:** JSON RPC over TCP socket to signal-cli daemon. Auto-starts daemon if not running (`signal-cli -u +NUMBER daemon --socket localhost:PORT`). Listens for `receive` notifications. Sends via `send` RPC method. Group detection via `groupInfo`. Mention detection via UUID matching. No public IP needed. Text chunk limit: 4096 chars.
+
+---
+
+### Email
+
+**Install:** `pip install evoscientist[email]` (core dependencies included, no extras needed)
+
+**Prerequisites:**
+
+1. Prepare an email account with IMAP + SMTP support (Gmail, Outlook, self-hosted, etc.).
+2. **Gmail:** Enable 2FA -> generate an App Password. IMAP: `imap.gmail.com:993` (SSL), SMTP: `smtp.gmail.com:587` (STARTTLS).
+3. **Outlook/Office 365:** IMAP: `outlook.office365.com:993` (SSL), SMTP: `smtp.office365.com:587` (STARTTLS).
+4. Ensure IMAP access is enabled in your email settings.
+
+**Configuration:**
+
+```yaml
+channel_enabled: "email"
+email_imap_host: "imap.gmail.com"
+email_imap_port: 993
+email_imap_username: "bot@gmail.com"
+email_imap_password: "xxxx-xxxx-xxxx-xxxx"
+email_imap_mailbox: "INBOX"
+email_imap_use_ssl: true
+email_smtp_host: "smtp.gmail.com"
+email_smtp_port: 587
+email_smtp_username: "bot@gmail.com"
+email_smtp_password: "xxxx-xxxx-xxxx-xxxx"
+email_smtp_use_tls: true
+email_from_address: "bot@gmail.com"
+email_poll_interval: 30
+email_mark_seen: true
+email_allowed_senders: ""
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `email_imap_host` | `str` | `""` | **Required.** IMAP server address |
+| `email_imap_port` | `int` | `993` | IMAP port |
+| `email_imap_username` | `str` | `""` | **Required.** IMAP login username |
+| `email_imap_password` | `str` | `""` | **Required.** IMAP login password (or app password) |
+| `email_imap_mailbox` | `str` | `"INBOX"` | Mailbox folder to monitor |
+| `email_imap_use_ssl` | `bool` | `true` | Use SSL for IMAP connection |
+| `email_smtp_host` | `str` | `""` | **Required.** SMTP server address |
+| `email_smtp_port` | `int` | `587` | SMTP port |
+| `email_smtp_username` | `str` | `""` | **Required.** SMTP login username |
+| `email_smtp_password` | `str` | `""` | **Required.** SMTP login password |
+| `email_smtp_use_tls` | `bool` | `true` | Use STARTTLS (`true`) or SSL (`false`) |
+| `email_from_address` | `str` | `""` | Sender address (defaults to smtp_username) |
+| `email_poll_interval` | `int` | `30` | IMAP poll interval in seconds |
+| `email_mark_seen` | `bool` | `true` | Mark emails as read after processing |
+| `email_max_body_chars` | `int` | `12000` | Max email body chars (truncated beyond) |
+| `email_subject_prefix` | `str` | `"Re: "` | Reply subject prefix |
+| `email_allowed_senders` | `str` | `""` | Comma-separated sender email addresses |
+
+**Env vars:** `EVOSCIENTIST_EMAIL_IMAP_HOST`, `EVOSCIENTIST_EMAIL_IMAP_USERNAME`, `EVOSCIENTIST_EMAIL_IMAP_PASSWORD`, `EVOSCIENTIST_EMAIL_SMTP_HOST`, `EVOSCIENTIST_EMAIL_SMTP_USERNAME`, `EVOSCIENTIST_EMAIL_SMTP_PASSWORD`
+
+**Technical details:** IMAP polling mode, checks for UNSEEN emails periodically (max 20 per cycle). Supports SSL and STARTTLS. Auto-parses multipart emails (prefers text/plain, falls back text/html -> plain text). Attachments auto-downloaded. Replies set `In-Reply-To` and `References` headers to maintain email threads. Sends HTML + plain text dual format (multipart/alternative), falls back to plain text on HTML failure. IMAP auto-reconnects on disconnect. Auth errors (auth/login/credential) not retried. No public IP needed. Text chunk limit: no limit.
+
+---
+
+### iMessage
+
+**Install:** No extra Python dependencies. Requires the [imsg](https://github.com/anthropics/imsg) CLI tool.
+
+**Requirements:** macOS only (iMessage is Apple-proprietary). Requires a signed-in Apple ID with iMessage and Full Disk Access permission for the terminal app.
+
+**Prerequisites:**
+
+1. Install imsg CLI:
+   ```bash
+   brew install imsg
+   ```
+2. Verify: `imsg --version`
+3. Ensure Messages.app is signed in and working on macOS.
+
+**Configuration:**
+
+```yaml
+channel_enabled: "imessage"
+imessage_cli_path: "imsg"
+imessage_db_path: ""
+imessage_service: "auto"
+imessage_region: "US"
+imessage_allowed_senders: ""
+```
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `imessage_cli_path` | `str` | `"imsg"` | Path to imsg CLI binary |
+| `imessage_db_path` | `str` | `""` | iMessage database path (empty = default) |
+| `imessage_service` | `str` | `"auto"` | Send service: `imessage`, `sms`, or `auto` |
+| `imessage_region` | `str` | `"US"` | Phone number region code |
+| `imessage_allowed_senders` | `str` | `""` | Comma-separated allowlist (see below) |
+
+**Allowlist formats:** phone (`+1234567890`), email (`user@example.com`), `chat_id:123`, `chat_guid:iMessage;-;+1234567890`, wildcard `*`.
+
+**Env vars:** `EVOSCIENTIST_IMESSAGE_CLI_PATH`, `EVOSCIENTIST_IMESSAGE_SERVICE`, `EVOSCIENTIST_IMESSAGE_ALLOWED_SENDERS`
+
+**Technical details:** JSON-RPC over stdio with imsg CLI. Creates `watch.subscribe` on startup for real-time message streaming (not polling). Supports iMessage + SMS dual channel (`service: auto`). Target resolution supports chat_id, chat_guid, chat_identifier, and phone/email. Attachments read from local paths provided by imsg. Group detection via `is_group` field. RPC errors (AppleScript/permission/not found) not retried; only connection timeouts retried. Plain text format (no Markdown). No public IP needed. Text chunk limit: 4000 chars.
+
+---
+
+## Running Multiple Channels
+
+Comma-separate channel names in the config to enable multiple channels simultaneously:
+
+```yaml
+channel_enabled: "telegram,discord,slack"
+```
+
+All enabled channels run concurrently via the internal `MessageBus`. Each channel:
+- Has its own connection lifecycle (connect, reconnect, health check)
+- Shares the same `InboundConsumer` worker pool and agent instance
+- Routes outbound replies back to the originating channel automatically
+
+### Multi-Channel Architecture
+
+```
+                    ChannelManager
+                    ├── TelegramChannel  ──┐
+                    ├── DiscordChannel   ──┤
+                    ├── SlackChannel     ──┤──► MessageBus ──► InboundConsumer ──► Agent
+                    ├── FeishuChannel    ──┤                         │
+                    └── ...              ──┘                         ▼
+                                                              OutboundMessage
+                                                                    │
+                                                              Dispatcher routes
+                                                              to origin channel
+```
+
+### Health Monitoring
+
+The `ChannelManager` runs a background health check task that monitors all active channels. Access health status via:
+
+```bash
+# CLI
+EvoSci channel status
+
+# HTTP (if health endpoint is enabled)
+curl http://localhost:8080/healthz
+```
+
+## Docker Deployment
+
+For webhook-based channels (Feishu, WeChat), Docker simplifies port mapping and process management:
+
+```dockerfile
+FROM python:3.11-slim
+
+WORKDIR /app
+COPY . .
+RUN pip install evoscientist[feishu,wechat]
+
+# Expose webhook ports
+EXPOSE 9000 9001
+
+CMD ["EvoSci", "serve"]
+```
+
+```bash
+docker build -t evoscientist .
+docker run -d \
+  -p 9000:9000 \
+  -p 9001:9001 \
+  -e EVOSCIENTIST_CHANNEL_ENABLED="feishu,wechat" \
+  -e EVOSCIENTIST_FEISHU_APP_ID="cli_xxx" \
+  -e EVOSCIENTIST_FEISHU_APP_SECRET="xxx" \
+  -e EVOSCIENTIST_WECHAT_BACKEND="wecom" \
+  -e EVOSCIENTIST_WECHAT_WECOM_CORP_ID="ww..." \
+  -e EVOSCIENTIST_WECHAT_WECOM_SECRET="xxx" \
+  evoscientist
+```
+
+For polling/WebSocket channels (Telegram, Discord, Slack, DingTalk, QQ), no port mapping is needed:
+
+```bash
+docker run -d \
+  -e EVOSCIENTIST_CHANNEL_ENABLED="telegram" \
+  -e EVOSCIENTIST_TELEGRAM_BOT_TOKEN="123456:ABC-xxx" \
+  evoscientist
+```
+
+## Troubleshooting
+
+### Common Issues
+
+**Bot not responding to messages**
+
+1. Check that `channel_enabled` includes your channel name
+2. Verify the bot token/credentials are correct: `EvoSci config get telegram_bot_token`
+3. If using `allowed_senders`, ensure your user ID is listed
+4. For group chats, ensure the bot is @mentioned (default behavior)
+5. Check logs for middleware drops: `DedupMiddleware`, `AllowListMiddleware`, or `MentionGatingMiddleware`
+
+**"channel X not found" or import errors**
+
+Install the channel-specific dependencies:
+```bash
+pip install evoscientist[telegram]  # or discord, slack, feishu, etc.
+```
+
+**Webhook channels (Feishu, WeChat) not receiving messages**
+
+1. Ensure the webhook URL is publicly reachable (not behind NAT without port forwarding)
+2. For local development, use a tunnel: `ngrok http 9000` or [natapp](https://natapp.cn/) for China
+3. Verify the callback URL matches exactly (including path: `/webhook/event` for Feishu, `/wechat/callback` for WeChat)
+4. Check that signature verification tokens match between the platform config and your local config
+
+**WeChat API error `60020`**
+
+Add your server's public IP to the WeCom app's **Trusted IP** list in the admin console.
+
+**Token refresh failures**
+
+- Feishu/WeChat/DingTalk tokens auto-refresh with a 5-minute safety margin before expiry
+- If the refresh endpoint is unreachable (network issues), messages will fail until the next successful refresh
+- Check proxy settings if your server requires a proxy to reach external APIs
+
+**Duplicate responses**
+
+- The `DedupMiddleware` prevents most duplicates using a 60-second LRU cache
+- If you see duplicates, check if the platform is sending the same message with different IDs (some platforms retry delivery)
+
+**Messages truncated**
+
+- Each platform has a max text length (see Capability Matrix)
+- Long responses are automatically chunked at paragraph/code-block boundaries
+- Adjust `text_chunk_limit` in the channel config if needed
+
+### Debug Logging
+
+Enable debug logs for the channel subsystem:
+
+```bash
+export EVOSCIENTIST_LOG_LEVEL=DEBUG
+# or
+EvoSci config set log_level debug
+```
+
+Channel-specific log output is prefixed with the module path (e.g., `EvoScientist.channels.telegram.channel`).
diff --git a/EvoScientist/channels/__init__.py b/EvoScientist/channels/__init__.py
new file mode 100644
index 0000000..ce5f44d
--- /dev/null
+++ b/EvoScientist/channels/__init__.py
@@ -0,0 +1,49 @@
+"""Communication channels for EvoScientist.
+
+This module provides an extensible interface for different messaging channels
+(iMessage, Telegram, Discord, Slack, WeChat, DingTalk, Feishu, Email, QQ, Signal) to communicate with the EvoScientist agent.
+"""
+
+from .base import Channel, IncomingMessage, OutgoingMessage, RawIncoming, chunk_text
+from .bus import InboundMessage, MessageBus, OutboundMessage
+from .capabilities import ChannelCapabilities
+from .channel_manager import (
+    ChannelManager,
+    available_channels,
+    create_channel,
+    register_channel,
+)
+from .consumer import InboundConsumer
+from .formatter import UnifiedFormatter
+from .middleware import TypingManager
+from .plugin import ChannelMeta, ChannelPlugin, ReloadPolicy
+from .standalone import run_standalone
+
+# Backward compat: ChannelServer is now Channel itself
+ChannelServer = Channel
+
+__all__ = [
+    "Channel",
+    # New modules
+    "ChannelCapabilities",
+    "ChannelManager",
+    "ChannelMeta",
+    # Plugin architecture
+    "ChannelPlugin",
+    "ChannelServer",
+    "InboundConsumer",
+    "InboundMessage",
+    "IncomingMessage",
+    "MessageBus",
+    "OutboundMessage",
+    "OutgoingMessage",
+    "RawIncoming",
+    "ReloadPolicy",
+    "TypingManager",
+    "UnifiedFormatter",
+    "available_channels",
+    "chunk_text",
+    "create_channel",
+    "register_channel",
+    "run_standalone",
+]
diff --git a/EvoScientist/channels/base.py b/EvoScientist/channels/base.py
new file mode 100644
index 0000000..bd0ad61
--- /dev/null
+++ b/EvoScientist/channels/base.py
@@ -0,0 +1,1243 @@
+"""Abstract base class for communication channels.
+
+This module defines the Channel interface that all messaging channels
+(iMessage, WeChat, etc.) must implement.
+"""
+
+import asyncio
+import logging
+import re
+from abc import ABC, abstractmethod
+from collections import OrderedDict
+from collections.abc import AsyncIterator, Awaitable, Callable
+from collections.abc import Callable as CallableABC
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from ..paths import MEDIA_DIR
+from .bus.events import InboundMessage, OutboundMessage
+from .capabilities import ChannelCapabilities
+from .debug import TraceMixin, debug_trace_enabled
+from .formatter import UnifiedFormatter
+from .plugin import ChannelMeta, ChannelPlugin
+
+_logger = logging.getLogger(__name__)
+
+
+# ── Text chunking ────────────────────────────────────────────────────
+
+
+def chunk_text(text: str, limit: int) -> list[str]:
+    """Split text into chunks that respect logical boundaries and code fences.
+
+    If a code block is split across chunks, each chunk is automatically
+    wrapped in its own fences (```...```) to maintain formatting.
+
+    Args:
+        text: The text to split.
+        limit: Maximum characters per chunk.
+
+    Returns:
+        List of text chunks, each <= limit characters.
+    """
+    if not text:
+        return []
+    if len(text) <= limit:
+        return [text]
+
+    chunks: list[str] = []
+    remaining = text
+    in_code_block = False
+    code_block_lang = ""
+
+    while remaining:
+        # Effective limit is reduced if we need to add fences
+        # We reserve ~20 chars for fences (```lang\n and \n```)
+        effective_limit = limit - (20 if in_code_block else 0)
+
+        if len(remaining) <= effective_limit:
+            segment = remaining
+            best = len(remaining)
+        else:
+            segment = remaining[:effective_limit]
+            best = -1
+
+            # 1. Paragraph/Line/Word boundaries
+            if not in_code_block:
+                # Paragraph
+                pos = segment.rfind("\n\n")
+                if pos > 0:
+                    best = pos
+
+                # Line
+                if best == -1:
+                    pos = segment.rfind("\n")
+                    if pos > 0:
+                        best = pos
+
+                # Word
+                if best == -1:
+                    pos = segment.rfind(" ")
+                    if pos > 0:
+                        best = pos
+            else:
+                # INSIDE code block: ONLY split at newlines to avoid breaking lines of code
+                pos = segment.rfind("\n")
+                if pos > 0:
+                    best = pos
+
+            if best == -1:
+                best = effective_limit
+
+        chunk_raw = remaining[:best].rstrip()
+
+        # Track state transitions within this raw segment
+        starts_in_code = in_code_block
+        current_lang = code_block_lang
+
+        # We use a simple count of ``` to toggle state.
+        # Note: This handles both opening and closing fences.
+        fences = list(re.finditer(r"```(\w*)", chunk_raw))
+        for f in fences:
+            if not in_code_block:
+                in_code_block = True
+                code_block_lang = f.group(1) or ""
+            else:
+                in_code_block = False
+                code_block_lang = ""
+
+        ends_in_code = in_code_block
+
+        # Build the final chunk with necessary fences
+        prefix = f"```{current_lang}\n" if starts_in_code else ""
+        suffix = "\n```" if ends_in_code else ""
+
+        final_chunk = prefix + chunk_raw + suffix
+        if final_chunk.strip():
+            chunks.append(final_chunk)
+
+        remaining = remaining[best:].lstrip("\n")
+
+    return chunks
+
+
+# ── Attachment / media helpers ───────────────────────────────────────
+
+MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024  # 20 MB
+
+IMAGE_EXTS = frozenset({".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"})
+VIDEO_EXTS = frozenset({".mp4", ".mov", ".avi", ".webm"})
+AUDIO_EXTS = frozenset({".mp3", ".ogg", ".m4a", ".wav"})
+
+
+def classify_media(ext: str) -> str | None:
+    """Classify a file extension into a media type string.
+
+    Returns ``"image"``, ``"video"``, ``"audio"``, or ``None``.
+    """
+    ext = ext.lower()
+    if ext in IMAGE_EXTS:
+        return "image"
+    if ext in VIDEO_EXTS:
+        return "video"
+    if ext in AUDIO_EXTS:
+        return "audio"
+    return None
+
+
+def media_path(filename: str) -> Path:
+    """Ensure MEDIA_DIR exists and return a path inside it."""
+    MEDIA_DIR.mkdir(parents=True, exist_ok=True)
+    return MEDIA_DIR / filename
+
+
+def check_attachment_size(file_size: int, filename: str) -> str | None:
+    """Return a 'too large' annotation if *file_size* exceeds the limit.
+
+    Returns ``None`` when the file is within the allowed size.
+    """
+    if file_size > MAX_ATTACHMENT_BYTES:
+        return f"[attachment: {filename} - too large ({file_size} bytes)]"
+    return None
+
+
+async def download_attachment(
+    url: str,
+    filename: str,
+    *,
+    channel_name: str = "",
+    headers: dict[str, str] | None = None,
+    file_size: int | None = None,
+    proxy: str | None = None,
+) -> tuple[str | None, str | None]:
+    """Download an attachment via httpx.
+
+    Returns ``(local_path, annotation)``.
+
+    If *file_size* exceeds ``MAX_ATTACHMENT_BYTES``, returns
+    ``(None, too-large-annotation)`` without downloading.
+    On download failure returns ``(None, failure-annotation)``.
+    On success returns ``(local_path_str, success-annotation)``.
+    """
+    if file_size is not None:
+        too_large = check_attachment_size(file_size, filename)
+        if too_large:
+            return None, too_large
+
+    try:
+        import httpx
+
+        safe_name = filename.replace("/", "_")
+        prefix = f"{channel_name}_" if channel_name else ""
+        local_path = media_path(f"{prefix}{safe_name}")
+
+        async with httpx.AsyncClient(proxy=proxy) as client:
+            async with client.stream(
+                "GET", url, headers=headers or {}, timeout=30
+            ) as resp:
+                if resp.status_code != 200:
+                    return None, f"[attachment: {filename} - download failed]"
+
+                # Check Content-Length header before downloading body
+                if file_size is None:
+                    cl = resp.headers.get("content-length")
+                    if cl:
+                        try:
+                            too_large = check_attachment_size(int(cl), filename)
+                            if too_large:
+                                return None, too_large
+                        except (ValueError, TypeError):
+                            pass
+
+                # Stream body with incremental size check
+                chunks: list[bytes] = []
+                total = 0
+                async for chunk in resp.aiter_bytes():
+                    total += len(chunk)
+                    if total > MAX_ATTACHMENT_BYTES:
+                        return None, check_attachment_size(total, filename)
+                    chunks.append(chunk)
+
+        local_path.write_bytes(b"".join(chunks))
+        return str(local_path), f"[attachment: {local_path}]"
+    except Exception as e:
+        _logger.warning(f"Failed to download attachment: {e}")
+        return None, f"[attachment: {filename} - download failed]"
+
+
+# Deprecated aliases — use InboundMessage / OutboundMessage instead.
+IncomingMessage = InboundMessage
+OutgoingMessage = OutboundMessage
+
+
+@dataclass
+class RawIncoming:
+    """Raw data extracted from a platform-specific message event.
+
+    Each channel's ``_on_message`` populates this with platform data,
+    then calls ``_enqueue_raw()`` which handles allow-list checks,
+    content merging, and ``InboundMessage`` creation.
+    """
+
+    sender_id: str
+    chat_id: str
+    text: str = ""
+    media_files: list[str] = field(default_factory=list)
+    content_annotations: list[str] = field(default_factory=list)
+    timestamp: datetime = field(default_factory=datetime.now)
+    message_id: str = ""
+    metadata: dict = field(default_factory=dict)
+    is_group: bool = False
+    was_mentioned: bool = True  # default True so DMs always pass
+
+
+class Channel(TraceMixin, ChannelPlugin, ABC):
+    """Abstract base class for messaging channels.
+
+    Subclasses must implement:
+    - start(): Initialize the channel (connect, authenticate, etc.)
+    - _send_chunk(): Send a single text chunk (platform-specific)
+
+    Subclasses may optionally override:
+    - _cleanup(): Channel-specific teardown (called by stop())
+    - _format_chunk(): Convert Markdown to channel format
+    - _is_ready(): Return False if channel cannot send
+    - _resolve_chat_id(): Extract chat_id from message
+    - receive(): Only if custom exit conditions are needed
+
+    Subclasses should set ``name`` to a unique identifier (e.g. "telegram").
+    """
+
+    name: str = "base"
+    capabilities: ChannelCapabilities = ChannelCapabilities()
+    _typing_interval: float = 5.0
+    _ready_attrs: tuple[str, ...] = ()
+
+    def __init__(self, config, *, queue_maxsize: int = 1000):
+        ChannelPlugin.__init__(self)
+        self.id = self.name
+        self.meta = ChannelMeta(id=self.name, label=self.name.title())
+
+        self.config = config
+
+        # Cache STT config at startup to avoid loading it on every message
+        from ..config.settings import load_config as _load_cfg
+
+        _global = _load_cfg()
+        self._stt_enabled: bool = _global.stt_enabled
+        self._stt_language: str = _global.stt_language
+        self._stt_model: str = _global.stt_model
+        self._stt_device: str = _global.stt_device
+        self._stt_compute_type: str = _global.stt_compute_type
+
+        # Auto-configure formatter from capabilities
+        self._formatter = UnifiedFormatter.for_channel(self.capabilities.format_type)
+        self._queue: asyncio.Queue[InboundMessage] = asyncio.Queue(
+            maxsize=queue_maxsize
+        )
+        self._running = False
+
+        # Global tracing can be enabled via shared config/env even when
+        # individual channel factories have not been updated yet.
+        self._debug_trace: bool = bool(getattr(config, "debug_trace", False)) or (
+            debug_trace_enabled()
+        )
+        self._trace_logger = _logger
+
+        # Typing indicator — delegated to TypingManager
+        from .middleware import TypingManager
+
+        self._typing_manager = TypingManager(
+            self._send_typing_action,
+            interval=self._typing_interval,
+            debug_trace=self._debug_trace,
+            channel_name=self.name,
+        )
+        # Keep legacy dict reference for any subclass that touches it directly
+        self._typing_tasks = self._typing_manager._tasks
+
+        # Bus integration (injected by ChannelManager.register / set_bus)
+        self._bus: Any = None
+        self.send_thinking: bool = False
+        self._on_activity: Callable | None = None
+
+        # Debounce settings
+        self.initial_debounce: float = 2.0
+        self.debounce_step: float = 0.5
+        self.max_debounce: float = 5.0
+
+        # Per-sender message buffers for debouncing
+        self._message_buffers: dict[str, list[str]] = {}
+        self._message_metadata: dict[str, dict] = {}
+        self._message_media: dict[str, list[str]] = {}
+        self._message_ids: dict[str, str] = {}
+        self._debounce_tasks: dict[str, asyncio.Task] = {}
+
+        # Mention gating: "always" | "group" | "off"
+        self.require_mention: str = getattr(config, "require_mention", "group")
+
+        # DM policy: "open" | "allowlist" | "pairing"
+        self.dm_policy: str = getattr(config, "dm_policy", "allowlist")
+
+        # Per-sender is_group / was_mentioned for debounce merge
+        self._message_is_group: dict[str, bool] = {}
+        self._message_was_mentioned: dict[str, bool] = {}
+
+        # Retry configuration (auto-resolved from channel name)
+        from .retry import DEFAULT_RETRY, RETRY_PRESETS, RetryConfig
+
+        self._retry_config: RetryConfig = RETRY_PRESETS.get(self.name, DEFAULT_RETRY)
+
+        # Per-chat send locks to prevent message reordering.
+        # Uses an OrderedDict as a bounded LRU cache to avoid unbounded growth.
+        self._send_locks: OrderedDict[str, asyncio.Lock] = OrderedDict()
+        self._send_locks_max: int = 1024
+
+        # Build inbound middleware pipeline
+        self._inbound_middlewares = self._build_inbound_middlewares()
+
+    def _build_inbound_middlewares(self) -> list:
+        """Build the inbound middleware chain from config and capabilities.
+
+        Middleware order:
+        1. DedupMiddleware — drop duplicates early
+        2. AllowListMiddleware — enforce sender/channel restrictions
+        3. PairingMiddleware — handle DM pairing (if applicable)
+        4. GroupHistoryMiddleware — buffer/inject group history
+        5. MentionGatingMiddleware — filter by mention policy
+        """
+        from .middleware import (
+            AllowListMiddleware,
+            DedupMiddleware,
+            GroupHistoryMiddleware,
+            MentionGatingMiddleware,
+            PairingMiddleware,
+        )
+
+        middlewares = []
+        middlewares.append(DedupMiddleware())
+        # AllowList
+        allowed_senders = getattr(self.config, "allowed_senders", None)
+        allowed_channels = getattr(self.config, "allowed_channels", None)
+        if allowed_senders and not isinstance(allowed_senders, set):
+            allowed_senders = set(allowed_senders)
+        if allowed_channels and not isinstance(allowed_channels, set):
+            allowed_channels = set(allowed_channels)
+        middlewares.append(
+            AllowListMiddleware(
+                allowed_senders=allowed_senders,
+                allowed_channels=allowed_channels,
+                dm_policy=self.dm_policy,
+            )
+        )
+        # Pairing
+        if self.dm_policy == "pairing":
+
+            async def _send_pair(chat_id, text):
+                await self._send_chunk(chat_id, text, text, None, {})
+
+            middlewares.append(
+                PairingMiddleware(
+                    channel_name=self.name,
+                    send_response_fn=_send_pair,
+                    dm_policy=self.dm_policy,
+                )
+            )
+        # GroupHistory
+        if self.capabilities.groups:
+            middlewares.append(GroupHistoryMiddleware())
+        # MentionGating
+        if self.capabilities.mentions:
+            middlewares.append(
+                MentionGatingMiddleware(
+                    require_mention=self.require_mention,
+                    strip_fn=self._strip_mention,
+                )
+            )
+        return middlewares
+
+    def is_debug_trace_enabled(self) -> bool:
+        """Return whether extra per-message diagnostics should be emitted."""
+        return self._debug_trace
+
+    @abstractmethod
+    async def start(self) -> None:
+        """Initialize and start the channel.
+
+        This method should:
+        - Establish connections
+        - Verify permissions/authentication
+        - Start any background tasks needed
+
+        Raises:
+            ChannelError: If initialization fails
+        """
+        pass
+
+    async def stop(self) -> None:
+        """Stop the channel and flush pending debounce buffers."""
+        self._running = False
+
+        # Cancel pending debounce timers, then flush buffered messages so they
+        # are not lost when stopping within the debounce window.
+        pending_tasks = list(self._debounce_tasks.values())
+        for task in pending_tasks:
+            task.cancel()
+        for task in pending_tasks:
+            try:
+                await task
+            except asyncio.CancelledError:
+                pass
+            except Exception as e:
+                _logger.debug(f"{self.name} debounce task shutdown error: {e}")
+        self._debounce_tasks.clear()
+
+        for sender in list(self._message_buffers.keys()):
+            try:
+                await self._process_buffered_messages(sender)
+            except Exception as e:
+                _logger.error(
+                    f"{self.name} failed to flush buffered messages for {sender}: {e}"
+                )
+
+        await self._typing_manager.stop_all()
+        await self._cleanup()
+
+    async def _cleanup(self) -> None:
+        """Channel-specific teardown. Override in subclasses."""
+
+    async def receive(self) -> AsyncIterator[InboundMessage]:
+        """Yield incoming messages from the queue.
+
+        Default implementation polls ``self._queue``. Override only if
+        the channel needs custom exit conditions.
+        """
+        while self._running:
+            try:
+                msg = await asyncio.wait_for(self._queue.get(), timeout=1.0)
+                yield msg
+            except TimeoutError:
+                continue
+
+    def _acquire_send_lock(self, chat_id: str) -> asyncio.Lock:
+        """Get or create a per-chat send lock with LRU eviction.
+
+        Moves the accessed entry to the end (most-recently-used).
+        When the cache exceeds ``_send_locks_max``, the least-recently-used
+        entry is evicted — but only if its lock is not currently held.
+        """
+        if chat_id in self._send_locks:
+            self._send_locks.move_to_end(chat_id)
+        else:
+            self._send_locks[chat_id] = asyncio.Lock()
+            # Evict oldest unlocked entries when over capacity.
+            # Skip locked entries instead of giving up entirely,
+            # to prevent unbounded growth.
+            if len(self._send_locks) > self._send_locks_max:
+                to_evict = [
+                    k
+                    for k, lock in self._send_locks.items()
+                    if not lock.locked() and k != chat_id
+                ]
+                for k in to_evict:
+                    if len(self._send_locks) <= self._send_locks_max:
+                        break
+                    del self._send_locks[k]
+        return self._send_locks[chat_id]
+
+    async def send(self, message: OutboundMessage) -> bool:
+        """Send a message. Handles chunking, retry, and error logging.
+
+        Subclasses override ``_send_chunk()`` for the platform-specific call.
+        Override ``_format_chunk()`` to convert Markdown to channel format.
+
+        A per-chat lock ensures messages to the same chat are serialised,
+        preventing out-of-order delivery when multiple sends overlap.
+
+        If formatting expands a chunk beyond the platform limit (e.g. Markdown
+        → HTML), the chunk is automatically re-split at a smaller size.  Per-
+        chunk errors are logged but do not abort delivery of remaining chunks.
+
+        When the channel satisfies ``ThreadingAdapter``, its ``reply_to_mode``
+        controls which chunks carry a ``reply_to`` reference.
+        """
+        if not self._is_ready():
+            return False
+        try:
+            chat_id = self._resolve_chat_id(message)
+            limit = self._get_chunk_limit()
+            async with self._acquire_send_lock(chat_id):
+                had_error = False
+                for i, (formatted, raw) in enumerate(
+                    self._prepare_chunks(message.content, limit)
+                ):
+                    reply_to = self._resolve_reply_to(message.reply_to, i)
+                    try:
+                        await self._send_with_retry(
+                            lambda _cid=chat_id, _fmt=formatted, _raw=raw, _reply=reply_to, _meta=message.metadata: (
+                                self._send_chunk(_cid, _fmt, _raw, _reply, _meta)
+                            )
+                        )
+                    except Exception as chunk_err:
+                        self._trace_event(
+                            "outbound_send_chunk_error",
+                            chat_id=chat_id,
+                            reply_to=reply_to,
+                            chunk_index=i,
+                            error_type=type(chunk_err).__name__,
+                        )
+                        _logger.error(f"{self.name} chunk {i} send error: {chunk_err}")
+                        had_error = True
+            return not had_error
+        except Exception as e:
+            _logger.error(f"{self.name} send error: {e}")
+            return False
+
+    def _resolve_reply_to(self, reply_to: str | None, chunk_index: int) -> str | None:
+        """Determine the reply_to value for a given chunk index.
+
+        Legacy: reply_to on first chunk only.
+        """
+        if not reply_to:
+            return None
+        return reply_to if chunk_index == 0 else None
+
+    def _prepare_chunks(
+        self,
+        content: str,
+        limit: int,
+    ) -> list[tuple[str, str]]:
+        """Build ``(formatted, raw)`` pairs, re-splitting when formatting
+        expands a chunk beyond *limit*.
+
+        Returns a list of ``(formatted_text, raw_text)`` tuples ready
+        for ``_send_chunk()``.
+        """
+        raw_chunks = chunk_text(content, limit)
+        pairs: list[tuple[str, str]] = []
+        for raw in raw_chunks:
+            formatted = self._format_chunk(raw)
+            if len(formatted) <= limit:
+                pairs.append((formatted, raw))
+            else:
+                # Re-chunk at half the limit to leave room for format expansion
+                sub_limit = max(limit // 2, 500)
+                for sub_raw in chunk_text(raw, sub_limit):
+                    sub_fmt = self._format_chunk(sub_raw)
+                    if len(sub_fmt) <= limit:
+                        pairs.append((sub_fmt, sub_raw))
+                    else:
+                        # Still too long — send raw text (guaranteed to fit)
+                        pairs.append((sub_raw, sub_raw))
+        return pairs
+
+    def _is_ready(self) -> bool:
+        """Return False if the channel cannot send (e.g. client not connected).
+
+        Default checks that every attribute named in ``_ready_attrs`` is truthy.
+        Override for channels with more complex readiness logic.
+        """
+        if not self._ready_attrs:
+            return True
+        return all(getattr(self, attr, None) for attr in self._ready_attrs)
+
+    def _resolve_chat_id(self, message: OutboundMessage) -> str:
+        """Extract chat_id from metadata or recipient. Override if needed."""
+        return message.metadata.get("chat_id", message.recipient)
+
+    def _get_chunk_limit(self) -> int:
+        config_limit = getattr(self.config, "text_chunk_limit", 0)
+        cap_limit = self.capabilities.max_text_length
+        return config_limit or cap_limit or 4096
+
+    def _format_chunk(self, text: str) -> str:
+        """Convert Markdown to channel format via UnifiedFormatter.
+
+        Uses the formatter auto-configured from ``capabilities.format_type``.
+        Subclasses rarely need to override this — set ``capabilities`` instead.
+        """
+        return self._formatter.format(text)
+
+    @abstractmethod
+    async def _send_chunk(
+        self,
+        chat_id: str,
+        formatted_text: str,
+        raw_text: str,
+        reply_to: str | None,
+        metadata: dict,
+    ) -> None:
+        """Send a single text chunk. Platform-specific implementation."""
+        ...
+
+    _format_fallback_patterns: tuple[str, ...] = ("parse", "invalid")
+
+    async def _send_with_format_fallback(
+        self,
+        send_fn: CallableABC[[str], Awaitable],
+        formatted: str,
+        raw: str,
+    ) -> None:
+        """Try *send_fn(formatted)*; on format-related errors retry with *raw*.
+
+        Channels whose ``_send_chunk`` follows the try-formatted / except-fallback
+        pattern can delegate to this helper instead of duplicating the logic.
+        """
+        try:
+            await send_fn(formatted)
+        except Exception as e:
+            if formatted != raw and any(
+                p in str(e).lower() for p in self._format_fallback_patterns
+            ):
+                self._trace_event(
+                    "outbound_format_fallback",
+                    error=str(e),
+                    formatted_len=len(formatted),
+                    raw_len=len(raw),
+                )
+                await send_fn(raw)
+            else:
+                raise
+
+    async def send_media(
+        self,
+        recipient: str,
+        file_path: str,
+        caption: str = "",
+        metadata: dict | None = None,
+    ) -> bool:
+        """Send a media file through the channel.
+
+        Handles the ready-check guard and error logging.  Subclasses
+        override ``_send_media_impl()`` with platform-specific logic.
+
+        Args:
+            recipient: Target recipient or chat identifier.
+            file_path: Local path to the media file.
+            caption: Optional caption text.
+            metadata: Optional channel-specific metadata.
+
+        Returns:
+            True if sent successfully, False otherwise.
+        """
+        if not self._is_ready():
+            return False
+        try:
+            return await self._send_media_impl(recipient, file_path, caption, metadata)
+        except Exception as e:
+            _logger.error(f"{self.name} send_media error: {e}")
+            return False
+
+    async def _send_media_impl(
+        self,
+        recipient: str,
+        file_path: str,
+        caption: str = "",
+        metadata: dict | None = None,
+    ) -> bool:
+        """Platform-specific media send.  Override in subclasses."""
+        return False
+
+    # ── Attachment / proxy helpers ─────────────────────────────────
+
+    def _media_path(self, filename: str) -> Path:
+        """Ensure MEDIA_DIR exists and return a path inside it."""
+        return media_path(filename)
+
+    def _resolve_media_chat_id(self, recipient: str, metadata: dict | None) -> str:
+        """Extract chat_id from metadata, falling back to recipient."""
+        return (metadata or {}).get("chat_id", recipient)
+
+    def _get_proxy(self) -> str | None:
+        """Return the configured proxy URL, or ``None`` if unset/empty."""
+        return getattr(self.config, "proxy", None) or None
+
+    def _check_attachment_size(self, file_size: int, filename: str) -> str | None:
+        """Return a 'too large' annotation string if *file_size* exceeds the limit."""
+        return check_attachment_size(file_size, filename)
+
+    async def _download_attachment(
+        self,
+        url: str,
+        filename: str,
+        *,
+        headers: dict[str, str] | None = None,
+        file_size: int | None = None,
+    ) -> tuple[str | None, str | None]:
+        """Download an attachment via httpx.  Returns ``(local_path, annotation)``.
+
+        Delegates to :func:`download_attachment`.
+        """
+        return await download_attachment(
+            url,
+            filename,
+            channel_name=self.name,
+            headers=headers,
+            file_size=file_size,
+            proxy=self._get_proxy(),
+        )
+
+    # ── Send retry abstraction ──────────────────────────────────────
+
+    _non_retryable_patterns: tuple[str, ...] = ()
+    _rate_limit_patterns: tuple[str, ...] = ("429", "ratelimit")
+    _rate_limit_delay: float = 1.0
+
+    def _extract_retry_after(self, exc: Exception) -> float | None:
+        """Extract retry-wait seconds from an exception.
+
+        Returns ``None`` to signal that the error is **not retryable**.
+
+        Pipeline:
+        1. SDK-provided ``retry_after`` attribute (Telegram / Slack SDKs).
+        2. HTTP ``Retry-After`` header via :meth:`_parse_retry_after_header`.
+        3. Non-retryable pattern match → ``None``.
+        4. Rate-limit pattern match → ``_rate_limit_delay``.
+        5. Default ``1.0`` s (generic transient-error retry).
+
+        Channels can customize behavior declaratively via class attributes
+        ``_non_retryable_patterns``, ``_rate_limit_patterns``, and
+        ``_rate_limit_delay``, or override this method entirely.
+        """
+        # 1. SDK retry_after attribute
+        retry = getattr(exc, "retry_after", None)
+        if retry is not None:
+            return float(retry)
+
+        # 2. HTTP Retry-After header
+        header_val = self._parse_retry_after_header(exc)
+        if header_val is not None:
+            return header_val
+
+        msg = str(exc).lower()
+
+        # 3. Non-retryable patterns
+        if self._non_retryable_patterns and any(
+            p in msg for p in self._non_retryable_patterns
+        ):
+            return None
+
+        # 4. Rate-limit patterns
+        if self._rate_limit_patterns and any(
+            p in msg for p in self._rate_limit_patterns
+        ):
+            return self._rate_limit_delay
+
+        # 5. Default
+        return 1.0
+
+    def _parse_retry_after_header(self, exc: Exception) -> float | None:
+        """Try to extract a ``Retry-After`` value from an HTTP response."""
+        resp = getattr(exc, "response", None)
+        if resp is None:
+            return None
+        headers = getattr(resp, "headers", None)
+        if not headers:
+            return None
+        raw = headers.get("Retry-After") or headers.get("retry-after")
+        if raw is None:
+            return None
+        try:
+            return float(raw)
+        except (ValueError, TypeError):
+            return None
+
+    async def _send_with_retry(
+        self,
+        coro_factory: CallableABC[[], Awaitable],
+        max_retries: int = 3,
+    ) -> Any:
+        """Send helper with automatic exponential-backoff retry.
+
+        *coro_factory* is called on every attempt so that the awaitable is
+        fresh.  Uses :func:`retry.retry_async` for backoff, jitter, and
+        server-supplied ``Retry-After`` support.
+
+        The *max_retries* parameter is accepted for backward compatibility
+        but the attempt count is taken from ``self._retry_config``.
+        """
+        from .retry import retry_async
+
+        def _on_retry(info):
+            self._trace_event(
+                "outbound_send_retry",
+                attempt=info.attempt,
+                max_attempts=info.max_attempts,
+                backoff_s=round(info.delay_s, 2),
+                error_type=type(info.error).__name__,
+            )
+            _logger.warning(
+                f"{self.name} send retry {info.attempt}/{info.max_attempts} "
+                f"in {info.delay_s:.2f}s: {info.error}"
+            )
+
+        return await retry_async(
+            coro_factory,
+            config=self._retry_config,
+            should_retry=lambda exc, _: self._extract_retry_after(exc) is not None,
+            retry_after_s=self._extract_retry_after,
+            on_retry=_on_retry,
+            label=f"{self.name}.send",
+        )
+
+    # ── Typing indicator abstraction ─────────────────────────────────
+
+    async def _send_typing_action(self, chat_id: str) -> None:
+        """Send a single typing indicator.  Override in sub-classes."""
+
+    async def start_typing(self, chat_id: str) -> None:
+        """Start a background typing-indicator loop for *chat_id*."""
+        await self._typing_manager.start(chat_id)
+
+    async def stop_typing(self, chat_id: str) -> None:
+        """Cancel the typing-indicator loop for *chat_id*."""
+        await self._typing_manager.stop(chat_id)
+
+    # ── Mention gating ──────────────────────────────────────────────
+
+    def _should_process(self, raw: RawIncoming) -> bool:
+        """Decide whether to process a message based on mention gating."""
+        if self.require_mention == "off":
+            return True
+        # Both "always" and "group" allow DMs through unconditionally
+        if not raw.is_group:
+            return True
+        if self.require_mention == "always":
+            return raw.was_mentioned
+        # "group" — require mention only in groups
+        return raw.was_mentioned
+
+    _mention_pattern: str | None = None
+    _mention_strip_count: int = 0  # 0 = all occurrences, 1 = first only
+
+    def _get_bot_identifier(self) -> str | None:
+        """Return the bot's identifier for mention pattern substitution.
+
+        Override in subclasses where ``_mention_pattern`` contains
+        ``{bot_id}`` placeholder.
+        """
+        return None
+
+    def _strip_mention(self, text: str) -> str:
+        """Strip bot mention from text using the ``_mention_pattern`` approach."""
+        if not self._mention_pattern:
+            return text
+        pattern = self._mention_pattern
+        if "{bot_id}" in pattern:
+            bot_id = self._get_bot_identifier()
+            if not bot_id:
+                return text
+            pattern = pattern.replace("{bot_id}", re.escape(bot_id))
+        return re.sub(pattern, "", text, count=self._mention_strip_count).strip()
+
+    # ── ACK reaction ─────────────────────────────────────────────────
+
+    async def _send_ack_reaction(
+        self, chat_id: str, message_id: str, emoji: str = "👀"
+    ) -> None:
+        """Send an acknowledgment reaction to a message. Override in subclasses that support reactions."""
+        pass  # Default no-op; channels override if they support reactions
+
+    async def _remove_ack_reaction(
+        self, chat_id: str, message_id: str, emoji: str = "👀"
+    ) -> None:
+        """Remove the ack reaction after replying. Override in subclasses."""
+        pass
+
+    # ── Inbound message pipeline ──────────────────────────────────────
+
+    async def _build_inbound_async(self, raw: RawIncoming) -> InboundMessage | None:
+        """Async version: run *raw* through inbound middlewares and convert."""
+        context: dict = {"channel": self}
+        current: RawIncoming | None = raw
+        for mw in self._inbound_middlewares:
+            if current is None:
+                return None
+            result = await mw.process_inbound(current, context)
+            if result is None:
+                return None
+            current = result
+        if current is None:
+            return None
+        return self._raw_to_inbound(current)
+
+    def _build_inbound(self, raw: RawIncoming) -> InboundMessage | None:
+        """Run *raw* through inbound middlewares and convert to InboundMessage.
+
+        Synchronous wrapper around :meth:`_build_inbound_async`.  When an
+        event loop is already running, the coroutine is scheduled on that
+        loop via :func:`asyncio.run_coroutine_threadsafe` to avoid
+        thread-safety issues with middleware state (DedupCache,
+        GroupHistoryBuffer, etc.).
+        """
+        import asyncio
+
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            loop = None
+
+        if loop is not None and loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(
+                self._build_inbound_async(raw),
+                loop,
+            )
+            return future.result()
+        else:
+            new_loop = asyncio.new_event_loop()
+            try:
+                return new_loop.run_until_complete(self._build_inbound_async(raw))
+            finally:
+                new_loop.close()
+
+    def _raw_to_inbound(self, raw: RawIncoming) -> InboundMessage | None:
+        """Convert a RawIncoming to InboundMessage (pure transformation, no filtering).
+
+        Merges text + annotations into content, sets metadata.
+        Returns None only if there is no content and no media.
+        """
+        parts = []
+        if raw.text:
+            parts.append(raw.text)
+        parts.extend(raw.content_annotations)
+        content = "\n".join(p for p in parts if p)
+        if not content and not raw.media_files:
+            return None
+        meta = dict(raw.metadata)
+        meta.setdefault("chat_id", raw.chat_id)
+        return InboundMessage(
+            channel=self.name,
+            sender_id=raw.sender_id,
+            chat_id=raw.chat_id,
+            content=content or "[media only]",
+            timestamp=raw.timestamp,
+            message_id=raw.message_id,
+            media=raw.media_files,
+            metadata=meta,
+            is_group=raw.is_group,
+            was_mentioned=raw.was_mentioned,
+        )
+
+    async def _enqueue_raw(self, raw: RawIncoming) -> None:
+        """Run *raw* through the inbound middleware pipeline, convert to
+        InboundMessage, and put it on the queue.
+
+        Convenience method for subclass ``_on_message`` handlers.
+        If STT is enabled and the message contains audio files, each audio
+        file is transcribed and the result is prepended to ``raw.text``.
+        """
+        self._trace_event(
+            "inbound_raw",
+            sender_id=raw.sender_id,
+            chat_id=raw.chat_id,
+            message_id=raw.message_id or "-",
+            has_text=bool(raw.text),
+            media_count=len(raw.media_files),
+            is_group=raw.is_group,
+        )
+        if raw.media_files and self._stt_enabled:
+            from ..stt import is_audio_file, transcribe_file
+
+            transcripts: list[str] = []
+            transcribed_files: set[str] = set()
+            for fp in raw.media_files:
+                if is_audio_file(fp):
+                    try:
+                        text = await transcribe_file(
+                            fp,
+                            language=self._stt_language,
+                            model=self._stt_model,
+                            device=self._stt_device,
+                            compute_type=self._stt_compute_type,
+                        )
+                    except Exception as exc:
+                        self._trace_event("stt_error", file_path=fp, error=str(exc))
+                        continue
+                    if text:
+                        transcripts.append(text)
+                        transcribed_files.add(fp)
+                # Non-audio files are silently skipped — no trace event
+                # to avoid log noise when messages contain many images.
+            if transcripts:
+                prefix = "\n".join(transcripts)
+                raw.text = (prefix + "\n" + raw.text).strip() if raw.text else prefix
+                # Remove annotations for transcribed files (exact path match)
+                # so the agent does not attempt to process the audio file itself
+                raw.content_annotations = [
+                    a
+                    for a in raw.content_annotations
+                    if not any(
+                        fp == a or a.endswith(f": {fp}]") or a == f"[voice: {fp}]"
+                        for fp in transcribed_files
+                    )
+                ]
+
+        msg = await self._build_inbound_async(raw)
+        if msg is None:
+            return
+        if raw.message_id:
+            try:
+                await self._send_ack_reaction(raw.chat_id, raw.message_id)
+            except Exception:
+                pass
+        await self._queue.put(msg)
+
+    # ── Bus integration ──────────────────────────────────────────────
+
+    def set_bus(self, bus) -> None:
+        """Inject the MessageBus reference (called by ChannelManager)."""
+        self._bus = bus
+
+    async def queue_message(self, msg: InboundMessage) -> None:
+        """Buffer *msg* with debounce, then publish to bus."""
+        sender = msg.sender_id
+
+        if sender not in self._message_buffers:
+            self._message_buffers[sender] = []
+            self._message_metadata[sender] = msg.metadata
+            self._message_media[sender] = []
+            self._message_is_group[sender] = msg.is_group
+            self._message_was_mentioned[sender] = msg.was_mentioned
+        self._message_buffers[sender].append(msg.content)
+        if msg.message_id:
+            self._message_ids[sender] = msg.message_id
+        if msg.media:
+            self._message_media[sender].extend(msg.media)
+
+        if self._on_activity:
+            try:
+                self._on_activity(sender, "received")
+            except Exception:
+                pass
+
+        if sender in self._debounce_tasks:
+            self._debounce_tasks[sender].cancel()
+
+        msg_count = len(self._message_buffers[sender])
+        wait = min(
+            self.initial_debounce + (msg_count - 1) * self.debounce_step,
+            self.max_debounce,
+        )
+        _logger.debug(f"Debounce for {sender}: {wait:.1f}s (message #{msg_count})")
+
+        async def debounce_callback(_s=sender, _w=wait):
+            await asyncio.sleep(_w)
+            try:
+                await self._process_buffered_messages(_s)
+            except Exception as e:
+                _logger.error(f"{self.name} debounce flush error for {_s}: {e}")
+
+        self._debounce_tasks[sender] = asyncio.create_task(debounce_callback())
+
+    async def _process_buffered_messages(self, sender: str) -> None:
+        """Flush buffered messages for *sender* and publish to bus."""
+        if sender not in self._message_buffers:
+            return
+
+        messages = self._message_buffers.pop(sender, [])
+        metadata = self._message_metadata.pop(sender, None)
+        media = self._message_media.pop(sender, [])
+        message_id = self._message_ids.pop(sender, "")
+        is_group = self._message_is_group.pop(sender, False)
+        was_mentioned = self._message_was_mentioned.pop(sender, True)
+        self._debounce_tasks.pop(sender, None)
+        if not messages:
+            return
+
+        merged_content = "\n".join(messages)
+        _logger.info(f"Processing {len(messages)} merged message(s) from {sender}")
+
+        if self._bus:
+            chat_id = (metadata or {}).get("chat_id", sender)
+            inbound = InboundMessage(
+                channel=self.name,
+                sender_id=sender,
+                chat_id=str(chat_id),
+                content=merged_content,
+                media=media,
+                metadata=metadata or {},
+                message_id=message_id,
+                is_group=is_group,
+                was_mentioned=was_mentioned,
+            )
+            await self._bus.publish_inbound(inbound)
+
+    async def _send_status_message(
+        self,
+        sender: str,
+        content: str,
+        metadata: dict | None = None,
+    ) -> None:
+        """Send a status/intermediate message to the channel."""
+        chat_id = (metadata or {}).get("chat_id", sender)
+        await self.send(
+            OutboundMessage(
+                channel=self.name,
+                chat_id=str(chat_id),
+                content=content,
+                metadata=metadata or {},
+            )
+        )
+
+    async def send_thinking_message(
+        self,
+        sender: str,
+        thinking: str,
+        metadata: dict | None = None,
+    ) -> None:
+        """Send a thinking intermediate message to the channel."""
+        if not self.send_thinking:
+            return
+        await self._send_status_message(
+            sender, f"\U0001f9e0\n{thinking}\n\u23f3", metadata
+        )
+
+    async def send_todo_message(
+        self,
+        sender: str,
+        content: str,
+        metadata: dict | None = None,
+    ) -> None:
+        """Send a todo list intermediate message to the channel."""
+        await self._send_status_message(sender, content, metadata)
+
+    async def run(self) -> None:
+        """Run the channel with auto-reconnect (exponential backoff)."""
+        backoff = 1.0
+        max_backoff = 60.0
+        self._running = True
+        while self._running:
+            try:
+                await self.start()
+                backoff = 1.0
+                async for msg in self.receive():
+                    await self.queue_message(msg)
+            except asyncio.CancelledError:
+                break
+            except ChannelError as e:
+                self._trace_event(
+                    "channel_fatal_error",
+                    error_type=type(e).__name__,
+                )
+                _logger.error(f"Channel {self.name} fatal error: {e}")
+                self._running = False
+                break
+            except Exception as e:
+                self._trace_event(
+                    "channel_runtime_error",
+                    error_type=type(e).__name__,
+                )
+                _logger.error(f"Channel {self.name} error: {e}")
+            finally:
+                # Preserve reconnect intent across stop()
+                should_reconnect = self._running
+                try:
+                    await self.stop()
+                except Exception:
+                    pass
+                self._running = should_reconnect
+
+            if self._running:
+                _logger.info(f"Reconnecting {self.name} in {backoff:.1f}s...")
+                await asyncio.sleep(backoff)
+                backoff = min(backoff * 2, max_backoff)
+
+    # ── Channel allow-list check ─────────────────────────────────────
+
+    def is_channel_allowed(self, channel_id: str) -> bool:
+        """Return ``True`` if *channel_id* is permitted by config.
+
+        When the allow-list is empty or absent every channel is allowed.
+        """
+        allowed = getattr(self.config, "allowed_channels", None)
+        return not allowed or str(channel_id) in allowed
+
+    # ── Sender allow-list check ──────────────────────────────────────
+
+    def is_allowed(self, sender: str) -> bool:
+        """Check if *sender* is permitted by ``self.config.allowed_senders``.
+
+        Returns ``True`` when the allow-list is empty / None (open access).
+        Supports ``|``-separated composite IDs (e.g. ``"uid|gid"``).
+        Subclasses with richer filtering (iMessage) may override.
+        """
+        config = getattr(self, "config", None)
+        allowed = getattr(config, "allowed_senders", None) if config else None
+        if not allowed:
+            return True
+        sender_str = str(sender)
+        if sender_str in allowed:
+            return True
+        if "|" in sender_str:
+            for part in sender_str.split("|"):
+                if part and part in allowed:
+                    return True
+        return False
+
+
+class ChannelError(Exception):
+    """Base exception for channel-related errors."""
+
+    pass
diff --git a/EvoScientist/channels/bus/__init__.py b/EvoScientist/channels/bus/__init__.py
new file mode 100644
index 0000000..d2ee6e3
--- /dev/null
+++ b/EvoScientist/channels/bus/__init__.py
@@ -0,0 +1,6 @@
+"""Message bus for decoupled channel-agent communication."""
+
+from .events import InboundMessage, OutboundMessage
+from .message_bus import MessageBus
+
+__all__ = ["InboundMessage", "MessageBus", "OutboundMessage"]
diff --git a/EvoScientist/channels/bus/events.py b/EvoScientist/channels/bus/events.py
new file mode 100644
index 0000000..411006f
--- /dev/null
+++ b/EvoScientist/channels/bus/events.py
@@ -0,0 +1,52 @@
+"""Event types for the message bus."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class InboundMessage:
+    """Message received from a chat channel.
+
+    Carries enough context for the bus to route and for the agent
+    to build a session: which channel, who sent it, which chat.
+    """
+
+    channel: str
+    sender_id: str
+    chat_id: str
+    content: str
+    timestamp: datetime = field(default_factory=datetime.now)
+    message_id: str = ""
+    media: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+    is_group: bool = False
+    was_mentioned: bool = True
+
+    @property
+    def sender(self) -> str:
+        """Alias for ``sender_id`` (compatibility with IncomingMessage)."""
+        return self.sender_id
+
+    @property
+    def session_key(self) -> str:
+        """Unique key for session identification: ``channel:chat_id``."""
+        return f"{self.channel}:{self.chat_id}"
+
+
+@dataclass
+class OutboundMessage:
+    """Message to send to a chat channel."""
+
+    channel: str
+    chat_id: str
+    content: str
+    reply_to: str | None = None
+    media: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def recipient(self) -> str:
+        """Alias for ``chat_id`` (compatibility with OutgoingMessage)."""
+        return self.chat_id
diff --git a/EvoScientist/channels/bus/message_bus.py b/EvoScientist/channels/bus/message_bus.py
new file mode 100644
index 0000000..89afcc3
--- /dev/null
+++ b/EvoScientist/channels/bus/message_bus.py
@@ -0,0 +1,114 @@
+"""Async message bus that decouples chat channels from the agent core.
+
+Channels push messages to the inbound queue; the agent (or any consumer)
+reads from inbound, processes, and pushes responses to the outbound queue.
+A background dispatcher routes outbound messages to the correct channel
+via subscriber callbacks.
+
+Deduplication is handled at the Channel level (single dedup point).
+"""
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+
+from ..debug import TraceMixin, debug_trace_enabled
+from .events import InboundMessage, OutboundMessage
+
+logger = logging.getLogger(__name__)
+
+OutboundCallback = Callable[[OutboundMessage], Awaitable[None]]
+
+
+class MessageBus(TraceMixin):
+    """Async message bus that decouples chat channels from the agent core."""
+
+    name = "bus"
+
+    def __init__(self):
+        self.inbound: asyncio.Queue[InboundMessage] = asyncio.Queue(maxsize=5000)
+        self.outbound: asyncio.Queue[OutboundMessage] = asyncio.Queue(maxsize=5000)
+        self._outbound_subscribers: dict[str, list[OutboundCallback]] = {}
+        self._running = False
+        self._debug_trace = debug_trace_enabled()
+        self._trace_logger = logger
+
+    # ── inbound (channel → agent) ──
+
+    async def publish_inbound(self, msg: InboundMessage) -> None:
+        """Publish a message from a channel to the agent."""
+        await self.inbound.put(msg)
+
+    async def consume_inbound(self) -> InboundMessage:
+        """Consume the next inbound message (blocks until available)."""
+        return await self.inbound.get()
+
+    # ── outbound (agent → channel) ──
+
+    async def publish_outbound(self, msg: OutboundMessage) -> None:
+        """Publish a response from the agent to channels."""
+        await self.outbound.put(msg)
+
+    async def consume_outbound(self) -> OutboundMessage:
+        """Consume the next outbound message (blocks until available)."""
+        return await self.outbound.get()
+
+    # ── subscriber routing ──
+
+    def subscribe_outbound(
+        self,
+        channel: str,
+        callback: OutboundCallback,
+    ) -> None:
+        """Register a callback for outbound messages targeting *channel*."""
+        if channel not in self._outbound_subscribers:
+            self._outbound_subscribers[channel] = []
+        self._outbound_subscribers[channel].append(callback)
+
+    async def dispatch_outbound(self) -> None:
+        """Route outbound messages to subscribed channels.
+
+        Run as a background task — loops until :meth:`stop` is called.
+        """
+        self._running = True
+        while self._running:
+            try:
+                msg = await asyncio.wait_for(
+                    self.outbound.get(),
+                    timeout=1.0,
+                )
+            except TimeoutError:
+                continue
+            subscribers = self._outbound_subscribers.get(msg.channel, [])
+            if not subscribers:
+                self._trace_event(
+                    "bus_dispatch_drop",
+                    target_channel=msg.channel,
+                    reason="no_subscriber",
+                    chat_id=msg.chat_id,
+                )
+                logger.warning(f"No subscriber for channel: {msg.channel}")
+                continue
+            for callback in subscribers:
+                try:
+                    await callback(msg)
+                except Exception as e:
+                    self._trace_event(
+                        "bus_dispatch_error",
+                        target_channel=msg.channel,
+                        chat_id=msg.chat_id,
+                        error_type=type(e).__name__,
+                    )
+                    logger.error(f"Error dispatching to {msg.channel}: {e}")
+
+    def stop(self) -> None:
+        """Stop the dispatcher loop."""
+        self._running = False
+
+    @property
+    def inbound_size(self) -> int:
+        return self.inbound.qsize()
+
+    @property
+    def outbound_size(self) -> int:
+        return self.outbound.qsize()
diff --git a/EvoScientist/channels/capabilities.py b/EvoScientist/channels/capabilities.py
new file mode 100644
index 0000000..7846ca4
--- /dev/null
+++ b/EvoScientist/channels/capabilities.py
@@ -0,0 +1,220 @@
+"""Channel capabilities declaration system.
+
+Each channel declares its capabilities via a ChannelCapabilities dataclass,
+enabling the framework to adapt behavior automatically (formatting, reactions,
+streaming, threading, etc.) without per-channel branching in core logic.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+FormatType = Literal["html", "markdown", "slack_mrkdwn", "discord", "plain"]
+
+
+@dataclass(frozen=True)
+class ChannelCapabilities:
+    """Immutable declaration of what a channel supports.
+
+    Set once as a class attribute on each Channel subclass.
+    The framework inspects these at runtime to auto-configure behavior.
+    """
+
+    # ── Messaging features ──────────────────────────────────────────
+    format_type: FormatType = "plain"
+    max_text_length: int = 4096
+    max_file_size: int = 20 * 1024 * 1024  # 20 MB
+
+    # ── Interaction capabilities ────────────────────────────────────
+    streaming: bool = False  # edit-in-place streaming output
+    threading: bool = False  # message threads / topics
+    reactions: bool = False  # emoji reactions on messages
+    typing: bool = False  # typing indicator API
+    inline_buttons: bool = False  # inline keyboard / action buttons
+
+    # ── Media capabilities ──────────────────────────────────────────
+    media_send: bool = False  # can send files/images
+    media_receive: bool = False  # can receive files/images
+    voice: bool = False  # platform has voice/audio messages that arrive as downloadable files (receive only, not bot sending)
+    stickers: bool = False  # supports sticker receive (not bot sending)
+    location: bool = False  # supports location message receive (not bot sending)
+    video: bool = False  # video messages
+
+    # ── Group features ──────────────────────────────────────────────
+    groups: bool = False  # group chat support
+    mentions: bool = False  # @mention detection
+
+    # ── Rich text ───────────────────────────────────────────────────
+    markdown: bool = False  # supports Markdown rendering
+    html: bool = False  # supports HTML rendering
+
+    # ── Extended capabilities ────────────────────────────────────────
+    chat_types: tuple[str, ...] = ()  # ("direct", "group", "channel", "thread")
+    edit: bool = False  # message editing after send
+    unsend: bool = False  # message recall / unsend
+    block_streaming: bool = False  # block edit-in-place streaming
+    native_commands: bool = False  # platform-native slash commands
+    polls: bool = False  # poll / vote messages
+
+    def supports(self, feature: str) -> bool:
+        """Check if a feature is supported by name."""
+        return getattr(self, feature, False)
+
+
+# ═════════════════════════════════════════════════════════════════════
+# Pre-built capability profiles for each channel
+# ═════════════════════════════════════════════════════════════════════
+
+TELEGRAM = ChannelCapabilities(
+    format_type="html",
+    max_text_length=4000,
+    streaming=False,  # could edit messages, but not implemented yet
+    threading=False,  # topics exist but not used yet
+    reactions=True,
+    typing=True,
+    media_send=True,
+    media_receive=True,
+    voice=True,
+    stickers=True,
+    location=True,
+    groups=True,
+    mentions=True,
+    html=True,
+    chat_types=("direct", "group", "channel"),
+    edit=True,
+    unsend=True,
+    native_commands=True,
+    polls=True,
+)
+
+DISCORD = ChannelCapabilities(
+    format_type="discord",
+    max_text_length=2000,
+    streaming=False,
+    threading=True,
+    reactions=True,
+    typing=True,
+    media_send=True,
+    media_receive=True,
+    voice=False,  # no distinct voice message type in Discord bot API
+    groups=True,
+    mentions=True,
+    markdown=True,
+    chat_types=("direct", "group", "thread"),
+    edit=True,
+    unsend=True,
+    native_commands=True,
+    polls=True,
+)
+
+SLACK = ChannelCapabilities(
+    format_type="slack_mrkdwn",
+    max_text_length=4000,
+    streaming=False,
+    threading=True,
+    reactions=True,
+    typing=False,  # no native typing API; workaround via post+delete "..." message
+    media_send=True,
+    media_receive=True,
+    voice=False,  # no distinct voice message type in Slack bot API
+    groups=True,
+    mentions=True,
+    chat_types=("direct", "group", "thread"),
+    edit=True,
+    unsend=True,
+    native_commands=True,
+)
+
+FEISHU = ChannelCapabilities(
+    format_type="markdown",
+    max_text_length=4096,
+    reactions=True,
+    typing=False,  # no typing API
+    media_send=True,
+    media_receive=True,
+    voice=True,
+    stickers=True,
+    groups=True,
+    mentions=True,
+    markdown=True,
+    chat_types=("direct", "group"),
+    edit=True,
+    unsend=True,
+)
+
+DINGTALK = ChannelCapabilities(
+    format_type="markdown",
+    max_text_length=4096,
+    typing=False,  # no typing API for bots
+    media_send=True,
+    media_receive=True,
+    voice=True,
+    groups=True,
+    mentions=True,
+    markdown=True,
+    chat_types=("direct", "group"),
+)
+
+QQ = ChannelCapabilities(
+    format_type="plain",
+    max_text_length=4096,
+    typing=False,  # no typing API for QQ bots
+    media_send=True,
+    media_receive=True,
+    voice=False,  # qq-botpy does not expose voice as a distinct message type
+    groups=True,
+    mentions=True,
+    chat_types=("direct", "group", "channel"),
+    unsend=True,
+)
+
+WECHAT = ChannelCapabilities(
+    format_type="markdown",  # WeCom supports markdown
+    max_text_length=4096,
+    typing=False,  # no typing API
+    media_send=True,
+    media_receive=True,
+    voice=True,
+    location=True,
+    groups=True,
+    mentions=True,
+    markdown=True,
+    chat_types=("direct", "group"),
+    unsend=True,
+)
+
+SIGNAL = ChannelCapabilities(
+    format_type="plain",
+    max_text_length=4096,
+    reactions=True,
+    typing=True,
+    media_send=True,
+    media_receive=True,
+    voice=True,
+    groups=True,
+    mentions=True,
+    chat_types=("direct", "group"),
+)
+
+EMAIL = ChannelCapabilities(
+    format_type="html",
+    max_text_length=999_999,  # no practical limit
+    media_send=True,
+    media_receive=True,
+    html=True,
+    chat_types=("direct",),
+)
+
+IMESSAGE = ChannelCapabilities(
+    format_type="plain",
+    max_text_length=999_999,
+    typing=False,  # Apple does not expose typing indicator API
+    media_send=True,
+    media_receive=True,
+    voice=True,
+    groups=True,
+    mentions=False,  # iMessage has no @mention concept
+    reactions=False,  # imsg CLI cannot send tapback reactions
+    chat_types=("direct", "group"),
+)
diff --git a/EvoScientist/channels/channel_manager.py b/EvoScientist/channels/channel_manager.py
new file mode 100644
index 0000000..c548a2f
--- /dev/null
+++ b/EvoScientist/channels/channel_manager.py
@@ -0,0 +1,1051 @@
+"""Unified channel manager for coordinating chat channels.
+
+Manages channel lifecycle (start/stop), wires each channel to the
+message bus, and routes outbound messages to the correct channel.
+
+Also provides the global channel registry (formerly in ``registry.py``),
+account management (formerly ``account.py``), and pipeline assembly
+(formerly ``pipeline.py``).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import importlib
+import json
+import logging
+import pkgutil
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from .base import Channel, OutboundMessage
+from .bus import MessageBus
+from .middleware import OutboundMiddlewareBase
+from .plugin import ChannelPlugin
+
+logger = logging.getLogger(__name__)
+
+
+# ═════════════════════════════════════════════════════════════════════
+# Account management (formerly account.py)
+# ═════════════════════════════════════════════════════════════════════
+
+
+@dataclass
+class ChannelAccountSnapshot:
+    """Point-in-time snapshot of a single account's connection state."""
+
+    account_id: str
+    channel: str
+    connected: bool = False
+    started_at: float = 0.0
+    last_outbound_at: float = 0.0
+    error: str | None = None
+
+    def mark_connected(self) -> None:
+        self.connected = True
+        self.started_at = time.monotonic()
+        self.error = None
+
+    def mark_disconnected(self, error: str | None = None) -> None:
+        self.connected = False
+        self.error = error
+
+    def mark_outbound(self) -> None:
+        self.last_outbound_at = time.monotonic()
+
+
+@dataclass
+class AccountConfig:
+    """Per-account configuration wrapper."""
+
+    account_id: str
+    channel_id: str  # which plugin
+    enabled: bool = True
+    config: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class AccountState:
+    """Runtime state for a single account."""
+
+    account_id: str
+    channel_id: str
+    status: str = "stopped"  # stopped | starting | running | error
+    snapshot: ChannelAccountSnapshot | None = None
+    error: str | None = None
+    started_at: float = 0.0
+
+
+class AccountManager:
+    """Manages multiple accounts across channel plugins.
+
+    Works with the ``ConfigAdapter`` protocol on each plugin to discover
+    accounts and manage their lifecycle independently.
+    """
+
+    def __init__(self) -> None:
+        self._plugins: dict[str, ChannelPlugin] = {}
+        self._states: dict[str, AccountState] = {}  # key: "{channel_id}:{account_id}"
+
+    @staticmethod
+    def _key(channel_id: str, account_id: str) -> str:
+        return f"{channel_id}:{account_id}"
+
+    def register_plugin(self, plugin: ChannelPlugin) -> None:
+        """Register a plugin that supports multi-account."""
+        self._plugins[plugin.id] = plugin
+        logger.info(f"AccountManager: registered plugin '{plugin.id}'")
+
+    async def start_account(
+        self,
+        channel_id: str,
+        account_id: str,
+        config: Any = None,
+    ) -> None:
+        """Start a specific account on a plugin."""
+        plugin = self._plugins.get(channel_id)
+        if plugin is None:
+            raise ValueError(f"No plugin registered for channel '{channel_id}'")
+
+        key = self._key(channel_id, account_id)
+        state = self._states.get(key)
+        if state is None:
+            state = AccountState(account_id=account_id, channel_id=channel_id)
+            self._states[key] = state
+
+        if state.status == "running":
+            logger.warning(f"Account {key} is already running")
+            return
+
+        state.status = "starting"
+        state.error = None
+        try:
+            account_config = config
+            if plugin.config_adapter is not None and config is not None:
+                account_config = plugin.config_adapter.resolve_account(
+                    config, account_id
+                )
+
+            await plugin.start(account_config, account_id=account_id)
+            state.status = "running"
+            state.started_at = time.monotonic()
+            state.snapshot = ChannelAccountSnapshot(
+                account_id=account_id,
+                channel=channel_id,
+            )
+            state.snapshot.mark_connected()
+            logger.info(f"Account {key} started")
+        except Exception as e:
+            state.status = "error"
+            state.error = str(e)
+            logger.error(f"Failed to start account {key}: {e}")
+            raise
+
+    async def stop_account(self, channel_id: str, account_id: str) -> None:
+        """Stop a specific account on a plugin."""
+        plugin = self._plugins.get(channel_id)
+        if plugin is None:
+            raise ValueError(f"No plugin registered for channel '{channel_id}'")
+
+        key = self._key(channel_id, account_id)
+        state = self._states.get(key)
+        if state is None or state.status == "stopped":
+            logger.debug(f"Account {key} is already stopped")
+            return
+
+        try:
+            await plugin.stop(account_id=account_id)
+            state.status = "stopped"
+            if state.snapshot is not None:
+                state.snapshot.mark_disconnected()
+            logger.info(f"Account {key} stopped")
+        except Exception as e:
+            state.status = "error"
+            state.error = str(e)
+            if state.snapshot is not None:
+                state.snapshot.mark_disconnected(error=str(e))
+            logger.error(f"Error stopping account {key}: {e}")
+            raise
+
+    async def restart_account(
+        self,
+        channel_id: str,
+        account_id: str,
+        config: Any = None,
+    ) -> None:
+        """Restart a specific account."""
+        await self.stop_account(channel_id, account_id)
+        await self.start_account(channel_id, account_id, config)
+
+    async def start_all(self, channel_id: str, config: Any = None) -> None:
+        """Start all accounts for a given channel plugin."""
+        plugin = self._plugins.get(channel_id)
+        if plugin is None:
+            raise ValueError(f"No plugin registered for channel '{channel_id}'")
+
+        adapter = plugin.config_adapter
+        if adapter is None:
+            await self.start_account(channel_id, "default", config)
+            return
+
+        if config is None:
+            logger.warning(f"No config provided for start_all on '{channel_id}'")
+            return
+
+        for account_id in adapter.list_account_ids(config):
+            if adapter.is_enabled(
+                adapter.resolve_account(config, account_id),
+                config,
+            ):
+                try:
+                    await self.start_account(channel_id, account_id, config)
+                except Exception as e:
+                    logger.error(
+                        f"Failed to start account {channel_id}:{account_id}: {e}"
+                    )
+
+    async def stop_all(self, channel_id: str) -> None:
+        """Stop all accounts for a given channel plugin."""
+        keys_to_stop = [
+            (state.channel_id, state.account_id)
+            for state in self._states.values()
+            if state.channel_id == channel_id and state.status != "stopped"
+        ]
+        for cid, aid in keys_to_stop:
+            try:
+                await self.stop_account(cid, aid)
+            except Exception as e:
+                logger.error(f"Failed to stop account {cid}:{aid}: {e}")
+
+    def get_state(
+        self,
+        channel_id: str,
+        account_id: str,
+    ) -> AccountState | None:
+        """Get the runtime state for a specific account."""
+        return self._states.get(self._key(channel_id, account_id))
+
+    def list_accounts(
+        self,
+        channel_id: str | None = None,
+    ) -> list[AccountState]:
+        """List account states, optionally filtered by channel."""
+        if channel_id is None:
+            return list(self._states.values())
+        return [s for s in self._states.values() if s.channel_id == channel_id]
+
+    def get_snapshot(
+        self,
+        channel_id: str,
+        account_id: str,
+    ) -> ChannelAccountSnapshot | None:
+        """Get the connection snapshot for a specific account."""
+        state = self._states.get(self._key(channel_id, account_id))
+        return state.snapshot if state else None
+
+
+# ═════════════════════════════════════════════════════════════════════
+# Inbound / outbound pipelines (formerly pipeline.py)
+# ═════════════════════════════════════════════════════════════════════
+
+
+class OutboundPipeline:
+    """Processes outgoing messages through a middleware chain."""
+
+    def __init__(
+        self,
+        plugin: ChannelPlugin,
+        middlewares: list[OutboundMiddlewareBase],
+    ) -> None:
+        self.plugin = plugin
+        self.middlewares = middlewares
+
+    async def process(
+        self,
+        message: OutboundMessage,
+        context: dict[str, Any] | None = None,
+    ) -> OutboundMessage | None:
+        """Run *message* through each middleware.  Returns ``None`` if dropped."""
+        ctx = context or {}
+        current: OutboundMessage | None = message
+        for mw in self.middlewares:
+            if current is None:
+                return None
+            current = await mw.process_outbound(current, ctx)
+        return current
+
+
+def build_outbound_pipeline(
+    plugin: ChannelPlugin,
+    config: Any,
+) -> OutboundPipeline:
+    """Auto-assemble outbound pipeline based on plugin capabilities.
+
+    FormattingMiddleware has been removed — Channel.send() handles
+    formatting + chunking via _format_chunk() / _prepare_chunks().
+    """
+    middlewares: list[OutboundMiddlewareBase] = []
+    return OutboundPipeline(plugin, middlewares)
+
+
+# ── Per-channel health tracking ──────────────────────────────────────
+
+
+@dataclass
+class ChannelHealth:
+    """Tracks send success / failure metrics for a single channel."""
+
+    consecutive_failures: int = 0
+    last_failure_time: float | None = None
+    last_failure_error: str | None = None
+    total_failures: int = 0
+    total_successes: int = 0
+
+
+# ── Minimal HTTP health-check server ────────────────────────────────
+
+
+class _HealthServer:
+    """Zero-dependency HTTP health-check endpoint using ``asyncio.start_server``.
+
+    Responds to ``GET /healthz`` with a JSON status payload; all other
+    requests receive a 404.  A per-connection timeout prevents slow
+    clients from tying up the server.
+    """
+
+    _CONNECTION_TIMEOUT = 5.0  # seconds
+
+    def __init__(self, manager: ChannelManager, port: int) -> None:
+        self._manager = manager
+        self._port = port
+        self._server: asyncio.AbstractServer | None = None
+        self._start_time: float = 0.0
+
+    async def start(self) -> None:
+        self._start_time = time.monotonic()
+        self._server = await asyncio.start_server(
+            self._handle_connection,
+            "0.0.0.0",
+            self._port,
+        )
+        addrs = [s.getsockname() for s in self._server.sockets]
+        logger.info(f"Health server listening on {addrs}")
+
+    async def stop(self) -> None:
+        if self._server is not None:
+            self._server.close()
+            await self._server.wait_closed()
+            self._server = None
+            logger.info("Health server stopped")
+
+    async def _handle_connection(
+        self,
+        reader: asyncio.StreamReader,
+        writer: asyncio.StreamWriter,
+    ) -> None:
+        try:
+            await asyncio.wait_for(
+                self._process_request(reader, writer),
+                timeout=self._CONNECTION_TIMEOUT,
+            )
+        except (TimeoutError, ConnectionError, OSError):
+            pass
+        finally:
+            try:
+                writer.close()
+                await writer.wait_closed()
+            except (ConnectionError, OSError):
+                pass
+
+    async def _process_request(
+        self,
+        reader: asyncio.StreamReader,
+        writer: asyncio.StreamWriter,
+    ) -> None:
+        request_line = await reader.readline()
+        # Consume remaining headers
+        while True:
+            line = await reader.readline()
+            if line in (b"\r\n", b"\n", b""):
+                break
+
+        parts = request_line.decode("utf-8", errors="replace").split()
+        if len(parts) >= 2 and parts[0] == "GET" and parts[1] == "/healthz":
+            body = self._build_response()
+            payload = json.dumps(body).encode()
+            header = (
+                "HTTP/1.1 200 OK\r\n"
+                "Content-Type: application/json\r\n"
+                f"Content-Length: {len(payload)}\r\n"
+                "Connection: close\r\n"
+                "\r\n"
+            )
+        else:
+            payload = b'{"error":"not found"}'
+            header = (
+                "HTTP/1.1 404 Not Found\r\n"
+                "Content-Type: application/json\r\n"
+                f"Content-Length: {len(payload)}\r\n"
+                "Connection: close\r\n"
+                "\r\n"
+            )
+        writer.write(header.encode() + payload)
+        await writer.drain()
+
+    def _build_response(self) -> dict[str, Any]:
+        mgr = self._manager
+        health_map: dict[str, Any] = {}
+        for name, h in mgr._health.items():
+            health_map[name] = {
+                "consecutive_failures": h.consecutive_failures,
+                "total_successes": h.total_successes,
+                "total_failures": h.total_failures,
+            }
+        accounts_map: dict[str, Any] = {}
+        for state in mgr._account_manager.list_accounts():
+            key = f"{state.channel_id}:{state.account_id}"
+            accounts_map[key] = {
+                "account_id": state.account_id,
+                "channel": state.channel_id,
+                "status": state.status,
+                "error": state.error,
+            }
+        resp: dict[str, Any] = {
+            "status": "healthy",
+            "uptime_seconds": round(time.monotonic() - self._start_time, 1),
+            "channels": {
+                "enabled": mgr.enabled_channels,
+                "running": mgr.running_channels(),
+            },
+            "queues": {
+                "inbound_size": mgr.bus.inbound_size,
+                "outbound_size": mgr.bus.outbound_size,
+            },
+            "health": health_map,
+            "accounts": accounts_map,
+        }
+        for pname, provider in mgr._health_providers.items():
+            try:
+                resp[pname] = provider()
+            except Exception:
+                resp[pname] = {"error": "provider failed"}
+        return resp
+
+
+# ── Channel registry ──────────────────────────────────────────────────
+
+ChannelFactory = Callable[..., Channel]
+
+_CHANNEL_REGISTRY: dict[str, ChannelFactory] = {}
+
+
+def _parse_csv(value: str) -> set[str] | None:
+    """Parse comma-separated string into a set, or ``None`` if empty."""
+    if not value or not value.strip():
+        return None
+    items = {s.strip() for s in value.split(",") if s.strip()}
+    return items or None
+
+
+def register_channel(name: str, factory: ChannelFactory) -> None:
+    """Register a channel factory under *name*."""
+    _CHANNEL_REGISTRY[name] = factory
+
+
+def create_channel(name: str, config) -> Channel:
+    """Create a channel instance using the registered factory for *name*."""
+    factory = _CHANNEL_REGISTRY.get(name)
+    if not factory:
+        raise ValueError(
+            f"Unknown channel type: {name}. Available: {list(_CHANNEL_REGISTRY.keys())}"
+        )
+    return factory(config)
+
+
+def available_channels() -> list[str]:
+    """Return the names of all available channel types.
+
+    Triggers auto-discovery if the registry is empty.
+    """
+    if not _CHANNEL_REGISTRY:
+        _ensure_channels_registered()
+    return list(_CHANNEL_REGISTRY.keys())
+
+
+def _discover_channel_subpackages() -> list[str]:
+    """Discover all channel sub-packages under the channels directory.
+
+    Returns a list of sub-package names (e.g. ["telegram", "discord", ...]).
+    Excludes non-channel directories (bus, __pycache__) and plain modules.
+    """
+    channels_dir = Path(__file__).parent
+    _EXCLUDED = {"bus", "__pycache__"}
+    names = []
+    for info in pkgutil.iter_modules([str(channels_dir)]):
+        if info.ispkg and info.name not in _EXCLUDED:
+            names.append(info.name)
+    return sorted(names)
+
+
+def _ensure_channels_registered(types: list[str] | None = None) -> None:
+    """Lazily import channel sub-packages to trigger registration.
+
+    If *types* is given, only those channels are imported.
+    If *types* is ``None``, all discovered channel sub-packages are imported.
+    """
+    if types is None:
+        targets = _discover_channel_subpackages()
+    else:
+        # Only import the ones that exist as sub-packages
+        available = set(_discover_channel_subpackages())
+        targets = [t for t in types if t in available]
+
+    for t in targets:
+        module_name = f"EvoScientist.channels.{t}"
+        if t not in _CHANNEL_REGISTRY:
+            try:
+                importlib.import_module(module_name)
+            except ImportError as e:
+                logger.debug(f"Could not import channel {t}: {e}")
+
+
+# ── Shared webhook server ─────────────────────────────────────────
+
+
+class SharedWebhookServer:
+    """Single aiohttp server that hosts routes from multiple HTTP channels.
+
+    When ``shared_webhook_port`` is configured, ``ChannelManager`` collects
+    routes from every channel that exposes ``_webhook_routes()`` and starts
+    one server instead of letting each channel bind its own port.
+    """
+
+    def __init__(self, port: int) -> None:
+        self._port = port
+        self._app: Any = None
+        self._runner: Any = None
+        self._site: Any = None
+
+    async def start(self, routes: list[tuple[str, str, Any]]) -> None:
+        from aiohttp import web
+
+        self._app = web.Application()
+        for method, path, handler in routes:
+            if method.upper() == "GET":
+                self._app.router.add_get(path, handler)
+            else:
+                self._app.router.add_post(path, handler)
+
+        self._runner = web.AppRunner(self._app)
+        await self._runner.setup()
+        self._site = web.TCPSite(self._runner, "0.0.0.0", self._port)
+        await self._site.start()
+        logger.info(
+            f"Shared webhook server started on 0.0.0.0:{self._port} "
+            f"with {len(routes)} route(s)"
+        )
+
+    async def stop(self) -> None:
+        if self._site:
+            await self._site.stop()
+            self._site = None
+        if self._runner:
+            await self._runner.cleanup()
+            self._runner = None
+        logger.info("Shared webhook server stopped")
+
+
+class ChannelManager:
+    """Manages all chat channels and coordinates message routing.
+
+    Responsibilities:
+    - Register channels and inject bus reference
+    - Start / stop all channels
+    - Route outbound messages from the bus to the correct channel
+    """
+
+    def __init__(
+        self,
+        bus: MessageBus,
+        *,
+        health_port: int = 8080,
+        drain_timeout: float = 30.0,
+        shared_webhook_port: int = 0,
+    ):
+        self.bus = bus
+        self._channels: dict[str, Channel] = {}
+        self._tasks: list[asyncio.Task] = []
+        self._dispatch_task: asyncio.Task | None = None
+        self._start_times: dict[str, datetime] = {}
+        self._message_counts: dict[str, dict[str, int]] = {}
+        self._health: dict[str, ChannelHealth] = {}
+        self._is_running: bool = False
+        self._health_port = health_port
+        self._health_server: _HealthServer | None = None
+        self._drain_timeout = drain_timeout
+        self._health_providers: dict[str, Callable[[], dict]] = {}
+        self._account_manager = AccountManager()
+        # Pipelines (built during registration)
+        self._outbound_pipelines: dict[str, OutboundPipeline] = {}
+        # Shared webhook
+        self._shared_webhook_port = shared_webhook_port
+        self._shared_webhook_server: SharedWebhookServer | None = None
+
+    @classmethod
+    def from_config(cls, config, bus: MessageBus | None = None) -> ChannelManager:
+        """Create a ChannelManager from application config.
+
+        Parses ``config.channel_enabled`` (comma-separated channel types),
+        creates each Channel instance, and registers them.
+
+        Args:
+            config: Application config with channel settings.
+            bus: Optional MessageBus instance. A new one is created if not provided.
+
+        Returns:
+            A fully configured ChannelManager.
+        """
+        if bus is None:
+            bus = MessageBus()
+        shared_webhook_port = getattr(config, "shared_webhook_port", 0) or 0
+        manager = cls(bus, shared_webhook_port=shared_webhook_port)
+        types = [
+            t.strip() for t in (config.channel_enabled or "").split(",") if t.strip()
+        ]
+        if not types:
+            raise ValueError("No channels enabled")
+        _ensure_channels_registered(types)
+        for ct in types:
+            channel = create_channel(ct, config)
+            manager.register(channel, config=config)
+        return manager
+
+    # ── registration ──
+
+    def register(
+        self,
+        channel: Channel,
+        *,
+        config: Any = None,
+        **kwargs: Any,
+    ) -> Channel:
+        """Register a channel and inject the bus reference.
+
+        Since Channel IS-A ChannelPlugin, the channel is also registered
+        in the plugin registry.  If *config* is provided, inbound/outbound
+        pipelines are built for the channel.
+
+        Args:
+            channel: The channel instance (must have a unique ``name``).
+            config: Optional app config for building pipelines.
+            **kwargs: Extra kwargs applied to the channel
+                (e.g. ``send_thinking=True``, ``initial_debounce=3.0``).
+
+        Returns:
+            The channel instance.
+        """
+        name = channel.name
+        if name in self._channels:
+            raise ValueError(f"Channel '{name}' already registered")
+
+        channel.set_bus(self.bus)
+        for key, value in kwargs.items():
+            if hasattr(channel, key):
+                setattr(channel, key, value)
+        self._channels[name] = channel
+        self._health[name] = ChannelHealth()
+        if channel.config_adapter is not None:
+            self._account_manager.register_plugin(channel)
+        if config is not None:
+            self._outbound_pipelines[name] = build_outbound_pipeline(channel, config)
+        logger.info(f"Registered channel: {name} (slots: {channel.filled_slots()})")
+        return channel
+
+    # ── lifecycle ──
+
+    async def start_all(self) -> None:
+        """Start the outbound dispatcher and all registered channels."""
+        if not self._channels:
+            logger.warning("No channels registered")
+            return
+
+        self._is_running = True
+
+        await self.start_health()
+
+        # Start shared webhook server before individual channels
+        await self._setup_shared_webhook()
+
+        self._dispatch_task = asyncio.create_task(self._dispatch_outbound())
+
+        now = datetime.now()
+        for name, channel in self._channels.items():
+            logger.info(f"Starting channel: {name}")
+            self._start_times[name] = now
+            if name not in self._message_counts:
+                self._message_counts[name] = {"received": 0, "sent": 0}
+            task = asyncio.create_task(channel.run())
+            self._tasks.append(task)
+
+        await asyncio.gather(*self._tasks, return_exceptions=True)
+
+    async def stop_all(self) -> None:
+        """Stop all channels and the outbound dispatcher.
+
+        Before shutting down channels, attempts to drain the outbound
+        queue so that pending replies are delivered.
+        """
+        logger.info("Stopping all channels...")
+        self._is_running = False
+
+        # Drain outbound queue — try to send pending replies
+        drained = 0
+        deadline = time.monotonic() + self._drain_timeout
+        while time.monotonic() < deadline:
+            try:
+                msg = self.bus.outbound.get_nowait()
+            except asyncio.QueueEmpty:
+                break
+            channel = self._channels.get(msg.channel)
+            if not channel:
+                continue
+            delivery_failed = False
+            if msg.content:
+                try:
+                    text_ok = await asyncio.wait_for(
+                        channel.send(msg),
+                        timeout=max(1.0, deadline - time.monotonic()),
+                    )
+                    if not text_ok:
+                        delivery_failed = True
+                except Exception:
+                    delivery_failed = True
+            for media_path in msg.media:
+                try:
+                    media_ok = await asyncio.wait_for(
+                        channel.send_media(
+                            recipient=msg.chat_id,
+                            file_path=media_path,
+                            metadata=msg.metadata,
+                        ),
+                        timeout=max(1.0, deadline - time.monotonic()),
+                    )
+                    if not media_ok:
+                        delivery_failed = True
+                except Exception:
+                    delivery_failed = True
+            if not delivery_failed and (msg.content or msg.media):
+                drained += 1
+        dropped = self.bus.outbound.qsize()
+        if drained or dropped:
+            logger.info(f"Outbound drain: {drained} sent, {dropped} dropped")
+
+        if self._dispatch_task:
+            self._dispatch_task.cancel()
+            try:
+                await self._dispatch_task
+            except asyncio.CancelledError:
+                pass
+
+        for name, channel in self._channels.items():
+            try:
+                channel._running = False
+                await channel.stop()
+                logger.info(f"Stopped channel: {name}")
+            except Exception as e:
+                logger.error(f"Error stopping {name}: {e}")
+
+        for task in self._tasks:
+            task.cancel()
+        self._tasks.clear()
+
+        # Stop shared webhook server
+        if self._shared_webhook_server is not None:
+            await self._shared_webhook_server.stop()
+            self._shared_webhook_server = None
+
+        await self.stop_health()
+
+    # ── health server ──
+
+    async def start_health(self) -> None:
+        """Start the HTTP health-check endpoint (if configured)."""
+        if self._health_port and self._health_server is None:
+            self._health_server = _HealthServer(self, self._health_port)
+            try:
+                await self._health_server.start()
+            except OSError as e:
+                logger.warning(
+                    "Health server failed to bind on port %s: %s — "
+                    "health endpoint disabled, channel will still start normally.",
+                    self._health_port,
+                    e,
+                )
+                self._health_server = None
+
+    async def stop_health(self) -> None:
+        """Stop the HTTP health-check endpoint."""
+        if self._health_server is not None:
+            await self._health_server.stop()
+            self._health_server = None
+
+    # ── shared webhook ──
+
+    async def _setup_shared_webhook(self) -> None:
+        """Collect routes from HTTP channels and start a shared server.
+
+        Only active when ``shared_webhook_port > 0``.  For each channel
+        that exposes ``_webhook_routes()``, the routes are gathered and
+        a sentinel attribute (``_shared_webhook_server``) is set so the
+        channel's own ``start()`` skips creating its own aiohttp server.
+        """
+        if not self._shared_webhook_port:
+            return
+
+        all_routes: list[tuple[str, str, Any]] = []
+        for name, channel in self._channels.items():
+            routes_fn = getattr(channel, "_webhook_routes", None)
+            if routes_fn is None:
+                continue
+            routes = routes_fn()
+            if not routes:
+                continue
+            # Set sentinel so the channel skips its own server
+            channel._shared_webhook_server = True  # type: ignore[attr-defined]
+            all_routes.extend(routes)
+            logger.debug(
+                f"Shared webhook: collected {len(routes)} route(s) from '{name}'"
+            )
+
+        if not all_routes:
+            logger.info("Shared webhook: no HTTP channels found, skipping")
+            return
+
+        self._shared_webhook_server = SharedWebhookServer(
+            self._shared_webhook_port,
+        )
+        await self._shared_webhook_server.start(all_routes)
+
+    def register_health_provider(
+        self,
+        name: str,
+        provider: Callable[[], dict],
+    ) -> None:
+        """Register a callable that returns extra data for ``/healthz``."""
+        self._health_providers[name] = provider
+
+    # ── outbound routing ──
+
+    async def _dispatch_outbound(self) -> None:
+        """Route outbound messages from the bus to the correct channel."""
+        logger.info("Outbound dispatcher started")
+        while True:
+            try:
+                msg: OutboundMessage = await asyncio.wait_for(
+                    self.bus.consume_outbound(),
+                    timeout=1.0,
+                )
+            except TimeoutError:
+                continue
+            except asyncio.CancelledError:
+                break
+
+            channel = self._channels.get(msg.channel)
+
+            if not channel:
+                logger.warning(f"Unknown channel: {msg.channel}")
+                continue
+
+            try:
+                # Run outbound pipeline if available (formatting, etc.)
+                if msg.channel in self._outbound_pipelines:
+                    processed = await self._outbound_pipelines[msg.channel].process(msg)
+                    if processed is None:
+                        continue  # dropped by pipeline
+                    msg = processed
+
+                delivery_failed = False
+                if msg.content:
+                    text_ok = await channel.send(msg)
+                    if not text_ok:
+                        logger.error(
+                            f"Error sending to {msg.channel}: send() returned False"
+                        )
+                        delivery_failed = True
+
+                for media_path in msg.media:
+                    try:
+                        media_ok = await channel.send_media(
+                            recipient=msg.chat_id,
+                            file_path=media_path,
+                            metadata=msg.metadata,
+                        )
+                        if not media_ok:
+                            logger.error(
+                                f"Error sending media to {msg.channel}: send_media() "
+                                f"returned False for {media_path}"
+                            )
+                            delivery_failed = True
+                    except Exception as e:
+                        logger.error(f"Error sending media to {msg.channel}: {e}")
+                        delivery_failed = True
+
+                if delivery_failed:
+                    raise RuntimeError("one or more outbound deliveries failed")
+
+                # Success
+                health = self._health.get(msg.channel)
+                if health is not None:
+                    health.consecutive_failures = 0
+                    health.total_successes += 1
+            except Exception as e:
+                logger.error(f"Error sending to {msg.channel}: {e}")
+                health = self._health.get(msg.channel)
+                if health is not None:
+                    health.consecutive_failures += 1
+                    health.total_failures += 1
+                    health.last_failure_time = time.monotonic()
+                    health.last_failure_error = str(e)
+
+    # ── per-account lifecycle ──
+
+    async def start_account(
+        self,
+        channel_id: str,
+        account_id: str,
+        config: Any = None,
+    ) -> None:
+        """Start a specific account on a registered plugin."""
+        await self._account_manager.start_account(channel_id, account_id, config)
+
+    async def stop_account(
+        self,
+        channel_id: str,
+        account_id: str,
+    ) -> None:
+        """Stop a specific account on a registered plugin."""
+        await self._account_manager.stop_account(channel_id, account_id)
+
+    def list_accounts(
+        self,
+        channel_id: str | None = None,
+    ) -> list[AccountState]:
+        """List account states, optionally filtered by channel."""
+        return self._account_manager.list_accounts(channel_id)
+
+    @property
+    def account_manager(self) -> AccountManager:
+        """Access the underlying AccountManager."""
+        return self._account_manager
+
+    # ── queries ──
+
+    def get_channel(self, name: str) -> Channel | None:
+        """Get a channel by name."""
+        return self._channels.get(name)
+
+    def get_server(self, name: str) -> Channel | None:
+        """Backward compat: returns the Channel (was ChannelServer)."""
+        return self._channels.get(name)
+
+    def get_status(self) -> dict[str, Any]:
+        """Get status of all registered channels."""
+        return {
+            name: {
+                "registered": True,
+                "running": channel._running,
+                "slots": channel.filled_slots(),
+            }
+            for name, channel in self._channels.items()
+        }
+
+    @property
+    def is_running(self) -> bool:
+        """Whether the manager is currently running."""
+        return self._is_running
+
+    @property
+    def enabled_channels(self) -> list[str]:
+        """List of registered channel names."""
+        return list(self._channels.keys())
+
+    def running_channels(self) -> list[str]:
+        """Return names of currently running channels."""
+        return [name for name, ch in self._channels.items() if ch._running]
+
+    def get_stats(self) -> dict:
+        """Return summary stats for all channels."""
+        return {
+            "channels": self.enabled_channels,
+            "running": self.running_channels(),
+            "message_counts": dict(self._message_counts),
+        }
+
+    async def add_channel(self, channel_type: str, config) -> Channel:
+        """Dynamically add and start a channel at runtime."""
+        _ensure_channels_registered([channel_type])
+        channel = create_channel(channel_type, config)
+        self.register(channel)
+        self._start_times[channel_type] = datetime.now()
+        if channel_type not in self._message_counts:
+            self._message_counts[channel_type] = {"received": 0, "sent": 0}
+        task = asyncio.create_task(channel.run())
+        self._tasks.append(task)
+        return channel
+
+    async def remove_channel(self, channel_type: str) -> None:
+        """Stop and remove a channel at runtime."""
+        channel = self._channels.pop(channel_type, None)
+        if channel:
+            channel._running = False
+            await channel.stop()
+            logger.info(f"Removed channel: {channel_type}")
+
+    def record_message(self, channel_name: str, direction: str) -> None:
+        """Record a message for tracking.
+
+        Args:
+            channel_name: Channel name (e.g. "telegram").
+            direction: "received" or "sent".
+        """
+        if channel_name not in self._message_counts:
+            self._message_counts[channel_name] = {"received": 0, "sent": 0}
+        if direction in self._message_counts[channel_name]:
+            self._message_counts[channel_name][direction] += 1
+
+    def get_detailed_status(self) -> dict[str, Any]:
+        """Get detailed status of all registered channels.
+
+        Returns:
+            Dict keyed by channel name with running, start_time, message
+            counts, health, and plugin information.
+        """
+        now = datetime.now()
+        result = {}
+        for name, channel in self._channels.items():
+            start = self._start_times.get(name)
+            counts = self._message_counts.get(name, {"received": 0, "sent": 0})
+            health = self._health.get(name, ChannelHealth())
+            result[name] = {
+                "registered": True,
+                "running": channel._running,
+                "start_time": start,
+                "uptime_seconds": (now - start).total_seconds() if start else 0,
+                "received": counts["received"],
+                "sent": counts["sent"],
+                "health": {
+                    "consecutive_failures": health.consecutive_failures,
+                    "last_failure_time": health.last_failure_time,
+                    "last_failure_error": health.last_failure_error,
+                    "total_failures": health.total_failures,
+                    "total_successes": health.total_successes,
+                },
+                "plugin_slots": channel.filled_slots(),
+                "has_outbound_pipeline": name in self._outbound_pipelines,
+            }
+        return result
diff --git a/EvoScientist/channels/config.py b/EvoScientist/channels/config.py
new file mode 100644
index 0000000..e51a357
--- /dev/null
+++ b/EvoScientist/channels/config.py
@@ -0,0 +1,128 @@
+"""Base configuration for all channel implementations.
+
+Provides common fields shared across channels, reducing duplication.
+Channel-specific configs inherit from BaseChannelConfig.
+
+Also provides ready-made ConfigAdapter implementations for the two most
+common account patterns:
+
+- ``SingleAccountConfigAdapter`` — one account per channel (default).
+- ``MultiAccountConfigAdapter`` — multiple accounts from a config dict.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class BaseChannelConfig:
+    """Common configuration fields for all channels.
+
+    Subclass this for channel-specific configs. Only add fields
+    here that are used by 3+ channels.
+    """
+
+    allowed_senders: set[str] | None = None
+    allowed_channels: set[str] | None = None
+    text_chunk_limit: int = 4096
+    proxy: str | None = None
+    include_attachments: bool = True
+    debug_trace: bool = False
+    accounts: dict | None = None  # multi-account config mapping
+
+
+class SingleAccountConfigAdapter:
+    """For channels that only ever have one account (most channels).
+
+    Returns a single ``"default"`` account whose config is the entire
+    channel config object.  This is the zero-change default: existing
+    single-account channels get multi-account support for free.
+    """
+
+    def list_account_ids(self, config: Any) -> list[str]:
+        return ["default"]
+
+    def resolve_account(
+        self,
+        config: Any,
+        account_id: str | None = None,
+    ) -> Any:
+        return config
+
+    def is_enabled(self, account: Any, config: Any) -> bool:
+        return True
+
+    def is_configured(self, account: Any, config: Any) -> bool:
+        """Check that the account has at least some non-None values."""
+        if account is None:
+            return False
+        if isinstance(account, dict):
+            return bool(account)
+        # dataclass / object — check that at least one field is truthy
+        if hasattr(account, "__dataclass_fields__"):
+            return any(getattr(account, f, None) for f in account.__dataclass_fields__)
+        return True
+
+
+class MultiAccountConfigAdapter:
+    """For channels that support multiple accounts.
+
+    Expects the channel config to contain a mapping of accounts under
+    a configurable key (default ``"accounts"``).  Each entry is keyed
+    by account id and holds account-specific settings.
+
+    Example config structure::
+
+        {
+            "accounts": {
+                "bot1": {"token": "...", "enabled": true},
+                "bot2": {"token": "...", "enabled": false},
+            }
+        }
+    """
+
+    def __init__(
+        self,
+        accounts_key: str = "accounts",
+        required_fields: list[str] | None = None,
+    ) -> None:
+        self._accounts_key = accounts_key
+        self._required_fields = required_fields or []
+
+    def _get_accounts_map(self, config: Any) -> dict[str, Any]:
+        """Extract the accounts mapping from config."""
+        if isinstance(config, dict):
+            return config.get(self._accounts_key, {})
+        return getattr(config, self._accounts_key, None) or {}
+
+    def list_account_ids(self, config: Any) -> list[str]:
+        return list(self._get_accounts_map(config).keys())
+
+    def resolve_account(
+        self,
+        config: Any,
+        account_id: str | None = None,
+    ) -> Any:
+        accounts = self._get_accounts_map(config)
+        if account_id is None:
+            # Return the first account, or empty dict
+            return next(iter(accounts.values()), {})
+        return accounts.get(account_id, {})
+
+    def is_enabled(self, account: Any, config: Any) -> bool:
+        if isinstance(account, dict):
+            return account.get("enabled", True)
+        return getattr(account, "enabled", True)
+
+    def is_configured(self, account: Any, config: Any) -> bool:
+        if not account:
+            return False
+        for f in self._required_fields:
+            if isinstance(account, dict):
+                if not account.get(f):
+                    return False
+            elif not getattr(account, f, None):
+                return False
+        return True
diff --git a/EvoScientist/channels/consumer.py b/EvoScientist/channels/consumer.py
new file mode 100644
index 0000000..1c8d6e7
--- /dev/null
+++ b/EvoScientist/channels/consumer.py
@@ -0,0 +1,888 @@
+"""Unified inbound message consumer.
+
+Provides :class:`InboundConsumer` — a single class that consumes
+inbound messages from the :class:`MessageBus`, runs them through
+the agent, and publishes outbound responses.  This replaces the
+inline consumer loops that were duplicated in ``cli.py`` and
+``standalone.py``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+import uuid
+from collections import OrderedDict
+from collections.abc import AsyncIterator, Callable
+from dataclasses import dataclass
+from typing import Any, TypeVar
+
+from .base import Channel
+from .bus import MessageBus
+from .bus.events import InboundMessage, OutboundMessage
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+_MAX_CHAT_LOCKS = 10_000
+_MAX_SESSIONS = 10_000
+_MAX_HITL_ROUNDS = 50
+_HITL_APPROVAL_TIMEOUT = 120.0  # seconds to wait for HITL approval reply
+_ASK_USER_TIMEOUT = (
+    300.0  # seconds to wait for ask_user reply (longer for thinking time)
+)
+
+
+@dataclass
+class ConsumerMetrics:
+    """Cumulative processing counters for the consumer."""
+
+    total_processed: int = 0
+    total_successes: int = 0
+    total_failures: int = 0
+    total_timeouts: int = 0
+
+
+async def _timeout_aiter(
+    agen: AsyncIterator[T],
+    idle_timeout: float,
+) -> AsyncIterator[T]:
+    """Wrap an async iterator with a per-yield idle timeout.
+
+    If ``__anext__()`` does not produce a value within *idle_timeout*
+    seconds, :class:`asyncio.TimeoutError` is raised.  Continuous
+    yielding resets the timer each time, so only a truly stalled
+    generator will trigger the timeout.
+    """
+    ait = agen.__aiter__()
+    try:
+        while True:
+            try:
+                item = await asyncio.wait_for(ait.__anext__(), timeout=idle_timeout)
+            except StopAsyncIteration:
+                return
+            yield item
+    finally:
+        if hasattr(ait, "aclose"):
+            await ait.aclose()
+
+
+def _format_todo_list(todos: list[dict]) -> str:
+    """Format todo items as a numbered list."""
+    lines = ["\U0001f4cb Todo List\n"]  # 📋
+    for i, item in enumerate(todos, 1):
+        content = item.get("content", "")
+        lines.append(f"{i}. {content}")
+    lines.append(f"\n\U0001f680 {len(todos)} tasks")  # 🚀
+    return "\n".join(lines)
+
+
+def _join_subagent_text(buffers: dict[str, tuple[str, list[str]]]) -> str:
+    """Join sub-agent text buffers into a single fallback string.
+
+    *buffers* maps ``instance_id`` → ``(display_name, chunks)``.
+
+    When only one instance produced text, return its content directly.
+    When multiple instances share the same display name, number them
+    (e.g. ``[research-agent #1]``, ``[research-agent #2]``).
+    """
+    if not buffers:
+        return ""
+    if len(buffers) == 1:
+        _display_name, chunks = next(iter(buffers.values()))
+        return "".join(chunks)
+
+    # Group by display_name to detect same-name instances
+    name_groups: dict[str, list[list[str]]] = {}
+    for _instance_id, (display_name, chunks) in buffers.items():
+        name_groups.setdefault(display_name, []).append(chunks)
+
+    sections: list[str] = []
+    for display_name, chunk_lists in name_groups.items():
+        if len(chunk_lists) == 1:
+            sections.append(f"[{display_name}]: {''.join(chunk_lists[0])}")
+        else:
+            for i, chs in enumerate(chunk_lists, 1):
+                sections.append(f"[{display_name} #{i}]: {''.join(chs)}")
+    return "\n\n".join(sections)
+
+
+def _should_auto_approve(action_requests: list[dict]) -> bool:
+    """Check if all action requests can be auto-approved via config.
+
+    Returns True if no manual approval is needed (config auto_approve,
+    non-execute tools, or shell_allow_list match).
+    """
+    if not action_requests:
+        return True
+
+    try:
+        from ..config.settings import load_config
+
+        cfg = load_config()
+    except Exception:
+        return False  # fail-closed
+
+    if cfg.auto_approve:
+        return True
+
+    shell_allow_list = (
+        [s.strip() for s in cfg.shell_allow_list.split(",") if s.strip()]
+        if cfg.shell_allow_list
+        else []
+    )
+
+    for req in action_requests:
+        name = (
+            req.get("name", "") if isinstance(req, dict) else getattr(req, "name", "")
+        )
+        if name != "execute":
+            continue
+        args = (
+            req.get("args", {}) if isinstance(req, dict) else getattr(req, "args", {})
+        )
+        command = args.get("command", "") if isinstance(args, dict) else ""
+        cmd = command.strip()
+        if not any(cmd.startswith(prefix) for prefix in shell_allow_list):
+            return False
+    return True
+
+
+def _format_approval_prompt(action_requests: list[dict]) -> str:
+    """Format an approval prompt as a text message for channel users."""
+    lines = ["\u26a0\ufe0f Approval Required\n"]
+    for i, req in enumerate(action_requests, 1):
+        name = (
+            req.get("name", "") if isinstance(req, dict) else getattr(req, "name", "")
+        )
+        args = (
+            req.get("args", {}) if isinstance(req, dict) else getattr(req, "args", {})
+        )
+        if isinstance(args, dict):
+            command = args.get("command", args.get("path", ""))
+        else:
+            command = ""
+        if command:
+            lines.append(f"  {i}. {name}: {command}")
+        else:
+            lines.append(f"  {i}. {name}")
+    lines.append("")
+    lines.append("Reply: 1=Approve, 2=Reject, 3=Approve all")
+    lines.append("(Auto-reject in 2 min if no reply)")
+    return "\n".join(lines)
+
+
+def _parse_approval_reply(text: str) -> str | None:
+    """Parse a channel user's reply as an approval decision.
+
+    Returns "approve", "reject", "auto", or None if not recognized.
+    """
+    t = text.strip().lower()
+    if t in ("1", "y", "yes", "approve", "ok"):
+        return "approve"
+    if t in ("2", "n", "no", "reject"):
+        return "reject"
+    if t in ("3", "a", "auto", "approve all"):
+        return "auto"
+    return None
+
+
+@dataclass
+class _PendingInterrupt:
+    """Stored state for a pending HITL interrupt awaiting channel user reply."""
+
+    thread_id: str
+    action_requests: list
+    event: asyncio.Event  # set when user replies
+    decision: str | None = None  # "approve", "reject", "auto"
+
+
+@dataclass
+class _PendingAskUserReply:
+    """Stored state for a pending ask_user question awaiting channel user reply."""
+
+    event: asyncio.Event  # set when user replies
+    reply: str | None = None  # raw reply text
+
+
+class InboundConsumer:
+    """Consume inbound messages from the bus, process via agent, publish outbound.
+
+    Parameters
+    ----------
+    bus:
+        The MessageBus to consume from / publish to.
+    manager:
+        The ChannelManager (used to look up channel instances).
+    agent:
+        The agent object (must support ``stream_agent_events``).
+    thread_id:
+        Default thread ID for agent conversations.
+    send_thinking:
+        Whether to forward thinking messages to the channel.
+    on_message_received:
+        Optional callback ``(msg: InboundMessage) -> None`` invoked when
+        a message is consumed (e.g. for CLI Rich display).
+    on_streaming_event:
+        Optional callback ``(event: dict) -> None`` invoked for each
+        streaming event from the agent.
+    on_message_sent:
+        Optional callback ``(msg: OutboundMessage) -> None`` invoked when
+        the outbound message is published.
+    inference_timeout:
+        Per-yield idle timeout in seconds for the agent stream.  If the
+        agent produces no event for this long, the inference is aborted.
+    max_concurrent:
+        Number of worker coroutines (= max parallel inferences).
+    max_pending:
+        Maximum depth of the internal work queue.  When full, the
+        consumer loop blocks (back-pressure).
+    drain_timeout:
+        Seconds to wait for in-flight workers to finish during ``stop()``.
+    """
+
+    def __init__(
+        self,
+        bus: MessageBus,
+        manager: Any,
+        agent: Any,
+        thread_id: str,
+        *,
+        send_thinking: bool = False,
+        on_message_received: Callable[[InboundMessage], None] | None = None,
+        on_streaming_event: Callable[[dict], None] | None = None,
+        on_message_sent: Callable[[OutboundMessage], None] | None = None,
+        inference_timeout: float = 300.0,
+        max_concurrent: int = 5,
+        max_pending: int = 50,
+        drain_timeout: float = 30.0,
+    ):
+        self.bus = bus
+        self.manager = manager
+        self.agent = agent
+        self.thread_id = thread_id
+        self.send_thinking = send_thinking
+        self._on_message_received = on_message_received
+        self._on_streaming_event = on_streaming_event
+        self._on_message_sent = on_message_sent
+        self._sessions: OrderedDict[str, str] = (
+            OrderedDict()
+        )  # sender_id -> thread_id (LRU)
+
+        # Per-chat locks: same chat is processed serially (bounded)
+        self._chat_locks: dict[str, asyncio.Lock] = {}
+
+        # Inference timeout
+        self._inference_timeout = inference_timeout
+
+        # Worker pool
+        self._max_concurrent = max_concurrent
+        self._work_queue: asyncio.Queue[InboundMessage | None] = asyncio.Queue(
+            maxsize=max_pending,
+        )
+        self._workers: list[asyncio.Task] = []
+        self._stopping = False
+        self._drain_timeout = drain_timeout
+
+        # Metrics
+        self._metrics = ConsumerMetrics()
+
+        # HITL: pending interrupts per session_key, and auto-approve sessions
+        self._pending_interrupts: dict[str, _PendingInterrupt] = {}
+        self._auto_approve_sessions: set[str] = set()
+
+        # ask_user: pending reply per session_key
+        self._pending_ask_user_replies: dict[str, _PendingAskUserReply] = {}
+
+    def _get_thread_id(self, sender_id: str) -> str:
+        """Get or create a thread ID for the given sender.
+
+        Uses LRU ordering: recently accessed senders are moved to the
+        end, so eviction always removes the least-recently-active sender.
+        """
+        if sender_id in self._sessions:
+            self._sessions.move_to_end(sender_id)
+            return self._sessions[sender_id]
+
+        if len(self._sessions) >= _MAX_SESSIONS:
+            # Evict the least-recently-used entry
+            self._sessions.popitem(last=False)
+        if self.thread_id:
+            self._sessions[sender_id] = f"{self.thread_id}:{sender_id}"
+        else:
+            self._sessions[sender_id] = str(uuid.uuid4())
+        return self._sessions[sender_id]
+
+    def _get_channel(self, channel_name: str) -> Channel | None:
+        """Look up the channel by name from the manager."""
+        return self.manager.get_channel(channel_name)
+
+    # ── lifecycle ──
+
+    async def run(self) -> None:
+        """Main consumer loop — runs until ``stop()`` or cancellation.
+
+        Spawns *max_concurrent* worker coroutines that pull from an
+        internal bounded queue.  The loop reads from the bus and feeds
+        the queue; when the queue is full the loop blocks (back-pressure).
+        """
+        self._stopping = False
+        self._workers = [
+            asyncio.create_task(self._worker(i)) for i in range(self._max_concurrent)
+        ]
+        try:
+            while not self._stopping:
+                try:
+                    msg = await asyncio.wait_for(
+                        self.bus.consume_inbound(),
+                        timeout=1.0,
+                    )
+                except TimeoutError:
+                    continue
+                except asyncio.CancelledError:
+                    break
+                if self._stopping:
+                    break
+                await self._work_queue.put(msg)  # blocks when full (back-pressure)
+        finally:
+            if not self._stopping:
+                await self.stop()
+
+    async def stop(self) -> None:
+        """Gracefully drain in-flight work and shut down workers."""
+        self._stopping = True
+        logger.info("Consumer stopping: draining in-flight messages...")
+        pending_count = self._work_queue.qsize()
+
+        # Send a None sentinel per worker so each exits its loop
+        for _ in self._workers:
+            try:
+                self._work_queue.put_nowait(None)
+            except asyncio.QueueFull:
+                pass
+
+        # Wait for workers to finish, then force-cancel stragglers
+        if self._workers:
+            done, still_running = await asyncio.wait(
+                self._workers,
+                timeout=self._drain_timeout,
+            )
+            for task in still_running:
+                task.cancel()
+                try:
+                    await task
+                except (asyncio.CancelledError, Exception):
+                    pass
+            logger.info(
+                f"Consumer drain: {len(done)} finished, "
+                f"{len(still_running)} force-cancelled, "
+                f"{pending_count} were pending"
+            )
+        self._workers.clear()
+
+    # ── workers ──
+
+    async def _worker(self, worker_id: int) -> None:
+        """Pull messages from the work queue and process them."""
+        while True:
+            msg = await self._work_queue.get()
+            if msg is None:
+                break  # shutdown sentinel
+            try:
+                await self._handle_message(msg)
+            except Exception:
+                logger.exception(f"Worker {worker_id} unhandled error")
+            finally:
+                self._work_queue.task_done()
+
+    async def _handle_message(self, msg: InboundMessage) -> None:
+        """Process a single inbound message."""
+        if self._on_message_received:
+            try:
+                self._on_message_received(msg)
+            except Exception:
+                pass
+
+        channel = self._get_channel(msg.channel)
+        thread_id = self._get_thread_id(msg.sender_id)
+        session_key = msg.session_key  # "channel:chat_id"
+
+        # Lazily create per-chat lock; evict stale locks when too many
+        if session_key not in self._chat_locks:
+            self._chat_locks[session_key] = asyncio.Lock()
+            if len(self._chat_locks) > _MAX_CHAT_LOCKS:
+                self._evict_chat_locks()
+
+        self._metrics.total_processed += 1
+
+        # ask_user: check if this message is a reply to a pending question.
+        # Must be checked BEFORE HITL approval — any text is a valid answer.
+        if session_key in self._pending_ask_user_replies:
+            pending_ask = self._pending_ask_user_replies[session_key]
+            pending_ask.reply = msg.content
+            pending_ask.event.set()
+            return  # consumed as ask_user answer
+
+        # HITL: check if this message is a reply to a pending approval
+        if session_key in self._pending_interrupts:
+            pending = self._pending_interrupts[session_key]
+            decision = _parse_approval_reply(msg.content)
+            if decision is not None:
+                pending.decision = decision
+                pending.event.set()
+                return  # don't process as a new agent message
+            # Unrecognized reply — treat as new message, cancel pending
+            pending.decision = "reject"
+            pending.event.set()
+            del self._pending_interrupts[session_key]
+
+        async with self._chat_locks[session_key]:
+            await self._stream_with_hitl(msg, channel, thread_id, session_key)
+
+    async def _stream_with_hitl(
+        self,
+        msg: InboundMessage,
+        channel: Channel | None,
+        thread_id: str,
+        session_key: str,
+    ) -> None:
+        """Stream agent events with HITL interrupt handling."""
+        from ..stream.events import stream_agent_events
+
+        stream_input: Any = msg.content
+        _t0 = time.monotonic()
+
+        try:
+            if channel:
+                await channel.start_typing(msg.chat_id)
+
+            for _hitl_round in range(_MAX_HITL_ROUNDS):
+                final_content = ""
+                thinking_buffer: list[str] = []
+                todo_sent = False
+                subagent_text_buffers: dict[str, tuple[str, list[str]]] = {}
+                thinking_sent = False
+                interrupt_data: dict | None = None
+
+                async for event in _timeout_aiter(
+                    stream_agent_events(
+                        self.agent,
+                        stream_input,
+                        thread_id,
+                        media=msg.media or None
+                        if isinstance(stream_input, str)
+                        else None,
+                    ),
+                    self._inference_timeout,
+                ):
+                    event_type = event.get("type")
+
+                    if self._on_streaming_event:
+                        try:
+                            self._on_streaming_event(event)
+                        except Exception:
+                            pass
+
+                    if event_type == "thinking":
+                        thinking_text = event.get("content", "")
+                        if thinking_text:
+                            thinking_buffer.append(thinking_text)
+
+                    elif event_type == "tool_call":
+                        if event.get("name") == "write_todos" and not todo_sent:
+                            todos = event.get("args", {}).get("todos", [])
+                            if todos and channel:
+                                if thinking_buffer and not thinking_sent:
+                                    full_thinking = "".join(thinking_buffer)
+                                    if full_thinking:
+                                        await channel.send_thinking_message(
+                                            msg.sender_id,
+                                            full_thinking,
+                                            msg.metadata,
+                                        )
+                                        thinking_sent = True
+                                    thinking_buffer.clear()
+                                await channel.send_todo_message(
+                                    msg.sender_id,
+                                    _format_todo_list(todos),
+                                    msg.metadata,
+                                )
+                                todo_sent = True
+
+                    elif event_type == "text":
+                        final_content += event.get("content", "")
+
+                    elif event_type == "progress":
+                        # Internal agent planning — ignore for channel delivery.
+                        # This text should NOT appear in the user-facing response.
+                        pass
+
+                    elif event_type == "subagent_text":
+                        sa_name = event.get("subagent", "unknown")
+                        instance_id = event.get("instance_id") or sa_name
+                        if instance_id not in subagent_text_buffers:
+                            subagent_text_buffers[instance_id] = (sa_name, [])
+                        subagent_text_buffers[instance_id][1].append(
+                            event.get("content", "")
+                        )
+
+                    elif event_type == "done":
+                        final_content = event.get("content", "") or final_content
+
+                    elif event_type == "interrupt":
+                        interrupt_data = event
+                        break  # exit async for to handle interrupt
+
+                    elif event_type == "ask_user":
+                        interrupt_data = event
+                        break  # exit async for to handle ask_user
+
+                # Flush thinking
+                if thinking_buffer and not thinking_sent and channel:
+                    full_thinking = "".join(thinking_buffer)
+                    if full_thinking:
+                        await channel.send_thinking_message(
+                            msg.sender_id,
+                            full_thinking,
+                            msg.metadata,
+                        )
+
+                # No interrupt — normal completion
+                if interrupt_data is None:
+                    outbound = OutboundMessage(
+                        channel=msg.channel,
+                        chat_id=msg.chat_id,
+                        content=final_content
+                        or _join_subagent_text(subagent_text_buffers)
+                        or "No response",
+                        reply_to=msg.message_id or None,
+                        metadata=msg.metadata,
+                    )
+                    await self.bus.publish_outbound(outbound)
+                    self._metrics.total_successes += 1
+                    elapsed = time.monotonic() - _t0
+                    logger.info(
+                        "stream completed: %d chars, %.2fs, session=%s",
+                        len(outbound.content),
+                        elapsed,
+                        session_key,
+                    )
+                    if self._on_message_sent:
+                        try:
+                            self._on_message_sent(outbound)
+                        except Exception:
+                            pass
+                    return  # done
+
+                # ask_user: send questions to channel user, collect answers
+                if interrupt_data.get("type") == "ask_user":
+                    result = await self._resolve_ask_user(
+                        msg,
+                        interrupt_data,
+                        session_key,
+                    )
+                    from langgraph.types import Command  # type: ignore[import-untyped]
+
+                    stream_input = Command(resume=result)
+                    continue
+
+                # HITL: resolve the interrupt
+                action_reqs = interrupt_data.get("action_requests", [])
+                n = len(action_reqs) or 1
+
+                # Session auto-approve (user previously chose "Approve all")
+                if session_key in self._auto_approve_sessions:
+                    from langgraph.types import Command  # type: ignore[import-untyped]
+
+                    stream_input = Command(
+                        resume={"decisions": [{"type": "approve"} for _ in range(n)]}
+                    )
+                    continue
+
+                # Config auto-approve (auto_approve, non-execute, allow_list)
+                if _should_auto_approve(action_reqs):
+                    from langgraph.types import Command  # type: ignore[import-untyped]
+
+                    stream_input = Command(
+                        resume={"decisions": [{"type": "approve"} for _ in range(n)]}
+                    )
+                    continue
+
+                # Needs user approval — send prompt to channel
+                prompt_text = _format_approval_prompt(action_reqs)
+                await self.bus.publish_outbound(
+                    OutboundMessage(
+                        channel=msg.channel,
+                        chat_id=msg.chat_id,
+                        content=prompt_text,
+                        metadata=msg.metadata,
+                    )
+                )
+
+                # Wait for user reply
+                pending = _PendingInterrupt(
+                    thread_id=thread_id,
+                    action_requests=action_reqs,
+                    event=asyncio.Event(),
+                )
+                self._pending_interrupts[session_key] = pending
+
+                try:
+                    await asyncio.wait_for(
+                        pending.event.wait(),
+                        timeout=_HITL_APPROVAL_TIMEOUT,
+                    )
+                except TimeoutError:
+                    # Auto-approve on timeout
+                    pending.decision = "approve"
+                finally:
+                    self._pending_interrupts.pop(session_key, None)
+
+                decision = pending.decision or "approve"
+
+                if decision == "reject":
+                    await self.bus.publish_outbound(
+                        OutboundMessage(
+                            channel=msg.channel,
+                            chat_id=msg.chat_id,
+                            content="Tool execution rejected.",
+                            metadata=msg.metadata,
+                        )
+                    )
+                    return
+
+                if decision == "auto":
+                    self._auto_approve_sessions.add(session_key)
+
+                from langgraph.types import Command  # type: ignore[import-untyped]
+
+                stream_input = Command(
+                    resume={"decisions": [{"type": "approve"} for _ in range(n)]}
+                )
+                # continue to next HITL round
+
+        except TimeoutError:
+            self._metrics.total_timeouts += 1
+            elapsed = time.monotonic() - _t0
+            logger.error(
+                "Inference timeout (%ds idle) for %s in %s, elapsed=%.2fs, response=%d chars",
+                self._inference_timeout,
+                msg.sender_id,
+                session_key,
+                elapsed,
+                len(final_content),
+            )
+            await self.bus.publish_outbound(
+                OutboundMessage(
+                    channel=msg.channel,
+                    chat_id=msg.chat_id,
+                    content="Sorry, the response timed out. Please try again.",
+                    metadata=msg.metadata,
+                )
+            )
+
+        except Exception as e:
+            self._metrics.total_failures += 1
+            elapsed = time.monotonic() - _t0
+            logger.error(
+                "Agent error: %s | elapsed=%.2fs, response=%d chars, session=%s",
+                e,
+                elapsed,
+                len(final_content),
+                session_key,
+            )
+            await self.bus.publish_outbound(
+                OutboundMessage(
+                    channel=msg.channel,
+                    chat_id=msg.chat_id,
+                    content="Sorry, something went wrong. Please try again later.",
+                    metadata=msg.metadata,
+                )
+            )
+        finally:
+            if channel:
+                await channel.stop_typing(msg.chat_id)
+
+    # ── observability ──
+
+    @property
+    def pending_count(self) -> int:
+        """Number of messages waiting in the work queue."""
+        return self._work_queue.qsize()
+
+    @property
+    def active_workers(self) -> int:
+        """Number of worker tasks that are still alive."""
+        return sum(1 for w in self._workers if not w.done())
+
+    @property
+    def metrics(self) -> dict[str, int]:
+        """Cumulative processing counters."""
+        m = self._metrics
+        return {
+            "total_processed": m.total_processed,
+            "total_successes": m.total_successes,
+            "total_failures": m.total_failures,
+            "total_timeouts": m.total_timeouts,
+            "pending": self.pending_count,
+            "active_workers": self.active_workers,
+            "chat_locks": len(self._chat_locks),
+            "sessions": len(self._sessions),
+        }
+
+    # ── ask_user helpers ──
+
+    async def _wait_for_ask_user_reply(
+        self,
+        session_key: str,
+        timeout: float,
+    ) -> str | None:
+        """Register a pending ask_user slot and wait for the user to reply.
+
+        Returns the raw reply text, or ``None`` on timeout.
+        """
+        pending = _PendingAskUserReply(event=asyncio.Event())
+        self._pending_ask_user_replies[session_key] = pending
+        try:
+            await asyncio.wait_for(pending.event.wait(), timeout=timeout)
+        except TimeoutError:
+            pass
+        finally:
+            self._pending_ask_user_replies.pop(session_key, None)
+        return pending.reply
+
+    async def _resolve_ask_user(
+        self,
+        msg: InboundMessage,
+        event_data: dict,
+        session_key: str,
+    ) -> dict:
+        """Handle an ask_user interrupt: send questions to channel, collect answers.
+
+        Mirrors the logic of ``cli.channel.channel_ask_user_prompt`` but runs
+        fully async inside the consumer event loop.
+
+        Returns a dict suitable for ``Command(resume=...)``:
+        ``{"answers": [...], "status": "answered"}`` or
+        ``{"status": "cancelled"}``.
+        """
+        questions = event_data.get("questions", [])
+        if not questions:
+            return {"answers": [], "status": "answered"}
+
+        total = len(questions)
+        answers: list[str] = []
+
+        for i, q in enumerate(questions):
+            q_text = q.get("question", "")
+            q_type = q.get("type", "text")
+            required = q.get("required", True)
+
+            # -- Format question header --
+            if total == 1:
+                header = "\u2753 Quick check-in from EvoScientist\n"
+            else:
+                header = f"\u2753 Question {i + 1}/{total}\n"
+
+            lines: list[str] = [header, f"{i + 1}. {q_text}"]
+            if not required:
+                lines[-1] += " (optional)"
+
+            if q_type == "multiple_choice":
+                choices = q.get("choices", [])
+                for j, choice in enumerate(choices):
+                    label = choice.get("value", str(choice))
+                    letter = chr(ord("A") + j)
+                    lines.append(f"   {letter}. {label}")
+                other_letter = chr(ord("A") + len(choices))
+                lines.append(f"   {other_letter}. Other")
+                letters = "/".join(chr(ord("A") + k) for k in range(len(choices) + 1))
+                lines.append(f"\nReply with a letter ({letters}), or 'cancel'.")
+            else:
+                skip_hint = " Leave empty to skip." if not required else ""
+                lines.append(f"\nReply with your answer, or 'cancel'.{skip_hint}")
+
+            # -- Send question --
+            await self.bus.publish_outbound(
+                OutboundMessage(
+                    channel=msg.channel,
+                    chat_id=msg.chat_id,
+                    content="\n".join(lines),
+                    metadata=msg.metadata,
+                )
+            )
+
+            # -- Wait for user reply --
+            reply = await self._wait_for_ask_user_reply(
+                session_key,
+                _ASK_USER_TIMEOUT,
+            )
+
+            if not reply:
+                await self.bus.publish_outbound(
+                    OutboundMessage(
+                        channel=msg.channel,
+                        chat_id=msg.chat_id,
+                        content="\u23f0 Response timed out.",
+                        metadata=msg.metadata,
+                    )
+                )
+                return {"status": "cancelled"}
+
+            raw = reply.strip()
+            if raw.lower() == "cancel":
+                return {"status": "cancelled"}
+
+            # -- Parse answer --
+            if q_type == "multiple_choice":
+                choices = q.get("choices", [])
+                other_letter = chr(ord("A") + len(choices))
+                if len(raw) == 1 and raw.upper() == other_letter:
+                    # "Other" selected — ask for free-form input
+                    await self.bus.publish_outbound(
+                        OutboundMessage(
+                            channel=msg.channel,
+                            chat_id=msg.chat_id,
+                            content="Please type your answer:",
+                            metadata=msg.metadata,
+                        )
+                    )
+                    other_reply = await self._wait_for_ask_user_reply(
+                        session_key,
+                        _ASK_USER_TIMEOUT,
+                    )
+                    if not other_reply:
+                        await self.bus.publish_outbound(
+                            OutboundMessage(
+                                channel=msg.channel,
+                                chat_id=msg.chat_id,
+                                content="\u23f0 Response timed out.",
+                                metadata=msg.metadata,
+                            )
+                        )
+                        return {"status": "cancelled"}
+                    if other_reply.strip().lower() == "cancel":
+                        return {"status": "cancelled"}
+                    answers.append(other_reply.strip())
+                elif len(raw) == 1 and raw.upper().isalpha():
+                    idx = ord(raw.upper()) - ord("A")
+                    if 0 <= idx < len(choices):
+                        answers.append(choices[idx].get("value", raw))
+                    else:
+                        answers.append(raw)
+                else:
+                    answers.append(raw)
+            else:
+                answers.append(raw)
+
+        return {"answers": answers, "status": "answered"}
+
+    # ── internal ──
+
+    def _evict_chat_locks(self) -> None:
+        """Remove chat locks that are not currently held."""
+        stale = [k for k, lock in self._chat_locks.items() if not lock.locked()]
+        for k in stale[: max(1, len(stale) // 2)]:
+            del self._chat_locks[k]
diff --git a/EvoScientist/channels/debug.py b/EvoScientist/channels/debug.py
new file mode 100644
index 0000000..088209e
--- /dev/null
+++ b/EvoScientist/channels/debug.py
@@ -0,0 +1,174 @@
+"""Shared debug logging helpers for channel integrations.
+
+This module is intentionally channel-agnostic. Future per-channel PRs should
+reuse these helpers instead of reintroducing ad-hoc logging formats or
+standalone ``basicConfig`` calls.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+_REDACTED = "***"
+_SECRET_TOKENS = (
+    "token",
+    "secret",
+    "password",
+    "authorization",
+    "cookie",
+    "api_key",
+    "apikey",
+    "access_key",
+    "private_key",
+    "signature",
+)
+
+
+def _load_debug_trace_flag() -> bool:
+    """Load the trace feature switch from config as a fallback to env vars."""
+    try:
+        from ..config.settings import load_config
+
+        return bool(getattr(load_config(), "channel_debug_tracing", False))
+    except Exception:
+        return False
+
+
+def debug_trace_enabled(enabled: bool | None = None) -> bool:
+    """Resolve the channel debug tracing switch.
+
+    Explicit ``enabled`` takes precedence; otherwise the helper falls back to
+    ``EVOSCIENTIST_CHANNEL_DEBUG_TRACING``.
+    """
+
+    if enabled is not None:
+        return bool(enabled)
+    raw = os.environ.get("EVOSCIENTIST_CHANNEL_DEBUG_TRACING", "")
+    if raw.strip():
+        return raw.strip().lower() in {"1", "true", "yes", "on"}
+    return _load_debug_trace_flag()
+
+
+def _should_redact_key(key: str) -> bool:
+    lowered = key.lower()
+    return any(token in lowered for token in _SECRET_TOKENS)
+
+
+def _stringify(value: Any) -> str:
+    if value is None:
+        return "-"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, bytes):
+        return f""
+    if isinstance(value, str):
+        return value.replace("\n", "\\n")
+    if isinstance(value, Mapping):
+        return f""
+    if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray):
+        return f""
+    return str(value).replace("\n", "\\n")
+
+
+def _format_fields(fields: Mapping[str, Any]) -> str:
+    parts: list[str] = []
+    for key, value in fields.items():
+        if value is None:
+            continue
+        safe_value = _REDACTED if _should_redact_key(key) else _stringify(value)
+        parts.append(f"{key}={safe_value}")
+    return " ".join(parts)
+
+
+_warned_debug_level_mismatch = False
+
+
+def _warn_debug_level_mismatch(logger: logging.Logger) -> None:
+    """Emit a one-time warning when tracing is enabled but DEBUG logs are hidden."""
+    global _warned_debug_level_mismatch
+    if _warned_debug_level_mismatch:
+        return
+    _warned_debug_level_mismatch = True
+    logger.warning(
+        "channel debug tracing is enabled but logger level is above DEBUG; "
+        "set EVOSCIENTIST_LOG_LEVEL=DEBUG, configure log_level=debug, "
+        "or use 'serve --debug' to see trace events"
+    )
+
+
+def emit_debug_event(
+    logger: logging.Logger,
+    event: str,
+    *,
+    channel: str,
+    enabled: bool,
+    **fields: Any,
+) -> None:
+    """Emit a structured channel debug event.
+
+    Example output:
+    ``event=inbound_raw channel=telegram message_id=123 chat_id=-1001``
+    """
+
+    if not enabled:
+        return
+    if not logger.isEnabledFor(logging.DEBUG):
+        _warn_debug_level_mismatch(logger)
+        return
+    base_fields = {"event": event, "channel": channel}
+    base_fields.update(fields)
+    logger.debug(_format_fields(base_fields))
+
+
+def emit_debug_event_if(
+    logger: logging.Logger,
+    event: str,
+    enabled: bool,
+    **fields: Any,
+) -> None:
+    """Convenience wrapper for code that lacks a :class:`Channel` instance.
+
+    Unlike :func:`emit_debug_event`, the ``channel`` field is not required —
+    pass it via *fields* when available.  This is intended for middleware
+    classes, managers, and standalone helpers.
+    """
+
+    if not enabled:
+        return
+    if not logger.isEnabledFor(logging.DEBUG):
+        _warn_debug_level_mismatch(logger)
+        return
+    base_fields: dict[str, Any] = {"event": event}
+    base_fields.update(fields)
+    logger.debug(_format_fields(base_fields))
+
+
+class TraceMixin:
+    """Mixin providing unified structured trace helpers.
+
+    Classes using this mixin must set ``_debug_trace`` (bool) and
+    ``_trace_logger`` attribute returning a :class:`logging.Logger`.
+
+    The trace name defaults to ``self.name`` if present, otherwise
+    ``"unknown"``.  Override ``_trace_name`` to customise.
+    """
+
+    _debug_trace: bool
+    _trace_logger: logging.Logger
+
+    @property
+    def _trace_name(self) -> str:
+        return getattr(self, "name", "unknown")
+
+    def _trace_event(self, event: str, **fields: Any) -> None:
+        """Emit a structured debug event when tracing is enabled."""
+        emit_debug_event(
+            self._trace_logger,
+            event,
+            channel=self._trace_name,
+            enabled=self._debug_trace,
+            **fields,
+        )
diff --git a/EvoScientist/channels/dingtalk/__init__.py b/EvoScientist/channels/dingtalk/__init__.py
new file mode 100644
index 0000000..6a9f0a6
--- /dev/null
+++ b/EvoScientist/channels/dingtalk/__init__.py
@@ -0,0 +1,31 @@
+"""DingTalk (钉钉) channel for EvoScientist.
+
+Uses Stream Mode (WebSocket) for receiving messages — no public IP needed.
+Sends replies via HTTP API.
+
+Usage in config:
+    channel_enabled = "dingtalk"
+    dingtalk_client_id = "your_app_key"
+    dingtalk_client_secret = "your_app_secret"
+"""
+
+from ..channel_manager import _parse_csv, register_channel
+from .channel import DingTalkChannel, DingTalkConfig
+
+__all__ = ["DingTalkChannel", "DingTalkConfig"]
+
+
+def create_from_config(config) -> DingTalkChannel:
+    allowed = _parse_csv(config.dingtalk_allowed_senders)
+    proxy = config.dingtalk_proxy or None
+    return DingTalkChannel(
+        DingTalkConfig(
+            client_id=config.dingtalk_client_id,
+            client_secret=config.dingtalk_client_secret,
+            allowed_senders=allowed,
+            proxy=proxy,
+        )
+    )
+
+
+register_channel("dingtalk", create_from_config)
diff --git a/EvoScientist/channels/dingtalk/channel.py b/EvoScientist/channels/dingtalk/channel.py
new file mode 100644
index 0000000..e875741
--- /dev/null
+++ b/EvoScientist/channels/dingtalk/channel.py
@@ -0,0 +1,437 @@
+"""DingTalk channel — refactored with WebSocketMixin + TokenMixin."""
+
+import asyncio
+import json
+import logging
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import ClassVar
+from urllib.parse import quote_plus
+
+from ..base import Channel, ChannelError, RawIncoming
+from ..capabilities import DINGTALK as DINGTALK_CAPS
+from ..config import BaseChannelConfig
+from ..mixins import TokenMixin, WebSocketMixin
+
+logger = logging.getLogger(__name__)
+
+GATEWAY_URL = "https://api.dingtalk.com/v1.0/gateway/connections/open"
+TOKEN_URL = "https://api.dingtalk.com/v1.0/oauth2/accessToken"
+SEND_URL = "https://api.dingtalk.com/v1.0/robot/oToMessages/batchSend"
+MEDIA_SEND_URL = "https://api.dingtalk.com/v1.0/robot/oToMessages/batchSend"
+MEDIA_UPLOAD_URL = "https://oapi.dingtalk.com/media/upload"
+FILE_DOWNLOAD_URL = "https://api.dingtalk.com/v1.0/robot/messageFiles/download"
+
+
+@dataclass
+class DingTalkConfig(BaseChannelConfig):
+    client_id: str = ""
+    client_secret: str = ""
+    text_chunk_limit: int = 4096
+
+
+class DingTalkChannel(Channel, WebSocketMixin, TokenMixin):
+    capabilities = DINGTALK_CAPS
+    name = "dingtalk"
+    _ready_attrs = ("_http_client", "_access_token")
+    _non_retryable_patterns = ("invalidauthentication", "forbidden", "40014")
+    _mention_pattern = r"@\S+\s*"
+    _mention_strip_count = 1
+
+    def __init__(self, config: DingTalkConfig):
+        super().__init__(config)
+
+    async def start(self) -> None:
+        import httpx
+
+        if not self.config.client_id or not self.config.client_secret:
+            raise ChannelError("DingTalk client_id and client_secret are required")
+        self._http_client = httpx.AsyncClient(timeout=15, proxy=self.config.proxy)
+        await self._refresh_token()
+        self._running = True
+        logger.info("DingTalk channel starting (Stream Mode)...")
+        self._ws_task = asyncio.create_task(self._ws_loop())
+
+    # ── TokenMixin ────────────────────────────────────────────────
+
+    async def _fetch_token(self) -> tuple[str, int]:
+        data = await self._api_post(
+            TOKEN_URL,
+            {
+                "appKey": self.config.client_id,
+                "appSecret": self.config.client_secret,
+            },
+        )
+        token = data.get("accessToken")
+        if not token:
+            raise ChannelError(f"DingTalk auth error: {data}")
+        return token, int(data.get("expireIn", 7200))
+
+    async def _api_post(self, url, body, headers=None):
+        resp = await self._http_client.post(url, json=body, headers=headers)
+        return resp.json()
+
+    async def _resolve_download_code(self, download_code: str) -> str | None:
+        """Exchange a DingTalk downloadCode for a real download URL."""
+        try:
+            token = await self._ensure_token()
+            data = await self._api_post(
+                FILE_DOWNLOAD_URL,
+                {"downloadCode": download_code, "robotCode": self.config.client_id},
+                headers={"x-acs-dingtalk-access-token": token},
+            )
+            url = data.get("downloadUrl") or ""
+            if url:
+                return url
+            logger.warning(f"DingTalk downloadCode resolve failed: {data}")
+        except Exception as e:
+            logger.warning(f"DingTalk downloadCode resolve error: {e}")
+        return None
+
+    # ── WebSocketMixin ────────────────────────────────────────────
+
+    async def _get_ws_url(self) -> str:
+        resp = await self._http_client.post(
+            GATEWAY_URL,
+            json={
+                "clientId": self.config.client_id,
+                "clientSecret": self.config.client_secret,
+                "subscriptions": [
+                    {"type": "CALLBACK", "topic": "/v1.0/im/bot/messages/get"}
+                ],
+                "ua": "dingtalk-sdk-python/v0.24.3-union",
+            },
+        )
+        data = resp.json()
+        endpoint, ticket = data.get("endpoint"), data.get("ticket")
+        if not endpoint or not ticket:
+            raise ChannelError(f"DingTalk gateway failed: {data}")
+        return f"{endpoint}?ticket={quote_plus(ticket)}"
+
+    async def _on_ws_message(self, data) -> None:
+        if not isinstance(data, dict):
+            return
+        headers = data.get("headers", {})
+        msg_id = headers.get("messageId", "")
+
+        # System ping
+        if data.get("type") == "SYSTEM" and headers.get("topic") == "ping":
+            await self._ws_send_json(
+                {
+                    "code": 200,
+                    "headers": headers,
+                    "message": "OK",
+                    "data": data.get("data", ""),
+                }
+            )
+            return
+
+        # ACK
+        await self._ws_send_json(
+            {
+                "code": 200,
+                "headers": {"contentType": "application/json", "messageId": msg_id},
+                "message": "OK",
+                "data": "{}",
+            }
+        )
+
+        if data.get("type") != "CALLBACK":
+            return
+
+        payload = data.get("data", "{}")
+        payload = json.loads(payload) if isinstance(payload, str) else payload
+        text_obj = payload.get("text", {})
+        content = (
+            text_obj.get("content", "") if isinstance(text_obj, dict) else str(text_obj)
+        ).strip()
+        if not content:
+            raw_content = payload.get("content", "")
+            content = raw_content.strip() if isinstance(raw_content, str) else ""
+
+        # Download attachments if present
+        annotations: list[str] = []
+        media_paths: list[str] = []
+
+        # DingTalk file/image messages may put download info in
+        # payload["content"] (as a dict) instead of in a dedicated
+        # "fileContent"/"imageContent" key.
+        raw_content_obj = payload.get("content")
+        if isinstance(raw_content_obj, dict) and raw_content_obj not in [
+            payload.get(k)
+            for k in ("imageContent", "fileContent", "videoContent", "audioContent")
+        ]:
+            msg_type = payload.get("msgtype") or payload.get("msgType") or ""
+            media_label = msg_type or "file"
+            file_size = (
+                raw_content_obj.get("fileSize")
+                or raw_content_obj.get("downloadSize")
+                or 0
+            )
+            file_name = (
+                raw_content_obj.get("fileName")
+                or raw_content_obj.get("name")
+                or f"dingtalk_{msg_type}"
+            )
+            download_code = raw_content_obj.get("downloadCode") or ""
+            download_url = raw_content_obj.get("downloadUrl") or ""
+            # downloadCode is NOT a URL — resolve it via DingTalk API first
+            if download_code and not download_code.startswith("http"):
+                resolved = await self._resolve_download_code(download_code)
+                if resolved:
+                    download_url = resolved
+            elif download_code:
+                download_url = download_code
+            if download_url:
+                try:
+                    dl_token = await self._ensure_token()
+                    dl_headers = {"x-acs-dingtalk-access-token": dl_token}
+                except Exception:
+                    dl_headers = None
+                local, ann = await self._download_attachment(
+                    download_url,
+                    f"dingtalk_{file_name}",
+                    headers=dl_headers,
+                    file_size=int(file_size) if file_size else None,
+                )
+                if local:
+                    media_paths.append(local)
+                if ann:
+                    ann = ann.replace("[attachment:", f"[{media_label}:")
+                    annotations.append(ann)
+            elif file_name:
+                annotations.append(f"[{media_label}: {file_name}]")
+
+        for att_key in ("imageContent", "fileContent", "videoContent", "audioContent"):
+            att = payload.get(att_key)
+            if att and isinstance(att, dict):
+                file_size = att.get("fileSize") or att.get("downloadSize") or 0
+                file_name = att.get("fileName", att_key)
+                download_code = att.get("downloadCode") or ""
+                download_url = att.get("downloadUrl") or ""
+                # Resolve downloadCode via API if it's not a URL
+                if download_code and not download_code.startswith("http"):
+                    resolved = await self._resolve_download_code(download_code)
+                    if resolved:
+                        download_url = resolved
+                elif download_code:
+                    download_url = download_code
+                # DingTalk audioContent is voice messages
+                media_label = "voice" if att_key == "audioContent" else att_key
+                if download_url and (
+                    self.config.include_attachments
+                    if hasattr(self.config, "include_attachments")
+                    else True
+                ):
+                    # DingTalk download URLs require access token
+                    try:
+                        dl_token = await self._ensure_token()
+                        dl_headers = {"x-acs-dingtalk-access-token": dl_token}
+                    except Exception:
+                        dl_headers = None
+                    local, ann = await self._download_attachment(
+                        download_url,
+                        f"dingtalk_{file_name}",
+                        headers=dl_headers,
+                        file_size=int(file_size) if file_size else None,
+                    )
+                    if local:
+                        media_paths.append(local)
+                    if ann:
+                        ann = ann.replace("[attachment:", f"[{media_label}:")
+                        annotations.append(ann)
+                elif file_size:
+                    too_large = self._check_attachment_size(int(file_size), file_name)
+                    if too_large:
+                        annotations.append(too_large)
+                    else:
+                        annotations.append(f"[{media_label}: {file_name}]")
+
+        if not content and not media_paths and not annotations:
+            return
+
+        sender_id = payload.get("senderStaffId") or payload.get("senderId", "")
+        is_group = payload.get("conversationType") == "2"
+        # For send API (oToMessages/batchSend), userIds needs staffId, not conversationId
+        chat_id = sender_id
+        create_time = payload.get("createAt") or payload.get("createTime", "")
+
+        # Mention gating: DMs always pass; groups require @bot
+        was_mentioned = not is_group
+        if is_group:
+            # isInAtList is set by DingTalk when bot is @mentioned
+            if payload.get("isInAtList"):
+                was_mentioned = True
+            else:
+                # Fallback: check atUsers array
+                at_users = payload.get("atUsers") or []
+                for u in at_users:
+                    if u.get("dingtalkId") == self.config.client_id:
+                        was_mentioned = True
+                        break
+
+        try:
+            ts = (
+                datetime.fromtimestamp(int(create_time) / 1000)
+                if create_time
+                else datetime.now()
+            )
+        except (ValueError, TypeError, OSError):
+            ts = datetime.now()
+
+        await self._enqueue_raw(
+            RawIncoming(
+                sender_id=sender_id,
+                chat_id=chat_id,
+                text=content,
+                timestamp=ts,
+                message_id=msg_id,
+                is_group=is_group,
+                was_mentioned=was_mentioned,
+                media_files=media_paths,
+                content_annotations=annotations,
+                metadata={
+                    "chat_id": chat_id,
+                    "sender_nick": payload.get("senderNick", ""),
+                    "backend": "dingtalk",
+                },
+            )
+        )
+
+    # _send_typing_action: inherited no-op (DingTalk has no typing API)
+    # _format_chunk: inherited from base (UnifiedFormatter)
+
+    # ── Send ──────────────────────────────────────────────────────
+
+    async def _send_chunk(self, chat_id, formatted_text, raw_text, reply_to, metadata):
+        token = await self._ensure_token()
+        data = await self._api_post(
+            SEND_URL,
+            {
+                "robotCode": self.config.client_id,
+                "userIds": [chat_id],
+                "msgKey": "sampleMarkdown",
+                "msgParam": json.dumps({"text": raw_text, "title": "EvoScientist"}),
+            },
+            headers={"x-acs-dingtalk-access-token": token},
+        )
+        return data
+
+    # ── Media send ────────────────────────────────────────────────
+
+    _IMAGE_EXTS: ClassVar[set[str]] = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp"}
+
+    async def _send_media_impl(
+        self,
+        recipient: str,
+        file_path: str,
+        caption: str = "",
+        metadata: dict | None = None,
+    ) -> bool:
+        """Send a media file through DingTalk.
+
+        For images: uploads via /media/upload to get media_id, then sends
+        as sampleImageMsg.  Non-image files are sent as markdown links
+        (DingTalk robot API does not support arbitrary file uploads).
+        """
+        token = await self._ensure_token()
+        chat_id = self._resolve_media_chat_id(recipient, metadata)
+        headers = {"x-acs-dingtalk-access-token": token}
+        ext = Path(file_path).suffix.lower()
+
+        if ext in self._IMAGE_EXTS:
+            # Try uploading image to get media_id for native image message
+            media_id = await self._upload_dingtalk_media(token, file_path, "image")
+            if media_id:
+                await self._api_post(
+                    MEDIA_SEND_URL,
+                    {
+                        "robotCode": self.config.client_id,
+                        "userIds": [chat_id],
+                        "msgKey": "sampleImageMsg",
+                        "msgParam": json.dumps({"photoURL": media_id}),
+                    },
+                    headers=headers,
+                )
+            else:
+                # Fallback to markdown with file path
+                await self._api_post(
+                    MEDIA_SEND_URL,
+                    {
+                        "robotCode": self.config.client_id,
+                        "userIds": [chat_id],
+                        "msgKey": "sampleMarkdown",
+                        "msgParam": json.dumps(
+                            {
+                                "text": f"![image]({file_path})"
+                                + (f"\n{caption}" if caption else ""),
+                                "title": caption or "Image",
+                            }
+                        ),
+                    },
+                    headers=headers,
+                )
+        else:
+            # Non-image: send as markdown with filename
+            name = Path(file_path).name
+            text = f"[文件] {name}" + (f"\n{caption}" if caption else "")
+            await self._api_post(
+                MEDIA_SEND_URL,
+                {
+                    "robotCode": self.config.client_id,
+                    "userIds": [chat_id],
+                    "msgKey": "sampleMarkdown",
+                    "msgParam": json.dumps({"text": text, "title": name}),
+                },
+                headers=headers,
+            )
+
+        if caption and ext in self._IMAGE_EXTS:
+            # Send caption separately for image messages
+            await self._api_post(
+                MEDIA_SEND_URL,
+                {
+                    "robotCode": self.config.client_id,
+                    "userIds": [chat_id],
+                    "msgKey": "sampleMarkdown",
+                    "msgParam": json.dumps({"text": caption, "title": "Caption"}),
+                },
+                headers=headers,
+            )
+        return True
+
+    async def _upload_dingtalk_media(
+        self,
+        token: str,
+        file_path: str,
+        media_type: str = "image",
+    ) -> str | None:
+        """Upload a file to DingTalk media API and return the media_id."""
+        try:
+            url = f"{MEDIA_UPLOAD_URL}?access_token={token}&type={media_type}"
+            with open(file_path, "rb") as f:
+                resp = await self._http_client.post(
+                    url,
+                    files={"media": (Path(file_path).name, f)},
+                )
+            data = resp.json()
+            return data.get("media_id")
+        except Exception as e:
+            logger.warning(f"DingTalk media upload failed: {e}")
+            return None
+
+    async def _cleanup(self) -> None:
+        if hasattr(self, "_ws_task") and self._ws_task:
+            self._ws_task.cancel()
+            try:
+                await self._ws_task
+            except (asyncio.CancelledError, Exception):
+                pass
+            self._ws_task = None
+        await self._stop_ws()
+        if hasattr(self, "_http_client") and self._http_client:
+            await self._http_client.aclose()
+            self._http_client = None
+        self._access_token = None
+        logger.info("DingTalk channel stopped")
diff --git a/EvoScientist/channels/dingtalk/probe.py b/EvoScientist/channels/dingtalk/probe.py
new file mode 100644
index 0000000..eb0eeb2
--- /dev/null
+++ b/EvoScientist/channels/dingtalk/probe.py
@@ -0,0 +1,33 @@
+"""DingTalk credential validation."""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+async def validate_dingtalk(
+    client_id: str,
+    client_secret: str,
+    proxy: str | None = None,
+) -> tuple[bool, str]:
+    """Validate DingTalk credentials by fetching an access token."""
+    if not client_id or not client_secret:
+        return False, "client_id and client_secret are required"
+
+    try:
+        import httpx
+    except ImportError:
+        return False, "httpx not installed"
+
+    url = "https://api.dingtalk.com/v1.0/oauth2/accessToken"
+    body = {"appKey": client_id, "appSecret": client_secret}
+
+    try:
+        async with httpx.AsyncClient(proxy=proxy) as client:
+            resp = await client.post(url, json=body, timeout=10)
+        data = resp.json()
+        if data.get("accessToken"):
+            return True, "DingTalk credentials valid"
+        return False, f"Error: {data.get('message', data)}"
+    except Exception as e:
+        return False, f"Error: {e}"
diff --git a/EvoScientist/channels/dingtalk/serve.py b/EvoScientist/channels/dingtalk/serve.py
new file mode 100644
index 0000000..57d5464
--- /dev/null
+++ b/EvoScientist/channels/dingtalk/serve.py
@@ -0,0 +1,89 @@
+"""DingTalk channel server.
+
+Standalone script to run the DingTalk channel with CLI options.
+
+Usage:
+    python -m EvoScientist.channels.dingtalk.serve --client-id ID --client-secret SECRET [OPTIONS]
+
+Examples:
+    # Basic usage
+    python -m EvoScientist.channels.dingtalk.serve --client-id ID --client-secret SECRET
+
+    # With proxy and allowed senders
+    python -m EvoScientist.channels.dingtalk.serve --client-id ID --client-secret SECRET --proxy http://proxy:8080 --allow user123
+
+    # With agent and thinking
+    python -m EvoScientist.channels.dingtalk.serve --client-id ID --client-secret SECRET --agent --thinking
+"""
+
+import argparse
+import logging
+
+from ...logging_config import configure_logging_from_settings
+from ..bus import MessageBus
+from ..standalone import run_standalone
+from .channel import DingTalkChannel, DingTalkConfig
+
+logger = logging.getLogger(__name__)
+
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="DingTalk channel server",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--client-id",
+        required=True,
+        help="DingTalk app client ID",
+    )
+    parser.add_argument(
+        "--client-secret",
+        required=True,
+        help="DingTalk app client secret",
+    )
+    parser.add_argument(
+        "--allow",
+        action="append",
+        dest="allowed_senders",
+        help="Allowed sender (DingTalk user ID). Can be used multiple times.",
+    )
+    parser.add_argument(
+        "--proxy",
+        help="HTTP proxy URL",
+    )
+    parser.add_argument(
+        "--agent",
+        action="store_true",
+        help="Use EvoScientist agent as handler (default: echo)",
+    )
+    parser.add_argument(
+        "--thinking",
+        action="store_true",
+        help="Send thinking content as intermediate messages (requires --agent)",
+    )
+    return parser.parse_args()
+
+
+def main():
+    """Entry point."""
+    configure_logging_from_settings(default_level=logging.INFO)
+    args = parse_args()
+
+    config = DingTalkConfig(
+        client_id=args.client_id,
+        client_secret=args.client_secret,
+        allowed_senders=set(args.allowed_senders) if args.allowed_senders else None,
+        proxy=args.proxy,
+    )
+
+    send_thinking = args.thinking and args.agent
+    bus = MessageBus()
+    channel = DingTalkChannel(config)
+
+    run_standalone(channel, bus, use_agent=args.agent, send_thinking=send_thinking)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/EvoScientist/channels/discord/__init__.py b/EvoScientist/channels/discord/__init__.py
new file mode 100644
index 0000000..8d79e72
--- /dev/null
+++ b/EvoScientist/channels/discord/__init__.py
@@ -0,0 +1,21 @@
+from ..channel_manager import _parse_csv, register_channel
+from .channel import DiscordChannel, DiscordConfig
+
+__all__ = ["DiscordChannel", "DiscordConfig"]
+
+
+def create_from_config(config) -> DiscordChannel:
+    allowed = _parse_csv(config.discord_allowed_senders)
+    channels = _parse_csv(config.discord_allowed_channels)
+    proxy = config.discord_proxy or None
+    return DiscordChannel(
+        DiscordConfig(
+            bot_token=config.discord_bot_token,
+            allowed_senders=allowed,
+            allowed_channels=channels,
+            proxy=proxy,
+        )
+    )
+
+
+register_channel("discord", create_from_config)
diff --git a/EvoScientist/channels/discord/channel.py b/EvoScientist/channels/discord/channel.py
new file mode 100644
index 0000000..d86611d
--- /dev/null
+++ b/EvoScientist/channels/discord/channel.py
@@ -0,0 +1,274 @@
+"""Discord channel implementation using discord.py."""
+
+import asyncio
+import logging
+import os
+from dataclasses import dataclass
+from datetime import datetime
+
+from ..base import Channel, ChannelError, RawIncoming
+from ..capabilities import DISCORD as DISCORD_CAPS
+from ..config import BaseChannelConfig
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DiscordConfig(BaseChannelConfig):
+    bot_token: str = ""
+    text_chunk_limit: int = 2000
+
+
+class DiscordChannel(Channel):
+    """Discord channel using discord.py."""
+
+    name = "discord"
+
+    capabilities = DISCORD_CAPS
+    _typing_interval: float = 8.0
+    _ready_attrs = ("_client",)
+    _mention_pattern = r"<@!?{bot_id}>\s*"
+
+    def __init__(self, config: DiscordConfig):
+        super().__init__(config)
+        self._client = None
+        self._ready = asyncio.Event()
+        # Cache message objects for ACK reactions
+        self._message_cache: dict[str, object] = {}
+        self._MESSAGE_CACHE_MAX = 200
+        self._background_tasks: set[asyncio.Task] = set()
+
+    async def start(self) -> None:
+        try:
+            import discord
+        except ImportError:
+            raise ChannelError(
+                "discord.py not installed. "
+                "Install with: pip install evoscientist[discord]"
+            ) from None
+
+        if not self.config.bot_token:
+            raise ChannelError("Discord bot token is required")
+
+        proxy = (
+            self.config.proxy
+            or os.environ.get("https_proxy")
+            or os.environ.get("HTTPS_PROXY")
+            or os.environ.get("http_proxy")
+            or os.environ.get("HTTP_PROXY")
+            or None
+        )
+
+        logger.info(
+            "Discord connect: token=%s...%s proxy=%s",
+            self.config.bot_token[:8],
+            self.config.bot_token[-4:],
+            proxy or "(none)",
+        )
+
+        intents = discord.Intents.default()
+        intents.message_content = True
+        client_kwargs = {"intents": intents}
+        if proxy:
+            client_kwargs["proxy"] = proxy
+        self._client = discord.Client(**client_kwargs)
+
+        self._start_task_error: BaseException | None = None
+
+        @self._client.event
+        async def on_ready():
+            logger.info(f"Discord bot ready: {self._client.user}")
+            self._ready.set()
+
+        @self._client.event
+        async def on_message(message):
+            await self._on_message(message)
+
+        async def _guarded_start():
+            try:
+                logger.info("Discord gateway: starting client.start()...")
+                await self._client.start(self.config.bot_token)
+            except Exception as exc:
+                logger.error("Discord gateway error: %s: %s", type(exc).__name__, exc)
+                self._start_task_error = exc
+                self._ready.set()  # unblock the waiter so it doesn't hang
+
+        logger.info("Discord connect: launching gateway task")
+        _task = asyncio.create_task(_guarded_start())
+        self._background_tasks.add(_task)
+        _task.add_done_callback(self._background_tasks.discard)
+
+        try:
+            await asyncio.wait_for(self._ready.wait(), timeout=60)
+        except TimeoutError:
+            raise ChannelError(
+                "Discord bot failed to connect within 60s. "
+                "Check network/proxy connectivity to gateway.discord.gg"
+            ) from None
+
+        if self._start_task_error:
+            raise ChannelError(
+                f"Discord bot failed to connect: {self._start_task_error}"
+            )
+
+        self._running = True
+        logger.info("Discord channel started")
+
+    async def _cleanup(self) -> None:
+        if self._client:
+            await self._client.close()
+            logger.info("Discord channel stopped")
+
+    # ── Typing indicator ────────────────────────────────────────────
+
+    async def _send_typing_action(self, chat_id: str) -> None:
+        if not self._client:
+            return
+        ch = self._client.get_channel(int(chat_id))
+        if ch:
+            await ch.trigger_typing()
+
+    # ── ACK Reactions ───────────────────────────────────────────────
+
+    async def _send_ack_reaction(
+        self, chat_id: str, message_id: str, emoji: str = "👀"
+    ) -> None:
+        msg = self._message_cache.get(message_id)
+        if msg:
+            try:
+                await msg.add_reaction(emoji)
+            except Exception as e:
+                logger.debug(f"Discord ACK reaction failed: {e}")
+
+    async def _remove_ack_reaction(
+        self, chat_id: str, message_id: str, emoji: str = "👀"
+    ) -> None:
+        msg = self._message_cache.get(message_id)
+        if msg and self._client and self._client.user:
+            try:
+                await msg.remove_reaction(emoji, self._client.user)
+            except Exception as e:
+                logger.debug(f"Discord remove ACK reaction failed: {e}")
+
+    def _cache_message(self, message) -> None:
+        """Cache a discord message object for later reaction use."""
+        mid = str(message.id)
+        self._message_cache[mid] = message
+        # Evict oldest entries if cache is too large
+        if len(self._message_cache) > self._MESSAGE_CACHE_MAX:
+            oldest = list(self._message_cache.keys())[: self._MESSAGE_CACHE_MAX // 2]
+            for k in oldest:
+                self._message_cache.pop(k, None)
+
+    # ── Send ────────────────────────────────────────────────────────
+
+    async def _send_chunk(self, chat_id, formatted_text, raw_text, reply_to, metadata):
+        import discord
+
+        thread_id = (metadata or {}).get("thread_id", "")
+        target_id = int(thread_id) if thread_id else int(chat_id)
+        ch = self._client.get_channel(target_id)
+        if not ch:
+            raise RuntimeError(f"Discord channel {target_id} not found")
+        ref = None
+        if reply_to:
+            try:
+                ref = discord.MessageReference(
+                    message_id=int(reply_to),
+                    channel_id=target_id,
+                )
+            except (ValueError, TypeError):
+                pass
+
+        async def _send(text):
+            await ch.send(text, reference=ref)
+
+        await self._send_with_format_fallback(_send, formatted_text, raw_text)
+
+    async def _send_media_impl(
+        self,
+        recipient: str,
+        file_path: str,
+        caption: str = "",
+        metadata: dict | None = None,
+    ) -> bool:
+        import discord
+
+        channel_id = self._resolve_media_chat_id(recipient, metadata)
+        ch = self._client.get_channel(int(channel_id))
+        if not ch:
+            logger.error(f"Discord channel {channel_id} not found")
+            return False
+        file = discord.File(file_path)
+        await ch.send(content=caption or None, file=file)
+        return True
+
+    def _get_bot_identifier(self) -> str | None:
+        if self._client and self._client.user:
+            return str(self._client.user.id)
+        return None
+
+    # ── Inbound ─────────────────────────────────────────────────────
+
+    async def _on_message(self, message) -> None:
+        import discord
+
+        if message.author == self._client.user:
+            return
+
+        # Cache for ACK reactions
+        self._cache_message(message)
+
+        user_id = str(message.author.id)
+        channel_id = str(message.channel.id)
+
+        is_dm = isinstance(message.channel, discord.DMChannel)
+        was_mentioned = is_dm or (self._client.user in message.mentions)
+
+        text = message.content or ""
+        annotations: list[str] = []
+        media_paths: list[str] = []
+
+        if self.config.include_attachments and message.attachments:
+            for attachment in message.attachments:
+                too_large = self._check_attachment_size(
+                    attachment.size or 0,
+                    attachment.filename,
+                )
+                if too_large:
+                    annotations.append(too_large)
+                    continue
+                try:
+                    safe_name = attachment.filename.replace("/", "_")
+                    file_path = self._media_path(f"{attachment.id}_{safe_name}")
+                    await attachment.save(file_path)
+                    media_paths.append(str(file_path))
+                    annotations.append(f"[attachment: {file_path}]")
+                except Exception as e:
+                    logger.warning(f"Failed to download Discord attachment: {e}")
+                    annotations.append(
+                        f"[attachment: {attachment.filename} - download failed]"
+                    )
+
+        # Detect thread context
+        thread_id = ""
+        parent_channel_id = channel_id
+        if hasattr(message.channel, "parent") and message.channel.parent:
+            # Message is inside a Thread — store thread info
+            thread_id = channel_id  # the thread IS the channel
+            parent_channel_id = str(message.channel.parent.id)
+
+        await self._enqueue_raw(
+            RawIncoming(
+                sender_id=user_id,
+                chat_id=parent_channel_id,
+                text=text,
+                media_files=media_paths,
+                content_annotations=annotations,
+                timestamp=message.created_at or datetime.now(),
+                message_id=str(message.id),
+                metadata={"chat_id": parent_channel_id, "thread_id": thread_id},
+                is_group=not is_dm,
+                was_mentioned=was_mentioned,
+            )
+        )
diff --git a/EvoScientist/channels/discord/probe.py b/EvoScientist/channels/discord/probe.py
new file mode 100644
index 0000000..eb56cd3
--- /dev/null
+++ b/EvoScientist/channels/discord/probe.py
@@ -0,0 +1,35 @@
+"""Discord bot token validation."""
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+async def validate_discord_token(
+    token: str, proxy: str | None = None
+) -> tuple[bool, str]:
+    """Validate a Discord bot token via the REST API.
+
+    Returns:
+        Tuple of (is_valid, message).
+    """
+    if not token:
+        return False, "No token provided"
+
+    try:
+        import httpx
+    except ImportError:
+        return False, "httpx not installed"
+
+    url = "https://discord.com/api/v10/users/@me"
+    headers = {"Authorization": f"Bot {token}"}
+    try:
+        async with httpx.AsyncClient(proxy=proxy) as client:
+            resp = await client.get(url, headers=headers, timeout=10)
+        if resp.status_code == 200:
+            data = resp.json()
+            username = data.get("username", "unknown")
+            return True, f"Bot: {username}"
+        return False, "Invalid token"
+    except Exception as e:
+        return False, f"Error: {e}"
diff --git a/EvoScientist/channels/discord/serve.py b/EvoScientist/channels/discord/serve.py
new file mode 100644
index 0000000..412a880
--- /dev/null
+++ b/EvoScientist/channels/discord/serve.py
@@ -0,0 +1,90 @@
+"""Discord channel server.
+
+Standalone script to run the Discord channel with CLI options.
+
+Usage:
+    python -m EvoScientist.channels.discord.serve --bot-token TOKEN [OPTIONS]
+
+Examples:
+    # Allow all senders (default)
+    python -m EvoScientist.channels.discord.serve --bot-token TOKEN
+
+    # Only allow specific senders and channels
+    python -m EvoScientist.channels.discord.serve --bot-token TOKEN --allow 123 --allow-channel 456
+
+    # With proxy, agent and thinking
+    python -m EvoScientist.channels.discord.serve --bot-token TOKEN --proxy http://proxy:8080 --agent --thinking
+"""
+
+import argparse
+import logging
+
+from ...logging_config import configure_logging_from_settings
+from ..bus import MessageBus
+from ..standalone import run_standalone
+from .channel import DiscordChannel, DiscordConfig
+
+logger = logging.getLogger(__name__)
+
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Discord channel server",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--bot-token",
+        required=True,
+        help="Discord bot token",
+    )
+    parser.add_argument(
+        "--allow",
+        action="append",
+        dest="allowed_senders",
+        help="Allowed sender (Discord user ID). Can be used multiple times.",
+    )
+    parser.add_argument(
+        "--allow-channel",
+        action="append",
+        dest="allowed_channels",
+        help="Allowed channel ID. Can be used multiple times.",
+    )
+    parser.add_argument(
+        "--proxy",
+        help="HTTP proxy URL for Discord API requests",
+    )
+    parser.add_argument(
+        "--agent",
+        action="store_true",
+        help="Use EvoScientist agent as handler (default: echo)",
+    )
+    parser.add_argument(
+        "--thinking",
+        action="store_true",
+        help="Send thinking content as intermediate messages (requires --agent)",
+    )
+    return parser.parse_args()
+
+
+def main():
+    """Entry point."""
+    configure_logging_from_settings(default_level=logging.INFO)
+    args = parse_args()
+
+    config = DiscordConfig(
+        bot_token=args.bot_token,
+        allowed_senders=set(args.allowed_senders) if args.allowed_senders else None,
+        allowed_channels=set(args.allowed_channels) if args.allowed_channels else None,
+        proxy=args.proxy,
+    )
+
+    send_thinking = args.thinking and args.agent
+    bus = MessageBus()
+    channel = DiscordChannel(config)
+
+    run_standalone(channel, bus, use_agent=args.agent, send_thinking=send_thinking)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/EvoScientist/channels/email/__init__.py b/EvoScientist/channels/email/__init__.py
new file mode 100644
index 0000000..fff3e2b
--- /dev/null
+++ b/EvoScientist/channels/email/__init__.py
@@ -0,0 +1,43 @@
+"""Email channel for EvoScientist.
+
+Uses IMAP polling for inbound + SMTP for outbound. Pure Python, no extra deps.
+
+Usage in config:
+    channel_enabled = "email"
+    email_imap_host = "imap.gmail.com"
+    email_smtp_host = "smtp.gmail.com"
+    ...
+"""
+
+from ..channel_manager import _parse_csv, register_channel
+from .channel import EmailChannel, EmailConfig
+
+__all__ = ["EmailChannel", "EmailConfig"]
+
+
+def create_from_config(config) -> EmailChannel:
+    allowed = _parse_csv(config.email_allowed_senders)
+    return EmailChannel(
+        EmailConfig(
+            imap_host=config.email_imap_host,
+            imap_port=config.email_imap_port,
+            imap_username=config.email_imap_username,
+            imap_password=config.email_imap_password,
+            imap_mailbox=config.email_imap_mailbox,
+            imap_use_ssl=config.email_imap_use_ssl,
+            smtp_host=config.email_smtp_host,
+            smtp_port=config.email_smtp_port,
+            smtp_username=config.email_smtp_username,
+            smtp_password=config.email_smtp_password,
+            smtp_starttls=config.email_smtp_use_tls,
+            from_address=config.email_from_address,
+            poll_interval=config.email_poll_interval,
+            mark_seen=config.email_mark_seen,
+            max_body_chars=config.email_max_body_chars,
+            subject_prefix=config.email_subject_prefix,
+            allowed_senders=allowed,
+        )
+    )
+
+
+register_channel("email", create_from_config)
diff --git a/EvoScientist/channels/email/channel.py b/EvoScientist/channels/email/channel.py
new file mode 100644
index 0000000..6d5e426
--- /dev/null
+++ b/EvoScientist/channels/email/channel.py
@@ -0,0 +1,454 @@
+"""Email channel implementation using IMAP + SMTP."""
+
+import asyncio
+import contextlib
+import email as email_lib
+import email.utils
+import html
+import imaplib
+import logging
+import re
+import smtplib
+import ssl
+from dataclasses import dataclass
+from datetime import datetime
+from email import encoders
+from email.header import decode_header, make_header
+from email.message import EmailMessage
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.utils import parseaddr
+from pathlib import Path
+
+from ..base import Channel, ChannelError, RawIncoming
+from ..capabilities import EMAIL as EMAIL_CAPS
+from ..config import BaseChannelConfig
+from ..mixins import PollingMixin
+
+logger = logging.getLogger(__name__)
+
+
+def _decode_hdr(raw: str) -> str:
+    try:
+        return str(make_header(decode_header(raw))) if raw else ""
+    except Exception:
+        return raw or ""
+
+
+def _strip_html(text: str) -> str:
+    text = re.sub(r"", "\n", text, flags=re.IGNORECASE)
+    text = re.sub(r"]*>", "\n", text, flags=re.IGNORECASE)
+    text = re.sub(r"

", "\n", text, flags=re.IGNORECASE) + text = re.sub(r"<[^>]+>", "", text) + return html.unescape(text).strip() + + +@dataclass +class EmailConfig(BaseChannelConfig): + imap_host: str = "" + imap_port: int = 993 + imap_username: str = "" + imap_password: str = "" + imap_mailbox: str = "INBOX" + imap_use_ssl: bool = True + smtp_host: str = "" + smtp_port: int = 587 + smtp_username: str = "" + smtp_password: str = "" + smtp_starttls: bool = ( + True # True=STARTTLS (port 587), False=implicit SSL (port 465) + ) + from_address: str = "" + poll_interval: int = 30 + mark_seen: bool = True + max_body_chars: int = 12000 + subject_prefix: str = "Re: " + text_chunk_limit: int = 4096 + + +class EmailChannel(Channel, PollingMixin): + """Email channel using IMAP polling + SMTP.""" + + name = "email" + + capabilities = EMAIL_CAPS + _non_retryable_patterns = ("auth", "login", "credential") + + def __init__(self, config: EmailConfig): + super().__init__(config) + self._imap: imaplib.IMAP4_SSL | imaplib.IMAP4 | None = None + + async def start(self) -> None: + cfg = self.config + if not cfg.imap_host or not cfg.imap_username: + raise ChannelError("Email imap_host and imap_username are required") + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, self._connect_imap) + self._running = True + logger.info( + f"Email channel started (IMAP: {cfg.imap_host}, poll {cfg.poll_interval}s)" + ) + await self._start_polling() + + async def _cleanup(self) -> None: + await self._stop_polling() + if self._imap: + try: + self._imap.close() + self._imap.logout() + except Exception: + pass + self._imap = None + logger.info("Email channel stopped") + + def _connect_imap(self) -> None: + cfg = self.config + try: + if cfg.imap_use_ssl: + self._imap = imaplib.IMAP4_SSL( + cfg.imap_host, + cfg.imap_port, + ssl_context=ssl.create_default_context(), + ) + else: + self._imap = imaplib.IMAP4(cfg.imap_host, cfg.imap_port) + self._imap.login(cfg.imap_username, cfg.imap_password) + self._imap.select(cfg.imap_mailbox) + except Exception as e: + raise ChannelError(f"IMAP failed: {e}") from e + + def _reconnect_imap(self) -> None: + try: + if self._imap: + self._imap.noop() + return + except Exception: + pass + self._connect_imap() + + async def _poll_once(self) -> None: + loop = asyncio.get_running_loop() + messages = await loop.run_in_executor(None, self._fetch_unseen) + for m in messages: + await self._process_email(m) + + def _fetch_unseen(self) -> list[dict]: + self._reconnect_imap() + results = [] + try: + st, data = self._imap.search(None, "UNSEEN") + if st != "OK": + return [] + for mid in data[0].split()[-20:]: + st, msg_data = self._imap.fetch(mid, "(RFC822)") + if st != "OK": + continue + msg = email_lib.message_from_bytes(msg_data[0][1]) + from_name, from_addr = parseaddr(msg.get("From", "")) + body = self._extract_body(msg) + if len(body) > self.config.max_body_chars: + body = body[: self.config.max_body_chars] + "\n[...truncated]" + # Extract attachments and inline images + attachments = [] + if msg.is_multipart(): + for part in msg.walk(): + content_disp = part.get("Content-Disposition") or "" + content_type = part.get_content_type() or "" + is_attachment = "attachment" in content_disp.lower() + is_inline_image = ( + "inline" in content_disp.lower() + and content_type.startswith("image/") + ) + # Also detect non-text parts with a filename but no + # Content-Disposition header (common for PDFs, docs, + # etc. sent by some email clients). + is_named_file = ( + not is_attachment + and not is_inline_image + and part.get_filename() + and not content_type.startswith("multipart/") + and not content_type.startswith("text/") + ) + if is_attachment or is_inline_image or is_named_file: + filename = part.get_filename() or "attachment" + filename = _decode_hdr(filename) + payload_data = part.get_payload(decode=True) + if payload_data: + from ..base import MAX_ATTACHMENT_BYTES, MEDIA_DIR + + if len(payload_data) > MAX_ATTACHMENT_BYTES: + attachments.append( + { + "annotation": f"[attachment: {filename} - too large ({len(payload_data)} bytes)]" + } + ) + else: + MEDIA_DIR.mkdir(parents=True, exist_ok=True) + local_path = ( + MEDIA_DIR / f"email_{mid.decode()}_{filename}" + ) + local_path.write_bytes(payload_data) + label = ( + "inline-image" + if is_inline_image + else "attachment" + ) + attachments.append( + { + "path": str(local_path), + "annotation": f"[{label}: {local_path}]", + } + ) + if self.config.mark_seen: + self._imap.store(mid, "+FLAGS", "\\Seen") + results.append( + { + "from_addr": from_addr, + "from_name": _decode_hdr(from_name), + "subject": _decode_hdr(msg.get("Subject", "")), + "body": body, + "message_id": msg.get("Message-ID", ""), + "date": msg.get("Date", ""), + "references": msg.get("References", ""), + "attachments": attachments, + } + ) + except Exception as e: + logger.error(f"IMAP fetch: {e}") + return results + + def _extract_body(self, msg) -> str: + if msg.is_multipart(): + for part in msg.walk(): + ct = part.get_content_type() + if ct == "text/plain": + return self._decode_payload(part) + for part in msg.walk(): + if part.get_content_type() == "text/html": + return _strip_html(self._decode_payload(part)) + return "[no text content]" + text = self._decode_payload(msg) + return _strip_html(text) if msg.get_content_type() == "text/html" else text + + @staticmethod + def _decode_payload(part) -> str: + payload = part.get_payload(decode=True) + if not payload: + return "" + charset = part.get_content_charset() or "utf-8" + return payload.decode(charset, errors="replace") + + async def _process_email(self, m: dict) -> None: + subject = m["subject"] + text = f"[邮件] 主题: {subject}\n\n{m['body']}" if subject else m["body"] + try: + ts = email_lib.utils.parsedate_to_datetime(m["date"]) + except Exception: + ts = datetime.now() + # Process attachments + media_paths: list[str] = [] + annotations: list[str] = [] + for att in m.get("attachments", []): + if att.get("path"): + media_paths.append(att["path"]) + if att.get("annotation"): + annotations.append(att["annotation"]) + await self._enqueue_raw( + RawIncoming( + sender_id=m["from_addr"], + chat_id=m["from_addr"], + text=text, + timestamp=ts, + message_id=m["message_id"], + media_files=media_paths, + content_annotations=annotations, + metadata={ + "chat_id": m["from_addr"], + "subject": subject, + "original_message_id": m["message_id"], + "references": m["references"], + "backend": "email", + }, + ) + ) + + # ── Send ────────────────────────────────────────────────────── + + def _is_ready(self) -> bool: + return bool(self.config.smtp_host) + + @contextlib.contextmanager + def _smtp_connect(self): + """Open an SMTP connection as a context manager. + + Ensures the connection is closed even if login or send raises. + Uses STARTTLS (port 587) when smtp_starttls=True, otherwise + implicit SSL (port 465). + """ + cfg = self.config + srv = None + try: + if cfg.smtp_starttls: + srv = smtplib.SMTP(cfg.smtp_host, cfg.smtp_port, timeout=30) + srv.starttls() + else: + srv = smtplib.SMTP_SSL( + cfg.smtp_host, + cfg.smtp_port, + context=ssl.create_default_context(), + timeout=30, + ) + srv.login(cfg.smtp_username, cfg.smtp_password) + yield srv + finally: + if srv is not None: + try: + srv.quit() + except Exception: + try: + srv.close() + except Exception: + pass + + async def _send_chunk(self, chat_id, formatted_text, raw_text, reply_to, metadata): + loop = asyncio.get_running_loop() + try: + await loop.run_in_executor( + None, + self._smtp_send_html, + chat_id, + formatted_text, + raw_text, + metadata or {}, + ) + except Exception as e: + err_str = str(e).lower() + # Only fall back to plain text for format-related errors, not server rejections + if any( + code in err_str for code in ("550", "553", "554", "auth", "rejected") + ): + raise + logger.warning(f"HTML email failed ({e}), falling back to plain text") + await loop.run_in_executor( + None, + self._smtp_send, + chat_id, + raw_text, + metadata or {}, + ) + + def _smtp_send(self, to: str, content: str, meta: dict) -> None: + cfg = self.config + from_addr = cfg.from_address or cfg.smtp_username + logger.debug(f"SMTP plain send: from={from_addr} to={to}") + msg = EmailMessage() + orig_subj = meta.get("subject", "") + msg["Subject"] = ( + f"{cfg.subject_prefix}{orig_subj}" + if orig_subj and not orig_subj.lower().startswith("re:") + else (orig_subj or "EvoScientist Reply") + ) + msg["From"] = from_addr + msg["To"] = to + orig_id = meta.get("original_message_id", "") + if orig_id: + msg["In-Reply-To"] = orig_id + msg["References"] = f"{meta.get('references', '')} {orig_id}".strip() + msg.set_content(content) + try: + with self._smtp_connect() as srv: + srv.sendmail(from_addr, [to], msg.as_string()) + except Exception as e: + logger.error(f"SMTP send failed: from={from_addr} to={to}") + raise RuntimeError("SMTP send failed") from e + + def _smtp_send_html( + self, to: str, html_content: str, plain_content: str, meta: dict + ) -> None: + """Send an email with both HTML and plain-text parts.""" + cfg = self.config + from_addr = cfg.from_address or cfg.smtp_username + logger.debug(f"SMTP HTML send: from={from_addr} to={to}") + msg = MIMEMultipart("alternative") + orig_subj = meta.get("subject", "") + msg["Subject"] = ( + f"{cfg.subject_prefix}{orig_subj}" + if orig_subj and not orig_subj.lower().startswith("re:") + else (orig_subj or "EvoScientist Reply") + ) + msg["From"] = from_addr + msg["To"] = to + orig_id = meta.get("original_message_id", "") + if orig_id: + msg["In-Reply-To"] = orig_id + msg["References"] = f"{meta.get('references', '')} {orig_id}".strip() + msg.attach(MIMEText(plain_content, "plain", "utf-8")) + msg.attach(MIMEText(html_content, "html", "utf-8")) + try: + with self._smtp_connect() as srv: + srv.sendmail(from_addr, [to], msg.as_string()) + except Exception as e: + logger.error(f"SMTP HTML send failed: from={from_addr} to={to}") + raise RuntimeError("SMTP HTML send failed") from e + + # ── Media send (email attachment) ───────────────────────────── + + async def _send_media_impl( + self, + recipient: str, + file_path: str, + caption: str = "", + metadata: dict | None = None, + ) -> bool: + """Send a file as an email attachment via SMTP.""" + loop = asyncio.get_running_loop() + await loop.run_in_executor( + None, + self._smtp_send_attachment, + recipient, + file_path, + caption, + metadata or {}, + ) + return True + + def _smtp_send_attachment( + self, to: str, file_path: str, caption: str, meta: dict + ) -> None: + """Send an email with a file attachment.""" + cfg = self.config + from_addr = cfg.from_address or cfg.smtp_username + logger.debug(f"SMTP attachment send: from={from_addr} to={to} file={file_path}") + msg = MIMEMultipart() + orig_subj = meta.get("subject", "") + msg["Subject"] = ( + f"{cfg.subject_prefix}{orig_subj}" + if orig_subj and not orig_subj.lower().startswith("re:") + else (orig_subj or "EvoScientist Reply") + ) + msg["From"] = from_addr + msg["To"] = to + orig_id = meta.get("original_message_id", "") + if orig_id: + msg["In-Reply-To"] = orig_id + msg["References"] = f"{meta.get('references', '')} {orig_id}".strip() + + # Text body + if caption: + msg.attach(MIMEText(caption, "plain", "utf-8")) + + # Attachment + path = Path(file_path) + part = MIMEBase("application", "octet-stream") + part.set_payload(path.read_bytes()) + encoders.encode_base64(part) + part.add_header("Content-Disposition", f"attachment; filename={path.name}") + msg.attach(part) + + try: + with self._smtp_connect() as srv: + srv.sendmail(from_addr, [to], msg.as_string()) + except Exception as e: + logger.error(f"SMTP attachment send failed: from={from_addr} to={to}") + raise RuntimeError("SMTP attachment send failed") from e diff --git a/EvoScientist/channels/email/probe.py b/EvoScientist/channels/email/probe.py new file mode 100644 index 0000000..c2a09d0 --- /dev/null +++ b/EvoScientist/channels/email/probe.py @@ -0,0 +1,92 @@ +"""Email credential validation.""" + +import imaplib +import logging +import smtplib +import ssl + +logger = logging.getLogger(__name__) + + +async def validate_email_imap( + host: str, + port: int, + username: str, + password: str, + use_ssl: bool = True, +) -> tuple[bool, str]: + """Validate IMAP credentials. + + Returns: + Tuple of (is_valid, message). + """ + if not host or not username or not password: + return False, "host, username, and password are required" + + import asyncio + + loop = asyncio.get_event_loop() + + def _check(): + try: + if use_ssl: + ctx = ssl.create_default_context() + conn = imaplib.IMAP4_SSL(host, port, ssl_context=ctx) + else: + conn = imaplib.IMAP4(host, port) + conn.login(username, password) + conn.logout() + return True, "IMAP credentials valid" + except imaplib.IMAP4.error as e: + return False, f"IMAP auth failed: {e}" + except Exception as e: + return False, f"IMAP error: {e}" + + return await loop.run_in_executor(None, _check) + + +async def validate_email_smtp( + host: str, + port: int, + username: str, + password: str, + use_tls: bool = True, +) -> tuple[bool, str]: + """Validate SMTP credentials. + + Returns: + Tuple of (is_valid, message). + """ + if not host or not username or not password: + return False, "host, username, and password are required" + + import asyncio + + loop = asyncio.get_event_loop() + + def _check(): + server = None + try: + if use_tls: + server = smtplib.SMTP(host, port, timeout=10) + server.starttls() + else: + ctx = ssl.create_default_context() + server = smtplib.SMTP_SSL(host, port, context=ctx, timeout=10) + server.login(username, password) + return True, "SMTP credentials valid" + except smtplib.SMTPAuthenticationError: + return False, "SMTP auth failed" + except Exception as e: + return False, f"SMTP error: {e}" + finally: + if server is not None: + try: + server.quit() + except Exception: + try: + server.close() + except Exception: + pass + + return await loop.run_in_executor(None, _check) diff --git a/EvoScientist/channels/email/serve.py b/EvoScientist/channels/email/serve.py new file mode 100644 index 0000000..a5181a3 --- /dev/null +++ b/EvoScientist/channels/email/serve.py @@ -0,0 +1,121 @@ +"""Email channel server. + +Standalone script to run the Email channel with CLI options. + +Usage: + python -m EvoScientist.channels.email.serve --imap-host HOST --imap-username USER --imap-password PASS --smtp-host HOST --smtp-username USER --smtp-password PASS --from-address ADDR [OPTIONS] + +Examples: + # Basic usage + python -m EvoScientist.channels.email.serve --imap-host imap.gmail.com --imap-username bot@gmail.com --imap-password PASS --smtp-host smtp.gmail.com --smtp-username bot@gmail.com --smtp-password PASS --from-address bot@gmail.com + + # With allowed senders and custom poll interval + python -m EvoScientist.channels.email.serve --imap-host imap.gmail.com --imap-username bot@gmail.com --imap-password PASS --smtp-host smtp.gmail.com --smtp-username bot@gmail.com --smtp-password PASS --from-address bot@gmail.com --allow user@example.com --poll-interval 60 + + # With agent and thinking + python -m EvoScientist.channels.email.serve --imap-host imap.gmail.com --imap-username bot@gmail.com --imap-password PASS --smtp-host smtp.gmail.com --smtp-username bot@gmail.com --smtp-password PASS --from-address bot@gmail.com --agent --thinking +""" + +import argparse +import logging + +from ...logging_config import configure_logging_from_settings +from ..bus import MessageBus +from ..standalone import run_standalone +from .channel import EmailChannel, EmailConfig + +logger = logging.getLogger(__name__) + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Email channel server", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--imap-host", + required=True, + help="IMAP server hostname", + ) + parser.add_argument( + "--imap-username", + required=True, + help="IMAP username", + ) + parser.add_argument( + "--imap-password", + required=True, + help="IMAP password", + ) + parser.add_argument( + "--smtp-host", + required=True, + help="SMTP server hostname", + ) + parser.add_argument( + "--smtp-username", + required=True, + help="SMTP username", + ) + parser.add_argument( + "--smtp-password", + required=True, + help="SMTP password", + ) + parser.add_argument( + "--from-address", + required=True, + help="From email address for outgoing messages", + ) + parser.add_argument( + "--allow", + action="append", + dest="allowed_senders", + help="Allowed sender (email address). Can be used multiple times.", + ) + parser.add_argument( + "--poll-interval", + type=int, + default=30, + help="IMAP poll interval in seconds (default: 30)", + ) + parser.add_argument( + "--agent", + action="store_true", + help="Use EvoScientist agent as handler (default: echo)", + ) + parser.add_argument( + "--thinking", + action="store_true", + help="Send thinking content as intermediate messages (requires --agent)", + ) + return parser.parse_args() + + +def main(): + """Entry point.""" + configure_logging_from_settings(default_level=logging.INFO) + args = parse_args() + + config = EmailConfig( + imap_host=args.imap_host, + imap_username=args.imap_username, + imap_password=args.imap_password, + smtp_host=args.smtp_host, + smtp_username=args.smtp_username, + smtp_password=args.smtp_password, + from_address=args.from_address, + allowed_senders=set(args.allowed_senders) if args.allowed_senders else None, + poll_interval=args.poll_interval, + ) + + send_thinking = args.thinking and args.agent + bus = MessageBus() + channel = EmailChannel(config) + + run_standalone(channel, bus, use_agent=args.agent, send_thinking=send_thinking) + + +if __name__ == "__main__": + main() diff --git a/EvoScientist/channels/feishu/__init__.py b/EvoScientist/channels/feishu/__init__.py new file mode 100644 index 0000000..55c6ed4 --- /dev/null +++ b/EvoScientist/channels/feishu/__init__.py @@ -0,0 +1,25 @@ +from ..channel_manager import _parse_csv, register_channel +from .channel import FeishuChannel, FeishuConfig + +__all__ = ["FeishuChannel", "FeishuConfig"] + + +def create_from_config(config) -> FeishuChannel: + allowed = _parse_csv(config.feishu_allowed_senders) + proxy = config.feishu_proxy or None + return FeishuChannel( + FeishuConfig( + app_id=config.feishu_app_id, + app_secret=config.feishu_app_secret, + verification_token=config.feishu_verification_token, + encrypt_key=config.feishu_encrypt_key, + webhook_port=config.feishu_webhook_port, + allowed_senders=allowed, + feishu_domain=config.feishu_domain, + proxy=proxy, + subscription_mode=getattr(config, "feishu_subscription_mode", "webhook"), + ) + ) + + +register_channel("feishu", create_from_config) diff --git a/EvoScientist/channels/feishu/channel.py b/EvoScientist/channels/feishu/channel.py new file mode 100644 index 0000000..c3e9016 --- /dev/null +++ b/EvoScientist/channels/feishu/channel.py @@ -0,0 +1,1055 @@ +"""Feishu (飞书/Lark) channel implementation. + +Receives messages via HTTP event subscription webhook (aiohttp) or +WebSocket long connection (lark-oapi SDK), sends replies via Feishu +Open API REST endpoints. + +Feishu Open API docs: https://open.feishu.cn/document + +Authentication: + - App ID + App Secret → tenant_access_token (2-hour TTL, auto-refreshed) + +Event subscription (two modes): + - **Webhook**: URL verification challenge on first request, + ``im.message.receive_v1`` events via HTTP POST callback. + Requires a publicly reachable URL. + - **WebSocket**: outbound long connection via ``lark-oapi`` SDK. + No public IP required. + +Send API: + - ``POST /open-apis/im/v1/messages?receive_id_type=chat_id`` +""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import json +import logging +import queue +import re +import threading +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar + +if TYPE_CHECKING: + from aiohttp import web + +from ..base import Channel, ChannelError, RawIncoming +from ..capabilities import FEISHU as FEISHU_CAPS +from ..config import BaseChannelConfig +from ..mixins import TokenMixin, WebhookMixin + +logger = logging.getLogger(__name__) + + +# ── Markdown → Feishu Post conversion ──────────────────────────── + + +def _parse_inline_text(text: str) -> list[dict]: + """Parse inline Markdown elements into Feishu post tag dicts. + + Handles: `code`, **bold**, ~~strikethrough~~, [link](url), _italic_. + """ + elements: list[dict] = [] + # Pattern order matters: code first (protect content), then bold, strikethrough, link, italic + pattern = re.compile( + r"`([^`]+)`" # inline code + r"|\*\*(.+?)\*\*" # bold + r"|~~(.+?)~~" # strikethrough + r"|\[([^\]]+)\]\(([^)]+)\)" # link + r"|_(.+?)_" # italic + ) + pos = 0 + for m in pattern.finditer(text): + # Plain text before this match + if m.start() > pos: + elements.append({"tag": "text", "text": text[pos : m.start()]}) + + if m.group(1) is not None: + # inline code → code_block would be block-level; use text with style + elements.append( + { + "tag": "text", + "text": m.group(1), + "style": ["code_block"], + } + ) + elif m.group(2) is not None: + elements.append( + { + "tag": "text", + "text": m.group(2), + "style": ["bold"], + } + ) + elif m.group(3) is not None: + elements.append( + { + "tag": "text", + "text": m.group(3), + "style": ["strikethrough"], + } + ) + elif m.group(4) is not None: + elements.append( + { + "tag": "a", + "text": m.group(4), + "href": m.group(5), + } + ) + elif m.group(6) is not None: + elements.append( + { + "tag": "text", + "text": m.group(6), + "style": ["italic"], + } + ) + pos = m.end() + + # Remaining plain text + if pos < len(text): + elements.append({"tag": "text", "text": text[pos:]}) + return elements + + +def _parse_inline_elements(line: str) -> list[dict]: + """Parse a single Markdown line into a list of Feishu post elements. + + Handles headings (→ bold), blockquotes (→ italic with prefix), + list items (→ bullet prefix), and plain lines. + """ + # Heading: # Title → bold text + heading_match = re.match(r"^(#{1,6})\s+(.+)$", line) + if heading_match: + return [{"tag": "text", "text": heading_match.group(2), "style": ["bold"]}] + + # Blockquote: > text → italic with "▎" prefix + quote_match = re.match(r"^>\s*(.*)$", line) + if quote_match: + inner = quote_match.group(1) + elements = [{"tag": "text", "text": "▎", "style": ["italic"]}] + elements.extend(_parse_inline_text(inner)) + return elements + + # Unordered list: - item or * item → "• " prefix + list_match = re.match(r"^[\-\*]\s+(.+)$", line) + if list_match: + elements = [{"tag": "text", "text": "• "}] + elements.extend(_parse_inline_text(list_match.group(1))) + return elements + + # Ordered list: 1. item → keep number prefix + ol_match = re.match(r"^(\d+)\.\s+(.+)$", line) + if ol_match: + elements = [{"tag": "text", "text": f"{ol_match.group(1)}. "}] + elements.extend(_parse_inline_text(ol_match.group(2))) + return elements + + # Plain line + return _parse_inline_text(line) + + +def _markdown_to_feishu_post(text: str) -> dict | None: + """Convert Markdown text to Feishu post (rich text) JSON structure. + + Returns a dict like {"zh_cn": {"content": [[...]]}} suitable for + Feishu msg_type="post", or None if the text is empty. + """ + if not text or not text.strip(): + return None + + paragraphs: list[list[dict]] = [] + current_paragraph: list[dict] = [] + in_code_block = False + code_lines: list[str] = [] + code_lang = "" + + for line in text.split("\n"): + # Code block fences + if line.startswith("```"): + if not in_code_block: + # Flush any pending paragraph + if current_paragraph: + paragraphs.append(current_paragraph) + current_paragraph = [] + in_code_block = True + code_lang = line[3:].strip() + code_lines = [] + else: + # End of code block + code_text = "\n".join(code_lines) + paragraphs.append( + [ + { + "tag": "code_block", + "language": code_lang or "plain", + "text": code_text, + } + ] + ) + in_code_block = False + code_lines = [] + code_lang = "" + continue + + if in_code_block: + code_lines.append(line) + continue + + # Empty line → new paragraph + if not line.strip(): + if current_paragraph: + paragraphs.append(current_paragraph) + current_paragraph = [] + continue + + # Non-empty line + elements = _parse_inline_elements(line) + if elements: + # Each visual line becomes its own paragraph in Feishu post + if current_paragraph: + paragraphs.append(current_paragraph) + current_paragraph = elements + + # Flush remaining + if in_code_block and code_lines: + code_text = "\n".join(code_lines) + paragraphs.append( + [ + { + "tag": "code_block", + "language": code_lang or "plain", + "text": code_text, + } + ] + ) + elif current_paragraph: + paragraphs.append(current_paragraph) + + if not paragraphs: + return None + + return {"zh_cn": {"content": paragraphs}} + + +@dataclass +class FeishuConfig(BaseChannelConfig): + app_id: str = "" + app_secret: str = "" + verification_token: str = "" + encrypt_key: str = "" + webhook_port: int = 9000 + text_chunk_limit: int = 4096 + feishu_domain: str = "https://open.feishu.cn" + subscription_mode: str = "webhook" # "webhook" | "websocket" + + +class FeishuChannel(Channel, WebhookMixin, TokenMixin): + capabilities = FEISHU_CAPS + """Feishu channel using Open API + event subscription webhook.""" + + name = "feishu" + _ready_attrs = ("_http_client", "_access_token") + _non_retryable_patterns = ( + "app_access_token is empty", # invalid credentials + "10003", # invalid app_id + "10014", # invalid app_secret + "99991401", # permission denied + "99991663", # no permission + "99991672", # feature not enabled + ) + _rate_limit_patterns = ("99991400", "rate limit", "频率限制") + _rate_limit_delay = 2.0 + + def __init__(self, config: FeishuConfig): + super().__init__(config) + self._mention_names: list[str] = [] # bot mention keys from events + self._main_loop: asyncio.AbstractEventLoop | None = None + self._lark_ws_thread: threading.Thread | None = None + self._ws_event_queue: queue.Queue | None = None + self._ws_consumer_task: asyncio.Task | None = None + + # ── WebhookMixin overrides ──────────────────────────────────── + + def _get_webhook_port(self) -> int: + return self.config.webhook_port + + def _webhook_routes(self) -> list[tuple[str, Any]]: + return [("POST", "/webhook/event", self._handle_event)] + + # ── TokenMixin overrides ────────────────────────────────────── + + async def _fetch_token(self) -> tuple[str, int]: + """Fetch Feishu tenant_access_token.""" + url = f"{self.config.feishu_domain}/open-apis/auth/v3/tenant_access_token/internal" + body = { + "app_id": self.config.app_id, + "app_secret": self.config.app_secret, + } + try: + resp = await self._http_client.post(url, json=body) + data = resp.json() + except Exception as e: + raise ChannelError(f"Failed to get Feishu access token: {e}") from e + + if data.get("code") != 0: + raise ChannelError(f"Feishu auth error: {data.get('msg', 'unknown')}") + return data["tenant_access_token"], data.get("expire", 7200) + + # ── Lifecycle ───────────────────────────────────────────────── + + _VALID_SUBSCRIPTION_MODES = ("webhook", "websocket") + + async def start(self) -> None: + if not self.config.app_id: + raise ChannelError("Feishu app_id is required") + if not self.config.app_secret: + raise ChannelError("Feishu app_secret is required") + if self.config.subscription_mode not in self._VALID_SUBSCRIPTION_MODES: + raise ChannelError( + f"Invalid feishu_subscription_mode: {self.config.subscription_mode!r}. " + f"Must be one of {self._VALID_SUBSCRIPTION_MODES}" + ) + + if self.config.subscription_mode == "websocket": + await self._start_websocket_mode() + else: + await self._start_webhook_mode() + + async def _start_webhook_mode(self) -> None: + try: + import httpx # noqa: F401 + from aiohttp import web # noqa: F401 + except ImportError: + raise ChannelError( + "aiohttp or httpx not installed. " + "Install with: pip install aiohttp httpx" + ) from None + + # Start webhook server (sets up self._http_client) + await self._start_webhook_server() + + # Verify credentials by fetching initial token + await self._refresh_token() + + self._running = True + logger.info( + f"Feishu channel started (webhook on port {self.config.webhook_port})" + ) + + async def _start_websocket_mode(self) -> None: + try: + import lark_oapi as lark + except ImportError: + raise ChannelError( + "lark-oapi not installed. Install with: pip install 'lark-oapi>=1.4.0'" + ) from None + + import httpx + + proxy = getattr(self.config, "proxy", None) or None + self._http_client = httpx.AsyncClient(timeout=15, proxy=proxy) + + # Verify credentials by fetching initial token + await self._refresh_token() + + self._main_loop = asyncio.get_running_loop() + + # Thread-safe queue: SDK thread puts events, main loop consumes + self._ws_event_queue = queue.Queue() + + # Set _running BEFORE creating consumer task — the task checks + # `while self._running` and would exit immediately otherwise. + self._running = True + self._ws_consumer_task = asyncio.create_task(self._consume_ws_events()) + + # Build SDK event handler + handler = ( + lark.EventDispatcherHandler.builder("", "") + .register_p2_im_message_receive_v1(self._on_lark_sdk_message) + .build() + ) + + ws_client = lark.ws.Client( + self.config.app_id, + self.config.app_secret, + event_handler=handler, + log_level=lark.LogLevel.WARNING, + ) + + def _run_ws(): + # Root cause: lark_oapi.ws.client stores the event loop in a + # *module-level* variable at import time. When imported from + # the main thread this captures the main loop (which has + # nest_asyncio patches). The SDK then calls + # loop.run_until_complete() from THIS thread on that *main* + # loop, causing cross-thread task-tracking conflicts: + # RuntimeError: Leaving task … does not match the current task + # AttributeError: 'NoneType' object has no attribute 'select' + # + # Fix: create a fresh event loop for this thread and replace + # the module-level ``loop`` variable so the SDK uses an + # isolated loop with no cross-thread interaction. + import lark_oapi.ws.client as _ws_mod + + fresh_loop = asyncio.new_event_loop() + asyncio.set_event_loop(fresh_loop) + _ws_mod.loop = fresh_loop + + try: + ws_client.start() + except Exception: + logger.exception( + "Feishu WebSocket SDK thread exited unexpectedly. " + "The channel will no longer receive messages. " + "Check app_id/app_secret and connection limits." + ) + + self._lark_ws_thread = threading.Thread(target=_run_ws, daemon=True) + self._lark_ws_thread.start() + + logger.info("Feishu channel started (WebSocket long connection mode)") + + def _on_lark_sdk_message(self, data) -> None: + """Sync callback invoked in lark-oapi SDK thread. + + Converts the SDK event object to a dict and puts it on a + thread-safe queue. The ``_consume_ws_events`` task on the main + asyncio loop picks it up — no asyncio cross-thread calls needed, + avoiding the nest_asyncio + Python 3.11 contextvars conflict. + """ + try: + event = data.event + msg = event.message + sender = event.sender + + # Rebuild mentions list from SDK objects + mentions_list = [] + if msg.mentions: + for m in msg.mentions: + mention_dict: dict[str, Any] = {"key": m.key, "id": {}} + if m.id: + mention_dict["id"] = { + "open_id": getattr(m.id, "open_id", ""), + "user_id": getattr(m.id, "user_id", ""), + } + mentions_list.append(mention_dict) + + event_dict = { + "sender": { + "sender_id": { + "open_id": sender.sender_id.open_id if sender.sender_id else "", + "user_id": getattr(sender.sender_id, "user_id", "") + if sender.sender_id + else "", + }, + "sender_type": sender.sender_type or "", + }, + "message": { + "chat_id": msg.chat_id or "", + "message_type": msg.message_type or "", + "message_id": msg.message_id or "", + "chat_type": msg.chat_type or "", + "content": msg.content or "{}", + "create_time": msg.create_time or "", + "mentions": mentions_list, + }, + } + + self._ws_event_queue.put(event_dict) + except Exception: + logger.exception("Feishu SDK message handler error") + + async def _consume_ws_events(self) -> None: + """Main-loop task that drains the thread-safe event queue.""" + while self._running: + try: + event_dict = self._ws_event_queue.get_nowait() + try: + await self._on_message(event_dict) + except Exception: + logger.exception("Feishu WS event processing error") + except queue.Empty: + await asyncio.sleep(0.05) + + async def _cleanup(self) -> None: + if self.config.subscription_mode == "websocket": + if self._ws_consumer_task: + self._ws_consumer_task.cancel() + try: + await self._ws_consumer_task + except asyncio.CancelledError: + pass + self._ws_consumer_task = None + if self._http_client: + await self._http_client.aclose() + self._http_client = None + # Daemon thread exits with the process; no explicit stop needed + self._lark_ws_thread = None + self._main_loop = None + self._ws_event_queue = None + else: + await self._stop_webhook_server() + self._access_token = None + logger.info("Feishu channel stopped") + + # ── Token helpers (adapt old API to mixin) ──────────────────── + + async def _ensure_token(self) -> str: + """Return a valid access token, refreshing if needed.""" + return await TokenMixin._ensure_token(self) + + # ── Send (template method overrides) ────────────────────────── + + async def _feishu_send(self, url: str, body: dict, headers: dict) -> bool: + """POST to Feishu API and return True if code==0.""" + try: + resp = await self._http_client.post(url, json=body, headers=headers) + return resp.json().get("code") == 0 + except Exception as e: + logger.warning(f"Feishu send error: {e}") + return False + + async def _send_chunk( + self, + chat_id, + formatted_text, + raw_text, + reply_to, + metadata, + ): + token = await self._ensure_token() + headers = {"Authorization": f"Bearer {token}"} + post_content = _markdown_to_feishu_post(raw_text) + + # If reply_to is set, try the reply API first + if reply_to: + reply_url = ( + f"{self.config.feishu_domain}/open-apis/im/v1/messages/{reply_to}/reply" + ) + if post_content is not None: + body = {"msg_type": "post", "content": json.dumps(post_content)} + else: + body = { + "msg_type": "text", + "content": json.dumps({"text": formatted_text}), + } + if await self._feishu_send(reply_url, body, headers): + return + + # Normal send (non-reply or reply fallback) + url = ( + f"{self.config.feishu_domain}" + f"/open-apis/im/v1/messages?receive_id_type=chat_id" + ) + + # Try post format first + if post_content is not None: + body = { + "receive_id": chat_id, + "msg_type": "post", + "content": json.dumps(post_content), + } + if await self._feishu_send(url, body, headers): + return + + # Fallback: plain text + body = { + "receive_id": chat_id, + "msg_type": "text", + "content": json.dumps({"text": formatted_text}), + } + if not await self._feishu_send(url, body, headers): + raise RuntimeError("Feishu send failed") + + # ── Media helpers ────────────────────────────────────────────── + + _IMAGE_EXTENSIONS: ClassVar[set[str]] = { + ".jpg", + ".jpeg", + ".png", + ".gif", + ".bmp", + ".webp", + } + + async def _download_media( + self, + message_id: str, + file_key: str, + msg_type: str, + ) -> str | None: + """Download an image or file attachment from Feishu. + + Returns the local file path on success, or None on failure. + """ + token = await self._ensure_token() + resource_type = "image" if msg_type == "image" else "file" + url = ( + f"{self.config.feishu_domain}" + f"/open-apis/im/v1/messages/{message_id}" + f"/resources/{file_key}?type={resource_type}" + ) + headers = {"Authorization": f"Bearer {token}"} + try: + resp = await self._http_client.get(url, headers=headers, timeout=30) + if resp.status_code != 200: + logger.warning(f"Feishu media download failed: HTTP {resp.status_code}") + return None + + # Check attachment size before writing to disk + cl = resp.headers.get("content-length") + if cl: + try: + too_large = self._check_attachment_size(int(cl), file_key) + if too_large: + logger.warning(too_large) + return None + except (ValueError, TypeError): + pass + from ..base import MAX_ATTACHMENT_BYTES + + if len(resp.content) > MAX_ATTACHMENT_BYTES: + logger.warning(f"Feishu media too large: {len(resp.content)} bytes") + return None + + # Determine extension from Content-Type or default + content_type = resp.headers.get("content-type", "") + ext_map = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/gif": ".gif", + "image/webp": ".webp", + "image/bmp": ".bmp", + } + ext = ext_map.get(content_type, ".bin") + local_path = self._media_path(f"feishu_{message_id}_{file_key}{ext}") + local_path.write_bytes(resp.content) + return str(local_path) + except Exception as e: + logger.warning(f"Failed to download Feishu media: {e}") + return None + + async def _upload_feishu_resource( + self, + url: str, + headers: dict, + file_path: str, + field_name: str, + extra_data: dict, + ) -> dict | None: + """Upload a file to Feishu API. Returns response data or None on failure.""" + with open(file_path, "rb") as f: + resp = await self._http_client.post( + url, + headers=headers, + data=extra_data, + files={field_name: (Path(file_path).name, f)}, + ) + data = resp.json() + if data.get("code") != 0: + logger.error(f"Feishu upload failed: {data.get('msg')}") + return None + return data["data"] + + async def _send_media_impl( + self, + recipient: str, + file_path: str, + caption: str = "", + metadata: dict | None = None, + ) -> bool: + """Send a media file through Feishu.""" + token = await self._ensure_token() + headers = {"Authorization": f"Bearer {token}"} + chat_id = self._resolve_media_chat_id(recipient, metadata) + + path = Path(file_path) + ext = path.suffix.lower() + is_image = ext in self._IMAGE_EXTENSIONS + + send_url = ( + f"{self.config.feishu_domain}" + f"/open-apis/im/v1/messages?receive_id_type=chat_id" + ) + + if is_image: + upload_url = f"{self.config.feishu_domain}/open-apis/im/v1/images" + data = await self._upload_feishu_resource( + upload_url, + headers, + file_path, + "image", + {"image_type": "message"}, + ) + if not data: + return False + body = { + "receive_id": chat_id, + "msg_type": "image", + "content": json.dumps({"image_key": data["image_key"]}), + } + else: + upload_url = f"{self.config.feishu_domain}/open-apis/im/v1/files" + data = await self._upload_feishu_resource( + upload_url, + headers, + file_path, + "file", + {"file_type": "stream", "file_name": path.name}, + ) + if not data: + return False + body = { + "receive_id": chat_id, + "msg_type": "file", + "content": json.dumps({"file_key": data["file_key"]}), + } + + if not await self._feishu_send(send_url, body, headers): + return False + + # Send caption as a separate text message if provided + if caption: + cap_body = { + "receive_id": chat_id, + "msg_type": "text", + "content": json.dumps({"text": caption}), + } + await self._feishu_send(send_url, cap_body, headers) + + return True + + # ── ACK reaction ─────────────────────────────────────────────── + + async def _send_ack_reaction( + self, chat_id: str, message_id: str, emoji: str = "THUMBSUP" + ) -> None: + """Send an acknowledgment reaction via Feishu Open API.""" + try: + token = await self._ensure_token() + url = f"{self.config.feishu_domain}/open-apis/im/v1/messages/{message_id}/reactions" + await self._http_client.post( + url, + json={"reaction_type": {"emoji_type": emoji}}, + headers={"Authorization": f"Bearer {token}"}, + ) + except Exception as e: + logger.debug(f"Feishu ack reaction failed: {e}") + + async def _remove_ack_reaction( + self, chat_id: str, message_id: str, emoji: str = "THUMBSUP" + ) -> None: + """Remove ACK reaction via Feishu Open API. + + Feishu's DELETE /reactions endpoint requires the reaction_id, which + we don't track. No-op for now. + """ + pass + + # ── Mention stripping ───────────────────────────────────────── + + def _strip_mention(self, text: str) -> str: + """Strip bot @mention placeholders from Feishu text. + + In Feishu v2 events the text contains placeholders like ``@_user_1`` + for each mention. ``_mention_names`` caches the placeholder keys that + belong to the bot (identified during ``_on_message``). + """ + result = text + for key in self._mention_names: + result = result.replace(key, "") + # Clean up extra whitespace left behind + return re.sub(r" +", " ", result).strip() + + # ── Event decryption ───────────────────────────────────────── + + def _decrypt_event(self, encrypted: str) -> dict: + """Decrypt a Feishu encrypted event payload (AES-256-CBC). + + Feishu encryption spec: + key = SHA256(encrypt_key) + data = base64_decode(encrypted) + iv = data[:16] + plain = AES_CBC_decrypt(data[16:], key, iv) # PKCS7 padded + """ + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + key = hashlib.sha256(self.config.encrypt_key.encode()).digest() + data = base64.b64decode(encrypted) + iv, ciphertext = data[:16], data[16:] + cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) + decryptor = cipher.decryptor() + padded = decryptor.update(ciphertext) + decryptor.finalize() + # Remove PKCS7 padding + pad_len = padded[-1] + plaintext = padded[:-pad_len].decode() + return json.loads(plaintext) + + # ── Webhook event handler ───────────────────────────────────── + + async def _handle_event(self, request) -> web.Response: + """Handle POST /webhook/event from Feishu.""" + from aiohttp import web + + try: + body = await request.json() + except Exception: + return web.Response(status=400) + + # ── Decrypt if encrypt_key is configured ── + if self.config.encrypt_key and "encrypt" in body: + try: + body = self._decrypt_event(body["encrypt"]) + except Exception: + logger.exception("Feishu event decryption failed") + return web.Response(status=400) + + # ── URL verification challenge ── + if body.get("type") == "url_verification": + challenge = body.get("challenge", "") + return web.json_response({"challenge": challenge}) + + # ── v2 event schema ── + schema = body.get("schema") + if schema == "2.0": + header = body.get("header", {}) + + # Verify token if configured + if self.config.verification_token: + token = header.get("token", "") + if token != self.config.verification_token: + logger.warning("Feishu event token mismatch") + return web.Response(status=403) + + event_type = header.get("event_type", "") + logger.info(f"Feishu v2 event received: {event_type}") + if event_type == "im.message.receive_v1": + try: + await self._on_message(body.get("event", {})) + except Exception: + logger.exception("Feishu _on_message failed") + + # ── v1 event schema (legacy) ── + elif "event" in body: + if self.config.verification_token: + token = body.get("token", "") + if token != self.config.verification_token: + logger.warning("Feishu event token mismatch (v1)") + return web.Response(status=403) + + event = body["event"] + msg_type = event.get("type", "") + logger.info(f"Feishu v1 event received: type={msg_type}") + if msg_type == "message": + try: + await self._on_message_v1(event) + except Exception: + logger.exception("Feishu _on_message_v1 failed") + else: + logger.info(f"Feishu event ignored: schema={schema}") + + return web.Response(status=200) + + async def _on_message(self, event: dict) -> None: + """Handle im.message.receive_v1 event (v2 schema).""" + sender_info = event.get("sender", {}) + sender_id_info = sender_info.get("sender_id", {}) + sender_id = sender_id_info.get("open_id") or sender_id_info.get("user_id") or "" + sender_type = sender_info.get("sender_type", "") + + # Skip bot's own messages + if sender_type == "app": + return + + message = event.get("message", {}) + chat_id = message.get("chat_id", "") + msg_type = message.get("message_type", "") + message_id = message.get("message_id", "") + + # In group chats, detect mention status for centralized gating + chat_type = message.get("chat_type", "") + is_group = chat_type == "group" + was_mentioned = True + if is_group: + mentions = message.get("mentions", []) + was_mentioned = bool(mentions) + # Cache bot mention keys — bot mentions have empty user IDs + bot_keys = [] + for m in mentions: + m_id = m.get("id", {}) + # Bot/app mentions have no open_id / user_id + if not m_id.get("open_id") and not m_id.get("user_id"): + key = m.get("key", "") + if key: + bot_keys.append(key) + if bot_keys: + self._mention_names = bot_keys + + # Parse content JSON + content_str = message.get("content", "{}") + try: + content_data = json.loads(content_str) + except json.JSONDecodeError: + content_data = {} + + text = "" + annotations: list[str] = [] + media_paths: list[str] = [] + + if msg_type == "text": + text = content_data.get("text", "") + elif msg_type == "post": + text = self._extract_post_text(content_data) + elif msg_type == "image" and self.config.include_attachments: + image_key = content_data.get("image_key", "") + if image_key: + local = await self._download_media(message_id, image_key, "image") + if local: + media_paths.append(local) + annotations.append(f"[attachment: {local}]") + else: + annotations.append("[image message - download failed]") + else: + annotations.append("[image message]") + elif msg_type == "file" and self.config.include_attachments: + file_key = content_data.get("file_key", "") + file_name = content_data.get("file_name", "unknown") + if file_key: + local = await self._download_media(message_id, file_key, "file") + if local: + media_paths.append(local) + annotations.append(f"[attachment: {local}]") + else: + annotations.append(f"[file: {file_name} - download failed]") + else: + annotations.append(f"[file message: {file_name}]") + elif msg_type in ("audio", "media") and self.config.include_attachments: + # Feishu audio messages are voice recordings + media_label = "voice" if msg_type == "audio" else msg_type + file_key = content_data.get("file_key", "") + if file_key: + local = await self._download_media(message_id, file_key, "file") + if local: + media_paths.append(local) + annotations.append(f"[{media_label}: {local}]") + else: + annotations.append(f"[{media_label} message - download failed]") + else: + annotations.append(f"[{media_label} message]") + elif msg_type == "sticker": + sticker_key = content_data.get("file_key", "") + if sticker_key and self.config.include_attachments: + local = await self._download_media(message_id, sticker_key, "image") + if local: + media_paths.append(local) + annotations.append(f"[sticker: {local}]") + else: + annotations.append("[sticker message]") + else: + annotations.append("[sticker message]") + else: + text = f"[{msg_type} message]" + + if not text and not media_paths and not annotations: + return + + # Parse timestamp (milliseconds) + create_time = message.get("create_time", "") + try: + timestamp = ( + datetime.fromtimestamp(int(create_time) / 1000) + if create_time + else datetime.now() + ) + except (ValueError, TypeError, OSError): + timestamp = datetime.now() + + await self._enqueue_raw( + RawIncoming( + sender_id=sender_id, + chat_id=chat_id, + text=text, + media_files=media_paths, + content_annotations=annotations, + timestamp=timestamp, + message_id=message_id, + metadata={ + "chat_id": chat_id, + "chat_type": message.get("chat_type", ""), + }, + is_group=is_group, + was_mentioned=was_mentioned, + ) + ) + + async def _on_message_v1(self, event: dict) -> None: + """Handle v1 schema message event (legacy).""" + sender_id = event.get("open_id", "") + if not sender_id: + return + + # Detect group and mention status for centralized gating + chat_type = event.get("chat_type", "") + is_group = chat_type == "group" + was_mentioned = True + if is_group: + text_without_at = event.get("text_without_at_bot", "") + was_mentioned = bool(text_without_at) + + text = event.get("text_without_at_bot", "") or event.get("text", "") + if not text: + return + + chat_id = event.get("open_chat_id", "") + message_id = event.get("open_message_id", "") + + await self._enqueue_raw( + RawIncoming( + sender_id=sender_id, + chat_id=chat_id, + text=text, + timestamp=datetime.now(), + message_id=message_id, + metadata={ + "chat_id": chat_id, + "chat_type": event.get("chat_type", ""), + }, + is_group=is_group, + was_mentioned=was_mentioned, + ) + ) + + @staticmethod + def _extract_post_text(content: dict) -> str: + """Extract plain text from Feishu post (rich text) content.""" + parts: list[str] = [] + # Post content has locale keys like "zh_cn", "en_us" + for locale_key in ("zh_cn", "en_us", "ja_jp"): + locale_content = content.get(locale_key) + if locale_content: + title = locale_content.get("title", "") + if title: + parts.append(title) + for paragraph in locale_content.get("content", []): + line_parts: list[str] = [] + for element in paragraph: + tag = element.get("tag", "") + if tag == "text": + line_parts.append(element.get("text", "")) + elif tag == "a": + line_parts.append(element.get("text", "")) + elif tag == "at": + # Skip @mentions of the bot + pass + line = "".join(line_parts).strip() + if line: + parts.append(line) + break # Use first available locale + return "\n".join(parts) diff --git a/EvoScientist/channels/feishu/probe.py b/EvoScientist/channels/feishu/probe.py new file mode 100644 index 0000000..46d962e --- /dev/null +++ b/EvoScientist/channels/feishu/probe.py @@ -0,0 +1,39 @@ +"""Feishu (飞书/Lark) app credential validation.""" + +import logging + +logger = logging.getLogger(__name__) + + +async def validate_feishu_credentials( + app_id: str, + app_secret: str, + domain: str = "https://open.feishu.cn", +) -> tuple[bool, str]: + """Validate Feishu app credentials by requesting a tenant_access_token. + + Returns: + Tuple of (is_valid, message). + """ + if not app_id: + return False, "No app_id provided" + if not app_secret: + return False, "No app_secret provided" + + try: + import httpx + except ImportError: + return False, "httpx not installed" + + url = f"{domain}/open-apis/auth/v3/tenant_access_token/internal" + body = {"app_id": app_id, "app_secret": app_secret} + try: + async with httpx.AsyncClient() as client: + resp = await client.post(url, json=body, timeout=10) + data = resp.json() + if data.get("code") == 0: + return True, f"App: {app_id}" + msg = data.get("msg", "unknown error") + return False, f"Auth failed: {msg}" + except Exception as e: + return False, f"Error: {e}" diff --git a/EvoScientist/channels/feishu/serve.py b/EvoScientist/channels/feishu/serve.py new file mode 100644 index 0000000..0bb0678 --- /dev/null +++ b/EvoScientist/channels/feishu/serve.py @@ -0,0 +1,117 @@ +"""Feishu (飞书/Lark) channel server. + +Standalone script to run the Feishu channel with CLI options. + +Usage: + python -m EvoScientist.channels.feishu.serve --app-id ID --app-secret SECRET [OPTIONS] + +Examples: + # Basic setup + python -m EvoScientist.channels.feishu.serve --app-id ID --app-secret SECRET + + # With verification token and custom port + python -m EvoScientist.channels.feishu.serve --app-id ID --app-secret SECRET \\ + --verification-token TOKEN --webhook-port 9000 + + # With agent and thinking + python -m EvoScientist.channels.feishu.serve --app-id ID --app-secret SECRET --agent --thinking +""" + +import argparse +import logging + +from ...logging_config import configure_logging_from_settings +from ..bus import MessageBus +from ..standalone import run_standalone +from .channel import FeishuChannel, FeishuConfig + +logger = logging.getLogger(__name__) + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Feishu (飞书/Lark) channel server", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--app-id", + required=True, + help="Feishu App ID", + ) + parser.add_argument( + "--app-secret", + required=True, + help="Feishu App Secret", + ) + parser.add_argument( + "--verification-token", + default="", + help="Feishu event verification token", + ) + parser.add_argument( + "--encrypt-key", + default="", + help="Feishu event encrypt key", + ) + parser.add_argument( + "--webhook-port", + type=int, + default=9000, + help="Port for webhook HTTP server (default: 9000)", + ) + parser.add_argument( + "--domain", + default="https://open.feishu.cn", + help="Feishu API domain (use https://open.larksuite.com for Lark)", + ) + parser.add_argument( + "--allow", + action="append", + dest="allowed_senders", + help="Allowed sender (Feishu open_id). Can be used multiple times.", + ) + parser.add_argument( + "--mode", + choices=["webhook", "websocket"], + default="webhook", + help="Subscription mode: webhook (default) or websocket (long connection, no public IP needed)", + ) + parser.add_argument( + "--agent", + action="store_true", + help="Use EvoScientist agent as handler (default: echo)", + ) + parser.add_argument( + "--thinking", + action="store_true", + help="Send thinking content as intermediate messages (requires --agent)", + ) + return parser.parse_args() + + +def main(): + """Entry point.""" + configure_logging_from_settings(default_level=logging.INFO) + args = parse_args() + + config = FeishuConfig( + app_id=args.app_id, + app_secret=args.app_secret, + verification_token=args.verification_token, + encrypt_key=args.encrypt_key, + webhook_port=args.webhook_port, + feishu_domain=args.domain, + allowed_senders=set(args.allowed_senders) if args.allowed_senders else None, + subscription_mode=args.mode, + ) + + send_thinking = args.thinking and args.agent + bus = MessageBus() + channel = FeishuChannel(config) + + run_standalone(channel, bus, use_agent=args.agent, send_thinking=send_thinking) + + +if __name__ == "__main__": + main() diff --git a/EvoScientist/channels/formatter.py b/EvoScientist/channels/formatter.py new file mode 100644 index 0000000..755ff83 --- /dev/null +++ b/EvoScientist/channels/formatter.py @@ -0,0 +1,295 @@ +"""Unified formatting pipeline for all channels. + +Internal representation is Markdown. This module converts Markdown to +each platform's native format: HTML, Slack mrkdwn, Discord Markdown, +or plain text. + +Channels no longer need per-file format functions — they just declare +``capabilities.format_type`` and the base class auto-configures a +``UnifiedFormatter`` instance. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from typing import ClassVar + +# ═════════════════════════════════════════════════════════════════════ +# Markdown conversion engine (formerly markdown_utils.py) +# ═════════════════════════════════════════════════════════════════════ + +_PLACEHOLDER_PREFIX = "\x00BLOCK" +_INLINE_PREFIX = "\x00INLINE" + +# A formatting rule: (regex_pattern, replacement) +InlineRule = tuple[str, str] + + +def convert_markdown( + text: str, + *, + code_block_formatter: Callable[[str, str], str], + inline_code_formatter: Callable[[str], str], + inline_rules: list[InlineRule], + escape_fn: Callable[[str], str] | None = None, +) -> str: + """Convert Markdown to a channel-specific format. + + Parameters + ---------- + text: + Input Markdown text. + code_block_formatter: + ``(language, code) -> str`` — format a fenced code block. + inline_code_formatter: + ``(code) -> str`` — format an inline code span. + inline_rules: + List of ``(pattern, replacement)`` pairs applied in order to the + remaining text (after code extraction and optional escaping). + escape_fn: + Optional function applied to the non-code text *before* inline + rules. Useful for HTML-escaping (Telegram) or other channel- + specific character escaping. + + Returns + ------- + str + The converted text. + """ + # 1. Extract and protect fenced code blocks (```...```) + code_blocks: list[str] = [] + + def _save_code_block(m: re.Match) -> str: + lang = m.group(1) or "" + code = m.group(2) + formatted = code_block_formatter(lang, code) + idx = len(code_blocks) + code_blocks.append(formatted) + return f"{_PLACEHOLDER_PREFIX}{idx}\x00" + + text = re.sub(r"```(\w*)\n?(.*?)```", _save_code_block, text, flags=re.DOTALL) + + # 2. Extract and protect inline code (`...`) + inline_codes: list[str] = [] + + def _save_inline(m: re.Match) -> str: + code = m.group(1) + formatted = inline_code_formatter(code) + idx = len(inline_codes) + inline_codes.append(formatted) + return f"{_INLINE_PREFIX}{idx}\x00" + + text = re.sub(r"`([^`]+)`", _save_inline, text) + + # 3. Optional escaping of remaining text + if escape_fn is not None: + text = escape_fn(text) + + # 4. Apply inline formatting rules + for pattern, replacement in inline_rules: + text = re.sub(pattern, replacement, text, flags=re.MULTILINE) + + # 5. Restore code blocks and inline code + for idx, html in enumerate(code_blocks): + text = text.replace(f"{_PLACEHOLDER_PREFIX}{idx}\x00", html) + for idx, code in enumerate(inline_codes): + text = text.replace(f"{_INLINE_PREFIX}{idx}\x00", code) + + return text + + +# ═════════════════════════════════════════════════════════════════════ +# Shared helpers +# ═════════════════════════════════════════════════════════════════════ + + +def _escape_html(text: str) -> str: + return text.replace("&", "&").replace("<", "<").replace(">", ">") + + +def _noop_escape(text: str) -> str: + return text + + +# ═════════════════════════════════════════════════════════════════════ +# HTML profile (Telegram, Email, Teams) +# ═════════════════════════════════════════════════════════════════════ + + +def _html_code_block(lang: str, code: str) -> str: + escaped = _escape_html(code) + if lang: + return f'
{escaped}
' + return f"
{escaped}
" + + +def _html_inline_code(code: str) -> str: + return f"{_escape_html(code)}" + + +_HTML_INLINE_RULES: list[InlineRule] = [ + # Headings → bold + (r"^#{1,6}\s+(.+)$", r"\1"), + # Blockquote markers (already escaped to >) + (r"^>\s?", ""), + # Links [text](url) → + (r"\[([^\]]+)\]\(([^)]+)\)", r'\1'), + # Bold **text** → + (r"\*\*(.+?)\*\*", r"\1"), + # Italic _text_ → + (r"(?\1"), + # Strikethrough ~~text~~ → + (r"~~(.+?)~~", r"\1"), + # List items + (r"^[\-\*]\s+", "• "), +] + + +# ═════════════════════════════════════════════════════════════════════ +# Slack mrkdwn profile +# ═════════════════════════════════════════════════════════════════════ + + +def _slack_code_block(lang: str, code: str) -> str: + return f"```\n{code}```" + + +def _slack_inline_code(code: str) -> str: + return f"`{code}`" + + +_SLACK_INLINE_RULES: list[InlineRule] = [ + (r"^#{1,6}\s+(.+)$", r"*\1*"), + (r"\[([^\]]+)\]\(([^)]+)\)", r"<\2|\1>"), + (r"\*\*(.+?)\*\*", r"*\1*"), + (r"~~(.+?)~~", r"~\1~"), + (r"^[\-\*]\s+", "• "), +] + + +# ═════════════════════════════════════════════════════════════════════ +# Discord profile (mostly passthrough, headings → bold) +# ═════════════════════════════════════════════════════════════════════ + + +def _discord_code_block(lang: str, code: str) -> str: + return f"```{lang}\n{code}```" + + +def _discord_inline_code(code: str) -> str: + return f"`{code}`" + + +_DISCORD_INLINE_RULES: list[InlineRule] = [ + (r"^#{1,6}\s+(.+)$", r"**\1**"), +] + + +# ═════════════════════════════════════════════════════════════════════ +# Plain text profile (strip all formatting) +# ═════════════════════════════════════════════════════════════════════ + + +def _plain_code_block(lang: str, code: str) -> str: + return code + + +def _plain_inline_code(code: str) -> str: + return code + + +_PLAIN_INLINE_RULES: list[InlineRule] = [ + (r"^#{1,6}\s+", ""), + (r"\[([^\]]+)\]\(([^)]+)\)", r"\1 (\2)"), + (r"\*\*(.+?)\*\*", r"\1"), + (r"(? str: + return f"```{lang}\n{code}```" + + +def _md_inline_code(code: str) -> str: + return f"`{code}`" + + +_MD_INLINE_RULES: list[InlineRule] = [] # passthrough — already Markdown + + +# ═════════════════════════════════════════════════════════════════════ +# Unified Formatter +# ═════════════════════════════════════════════════════════════════════ + + +class UnifiedFormatter: + """Converts internal Markdown to a target platform format. + + Instantiated once per channel based on its ``capabilities.format_type``. + """ + + _PROFILES: ClassVar[dict[str, dict]] = { + "html": { + "code_block_formatter": _html_code_block, + "inline_code_formatter": _html_inline_code, + "inline_rules": _HTML_INLINE_RULES, + "escape_fn": _escape_html, + }, + "slack_mrkdwn": { + "code_block_formatter": _slack_code_block, + "inline_code_formatter": _slack_inline_code, + "inline_rules": _SLACK_INLINE_RULES, + "escape_fn": None, + }, + "discord": { + "code_block_formatter": _discord_code_block, + "inline_code_formatter": _discord_inline_code, + "inline_rules": _DISCORD_INLINE_RULES, + "escape_fn": None, + }, + "markdown": { + "code_block_formatter": _md_code_block, + "inline_code_formatter": _md_inline_code, + "inline_rules": _MD_INLINE_RULES, + "escape_fn": None, + }, + "plain": { + "code_block_formatter": _plain_code_block, + "inline_code_formatter": _plain_inline_code, + "inline_rules": _PLAIN_INLINE_RULES, + "escape_fn": None, + }, + } + + def __init__(self, format_type: str = "plain") -> None: + self._format_type = format_type + profile = self._PROFILES.get(format_type) + if profile is None: + raise ValueError( + f"Unknown format_type: {format_type!r}. " + f"Available: {list(self._PROFILES.keys())}" + ) + self._profile = profile + + @property + def format_type(self) -> str: + return self._format_type + + def format(self, text: str) -> str: + """Convert Markdown *text* to the target format.""" + if not text: + return text + return convert_markdown(text, **self._profile) + + @classmethod + def for_channel(cls, format_type: str) -> UnifiedFormatter: + """Factory: create a formatter for the given format type.""" + return cls(format_type) diff --git a/EvoScientist/channels/imessage/__init__.py b/EvoScientist/channels/imessage/__init__.py new file mode 100644 index 0000000..aa7becd --- /dev/null +++ b/EvoScientist/channels/imessage/__init__.py @@ -0,0 +1,42 @@ +"""iMessage channel implementation for EvoScientist. + +Uses imsg CLI via JSON-RPC for real-time message streaming. + +Requirements: +- macOS only +- imsg CLI: brew install steipete/tap/imsg +- Full Disk Access permission +- Messages.app logged into iCloud +""" + +from ..channel_manager import _parse_csv, register_channel +from .channel_rpc import IMessageChannelRpc as IMessageChannel +from .channel_rpc import IMessageConfig +from .probe import ProbeResult, probe_imessage +from .targets import ( + IMessageService, + IMessageTarget, + normalize_e164, + normalize_handle, + parse_target, +) + +__all__ = [ + "IMessageChannel", + "IMessageConfig", + "IMessageService", + "IMessageTarget", + "ProbeResult", + "normalize_e164", + "normalize_handle", + "parse_target", + "probe_imessage", +] + + +def create_from_config(config) -> IMessageChannel: + allowed = _parse_csv(config.imessage_allowed_senders) + return IMessageChannel(IMessageConfig(allowed_senders=allowed)) + + +register_channel("imessage", create_from_config) diff --git a/EvoScientist/channels/imessage/channel_rpc.py b/EvoScientist/channels/imessage/channel_rpc.py new file mode 100644 index 0000000..f7e369b --- /dev/null +++ b/EvoScientist/channels/imessage/channel_rpc.py @@ -0,0 +1,416 @@ +"""iMessage channel using imsg JSON-RPC. + +This is an improved implementation that uses the imsg CLI +via JSON-RPC, similar to OpenClaw's approach. +""" + +import asyncio +import logging +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +from ..base import Channel, ChannelError, RawIncoming +from ..config import BaseChannelConfig +from .rpc_client import ImsgRpcClient, RpcNotification +from .targets import ( + ChatGuidTarget, + ChatIdentifierTarget, + ChatIdTarget, + normalize_handle, + parse_target, +) + +logger = logging.getLogger(__name__) + + +class _IMessageAllowListMiddleware: + """Custom allow-list middleware for iMessage's rich sender filtering. + + Supports chat_id/chat_guid matching, wildcard, and normalized + phone/email matching — logic that the generic AllowListMiddleware + does not cover. + """ + + def __init__(self, channel: "IMessageChannelRpc"): + self._channel = channel + + async def process_inbound(self, raw, context): + chat_id = raw.metadata.get("chat_id") + chat_guid = raw.metadata.get("chat_guid") + if not self._channel._is_sender_allowed(raw.sender_id, chat_id, chat_guid): + return None + return raw + + +@dataclass +class IMessageConfig(BaseChannelConfig): + """Configuration for iMessage channel.""" + + cli_path: str = "imsg" + db_path: str | None = None + text_chunk_limit: int = 4096 + service: str = "auto" # imessage, sms, or auto + region: str = "US" + + +class IMessageChannelRpc(Channel): + """iMessage channel using imsg JSON-RPC. + + This implementation uses the imsg CLI via JSON-RPC over stdio, + providing real-time message streaming instead of polling. + + Args: + config: Channel configuration + """ + + name = "imessage" + _ready_attrs = ("_client",) + + def __init__(self, config: IMessageConfig | None = None): + super().__init__(config or IMessageConfig()) + self._client: ImsgRpcClient | None = None + self._subscription_id: int | None = None + self._background_tasks: set[asyncio.Task] = set() + + # ── Pipeline overrides ──────────────────────────────────────── + + def _build_inbound_middlewares(self): + """Use iMessage-specific allow-list middleware. + + iMessage doesn't need MentionGating (always sets was_mentioned=True). + """ + from ..middleware import DedupMiddleware, GroupHistoryMiddleware + + middlewares = [] + middlewares.append(DedupMiddleware()) + middlewares.append(_IMessageAllowListMiddleware(self)) + if self.capabilities.groups: + middlewares.append(GroupHistoryMiddleware()) + return middlewares + + # ── Incoming message handling ───────────────────────────────── + + def _handle_notification(self, notification: RpcNotification) -> None: + """Handle incoming RPC notifications.""" + if notification.method == "message": + _task = asyncio.create_task(self._handle_message(notification.params)) + self._background_tasks.add(_task) + _task.add_done_callback(self._background_tasks.discard) + elif notification.method == "error": + logger.error(f"imsg error: {notification.params}") + + async def _handle_message(self, params: dict | None) -> None: + """Process incoming message notification.""" + if not params: + return + + message = params.get("message", {}) + if not message: + return + + # Skip messages from self + if message.get("is_from_me"): + return + + sender = message.get("sender", "").strip() + if not sender: + return + + text = message.get("text", "").strip() + + # Parse timestamp + timestamp = datetime.now() + if created_at := message.get("created_at"): + try: + timestamp = datetime.fromisoformat(created_at) + except ValueError: + pass + + # Build metadata + metadata = { + "chat_id": message.get("chat_id"), + "chat_guid": message.get("chat_guid"), + "is_group": message.get("is_group", False), + "chat_name": message.get("chat_name"), + } + + # Handle attachments if enabled + annotations: list[str] = [] + media_paths: list[str] = [] + _VOICE_EXTS = {".caf", ".m4a", ".aac", ".ogg", ".opus", ".mp3", ".amr"} + if self.config.include_attachments: + attachments = message.get("attachments", []) + for att in attachments: + # imsg CLI provides local file paths for attachments + file_path = att if isinstance(att, str) else att.get("path", "") + if not file_path: + annotations.append("[attachment: missing path]") + continue + att_path = Path(file_path) + is_voice = att_path.suffix.lower() in _VOICE_EXTS + media_label = "voice" if is_voice else "attachment" + if att_path.exists(): + fname = att_path.name + # Check file size before copying + from ..base import MAX_ATTACHMENT_BYTES + + if att_path.stat().st_size > MAX_ATTACHMENT_BYTES: + annotations.append( + f"[{media_label}: {fname} - too large " + f"({att_path.stat().st_size} bytes)]" + ) + else: + local = self._media_path(f"imsg_{fname}") + try: + import shutil + + shutil.copy2(str(att_path), str(local)) + media_paths.append(str(local)) + annotations.append(f"[{media_label}: {local}]") + except Exception as e: + logger.warning(f"Failed to copy iMessage attachment: {e}") + annotations.append( + f"[{media_label}: {fname} - copy failed]" + ) + else: + annotations.append(f"[{media_label}: {file_path} - not found]") + + if not text and not media_paths and not annotations: + return + + is_group = message.get("is_group", False) + + await self._enqueue_raw( + RawIncoming( + sender_id=sender, + chat_id=str(metadata.get("chat_id", sender)), + text=text, + media_files=media_paths, + content_annotations=annotations, + timestamp=timestamp, + message_id=str(message.get("id", "")), + metadata=metadata, + is_group=is_group, + was_mentioned=True, # iMessage has no mention concept + ) + ) + + # ── Sender filtering ────────────────────────────────────────── + + def _is_sender_allowed( + self, + sender: str, + chat_id: int | None = None, + chat_guid: str | None = None, + ) -> bool: + """Check if sender is in allowed list. + + Supports: + - Wildcard "*" to allow all + - chat_id:123 to match by chat ID + - chat_guid:abc to match by chat GUID + - Normalized phone/email matching + """ + if not self.config.allowed_senders: + return True + + # Wildcard allows all + if "*" in self.config.allowed_senders: + return True + + sender_normalized = normalize_handle(sender) + + for entry in self.config.allowed_senders: + entry = entry.strip() + if not entry: + continue + + lower = entry.lower() + + # Check chat_id match + if lower.startswith("chat_id:") or lower.startswith("chatid:"): + if chat_id is not None: + try: + allowed_id = int(entry.split(":", 1)[1].strip()) + if allowed_id == chat_id: + return True + except ValueError: + pass + continue + + # Check chat_guid match + if lower.startswith("chat_guid:") or lower.startswith("chatguid:"): + if chat_guid: + allowed_guid = entry.split(":", 1)[1].strip() + if allowed_guid == chat_guid: + return True + continue + + # Normalize and compare handle + entry_normalized = normalize_handle(entry) + if entry_normalized == sender_normalized: + return True + + return False + + def _normalize_sender(self, sender: str) -> str: + """Normalize a sender identifier.""" + return sender if sender.startswith("chat") else normalize_handle(sender) + + def add_allowed_sender(self, sender: str) -> None: + """Add a sender to the allowed list.""" + normalized = self._normalize_sender(sender) + if self.config.allowed_senders is None: + self.config.allowed_senders = set() + self.config.allowed_senders.add(normalized) + logger.info(f"Added allowed sender: {normalized}") + + def remove_allowed_sender(self, sender: str) -> None: + """Remove a sender from the allowed list.""" + normalized = self._normalize_sender(sender) + if self.config.allowed_senders: + self.config.allowed_senders.discard(normalized) + logger.info(f"Removed allowed sender: {normalized}") + + def clear_allowed_senders(self) -> None: + """Clear allowed list (allow all).""" + self.config.allowed_senders = None + logger.info("Cleared allowed senders (allowing all)") + + def list_allowed_senders(self) -> list[str]: + """Get current allowed senders.""" + return list(self.config.allowed_senders) if self.config.allowed_senders else [] + + # ── Lifecycle ───────────────────────────────────────────────── + + async def start(self) -> None: + """Initialize and start the channel.""" + logger.info("Starting iMessage channel (RPC)...") + + self._client = ImsgRpcClient( + cli_path=self.config.cli_path, + db_path=self.config.db_path, + on_notification=self._handle_notification, + ) + + try: + await self._client.start() + except Exception as e: + raise ChannelError(f"Failed to start imsg: {e}") from e + + # Subscribe to message events + try: + result = await self._client.request( + "watch.subscribe", + {"attachments": self.config.include_attachments}, + ) + self._subscription_id = result.get("subscription") + except Exception as e: + await self._client.stop() + raise ChannelError(f"Failed to subscribe: {e}") from e + + self._running = True + logger.info("iMessage channel started") + + async def _cleanup(self) -> None: + if self._client and self._subscription_id: + try: + await self._client.request( + "watch.unsubscribe", + {"subscription": self._subscription_id}, + ) + except Exception: + pass + if self._client: + await self._client.stop() + self._client = None + logger.info("iMessage channel stopped") + + # ── Send (template method overrides) ────────────────────────── + + def _resolve_target(self, chat_id: str | None, metadata: dict | None) -> dict: + """Resolve send target from metadata or chat_id string.""" + meta = metadata or {} + for key in ("chat_id", "chat_guid", "chat_identifier"): + if meta.get(key): + return {key: meta[key]} + if chat_id: + try: + target = parse_target(chat_id) + if isinstance(target, ChatIdTarget): + return {"chat_id": target.chat_id} + elif isinstance(target, ChatGuidTarget): + return {"chat_guid": target.chat_guid} + elif isinstance(target, ChatIdentifierTarget): + return {"chat_identifier": target.chat_identifier} + else: + return {"to": target.to, "service": target.service.value} + except ValueError: + return {"to": chat_id} + return {} + + async def _send_chunk(self, chat_id, formatted_text, raw_text, reply_to, metadata): + """Send a single text chunk via iMessage RPC.""" + if not self._client: + raise RuntimeError("iMessage client not running") + + params: dict = { + "text": formatted_text, + "service": self.config.service, + "region": self.config.region, + } + params.update(self._resolve_target(chat_id, metadata)) + + if reply_to: + params["reply_to"] = reply_to + + await self._client.request("send", params) + + # ── Retry logic (override base) ─────────────────────────────── + + def _format_chunk(self, text: str) -> str: + """iMessage uses plain text; no formatting conversion needed.""" + return text + + def _extract_retry_after(self, exc: Exception) -> float | None: + """iMessage-specific retry logic. + + RPC errors (e.g. AppleScript failures) are generally not + retryable. Transient connection issues get a short retry. + """ + msg = str(exc).lower() + if "not found" in msg or "applescript" in msg or "permission" in msg: + return None # not retryable + if "timeout" in msg or "connection" in msg: + return 1.0 + return None # default: don't retry RPC errors + + async def _send_media_impl( + self, + recipient: str, + file_path: str, + caption: str = "", + metadata: dict | None = None, + ) -> bool: + """Send a media file via iMessage.""" + if not self._client: + return False + + params: dict = { + "file": file_path, + "service": self.config.service, + "region": self.config.region, + } + + if caption: + params["text"] = caption + + target = self._resolve_target(recipient, metadata) + if not target: + logger.error("Cannot send media: no recipient") + return False + params.update(target) + + await self._client.request("send", params) + return True diff --git a/EvoScientist/channels/imessage/probe.py b/EvoScientist/channels/imessage/probe.py new file mode 100644 index 0000000..4c8e3a0 --- /dev/null +++ b/EvoScientist/channels/imessage/probe.py @@ -0,0 +1,109 @@ +"""iMessage environment probe/diagnostics. + +Provides utilities to detect and verify the imsg CLI environment. +""" + +import asyncio +import shutil +from dataclasses import dataclass + + +@dataclass +class ProbeResult: + """Result of iMessage environment probe.""" + + available: bool = False + cli_path: str | None = None + cli_version: str | None = None + rpc_supported: bool = False + error: str | None = None + + +def find_cli(cli_path: str = "imsg") -> str | None: + """Find the imsg CLI binary. + + Args: + cli_path: Path or command name to search + + Returns: + Full path to CLI or None if not found + """ + return shutil.which(cli_path) + + +async def get_cli_version(cli_path: str) -> str | None: + """Get the imsg CLI version. + + Args: + cli_path: Path to CLI binary + + Returns: + Version string or None + """ + try: + proc = await asyncio.create_subprocess_exec( + cli_path, + "--version", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=5.0) + return stdout.decode().strip() or None + except Exception: + return None + + +async def check_rpc_support(cli_path: str) -> bool: + """Check if CLI supports RPC mode. + + Args: + cli_path: Path to CLI binary + + Returns: + True if RPC is supported + """ + try: + proc = await asyncio.create_subprocess_exec( + cli_path, + "rpc", + "--help", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await asyncio.wait_for(proc.communicate(), timeout=5.0) + return proc.returncode == 0 + except Exception: + return False + + +async def probe_imessage( + cli_path: str = "imsg", + timeout_ms: int = 10000, +) -> ProbeResult: + """Probe the iMessage environment. + + Args: + cli_path: Path or name of imsg CLI + timeout_ms: Timeout in milliseconds + + Returns: + ProbeResult with environment details + """ + result = ProbeResult() + + # Find CLI + found_path = find_cli(cli_path) + if not found_path: + result.error = f"imsg CLI not found: {cli_path}" + return result + + result.cli_path = found_path + result.available = True + + # Get version + result.cli_version = await get_cli_version(found_path) + + # Check RPC support + result.rpc_supported = await check_rpc_support(found_path) + + return result diff --git a/EvoScientist/channels/imessage/rpc_client.py b/EvoScientist/channels/imessage/rpc_client.py new file mode 100644 index 0000000..9e18091 --- /dev/null +++ b/EvoScientist/channels/imessage/rpc_client.py @@ -0,0 +1,246 @@ +"""JSON-RPC client for imsg CLI. + +Communicates with the imsg CLI via JSON-RPC over stdio, +similar to OpenClaw's approach. +""" + +import asyncio +import json +import logging +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +logger = logging.getLogger(__name__) + + +@dataclass +class RpcError: + """RPC error response.""" + + code: int | None = None + message: str | None = None + data: Any = None + + +@dataclass +class RpcNotification: + """RPC notification (no id, server-initiated).""" + + method: str + params: Any = None + + +class ImsgRpcClient: + """JSON-RPC client for imsg CLI. + + Spawns `imsg rpc` as a subprocess and communicates via stdin/stdout. + + Example: + client = ImsgRpcClient(cli_path="/usr/local/bin/imsg") + await client.start() + result = await client.request("send", {"to": "+1234", "text": "Hello"}) + await client.stop() + """ + + def __init__( + self, + cli_path: str = "imsg", + db_path: str | None = None, + on_notification: Callable[[RpcNotification], None] | None = None, + ): + self.cli_path = cli_path + self.db_path = db_path + self.on_notification = on_notification + + self._process: asyncio.subprocess.Process | None = None + self._next_id = 1 + self._pending: dict[int, asyncio.Future] = {} + self._reader_task: asyncio.Task | None = None + self._stderr_task: asyncio.Task | None = None + self._closed = asyncio.Event() + + async def start(self) -> None: + """Start the imsg rpc subprocess.""" + if self._process is not None: + return + + args = [self.cli_path, "rpc"] + if self.db_path: + args.extend(["--db", self.db_path]) + + self._process = await asyncio.create_subprocess_exec( + *args, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + self._reader_task = asyncio.create_task(self._read_loop()) + self._stderr_task = asyncio.create_task(self._stderr_loop()) + logger.info(f"Started imsg rpc (pid={self._process.pid})") + + async def stop(self) -> None: + """Stop the imsg rpc subprocess.""" + if self._process is None: + return + + if self._process.stdin: + self._process.stdin.close() + + if self._reader_task: + self._reader_task.cancel() + try: + await self._reader_task + except asyncio.CancelledError: + pass + + if self._stderr_task: + self._stderr_task.cancel() + try: + await self._stderr_task + except asyncio.CancelledError: + pass + + try: + self._process.terminate() + await asyncio.wait_for(self._process.wait(), timeout=2.0) + except TimeoutError: + self._process.kill() + await self._process.wait() + + self._fail_all_pending(Exception("RPC client stopped")) + self._process = None + self._closed.set() + logger.info("Stopped imsg rpc") + + async def wait_closed(self) -> None: + """Wait for the client to close.""" + await self._closed.wait() + + async def request( + self, + method: str, + params: dict[str, Any] | None = None, + timeout: float = 10.0, + ) -> Any: + """Send a JSON-RPC request and wait for response. + + Args: + method: RPC method name + params: Method parameters + timeout: Request timeout in seconds + + Returns: + The result from the RPC response + + Raises: + Exception: If request fails or times out + """ + if self._process is None or self._process.stdin is None: + raise Exception("RPC client not running") + + request_id = self._next_id + self._next_id += 1 + + payload = { + "jsonrpc": "2.0", + "id": request_id, + "method": method, + "params": params or {}, + } + + future: asyncio.Future = asyncio.get_event_loop().create_future() + self._pending[request_id] = future + + line = json.dumps(payload) + "\n" + self._process.stdin.write(line.encode()) + await self._process.stdin.drain() + + try: + return await asyncio.wait_for(future, timeout=timeout) + except TimeoutError: + self._pending.pop(request_id, None) + raise Exception(f"RPC request timeout: {method}") from None + + async def _read_loop(self) -> None: + """Read and process responses from stdout.""" + if self._process is None or self._process.stdout is None: + return + + while True: + try: + line = await self._process.stdout.readline() + if not line: + break + + self._handle_line(line.decode().strip()) + except asyncio.CancelledError: + break + except Exception as e: + logger.error(f"Error reading from imsg: {e}") + break + + self._fail_all_pending(Exception("RPC connection closed")) + self._closed.set() + + async def _stderr_loop(self) -> None: + """Log stderr output.""" + if self._process is None or self._process.stderr is None: + return + + while True: + try: + line = await self._process.stderr.readline() + if not line: + break + logger.warning(f"imsg stderr: {line.decode().strip()}") + except asyncio.CancelledError: + break + except Exception: + break + + def _handle_line(self, line: str) -> None: + """Handle a single JSON-RPC response line.""" + if not line: + return + + try: + data = json.loads(line) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse RPC response: {e}") + return + + # Check if it's a response (has id) + if "id" in data and data["id"] is not None: + request_id = data["id"] + future = self._pending.pop(request_id, None) + if future is None: + return + + if data.get("error"): + error = data["error"] + msg = error.get("message", "RPC error") + future.set_exception(Exception(msg)) + else: + future.set_result(data.get("result")) + return + + # It's a notification + if "method" in data: + notification = RpcNotification( + method=data["method"], + params=data.get("params"), + ) + if self.on_notification: + try: + self.on_notification(notification) + except Exception as e: + logger.error(f"Notification handler error: {e}") + + def _fail_all_pending(self, error: Exception) -> None: + """Fail all pending requests with an error.""" + for future in self._pending.values(): + if not future.done(): + future.set_exception(error) + self._pending.clear() diff --git a/EvoScientist/channels/imessage/serve.py b/EvoScientist/channels/imessage/serve.py new file mode 100644 index 0000000..30966b9 --- /dev/null +++ b/EvoScientist/channels/imessage/serve.py @@ -0,0 +1,89 @@ +"""iMessage channel server. + +Standalone script to run the iMessage channel with CLI options. + +Usage: + python -m EvoScientist.channels.imessage.serve [OPTIONS] + +Examples: + # Allow all senders (default) + python -m EvoScientist.channels.imessage.serve + + # Only allow specific senders + python -m EvoScientist.channels.imessage.serve --allow +1234567890 --allow user@example.com + + # Custom imsg path + python -m EvoScientist.channels.imessage.serve --cli-path /usr/local/bin/imsg +""" + +import argparse +import logging + +from ...logging_config import configure_logging_from_settings +from ..bus import MessageBus +from ..standalone import run_standalone +from . import IMessageChannel, IMessageConfig + +logger = logging.getLogger(__name__) + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="iMessage channel server", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--allow", + action="append", + dest="allowed_senders", + help="Allowed sender (phone/email). Can be used multiple times.", + ) + parser.add_argument( + "--cli-path", + default="imsg", + help="Path to imsg CLI (default: imsg)", + ) + parser.add_argument( + "--db-path", + help="Path to Messages database", + ) + parser.add_argument( + "--attachments", + action="store_true", + help="Include attachments in messages", + ) + parser.add_argument( + "--agent", + action="store_true", + help="Use EvoScientist agent as handler (default: echo)", + ) + parser.add_argument( + "--thinking", + action="store_true", + help="Send thinking content as intermediate messages (requires --agent)", + ) + return parser.parse_args() + + +def main(): + """Entry point.""" + configure_logging_from_settings(default_level=logging.INFO) + args = parse_args() + + config = IMessageConfig( + cli_path=args.cli_path, + db_path=args.db_path, + allowed_senders=list(args.allowed_senders) if args.allowed_senders else [], + include_attachments=args.attachments, + ) + + send_thinking = args.thinking and args.agent + bus = MessageBus() + channel = IMessageChannel(config) + + run_standalone(channel, bus, use_agent=args.agent, send_thinking=send_thinking) + + +if __name__ == "__main__": + main() diff --git a/EvoScientist/channels/imessage/targets.py b/EvoScientist/channels/imessage/targets.py new file mode 100644 index 0000000..02f9fe0 --- /dev/null +++ b/EvoScientist/channels/imessage/targets.py @@ -0,0 +1,238 @@ +"""iMessage target parsing and normalization. + +Provides utilities for parsing iMessage targets and normalizing +phone numbers and email addresses, similar to OpenClaw's approach. +""" + +import re +from dataclasses import dataclass +from enum import Enum + + +class IMessageService(Enum): + """iMessage service type.""" + + IMESSAGE = "imessage" + SMS = "sms" + AUTO = "auto" + + +@dataclass +class ChatIdTarget: + """Target by chat ID.""" + + kind: str = "chat_id" + chat_id: int = 0 + + +@dataclass +class ChatGuidTarget: + """Target by chat GUID.""" + + kind: str = "chat_guid" + chat_guid: str = "" + + +@dataclass +class ChatIdentifierTarget: + """Target by chat identifier.""" + + kind: str = "chat_identifier" + chat_identifier: str = "" + + +@dataclass +class HandleTarget: + """Target by handle (phone/email).""" + + kind: str = "handle" + to: str = "" + service: IMessageService = IMessageService.AUTO + + +IMessageTarget = ChatIdTarget | ChatGuidTarget | ChatIdentifierTarget | HandleTarget + + +# Prefix constants +CHAT_ID_PREFIXES = ["chat_id:", "chatid:", "chat:"] +CHAT_GUID_PREFIXES = ["chat_guid:", "chatguid:", "guid:"] +CHAT_IDENTIFIER_PREFIXES = ["chat_identifier:", "chatidentifier:", "chatident:"] +SERVICE_PREFIXES = [ + ("imessage:", IMessageService.IMESSAGE), + ("sms:", IMessageService.SMS), + ("auto:", IMessageService.AUTO), +] + + +def normalize_e164(phone: str) -> str | None: + """Normalize phone number to E.164 format. + + Args: + phone: Raw phone number string + + Returns: + Normalized E.164 format or None if invalid + """ + # Remove all non-digit characters except leading + + cleaned = re.sub(r"[^\d+]", "", phone) + + if not cleaned: + return None + + # Already has + prefix + if cleaned.startswith("+"): + digits = cleaned[1:] + if len(digits) >= 10 and len(digits) <= 15: + return cleaned + return None + + # US/Canada number without country code + if len(cleaned) == 10: + return f"+1{cleaned}" + + # Has country code + if len(cleaned) >= 11 and len(cleaned) <= 15: + return f"+{cleaned}" + + return None + + +def normalize_handle(raw: str) -> str: + """Normalize an iMessage handle (phone or email). + + Args: + raw: Raw handle string + + Returns: + Normalized handle + """ + trimmed = raw.strip() + if not trimmed: + return "" + + lowered = trimmed.lower() + + # Strip service prefixes + for prefix, _ in SERVICE_PREFIXES: + if lowered.startswith(prefix): + return normalize_handle(trimmed[len(prefix) :]) + + # Normalize chat_id/chat_guid/chat_identifier prefixes + for prefix in CHAT_ID_PREFIXES: + if lowered.startswith(prefix): + value = trimmed[len(prefix) :].strip() + return f"chat_id:{value}" + + for prefix in CHAT_GUID_PREFIXES: + if lowered.startswith(prefix): + value = trimmed[len(prefix) :].strip() + return f"chat_guid:{value}" + + for prefix in CHAT_IDENTIFIER_PREFIXES: + if lowered.startswith(prefix): + value = trimmed[len(prefix) :].strip() + return f"chat_identifier:{value}" + + # Email - lowercase + if "@" in trimmed: + return trimmed.lower() + + # Phone number - normalize to E.164 + normalized = normalize_e164(trimmed) + if normalized: + return normalized + + # Fallback: remove whitespace + return re.sub(r"\s+", "", trimmed) + + +def parse_target(raw: str) -> IMessageTarget: + """Parse an iMessage target string. + + Supports formats: + - chat_id:123 + - chat_guid:abc-def + - chat_identifier:iMessage;+;chat123 + - imessage:+1234567890 + - sms:+1234567890 + - +1234567890 (auto service) + - email@example.com (auto service) + + Args: + raw: Raw target string + + Returns: + Parsed IMessageTarget + + Raises: + ValueError: If target is invalid + """ + trimmed = raw.strip() + if not trimmed: + raise ValueError("iMessage target is required") + + lower = trimmed.lower() + + # Check service prefixes first + for prefix, service in SERVICE_PREFIXES: + if lower.startswith(prefix): + remainder = trimmed[len(prefix) :].strip() + if not remainder: + raise ValueError(f"{prefix} target is required") + + remainder_lower = remainder.lower() + + # Check if remainder is a chat target + is_chat = any( + remainder_lower.startswith(p) + for p in CHAT_ID_PREFIXES + + CHAT_GUID_PREFIXES + + CHAT_IDENTIFIER_PREFIXES + ) + if is_chat: + return parse_target(remainder) + + return HandleTarget(to=remainder, service=service) + + # Check chat_id prefixes + for prefix in CHAT_ID_PREFIXES: + if lower.startswith(prefix): + value = trimmed[len(prefix) :].strip() + try: + chat_id = int(value) + return ChatIdTarget(chat_id=chat_id) + except ValueError as e: + raise ValueError(f"Invalid chat_id: {value}") from e + + # Check chat_guid prefixes + for prefix in CHAT_GUID_PREFIXES: + if lower.startswith(prefix): + value = trimmed[len(prefix) :].strip() + if not value: + raise ValueError("chat_guid is required") + return ChatGuidTarget(chat_guid=value) + + # Check chat_identifier prefixes + for prefix in CHAT_IDENTIFIER_PREFIXES: + if lower.startswith(prefix): + value = trimmed[len(prefix) :].strip() + if not value: + raise ValueError("chat_identifier is required") + return ChatIdentifierTarget(chat_identifier=value) + + # Default: handle with auto service + return HandleTarget(to=trimmed, service=IMessageService.AUTO) + + +def format_chat_target(chat_id: int | None) -> str: + """Format a chat ID as a target string. + + Args: + chat_id: Chat ID number + + Returns: + Formatted target string or empty string + """ + if chat_id is None: + return "" + return f"chat_id:{chat_id}" diff --git a/EvoScientist/channels/middleware.py b/EvoScientist/channels/middleware.py new file mode 100644 index 0000000..47de4fc --- /dev/null +++ b/EvoScientist/channels/middleware.py @@ -0,0 +1,989 @@ +"""Composable message processing middleware. + +Each middleware is a standalone class that can be composed into a pipeline. +They extract logic that was previously baked into the Channel base class, +making it reusable across both legacy and plugin-based channels. + +Also contains the supporting data structures (DedupCache, GroupHistoryBuffer, +TypingManager, PairingManager) that were previously in separate files. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import logging +import random +import time +from collections import OrderedDict, deque +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any + +from .base import RawIncoming +from .bus.events import InboundMessage, OutboundMessage +from .debug import emit_debug_event_if + +_logger = logging.getLogger(__name__) + + +# ── Task cancellation helper ───────────────────────────────────────── + + +async def _cancel_task(task: asyncio.Task) -> None: + """Cancel an asyncio task and await its completion. + + Suppresses ``CancelledError`` from the cancelled *task* but re-raises + if the **current** task was itself cancelled (to avoid swallowing an + outer cancellation signal — required for correct behavior on + Python 3.12+ where ``_must_cancel`` no longer auto-re-delivers). + """ + task.cancel() + try: + await task + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None and current.cancelling() > 0: + raise + except Exception: + pass # Already logged elsewhere; prevent unhandled propagation + + +# ═══════════════════════════════════════════════════════════════════════ +# Supporting data structures +# ═══════════════════════════════════════════════════════════════════════ + + +# ── Dedup cache ────────────────────────────────────────────────────── + +_DEDUP_MAX = 1000 +_DEDUP_TRIM = 500 +_DEDUP_TTL = 3600 # 1 hour + + +class DedupCache: + """Bounded ordered cache with TTL for detecting duplicate message IDs. + + Entries expire after *ttl_seconds* and are pruned lazily on each + lookup. When the cache exceeds *max_size* entries it is trimmed + down to *trim_to* by evicting the oldest entries. Accessed entries + are moved to the end (LRU behavior). + """ + + def __init__( + self, + max_size: int = _DEDUP_MAX, + trim_to: int = _DEDUP_TRIM, + ttl_seconds: float = _DEDUP_TTL, + ) -> None: + self._seen: OrderedDict[str, float] = OrderedDict() + self._max = max_size + self._trim = trim_to + self._ttl = ttl_seconds + + # ── public API ────────────────────────────────────────────────── + + def is_duplicate(self, msg_id: str) -> bool: + """Return ``True`` if *msg_id* has been seen before. + + First-time IDs are recorded and ``False`` is returned. + Empty / falsy IDs are never considered duplicates. + Expired entries are pruned before the check. + """ + if not msg_id: + return False + + self._prune() + + if msg_id in self._seen: + # LRU: refresh position and timestamp + self._seen.move_to_end(msg_id) + self._seen[msg_id] = time.monotonic() + return True + + self._seen[msg_id] = time.monotonic() + if len(self._seen) > self._max: + while len(self._seen) > self._trim: + self._seen.popitem(last=False) + return False + + def clear(self) -> None: + """Remove all entries.""" + self._seen.clear() + + @property + def size(self) -> int: + """Number of entries currently in the cache.""" + return len(self._seen) + + # ── internal ──────────────────────────────────────────────────── + + def _prune(self) -> None: + """Remove entries older than *ttl_seconds*.""" + cutoff = time.monotonic() - self._ttl + # OrderedDict is insertion-ordered; oldest entries are first. + while self._seen: + _key, ts = next(iter(self._seen.items())) + if ts > cutoff: + break + self._seen.popitem(last=False) + + +# ── Group history buffer ───────────────────────────────────────────── + + +@dataclass +class HistoryEntry: + sender_id: str + text: str + timestamp: float + message_id: str = "" + + +class GroupHistoryBuffer: + """Per-chat circular buffer of recent messages.""" + + def __init__(self, max_per_chat: int = 50, max_age_seconds: int = 3600): + self._buffers: dict[str, deque[HistoryEntry]] = {} + self._max = max_per_chat + self._max_age = max_age_seconds + + def add(self, chat_id: str, entry: HistoryEntry) -> None: + """Add a message to the chat's history buffer.""" + if chat_id not in self._buffers: + self._buffers[chat_id] = deque(maxlen=self._max) + self._buffers[chat_id].append(entry) + + def get_recent(self, chat_id: str, limit: int = 20) -> list[HistoryEntry]: + """Get recent messages for context injection, excluding expired ones.""" + buf = self._buffers.get(chat_id) + if not buf: + return [] + now = time.monotonic() + recent = [e for e in buf if now - e.timestamp < self._max_age] + return recent[-limit:] + + def format_context(self, chat_id: str, limit: int = 20) -> str: + """Format recent messages as context block for the agent.""" + entries = self.get_recent(chat_id, limit) + if not entries: + return "" + lines = ["[Chat messages since your last reply - for context]"] + for e in entries: + lines.append(f"[from: {e.sender_id}] {e.text}") + lines.append("[/Chat context]") + return "\n".join(lines) + + def clear(self, chat_id: str) -> None: + """Clear history for a chat (e.g., after the bot replies).""" + self._buffers.pop(chat_id, None) + + +# ── Typing indicator manager ───────────────────────────────────────── + + +class TypingManager: + """Manages background typing-indicator loops per chat_id. + + Args: + send_action: Async callable that sends a single typing indicator + for a given chat_id. + interval: Seconds between typing indicator sends. + """ + + def __init__( + self, + send_action: Callable[[str], Awaitable[None]], + interval: float = 5.0, + debug_trace: bool = False, + channel_name: str = "unknown", + ) -> None: + self._send_action = send_action + self._interval = interval + self._tasks: dict[str, asyncio.Task] = {} + self._debug_trace = debug_trace + self._channel_name = channel_name + + async def start(self, chat_id: str) -> None: + """Start a background typing-indicator loop for *chat_id*.""" + await self.stop(chat_id) + + async def _loop() -> None: + while True: + try: + await self._send_action(chat_id) + except Exception as exc: + _trace_named_event( + "typing_error", + enabled=self._debug_trace, + channel_name=self._channel_name, + chat_id=chat_id, + error=str(exc), + ) + await asyncio.sleep(self._interval) + + self._tasks[chat_id] = asyncio.create_task(_loop()) + + async def stop(self, chat_id: str) -> None: + """Cancel the typing-indicator loop for *chat_id*.""" + task = self._tasks.pop(chat_id, None) + if task: + await _cancel_task(task) + + async def stop_all(self) -> None: + """Cancel all active typing-indicator loops.""" + for cid in list(self._tasks): + await self.stop(cid) + + @property + def active_chats(self) -> list[str]: + """Return chat_ids with active typing loops.""" + return list(self._tasks) + + +# ── Pairing manager ───────────────────────────────────────────────── + + +@dataclass +class PairingRequest: + sender_id: str + channel: str + code: str + created_at: float + approved: bool = False + + +class PairingManager: + """Manages DM pairing codes for channel access control.""" + + CODE_EXPIRY = 3600 # 1 hour + MAX_PENDING = 50 # max pending requests + + def __init__(self): + self._pending: dict[str, PairingRequest] = {} # code -> request + self._approved: set[str] = set() # "channel:sender_id" keys + + def is_approved(self, channel: str, sender_id: str) -> bool: + """Check if sender is already approved.""" + return f"{channel}:{sender_id}" in self._approved + + def request_pairing(self, channel: str, sender_id: str) -> str: + """Generate a pairing code for a new sender. Returns the code.""" + # Check if already has pending request + for code, req in list(self._pending.items()): + if req.sender_id == sender_id and req.channel == channel: + if time.monotonic() - req.created_at < self.CODE_EXPIRY: + return code # return existing code + else: + del self._pending[code] + break + + # Cleanup expired + self._cleanup_expired() + + # Generate new code + code = f"{random.randint(100000, 999999)}" + while code in self._pending: + code = f"{random.randint(100000, 999999)}" + + self._pending[code] = PairingRequest( + sender_id=sender_id, + channel=channel, + code=code, + created_at=time.monotonic(), + ) + _logger.info(f"Pairing code {code} generated for {channel}:{sender_id}") + return code + + def approve(self, code: str) -> tuple[bool, str]: + """Approve a pairing code. Returns (success, message).""" + req = self._pending.get(code) + if not req: + return False, f"Unknown code: {code}" + if time.monotonic() - req.created_at > self.CODE_EXPIRY: + del self._pending[code] + return False, f"Code {code} expired" + + key = f"{req.channel}:{req.sender_id}" + self._approved.add(key) + del self._pending[code] + _logger.info(f"Approved pairing for {key}") + return True, f"Approved {req.sender_id} on {req.channel}" + + def reject(self, code: str) -> tuple[bool, str]: + """Reject a pairing code.""" + if code in self._pending: + del self._pending[code] + return True, f"Rejected code {code}" + return False, f"Unknown code: {code}" + + def list_pending(self) -> list[PairingRequest]: + """List all pending (non-expired) requests.""" + self._cleanup_expired() + return list(self._pending.values()) + + def _cleanup_expired(self): + now = time.monotonic() + expired = [ + c for c, r in self._pending.items() if now - r.created_at > self.CODE_EXPIRY + ] + for c in expired: + del self._pending[c] + + +# ═══════════════════════════════════════════════════════════════════════ +# Middleware classes +# ═══════════════════════════════════════════════════════════════════════ + + +# ── Inbound middleware base ────────────────────────────────────────── + + +class InboundMiddleware: + """Base class for inbound message processing middleware.""" + + async def process_inbound( + self, + raw: RawIncoming, + context: dict[str, Any], + ) -> RawIncoming | None: + """Process an inbound raw message. + + Return the (possibly modified) RawIncoming to continue the + pipeline, or ``None`` to drop the message. + """ + return raw + + +class OutboundMiddlewareBase: + """Base class for outbound message processing middleware.""" + + async def process_outbound( + self, + message: OutboundMessage, + context: dict[str, Any], + ) -> OutboundMessage | None: + """Process an outbound message. + + Return the (possibly modified) OutboundMessage to continue, + or ``None`` to drop it. + """ + return message + + +def _debug_trace_enabled(context: dict[str, Any]) -> bool: + """Check whether channel-level debug tracing is enabled for this message.""" + channel = context.get("channel") + if channel is None: + return False + return channel.is_debug_trace_enabled() + + +def _ctx_channel_name(context: dict[str, Any]) -> str: + """Extract the channel name from middleware context.""" + ch = context.get("channel") + return getattr(ch, "name", "unknown") if ch else "unknown" + + +def _trace_context_event( + context: dict[str, Any], + event: str, + **fields: Any, +) -> None: + """Emit a middleware trace event using the shared channel context.""" + emit_debug_event_if( + _logger, + event, + _debug_trace_enabled(context), + channel=_ctx_channel_name(context), + **fields, + ) + + +def _trace_named_event( + event: str, + *, + enabled: bool, + channel_name: str, + **fields: Any, +) -> None: + """Emit a trace event for helpers that already carry trace state.""" + emit_debug_event_if( + _logger, + event, + enabled, + channel=channel_name, + **fields, + ) + + +# ── Dedup ──────────────────────────────────────────────────────────── + + +class DedupMiddleware(InboundMiddleware): + """Message deduplication using a bounded TTL cache.""" + + def __init__( + self, + max_size: int = 1000, + trim_to: int = 500, + ttl_seconds: float = 3600.0, + ) -> None: + self._cache = DedupCache( + max_size=max_size, + trim_to=trim_to, + ttl_seconds=ttl_seconds, + ) + + async def process_inbound( + self, + raw: RawIncoming, + context: dict[str, Any], + ) -> RawIncoming | None: + if raw.message_id and self._cache.is_duplicate(raw.message_id): + _trace_context_event( + context, + "middleware_dedup_drop", + message_id=raw.message_id, + sender_id=raw.sender_id, + ) + return None + return raw + + +# ── Debounce ───────────────────────────────────────────────────────── + + +class DebounceMiddleware: + """Per-sender message batching with configurable timing. + + This middleware collects messages from the same sender and merges + them after a debounce delay. It does not follow the simple + process_inbound pattern because it needs to buffer across calls. + + Usage: call ``submit()`` for each message; merged results are + delivered via the ``on_ready`` callback. + """ + + def __init__( + self, + *, + initial_debounce: float = 2.0, + debounce_step: float = 0.5, + max_debounce: float = 5.0, + on_ready: Callable[[InboundMessage], Any] | None = None, + ) -> None: + self.initial_debounce = initial_debounce + self.debounce_step = debounce_step + self.max_debounce = max_debounce + self.on_ready = on_ready + + self._buffers: dict[str, list[str]] = {} + self._metadata: dict[str, dict] = {} + self._media: dict[str, list[str]] = {} + self._message_ids: dict[str, str] = {} + self._tasks: dict[str, asyncio.Task] = {} + self._channel_name: str = "" + + def set_channel_name(self, name: str) -> None: + self._channel_name = name + + async def submit(self, msg: InboundMessage) -> None: + """Buffer *msg* and schedule flush after debounce delay.""" + sender = msg.sender_id + + if sender not in self._buffers: + self._buffers[sender] = [] + self._metadata[sender] = msg.metadata + self._media[sender] = [] + self._buffers[sender].append(msg.content) + if msg.message_id: + self._message_ids[sender] = msg.message_id + if msg.media: + self._media[sender].extend(msg.media) + + if sender in self._tasks: + self._tasks[sender].cancel() + + count = len(self._buffers[sender]) + wait = min( + self.initial_debounce + (count - 1) * self.debounce_step, + self.max_debounce, + ) + + async def _flush(_s: str = sender, _w: float = wait) -> None: + await asyncio.sleep(_w) + await self._flush_sender(_s) + + self._tasks[sender] = asyncio.create_task(_flush()) + + async def _flush_sender(self, sender: str) -> None: + messages = self._buffers.pop(sender, []) + metadata = self._metadata.pop(sender, None) + media = self._media.pop(sender, []) + message_id = self._message_ids.pop(sender, "") + self._tasks.pop(sender, None) + if not messages: + return + + merged = "\n".join(messages) + chat_id = (metadata or {}).get("chat_id", sender) + inbound = InboundMessage( + channel=self._channel_name, + sender_id=sender, + chat_id=str(chat_id), + content=merged, + media=media, + metadata=metadata or {}, + message_id=message_id, + ) + if self.on_ready: + await self.on_ready(inbound) + + async def cancel_all(self) -> None: + """Cancel all pending debounce tasks and await their completion.""" + tasks = list(self._tasks.values()) + self._tasks.clear() + for task in tasks: + await _cancel_task(task) + + +# ── Chunking ───────────────────────────────────────────────────────── + + +class ChunkingMiddleware(OutboundMiddlewareBase): + """Auto-split messages respecting format expansion. + + Wraps the existing ``chunking.chunk_text`` utility and the + re-splitting logic from ``Channel._prepare_chunks``. + """ + + def __init__(self, capabilities: Any) -> None: + from .capabilities import ChannelCapabilities + + self._capabilities: ChannelCapabilities = capabilities + + def prepare_chunks( + self, + content: str, + limit: int, + format_fn: Callable[[str], str] | None = None, + ) -> list[tuple[str, str]]: + """Build ``(formatted, raw)`` pairs, re-splitting when needed. + + If *format_fn* is None, formatted == raw. + """ + from .base import chunk_text + + if format_fn is None: + format_fn = lambda t: t # noqa: E731 + + raw_chunks = chunk_text(content, limit) + pairs: list[tuple[str, str]] = [] + for raw in raw_chunks: + formatted = format_fn(raw) + if len(formatted) <= limit: + pairs.append((formatted, raw)) + else: + sub_limit = max(limit // 2, 500) + for sub_raw in chunk_text(raw, sub_limit): + sub_fmt = format_fn(sub_raw) + if len(sub_fmt) <= limit: + pairs.append((sub_fmt, sub_raw)) + else: + pairs.append((sub_raw, sub_raw)) + return pairs + + +# ── Formatting ─────────────────────────────────────────────────────── + + +class FormattingMiddleware(OutboundMiddlewareBase): + """Markdown -> channel format conversion. + + Uses ``UnifiedFormatter`` configured from capabilities. + """ + + def __init__(self, capabilities: Any) -> None: + from .capabilities import ChannelCapabilities + from .formatter import UnifiedFormatter + + caps: ChannelCapabilities = capabilities + self._formatter = UnifiedFormatter.for_channel(caps.format_type) + + def format(self, text: str) -> str: + """Convert text to channel format.""" + return self._formatter.format(text) + + async def process_outbound( + self, + message: OutboundMessage, + context: dict[str, Any], + ) -> OutboundMessage | None: + formatted = self._formatter.format(message.content) + return dataclasses.replace(message, content=formatted) + + +# ── Retry ──────────────────────────────────────────────────────────── + + +class RetryMiddleware: + """Exponential backoff send retry. + + Wraps ``retry.retry_async`` with channel-appropriate configuration. + """ + + def __init__(self, channel_name: str = "unknown") -> None: + from .retry import DEFAULT_RETRY, RETRY_PRESETS + + self._config = RETRY_PRESETS.get(channel_name, DEFAULT_RETRY) + self._channel_name = channel_name + + async def execute( + self, + coro_factory: Callable[[], Any], + should_retry: Callable[[Exception, int], bool] | None = None, + retry_after_s: Callable[[Exception], float | None] | None = None, + ) -> Any: + """Execute *coro_factory* with retry logic.""" + from .retry import retry_async + + return await retry_async( + coro_factory, + config=self._config, + should_retry=should_retry or (lambda exc, _: True), + retry_after_s=retry_after_s, + on_retry=lambda info: _logger.warning( + f"{self._channel_name} retry {info.attempt}/{info.max_attempts} " + f"in {info.delay_s:.2f}s: {info.error}" + ), + label=f"{self._channel_name}.send", + ) + + +# ── Typing ─────────────────────────────────────────────────────────── + + +class TypingMiddleware: + """Typing indicator management. + + Wraps ``TypingManager`` for use as a standalone middleware component. + """ + + def __init__( + self, + send_typing_fn: Callable[[str], Any], + interval: float = 5.0, + debug_trace: bool = False, + channel_name: str = "unknown", + ) -> None: + self._manager = TypingManager( + send_typing_fn, + interval=interval, + debug_trace=debug_trace, + channel_name=channel_name, + ) + + async def start(self, chat_id: str) -> None: + await self._manager.start(chat_id) + + async def stop(self, chat_id: str) -> None: + await self._manager.stop(chat_id) + + async def stop_all(self) -> None: + await self._manager.stop_all() + + +# ── ACK Reaction ───────────────────────────────────────────────────── + + +class AckReactionMiddleware: + """ACK emoji reaction with configurable scope. + + Scope controls when reactions are sent: + - ``"all"``: react to every message + - ``"direct"``: react only in DMs + - ``"group-all"``: react in group chats (all messages) + - ``"group-mentions"``: react in groups only when mentioned + - ``"off"``: disable reactions + """ + + def __init__( + self, + *, + scope: str = "all", + emoji: str = "\U0001f440", + remove_after_reply: bool = False, + send_fn: Callable[[str, str, str], Any] | None = None, + remove_fn: Callable[[str, str, str], Any] | None = None, + debug_trace: bool = False, + channel_name: str = "unknown", + ) -> None: + self.scope = scope + self.emoji = emoji + self.remove_after_reply = remove_after_reply + self._send_fn = send_fn + self._remove_fn = remove_fn + self._pending: dict[str, str] = {} # chat_id -> message_id + self._debug_trace = debug_trace + self._channel_name = channel_name + + def should_react(self, *, is_group: bool, was_mentioned: bool) -> bool: + if self.scope == "off": + return False + if self.scope == "all": + return True + if self.scope == "direct": + return not is_group + if self.scope == "group-all": + return is_group + if self.scope == "group-mentions": + return is_group and was_mentioned + return False + + async def send_ack(self, chat_id: str, message_id: str) -> None: + if self._send_fn and message_id: + try: + await self._send_fn(chat_id, message_id, self.emoji) + if self.remove_after_reply: + self._pending[chat_id] = message_id + except Exception as exc: + _trace_named_event( + "ack_send_error", + enabled=self._debug_trace, + channel_name=self._channel_name, + chat_id=chat_id, + message_id=message_id, + error=str(exc), + ) + + async def remove_ack(self, chat_id: str) -> None: + message_id = self._pending.pop(chat_id, None) + if message_id and self._remove_fn: + try: + await self._remove_fn(chat_id, message_id, self.emoji) + except Exception as exc: + _trace_named_event( + "ack_remove_error", + enabled=self._debug_trace, + channel_name=self._channel_name, + chat_id=chat_id, + message_id=message_id, + error=str(exc), + ) + + +# ── Mention Gating ─────────────────────────────────────────────────── + + +class MentionGatingMiddleware(InboundMiddleware): + """Filter messages based on mention policy. + + Policy values: + - ``"always"``: require mention in all chats + - ``"group"``: require mention only in groups (default) + - ``"off"``: never require mention + """ + + def __init__( + self, + require_mention: str = "group", + strip_fn: Callable[[str], str] | None = None, + ) -> None: + self.require_mention = require_mention + self._strip_fn = strip_fn + + async def process_inbound( + self, + raw: RawIncoming, + context: dict[str, Any], + ) -> RawIncoming | None: + if not self._should_process(raw): + _trace_context_event( + context, + "middleware_mention_drop", + chat_id=raw.chat_id, + policy=self.require_mention, + ) + return None + # Strip mentions from group messages + if raw.is_group and self._strip_fn: + raw = dataclasses.replace(raw, text=self._strip_fn(raw.text)) + return raw + + def _should_process(self, raw: RawIncoming) -> bool: + if self.require_mention == "off": + return True + if self.require_mention == "always": + return raw.was_mentioned + # "group" — require mention only in groups + if not raw.is_group: + return True + return raw.was_mentioned + + +# ── AllowList ──────────────────────────────────────────────────────── + + +class AllowListMiddleware(InboundMiddleware): + """Sender and channel allow-list enforcement.""" + + def __init__( + self, + allowed_senders: set[str] | None = None, + allowed_channels: set[str] | None = None, + dm_policy: str = "allowlist", + ) -> None: + self.allowed_senders = allowed_senders + self.allowed_channels = allowed_channels + self.dm_policy = dm_policy + + async def process_inbound( + self, + raw: RawIncoming, + context: dict[str, Any], + ) -> RawIncoming | None: + # Channel allow-list + if self.allowed_channels and str(raw.chat_id) not in self.allowed_channels: + _trace_context_event( + context, + "middleware_allowlist_drop", + sender_id=raw.sender_id, + chat_id=raw.chat_id, + reason="chat_not_allowed", + ) + return None + + # Sender allow-list + if not raw.is_group and self.dm_policy == "open": + return raw # open DMs bypass sender checks + + if not self._is_sender_allowed(raw.sender_id): + _trace_context_event( + context, + "middleware_allowlist_drop", + sender_id=raw.sender_id, + chat_id=raw.chat_id, + reason="sender_not_allowed", + ) + return None + + return raw + + def _is_sender_allowed(self, sender: str) -> bool: + if not self.allowed_senders: + return True + sender_str = str(sender) + if sender_str in self.allowed_senders: + return True + if "|" in sender_str: + for part in sender_str.split("|"): + if part and part in self.allowed_senders: + return True + return False + + +# ── Group History ──────────────────────────────────────────────────── + + +class GroupHistoryMiddleware(InboundMiddleware): + """Buffer non-mentioned group messages, inject as context when mentioned.""" + + def __init__( + self, + max_per_chat: int = 50, + max_age_seconds: int = 3600, + ) -> None: + self._buffer = GroupHistoryBuffer( + max_per_chat=max_per_chat, + max_age_seconds=max_age_seconds, + ) + + async def process_inbound( + self, + raw: RawIncoming, + context: dict[str, Any], + ) -> RawIncoming | None: + if not raw.is_group: + return raw + + # Use monotonic clock for consistent expiry calculation + ts = time.monotonic() + + if not raw.was_mentioned: + self._buffer.add( + raw.chat_id, + HistoryEntry( + sender_id=raw.sender_id, + text=raw.text, + timestamp=ts, + message_id=raw.message_id, + ), + ) + # Don't drop here — let MentionGatingMiddleware handle that + return raw + + # Mentioned: inject history context + history_context = self._buffer.format_context(raw.chat_id) + if history_context: + raw = dataclasses.replace( + raw, + text=history_context + + "\n\n[Current message - respond to this]\n" + + raw.text, + ) + self._buffer.clear(raw.chat_id) + return raw + + +# ── Pairing ────────────────────────────────────────────────────────── + + +class PairingMiddleware(InboundMiddleware): + """DM pairing flow management. + + When dm_policy is "pairing", unapproved DM senders receive a + pairing code. Approved senders pass through normally. + """ + + def __init__( + self, + channel_name: str, + send_response_fn: Callable[[str, str], Any] | None = None, + dm_policy: str = "allowlist", + ) -> None: + self._manager = PairingManager() + self._channel_name = channel_name + self._send_response_fn = send_response_fn + self._dm_policy = dm_policy + self._background_tasks: set[asyncio.Task] = set() + + async def process_inbound( + self, + raw: RawIncoming, + context: dict[str, Any], + ) -> RawIncoming | None: + if raw.is_group: + return raw # pairing only applies to DMs + + if self._dm_policy != "pairing": + return raw + + if self._manager.is_approved(self._channel_name, raw.sender_id): + return raw + + # Request pairing + code = self._manager.request_pairing(self._channel_name, raw.sender_id) + if self._send_response_fn: + text = f"\U0001f510 Pairing required. Your code: {code}\nThis code expires in 1 hour." + task = asyncio.create_task(self._send_response_fn(raw.chat_id, text)) + # Track the task to prevent GC and handle exceptions + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) + _trace_context_event( + context, + "middleware_pairing_required", + sender_id=raw.sender_id, + ) + _logger.info(f"Pairing required for {raw.sender_id}, code sent") + return None diff --git a/EvoScientist/channels/mixins.py b/EvoScientist/channels/mixins.py new file mode 100644 index 0000000..d7eeb0c --- /dev/null +++ b/EvoScientist/channels/mixins.py @@ -0,0 +1,342 @@ +"""Reusable channel mixins for common architecture patterns. + +Three mixins that eliminate boilerplate across channels: + +- ``WebhookMixin`` — aiohttp webhook server + httpx client + token refresh +- ``WebSocketMixin`` — WS connect/reconnect/heartbeat loop +- ``PollingMixin`` — async poll loop with backoff + +Each mixin works with the Channel base class. Subclasses override +a small set of abstract/hook methods to define platform-specific behavior. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import time +from typing import Any + +logger = logging.getLogger(__name__) + + +# ═════════════════════════════════════════════════════════════════════ +# Token refresh mixin (shared by Webhook & WebSocket channels) +# ═════════════════════════════════════════════════════════════════════ + + +class TokenMixin: + """Mixin for channels that need OAuth-style token management. + + Subclass must implement ``_fetch_token()`` returning + ``(access_token, expires_in_seconds)``. + """ + + _access_token: str | None = None + _token_expires: float = 0 + _http_client: Any = None # httpx.AsyncClient + + async def _fetch_token(self) -> tuple[str, int]: + """Fetch a new access token. Return (token, expires_in_seconds). + + Must be implemented by the channel. + """ + raise NotImplementedError + + async def _refresh_token(self) -> None: + token, expire = await self._fetch_token() + self._access_token = token + self._token_expires = time.monotonic() + expire - 300 + logger.debug( + f"{getattr(self, 'name', '?')} token refreshed, expires in {expire}s" + ) + + async def _ensure_token(self) -> str: + if not self._access_token or time.monotonic() >= self._token_expires: + await self._refresh_token() + return self._access_token + + +# ═════════════════════════════════════════════════════════════════════ +# Webhook + REST mixin +# ═════════════════════════════════════════════════════════════════════ + + +class WebhookMixin: + """Mixin for channels that use an HTTP webhook server for inbound + and REST API for outbound. + + Provides: + - aiohttp web server lifecycle (start/stop) + - httpx async client lifecycle + - Route registration via ``_webhook_routes()`` + + Subclass must implement: + - ``_webhook_routes()`` → list of (method, path, handler) + - ``_get_webhook_port()`` → int + """ + + _http_client: Any = None + _runner: Any = None + _site: Any = None + + def _get_webhook_port(self) -> int: + return getattr(self.config, "webhook_port", 9000) + + def _webhook_routes(self) -> list[tuple[str, str, Any]]: + """Return [(method, path, handler), ...]. Override in subclass.""" + return [] + + async def _start_webhook_server(self) -> None: + """Start aiohttp webhook server + httpx client. + + If ``_shared_webhook_server`` is set (by ChannelManager), the + aiohttp server is already running on the shared port — only + create the httpx outbound client. + """ + import httpx + + proxy = getattr(self.config, "proxy", None) or None + self._http_client = httpx.AsyncClient(timeout=15, proxy=proxy) + + # Shared webhook mode: routes already registered on shared server + if getattr(self, "_shared_webhook_server", None): + logger.info(f"{getattr(self, 'name', '?')} using shared webhook server") + return + + from aiohttp import web + + app = web.Application() + for method, path, handler in self._webhook_routes(): + if method.upper() == "GET": + app.router.add_get(path, handler) + else: + app.router.add_post(path, handler) + + self._runner = web.AppRunner(app) + await self._runner.setup() + port = self._get_webhook_port() + self._site = web.TCPSite(self._runner, "0.0.0.0", port) + await self._site.start() + logger.info(f"{getattr(self, 'name', '?')} webhook on port {port}") + + async def _stop_webhook_server(self) -> None: + if self._site: + await self._site.stop() + self._site = None + if self._runner: + await self._runner.cleanup() + self._runner = None + if self._http_client: + await self._http_client.aclose() + self._http_client = None + + async def _api_post( + self, url: str, body: dict, headers: dict | None = None + ) -> dict: + """POST JSON to API, return parsed response. Raises on HTTP error.""" + resp = await self._http_client.post(url, json=body, headers=headers) + data = resp.json() + return data + + async def _api_get(self, url: str, headers: dict | None = None) -> dict: + resp = await self._http_client.get(url, headers=headers) + return resp.json() + + +# ═════════════════════════════════════════════════════════════════════ +# WebSocket mixin +# ═════════════════════════════════════════════════════════════════════ + + +class WebSocketMixin: + """Mixin for channels that receive messages via WebSocket. + + Provides: + - Connect/reconnect loop with exponential backoff + - Heartbeat task management + - Message dispatch + + Subclass must implement: + - ``_get_ws_url()`` → WebSocket URL to connect to + - ``_on_ws_message(data)`` → handle a parsed message dict + - ``_on_ws_connected(ws)`` → called after connection (send identify, etc.) + + Optional overrides: + - ``_ws_heartbeat_interval`` → seconds between heartbeats (0 = disabled) + - ``_on_ws_heartbeat(ws)`` → send heartbeat + """ + + _ws_session: Any = None + _ws_heartbeat_task: asyncio.Task | None = None + _ws_heartbeat_interval: float = 0 # 0 = no heartbeat + _ws_reconnect_delay: float = 5.0 + + async def _get_ws_url(self) -> str: + raise NotImplementedError + + async def _on_ws_connected(self, ws) -> None: + """Called after WebSocket connects. Send identify/auth here.""" + pass + + async def _on_ws_message(self, data: dict | str) -> None: + """Handle a single WebSocket message.""" + raise NotImplementedError + + async def _on_ws_heartbeat(self, ws) -> None: + """Send a heartbeat. Override if needed.""" + pass + + async def _ws_loop(self) -> None: + """Main WebSocket loop with auto-reconnect.""" + import os + + import aiohttp + + while getattr(self, "_running", False): + try: + ws_url = await self._get_ws_url() + # Resolve proxy: channel config > environment variable + proxy = getattr(getattr(self, "config", None), "proxy", None) + if not proxy: + proxy = ( + os.environ.get("https_proxy") + or os.environ.get("HTTPS_PROXY") + or os.environ.get("http_proxy") + or os.environ.get("HTTP_PROXY") + or None + ) + logger.debug( + f"{getattr(self, 'name', '?')} WS connecting to {ws_url[:60]}... proxy={proxy}" + ) + async with aiohttp.ClientSession() as session: + async with session.ws_connect( + ws_url, + proxy=proxy, + timeout=aiohttp.ClientWSTimeout(ws_close=30), + ) as ws: + logger.info(f"{getattr(self, 'name', '?')} WebSocket connected") + self._ws_session = ws + await self._on_ws_connected(ws) + + # Start heartbeat if configured + if self._ws_heartbeat_interval > 0: + self._ws_heartbeat_task = asyncio.create_task( + self._ws_heartbeat_loop(ws) + ) + + async for msg in ws: + if msg.type == aiohttp.WSMsgType.TEXT: + try: + data = json.loads(msg.data) + except (json.JSONDecodeError, TypeError): + data = msg.data + await self._on_ws_message(data) + elif msg.type in ( + aiohttp.WSMsgType.CLOSED, + aiohttp.WSMsgType.ERROR, + ): + break + + except asyncio.CancelledError: + break + except Exception as e: + logger.error(f"{getattr(self, 'name', '?')} WS error: {e}") + + await self._ws_cleanup_heartbeat() + self._ws_session = None + + if getattr(self, "_running", False): + logger.info( + f"{getattr(self, 'name', '?')} reconnecting in {self._ws_reconnect_delay}s..." + ) + await asyncio.sleep(self._ws_reconnect_delay) + + async def _ws_heartbeat_loop(self, ws) -> None: + while True: + try: + await self._on_ws_heartbeat(ws) + except Exception: + break + await asyncio.sleep(self._ws_heartbeat_interval) + + async def _ws_cleanup_heartbeat(self) -> None: + if self._ws_heartbeat_task: + task = self._ws_heartbeat_task + self._ws_heartbeat_task = None + task.cancel() + try: + await task + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None and current.cancelling() > 0: + raise + except Exception: + pass + + async def _ws_send_json(self, data: dict) -> None: + """Send JSON to the active WebSocket.""" + if self._ws_session: + await self._ws_session.send_str(json.dumps(data)) + + async def _stop_ws(self) -> None: + await self._ws_cleanup_heartbeat() + if self._ws_session: + await self._ws_session.close() + self._ws_session = None + + +# ═════════════════════════════════════════════════════════════════════ +# Polling mixin +# ═════════════════════════════════════════════════════════════════════ + + +class PollingMixin: + """Mixin for channels that poll for new messages. + + Provides: + - Poll loop with configurable interval + - Error handling + reconnect + + Subclass must implement: + - ``_poll_once()`` → fetch and enqueue new messages + - ``_get_poll_interval()`` → seconds between polls + """ + + _poll_task: asyncio.Task | None = None + + def _get_poll_interval(self) -> float: + return getattr(self.config, "poll_interval", 30) + + async def _poll_once(self) -> None: + """Fetch new messages and enqueue them. Override in subclass.""" + raise NotImplementedError + + async def _start_polling(self) -> None: + self._poll_task = asyncio.create_task(self._poll_loop()) + + async def _poll_loop(self) -> None: + interval = self._get_poll_interval() + while getattr(self, "_running", False): + try: + await self._poll_once() + except asyncio.CancelledError: + break + except Exception as e: + logger.error(f"{getattr(self, 'name', '?')} poll error: {e}") + await asyncio.sleep(interval) + + async def _stop_polling(self) -> None: + if self._poll_task: + task = self._poll_task + self._poll_task = None + task.cancel() + try: + await task + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None and current.cancelling() > 0: + raise + except Exception: + pass diff --git a/EvoScientist/channels/plugin.py b/EvoScientist/channels/plugin.py new file mode 100644 index 0000000..752583e --- /dev/null +++ b/EvoScientist/channels/plugin.py @@ -0,0 +1,243 @@ +"""Plugin-based channel interface. + +A ChannelPlugin is a declarative object with optional adapter slots. +The framework inspects which slots are filled and auto-assembles +the message processing pipeline. + +The ``Channel`` base class extends ``ChannelPlugin``, so all channel +implementations are automatically ChannelPlugin instances. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Protocol, runtime_checkable + +from .capabilities import ChannelCapabilities + +# ── Channel metadata ───────────────────────────────────────────────── + + +@dataclass +class ChannelMeta: + """Channel metadata for registry and UI.""" + + id: str + label: str + description: str = "" + docs_path: str = "" + system_image: str = "" # icon name + + +# ── Adapter Protocols (slots) ──────────────────────────────────────── + + +@runtime_checkable +class ConfigAdapter(Protocol): + """Account configuration management.""" + + def list_account_ids(self, config: Any) -> list[str]: ... + def resolve_account(self, config: Any, account_id: str | None = None) -> Any: ... + def is_enabled(self, account: Any, config: Any) -> bool: ... + def is_configured(self, account: Any, config: Any) -> bool: ... + + +@runtime_checkable +class SecurityAdapter(Protocol): + """DM policy and security warnings.""" + + def resolve_dm_policy( + self, ctx: Any + ) -> str: ... # "open" | "allowlist" | "pairing" + def collect_warnings(self, ctx: Any) -> list[str]: ... + + +@runtime_checkable +class GroupAdapter(Protocol): + """Per-group policy resolution.""" + + def resolve_require_mention(self, ctx: Any) -> bool | None: ... + def resolve_tool_policy(self, ctx: Any) -> dict[str, Any] | None: ... + def resolve_intro_hint(self, ctx: Any) -> str | None: ... + + +@runtime_checkable +class MentionAdapter(Protocol): + """Bot mention detection and stripping.""" + + def strip_mentions(self, text: str, ctx: Any) -> str: ... + + +@runtime_checkable +class OutboundAdapter(Protocol): + """Outbound message delivery.""" + + delivery_mode: str # "direct" | "gateway" | "hybrid" + + async def send_text(self, ctx: Any) -> bool: ... + async def send_media(self, ctx: Any) -> bool: ... + + +@runtime_checkable +class ThreadingAdapter(Protocol): + """Reply threading behavior.""" + + def resolve_reply_to_mode(self, ctx: Any) -> str: ... # "off" | "first" | "all" + + +@runtime_checkable +class StreamingAdapter(Protocol): + """Edit-in-place streaming output.""" + + async def edit_message(self, chat_id: str, message_id: str, text: str) -> bool: ... + + +@runtime_checkable +class DirectoryAdapter(Protocol): + """Contact/group directory queries.""" + + async def list_peers(self, ctx: Any) -> list[dict]: ... + async def list_groups(self, ctx: Any) -> list[dict]: ... + async def list_group_members(self, ctx: Any) -> list[dict]: ... + + +@runtime_checkable +class StatusAdapter(Protocol): + """Health probing and status reporting.""" + + async def probe_account(self, ctx: Any) -> Any: ... + async def audit_account(self, ctx: Any) -> Any: ... + def collect_status_issues(self, accounts: list) -> list[dict]: ... + + +@runtime_checkable +class HeartbeatAdapter(Protocol): + """Channel heartbeat / readiness checks.""" + + async def check_ready(self, ctx: Any) -> tuple[bool, str]: ... + + +@runtime_checkable +class ActionsAdapter(Protocol): + """Message actions (react, edit, delete, poll, etc.).""" + + def list_actions(self) -> list[str]: ... + async def handle_action(self, action: str, ctx: Any) -> Any: ... + + +@runtime_checkable +class PairingAdapter(Protocol): + """DM pairing flow.""" + + id_label: str + + def normalize_entry(self, entry: str) -> str: ... + async def notify_approval(self, ctx: Any) -> None: ... + + +@runtime_checkable +class OnboardingAdapter(Protocol): + """Interactive setup wizard hooks.""" + + async def wizard_steps(self, ctx: Any) -> list[dict]: ... + async def validate_step(self, step: str, value: Any) -> str | None: ... + + +# ── Reload policy ──────────────────────────────────────────────────── + + +@dataclass +class ReloadPolicy: + """Declares which config prefixes trigger a channel reload.""" + + config_prefixes: list[str] = field(default_factory=list) + noop_prefixes: list[str] = field(default_factory=list) + + +# ── ChannelPlugin ──────────────────────────────────────────────────── + + +class ChannelPlugin: + """Declarative channel plugin with optional adapter slots. + + Replaces the monolithic Channel base class. Each slot is optional — + the framework adapts behavior based on which are present. + + Usage:: + + class MyPlugin(ChannelPlugin): + id = "my_channel" + meta = ChannelMeta(id="my_channel", label="My Channel") + capabilities = ChannelCapabilities(...) + + def __init__(self): + self.outbound = MyOutboundAdapter() + self.config_adapter = MyConfigAdapter() + + async def start(self, config, account_id=None): + ... + + async def stop(self, account_id=None): + ... + """ + + id: str = "" + meta: ChannelMeta | None = None + capabilities: ChannelCapabilities = ChannelCapabilities() + + # Optional adapter slots — fill what you need + # Default: SingleAccountConfigAdapter so every plugin has multi-account + # support out of the box (returns a single "default" account). + config_adapter: ConfigAdapter | None = None + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + + def __init__(self) -> None: + # Provide default SingleAccountConfigAdapter if not overridden + if self.config_adapter is None: + from .config import SingleAccountConfigAdapter + + self.config_adapter = SingleAccountConfigAdapter() + + security: SecurityAdapter | None = None + groups: GroupAdapter | None = None + mentions: MentionAdapter | None = None + outbound: OutboundAdapter | None = None + threading: ThreadingAdapter | None = None + streaming: StreamingAdapter | None = None + directory: DirectoryAdapter | None = None + status: StatusAdapter | None = None + heartbeat: HeartbeatAdapter | None = None + actions: ActionsAdapter | None = None + pairing: PairingAdapter | None = None + onboarding: OnboardingAdapter | None = None + + # Lifecycle + reload: ReloadPolicy | None = None + + # Connection management + async def start(self, config: Any, account_id: str | None = None) -> None: + """Start the channel (or a specific account).""" + + async def stop(self, account_id: str | None = None) -> None: + """Stop the channel (or a specific account).""" + + def filled_slots(self) -> list[str]: + """Return names of adapter slots that are not None.""" + slot_names = [ + "config_adapter", + "security", + "groups", + "mentions", + "outbound", + "threading", + "streaming", + "directory", + "status", + "heartbeat", + "actions", + "pairing", + "onboarding", + ] + return [s for s in slot_names if getattr(self, s, None) is not None] diff --git a/EvoScientist/channels/qq/__init__.py b/EvoScientist/channels/qq/__init__.py new file mode 100644 index 0000000..7f560c2 --- /dev/null +++ b/EvoScientist/channels/qq/__init__.py @@ -0,0 +1,28 @@ +"""QQ channel for EvoScientist. + +Uses the official qq-botpy SDK for WebSocket connection. + +Usage in config: + channel_enabled = "qq" + qq_app_id = "your_app_id" + qq_app_secret = "your_app_secret" +""" + +from ..channel_manager import _parse_csv, register_channel +from .channel import QQChannel, QQConfig + +__all__ = ["QQChannel", "QQConfig"] + + +def create_from_config(config) -> QQChannel: + allowed = _parse_csv(config.qq_allowed_senders) + return QQChannel( + QQConfig( + app_id=config.qq_app_id, + app_secret=config.qq_app_secret, + allowed_senders=allowed, + ) + ) + + +register_channel("qq", create_from_config) diff --git a/EvoScientist/channels/qq/channel.py b/EvoScientist/channels/qq/channel.py new file mode 100644 index 0000000..eb8753b --- /dev/null +++ b/EvoScientist/channels/qq/channel.py @@ -0,0 +1,374 @@ +"""QQ channel implementation using botpy SDK.""" + +import asyncio +import logging +from collections import deque +from dataclasses import dataclass +from datetime import datetime +from typing import ClassVar + +from ..base import Channel, ChannelError, RawIncoming +from ..capabilities import QQ as QQ_CAPS +from ..config import BaseChannelConfig +from ..formatter import UnifiedFormatter + +logger = logging.getLogger(__name__) + +try: + import botpy + from botpy.message import C2CMessage, GroupMessage + + QQ_AVAILABLE = True +except ImportError: + QQ_AVAILABLE = False + botpy = None + C2CMessage = None + GroupMessage = None + + +@dataclass +class QQConfig(BaseChannelConfig): + app_id: str = "" + app_secret: str = "" + text_chunk_limit: int = 4096 + + +def _make_bot_class(channel: "QQChannel") -> "type[botpy.Client]": + """Create a botpy Client subclass bound to the given channel.""" + intents = botpy.Intents(public_messages=True, direct_message=True) + + class _Bot(botpy.Client): + def __init__(self): + super().__init__(intents=intents) + + async def on_ready(self): + logger.info(f"QQ bot ready: {self.robot.name}") + + async def on_c2c_message_create(self, message: "C2CMessage"): + await channel._on_msg(message, "c2c") + + async def on_group_at_message_create(self, message: "GroupMessage"): + await channel._on_msg(message, "group") + + return _Bot + + +class QQChannel(Channel): + """QQ channel using botpy SDK.""" + + name = "qq" + + capabilities = QQ_CAPS + _ready_attrs = ("_client", "_running") + _non_retryable_patterns = () + _mention_pattern = r"@\S+\s*" + _mention_strip_count = 1 + _markdown_fallback_exc_types: ClassVar[tuple[type[Exception], ...]] = ( + TypeError, + ValueError, + ) + + def __init__(self, config: QQConfig): + super().__init__(config) + self._client: botpy.Client | None = None + self._bot_task: asyncio.Task | None = None + self._processed_ids: deque = deque(maxlen=1000) + self._msg_seq: dict[str, int] = {} # msg_id -> next seq number + self._msg_seq_order: deque = deque(maxlen=500) + self._msg_seq_ids: set[str] = set() # companion set for O(1) lookup + self._plain_formatter = UnifiedFormatter.for_channel("plain") + + # ── Lifecycle ───────────────────────────────────────────────── + + async def start(self) -> None: + if not QQ_AVAILABLE: + raise ChannelError("QQ SDK not installed. Run: pip install qq-botpy") + if not self.config.app_id or not self.config.app_secret: + raise ChannelError("QQ app_id and app_secret are required") + self._running = True + BotClass = _make_bot_class(self) + self._client = BotClass() + self._bot_task = asyncio.create_task(self._run_bot()) + logger.info("QQ channel starting...") + + async def _run_bot(self) -> None: + try: + await self._client.start( + appid=self.config.app_id, secret=self.config.app_secret + ) + except Exception as e: + logger.error(f"QQ auth failed: {e}") + self._running = False + + async def _cleanup(self) -> None: + self._running = False + if self._bot_task: + self._bot_task.cancel() + try: + await self._bot_task + except asyncio.CancelledError: + pass + self._client = None + logger.info("QQ channel stopped") + + # ── Incoming ────────────────────────────────────────────────── + + async def _on_msg(self, message, msg_type: str) -> None: + try: + if message.id in self._processed_ids: + return + self._processed_ids.append(message.id) + + author = message.author + content = (message.content or "").strip() + + if msg_type == "c2c": + sender_id = str(getattr(author, "user_openid", "")) + chat_id = sender_id + else: + sender_id = str(getattr(author, "member_openid", "")) + chat_id = str(getattr(message, "group_openid", "")) + + # Handle attachments (images, files, audio, video) + annotations: list[str] = [] + media_paths: list[str] = [] + attachments = getattr(message, "attachments", None) or [] + for att in attachments: + url = getattr(att, "url", "") or "" + filename = getattr(att, "filename", "attachment") or "attachment" + content_type = getattr(att, "content_type", "") or "" + if url: + local, ann = await self._download_attachment( + url, + f"qq_{filename}", + ) + if local: + media_paths.append(local) + if ann: + annotations.append(ann) + else: + annotations.append(f"[{content_type or 'attachment'}: {filename}]") + + if not content and not media_paths and not annotations: + return + + await self._enqueue_raw( + RawIncoming( + sender_id=sender_id, + chat_id=chat_id, + text=content, + media_files=media_paths, + content_annotations=annotations, + timestamp=datetime.now(), + message_id=message.id, + is_group=(msg_type == "group"), + was_mentioned=True, + metadata={ + "chat_id": chat_id, + "msg_type": msg_type, + "event_id": message.id, + "backend": "qq", + }, + ) + ) + except Exception as e: + logger.error(f"Error handling QQ message: {e}") + + # ── Send ────────────────────────────────────────────────────── + + def _next_msg_seq(self, msg_id: str) -> int: + """Return the next msg_seq for *msg_id* and increment the counter.""" + seq = self._msg_seq.get(msg_id, 1) + self._msg_seq[msg_id] = seq + 1 + if msg_id not in self._msg_seq_ids: + self._msg_seq_order.append(msg_id) + self._msg_seq_ids.add(msg_id) + if len(self._msg_seq_order) > 500: + oldest = self._msg_seq_order.popleft() + self._msg_seq_ids.discard(oldest) + self._msg_seq.pop(oldest, None) + return seq + + async def _send_chunk(self, chat_id, formatted_text, raw_text, reply_to, metadata): + if not self._client: + raise ChannelError("QQ client not initialized") + msg_type = (metadata or {}).get("msg_type", "c2c") + msg_id = (metadata or {}).get("event_id", "") + seq = self._next_msg_seq(msg_id) + try: + await self._post_markdown_message(chat_id, raw_text, msg_type, msg_id, seq) + return + except Exception as exc: + if not self._should_fallback_to_plain_text(exc): + raise + self._record_markdown_fallback(chat_id, raw_text, exc) + logger.debug("QQ markdown send failed, falling back to plain text: %s", exc) + + plain_text = self._plain_formatter.format(raw_text) + await self._post_plain_message(chat_id, plain_text, msg_type, msg_id, seq) + + def _should_fallback_to_plain_text(self, exc: Exception) -> bool: + """Return True only for markdown compatibility/validation failures.""" + if isinstance(exc, self._markdown_fallback_exc_types): + return True + + msg = str(exc).lower() + compatibility_tokens = ("unsupported", "unexpected", "unknown", "invalid") + return ( + "unexpected keyword argument" in msg + or ( + "markdown" in msg + and any(token in msg for token in compatibility_tokens) + ) + or ( + "msg_type" in msg + and any(token in msg for token in compatibility_tokens) + ) + ) + + def _record_markdown_fallback( + self, + chat_id: str, + raw_text: str, + exc: Exception, + ) -> None: + """Emit optional debug trace for markdown fallback without blocking send.""" + trace_event = getattr(self, "_trace_event", None) + if not callable(trace_event): + return + try: + trace_event( + "outbound_format_fallback", + chat_id=chat_id, + error=str(exc), + formatted_len=len(raw_text), + raw_len=len(raw_text), + ) + except Exception as trace_exc: + logger.debug("QQ fallback trace failed: %s", trace_exc) + + async def _post_markdown_message( + self, + chat_id: str, + text: str, + msg_type: str, + msg_id: str, + seq: int, + ) -> None: + payload = { + "msg_type": 2, + "markdown": {"content": text}, + "msg_id": msg_id, + "msg_seq": seq, + } + if msg_type == "group": + await self._client.api.post_group_message( + group_openid=chat_id, + **payload, + ) + else: + await self._client.api.post_c2c_message( + openid=chat_id, + **payload, + ) + + async def _post_plain_message( + self, + chat_id: str, + text: str, + msg_type: str, + msg_id: str, + seq: int, + ) -> None: + payload = { + "msg_type": 0, + "content": text, + "msg_id": msg_id, + "msg_seq": seq, + } + if msg_type == "group": + await self._client.api.post_group_message( + group_openid=chat_id, + **payload, + ) + else: + await self._client.api.post_c2c_message( + openid=chat_id, + **payload, + ) + + # _send_typing_action: inherited no-op (QQ Bot API has no typing indicator) + + # ── Media send ──────────────────────────────────────────────── + + # qq-botpy file_type constants: 1=image, 2=video, 3=audio + _FILE_TYPE_MAP: ClassVar[dict[str, int]] = { + ".jpg": 1, + ".jpeg": 1, + ".png": 1, + ".gif": 1, + ".webp": 1, + ".bmp": 1, + ".mp4": 2, + ".mov": 2, + ".avi": 2, + ".mp3": 3, + ".ogg": 3, + ".m4a": 3, + ".wav": 3, + ".silk": 3, + } + + async def _send_media_impl( + self, + recipient: str, + file_path: str, + caption: str = "", + metadata: dict | None = None, + ) -> bool: + """Send a media file through QQ Bot API. + + Uses post_group_file / post_c2c_file with a URL. Local files + without a public URL are not supported — falls back to a text hint. + """ + if not self._client: + raise ChannelError("QQ client not initialized") + + from pathlib import Path + + chat_id = self._resolve_media_chat_id(recipient, metadata) + msg_type = (metadata or {}).get("msg_type", "c2c") + ext = Path(file_path).suffix.lower() + file_type = self._FILE_TYPE_MAP.get(ext, 1) # default to image + + # qq-botpy file API requires a URL, not a local path + is_url = file_path.startswith("http://") or file_path.startswith("https://") + if not is_url: + # Fallback: send text hint for local files + name = Path(file_path).name + hint = f"[文件] {name}" + (f"\n{caption}" if caption else "") + await self._send_chunk(chat_id, hint, hint, None, metadata or {}) + return True + + try: + if msg_type == "group": + await self._client.api.post_group_file( + group_openid=chat_id, + file_type=file_type, + url=file_path, + srv_send_msg=True, + ) + else: + await self._client.api.post_c2c_file( + openid=chat_id, + file_type=file_type, + url=file_path, + srv_send_msg=True, + ) + except Exception as e: + logger.warning(f"QQ media send failed: {e}") + return False + + if caption: + await self._send_chunk(chat_id, caption, caption, None, metadata or {}) + return True diff --git a/EvoScientist/channels/qq/probe.py b/EvoScientist/channels/qq/probe.py new file mode 100644 index 0000000..c84b673 --- /dev/null +++ b/EvoScientist/channels/qq/probe.py @@ -0,0 +1,37 @@ +"""QQ Bot credential validation.""" + +import logging + +logger = logging.getLogger(__name__) + +QQ_TOKEN_URL = "https://bots.qq.com/app/getAppAccessToken" + + +async def validate_qq( + app_id: str, + app_secret: str, +) -> tuple[bool, str]: + """Validate QQ Bot credentials by fetching an access token. + + Returns: + Tuple of (is_valid, message). + """ + if not app_id or not app_secret: + return False, "app_id and app_secret are required" + + try: + import httpx + except ImportError: + return False, "httpx not installed" + + body = {"appId": app_id, "clientSecret": app_secret} + + try: + async with httpx.AsyncClient() as client: + resp = await client.post(QQ_TOKEN_URL, json=body, timeout=10) + data = resp.json() + if data.get("access_token"): + return True, "QQ Bot credentials valid" + return False, f"Error: {data.get('message', data)}" + except Exception as e: + return False, f"Error: {e}" diff --git a/EvoScientist/channels/qq/serve.py b/EvoScientist/channels/qq/serve.py new file mode 100644 index 0000000..1a43ab7 --- /dev/null +++ b/EvoScientist/channels/qq/serve.py @@ -0,0 +1,84 @@ +"""QQ channel server. + +Standalone script to run the QQ channel with CLI options. + +Usage: + python -m EvoScientist.channels.qq.serve --app-id ID --app-secret SECRET [OPTIONS] + +Examples: + # Basic usage + python -m EvoScientist.channels.qq.serve --app-id ID --app-secret SECRET + + # Sandbox mode with allowed senders + python -m EvoScientist.channels.qq.serve --app-id ID --app-secret SECRET --allow user123 + + # With agent and thinking + python -m EvoScientist.channels.qq.serve --app-id ID --app-secret SECRET --agent --thinking +""" + +import argparse +import logging + +from ...logging_config import configure_logging_from_settings +from ..bus import MessageBus +from ..standalone import run_standalone +from .channel import QQChannel, QQConfig + +logger = logging.getLogger(__name__) + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="QQ channel server", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--app-id", + required=True, + help="QQ bot app ID", + ) + parser.add_argument( + "--app-secret", + required=True, + help="QQ bot app secret", + ) + parser.add_argument( + "--allow", + action="append", + dest="allowed_senders", + help="Allowed sender (QQ user ID). Can be used multiple times.", + ) + parser.add_argument( + "--agent", + action="store_true", + help="Use EvoScientist agent as handler (default: echo)", + ) + parser.add_argument( + "--thinking", + action="store_true", + help="Send thinking content as intermediate messages (requires --agent)", + ) + return parser.parse_args() + + +def main(): + """Entry point.""" + configure_logging_from_settings(default_level=logging.INFO) + args = parse_args() + + config = QQConfig( + app_id=args.app_id, + app_secret=args.app_secret, + allowed_senders=set(args.allowed_senders) if args.allowed_senders else None, + ) + + send_thinking = args.thinking and args.agent + bus = MessageBus() + channel = QQChannel(config) + + run_standalone(channel, bus, use_agent=args.agent, send_thinking=send_thinking) + + +if __name__ == "__main__": + main() diff --git a/EvoScientist/channels/retry.py b/EvoScientist/channels/retry.py new file mode 100644 index 0000000..04b55ec --- /dev/null +++ b/EvoScientist/channels/retry.py @@ -0,0 +1,114 @@ +"""Configurable exponential-backoff retry for async callables.""" + +from __future__ import annotations + +import asyncio +import random +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import TypeVar + +T = TypeVar("T") + + +@dataclass +class RetryConfig: + """Configuration for retry behavior.""" + + attempts: int = 3 + min_delay_s: float = 0.3 + max_delay_s: float = 30.0 + jitter: float = 0.1 # ±10 % random offset + + +@dataclass +class RetryInfo: + """Information passed to the *on_retry* callback.""" + + attempt: int + max_attempts: int + delay_s: float + error: Exception + label: str | None = None + + +async def retry_async( + fn: Callable[[], Awaitable[T]], + config: RetryConfig | None = None, + *, + should_retry: Callable[[Exception, int], bool] | None = None, + retry_after_s: Callable[[Exception], float | None] | None = None, + on_retry: Callable[[RetryInfo], None] | None = None, + label: str | None = None, +) -> T: + """Execute *fn* with exponential-backoff retry. + + Parameters + ---------- + fn: + Zero-argument async factory — called on every attempt so the + awaitable is always fresh. + """ + if config is None: + config = RetryConfig() + + last_exc: Exception | None = None + for attempt in range(1, config.attempts + 1): + try: + return await fn() + except Exception as exc: + last_exc = exc + + if attempt >= config.attempts: + raise + + if should_retry is not None and not should_retry(exc, attempt): + raise + + # Compute delay + server_delay: float | None = None + if retry_after_s is not None: + server_delay = retry_after_s(exc) + + if server_delay is not None: + base_delay = max(server_delay, config.min_delay_s) + else: + base_delay = config.min_delay_s * (2 ** (attempt - 1)) + + # Apply jitter + jittered = base_delay * (1 + random.uniform(-config.jitter, config.jitter)) + + # Clamp to [min_delay_s, max_delay_s] + delay = max(config.min_delay_s, min(jittered, config.max_delay_s)) + + if on_retry is not None: + on_retry( + RetryInfo( + attempt=attempt, + max_attempts=config.attempts, + delay_s=delay, + error=exc, + label=label, + ) + ) + + await asyncio.sleep(delay) + + # Should never reach here, but satisfy the type checker. + assert last_exc is not None + raise last_exc + + +# ── Presets ────────────────────────────────────────────────────────── + +TELEGRAM_RETRY = RetryConfig(attempts=3, min_delay_s=0.4, max_delay_s=30.0, jitter=0.1) +DEFAULT_RETRY = RetryConfig() + +# Discord, Slack, Teams, Feishu all use the same config (attempts=3, +# min_delay_s=0.5, max_delay_s=30.0, jitter=0.1) — close enough to +# DEFAULT_RETRY that separate presets add no value. Channels that +# don't appear in RETRY_PRESETS already fall back to DEFAULT_RETRY. + +RETRY_PRESETS: dict[str, RetryConfig] = { + "telegram": TELEGRAM_RETRY, +} diff --git a/EvoScientist/channels/signal/__init__.py b/EvoScientist/channels/signal/__init__.py new file mode 100644 index 0000000..ecc05fb --- /dev/null +++ b/EvoScientist/channels/signal/__init__.py @@ -0,0 +1,29 @@ +"""Signal channel for EvoScientist. + +Uses signal-cli in JSON RPC mode — no public IP needed. + +Usage in config: + channel_enabled = "signal" + signal_phone_number = "+1234567890" +""" + +from ..channel_manager import _parse_csv, register_channel +from .channel import SignalChannel, SignalConfig + +__all__ = ["SignalChannel", "SignalConfig"] + + +def create_from_config(config) -> SignalChannel: + allowed = _parse_csv(config.signal_allowed_senders) + return SignalChannel( + SignalConfig( + phone_number=config.signal_phone_number, + cli_path=config.signal_cli_path, + config_dir=config.signal_config_dir or None, + rpc_port=config.signal_rpc_port, + allowed_senders=allowed, + ) + ) + + +register_channel("signal", create_from_config) diff --git a/EvoScientist/channels/signal/channel.py b/EvoScientist/channels/signal/channel.py new file mode 100644 index 0000000..a7aa31e --- /dev/null +++ b/EvoScientist/channels/signal/channel.py @@ -0,0 +1,510 @@ +"""Signal channel implementation using signal-cli JSON RPC.""" + +import asyncio +import json +import logging +import re +import subprocess +from collections import deque +from dataclasses import dataclass +from datetime import datetime +from typing import Any + +from ..base import Channel, ChannelError, RawIncoming +from ..capabilities import SIGNAL as SIGNAL_CAPS +from ..config import BaseChannelConfig + +logger = logging.getLogger(__name__) + + +@dataclass +class SignalConfig(BaseChannelConfig): + phone_number: str = "" + cli_path: str = "signal-cli" + config_dir: str | None = None + rpc_port: int = 7583 + text_chunk_limit: int = 4096 + + +class SignalChannel(Channel): + """Signal channel using signal-cli JSON RPC.""" + + name = "signal" + + capabilities = SIGNAL_CAPS + _non_retryable_patterns = ("unregistered", "auth") + + def __init__(self, config: SignalConfig): + super().__init__(config) + self._reader: asyncio.StreamReader | None = None + self._writer: asyncio.StreamWriter | None = None + self._rpc_id = 0 + self._daemon_proc = None + # Pending RPC responses: rpc_id -> Future + self._pending_rpcs: dict[int, asyncio.Future] = {} + # Cache message_id → sender for reaction targetAuthor (bounded) + self._msg_senders: dict[str, str] = {} + self._msg_senders_order: deque = deque(maxlen=200) + self._listen_task: asyncio.Task | None = None + + async def start(self) -> None: + if not self.config.phone_number: + raise ChannelError("Signal phone_number is required") + + # Try to start signal-cli daemon if not already running + await self._ensure_daemon() + + try: + # Connect to JSON RPC socket + await self._connect() + except Exception: + # If connect fails after daemon was started, clean up the daemon + await self._cleanup() + raise + + self._running = True + logger.info(f"Signal channel started (phone: {self.config.phone_number})") + + # Listen for incoming messages in background task + # (start() must return so that run() can iterate receive()) + self._listen_task = asyncio.create_task(self._listen_loop()) + + async def _cleanup(self) -> None: + if self._listen_task: + self._listen_task.cancel() + self._listen_task = None + # Cancel any pending RPC futures + for fut in self._pending_rpcs.values(): + if not fut.done(): + fut.cancel() + self._pending_rpcs.clear() + if self._writer: + self._writer.close() + try: + await self._writer.wait_closed() + except Exception: + pass + self._writer = None + self._reader = None + if self._daemon_proc: + self._daemon_proc.terminate() + self._daemon_proc = None + logger.info("Signal channel stopped") + + async def _ensure_daemon(self) -> None: + """Start signal-cli daemon if not already running.""" + try: + _reader, writer = await asyncio.wait_for( + asyncio.open_connection("localhost", self.config.rpc_port), + timeout=2, + ) + writer.close() + await writer.wait_closed() + logger.info("signal-cli daemon already running") + return + except (TimeoutError, ConnectionRefusedError, OSError): + pass + + # Start daemon + cmd = [self.config.cli_path, "-u", self.config.phone_number] + if self.config.config_dir: + cmd.extend(["--config", self.config.config_dir]) + cmd.extend( + [ + "daemon", + "--tcp", + f"localhost:{self.config.rpc_port}", + "--no-receive-stdout", + ] + ) + + logger.info(f"Starting signal-cli daemon: {' '.join(cmd)}") + try: + self._daemon_proc = subprocess.Popen( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except FileNotFoundError: + raise ChannelError( + f"signal-cli not found at '{self.config.cli_path}'. " + "Install: https://github.com/AsamK/signal-cli" + ) from None + + # Wait for daemon to be ready + for _ in range(30): + await asyncio.sleep(1) + try: + _reader, writer = await asyncio.open_connection( + "localhost", + self.config.rpc_port, + ) + writer.close() + await writer.wait_closed() + logger.info("signal-cli daemon started") + return + except (ConnectionRefusedError, OSError): + continue + + raise ChannelError("signal-cli daemon failed to start within 30s") + + async def _connect(self) -> None: + """Connect to signal-cli JSON RPC socket.""" + try: + self._reader, self._writer = await asyncio.open_connection( + "localhost", + self.config.rpc_port, + ) + except Exception as e: + raise ChannelError(f"Cannot connect to signal-cli: {e}") from e + + async def _listen_loop(self) -> None: + """Listen for incoming JSON RPC notifications and responses.""" + while self._running and self._reader: + try: + line = await self._reader.readline() + if not line: + break + data = json.loads(line.decode()) + # Dispatch RPC response if it has an 'id' matching a pending call + rpc_id = data.get("id") + if rpc_id is not None and rpc_id in self._pending_rpcs: + fut = self._pending_rpcs.pop(rpc_id) + if not fut.done(): + if "error" in data: + fut.set_exception( + RuntimeError(f"signal-cli RPC error: {data['error']}") + ) + else: + fut.set_result(data.get("result")) + continue + await self._handle_rpc(data) + except asyncio.CancelledError: + break + except json.JSONDecodeError: + continue + except Exception as e: + logger.error(f"Signal listen error: {e}") + # Reconnect + if self._running: + await asyncio.sleep(2) + try: + await self._connect() + except Exception: + logger.warning("Signal reconnect failed, exiting listen loop") + break + + async def _handle_rpc(self, data: dict) -> None: + """Handle a JSON RPC message from signal-cli.""" + method = data.get("method", "") + + if method != "receive": + return + + params = data.get("params", {}) + envelope = params.get("envelope", {}) + source = envelope.get("source") or envelope.get("sourceUuid") or "" + source_number = envelope.get("sourceNumber") or source + source_name = envelope.get("sourceName") or "" + timestamp = envelope.get("timestamp", 0) + + # Ignore messages from self + if ( + source_number == self.config.phone_number + or source == self.config.phone_number + ): + logger.debug("Ignoring message from self") + return + + # Data message (text) + data_msg = envelope.get("dataMessage", {}) + if data_msg: + text = data_msg.get("message", "") + group_info = data_msg.get("groupInfo", {}) + is_group = bool(group_info) + chat_id = ( + group_info.get("groupId", source_number) if is_group else source_number + ) + msg_ts = data_msg.get("timestamp", timestamp) + + media_paths: list[str] = [] + annotations: list[str] = [] + _VOICE_TYPES = { + "audio/aac", + "audio/ogg", + "audio/mp4", + "audio/mpeg", + "audio/opus", + } + attachments = data_msg.get("attachments", []) + for att in attachments: + att_size = att.get("size", 0) + att_name = att.get("filename", "attachment") + att_file = att.get("file") # signal-cli provides local path + content_type = att.get("contentType", "") + is_voice = content_type in _VOICE_TYPES or att.get("voiceNote", False) + media_label = "voice" if is_voice else "attachment" + if att_file: + from pathlib import Path as _Path + + att_path = _Path(att_file) + if att_path.exists(): + from ..base import MAX_ATTACHMENT_BYTES + + if att_path.stat().st_size > MAX_ATTACHMENT_BYTES: + annotations.append( + f"[{media_label}: {att_name} - too large ({att_path.stat().st_size} bytes)]" + ) + else: + local = self._media_path(f"signal_{att_name}") + import shutil + + shutil.copy2(str(att_path), str(local)) + media_paths.append(str(local)) + annotations.append(f"[{media_label}: {local}]") + else: + annotations.append( + f"[{media_label}: {att_name} - file not found]" + ) + elif att_size: + too_large = self._check_attachment_size(att_size, att_name) + if too_large: + annotations.append(too_large) + else: + annotations.append(f"[{media_label}: {att_name}]") + + if not text and not media_paths and not annotations: + if not attachments: + return + # Had attachments but none downloaded successfully + if not annotations: + text = "[attachment]" + + try: + ts = datetime.fromtimestamp(msg_ts / 1000) if msg_ts else datetime.now() + except (ValueError, TypeError, OSError): + ts = datetime.now() + + was_mentioned = not is_group # DMs always pass + if is_group: + mentions = data_msg.get("mentions", []) + for m in mentions: + if ( + m.get("uuid") == self.config.phone_number + or m.get("number") == self.config.phone_number + ): + was_mentioned = True + break + + # Cache message_id → sender for reaction targetAuthor + self._cache_msg_sender(str(msg_ts), source_number) + + logger.info( + "Signal message from %s: %s", + source_number, + text[:50] if text else "[media]", + ) + await self._enqueue_raw( + RawIncoming( + sender_id=source_number, + chat_id=chat_id, + text=text, + content_annotations=annotations, + media_files=media_paths, + timestamp=ts, + message_id=str(msg_ts), + is_group=is_group, + was_mentioned=was_mentioned, + metadata={ + "chat_id": chat_id, + "source_name": source_name, + "sender_id": source_number, + "backend": "signal", + }, + ) + ) + + # ── Typing indicator ──────────────────────────────────────────── + + async def _send_typing_action(self, chat_id: str) -> None: + """Send typing indicator via signal-cli JSON RPC.""" + params: dict[str, Any] = { + "account": self.config.phone_number, + } + if self._is_group_id(chat_id): + params["groupId"] = chat_id + else: + params["recipient"] = [chat_id] + try: + await self._rpc_call("sendTyping", params) + except Exception: + pass # typing indicator is best-effort + + # ── ACK reaction ───────────────────────────────────────────── + + def _cache_msg_sender(self, message_id: str, sender: str) -> None: + """Store message_id → sender mapping for reaction targetAuthor.""" + if len(self._msg_senders) >= 200: + oldest = self._msg_senders_order.popleft() + self._msg_senders.pop(oldest, None) + self._msg_senders[message_id] = sender + self._msg_senders_order.append(message_id) + + async def _send_ack_reaction( + self, chat_id: str, message_id: str, emoji: str = "👀" + ) -> None: + """Send an acknowledgment reaction via signal-cli sendReaction.""" + target_author = self._msg_senders.get(message_id, "") + if not target_author: + return # cannot send reaction without knowing the original sender + try: + params: dict[str, Any] = { + "account": self.config.phone_number, + "emoji": emoji, + "targetAuthor": target_author, + "targetTimestamp": int(message_id), + } + if self._is_group_id(chat_id): + params["groupId"] = chat_id + else: + params["recipient"] = [chat_id] + await self._rpc_call("sendReaction", params) + except Exception as e: + logger.debug(f"Signal ack reaction failed: {e}") + + async def _remove_ack_reaction( + self, chat_id: str, message_id: str, emoji: str = "👀" + ) -> None: + """Remove ACK reaction via signal-cli sendReaction --remove.""" + target_author = self._msg_senders.get(message_id, "") + if not target_author: + return + try: + params: dict[str, Any] = { + "account": self.config.phone_number, + "emoji": emoji, + "targetAuthor": target_author, + "targetTimestamp": int(message_id), + "remove": True, + } + if self._is_group_id(chat_id): + params["groupId"] = chat_id + else: + params["recipient"] = [chat_id] + await self._rpc_call("sendReaction", params) + except Exception as e: + logger.debug(f"Signal remove ACK reaction failed: {e}") + + # ── Send ────────────────────────────────────────────────────── + + @staticmethod + def _is_group_id(chat_id: str) -> bool: + """Return True if *chat_id* looks like a Signal group ID. + + Group IDs are base64-encoded strings (e.g. ``"aB3d...=="``). + Individual recipients are either phone numbers (``"+1234..."``) + or UUIDs (``"817ab5e9-..."``) — neither of which is a group. + """ + return not chat_id.startswith("+") and "-" not in chat_id + + def _is_ready(self) -> bool: + return self._writer is not None and not self._writer.is_closing() + + async def _rpc_call( + self, method: str, params: dict, timeout: float = 10.0 + ) -> dict | None: + """Send a JSON RPC call to signal-cli and wait for the response.""" + if not self._writer: + return None + + self._rpc_id += 1 + rpc_id = self._rpc_id + request = { + "jsonrpc": "2.0", + "id": rpc_id, + "method": method, + "params": params, + } + + # Register a Future before sending so the listen loop can resolve it + loop = asyncio.get_running_loop() + fut: asyncio.Future = loop.create_future() + self._pending_rpcs[rpc_id] = fut + + line = json.dumps(request) + "\n" + self._writer.write(line.encode()) + await self._writer.drain() + + try: + return await asyncio.wait_for(fut, timeout=timeout) + except TimeoutError: + self._pending_rpcs.pop(rpc_id, None) + logger.warning(f"Signal RPC '{method}' timed out after {timeout}s") + return None + + async def _send_chunk( + self, + chat_id, + formatted_text, + raw_text, + reply_to, + metadata, + ): + # Determine if group or individual + params: dict[str, Any] = { + "message": raw_text, + "account": self.config.phone_number, + } + + if self._is_group_id(chat_id): + params["groupId"] = chat_id + else: + params["recipient"] = [chat_id] + + await self._rpc_call("send", params) + + # ── Mention stripping ──────────────────────────────────────────── + + def _strip_mention(self, text: str) -> str: + """Strip bot mention from Signal messages. + + Signal mentions are embedded as special objects that reference + the phone number. The text contains a placeholder character (U+FFFC) + at the mention position. + """ + phone = self.config.phone_number + if phone: + # Remove phone number if directly mentioned as text + text = re.sub(rf"@?{re.escape(phone)}\s*", "", text).strip() + # Remove Unicode Object Replacement Character used as mention placeholder + text = text.replace("\ufffc", "").strip() + return text + + # ── Media send ──────────────────────────────────────────────── + + async def _send_media_impl( + self, + recipient: str, + file_path: str, + caption: str = "", + metadata: dict | None = None, + ) -> bool: + """Send a media file via signal-cli JSON RPC. + + Uses the "send" RPC method with the attachments parameter. + """ + chat_id = self._resolve_media_chat_id(recipient, metadata) + params: dict[str, Any] = { + "account": self.config.phone_number, + "attachments": [file_path], + } + if caption: + params["message"] = caption + + if self._is_group_id(chat_id): + params["groupId"] = chat_id + else: + params["recipient"] = [chat_id] + + await self._rpc_call("send", params) + return True diff --git a/EvoScientist/channels/signal/probe.py b/EvoScientist/channels/signal/probe.py new file mode 100644 index 0000000..bcc73b9 --- /dev/null +++ b/EvoScientist/channels/signal/probe.py @@ -0,0 +1,43 @@ +"""Signal credential validation.""" + +import logging + +logger = logging.getLogger(__name__) + + +async def validate_signal( + phone_number: str, + cli_path: str = "signal-cli", + rpc_port: int = 7583, +) -> tuple[bool, str]: + """Validate Signal setup by checking signal-cli availability. + + Returns: + Tuple of (is_valid, message). + """ + import asyncio + import subprocess + + if not phone_number: + return False, "phone_number is required" + + # Check signal-cli binary + loop = asyncio.get_event_loop() + + def _check(): + try: + result = subprocess.run( + [cli_path, "--version"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + return True, f"signal-cli {result.stdout.strip()}" + return False, "signal-cli returned error" + except FileNotFoundError: + return False, f"signal-cli not found at '{cli_path}'" + except Exception as e: + return False, f"Error: {e}" + + return await loop.run_in_executor(None, _check) diff --git a/EvoScientist/channels/signal/serve.py b/EvoScientist/channels/signal/serve.py new file mode 100644 index 0000000..1d3e199 --- /dev/null +++ b/EvoScientist/channels/signal/serve.py @@ -0,0 +1,96 @@ +"""Signal channel server. + +Standalone script to run the Signal channel with CLI options. + +Usage: + python -m EvoScientist.channels.signal.serve --phone-number NUMBER [OPTIONS] + +Examples: + # Basic usage + python -m EvoScientist.channels.signal.serve --phone-number +1234567890 + + # With custom signal-cli path and allowed senders + python -m EvoScientist.channels.signal.serve --phone-number +1234567890 --cli-path /usr/local/bin/signal-cli --allow +9876543210 + + # With agent and thinking + python -m EvoScientist.channels.signal.serve --phone-number +1234567890 --agent --thinking +""" + +import argparse +import logging + +from ...logging_config import configure_logging_from_settings +from ..bus import MessageBus +from ..standalone import run_standalone +from .channel import SignalChannel, SignalConfig + +logger = logging.getLogger(__name__) + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Signal channel server", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--phone-number", + required=True, + help="Signal phone number (e.g. +1234567890)", + ) + parser.add_argument( + "--cli-path", + default="signal-cli", + help="Path to signal-cli binary (default: signal-cli)", + ) + parser.add_argument( + "--config-dir", + help="signal-cli config directory", + ) + parser.add_argument( + "--rpc-port", + type=int, + default=7583, + help="signal-cli JSON RPC port (default: 7583)", + ) + parser.add_argument( + "--allow", + action="append", + dest="allowed_senders", + help="Allowed sender (phone number). Can be used multiple times.", + ) + parser.add_argument( + "--agent", + action="store_true", + help="Use EvoScientist agent as handler (default: echo)", + ) + parser.add_argument( + "--thinking", + action="store_true", + help="Send thinking content as intermediate messages (requires --agent)", + ) + return parser.parse_args() + + +def main(): + """Entry point.""" + configure_logging_from_settings(default_level=logging.INFO) + args = parse_args() + + config = SignalConfig( + phone_number=args.phone_number, + cli_path=args.cli_path, + config_dir=args.config_dir, + rpc_port=args.rpc_port, + allowed_senders=set(args.allowed_senders) if args.allowed_senders else None, + ) + + send_thinking = args.thinking and args.agent + bus = MessageBus() + channel = SignalChannel(config) + + run_standalone(channel, bus, use_agent=args.agent, send_thinking=send_thinking) + + +if __name__ == "__main__": + main() diff --git a/EvoScientist/channels/slack/__init__.py b/EvoScientist/channels/slack/__init__.py new file mode 100644 index 0000000..0520371 --- /dev/null +++ b/EvoScientist/channels/slack/__init__.py @@ -0,0 +1,22 @@ +from ..channel_manager import _parse_csv, register_channel +from .channel import SlackChannel, SlackConfig + +__all__ = ["SlackChannel", "SlackConfig"] + + +def create_from_config(config) -> SlackChannel: + allowed = _parse_csv(config.slack_allowed_senders) + channels = _parse_csv(config.slack_allowed_channels) + proxy = config.slack_proxy or None + return SlackChannel( + SlackConfig( + bot_token=config.slack_bot_token, + app_token=config.slack_app_token, + allowed_senders=allowed, + allowed_channels=channels, + proxy=proxy, + ) + ) + + +register_channel("slack", create_from_config) diff --git a/EvoScientist/channels/slack/channel.py b/EvoScientist/channels/slack/channel.py new file mode 100644 index 0000000..ff2a031 --- /dev/null +++ b/EvoScientist/channels/slack/channel.py @@ -0,0 +1,302 @@ +"""Slack channel implementation using slack-sdk Socket Mode.""" + +import asyncio +import logging +from dataclasses import dataclass +from datetime import datetime + +from ..base import Channel, ChannelError, RawIncoming +from ..capabilities import SLACK as SLACK_CAPS +from ..config import BaseChannelConfig + +logger = logging.getLogger(__name__) + + +@dataclass +class SlackConfig(BaseChannelConfig): + bot_token: str = "" + app_token: str = "" + text_chunk_limit: int = 4096 + + +class SlackChannel(Channel): + """Slack channel using slack-sdk Socket Mode.""" + + name = "slack" + + capabilities = SLACK_CAPS + _ready_attrs = ("_web_client",) + _mention_pattern = r"<@{bot_id}>\s*" + + def __init__(self, config: SlackConfig): + super().__init__(config) + self._socket_client = None + self._web_client = None + self._typing_message_ts: dict[str, str] = {} + + async def start(self) -> None: + if not self.config.bot_token: + raise ChannelError("Slack bot token is required") + if not self.config.app_token: + raise ChannelError( + "Slack app token is required for Socket Mode (starts with xapp-)" + ) + + try: + from slack_sdk.socket_mode.aiohttp import SocketModeClient + from slack_sdk.socket_mode.request import SocketModeRequest + from slack_sdk.socket_mode.response import SocketModeResponse + from slack_sdk.web.async_client import AsyncWebClient + except ImportError: + raise ChannelError( + "slack-sdk or aiohttp not installed. " + "Install with: pip install evoscientist[slack]" + ) from None + + self._web_client = AsyncWebClient( + token=self.config.bot_token, + proxy=self._get_proxy(), + ) + + # Get bot user ID for filtering own messages + try: + auth = await asyncio.wait_for( + self._web_client.auth_test(), + timeout=15, + ) + self._bot_user_id = auth["user_id"] + except TimeoutError: + raise ChannelError( + "Slack auth_test timed out — check network and bot token" + ) from None + except Exception as e: + raise ChannelError(f"Failed to authenticate Slack bot: {e}") from e + + self._socket_client = SocketModeClient( + app_token=self.config.app_token, + web_client=self._web_client, + ) + + async def _event_handler( + client: SocketModeClient, + req: SocketModeRequest, + ) -> None: + # Acknowledge immediately + resp = SocketModeResponse(envelope_id=req.envelope_id) + await client.send_socket_mode_response(resp) + + logger.debug(f"Slack socket event: type={req.type}") + + if req.type == "events_api": + event = req.payload.get("event", {}) + event_type = event.get("type", "") + if event_type == "message" and "subtype" not in event: + is_dm = event.get("channel_type") == "im" + await self._on_message( + event, + is_group=not is_dm, + was_mentioned=is_dm, + ) + elif event_type == "app_mention": + await self._on_message( + event, + is_group=True, + was_mentioned=True, + ) + + self._socket_client.socket_mode_request_listeners.append(_event_handler) + try: + await asyncio.wait_for( + self._socket_client.connect(), + timeout=30, + ) + except TimeoutError: + raise ChannelError( + "Slack Socket Mode connection timed out — " + "check app token (must start with xapp-) and " + "ensure Socket Mode is enabled in your Slack app settings" + ) from None + self._running = True + logger.info("Slack channel started (Socket Mode)") + + async def _cleanup(self) -> None: + if self._socket_client: + await self._socket_client.close() + logger.info("Slack channel stopped") + + # ── Typing indicator (override base) ──────────────────────────── + + async def _send_typing_action(self, chat_id: str) -> None: + """Send typing indicator via Slack. + + Slack's Web API and Socket Mode do not expose a dedicated + typing-indicator endpoint for bot tokens. We approximate + the experience by posting a short-lived status message that + is deleted once the real reply is sent (handled by + ``stop_typing``). When the status post fails we silently + fall back to no indicator. + """ + if not self._web_client: + return + try: + resp = await self._web_client.chat_postMessage( + channel=chat_id, + text="\u2026", # "…" ellipsis as minimal typing hint + ) + ts = resp.get("ts") + if ts: + self._typing_message_ts[chat_id] = ts + except Exception: + pass + + async def stop_typing(self, chat_id: str) -> None: + """Cancel typing loop and clean up the status message.""" + # Delete the ephemeral "…" message if we posted one + ts = self._typing_message_ts.pop(chat_id, None) + if ts and self._web_client: + try: + await self._web_client.chat_delete(channel=chat_id, ts=ts) + except Exception: + pass + await super().stop_typing(chat_id) + + # ── Send (template method overrides) ────────────────────────── + + async def _send_chunk(self, chat_id, formatted_text, raw_text, reply_to, metadata): + kwargs = {"channel": chat_id} + # Always route to thread if thread_ts is present in metadata, + # not just for the first chunk (reply_to is only set for chunk 0). + if metadata: + thread_ts = metadata.get("thread_ts") + if thread_ts: + kwargs["thread_ts"] = thread_ts + + async def _send(text): + await self._web_client.chat_postMessage(text=text, **kwargs) + + await self._send_with_format_fallback(_send, formatted_text, raw_text) + + async def _send_media_impl( + self, + recipient: str, + file_path: str, + caption: str = "", + metadata: dict | None = None, + ) -> bool: + """Send a media file through Slack.""" + channel_id = self._resolve_media_chat_id(recipient, metadata) + await self._web_client.files_upload_v2( + channel=channel_id, + file=file_path, + initial_comment=caption or None, + ) + return True + + def _get_bot_identifier(self) -> str | None: + return getattr(self, "_bot_user_id", None) + + # ── ACK Reactions ─────────────────────────────────────────────── + + async def _send_ack_reaction( + self, chat_id: str, message_id: str, emoji: str = "eyes" + ) -> None: + """Add an emoji reaction to acknowledge receipt.""" + if self._web_client and message_id: + try: + await self._web_client.reactions_add( + channel=chat_id, + timestamp=message_id, + name=emoji, + ) + except Exception as e: + logger.debug(f"Slack ACK reaction failed: {e}") + + async def _remove_ack_reaction( + self, chat_id: str, message_id: str, emoji: str = "eyes" + ) -> None: + """Remove the ACK reaction after replying.""" + if self._web_client and message_id: + try: + await self._web_client.reactions_remove( + channel=chat_id, + timestamp=message_id, + name=emoji, + ) + except Exception as e: + logger.debug(f"Slack remove ACK reaction failed: {e}") + + async def _on_message( + self, + event: dict, + *, + is_group: bool = False, + was_mentioned: bool = True, + ) -> None: + """Handle an incoming Slack message event.""" + user_id = event.get("user", "") + + # Skip bot's own messages + if user_id == getattr(self, "_bot_user_id", None): + logger.debug("Skipping own bot message") + return + + # Skip bot messages (e.g. from other bots) + if event.get("bot_id"): + logger.debug(f"Skipping bot message from bot_id={event.get('bot_id')}") + return + + channel_id = event.get("channel", "") + + text = event.get("text", "") + + annotations: list[str] = [] + media_paths: list[str] = [] + + # Handle file attachments + if self.config.include_attachments: + files = event.get("files", []) + for file_info in files: + file_size = file_info.get("size", 0) + filename = file_info.get("name", "unknown") + + url = file_info.get("url_private_download") or file_info.get( + "url_private" + ) + if url and self._web_client: + headers = {"Authorization": f"Bearer {self.config.bot_token}"} + local_path, annotation = await self._download_attachment( + url, + f"{file_info.get('id', 'unknown')}_{filename}", + headers=headers, + file_size=file_size, + ) + if local_path: + media_paths.append(local_path) + if annotation: + annotations.append(annotation) + + ts = event.get("ts", "") + thread_ts = event.get("thread_ts") or ts + try: + timestamp = datetime.fromtimestamp(float(ts)) if ts else datetime.now() + except (ValueError, TypeError): + timestamp = datetime.now() + + await self._enqueue_raw( + RawIncoming( + sender_id=user_id, + chat_id=channel_id, + text=text, + media_files=media_paths, + content_annotations=annotations, + timestamp=timestamp, + message_id=ts, + metadata={"chat_id": channel_id, "thread_ts": thread_ts}, + is_group=is_group, + was_mentioned=was_mentioned, + ) + ) + logger.info( + f"Slack message queued: sender={user_id}, " + f"channel={channel_id}, content={text[:50]}" + ) diff --git a/EvoScientist/channels/slack/probe.py b/EvoScientist/channels/slack/probe.py new file mode 100644 index 0000000..fa6d260 --- /dev/null +++ b/EvoScientist/channels/slack/probe.py @@ -0,0 +1,48 @@ +"""Slack bot token validation.""" + +import logging + +logger = logging.getLogger(__name__) + + +async def validate_slack_tokens( + bot_token: str, + app_token: str | None = None, + proxy: str | None = None, +) -> tuple[bool, str]: + """Validate Slack bot token via the auth.test API. + + Optionally checks the app-level token format (must start with ``xapp-``). + + Returns: + Tuple of (is_valid, message). + """ + if not bot_token: + return False, "No bot token provided" + + try: + import httpx + except ImportError: + return False, "httpx not installed" + + # Validate bot token via auth.test + url = "https://slack.com/api/auth.test" + headers = {"Authorization": f"Bearer {bot_token}"} + try: + async with httpx.AsyncClient(proxy=proxy) as client: + resp = await client.post(url, headers=headers, timeout=10) + data = resp.json() + if not data.get("ok"): + error = data.get("error", "unknown error") + return False, f"Invalid bot token: {error}" + bot_name = data.get("user", "unknown") + team = data.get("team", "unknown") + except Exception as e: + return False, f"Error: {e}" + + # Optionally validate app token format + if app_token: + if not app_token.startswith("xapp-"): + return False, "App token must start with 'xapp-'" + + return True, f"Bot: {bot_name} (team: {team})" diff --git a/EvoScientist/channels/slack/serve.py b/EvoScientist/channels/slack/serve.py new file mode 100644 index 0000000..57920f7 --- /dev/null +++ b/EvoScientist/channels/slack/serve.py @@ -0,0 +1,96 @@ +"""Slack channel server. + +Standalone script to run the Slack channel with CLI options. + +Usage: + python -m EvoScientist.channels.slack.serve --bot-token TOKEN --app-token TOKEN [OPTIONS] + +Examples: + # Allow all senders (default) + python -m EvoScientist.channels.slack.serve --bot-token xoxb-... --app-token xapp-... + + # Only allow specific senders and channels + python -m EvoScientist.channels.slack.serve --bot-token xoxb-... --app-token xapp-... --allow U123 --allow-channel C456 + + # With proxy, agent and thinking + python -m EvoScientist.channels.slack.serve --bot-token xoxb-... --app-token xapp-... --proxy http://proxy:8080 --agent --thinking +""" + +import argparse +import logging + +from ...logging_config import configure_logging_from_settings +from ..bus import MessageBus +from ..standalone import run_standalone +from .channel import SlackChannel, SlackConfig + +logger = logging.getLogger(__name__) + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Slack channel server", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--bot-token", + required=True, + help="Slack bot token (xoxb-...)", + ) + parser.add_argument( + "--app-token", + required=True, + help="Slack app-level token for Socket Mode (xapp-...)", + ) + parser.add_argument( + "--allow", + action="append", + dest="allowed_senders", + help="Allowed sender (Slack user ID). Can be used multiple times.", + ) + parser.add_argument( + "--allow-channel", + action="append", + dest="allowed_channels", + help="Allowed channel ID. Can be used multiple times.", + ) + parser.add_argument( + "--proxy", + help="HTTP proxy URL for Slack API requests", + ) + parser.add_argument( + "--agent", + action="store_true", + help="Use EvoScientist agent as handler (default: echo)", + ) + parser.add_argument( + "--thinking", + action="store_true", + help="Send thinking content as intermediate messages (requires --agent)", + ) + return parser.parse_args() + + +def main(): + """Entry point.""" + configure_logging_from_settings(default_level=logging.INFO) + args = parse_args() + + config = SlackConfig( + bot_token=args.bot_token, + app_token=args.app_token, + allowed_senders=set(args.allowed_senders) if args.allowed_senders else None, + allowed_channels=set(args.allowed_channels) if args.allowed_channels else None, + proxy=args.proxy, + ) + + send_thinking = args.thinking and args.agent + bus = MessageBus() + channel = SlackChannel(config) + + run_standalone(channel, bus, use_agent=args.agent, send_thinking=send_thinking) + + +if __name__ == "__main__": + main() diff --git a/EvoScientist/channels/standalone.py b/EvoScientist/channels/standalone.py new file mode 100644 index 0000000..cdd8d10 --- /dev/null +++ b/EvoScientist/channels/standalone.py @@ -0,0 +1,183 @@ +"""Shared standalone runner for channel servers. + +Provides the channel-agnostic agent loop that any channel can use to +run headless — consuming inbound messages from the bus, streaming +agent events, and dispatching outbound replies. + +Usage from a channel's ``main()``:: + + from EvoScientist.channels.standalone import run_standalone + + channel = SomeChannel(config) + bus = MessageBus() + run_standalone(channel, bus, use_agent=True, send_thinking=True) +""" + +import asyncio +import logging +import signal + +from .base import Channel +from .bus import MessageBus +from .bus.events import OutboundMessage +from .consumer import InboundConsumer +from .debug import emit_debug_event + +logger = logging.getLogger(__name__) + + +def _channel_trace_enabled(channel: Channel) -> bool: + """Check if debug tracing is enabled on the channel.""" + try: + return channel.is_debug_trace_enabled() + except Exception: + return False + + +async def _deliver_outbound(channel: Channel, msg: OutboundMessage) -> None: + """Deliver an outbound message, including any media attachments.""" + if msg.content: + sent = await channel.send(msg) + if not sent: + raise RuntimeError("send() returned False") + for media_path in msg.media: + media_ok = await channel.send_media( + recipient=msg.chat_id, + file_path=media_path, + metadata=msg.metadata, + ) + if not media_ok: + raise RuntimeError(f"send_media() returned False for {media_path}") + + +async def standalone_outbound_dispatcher( + bus: MessageBus, + channel: Channel, +) -> None: + """Consume outbound messages from the bus and send via channel.""" + while True: + try: + msg: OutboundMessage = await asyncio.wait_for( + bus.consume_outbound(), + timeout=1.0, + ) + except TimeoutError: + continue + except asyncio.CancelledError: + break + + try: + await _deliver_outbound(channel, msg) + except Exception as e: + emit_debug_event( + logger, + "standalone_dispatch_error", + channel=channel.name, + enabled=_channel_trace_enabled(channel), + recipient=msg.recipient, + error=str(e), + ) + logger.error(f"Error sending outbound: {e}") + + +async def _async_main( + channel: Channel, + bus: MessageBus, + use_agent: bool, + send_thinking: bool, +) -> None: + """Async entry point — gather channel, dispatcher and optional consumer.""" + from .channel_manager import ChannelManager + + channel.set_bus(bus) + if send_thinking: + channel.send_thinking = True + + # Create a lightweight manager for the consumer to use + manager = ChannelManager(bus) + manager._channels[channel.name] = channel + + await manager.start_health() + + tasks = [channel.run()] + + dispatcher = standalone_outbound_dispatcher(bus, channel) + tasks.append(dispatcher) + + consumer: InboundConsumer | None = None + if use_agent: + logger.info("Loading EvoScientist agent...") + from ..EvoScientist import create_cli_agent + + agent = create_cli_agent() + logger.info("Agent loaded") + + consumer = InboundConsumer( + bus=bus, + manager=manager, + agent=agent, + thread_id="", + send_thinking=send_thinking, + ) + manager.register_health_provider("consumer", lambda: consumer.metrics) + tasks.append(consumer.run()) + if send_thinking: + logger.info("Thinking messages enabled") + + async def _graceful_shutdown() -> None: + """Graceful shutdown: drain consumer, flush outbound, stop channel.""" + logger.info("Graceful shutdown initiated...") + if consumer is not None: + await consumer.stop() + # Drain outbound queue before stopping the channel + drained = 0 + while True: + try: + msg = bus.outbound.get_nowait() + except asyncio.QueueEmpty: + break + try: + await asyncio.wait_for(_deliver_outbound(channel, msg), timeout=5.0) + if msg.content or msg.media: + drained += 1 + except Exception: + pass + if drained: + logger.info(f"Outbound drain: {drained} sent") + channel._running = False + await channel.stop() + await manager.stop_health() + + loop = asyncio.get_event_loop() + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler( + sig, + lambda s=sig: asyncio.create_task(_graceful_shutdown()), + ) + + await asyncio.gather(*tasks) + + +def run_standalone( + channel: Channel, + bus: MessageBus, + *, + use_agent: bool = False, + send_thinking: bool = False, +) -> None: + """Synchronous entry point that spins up the standalone runner. + + Parameters + ---------- + channel: + A fully-configured :class:`Channel` instance. + bus: + The :class:`MessageBus` shared with *channel*. + use_agent: + When ``True``, load the EvoScientist agent and process inbound + messages through it. + send_thinking: + When ``True`` **and** *use_agent* is set, forward intermediate + thinking messages to the channel. + """ + asyncio.run(_async_main(channel, bus, use_agent, send_thinking)) diff --git a/EvoScientist/channels/telegram/__init__.py b/EvoScientist/channels/telegram/__init__.py new file mode 100644 index 0000000..82277b5 --- /dev/null +++ b/EvoScientist/channels/telegram/__init__.py @@ -0,0 +1,19 @@ +from ..channel_manager import _parse_csv, register_channel +from .channel import TelegramChannel, TelegramConfig + +__all__ = ["TelegramChannel", "TelegramConfig"] + + +def create_from_config(config) -> TelegramChannel: + allowed = _parse_csv(config.telegram_allowed_senders) + proxy = config.telegram_proxy or None + return TelegramChannel( + TelegramConfig( + bot_token=config.telegram_bot_token, + allowed_senders=allowed, + proxy=proxy, + ) + ) + + +register_channel("telegram", create_from_config) diff --git a/EvoScientist/channels/telegram/channel.py b/EvoScientist/channels/telegram/channel.py new file mode 100644 index 0000000..e29059a --- /dev/null +++ b/EvoScientist/channels/telegram/channel.py @@ -0,0 +1,315 @@ +"""Telegram channel implementation using python-telegram-bot.""" + +import logging +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import ClassVar + +from ..base import ( + AUDIO_EXTS, + IMAGE_EXTS, + VIDEO_EXTS, + Channel, + ChannelError, + RawIncoming, +) +from ..capabilities import TELEGRAM as TELEGRAM_CAPS +from ..config import BaseChannelConfig + +logger = logging.getLogger(__name__) + + +@dataclass +class TelegramConfig(BaseChannelConfig): + bot_token: str = "" + text_chunk_limit: int = 4096 + + +class TelegramChannel(Channel): + """Telegram channel using python-telegram-bot with long polling.""" + + name = "telegram" + + capabilities = TELEGRAM_CAPS + _typing_interval: float = 4.0 + _ready_attrs = ("_app",) + _non_retryable_patterns = ("parse", "can't parse") + _mention_pattern = r"(?i)@{bot_id}\s*" + + def __init__(self, config: TelegramConfig): + super().__init__(config) + self._app = None + self._bot_username: str = "" + + async def start(self) -> None: + if not self.config.bot_token: + raise ChannelError("Telegram bot token is required") + + try: + from telegram.ext import ( + ApplicationBuilder, + MessageHandler, + filters, + ) + except ImportError: + raise ChannelError( + "python-telegram-bot not installed. " + "Install with: pip install evoscientist[telegram]" + ) from None + + builder = ApplicationBuilder().token(self.config.bot_token) + + if self.config.proxy: + builder = builder.proxy(self.config.proxy).get_updates_proxy( + self.config.proxy + ) + self._app = builder.build() + + # Accept text and media message types + media_filter = filters.TEXT + if self.config.include_attachments: + media_filter = ( + filters.TEXT + | filters.PHOTO + | filters.VOICE + | filters.AUDIO + | filters.Document.ALL + | filters.VIDEO + | filters.Sticker.ALL + | filters.LOCATION + ) + + self._app.add_handler( + MessageHandler(media_filter & ~filters.COMMAND, self._on_message) + ) + + await self._app.initialize() + # Cache bot username for @mention detection in groups + bot_info = await self._app.bot.get_me() + self._bot_username = (bot_info.username or "").lower() + await self._app.start() + await self._app.updater.start_polling(drop_pending_updates=True) + self._running = True + logger.info("Telegram channel started (polling)") + + async def _cleanup(self) -> None: + if self._app: + if self._app.updater and self._app.updater.running: + await self._app.updater.stop() + await self._app.stop() + await self._app.shutdown() + logger.info("Telegram channel stopped") + + # ── Typing indicator (override base) ──────────────────────────── + + async def _send_typing_action(self, chat_id: str) -> None: + """Send typing action via Telegram Bot API.""" + if self._app: + await self._app.bot.send_chat_action( + chat_id=int(chat_id), + action="typing", + ) + + # ── Send (template method overrides) ────────────────────────── + + async def _send_chunk(self, chat_id, formatted_text, raw_text, reply_to, metadata): + reply_id = int(reply_to) if reply_to else None + + async def _send(text): + await self._app.bot.send_message( + chat_id=int(chat_id), + text=text, + parse_mode="HTML" if text == formatted_text else None, + reply_to_message_id=reply_id, + ) + + await self._send_with_format_fallback(_send, formatted_text, raw_text) + + _MEDIA_SENDERS: ClassVar[dict] = { + IMAGE_EXTS: ("send_photo", "photo"), + VIDEO_EXTS: ("send_video", "video"), + AUDIO_EXTS: ("send_audio", "audio"), + } + + async def _send_media_impl( + self, + recipient: str, + file_path: str, + caption: str = "", + metadata: dict | None = None, + ) -> bool: + """Send a media file through Telegram.""" + chat_id = int(self._resolve_media_chat_id(recipient, metadata)) + cap = caption or None + ext = Path(file_path).suffix.lower() + for exts, (method, param) in self._MEDIA_SENDERS.items(): + if ext in exts: + await getattr(self._app.bot, method)( + chat_id=chat_id, + caption=cap, + **{param: file_path}, + ) + return True + await self._app.bot.send_document( + chat_id=chat_id, + document=file_path, + caption=cap, + ) + return True + + def _get_bot_identifier(self) -> str | None: + return self._bot_username or None + + async def _send_ack_reaction( + self, chat_id: str, message_id: str, emoji: str = "👀" + ) -> None: + """Send an acknowledgment reaction via Telegram.""" + if self._app: + try: + from telegram import ReactionTypeEmoji + + await self._app.bot.set_message_reaction( + chat_id=int(chat_id), + message_id=int(message_id), + reaction=[ReactionTypeEmoji(emoji)], + ) + except Exception as e: + logger.debug(f"Telegram ACK reaction failed: {e}") + + async def _remove_ack_reaction( + self, chat_id: str, message_id: str, emoji: str = "👀" + ) -> None: + """Remove the ack reaction by setting empty reaction list.""" + if self._app: + try: + await self._app.bot.set_message_reaction( + chat_id=int(chat_id), + message_id=int(message_id), + reaction=[], + ) + except Exception as e: + logger.debug(f"Telegram remove ACK reaction failed: {e}") + + async def _on_message(self, update, context) -> None: + """Handler callback for text, photos, voice, audio, documents, video.""" + if not update.message: + return + + message = update.message + user_id = str(message.from_user.id) + chat_id = str(message.chat_id) + + # Detect group and mention status for centralized gating + is_group = message.chat.type in ("group", "supergroup") + was_mentioned = True # DM default + if is_group and self._bot_username: + text_check = (message.text or message.caption or "").lower() + was_mentioned = f"@{self._bot_username}" in text_check + + content_parts: list[str] = [] + media_paths: list[str] = [] + + # Text content + if message.text: + content_parts.append(message.text) + if message.caption: + content_parts.append(message.caption) + + # Handle media files + annotations: list[str] = [] + if self.config.include_attachments: + media_file = None + media_type = None + + if message.photo: + media_file = message.photo[-1] # Largest size + media_type = "image" + elif message.voice: + media_file = message.voice + media_type = "voice" + elif message.audio: + media_file = message.audio + media_type = "audio" + elif message.video: + media_file = message.video + media_type = "video" + elif message.document: + media_file = message.document + media_type = "file" + elif message.sticker: + media_file = message.sticker + media_type = "sticker" + + # Location is not a downloadable file — handle separately + if message.location and not media_file: + loc = message.location + annotations.append(f"[位置] ({loc.latitude}, {loc.longitude})") + + if media_file and self._app: + file_size = getattr(media_file, "file_size", 0) or 0 + too_large = self._check_attachment_size(file_size, media_type) + if too_large: + annotations.append(too_large) + else: + try: + file = await self._app.bot.get_file( + media_file.file_id, + ) + ext = self._get_extension( + media_type, + getattr(media_file, "mime_type", None), + ) + file_path = self._media_path(f"{media_file.file_id[:16]}{ext}") + await file.download_to_drive(str(file_path)) + + media_paths.append(str(file_path)) + annotations.append(f"[{media_type}: {file_path}]") + logger.debug(f"Downloaded {media_type} to {file_path}") + except Exception as e: + logger.error(f"Failed to download media: {e}") + annotations.append(f"[{media_type}: download failed]") + + text_content = "\n".join(content_parts) if content_parts else "" + + await self._enqueue_raw( + RawIncoming( + sender_id=user_id, + chat_id=chat_id, + text=text_content, + media_files=media_paths, + content_annotations=annotations, + timestamp=message.date or datetime.now(), + message_id=str(message.message_id), + metadata={"chat_id": chat_id}, + is_group=is_group, + was_mentioned=was_mentioned, + ) + ) + + _MIME_TO_EXT: ClassVar[dict[str, str]] = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/gif": ".gif", + "image/webp": ".webp", + "audio/ogg": ".ogg", + "audio/mpeg": ".mp3", + "audio/mp4": ".m4a", + "video/mp4": ".mp4", + "video/quicktime": ".mov", + } + _TYPE_TO_EXT: ClassVar[dict[str, str]] = { + "image": ".jpg", + "voice": ".ogg", + "audio": ".mp3", + "video": ".mp4", + "file": "", + "sticker": ".webp", + } + + @staticmethod + def _get_extension(media_type: str, mime_type: str | None) -> str: + """Get file extension based on media type and MIME type.""" + if mime_type and mime_type in TelegramChannel._MIME_TO_EXT: + return TelegramChannel._MIME_TO_EXT[mime_type] + return TelegramChannel._TYPE_TO_EXT.get(media_type, "") diff --git a/EvoScientist/channels/telegram/probe.py b/EvoScientist/channels/telegram/probe.py new file mode 100644 index 0000000..037d6bf --- /dev/null +++ b/EvoScientist/channels/telegram/probe.py @@ -0,0 +1,34 @@ +"""Telegram bot token validation.""" + +import logging + +logger = logging.getLogger(__name__) + + +async def validate_telegram_token( + token: str, proxy: str | None = None +) -> tuple[bool, str]: + """Validate a Telegram bot token via the getMe API. + + Returns: + Tuple of (is_valid, message). + """ + if not token: + return False, "No token provided" + + try: + import httpx + except ImportError: + return False, "httpx not installed" + + url = f"https://api.telegram.org/bot{token}/getMe" + try: + async with httpx.AsyncClient(proxy=proxy) as client: + resp = await client.get(url, timeout=10) + data = resp.json() + if data.get("ok"): + username = data["result"].get("username", "unknown") + return True, f"Bot: @{username}" + return False, "Invalid token" + except Exception as e: + return False, f"Error: {e}" diff --git a/EvoScientist/channels/telegram/serve.py b/EvoScientist/channels/telegram/serve.py new file mode 100644 index 0000000..6348b42 --- /dev/null +++ b/EvoScientist/channels/telegram/serve.py @@ -0,0 +1,78 @@ +"""Telegram channel server. + +Standalone script to run the Telegram channel with CLI options. + +Usage: + python -m EvoScientist.channels.telegram.serve --bot-token TOKEN [OPTIONS] + +Examples: + # Allow all senders (default) + python -m EvoScientist.channels.telegram.serve --bot-token TOKEN + + # Only allow specific senders + python -m EvoScientist.channels.telegram.serve --bot-token TOKEN --allow 123456 --allow 789012 + + # With agent and thinking + python -m EvoScientist.channels.telegram.serve --bot-token TOKEN --agent --thinking +""" + +import argparse +import logging + +from ...logging_config import configure_logging_from_settings +from ..bus import MessageBus +from ..standalone import run_standalone +from .channel import TelegramChannel, TelegramConfig + +logger = logging.getLogger(__name__) + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Telegram channel server", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--bot-token", + required=True, + help="Telegram bot token from @BotFather", + ) + parser.add_argument( + "--allow", + action="append", + dest="allowed_senders", + help="Allowed sender (Telegram user ID). Can be used multiple times.", + ) + parser.add_argument( + "--agent", + action="store_true", + help="Use EvoScientist agent as handler (default: echo)", + ) + parser.add_argument( + "--thinking", + action="store_true", + help="Send thinking content as intermediate messages (requires --agent)", + ) + return parser.parse_args() + + +def main(): + """Entry point.""" + configure_logging_from_settings(default_level=logging.INFO) + args = parse_args() + + config = TelegramConfig( + bot_token=args.bot_token, + allowed_senders=set(args.allowed_senders) if args.allowed_senders else None, + ) + + send_thinking = args.thinking and args.agent + bus = MessageBus() + channel = TelegramChannel(config) + + run_standalone(channel, bus, use_agent=args.agent, send_thinking=send_thinking) + + +if __name__ == "__main__": + main() diff --git a/EvoScientist/channels/wechat/__init__.py b/EvoScientist/channels/wechat/__init__.py new file mode 100644 index 0000000..9534812 --- /dev/null +++ b/EvoScientist/channels/wechat/__init__.py @@ -0,0 +1,69 @@ +"""WeChat channel implementations for EvoScientist. + +Supports multiple WeChat backends: + - **wecom**: 企业微信应用 (WeCom / WeChat Work) via official API + — Most stable, pure HTTP, no third-party dependencies + - **wechatmp**: 微信公众号 (WeChat Official Account) via official API + — Pure HTTP webhook, suitable for public-facing bots + +Both backends use httpx (already a core dependency) and receive messages +via HTTP webhook, send replies via REST API. + +Usage in config: + channel_enabled = "wechat" + wechat_backend = "wecom" # or "wechatmp" + + # WeCom settings + wechat_wecom_corp_id = "..." + wechat_wecom_agent_id = "..." + wechat_wecom_secret = "..." + wechat_wecom_token = "..." + wechat_wecom_encoding_aes_key = "..." + wechat_webhook_port = 9001 + + # OR: Official Account settings + wechat_mp_app_id = "..." + wechat_mp_app_secret = "..." + wechat_mp_token = "..." + wechat_mp_encoding_aes_key = "..." + wechat_webhook_port = 9001 +""" + +from ..channel_manager import _parse_csv, register_channel +from .channel import WeChatChannel, WeChatMPConfig, WeComConfig + +__all__ = ["WeChatChannel", "WeChatMPConfig", "WeComConfig"] + + +def create_from_config(config) -> WeChatChannel: + backend = config.wechat_backend or "wecom" + allowed = _parse_csv(config.wechat_allowed_senders) + proxy = config.wechat_proxy or None + port = int(config.wechat_webhook_port or 9001) + + if backend == "wechatmp": + mp_config = WeChatMPConfig( + app_id=config.wechat_mp_app_id, + app_secret=config.wechat_mp_app_secret, + token=config.wechat_mp_token, + encoding_aes_key=config.wechat_mp_encoding_aes_key, + webhook_port=port, + allowed_senders=allowed, + proxy=proxy, + ) + return WeChatChannel(mp_config, backend="wechatmp") + else: + wecom_config = WeComConfig( + corp_id=config.wechat_wecom_corp_id, + agent_id=config.wechat_wecom_agent_id, + secret=config.wechat_wecom_secret, + token=config.wechat_wecom_token, + encoding_aes_key=config.wechat_wecom_encoding_aes_key, + webhook_port=port, + allowed_senders=allowed, + proxy=proxy, + ) + return WeChatChannel(wecom_config, backend="wecom") + + +register_channel("wechat", create_from_config) diff --git a/EvoScientist/channels/wechat/channel.py b/EvoScientist/channels/wechat/channel.py new file mode 100644 index 0000000..a39ed4c --- /dev/null +++ b/EvoScientist/channels/wechat/channel.py @@ -0,0 +1,906 @@ +"""WeChat channel implementation. + +Supports two backends via a unified Channel interface: + +1. **wecom** (企业微信应用): Corporate WeChat official API + - Receives messages via HTTP callback (XML + optional AES encryption) + - Sends replies via REST API (POST /cgi-bin/message/send) + - Supports text, image, file, markdown messages + - Token auto-refresh with 2-hour TTL + +2. **wechatmp** (微信公众号): WeChat Official Account API + - Receives messages via HTTP callback (XML + optional AES encryption) + - Sends replies via REST API (POST /cgi-bin/message/custom/send) + - Supports text, image, news messages + +Both backends use httpx (already a core dependency) and aiohttp for +webhook server — matching the Feishu channel pattern. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +import re +import time +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from aiohttp import web + +from ..base import Channel, ChannelError, RawIncoming +from ..capabilities import WECHAT as WECHAT_CAPS +from ..config import BaseChannelConfig +from ..mixins import TokenMixin, WebhookMixin + +logger = logging.getLogger(__name__) + + +# ── Markdown → plain text (fallback for WeChat text messages) ──── + + +def _strip_markdown(text: str) -> str: + """Strip Markdown formatting for plain-text WeChat messages.""" + # Remove code blocks + text = re.sub(r"```[\s\S]*?```", lambda m: m.group(0).strip("`").strip(), text) + # Remove inline code + text = re.sub(r"`([^`]+)`", r"\1", text) + # Remove bold + text = re.sub(r"\*\*(.+?)\*\*", r"\1", text) + # Remove italic + text = re.sub(r"(? list[tuple[str, str, Any]]: + """Return HTTP routes for the shared webhook server.""" + return [ + ("GET", "/wechat/callback", self._handle_verify), + ("POST", "/wechat/callback", self._handle_message), + ] + + async def start(self) -> None: + try: + import httpx + from aiohttp import web + except ImportError: + raise ChannelError( + "aiohttp or httpx not installed. " + "Install with: pip install aiohttp httpx" + ) from None + + self._validate_config() + + import httpx + + self._http_client = httpx.AsyncClient( + timeout=15, + proxy=self._get_proxy(), + ) + + # Set up message encryption if configured + if self.config.encoding_aes_key and self.config.token: + from .crypto import WeChatCrypto + + app_id = self._get_app_id() + self._crypto = WeChatCrypto( + token=self.config.token, + encoding_aes_key=self.config.encoding_aes_key, + app_id=app_id, + ) + + # Verify credentials by fetching initial token + await self._refresh_token() + + if not getattr(self, "_shared_webhook_server", None): + app = web.Application() + app.router.add_get("/wechat/callback", self._handle_verify) + app.router.add_post("/wechat/callback", self._handle_message) + + self._runner = web.AppRunner(app) + await self._runner.setup() + self._site = web.TCPSite( + self._runner, + "0.0.0.0", + self.config.webhook_port, + ) + await self._site.start() + + self._running = True + logger.info( + f"WeChat channel started " + f"(backend={self._backend}, " + f"webhook on port {self.config.webhook_port})" + ) + + async def _cleanup(self) -> None: + if self._site: + await self._site.stop() + if self._runner: + await self._runner.cleanup() + if self._http_client: + await self._http_client.aclose() + self._http_client = None + self._access_token = None + logger.info("WeChat channel stopped") + + def _validate_config(self) -> None: + """Validate required config fields based on backend.""" + if self._backend == "wecom": + cfg = self.config + if not cfg.corp_id: + raise ChannelError("WeCom corp_id is required") + if not cfg.secret: + raise ChannelError("WeCom secret is required") + if not cfg.agent_id: + raise ChannelError("WeCom agent_id is required") + elif self._backend == "wechatmp": + cfg = self.config + if not cfg.app_id: + raise ChannelError("WeChat MP app_id is required") + if not cfg.app_secret: + raise ChannelError("WeChat MP app_secret is required") + + def _get_app_id(self) -> str: + """Return the app identifier for crypto operations.""" + if self._backend == "wecom": + return self.config.corp_id + return self.config.app_id + + # ── Token management ────────────────────────────────────────── + + async def _refresh_token(self) -> None: + """Fetch or refresh the access_token.""" + if self._backend == "wecom": + url = ( + f"https://qyapi.weixin.qq.com/cgi-bin/gettoken" + f"?corpid={self.config.corp_id}" + f"&corpsecret={self.config.secret}" + ) + else: + url = ( + f"https://api.weixin.qq.com/cgi-bin/token" + f"?grant_type=client_credential" + f"&appid={self.config.app_id}" + f"&secret={self.config.app_secret}" + ) + + try: + resp = await self._http_client.get(url) + data = resp.json() + except Exception as e: + if not self._running: + raise ChannelError(f"Failed to get WeChat access token: {e}") from e + raise RuntimeError(f"Failed to get WeChat access token: {e}") from e + + if data.get("errcode", 0) != 0: + err_msg = ( + f"WeChat auth error ({data.get('errcode')}): " + f"{data.get('errmsg', 'unknown')}" + ) + if not self._running: + raise ChannelError(err_msg) + raise RuntimeError(err_msg) + + self._access_token = data["access_token"] + expire = data.get("expires_in", 7200) + # Refresh 5 minutes before expiry + self._token_expires = time.monotonic() + expire - 300 + logger.debug(f"WeChat token refreshed, expires in {expire}s") + + async def _ensure_token(self) -> str: + """Return a valid access token, refreshing if needed.""" + if not self._access_token or time.monotonic() >= self._token_expires: + await self._refresh_token() + return self._access_token + + # ── Signature verification (GET callback) ───────────────────── + + async def _handle_verify(self, request) -> web.Response: + """Handle GET /wechat/callback for URL verification. + + WeChat/WeCom sends: msg_signature, timestamp, nonce, echostr + We decrypt echostr (encrypted mode) or verify signature (plain mode) + and return the plain echostr. + """ + from aiohttp import web + + signature = request.query.get("msg_signature") or request.query.get( + "signature", "" + ) + timestamp = request.query.get("timestamp", "") + nonce = request.query.get("nonce", "") + echostr = request.query.get("echostr", "") + + logger.info(f"Verify request received: timestamp={timestamp}") + + if not echostr: + return web.Response(status=400, text="missing echostr") + + # Encrypted mode: WeCom sends msg_signature and encrypted echostr + if self._crypto and request.query.get("msg_signature"): + # Verify signature first + sig_ok = self._crypto.verify_signature(signature, timestamp, nonce, echostr) + if not sig_ok: + logger.warning("WeChat verify: signature mismatch") + # Try to decrypt regardless — the decrypted echostr must be returned + try: + plain_echostr, _ = self._crypto.decrypt(echostr) + logger.info("WeChat verify: echostr decrypted successfully") + return web.Response(text=plain_echostr) + except Exception as e: + logger.error(f"WeChat verify: echostr decrypt failed: {e}") + return web.Response(status=500) + else: + # Plain mode verification + token = self.config.token + if token: + parts = sorted([token, timestamp, nonce]) + expected = hashlib.sha1("".join(parts).encode()).hexdigest() + if expected != signature: + logger.warning("WeChat verify: signature mismatch (plain)") + return web.Response(status=403) + return web.Response(text=echostr) + + # ── Inbound message handling (POST callback) ────────────────── + + async def _handle_message(self, request) -> web.Response: + """Handle POST /wechat/callback for incoming messages.""" + from aiohttp import web + + from .crypto import parse_xml + + try: + body = await request.text() + except Exception: + return web.Response(status=400) + + logger.info(f"WeChat callback POST received, body length={len(body)}") + xml_data = parse_xml(body) + + # If encrypted, decrypt first + encrypt = xml_data.get("Encrypt", "") + if encrypt and self._crypto: + signature = request.query.get("msg_signature", "") + timestamp = request.query.get("timestamp", "") + nonce = request.query.get("nonce", "") + + if not self._crypto.verify_signature(signature, timestamp, nonce, encrypt): + logger.warning("WeChat message signature mismatch") + return web.Response(status=403) + + try: + decrypted_xml, _from_id = self._crypto.decrypt(encrypt) + xml_data = parse_xml(decrypted_xml) + except Exception as e: + logger.error(f"WeChat decrypt failed: {e}") + return web.Response(status=500) + + # Process message asynchronously — WeCom requires a response within + # 5 seconds, but media downloads can take much longer. Return + # "success" immediately and handle the message in the background. + _task = asyncio.create_task(self._safe_process_message(xml_data)) + self._background_tasks.add(_task) + _task.add_done_callback(self._background_tasks.discard) + + return web.Response(text="success") + + async def _safe_process_message(self, xml_data: dict[str, str]) -> None: + """Wrapper that catches exceptions so fire-and-forget tasks don't leak.""" + try: + await self._process_message(xml_data) + except Exception: + logger.exception("Error processing WeChat message") + + async def _process_message(self, xml_data: dict[str, str]) -> None: + """Process a parsed XML message from WeChat/WeCom callback.""" + msg_type = xml_data.get("MsgType", "") + from_user = xml_data.get("FromUserName", "") + to_user = xml_data.get("ToUserName", "") + content = xml_data.get("Content", "") + msg_id = xml_data.get("MsgId", "") + create_time = xml_data.get("CreateTime", "") + + logger.info( + f"WeChat message received: type={msg_type}, from={from_user}, id={msg_id}, keys={list(xml_data.keys())}" + ) + + if not from_user: + return + + # Determine chat_id + # For WeCom: FromUserName is the user's UserID + # For MP: FromUserName is the user's OpenID + chat_id = from_user + + # Group chat detection + is_group = False + was_mentioned = True # Default: treat as mentioned (DMs) + + # WeCom group detection: ChatId field indicates a group message + if self._backend == "wecom": + group_chat_id = xml_data.get("ChatId", "") + if group_chat_id: + is_group = True + chat_id = group_chat_id + # WeCom sets MsgType=event with Event=sys when bot is @mentioned, + # but for text messages we check the XML AtUserList field + at_user_list = xml_data.get("AtUserList", "") + was_mentioned = bool(at_user_list) + + # Handle different message types + text = "" + annotations: list[str] = [] + media_paths: list[str] = [] + + if msg_type == "text": + text = content + elif msg_type == "image": + pic_url = xml_data.get("PicUrl", "") + media_id = xml_data.get("MediaId", "") + if pic_url: + local, ann = await self._download_attachment( + pic_url, + f"wechat_{msg_id}.jpg", + ) + if local: + media_paths.append(local) + if ann: + annotations.append(ann) + elif media_id: + local, ann = await self._download_wechat_media( + media_id, + f"wechat_image_{msg_id}", + ) + if local: + media_paths.append(local) + if ann: + annotations.append(ann) + else: + annotations.append("[image: no download source]") + elif msg_type == "voice": + recognition = xml_data.get("Recognition", "") + media_id = xml_data.get("MediaId", "") + if media_id: + local, ann = await self._download_wechat_media( + media_id, f"wechat_voice_{msg_id}" + ) + if local: + media_paths.append(local) + if ann: + ann = ann.replace("[attachment:", "[voice:") + annotations.append(ann) + if recognition: + text = f"[语音识别] {recognition}" + elif not media_paths: + annotations.append("[voice message]") + elif msg_type in ("video", "shortvideo"): + media_id = xml_data.get("MediaId", "") + if media_id: + local, ann = await self._download_wechat_media( + media_id, f"wechat_{msg_type}_{msg_id}" + ) + if local: + media_paths.append(local) + if ann: + annotations.append(ann) + if not media_paths: + annotations.append(f"[{msg_type} message]") + elif msg_type == "location": + label = xml_data.get("Label", "") + lat = xml_data.get("Location_X", "") + lon = xml_data.get("Location_Y", "") + text = f"[位置] {label} ({lat}, {lon})" + elif msg_type == "file": + media_id = xml_data.get("MediaId", "") + file_name = xml_data.get("FileName", "") or xml_data.get( + "Title", f"wechat_file_{msg_id}" + ) + logger.info( + f"WeChat file message: name={file_name}, media_id={media_id!r}, keys={list(xml_data.keys())}" + ) + if media_id: + local, ann = await self._download_wechat_media( + media_id, + f"wechat_file_{msg_id}_{file_name}", + ) + logger.info(f"WeChat file download result: local={local}, ann={ann}") + if local: + media_paths.append(local) + if ann: + annotations.append(ann) + if not media_paths: + annotations.append(f"[file: {file_name}]") + elif msg_type == "link": + title = xml_data.get("Title", "") + description = xml_data.get("Description", "") + url = xml_data.get("Url", "") + text = f"[链接] {title}\n{description}\n{url}" + elif msg_type == "event": + event_type = xml_data.get("Event", "") + if event_type == "subscribe": + text = "[用户关注]" + elif event_type == "unsubscribe": + logger.info(f"User {from_user} unsubscribed") + return # Don't process + elif event_type == "CLICK": + event_key = xml_data.get("EventKey", "") + text = f"[菜单点击] {event_key}" + elif event_type in ("LOCATION", "VIEW"): + # Periodic location reports and menu-link clicks — ignore + return + else: + logger.debug(f"Ignoring WeChat event: {event_type}") + return + else: + text = f"[{msg_type} message]" + + if not text and not media_paths and not annotations: + return + + # Parse timestamp + try: + timestamp = ( + datetime.fromtimestamp(int(create_time)) + if create_time + else datetime.now() + ) + except (ValueError, TypeError, OSError): + timestamp = datetime.now() + + await self._enqueue_raw( + RawIncoming( + sender_id=from_user, + chat_id=chat_id, + text=text, + media_files=media_paths, + content_annotations=annotations, + timestamp=timestamp, + message_id=msg_id, + is_group=is_group, + was_mentioned=was_mentioned, + metadata={ + "chat_id": chat_id, + "to_user": to_user, + "backend": self._backend, + }, + ) + ) + + # ── Send (template method overrides) ────────────────────────── + + def _format_chunk(self, text: str) -> str: + """WeCom uses markdown formatter; MP uses plain text.""" + if self._backend == "wecom": + return self._formatter.format(text) # markdown profile + return _strip_markdown(text) + + async def _send_chunk( + self, + chat_id, + formatted_text, + raw_text, + reply_to, + metadata, + ): + token = await self._ensure_token() + + if self._backend == "wecom": + # Group chat: use appchat/send endpoint + if chat_id.startswith("wr"): + try: + await self._wecom_send_group_markdown(token, chat_id, raw_text) + return + except Exception: + pass + await self._wecom_send_group_text(token, chat_id, raw_text) + else: + # DM: Try markdown first, fall back to plain text + try: + await self._wecom_send_markdown(token, chat_id, raw_text) + return + except Exception: + pass + await self._wecom_send_text(token, chat_id, raw_text) + else: + await self._mp_send_text(token, chat_id, raw_text) + + # ── WeCom send ──────────────────────────────────────────────── + + async def _wecom_send_text( + self, + token: str, + user_id: str, + text: str, + ) -> None: + """Send a text message via WeCom API.""" + url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}" + body = { + "touser": user_id, + "msgtype": "text", + "agentid": int(self.config.agent_id), + "text": {"content": _strip_markdown(text)}, + } + await self._post_api(url, body) + + async def _wecom_send_markdown( + self, + token: str, + user_id: str, + text: str, + ) -> None: + """Send a markdown message via WeCom API. + + Note: WeCom markdown only supports a subset of Markdown + (no code blocks, no images). Falls back to text if the + message is too complex. + """ + url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}" + body = { + "touser": user_id, + "msgtype": "markdown", + "agentid": int(self.config.agent_id), + "markdown": {"content": text}, + } + await self._post_api(url, body) + + # ── WeCom group send ──────────────────────────────────────────── + + async def _wecom_send_group_text( + self, + token: str, + chatid: str, + text: str, + ) -> None: + """Send a text message to a WeCom group chat.""" + url = f"https://qyapi.weixin.qq.com/cgi-bin/appchat/send?access_token={token}" + body = { + "chatid": chatid, + "msgtype": "text", + "text": {"content": _strip_markdown(text)}, + } + await self._post_api(url, body) + + async def _wecom_send_group_markdown( + self, + token: str, + chatid: str, + text: str, + ) -> None: + """Send a markdown message to a WeCom group chat.""" + url = f"https://qyapi.weixin.qq.com/cgi-bin/appchat/send?access_token={token}" + body = { + "chatid": chatid, + "msgtype": "markdown", + "markdown": {"content": text}, + } + await self._post_api(url, body) + + # ── MP send ─────────────────────────────────────────────────── + + async def _mp_send_text( + self, + token: str, + openid: str, + text: str, + ) -> None: + """Send a text message via WeChat MP customer service API.""" + url = ( + f"https://api.weixin.qq.com/cgi-bin/message/custom/send" + f"?access_token={token}" + ) + body = { + "touser": openid, + "msgtype": "text", + "text": {"content": _strip_markdown(text)}, + } + await self._post_api(url, body) + + # ── Media send ──────────────────────────────────────────────── + + async def _send_media_impl( + self, + recipient: str, + file_path: str, + caption: str = "", + metadata: dict | None = None, + ) -> bool: + """Send a media file via WeChat/WeCom.""" + token = await self._ensure_token() + chat_id = self._resolve_media_chat_id(recipient, metadata) + + # Upload media to get media_id + media_id = await self._upload_media(token, file_path) + if not media_id: + return False + + path = Path(file_path) + ext = path.suffix.lower() + is_image = ext in {".jpg", ".jpeg", ".png", ".gif", ".bmp"} + + if self._backend == "wecom": + msg_type = "image" if is_image else "file" + # Group chat: use appchat/send endpoint + if chat_id.startswith("wr"): + url = ( + f"https://qyapi.weixin.qq.com/cgi-bin/appchat/send" + f"?access_token={token}" + ) + body = { + "chatid": chat_id, + "msgtype": msg_type, + msg_type: {"media_id": media_id}, + } + else: + url = ( + f"https://qyapi.weixin.qq.com/cgi-bin/message/send" + f"?access_token={token}" + ) + body = { + "touser": chat_id, + "msgtype": msg_type, + "agentid": int(self.config.agent_id), + msg_type: {"media_id": media_id}, + } + else: + url = ( + f"https://api.weixin.qq.com/cgi-bin/message/custom/send" + f"?access_token={token}" + ) + msg_type = "image" if is_image else "file" # MP only supports image + if not is_image: + # MP doesn't support file via customer service API; + # send caption as text instead + if caption: + await self._mp_send_text( + token, chat_id, f"[文件] {path.name}\n{caption}" + ) + return True + body = { + "touser": chat_id, + "msgtype": "image", + "image": {"media_id": media_id}, + } + + await self._post_api(url, body) + + # Send caption separately if provided + if caption: + if self._backend == "wecom": + if chat_id.startswith("wr"): + await self._wecom_send_group_text(token, chat_id, caption) + else: + await self._wecom_send_text(token, chat_id, caption) + else: + await self._mp_send_text(token, chat_id, caption) + + return True + + async def _upload_media( + self, + token: str, + file_path: str, + ) -> str | None: + """Upload a media file and return the media_id.""" + path = Path(file_path) + ext = path.suffix.lower() + is_image = ext in {".jpg", ".jpeg", ".png", ".gif", ".bmp"} + media_type = "image" if is_image else "file" + + if self._backend == "wecom": + url = ( + f"https://qyapi.weixin.qq.com/cgi-bin/media/upload" + f"?access_token={token}&type={media_type}" + ) + else: + url = ( + f"https://api.weixin.qq.com/cgi-bin/media/upload" + f"?access_token={token}&type={media_type}" + ) + + try: + with open(file_path, "rb") as f: + resp = await self._http_client.post( + url, + files={"media": (path.name, f)}, + ) + data = resp.json() + if data.get("errcode", 0) != 0 and "media_id" not in data: + logger.error(f"WeChat media upload failed: {data.get('errmsg')}") + return None + return data.get("media_id") + except Exception as e: + logger.error(f"WeChat media upload error: {e}") + return None + + # ── Media download helper ──────────────────────────────────── + + async def _download_wechat_media( + self, + media_id: str, + filename: str, + ) -> tuple[str | None, str | None]: + """Download media by media_id via WeChat/WeCom media API.""" + token = await self._ensure_token() + if self._backend == "wecom": + url = f"https://qyapi.weixin.qq.com/cgi-bin/media/get?access_token={token}&media_id={media_id}" + else: + url = f"https://api.weixin.qq.com/cgi-bin/media/get?access_token={token}&media_id={media_id}" + return await self._download_attachment(url, filename) + + # ── Shared API helper ───────────────────────────────────────── + + async def _post_api(self, url: str, body: dict) -> dict: + """POST to WeChat/WeCom API, check errcode, return response.""" + try: + resp = await self._http_client.post(url, json=body) + data = resp.json() + except Exception as e: + raise RuntimeError(f"WeChat API error: {e}") from e + + errcode = data.get("errcode", 0) + if errcode != 0: + errmsg = data.get("errmsg", "unknown") + # Token expired — refresh and retry once + if errcode in (40014, 42001): + logger.warning("WeChat token expired, refreshing...") + await self._refresh_token() + token = self._access_token + # Replace token in URL + if "access_token=" in url: + url = re.sub( + r"access_token=[^&]+", + f"access_token={token}", + url, + ) + resp = await self._http_client.post(url, json=body) + data = resp.json() + if data.get("errcode", 0) != 0: + raise RuntimeError( + f"WeChat API error after retry: {data.get('errmsg')}" + ) + return data + else: + raise RuntimeError(f"WeChat API error ({errcode}): {errmsg}") + + return data + + # ── Typing indicator (WeCom only) ──────────────────────────── + + async def _send_typing_action(self, chat_id: str) -> None: + """Send typing indicator via WeChat. + + WeChat has no native typing API. For the WeCom backend we + approximate the experience by posting a short-lived "…" message + that is recalled once the real reply is sent (handled by + ``stop_typing``). WeChat MP has no recall API so we skip it. + """ + if self._backend != "wecom" or not self._http_client: + return + try: + token = await self._ensure_token() + if chat_id.startswith("wr"): + url = ( + f"https://qyapi.weixin.qq.com/cgi-bin/appchat/send" + f"?access_token={token}" + ) + body = { + "chatid": chat_id, + "msgtype": "text", + "text": {"content": "\u2026"}, + } + else: + url = ( + f"https://qyapi.weixin.qq.com/cgi-bin/message/send" + f"?access_token={token}" + ) + body = { + "touser": chat_id, + "msgtype": "text", + "agentid": int(self.config.agent_id), + "text": {"content": "\u2026"}, + } + data = await self._post_api(url, body) + msgid = data.get("msgid") + if msgid: + self._typing_message_ids.setdefault(chat_id, []).append(msgid) + except Exception: + pass + + async def stop_typing(self, chat_id: str) -> None: + """Cancel typing loop and recall all status messages.""" + msgids = self._typing_message_ids.pop(chat_id, []) + if msgids and self._http_client and self._backend == "wecom": + try: + token = await self._ensure_token() + url = ( + f"https://qyapi.weixin.qq.com/cgi-bin/message/recall" + f"?access_token={token}" + ) + for msgid in msgids: + try: + await self._http_client.post(url, json={"msgid": msgid}) + except Exception: + pass + except Exception: + pass + await super().stop_typing(chat_id) diff --git a/EvoScientist/channels/wechat/crypto.py b/EvoScientist/channels/wechat/crypto.py new file mode 100644 index 0000000..88637a2 --- /dev/null +++ b/EvoScientist/channels/wechat/crypto.py @@ -0,0 +1,193 @@ +"""WeChat / WeCom crypto helpers. + +Implements the message encryption/decryption protocol used by both +WeCom (企业微信) and WeChat Official Account (公众号) callback APIs. + +The protocol uses AES-256-CBC with a key derived from the EncodingAESKey +(base64-encoded 43-char string → 32-byte AES key). + +References: + - WeCom: https://developer.work.weixin.qq.com/document/path/90930 + - MP: https://developers.weixin.qq.com/doc/offiaccount/Message_Management/Message_Encryption_and_Decryption_Instructions.html +""" + +import base64 +import hashlib +import struct +import time +import xml.etree.ElementTree as ET + +# Crypto imports — all from the Python standard library + pycryptodome +# (but we'll use a pure-Python fallback if not available) +try: + from Crypto.Cipher import AES + + _HAS_PYCRYPTO = True +except ImportError: + _HAS_PYCRYPTO = False + + +def _pkcs7_pad(data: bytes, block_size: int = 32) -> bytes: + """PKCS#7 padding.""" + pad_len = block_size - (len(data) % block_size) + return data + bytes([pad_len]) * pad_len + + +def _pkcs7_unpad(data: bytes) -> bytes: + """PKCS#7 unpadding.""" + pad_len = data[-1] + if pad_len < 1 or pad_len > 32: + return data + return data[:-pad_len] + + +def _aes_decrypt(key: bytes, iv: bytes, ciphertext: bytes) -> bytes: + """AES-256-CBC decryption.""" + if _HAS_PYCRYPTO: + cipher = AES.new(key, AES.MODE_CBC, iv) + return cipher.decrypt(ciphertext) + else: + # Pure-Python AES fallback (slower but no C deps) + # We'll try pyaes as a fallback + try: + import pyaes + + decrypter = pyaes.Decrypter(pyaes.AESModeOfOperationCBC(key, iv=iv)) + decrypted = decrypter.feed(ciphertext) + decrypted += decrypter.feed() + return decrypted + except ImportError: + raise ImportError( + "WeChat message decryption requires pycryptodome or pyaes. " + "Install with: pip install pycryptodome" + ) from None + + +def _aes_encrypt(key: bytes, iv: bytes, plaintext: bytes) -> bytes: + """AES-256-CBC encryption.""" + if _HAS_PYCRYPTO: + cipher = AES.new(key, AES.MODE_CBC, iv) + return cipher.encrypt(plaintext) + else: + try: + import pyaes + + encrypter = pyaes.Encrypter(pyaes.AESModeOfOperationCBC(key, iv=iv)) + encrypted = encrypter.feed(plaintext) + encrypted += encrypter.feed() + return encrypted + except ImportError: + raise ImportError( + "WeChat message encryption requires pycryptodome or pyaes. " + "Install with: pip install pycryptodome" + ) from None + + +class WeChatCrypto: + """Handles WeChat/WeCom message encryption and decryption. + + Parameters + ---------- + token: + The Token configured in the WeChat/WeCom callback URL settings. + encoding_aes_key: + The 43-character EncodingAESKey (base64-encoded). + app_id: + The AppID (for MP) or CorpID (for WeCom). + """ + + def __init__(self, token: str, encoding_aes_key: str, app_id: str): + self.token = token + self.app_id = app_id + # Decode the AES key: EncodingAESKey + "=" → base64 decode → 32 bytes + self.aes_key = base64.b64decode(encoding_aes_key + "=") + # IV is the first 16 bytes of the key + self.iv = self.aes_key[:16] + + def verify_signature( + self, + signature: str, + timestamp: str, + nonce: str, + encrypt: str = "", + ) -> bool: + """Verify the callback signature. + + For plain-mode verification (no encryption), *encrypt* can be empty. + """ + parts = sorted([self.token, timestamp, nonce] + ([encrypt] if encrypt else [])) + sha1 = hashlib.sha1("".join(parts).encode()).hexdigest() + return sha1 == signature + + def decrypt(self, encrypt: str) -> tuple[str, str]: + """Decrypt an encrypted message. + + Returns ``(xml_content, from_app_id)`` tuple. + """ + ciphertext = base64.b64decode(encrypt) + plaintext = _aes_decrypt(self.aes_key, self.iv, ciphertext) + plaintext = _pkcs7_unpad(plaintext) + + # plaintext layout: + # 16 bytes random + 4 bytes msg_len (big-endian) + msg + app_id + msg_len = struct.unpack("!I", plaintext[16:20])[0] + msg = plaintext[20 : 20 + msg_len].decode("utf-8") + from_app_id = plaintext[20 + msg_len :].decode("utf-8") + return msg, from_app_id + + def encrypt(self, reply_msg: str) -> str: + """Encrypt a reply message. + + Returns the base64-encoded ciphertext. + """ + msg_bytes = reply_msg.encode("utf-8") + app_id_bytes = self.app_id.encode("utf-8") + + # Random 16 bytes + msg_len (4 bytes big-endian) + msg + app_id + import os + + random_bytes = os.urandom(16) + msg_len = struct.pack("!I", len(msg_bytes)) + plaintext = random_bytes + msg_len + msg_bytes + app_id_bytes + plaintext = _pkcs7_pad(plaintext) + + ciphertext = _aes_encrypt(self.aes_key, self.iv, plaintext) + return base64.b64encode(ciphertext).decode("utf-8") + + def generate_signature( + self, + encrypt: str, + timestamp: str, + nonce: str, + ) -> str: + """Generate the msg_signature for an encrypted reply.""" + parts = sorted([self.token, timestamp, nonce, encrypt]) + return hashlib.sha1("".join(parts).encode()).hexdigest() + + def wrap_encrypted_reply(self, reply_msg: str) -> str: + """Encrypt a reply and wrap it in the XML envelope. + + Returns the full XML string to return in the HTTP response. + """ + encrypt = self.encrypt(reply_msg) + timestamp = str(int(time.time())) + nonce = hashlib.md5(str(time.time()).encode()).hexdigest()[:10] + signature = self.generate_signature(encrypt, timestamp, nonce) + + return ( + f"" + f"" + f"" + f"{timestamp}" + f"" + f"" + ) + + +def parse_xml(xml_str: str) -> dict[str, str]: + """Parse a WeChat callback XML into a flat dict.""" + root = ET.fromstring(xml_str) + result = {} + for child in root: + result[child.tag] = child.text or "" + return result diff --git a/EvoScientist/channels/wechat/probe.py b/EvoScientist/channels/wechat/probe.py new file mode 100644 index 0000000..ffa26c2 --- /dev/null +++ b/EvoScientist/channels/wechat/probe.py @@ -0,0 +1,72 @@ +"""WeChat/WeCom credential validation.""" + +import logging + +logger = logging.getLogger(__name__) + + +async def validate_wecom( + corp_id: str, + secret: str, + proxy: str | None = None, +) -> tuple[bool, str]: + """Validate WeCom credentials by fetching an access token. + + Returns: + Tuple of (is_valid, message). + """ + if not corp_id or not secret: + return False, "corp_id and secret are required" + + try: + import httpx + except ImportError: + return False, "httpx not installed" + + url = ( + f"https://qyapi.weixin.qq.com/cgi-bin/gettoken" + f"?corpid={corp_id}&corpsecret={secret}" + ) + try: + async with httpx.AsyncClient(proxy=proxy) as client: + resp = await client.get(url, timeout=10) + data = resp.json() + if data.get("errcode", 0) == 0: + return True, "WeCom credentials valid" + return False, f"Error ({data.get('errcode')}): {data.get('errmsg')}" + except Exception as e: + return False, f"Error: {e}" + + +async def validate_wechat_mp( + app_id: str, + app_secret: str, + proxy: str | None = None, +) -> tuple[bool, str]: + """Validate WeChat Official Account credentials. + + Returns: + Tuple of (is_valid, message). + """ + if not app_id or not app_secret: + return False, "app_id and app_secret are required" + + try: + import httpx + except ImportError: + return False, "httpx not installed" + + url = ( + f"https://api.weixin.qq.com/cgi-bin/token" + f"?grant_type=client_credential" + f"&appid={app_id}&secret={app_secret}" + ) + try: + async with httpx.AsyncClient(proxy=proxy) as client: + resp = await client.get(url, timeout=10) + data = resp.json() + if "access_token" in data: + return True, "WeChat MP credentials valid" + return False, f"Error ({data.get('errcode')}): {data.get('errmsg')}" + except Exception as e: + return False, f"Error: {e}" diff --git a/EvoScientist/channels/wechat/serve.py b/EvoScientist/channels/wechat/serve.py new file mode 100644 index 0000000..4d137e3 --- /dev/null +++ b/EvoScientist/channels/wechat/serve.py @@ -0,0 +1,136 @@ +"""WeChat channel server. + +Standalone script to run the WeChat channel with CLI options. + +Usage: + # WeCom (企业微信应用) + python -m EvoScientist.channels.wechat.serve \\ + --backend wecom \\ + --corp-id CORP_ID \\ + --agent-id AGENT_ID \\ + --secret SECRET \\ + --token TOKEN \\ + --aes-key AES_KEY + + # WeChat Official Account (公众号) + python -m EvoScientist.channels.wechat.serve \\ + --backend wechatmp \\ + --app-id APP_ID \\ + --app-secret APP_SECRET \\ + --token TOKEN \\ + --aes-key AES_KEY + +Options: + --port PORT Webhook listen port (default: 9001) + --allow USER_ID Allowed sender (repeatable) + --agent Use EvoScientist agent as handler + --thinking Send thinking content as intermediate messages +""" + +import argparse +import logging + +from ...logging_config import configure_logging_from_settings +from ..bus import MessageBus +from ..standalone import run_standalone +from .channel import WeChatChannel, WeChatMPConfig, WeComConfig + +logger = logging.getLogger(__name__) + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="WeChat channel server", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--backend", + choices=["wecom", "wechatmp"], + default="wecom", + help="WeChat backend type (default: wecom)", + ) + parser.add_argument("--port", type=int, default=9001, help="Webhook port") + parser.add_argument( + "--allow", + action="append", + dest="allowed_senders", + help="Allowed sender ID (repeatable)", + ) + parser.add_argument( + "--allow-channel", + action="append", + dest="allowed_channels", + help="Allowed channel ID. Can be used multiple times.", + ) + parser.add_argument( + "--agent", + action="store_true", + help="Use EvoScientist agent as handler", + ) + parser.add_argument( + "--thinking", + action="store_true", + help="Send thinking content (requires --agent)", + ) + + # WeCom settings + wecom = parser.add_argument_group("WeCom (企业微信)") + wecom.add_argument("--corp-id", default="", help="WeCom Corp ID") + wecom.add_argument("--agent-id", default="", help="WeCom Agent ID") + wecom.add_argument("--secret", default="", help="WeCom Secret") + + # MP settings + mp = parser.add_argument_group("WeChat Official Account (公众号)") + mp.add_argument("--app-id", default="", help="MP App ID") + mp.add_argument("--app-secret", default="", help="MP App Secret") + + # Shared settings + parser.add_argument("--token", default="", help="Callback verification token") + parser.add_argument("--aes-key", default="", help="EncodingAESKey") + parser.add_argument("--proxy", default="", help="HTTP proxy URL") + + return parser.parse_args() + + +def main(): + """Entry point.""" + configure_logging_from_settings(default_level=logging.INFO) + args = parse_args() + allowed = set(args.allowed_senders) if args.allowed_senders else None + allowed_channels = set(args.allowed_channels) if args.allowed_channels else None + proxy = args.proxy or None + + if args.backend == "wecom": + config = WeComConfig( + corp_id=args.corp_id, + agent_id=args.agent_id, + secret=args.secret, + token=args.token, + encoding_aes_key=args.aes_key, + webhook_port=args.port, + allowed_senders=allowed, + allowed_channels=allowed_channels, + proxy=proxy, + ) + else: + config = WeChatMPConfig( + app_id=args.app_id, + app_secret=args.app_secret, + token=args.token, + encoding_aes_key=args.aes_key, + webhook_port=args.port, + allowed_senders=allowed, + allowed_channels=allowed_channels, + proxy=proxy, + ) + + send_thinking = args.thinking and args.agent + bus = MessageBus() + channel = WeChatChannel(config, backend=args.backend) + + run_standalone(channel, bus, use_agent=args.agent, send_thinking=send_thinking) + + +if __name__ == "__main__": + main() diff --git a/EvoScientist/channels/wechat/verify_server.py b/EvoScientist/channels/wechat/verify_server.py new file mode 100644 index 0000000..f8e8572 --- /dev/null +++ b/EvoScientist/channels/wechat/verify_server.py @@ -0,0 +1,179 @@ +"""WeChat callback verification server. + +Provides a lightweight temporary HTTP server that handles the WeChat/WeCom +URL verification handshake during onboarding. This solves the chicken-and-egg +problem: WeChat requires a live server to verify the callback URL before +saving, but the main EvoScientist service isn't running during onboard. + +Usage: + server = VerifyServer(port, token, encoding_aes_key, corp_id) + await server.start() + # ... user clicks "Save" in WeCom admin console ... + # ... server auto-responds to the verification GET request ... + await server.wait_for_verify(timeout=120) + await server.stop() +""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from aiohttp import web + +logger = logging.getLogger(__name__) + + +class VerifyServer: + """Temporary HTTP server for WeChat/WeCom callback URL verification. + + Handles the GET verification request (signature + echostr) and + signals when verification succeeds. + """ + + def __init__( + self, + port: int, + token: str, + encoding_aes_key: str = "", + app_id: str = "", + ): + self.port = port + self.token = token + self._crypto = None + self._runner = None + self._site = None + self._verified = asyncio.Event() + + if encoding_aes_key and token and app_id: + from .crypto import WeChatCrypto + + self._crypto = WeChatCrypto( + token=token, + encoding_aes_key=encoding_aes_key, + app_id=app_id, + ) + + async def start(self) -> None: + """Start the verification server.""" + from aiohttp import web + + app = web.Application() + app.router.add_get("/wechat/callback", self._handle) + # Also handle POST in case WeCom sends a POST for some reason + app.router.add_post("/wechat/callback", self._handle_post) + + self._runner = web.AppRunner(app) + await self._runner.setup() + self._site = web.TCPSite(self._runner, "0.0.0.0", self.port) + await self._site.start() + logger.info(f"Verify server listening on port {self.port}") + + async def stop(self) -> None: + """Stop the verification server.""" + if self._site: + await self._site.stop() + if self._runner: + await self._runner.cleanup() + self._site = None + self._runner = None + + async def wait_for_verify(self, timeout: float = 120) -> bool: + """Wait for verification to succeed. + + Returns True if verified within timeout, False otherwise. + """ + try: + await asyncio.wait_for(self._verified.wait(), timeout=timeout) + return True + except TimeoutError: + return False + + @property + def is_verified(self) -> bool: + return self._verified.is_set() + + async def _handle(self, request) -> web.Response: + """Handle GET verification request. + + During onboarding we use a lenient approach: + 1. Try strict crypto verification (encrypted mode) + 2. Try strict plain-mode signature check + 3. If both fail, fall back to decrypting echostr without + signature check (WeCom requires the decrypted echostr) + 4. Last resort: echo back raw echostr + + This ensures the callback URL can be saved even if Token/AESKey + have minor issues, while still attempting proper verification. + """ + from aiohttp import web + + signature = request.query.get("msg_signature") or request.query.get( + "signature", "" + ) + timestamp = request.query.get("timestamp", "") + nonce = request.query.get("nonce", "") + echostr = request.query.get("echostr", "") + + logger.info( + f"Verify request: msg_signature={signature[:16]}... " + f"timestamp={timestamp} nonce={nonce} " + f"echostr={echostr[:32]}..." + ) + + if not echostr: + return web.Response(status=400, text="missing echostr") + + # Attempt 1: Encrypted mode with full signature verification + if self._crypto and request.query.get("msg_signature"): + sig_ok = self._crypto.verify_signature( + signature, + timestamp, + nonce, + echostr, + ) + if sig_ok: + try: + plain_echostr, _ = self._crypto.decrypt(echostr) + self._verified.set() + logger.info("✓ Verified (encrypted, signature OK)") + return web.Response(text=plain_echostr) + except Exception as e: + logger.warning(f"Signature OK but decrypt failed: {e}") + else: + logger.warning("Signature mismatch, trying decrypt anyway...") + + # Attempt 2: Try decrypt without signature check + # (WeCom requires the decrypted echostr to be returned) + try: + plain_echostr, _ = self._crypto.decrypt(echostr) + self._verified.set() + logger.info("✓ Verified (decrypted, signature skipped)") + return web.Response(text=plain_echostr) + except Exception as e: + logger.warning(f"Decrypt also failed: {e}") + + # Attempt 3: Plain mode signature check + if self.token: + parts = sorted([self.token, timestamp, nonce]) + expected = hashlib.sha1("".join(parts).encode()).hexdigest() + if expected == signature: + self._verified.set() + logger.info("✓ Verified (plain mode)") + return web.Response(text=echostr) + + # Attempt 4: Last resort — just echo back the echostr + # This won't work for encrypted mode (WeCom expects decrypted), + # but works for plain mode with wrong token. + logger.warning("All verification methods failed, echoing raw echostr") + self._verified.set() + return web.Response(text=echostr) + + async def _handle_post(self, request) -> web.Response: + """Handle POST — just acknowledge during verification phase.""" + from aiohttp import web + + return web.Response(text="success") diff --git a/EvoScientist/cli/__init__.py b/EvoScientist/cli/__init__.py new file mode 100644 index 0000000..d708940 --- /dev/null +++ b/EvoScientist/cli/__init__.py @@ -0,0 +1,49 @@ +"""EvoScientist CLI package.""" + +# Backward-compat re-exports (tests import these from EvoScientist.cli) +from ..stream.state import ( # noqa: F401 + StreamState, + SubAgentState, + _build_todo_stats, + _parse_todo_items, +) +from . import commands # noqa: F401 — registers @app.command decorators +from ._app import app +from ._constants import WELCOME_SLOGANS # noqa: F401 +from .agent import _deduplicate_run_name # noqa: F401 +from .channel import _channels_is_running, _channels_stop # noqa: F401 + +# UI runtime re-exports (merged from former tui/ package) +from .tui_runtime import ( # noqa: F401 + DEFAULT_UI_BACKEND, + SUPPORTED_UI_BACKENDS, + get_backend, + normalize_ui_backend, + resolve_ui_backend, + run_streaming, +) + + +def main(): + """CLI entry point.""" + import os + import warnings + + warnings.filterwarnings("ignore", message=".*not known to support tools.*") + warnings.filterwarnings( + "ignore", message=".*type is unknown and inference may fail.*" + ) + from ..config import load_config + from .commands import _configure_logging + + # Priority: env var > config file > default (WARNING) + config = load_config() + _log_level = os.environ.get("EVOSCIENTIST_LOG_LEVEL", "") or config.log_level + _configure_logging() + app() + + +def admin_main(): + """evo-admin CLI entry point — admin management commands.""" + from ._app import admin_app + admin_app() diff --git a/EvoScientist/cli/_app.py b/EvoScientist/cli/_app.py new file mode 100644 index 0000000..940d1bc --- /dev/null +++ b/EvoScientist/cli/_app.py @@ -0,0 +1,57 @@ +"""Typer application objects — no intra-package imports to avoid circular deps.""" + +import typer # type: ignore[import-untyped] + +app = typer.Typer( + no_args_is_help=False, + add_completion=False, + context_settings={"help_option_names": ["-h", "--help"]}, +) + +# Config subcommand group +config_app = typer.Typer( + help="Configuration management commands", invoke_without_command=True +) +app.add_typer(config_app, name="config") + +# MCP subcommand group +_MCP_HELP = """\ +Configure and manage MCP servers + +Examples: + # Add a local MCP server (stdio auto-detected): + EvoSci mcp add local-server python -- /path/to/server.py + + # Add an npx-based server: + EvoSci mcp add sequential-thinking npx -- -y @modelcontextprotocol/server-sequential-thinking + + # Add an HTTP server (http auto-detected from URL): + EvoSci mcp add docs-langchain https://docs.langchain.com/mcp + + # Add a stdio server with env vars (hardcoded): + EvoSci mcp add my-server node --env API_KEY=xxx -- server.js + + # Add a server with runtime env ref (resolved from .env at startup): + EvoSci mcp add brave-search npx --env-ref BRAVE_API_KEY -- -y @modelcontextprotocol/server-brave-search + + # Expose to a specific sub-agent (e.g. research-agent): + EvoSci mcp add brave-search npx --env-ref BRAVE_API_KEY -e research-agent -- -y @modelcontextprotocol/server-brave-search + + # Expose to multiple agents: + EvoSci mcp add local-server python -e main,research-agent,code-agent -- /path/to/server.py + + # Explicit transport override: + EvoSci mcp add my-sse https://example.com/sse --transport sse + +Sub-agents (-e): planner-agent | research-agent | code-agent | debug-agent | data-analysis-agent | writing-agent +""" +mcp_app = typer.Typer(help=_MCP_HELP, invoke_without_command=True) +app.add_typer(mcp_app, name="mcp") + +# Channel subcommand group +channel_app = typer.Typer(help="Channel management commands") +app.add_typer(channel_app, name="channel") + +# Admin subcommand group +admin_app = typer.Typer(help="Admin management commands") +app.add_typer(admin_app, name="admin") diff --git a/EvoScientist/cli/_constants.py b/EvoScientist/cli/_constants.py new file mode 100644 index 0000000..8c507a5 --- /dev/null +++ b/EvoScientist/cli/_constants.py @@ -0,0 +1,41 @@ +"""Shared constants and utilities for CLI and TUI modules.""" + +from datetime import UTC, datetime + +from ..sessions import AGENT_NAME + +WELCOME_SLOGANS = [ + "Ready for vibe research? What do you want cooking?", + "Science doesn't sleep. Neither do your sub-agents.", + "From hypothesis to paper — let's cook.", + "Your research kitchen is ready. What's on the menu?", + "Experiments don't run themselves. Oh wait — they do now.", + "Drop a question. We'll bring the citations.", + "Vibe first. Discovery follows.", + "What breakthrough are we cooking today?", + "Harness the vibe. Start the research.", + "Ideas in. Discoveries out.", +] + +# ASCII art logo — shared by both Rich CLI and Textual TUI banners. +LOGO_LINES = ( + r" ███████╗ ██╗ ██╗ ██████╗ ███████╗ ██████╗ ██╗ ███████╗ ███╗ ██╗ ████████╗ ██╗ ███████╗ ████████╗", + r" ██╔════╝ ██║ ██║ ██╔═══██╗ ██╔════╝ ██╔════╝ ██║ ██╔════╝ ████╗ ██║ ╚══██╔══╝ ██║ ██╔════╝ ╚══██╔══╝", + r" █████╗ ██║ ██║ ██║ ██║ ███████╗ ██║ ██║ █████╗ ██╔██╗ ██║ ██║ ██║ ███████╗ ██║ ", + r" ██╔══╝ ╚██╗ ██╔╝ ██║ ██║ ╚════██║ ██║ ██║ ██╔══╝ ██║╚██╗██║ ██║ ██║ ╚════██║ ██║ ", + r" ███████╗ ╚████╔╝ ╚██████╔╝ ███████║ ╚██████╗ ██║ ███████╗ ██║ ╚████║ ██║ ██║ ███████║ ██║ ", + r" ╚══════╝ ╚═══╝ ╚═════╝ ╚══════╝ ╚═════╝ ╚═╝ ╚══════╝ ╚═╝ ╚═══╝ ╚═╝ ╚═╝ ╚══════╝ ╚═╝ ", +) + +# Blue gradient: deep navy -> royal blue -> sky blue -> cyan +LOGO_GRADIENT = ["#1a237e", "#1565c0", "#1e88e5", "#42a5f5", "#64b5f6", "#90caf9"] + + +def build_metadata(workspace_dir: str | None, model: str | None) -> dict: + """Build metadata dict for LangGraph checkpoint persistence.""" + return { + "agent_name": AGENT_NAME, + "updated_at": datetime.now(UTC).isoformat(), + "workspace_dir": workspace_dir or "", + "model": model or "", + } diff --git a/EvoScientist/cli/agent.py b/EvoScientist/cli/agent.py new file mode 100644 index 0000000..393fcf2 --- /dev/null +++ b/EvoScientist/cli/agent.py @@ -0,0 +1,75 @@ +"""Agent loading and workspace helpers.""" + +import os +from datetime import datetime +from pathlib import Path + +from ..paths import new_run_dir + + +def _shorten_path(path: str) -> str: + """Shorten absolute path to relative path from current directory.""" + if not path: + return path + try: + cwd = os.getcwd() + if path.startswith(cwd): + rel = path[len(cwd) :].lstrip(os.sep) + return ( + os.path.join(os.path.basename(cwd), rel) + if rel + else os.path.basename(cwd) + ) + return path + except Exception: + return path + + +def _deduplicate_run_name(name: str, runs_dir: Path | None = None) -> str: + """Return *name* if available, otherwise *name_1*, *name_2*, etc.""" + if runs_dir is None: + from ..paths import RUNS_DIR + + runs_dir = RUNS_DIR + if not (runs_dir / name).exists(): + return name + i = 1 + while (runs_dir / f"{name}_{i}").exists(): + i += 1 + return f"{name}_{i}" + + +def _create_session_workspace(name: str | None = None) -> str: + """Create a per-session workspace directory and return its path. + + Args: + name: Optional human-friendly run name. Duplicates are resolved + by appending ``_1``, ``_2``, etc. Falls back to a timestamp + if *name* is None. + """ + if name: + from ..paths import RUNS_DIR + + session_id = _deduplicate_run_name(name, RUNS_DIR) + else: + session_id = datetime.now().strftime("%Y%m%d_%H%M%S") + workspace_dir = str(new_run_dir(session_id)) + os.makedirs(workspace_dir, exist_ok=True) + return workspace_dir + + +def _load_agent(workspace_dir: str | None = None, checkpointer=None, config=None): + """Load the CLI agent with optional persistent checkpointer. + + Args: + workspace_dir: Optional per-session workspace directory. + checkpointer: Optional LangGraph checkpointer. + Falls back to ``InMemorySaver`` when ``None``. + config: Optional pre-loaded ``EvoScientistConfig``. Forwarded to + ``create_cli_agent`` to avoid double config loading. + """ + from ..EvoScientist import create_cli_agent + + return create_cli_agent( + workspace_dir=workspace_dir, checkpointer=checkpointer, config=config + ) diff --git a/EvoScientist/cli/channel.py b/EvoScientist/cli/channel.py new file mode 100644 index 0000000..afc9a9c --- /dev/null +++ b/EvoScientist/cli/channel.py @@ -0,0 +1,834 @@ +"""Background channel management — bus mode with ChannelManager. + +Architecture: + Bus thread: runs ChannelManager + all channels + inbound consumer. + Main CLI thread: runs agent invocations (to avoid event-loop conflicts). + +The inbound consumer does NOT call the agent directly. Instead it +enqueues a ``ChannelMessage`` on a thread-safe ``queue.Queue`` and waits +for the main thread to set a response via ``_set_channel_response()``. +""" + +import asyncio +import logging +import queue +import threading +import time +import uuid +from dataclasses import dataclass +from typing import Any + +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +from ..stream.display import console + +_channel_logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Queue bridge: bus thread ⇄ main CLI thread +# --------------------------------------------------------------------------- + + +@dataclass +class ChannelMessage: + """A message from a channel, enqueued for the main CLI thread.""" + + msg_id: str + content: str + sender: str + channel_type: str + metadata: dict | None = None + # Filled by the bus consumer so the main thread can send callbacks + channel_ref: Any = None # Channel instance (for thinking / todo / file) + bus_ref: Any = None # MessageBus (for publishing outbound) + chat_id: str = "" + message_id: str | None = None + + +# Thread-safe queue: bus → main +_message_queue: queue.Queue[ChannelMessage] = queue.Queue() + +# Pending responses: +# main → bus (msg_id → {"future": Future[str], "loop": loop, "response": str|None}) +_pending_responses: dict[str, dict] = {} +_response_lock = threading.Lock() + +_RESPONSE_TIMEOUT = 600.0 +_LATE_RESPONSE_TIMEOUT = 86400.0 +_LATE_RESPONSE_NOTICE = "Still working on it. I'll send the result when it's ready." + + +def _enqueue_channel_message(msg: ChannelMessage) -> asyncio.Future[str]: + """Enqueue a channel message for the main thread and return a wait future.""" + loop = asyncio.get_running_loop() + future: asyncio.Future[str] = loop.create_future() + with _response_lock: + _pending_responses[msg.msg_id] = { + "future": future, + "loop": loop, + "response": None, + } + _message_queue.put(msg) + return future + + +def _set_channel_response(msg_id: str, response: str) -> None: + """Set the response for a channel message and unblock the bus consumer.""" + with _response_lock: + slot = _pending_responses.get(msg_id) + if slot: + slot["response"] = response + future = slot["future"] + loop = slot["loop"] + else: + return + + def _resolve_future() -> None: + if not future.done(): + future.set_result(response) + + loop.call_soon_threadsafe(_resolve_future) + + +def _pop_channel_response(msg_id: str, *, cancel_pending: bool = False) -> str | None: + """Retrieve and remove the response for a channel message.""" + with _response_lock: + slot = _pending_responses.pop(msg_id, None) + if not slot: + return None + + future = slot["future"] + if cancel_pending and not future.done(): + future.cancel() + return slot["response"] + + +# --------------------------------------------------------------------------- +# HITL approval intercept: bus thread ⇄ main CLI thread +# --------------------------------------------------------------------------- +# When the main thread needs HITL approval from a channel user, it registers +# a pending HITL wait for (channel, chat_id). The bus consumer checks this +# BEFORE normal enqueue, so the next reply from that user is intercepted. + +_pending_hitl: dict[str, dict] = {} # "channel:chat_id" -> {event, reply} +_hitl_lock = threading.Lock() +_hitl_auto_approve: set[str] = set() # "channel:chat_id" keys with auto-approve + +_HITL_APPROVAL_TIMEOUT = 120.0 # seconds to wait for HITL approval reply +_ASK_USER_TIMEOUT = ( + 300.0 # seconds to wait for ask_user reply (longer for thinking time) +) + + +def _register_hitl_wait(channel_type: str, chat_id: str) -> threading.Event: + """Register a pending HITL wait. Returns a threading.Event to block on.""" + key = f"{channel_type}:{chat_id}" + event = threading.Event() + with _hitl_lock: + _pending_hitl[key] = {"event": event, "reply": None} + return event + + +def _pop_hitl_reply(channel_type: str, chat_id: str) -> str | None: + """Pop and return the HITL reply (or None if not set).""" + key = f"{channel_type}:{chat_id}" + with _hitl_lock: + slot = _pending_hitl.pop(key, None) + return slot["reply"] if slot else None + + +def _try_set_hitl_reply(channel_type: str, chat_id: str, content: str) -> bool: + """Try to intercept a message as a HITL reply. Returns True if consumed.""" + key = f"{channel_type}:{chat_id}" + with _hitl_lock: + slot = _pending_hitl.get(key) + if slot: + slot["reply"] = content + slot["event"].set() + return True + return False + + +def channel_ask_user_prompt( + ask_user_data: dict, + msg: "ChannelMessage | None" = None, +) -> dict: + """Format ask_user questions and collect answers from a channel user. + + If *msg* is provided, sends questions via the bus and waits for a reply. + Otherwise falls back to returning a cancelled result. + + Returns: + ``{"answers": [...], "status": "answered"}`` or + ``{"status": "cancelled"}``. + """ + from ..channels.bus.events import OutboundMessage + + questions = ask_user_data.get("questions", []) + if not questions: + return {"answers": [], "status": "answered"} + + if msg is None or not msg.bus_ref: + return {"status": "cancelled"} + + bus_loop = _bus_loop + if not bus_loop: + return {"status": "cancelled"} + + def _send(content: str) -> bool: + try: + asyncio.run_coroutine_threadsafe( + msg.bus_ref.publish_outbound( + OutboundMessage( + channel=msg.channel_type, + chat_id=msg.chat_id, + content=content, + metadata=msg.metadata, + ) + ), + bus_loop, + ).result(timeout=15) + return True + except Exception as exc: + _channel_logger.debug("ask_user send failed: %s", exc) + return False + + # Ask one question at a time (consistent with Rich CLI / TUI) + total = len(questions) + answers: list[str] = [] + + for i, q in enumerate(questions): + q_text = q.get("question", "") + q_type = q.get("type", "text") + required = q.get("required", True) + + # Format single question + if total == 1: + header = "\u2753 Quick check-in from EvoScientist\n" + else: + header = f"\u2753 Question {i + 1}/{total}\n" + + lines = [header, f"{i + 1}. {q_text}"] + if not required: + lines[-1] += " (optional)" + + if q_type == "multiple_choice": + choices = q.get("choices", []) + for j, choice in enumerate(choices): + label = choice.get("value", str(choice)) + letter = chr(ord("A") + j) + lines.append(f" {letter}. {label}") + other_letter = chr(ord("A") + len(choices)) + lines.append(f" {other_letter}. Other") + lines.append( + f"\nReply with a letter ({'/'.join(chr(ord('A') + k) for k in range(len(choices) + 1))}), or 'cancel'." + ) + else: + skip_hint = " Leave empty to skip." if not required else "" + lines.append(f"\nReply with your answer, or 'cancel'.{skip_hint}") + + if not _send("\n".join(lines)): + return {"status": "cancelled"} + + # Wait for reply + hitl_event = _register_hitl_wait(msg.channel_type, msg.chat_id) + replied = hitl_event.wait(timeout=_ASK_USER_TIMEOUT) + reply_text = _pop_hitl_reply(msg.channel_type, msg.chat_id) + + if not replied or not reply_text: + _send("\u23f0 Response timed out.") + return {"status": "cancelled"} + + raw = reply_text.strip() + if raw.lower() == "cancel": + return {"status": "cancelled"} + + # Parse answer + if q_type == "multiple_choice": + choices = q.get("choices", []) + other_letter = chr(ord("A") + len(choices)) + if len(raw) == 1 and raw.upper() == other_letter: + # Other selected — ask for free-form input + if not _send("Please type your answer:"): + return {"status": "cancelled"} + hitl_event = _register_hitl_wait(msg.channel_type, msg.chat_id) + replied = hitl_event.wait(timeout=_ASK_USER_TIMEOUT) + other_text = _pop_hitl_reply(msg.channel_type, msg.chat_id) + if not replied or not other_text: + _send("\u23f0 Response timed out.") + return {"status": "cancelled"} + if other_text.strip().lower() == "cancel": + return {"status": "cancelled"} + answers.append(other_text.strip()) + elif len(raw) == 1 and raw.upper().isalpha(): + idx = ord(raw.upper()) - ord("A") + if 0 <= idx < len(choices): + answers.append(choices[idx].get("value", raw)) + else: + answers.append(raw) + else: + answers.append(raw) + else: + answers.append(raw) + + return {"answers": answers, "status": "answered"} + + +def channel_hitl_prompt( + action_requests: list, + msg: "ChannelMessage", +) -> list[dict] | None: + """Send HITL approval prompt to channel user and wait for reply. + + Blocking function — uses threading.Event.wait(). Safe to call from a + background thread (CLI channel processing or asyncio.to_thread in TUI). + + Returns approval decisions list on approve/auto, or None on reject/timeout. + """ + from ..channels.bus.events import OutboundMessage + from ..channels.consumer import ( + _format_approval_prompt, + _parse_approval_reply, + ) + + # Check session auto-approve (set by a previous "3" reply) + session_key = f"{msg.channel_type}:{msg.chat_id}" + if session_key in _hitl_auto_approve: + return [{"type": "approve"} for _ in action_requests] + + bus_loop = _bus_loop + if not (bus_loop and msg.bus_ref): + _channel_logger.debug("HITL: no bus_loop or bus_ref, rejecting") + return None + + def _send(content: str) -> bool: + """Send a message to the channel user. Returns True on success.""" + try: + asyncio.run_coroutine_threadsafe( + msg.bus_ref.publish_outbound( + OutboundMessage( + channel=msg.channel_type, + chat_id=msg.chat_id, + content=content, + metadata=msg.metadata, + ) + ), + bus_loop, + ).result(timeout=15) + return True + except Exception as exc: + _channel_logger.debug("HITL send failed: %s", exc) + return False + + # 1. Send approval prompt + prompt_text = _format_approval_prompt(action_requests) + if not _send(prompt_text): + return None + + # 2. Wait for channel user's reply + hitl_event = _register_hitl_wait(msg.channel_type, msg.chat_id) + replied = hitl_event.wait(timeout=_HITL_APPROVAL_TIMEOUT) + reply_text = _pop_hitl_reply(msg.channel_type, msg.chat_id) + + if not replied or not reply_text: + _send("\u23f0 Approval timed out. Action rejected.") + return None + + # 3. Parse decision + decision = _parse_approval_reply(reply_text) + if decision == "auto": + _hitl_auto_approve.add(session_key) + return [{"type": "approve"} for _ in action_requests] + if decision == "approve": + return [{"type": "approve"} for _ in action_requests] + + feedback = ( + "Action rejected." + if decision == "reject" + else "Unrecognized reply. Action rejected." + ) + _send(feedback) + return None + + +# --------------------------------------------------------------------------- +# Module-level channel state (bus mode) +# --------------------------------------------------------------------------- + +_manager: Any | None = None # ChannelManager +_bus_loop: asyncio.AbstractEventLoop | None = None +_bus_thread: threading.Thread | None = None +_cli_agent: Any = None # shared agent reference (same as CLI) +_cli_thread_id: str | None = None # shared thread_id (same conversation) + + +def _channels_is_running(channel_type: str | None = None) -> bool: + """Check whether channels are running.""" + if _manager is None: + return False + if channel_type: + ch = _manager.get_channel(channel_type) + return ch is not None and ch._running + return _manager.is_running and bool(_manager.running_channels()) + + +def _channels_running_list() -> list[str]: + """Return names of running channels.""" + return _manager.running_channels() if _manager else [] + + +def _channels_stop(channel_type: str | None = None) -> None: + """Stop channel(s) and clean up module-level state.""" + global _manager, _bus_loop, _bus_thread, _cli_agent, _cli_thread_id + + if channel_type is None: + # Stop everything + if _bus_loop and _manager: + try: + future = asyncio.run_coroutine_threadsafe( + _manager.stop_all(), + _bus_loop, + ) + future.result(timeout=10) + except Exception as e: + _channel_logger.debug(f"Error stopping channels: {e}") + if _manager: + _manager.bus.stop() + if _bus_thread: + _bus_thread.join(timeout=5) + _manager = None + _bus_loop = None + _bus_thread = None + _cli_agent = None + _cli_thread_id = None + return + + # Stop a specific channel + if _manager and _bus_loop: + try: + future = asyncio.run_coroutine_threadsafe( + _manager.remove_channel(channel_type), + _bus_loop, + ) + future.result(timeout=5) + except Exception as e: + _channel_logger.debug(f"Error removing channel {channel_type}: {e}") + + if _manager and not _manager.running_channels(): + _cli_agent = None + _cli_thread_id = None + + +def _start_channels_bus_mode( + config, + agent, + thread_id: str, + *, + send_thinking: bool | None = None, +) -> None: + """Start all channels in bus mode with MessageBus + ChannelManager. + + Creates a single event loop in a daemon thread running the bus, + ChannelManager, and the inbound consumer. + """ + global _manager, _bus_loop, _bus_thread + + from ..channels.channel_manager import ChannelManager + + mgr = ChannelManager.from_config(config) + + effective_send_thinking = ( + getattr(config, "channel_send_thinking", True) + if send_thinking is None + else send_thinking + ) + for channel in mgr._channels.values(): + channel.send_thinking = bool(effective_send_thinking) + + _manager = mgr + + def _bus_thread_entry(): + global _bus_loop + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + _bus_loop = loop + + async def _run(): + consumer = asyncio.create_task(_bus_inbound_consumer(mgr.bus, mgr)) + try: + await mgr.start_all() + finally: + consumer.cancel() + try: + await consumer + except asyncio.CancelledError: + pass + + try: + loop.run_until_complete(_run()) + except Exception as e: + _channel_logger.error( + "Bus thread terminated with error: %s", e, exc_info=True + ) + finally: + _channel_logger.debug("Bus thread event loop closed") + loop.close() + + thread = threading.Thread(target=_bus_thread_entry, daemon=True) + _bus_thread = thread + thread.start() + + # Wait briefly for the loop to start + for _ in range(20): + if _bus_loop is not None: + break + time.sleep(0.1) + + +def _add_channel_to_running_bus( + channel_type: str, + config, + *, + send_thinking: bool | None = None, +) -> None: + """Dynamically add a single channel to the already-running bus. + + Raises: + RuntimeError: If the bus loop or manager is not initialised. + ValueError: If the channel type is unknown or already registered. + """ + if not _manager or not _bus_loop: + raise RuntimeError("Bus not initialised") + + effective_send_thinking = ( + getattr(config, "channel_send_thinking", True) + if send_thinking is None + else send_thinking + ) + + async def _do_add(): + channel = await _manager.add_channel(channel_type, config) + channel.send_thinking = bool(effective_send_thinking) + + future = asyncio.run_coroutine_threadsafe(_do_add(), _bus_loop) + future.result(timeout=10) + + +async def _bus_inbound_consumer(bus, manager) -> None: + """Consume inbound messages from bus and bridge to the main CLI thread. + + Task-based: each inbound message is handled in its own asyncio task + so the consumer loop stays responsive for HITL approval replies. + """ + _tasks: set[asyncio.Task] = set() + try: + while True: + try: + msg = await asyncio.wait_for(bus.consume_inbound(), timeout=1.0) + except TimeoutError: + continue + except asyncio.CancelledError: + break + + # Check if this message is a HITL approval reply + if _try_set_hitl_reply(msg.channel, msg.chat_id, msg.content): + _channel_logger.info( + f"[bus] HITL reply from {msg.channel}:{msg.sender_id}: " + f"{msg.content[:60]}" + ) + continue + + # Regular message — handle in a separate task + _task = asyncio.create_task(_handle_bus_message(bus, manager, msg)) + _tasks.add(_task) + _task.add_done_callback(_tasks.discard) + finally: + for task in list(_tasks): + task.cancel() + if _tasks: + await asyncio.gather(*_tasks, return_exceptions=True) + + +async def _handle_bus_message(bus, manager, msg) -> None: + """Handle a single inbound bus message (runs as an independent task).""" + from ..channels.bus.events import OutboundMessage + + _channel_logger.info( + f"[bus] Received from {msg.channel}:{msg.sender_id}: {msg.content[:60]}..." + ) + manager.record_message(msg.channel, "received") + + channel = manager.get_channel(msg.channel) + typing_active = False + if channel: + await channel.start_typing(msg.chat_id) + typing_active = True + + # Enqueue for main CLI thread to process with its own event loop + cm = ChannelMessage( + msg_id=str(uuid.uuid4()), + content=msg.content, + sender=msg.sender_id, + channel_type=msg.channel, + metadata=msg.metadata, + channel_ref=channel, + bus_ref=bus, + chat_id=msg.chat_id, + message_id=msg.message_id, + ) + response_waiter = _enqueue_channel_message(cm) + + try: + # Two-stage wait: first stage with timeout, then extended wait for late reply + try: + await asyncio.wait_for( + asyncio.shield(response_waiter), + timeout=_RESPONSE_TIMEOUT, + ) + replied = True + except TimeoutError: + replied = False + + if not replied: + _channel_logger.warning( + f"[bus] Response timeout ({_RESPONSE_TIMEOUT}s) for {cm.msg_id}; " + "keeping late-reply delivery active" + ) + try: + await bus.publish_outbound( + OutboundMessage( + channel=msg.channel, + chat_id=msg.chat_id, + content=_LATE_RESPONSE_NOTICE, + reply_to=msg.message_id or None, + metadata=msg.metadata, + ) + ) + manager.record_message(msg.channel, "sent") + except Exception as e: + _channel_logger.error(f"[bus] Late notice send error: {e}") + if channel and typing_active: + await channel.stop_typing(msg.chat_id) + typing_active = False + + # Keep waiting for the actual response + try: + await asyncio.wait_for( + asyncio.shield(response_waiter), + timeout=_LATE_RESPONSE_TIMEOUT, + ) + replied = True + except TimeoutError: + replied = False + + if not replied: + _channel_logger.warning( + f"[bus] Late response timeout ({_LATE_RESPONSE_TIMEOUT}s) " + f"for {cm.msg_id}" + ) + _pop_channel_response(cm.msg_id, cancel_pending=True) + return + + response = _pop_channel_response(cm.msg_id) or "No response" + await bus.publish_outbound( + OutboundMessage( + channel=msg.channel, + chat_id=msg.chat_id, + content=response, + reply_to=msg.message_id or None, + metadata=msg.metadata, + ) + ) + manager.record_message(msg.channel, "sent") + except asyncio.CancelledError: + _pop_channel_response(cm.msg_id, cancel_pending=True) + raise + except Exception as e: + _channel_logger.error(f"[bus] Outbound error: {e}") + finally: + if channel and typing_active: + await channel.stop_typing(msg.chat_id) + + +def _print_channel_panel(channels: list[tuple[str, bool, str]]) -> None: + """Print a summary panel for active channels. + + Args: + channels: List of (name, ok, detail) tuples. + """ + lines: list[Text] = [] + all_ok = True + for name, ok, detail in channels: + line = Text() + if ok: + line.append("\u25cf ", style="green") + line.append(name, style="bold") + else: + line.append("\u2717 ", style="yellow") + line.append(name, style="bold yellow") + all_ok = False + if detail: + line.append(f" {detail}", style="dim") + lines.append(line) + + body = Text("\n").join(lines) + border = "green" if all_ok else "yellow" + console.print( + Panel(body, title="[bold]Channels[/bold]", border_style=border, expand=False) + ) + console.print() + + +def _cmd_channel( + args: str, + agent: Any, + thread_id: str, + *, + send_thinking: bool | None = None, +) -> None: + """Start a channel in background using bus mode. + + Usage: + /channel [telegram|discord|imessage] -- start channel (default from config) + /channel status -- show current channel status + /channel stop -- stop running channel + """ + global _cli_agent, _cli_thread_id + + from ..config import load_config + + app_config = load_config() + + channel_type = args.strip().lower() if args and args.strip() else "" + if channel_type == "status": + running = _channels_running_list() + if running and _manager: + detailed = _manager.get_detailed_status() + table = Table(title="Channel Status", show_header=True, expand=False) + table.add_column("Channel", style="cyan") + table.add_column("Status") + table.add_column("Uptime", style="dim") + table.add_column("Rx", justify="right") + table.add_column("Tx", justify="right") + for ch_name in running: + info = detailed.get(ch_name, {}) + secs = info.get("uptime_seconds", 0) + mins, s = divmod(int(secs), 60) + hours, mins = divmod(mins, 60) + uptime = f"{hours}h{mins:02d}m" if hours else f"{mins}m{s:02d}s" + rx = str(info.get("received", 0)) + tx = str(info.get("sent", 0)) + table.add_row(ch_name, "[green]running[/green]", uptime, rx, tx) + console.print(table) + console.print() + else: + console.print("[dim]No channel running[/dim]\n") + return + + if not channel_type: + channel_type = app_config.channel_enabled + if not channel_type: + console.print("[yellow]No channel configured.[/yellow]") + console.print( + "[dim]Run[/dim] evosci onboard [dim]or specify:[/dim] /channel telegram\n" + ) + return + + requested = [t.strip() for t in channel_type.split(",") if t.strip()] + + if _channels_is_running(): + running = _channels_running_list() + results: list[tuple[str, bool, str]] = [] + for ct in requested: + if ct in running: + results.append((ct, True, "already running")) + else: + try: + _add_channel_to_running_bus( + ct, + app_config, + send_thinking=send_thinking, + ) + results.append((ct, True, "connected (bus)")) + except Exception as e: + results.append((ct, False, str(e))) + _print_channel_panel(results) + return + + _cli_agent = agent + _cli_thread_id = thread_id + + # Override channel_enabled for this invocation + original = app_config.channel_enabled + app_config.channel_enabled = channel_type + try: + _start_channels_bus_mode( + app_config, + agent, + thread_id, + send_thinking=send_thinking, + ) + results = [(ct, True, "connected (bus)") for ct in requested] + except Exception as e: + results = [(ct, False, str(e)) for ct in requested] + finally: + app_config.channel_enabled = original + + _print_channel_panel(results) + + +def _cmd_channel_stop(channel_type: str | None = None) -> None: + """Stop background channel(s). + + Args: + channel_type: Specific channel to stop, or None to stop all. + """ + if not _channels_is_running(): + console.print("[dim]No channel running[/dim]\n") + return + if channel_type: + if not _channels_is_running(channel_type): + console.print(f"[dim]{channel_type} is not running[/dim]\n") + return + _channels_stop(channel_type) + console.print(f"[dim]{channel_type} stopped[/dim]\n") + else: + running = _channels_running_list() + _channels_stop() + console.print(f"[dim]{', '.join(running)} stopped[/dim]\n") + + +def _auto_start_channel( + agent: Any, + thread_id: str, + config, + *, + send_thinking: bool | None = None, +) -> None: + """Start channels automatically from config (bus mode). + + Args: + agent: Compiled agent graph. + thread_id: Current thread ID. + config: EvoScientistConfig with channel settings. + """ + global _cli_agent, _cli_thread_id + + if not config.channel_enabled: + return + + _cli_agent = agent + _cli_thread_id = thread_id + + _start_channels_bus_mode( + config, + agent, + thread_id, + send_thinking=send_thinking, + ) + types = [t.strip() for t in config.channel_enabled.split(",") if t.strip()] + results = [(ct, True, "connected (bus)") for ct in types] + _print_channel_panel(results) diff --git a/EvoScientist/cli/clipboard.py b/EvoScientist/cli/clipboard.py new file mode 100644 index 0000000..19cedfd --- /dev/null +++ b/EvoScientist/cli/clipboard.py @@ -0,0 +1,220 @@ +"""Clipboard utilities for EvoScientist TUI. + +Provides copy-on-select and paste for the Textual TUI with fallback methods: + +Copy (3 methods): + 1. pyperclip — preferred on local machines (uses pbcopy on macOS) + 2. Textual — built-in app.copy_to_clipboard() + 3. OSC 52 — escape sequence for SSH / tmux remote sessions + +Paste (3 methods): + 1. pyperclip — preferred on local machines + 2. Platform-native — pbpaste (macOS), xclip/xsel (Linux), PowerShell (Windows) + 3. Textual — built-in app.paste() on supported terminals +""" + +from __future__ import annotations + +import base64 +import logging +import os +import pathlib +import subprocess +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from textual.app import App + +logger = logging.getLogger(__name__) + +_PREVIEW_MAX = 40 + + +# ── Platform-native clipboard read ──────────────────────────────── + + +def _paste_native() -> str | None: + """Read clipboard using platform-native commands. + + Returns: + Clipboard text, or None if unavailable. + """ + if sys.platform == "darwin": + # macOS: pbpaste + try: + result = subprocess.run( + ["pbpaste"], + capture_output=True, + text=True, + timeout=2, + ) + if result.returncode == 0: + return result.stdout + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + elif sys.platform == "win32": + # Windows: PowerShell Get-Clipboard + try: + result = subprocess.run( + ["powershell", "-command", "Get-Clipboard"], + capture_output=True, + timeout=2, + ) + if result.returncode == 0: + # Try UTF-8 first, fall back to system encoding + for encoding in ("utf-8", "gbk", "cp936"): + try: + return result.stdout.decode(encoding).rstrip("\r\n") + except UnicodeDecodeError: + continue + # Last resort: decode with errors ignored + return result.stdout.decode("utf-8", errors="ignore").rstrip("\r\n") + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + else: + # Linux: try xclip, then xsel + for cmd in ( + ["xclip", "-selection", "clipboard", "-o"], + ["xsel", "--clipboard", "--output"], + ): + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=2, + ) + if result.returncode == 0: + return result.stdout + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + continue + return None + + +# ── OSC 52 (remote / SSH / tmux) ────────────────────────────────── + + +def _copy_osc52(text: str) -> None: + """Copy text using OSC 52 escape sequence (works over SSH/tmux).""" + encoded = base64.b64encode(text.encode("utf-8")).decode("ascii") + seq = f"\033]52;c;{encoded}\a" + if os.environ.get("TMUX"): + seq = f"\033Ptmux;\033{seq}\033\\" + with pathlib.Path("/dev/tty").open("w", encoding="utf-8") as tty: + tty.write(seq) + tty.flush() + + +# ── Preview helper ───────────────────────────────────────────────── + + +def _shorten(texts: list[str]) -> str: + """Return a short preview string for the notification toast.""" + dense = "⏎".join(texts).replace("\n", "⏎") + if len(dense) > _PREVIEW_MAX: + return dense[: _PREVIEW_MAX - 1] + "…" + return dense + + +# ── Public API ───────────────────────────────────────────────────── + + +def copy_selection_to_clipboard(app: App) -> None: + """Copy mouse-selected text from any widget to the system clipboard. + + Called from ``on_mouse_up`` in the Textual app so that selecting + text with the mouse automatically copies it. + """ + selected_texts: list[str] = [] + + for widget in app.query("*"): + if not hasattr(widget, "text_selection") or not widget.text_selection: + continue + selection = widget.text_selection + if selection.end is None: + continue + try: + result = widget.get_selection(selection) + except (AttributeError, TypeError, ValueError, IndexError) as exc: + logger.debug( + "Failed to get selection from %s: %s", + type(widget).__name__, + exc, + ) + continue + if not result: + continue + text, _ = result + if text.strip(): + selected_texts.append(text) + + if not selected_texts: + return + + combined = "\n".join(selected_texts) + + # Try methods in priority order + copy_methods = [app.copy_to_clipboard] + + try: + import pyperclip + + copy_methods.insert(0, pyperclip.copy) + except ImportError: + pass + + copy_methods.append(_copy_osc52) + + for fn in copy_methods: + try: + fn(combined) + app.notify( + f'"{_shorten(selected_texts)}" copied', + severity="information", + timeout=2, + markup=False, + ) + except (OSError, RuntimeError, TypeError) as exc: + logger.debug( + "Clipboard method %s failed: %s", getattr(fn, "__name__", repr(fn)), exc + ) + continue + else: + return + + app.notify( + "Failed to copy — no clipboard method available", + severity="warning", + timeout=3, + ) + + +def get_clipboard_text() -> str | None: + """Read text from the system clipboard. + + Tries multiple methods in priority order: + 1. pyperclip (if installed) + 2. Platform-native commands (pbpaste, xclip, PowerShell) + + Returns: + Clipboard text, or None if unavailable or empty. + """ + # 1. Try pyperclip first + try: + import pyperclip + + text = pyperclip.paste() + if text: + return text + except ImportError: + pass + except Exception as exc: + logger.debug("pyperclip.paste() failed: %s", exc) + + # 2. Try platform-native commands + text = _paste_native() + if text: + return text + + return None diff --git a/EvoScientist/cli/commands.py b/EvoScientist/cli/commands.py new file mode 100644 index 0000000..dce6ce0 --- /dev/null +++ b/EvoScientist/cli/commands.py @@ -0,0 +1,1554 @@ +"""Typer command registrations — onboard, config, mcp, main callback.""" + +import asyncio +import logging +import os +import queue +import re +from datetime import datetime +from importlib.metadata import version as _pkg_version +from pathlib import Path +from typing import Annotated, Any + +import asyncpg # type: ignore[import-untyped] +import typer # type: ignore[import-untyped] +from rich.markup import escape +from rich.table import Table + +from ..llm.context_window import DEFAULT_CONTEXT_WINDOW_FALLBACK, resolve_context_window +from ..paths import ensure_dirs, set_workspace_root +from ..stream.display import console +from ._app import admin_app, app, channel_app, config_app, mcp_app +from ._constants import build_metadata +from .agent import ( + _create_session_workspace, + _deduplicate_run_name, + _load_agent, + _shorten_path, +) +from .channel import ( + ChannelMessage, + _channels_stop, + _message_queue, + _set_channel_response, + _start_channels_bus_mode, + channel_ask_user_prompt, + channel_hitl_prompt, +) +from .interactive import cmd_interactive, cmd_run +from .mcp_ui import ( + _mcp_add_server_from_kwargs, + _mcp_edit_server_fields, + _mcp_list_servers, + _mcp_remove_server, + _show_mcp_config, +) +from .tui_runtime import run_streaming + +# ============================================================================= +# Onboard command +# ============================================================================= + + +@app.command() +def onboard( + skip_validation: bool = typer.Option( + False, "--skip-validation", help="Skip API key validation during setup" + ), +): + """Interactive setup wizard for EvoScientist + + Guides you through configuring API keys, model selection, + workspace settings, and agent parameters. + """ + from ..config import run_onboard + + run_onboard(skip_validation=skip_validation) + + +# ============================================================================= +# Channel setup command +# ============================================================================= + + +@channel_app.command("setup") +def channel_setup(): + """Interactive channel configuration wizard. + + Guides you through selecting and configuring messaging channels + (Telegram, Discord, or iMessage). + """ + import asyncio + + try: + asyncio.get_event_loop() + except RuntimeError: + asyncio.set_event_loop(asyncio.new_event_loop()) + + from ..config import load_config, save_config + from ..config.onboard import _step_channels + + config = load_config() + updates = _step_channels(config) + if updates: + for key, value in updates.items(): + setattr(config, key, value) + save_config(config) + console.print("[green]Channel configuration saved.[/green]") + else: + console.print("[dim]No changes made.[/dim]") + + +# ============================================================================= +# Compact helper +# ============================================================================= + +_COMPACT_CONTEXT_WINDOW_FALLBACK = DEFAULT_CONTEXT_WINDOW_FALLBACK +_MANUAL_COMPACT_MIN_FRACTION = 0.40 +_MANUAL_COMPACT_MIN_PERCENT = int(_MANUAL_COMPACT_MIN_FRACTION * 100) + + +class CompactResult: + """Structured result from compact_conversation. + + Attributes: + status: "noop" (nothing to compact), "ok" (compacted), or "error". + message: Short human-readable message (used as fallback / TUI text). + messages_compacted: Number of messages summarized (0 for noop/error). + messages_kept: Number of messages unchanged. + tokens_before: Total tokens before compaction. + tokens_after: Total tokens after compaction. + tokens_summarized: Tokens in the summarized portion (before). + tokens_summary: Tokens in the summary message (after). + pct_decrease: Percentage decrease. + context_window: Model context window used for thresholding. + context_percent: Effective context utilization percent. + summary_text: Human-readable compact summary content for UI display. + """ + + __slots__ = ( + "context_percent", + "context_window", + "message", + "messages_compacted", + "messages_kept", + "pct_decrease", + "status", + "summary_text", + "tokens_after", + "tokens_before", + "tokens_summarized", + "tokens_summary", + ) + + def __init__( + self, + status: str, + message: str, + *, + messages_compacted: int = 0, + messages_kept: int = 0, + tokens_before: int = 0, + tokens_after: int = 0, + tokens_summarized: int = 0, + tokens_summary: int = 0, + pct_decrease: int = 0, + context_window: int = 0, + context_percent: int = 0, + summary_text: str = "", + ): + self.status = status + self.message = message + self.messages_compacted = messages_compacted + self.messages_kept = messages_kept + self.tokens_before = tokens_before + self.tokens_after = tokens_after + self.tokens_summarized = tokens_summarized + self.tokens_summary = tokens_summary + self.pct_decrease = pct_decrease + self.context_window = context_window + self.context_percent = context_percent + self.summary_text = summary_text + + def __str__(self) -> str: + return self.message + + +class CompactSummaryRenderable: + """Rich renderable payload for the manual compact summary content.""" + + __slots__ = ("summary_text",) + + def __init__(self, summary_text: str): + self.summary_text = (summary_text or "").strip() + + def __rich_console__(self, console, options): + yield render_compact_summary_panel(self.summary_text) + + +def _resolve_context_window( + model: Any, fallback: int = _COMPACT_CONTEXT_WINDOW_FALLBACK +) -> int: + """Resolve a model context window with a stable fallback.""" + return resolve_context_window(model, fallback=fallback) + + +def _percent_used(tokens: int, context_window: int) -> int: + """Return a clamped utilization percent.""" + if context_window <= 0: + return 0 + return max(0, min(100, round((tokens / context_window) * 100))) + + +def render_compact_result(result: CompactResult): # -> rich.text.Text + """Render a CompactResult as styled Rich Text. + + Uses the same visual language as the token usage display: + cyan for numbers, green for savings, dim for labels. + """ + from rich.text import Text + + output = Text() + + if result.status == "noop": + output.append("○ ", style="dim") + output.append("Manual compact not needed", style="dim") + if result.tokens_before > 0: + output.append(" [", style="dim") + output.append(f"{result.tokens_before:,}", style="cyan") + if result.context_window > 0: + output.append(" / ", style="dim") + output.append(f"{result.context_window:,}", style="cyan") + output.append(" tokens", style="dim") + output.append(" │ ", style="dim") + output.append(f"{result.context_percent}%", style="cyan") + output.append(" of window", style="dim") + else: + output.append(" tokens", style="dim") + output.append("]", style="dim") + if result.message: + output.append("\n ", style="") + output.append(result.message, style="dim") + return output + + if result.status == "error": + output.append("✗ ", style="red") + output.append(result.message, style="red") + return output + + # status == "ok" + output.append("✓ ", style="green") + output.append("Compacted ", style="dim") + output.append(f"{result.messages_compacted}", style="bold") + output.append(" messages", style="dim") + output.append(" [", style="dim") + output.append(f"{result.tokens_before:,}", style="cyan") + output.append(" → ", style="dim") + output.append(f"{result.tokens_after:,}", style="green") + output.append(" tokens", style="dim") + output.append(f" ↓{result.pct_decrease}%", style="green bold") + output.append("]", style="dim") + + # Second line: detail breakdown + output.append("\n ", style="") + output.append("Summarized: ", style="dim") + output.append(f"{result.tokens_summarized:,}", style="cyan") + output.append(" → ", style="dim") + output.append(f"{result.tokens_summary:,}", style="green") + output.append(" │ ", style="dim") + output.append("Kept: ", style="dim") + output.append(f"{result.messages_kept}", style="cyan") + output.append(" messages unchanged", style="dim") + if result.context_window > 0: + output.append(" │ ", style="dim") + output.append("Window: ", style="dim") + output.append(f"{result.context_percent}%", style="cyan") + output.append(" used", style="dim") + + return output + + +def render_compact_summary_panel(summary_text: str): + """Render the compacted summary content as a Rich panel.""" + from rich.panel import Panel + from rich.text import Text + + content = (summary_text or "").strip() + body = Text(content or "(empty summary)", style="dim italic") + return Panel( + body, + title="Context Compacted", + border_style="#f59e0b", + padding=(0, 1), + ) + + +def build_compact_summary_renderable( + result: CompactResult, +) -> CompactSummaryRenderable | None: + """Build the UI summary payload for a successful compact operation.""" + if result.status != "ok" or not result.summary_text.strip(): + return None + return CompactSummaryRenderable(result.summary_text) + + +async def compact_conversation( + agent: Any, + thread_id: str | None, + *, + input_tokens_hint: int | None = None, +) -> CompactResult: + """Compact the conversation by summarizing old messages. + + Reads the agent's checkpointed state, creates a temporary + ``SummarizationMiddleware``, generates a summary, and writes + the compacted state back via ``aupdate_state``. + + ``input_tokens_hint`` is the real LLM input token count from the last + ``usage_metadata`` (includes system prompt + tool schemas). When + provided it is used for the display values in ``CompactResult`` so the + panel stays in sync with the status bar; the internal compact logic + (cutoff determination) still uses message-level token counts. + + Returns a structured ``CompactResult``. + """ + if not agent or not thread_id: + return CompactResult("noop", "Nothing to compact — start a conversation first.") + + from langchain_core.messages.utils import count_tokens_approximately + + config = {"configurable": {"thread_id": thread_id}} + + try: + state_snapshot = await agent.aget_state(config) + except Exception as exc: + return CompactResult("error", f"Failed to read state: {exc}") + + messages = state_snapshot.values.get("messages", []) + if not messages: + return CompactResult( + "noop", "Nothing to compact — no messages in conversation." + ) + + from deepagents.middleware.summarization import ( + SummarizationEvent, + SummarizationMiddleware, + compute_summarization_defaults, + ) + + from ..EvoScientist import _ensure_chat_model, _get_default_backend + + try: + model = _ensure_chat_model() + except Exception as exc: + return CompactResult( + "error", f"Compaction requires a working model configuration: {exc}" + ) + + backend = _get_default_backend() + context_window = _resolve_context_window(model) + + defaults = compute_summarization_defaults(model) + middleware = SummarizationMiddleware( + model=model, + backend=backend, + keep=defaults["keep"], + trim_tokens_to_summarize=None, + ) + + # Rebuild effective message list accounting for prior compaction + event = state_snapshot.values.get("_summarization_event") + effective = middleware._apply_event_to_messages(messages, event) + effective_tokens = count_tokens_approximately(effective) + + # For display and threshold we prefer the real LLM input token count + # (includes system prompt + tool schemas) so the panel stays in sync with + # the status bar. The internal compact logic (cutoff, partition, savings) + # still uses effective_tokens (message-level) because compact only reduces + # messages, not the constant system/tool overhead. + display_tokens = ( + input_tokens_hint + if input_tokens_hint is not None and input_tokens_hint > 0 + else effective_tokens + ) + display_percent = _percent_used(display_tokens, context_window) + + if display_percent < _MANUAL_COMPACT_MIN_PERCENT: + return CompactResult( + "noop", + "Conversation is below the manual compact threshold " + f"({display_percent}% < {_MANUAL_COMPACT_MIN_PERCENT}%).", + tokens_before=display_tokens, + context_window=context_window, + context_percent=display_percent, + ) + + cutoff = middleware._determine_cutoff_index(effective) + if cutoff == 0: + return CompactResult( + "noop", + f"Conversation (~{display_tokens:,} tokens) is within the retention budget.", + tokens_before=display_tokens, + context_window=context_window, + context_percent=display_percent, + ) + + to_summarize, to_keep = middleware._partition_messages(effective, cutoff) + + tokens_summarized = count_tokens_approximately(to_summarize) + tokens_kept = count_tokens_approximately(to_keep) + tokens_before = tokens_summarized + tokens_kept + + # Skip if savings would be negligible — compacting ≤2 messages with + # <2% of total tokens prevents the infinite 1-message-at-a-time loop + # that occurs when the conversation sits just above the keep budget. + _MIN_COMPACT_MESSAGES = 3 + _MIN_COMPACT_TOKEN_FRACTION = 0.02 + if ( + len(to_summarize) < _MIN_COMPACT_MESSAGES + and tokens_summarized < tokens_before * _MIN_COMPACT_TOKEN_FRACTION + ): + return CompactResult( + "noop", + f"Nothing to compact — only {len(to_summarize)} message(s) " + f"({tokens_summarized:,} tokens) would be summarized, " + f"not worth the overhead.", + tokens_before=display_tokens, + context_window=context_window, + context_percent=display_percent, + ) + + # Generate summary (LLM call) + summary = await middleware._acreate_summary(to_summarize) + + # Inject thread_id into LangGraph contextvar so _get_thread_id() finds it + # (compact runs outside a runnable context, so get_config() would fail + # and the middleware would generate a random "session_xxx" filename instead + # of reusing the real thread_id). + from langgraph.config import var_child_runnable_config + + _token = var_child_runnable_config.set(config) + + # Offload old messages to backend + file_path: str | None = None + try: + file_path = await middleware._aoffload_to_backend(backend, to_summarize) + except Exception: + pass # non-fatal — proceed without offloaded history + finally: + var_child_runnable_config.reset(_token) + + summary_msg = middleware._build_new_messages_with_path(summary, file_path)[0] + + # Compute token savings (message-level, used for pct calculation) + tokens_summary = count_tokens_approximately([summary_msg]) + tokens_after = tokens_summary + tokens_kept + pct = ( + round((tokens_before - tokens_after) / tokens_before * 100) + if tokens_before > 0 + else 0 + ) + + # Adjust display totals: preserve real overhead (system + tools) by + # offsetting from input_tokens_hint rather than using bare message counts. + msg_reduction = tokens_before - tokens_after # how many message tokens saved + display_before = display_tokens + display_after = max(0, display_tokens - msg_reduction) + display_after_percent = _percent_used(display_after, context_window) + + # Append savings note to summary message for model awareness + savings_note = ( + f"\n\n{len(to_summarize)} messages were compacted " + f"({tokens_summarized:,} → {tokens_summary:,} tokens). " + f"Total context: {display_before:,} → {display_after:,} tokens " + f"({pct}% decrease), " + f"{len(to_keep)} messages unchanged." + ) + summary_msg.content += savings_note + + state_cutoff = middleware._compute_state_cutoff(event, cutoff) + + new_event: SummarizationEvent = { + "cutoff_index": state_cutoff, + "summary_message": summary_msg, + "file_path": file_path, + } + + await agent.aupdate_state(config, {"_summarization_event": new_event}) + + return CompactResult( + "ok", + f"Compacted {len(to_summarize)} messages " + f"({display_before:,} → {display_after:,} tokens, {pct}% decrease)", + messages_compacted=len(to_summarize), + messages_kept=len(to_keep), + tokens_before=display_before, + tokens_after=display_after, + tokens_summarized=tokens_summarized, + tokens_summary=tokens_summary, + pct_decrease=pct, + context_window=context_window, + context_percent=display_after_percent, + summary_text=summary, + ) + + +# ============================================================================= +# Serve helpers +# ============================================================================= + +_serve_logger = logging.getLogger(__name__) + + +def _serve_process_message( + msg: ChannelMessage, + *, + agent: Any, + thread_id: str, + model: str | None, + workspace_dir: str, + show_thinking: bool, +) -> None: + """Process a single channel message in headless serve mode. + + Headless equivalent of interactive.py's ``_process_channel_message``. + No CLI prompt manipulation — just log lines for monitoring. + """ + import asyncio + + from .channel import _bus_loop + + console.print( + f"[dim][{msg.channel_type}] {msg.sender}: {escape(msg.content[:80])}[/dim]" + ) + + # -- channel callback helpers (same pattern as interactive.py) -- + + def _send_to_channel(coro, label: str, timeout: int = 15) -> None: + loop = _bus_loop + if not loop: + return + try: + asyncio.run_coroutine_threadsafe(coro, loop).result(timeout=timeout) + except Exception as e: + _serve_logger.debug(f"{label} send failed: {e}") + + def _send_thinking(thinking: str) -> None: + ch = msg.channel_ref + if ch and ch.send_thinking: + _send_to_channel( + ch.send_thinking_message( + sender=msg.chat_id, + thinking=thinking, + metadata=msg.metadata, + ), + "Thinking", + ) + + def _send_todo(items: list[dict]) -> None: + from ..channels.consumer import _format_todo_list + + if msg.channel_ref: + _send_to_channel( + msg.channel_ref.send_todo_message( + sender=msg.chat_id, + content=_format_todo_list(items), + metadata=msg.metadata, + ), + "Todo", + ) + + def _send_media(file_path: str) -> None: + if msg.channel_ref: + _send_to_channel( + msg.channel_ref.send_media( + recipient=msg.chat_id, + file_path=file_path, + metadata=msg.metadata, + ), + "Media", + timeout=30, + ) + + def _hitl_prompt(action_requests: list) -> list[dict] | None: + return channel_hitl_prompt(action_requests, msg) + + def _ask_user_prompt(ask_user_data: dict) -> dict: + return channel_ask_user_prompt(ask_user_data, msg) + + meta = build_metadata(workspace_dir, model) + try: + response = run_streaming( + ui_backend="cli", + agent=agent, + message=msg.content, + thread_id=thread_id, + show_thinking=show_thinking, + interactive=True, + metadata=meta, + on_thinking=_send_thinking, + on_todo=_send_todo, + on_file_write=_send_media, + hitl_prompt_fn=_hitl_prompt, + ask_user_prompt_fn=_ask_user_prompt, + ) + except Exception as e: + response = f"Error: {e}" + console.print(f"[red]Serve error: {e}[/red]") + + _set_channel_response(msg.msg_id, response) + console.print(f"[dim][{msg.channel_type}] Replied to {msg.sender}[/dim]") + + +# ============================================================================= +# Serve command (headless mode) +# ============================================================================= + + +@app.command() +def serve( + no_thinking: bool = typer.Option( + False, "--no-thinking", help="Disable thinking relay to channels" + ), + workdir: str | None = typer.Option( + None, "--workdir", help="Override workspace directory" + ), + auto_approve: bool = typer.Option( + False, + "--auto-approve", + help="Skip tool approval prompts for HITL actions", + ), + auto_mode: bool = typer.Option( + False, + "--auto-mode", + help="Run unattended: skip ask_user and tool approval prompts", + ), + ask_user: bool = typer.Option( + False, + "--ask-user", + help="Enable agent to ask clarifying questions about your research preferences", + ), + debug: bool = typer.Option( + False, + "--debug", + help="Enable debug logging and channel trace output in serve mode", + ), +): + """Run EvoScientist in headless mode -- channels only, no interactive prompt. + + Starts all configured channels and processes messages via the agent. + Press Ctrl+C to shut down. + """ + from ..config import apply_config_to_env, get_effective_config + + cli_overrides = {} + if auto_approve: + cli_overrides["auto_approve"] = True + if auto_mode: + cli_overrides["auto_mode"] = True + cli_overrides["auto_approve"] = True + cli_overrides["enable_ask_user"] = False + elif ask_user: + cli_overrides["enable_ask_user"] = True + if debug: + cli_overrides["log_level"] = "DEBUG" + cli_overrides["channel_debug_tracing"] = True + config = get_effective_config(cli_overrides) + if debug: + os.environ["EVOSCIENTIST_LOG_LEVEL"] = "DEBUG" + os.environ["EVOSCIENTIST_CHANNEL_DEBUG_TRACING"] = "true" + apply_config_to_env(config) + if debug: + _configure_logging() + + # Auto-start ccproxy if any provider uses OAuth mode + _ccproxy_proc_serve = None + if config.anthropic_auth_mode == "oauth" or config.openai_auth_mode == "oauth": + try: + from ..ccproxy_manager import maybe_start_ccproxy, stop_ccproxy + + _ccproxy_proc_serve = maybe_start_ccproxy(config) + if _ccproxy_proc_serve: + import atexit + + atexit.register(stop_ccproxy, _ccproxy_proc_serve) + except RuntimeError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + + if not config.channel_enabled: + console.print("[red]No channels configured.[/red]") + console.print("[dim]Run [bold]evosci channel setup[/bold] first.[/dim]") + raise typer.Exit(1) + + effective_channel_thinking = config.channel_send_thinking and (not no_thinking) + if workdir: + ws = os.path.abspath(os.path.expanduser(workdir)) + elif config.default_workdir: + ws = os.path.abspath(os.path.expanduser(config.default_workdir)) + else: + ws = os.getcwd() + os.makedirs(ws, exist_ok=True) + set_workspace_root(ws) + ensure_dirs() + + console.print("[dim]Loading agent...[/dim]") + agent = _load_agent(workspace_dir=ws, config=config) + from ..sessions import generate_thread_id + + tid = generate_thread_id() + + _start_channels_bus_mode( + config, + agent, + tid, + send_thinking=effective_channel_thinking, + ) + console.print("[green]Serve mode started (bus mode).[/green]") + + console.print(f"[dim]Thread: {tid}[/dim]") + console.print(f"[dim]Workspace: {_shorten_path(ws)}[/dim]") + console.print("[dim]Press Ctrl+C to stop.[/dim]\n") + + try: + while True: + try: + msg = _message_queue.get(timeout=1.0) + except queue.Empty: + continue + _serve_process_message( + msg, + agent=agent, + thread_id=tid, + model=config.model, + workspace_dir=ws, + show_thinking=effective_channel_thinking, + ) + except KeyboardInterrupt: + console.print("\n[dim]Shutting down...[/dim]") + finally: + _channels_stop() + console.print("[dim]Stopped.[/dim]") + + +# ============================================================================= +# Config commands +# ============================================================================= + + +@config_app.callback(invoke_without_command=True) +def config_callback(ctx: typer.Context): + """Configuration management commands""" + if ctx.invoked_subcommand is None: + config_list() + + +@config_app.command("list") +def config_list(): + """List all configuration values""" + from ..config import get_config_path, list_config + + config_data = list_config() + + table = Table(title="EvoScientist Configuration", show_header=True) + table.add_column("Setting", style="cyan") + table.add_column("Value") + + # Mask API keys + def format_value(key: str, value: Any) -> str: + if "api_key" in key and value: + return "***" + str(value)[-4:] if len(str(value)) > 4 else "***" + if value == "": + return "[dim](not set)[/dim]" + return str(value) + + for key, value in config_data.items(): + table.add_row(key, format_value(key, value)) + + console.print(table) + console.print(f"\n[dim]Config file: {get_config_path()}[/dim]") + + +@config_app.command("get") +def config_get(key: str = typer.Argument(..., help="Configuration key to get")): + """Get a single configuration value""" + from ..config import get_config_value + + value = get_config_value(key) + if value is None: + console.print(f"[red]Unknown key: {key}[/red]") + raise typer.Exit(1) + + # Mask API keys + if "api_key" in key and value: + display_value = "***" + str(value)[-4:] if len(str(value)) > 4 else "***" + elif value == "": + display_value = "(not set)" + else: + display_value = str(value) + + console.print(f"[cyan]{key}[/cyan]: {display_value}") + + +@config_app.command("set") +def config_set( + key: str = typer.Argument(..., help="Configuration key to set"), + value: str = typer.Argument(..., help="New value"), +): + """Set a single configuration value""" + from ..config import set_config_value + + if set_config_value(key, value): + console.print(f"[green]Set {escape(key)}[/green]") + else: + console.print(f"[red]Invalid key: {escape(key)}[/red]") + raise typer.Exit(1) + + +@config_app.command("reset") +def config_reset( + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +): + """Reset configuration to defaults""" + from ..config import get_config_path, reset_config + + config_path = get_config_path() + + if not config_path.exists(): + console.print("[yellow]No config file to reset.[/yellow]") + return + + if not yes: + confirm = typer.confirm("Reset configuration to defaults?") + if not confirm: + console.print("[dim]Cancelled.[/dim]") + return + + reset_config() + console.print("[green]Configuration reset to defaults.[/green]") + + +@config_app.command("path") +def config_path(): + """Show the configuration file path""" + from ..config import get_config_path + + path = get_config_path() + exists = path.exists() + status = "[green]exists[/green]" if exists else "[dim]not created yet[/dim]" + console.print(f"{path} ({status})") + + +# ============================================================================= +# Admin commands — operate on the admins table (separate from users) +# ============================================================================= + +def _hash_password(pw: str) -> str: + """Hash password — try bcrypt, fallback to stdlib.""" + import hashlib + try: + import bcrypt + return bcrypt.hashpw(pw.encode(), bcrypt.gensalt(12)).decode() + except ImportError: + salt = hashlib.sha256(pw.encode()).hexdigest()[:16] + h = hashlib.pbkdf2_hmac("sha256", pw.encode(), salt.encode(), 100_000) + return f"pbkdf2:{salt}:{h.hex()}" + + +def _get_pg_dsn() -> str: + """Return the PostgreSQL DSN from GATEWAY_DB_URL env var.""" + from dotenv import load_dotenv + + load_dotenv() + dsn = os.environ.get("GATEWAY_DB_URL", "").strip() + if not dsn.startswith("postgresql://"): + console.print("[red]GATEWAY_DB_URL not set or not PostgreSQL[/red]") + console.print("[dim]Set GATEWAY_DB_URL=postgresql://... in .env[/dim]") + raise typer.Exit(1) + return dsn + + +async def _async_pg_execute(dsn: str, sql: str, *args) -> list[asyncpg.Record]: + """Execute a PostgreSQL query and return all rows.""" + conn = await asyncpg.connect(dsn=dsn, timeout=10) + try: + rows = await conn.fetch(sql, *args) + if sql.strip().upper().startswith(("INSERT", "UPDATE", "DELETE")): + await conn.execute("COMMIT") + return rows + finally: + await conn.close() + + +async def _async_pg_fetchone(dsn: str, sql: str, *args) -> asyncpg.Record | None: + """Execute a PostgreSQL query and return one row or None.""" + conn = await asyncpg.connect(dsn=dsn, timeout=10) + try: + row = await conn.fetchrow(sql, *args) + return row + finally: + await conn.close() + + +async def _async_pg_execute_write(dsn: str, sql: str, *args) -> None: + """Execute a PostgreSQL write (INSERT/UPDATE) with commit.""" + conn = await asyncpg.connect(dsn=dsn, timeout=10) + try: + await conn.execute(sql, *args) + finally: + await conn.close() + + +@admin_app.command("create-admin") +def admin_create_admin( + email: str = typer.Option(..., "--email", help="管理员邮箱"), + username: str = typer.Option(..., "--username", help="用户名"), + password: str = typer.Option( + ..., "--password", prompt=True, hide_input=True, help="密码" + ), +): + """创建管理员账户 — 写入 PostgreSQL admins 表""" + import asyncio + import uuid + + dsn = _get_pg_dsn() + uid = str(uuid.uuid4())[:8] + hashed = _hash_password(password) + + async def _run(): + existing = await _async_pg_fetchone( + dsn, "SELECT id FROM admins WHERE email = $1", email + ) + if existing: + console.print(f"[yellow]Admin {email} already exists[/yellow]") + return + await _async_pg_execute_write( + dsn, + "INSERT INTO admins (uid, username, email, password, status) VALUES ($1, $2, $3, $4, 'active')", + uid, username, email, hashed, + ) + console.print(f"[green]Admin '{username}' created ({email}, uid={uid})[/green]") + + try: + asyncio.run(_run()) + except Exception as e: + console.print(f"[red]Failed: {e}[/red]") + raise typer.Exit(1) + + +@admin_app.command("reset-password") +def admin_reset_password( + email: str = typer.Option(..., "--email", help="管理员邮箱"), + password: str = typer.Option( + ..., "--password", prompt=True, hide_input=True, help="新密码" + ), +): + """重置管理员密码 — PostgreSQL""" + import asyncio + + dsn = _get_pg_dsn() + hashed = _hash_password(password) + + async def _run(): + existing = await _async_pg_fetchone( + dsn, "SELECT id, username FROM admins WHERE email = $1", email + ) + if not existing: + console.print(f"[red]Admin {email} not found[/red]") + raise typer.Exit(1) + await _async_pg_execute_write( + dsn, + "UPDATE admins SET password = $1, updated_at = NOW() WHERE email = $2", + hashed, email, + ) + console.print( + f"[green]Password reset for {existing['username']} ({email})[/green]" + ) + + try: + asyncio.run(_run()) + except Exception as e: + console.print(f"[red]Failed: {e}[/red]") + raise typer.Exit(1) + + +@admin_app.command("list") +def admin_list(): + """列出所有管理员 — PostgreSQL""" + import asyncio + + dsn = _get_pg_dsn() + + async def _run(): + rows = await _async_pg_execute( + dsn, + "SELECT uid, username, email, status, created_at FROM admins ORDER BY created_at", + ) + if not rows: + console.print("[dim]No admins found[/dim]") + return + for r in rows: + console.print( + f" {r['uid']} {r['username']:20s} {r['email']:35s} {r['status']:10s} {r['created_at']}" + ) + + asyncio.run(_run()) + + +@admin_app.command("reset-user-password") +def admin_reset_user_password( + email: str = typer.Option(..., "--email", help="用户邮箱"), + password: str = typer.Option( + ..., "--password", prompt=True, hide_input=True, help="新密码" + ), +): + """重置普通用户密码 — PostgreSQL""" + import asyncio + + dsn = _get_pg_dsn() + hashed = _hash_password(password) + + async def _run(): + existing = await _async_pg_fetchone( + dsn, "SELECT id, username FROM users WHERE email = $1", email + ) + if not existing: + console.print(f"[red]User {email} not found[/red]") + raise typer.Exit(1) + await _async_pg_execute_write( + dsn, + "UPDATE users SET password = $1, updated_at = NOW() WHERE email = $2", + hashed, email, + ) + console.print( + f"[green]Password reset for user {existing['username']} ({email})[/green]" + ) + + try: + asyncio.run(_run()) + except Exception as e: + console.print(f"[red]Failed: {e}[/red]") + raise typer.Exit(1) + + +# ============================================================================= +# MCP commands +# ============================================================================= + + +@mcp_app.callback(invoke_without_command=True) +def mcp_callback(ctx: typer.Context): + """MCP server management commands""" + if ctx.invoked_subcommand is None: + mcp_list() + + +@mcp_app.command("list") +def mcp_list(): + """List configured MCP servers""" + _mcp_list_servers() + + +@mcp_app.command("config") +def mcp_config( + name: str | None = typer.Argument(None, help="Server name (omit to show all)"), +): + """Show detailed configuration for MCP servers + + \b + Examples: + evosci mcp config # Show all servers in detail + evosci mcp config filesystem # Show one server + """ + status = _show_mcp_config(name or "", show_blank_line=False) + if status == "empty": + console.print( + "[dim]Add one with:[/dim] EvoSci mcp add [args...]" + ) + return + if status == "missing": + raise typer.Exit(1) + + +@mcp_app.command("add") +def mcp_add( + name: Annotated[str, typer.Argument(help="Server name")], + target: Annotated[str, typer.Argument(help="Command (stdio) or URL (http/sse)")], + args: Annotated[ + list[str] | None, typer.Argument(help="Extra args for stdio command") + ] = None, + transport: Annotated[ + str | None, + typer.Option("--transport", "-T", help="Transport type (default: auto-detect)"), + ] = None, + tools: Annotated[ + str | None, + typer.Option( + "--tools", + "-t", + help="Comma-separated tool allowlist (supports wildcards: *_exa, read_*)", + ), + ] = None, + expose_to: Annotated[ + str | None, + typer.Option("--expose-to", "-e", help="Comma-separated target agents"), + ] = None, + header: Annotated[ + list[str] | None, + typer.Option("--header", "-H", help="HTTP header as Key:Value (repeatable)"), + ] = None, + env: Annotated[ + list[str] | None, + typer.Option("--env", help="Env var as KEY=VALUE for stdio (repeatable)"), + ] = None, + env_ref: Annotated[ + list[str] | None, + typer.Option( + "--env-ref", help="Env var name as ${NAME} runtime ref (repeatable)" + ), + ] = None, +): + """Add an MCP server to user config + + \b + Transport is auto-detected: URLs default to http, commands default to stdio. + + \b + Examples: + evosci mcp add sequential-thinking npx -- -y @modelcontextprotocol/server-sequential-thinking + evosci mcp add docs-langchain https://docs.langchain.com/mcp + evosci mcp add my-sse https://example.com/sse --transport sse -e research-agent + evosci mcp add brave-search npx --env-ref BRAVE_API_KEY -- -y @modelcontextprotocol/server-brave-search + """ + from ..mcp import build_mcp_add_kwargs + + # Merge env and env_ref into a single dict + env_dict: dict[str, str] = {} + for e in env or []: + if "=" in e: + k, v = e.split("=", 1) + env_dict[k.strip()] = v.strip() + for ref in env_ref or []: + env_dict[ref] = "${" + ref + "}" + + kwargs = build_mcp_add_kwargs( + name=name, + target=target, + extra_args=list(args) if args else None, + transport=transport, + tools=[t.strip() for t in tools.split(",") if t.strip()] if tools else None, + expose_to=[a.strip() for a in expose_to.split(",") if a.strip()] + if expose_to + else None, + headers={ + k.strip(): v.strip() + for h in (header or []) + for k, v in [h.split(":", 1)] + if ":" in h + } + or None, + env=env_dict or None, + ) + + if not _mcp_add_server_from_kwargs(kwargs, show_reload_hint=False): + raise typer.Exit(1) + + +@mcp_app.command("edit") +def mcp_edit( + name: Annotated[str, typer.Argument(help="Server name to edit")], + transport: Annotated[ + str | None, typer.Option("--transport", help="New transport type") + ] = None, + command: Annotated[ + str | None, typer.Option("--command", help="New command (stdio)") + ] = None, + url: Annotated[ + str | None, typer.Option("--url", help="New URL (http/sse/websocket)") + ] = None, + tools: Annotated[ + str | None, + typer.Option( + "--tools", + "-t", + help="Comma-separated tool allowlist, supports wildcards ('none' to clear)", + ), + ] = None, + expose_to: Annotated[ + str | None, + typer.Option( + "--expose-to", + "-e", + help="Comma-separated target agents ('none' to clear)", + ), + ] = None, + header: Annotated[ + list[str] | None, + typer.Option("--header", "-H", help="HTTP header as Key:Value (repeatable)"), + ] = None, + env: Annotated[ + list[str] | None, + typer.Option("--env", help="Env var as KEY=VALUE for stdio (repeatable)"), + ] = None, +): + """Edit an existing MCP server in user config + + \b + Examples: + evosci mcp edit filesystem --expose-to main,code-agent + evosci mcp edit filesystem -t read_file,write_file + evosci mcp edit my-api --url http://new-host:9090/mcp + evosci mcp edit my-api --tools none + """ + from ..mcp import build_mcp_edit_fields + + fields = build_mcp_edit_fields( + transport=transport, + command=command, + url=url, + tools=tools, + expose_to=expose_to, + headers=header, + env=env, + ) + + if not _mcp_edit_server_fields(name, fields, show_reload_hint=False): + raise typer.Exit(1) + + +@mcp_app.command("remove") +def mcp_remove( + name: str = typer.Argument(..., help="Server name to remove"), +): + """Remove an MCP server from user config""" + if not _mcp_remove_server(name, show_reload_hint=False): + raise typer.Exit(1) + + +@mcp_app.command("install") +def mcp_install( + source: Annotated[ + str | None, typer.Argument(help="Server name or tag filter") + ] = None, +): + """Browse and install MCP servers from the registry and marketplace + + \b + Examples: + evosci mcp install # Interactive browser + evosci mcp install search # Filter by 'search' tag + evosci mcp install sequential-thinking # Install by name + """ + from .mcp_install_cmd import _cmd_install_mcp + + _cmd_install_mcp(source or "") + + +# ============================================================================= +# Main callback (default behavior) +# ============================================================================= + + +def _version_callback(value: bool): + if value: + typer.echo(f"EvoScientist {_pkg_version('EvoScientist')}") + raise typer.Exit() + + +@app.callback(invoke_without_command=True) +def _main_callback( + ctx: typer.Context, + version: bool | None = typer.Option( + None, + "-V", + "--version", + callback=_version_callback, + is_eager=True, + help="Show version and exit.", + ), + mode: str | None = typer.Option( + None, + "-m", + "--mode", + help="Workspace mode: 'daemon' (persistent, default) or 'run' (isolated per-session)", + ), + name: str | None = typer.Option( + None, + "-n", + "--name", + help="Name for this run (used as directory name instead of timestamp; requires --mode run)", + ), + prompt: str | None = typer.Option( + None, "-p", "--prompt", help="Query to execute (single-shot mode)" + ), + thread_id: str | None = typer.Option( + None, "--thread-id", help="Thread ID for conversation persistence" + ), + workdir: str | None = typer.Option( + None, "--workdir", help="Override workspace directory for this session" + ), + use_cwd: bool = typer.Option( + False, "--use-cwd", help="Use current working directory as workspace" + ), + no_thinking: bool = typer.Option( + False, "--no-thinking", help="Disable thinking display" + ), + auto_approve: bool = typer.Option( + False, + "--auto-approve", + help="Skip tool approval prompts for HITL actions", + ), + auto_mode: bool = typer.Option( + False, + "--auto-mode", + help="Run unattended: skip ask_user and tool approval prompts", + ), + ask_user: bool = typer.Option( + False, + "--ask-user", + help="Enable agent to ask clarifying questions about your research preferences", + ), + auth_mode: str | None = typer.Option( + None, + "--auth-mode", + help="Auth mode for Anthropic/OpenAI: api_key (default) or oauth (ccproxy).", + ), + ui: str | None = typer.Option( + None, + "--ui", + help="UI backend: tui (default) or cli.", + ), +): + """EvoScientist Agent - AI-powered research & code execution CLI""" + # If a subcommand was invoked, don't run the default behavior + if ctx.invoked_subcommand is not None: + return + + # Load and apply configuration + from ..config import apply_config_to_env, get_effective_config + + # Build CLI overrides dict + cli_overrides = {} + if mode: + cli_overrides["default_mode"] = mode + if workdir: + cli_overrides["default_workdir"] = workdir + if no_thinking: + cli_overrides["show_thinking"] = False + if ui: + cli_overrides["ui_backend"] = ui + if auto_approve: + cli_overrides["auto_approve"] = True + if auto_mode: + cli_overrides["auto_mode"] = True + cli_overrides["auto_approve"] = True + cli_overrides["enable_ask_user"] = False + elif ask_user: + cli_overrides["enable_ask_user"] = True + if auth_mode: + if auth_mode not in ("api_key", "oauth"): + raise typer.BadParameter("--auth-mode must be 'api_key' or 'oauth'") + cli_overrides["anthropic_auth_mode"] = auth_mode + cli_overrides["openai_auth_mode"] = auth_mode + + config = get_effective_config(cli_overrides) + apply_config_to_env(config) + + # Auto-start ccproxy if any provider uses OAuth mode + _ccproxy_proc = None + if config.anthropic_auth_mode == "oauth" or config.openai_auth_mode == "oauth": + try: + from ..ccproxy_manager import maybe_start_ccproxy, stop_ccproxy + + _ccproxy_proc = maybe_start_ccproxy(config) + if _ccproxy_proc: + import atexit + + atexit.register(stop_ccproxy, _ccproxy_proc) + except RuntimeError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + + show_thinking = config.show_thinking if not no_thinking else False + effective_channel_thinking = config.channel_send_thinking and (not no_thinking) + + # Validate mutually exclusive options + if workdir and use_cwd: + raise typer.BadParameter("Use either --workdir or --use-cwd, not both.") + + if mode and (workdir or use_cwd): + raise typer.BadParameter( + "--mode cannot be combined with --workdir or --use-cwd" + ) + + if mode and mode not in ("run", "daemon"): + raise typer.BadParameter("--mode must be 'run' or 'daemon'") + if ui and ui.lower() not in ("cli", "tui"): + raise typer.BadParameter("--ui must be 'tui' or 'cli'") + + # --name only makes sense in run mode + if name and not ( + mode == "run" + or (not mode and not workdir and not use_cwd and config.default_mode == "run") + ): + raise typer.BadParameter("--name can only be used with --mode run") + + # Sanitize run name: allow alphanumeric, hyphens, underscores + if name: + if not re.fullmatch(r"[A-Za-z0-9_-]+", name): + raise typer.BadParameter( + "--name may only contain letters, digits, hyphens, and underscores" + ) + + # Resolve effective mode from config (CLI mode already applied via overrides) + effective_mode: str | None = ( + None # None means explicit --workdir/--use-cwd was used + ) + + # Resolve workspace directory for this session + # Priority: --workdir > --mode (explicit) > default_workdir > default_mode > cwd + # --use-cwd is kept for backward compat but is now the default behavior + if use_cwd: + workspace_dir = os.getcwd() + set_workspace_root(workspace_dir) + workspace_fixed = True + elif workdir: + workspace_dir = os.path.abspath(os.path.expanduser(workdir)) + os.makedirs(workspace_dir, exist_ok=True) + set_workspace_root(workspace_dir) + workspace_fixed = True + elif mode: + # Explicit --mode overrides default_workdir + effective_mode = mode + workspace_root = config.default_workdir or os.getcwd() + workspace_root = os.path.abspath(os.path.expanduser(workspace_root)) + set_workspace_root(workspace_root) + if effective_mode == "run": + runs_dir = Path(workspace_root, "runs") + session_id = ( + _deduplicate_run_name(name, runs_dir) + if name + else datetime.now().strftime("%Y%m%d_%H%M%S") + ) + workspace_dir = os.path.join(runs_dir, session_id) + os.makedirs(workspace_dir, exist_ok=True) + workspace_fixed = False + else: # daemon + workspace_dir = workspace_root + workspace_fixed = True + elif config.default_workdir: + # Use configured default workdir with configured mode + workspace_root = os.path.abspath(os.path.expanduser(config.default_workdir)) + set_workspace_root(workspace_root) + effective_mode = config.default_mode + if effective_mode == "run": + runs_dir = Path(workspace_root, "runs") + session_id = ( + _deduplicate_run_name(name, runs_dir) + if name + else datetime.now().strftime("%Y%m%d_%H%M%S") + ) + workspace_dir = os.path.join(runs_dir, session_id) + os.makedirs(workspace_dir, exist_ok=True) + workspace_fixed = False + else: # daemon + workspace_dir = workspace_root + workspace_fixed = True + else: + effective_mode = config.default_mode + workspace_root = os.getcwd() + set_workspace_root(workspace_root) + if effective_mode == "run": + workspace_dir = _create_session_workspace(name) + workspace_fixed = False + else: # daemon mode (default) — use current directory + workspace_dir = workspace_root + workspace_fixed = True + + # Ensure memory and skills subdirs exist in workspace + ensure_dirs() + + if prompt: + # Single-shot mode: wrap in persistent checkpointer + import asyncio + + from ..sessions import generate_thread_id, get_checkpointer + + async def _single_shot(): + async with get_checkpointer() as checkpointer: + console.print("[dim]Loading agent...[/dim]") + agent = _load_agent( + workspace_dir=workspace_dir, + checkpointer=checkpointer, + config=config, + ) + tid = thread_id or generate_thread_id() + cmd_run( + agent, + prompt, + thread_id=tid, + show_thinking=show_thinking, + workspace_dir=workspace_dir, + model=config.model, + ui_backend=config.ui_backend, + ) + + import nest_asyncio # type: ignore[import-untyped] + + nest_asyncio.apply() + asyncio.get_event_loop().run_until_complete(_single_shot()) + else: + # Interactive mode (default) — checkpointer managed inside cmd_interactive + cmd_interactive( + show_thinking=show_thinking, + channel_send_thinking=effective_channel_thinking, + workspace_dir=workspace_dir, + workspace_fixed=workspace_fixed, + mode=effective_mode, + model=config.model, + provider=config.provider, + run_name=name, + thread_id=thread_id, + ui_backend=config.ui_backend, + config=config, + ) + + +def _configure_logging(): + """Configure logging with warning symbols for better visibility.""" + from ..config import get_effective_config + from ..logging_config import ( + configure_daily_file_logging, + configure_rich_console_logging, + resolve_log_level, + ) + + try: + cfg = get_effective_config() + raw_level = cfg.log_level + except Exception: + cfg = None + raw_level = "" + resolved_level = resolve_log_level(raw_level, default=logging.WARNING) + + # Apply to root logger (catches all loggers including deepagents) + root_logger = logging.getLogger() + configure_rich_console_logging( + root_logger, + level=resolved_level, + console=console, + dim_warnings=True, + ) + root_logger.setLevel(resolved_level) + + if cfg is not None: + configure_daily_file_logging( + root_logger, + log_dir=cfg.log_dir or None, + level=resolved_level, + retention_days=cfg.log_retention_days, + ) + + # Suppress noisy schema warnings from langchain_google_genai + # (e.g. "Key '$schema' is not supported in schema, ignoring") + logging.getLogger("langchain_google_genai._function_utils").setLevel(logging.ERROR) diff --git a/EvoScientist/cli/file_mentions.py b/EvoScientist/cli/file_mentions.py new file mode 100644 index 0000000..7bc5591 --- /dev/null +++ b/EvoScientist/cli/file_mentions.py @@ -0,0 +1,378 @@ +"""@file mention parsing and injection for CLI and TUI input. + +Usage:: + + text, injected = resolve_file_mentions(user_input, workspace_dir) + # text — original input unchanged + # injected — full prompt with file contents appended (or original if no mentions) +""" + +from __future__ import annotations + +import re +from difflib import SequenceMatcher +from pathlib import Path + +# --------------------------------------------------------------------------- +# Patterns +# --------------------------------------------------------------------------- + +_PATH_CHARS = r"A-Za-z0-9._~/\\:-" + +FILE_MENTION_PATTERN = re.compile(r"@(?P(?:\\.|[" + _PATH_CHARS + r"])+)") +"""Matches ``@path/to/file`` in user input. + +Escaped spaces (``@my\\\\ folder/file``) are supported. Bare ``@`` with no +path characters is not matched (uses ``+`` not ``*``). +""" + +_EMAIL_PREFIX = re.compile(r"[a-zA-Z0-9._%+-]$") +"""If the character immediately before ``@`` matches this, it's an email address.""" + +# Files larger than this are referenced by path only (not embedded inline). +_MAX_EMBED_BYTES = 256 * 1024 # 256 KB + +# Bytes to sample for binary detection (null byte check). +_BINARY_PROBE_BYTES = 8192 + +# Fuzzy search thresholds (ported from DeepAgents FuzzyFileController) +_MIN_FUZZY_SCORE = 15 +_MIN_FUZZY_RATIO = 0.4 + +# Max files to index per workspace +_MAX_WORKSPACE_FILES = 1000 + + +# --------------------------------------------------------------------------- +# Module-level file cache +# --------------------------------------------------------------------------- + +_file_cache: dict[str, list[str]] = {} +"""workspace_dir -> sorted list of relative POSIX paths""" + + +def _get_workspace_files(root: Path) -> list[str]: + """Glob workspace files up to 4 levels deep, skipping hidden entries.""" + files: list[str] = [] + for pattern in ["*", "*/*", "*/*/*", "*/*/*/*"]: + for p in root.glob(pattern): + if not p.is_file(): + continue + rel = p.relative_to(root) + # Skip any part that starts with '.' + if any(part.startswith(".") for part in rel.parts): + continue + files.append(rel.as_posix()) + if len(files) >= _MAX_WORKSPACE_FILES: + return files + return files + + +def _get_cached_files(workspace_dir: str) -> list[str]: + """Return cached file list for *workspace_dir*, scanning if necessary.""" + if workspace_dir not in _file_cache: + _file_cache[workspace_dir] = _get_workspace_files(Path(workspace_dir)) + return _file_cache[workspace_dir] + + +def invalidate_file_cache(workspace_dir: str | None = None) -> None: + """Invalidate the workspace file cache. + + Call when the workspace changes (e.g. ``/new``, ``/resume``). + + Args: + workspace_dir: If given, invalidate only that workspace entry. + If ``None``, clear the entire cache. + """ + if workspace_dir: + _file_cache.pop(workspace_dir, None) + else: + _file_cache.clear() + + +# --------------------------------------------------------------------------- +# Fuzzy scoring (ported from DeepAgents FuzzyFileController) +# --------------------------------------------------------------------------- + + +def _fuzzy_score(query: str, candidate: str) -> float: + """Score how well *query* matches *candidate* path. + + Four-level priority (higher = better match): + + 1. Filename starts with query (150 base + length bonus) + 2. Filename contains query as substring (100–120) + 3. Full path contains query as substring (40–80) + 4. SequenceMatcher ratio on filename (15–30) + + Returns 0 when below ``_MIN_FUZZY_SCORE``. + """ + q = query.lower() + c = candidate.lower() + filename = c.split("/")[-1] + + # Level 1: filename starts with query + if filename.startswith(q): + return 150 + len(q) + + # Level 2: filename contains query + if q in filename: + bonus = 20 if filename.startswith(q[:1]) else 0 + return 100 + bonus + + # Level 3: full path contains query + if q in c: + depth_bonus = max(0, 40 - candidate.count("/") * 5) + return 40 + depth_bonus + + # Level 4: SequenceMatcher on filename + ratio = SequenceMatcher(None, q, filename).ratio() + if ratio >= _MIN_FUZZY_RATIO: + return 15 + ratio * 15 + + return 0 + + +def _fuzzy_search( + query: str, + candidates: list[str], + limit: int = 10, +) -> list[str]: + """Return up to *limit* candidates from *candidates* ranked by fuzzy score. + + When *query* is empty, returns the first *limit* candidates sorted by + depth then name (shallowest, alphabetical first). + """ + if not query: + # Tree order: group by top-level component, dir entry before its children, + # root-level files sorted among top-level dirs alphabetically. + def _tree_key(p: str) -> tuple: + top = p.split("/")[0] # first path component (no slash) + is_file_entry = 0 if p.endswith("/") else 1 # dir entry sorts first + return (top.lower(), is_file_entry, p.lower()) + + return sorted(candidates, key=_tree_key)[:limit] + + scored = [ + (score, c) + for c in candidates + if (score := _fuzzy_score(query, c)) >= _MIN_FUZZY_SCORE + ] + return [c for _, c in sorted(scored, key=lambda x: -x[0])[:limit]] + + +# --------------------------------------------------------------------------- +# Core helpers +# --------------------------------------------------------------------------- + + +def _read_file(path: Path) -> str: + """Return a Markdown snippet for embedding the file inline. + + Files larger than ``_MAX_EMBED_BYTES`` get a path-only reference with a + hint to use the ``read_file`` tool instead. + """ + size = path.stat().st_size + # Binary detection: sample first bytes for null byte (covers all formats). + with open(path, "rb") as fh: + if b"\x00" in fh.read(_BINARY_PROBE_BYTES): + return ( + f"\n### {path.name}\n" + f"Path: `{path}`\n" + "(binary file — use the read_file tool to view it)" + ) + if size > _MAX_EMBED_BYTES: + size_kb = size // 1024 + return ( + f"\n### {path.name}\n" + f"Path: `{path}`\n" + f"Size: {size_kb} KB (too large to embed inline — " + "use the read_file tool to view it)" + ) + content = path.read_text(encoding="utf-8", errors="replace") + return f"\n### {path.name}\nPath: `{path}`\n```\n{content}\n```" + + +def parse_file_mentions( + text: str, + cwd: Path | None = None, +) -> tuple[list[Path], list[str]]: + """Extract resolved ``@file`` paths from *text*. + + Args: + text: Raw user input that may contain ``@path`` mentions. + cwd: Base directory for resolving relative paths. Defaults to the + process working directory. + + Returns: + ``(files, warnings)`` — deduplicated list of resolved, existing + ``Path`` objects (directories excluded) in order of first appearance, + and a list of human-readable warning strings to be displayed by the + caller. Callers must display the warnings themselves using the + appropriate UI mechanism (Rich console, Textual widget, etc.). + """ + if cwd is None: + cwd = Path.cwd() + + workspace_root = cwd.resolve() + files: list[Path] = [] + warnings: list[str] = [] + seen: set[Path] = set() + for match in FILE_MENTION_PATTERN.finditer(text): + # Skip email addresses — character immediately before @ is alphanumeric + before = text[: match.start()] + if before and _EMAIL_PREFIX.search(before): + continue + + raw = match.group("path") + clean = raw.replace("\\ ", " ") + + try: + p = Path(clean).expanduser() + if not p.is_absolute(): + p = cwd / p + resolved = p.resolve() + if not resolved.exists() or not resolved.is_file(): + warnings.append(f"@file not found: {raw}") + continue + # Deduplicate: skip paths already seen in this message. + if resolved in seen: + continue + seen.add(resolved) + files.append(resolved) + # Warn when the file lives outside the workspace root — it may + # contain sensitive content (e.g. @~/.ssh/id_rsa). + # Checked after dedup so a repeated mention only warns once. + try: + resolved.relative_to(workspace_root) + except ValueError: + warnings.append( + f"@{raw} is outside the workspace " + f"({workspace_root}) — embedding may expose sensitive files" + ) + except (OSError, RuntimeError) as exc: + warnings.append(f"invalid @file path {raw!r}: {exc}") + + return files, warnings + + +def resolve_file_mentions( + text: str, + workspace_dir: str | None = None, +) -> tuple[str, str, list[str]]: + """Parse ``@file`` mentions and return *(original_text, final_prompt, warnings)*. + + *final_prompt* equals *original_text* when no valid mentions are found, + otherwise it appends a ``## Referenced Files`` section with the file + contents embedded as fenced code blocks. + + Args: + text: Raw user input. + workspace_dir: Workspace root used for resolving relative paths. + + Returns: + ``(original_text, final_prompt, warnings)`` — the first element is + always the unchanged input; the second is the prompt to send to the + agent; the third is a list of warning strings to display to the user. + """ + cwd = Path(workspace_dir) if workspace_dir else None + files, warnings = parse_file_mentions(text, cwd=cwd) + + if not files: + return text, text, warnings + + parts = [text, "\n\n## Referenced Files\n"] + for path in files: + try: + parts.append(_read_file(path)) + except (OSError, UnicodeDecodeError) as exc: + parts.append(f"\n### {path.name}\n[Error reading file: {exc}]") + + return text, "\n".join(parts), warnings + + +# --------------------------------------------------------------------------- +# Autocomplete helpers (used by CLI completer and TUI) +# --------------------------------------------------------------------------- + + +def _type_hint(rel_path: str) -> str: + """Return a short type label for *rel_path* (extension or ``'file'``).""" + suffix = rel_path.rsplit(".", 1)[-1] if "." in rel_path.split("/")[-1] else "" + return suffix or "file" + + +def complete_file_mention( + text: str, + workspace_dir: str | None = None, +) -> list[tuple[str, str]]: + """Return candidate file paths for the ``@`` prefix at the end of *text*. + + Scans the workspace (up to 4 levels deep) and returns fuzzy-matched + file/dir names relative to *workspace_dir* (or cwd). Returns ``[]`` + when *text* does not end with an ``@``-started token. + + Args: + text: Current input text (up to cursor position). + workspace_dir: Root directory to scan for completions. + + Returns: + List of ``(completion_string, type_hint)`` tuples, e.g. + ``[("@results/v2.json", "json"), ("@README.md", "md")]``. + Directories have a trailing ``/`` and type hint ``"dir"``. + """ + # Find the last @token + match = re.search(r"@([^\s]*)$", text) + if not match: + return [] + + partial = match.group(1).replace("\\ ", " ") + base_str = workspace_dir or str(Path.cwd()) + base = Path(base_str) + + # If partial contains a path separator, check for subdirectory listing + if partial.endswith("/"): + # List directory contents + sub = (base / partial.rstrip("/")).resolve() + if not sub.is_dir(): + return [] + candidates_raw: list[str] = [] + try: + for entry in sorted(sub.iterdir()): + if entry.name.startswith("."): + continue + rel = entry.relative_to(base) + suffix = "/" if entry.is_dir() else "" + candidates_raw.append(rel.as_posix() + suffix) + except OSError: + return [] + return [ + (f"@{r}", "dir" if r.endswith("/") else _type_hint(r)) + for r in candidates_raw[:10] + ] + + # Fuzzy search over cached workspace files + all_files = _get_cached_files(base_str) + + # Also add top-level directories (for dir completion) + dir_candidates: list[str] = [] + try: + for entry in sorted(base.iterdir()): + if entry.is_dir() and not entry.name.startswith("."): + dir_candidates.append(entry.name + "/") + except OSError: + pass + + combined = all_files + dir_candidates + + # Determine query: if partial has a slash, search within that subtree + if "/" in partial: + # Filter candidates to those starting with the directory prefix + dir_prefix = partial.rsplit("/", 1)[0] + "/" + file_query = partial.rsplit("/", 1)[1] + subtree = [c for c in combined if c.startswith(dir_prefix)] + results = _fuzzy_search(file_query, subtree) + else: + results = _fuzzy_search(partial, combined) + + return [(f"@{r}", "dir" if r.endswith("/") else _type_hint(r)) for r in results] diff --git a/EvoScientist/cli/history_suggester.py b/EvoScientist/cli/history_suggester.py new file mode 100644 index 0000000..3b0fb1a --- /dev/null +++ b/EvoScientist/cli/history_suggester.py @@ -0,0 +1,85 @@ +"""History-based auto-suggest for Textual TUI Input widget. + +Reads prompt_toolkit FileHistory format so Rich CLI and TUI share the same +history file at ~/.config/ai4scientist/history. +""" + +from __future__ import annotations + +import time +from pathlib import Path + +from textual.suggester import Suggester + + +class HistorySuggester(Suggester): + """Suggest completions from a shared prompt_toolkit FileHistory file.""" + + def __init__(self, history_file: Path) -> None: + super().__init__(use_cache=False, case_sensitive=True) + self._history_file = history_file + self._entries: list[str] = self._load_history() + + def _load_history(self) -> list[str]: + """Parse prompt_toolkit FileHistory format into a list (newest-first). + + Format: blocks separated by blank lines. Each block has a ``#`` comment + line (timestamp) followed by one or more ``+`` prefixed content lines. + """ + if not self._history_file.exists(): + return [] + + entries: list[str] = [] + current_lines: list[str] = [] + try: + for raw in self._history_file.read_text(encoding="utf-8").splitlines(): + if raw.startswith("+"): + current_lines.append(raw[1:]) + elif raw.startswith("#"): + # Comment/timestamp line — flush any accumulated entry + if current_lines: + entries.append("\n".join(current_lines)) + current_lines = [] + else: + # Blank or unknown line — flush + if current_lines: + entries.append("\n".join(current_lines)) + current_lines = [] + + if current_lines: + entries.append("\n".join(current_lines)) + except OSError: + return [] + + # Newest-first for matching + entries.reverse() + return entries + + async def get_suggestion(self, value: str) -> str | None: + """Return the full history entry whose prefix matches *value*.""" + if not value: + return None + for entry in self._entries: + if entry.startswith(value) and entry != value: + return entry + return None + + def append_entry(self, text: str) -> None: + """Record a new entry in-memory and persist to the history file.""" + text = text.strip() + if not text: + return + + # Prepend so it's found first on next suggestion lookup + self._entries.insert(0, text) + + # Append to file in prompt_toolkit FileHistory format + try: + self._history_file.parent.mkdir(parents=True, exist_ok=True) + with self._history_file.open("a", encoding="utf-8") as f: + ts = time.strftime("%Y-%m-%d %H:%M:%S") + f.write(f"\n# {ts}\n") + for line in text.split("\n"): + f.write(f"+{line}\n") + except OSError: + pass diff --git a/EvoScientist/cli/interactive.py b/EvoScientist/cli/interactive.py new file mode 100644 index 0000000..88b9412 --- /dev/null +++ b/EvoScientist/cli/interactive.py @@ -0,0 +1,1202 @@ +"""Interactive CLI mode and single-shot execution.""" + +import asyncio +import logging +import queue +import random +import sys +from datetime import datetime +from typing import Any + +import typer # type: ignore[import-untyped] +from prompt_toolkit import PromptSession # type: ignore[import-untyped] +from prompt_toolkit.auto_suggest import ( + AutoSuggestFromHistory, # type: ignore[import-untyped] +) +from prompt_toolkit.completion import ( # type: ignore[import-untyped] + Completer, + Completion, +) +from prompt_toolkit.formatted_text import HTML # type: ignore[import-untyped] +from prompt_toolkit.history import FileHistory # type: ignore[import-untyped] +from prompt_toolkit.key_binding import KeyBindings # type: ignore[import-untyped] +from prompt_toolkit.shortcuts import CompleteStyle # type: ignore[import-untyped] +from prompt_toolkit.styles import Style as PtStyle # type: ignore[import-untyped] +from rich.markdown import Markdown +from rich.markup import escape +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +import EvoScientist.cli.channel as _ch_mod + +from ..sessions import ( + _format_relative_time, + delete_thread, + find_similar_threads, + generate_thread_id, + get_checkpointer, + get_thread_messages, + get_thread_metadata, + list_threads, + thread_exists, +) +from ..stream.display import console +from ._constants import LOGO_GRADIENT, LOGO_LINES, WELCOME_SLOGANS, build_metadata +from .agent import _create_session_workspace, _load_agent, _shorten_path +from .channel import ( + ChannelMessage, + _auto_start_channel, + _channels_is_running, + _cmd_channel, + _cmd_channel_stop, + _message_queue, + _set_channel_response, +) +from .file_mentions import complete_file_mention, resolve_file_mentions +from .mcp_ui import _cmd_mcp +from .skills_cmd import ( + _cmd_install_skill, + _cmd_install_skills, + _cmd_list_skills, + _cmd_uninstall_skill, +) +from .status_bar import ( + STATUS_BAD, + STATUS_BAR_BG, + STATUS_CRITICAL, + STATUS_DIM, + STATUS_GOOD, + STATUS_STRONG, + STATUS_TEXT, + STATUS_WARN, + apply_assistant_text_to_snapshot, + apply_user_text_to_snapshot, + build_session_status_snapshot, + build_status_fragments, + build_status_text, + make_empty_status_snapshot, + make_usage_status_snapshot, +) +from .tui_interactive import run_textual_interactive +from .tui_runtime import resolve_ui_backend, run_streaming + +_channel_logger = logging.getLogger(__name__) + + +# ============================================================================= +# Banner +# ============================================================================= + + +def print_banner( + thread_id: str, + workspace_dir: str | None = None, + memory_dir: str | None = None, + mode: str | None = None, + model: str | None = None, + provider: str | None = None, + ui_backend: str | None = None, +): + """Print welcome banner with ASCII art logo, info line, and hint.""" + for line, color in zip(LOGO_LINES, LOGO_GRADIENT, strict=False): + console.print(Text(line, style=f"{color} bold")) + info = Text() + info.append(" ", style="dim") + parts: list[tuple[str, str]] = [] + if model: + parts.append(("Model: ", model)) + if provider: + parts.append(("Provider: ", provider)) + if mode: + parts.append(("Mode: ", mode)) + if ui_backend: + parts.append(("UI: ", ui_backend)) + for i, (label, value) in enumerate(parts): + if i > 0: + info.append(" ", style="dim") + info.append(label, style="dim") + info.append(value, style="magenta") + # Directory line + import os + + effective_dir = workspace_dir or os.getcwd() + home = os.path.expanduser("~") + dir_display = ( + effective_dir.replace(home, "~", 1) + if effective_dir.startswith(home) + else effective_dir + ) + info.append("\n ", style="dim") + info.append("Directory: ", style="dim") + info.append(dir_display, style="magenta") + _nl_key = "Option+Enter" if sys.platform == "darwin" else "Ctrl+J" + info.append("\n Enter ", style="#ffe082") + info.append("send", style="#ffe082 bold") + info.append(f" \u2022 {_nl_key} ", style="#ffe082") + info.append("newline", style="#ffe082 bold") + info.append(" \u2022 Type ", style="#ffe082") + info.append("/", style="#ffe082 bold") + info.append(" for commands", style="#ffe082") + info.append(" \u2022 ", style="#ffe082") + info.append("@ files", style="#ffe082 bold") + info.append(" \u2022 Ctrl+C ", style="#ffe082") + info.append("interrupt", style="#ffe082 bold") + console.print(info) + + +# ============================================================================= +# Slash-command completer +# ============================================================================= + +_SLASH_COMMANDS = [ + ("/current", "Show current session info"), + ("/threads", "List recent sessions"), + ("/resume", "Resume a previous session (prefix match)"), + ("/delete", "Delete a saved session"), + ("/new", "Start a new session"), + ("/skills", "List installed skills"), + ("/install-skill", "Add a skill from path or GitHub"), + ("/uninstall-skill", "Remove an installed skill"), + ("/evoskills", "Browse and install EvoSkills (optional: /evoskills )"), + ("/mcp", "Manage MCP servers"), + ("/channel", "Configure messaging channels"), + ("/compact", "Compact conversation to free context"), + ("/exit", "Quit EvoScientist"), +] + +_COMPLETION_STYLE = PtStyle.from_dict( + { + "completion-menu": "bg:default noreverse nounderline noitalic", + "completion-menu.completion": "bg:default #888888 noreverse", + "completion-menu.completion.current": "bg:default default bold noreverse", + "completion-menu.meta.completion": "bg:default #888888 noreverse", + "completion-menu.meta.completion.current": "bg:default default bold noreverse", + "scrollbar.background": "bg:default", + "scrollbar.button": "bg:default", + "status-bar": f"bg:{STATUS_BAR_BG} {STATUS_TEXT}", + "status-bar-strong": f"bg:{STATUS_BAR_BG} {STATUS_STRONG} bold", + "status-bar-dim": f"bg:{STATUS_BAR_BG} {STATUS_DIM}", + "status-bar-good": f"bg:{STATUS_BAR_BG} {STATUS_GOOD} bold", + "status-bar-warn": f"bg:{STATUS_BAR_BG} {STATUS_WARN} bold", + "status-bar-bad": f"bg:{STATUS_BAR_BG} {STATUS_BAD} bold", + "status-bar-critical": f"bg:{STATUS_BAR_BG} {STATUS_CRITICAL} bold", + } +) + +# Style for questionary pickers — matches _COMPLETION_STYLE visual language: +# gray (#888888) for non-selected, bold for selected, no background changes. +_PICKER_STYLE = PtStyle.from_dict( + { + "questionmark": "#888888", + "question": "", + "pointer": "bold", + "highlighted": "bold", + "text": "#888888", + "answer": "bold", + } +) + + +class SlashCommandCompleter(Completer): + """Autocomplete for slash commands and ``@file`` mentions.""" + + def __init__(self, workspace_dir: str | None = None) -> None: + self._workspace_dir = workspace_dir + + def get_completions(self, document, complete_event): + text = document.text_before_cursor + + # @file mention completion + if "@" in text: + candidates = complete_file_mention(text, self._workspace_dir) + if candidates: + # Replace from the last '@' token + import re as _re + + m = _re.search(r"@[^\s]*$", text) + start = -len(m.group(0)) if m else 0 + for path, type_hint in candidates: + yield Completion(path, start_position=start, display_meta=type_hint) + return + + # Slash command completion + if not text.startswith("/"): + return + for cmd, desc in _SLASH_COMMANDS: + if cmd.startswith(text): + yield Completion( + cmd, + start_position=-len(text), + display=f"{cmd:<40}", + display_meta=desc, + ) + + +# ============================================================================= +# Interactive & single-shot modes +# ============================================================================= + + +def cmd_interactive( + show_thinking: bool = True, + channel_send_thinking: bool = True, + workspace_dir: str | None = None, + workspace_fixed: bool = False, + mode: str | None = None, + model: str | None = None, + provider: str | None = None, + run_name: str | None = None, + thread_id: str | None = None, + ui_backend: str = "cli", + config=None, +) -> None: + """Interactive conversation mode with streaming output. + + The persistent LangGraph checkpointer is opened here and + shared for the entire interactive session lifetime. + + Args: + show_thinking: Whether to display thinking panels + channel_send_thinking: Whether channels should receive thinking messages + workspace_dir: Per-session workspace directory path + workspace_fixed: If True, /new keeps the same workspace directory + mode: Workspace mode ('daemon' or 'run'), displayed in banner + model: Model name to display in banner + provider: LLM provider name to display in banner + run_name: Optional run name for /new session deduplication + thread_id: Optional thread ID to resume a previous session + ui_backend: UI backend ('cli' or 'tui') + """ + import nest_asyncio + + nest_asyncio.apply() + + resolved_ui_backend = resolve_ui_backend(ui_backend, warn_fallback=True) + if resolved_ui_backend == "tui": + from functools import partial + + load_agent = partial(_load_agent, config=config) + run_textual_interactive( + show_thinking=show_thinking, + channel_send_thinking=channel_send_thinking, + workspace_dir=workspace_dir, + workspace_fixed=workspace_fixed, + mode=mode, + model=model, + provider=provider, + run_name=run_name, + thread_id=thread_id, + load_agent=load_agent, + create_session_workspace=_create_session_workspace, + ) + return + + from .. import paths + + memory_dir = str(paths.MEMORY_DIR) + + from ..config.settings import get_config_dir + + config_dir = get_config_dir() + config_dir.mkdir(parents=True, exist_ok=True) + history_file = str(config_dir / "history") + + # Key bindings: Enter submits, Alt+Enter (Option+Enter) inserts newline + _kb = KeyBindings() + + @_kb.add("escape", "enter") # Alt+Enter / Option+Enter on macOS + def _insert_newline(event): + event.current_buffer.insert_text("\n") + + @_kb.add("enter") + def _submit(event): + event.current_buffer.validate_and_handle() + + session = PromptSession( + history=FileHistory(history_file), + auto_suggest=AutoSuggestFromHistory(), + completer=SlashCommandCompleter(workspace_dir=workspace_dir), + complete_style=CompleteStyle.COLUMN, + complete_while_typing=True, + style=_COMPLETION_STYLE, + multiline=True, + key_bindings=_kb, + ) + + def _print_separator(): + """Print a horizontal separator line spanning the terminal width.""" + width = console.size.width + console.print(Text("\u2500" * width, style="dim")) + + # Mutable state for async loop + state: dict[str, Any] = { + "agent": None, + "thread_id": thread_id or generate_thread_id(), + "workspace_dir": workspace_dir, + "running": True, + "resumed": False, + "ui_backend": resolved_ui_backend, + "status_started_at": datetime.now(), + "status_base_snapshot": make_empty_status_snapshot(model), + "status_snapshot": make_empty_status_snapshot(model), + "status_streaming_text": "", + "status_last_input_tokens": None, + } + + def _rebuild_status_snapshot() -> None: + """Compose the visible snapshot from thread state + live output.""" + state["status_snapshot"] = apply_assistant_text_to_snapshot( + state["status_base_snapshot"], + state["status_streaming_text"], + ) + + def _set_status_streaming_text(text: str | None) -> None: + """Update the in-flight assistant overlay used by the status bar.""" + new_text = text or "" + if new_text == state["status_streaming_text"]: + return + state["status_streaming_text"] = new_text + _rebuild_status_snapshot() + + async def _refresh_status_snapshot( + pending_user_text: str | None = None, + *, + reset_streaming_text: bool = True, + ) -> None: + """Recompute the persistent status-bar snapshot for the active thread.""" + pending = (pending_user_text or "").strip() + if pending: + if state["status_last_input_tokens"] is not None: + state["status_base_snapshot"] = apply_user_text_to_snapshot( + make_usage_status_snapshot( + state["status_last_input_tokens"], + model_name=model, + ), + pending, + ) + else: + state["status_base_snapshot"] = await build_session_status_snapshot( + state["thread_id"], + model_name=model, + pending_user_text=pending, + ) + elif state["status_last_input_tokens"] is not None: + state["status_base_snapshot"] = make_usage_status_snapshot( + state["status_last_input_tokens"], + model_name=model, + ) + else: + state["status_base_snapshot"] = await build_session_status_snapshot( + state["thread_id"], + model_name=model, + ) + if reset_streaming_text: + state["status_streaming_text"] = "" + _rebuild_status_snapshot() + + def _bottom_toolbar(): + """Render the persistent bottom status bar for prompt_toolkit.""" + try: + from prompt_toolkit.application import get_app + + width = get_app().output.get_size().columns + except Exception: + width = console.size.width + return build_status_fragments( + state["status_snapshot"], + state["status_started_at"], + width, + ) + + def _stream_status_footer(): + """Render the live Rich footer used during streaming output.""" + return build_status_text( + state["status_snapshot"], + state["status_started_at"], + console.size.width, + ) + + async def _handle_stream_status_event(event_type: str, stream_state) -> None: + """Keep the CLI status bar aligned with live stream progress.""" + if event_type == "usage_stats": + last_input_tokens = getattr(stream_state, "last_input_tokens", 0) + if last_input_tokens > 0: + state["status_last_input_tokens"] = last_input_tokens + state["status_base_snapshot"] = make_usage_status_snapshot( + last_input_tokens, + model_name=model, + ) + _rebuild_status_snapshot() + elif event_type == "text": + _set_status_streaming_text(stream_state.response_text) + elif event_type in ("done", "error"): + _set_status_streaming_text("") + + async def _resolve_thread_id(tid: str) -> str | None: + """Resolve a (possibly partial) thread ID. Returns full ID or None.""" + if await thread_exists(tid): + return tid + similar = await find_similar_threads(tid) + if len(similar) == 1: + return similar[0] + if len(similar) > 1: + console.print( + f"[yellow]Ambiguous thread ID '{escape(tid)}'. Matches:[/yellow]" + ) + for s in similar: + console.print(f" [cyan]{s}[/cyan]") + return None + console.print(f"[red]Thread '{escape(tid)}' not found.[/red]") + return None + + async def _cmd_threads(): + """Handle /threads command — show recent sessions.""" + threads = await list_threads( + limit=0, + include_message_count=True, + include_preview=True, + ) + if not threads: + console.print("[yellow]No saved sessions.[/yellow]") + return + table = Table(title="Sessions", show_header=True, header_style="bold cyan") + table.add_column("ID", style="bold") + table.add_column("Preview", style="dim", max_width=50, no_wrap=True) + table.add_column("Messages", justify="right") + table.add_column("Model", style="dim") + table.add_column("Last Used", style="dim") + for t in threads: + tid = t["thread_id"] + marker = " *" if tid == state["thread_id"] else "" + table.add_row( + f"{tid}{marker}", + t.get("preview", "") or "", + str(t.get("message_count", 0)), + t.get("model", "") or "", + _format_relative_time(t.get("updated_at")), + ) + console.print() + console.print(table) + console.print( + "[dim] /resume[/dim] to continue a session [dim]/delete [/dim] to remove [dim]/new[/dim] to start fresh" + ) + console.print() + + async def _render_history(thread_id: str): + """Display conversation history for a resumed session.""" + messages = await get_thread_messages(thread_id) + if not messages: + return + + HISTORY_WINDOW = 50 + + # Only human and ai messages; skip tool/system + display = [m for m in messages if getattr(m, "type", None) in ("human", "ai")] + + if len(display) > HISTORY_WINDOW: + skipped = len(display) - HISTORY_WINDOW + display = display[-HISTORY_WINDOW:] + console.print(f"[dim]── ... {skipped} earlier messages ──[/dim]") + else: + console.print("[dim]── Conversation history ──[/dim]") + + for msg in display: + msg_type = getattr(msg, "type", None) + content = getattr(msg, "content", "") or "" + + if msg_type == "human": + # Extract text from multimodal list + if isinstance(content, list): + parts = [ + b.get("text", "") + for b in content + if isinstance(b, dict) and b.get("type") == "text" + ] + content = " ".join(parts) if parts else "" + content = content.strip() + if content: + console.print( + Text.assemble(("\u276f ", "bold blue"), (content, "")) + ) + + elif msg_type == "ai": + thinking_text = "" + text_content = "" + + if isinstance(content, list): + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") == "thinking": + thinking_text += block.get("thinking", "") + elif block.get("type") == "text": + text_content += block.get("text", "") + else: + text_content = content or "" + + text_content = text_content.strip() + + # Thinking panel (only when show_thinking is enabled) + if thinking_text.strip() and show_thinking: + console.print( + Panel( + thinking_text.strip(), + title="[bold blue]\U0001f4ad Thinking[/bold blue]", + border_style="blue", + expand=False, + ) + ) + + # AI response — full Markdown rendering + if text_content: + console.print(Markdown(text_content)) + + # Skip tool messages — verbose and not useful in replay + + console.print("[dim]── End of history ──[/dim]") + console.print() + + async def _cmd_resume(arg: str, checkpointer): + """Handle /resume [id] — resume a previous session.""" + if not arg: + # Show interactive session picker with conversation previews + threads = await list_threads( + limit=0, + include_message_count=True, + include_preview=True, + ) + if not threads: + console.print("[yellow]No sessions to resume.[/yellow]") + return + + import questionary + + from .widgets.thread_selector import _build_items + + choices = [] + items = _build_items(threads) + for item in items: + if item["type"] == "header": + choices.append( + questionary.Separator( + f"\u2500\u2500 \U0001f4c2 {item['label']}" + ) + ) + elif item["type"] == "subheader": + choices.append(questionary.Separator(f" {item['label']}")) + else: + t = item["thread"] + tid = t["thread_id"] + preview = t.get("preview", "") or "" + msgs = t.get("message_count", 0) + model = t.get("model", "") or "" + when = _format_relative_time(t.get("updated_at")) + indent = " " if item.get("indented") else " " + parts = [f"{indent}{tid}"] + if preview: + parts.append( + preview[:40] + "\u2026" if len(preview) > 40 else preview + ) + parts.append(f"({msgs} msgs)") + if model: + parts.append(model) + if when: + parts.append(when) + label = " ".join(parts) + choices.append(questionary.Choice(title=label, value=tid)) + + from prompt_toolkit.layout.dimension import Dimension + from questionary.prompts.common import InquirerControl + + prompt = questionary.select( + "Select session to resume:", + choices=choices, + style=_PICKER_STYLE, + ) + # Limit visible list to 10 rows with scrolling + for window in prompt.application.layout.find_all_windows(): + if isinstance(window.content, InquirerControl): + window.height = Dimension(max=10) + break + selected = prompt.ask() + + if selected is None: + return + arg = selected + + resolved = await _resolve_thread_id(arg) + if not resolved: + return + + meta = await get_thread_metadata(resolved) + ws = (meta or {}).get("workspace_dir", "") or state["workspace_dir"] + + state["thread_id"] = resolved + state["resumed"] = True + if ws: + state["workspace_dir"] = ws + state["status_started_at"] = datetime.now() + state["status_last_input_tokens"] = None + console.print("[dim]Loading session...[/dim]") + state["agent"] = _load_agent( + workspace_dir=state["workspace_dir"], + checkpointer=checkpointer, + config=config, + ) + await _refresh_status_snapshot(reset_streaming_text=True) + # Sync shared refs if channel is running + if _channels_is_running(): + _ch_mod._cli_agent = state["agent"] + _ch_mod._cli_thread_id = state["thread_id"] + console.print(f"[green]Resumed session:[/green] [yellow]{resolved}[/yellow]") + if state["workspace_dir"]: + console.print( + f"[dim]Workspace:[/dim] [cyan]{_shorten_path(state['workspace_dir'])}[/cyan]" + ) + console.print() + await _render_history(resolved) + + async def _cmd_delete(arg: str): + """Handle /delete — delete a saved session.""" + if not arg: + console.print("[red]Usage: /delete [/red]") + return + resolved = await _resolve_thread_id(arg) + if not resolved: + return + if resolved == state["thread_id"]: + console.print("[red]Cannot delete the current session.[/red]") + return + deleted = await delete_thread(resolved) + if deleted: + console.print(f"[green]Deleted session {resolved}.[/green]") + else: + console.print(f"[red]Session {resolved} not found.[/red]") + + async def _async_main_loop(): + """Async main loop with prompt_async and channel queue checking.""" + async with get_checkpointer() as checkpointer: + # Handle --thread-id resume + if thread_id: + resolved = await _resolve_thread_id(thread_id) + if resolved: + meta = await get_thread_metadata(resolved) + ws = (meta or {}).get("workspace_dir", "") or state["workspace_dir"] + state["thread_id"] = resolved + state["resumed"] = True + state["status_started_at"] = datetime.now() + state["status_last_input_tokens"] = None + if ws: + state["workspace_dir"] = ws + + console.print("[dim]Loading agent...[/dim]") + state["agent"] = _load_agent( + workspace_dir=state["workspace_dir"], + checkpointer=checkpointer, + config=config, + ) + await _refresh_status_snapshot(reset_streaming_text=True) + + # Print banner + if state["resumed"]: + print_banner( + state["thread_id"], + state["workspace_dir"], + memory_dir, + mode, + model, + provider, + state["ui_backend"], + ) + console.print( + f"[green]Resumed session [yellow]{state['thread_id']}[/yellow][/green]\n" + ) + else: + print_banner( + state["thread_id"], + state["workspace_dir"], + memory_dir, + mode, + model, + provider, + state["ui_backend"], + ) + + # ---- Channel queue processing (bus → main thread) ---- + + async def _process_channel_message(msg: ChannelMessage) -> None: + """Process a single channel message with real-time streaming. + + Clears the waiting prompt line and reprints the message as if + the user typed it after ❯, then streams the agent response + with Rich Live display. + + Display: + ❯ message content + [channel: Received from sender] + ───────────────── + (real-time streaming output) + [channel: Replied to sender] + ───────────────── + """ + # Clear the waiting ❯ prompt line + sys.stdout.write("\r\033[2K") + sys.stdout.flush() + + # Reprint as if user typed it after ❯ + prompt_line = Text() + prompt_line.append("\u276f ", style="bold blue") + prompt_line.append(msg.content) + console.print(prompt_line) + rx = Text() + rx.append(f"[{msg.channel_type}: Received from ", style="dim") + rx.append(msg.sender, style="cyan") + rx.append("]", style="dim") + console.print(rx) + _print_separator() + + def _send_to_channel(coro, label: str, timeout: int = 15) -> None: + """Schedule an async channel send on the bus loop.""" + loop = _ch_mod._bus_loop + if not loop: + return + try: + asyncio.run_coroutine_threadsafe(coro, loop).result( + timeout=timeout + ) + except Exception as e: + _channel_logger.debug(f"{label} send failed: {e}") + + def _send_thinking_to_channel(thinking: str) -> None: + ch = msg.channel_ref + if ch and ch.send_thinking: + _send_to_channel( + ch.send_thinking_message( + sender=msg.chat_id, + thinking=thinking, + metadata=msg.metadata, + ), + "Thinking", + ) + + def _send_todo_to_channel(items: list[dict]) -> None: + from ..channels.consumer import _format_todo_list + + if msg.channel_ref: + _send_to_channel( + msg.channel_ref.send_todo_message( + sender=msg.chat_id, + content=_format_todo_list(items), + metadata=msg.metadata, + ), + "Todo", + ) + + def _send_media_to_channel(file_path: str) -> None: + if msg.channel_ref: + _send_to_channel( + msg.channel_ref.send_media( + recipient=msg.chat_id, + file_path=file_path, + metadata=msg.metadata, + ), + "Media", + timeout=30, + ) + + def _channel_hitl_prompt(action_requests: list) -> list[dict] | None: + """Send HITL approval prompt to channel user and wait for reply.""" + return _ch_mod.channel_hitl_prompt(action_requests, msg) + + def _channel_ask_user(ask_user_data: dict) -> dict: + """Send ask_user questions to channel user and wait for reply.""" + return _ch_mod.channel_ask_user_prompt(ask_user_data, msg) + + meta = build_metadata(state["workspace_dir"], model) + try: + await _refresh_status_snapshot( + msg.content, reset_streaming_text=True + ) + response = run_streaming( + ui_backend=state["ui_backend"], + agent=state["agent"], + message=msg.content, + thread_id=state["thread_id"], + show_thinking=show_thinking, + interactive=True, + metadata=meta, + on_thinking=_send_thinking_to_channel, + on_todo=_send_todo_to_channel, + on_file_write=_send_media_to_channel, + hitl_prompt_fn=_channel_hitl_prompt, + ask_user_prompt_fn=_channel_ask_user, + on_stream_event=_handle_stream_status_event, + status_footer_builder=_stream_status_footer, + ) + except Exception as e: + response = f"Error: {e}" + console.print(f"[red]Channel error: {e}[/red]") + + _set_channel_response(msg.msg_id, response) + await _refresh_status_snapshot(reset_streaming_text=True) + + tx = Text() + tx.append(f"[{msg.channel_type}: Replied to ", style="dim") + tx.append(msg.sender, style="cyan") + tx.append("]", style="dim") + console.print(tx) + _print_separator() + + # Redraw the ❯ prompt on a new line after separator + sys.stdout.write("\033[34;1m\u276f\033[0m ") + sys.stdout.flush() + + async def _check_channel_queue() -> None: + """Poll the channel message queue and dispatch to the agent.""" + while True: + try: + msg = _message_queue.get_nowait() + except queue.Empty: + await asyncio.sleep(0.1) + continue + await _process_channel_message(msg) + + queue_task = asyncio.create_task(_check_channel_queue()) + + # Startup hint + console.print( + Text( + " EvoScientist is your research buddy.\n" + " Tell it about your taste before cooking some meal!", + style="yellow", + ) + ) + + # Auto-start channel if enabled in config + from ..config import load_config + + _channel_cfg = load_config() + if ( + _channel_cfg + and _channel_cfg.channel_enabled + and not _channels_is_running() + ): + _auto_start_channel( + state["agent"], + state["thread_id"], + _channel_cfg, + send_thinking=channel_send_thinking, + ) + + # Update check — non-blocking, runs in background thread + import concurrent.futures + + _update_executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) + + def _show_update_hint() -> None: + try: + from ..update_check import _installed_version, is_update_available + + available, latest = is_update_available() + if available: + current = _installed_version() + console.print( + Text( + f" Update available: v{latest} (current: v{current}).\n" + " Run: uv tool upgrade EvoScientist", + style="yellow", + ) + ) + except Exception: + pass + + _update_executor.submit(_show_update_hint) + + # Slogan — after channels, right before user input + console.print( + Text(f" {random.choice(WELCOME_SLOGANS)}", style="dim italic") + ) + console.print() + + try: + _print_separator() + while state["running"]: + try: + user_input = await session.prompt_async( + HTML("\u276f "), + bottom_toolbar=_bottom_toolbar, + refresh_interval=1.0, + ) + user_input = user_input.strip() + + if not user_input: + # Erase the empty prompt line so it looks like nothing happened + sys.stdout.write("\033[A\033[2K\r") + sys.stdout.flush() + continue + + _print_separator() + + # Special commands + if user_input.lower() in ("/exit", "/quit", "/q"): + console.print("[dim]Goodbye![/dim]") + state["running"] = False + break + + if user_input.lower() == "/threads": + await _cmd_threads() + continue + + if user_input.lower().startswith("/resume"): + arg = user_input[len("/resume") :].strip() + await _cmd_resume(arg, checkpointer) + continue + + if user_input.lower().startswith("/delete"): + arg = user_input[len("/delete") :].strip() + await _cmd_delete(arg) + continue + + if user_input.lower() == "/new": + # New session: new thread; workspace only changes if not fixed + if not workspace_fixed: + state["workspace_dir"] = _create_session_workspace( + run_name + ) + console.print("[dim]Loading new session...[/dim]") + state["agent"] = _load_agent( + workspace_dir=state["workspace_dir"], + checkpointer=checkpointer, + config=config, + ) + state["thread_id"] = generate_thread_id() + state["resumed"] = False + state["status_started_at"] = datetime.now() + state["status_last_input_tokens"] = None + await _refresh_status_snapshot(reset_streaming_text=True) + # Sync channel refs so the queue checker uses the new agent + if _channels_is_running(): + _ch_mod._cli_agent = state["agent"] + _ch_mod._cli_thread_id = state["thread_id"] + console.print( + f"[green]New session:[/green] [yellow]{state['thread_id']}[/yellow]" + ) + if state["workspace_dir"]: + console.print( + f"[dim]Workspace:[/dim] [cyan]{_shorten_path(state['workspace_dir'])}[/cyan]\n" + ) + continue + + if user_input.lower() == "/current": + console.print( + f"[dim]Thread:[/dim] [yellow]{state['thread_id']}[/yellow]" + ) + if state["workspace_dir"]: + console.print( + f"[dim]Workspace:[/dim] [cyan]{_shorten_path(state['workspace_dir'])}[/cyan]" + ) + if memory_dir: + console.print( + f"[dim]Memory dir:[/dim] [cyan]{_shorten_path(memory_dir)}[/cyan]" + ) + console.print( + f"[dim]UI:[/dim] [cyan]{state['ui_backend']}[/cyan]" + ) + console.print() + continue + + if user_input.lower() == "/skills": + _cmd_list_skills() + continue + + if user_input.lower().startswith("/install-skill"): + source = user_input[len("/install-skill") :].strip() + _cmd_install_skill(source) + continue + + if user_input.lower().startswith("/uninstall-skill"): + name = user_input[len("/uninstall-skill") :].strip() + _cmd_uninstall_skill(name) + continue + + if user_input.lower().startswith("/evoskills"): + browse_args = user_input[len("/evoskills") :].strip() + _cmd_install_skills(browse_args) + continue + + if user_input.lower().startswith("/mcp"): + _cmd_mcp(user_input[len("/mcp") :]) + continue + + if user_input.lower().startswith("/channel"): + args = user_input[len("/channel") :].strip() + if args.lower().startswith("stop"): + stop_arg = args[len("stop") :].strip() + _cmd_channel_stop(stop_arg or None) + else: + _cmd_channel( + args, + state["agent"], + state["thread_id"], + send_thinking=channel_send_thinking, + ) + continue + + if user_input.lower() == "/compact": + from .commands import ( + build_compact_summary_renderable, + compact_conversation, + render_compact_result, + ) + + with console.status( + "[cyan]Compacting conversation...[/cyan]" + ): + result = await compact_conversation( + agent=state["agent"], + thread_id=state["thread_id"], + input_tokens_hint=state.get( + "status_last_input_tokens" + ), + ) + console.print(render_compact_result(result)) + summary_renderable = build_compact_summary_renderable( + result + ) + if summary_renderable is not None: + console.print(summary_renderable) + if result.status == "ok" and result.tokens_after > 0: + state["status_last_input_tokens"] = result.tokens_after + state["status_base_snapshot"] = ( + make_usage_status_snapshot( + result.tokens_after, + model_name=model, + ) + ) + await _refresh_status_snapshot( + reset_streaming_text=True, + ) + continue + + # Resolve @file mentions — inject file contents inline + _, message_to_send, file_warnings = resolve_file_mentions( + user_input, state["workspace_dir"] + ) + + # Stream agent response with metadata for persistence + # Warnings printed here so they appear just before the + # model response, not before the user input echo. + for w in file_warnings: + console.print(f"[yellow]⚠ {escape(w)}[/yellow]") + console.print() + meta = build_metadata(state["workspace_dir"], model) + await _refresh_status_snapshot( + message_to_send, reset_streaming_text=True + ) + run_streaming( + ui_backend=state["ui_backend"], + agent=state["agent"], + message=message_to_send, + thread_id=state["thread_id"], + show_thinking=show_thinking, + interactive=True, + metadata=meta, + on_stream_event=_handle_stream_status_event, + status_footer_builder=_stream_status_footer, + ) + await _refresh_status_snapshot(reset_streaming_text=True) + console.print() + _print_separator() + + except KeyboardInterrupt: + console.print("\n[dim]Goodbye![/dim]") + state["running"] = False + break + except EOFError: + # Handle Ctrl+D + console.print("\n[dim]Goodbye![/dim]") + state["running"] = False + break + except Exception as e: + error_msg = str(e) + if ( + "authentication" in error_msg.lower() + or "api_key" in error_msg.lower() + ): + console.print("[red]Error: API key not configured.[/red]") + console.print( + "[dim]Run [bold]EvoSci onboard[/bold] to set up your API key.[/dim]" + ) + state["running"] = False + break + else: + console.print(f"[red]Error: {escape(str(e))}[/red]") + finally: + queue_task.cancel() + try: + await queue_task + except asyncio.CancelledError: + pass + + # Run the async main loop + try: + asyncio.run(_async_main_loop()) + except KeyboardInterrupt: + console.print("\n[dim]Goodbye![/dim]") + + +def cmd_run( + agent: Any, + prompt: str, + thread_id: str | None = None, + show_thinking: bool = True, + workspace_dir: str | None = None, + model: str | None = None, + ui_backend: str = "cli", +) -> None: + """Single-shot execution with streaming display. + + Args: + agent: Compiled agent graph + prompt: User prompt + thread_id: Optional thread ID (generates new one if None) + show_thinking: Whether to display thinking panels + workspace_dir: Per-session workspace directory path + model: Model name for checkpoint metadata + ui_backend: UI backend ('cli' or 'tui') + """ + thread_id = thread_id or generate_thread_id() + + width = console.size.width + sep = Text("\u2500" * width, style="dim") + console.print(sep) + console.print(Text(f"> {prompt}")) + console.print(sep) + console.print(f"[dim]Thread: {thread_id}[/dim]") + if workspace_dir: + console.print(f"[dim]Workspace: {_shorten_path(workspace_dir)}[/dim]") + console.print() + + meta = build_metadata(workspace_dir, model) + try: + run_streaming( + ui_backend=resolve_ui_backend(ui_backend, warn_fallback=True), + agent=agent, + message=prompt, + thread_id=thread_id, + show_thinking=show_thinking, + interactive=False, + metadata=meta, + ) + except Exception as e: + error_msg = str(e) + if "authentication" in error_msg.lower() or "api_key" in error_msg.lower(): + console.print("[red]Error: API key not configured.[/red]") + console.print( + "[dim]Run [bold]EvoSci onboard[/bold] to set up your API key.[/dim]" + ) + raise typer.Exit(1) from e + else: + console.print(f"[red]Error: {e}[/red]") + raise diff --git a/EvoScientist/cli/mcp_install_cmd.py b/EvoScientist/cli/mcp_install_cmd.py new file mode 100644 index 0000000..3d4e060 --- /dev/null +++ b/EvoScientist/cli/mcp_install_cmd.py @@ -0,0 +1,212 @@ +"""CLI /install-mcp — questionary-based interactive browser. + +Uses the same logic as the TUI command (``commands.implementation.mcp_install``) +but provides a questionary-based UI for the plain CLI interactive mode. +""" + +from __future__ import annotations + +from collections import Counter + +import questionary +from prompt_toolkit.styles import Style as PtStyle +from questionary import Choice + +from ..mcp.registry import ( + MCPServerEntry, + fetch_marketplace_index, + find_server_by_name, + get_all_tags, + get_installed_names, + install_mcp_server, + install_mcp_servers, +) +from ..stream.display import console + +_PICKER_STYLE = PtStyle.from_dict( + { + "questionmark": "#888888", + "question": "", + "pointer": "bold", + "highlighted": "bold", + "text": "#888888", + "answer": "bold", + } +) + +_INSTALLED_INDICATOR = ("fg:#4caf50", "\u2713 ") + + +def _checkbox_ask(choices, message: str, **kwargs): + """questionary.checkbox that renders disabled items with checkmark.""" + from questionary.prompts.common import InquirerControl + + original = InquirerControl._get_choice_tokens + + def _patched(self): + tokens = original(self) + return [ + _INSTALLED_INDICATOR + if cls == "class:disabled" and text == "- " + else (cls, text) + for cls, text in tokens + ] + + InquirerControl._get_choice_tokens = _patched + try: + return questionary.checkbox( + message, + choices=choices, + style=_PICKER_STYLE, + qmark="\u276f", + **kwargs, + ).ask() + finally: + InquirerControl._get_choice_tokens = original + + +def _browse_and_select( + servers: list[MCPServerEntry], + installed_names: set[str], + pre_filter_tag: str = "", +) -> list[MCPServerEntry] | None: + """Questionary-based tag picker + checkbox selection. + + Returns selected entries, empty list if none selected, or None on cancel. + """ + tag_counter: Counter[str] = Counter() + for entry in servers: + for t in entry.tags: + tag_counter[t.lower()] += 1 + + if pre_filter_tag: + pre_filter_tag = pre_filter_tag.lower() + filtered = [e for e in servers if pre_filter_tag in [t.lower() for t in e.tags]] + if not filtered: + console.print( + f"[yellow]No servers found with tag: {pre_filter_tag}[/yellow]" + ) + if tag_counter: + sorted_tags = sorted(tag_counter.items(), key=lambda x: (-x[1], x[0])) + tags_str = ", ".join(f"{tag} ({count})" for tag, count in sorted_tags) + console.print(f"[dim]Available tags: {tags_str}[/dim]") + return None + else: + sorted_tags = sorted(tag_counter.items(), key=lambda x: (-x[1], x[0])) + tag_choices = [Choice(title=f"All servers ({len(servers)})", value="__all__")] + for tag, count in sorted_tags: + tag_choices.append(Choice(title=f"{tag} ({count})", value=tag)) + + selected_tag = questionary.select( + "Filter by tag:", + choices=tag_choices, + style=_PICKER_STYLE, + qmark="\u276f", + ).ask() + + if selected_tag is None: + return None + + if selected_tag == "__all__": + filtered = servers + else: + filtered = [ + e for e in servers if selected_tag in [t.lower() for t in e.tags] + ] + + if all(e.name in installed_names for e in filtered): + console.print( + "[green]All servers in this category are already configured.[/green]" + ) + return None + + choices = [] + for entry in filtered: + if entry.name in installed_names: + choices.append( + Choice( + title=[ + ("", f"{entry.name} \u2014 {entry.description[:80]}"), + ("class:instruction", " (configured)"), + ], + value=entry, + disabled=True, + ) + ) + else: + choices.append( + Choice( + title=f"{entry.name} \u2014 {entry.description[:80]}", + value=entry, + ) + ) + + selected = _checkbox_ask(choices, "Select MCP servers to install:") + if selected is None: + return None + return selected + + +def _cmd_install_mcp(args: str = "") -> None: + """Entry point for ``/install-mcp`` in CLI mode.""" + args = args.strip() + + console.print("[dim]Fetching MCP server index...[/dim]") + try: + servers = fetch_marketplace_index() + except Exception as e: + console.print(f"[red]Failed to fetch server index: {e}[/red]") + console.print() + return + + if not servers: + console.print("[yellow]No MCP servers found.[/yellow]") + console.print() + return + + # Direct name match + if args: + match = find_server_by_name(args, servers) + if match: + installed = get_installed_names() + if match.name in installed: + console.print(f"[yellow]{match.name} is already configured.[/yellow]") + console.print() + return + if install_mcp_server(match): + console.print(f"[green]Configured:[/green] [cyan]{match.name}[/cyan]") + console.print("[dim]Reload with /new to apply.[/dim]") + else: + console.print(f"[red]Failed to configure {match.name}.[/red]") + console.print() + return + + # Tag match — fall through to browser + if args.lower() not in get_all_tags(servers): + console.print(f"[red]No server or tag found matching: {args}[/red]") + close = [s.name for s in servers if args.lower() in s.name.lower()] + if close: + console.print(f"[dim]Did you mean: {', '.join(close)}?[/dim]") + console.print() + return + + # Interactive browse + installed_names = get_installed_names() + selected = _browse_and_select(servers, installed_names, pre_filter_tag=args) + + if not selected: + if selected is None: + console.print("[dim]Cancelled.[/dim]") + else: + console.print("[dim]No servers selected.[/dim]") + console.print() + return + + def _print(text: str, style: str = "") -> None: + console.print(f"[{style}]{text}[/{style}]" if style else text) + + count = install_mcp_servers(selected, print_fn=_print) + if count: + console.print(f"\n[green]{count} server(s) configured.[/green]") + console.print("[dim]Reload with /new to apply.[/dim]") + console.print() diff --git a/EvoScientist/cli/mcp_ui.py b/EvoScientist/cli/mcp_ui.py new file mode 100644 index 0000000..5bd08b9 --- /dev/null +++ b/EvoScientist/cli/mcp_ui.py @@ -0,0 +1,309 @@ +"""MCP server display, operations, and /mcp slash-command dispatcher.""" + +from typing import Any + +from rich.table import Table + +from ..stream.display import console + + +def _mcp_list_servers() -> None: + """Print a table of configured MCP servers.""" + from ..mcp import load_mcp_config + from ..mcp.client import USER_MCP_CONFIG + + config = load_mcp_config() + + if not config: + console.print("[dim]No MCP servers configured.[/dim]") + console.print( + "[dim]Add one with:[/dim] /mcp add [args...]" + ) + console.print() + return + + table = Table(title="MCP Servers", show_header=True) + table.add_column("Server", style="cyan") + table.add_column("Transport", style="green") + table.add_column("Tools", style="yellow") + table.add_column("Expose To", style="magenta") + + for name, server in config.items(): + transport = server.get("transport", "?") + tools = server.get("tools") + tools_str = ", ".join(tools) if tools else "(all)" + expose_to = server.get("expose_to", ["main"]) + if isinstance(expose_to, str): + expose_to = [expose_to] + expose_str = ", ".join(expose_to) + table.add_row(name, transport, tools_str, expose_str) + + console.print(table) + console.print(f"\n[dim]Config file: {USER_MCP_CONFIG}[/dim]") + console.print() + + +def _mcp_add_server_from_kwargs( + kwargs: dict[str, Any], + *, + show_reload_hint: bool = False, +) -> bool: + """Add an MCP server from prepared kwargs.""" + from ..mcp import add_mcp_server + + try: + entry = add_mcp_server(**kwargs) + console.print( + f"[green]Added MCP server:[/green] [cyan]{kwargs['name']}[/cyan] ({entry['transport']})" + ) + if show_reload_hint: + console.print("[dim]Reload with /new to apply.[/dim]") + return True + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + return False + + +def _mcp_edit_server_fields( + name: str, + fields: dict[str, Any], + *, + show_reload_hint: bool = False, +) -> bool: + """Edit an MCP server from prepared field updates.""" + from ..mcp import edit_mcp_server + + if not fields: + console.print( + "[red]No fields to edit. Use --transport, --command, --url, --tools, --expose-to, etc.[/red]" + ) + return False + + try: + edit_mcp_server(name, **fields) + console.print(f"[green]Updated MCP server:[/green] [cyan]{name}[/cyan]") + for k, v in fields.items(): + console.print(f" [dim]{k}:[/dim] {v}") + if show_reload_hint: + console.print("[dim]Reload with /new to apply.[/dim]") + return True + except KeyError as exc: + console.print(f"[red]{exc}[/red]") + return False + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + return False + + +def _mcp_remove_server(name: str, *, show_reload_hint: bool = False) -> bool: + """Remove an MCP server by name.""" + from ..mcp import remove_mcp_server + + clean_name = name.strip() + if not clean_name: + console.print("[red]Usage:[/red] /mcp remove ") + return False + + if remove_mcp_server(clean_name): + console.print(f"[green]Removed MCP server:[/green] [cyan]{clean_name}[/cyan]") + if show_reload_hint: + console.print("[dim]Reload with /new to apply.[/dim]") + return True + + console.print(f"[red]Server not found:[/red] {clean_name}") + return False + + +def _render_mcp_server_config_table(name: str, server: dict[str, Any]) -> None: + """Render one MCP server config table.""" + table = Table( + title=f"MCP Server: {name}", + show_header=True, + title_style="bold cyan", + ) + table.add_column("Setting", style="cyan") + table.add_column("Value") + + table.add_row("transport", str(server.get("transport", "(not set)"))) + if server.get("command"): + table.add_row("command", str(server["command"])) + if server.get("args"): + table.add_row("args", " ".join(str(a) for a in server["args"])) + if server.get("url"): + table.add_row("url", str(server["url"])) + if server.get("headers"): + for k, v in server["headers"].items(): + table.add_row(f"header: {k}", str(v)) + if server.get("env"): + for k, v in server["env"].items(): + table.add_row(f"env: {k}", str(v)) + + tools = server.get("tools") + table.add_row("tools", ", ".join(tools) if tools else "[dim](all)[/dim]") + expose_to = server.get("expose_to", ["main"]) + if isinstance(expose_to, str): + expose_to = [expose_to] + table.add_row("expose_to", ", ".join(expose_to)) + + console.print(table) + console.print() + + +def _show_mcp_config(name: str = "", *, show_blank_line: bool = True) -> str: + """Show MCP config details. + + Returns: + "ok" when rendered, "empty" when no config exists, "missing" when + a specific server name is requested but not found. + """ + from ..mcp import load_mcp_config + from ..mcp.client import USER_MCP_CONFIG + + config = load_mcp_config() + if not config: + console.print("[dim]No MCP servers configured.[/dim]") + if show_blank_line: + console.print() + return "empty" + + name = name.strip() + if name and name not in config: + console.print(f"[red]Server not found:[/red] {name}") + if show_blank_line: + console.print() + return "missing" + + servers = {name: config[name]} if name else config + for srv_name, srv in servers.items(): + _render_mcp_server_config_table(srv_name, srv) + + console.print(f"[dim]Config file: {USER_MCP_CONFIG}[/dim]") + if show_blank_line: + console.print() + return "ok" + + +def _cmd_mcp_add(args_str: str) -> None: + """Handle ``/mcp add ...``.""" + import shlex + + from ..mcp import parse_mcp_add_args + + if not args_str.strip(): + console.print("[bold]Usage:[/bold] /mcp add [args...]") + console.print() + console.print( + "[dim]Transport is auto-detected: URLs \u2192 http, commands \u2192 stdio[/dim]" + ) + console.print() + console.print("[bold]Examples:[/bold]") + console.print( + " /mcp add sequential-thinking npx -y @modelcontextprotocol/server-sequential-thinking" + ) + console.print(" /mcp add docs-langchain https://docs.langchain.com/mcp") + console.print( + " /mcp add my-sse http://localhost:9090/sse --transport sse --expose-to research-agent" + ) + console.print() + console.print("[dim]Options:[/dim]") + console.print(" --transport T Transport type (default: auto-detect)") + console.print( + " --tools t1,t2 Tool allowlist (supports wildcards: *_exa, read_*)" + ) + console.print(" --expose-to a1,a2 Target agents (default: main)") + console.print(" --header Key:Value HTTP header (repeatable)") + console.print(" --env KEY=VALUE Env var for stdio (repeatable)") + console.print( + " --env-ref KEY Env var as runtime ${KEY} reference (repeatable)" + ) + console.print() + return + + try: + tokens = shlex.split(args_str) + kwargs = parse_mcp_add_args(tokens) + _mcp_add_server_from_kwargs(kwargs, show_reload_hint=True) + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + console.print() + + +def _cmd_mcp_edit(args_str: str) -> None: + """Handle ``/mcp edit --field value ...``.""" + import shlex + + from ..mcp import parse_mcp_edit_args + + if not args_str.strip(): + console.print("[bold]Usage:[/bold] /mcp edit -- ...") + console.print() + console.print( + "[dim]Fields:[/dim] --transport, --command, --url, --args, --tools, --expose-to, --header, --env" + ) + console.print( + "[dim]Use[/dim] --tools none [dim]or[/dim] --expose-to none [dim]to clear a field.[/dim]" + ) + console.print() + console.print("[bold]Examples:[/bold]") + console.print(" /mcp edit filesystem --expose-to main,code-agent") + console.print(" /mcp edit filesystem --tools read_file,write_file") + console.print(" /mcp edit my-api --url http://new-host:8080/mcp") + console.print(" /mcp edit my-api --tools none") + console.print() + return + + try: + tokens = shlex.split(args_str) + name, fields = parse_mcp_edit_args(tokens) + _mcp_edit_server_fields(name, fields, show_reload_hint=True) + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + console.print() + + +def _cmd_mcp_remove(name: str) -> None: + """Handle ``/mcp remove ``.""" + _mcp_remove_server(name, show_reload_hint=True) + console.print() + + +def _cmd_mcp_config(name: str) -> None: + """Handle ``/mcp config [name]``.""" + _show_mcp_config(name, show_blank_line=True) + + +def _cmd_mcp(args: str) -> None: + """Dispatch ``/mcp`` subcommands.""" + args = args.strip() + if not args: + _mcp_list_servers() + return + + parts = args.split(maxsplit=1) + subcmd = parts[0].lower() + subargs = parts[1] if len(parts) > 1 else "" + + if subcmd == "list": + _mcp_list_servers() + elif subcmd == "add": + _cmd_mcp_add(subargs) + elif subcmd == "edit": + _cmd_mcp_edit(subargs) + elif subcmd == "remove": + _cmd_mcp_remove(subargs) + elif subcmd == "config": + _cmd_mcp_config(subargs) + elif subcmd == "install": + from .mcp_install_cmd import _cmd_install_mcp + + _cmd_install_mcp(subargs) + else: + console.print("[bold]MCP commands:[/bold]") + console.print(" /mcp List configured servers") + console.print(" /mcp list List configured servers") + console.print(" /mcp config Show detailed server config") + console.print(" /mcp add ... Add a server") + console.print(" /mcp edit ... Edit an existing server") + console.print(" /mcp remove ... Remove a server") + console.print(" /mcp install ... Browse and install servers") + console.print() diff --git a/EvoScientist/cli/skills_cmd.py b/EvoScientist/cli/skills_cmd.py new file mode 100644 index 0000000..c2ddf0b --- /dev/null +++ b/EvoScientist/cli/skills_cmd.py @@ -0,0 +1,335 @@ +"""Slash commands for skill management: /skills, /install-skill, /uninstall-skill, /evoskills.""" + +from pathlib import Path + +from ..stream.display import console +from .agent import _shorten_path + + +def _cmd_list_skills() -> None: + """List all available skills (workspace, global, and built-in).""" + from ..paths import GLOBAL_SKILLS_DIR, USER_SKILLS_DIR + from ..tools.skills_manager import list_skills + + skills = list_skills(include_system=True) + + if not skills: + console.print("[dim]No skills available.[/dim]") + console.print("[dim]Install with:[/dim] /install-skill ") + console.print( + f"[dim]Global skills:[/dim] [cyan]{_shorten_path(str(GLOBAL_SKILLS_DIR))}[/cyan]" + ) + console.print() + return + + workspace_skills = [s for s in skills if s.source == "workspace"] + global_skills = [s for s in skills if s.source == "global"] + builtin_skills = [s for s in skills if s.source == "builtin"] + + sections = [ + ("Workspace Skills", workspace_skills, "green"), + ("Global Skills", global_skills, "cyan"), + ("Built-in Skills", builtin_skills, "blue"), + ] + + printed = False + for title, group, color in sections: + if not group: + continue + if printed: + console.print() + console.print(f"[bold]{title}[/bold] ({len(group)}):") + for skill in group: + tags_str = f" [dim]({', '.join(skill.tags)})[/dim]" if skill.tags else "" + console.print( + f" [{color}]{skill.name}[/{color}] - {skill.description}{tags_str}" + ) + printed = True + + console.print( + f"\n[dim]Global skills:[/dim] [cyan]{_shorten_path(str(GLOBAL_SKILLS_DIR))}[/cyan]" + ) + console.print( + f"[dim]Workspace skills:[/dim] [green]{_shorten_path(str(USER_SKILLS_DIR))}[/green]" + ) + console.print() + + +def _cmd_install_skill(args: str) -> None: + """Install a skill from local path or GitHub URL. + + By default, installs to the global skills directory (~/.config/ai4scientist/skills/). + Append --local to install to the current workspace instead. + + Usage: /install-skill [--local] + """ + from ..paths import GLOBAL_SKILLS_DIR, USER_SKILLS_DIR + from ..tools.skills_manager import install_skill + + # Parse --local flag out of the args string + local = "--local" in args.split() + source = args.replace("--local", "").strip() + + if not source: + console.print("[red]Usage:[/red] /install-skill [--local]") + console.print("[dim]Examples:[/dim]") + console.print(" /install-skill ./my-skill") + console.print( + " /install-skill https://github.com/user/repo/tree/main/skill-name" + ) + console.print(" /install-skill user/repo@skill-name") + console.print( + " /install-skill ./my-skill --local [dim](workspace only)[/dim]" + ) + console.print() + return + + dest_label = ( + f"[cyan]{_shorten_path(str(USER_SKILLS_DIR))}[/cyan] [dim](workspace)[/dim]" + if local + else f"[cyan]{_shorten_path(str(GLOBAL_SKILLS_DIR))}[/cyan] [dim](global)[/dim]" + ) + console.print(f"[dim]Installing skill from:[/dim] {source}") + console.print(f"[dim]Destination:[/dim] {dest_label}") + + result = install_skill(source, global_install=not local) + + if result.get("batch"): + # Batch install — multiple skills + for item in result.get("installed", []): + console.print(f"[green]Installed:[/green] {item['name']}") + console.print( + f" [dim]Description:[/dim] {item.get('description', '(none)')}" + ) + console.print( + f" [dim]Path:[/dim] [cyan]{_shorten_path(item['path'])}[/cyan]" + ) + for item in result.get("failed", []): + console.print(f"[red]Failed:[/red] {item['name']} — {item['error']}") + installed_count = len(result.get("installed", [])) + if installed_count: + console.print(f"\n[green]{installed_count} skill(s) installed.[/green]") + console.print("[dim]Reload with /new to apply.[/dim]") + elif result["success"]: + console.print(f"[green]Installed:[/green] {result['name']}") + console.print(f"[dim]Description:[/dim] {result.get('description', '(none)')}") + console.print(f"[dim]Path:[/dim] [cyan]{_shorten_path(result['path'])}[/cyan]") + console.print() + console.print("[dim]Reload with /new to apply.[/dim]") + else: + console.print(f"[red]Failed:[/red] {result['error']}") + console.print() + + +def _cmd_uninstall_skill(name: str) -> None: + """Uninstall a user-installed skill.""" + from ..tools.skills_manager import uninstall_skill + + if not name: + console.print("[red]Usage:[/red] /uninstall-skill ") + console.print("[dim]Use /skills to see installed skills.[/dim]") + console.print() + return + + result = uninstall_skill(name) + + if result["success"]: + console.print(f"[green]Uninstalled:[/green] {name}") + console.print("[dim]Reload with /new to apply.[/dim]") + else: + console.print(f"[red]Failed:[/red] {result['error']}") + console.print() + + +def _cmd_install_skills(args: str = "") -> None: + """Browse and install skills from the EvoSkills repository. + + Args: + args: Optional tag name to pre-filter (e.g. "core"). + """ + from collections import Counter + + import questionary + from prompt_toolkit.styles import Style as PtStyle + from questionary import Choice + + from ..paths import GLOBAL_SKILLS_DIR, USER_SKILLS_DIR + from ..tools.skills_manager import fetch_remote_skill_index, install_skill + + _PICKER_STYLE = PtStyle.from_dict( + { + "questionmark": "#888888", + "question": "", + "pointer": "bold", + "highlighted": "bold", + "text": "#888888", + "answer": "bold", + } + ) + + # Installed-item indicator style for disabled checkbox choices. + _INSTALLED_INDICATOR = ("fg:#4caf50", "✓ ") + + def _checkbox_ask(choices, message: str, **kwargs): + """questionary.checkbox that renders disabled items with checkmark.""" + from questionary.prompts.common import InquirerControl + + original = InquirerControl._get_choice_tokens + + def _patched(self): + tokens = original(self) + return [ + _INSTALLED_INDICATOR + if cls == "class:disabled" and text == "- " + else (cls, text) + for cls, text in tokens + ] + + InquirerControl._get_choice_tokens = _patched + try: + return questionary.checkbox( + message, + choices=choices, + style=_PICKER_STYLE, + qmark="❯", + **kwargs, + ).ask() + finally: + InquirerControl._get_choice_tokens = original + + # Step 1: Fetch remote index + console.print("[dim]Fetching skill index...[/dim]") + try: + index = fetch_remote_skill_index() + except Exception as e: + console.print(f"[red]Failed to fetch skill index: {e}[/red]") + console.print( + "[dim]Try installing directly: /install-skill EvoScientist/EvoSkills@skills[/dim]" + ) + console.print() + return + + if not index: + console.print("[yellow]No skills found in the repository.[/yellow]") + console.print() + return + + # Detect already-installed skills (both global and workspace tiers) + installed_names: set[str] = set() + for skills_dir in (Path(GLOBAL_SKILLS_DIR), Path(USER_SKILLS_DIR)): + if skills_dir.exists(): + installed_names.update(e.name for e in skills_dir.iterdir() if e.is_dir()) + + pre_filter_tag = args.strip().lower() if args else "" + + # Step 2: Tag filter (skip if pre-filtered via args) + if pre_filter_tag: + filtered = [ + s for s in index if pre_filter_tag in [t.lower() for t in s.get("tags", [])] + ] + if not filtered: + console.print(f"[yellow]No skills found with tag: {args.strip()}[/yellow]") + # Show available tags + tag_counter: Counter[str] = Counter() + for s in index: + for t in s.get("tags", []): + tag_counter[t.lower()] += 1 + if tag_counter: + sorted_tags = sorted(tag_counter.items(), key=lambda x: (-x[1], x[0])) + tags_str = ", ".join(f"{tag} ({count})" for tag, count in sorted_tags) + console.print(f"[dim]Available tags: {tags_str}[/dim]") + console.print() + return + else: + # Build tag choices for interactive picker + tag_counter = Counter() + for s in index: + for t in s.get("tags", []): + tag_counter[t.lower()] += 1 + + sorted_tags = sorted(tag_counter.items(), key=lambda x: (-x[1], x[0])) + tag_choices = [Choice(title=f"All skills ({len(index)})", value="__all__")] + for tag, count in sorted_tags: + tag_choices.append(Choice(title=f"{tag} ({count})", value=tag)) + + selected_tag = questionary.select( + "Filter by tag:", + choices=tag_choices, + style=_PICKER_STYLE, + qmark="❯", + ).ask() + + if selected_tag is None: + console.print() + return + + if selected_tag == "__all__": + filtered = index + else: + filtered = [ + s + for s in index + if selected_tag in [t.lower() for t in s.get("tags", [])] + ] + + # Step 3: Skill selection checkbox + all_installed = all(s["name"] in installed_names for s in filtered) + if all_installed: + console.print( + "[green]All skills in this category are already installed.[/green]" + ) + console.print() + return + + choices = [] + for s in filtered: + if s["name"] in installed_names: + choices.append( + Choice( + title=[ + ("", f"{s['name']} — {s['description'][:80]}"), + ("class:instruction", " (installed)"), + ], + value=s["install_source"], + disabled=True, + ) + ) + else: + choices.append( + Choice( + title=f"{s['name']} — {s['description'][:80]}", + value=s["install_source"], + ) + ) + + selected = _checkbox_ask(choices, "Select skills to install:") + + if selected is None: + console.print() + return + + if not selected: + console.print("[dim]No skills selected.[/dim]") + console.print() + return + + # Step 4: Install selected skills (default: global) + installed_count = 0 + for source in selected: + result = install_skill(source, global_install=True) + if result.get("batch"): + for item in result.get("installed", []): + console.print(f"[green]Installed:[/green] {item['name']}") + installed_count += 1 + for item in result.get("failed", []): + console.print(f"[red]Failed:[/red] {item['name']} — {item['error']}") + elif result.get("success"): + console.print(f"[green]Installed:[/green] {result['name']}") + installed_count += 1 + else: + console.print(f"[red]Failed:[/red] {result.get('error', 'unknown')}") + + if installed_count: + console.print(f"\n[green]{installed_count} skill(s) installed.[/green]") + console.print("[dim]Reload with /new to apply.[/dim]") + console.print() diff --git a/EvoScientist/cli/status_bar.py b/EvoScientist/cli/status_bar.py new file mode 100644 index 0000000..04696e2 --- /dev/null +++ b/EvoScientist/cli/status_bar.py @@ -0,0 +1,372 @@ +"""Shared session status bar helpers for CLI and TUI frontends.""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from datetime import datetime +from typing import Any + +from langchain_core.messages import AIMessage, HumanMessage +from langchain_core.messages.utils import count_tokens_approximately + +from ..llm.context_window import ( + DEFAULT_CONTEXT_WINDOW_FALLBACK, + resolve_context_window, +) +from ..sessions import get_thread_messages + +_FALLBACK_CONTEXT_WINDOW = DEFAULT_CONTEXT_WINDOW_FALLBACK +STATUS_BAR_BG = "#171a20" +STATUS_TEXT = "#cbd5e1" +STATUS_STRONG = "#e5e7eb" +STATUS_DIM = "#7c8594" +STATUS_GOOD = "#5fcf8b" +STATUS_WARN = "#d7b45a" +STATUS_BAD = "#d08c61" +STATUS_CRITICAL = "#d86f6f" +STATUS_HINT_IDLE = "#8b9bb0" +STATUS_HINT_BUSY = "#f0c36a" + + +@dataclass(slots=True) +class SessionStatusSnapshot: + """Current session metrics shown in the persistent status bar.""" + + model_full: str + model_short: str + context_tokens: int + context_window: int + context_percent: int + context_source: str = "estimated" + + +def _percent_from_context(context_tokens: int, context_window: int) -> int: + """Convert token counts into a clamped percent value.""" + if context_window <= 0: + return 0 + return max(0, min(100, round((context_tokens / context_window) * 100))) + + +def _get_default_chat_model() -> Any: + """Resolve the default chat model lazily to avoid import cycles.""" + from ..EvoScientist import _ensure_chat_model + + return _ensure_chat_model() + + +def _resolve_model_name(model_name: str | None, model_obj: Any | None) -> str: + """Best-effort model name resolution for display.""" + if model_name: + return str(model_name) + if model_obj is None: + model_obj = _get_default_chat_model() + for attr in ("model_name", "model", "name"): + value = getattr(model_obj, attr, None) + if value: + return str(value) + return "unknown" + + +def _resolve_context_window(model_obj: Any | None) -> int: + """Resolve the model context window with a safe fallback.""" + if model_obj is None: + model_obj = _get_default_chat_model() + return resolve_context_window(model_obj, fallback=_FALLBACK_CONTEXT_WINDOW) + + +def shorten_model_name(model_name: str, max_len: int = 26) -> str: + """Shorten provider-prefixed model names for compact display.""" + short = (model_name or "unknown").split("/")[-1] + if len(short) > max_len: + return f"{short[: max_len - 3]}..." + return short + + +def format_token_count_compact(value: int) -> str: + """Format large token counts into a compact human-readable form.""" + abs_value = abs(int(value)) + if abs_value >= 1_000_000: + num = value / 1_000_000 + suffix = "M" + elif abs_value >= 1_000: + num = value / 1_000 + suffix = "K" + else: + return str(value) + + if num.is_integer(): + return f"{int(num)}{suffix}" + return f"{num:.1f}{suffix}" + + +def format_duration_compact(started_at: datetime, now: datetime | None = None) -> str: + """Format elapsed wall time into a compact duration string.""" + current = now or datetime.now() + seconds = max(0, int((current - started_at).total_seconds())) + if seconds < 60: + return f"{seconds}s" + minutes = seconds // 60 + if minutes < 60: + return f"{minutes}m" + hours = minutes // 60 + if hours < 24: + return f"{hours}h" + days = hours // 24 + return f"{days}d" + + +def status_style_name(percent_used: int) -> str: + """Map utilization percent to shared status bar style buckets.""" + if percent_used >= 95: + return "critical" + if percent_used > 80: + return "bad" + if percent_used >= 50: + return "warn" + return "good" + + +def build_context_bar(percent_used: int, width: int = 10) -> str: + """Build a compact visual context progress bar.""" + safe_percent = max(0, min(100, int(percent_used))) + filled = round((safe_percent / 100) * width) + body = ("█" * filled) + ("░" * max(0, width - filled)) + return f"[{body}]" + + +def _display_width(text: str) -> int: + try: + from prompt_toolkit.utils import get_cwidth + + return get_cwidth(text or "") + except Exception: + return len(text or "") + + +def trim_status_text(text: str, max_width: int) -> str: + """Trim status-bar content to fit a single terminal row.""" + if max_width <= 0: + return "" + if _display_width(text) <= max_width: + return text + + ellipsis = "..." + ellipsis_width = _display_width(ellipsis) + if max_width <= ellipsis_width: + return ellipsis[:max_width] + + try: + from prompt_toolkit.utils import get_cwidth + except Exception: + get_cwidth = None + + out: list[str] = [] + width = 0 + for ch in text: + ch_width = get_cwidth(ch) if get_cwidth else len(ch) + if width + ch_width + ellipsis_width > max_width: + break + out.append(ch) + width += ch_width + return "".join(out).rstrip() + ellipsis + + +def build_status_fragments( + snapshot: SessionStatusSnapshot, + started_at: datetime, + width: int, +) -> list[tuple[str, str]]: + """Build prompt_toolkit formatted-text fragments for the status bar.""" + duration_label = format_duration_compact(started_at) + percent = snapshot.context_percent + percent_label = f"{percent}%" + if width < 52: + frags = [ + ("class:status-bar-strong", snapshot.model_short), + ("class:status-bar-dim", " · "), + ("class:status-bar-dim", duration_label), + ("class:status-bar", " "), + ] + elif width < 76: + frags = [ + ("class:status-bar-strong", snapshot.model_short), + ("class:status-bar-dim", " · "), + (f"class:status-bar-{status_style_name(percent)}", percent_label), + ("class:status-bar-dim", " · "), + ("class:status-bar-dim", duration_label), + ("class:status-bar", " "), + ] + else: + context_label = ( + f"{format_token_count_compact(snapshot.context_tokens)}/" + f"{format_token_count_compact(snapshot.context_window)}" + ) + bucket = status_style_name(percent) + frags = [ + ("class:status-bar-strong", snapshot.model_short), + ("class:status-bar-dim", " │ "), + ("class:status-bar-dim", context_label), + ("class:status-bar-dim", " │ "), + (f"class:status-bar-{bucket}", build_context_bar(percent)), + ("class:status-bar-dim", " "), + (f"class:status-bar-{bucket}", percent_label), + ("class:status-bar-dim", " │ "), + ("class:status-bar-dim", duration_label), + ("class:status-bar", " "), + ] + + total_width = sum(_display_width(text) for _, text in frags) + if total_width > width: + plain_text = "".join(text for _, text in frags) + return [("class:status-bar", trim_status_text(plain_text, width))] + return frags + + +def build_status_text( + snapshot: SessionStatusSnapshot, + started_at: datetime, + width: int, +): + """Build a Rich Text object for the persistent TUI status bar.""" + from rich.text import Text + + rich_styles = { + "status-bar": f"on {STATUS_BAR_BG} {STATUS_TEXT}", + "status-bar-strong": f"on {STATUS_BAR_BG} {STATUS_STRONG} bold", + "status-bar-dim": f"on {STATUS_BAR_BG} {STATUS_DIM}", + "status-bar-good": f"on {STATUS_BAR_BG} {STATUS_GOOD} bold", + "status-bar-warn": f"on {STATUS_BAR_BG} {STATUS_WARN} bold", + "status-bar-bad": f"on {STATUS_BAR_BG} {STATUS_BAD} bold", + "status-bar-critical": f"on {STATUS_BAR_BG} {STATUS_CRITICAL} bold", + } + text = Text(no_wrap=True, overflow="crop") + for style, content in build_status_fragments(snapshot, started_at, width): + rich_style = rich_styles.get( + style.removeprefix("class:"), + f"on {STATUS_BAR_BG} {STATUS_TEXT}", + ) + text.append(content, style=rich_style) + return text + + +def make_empty_status_snapshot( + model_name: str | None = None, model_obj: Any | None = None +) -> SessionStatusSnapshot: + """Build a placeholder snapshot before async context counting completes.""" + resolved_name = _resolve_model_name(model_name, model_obj) + window = _resolve_context_window(model_obj) + return SessionStatusSnapshot( + model_full=resolved_name, + model_short=shorten_model_name(resolved_name), + context_tokens=0, + context_window=window, + context_percent=0, + context_source="estimated", + ) + + +def make_usage_status_snapshot( + input_tokens: int, + *, + model_name: str | None = None, + model_obj: Any | None = None, +) -> SessionStatusSnapshot: + """Build a snapshot from the last real model input usage.""" + resolved_name = _resolve_model_name(model_name, model_obj) + window = _resolve_context_window(model_obj) + context_tokens = max(0, int(input_tokens)) + return SessionStatusSnapshot( + model_full=resolved_name, + model_short=shorten_model_name(resolved_name), + context_tokens=context_tokens, + context_window=window, + context_percent=_percent_from_context(context_tokens, window), + context_source="usage", + ) + + +def estimate_message_tokens( + text: str, + *, + message_type: str = "ai", +) -> int: + """Estimate tokens for a single in-flight message fragment.""" + content = (text or "").strip() + if not content: + return 0 + + try: + if message_type == "human": + messages = [HumanMessage(content=content)] + else: + messages = [AIMessage(content=content)] + return int(count_tokens_approximately(messages)) + except Exception: + return 0 + + +def apply_assistant_text_to_snapshot( + snapshot: SessionStatusSnapshot, + assistant_text: str | None, +) -> SessionStatusSnapshot: + """Overlay in-flight assistant output on top of a base snapshot.""" + extra_tokens = estimate_message_tokens(assistant_text or "", message_type="ai") + if extra_tokens <= 0: + return snapshot + + context_tokens = snapshot.context_tokens + extra_tokens + return replace( + snapshot, + context_tokens=context_tokens, + context_percent=_percent_from_context(context_tokens, snapshot.context_window), + ) + + +def apply_user_text_to_snapshot( + snapshot: SessionStatusSnapshot, + user_text: str | None, +) -> SessionStatusSnapshot: + """Overlay pending user input on top of an existing snapshot.""" + extra_tokens = estimate_message_tokens(user_text or "", message_type="human") + if extra_tokens <= 0: + return snapshot + + context_tokens = snapshot.context_tokens + extra_tokens + return replace( + snapshot, + context_tokens=context_tokens, + context_percent=_percent_from_context(context_tokens, snapshot.context_window), + ) + + +async def build_session_status_snapshot( + thread_id: str, + *, + model_name: str | None = None, + model_obj: Any | None = None, + pending_user_text: str | None = None, +) -> SessionStatusSnapshot: + """Count current thread context and return a display snapshot.""" + resolved_name = _resolve_model_name(model_name, model_obj) + window = _resolve_context_window(model_obj) + messages = list(await get_thread_messages(thread_id)) + + pending = (pending_user_text or "").strip() + if pending: + messages.append(HumanMessage(content=pending)) + + try: + context_tokens = int(count_tokens_approximately(messages)) if messages else 0 + except Exception: + context_tokens = 0 + + percent = _percent_from_context(context_tokens, window) + + return SessionStatusSnapshot( + model_full=resolved_name, + model_short=shorten_model_name(resolved_name), + context_tokens=context_tokens, + context_window=window, + context_percent=percent, + context_source="estimated", + ) diff --git a/EvoScientist/cli/tui_backends.py b/EvoScientist/cli/tui_backends.py new file mode 100644 index 0000000..9545c8b --- /dev/null +++ b/EvoScientist/cli/tui_backends.py @@ -0,0 +1,74 @@ +"""TUI backend abstractions for streaming output.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, Protocol + +from ..stream.display import _run_streaming + + +class StreamingTUIBackend(Protocol): + """Protocol for TUI backends that can render agent streaming output.""" + + name: str + + def run_streaming( + self, + *, + agent: Any, + message: str, + thread_id: str, + show_thinking: bool, + interactive: bool, + on_thinking: Callable[[str], None] | None = None, + on_todo: Callable[[list[dict]], None] | None = None, + on_file_write: Callable[[str], None] | None = None, + on_stream_event: Callable[[str, Any], Any] | None = None, + status_footer_builder: Callable[[], Any] | None = None, + metadata: dict | None = None, + hitl_prompt_fn: Callable[[list], list[dict] | None] | None = None, + ask_user_prompt_fn: Callable[[dict], dict] | None = None, + ) -> str: + """Run streaming and return final response text.""" + + +@dataclass(slots=True) +class RichStreamingBackend: + """Default Rich backend wrapper around the existing streaming renderer.""" + + name: str = "cli" + + def run_streaming( + self, + *, + agent: Any, + message: str, + thread_id: str, + show_thinking: bool, + interactive: bool, + on_thinking: Callable[[str], None] | None = None, + on_todo: Callable[[list[dict]], None] | None = None, + on_file_write: Callable[[str], None] | None = None, + on_stream_event: Callable[[str, Any], Any] | None = None, + status_footer_builder: Callable[[], Any] | None = None, + metadata: dict | None = None, + hitl_prompt_fn: Callable[[list], list[dict] | None] | None = None, + ask_user_prompt_fn: Callable[[dict], dict] | None = None, + ) -> str: + return _run_streaming( + agent=agent, + message=message, + thread_id=thread_id, + show_thinking=show_thinking, + interactive=interactive, + on_thinking=on_thinking, + on_todo=on_todo, + on_file_write=on_file_write, + on_stream_event=on_stream_event, + status_footer_builder=status_footer_builder, + metadata=metadata, + hitl_prompt_fn=hitl_prompt_fn, + ask_user_prompt_fn=ask_user_prompt_fn, + ) diff --git a/EvoScientist/cli/tui_interactive.py b/EvoScientist/cli/tui_interactive.py new file mode 100644 index 0000000..582a94c --- /dev/null +++ b/EvoScientist/cli/tui_interactive.py @@ -0,0 +1,2463 @@ +"""Full-screen Textual interactive TUI for EvoScientist. + +Widget-based rendering: each message/tool/sub-agent is an independent widget +mounted into a VerticalScroll container. No timer-based Group rebuilds. +""" + +from __future__ import annotations + +import asyncio +import logging +import queue +import random +import sys +from collections.abc import Callable +from datetime import datetime +from typing import Any, ClassVar + +from rich.console import Group +from rich.text import Text + +import EvoScientist.cli.channel as _ch_mod + +from ..commands import CommandContext +from ..commands import manager as cmd_manager +from ..config.settings import get_config_dir +from ..sessions import ( + find_similar_threads, + generate_thread_id, + get_checkpointer, + get_thread_messages, + get_thread_metadata, + thread_exists, +) +from ..stream.events import stream_agent_events +from ..stream.state import _INTERNAL_TOOLS, StreamState +from ._constants import LOGO_GRADIENT, LOGO_LINES, WELCOME_SLOGANS, build_metadata +from .channel import ( + ChannelMessage, + _auto_start_channel, + _channels_is_running, + _channels_running_list, + _channels_stop, + _message_queue, + _set_channel_response, +) +from .file_mentions import complete_file_mention, resolve_file_mentions +from .history_suggester import HistorySuggester +from .status_bar import ( + STATUS_BAR_BG, + STATUS_DIM, + STATUS_HINT_BUSY, + STATUS_HINT_IDLE, + apply_assistant_text_to_snapshot, + apply_user_text_to_snapshot, + build_session_status_snapshot, + build_status_text, + make_empty_status_snapshot, + make_usage_status_snapshot, +) + +_channel_logger = logging.getLogger(__name__) + + +def _shorten_path(path: str) -> str: + """Shorten absolute path to a cwd-relative form (consistent with Rich CLI).""" + if not path: + return path + from .agent import _shorten_path as _sp + + return _sp(path) + + +def _build_welcome_banner( + *, + thread_id: str, + workspace_dir: str | None, + mode: str | None, + model: str | None, + provider: str | None, + ui_backend: str | None = None, + channels: list[tuple[str, bool, str]] | None = None, +) -> Any: + """Build CLI-matching welcome banner with logo, info line, and channels. + + Args: + channels: List of (name, ok, detail) tuples for the channels panel. + """ + banner = Text() + for line, color in zip(LOGO_LINES, LOGO_GRADIENT, strict=False): + banner.append(f"{line}\n", style=f"bold {color}") + + # Info line — matches CLI print_banner format + info = Text() + parts: list[tuple[str, str]] = [] + if model: + parts.append(("Model: ", model)) + if provider: + parts.append(("Provider: ", provider)) + if mode: + parts.append(("Mode: ", mode)) + if ui_backend: + parts.append(("UI: ", ui_backend)) + if parts: + info.append(" ", style="dim") + for i, (label, value) in enumerate(parts): + if i > 0: + info.append(" ", style="dim") + info.append(label, style="dim") + info.append(value, style="magenta") + # Directory line + import os + + effective_dir = workspace_dir or os.getcwd() + home = os.path.expanduser("~") + dir_display = ( + effective_dir.replace(home, "~", 1) + if effective_dir.startswith(home) + else effective_dir + ) + info.append("\n ", style="dim") + info.append("Directory: ", style="dim") + info.append(dir_display, style="magenta") + _nl_key = "Option+Enter" if sys.platform == "darwin" else "Ctrl+J" + info.append("\n Enter ", style="#ffe082") + info.append("send", style="#ffe082 bold") + info.append(f" \u2022 {_nl_key} ", style="#ffe082") + info.append("newline", style="#ffe082 bold") + info.append(" \u2022 Type ", style="#ffe082") + info.append("/", style="#ffe082 bold") + info.append(" for commands", style="#ffe082") + info.append(" \u2022 ", style="#ffe082") + info.append("@ files", style="#ffe082 bold") + info.append(" \u2022 Ctrl+C ", style="#ffe082") + info.append("interrupt", style="#ffe082 bold") + banner.append_text(info) + + slogan = Text(f"\n {random.choice(WELCOME_SLOGANS)}", style="dim italic") + + # Channels panel + if channels: + from rich.panel import Panel + + lines: list[Text] = [] + all_ok = True + for name, ok, detail in channels: + line = Text() + if ok: + line.append("\u25cf ", style="green") + line.append(name, style="bold") + else: + line.append("\u25cb ", style="dim") + line.append(name, style="bold dim") + all_ok = False + if detail: + line.append(f" {detail}", style="dim") + lines.append(line) + body = Text("\n").join(lines) + border = "green" if all_ok else "dim" + panel = Panel( + body, title="[bold]Channels[/bold]", border_style=border, expand=False + ) + return Group(banner, Text(""), panel, slogan) + + # No channels — append slogan directly to banner + banner.append_text(slogan) + return banner + + +def _is_final_response(state: StreamState) -> bool: + """Check if all tools are done and no sub-agents are active.""" + n_visible = 0 + n_done = 0 + for i, tc in enumerate(state.tool_calls): + if tc.get("name") in _INTERNAL_TOOLS: + continue + n_visible += 1 + if i < len(state.tool_results): + n_done += 1 + has_pending = n_visible > n_done + any_active_sa = any(sa.is_active for sa in state.subagents) + return not has_pending and not any_active_sa and not state.is_processing + + +_SUMMARY_CONTINUATION_EVENTS = { + "summarization_start", + "summarization", + "usage_stats", +} + + +def _should_finalize_active_summarization(event_type: str) -> bool: + """Return whether an active summary panel should stop for this event.""" + return bool(event_type) and event_type not in _SUMMARY_CONTINUATION_EVENTS + + +def run_textual_interactive( + *, + show_thinking: bool, + channel_send_thinking: bool = True, + workspace_dir: str | None, + workspace_fixed: bool, + mode: str | None, + model: str | None, + provider: str | None, + run_name: str | None, + thread_id: str | None, + load_agent: Callable[..., Any], + create_session_workspace: Callable[[str | None], str], +) -> None: + """Run full-screen Textual interactive chat loop.""" + try: + from textual.app import App, ComposeResult + from textual.binding import Binding + from textual.containers import Container, Horizontal, VerticalScroll + from textual.events import MouseUp + from textual.widgets import Static + + from .clipboard import copy_selection_to_clipboard, get_clipboard_text + from .widgets import ( + AssistantMessage, + CompactingWidget, + LoadingWidget, + SubAgentWidget, + SummarizationWidget, + SystemMessage, + ThinkingWidget, + TodoWidget, + ToolCallWidget, + UsageWidget, + UserMessage, + ) + from .widgets.chat_input import ChatTextArea + except Exception as e: # pragma: no cover - runtime fallback path + raise RuntimeError( + "Textual TUI backend requires 'textual'. Run: pip install textual" + ) from e + + class EvoTextualInteractiveApp(App[None]): # type: ignore[type-arg] + """Deep-Agents-style full-screen TUI with independent widget rendering.""" + + @property + def supports_interactive(self) -> bool: + return True + + CSS = """ + Screen { + layout: vertical; + background: #16161a; + color: #d1d5db; + } + #chat { + height: 1fr; + padding: 1 2; + background: #16161a; + } + #welcome { + height: auto; + margin-bottom: 1; + } + #input-shell { + height: auto; + padding: 0 2 0 2; + background: #16161a; + } + #input-row { + height: auto; + min-height: 3; + max-height: 10; + border: solid #0284c7; + background: #1e1f26; + padding: 0 1; + } + #input-cursor { + width: 2; + content-align: center middle; + color: #0284c7; + text-style: bold; + } + #prompt { + width: 1fr; + min-height: 1; + max-height: 8; + border: none; + background: transparent; + color: #e5e7eb; + } + #prompt:focus { + border: none; + } + #queued-message { + display: none; + height: auto; + background: #1e1f26; + padding: 0 2; + color: #9ca3af; + } + #completions { + display: none; + height: auto; + max-height: 15; + background: #1e1f26; + padding: 0 1; + border-bottom: solid #0284c7; + } + #status { + height: 1; + min-height: 1; + background: #171a20; + color: #cbd5e1; + padding: 0 1; + } + """ + BINDINGS: ClassVar[list[Binding]] = [ + Binding("ctrl+c", "request_quit", "Quit", show=False, priority=True), + Binding("ctrl+v", "paste_clipboard", "Paste", show=False), + Binding("tab", "tab_complete", show=False, priority=True), + Binding("up", "edit_queued", show=False, priority=True), + Binding("down", "down_delegate", show=False, priority=True), + Binding("escape", "cancel_queued", show=False, priority=True), + ] + + def __init__( + self, + *, + agent: Any, + thread_id_value: str, + workspace: str | None, + checkpointer: Any, + channel_send_thinking_value: bool = True, + resumed: bool = False, + resume_warning: str = "", + ) -> None: + super().__init__() + self._agent = agent + self._conversation_tid = thread_id_value + self._workspace_dir = workspace + self._checkpointer = checkpointer + self._channel_send_thinking = channel_send_thinking_value + self._resumed = resumed + self._resume_warning = resume_warning + self._channel_timer: Any = None + self._started_channel_types: list[str] = [] + self._busy = False + self._run_task: Any = None # asyncio.Task for current _run_turn + self._queued_messages: list[ + str + ] = [] # queued messages to send after current turn + self._comp_items: list[tuple[str, str]] = [] + self._comp_index: int = -1 + self._hitl_auto_approve: bool = False + self._approval_future: asyncio.Future | None = None + self._ask_user_future: asyncio.Future | None = None + self._picker_future: asyncio.Future | None = None + self._browser_future: asyncio.Future | None = None + self._mcp_browser_future: asyncio.Future | None = None + self._history_suggester = HistorySuggester(get_config_dir() / "history") + self._history_index: int = -1 # -1 = not browsing history + self._history_saved_input: str = "" # saved current input before browsing + self._background_tasks: set[asyncio.Task] = set() + self._quit_pending: bool = False + self._status_started_at = datetime.now() + self._status_base_snapshot = make_empty_status_snapshot(model) + self._status_snapshot = self._status_base_snapshot + self._status_streaming_text = "" + self._status_last_input_tokens: int | None = None + self._compacting_widget: CompactingWidget | None = None + + # ── CommandUI implementation ───────────────────────── + + def append_system(self, text: str, style: str = "dim") -> None: + self._append_system(text, style) + + def mount_renderable(self, renderable: Any) -> None: + self._mount_renderable(renderable) + + async def start_compacting_indicator(self) -> None: + await self._start_compacting_indicator() + + async def stop_compacting_indicator(self) -> None: + await self._stop_compacting_indicator() + + async def wait_for_thread_pick( + self, threads: list[dict], current_thread: str, title: str + ) -> str | None: + from .widgets.thread_selector import ThreadPickerWidget + + container = self.query_one("#chat", VerticalScroll) + picker = ThreadPickerWidget( + threads, + current_thread=current_thread, + title=title, + ) + await container.mount(picker) + container.scroll_end(animate=False) + picker.focus() + + return await self._wait_for_thread_pick(picker) + + async def wait_for_skill_browse( + self, index: list[dict], installed_names: set[str], pre_filter_tag: str + ) -> list[str] | None: + from .widgets.skill_browser import SkillBrowserWidget + + container = self.query_one("#chat", VerticalScroll) + browser = SkillBrowserWidget( + index, + installed_names, + pre_filter_tag=pre_filter_tag, + ) + await container.mount(browser) + container.scroll_end(animate=False) + browser.focus() + + return await self._wait_for_skill_browse(browser) + + async def wait_for_mcp_browse( + self, servers: list, installed_names: set[str], pre_filter_tag: str + ) -> list | None: + from .widgets.mcp_browser import MCPBrowserWidget + + container = self.query_one("#chat", VerticalScroll) + browser = MCPBrowserWidget( + servers, + installed_names, + pre_filter_tag=pre_filter_tag, + ) + await container.mount(browser) + container.scroll_end(animate=False) + browser.focus() + + return await self._wait_for_mcp_browse(browser) + + def clear_chat(self) -> None: + container = self.query_one("#chat", VerticalScroll) + welcome = self.query_one("#welcome", Static) + for child in list(container.children): + if child is not welcome: + child.remove() + + def request_quit(self) -> None: + self.action_request_quit() + + def start_new_session(self) -> None: + # Clear all widgets except #welcome + self.clear_chat() + + if not workspace_fixed: + self._workspace_dir = create_session_workspace(run_name) + self._conversation_tid = generate_thread_id() + self._agent = load_agent( + workspace_dir=self._workspace_dir, + checkpointer=self._checkpointer, + ) + self._status_started_at = datetime.now() + self._status_base_snapshot = make_empty_status_snapshot(model) + self._status_snapshot = self._status_base_snapshot + self._status_streaming_text = "" + self._status_last_input_tokens = None + if _channels_is_running(): + _ch_mod._cli_agent = self._agent + _ch_mod._cli_thread_id = self._conversation_tid + self._render_welcome() + self._render_status() + refresh_task = asyncio.create_task(self._refresh_status_snapshot()) + self._background_tasks.add(refresh_task) + refresh_task.add_done_callback(self._background_tasks.discard) + self.append_system(f"New session: {self._conversation_tid}", style="green") + + async def handle_session_resume( + self, thread_id: str, workspace_dir: str | None = None + ) -> None: + if workspace_dir: + self._workspace_dir = workspace_dir + + self._conversation_tid = thread_id + self._agent = load_agent( + workspace_dir=self._workspace_dir, + checkpointer=self._checkpointer, + ) + self._status_started_at = datetime.now() + self._status_base_snapshot = make_empty_status_snapshot(model) + self._status_snapshot = self._status_base_snapshot + self._status_streaming_text = "" + self._status_last_input_tokens = None + if _channels_is_running(): + _ch_mod._cli_agent = self._agent + _ch_mod._cli_thread_id = self._conversation_tid + self._render_welcome() + await self._refresh_status_snapshot() + self._render_status() + self.append_system(f"Resumed session: {thread_id}", style="green") + await self._render_history(thread_id) + + async def flush(self) -> None: + """No-op for TUI, messages are already delivered incrementally.""" + pass + + # ── Layout ───────────────────────────────────────────── + + def compose(self) -> ComposeResult: + with VerticalScroll(id="chat"): + yield Static("", id="welcome") + # Widgets are mounted directly here by _stream_with_widgets, + # _append_system, _mount_renderable, etc. + + with Container(id="input-shell"): + yield Static("", id="queued-message") + yield Static("", id="completions") + with Horizontal(id="input-row"): + yield Static(">", id="input-cursor") + yield ChatTextArea( + placeholder="Type message (/ for commands)", + id="prompt", + ) + + yield Static("", id="status") + + def on_mount(self) -> None: + self._render_welcome() + self._render_status() + self.set_interval(1.0, self._render_status) + refresh_task = asyncio.create_task(self._refresh_status_snapshot()) + self._background_tasks.add(refresh_task) + refresh_task.add_done_callback(self._background_tasks.discard) + prompt = self.query_one("#prompt", ChatTextArea) + prompt.before_submit = self._handle_completion_enter + prompt.focus() + # Show resume status + if self._resume_warning: + self._append_system(self._resume_warning, style="yellow") + elif self._resumed: + self._append_system( + f"Resumed session: {self._conversation_tid}", + style="green", + ) + self.call_later( + lambda: asyncio.ensure_future( + self._render_history(self._conversation_tid) + ) + ) + # Startup notifications + self.notify( + "EvoScientist is your research buddy.\n" + "Tell it about your taste before cooking some meal!", + severity="warning", + timeout=10, + ) + self.run_worker( + self._check_for_updates, exclusive=True, group="update-check" + ) + # Auto-start channels + self._start_channels() + + # ── Update check ────────────────────────────────────── + + async def _check_for_updates(self) -> None: + """Check PyPI for a newer EvoScientist version and notify.""" + try: + from ..update_check import _installed_version, is_update_available + + available, latest = await asyncio.to_thread(is_update_available) + if available: + current = _installed_version() + self.notify( + f"Update available: v{latest} (current: v{current}).\n" + "Run: uv tool upgrade EvoScientist", + severity="information", + timeout=15, + ) + except Exception: + _channel_logger.debug("Background update check failed", exc_info=True) + + # ── Channel integration ──────────────────────────────── + + def _start_channels(self) -> None: + """Auto-start channels if enabled in config.""" + try: + from ..config import load_config + + cfg = load_config() + if cfg and cfg.channel_enabled and not _channels_is_running(): + _auto_start_channel( + self._agent, + self._conversation_tid, + cfg, + send_thinking=self._channel_send_thinking, + ) + types = [ + t.strip() for t in cfg.channel_enabled.split(",") if t.strip() + ] + self._started_channel_types = types + self._render_welcome() + except Exception as e: + _channel_logger.debug(f"Channel auto-start failed: {e}") + self._channel_timer = self.set_interval(0.1, self._poll_channel_queue) + + def _poll_channel_queue(self) -> None: + """Poll the channel message queue (called every 100ms).""" + try: + msg = _message_queue.get_nowait() + except queue.Empty: + return + if self._busy: + _message_queue.put(msg) + return + self.call_later( + lambda m=msg: asyncio.ensure_future(self._process_channel_message(m)) + ) + + # ── Widget helpers ───────────────────────────────────── + + def _append_system(self, text: str, style: str = "dim") -> None: + """Mount a SystemMessage widget into #chat.""" + container = self.query_one("#chat", VerticalScroll) + container.mount(SystemMessage(text, msg_style=style)) + container.scroll_end(animate=False) + + def _mount_renderable(self, renderable: Any) -> None: + """Mount a Rich renderable (e.g. Table) as a Static widget.""" + container = self.query_one("#chat", VerticalScroll) + try: + from .commands import CompactSummaryRenderable + from .widgets.compact_summary_widget import CompactSummaryWidget + except Exception: + CompactSummaryRenderable = None # type: ignore[assignment] + + if CompactSummaryRenderable is not None and isinstance( + renderable, CompactSummaryRenderable + ): + container.mount(CompactSummaryWidget(renderable.summary_text)) + else: + container.mount(Static(renderable)) + container.scroll_end(animate=False) + + async def _start_compacting_indicator(self) -> None: + """Show a transient timer widget while /compact is running.""" + await self._stop_compacting_indicator() + container = self.query_one("#chat", VerticalScroll) + widget = CompactingWidget() + self._compacting_widget = widget + await container.mount(widget) + container.scroll_end(animate=False) + + async def _stop_compacting_indicator(self) -> None: + """Remove the transient /compact progress widget, if present.""" + widget = self._compacting_widget + self._compacting_widget = None + if widget is not None: + try: + await widget.cleanup() + except Exception: + try: + await widget.remove() + except Exception: + pass + + async def _wait_for_approval(self, approval_widget) -> Any: + """Wait for user to interact with an ApprovalWidget. + + Returns the ``ApprovalWidget.Decided`` message, or ``None`` on + timeout / cancellation. + """ + self._approval_future = asyncio.get_event_loop().create_future() + try: + return await asyncio.wait_for(self._approval_future, timeout=300) + except (TimeoutError, asyncio.CancelledError): + return None + finally: + self._approval_future = None + + def on_approval_widget_decided(self, event) -> None: # type: ignore[override] + """Handle ApprovalWidget.Decided message.""" + if self._approval_future and not self._approval_future.done(): + self._approval_future.set_result(event) + + async def _wait_for_ask_user(self, ask_w) -> dict: + """Wait for the interactive ask_user widget to resolve via Future. + + Returns ``{"answers": [...], "status": "answered"}`` + or ``{"status": "cancelled"}``. + """ + loop = asyncio.get_running_loop() + self._ask_user_future = loop.create_future() + ask_w.set_future(self._ask_user_future) + + try: + result = await asyncio.wait_for(self._ask_user_future, timeout=300) + except (TimeoutError, asyncio.CancelledError): + ask_w.action_cancel() + return {"status": "cancelled"} + finally: + self._ask_user_future = None + + if not isinstance(result, dict): + return {"status": "cancelled"} + + result_type = result.get("type", "") + if result_type == "answered": + return {"answers": result.get("answers", []), "status": "answered"} + return {"status": "cancelled"} + + async def _wait_for_thread_pick(self, picker_widget) -> str | None: + """Wait for user to pick a thread from ThreadPickerWidget. + + Returns the selected thread_id, or ``None`` on cancel/timeout. + """ + self._picker_future = asyncio.get_event_loop().create_future() + try: + return await asyncio.wait_for(self._picker_future, timeout=120) + except (TimeoutError, asyncio.CancelledError): + return None + finally: + self._picker_future = None + try: + picker_widget.remove() + except Exception: + pass + self.query_one("#prompt", ChatTextArea).focus() + + def on_thread_picker_widget_picked(self, event) -> None: # type: ignore[override] + """Handle ThreadPickerWidget.Picked message.""" + if self._picker_future and not self._picker_future.done(): + self._picker_future.set_result(event.thread_id) + + def on_thread_picker_widget_cancelled(self, event) -> None: # type: ignore[override] + """Handle ThreadPickerWidget.Cancelled message.""" + if self._picker_future and not self._picker_future.done(): + self._picker_future.set_result(None) + + async def _wait_for_skill_browse(self, browser_widget) -> list[str] | None: + """Wait for user to complete skill browsing. + + Returns list of install sources, or None on cancel/timeout. + """ + self._browser_future = asyncio.get_event_loop().create_future() + try: + return await asyncio.wait_for(self._browser_future, timeout=300) + except (TimeoutError, asyncio.CancelledError): + return None + finally: + self._browser_future = None + try: + browser_widget.remove() + except Exception: + pass + self.query_one("#prompt", ChatTextArea).focus() + + def on_skill_browser_widget_confirmed(self, event) -> None: # type: ignore[override] + """Handle SkillBrowserWidget.Confirmed message.""" + if self._browser_future and not self._browser_future.done(): + self._browser_future.set_result(event.install_sources) + + def on_skill_browser_widget_cancelled(self, event) -> None: # type: ignore[override] + """Handle SkillBrowserWidget.Cancelled message.""" + if self._browser_future and not self._browser_future.done(): + self._browser_future.set_result(None) + + # ── MCP browser ─────────────────────────────────────── + + async def _wait_for_mcp_browse(self, browser_widget) -> list | None: + """Wait for user to complete MCP server browsing.""" + self._mcp_browser_future = asyncio.get_event_loop().create_future() + try: + return await asyncio.wait_for(self._mcp_browser_future, timeout=300) + except (TimeoutError, asyncio.CancelledError): + return None + finally: + self._mcp_browser_future = None + try: + browser_widget.remove() + except Exception: + pass + self.query_one("#prompt", ChatTextArea).focus() + + def on_mcpbrowser_widget_confirmed(self, event) -> None: # type: ignore[override] + """Handle MCPBrowserWidget.Confirmed message.""" + if self._mcp_browser_future and not self._mcp_browser_future.done(): + self._mcp_browser_future.set_result(event.entries) + + def on_mcpbrowser_widget_cancelled(self, event) -> None: # type: ignore[override] + """Handle MCPBrowserWidget.Cancelled message.""" + if self._mcp_browser_future and not self._mcp_browser_future.done(): + self._mcp_browser_future.set_result(None) + + # ── Streaming core ───────────────────────────────────── + + async def _stream_with_widgets( + self, + user_text: str, + *, + display_text: str | None = None, + on_thinking_cb: Callable[[str], None] | None = None, + on_todo_cb: Callable[[list[dict]], None] | None = None, + on_media_cb: Callable[[str], None] | None = None, + skip_user_message: bool = False, + file_warnings: list[str] | None = None, + channel_hitl_fn: Callable[[list], list[dict] | None] | None = None, + channel_ask_user_fn: Callable[[dict], dict] | None = None, + ) -> str: + """Stream agent events and mount widgets. Returns response text. + + Shared by ``_run_turn`` (interactive) and + ``_process_channel_message`` (channel). + + Args: + display_text: Text to show in UserMessage widget. When + ``None`` (default), falls back to *user_text*. This + allows callers to show the original user input while + sending the resolved (e.g. @file-expanded) text to + the agent. + skip_user_message: If True, don't mount UserMessage (caller + already mounted it — e.g. channel messages with labels). + channel_hitl_fn: Optional channel-based HITL approval function. + When provided (channel messages), this is called instead + of mounting the ApprovalWidget. + channel_ask_user_fn: Optional channel-based ask_user function. + When provided (channel messages), this is called instead + of mounting the AskUserWidget. + """ + container = self.query_one("#chat", VerticalScroll) + + # 1. Mount user message + loading spinner + if not skip_user_message: + await container.mount(UserMessage(display_text or user_text)) + # Mount file warnings after user message so they appear in the + # correct position (between user input and model response). + for w in file_warnings or []: + self._append_system(f"⚠ {w}", style="yellow") + loading = LoadingWidget() + await container.mount(loading) + container.scroll_end(animate=False) + + # 2. Event-driven widget rendering + state = StreamState() + loading_removed = False + thinking_w: ThinkingWidget | None = None + summarization_w: SummarizationWidget | None = None + assistant_w: AssistantMessage | None = None + todo_w: TodoWidget | None = None + tool_widgets: dict[str, ToolCallWidget] = {} + subagent_widgets: dict[str, SubAgentWidget] = {} + + # Transient indicator widgets (auto-removed on state transitions) + narration_w: Static | None = None # dim italic intermediate text + processing_w: Static | None = None # "Analyzing results..." + + # Tool collapsing (matches CLI MAX_VISIBLE_TOOLS) + _MAX_VISIBLE_TOOLS = 4 + completed_tool_order: list[str] = [] # tool_ids in completion order + collapse_summary_w: Static | None = None + has_used_tools = False + + _thinking_sent = False + _todo_sent = False + _media_sent: set[str] = set() + _MIN_THINKING_LEN = 200 + _scroll_pending = False + + def _schedule_scroll() -> None: + """Throttle scroll_end to at most once per 200ms. + + Uses call_after_refresh so the scroll happens after Textual + finishes its layout pass — otherwise scroll_end may see + stale widget heights and not scroll far enough. + """ + nonlocal _scroll_pending + if not _scroll_pending: + _scroll_pending = True + self.set_timer(0.2, _do_scroll) + + def _do_scroll() -> None: + nonlocal _scroll_pending + _scroll_pending = False + self.call_after_refresh( + lambda: container.scroll_end(animate=False), + ) + + metadata = build_metadata(self._workspace_dir, model) + response = "" + + async def _remove_w(w: Static | None) -> None: + """Safely remove a transient indicator widget.""" + if w is not None: + try: + await w.remove() + except Exception: + pass + + def _finalize_active_summarization() -> None: + """Stop the active summary timer once the stream moves on.""" + if summarization_w is not None and summarization_w._is_active: + summarization_w.finalize() + + async def _collapse_completed_tools() -> None: + """Hide older completed tool widgets; show summary line.""" + nonlocal collapse_summary_w + completed = [ + (tid, tool_widgets[tid]) + for tid in completed_tool_order + if tid in tool_widgets + ] + n = len(completed) + if n <= _MAX_VISIBLE_TOOLS: + if collapse_summary_w is not None: + collapse_summary_w.display = False + return + + to_hide = n - _MAX_VISIBLE_TOOLS + ok_count = 0 + fail_count = 0 + for i, (_, tw) in enumerate(completed): + if i < to_hide: + tw.display = False + if tw._status == "success": + ok_count += 1 + else: + fail_count += 1 + else: + tw.display = True + + summary = Text() + summary.append(f"\u2713 {ok_count} completed", style="dim green") + if fail_count > 0: + summary.append(f" | {fail_count} failed", style="dim red") + + if collapse_summary_w is None: + collapse_summary_w = Static(summary) + # Position before first visible tool widget + first_visible = None + for _, tw in completed[to_hide:]: + if tw.display: + first_visible = tw + break + if first_visible: + await container.mount(collapse_summary_w, before=first_visible) + else: + await container.mount(collapse_summary_w) + else: + collapse_summary_w.update(summary) + collapse_summary_w.display = True + + def _find_or_rename_sa_widget( + resolved_name: str, + description: str = "", + ) -> SubAgentWidget | None: + """Look up a sub-agent widget, renaming 'sub-agent' entry if needed.""" + if resolved_name in subagent_widgets: + w = subagent_widgets[resolved_name] + if description and not w._description: + w.update_name(w._sa_name, description) + return w + # Rename "sub-agent" → real name (mirrors state._get_or_create_subagent) + if resolved_name != "sub-agent" and "sub-agent" in subagent_widgets: + w = subagent_widgets.pop("sub-agent") + w.update_name(resolved_name, description) + subagent_widgets[resolved_name] = w + return w + return None + + _MAX_HITL_ROUNDS = 50 + _stream_input: Any = user_text # str or Command for HITL resume + + for _hitl_round in range(_MAX_HITL_ROUNDS): + state.pending_interrupt = None + state.pending_ask_user = None + _hitl_resuming = False + # Reset per-round widgets so resumed streams get fresh ones + if _hitl_round > 0: + thinking_w = None + summarization_w = None + try: + async for event in stream_agent_events( + self._agent, + _stream_input, + self._conversation_tid, + metadata=metadata, + ): + event_type = state.handle_event(event) + + if event_type == "usage_stats": + self._set_status_usage_baseline(state.last_input_tokens) + + if _should_finalize_active_summarization(event_type): + _finalize_active_summarization() + + # -- Channel callbacks (thinking, todo, media) -- + if ( + on_thinking_cb + and not _thinking_sent + and state.thinking_text + and event_type != "thinking" + and len(state.thinking_text) >= _MIN_THINKING_LEN + ): + on_thinking_cb(state.thinking_text.rstrip()) + _thinking_sent = True + + if ( + on_todo_cb + and not _todo_sent + and event_type == "tool_call" + and event.get("name") == "write_todos" + and state.todo_items + ): + if ( + on_thinking_cb + and not _thinking_sent + and state.thinking_text + and len(state.thinking_text) >= _MIN_THINKING_LEN + ): + on_thinking_cb(state.thinking_text.rstrip()) + _thinking_sent = True + on_todo_cb(state.todo_items) + _todo_sent = True + + if ( + on_media_cb + and event_type == "tool_result" + and event.get("success") + ): + tool_name = event.get("name", "") + if tool_name in ("write_file", "read_file"): + _forward_media_to_channel( + state, + tool_name, + _media_sent, + on_media_cb, + ) + + # -- Remove loading spinner on first content event -- + if not loading_removed and event_type in ( + "thinking", + "text", + "tool_call", + "summarization_start", + "summarization", + ): + await loading.cleanup() + loading_removed = True + + # -- Widget dispatch -- + if event_type == "thinking": + if thinking_w is None: + thinking_w = ThinkingWidget(show_thinking=show_thinking) + await container.mount(thinking_w) + thinking_w.append_text(event.get("content", "")) + + elif event_type == "summarization_start": + if ( + summarization_w is not None + and not summarization_w._is_active + ): + summarization_w = None + if summarization_w is None: + summarization_w = SummarizationWidget() + await container.mount(summarization_w) + + elif event_type == "summarization": + content = event.get("content", "") + if ( + summarization_w is not None + and not summarization_w._is_active + ): + summarization_w = None + if summarization_w is None: + summarization_w = SummarizationWidget() + await container.mount(summarization_w) + if content: + summarization_w.append_text(content) + + elif event_type == "tool_selection": + tools = event.get("tools", []) + if tools: + from .widgets.tool_selection_widget import ( + ToolSelectionWidget, + ) + + await container.mount(ToolSelectionWidget(tools)) + _schedule_scroll() + + elif event_type == "text": + if thinking_w is not None and thinking_w._is_active: + thinking_w.finalize() + # Clear processing indicator + await _remove_w(processing_w) + processing_w = None + + if has_used_tools and not _is_final_response(state): + # Tools still running — show intermediate narration + await _remove_w(narration_w) + narration_w = None + last_line = ( + state.latest_text.strip().split("\n")[-1].strip() + ) + if last_line: + if len(last_line) > 60: + last_line = last_line[:57] + "\u2026" + narration_w = Static( + Text(f" {last_line}", style="dim italic"), + ) + await container.mount(narration_w) + else: + # Stream final response incrementally (both + # text-only replies and post-tool responses). + await _remove_w(narration_w) + narration_w = None + if assistant_w is None: + assistant_w = AssistantMessage(state.response_text) + await container.mount(assistant_w) + else: + await assistant_w.append_content( + event.get("content", ""), + ) + self._set_status_streaming_text(state.response_text) + + elif event_type == "tool_call": + tool_name = event.get("name", "unknown") + tool_id = event.get("id", "") + tool_args = event.get("args", {}) + # Finalize thinking if still active + if thinking_w is not None and thinking_w._is_active: + thinking_w.finalize() + # Clear transient indicators + await _remove_w(narration_w) + narration_w = None + await _remove_w(processing_w) + processing_w = None + # Remove early AssistantMessage (text arrived before tools) + if assistant_w is not None: + try: + await assistant_w.remove() + except Exception: + pass + assistant_w = None + # Skip internal tools and task (handled by SubAgentWidget) + if tool_name not in _INTERNAL_TOOLS and tool_name != "task": + has_used_tools = True + if tool_id and tool_id in tool_widgets: + # Re-emitted with updated args — update in place + existing = tool_widgets[tool_id] + existing._tool_name = tool_name + existing._tool_args = tool_args + try: + existing._render_header() + except Exception: + pass + else: + w = ToolCallWidget(tool_name, tool_args, tool_id) + await container.mount(w) + if tool_id: + tool_widgets[tool_id] = w + # Update todo widget on write_todos. + # Insert before tool call widget so Task List + # panel appears above the tool call. + if tool_name == "write_todos" and state.todo_items: + if todo_w is None: + todo_w = TodoWidget(state.todo_items) + if tool_id and tool_id in tool_widgets: + await container.mount( + todo_w, + before=tool_widgets[tool_id], + ) + else: + await container.mount(todo_w) + else: + todo_w.update_items(state.todo_items) + + elif event_type == "tool_result": + result_name = event.get("name", "unknown") + result_content = event.get("content", "") + result_success = event.get("success", True) + # Match via state's deduplicated tool_calls (uses tool_id) + matched = False + matched_tid = "" + result_idx = len(state.tool_results) - 1 + if 0 <= result_idx < len(state.tool_calls): + tc = state.tool_calls[result_idx] + tid = tc.get("id", "") + if tid and tid in tool_widgets: + tw = tool_widgets[tid] + if tw._status == "running": + if result_success: + tw.set_success(result_content) + else: + tw.set_error(result_content) + matched = True + matched_tid = tid + # Fallback: match first running widget with same name + if not matched: + for fid, tw in tool_widgets.items(): + if ( + tw.tool_name == result_name + and tw._status == "running" + ): + if result_success: + tw.set_success(result_content) + else: + tw.set_error(result_content) + matched = True + matched_tid = fid + break + # Track completion order for collapsing + if matched_tid and matched_tid not in completed_tool_order: + completed_tool_order.append(matched_tid) + await _collapse_completed_tools() + # Update todo from results + if ( + result_name in ("write_todos", "read_todos") + and state.todo_items + ): + if todo_w is None: + todo_w = TodoWidget(state.todo_items) + await container.mount(todo_w) + else: + todo_w.update_items(state.todo_items) + # Show "Analyzing results..." if all tools done, no text yet + if ( + _is_final_response(state) + and not state.response_text + and processing_w is None + ): + processing_w = Static( + Text("\u25cf Analyzing results...", style="cyan"), + ) + await container.mount(processing_w) + + elif event_type == "subagent_start": + sa_name = event.get("name", "sub-agent") + sa_desc = event.get("description", "") + existing = _find_or_rename_sa_widget(sa_name, sa_desc) + if existing is None: + sa_w = SubAgentWidget(sa_name, sa_desc) + await container.mount(sa_w) + subagent_widgets[sa_name] = sa_w + + elif event_type == "subagent_tool_call": + sa_name = event.get("subagent", "sub-agent") + sa_name = state._resolve_subagent_name(sa_name) + sa_w = _find_or_rename_sa_widget(sa_name) + if sa_w is None: + sa_w = SubAgentWidget(sa_name) + await container.mount(sa_w) + subagent_widgets[sa_name] = sa_w + await sa_w.add_tool_call( + event.get("name", "unknown"), + event.get("args", {}), + event.get("id", ""), + ) + + elif event_type == "subagent_tool_result": + sa_name = event.get("subagent", "sub-agent") + sa_name = state._resolve_subagent_name(sa_name) + sa_w = _find_or_rename_sa_widget(sa_name) + if sa_w is not None: + sa_w.complete_tool( + event.get("name", "unknown"), + event.get("content", ""), + event.get("success", True), + event.get("id", ""), + ) + + elif event_type == "subagent_end": + sa_name = event.get("name", "sub-agent") + sa_name = state._resolve_subagent_name(sa_name) + sa_w = _find_or_rename_sa_widget(sa_name) + if sa_w is not None: + sa_w.finalize() + + elif event_type == "ask_user": + questions = event.get("questions", []) + if questions: + # Channel messages: use channel-based text prompt + if channel_ask_user_fn is not None: + self._append_system( + "Waiting for channel user input...", + style="dim italic", + ) + _ask_fn = channel_ask_user_fn + result = await asyncio.to_thread( + lambda f=_ask_fn, e=event: f(e), + ) + else: + # Interactive TUI: display widget, collect via arrow keys + from .widgets.ask_user_widget import AskUserWidget + + _prompt = self.query_one("#prompt", ChatTextArea) + _prompt.disabled = True + ask_w = AskUserWidget(questions) + await container.mount(ask_w) + _schedule_scroll() + self.call_after_refresh(ask_w.focus_active) + result = await self._wait_for_ask_user(ask_w) + try: + await ask_w.remove() + except Exception: + pass + _prompt.disabled = False + from langgraph.types import ( + Command, # type: ignore[import-untyped] + ) + + _stream_input = Command(resume=result) + _hitl_resuming = True + break # re-enter outer HITL loop + + elif event_type == "interrupt": + action_reqs = event.get("action_requests", []) + n = len(action_reqs) or 1 + + # HITL: check session auto-approve first + if self._hitl_auto_approve: + from langgraph.types import ( + Command, # type: ignore[import-untyped] + ) + + _stream_input = Command( + resume={ + "decisions": [ + {"type": "approve"} for _ in range(n) + ] + } + ) + _hitl_resuming = True + break # re-enter outer HITL loop + + # Channel messages: use channel-based text approval + if channel_hitl_fn is not None: + self._append_system( + "Waiting for channel user approval...", + style="dim italic", + ) + decisions = await asyncio.to_thread( + channel_hitl_fn, + action_reqs, + ) + if decisions is not None: + from langgraph.types import ( + Command, # type: ignore[import-untyped] + ) + + _stream_input = Command( + resume={"decisions": decisions} + ) + _hitl_resuming = True + break # re-enter outer HITL loop + else: + state.pending_interrupt = None + for tw in tool_widgets.values(): + if tw._status == "running": + tw.set_rejected() + self._append_system( + "Tool execution rejected by channel user.", + style="yellow", + ) + continue + + # Interactive TUI: mount approval widget + # Disable main prompt so it can't steal focus + _prompt = self.query_one("#prompt", ChatTextArea) + _prompt.disabled = True + from .widgets.approval_widget import ApprovalWidget + + approval_w = ApprovalWidget(action_reqs) + await container.mount(approval_w) + _schedule_scroll() + decided_event = await self._wait_for_approval(approval_w) + await approval_w.remove() + _prompt.disabled = False + if decided_event and decided_event.decisions is not None: + if decided_event.auto_approve_session: + self._hitl_auto_approve = True + from langgraph.types import ( + Command, # type: ignore[import-untyped] + ) + + _stream_input = Command( + resume={"decisions": decided_event.decisions} + ) + _hitl_resuming = True + break # re-enter outer HITL loop with resume + else: + state.pending_interrupt = None + for tw in tool_widgets.values(): + if tw._status == "running": + tw.set_rejected() + self._append_system( + "Tool execution rejected.", + style="yellow", + ) + + elif event_type == "done": + # Clean up transient indicators + await _remove_w(narration_w) + narration_w = None + await _remove_w(processing_w) + processing_w = None + # Mount final response + if assistant_w is None and state.response_text: + # Strip trailing standalone "..." + clean = state.response_text.strip() + while ( + clean.endswith("\n...") or clean.rstrip() == "..." + ): + clean = clean.rstrip().removesuffix("...").rstrip() + assistant_w = AssistantMessage( + clean or state.response_text + ) + await container.mount(assistant_w) + # Markdown rendering is async and needs multiple + # layout cycles to compute final height. Schedule + # repeated deferred scrolls so long content stays + # visible even when Markdown takes time to lay out. + for delay in (0.15, 0.4, 0.8, 1.5): + self.set_timer( + delay, + lambda: self.call_after_refresh( + lambda: container.scroll_end(animate=False), + ), + ) + # Mount token usage stats + if state.total_input_tokens or state.total_output_tokens: + await container.mount( + UsageWidget( + state.total_input_tokens, + state.total_output_tokens, + ) + ) + + elif event_type == "error": + error_msg = event.get("message", "Unknown error") + self._append_system(f"Error: {error_msg}", style="red") + + # Scroll after Textual processes the layout update + _schedule_scroll() + + response = (state.response_text or "").strip() + + except asyncio.CancelledError: + # Ctrl+C cancellation — re-raise so _run_turn can handle it + raise + except Exception as exc: + error_msg = str(exc) + if ( + "authentication" in error_msg.lower() + or "api_key" in error_msg.lower() + ): + self._append_system( + "Error: API key not configured.", + style="red", + ) + self._append_system( + "Run EvoSci onboard to set up your API key.", + style="dim", + ) + else: + self._append_system(f"Error: {exc}", style="red") + response = f"Error: {exc}" + finally: + # Clean up loading widget if it wasn't removed yet + if not loading_removed: + try: + await loading.cleanup() + except Exception: + pass + # Clean up transient indicators + for w in (narration_w, processing_w): + await _remove_w(w) + # Mark any still-running tool widgets as interrupted + # (skip if HITL approved — tools will continue next round) + if not _hitl_resuming: + for tw in tool_widgets.values(): + if tw._status == "running": + try: + tw.set_interrupted() + except Exception: + pass + # Finalize any still-active sub-agents + for sa_w in subagent_widgets.values(): + if sa_w._is_active: + try: + sa_w.finalize() + except Exception: + pass + # Finalize thinking widget + if thinking_w is not None and thinking_w._is_active: + try: + thinking_w.finalize() + except Exception: + pass + # Finalize assistant message stream + if assistant_w is not None: + await assistant_w.stop_stream() + # Flush remaining thinking callback + if ( + on_thinking_cb + and not _thinking_sent + and state.thinking_text + and len(state.thinking_text) >= _MIN_THINKING_LEN + ): + on_thinking_cb(state.thinking_text.rstrip()) + # Final scrolls to ensure last content is visible. + # Markdown layout is async — schedule multiple deferred + # scrolls so long content eventually scrolls into view. + self.call_after_refresh( + lambda: container.scroll_end(animate=False), + ) + for delay in (0.3, 0.8): + self.set_timer( + delay, + lambda: self.call_after_refresh( + lambda: container.scroll_end(animate=False), + ), + ) + + # HITL / ask_user: if interrupt was handled, loop back to resume stream + if state.pending_interrupt is None and state.pending_ask_user is None: + break # normal completion or rejection — exit HITL loop + # Otherwise _stream_input was set to Command(resume=...) + # by the interrupt handler above; loop continues. + + return response + + async def _run_turn(self, user_text: str) -> None: + """Handle a user turn: stream agent response with widgets.""" + cancelled = False + try: + self._busy = True + self._render_status() + + # Resolve @file mentions — inject file contents before sending to agent. + # Use self._workspace_dir (current session) not the startup-captured + # workspace_dir closure, which becomes stale after /new or /resume. + _, message_to_send, file_warnings = await asyncio.to_thread( + resolve_file_mentions, user_text, self._workspace_dir + ) + await self._refresh_status_snapshot(message_to_send) + + await self._stream_with_widgets( + message_to_send, + display_text=user_text, + file_warnings=file_warnings, + ) + except asyncio.CancelledError: + cancelled = True + self._append_system("\nInterrupted by user", style="dim italic #ffe082") + finally: + self._busy = False + self._run_task = None + await self._refresh_status_snapshot(reset_streaming_text=True) + self._render_status() + self.query_one("#prompt", ChatTextArea).focus() + + # Process next queued message (FIFO) — skip if interrupted + if not cancelled and self._queued_messages: + next_msg = self._queued_messages.pop(0) + self._render_queue_indicator() + self._run_task = asyncio.ensure_future(self._run_turn(next_msg)) + + async def _process_channel_message(self, msg: ChannelMessage) -> None: + """Process a channel message: stream agent response and reply. + + Display order (matches Rich CLI): + > message content + [channel: Received from sender] + (streaming response) + [channel: Replied to sender] + """ + prompt_widget = None + try: + self._busy = True + await self._refresh_status_snapshot(msg.content) + self._render_status() + + prompt_widget = self.query_one("#prompt", ChatTextArea) + prompt_widget.disabled = True + + # Mount user message first, then "Received" label + container = self.query_one("#chat", VerticalScroll) + await container.mount(UserMessage(msg.content)) + self._append_system( + f"[{msg.channel_type}: Received from {msg.sender}]", + style="dim", + ) + container.scroll_end(animate=False) + + # Build channel callbacks (fire-and-forget to avoid blocking UI) + def _send_to_channel(coro, label: str) -> None: + loop = _ch_mod._bus_loop + if not loop: + return + future = asyncio.run_coroutine_threadsafe(coro, loop) + future.add_done_callback( + lambda f: ( + _channel_logger.debug( + f"{label} send failed: {f.exception()}" + ) + if f.exception() + else None + ) + ) + + def _send_thinking(thinking: str) -> None: + ch = msg.channel_ref + if ch and ch.send_thinking: + _send_to_channel( + ch.send_thinking_message( + sender=msg.chat_id, + thinking=thinking, + metadata=msg.metadata, + ), + "Thinking", + ) + + def _send_todo(items: list[dict]) -> None: + from ..channels.consumer import _format_todo_list + + if msg.channel_ref: + _send_to_channel( + msg.channel_ref.send_todo_message( + sender=msg.chat_id, + content=_format_todo_list(items), + metadata=msg.metadata, + ), + "Todo", + ) + + def _send_media(file_path: str) -> None: + if msg.channel_ref: + _send_to_channel( + msg.channel_ref.send_media( + recipient=msg.chat_id, + file_path=file_path, + metadata=msg.metadata, + ), + "Media", + ) + + def _channel_hitl_prompt(action_requests: list) -> list[dict] | None: + """Send HITL approval prompt to channel user and wait for reply. + + This runs in a thread (called via asyncio.to_thread) so it can + block without freezing the Textual event loop. + """ + return _ch_mod.channel_hitl_prompt(action_requests, msg) + + def _channel_ask_user(ask_user_data: dict) -> dict: + """Send ask_user questions to channel user and wait for reply. + + This runs in a thread (called via asyncio.to_thread) so it can + block without freezing the Textual event loop. + """ + return _ch_mod.channel_ask_user_prompt(ask_user_data, msg) + + from ..commands.channel_ui import ChannelCommandUI + + # Handle slash commands from channel + if msg.content.strip().startswith("/"): + ctx = CommandContext( + agent=self._agent, + thread_id=self._conversation_tid, + ui=ChannelCommandUI( + msg, + append_system_callback=self._append_system, + start_new_session_callback=self.start_new_session, + handle_session_resume_callback=self.handle_session_resume, + ), + workspace_dir=self._workspace_dir, + checkpointer=self._checkpointer, + ) + try: + cmd_executed = await cmd_manager.execute(msg.content, ctx) + except Exception as _cmd_exc: + # Command raised — report the error and do NOT fall through + # to _stream_with_widgets (which would treat the slash + # command text as a plain user message to the agent). + _channel_logger.debug( + f"Channel command error: {_cmd_exc}", exc_info=True + ) + _set_channel_response(msg.msg_id, f"Command error: {_cmd_exc}") + return # outer finally handles _busy / widget cleanup + + if cmd_executed: + self._append_system( + f"[{msg.channel_type}: Executed command from {msg.sender}]", + style="dim", + ) + _set_channel_response( + msg.msg_id, f"Command executed: {msg.content}" + ) + return # outer finally handles _busy / widget cleanup + + response = "" + try: + response = await self._stream_with_widgets( + msg.content, + on_thinking_cb=_send_thinking + if self._channel_send_thinking + else None, + on_todo_cb=_send_todo, + on_media_cb=_send_media, + skip_user_message=True, + channel_hitl_fn=_channel_hitl_prompt, + channel_ask_user_fn=_channel_ask_user, + ) + except Exception as exc: + response = f"Error: {exc}" + self._append_system(f"Error: {exc}", style="red") + + _set_channel_response(msg.msg_id, response) + self._append_system( + f"[{msg.channel_type}: Replied to {msg.sender}]", + style="dim", + ) + + finally: + self._busy = False + await self._refresh_status_snapshot(reset_streaming_text=True) + self._render_status() + if prompt_widget is not None: + prompt_widget.disabled = False + prompt_widget.focus() + + # ── Clipboard (copy on mouse select) ───────────────── + + def on_mouse_up(self, event: MouseUp) -> None: + """Copy mouse-selected text to clipboard on release.""" + copy_selection_to_clipboard(self) + + # ── Input handling ───────────────────────────────────── + + async def on_chat_text_area_submitted( + self, event: ChatTextArea.Submitted + ) -> None: + text = event.value.strip() + prompt = self.query_one("#prompt", ChatTextArea) + prompt.value = "" + self._quit_pending = False + self._history_index = -1 + self._history_saved_input = "" + + if not text: + return + + if self._busy: + # Queue the message to send after current turn finishes + self._queued_messages.append(text) + self._render_queue_indicator() + return + + if text.startswith("/"): + self._hide_completions() + # Launch as independent task to free the message pump. + # Commands like /resume mount interactive widgets that need + # the pump to process key events and message bubbling. + _task = asyncio.create_task(self._handle_command(text)) + self._background_tasks.add(_task) + _task.add_done_callback(self._background_tasks.discard) + return + + self._history_suggester.append_entry(text) + self._run_task = asyncio.ensure_future(self._run_turn(text)) + + def on_text_area_changed(self, event: ChatTextArea.Changed) -> None: + text = event.text_area.text + comp_widget = self.query_one("#completions", Static) + + # @file mention completion + if "@" in text: + candidates = complete_file_mention(text, workspace_dir) + if candidates: + self._comp_items = candidates + self._comp_index = -1 + self._render_completions() + comp_widget.display = True + return + + if text.startswith("/"): + prefix = text.lower() + matches = [ + (cmd, desc) + for cmd, desc in cmd_manager.list_commands() + if cmd.startswith(prefix) + ] + if len(matches) == 1 and matches[0][0] == prefix: + self._hide_completions() + return + if matches: + self._comp_items = matches + self._comp_index = -1 + self._render_completions() + comp_widget.display = True + return + self._hide_completions() + + def _render_queue_indicator(self) -> None: + """Render the queued messages indicator above the input.""" + queued_w = self.query_one("#queued-message", Static) + if not self._queued_messages: + queued_w.display = False + return + parts: list[tuple[str, str]] = [] + for msg in self._queued_messages: + preview = msg if len(msg) <= 60 else msg[:57] + "\u2026" + parts.append(("\u276f ", "bold")) + parts.append((preview, "")) + parts.append(("\n", "")) + parts.append( + (" [press up to edit last \u00b7 esc to cancel last]", "dim italic") + ) + queued_w.update(Text.assemble(*parts)) + queued_w.display = True + + def action_cancel_queued(self) -> None: + """Cancel the last queued message on Esc.""" + # Cancel ask_user if active (widget handles Escape internally, + # but this is a safety fallback) + if self._ask_user_future and not self._ask_user_future.done(): + try: + from .widgets.ask_user_widget import AskUserWidget + + ask_w = self.query_one(AskUserWidget) + ask_w.action_cancel() + except Exception: + # Force-resolve the future + self._ask_user_future.set_result({"type": "cancelled"}) + return + # Delegate to ApprovalWidget, ThreadPickerWidget, or SkillBrowserWidget if focused + focused = self.focused + if focused is not None: + from .widgets.approval_widget import ApprovalWidget + from .widgets.mcp_browser import MCPBrowserWidget + from .widgets.skill_browser import SkillBrowserWidget + from .widgets.thread_selector import ThreadPickerWidget + + if isinstance(focused, ApprovalWidget): + focused.action_select_reject() + return + if isinstance(focused, ThreadPickerWidget): + focused.action_cancel() + return + if isinstance(focused, SkillBrowserWidget): + focused.action_cancel() + return + if isinstance(focused, MCPBrowserWidget): + focused.action_cancel() + return + if self._queued_messages: + self._queued_messages.pop() + self._render_queue_indicator() + + def action_edit_queued(self) -> None: + """Pop the last queued message back into input for editing.""" + # Handle completion list selection (up key) + comp_widget = self.query_one("#completions", Static) + if comp_widget.display and self._comp_items: + self._comp_index = (self._comp_index - 1) % len(self._comp_items) + self._render_completions() + return + + # Skip if an ApprovalWidget, AskUserWidget, ThreadPickerWidget, or SkillBrowserWidget has focus + focused = self.focused + if focused is not None: + from .widgets.approval_widget import ApprovalWidget + from .widgets.ask_user_widget import AskUserWidget + from .widgets.mcp_browser import MCPBrowserWidget + from .widgets.skill_browser import SkillBrowserWidget + from .widgets.thread_selector import ThreadPickerWidget + + if isinstance(focused, ApprovalWidget): + focused.action_move_up() + return + if isinstance(focused, AskUserWidget): + focused.action_move_up() + return + if isinstance(focused, ThreadPickerWidget): + focused.action_move_up() + return + if isinstance(focused, SkillBrowserWidget): + focused.action_move_up() + return + if isinstance(focused, MCPBrowserWidget): + focused.action_move_up() + return + if self._queued_messages: + last = self._queued_messages.pop() + prompt = self.query_one("#prompt", ChatTextArea) + prompt.value = last + prompt.focus() + self._render_queue_indicator() + return + + # History browsing (up key) + entries = self._history_suggester._entries + if not entries: + return + prompt = self.query_one("#prompt", ChatTextArea) + if self._history_index == -1: + # Save current input before entering history + self._history_saved_input = prompt.value + if self._history_index + 1 < len(entries): + self._history_index += 1 + prompt.value = entries[self._history_index] + prompt.focus() + + def action_down_delegate(self) -> None: + """Delegate down key to focused interactive widget.""" + # Handle completion list selection (down key) + comp_widget = self.query_one("#completions", Static) + if comp_widget.display and self._comp_items: + self._comp_index = (self._comp_index + 1) % len(self._comp_items) + self._render_completions() + return + + focused = self.focused + if focused is not None: + from .widgets.approval_widget import ApprovalWidget + from .widgets.ask_user_widget import AskUserWidget + from .widgets.mcp_browser import MCPBrowserWidget + from .widgets.skill_browser import SkillBrowserWidget + from .widgets.thread_selector import ThreadPickerWidget + + if isinstance(focused, ApprovalWidget): + focused.action_move_down() + return + if isinstance(focused, AskUserWidget): + focused.action_move_down() + return + if isinstance(focused, ThreadPickerWidget): + focused.action_move_down() + return + if isinstance(focused, SkillBrowserWidget): + focused.action_move_down() + return + if isinstance(focused, MCPBrowserWidget): + focused.action_move_down() + return + + # History browsing (down key) + if self._history_index >= 0: + prompt = self.query_one("#prompt", ChatTextArea) + self._history_index -= 1 + if self._history_index == -1: + # Back to saved input + prompt.value = self._history_saved_input + else: + prompt.value = self._history_suggester._entries[self._history_index] + prompt.focus() + + def action_paste_clipboard(self) -> None: + """Paste text from system clipboard into the input field.""" + text = get_clipboard_text() + if not text: + self.notify( + "Clipboard is empty or unavailable", + severity="warning", + timeout=2, + ) + return + + prompt = self.query_one("#prompt", ChatTextArea) + prompt.insert(text) + prompt.focus() + + def action_tab_complete(self) -> None: + """Handle TAB: cycle completions when visible, otherwise no-op. + + Registered as a priority binding so it intercepts before Textual's + default focus-next behaviour, which would steal focus from the input + and lose the cursor. + """ + comp_widget = self.query_one("#completions", Static) + if not (comp_widget.display and self._comp_items): + # No completions active — keep focus on the prompt. + self.query_one("#prompt", ChatTextArea).focus() + return + self._comp_index = (self._comp_index + 1) % len(self._comp_items) + self._apply_selected_completion() + + def _handle_completion_enter(self) -> bool: + """Called by ChatTextArea before submitting on Enter. + + If a completion is active and an item is selected, apply it + and suppress the submit. If the list is visible but nothing + is selected (index == -1), select the first item instead of + submitting the raw prefix. + + Returns: + True to suppress submit, False to allow it. + """ + comp_widget = self.query_one("#completions", Static) + if not (comp_widget.display and self._comp_items): + return False + + # If no item highlighted yet, select the first one + if self._comp_index < 0: + self._comp_index = 0 + + self._apply_selected_completion() + self._hide_completions() + return True + + def _apply_selected_completion(self) -> None: + """Apply the currently selected completion to the input field. + + For ``@file`` completions the last ``@token`` is replaced in-place; + for slash-command completions the entire input is replaced. + """ + if self._comp_index < 0 or self._comp_index >= len(self._comp_items): + return + selected = self._comp_items[self._comp_index][0] + prompt = self.query_one("#prompt", ChatTextArea) + + if selected.startswith("@"): + import re as _re + + current = prompt.value + m = _re.search(r"@[^\s]*$", current) + if m: + new_val = current[: m.start()] + selected + " " + else: + new_val = current + selected + " " + prompt.value = new_val + else: + prompt.value = selected + " " + + def _hide_completions(self) -> None: + self._comp_items = [] + self._comp_index = -1 + comp_widget = self.query_one("#completions", Static) + comp_widget.display = False + + def _render_completions(self) -> None: + comp_text = Text() + for i, (cmd, desc) in enumerate(self._comp_items): + if i == self._comp_index: + comp_text.append("\u25b8 ", style="bold") + comp_text.append(f"{cmd:<30}", style="bold") + comp_text.append(desc, style="bold") + else: + comp_text.append(" ", style="#888888") + comp_text.append(f"{cmd:<30}", style="#888888") + comp_text.append(desc, style="#888888") + if i < len(self._comp_items) - 1: + comp_text.append("\n") + self.query_one("#completions", Static).update(comp_text) + + # ── Slash commands ───────────────────────────────────── + + async def _handle_command(self, command: str) -> None: + # Echo the command so the user sees what they ran + self._append_system(command.strip(), style="cyan") + + # Block new user input while the command runs (important for slow + # commands like /compact that call an LLM internally). + prompt_widget = self.query_one("#prompt", ChatTextArea) + self._busy = True + prompt_widget.disabled = True + self._render_status() + + ctx = CommandContext( + agent=self._agent, + thread_id=self._conversation_tid, + ui=self, + workspace_dir=self._workspace_dir, + checkpointer=self._checkpointer, + input_tokens_hint=self._status_last_input_tokens, + ) + + try: + if await cmd_manager.execute(command, ctx): + # Do NOT invalidate the usage baseline after /compact. + # build_session_status_snapshot() only counts raw checkpoint + # messages (~46 tokens) and misses system prompt + tool + # definitions (~50K overhead). The stale pre-compact count + # is far more accurate; the next LLM call will correct it. + await self._refresh_status_snapshot( + reset_streaming_text=True, + ) + return + + self._append_system(f"Unknown command: {command}", style="yellow") + self._render_status() + finally: + self._busy = False + prompt_widget.disabled = False + prompt_widget.focus() + + async def _render_history(self, thread_id_value: str) -> None: + """Render conversation history from a saved thread. + + Restores human messages and AI responses (with Markdown and + thinking panels). Tool calls and other intermediate steps are + skipped — they are difficult to faithfully reproduce from + checkpoint data. + """ + messages = await get_thread_messages(thread_id_value) + if not messages: + return + + HISTORY_WINDOW = 50 + container = self.query_one("#chat", VerticalScroll) + + # Only human and ai messages; skip tool/system/other + display = [ + m for m in messages if getattr(m, "type", None) in ("human", "ai") + ] + + if len(display) > HISTORY_WINDOW: + skipped = len(display) - HISTORY_WINDOW + display = display[-HISTORY_WINDOW:] + await container.mount( + SystemMessage( + f"── ... {skipped} earlier messages ──", msg_style="dim" + ) + ) + else: + await container.mount( + SystemMessage("── Conversation history ──", msg_style="dim") + ) + + for message in display: + msg_type = getattr(message, "type", None) + content = getattr(message, "content", "") or "" + + if msg_type == "human": + if isinstance(content, list): + parts = [ + block.get("text", "") + for block in content + if isinstance(block, dict) and block.get("type") == "text" + ] + content = " ".join(parts) if parts else "" + content = content.strip() + if content: + await container.mount(UserMessage(content)) + + elif msg_type == "ai": + # Extract thinking and text blocks from content list + thinking_text = "" + text_content = "" + if isinstance(content, list): + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") == "thinking": + thinking_text += block.get("thinking", "") + elif block.get("type") == "text": + text_content += block.get("text", "") + else: + text_content = content or "" + text_content = text_content.strip() + + # Render thinking as collapsed panel (click to expand) + if thinking_text.strip() and show_thinking: + w = ThinkingWidget(show_thinking=True) + await container.mount(w) + w.append_text(thinking_text) + w.finalize() + + # Render AI response with full Markdown + if text_content: + await container.mount(AssistantMessage(text_content)) + + await container.mount( + SystemMessage("── End of history ──", msg_style="dim") + ) + container.scroll_end(animate=False) + + # ── Quit handling ────────────────────────────────────── + + def _arm_quit_pending(self, shortcut: str) -> None: + """Set the pending-quit flag and show a matching hint.""" + self._quit_pending = True + quit_timeout = 3 # seconds + self.notify(f"Press {shortcut} again to quit", timeout=quit_timeout) + self.set_timer( + quit_timeout, + lambda: setattr(self, "_quit_pending", False), + ) + + def force_quit(self) -> None: + """Exit immediately without double-press confirmation (used by /exit command).""" + self._do_exit() + + def _do_exit(self) -> None: + """Clean up channels and exit.""" + if self._channel_timer is not None: + self._channel_timer.stop() + self._channel_timer = None + self._started_channel_types.clear() + if _channels_is_running(): + try: + _channels_stop() + except Exception: + pass + self.exit() + + def action_request_quit(self) -> None: + if self._busy: + self._quit_pending = False + # Clear all queued messages on interrupt + if self._queued_messages: + self._queued_messages.clear() + self._render_queue_indicator() + if self._run_task is not None and not self._run_task.done(): + self._run_task.cancel() + else: + # Edge case: busy but no task — force reset + self._busy = False + self.query_one("#prompt", ChatTextArea).focus() + self._render_status() + self._append_system( + "\nInterrupted by user", style="dim italic #ffe082" + ) + return + # Double Ctrl+C to quit + if self._quit_pending: + self._do_exit() + else: + self._arm_quit_pending("Ctrl+C") + + # ── Banner & status ──────────────────────────────────── + + async def _refresh_status_snapshot( + self, + pending_user_text: str | None = None, + *, + reset_streaming_text: bool = True, + ) -> None: + """Recompute persistent status metrics for the active thread.""" + pending = (pending_user_text or "").strip() + if pending: + if self._status_last_input_tokens is not None: + self._status_base_snapshot = apply_user_text_to_snapshot( + make_usage_status_snapshot( + self._status_last_input_tokens, + model_name=model, + ), + pending, + ) + else: + self._status_base_snapshot = await build_session_status_snapshot( + self._conversation_tid, + model_name=model, + pending_user_text=pending, + ) + elif self._status_last_input_tokens is not None: + self._status_base_snapshot = make_usage_status_snapshot( + self._status_last_input_tokens, + model_name=model, + ) + else: + self._status_base_snapshot = await build_session_status_snapshot( + self._conversation_tid, + model_name=model, + ) + if reset_streaming_text: + self._status_streaming_text = "" + self._rebuild_status_snapshot() + + def _set_status_usage_baseline(self, input_tokens: int) -> None: + """Promote the latest real prompt usage into the status-bar base.""" + if input_tokens <= 0: + return + self._status_last_input_tokens = input_tokens + self._status_base_snapshot = make_usage_status_snapshot( + input_tokens, + model_name=model, + ) + self._rebuild_status_snapshot() + + def update_status_after_compact(self, tokens_after: int) -> None: + """Update the status bar immediately after a successful /compact. + + Called by CompactCommand so the bar reflects the reduced context + without waiting for the next LLM call. + """ + if tokens_after <= 0: + return + self._status_last_input_tokens = tokens_after + self._status_base_snapshot = make_usage_status_snapshot( + tokens_after, + model_name=model, + ) + self._rebuild_status_snapshot() + + def _set_status_streaming_text(self, text: str | None) -> None: + """Update in-flight assistant text shown in the context bar.""" + new_text = text or "" + if new_text == self._status_streaming_text: + return + self._status_streaming_text = new_text + self._rebuild_status_snapshot() + + def _rebuild_status_snapshot(self) -> None: + """Compose the displayed snapshot from base state + live overlay.""" + self._status_snapshot = apply_assistant_text_to_snapshot( + self._status_base_snapshot, + self._status_streaming_text, + ) + self._render_status() + + def _render_welcome(self) -> None: + channels_info: list[tuple[str, bool, str]] | None = None + try: + running = _channels_running_list() + started = self._started_channel_types + if running or started: + all_types = list(dict.fromkeys(running + started)) + channels_info = [(ct, True, "connected (bus)") for ct in all_types] + else: + from ..config import load_config + + cfg = load_config() + if cfg and cfg.channel_enabled: + types = [ + t.strip() + for t in cfg.channel_enabled.split(",") + if t.strip() + ] + if types: + channels_info = [(ct, False, "configured") for ct in types] + except Exception: + pass + + welcome = self.query_one("#welcome", Static) + welcome.update( + _build_welcome_banner( + thread_id=self._conversation_tid, + workspace_dir=self._workspace_dir, + mode=mode, + model=model, + provider=provider, + ui_backend="tui", + channels=channels_info, + ) + ) + + def _render_status(self) -> None: + status = self.query_one("#status", Static) + width = ( + getattr(status.size, "width", 0) + or getattr(status.content_region, "width", 0) + or getattr(self.screen.size, "width", 0) + or 80 + ) + if self._busy: + hint_label = "vibe researching..." + hint_style = f"on {STATUS_BAR_BG} {STATUS_HINT_BUSY} bold" + else: + hint_label = "/help for commands" + hint_style = f"on {STATUS_BAR_BG} {STATUS_HINT_IDLE}" + + hint = Text.assemble( + (hint_label, hint_style), + (" │ ", f"on {STATUS_BAR_BG} {STATUS_DIM}"), + ) + remaining_width = max(1, width - len(hint.plain)) + metrics = build_status_text( + self._status_snapshot, + self._status_started_at, + remaining_width, + ) + line = Text(no_wrap=True, overflow="crop") + line.append_text(hint) + line.append_text(metrics) + status.update(line) + + # ── Media forwarding helper (module-level) ────────────── + + _MEDIA_EXTENSIONS = { + ".png", + ".jpg", + ".jpeg", + ".gif", + ".bmp", + ".webp", + ".svg", + ".pdf", + ".mp3", + ".wav", + ".mp4", + } + + def _forward_media_to_channel( + state: StreamState, + tool_name: str, + media_sent: set[str], + send_fn: Any, + ) -> None: + """Check tool calls for media files and forward to channel.""" + import os + + from ..paths import resolve_virtual_path + + arg_key = "path" if tool_name == "write_file" else "file_path" + for tc in reversed(state.tool_calls): + if tc.get("name") == tool_name: + p = tc.get("args", {}).get(arg_key, "") + if not p: + p = tc.get("args", {}).get("path", "") + if p and p not in media_sent: + ext = os.path.splitext(p)[1].lower() + if ext in _MEDIA_EXTENSIONS: + real_path = str(resolve_virtual_path(p)) + if not os.path.isfile(real_path) and os.path.isfile(p): + real_path = p + if os.path.isfile(real_path): + media_sent.add(p) + send_fn(real_path) + break + + # ── Entry point ───────────────────────────────────────── + + async def _amain() -> None: + async with get_checkpointer() as checkpointer: + effective_workspace = workspace_dir + effective_thread_id = thread_id + resumed = False + resume_warning = "" + if thread_id: + if await thread_exists(thread_id): + resolved = thread_id + else: + similar = await find_similar_threads(thread_id) + resolved = similar[0] if len(similar) == 1 else None + if resolved: + meta = await get_thread_metadata(resolved) + ws = (meta or {}).get("workspace_dir", "") + if ws: + effective_workspace = ws + effective_thread_id = resolved + resumed = True + else: + resume_warning = ( + f"Thread '{thread_id}' not found. Starting new session." + ) + if not effective_thread_id: + effective_thread_id = generate_thread_id() + + initial_agent = load_agent( + workspace_dir=effective_workspace, + checkpointer=checkpointer, + ) + app = EvoTextualInteractiveApp( + agent=initial_agent, + thread_id_value=effective_thread_id, + workspace=effective_workspace, + checkpointer=checkpointer, + channel_send_thinking_value=channel_send_thinking, + resumed=resumed, + resume_warning=resume_warning, + ) + await app.run_async() + + import nest_asyncio # type: ignore[import-untyped] + + nest_asyncio.apply() + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(_amain()) diff --git a/EvoScientist/cli/tui_runtime.py b/EvoScientist/cli/tui_runtime.py new file mode 100644 index 0000000..b38fd7e --- /dev/null +++ b/EvoScientist/cli/tui_runtime.py @@ -0,0 +1,117 @@ +"""Runtime selection for streaming UI backends.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from ..stream.display import console +from .tui_backends import RichStreamingBackend, StreamingTUIBackend + +DEFAULT_UI_BACKEND = "cli" +SUPPORTED_UI_BACKENDS = ("cli", "tui") +_LEGACY_BACKEND_MAP = {"textual": "tui", "rich": "cli"} + + +def normalize_ui_backend(value: str | None) -> str: + """Normalize user-provided backend name with a safe default.""" + if not value: + return DEFAULT_UI_BACKEND + normalized = value.strip().lower() + normalized = _LEGACY_BACKEND_MAP.get(normalized, normalized) + if normalized in SUPPORTED_UI_BACKENDS: + return normalized + return DEFAULT_UI_BACKEND + + +def _has_textual_support() -> bool: + try: + import textual # noqa: F401 + + return True + except Exception: + return False + + +def resolve_ui_backend(value: str | None, *, warn_fallback: bool = False) -> str: + """Resolve requested backend and fallback safely when unavailable.""" + requested = normalize_ui_backend(value) + if requested == "tui" and not _has_textual_support(): + if warn_fallback: + console.print( + "[yellow]TUI is unavailable (missing textual package). " + "Falling back to CLI.[/yellow]" + ) + return DEFAULT_UI_BACKEND + return requested + + +def get_backend( + name: str | None, *, warn_fallback: bool = False +) -> StreamingTUIBackend: + """Instantiate a streaming backend by name. + + Note: The Textual TUI is now a full interactive app (tui_interactive.py), + not a streaming backend. The streaming backend is always Rich. + """ + resolve_ui_backend(name, warn_fallback=warn_fallback) + return RichStreamingBackend() + + +def run_streaming( + *, + ui_backend: str | None, + agent: Any, + message: str, + thread_id: str, + show_thinking: bool, + interactive: bool, + on_thinking: Callable[[str], None] | None = None, + on_todo: Callable[[list[dict]], None] | None = None, + on_file_write: Callable[[str], None] | None = None, + on_stream_event: Callable[[str, Any], Any] | None = None, + status_footer_builder: Callable[[], Any] | None = None, + metadata: dict | None = None, + hitl_prompt_fn: Callable[[list], list[dict] | None] | None = None, + ask_user_prompt_fn: Callable[[dict], dict] | None = None, +) -> str: + """Run streaming with the selected backend.""" + backend = get_backend(ui_backend, warn_fallback=True) + try: + return backend.run_streaming( + agent=agent, + message=message, + thread_id=thread_id, + show_thinking=show_thinking, + interactive=interactive, + on_thinking=on_thinking, + on_todo=on_todo, + on_file_write=on_file_write, + on_stream_event=on_stream_event, + status_footer_builder=status_footer_builder, + metadata=metadata, + hitl_prompt_fn=hitl_prompt_fn, + ask_user_prompt_fn=ask_user_prompt_fn, + ) + except RuntimeError: + requested = normalize_ui_backend(ui_backend) + if requested == "tui": + console.print( + "[yellow]TUI failed at runtime. Falling back to CLI for this request.[/yellow]" + ) + return RichStreamingBackend().run_streaming( + agent=agent, + message=message, + thread_id=thread_id, + show_thinking=show_thinking, + interactive=interactive, + on_thinking=on_thinking, + on_todo=on_todo, + on_file_write=on_file_write, + on_stream_event=on_stream_event, + status_footer_builder=status_footer_builder, + metadata=metadata, + hitl_prompt_fn=hitl_prompt_fn, + ask_user_prompt_fn=ask_user_prompt_fn, + ) + raise diff --git a/EvoScientist/cli/widgets/__init__.py b/EvoScientist/cli/widgets/__init__.py new file mode 100644 index 0000000..8249d36 --- /dev/null +++ b/EvoScientist/cli/widgets/__init__.py @@ -0,0 +1,35 @@ +"""TUI widgets for EvoScientist Textual interface.""" + +from .approval_widget import ApprovalWidget +from .ask_user_widget import AskUserWidget +from .assistant_message import AssistantMessage +from .compact_summary_widget import CompactSummaryWidget +from .compacting_widget import CompactingWidget +from .loading_widget import LoadingWidget +from .subagent_widget import SubAgentWidget +from .summarization_widget import SummarizationWidget +from .system_message import SystemMessage +from .thinking_widget import ThinkingWidget +from .thread_selector import ThreadPickerWidget +from .todo_widget import TodoWidget +from .tool_call_widget import ToolCallWidget +from .usage_widget import UsageWidget +from .user_message import UserMessage + +__all__ = [ + "ApprovalWidget", + "AskUserWidget", + "AssistantMessage", + "CompactSummaryWidget", + "CompactingWidget", + "LoadingWidget", + "SubAgentWidget", + "SummarizationWidget", + "SystemMessage", + "ThinkingWidget", + "ThreadPickerWidget", + "TodoWidget", + "ToolCallWidget", + "UsageWidget", + "UserMessage", +] diff --git a/EvoScientist/cli/widgets/approval_widget.py b/EvoScientist/cli/widgets/approval_widget.py new file mode 100644 index 0000000..b7e574e --- /dev/null +++ b/EvoScientist/cli/widgets/approval_widget.py @@ -0,0 +1,225 @@ +"""Textual widget for HITL (Human-in-the-Loop) approval prompts. + +Follows DeepAgents CLI pattern: keyboard-driven menu with Static text options, +not Button widgets. Compact height: auto layout. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + +from textual.binding import Binding, BindingType +from textual.containers import Container # type: ignore[import-untyped] +from textual.message import Message # type: ignore[import-untyped] +from textual.widget import Widget # type: ignore[import-untyped] +from textual.widgets import Static # type: ignore[import-untyped] + +if TYPE_CHECKING: + from textual import events + from textual.app import ComposeResult + +# Max length for truncated shell command display +_COMMAND_TRUNCATE_LENGTH: int = 120 + + +class ApprovalWidget(Widget): + """Widget that displays pending tool approvals and collects user decisions. + + Keyboard-driven: y/n/a or 1/2/3 quick keys, arrow keys to navigate, + Enter to confirm. Posts ``ApprovalWidget.Decided`` when user chooses. + """ + + can_focus = True + can_focus_children = False + + DEFAULT_CSS = """ + ApprovalWidget { + height: auto; + max-height: 12; + margin: 1 0; + padding: 0 1; + background: $surface; + border: solid $warning; + } + ApprovalWidget .approval-title { + height: 1; + text-style: bold; + color: $warning; + } + ApprovalWidget .approval-command { + height: 1; + margin: 0 0 0 2; + } + ApprovalWidget .approval-options { + height: auto; + } + ApprovalWidget .approval-option { + height: 1; + padding: 0 1; + } + ApprovalWidget .approval-option-selected { + background: $primary; + text-style: bold; + } + ApprovalWidget .approval-help { + height: 1; + color: $text-muted; + text-style: italic; + } + """ + + BINDINGS: ClassVar[list[BindingType]] = [ + Binding("up", "move_up", "Up", show=False), + Binding("k", "move_up", "Up", show=False), + Binding("down", "move_down", "Down", show=False), + Binding("j", "move_down", "Down", show=False), + Binding("enter", "select", "Select", show=False), + Binding("1", "select_approve", "Approve", show=False), + Binding("y", "select_approve", "Approve", show=False), + Binding("2", "select_reject", "Reject", show=False), + Binding("n", "select_reject", "Reject", show=False), + Binding("3", "select_auto", "Auto-approve", show=False), + Binding("a", "select_auto", "Auto-approve", show=False), + Binding("escape", "select_reject", "Reject", show=False), + ] + + class Decided(Message): + """Posted when the user approves, rejects, or auto-approves.""" + + def __init__( + self, + decisions: list[dict[str, Any]] | None, + auto_approve_session: bool = False, + ) -> None: + super().__init__() + self.decisions = decisions + self.auto_approve_session = auto_approve_session + + def __init__(self, action_requests: list, **kwargs) -> None: + super().__init__(**kwargs) + self._action_requests = action_requests + self._selected = 0 + self._option_widgets: list[Static] = [] + + def compose(self) -> ComposeResult: + self._option_widgets = [] + count = len(self._action_requests) + if count == 1: + name = ( + self._action_requests[0].get("name", "") + if isinstance(self._action_requests[0], dict) + else getattr(self._action_requests[0], "name", "") + ) + title = f">>> {name} Requires Approval <<<" + else: + title = f">>> {count} Tool Calls Require Approval <<<" + yield Static(title, classes="approval-title") + + # Show each action request as a compact line + for req in self._action_requests: + name = ( + req.get("name", "") + if isinstance(req, dict) + else getattr(req, "name", "") + ) + args = ( + req.get("args", {}) + if isinstance(req, dict) + else getattr(req, "args", {}) + ) + if isinstance(args, dict): + command = args.get("command", args.get("path", "")) + else: + command = "" + if command: + cmd_str = str(command) + if len(cmd_str) > _COMMAND_TRUNCATE_LENGTH: + cmd_str = cmd_str[:_COMMAND_TRUNCATE_LENGTH] + "..." + label = f"[bold #f59e0b]{cmd_str}[/bold #f59e0b]" + else: + label = f"[bold]{name}[/bold]" + yield Static(label, classes="approval-command") + + # Options as Static text (not buttons) + with Container(classes="approval-options"): + for _ in range(3): + widget = Static("", classes="approval-option") + self._option_widgets.append(widget) + yield widget + + yield Static( + "↑/↓ navigate · Enter select · y/n/a quick keys · Esc reject", + classes="approval-help", + ) + + def on_mount(self) -> None: + self._update_options() + self.focus() + + def _update_options(self) -> None: + n = len(self._action_requests) + if n == 1: + options = [ + "1. Approve (y)", + "2. Reject (n)", + "3. Auto-approve for this session (a)", + ] + else: + options = [ + f"1. Approve all {n} (y)", + f"2. Reject all {n} (n)", + "3. Auto-approve for this session (a)", + ] + + for i, (text, widget) in enumerate( + zip(options, self._option_widgets, strict=True) + ): + cursor = "▸ " if i == self._selected else " " + widget.update(f"{cursor}{text}") + widget.remove_class("approval-option-selected") + if i == self._selected: + widget.add_class("approval-option-selected") + + def action_move_up(self) -> None: + self._selected = (self._selected - 1) % 3 + self._update_options() + + def action_move_down(self) -> None: + self._selected = (self._selected + 1) % 3 + self._update_options() + + def action_select(self) -> None: + self._handle_selection(self._selected) + + def action_select_approve(self) -> None: + self._selected = 0 + self._update_options() + self._handle_selection(0) + + def action_select_reject(self) -> None: + self._selected = 1 + self._update_options() + self._handle_selection(1) + + def action_select_auto(self) -> None: + self._selected = 2 + self._update_options() + self._handle_selection(2) + + def _handle_selection(self, option: int) -> None: + n = len(self._action_requests) or 1 + if option == 0: + self.post_message(self.Decided([{"type": "approve"} for _ in range(n)])) + elif option == 2: + self.post_message( + self.Decided( + [{"type": "approve"} for _ in range(n)], + auto_approve_session=True, + ) + ) + else: + self.post_message(self.Decided(None)) + + def on_blur(self, event: events.Blur) -> None: + """Re-focus to keep focus trapped until decision is made.""" + self.call_after_refresh(self.focus) diff --git a/EvoScientist/cli/widgets/ask_user_widget.py b/EvoScientist/cli/widgets/ask_user_widget.py new file mode 100644 index 0000000..d70df61 --- /dev/null +++ b/EvoScientist/cli/widgets/ask_user_widget.py @@ -0,0 +1,377 @@ +"""Interactive ask_user widget for Textual TUI. + +Shows one question at a time with a progress indicator. Keyboard-driven +like ApprovalWidget: all bindings on the top-level widget, compact layout. + +The widget is self-contained: the main ``#prompt`` Input should be +**disabled** while this widget is mounted so it cannot steal focus. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING, Any, ClassVar, Literal + +from rich.markup import escape as escape_markup +from textual.binding import Binding, BindingType +from textual.message import Message +from textual.widget import Widget +from textual.widgets import Input, Static + +if TYPE_CHECKING: + from textual import events + from textual.app import ComposeResult + +logger = logging.getLogger(__name__) + +OTHER_CHOICE_LABEL = "Other (type your answer)" +_CURSOR = "▸" + + +class AskUserWidget(Widget): + """Interactive widget for asking the user questions one at a time. + + Extends Widget (not Container) to avoid built-in scroll behavior + that captures arrow keys. Same pattern as ApprovalWidget. + """ + + can_focus = True + can_focus_children = True # needed for text Input + + BINDINGS: ClassVar[list[BindingType]] = [ + Binding("up", "move_up", "Up", show=False), + Binding("k", "move_up", "Up", show=False), + Binding("down", "move_down", "Down", show=False), + Binding("j", "move_down", "Down", show=False), + Binding("enter", "confirm", "Confirm", show=False), + Binding("escape", "cancel", "Cancel", show=False), + ] + + DEFAULT_CSS = """ + AskUserWidget { + height: auto; + margin: 1 0; + padding: 0 1; + background: $surface; + border: solid $success; + } + AskUserWidget .ask-title { + height: 1; + text-style: bold; + color: $success; + } + AskUserWidget .ask-question-text { + height: auto; + margin: 0 0 0 1; + text-style: bold; + } + AskUserWidget .ask-choice { + height: 1; + padding: 0 2; + margin: 0; + } + AskUserWidget .ask-choice-selected { + background: $primary; + text-style: bold; + } + AskUserWidget .ask-text-input { + height: auto; + margin: 0 2; + } + AskUserWidget .ask-help { + height: 1; + color: $text-muted; + text-style: italic; + margin: 0; + } + """ + + class Answered(Message): + """Posted when the user submits all answers.""" + + def __init__(self, answers: list[str]) -> None: + super().__init__() + self.answers = answers + + class Cancelled(Message): + """Posted when the user cancels the ask_user prompt.""" + + def __init__(self) -> None: + super().__init__() + + def __init__( + self, + questions: list[dict], + id: str | None = None, + **kwargs: Any, + ) -> None: + super().__init__(id=id or "ask-user-widget", **kwargs) + self._questions = questions + self._answers: list[str] = [] + self._current_index = 0 + self._future: asyncio.Future | None = None + self._submitted = False + + # Current question state + self._q_type: Literal["text", "multiple_choice"] = "text" + self._choices: list[dict] = [] + self._required: bool = True + self._selected_choice: int = 0 + self._is_other: bool = False + + # Widgets (composed once, updated per question) + self._title_w: Static | None = None + self._question_w: Static | None = None + self._choice_widgets: list[Static] = [] + self._text_input: Input | None = None + self._other_input: Input | None = None + self._help_w: Static | None = None + + def set_future(self, future: asyncio.Future) -> None: + """Set the future to resolve when user answers.""" + self._future = future + + def compose(self) -> ComposeResult: + total = len(self._questions) + if total == 1: + title = ">>> Quick check-in from EvoScientist <<<" + else: + title = f">>> Question 1/{total} — Quick check-in from EvoScientist <<<" + + self._title_w = Static(title, classes="ask-title") + yield self._title_w + + self._question_w = Static("", classes="ask-question-text") + yield self._question_w + + # Pre-create max choice slots (choices + Other = up to ~10) + # Unused slots stay hidden. + self._choice_widgets = [] + for _ in range(12): + cw = Static("", classes="ask-choice") + cw.display = False + self._choice_widgets.append(cw) + yield cw + + self._text_input = Input( + placeholder="Type your answer...", + classes="ask-text-input", + ) + self._text_input.display = False + yield self._text_input + + self._other_input = Input( + placeholder="Type your answer...", + classes="ask-text-input", + ) + self._other_input.display = False + yield self._other_input + + self._help_w = Static("", classes="ask-help") + yield self._help_w + + async def on_mount(self) -> None: + self._show_question(0) + + def focus_active(self) -> None: + """Focus the appropriate element for the current question.""" + if self._q_type == "text": + if self._text_input: + self._text_input.focus() + elif self._is_other: + if self._other_input: + self._other_input.focus() + else: + self.focus() + + # ------------------------------------------------------------------ + # Show a question + # ------------------------------------------------------------------ + + def _show_question(self, index: int) -> None: + """Populate widgets for question at *index*.""" + q = self._questions[index] + q_text = q.get("question", "") + q_type = q.get("type", "text") + self._choices = q.get("choices", []) + self._required = q.get("required", True) + self._q_type = "multiple_choice" if q_type == "multiple_choice" else "text" + self._selected_choice = 0 + self._is_other = False + + # Title + total = len(self._questions) + if self._title_w: + if total == 1: + self._title_w.update(">>> Quick check-in from EvoScientist <<<") + else: + self._title_w.update( + f">>> Question {index + 1}/{total}" + " — Quick check-in from EvoScientist <<<" + ) + + # Question text + suffix = ( + " [dim](required)[/dim]" if self._required else " [dim](optional)[/dim]" + ) + if self._question_w: + self._question_w.update( + f"[bold]{index + 1}. {escape_markup(q_text)}[/bold]{suffix}" + ) + + # Reset all choice slots + for cw in self._choice_widgets: + cw.display = False + cw.remove_class("ask-choice-selected") + + if self._text_input: + self._text_input.display = False + self._text_input.value = "" + if self._other_input: + self._other_input.display = False + self._other_input.value = "" + + if self._q_type == "multiple_choice" and self._choices: + # Show choice options + Other + for i, choice in enumerate(self._choices): + if i < len(self._choice_widgets): + label = escape_markup(choice.get("value", str(choice))) + cursor = f"{_CURSOR} " if i == 0 else " " + self._choice_widgets[i].update(f"{cursor}{label}") + self._choice_widgets[i].display = True + if i == 0: + self._choice_widgets[i].add_class("ask-choice-selected") + + other_idx = len(self._choices) + if other_idx < len(self._choice_widgets): + self._choice_widgets[other_idx].update(f" {OTHER_CHOICE_LABEL}") + self._choice_widgets[other_idx].display = True + + # Help text + if self._help_w: + self._help_w.update("↑/↓ select · Enter confirm · Esc cancel") + self.focus() + else: + # Text input + if self._text_input: + self._text_input.display = True + self._text_input.focus() + if self._help_w: + self._help_w.update("Enter confirm · Esc cancel") + + # ------------------------------------------------------------------ + # Key bindings + # ------------------------------------------------------------------ + + def action_move_up(self) -> None: + if self._q_type != "multiple_choice": + return + if self._is_other and self._other_input and self._other_input.has_focus: + # Jump back from Other input to choice list + self._is_other = False + if self._other_input: + self._other_input.display = False + self._selected_choice = len(self._choices) # stay on Other option + self._update_choices() + self.focus() + return + total_opts = len(self._choices) + 1 # choices + Other + self._selected_choice = (self._selected_choice - 1) % total_opts + self._update_choices() + + def action_move_down(self) -> None: + if self._q_type != "multiple_choice": + return + total_opts = len(self._choices) + 1 + self._selected_choice = (self._selected_choice + 1) % total_opts + self._update_choices() + + def action_confirm(self) -> None: + if self._q_type == "multiple_choice": + is_other = self._selected_choice == len(self._choices) + if is_other and not self._is_other: + # Show Other text input + self._is_other = True + if self._other_input: + self._other_input.display = True + self._other_input.focus() + return + if is_other and self._is_other: + # Submit Other answer + answer = self._other_input.value if self._other_input else "" + if answer.strip() or not self._required: + self._advance(answer) + return + # Regular choice + if self._selected_choice < len(self._choices): + answer = self._choices[self._selected_choice].get("value", "") + self._advance(answer) + else: + # Text question — Enter on widget (not Input) acts as confirm + answer = self._text_input.value if self._text_input else "" + if answer.strip() or not self._required: + self._advance(answer) + + def on_input_submitted(self, event: Input.Submitted) -> None: + """Handle Enter in text Input widgets.""" + event.stop() + if event.input is self._text_input: + answer = self._text_input.value if self._text_input else "" + if answer.strip() or not self._required: + self._advance(answer) + elif event.input is self._other_input: + answer = self._other_input.value if self._other_input else "" + if answer.strip() or not self._required: + self._advance(answer) + + def action_cancel(self) -> None: + if self._submitted: + return + self._submitted = True + if self._future and not self._future.done(): + self._future.set_result({"type": "cancelled"}) + self.post_message(self.Cancelled()) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _update_choices(self) -> None: + """Update choice display to reflect current selection.""" + total_opts = len(self._choices) + 1 + for i in range(total_opts): + if i >= len(self._choice_widgets): + break + if i < len(self._choices): + label = escape_markup(self._choices[i].get("value", "")) + else: + label = OTHER_CHOICE_LABEL + cursor = f"{_CURSOR} " if i == self._selected_choice else " " + self._choice_widgets[i].update(f"{cursor}{label}") + self._choice_widgets[i].remove_class("ask-choice-selected") + if i == self._selected_choice: + self._choice_widgets[i].add_class("ask-choice-selected") + + def _advance(self, answer: str) -> None: + """Record answer, show next question or submit.""" + self._answers.append(answer) + self._current_index += 1 + + if self._current_index >= len(self._questions): + self._submit() + return + + self._show_question(self._current_index) + + def _submit(self) -> None: + if self._submitted: + return + self._submitted = True + if self._future and not self._future.done(): + self._future.set_result({"type": "answered", "answers": self._answers}) + self.post_message(self.Answered(self._answers)) + + def on_blur(self, event: events.Blur) -> None: + """Prevent blur from propagating and dismissing the widget.""" + event.stop() diff --git a/EvoScientist/cli/widgets/assistant_message.py b/EvoScientist/cli/widgets/assistant_message.py new file mode 100644 index 0000000..e56e8d7 --- /dev/null +++ b/EvoScientist/cli/widgets/assistant_message.py @@ -0,0 +1,61 @@ +"""Assistant message widget with incremental Markdown rendering.""" + +from __future__ import annotations + +from textual.containers import Vertical +from textual.widgets import Markdown + +from .timestamp_mixin import TimestampClickMixin + + +class AssistantMessage(TimestampClickMixin, Vertical): + """Displays the assistant's final Markdown response. + + Mount once, then call :meth:`append_content` for each text chunk. + When streaming finishes, call :meth:`stop_stream`. + + Each ``append_content`` call re-renders only *this* widget's Markdown — + not the entire chat history — which is the core improvement over the + old "rebuild Rich Group every 100 ms" approach. + """ + + DEFAULT_CSS = """ + AssistantMessage { + height: auto; + margin: 1 0 0 0; + } + AssistantMessage Markdown { + margin: 0; + padding: 0; + } + """ + + def __init__(self, initial_content: str = "") -> None: + super().__init__() + self._content = initial_content + self._flush_pending = False + + def compose(self): + yield Markdown("") + + def on_mount(self) -> None: + if self._content: + self.query_one(Markdown).update(self._content) + + async def append_content(self, text: str) -> None: + """Append text and schedule a debounced Markdown re-render.""" + self._content += text + if not self._flush_pending: + self._flush_pending = True + self.set_timer(0.1, self._flush_markdown) + + def _flush_markdown(self) -> None: + """Flush accumulated content to the Markdown widget.""" + self._flush_pending = False + self.query_one(Markdown).update(self._content) + + async def stop_stream(self) -> None: + """Finalize the stream — ensure final content is rendered.""" + self._flush_pending = False + if self._content: + self.query_one(Markdown).update(self._content) diff --git a/EvoScientist/cli/widgets/chat_input.py b/EvoScientist/cli/widgets/chat_input.py new file mode 100644 index 0000000..8dd83b6 --- /dev/null +++ b/EvoScientist/cli/widgets/chat_input.py @@ -0,0 +1,115 @@ +"""Multi-line chat input widget with Enter-to-submit and modifier+Enter newline.""" + +from __future__ import annotations + +import sys +from collections.abc import Callable +from typing import ClassVar + +from textual import events +from textual.binding import Binding +from textual.message import Message +from textual.widgets import TextArea + + +class ChatTextArea(TextArea): + """TextArea that submits on Enter and inserts newline on modifier+Enter. + + Emits :class:`ChatTextArea.Submitted` when the user presses Enter + with non-empty text. Modifier+Enter (Option+Enter on macOS, + Ctrl+J everywhere) inserts a literal newline instead. + + An optional *before_submit* callback can be set to intercept Enter + before submission. If it returns ``True`` the submit is suppressed + (the callback handled the event itself). + """ + + BINDINGS: ClassVar[list[Binding]] = [ + Binding( + "shift+enter,ctrl+j,alt+enter,ctrl+enter", + "insert_newline", + "New Line", + show=False, + priority=True, + ), + ] + + DEFAULT_CSS = """ + ChatTextArea { + height: auto; + min-height: 1; + max-height: 8; + border: none; + padding: 0; + background: transparent; + } + ChatTextArea:focus { + border: none; + } + ChatTextArea .text-area--cursor-line { + background: transparent; + } + """ + + class Submitted(Message): + """Posted when the user presses Enter to submit.""" + + def __init__(self, value: str) -> None: + super().__init__() + self.value = value + + def __init__( + self, + *, + placeholder: str = "", + id: str | None = None, + before_submit: Callable[[], bool] | None = None, + ) -> None: + super().__init__( + id=id, + language=None, + show_line_numbers=False, + soft_wrap=True, + ) + self._placeholder = placeholder + self.before_submit: Callable[[], bool] | None = before_submit + + @property + def value(self) -> str: + """Get the current text content.""" + return self.text + + @value.setter + def value(self, new_value: str) -> None: + """Set the text content (clears and replaces).""" + self.clear() + if new_value: + self.insert(new_value) + + def action_insert_newline(self) -> None: + """Insert a literal newline character.""" + self.insert("\n") + + async def _on_key(self, event: events.Key) -> None: + """Handle Enter as submit. + + If *before_submit* is set and returns ``True``, the submit is + suppressed (the callback already handled the Enter press, e.g. + to apply a completion selection). + """ + if event.key == "enter": + event.prevent_default() + event.stop() + # Let the host intercept Enter (e.g. for completion selection) + if self.before_submit and self.before_submit(): + return + value = self.text.strip() + if value: + self.post_message(self.Submitted(value)) + return + await super()._on_key(event) + + @staticmethod + def newline_shortcut_label() -> str: + """Return the platform-native label for the newline shortcut.""" + return "Option+Enter" if sys.platform == "darwin" else "Ctrl+J" diff --git a/EvoScientist/cli/widgets/compact_summary_widget.py b/EvoScientist/cli/widgets/compact_summary_widget.py new file mode 100644 index 0000000..60509d5 --- /dev/null +++ b/EvoScientist/cli/widgets/compact_summary_widget.py @@ -0,0 +1,72 @@ +"""Collapsible widget for manual /compact summary results.""" + +from __future__ import annotations + +from rich.panel import Panel +from rich.text import Text +from textual.events import Click +from textual.widgets import Static + +_MAX_COLLAPSED_CHARS = 80 +_MAX_EXPANDED_CHARS = 3000 + + +class CompactSummaryWidget(Static): + """Collapsible panel showing the generated manual compact summary.""" + + DEFAULT_CSS = """ + CompactSummaryWidget { + height: auto; + margin: 0 0 1 0; + } + """ + + def __init__(self, summary_text: str) -> None: + super().__init__("") + self._content = (summary_text or "").strip() + self._collapsed = True + self._refresh_display() + + def _char_count_label(self) -> str: + n = len(self._content) + if n >= 1000: + return f"{n / 1000:.1f}k chars" + return f"{n:,} chars" + + def _refresh_display(self) -> None: + if not self._content: + self.update( + Panel( + Text("(empty summary)", style="dim"), + title="Context Compacted", + border_style="#f59e0b", + padding=(0, 1), + ) + ) + return + + if self._collapsed: + title = f"Context Compacted ({self._char_count_label()})" + first_line = self._content.strip().split("\n")[0].strip() + if len(first_line) > _MAX_COLLAPSED_CHARS: + first_line = first_line[: _MAX_COLLAPSED_CHARS - 3] + "..." + preview = Text(first_line, style="dim italic") + preview.append(" [click to expand]", style="dim italic") + body = preview + else: + title = f"Context Compacted ({self._char_count_label()})" + display = self._content.rstrip() + if len(display) > _MAX_EXPANDED_CHARS: + half = _MAX_EXPANDED_CHARS // 2 + display = ( + display[:half] + "\n\n... (truncated) ...\n\n" + display[-half:] + ) + body = Text(display, style="dim italic") + + self.update(Panel(body, title=title, border_style="#f59e0b", padding=(0, 1))) + + def on_click(self, event: Click) -> None: + """Toggle collapsed/expanded state.""" + if self._content: + self._collapsed = not self._collapsed + self._refresh_display() diff --git a/EvoScientist/cli/widgets/compacting_widget.py b/EvoScientist/cli/widgets/compacting_widget.py new file mode 100644 index 0000000..3c8a661 --- /dev/null +++ b/EvoScientist/cli/widgets/compacting_widget.py @@ -0,0 +1,30 @@ +"""Transient widget shown while manual /compact is running in the TUI.""" + +from __future__ import annotations + +from .timed_status_widget import TimedStatusWidget + + +class CompactingWidget(TimedStatusWidget): + """Timer-backed status line for an in-progress manual compact.""" + + DEFAULT_CSS = """ + CompactingWidget { + height: auto; + color: #f59e0b; + padding: 0 0; + margin: 0 0 1 0; + } + """ + + def __init__(self) -> None: + super().__init__() + + def _refresh_display(self) -> None: + self.update(f"Compacting conversation... ({self.elapsed_seconds}s)") + + async def cleanup(self) -> None: + """Stop timer and remove from DOM.""" + self._stop_timer() + if self.is_mounted: + await self.remove() diff --git a/EvoScientist/cli/widgets/loading_widget.py b/EvoScientist/cli/widgets/loading_widget.py new file mode 100644 index 0000000..d549055 --- /dev/null +++ b/EvoScientist/cli/widgets/loading_widget.py @@ -0,0 +1,40 @@ +"""Loading spinner widget shown while waiting for the first token.""" + +from __future__ import annotations + +from .timed_status_widget import TimedStatusWidget + +_SPINNER_FRAMES = "\u280b\u2819\u2839\u2838\u283c\u2834\u2826\u2827\u2807\u280f" + + +class LoadingWidget(TimedStatusWidget): + """Spinner + 'Thinking...' with elapsed time counter. + + Mount when a turn starts; call ``remove()`` when the first + thinking/text/tool_call event arrives. + """ + + DEFAULT_CSS = """ + LoadingWidget { + height: auto; + color: #22d3ee; + padding: 0 0; + } + """ + + def __init__(self) -> None: + super().__init__() + self._frame = 0 + + def _tick(self) -> None: + self._frame = (self._frame + 1) % len(_SPINNER_FRAMES) + super()._tick() + + def _refresh_display(self) -> None: + char = _SPINNER_FRAMES[self._frame] + self.update(f"{char} Thinking... ({self.elapsed_seconds}s)") + + async def cleanup(self) -> None: + """Stop timer and remove from DOM.""" + self._stop_timer() + await self.remove() diff --git a/EvoScientist/cli/widgets/mcp_browser.py b/EvoScientist/cli/widgets/mcp_browser.py new file mode 100644 index 0000000..9efda52 --- /dev/null +++ b/EvoScientist/cli/widgets/mcp_browser.py @@ -0,0 +1,331 @@ +"""Inline MCP server browser widget for /install-mcp in TUI. + +Two-phase keyboard-driven widget (mirrors SkillBrowserWidget): + Phase 1 — tag picker (arrow keys + Enter to select, or Esc for all) + Phase 2 — server checkbox (arrow keys to navigate, Space to toggle, Enter to confirm) + +Posts ``MCPBrowserWidget.Confirmed`` with selected MCPServerEntry objects, +or ``MCPBrowserWidget.Cancelled`` on Esc. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + +from rich.text import Text +from textual.binding import Binding, BindingType +from textual.containers import Container +from textual.message import Message +from textual.widget import Widget +from textual.widgets import Static + +if TYPE_CHECKING: + from textual import events + from textual.app import ComposeResult + + from ...mcp.registry import MCPServerEntry + + +class MCPBrowserWidget(Widget): + """Inline MCP server browser — mounts in chat, keyboard-driven. + + Phase 1: Tag picker (select a tag filter or "All"). + Phase 2: Server checkbox (toggle servers, confirm to install). + """ + + can_focus = True + can_focus_children = False + + DEFAULT_CSS = """ + MCPBrowserWidget { + height: auto; + max-height: 30; + margin: 1 0; + padding: 0 1; + background: $surface; + border: solid $primary; + } + MCPBrowserWidget .browser-title { + height: 1; + text-style: bold; + color: $primary; + } + MCPBrowserWidget .browser-rows { + height: auto; + max-height: 20; + overflow-y: auto; + } + MCPBrowserWidget .browser-row { + height: 1; + padding: 0 1; + } + MCPBrowserWidget .browser-row-selected { + background: $primary; + text-style: bold; + } + MCPBrowserWidget .browser-help { + height: 1; + color: $text-muted; + text-style: italic; + } + """ + + BINDINGS: ClassVar[list[BindingType]] = [ + Binding("up", "move_up", "Up", show=False), + Binding("k", "move_up", "Up", show=False), + Binding("down", "move_down", "Down", show=False), + Binding("j", "move_down", "Down", show=False), + Binding("enter", "confirm", "Confirm", show=False), + Binding("space", "toggle", "Toggle", show=False), + Binding("escape", "cancel", "Cancel", show=False), + ] + + class Confirmed(Message): + """Posted when user confirms server selection.""" + + def __init__(self, entries: list[MCPServerEntry]) -> None: + super().__init__() + self.entries = entries + + class Cancelled(Message): + """Posted when user cancels.""" + + def __init__( + self, + servers: list[MCPServerEntry], + installed_names: set[str], + *, + pre_filter_tag: str = "", + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self._servers = servers + self._installed_names = installed_names + self._pre_filter_tag = pre_filter_tag.lower() + self._selected = 0 + self._row_widgets: list[Static] = [] + self._title_widget: Static | None = None + self._help_widget: Static | None = None + + # Phase 1: tag picker + # Phase 2: server checkbox + self._phase: int = 1 + self._tag_items: list[tuple[str, int]] = [] + self._server_items: list[MCPServerEntry] = [] + self._checked: set[int] = set() + + # Build tag list + from collections import Counter + + tag_counter: Counter[str] = Counter() + for s in self._servers: + for t in s.tags: + tag_counter[t.lower()] += 1 + sorted_tags = sorted(tag_counter.items(), key=lambda x: (-x[1], x[0])) + self._tag_items = [("all", len(self._servers)), *sorted_tags] + + # If pre-filtered, skip to phase 2 + if self._pre_filter_tag: + self._server_items = [ + s + for s in self._servers + if self._pre_filter_tag in [t.lower() for t in s.tags] + ] + if self._server_items: + self._phase = 2 + else: + self._pre_filter_tag = "" + + def compose(self) -> ComposeResult: + self._title_widget = Static("", classes="browser-title") + yield self._title_widget + with Container(classes="browser-rows"): + max_rows = max(len(self._tag_items), len(self._servers)) + for _ in range(max_rows): + widget = Static("", classes="browser-row") + self._row_widgets.append(widget) + yield widget + self._help_widget = Static("", classes="browser-help") + yield self._help_widget + + def on_mount(self) -> None: + self.call_after_refresh(self._update_display) + self.call_later(self.focus) + + def _update_display(self) -> None: + if self._phase == 1: + self._render_tag_picker() + else: + self._render_server_checkbox() + + def _render_tag_picker(self) -> None: + if self._title_widget: + self._title_widget.update("Filter by tag:") + if self._help_widget: + self._help_widget.update( + "\u2191/\u2193 navigate \u00b7 Enter select \u00b7 Esc cancel" + ) + + for i, widget in enumerate(self._row_widgets): + if i < len(self._tag_items): + tag, count = self._tag_items[i] + is_selected = i == self._selected + text = Text() + cursor = "\u25b8 " if is_selected else " " + text.append(cursor, style="bold cyan" if is_selected else "dim") + label = f"{tag} ({count})" + text.append(label, style="bold" if is_selected else "") + widget.update(text) + widget.display = True + widget.remove_class("browser-row-selected") + if is_selected: + widget.add_class("browser-row-selected") + widget.scroll_visible() + else: + widget.update("") + widget.display = False + + def _row_content_width(self) -> int: + try: + w = self.size.width + if w > 0: + return w - 6 + except Exception: + pass + try: + return self.app.size.width - 10 + except Exception: + return 100 + + def _truncate(self, desc: str, name: str, *, suffix: str = "") -> str: + overhead = 2 + 2 + len(name) + 3 + len(suffix) + max_len = max(20, self._row_content_width() - overhead) + if len(desc) <= max_len: + return desc + return desc[: max_len - 1] + "\u2026" + + def _render_server_checkbox(self) -> None: + n_checked = len( + [ + i + for i in self._checked + if self._server_items[i].name not in self._installed_names + ] + ) + if self._title_widget: + self._title_widget.update( + f"Select MCP servers to install ({n_checked} selected):" + ) + if self._help_widget: + self._help_widget.update( + "\u2191/\u2193 navigate \u00b7 Space toggle \u00b7 Enter install \u00b7 Esc cancel" + ) + + for i, widget in enumerate(self._row_widgets): + if i < len(self._server_items): + entry = self._server_items[i] + is_selected = i == self._selected + is_installed = entry.name in self._installed_names + is_checked = i in self._checked + + text = Text() + cursor = "\u25b8 " if is_selected else " " + text.append(cursor, style="bold cyan" if is_selected else "dim") + + desc = entry.description or entry.label + + if is_installed: + suffix = " (configured)" + desc = self._truncate(desc, entry.name, suffix=suffix) + text.append("\u2713 ", style="green") + text.append(entry.name, style="green dim") + text.append(f" \u2014 {desc}", style="dim") + text.append(suffix, style="dim italic") + elif is_checked: + desc = self._truncate(desc, entry.name) + text.append("\u25cf ", style="green bold") + text.append(entry.name, style="bold") + text.append(f" \u2014 {desc}", style="") + else: + desc = self._truncate(desc, entry.name) + text.append("\u25cb ", style="dim") + text.append(entry.name, style="bold" if is_selected else "") + text.append(f" \u2014 {desc}", style="dim") + + widget.update(text) + widget.display = True + widget.remove_class("browser-row-selected") + if is_selected: + widget.add_class("browser-row-selected") + widget.scroll_visible() + else: + widget.update("") + widget.display = False + + def _current_items_count(self) -> int: + if self._phase == 1: + return len(self._tag_items) + return len(self._server_items) + + def action_move_up(self) -> None: + n = self._current_items_count() + if not n: + return + self._selected = (self._selected - 1) % n + self._update_display() + + def action_move_down(self) -> None: + n = self._current_items_count() + if not n: + return + self._selected = (self._selected + 1) % n + self._update_display() + + def action_toggle(self) -> None: + if self._phase != 2: + return + if not self._server_items: + return + entry = self._server_items[self._selected] + if entry.name in self._installed_names: + return + if self._selected in self._checked: + self._checked.discard(self._selected) + else: + self._checked.add(self._selected) + self._update_display() + + def action_confirm(self) -> None: + if self._phase == 1: + if not self._tag_items: + return + tag, _ = self._tag_items[self._selected] + if tag == "all": + self._server_items = list(self._servers) + else: + self._server_items = [ + s for s in self._servers if tag in [t.lower() for t in s.tags] + ] + self._phase = 2 + self._selected = 0 + self._checked = set() + self._update_display() + else: + entries = [ + self._server_items[i] + for i in sorted(self._checked) + if self._server_items[i].name not in self._installed_names + ] + self.post_message(self.Confirmed(entries)) + + def action_cancel(self) -> None: + if self._phase == 2 and not self._pre_filter_tag: + self._phase = 1 + self._selected = 0 + self._checked = set() + self._update_display() + else: + self.post_message(self.Cancelled()) + + def on_blur(self, event: events.Blur) -> None: + self.call_after_refresh(self.focus) diff --git a/EvoScientist/cli/widgets/skill_browser.py b/EvoScientist/cli/widgets/skill_browser.py new file mode 100644 index 0000000..e6de017 --- /dev/null +++ b/EvoScientist/cli/widgets/skill_browser.py @@ -0,0 +1,348 @@ +"""Inline skill browser widget for /evoskills in TUI. + +Two-phase keyboard-driven widget: + Phase 1 — tag picker (arrow keys + Enter to select, or Esc for all) + Phase 2 — skill checkbox (arrow keys to navigate, Space to toggle, Enter to confirm) + +Posts ``SkillBrowserWidget.Confirmed`` with selected install sources, +or ``SkillBrowserWidget.Cancelled`` on Esc. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + +from rich.text import Text +from textual.binding import Binding, BindingType +from textual.containers import Container +from textual.message import Message +from textual.widget import Widget +from textual.widgets import Static + +if TYPE_CHECKING: + from textual import events + from textual.app import ComposeResult + + +class SkillBrowserWidget(Widget): + """Inline skill browser — mounts in chat, keyboard-driven. + + Phase 1: Tag picker (select a tag filter or "All"). + Phase 2: Skill checkbox (toggle skills, confirm to install). + """ + + can_focus = True + can_focus_children = False + + DEFAULT_CSS = """ + SkillBrowserWidget { + height: auto; + max-height: 30; + margin: 1 0; + padding: 0 1; + background: $surface; + border: solid $primary; + } + SkillBrowserWidget .browser-title { + height: 1; + text-style: bold; + color: $primary; + } + SkillBrowserWidget .browser-rows { + height: auto; + max-height: 20; + overflow-y: auto; + } + SkillBrowserWidget .browser-row { + height: 1; + padding: 0 1; + } + SkillBrowserWidget .browser-row-selected { + background: $primary; + text-style: bold; + } + SkillBrowserWidget .browser-help { + height: 1; + color: $text-muted; + text-style: italic; + } + """ + + BINDINGS: ClassVar[list[BindingType]] = [ + Binding("up", "move_up", "Up", show=False), + Binding("k", "move_up", "Up", show=False), + Binding("down", "move_down", "Down", show=False), + Binding("j", "move_down", "Down", show=False), + Binding("enter", "confirm", "Confirm", show=False), + Binding("space", "toggle", "Toggle", show=False), + Binding("escape", "cancel", "Cancel", show=False), + ] + + class Confirmed(Message): + """Posted when user confirms skill selection.""" + + def __init__(self, install_sources: list[str]) -> None: + super().__init__() + self.install_sources = install_sources + + class Cancelled(Message): + """Posted when user cancels.""" + + def __init__( + self, + index: list[dict], + installed_names: set[str], + *, + pre_filter_tag: str = "", + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self._index = index + self._installed_names = installed_names + self._pre_filter_tag = pre_filter_tag.lower() + self._selected = 0 + self._row_widgets: list[Static] = [] + self._title_widget: Static | None = None + self._help_widget: Static | None = None + + # Phase 1: tag picker + # Phase 2: skill checkbox + self._phase: int = 1 + self._tag_items: list[tuple[str, int]] = [] # (tag, count) + self._skill_items: list[dict] = [] # filtered skills + self._checked: set[int] = set() # indices of checked skills + + # Build tag list (sorted by count desc, then alphabetically) + from collections import Counter + + tag_counter: Counter[str] = Counter() + for s in self._index: + for t in s.get("tags", []): + tag_counter[t.lower()] += 1 + sorted_tags = sorted(tag_counter.items(), key=lambda x: (-x[1], x[0])) + self._tag_items = [("all", len(self._index)), *sorted_tags] + + # If pre-filtered, skip to phase 2 + if self._pre_filter_tag: + self._skill_items = [ + s + for s in self._index + if self._pre_filter_tag in [t.lower() for t in s.get("tags", [])] + ] + if self._skill_items: + self._phase = 2 + else: + # No matches — show tag picker anyway + self._pre_filter_tag = "" + + def compose(self) -> ComposeResult: + self._title_widget = Static("", classes="browser-title") + yield self._title_widget + with Container(classes="browser-rows"): + # Pre-allocate enough rows for the larger of tag list or skill list + max_rows = max(len(self._tag_items), len(self._index)) + for _ in range(max_rows): + widget = Static("", classes="browser-row") + self._row_widgets.append(widget) + yield widget + self._help_widget = Static("", classes="browser-help") + yield self._help_widget + + def on_mount(self) -> None: + # Defer rendering until after layout so self.size is populated + self.call_after_refresh(self._update_display) + self.call_later(self.focus) + + def _update_display(self) -> None: + if self._phase == 1: + self._render_tag_picker() + else: + self._render_skill_checkbox() + + def _render_tag_picker(self) -> None: + if self._title_widget: + self._title_widget.update("Filter by tag:") + if self._help_widget: + self._help_widget.update("↑/↓ navigate · Enter select · Esc cancel") + + for i, widget in enumerate(self._row_widgets): + if i < len(self._tag_items): + tag, count = self._tag_items[i] + is_selected = i == self._selected + text = Text() + cursor = "▸ " if is_selected else " " + text.append(cursor, style="bold cyan" if is_selected else "dim") + label = f"{tag} ({count})" + text.append(label, style="bold" if is_selected else "") + widget.update(text) + widget.display = True + widget.remove_class("browser-row-selected") + if is_selected: + widget.add_class("browser-row-selected") + widget.scroll_visible() + else: + widget.update("") + widget.display = False + + def _row_content_width(self) -> int: + """Get the usable character width for a row's text content. + + Accounts for widget border, widget padding, and row padding. + Falls back to terminal width if the widget hasn't been laid out yet. + """ + try: + w = self.size.width + if w > 0: + # border (2) + widget padding-left/right (2) + row padding-left/right (2) + return w - 6 + except Exception: + pass + # Fallback: use terminal width minus reasonable chrome + try: + return self.app.size.width - 10 + except Exception: + return 100 + + def _truncate(self, desc: str, name: str, *, suffix: str = "") -> str: + """Truncate a description to fit the row, adding ellipsis if needed.""" + # cursor(2) + indicator(2) + name + " — "(3) + suffix + overhead = 2 + 2 + len(name) + 3 + len(suffix) + max_len = max(20, self._row_content_width() - overhead) + if len(desc) <= max_len: + return desc + return desc[: max_len - 1] + "…" + + def _render_skill_checkbox(self) -> None: + n_checked = len( + [ + i + for i in self._checked + if self._skill_items[i]["name"] not in self._installed_names + ] + ) + if self._title_widget: + self._title_widget.update( + f"Select skills to install ({n_checked} selected):" + ) + if self._help_widget: + self._help_widget.update( + "↑/↓ navigate · Space toggle · Enter install · Esc cancel" + ) + + for i, widget in enumerate(self._row_widgets): + if i < len(self._skill_items): + skill = self._skill_items[i] + is_selected = i == self._selected + is_installed = skill["name"] in self._installed_names + is_checked = i in self._checked + + text = Text() + cursor = "▸ " if is_selected else " " + text.append(cursor, style="bold cyan" if is_selected else "dim") + + if is_installed: + suffix = " (installed)" + desc = self._truncate( + desc=skill["description"], + name=skill["name"], + suffix=suffix, + ) + text.append("✓ ", style="green") + text.append(skill["name"], style="green dim") + text.append(f" — {desc}", style="dim") + text.append(suffix, style="dim italic") + elif is_checked: + desc = self._truncate(skill["description"], skill["name"]) + text.append("● ", style="green bold") + text.append(skill["name"], style="bold") + text.append(f" — {desc}", style="") + else: + desc = self._truncate(skill["description"], skill["name"]) + text.append("○ ", style="dim") + text.append(skill["name"], style="bold" if is_selected else "") + text.append(f" — {desc}", style="dim") + + widget.update(text) + widget.display = True + widget.remove_class("browser-row-selected") + if is_selected: + widget.add_class("browser-row-selected") + widget.scroll_visible() + else: + widget.update("") + widget.display = False + + def _current_items_count(self) -> int: + if self._phase == 1: + return len(self._tag_items) + return len(self._skill_items) + + def action_move_up(self) -> None: + n = self._current_items_count() + if not n: + return + self._selected = (self._selected - 1) % n + self._update_display() + + def action_move_down(self) -> None: + n = self._current_items_count() + if not n: + return + self._selected = (self._selected + 1) % n + self._update_display() + + def action_toggle(self) -> None: + """Toggle skill selection (phase 2 only).""" + if self._phase != 2: + return + if not self._skill_items: + return + skill = self._skill_items[self._selected] + if skill["name"] in self._installed_names: + return # Can't toggle installed skills + if self._selected in self._checked: + self._checked.discard(self._selected) + else: + self._checked.add(self._selected) + self._update_display() + + def action_confirm(self) -> None: + if self._phase == 1: + # Transition to phase 2 + if not self._tag_items: + return + tag, _ = self._tag_items[self._selected] + if tag == "all": + self._skill_items = list(self._index) + else: + self._skill_items = [ + s + for s in self._index + if tag in [t.lower() for t in s.get("tags", [])] + ] + self._phase = 2 + self._selected = 0 + self._checked = set() + self._update_display() + else: + # Confirm selection + sources = [ + self._skill_items[i]["install_source"] + for i in sorted(self._checked) + if self._skill_items[i]["name"] not in self._installed_names + ] + self.post_message(self.Confirmed(sources)) + + def action_cancel(self) -> None: + if self._phase == 2 and not self._pre_filter_tag: + # Go back to tag picker + self._phase = 1 + self._selected = 0 + self._checked = set() + self._update_display() + else: + self.post_message(self.Cancelled()) + + def on_blur(self, event: events.Blur) -> None: + """Re-focus to keep focus trapped until decision is made.""" + self.call_after_refresh(self.focus) diff --git a/EvoScientist/cli/widgets/subagent_widget.py b/EvoScientist/cli/widgets/subagent_widget.py new file mode 100644 index 0000000..46c4e16 --- /dev/null +++ b/EvoScientist/cli/widgets/subagent_widget.py @@ -0,0 +1,306 @@ +"""Sub-agent widget — bordered area with nested tool calls.""" + +from __future__ import annotations + +from rich.text import Text +from textual.containers import Vertical +from textual.widgets import Static + +from .tool_call_widget import ToolCallWidget + +_SPINNER_FRAMES = "\u280b\u2819\u2839\u2838\u283c\u2834\u2826\u2827\u2807\u280f" + +# Keep in sync with display.py MAX_SA_VISIBLE / MAX_SA_RUNNING +_MAX_VISIBLE_COMPLETED = 3 +_MAX_VISIBLE_RUNNING = 2 + + +class SubAgentWidget(Vertical): + """Displays a sub-agent's activity with a bordered frame. + + Active state:: + + ┌ ▶ Cooking with research-agent — Search literature ─┐ + │ ✓ 8 completed │ + │ ● web_search query="LLM attention" │ + │ ✓ 3 results │ + └─────────────────────────────────────────────────────┘ + + Completed state:: + + ✓ Cooking with research-agent (3 tools) + """ + + DEFAULT_CSS = """ + SubAgentWidget { + height: auto; + margin: 0 0; + } + SubAgentWidget .sa-header { + height: auto; + color: #22d3ee; + } + SubAgentWidget .sa-tools { + height: auto; + padding: 0 0 0 2; + } + SubAgentWidget .sa-collapse-summary { + height: auto; + padding: 0 0 0 2; + display: none; + } + SubAgentWidget .sa-collapse-summary.--visible { + display: block; + } + SubAgentWidget .sa-footer { + height: auto; + color: #22d3ee; + } + SubAgentWidget.--completed .sa-header { + color: #4ade80; + } + SubAgentWidget.--completed .sa-footer { + color: #4ade80; + } + """ + + def __init__(self, name: str, description: str = "") -> None: + super().__init__() + self._sa_name = name + self._description = description + self._is_active = True + self._frame = 0 + self._tool_count = 0 + self._timer_handle = None + self._tool_widgets: dict[str, ToolCallWidget] = {} + # Ordered lists to track completed / running tools for collapsing + self._completed_ids: list[str] = [] + self._running_ids: list[str] = [] + + @property + def sa_name(self) -> str: + return self._sa_name + + def update_name(self, name: str, description: str = "") -> None: + """Update the sub-agent display name after resolution.""" + self._sa_name = name + if description: + self._description = description + try: + self._render_header() + except Exception: + pass # Widget may not be mounted yet + + def compose(self): + yield Static("", classes="sa-header") + yield Static("", classes="sa-collapse-summary") + yield Vertical(classes="sa-tools") + yield Static("", classes="sa-footer") + + def on_mount(self) -> None: + self._timer_handle = self.set_interval(0.1, self._tick) + self._render_header() + self._render_footer() + + def _tick(self) -> None: + if self._is_active: + self._frame = (self._frame + 1) % len(_SPINNER_FRAMES) + self._render_header() + + def _display_name(self) -> str: + name = f"Cooking with {self._sa_name}" + if self._description: + desc = self._description.split("\n")[0].strip() + if len(desc) > 50: + desc = desc[:47] + "\u2026" + name += f" \u2014 {desc}" + return name + + def _render_header(self) -> None: + header = self.query_one(".sa-header", Static) + line = Text() + if self._is_active: + char = _SPINNER_FRAMES[self._frame] + line.append( + f"\u250c \u25b6 {self._display_name()} {char}", style="bold cyan" + ) + else: + line.append(f"\u2713 {self._display_name()}", style="bold green") + line.append(f" ({self._tool_count} tools)", style="dim") + header.update(line) + + def _render_footer(self) -> None: + footer = self.query_one(".sa-footer", Static) + if self._is_active: + footer.update(Text("\u2514 running...", style="dim cyan")) + else: + footer.update(Text("")) + + async def add_tool_call( + self, + tool_name: str, + tool_args: dict | None = None, + tool_id: str = "", + ) -> ToolCallWidget: + """Mount a new ToolCallWidget inside this sub-agent. + + If a widget with the same *tool_id* already exists (re-emitted with + updated args during incremental streaming), update it in place instead + of creating a duplicate. + """ + if tool_id and tool_id in self._tool_widgets: + # Re-emitted with updated args — update in place + existing = self._tool_widgets[tool_id] + existing._tool_name = tool_name + existing._tool_args = tool_args or {} + try: + existing._render_header() + except Exception: + pass # Widget may not be mounted yet + return existing + + self._tool_count += 1 + w = ToolCallWidget(tool_name, tool_args, tool_id) + tools_container = self.query_one(".sa-tools", Vertical) + await tools_container.mount(w) + key = tool_id or f"_anon_{self._tool_count}" + self._tool_widgets[key] = w + self._running_ids.append(key) + self._update_visibility() + return w + + def complete_tool( + self, + tool_name: str, + content: str, + success: bool = True, + tool_id: str = "", + ) -> None: + """Update the matching ToolCallWidget with its result.""" + widget = None + matched_key = "" + if tool_id and tool_id in self._tool_widgets: + widget = self._tool_widgets[tool_id] + matched_key = tool_id + else: + # Match by name — find first running tool with this name + for key, w in self._tool_widgets.items(): + if w.tool_name == tool_name and w._status == "running": + widget = w + matched_key = key + break + if widget is None: + # Fallback: find any running tool + tools = self.query_one(".sa-tools", Vertical) + for child in tools.children: + if isinstance(child, ToolCallWidget) and child._status == "running": + if child.tool_name == tool_name: + widget = child + # Find the key for this widget + for key, w in self._tool_widgets.items(): + if w is widget: + matched_key = key + break + break + + if widget is not None: + if success: + widget.set_success(content) + else: + widget.set_error(content) + # Move from running to completed + if matched_key and matched_key in self._running_ids: + self._running_ids.remove(matched_key) + if matched_key: + self._completed_ids.append(matched_key) + self._update_visibility() + + def _update_visibility(self) -> None: + """Hide older completed tools, keep recent ones visible. + + Mirrors the Rich display.py collapsing logic: + - At most ``_MAX_VISIBLE_COMPLETED`` completed tools shown + (fewer if running tools take up slots). + - At most ``_MAX_VISIBLE_RUNNING`` running tools shown. + """ + # Determine how many completed slots are available + running_visible = self._running_ids[-_MAX_VISIBLE_RUNNING:] + completed_slots = max(0, _MAX_VISIBLE_COMPLETED - len(running_visible)) + completed_visible = ( + self._completed_ids[-completed_slots:] if completed_slots else [] + ) + completed_hidden = ( + self._completed_ids[:-completed_slots] + if completed_slots and len(self._completed_ids) > completed_slots + else (self._completed_ids if not completed_slots else []) + ) + + # Running tools to hide + running_hidden = ( + self._running_ids[:-_MAX_VISIBLE_RUNNING] + if len(self._running_ids) > _MAX_VISIBLE_RUNNING + else [] + ) + + # Apply visibility + visible_keys = set(completed_visible) | set(running_visible) + hidden_keys = set(completed_hidden) | set(running_hidden) + + for key in visible_keys: + w = self._tool_widgets.get(key) + if w is not None: + w.display = True + + for key in hidden_keys: + w = self._tool_widgets.get(key) + if w is not None: + w.display = False + + # Update collapse summary + total_hidden = len(completed_hidden) + hidden_running_count = len(running_hidden) + + summary_w = self.query_one(".sa-collapse-summary", Static) + if total_hidden > 0 or hidden_running_count > 0: + line = Text() + if total_hidden > 0: + # Count successes/failures among hidden completed + ok = 0 + fail = 0 + for key in completed_hidden: + w = self._tool_widgets.get(key) + if w is not None: + if w._status == "error": + fail += 1 + else: + ok += 1 + line.append(f"\u2713 {ok} completed", style="dim green") + if fail > 0: + line.append(f" | {fail} failed", style="dim red") + if hidden_running_count > 0: + if total_hidden > 0: + line.append(" | ", style="dim") + line.append( + f"\u25cf {hidden_running_count} more running...", style="dim yellow" + ) + summary_w.update(line) + summary_w.add_class("--visible") + else: + summary_w.remove_class("--visible") + + def finalize(self) -> None: + """Mark sub-agent as completed and stop all nested timers.""" + self._is_active = False + if self._timer_handle is not None: + self._timer_handle.stop() + self._timer_handle = None + # Mark any nested ToolCallWidgets still running as interrupted + for tw in self._tool_widgets.values(): + if tw._status == "running": + try: + tw.set_interrupted() + except Exception: + pass + self.add_class("--completed") + self._render_header() + self._render_footer() diff --git a/EvoScientist/cli/widgets/summarization_widget.py b/EvoScientist/cli/widgets/summarization_widget.py new file mode 100644 index 0000000..96e1571 --- /dev/null +++ b/EvoScientist/cli/widgets/summarization_widget.py @@ -0,0 +1,131 @@ +"""Summarization panel widget for context compression display. + +Renders a Rich Panel showing that LangGraph's summarization middleware +has compressed older conversation history. Yellow/amber border to +distinguish from the blue thinking panel. Default collapsed; click to +expand/collapse. + +Supports streaming: text arrives incrementally via ``append_text()`` +while the widget shows a live "Summarizing..." indicator, then switches +to a collapsed preview once ``finalize()`` is called. +""" + +from __future__ import annotations + +from rich.panel import Panel +from rich.text import Text +from textual.events import Click + +from .timed_status_widget import TimedStatusWidget + +_MAX_COLLAPSED_CHARS = 80 +_MAX_EXPANDED_CHARS = 3000 + + +class SummarizationWidget(TimedStatusWidget): + """Collapsible panel showing context summarization. + + Streams text via ``append_text()`` (shows live spinner while active). + Defaults to collapsed after streaming ends; click to expand/collapse. + + Usage:: + + w = SummarizationWidget() + await container.mount(w) + w.append_text("The conversation ") + w.append_text("covered ...") + w.finalize() # stop spinner, collapse + """ + + DEFAULT_CSS = """ + SummarizationWidget { + height: auto; + margin: 0 0 1 0; + } + """ + + def __init__(self) -> None: + super().__init__() + self._content = "" + self._collapsed = True + self._is_active = True # still receiving chunks + + def _should_tick(self) -> bool: + return self._is_active + + def _char_count_label(self) -> str: + n = len(self._content) + if n >= 1000: + return f"{n / 1000:.1f}k chars" + return f"{n:,} chars" + + def _refresh_display(self) -> None: + secs = self.elapsed_seconds + if not self._content: + if self._is_active: + self.update( + Panel( + Text("Summarizing...", style="dim italic"), + title=f"Context Summarizing... ({secs}s)", + border_style="#f59e0b", + padding=(0, 1), + ) + ) + else: + self.update("") + return + + if self._is_active: + # While streaming: show latest content tail (like thinking widget) + title = f"Context Summarizing... ({secs}s)" + tail = self._content.rstrip() + if len(tail) > 200: + tail = tail[-200:] + body = Text(tail, style="dim italic") + elif self._collapsed: + title = f"Context Summarized ({self._char_count_label()})" + first_line = self._content.strip().split("\n")[0].strip() + if len(first_line) > _MAX_COLLAPSED_CHARS: + first_line = first_line[: _MAX_COLLAPSED_CHARS - 3] + "\u2026" + preview = Text(first_line, style="dim italic") + preview.append(" [click to expand]", style="dim italic") + body = preview + else: + title = f"Context Summarized ({self._char_count_label()})" + display = self._content.rstrip() + if len(display) > _MAX_EXPANDED_CHARS: + half = _MAX_EXPANDED_CHARS // 2 + display = ( + display[:half] + "\n\n... (truncated) ...\n\n" + display[-half:] + ) + body = ( + Text(display, style="dim italic") + if display + else Text("(empty)", style="dim") + ) + + self.update(Panel(body, title=title, border_style="#f59e0b", padding=(0, 1))) + + def append_text(self, text: str) -> None: + """Append a chunk of summarization text (streaming).""" + self._content += text + self._refresh_display() + + def finalize(self) -> None: + """Mark streaming as complete — switch to collapsed preview.""" + self._is_active = False + self._collapsed = True + self._stop_timer() + self._refresh_display() + + def set_content(self, text: str) -> None: + """Set the full summarization text at once (non-streaming fallback).""" + self._content = text + self._is_active = False + self._refresh_display() + + def on_click(self, event: Click) -> None: + """Toggle collapsed/expanded state.""" + if self._content and not self._is_active: + self._collapsed = not self._collapsed + self._refresh_display() diff --git a/EvoScientist/cli/widgets/system_message.py b/EvoScientist/cli/widgets/system_message.py new file mode 100644 index 0000000..a7379c7 --- /dev/null +++ b/EvoScientist/cli/widgets/system_message.py @@ -0,0 +1,21 @@ +"""System message widget.""" + +from __future__ import annotations + +from rich.text import Text +from textual.widgets import Static + +from .timestamp_mixin import TimestampClickMixin + + +class SystemMessage(TimestampClickMixin, Static): + """Displays a system/status message (replaces ``_append_system``).""" + + DEFAULT_CSS = """ + SystemMessage { + height: auto; + } + """ + + def __init__(self, content: str, *, msg_style: str = "dim") -> None: + super().__init__(Text(content, style=msg_style)) diff --git a/EvoScientist/cli/widgets/thinking_widget.py b/EvoScientist/cli/widgets/thinking_widget.py new file mode 100644 index 0000000..9873f32 --- /dev/null +++ b/EvoScientist/cli/widgets/thinking_widget.py @@ -0,0 +1,120 @@ +"""Thinking panel widget for extended thinking display. + +Renders a Rich Panel matching the Rich CLI's style: +blue border, "Thinking" title with spinner, dim content. +Collapses to a one-line summary after finalize; click to expand/collapse. +""" + +from __future__ import annotations + +from rich.panel import Panel +from rich.text import Text +from textual.events import Click +from textual.widgets import Static + +_SPINNER_FRAMES = "\u280b\u2819\u2839\u2838\u283c\u2834\u2826\u2827\u2807\u280f" +_MAX_DISPLAY_CHARS = 1000 +_MAX_EXPANDED_CHARS = 5000 + + +class ThinkingWidget(Static): + """Collapsible panel showing the model's extended thinking. + + Uses Rich ``Panel`` for rendering, matching the Rich CLI output exactly. + While streaming, shows tail-truncated content with spinner. + After finalize, collapses to a one-line summary; click to expand/collapse. + + Usage:: + + w = ThinkingWidget(show_thinking=True) + await container.mount(w) + w.append_text("reasoning chunk...") + w.finalize() # stop spinner, collapse + """ + + DEFAULT_CSS = """ + ThinkingWidget { + height: auto; + margin: 0 0 1 0; + } + """ + + def __init__(self, *, show_thinking: bool = True) -> None: + super().__init__("") + self._content = "" + self._is_active = True + self._show = show_thinking + self._collapsed = False + self._frame = 0 + self._timer_handle = None + if not show_thinking: + self.display = False + + def on_mount(self) -> None: + self._timer_handle = self.set_interval(0.1, self._tick) + self._refresh_display() + + def _tick(self) -> None: + if self._is_active: + self._frame = (self._frame + 1) % len(_SPINNER_FRAMES) + self._refresh_display() + + def _char_count_label(self) -> str: + n = len(self._content) + if n >= 1000: + return f"{n / 1000:.1f}k chars" + return f"{n:,} chars" + + def _refresh_display(self) -> None: + if self._is_active: + # Streaming: show tail-truncated content with spinner + char = _SPINNER_FRAMES[self._frame] + title = f"Thinking {char}" + display = self._content.rstrip() + if len(display) > _MAX_DISPLAY_CHARS: + display = "..." + display[-_MAX_DISPLAY_CHARS:] + body = Text(display, style="dim") if display else Text("...", style="dim") + elif self._collapsed: + # Finalized + collapsed: first line preview + click hint + title = f"Thinking ({self._char_count_label()})" + first_line = self._content.strip().split("\n")[0].strip() + if len(first_line) > 80: + first_line = first_line[:77] + "\u2026" + preview = Text(first_line, style="dim") + preview.append(" [click to expand]", style="dim italic") + body = preview + else: + # Finalized + expanded: show with middle elision + title = f"Thinking ({self._char_count_label()})" + display = self._content.rstrip() + if len(display) > _MAX_EXPANDED_CHARS: + half = _MAX_EXPANDED_CHARS // 2 + display = ( + display[:half] + "\n\n... (truncated) ...\n\n" + display[-half:] + ) + body = ( + Text(display, style="dim") if display else Text("(empty)", style="dim") + ) + + self.update(Panel(body, title=title, border_style="blue", padding=(0, 1))) + + def append_text(self, chunk: str) -> None: + """Append a chunk of thinking text.""" + self._content += chunk + self._refresh_display() + + def finalize(self) -> None: + """Mark thinking as complete — stop spinner, collapse.""" + self._is_active = False + self._collapsed = True + if self._timer_handle is not None: + self._timer_handle.stop() + self._timer_handle = None + self._refresh_display() + + def on_click(self, event: Click) -> None: + """Toggle collapsed/expanded state after finalize.""" + if self._is_active: + return + self._collapsed = not self._collapsed + self._refresh_display() diff --git a/EvoScientist/cli/widgets/thread_selector.py b/EvoScientist/cli/widgets/thread_selector.py new file mode 100644 index 0000000..186b55a --- /dev/null +++ b/EvoScientist/cli/widgets/thread_selector.py @@ -0,0 +1,391 @@ +"""Inline thread picker widget for /resume and /delete in TUI. + +Keyboard-driven widget mounted directly into the chat container (like +ApprovalWidget). Posts ``ThreadPickerWidget.Picked`` when user selects +a thread, or ``ThreadPickerWidget.Cancelled`` on Esc. + +Threads are grouped into a two-level hierarchy: + + L1 header — common ancestor path shared by 2+ workspaces, or the + workspace path itself for standalone workspaces. + L2 subheader — relative sub-path shown only when a group contains + multiple workspaces. Run-mode dirs are marked with 🔁. + thread row — indented under their sub-path (or directly under L1 for + standalone groups). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + +from rich.text import Text +from textual.binding import Binding, BindingType +from textual.containers import Container +from textual.message import Message +from textual.widget import Widget +from textual.widgets import Static + +if TYPE_CHECKING: + from textual import events + from textual.app import ComposeResult + + +# --------------------------------------------------------------------------- +# Path helpers +# --------------------------------------------------------------------------- + + +def _normalize_path(path: str) -> str: + """Strip trailing slash and replace home directory with ~.""" + import os + + if not path: + return "" + path = path.rstrip("/") + home = os.path.expanduser("~") + if path.startswith(home): + path = "~" + path[len(home) :] + return path + + +def _common_prefix_depth(p1: str, p2: str) -> int: + """Return the number of leading path components shared by *p1* and *p2*.""" + depth = 0 + for a, b in zip(p1.split("/"), p2.split("/"), strict=False): + if a == b: + depth += 1 + else: + break + return depth + + +def _is_run_path(rel: str) -> bool: + """Return True if *rel* (relative to group ancestor) is a run-mode dir.""" + return "runs" in rel.split("/") + + +def _group_by_ancestor(norm_paths: list[str]) -> dict[str, list[str]]: + """Group normalized paths by their deepest common ancestor. + + Two paths are placed in the same group when they share a common prefix + of at least 2 components (e.g. ``~/Projects``). Paths with no such + shared prefix become standalone single-item groups keyed by their own + full path. + + The returned dict is ordered by first appearance in *norm_paths*. + + Complexity: O(n log n) — paths are sorted lexicographically so the + maximum common prefix depth for each path is found by comparing only + its immediate neighbours in sorted order (not all pairs). + """ + sorted_paths = sorted(norm_paths) + n = len(sorted_paths) + + # Single pass over sorted list: max common prefix is always with a neighbour. + sorted_best: dict[str, int] = {} + for i, p in enumerate(sorted_paths): + best = 1 # at minimum depth 1 (~) + if i > 0: + best = max(best, _common_prefix_depth(p, sorted_paths[i - 1])) + if i < n - 1: + best = max(best, _common_prefix_depth(p, sorted_paths[i + 1])) + sorted_best[p] = best + + path_to_ancestor: dict[str, str] = { + p: ("/".join(p.split("/")[:best]) if best >= 2 else p) + for p, best in sorted_best.items() + } + + groups: dict[str, list[str]] = {} + for p in norm_paths: + anc = path_to_ancestor[p] + if anc not in groups: + groups[anc] = [] + groups[anc].append(p) + return groups + + +# --------------------------------------------------------------------------- +# Item builders +# --------------------------------------------------------------------------- + + +def _build_items(threads: list[dict]) -> list[dict]: + """Build the flat item list rendered by ThreadPickerWidget. + + Returns a list whose elements are one of:: + + {"type": "header", "label": str} + {"type": "subheader", "label": str} + {"type": "thread", "thread": dict, "indented": bool} + + *indented* is True for thread rows that sit under a L2 subheader. + """ + if not threads: + return [] + + # Map normalized path -> list[thread dicts], preserving first-seen order + raw_to_threads: dict[str, list[dict]] = {} + seen_order: list[str] = [] + for t in threads: + raw = t.get("workspace_dir", "") or "" + norm = _normalize_path(raw) or raw + if norm not in raw_to_threads: + raw_to_threads[norm] = [] + seen_order.append(norm) + raw_to_threads[norm].append(t) + + groups = _group_by_ancestor(seen_order) + + items: list[dict] = [] + for ancestor, norm_paths in groups.items(): + multi = len(norm_paths) > 1 + + # L1 header — the common ancestor (or the sole workspace path) + items.append({"type": "header", "label": ancestor or "(no workspace)"}) + + for norm_path in norm_paths: + if multi: + # L2 subheader — relative path from ancestor + rel = norm_path[len(ancestor) :].lstrip("/") + if not rel: + # norm_path IS the ancestor (standalone group of 1 that shares + # an ancestor with others); show just the last path component + rel = norm_path.split("/")[-1] or norm_path + icon = "🔁" if _is_run_path(rel) else "📁" + items.append({"type": "subheader", "label": f"{icon} {rel}"}) + + for t in raw_to_threads[norm_path]: + items.append({"type": "thread", "thread": t, "indented": multi}) + + return items + + +def build_header_text(label: str) -> Text: + """L1 header: ``── 📂