m4
c2743251e9
Self-evolving AI scientist framework built on LangGraph/LangChain with CLI/TUI core, FastAPI gateway, and Next.js frontend. Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
616 lines
19 KiB
Python
616 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Standalone gpt-5.5 endpoint diagnostics.
|
|
|
|
This script intentionally stays outside the gateway/agent runtime. It checks:
|
|
1. structured config resolution for gpt-5.5
|
|
2. raw OpenAI-compatible Chat Completions
|
|
3. raw OpenAI Responses API
|
|
4. LangChain direct chat/responses clients
|
|
5. optional EvoScientist get_chat_model app path
|
|
|
|
It never prints API keys.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import traceback
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from openai import AsyncOpenAI, DefaultAsyncHttpxClient
|
|
from langchain.chat_models import init_chat_model
|
|
from langchain_core.messages import HumanMessage
|
|
|
|
|
|
def _load_dotenv_file(path: Path) -> None:
|
|
if not path.exists():
|
|
return
|
|
for raw in path.read_text(encoding="utf-8").splitlines():
|
|
line = raw.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
key = key.strip()
|
|
value = value.strip().strip('"').strip("'")
|
|
os.environ.setdefault(key, value)
|
|
|
|
|
|
def _short_body(text: str, limit: int = 500) -> str:
|
|
text = text.replace("\n", " ").replace("\r", " ")
|
|
if len(text) <= limit:
|
|
return text
|
|
return text[:limit] + "...<truncated>"
|
|
|
|
|
|
def _json_safe(value: Any) -> str:
|
|
try:
|
|
return json.dumps(value, ensure_ascii=False, default=str)
|
|
except Exception:
|
|
return repr(value)
|
|
|
|
|
|
def _print_result(label: str, status: int | str, elapsed: float, body: str) -> None:
|
|
print(f"[{label}] status={status} elapsed={elapsed:.2f}s body={_short_body(body)}")
|
|
|
|
|
|
def _post_json(
|
|
client: httpx.Client,
|
|
label: str,
|
|
url: str,
|
|
api_key: str,
|
|
payload: dict[str, Any],
|
|
) -> None:
|
|
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
|
start = time.perf_counter()
|
|
try:
|
|
resp = client.post(url, headers=headers, json=payload)
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(label, resp.status_code, elapsed, resp.text)
|
|
except Exception as exc:
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(label, type(exc).__name__, elapsed, str(exc))
|
|
|
|
|
|
def _get(
|
|
client: httpx.Client,
|
|
label: str,
|
|
url: str,
|
|
api_key: str,
|
|
) -> None:
|
|
headers = {"Authorization": f"Bearer {api_key}"}
|
|
start = time.perf_counter()
|
|
try:
|
|
resp = client.get(url, headers=headers)
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(label, resp.status_code, elapsed, resp.text)
|
|
except Exception as exc:
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(label, type(exc).__name__, elapsed, str(exc))
|
|
|
|
|
|
async def _try_langchain(
|
|
label: str,
|
|
*,
|
|
model: str,
|
|
api_key: str,
|
|
base_url: str,
|
|
use_responses_api: bool,
|
|
max_tokens: int,
|
|
reasoning: dict[str, Any] | None = None,
|
|
) -> None:
|
|
kwargs: dict[str, Any] = {
|
|
"model": model,
|
|
"model_provider": "openai",
|
|
"api_key": api_key,
|
|
"base_url": base_url,
|
|
"use_responses_api": use_responses_api,
|
|
"max_tokens": max_tokens,
|
|
"request_timeout": 30,
|
|
"stream_usage": True,
|
|
}
|
|
if reasoning is not None:
|
|
kwargs["reasoning"] = reasoning
|
|
|
|
start = time.perf_counter()
|
|
try:
|
|
llm = init_chat_model(**kwargs)
|
|
payload = llm._get_request_payload(
|
|
[HumanMessage(content="Say OK in one word.")], stop=None
|
|
)
|
|
print(f"[{label}] payload={_json_safe(payload)}")
|
|
msg = await llm.ainvoke([HumanMessage(content="Say OK in one word.")])
|
|
elapsed = time.perf_counter() - start
|
|
body = _json_safe(
|
|
{
|
|
"content": msg.content,
|
|
"usage": getattr(msg, "usage_metadata", None),
|
|
}
|
|
)
|
|
_print_result(label, "OK", elapsed, body)
|
|
except Exception as exc:
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(label, type(exc).__name__, elapsed, str(exc))
|
|
|
|
|
|
async def _try_openai_sdk(
|
|
label: str,
|
|
*,
|
|
model: str,
|
|
api_key: str,
|
|
base_url: str,
|
|
mode: str,
|
|
max_tokens: int,
|
|
sanitize_headers: bool = False,
|
|
) -> None:
|
|
async def on_request(request: httpx.Request) -> None:
|
|
if sanitize_headers:
|
|
for key in list(request.headers):
|
|
lower_key = key.lower()
|
|
if lower_key.startswith("x-stainless-") and lower_key != "x-stainless-raw-response":
|
|
request.headers.pop(key, None)
|
|
request.headers["User-Agent"] = "python-httpx"
|
|
headers = {
|
|
k: v
|
|
for k, v in request.headers.items()
|
|
if k.lower()
|
|
not in {
|
|
"authorization",
|
|
"cookie",
|
|
"x-api-key",
|
|
}
|
|
}
|
|
print(
|
|
f"[{label}] request method={request.method} url={request.url} "
|
|
f"headers={_json_safe(headers)}"
|
|
)
|
|
|
|
http_client = DefaultAsyncHttpxClient(
|
|
timeout=30.0,
|
|
event_hooks={"request": [on_request]},
|
|
)
|
|
client = AsyncOpenAI(
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
timeout=30.0,
|
|
max_retries=0,
|
|
http_client=http_client,
|
|
)
|
|
start = time.perf_counter()
|
|
try:
|
|
if mode == "chat":
|
|
result = await client.chat.completions.create(
|
|
model=model,
|
|
messages=[{"role": "user", "content": "Say OK in one word."}],
|
|
max_completion_tokens=max_tokens,
|
|
)
|
|
body = result.model_dump()
|
|
elif mode == "responses":
|
|
result = await client.responses.create(
|
|
model=model,
|
|
input=[
|
|
{
|
|
"role": "user",
|
|
"content": "Say OK in one word.",
|
|
"type": "message",
|
|
}
|
|
],
|
|
max_output_tokens=max_tokens,
|
|
)
|
|
body = result.model_dump()
|
|
else:
|
|
raise ValueError(f"unknown mode: {mode}")
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(label, "OK", elapsed, _json_safe(body))
|
|
except Exception as exc:
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(label, type(exc).__name__, elapsed, str(exc))
|
|
finally:
|
|
await http_client.aclose()
|
|
|
|
|
|
async def _try_app_path(model: str) -> None:
|
|
from EvoScientist.config.settings import apply_config_to_env, load_config
|
|
from EvoScientist.llm.models import _clear_structured_config_cache, get_chat_model
|
|
|
|
apply_config_to_env(load_config())
|
|
_clear_structured_config_cache()
|
|
start = time.perf_counter()
|
|
try:
|
|
llm = get_chat_model(model)
|
|
payload = llm._get_request_payload(
|
|
[HumanMessage(content="Say OK in one word.")], stop=None
|
|
)
|
|
print(f"[app:get_chat_model] payload={_json_safe(payload)}")
|
|
msg = await llm.ainvoke([HumanMessage(content="Say OK in one word.")])
|
|
elapsed = time.perf_counter() - start
|
|
body = _json_safe(
|
|
{
|
|
"content": msg.content,
|
|
"usage": getattr(msg, "usage_metadata", None),
|
|
"use_responses_api": getattr(llm, "use_responses_api", None),
|
|
"base_url": getattr(llm, "openai_api_base", None),
|
|
}
|
|
)
|
|
_print_result("app:get_chat_model", "OK", elapsed, body)
|
|
except Exception as exc:
|
|
elapsed = time.perf_counter() - start
|
|
cause = repr(getattr(exc, "__cause__", None))
|
|
context = repr(getattr(exc, "__context__", None))
|
|
_print_result(
|
|
"app:get_chat_model",
|
|
type(exc).__name__,
|
|
elapsed,
|
|
f"{exc} cause={cause} context={context}",
|
|
)
|
|
traceback.print_exception(type(exc), exc, exc.__traceback__, limit=6)
|
|
|
|
|
|
def _resolve_config(model_ref: str) -> tuple[str, str, str, dict[str, Any]]:
|
|
from EvoScientist.config.model_config import load_structured_config, resolve_model
|
|
from EvoScientist.config.settings import apply_config_to_env, load_config
|
|
|
|
apply_config_to_env(load_config())
|
|
cfg = load_structured_config()
|
|
resolved = resolve_model(model_ref, config=cfg)
|
|
api_key = resolved.api_key.strip() if resolved.api_key else ""
|
|
base_url = resolved.base_url.rstrip("/")
|
|
print(
|
|
"[config] "
|
|
f"provider={resolved.provider_name} protocol={resolved.protocol} "
|
|
f"model={resolved.model_id} base_url={base_url} "
|
|
f"api_key_set={bool(api_key)} params={_json_safe(resolved.params)}"
|
|
)
|
|
print(
|
|
"[env] "
|
|
f"EVOSCIENTIST_USE_RESPONSES_API="
|
|
f"{os.environ.get('EVOSCIENTIST_USE_RESPONSES_API')!r}"
|
|
)
|
|
return resolved.model_id, api_key, base_url, resolved.params
|
|
|
|
|
|
async def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--model", default="gpt-5.5")
|
|
parser.add_argument("--timeout", type=float, default=30.0)
|
|
parser.add_argument("--max-tokens", type=int, default=128)
|
|
parser.add_argument("--full-app-path", action="store_true")
|
|
parser.add_argument(
|
|
"--case",
|
|
choices=(
|
|
"all",
|
|
"raw-chat-exact",
|
|
"raw-chat-sdk-headers",
|
|
"raw-chat-sanitized-sdk-headers",
|
|
"raw-chat-accept-json",
|
|
"raw-chat-user-agent",
|
|
"raw-responses-exact",
|
|
"sdk-chat",
|
|
"sdk-chat-sanitized",
|
|
"sdk-responses",
|
|
"sdk-responses-sanitized",
|
|
"app",
|
|
),
|
|
default="all",
|
|
help="Run a single diagnostic case to avoid triggering provider cooldowns.",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
_load_dotenv_file(Path(".env"))
|
|
|
|
model_id, api_key, base_url, _params = _resolve_config(args.model)
|
|
if not api_key:
|
|
print("[fatal] API key is empty after config resolution")
|
|
return 2
|
|
if not base_url:
|
|
print("[fatal] base_url is empty after config resolution")
|
|
return 2
|
|
|
|
chat_payload = {
|
|
"model": model_id,
|
|
"messages": [{"role": "user", "content": "Say OK in one word."}],
|
|
"max_tokens": args.max_tokens,
|
|
}
|
|
responses_payload = {
|
|
"model": model_id,
|
|
"input": "Say OK in one word.",
|
|
"max_output_tokens": args.max_tokens,
|
|
}
|
|
lc_chat_payload = {
|
|
"model": model_id,
|
|
"messages": [{"role": "user", "content": "Say OK in one word."}],
|
|
"max_completion_tokens": args.max_tokens,
|
|
"stream": False,
|
|
}
|
|
lc_responses_payload = {
|
|
"model": model_id,
|
|
"input": [
|
|
{
|
|
"role": "user",
|
|
"content": "Say OK in one word.",
|
|
"type": "message",
|
|
}
|
|
],
|
|
"max_output_tokens": args.max_tokens,
|
|
"stream": False,
|
|
}
|
|
|
|
if args.case == "raw-chat-exact":
|
|
with httpx.Client(timeout=args.timeout) as client:
|
|
_post_json(
|
|
client,
|
|
"raw:POST /chat/completions exact-lc",
|
|
f"{base_url}/chat/completions",
|
|
api_key,
|
|
lc_chat_payload,
|
|
)
|
|
return 0
|
|
|
|
if args.case == "raw-chat-sdk-headers":
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Accept": "application/json",
|
|
"Content-Type": "application/json",
|
|
"User-Agent": "AsyncOpenAI/Python 2.31.0",
|
|
"X-Stainless-Lang": "python",
|
|
"X-Stainless-Package-Version": "2.31.0",
|
|
"X-Stainless-OS": "Linux",
|
|
"X-Stainless-Arch": "x64",
|
|
"X-Stainless-Runtime": "CPython",
|
|
"X-Stainless-Runtime-Version": "3.12.13",
|
|
"X-Stainless-Async": "async:asyncio",
|
|
"X-Stainless-Retry-Count": "0",
|
|
"X-Stainless-Read-Timeout": "30.0",
|
|
}
|
|
start = time.perf_counter()
|
|
try:
|
|
with httpx.Client(timeout=args.timeout) as client:
|
|
resp = client.post(
|
|
f"{base_url}/chat/completions",
|
|
headers=headers,
|
|
json=lc_chat_payload,
|
|
)
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(
|
|
"raw:POST /chat/completions sdk-headers",
|
|
resp.status_code,
|
|
elapsed,
|
|
resp.text,
|
|
)
|
|
except Exception as exc:
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(
|
|
"raw:POST /chat/completions sdk-headers",
|
|
type(exc).__name__,
|
|
elapsed,
|
|
str(exc),
|
|
)
|
|
return 0
|
|
|
|
if args.case == "raw-chat-sanitized-sdk-headers":
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Accept": "application/json",
|
|
"Content-Type": "application/json",
|
|
"User-Agent": "python-httpx",
|
|
}
|
|
start = time.perf_counter()
|
|
try:
|
|
with httpx.Client(timeout=args.timeout) as client:
|
|
resp = client.post(
|
|
f"{base_url}/chat/completions",
|
|
headers=headers,
|
|
json=lc_chat_payload,
|
|
)
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(
|
|
"raw:POST /chat/completions sanitized-sdk-headers",
|
|
resp.status_code,
|
|
elapsed,
|
|
resp.text,
|
|
)
|
|
except Exception as exc:
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(
|
|
"raw:POST /chat/completions sanitized-sdk-headers",
|
|
type(exc).__name__,
|
|
elapsed,
|
|
str(exc),
|
|
)
|
|
return 0
|
|
|
|
if args.case in ("raw-chat-accept-json", "raw-chat-user-agent"):
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Content-Type": "application/json",
|
|
}
|
|
if args.case == "raw-chat-accept-json":
|
|
headers["Accept"] = "application/json"
|
|
label = "raw:POST /chat/completions accept-json"
|
|
else:
|
|
headers["User-Agent"] = "python-httpx"
|
|
label = "raw:POST /chat/completions user-agent"
|
|
start = time.perf_counter()
|
|
try:
|
|
with httpx.Client(timeout=args.timeout) as client:
|
|
resp = client.post(
|
|
f"{base_url}/chat/completions",
|
|
headers=headers,
|
|
json=lc_chat_payload,
|
|
)
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(label, resp.status_code, elapsed, resp.text)
|
|
except Exception as exc:
|
|
elapsed = time.perf_counter() - start
|
|
_print_result(label, type(exc).__name__, elapsed, str(exc))
|
|
return 0
|
|
|
|
if args.case == "raw-responses-exact":
|
|
with httpx.Client(timeout=args.timeout) as client:
|
|
_post_json(
|
|
client,
|
|
"raw:POST /responses exact-lc",
|
|
f"{base_url}/responses",
|
|
api_key,
|
|
lc_responses_payload,
|
|
)
|
|
return 0
|
|
|
|
if args.case == "sdk-chat":
|
|
await _try_openai_sdk(
|
|
"sdk:chat",
|
|
model=model_id,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
mode="chat",
|
|
max_tokens=args.max_tokens,
|
|
)
|
|
return 0
|
|
|
|
if args.case == "sdk-chat-sanitized":
|
|
await _try_openai_sdk(
|
|
"sdk:chat:sanitized",
|
|
model=model_id,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
mode="chat",
|
|
max_tokens=args.max_tokens,
|
|
sanitize_headers=True,
|
|
)
|
|
return 0
|
|
|
|
if args.case == "sdk-responses":
|
|
await _try_openai_sdk(
|
|
"sdk:responses",
|
|
model=model_id,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
mode="responses",
|
|
max_tokens=args.max_tokens,
|
|
)
|
|
return 0
|
|
|
|
if args.case == "sdk-responses-sanitized":
|
|
await _try_openai_sdk(
|
|
"sdk:responses:sanitized",
|
|
model=model_id,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
mode="responses",
|
|
max_tokens=args.max_tokens,
|
|
sanitize_headers=True,
|
|
)
|
|
return 0
|
|
|
|
if args.case == "app":
|
|
await _try_app_path(args.model)
|
|
return 0
|
|
|
|
with httpx.Client(timeout=args.timeout) as client:
|
|
_get(client, "raw:GET /v1/models", f"{base_url}/v1/models", api_key)
|
|
_post_json(
|
|
client,
|
|
"raw:POST /chat/completions",
|
|
f"{base_url}/chat/completions",
|
|
api_key,
|
|
chat_payload,
|
|
)
|
|
_post_json(
|
|
client,
|
|
"raw:POST /v1/chat/completions",
|
|
f"{base_url}/v1/chat/completions",
|
|
api_key,
|
|
chat_payload,
|
|
)
|
|
_post_json(
|
|
client,
|
|
"raw:POST /chat/completions exact-lc",
|
|
f"{base_url}/chat/completions",
|
|
api_key,
|
|
lc_chat_payload,
|
|
)
|
|
_post_json(
|
|
client,
|
|
"raw:POST /responses",
|
|
f"{base_url}/responses",
|
|
api_key,
|
|
responses_payload,
|
|
)
|
|
_post_json(
|
|
client,
|
|
"raw:POST /v1/responses",
|
|
f"{base_url}/v1/responses",
|
|
api_key,
|
|
responses_payload,
|
|
)
|
|
_post_json(
|
|
client,
|
|
"raw:POST /responses exact-lc",
|
|
f"{base_url}/responses",
|
|
api_key,
|
|
lc_responses_payload,
|
|
)
|
|
_post_json(
|
|
client,
|
|
"raw:POST /v1/responses exact-lc",
|
|
f"{base_url}/v1/responses",
|
|
api_key,
|
|
lc_responses_payload,
|
|
)
|
|
|
|
await _try_langchain(
|
|
"lc:chat",
|
|
model=model_id,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
use_responses_api=False,
|
|
max_tokens=args.max_tokens,
|
|
)
|
|
await _try_langchain(
|
|
"lc:responses",
|
|
model=model_id,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
use_responses_api=True,
|
|
max_tokens=args.max_tokens,
|
|
)
|
|
await _try_langchain(
|
|
"lc:responses+reasoning",
|
|
model=model_id,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
use_responses_api=True,
|
|
max_tokens=args.max_tokens,
|
|
reasoning={"effort": "high", "summary": "auto"},
|
|
)
|
|
await _try_openai_sdk(
|
|
"sdk:chat",
|
|
model=model_id,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
mode="chat",
|
|
max_tokens=args.max_tokens,
|
|
)
|
|
await _try_openai_sdk(
|
|
"sdk:responses",
|
|
model=model_id,
|
|
api_key=api_key,
|
|
base_url=base_url,
|
|
mode="responses",
|
|
max_tokens=args.max_tokens,
|
|
)
|
|
|
|
if args.full_app_path:
|
|
await _try_app_path(args.model)
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(asyncio.run(main()))
|