From 78e0d296d6946d3789bc2f7cf56cbf0f8ef8ba5f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:45:21 -0700 Subject: [PATCH] test: publish credential-free fallback wire reproduction --- evals/provider_fallback/probe_104260.py | 235 ++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 evals/provider_fallback/probe_104260.py diff --git a/evals/provider_fallback/probe_104260.py b/evals/provider_fallback/probe_104260.py new file mode 100644 index 0000000000..c31e4e431e --- /dev/null +++ b/evals/provider_fallback/probe_104260.py @@ -0,0 +1,235 @@ +"""Offline production-import reproduction: no repo edits or real credentials.""" + +import os +import sys +import tempfile +from pathlib import Path + +REPO = sys.argv[1] +sys.dont_write_bytecode = True +sandbox = tempfile.TemporaryDirectory(prefix="hermes-104260-") +os.environ.clear() +os.environ.update( + HOME=sandbox.name, + HERMES_HOME=sandbox.name + "/hermes", + PATH="/usr/bin:/bin", + PYTHONDONTWRITEBYTECODE="1", + HERMES_DISABLE_MODEL_METADATA_FETCH="1", +) +Path(os.environ["HERMES_HOME"]).mkdir() +os.chdir(sandbox.name) +sys.path.insert(0, REPO) +import socket + +_original_connect = socket.socket.connect +blocked = [] + + +def loopback_only(self, address): + if isinstance(address, tuple) and address[0] not in ( + "127.0.0.1", + "::1", + "localhost", + ): + blocked.append(str(address)) + raise RuntimeError("Probe blocks non-loopback network") + return _original_connect(self, address) + + +socket.socket.connect = loopback_only +import asyncio +import json +import logging +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +logging.basicConfig(level=logging.WARNING) +requests = [] + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0")))) + requests.append({ + "endpoint": self.server.kind, + "path": self.path, + "model": body.get("model"), + }) + if self.server.kind == "cloud-simulator": + status = self.server.failure_status + data = { + "error": { + "message": "weekly usage limit exceeded", + "type": "insufficient_quota", + "code": "insufficient_quota", + } + } + else: + status = 200 + data = { + "id": "local-probe", + "object": "chat.completion", + "created": 1, + "model": body.get("model"), + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "LOCAL_OK"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + } + wire = json.dumps(data).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(wire))) + self.end_headers() + self.wfile.write(wire) + + +servers = [] +for kind in ("cloud-simulator", "healthy-local"): + srv = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + srv.kind = kind + srv.failure_status = 402 + threading.Thread(target=srv.serve_forever, daemon=True).start() + servers.append(srv) +cloud, local = [f"http://127.0.0.1:{s.server_port}/v1" for s in servers] +config = { + "model": { + "provider": "custom", + "default": "local-probe-model", + "base_url": local, + "api_key": "no-key-required", + "api_mode": "chat_completions", + }, + "auxiliary": { + "compression": { + "fallback_chain": [ + { + "provider": "custom", + "model": "local-probe-model", + "base_url": local, + "api_key": "no-key-required", + "api_mode": "chat_completions", + } + ] + } + }, +} +# JSON is valid YAML; only the temporary Hermes home is written. +Path(os.environ["HERMES_HOME"], "config.yaml").write_text( + json.dumps(config), encoding="utf-8" +) +from agent import auxiliary_client as aux +from agent.backend_identity import BackendIdentity, FailureScope, should_skip_candidate + +assert aux.__file__.startswith(REPO) +print("PRODUCTION_IMPORT", aux.__file__, flush=True) +args = dict( + provider="custom", + model="cloud-probe-model", + base_url=cloud, + api_key="no-key-required", + api_mode="chat_completions", + messages=[{"role": "user", "content": "probe"}], + timeout=2, + max_tokens=8, +) +local_args = dict(args, model="local-probe-model", base_url=local) +results = [] +for mode, status in [("sync", 402), ("async", 402), ("sync", 429)]: + aux._reset_aux_unhealthy_cache() + servers[0].failure_status = status + before = aux.call_llm(**local_args).choices[0].message.content + start = len(requests) + try: + if mode == "sync": + aux.call_llm(task="compression", **args) + else: + asyncio.run( + aux.async_call_llm( + task="compression", + **{k: v for k, v in args.items() if k != "api_mode"}, + ) + ) + error = None + except Exception as exc: + error = { + "type": type(exc).__name__, + "status": getattr(exc, "status_code", None), + "payment": aux._is_payment_error(exc), + } + failure_requests = requests[start:] + cache = { + str(k): round(v - time.time(), 2) for k, v in aux._aux_unhealthy_until.items() + } + route_after = aux._try_main_provider_route( + "custom", "local-probe-model", local, "no-key-required", "chat_completions" + ) + direct_after = aux.call_llm(**local_args).choices[0].message.content + aux._reset_aux_unhealthy_cache() + route_reset = aux._try_main_provider_route( + "custom", "local-probe-model", local, "no-key-required", "chat_completions" + ) + entry = { + "mode": mode, + "status": status, + "local_before": before, + "cloud_error": error, + "failure_requests": failure_requests, + "health_cache_remaining_seconds": cache, + "local_auto_route_after_payment": route_after is not None, + "direct_local_after_payment": direct_after, + "local_auto_route_after_cache_reset": route_reset is not None, + } + print("OBSERVED", json.dumps(entry), flush=True) + assert before == direct_after == "LOCAL_OK" + assert error is None + assert any(r["endpoint"] == "healthy-local" for r in failure_requests) + assert len(cache) == 1 and all(590 < v <= 600 for v in cache.values()) + assert route_after is not None and route_reset is not None + results.append(entry) + print("CASE", json.dumps(entry), flush=True) +ident_cloud = BackendIdentity.build("custom", "cloud-probe-model", cloud) +ident_local = BackendIdentity.build("custom", "local-probe-model", local) +identity = { + "credential_skip_distinct_custom_urls": should_skip_candidate( + ident_local, ident_cloud, FailureScope.CREDENTIAL + ), + "endpoint_skip_distinct_custom_urls": should_skip_candidate( + ident_local, ident_cloud, FailureScope.ENDPOINT + ), + "model_skip_distinct_custom_urls": should_skip_candidate( + ident_local, ident_cloud, FailureScope.MODEL + ), + "named_custom_distinct_labels_credential_skip": should_skip_candidate( + BackendIdentity.build("custom:local", "local-probe-model", local), + BackendIdentity.build("custom:cloud", "cloud-probe-model", cloud), + FailureScope.CREDENTIAL, + ), +} +assert identity == { + "credential_skip_distinct_custom_urls": True, + "endpoint_skip_distinct_custom_urls": False, + "model_skip_distinct_custom_urls": False, + "named_custom_distinct_labels_credential_skip": False, +} +print("IDENTITY", json.dumps(identity), flush=True) +print("BLOCKED_NON_LOOPBACK", json.dumps(blocked), flush=True) +print( + "PASS: sync/async local fallback and subsequent routing survive cloud payment failures.", + flush=True, +) +for srv in servers: + srv.shutdown() +sandbox.cleanup()