From 0012dd1e0c7efc7e86fa75b89567a492057f0c02 Mon Sep 17 00:00:00 2001 From: ethernet Date: Sat, 22 Aug 2026 01:14:19 -0400 Subject: [PATCH] perf(ci): set python test workers to one for each core, from measurement `run_tests.sh` defaults to twice the core count, and the value this branch started with came from a rule of thumb of 1.5x cores plus a measurement on a 16-core machine. A sweep on the real runner disagrees with both. Run 32549672063 on the 96-core runner (EPYC 7763, 377GB) timed the whole suite at six worker counts, two repetitions for each. A warmup run came first, and retries were off: workers x cores rep 1 rep 2 mean 48 0.5x 138s 139s 138s 96 1.0x 127s 126s 126s <- fastest 144 1.5x 130s 134s 132s 192 2.0x 132s 133s 132s 240 2.5x 140s 139s 140s 288 3.0x 143s 142s 142s One worker for each core wins. Both repetitions agree on the order. The shape is the more useful result. The range is 126s to 142s across a 6x range of worker counts. The suite has sufficient concurrency at this machine size, so nothing above the core count buys anything. The remaining time belongs to the slowest individual files and to the setup. A future gain must come from those, and not from this number. The sweep ran from a temporary workflow that this branch does not keep. --- .github/workflows/tests.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7286abf6a8..a0dca54ad0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -104,8 +104,22 @@ jobs: # from a single ThreadPoolExecutor, so this value IS the limit. The # default is cpu_count*2, which is 192 here. # - # A later commit sets this from a sweep on the real runner. - HERMES_TEST_WORKERS: 144 + # Measured on this runner (96-core EPYC 7763, 377GB). Whole suite, + # two repetitions for each value. See run 32549672063: + # + # workers x cores mean + # 48 0.5x 138s + # 96 1.0x 126s <- fastest + # 144 1.5x 132s + # 192 2.0x 132s + # 240 2.5x 140s + # 288 3.0x 142s + # + # One worker for each core wins. The curve is shallow: 126s to 142s + # across a 6x range. The suite has sufficient concurrency at this + # size. The remaining time is the slowest files plus the setup. + # Workers above the core count only add contention. + HERMES_TEST_WORKERS: 96 # Ensure tests don't accidentally call real APIs OPENROUTER_API_KEY: "" OPENAI_API_KEY: ""