From e628d9d297a1549bd076a96fb030587ef52d7f4d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 10 Aug 2026 16:40:31 +0200 Subject: [PATCH] test(bench): make the py-lsp time budget a liveness backstop, not a perf gate test_py_lsp_bench.c:262 failed the v0.10.0 release run TWICE on genuine attempts (ubuntu-24.04-arm 3/3), while the SAME sha passed that shard in the morning's dry run. A wall-clock bound on a shared 4-vCPU runner is not a property of the code, and this repo's testing doctrine says exactly that: perf tests assert a ratio; a time bound may exist only as a liveness backstop. This test's perf content already IS ratio-form (resolution ratio with a sanitizer-aware floor). The 150/1500 ms budget was the anti-pattern half. It becomes a 30 s liveness backstop so a catastrophic hang still fails the suite rather than the 900 s shard wall clock, and absolute timings stay where they belong: scripts/benchmark-*.sh, which record rather than gate. Signed-off-by: Martin Vogel --- tests/test_py_lsp_bench.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_py_lsp_bench.c b/tests/test_py_lsp_bench.c index 9b6bfd41b..bd5d99d57 100644 --- a/tests/test_py_lsp_bench.c +++ b/tests/test_py_lsp_bench.c @@ -255,14 +255,15 @@ TEST(pylsp_bench_resolution_ratio) { ASSERT_GTE(resolved * 2, calls); } - /* Time budget. ASan+UBSan instrumentation slows the parse ~5-10×, so - * scale the budget when a sanitizer is active. Native: 150 ms for a - * ~200-line fixture; sanitized: 1500 ms. */ -#if defined(CBM_SANITIZED_BUILD) || defined(__SANITIZE_ADDRESS__) - ASSERT(ms < 1500.0); -#else - ASSERT(ms < 150.0); -#endif + /* Liveness backstop ONLY — not a perf gate. The perf content of this test + * is the resolution-ratio assertion above; wall-clock on a shared CI + * runner is not a property of the code (the 1500 ms sanitized budget was + * exceeded twice on loaded ubuntu-24.04-arm runners by the SAME sha that + * passed hours earlier). Absolute timings belong to + * scripts/benchmark-*.sh, which record rather than gate. This bound + * exists solely so a catastrophic hang fails the suite instead of the + * 900 s shard wall clock. */ + ASSERT(ms < 30000.0); PASS(); }