From 78bf6a274505aafe662b49955376f2d3f9e9d47e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 2 Nov 2026 00:06:52 +0100 Subject: [PATCH] fix(test): hand over a kept benchmark runtime only after its daemon stopped With CBM_BENCH_KEEP_RUNTIME=1, bench_finish ran `daemon stop` with its output and exit status discarded and wrote runtime-root.txt regardless. `daemon stop` refuses while a committed client still uses the daemon and fails when the daemon does not answer; the handoff then promised a stopped daemon that was still running. A refused stop now prints the explanation of the product, writes no handoff, keeps the root without cleaning it up (the daemon may still be using it) and exits non-zero. A stop with nothing to stop still exits 0, so a clean run is unaffected. Section 7 step 8 of the evaluation plan follows the same rule: the kept root is removed only if its daemon stopped, and is kept with a message otherwise. Signed-off-by: Martin Vogel --- docs/EVALUATION_PLAN.md | 10 +++++++--- scripts/benchmark-index.sh | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md index 39d72848f..25a8f6aaf 100644 --- a/docs/EVALUATION_PLAN.md +++ b/docs/EVALUATION_PLAN.md @@ -299,9 +299,13 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list # --- step 8: delete THIS language's index so the next is cold, then mark done --- # The graph session (steps 4-7) may still have a daemon running under the kept - # CBM_RUNTIME_DIR (exported above); stop it before its root goes away. - ~/.local/bin/codebase-memory-mcp daemon stop || true - rm -rf -- "$CBM_BENCH_RUNTIME_ROOT" + # CBM_RUNTIME_DIR (exported above); stop it before its root goes away. A refused + # stop (a session still attached) means the root is still in use: keep it. + if ~/.local/bin/codebase-memory-mcp daemon stop; then + rm -rf -- "$CBM_BENCH_RUNTIME_ROOT" + else + echo "$lang: daemon under $CBM_RUNTIME_DIR did not stop; keeping $CBM_BENCH_RUNTIME_ROOT" >&2 + fi manifest_mark_done "$lang" "$index_ms" done ``` diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index a8db36828..792ec9feb 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -33,11 +33,21 @@ cbm_test_runtime_init # including its removal — passes to the caller. A failed run cleans up # regardless: there is no index worth keeping, and nothing must leak. Never on # by default, or an unattended run accumulates one root per language. +# +# The handoff promises a stopped daemon, so it is written only once +# `daemon stop` succeeded. A refused stop (a client still committed to the +# daemon, or no answer) fails the run with the product's explanation, writes +# no handoff, and keeps the root without cleaning it up: the daemon may still +# be using it, and the operator decides what happens to it. bench_finish() { - local rc=$? + local rc=$? stop_out="" [ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then - "$BINARY" daemon stop >/dev/null 2>&1 || true + if ! stop_out=$("$BINARY" daemon stop 2>&1); then + echo " $LANG: private daemon did not stop; runtime kept at $CBM_TEST_RUNTIME_ROOT, no handoff written" >&2 + [ -z "$stop_out" ] || printf '%s\n' "$stop_out" >&2 + exit 1 + fi printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \ "$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt" echo " $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2