From 567ed5d87e807f6463ed90f324137b6794fa23ae Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Mon, 21 Sep 2026 12:25:28 +0300 Subject: [PATCH] fix(test): hand the benchmark index to the caller on request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language with benchmark-index.sh and then answers graph questions against that index from its own MCP session. With a run-private runtime (#1696) the index was gone before that session could start, so the handoff is now explicit and opt-in. With CBM_BENCH_KEEP_RUNTIME set, a SUCCESSFUL run stops its daemon, leaves the private root in place and records the paths that reach it in //runtime-root.txt (sourceable: CBM_BENCH_RUNTIME_ROOT, CBM_RUNTIME_DIR, CBM_CACHE_DIR). Ownership of that root, including its removal, passes to the caller. A failed run cleans up regardless: there is no index worth keeping and nothing may leak. Nothing changes in the default path or in the three timing files. §7's skeleton sets the flag, sources the file before the graph session and removes the root in step 8 instead of the live-store *.db files. The contract test drives a kept run and requires the root to survive, the handoff file to be sourceable, and its paths to be the ones the product processes actually used; the unflagged run still asserts the root is gone. The kept-run assertions fail against the previous head. Signed-off-by: Anton Standrik --- docs/EVALUATION_PLAN.md | 7 +++++- scripts/benchmark-index.sh | 23 ++++++++++++++++- ...st_benchmark_runtime_isolation_contract.sh | 25 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md index efd56207d..01398daca 100644 --- a/docs/EVALUATION_PLAN.md +++ b/docs/EVALUATION_PLAN.md @@ -283,8 +283,13 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list # --- step 2: cold index in the main channel, TIMED (key metric) --- t0=$(now_ms) + CBM_BENCH_KEEP_RUNTIME=1 \ scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results index_ms=$(( $(now_ms) - t0 )) # clone+index wall-clock → manifest + report (§5) + # The harness indexes into a run-private root (#1696). CBM_BENCH_KEEP_RUNTIME leaves that + # root behind on success and records its paths; the graph session in steps 4-7 must be + # started with these CBM_RUNTIME_DIR / CBM_CACHE_DIR, and step 8 removes the root. + set -a; . /tmp/eval-results/"$lang"/runtime-root.txt; set +a # --- step 3: record per-type histograms (zeros back-filled) --- # node-types.json, edge-types.json (every label + all 32 edge types, zeros kept, §7 below) @@ -293,7 +298,7 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list # + per-language report + (deferred, blind) judge --- # --- step 8: delete THIS language's index so the next is cold, then mark done --- - rm -f ~/.cache/codebase-memory-mcp/*.db + rm -rf -- "$CBM_BENCH_RUNTIME_ROOT" manifest_mark_done "$lang" "$index_ms" done ``` diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 0bc24af2f..11d79ee1f 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -15,7 +15,28 @@ RESULTS_DIR="${4:?}" # shellcheck source=test-runtime.sh source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" cbm_test_runtime_init -trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + +# The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language here and +# then answers graph questions against that index from its own MCP session. +# A run-private root would be gone before that session starts, so the caller +# may ask for it to be kept: after a SUCCESSFUL run the harness stops its +# daemon, leaves the root in place, records the paths that reach it in +# //runtime-root.txt (sourceable), and ownership of the root — +# including its removal — passes to the caller. A failed run cleans up +# regardless: there is no index worth keeping, and nothing must leak. Never on +# by default, or an unattended run accumulates one root per language. +bench_finish() { + local rc=$? + if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then + "$BINARY" daemon stop >/dev/null 2>&1 || true + printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \ + "$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt" + echo " $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2 + return 0 + fi + cbm_test_runtime_cleanup "$BINARY" +} +trap bench_finish EXIT # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh index 1dc618ae6..73c1758f2 100755 --- a/tests/test_benchmark_runtime_isolation_contract.sh +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -90,6 +90,31 @@ for metric in setup-time total-time index-time; do fail "benchmark-index did not record $metric.txt" done +# The evaluation plan indexes a language and then reads that index from its own +# MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a +# successful run must leave its root behind and record, sourceably, the paths +# that reach it — and they must be the paths the product processes actually +# used. Unasked, the root is gone (asserted above). +KEEP_LOG="$WORKDIR/keep-environment.log" +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$KEEP_LOG" \ +CBM_BENCH_KEEP_RUNTIME=1 \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/keep.out" 2>&1 || true +HANDOFF="$WORKDIR/results/keep/runtime-root.txt" +[[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff" +KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF") +KEPT_RUNTIME=$(bash -c '. "$1" && printf "%s" "${CBM_RUNTIME_DIR-}"' _ "$HANDOFF") +KEPT_CACHE=$(bash -c '. "$1" && printf "%s" "${CBM_CACHE_DIR-}"' _ "$HANDOFF") +[[ -n "$KEPT_ROOT" && -d "$KEPT_ROOT" && ! -L "$KEPT_ROOT" ]] || + fail "benchmark-index did not keep its runtime root: ${KEPT_ROOT:-}" +[[ -d "$KEPT_ROOT/cache" && -d "$KEPT_ROOT/runtime" ]] || + fail "the kept root $KEPT_ROOT lost its cache or runtime directory" +grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" || + fail "runtime-root.txt does not name the runtime and cache the product processes used" +rm -rf -- "$KEPT_ROOT" + SEARCH_LOG="$WORKDIR/search-environment.log" CBM_CACHE_DIR="$CALLER_CACHE" \ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \