From a2ac0540c1d10cde927707717f8cec2088e245bb Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Tue, 22 Sep 2026 13:57:30 +0300 Subject: [PATCH] fix(test): fail the index benchmark when its index is refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bench_finish keeps the private root on a zero exit status, but the index call swallowed its failure (`|| echo '{"error":"index failed"}'`), so a refused index exited 0 and, with CBM_BENCH_KEEP_RUNTIME set, kept an empty root and wrote runtime-root.txt for an index that does not exist. benchmark-index.sh now records the CLI's exit status, keeps 00-index.json valid JSON on failure ({"error":"index failed","exit":N}), and exits 1 once every per-run file — the three timing files included — is written, when the index failed or reported no project (the search-graph twin's own test). bench_finish then cleans up as its comment always promised, and the caller's loop notices. The index call keeps its stderr in a run-private temp file, the twin's shape, and the failure prints it under `--- index stderr ---` with the first 500 bytes of the response: for a one-shot CLI, stderr is the only channel a refusal is reported on. A stale runtime-root.txt from an earlier run in the same results directory is removed at start, so a failed run leaves no handoff at all. The contract test's fixture answers `cli index_repository` with a minimal envelope when CBM_BENCH_PROBE_INDEX_OK is set, so the keep run covers the successful-index handoff and asserts project.txt; the unflagged run asserts a non-zero exit and the surfaced cause after a refused index; a new keep run with a refused index asserts a non-zero exit, the timing files, no surviving root and no handoff, a pre-seeded stale one included. Against the previous head the test fails on "benchmark-index exited 0 after a refused index". Signed-off-by: Anton Standrik --- scripts/benchmark-index.sh | 33 +++++++++++- ...st_benchmark_runtime_isolation_contract.sh | 54 +++++++++++++++++-- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 11d79ee1f..207944a85 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -27,6 +27,7 @@ cbm_test_runtime_init # by default, or an unattended run accumulates one root per language. bench_finish() { local rc=$? + [ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then "$BINARY" daemon stop >/dev/null 2>&1 || true printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \ @@ -36,7 +37,13 @@ bench_finish() { fi cbm_test_runtime_cleanup "$BINARY" } +BENCH_TMP="" trap bench_finish EXIT +# The index call keeps its stderr, the way the search-graph twin does: for a +# one-shot CLI that is the only channel a refusal is reported on, and without +# it the failure below could name only its symptom. +BENCH_TMP=$(mktemp -d) +INDEX_ERR="$BENCH_TMP/index-stderr.log" # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) @@ -54,6 +61,10 @@ bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)" OUT="$RESULTS_DIR/$LANG" mkdir -p "$OUT" +# A handoff names this run's index or none: the evaluation loop reuses the +# results directory, and a stale one would point the graph session at a root +# step 8 has already removed. +rm -f -- "$OUT/runtime-root.txt" echo "INDEX: $LANG ($REPO)" @@ -86,7 +97,12 @@ fi # Index via CLI and capture timing START_MS=$(bench_now_ms) -INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}') +# The CLI's exit status is the index's verdict. It is recorded here and acted +# on below, once every per-run file is written; an empty response stays valid +# JSON in 00-index.json so the failure is legible there as well. +INDEX_RC=0 +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>"$INDEX_ERR") || + { INDEX_RC=$?; INDEX_JSON="{\"error\":\"index failed\",\"exit\":$INDEX_RC}"; } END_MS=$(bench_now_ms) ELAPSED=$((END_MS - START_MS)) @@ -130,5 +146,20 @@ echo "$NODES" > "$OUT/nodes.txt" echo "$EDGES" > "$OUT/edges.txt" echo "$PROJECT" > "$OUT/project.txt" +# An index that failed, or that named no project, fails the run: after the +# timing files, so the caller keeps its figures, and through the exit status, +# so the caller's loop notices and bench_finish never keeps a root with no +# index in it. +if [ "$INDEX_RC" -ne 0 ] || [ -z "$PROJECT" ]; then + echo " $LANG: index failed (cli exit $INDEX_RC, project '$PROJECT'); response in $OUT/00-index.json" >&2 + if [ -s "$INDEX_ERR" ]; then + echo "--- index stderr ---" >&2 + cat "$INDEX_ERR" >&2 + fi + echo "--- index response (first 500 bytes) ---" >&2 + printf '%.500s\n' "$INDEX_JSON" >&2 + exit 1 +fi + printf " %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \ "$LANG" "$FILE_COUNT" "$LOC" "$ELAPSED" "$NODES" "$EDGES" diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh index 73c1758f2..f2e862d0d 100755 --- a/tests/test_benchmark_runtime_isolation_contract.sh +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -9,8 +9,9 @@ set -euo pipefail # operator's live store, every one-shot joined the operator's account daemon, # and the timings depended on whatever that daemon was doing. Drive both with # an environment-probe fixture and require that no product process ever -# receives the caller's runtime or cache, and that the index benchmark records -# the setup cost it now pays explicitly. +# receives the caller's runtime or cache, that the index benchmark records +# the setup cost it now pays explicitly, and that a refused index fails the +# run, names its cause, and leaves nothing behind. ROOT="$(cd "$(dirname "$0")/.." && pwd)" WORKDIR="$(mktemp -d)" @@ -35,6 +36,10 @@ cat > "$ENV_PROBE" <<'EOF' printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE" if [[ "${1-} ${2-}" == "cli index_repository" ]]; then [[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG" + if [[ -n "${CBM_BENCH_PROBE_INDEX_OK-}" ]]; then + printf '%s\n' '{"project":"probe"}' + exit 0 + fi echo "probe: index refused" >&2 exit 1 fi @@ -78,30 +83,46 @@ assert_isolated() { [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root" } +# The fixture refuses the index. That must fail the run — after the timing +# files are written, so the caller keeps its figures — and name its cause: the +# fixture's one stderr line has to reach the output. INDEX_LOG="$WORKDIR/index-environment.log" +index_rc=0 CBM_CACHE_DIR="$CALLER_CACHE" \ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ CBM_BENCH_ENV_PROBE="$INDEX_LOG" \ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \ - > "$WORKDIR/index.out" 2>&1 || true + > "$WORKDIR/index.out" 2>&1 || index_rc=$? +[[ "$index_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index" assert_isolated "benchmark-index" "$INDEX_LOG" for metric in setup-time total-time index-time; do [[ -s "$WORKDIR/results/probe/$metric.txt" ]] || fail "benchmark-index did not record $metric.txt" done +grep -q -- '--- index stderr ---' "$WORKDIR/index.out" || + fail "benchmark-index hid the index stderr behind its own message" +grep -q 'probe: index refused' "$WORKDIR/index.out" || + fail "benchmark-index did not surface the cause of the index failure" # The evaluation plan indexes a language and then reads that index from its own # MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a # successful run must leave its root behind and record, sourceably, the paths # that reach it — and they must be the paths the product processes actually -# used. Unasked, the root is gone (asserted above). +# used. Unasked, the root is gone (asserted above). The fixture answers this +# index with a minimal envelope, so what is kept is an index that succeeded. KEEP_LOG="$WORKDIR/keep-environment.log" +keep_rc=0 CBM_CACHE_DIR="$CALLER_CACHE" \ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ CBM_BENCH_ENV_PROBE="$KEEP_LOG" \ +CBM_BENCH_PROBE_INDEX_OK=1 \ CBM_BENCH_KEEP_RUNTIME=1 \ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \ - > "$WORKDIR/keep.out" 2>&1 || true + > "$WORKDIR/keep.out" 2>&1 || keep_rc=$? +[[ "$keep_rc" -eq 0 ]] || fail "benchmark-index failed a run whose index succeeded (exit $keep_rc)" +KEPT_PROJECT=$(cat "$WORKDIR/results/keep/project.txt" 2>/dev/null || true) +[[ "${KEPT_PROJECT%$'\r'}" == "probe" ]] || + fail "benchmark-index did not record the project the index reported: '${KEPT_PROJECT:-}'" HANDOFF="$WORKDIR/results/keep/runtime-root.txt" [[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff" KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF") @@ -115,6 +136,29 @@ grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" || fail "runtime-root.txt does not name the runtime and cache the product processes used" rm -rf -- "$KEPT_ROOT" +# Asked to keep the runtime and refused the index, the harness has nothing worth +# keeping: the run fails, no root survives, and no handoff is written — nor left +# over from an earlier run, since the evaluation loop reuses the results +# directory. +REFUSED_LOG="$WORKDIR/keep-refused-environment.log" +REFUSED_HANDOFF="$WORKDIR/results/keep-refused/runtime-root.txt" +mkdir -p "${REFUSED_HANDOFF%/*}" +echo 'CBM_BENCH_RUNTIME_ROOT=/stale/root/from/an/earlier/run' > "$REFUSED_HANDOFF" +refused_rc=0 +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$REFUSED_LOG" \ +CBM_BENCH_KEEP_RUNTIME=1 \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep-refused "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/keep-refused.out" 2>&1 || refused_rc=$? +[[ "$refused_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index it was asked to keep" +for metric in setup-time total-time index-time; do + [[ -s "$WORKDIR/results/keep-refused/$metric.txt" ]] || + fail "benchmark-index did not record $metric.txt for a refused index" +done +assert_isolated "benchmark-index (keep, refused index)" "$REFUSED_LOG" +[[ ! -e "$REFUSED_HANDOFF" ]] || fail "benchmark-index left a handoff for an index that failed" + SEARCH_LOG="$WORKDIR/search-environment.log" CBM_CACHE_DIR="$CALLER_CACHE" \ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \