fix(test): fail the index benchmark when its index is refused

bench_finish keeps the private root on a zero exit status, but the index
call swallowed its failure (`|| echo '{"error":"index failed"}'`), so a
refused index exited 0 and, with CBM_BENCH_KEEP_RUNTIME set, kept an
empty root and wrote runtime-root.txt for an index that does not exist.

benchmark-index.sh now records the CLI's exit status, keeps 00-index.json
valid JSON on failure ({"error":"index failed","exit":N}), and exits 1
once every per-run file — the three timing files included — is written,
when the index failed or reported no project (the search-graph twin's own
test). bench_finish then cleans up as its comment always promised, and
the caller's loop notices. The index call keeps its stderr in a
run-private temp file, the twin's shape, and the failure prints it under
`--- index stderr ---` with the first 500 bytes of the response: for a
one-shot CLI, stderr is the only channel a refusal is reported on. A
stale runtime-root.txt from an earlier run in the same results directory
is removed at start, so a failed run leaves no handoff at all.

The contract test's fixture answers `cli index_repository` with a
minimal envelope when CBM_BENCH_PROBE_INDEX_OK is set, so the keep run
covers the successful-index handoff and asserts project.txt; the
unflagged run asserts a non-zero exit and the surfaced cause after a
refused index; a new keep run with a refused index asserts a non-zero
exit, the timing files, no surviving root and no handoff, a pre-seeded
stale one included. Against the previous head the test fails on
"benchmark-index exited 0 after a refused index".

Signed-off-by: Anton Standrik <astandrik@yandex-team.ru>

scripts/benchmark-index.sh

@@ -27,6 +27,7 @@ cbm_test_runtime_init
 # by default, or an unattended run accumulates one root per language.
 bench_finish() {
   local rc=$?
+  [ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true
   if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then
     "$BINARY" daemon stop >/dev/null 2>&1 || true
     printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \
@@ -36,7 +37,13 @@ bench_finish() {
   fi
   cbm_test_runtime_cleanup "$BINARY"
 }
+BENCH_TMP=""
 trap bench_finish EXIT
+# The index call keeps its stderr, the way the search-graph twin does: for a
+# one-shot CLI that is the only channel a refusal is reported on, and without
+# it the failure below could name only its symptom.
+BENCH_TMP=$(mktemp -d)
+INDEX_ERR="$BENCH_TMP/index-stderr.log"
 
 # Resolve symlinks
 REPO=$(cd "$REPO" && pwd -P)
@@ -54,6 +61,10 @@ bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"
 
 OUT="$RESULTS_DIR/$LANG"
 mkdir -p "$OUT"
+# A handoff names this run's index or none: the evaluation loop reuses the
+# results directory, and a stale one would point the graph session at a root
+# step 8 has already removed.
+rm -f -- "$OUT/runtime-root.txt"
 
 echo "INDEX: $LANG ($REPO)"
 
@@ -86,7 +97,12 @@ fi
 # Index via CLI and capture timing
 START_MS=$(bench_now_ms)
 
-INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
+# The CLI's exit status is the index's verdict. It is recorded here and acted
+# on below, once every per-run file is written; an empty response stays valid
+# JSON in 00-index.json so the failure is legible there as well.
+INDEX_RC=0
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>"$INDEX_ERR") ||
+  { INDEX_RC=$?; INDEX_JSON="{\"error\":\"index failed\",\"exit\":$INDEX_RC}"; }
 
 END_MS=$(bench_now_ms)
 ELAPSED=$((END_MS - START_MS))
@@ -130,5 +146,20 @@ echo "$NODES" > "$OUT/nodes.txt"
 echo "$EDGES" > "$OUT/edges.txt"
 echo "$PROJECT" > "$OUT/project.txt"
 
+# An index that failed, or that named no project, fails the run: after the
+# timing files, so the caller keeps its figures, and through the exit status,
+# so the caller's loop notices and bench_finish never keeps a root with no
+# index in it.
+if [ "$INDEX_RC" -ne 0 ] || [ -z "$PROJECT" ]; then
+  echo "  $LANG: index failed (cli exit $INDEX_RC, project '$PROJECT'); response in $OUT/00-index.json" >&2
+  if [ -s "$INDEX_ERR" ]; then
+    echo "--- index stderr ---" >&2
+    cat "$INDEX_ERR" >&2
+  fi
+  echo "--- index response (first 500 bytes) ---" >&2
+  printf '%.500s\n' "$INDEX_JSON" >&2
+  exit 1
+fi
+
 printf "  %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \
   "$LANG" "$FILE_COUNT" "$LOC" "$ELAPSED" "$NODES" "$EDGES"

tests/test_benchmark_runtime_isolation_contract.sh

@@ -9,8 +9,9 @@ set -euo pipefail
 # operator's live store, every one-shot joined the operator's account daemon,
 # and the timings depended on whatever that daemon was doing. Drive both with
 # an environment-probe fixture and require that no product process ever
-# receives the caller's runtime or cache, and that the index benchmark records
-# the setup cost it now pays explicitly.
+# receives the caller's runtime or cache, that the index benchmark records
+# the setup cost it now pays explicitly, and that a refused index fails the
+# run, names its cause, and leaves nothing behind.
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 WORKDIR="$(mktemp -d)"
@@ -35,6 +36,10 @@ cat > "$ENV_PROBE" <<'EOF'
 printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE"
 if [[ "${1-} ${2-}" == "cli index_repository" ]]; then
     [[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG"
+    if [[ -n "${CBM_BENCH_PROBE_INDEX_OK-}" ]]; then
+        printf '%s\n' '{"project":"probe"}'
+        exit 0
+    fi
     echo "probe: index refused" >&2
     exit 1
 fi
@@ -78,30 +83,46 @@ assert_isolated() {
     [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root"
 }
 
+# The fixture refuses the index. That must fail the run — after the timing
+# files are written, so the caller keeps its figures — and name its cause: the
+# fixture's one stderr line has to reach the output.
 INDEX_LOG="$WORKDIR/index-environment.log"
+index_rc=0
 CBM_CACHE_DIR="$CALLER_CACHE" \
 CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
 CBM_BENCH_ENV_PROBE="$INDEX_LOG" \
     "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \
-    > "$WORKDIR/index.out" 2>&1 || true
+    > "$WORKDIR/index.out" 2>&1 || index_rc=$?
+[[ "$index_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index"
 assert_isolated "benchmark-index" "$INDEX_LOG"
 for metric in setup-time total-time index-time; do
     [[ -s "$WORKDIR/results/probe/$metric.txt" ]] ||
         fail "benchmark-index did not record $metric.txt"
 done
+grep -q -- '--- index stderr ---' "$WORKDIR/index.out" ||
+    fail "benchmark-index hid the index stderr behind its own message"
+grep -q 'probe: index refused' "$WORKDIR/index.out" ||
+    fail "benchmark-index did not surface the cause of the index failure"
 
 # The evaluation plan indexes a language and then reads that index from its own
 # MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a
 # successful run must leave its root behind and record, sourceably, the paths
 # that reach it — and they must be the paths the product processes actually
-# used. Unasked, the root is gone (asserted above).
+# used. Unasked, the root is gone (asserted above). The fixture answers this
+# index with a minimal envelope, so what is kept is an index that succeeded.
 KEEP_LOG="$WORKDIR/keep-environment.log"
+keep_rc=0
 CBM_CACHE_DIR="$CALLER_CACHE" \
 CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
 CBM_BENCH_ENV_PROBE="$KEEP_LOG" \
+CBM_BENCH_PROBE_INDEX_OK=1 \
 CBM_BENCH_KEEP_RUNTIME=1 \
     "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \
-    > "$WORKDIR/keep.out" 2>&1 || true
+    > "$WORKDIR/keep.out" 2>&1 || keep_rc=$?
+[[ "$keep_rc" -eq 0 ]] || fail "benchmark-index failed a run whose index succeeded (exit $keep_rc)"
+KEPT_PROJECT=$(cat "$WORKDIR/results/keep/project.txt" 2>/dev/null || true)
+[[ "${KEPT_PROJECT%$'\r'}" == "probe" ]] ||
+    fail "benchmark-index did not record the project the index reported: '${KEPT_PROJECT:-<none>}'"
 HANDOFF="$WORKDIR/results/keep/runtime-root.txt"
 [[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff"
 KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF")
@@ -115,6 +136,29 @@ grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" ||
     fail "runtime-root.txt does not name the runtime and cache the product processes used"
 rm -rf -- "$KEPT_ROOT"
 
+# Asked to keep the runtime and refused the index, the harness has nothing worth
+# keeping: the run fails, no root survives, and no handoff is written — nor left
+# over from an earlier run, since the evaluation loop reuses the results
+# directory.
+REFUSED_LOG="$WORKDIR/keep-refused-environment.log"
+REFUSED_HANDOFF="$WORKDIR/results/keep-refused/runtime-root.txt"
+mkdir -p "${REFUSED_HANDOFF%/*}"
+echo 'CBM_BENCH_RUNTIME_ROOT=/stale/root/from/an/earlier/run' > "$REFUSED_HANDOFF"
+refused_rc=0
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$REFUSED_LOG" \
+CBM_BENCH_KEEP_RUNTIME=1 \
+    "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep-refused "$REPO" "$WORKDIR/results" \
+    > "$WORKDIR/keep-refused.out" 2>&1 || refused_rc=$?
+[[ "$refused_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index it was asked to keep"
+for metric in setup-time total-time index-time; do
+    [[ -s "$WORKDIR/results/keep-refused/$metric.txt" ]] ||
+        fail "benchmark-index did not record $metric.txt for a refused index"
+done
+assert_isolated "benchmark-index (keep, refused index)" "$REFUSED_LOG"
+[[ ! -e "$REFUSED_HANDOFF" ]] || fail "benchmark-index left a handoff for an index that failed"
+
 SEARCH_LOG="$WORKDIR/search-environment.log"
 CBM_CACHE_DIR="$CALLER_CACHE" \
 CBM_RUNTIME_DIR="$CALLER_RUNTIME" \