diff --git a/.gitignore b/.gitignore
index 3705501c2..1533c93e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -51,6 +51,9 @@ graph-ui/dist/
# Generated by scripts/embed-frontend.sh into the source tree (the linker needs
# it beside the other UI sources). Removed by scripts/clean.sh; never committed.
src/ui/embedded_assets.c
+# Raw coverage profiles. A coverage-instrumented binary (make test-runner-cov,
+# cbm-cov) run without LLVM_PROFILE_FILE drops default.profraw in its cwd.
+*.profraw
# Generated reports
BENCHMARK_REPORT.md
diff --git a/Makefile.cbm b/Makefile.cbm
index ab5c89db7..b59c5ed00 100644
--- a/Makefile.cbm
+++ b/Makefile.cbm
@@ -71,13 +71,29 @@ ifeq ($(TEST_SEAMS),1)
TEST_SEAM_DEFINE := -DCBM_ENABLE_TEST_SEAMS=1
endif
+# Per-test coverage for the test-impact map. TEST_COVERAGE=1 compiles clang
+# source coverage into OUR code only -- the one-shot sources, the LSP unity
+# object and the preprocessor -- in the test runner and in the product binary,
+# and turns on the runner's per-test profile hooks (tests/test_framework.h).
+# Vendored objects stay uninstrumented: a vendored change selects tests by rule,
+# and every counter compiled in is written out again for every test. Like every
+# other flavour it needs its own BUILD_DIR; the test-runner-cov and cbm-cov
+# targets pin one.
+TEST_COVERAGE ?= 0
+TEST_COVERAGE_FLAGS :=
+TEST_COVERAGE_DEFINE :=
+ifeq ($(TEST_COVERAGE),1)
+TEST_COVERAGE_FLAGS := -fprofile-instr-generate -fcoverage-mapping
+TEST_COVERAGE_DEFINE := -DCBM_TEST_COVERAGE=1
+endif
+
# Production flags (CFLAGS_EXTRA allows CI to inject -DCBM_VERSION)
# CBM_BIND_TS_ALLOCATOR=1: bind the tree-sitter runtime to mimalloc (#424). Only
# the prod build uses mimalloc (MI_OVERRIDE=1); the test build is CRT+ASan, where
# binding would create an alloc/free mismatch, so the guard is prod-only.
CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(TEST_SEAM_DEFINE) \
- $(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(CFLAGS_EXTRA)
-CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS)
+ $(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(TEST_COVERAGE_FLAGS) $(CFLAGS_EXTRA)
+CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS) $(TEST_COVERAGE_FLAGS)
# Test flags: debug + sanitizers (override SANITIZE= to disable on Windows)
SANITIZE = -fsanitize=address,undefined -fno-omit-frame-pointer
@@ -102,8 +118,10 @@ INCREMENTAL_TEST_DEFINE = -DCBM_INCREMENTAL_TEST_API=1
COVERAGE_MARKER_TEST_DEFINE = -DCBM_COVERAGE_MARKER_TEST_API=1
CFLAGS_TEST = $(CFLAGS_COMMON) $(EDITOR_TEST_DEFINES) $(SANITIZED_DEFINE) \
$(KOTLIN_DEDUP_TEST_DEFINE) $(CALL_REFERENCE_LOOKUP_TEST_DEFINE) \
- $(INCREMENTAL_TEST_DEFINE) $(COVERAGE_MARKER_TEST_DEFINE) -g -O1 $(SANITIZE)
-CXXFLAGS_TEST = $(CXXFLAGS_COMMON) $(SANITIZED_DEFINE) -g -O1 $(SANITIZE) $(CXX_STDLIB_FLAGS)
+ $(INCREMENTAL_TEST_DEFINE) $(COVERAGE_MARKER_TEST_DEFINE) $(TEST_COVERAGE_DEFINE) \
+ -g -O1 $(SANITIZE) $(TEST_COVERAGE_FLAGS)
+CXXFLAGS_TEST = $(CXXFLAGS_COMMON) $(SANITIZED_DEFINE) -g -O1 $(SANITIZE) $(TEST_COVERAGE_FLAGS) \
+ $(CXX_STDLIB_FLAGS)
# TSan (can't combine with ASan)
TSAN_SANITIZE = -fsanitize=thread -fno-omit-frame-pointer
@@ -851,7 +869,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o
# ── Targets ──────────────────────────────────────────────────────
-.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz
+.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke test-runner-cov cbm-cov cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz
$(BUILD_DIR):
mkdir -p $(BUILD_DIR)
@@ -890,7 +908,7 @@ $(BUILD_DIR)/ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR)
$(CC) $(GRAMMAR_CFLAGS_TEST) -c -o $@ $<
$(BUILD_DIR)/lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)
- $(CC) $(GRAMMAR_CFLAGS_TEST) $(SANITIZE) -c -o $@ $<
+ $(CC) $(GRAMMAR_CFLAGS_TEST) $(SANITIZE) $(TEST_COVERAGE_FLAGS) -c -o $@ $<
$(BUILD_DIR)/preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
$(CXX) $(CXXFLAGS_TEST) -w -I$(CBM_DIR)/vendored -c -o $@ $<
@@ -1132,7 +1150,7 @@ $(BUILD_DIR)/prod_ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR)
$(CC) $(GRAMMAR_CFLAGS) -c -o $@ $<
$(BUILD_DIR)/prod_lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)
- $(CC) $(GRAMMAR_CFLAGS) -c -o $@ $<
+ $(CC) $(GRAMMAR_CFLAGS) $(TEST_COVERAGE_FLAGS) -c -o $@ $<
$(BUILD_DIR)/prod_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
$(CXX) $(CXXFLAGS_PROD) -w -I$(CBM_DIR)/vendored -c -o $@ $<
@@ -1169,7 +1187,7 @@ MAIN_SRC = src/main.c
# for trap debugging -- silently re-run the previously instrumented binary,
# which during this bisect produced a "crashes without sanitizers too" result
# that was pure artifact and sent the investigation down the wrong path.
-BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE)
+BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE)|TEST_COVERAGE=$(TEST_COVERAGE)
.PHONY: build-config-check
build-config-check:
$(BUILD_DIR)/.build-config: build-config-check | $(BUILD_DIR)
@@ -1365,6 +1383,30 @@ test-lsan:
cd $(CURDIR) && ASAN_OPTIONS="detect_leaks=1:halt_on_error=1" \
./build/lsan/test-runner $(LSAN_SUITES)
+# ── Per-test coverage build (test-impact map) ───────────────────
+# The instrumented runner and product binary (TEST_COVERAGE=1 above), in their
+# own build directory. Homebrew LLVM rather than Apple clang: the raw profile
+# format is versioned with the compiler, so llvm-profdata and llvm-cov have to
+# come from the toolchain that built the binary.
+# LLVM_PROFILE_FILE=/dev/null CBM_TEST_COVERAGE_DIR=
\
+# build/cov/test-runner [suite...]
+# leaves one profile set per test (tests/test_main.c). The runner names every
+# profile itself, but the profile runtime creates its default file before main
+# runs: started without LLVM_PROFILE_FILE it leaves an empty default.profraw in
+# the working directory.
+# No C suite runs the product binary -- they re-exec the runner itself. The
+# shell process regressions do, and they find the instrumented one the way they
+# find any other: CBM_TEST_BINARY=build/cov/codebase-memory-mcp, with
+# LLVM_PROFILE_FILE set by whoever collects the profiles.
+COV_LLVM_BIN ?= /opt/homebrew/opt/llvm/bin
+COV_BUILD_DIR ?= build/cov
+COV_MAKE = $(MAKE) -f Makefile.cbm CC=$(COV_LLVM_BIN)/clang CXX=$(COV_LLVM_BIN)/clang++ \
+ BUILD_DIR=$(COV_BUILD_DIR) SANITIZE= TEST_COVERAGE=1
+test-runner-cov:
+ $(COV_MAKE) $(COV_BUILD_DIR)/test-runner
+cbm-cov:
+ $(COV_MAKE) $(COV_BUILD_DIR)/codebase-memory-mcp TEST_SEAMS=1
+
# ── Coverage-guided fuzzing (libFuzzer) ─────────────────────────
# One binary, three targets (tests/fuzz/cbm_fuzz.c): the per-file extractor,
# the Cypher parser, the TOML/YAML/JSON config editors. ASan+UBSan as in the
diff --git a/scripts/test.sh b/scripts/test.sh
index eae756138..81483c115 100755
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -381,6 +381,15 @@ BUILD_DIR="$BUILD_DIR" scripts/clean.sh
# CBM_TEST_SEQUENTIAL=1 restores the single-process runner.
make -j"$NPROC" -f Makefile.cbm "$BUILD_DIR/test-runner" ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}
assert_test_runner_build_config "$BUILD_DIR/test-runner" "$EXPECTED_SANITIZED"
+
+# Step 2b: per-test selection process regression. CBM_TEST_ONLY must run exactly
+# the tests it names and fail on a token that matches nothing; a suite cannot
+# assert that about the runner executing it. Runs against the runner just built
+# and BEFORE the suites: it takes seconds, and a selection that silently drops
+# tests is a property of the harness every later result depends on.
+echo "=== Step 2b: per-test selection regression (CBM_TEST_ONLY) ==="
+CBM_TEST_RUNNER="$ROOT/$BUILD_DIR/test-runner" bash "$ROOT/tests/test_harness_test_only.sh"
+
if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; then
make -f Makefile.cbm test ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}
else
diff --git a/tests/repro/repro_main.c b/tests/repro/repro_main.c
index 7fc1cc20b..d164fce90 100644
--- a/tests/repro/repro_main.c
+++ b/tests/repro/repro_main.c
@@ -24,12 +24,29 @@
int tf_pass_count = 0;
int tf_fail_count = 0;
int tf_skip_count = 0;
+int tf_deselected_count = 0;
#include "test_framework.h"
#include "repro_runner.h"
#include "foundation/compat.h" /* cbm_setenv — #845 supervisor kill switch */
#include "daemon/bootstrap.h" /* refuse the developer's default daemon rendezvous */
+/* Per-test selection (CBM_TEST_ONLY) and per-test coverage belong to the gating
+ * runner, tests/test_main.c. This board selects whole suites with
+ * CBM_REPRO_ONLY, and every test of a selected suite runs. */
+bool tf_test_selected(const char *test) {
+ (void)test;
+ return true;
+}
+#ifdef CBM_TEST_COVERAGE
+void tf_coverage_test_begin(const char *test) {
+ (void)test;
+}
+void tf_coverage_test_end(const char *test) {
+ (void)test;
+}
+#endif
+
/* Per-suite summary + filter. RUN_SUITE prints a one-line
* "[SUITE] P passed, F failed" report (greppable for which suites still
* have reds). The shared selector implementation lives with its gating tests in
diff --git a/tests/test_framework.h b/tests/test_framework.h
index 2af654b4d..ac9539a6d 100644
--- a/tests/test_framework.h
+++ b/tests/test_framework.h
@@ -6,6 +6,7 @@
* - Test suite grouping
* - Pass/fail counting with summary
* - Color output (when isatty)
+ * - Per-test selection (CBM_TEST_ONLY) and per-test coverage profiles
*
* Usage:
* TEST(my_test) {
@@ -26,6 +27,7 @@
#ifndef TEST_FRAMEWORK_H
#define TEST_FRAMEWORK_H
+#include
#include
#include
#include
@@ -37,6 +39,32 @@
extern int tf_pass_count;
extern int tf_fail_count;
extern int tf_skip_count;
+extern int tf_deselected_count;
+
+/* ── Per-test selection (defined by the runner) ────────────────── */
+
+/* CBM_TEST_ONLY / CBM_TEST_ONLY_FILE narrow a run to `suite:test` tokens (see
+ * test_main.c). tf_suite_enter decides whether a suite's body executes at all;
+ * inside it tf_test_selected decides each RUN_TEST and counts the tests it
+ * leaves out. Without a selection both are always true. */
+extern const char *tf_current_suite;
+bool tf_suite_enter(const char *suite);
+void tf_suite_leave(const char *suite);
+bool tf_test_selected(const char *test);
+
+/* ── Per-test coverage (CBM_TEST_COVERAGE builds only) ─────────── */
+
+/* A coverage-instrumented runner (make test-runner-cov) writes one profile per
+ * test under CBM_TEST_COVERAGE_DIR. No other build compiles any of it. */
+#ifdef CBM_TEST_COVERAGE
+void tf_coverage_test_begin(const char *test);
+void tf_coverage_test_end(const char *test);
+#define TF_COVERAGE_TEST_BEGIN(test) tf_coverage_test_begin(test)
+#define TF_COVERAGE_TEST_END(test) tf_coverage_test_end(test)
+#else
+#define TF_COVERAGE_TEST_BEGIN(test) ((void)0)
+#define TF_COVERAGE_TEST_END(test) ((void)0)
+#endif
/* ── Color helpers ─────────────────────────────────────────────── */
@@ -69,10 +97,10 @@ static inline const char *tf_reset(void) {
/* Hard failure with a message — for setup/environment failures that must NOT be
* silently skipped (a test that cannot establish its preconditions has FAILED,
* not "skipped"). Mirrors the ASSERT failure path (red FAIL + file:line). */
-#define FAIL(reason) \
- do { \
+#define FAIL(reason) \
+ do { \
printf(" %sFAIL%s %s:%d: %s\n", tf_red(), tf_reset(), __FILE__, __LINE__, (reason)); \
- return 1; \
+ return 1; \
} while (0)
/* The ONLY tolerable skip: a test that is inherently platform-specific and does
@@ -224,9 +252,14 @@ static inline const char *tf_reset(void) {
#define RUN_TEST(name) \
do { \
+ if (!tf_test_selected(#name)) { \
+ break; \
+ } \
printf(" %-55s", #name); \
fflush(stdout); \
+ TF_COVERAGE_TEST_BEGIN(#name); \
int _result = test_##name(); \
+ TF_COVERAGE_TEST_END(#name); \
if (_result == 0) { \
printf("%sPASS%s\n", tf_green(), tf_reset()); \
tf_pass_count++; \
@@ -243,22 +276,28 @@ static inline const char *tf_reset(void) {
#define RUN_SUITE(name) \
do { \
+ if (!tf_suite_enter(#name)) { \
+ break; \
+ } \
printf("\n%s=== %s ===%s\n", tf_dim(), #name, tf_reset()); \
suite_##name(); \
+ tf_suite_leave(#name); \
} while (0)
/* ── Summary ───────────────────────────────────────────────────── */
-#define TEST_SUMMARY() \
- do { \
- printf("\n────────────────────────────────────────────\n"); \
- printf(" %s%d passed%s", tf_green(), tf_pass_count, tf_reset()); \
- if (tf_fail_count > 0) \
- printf(", %s%d failed%s", tf_red(), tf_fail_count, tf_reset()); \
- if (tf_skip_count > 0) \
- printf(", %s%d skipped%s", tf_dim(), tf_skip_count, tf_reset()); \
- printf("\n────────────────────────────────────────────\n\n"); \
- return tf_fail_count > 0 ? 1 : 0; \
+#define TEST_SUMMARY() \
+ do { \
+ printf("\n────────────────────────────────────────────\n"); \
+ printf(" %s%d passed%s", tf_green(), tf_pass_count, tf_reset()); \
+ if (tf_fail_count > 0) \
+ printf(", %s%d failed%s", tf_red(), tf_fail_count, tf_reset()); \
+ if (tf_skip_count > 0) \
+ printf(", %s%d skipped%s", tf_dim(), tf_skip_count, tf_reset()); \
+ if (tf_deselected_count > 0) \
+ printf(", %s%d deselected%s", tf_dim(), tf_deselected_count, tf_reset()); \
+ printf("\n────────────────────────────────────────────\n\n"); \
+ return tf_fail_count > 0 ? 1 : 0; \
} while (0)
#endif /* TEST_FRAMEWORK_H */
diff --git a/tests/test_harness_test_only.sh b/tests/test_harness_test_only.sh
new file mode 100644
index 000000000..04318308b
--- /dev/null
+++ b/tests/test_harness_test_only.sh
@@ -0,0 +1,249 @@
+#!/usr/bin/env bash
+# Per-test selection in the C test runner: CBM_TEST_ONLY / CBM_TEST_ONLY_FILE.
+#
+# The test-impact selector hands the runner `suite:test` tokens. Two properties
+# make that safe to gate on, and both are process-level behaviour a suite cannot
+# assert about the runner that is executing it:
+# 1. exactly the named tests run: the other tests of a named suite are counted
+# as deselected, and a suite no token names never executes its body;
+# 2. a token that matches nothing is an ERROR. A selection that quietly ran
+# fewer tests than it named would report green for work that never ran.
+#
+# Every case drives the built runner over two small in-memory suites, so the
+# whole script is a dozen runner starts. Expected counts are read from the suite
+# sources, never hard-coded, so adding a test to either suite cannot stale them.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+# scripts/test.sh builds into $BUILD_DIR, which is not build/c on every leg; it
+# passes the runner it built. The default keeps a bare manual invocation working.
+RUNNER="${CBM_TEST_RUNNER:-${ROOT}/build/c/test-runner}"
+if [[ ! -x "${RUNNER}" && -x "${RUNNER}.exe" ]]; then
+ RUNNER="${RUNNER}.exe"
+fi
+if [[ ! -x "${RUNNER}" ]]; then
+ echo "missing test runner: ${RUNNER}" >&2
+ exit 2
+fi
+
+# A selection inherited from the caller would decide every case below.
+unset CBM_TEST_ONLY CBM_TEST_ONLY_FILE
+
+NAMED_SUITE=dyn_array
+OTHER_SUITE=str_intern
+UNKNOWN_MESSAGE="selected test not compiled into this build or unknown:"
+
+run_test_count() {
+ awk '/^[[:space:]]*RUN_TEST\(/ { n++ } END { print n + 0 }' "$1"
+}
+first_run_test() {
+ awk -F'[()]' '/^[[:space:]]*RUN_TEST\(/ { print $2; exit }' "$1"
+}
+NAMED_TOTAL="$(run_test_count "${ROOT}/tests/test_${NAMED_SUITE}.c")"
+OTHER_TOTAL="$(run_test_count "${ROOT}/tests/test_${OTHER_SUITE}.c")"
+OTHER_TEST="$(first_run_test "${ROOT}/tests/test_${OTHER_SUITE}.c")"
+if [[ "${NAMED_TOTAL}" -lt 3 || "${OTHER_TOTAL}" -lt 1 || -z "${OTHER_TEST}" ]]; then
+ echo "fixture suites changed shape: ${NAMED_SUITE}=${NAMED_TOTAL} ${OTHER_SUITE}=${OTHER_TOTAL}" >&2
+ exit 2
+fi
+
+tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/cbm-test-only.XXXXXX")"
+trap 'rm -rf "${tmpdir}"' EXIT
+
+FAILURES=0
+RC=0
+OUT=""
+
+# run [suite...] — runs the runner under the caller's VAR=VAL prefix and
+# records its merged output and exit status. CR is stripped: the Windows CRT
+# ends every stdout line with CRLF.
+run() {
+ local name="$1"
+ shift
+ OUT="${tmpdir}/${name}.out"
+ set +e
+ "${RUNNER}" "$@" > "${OUT}.raw" 2>&1
+ RC=$?
+ set -e
+ tr -d '\r' < "${OUT}.raw" > "${OUT}"
+}
+
+# The summary line, e.g. " 1 passed, 25 deselected".
+summary_count() {
+ awk -v label="$1" '
+ /^ [0-9]+ passed/ {
+ n = 0
+ count = split($0, part, ", ")
+ for (i = 1; i <= count; i++) {
+ gsub(/^ +/, "", part[i])
+ if (split(part[i], field, " ") == 2 && field[2] == label) n = field[1]
+ }
+ found = 1
+ }
+ END { print found ? n : "none" }
+ ' "${OUT}"
+}
+pass_lines() {
+ awk '/PASS$/ { n++ } END { print n + 0 }' "${OUT}"
+}
+has_line() {
+ grep -F -- "$1" "${OUT}" > /dev/null
+}
+
+CASE=""
+CASE_OK=1
+begin() {
+ CASE="$1"
+ CASE_OK=1
+}
+expect() {
+ local what="$1" actual="$2" wanted="$3"
+ if [[ "${actual}" != "${wanted}" ]]; then
+ echo " FAIL ${CASE}: ${what} = ${actual}, expected ${wanted}"
+ CASE_OK=0
+ fi
+}
+expect_nonzero_exit() {
+ if [[ "${RC}" -eq 0 ]]; then
+ echo " FAIL ${CASE}: runner exited 0, expected a nonzero exit"
+ CASE_OK=0
+ fi
+}
+expect_line() {
+ if ! has_line "$1"; then
+ echo " FAIL ${CASE}: output lacks: $1"
+ CASE_OK=0
+ fi
+}
+expect_no_line() {
+ if has_line "$1"; then
+ echo " FAIL ${CASE}: output unexpectedly has: $1"
+ CASE_OK=0
+ fi
+}
+finish() {
+ if [[ "${CASE_OK}" -eq 1 ]]; then
+ echo " ok ${CASE}"
+ else
+ FAILURES=$((FAILURES + 1))
+ echo " ---- ${CASE}: runner exit ${RC}, output tail ----"
+ tail -6 "${OUT}" | sed 's/^/ | /'
+ fi
+}
+
+echo "per-test selection: ${RUNNER}"
+
+# ── No filter: every test of the requested suites runs, nothing is deselected ──
+begin "no filter runs every test"
+run unfiltered "${NAMED_SUITE}" "${OTHER_SUITE}"
+expect "exit status" "${RC}" 0
+expect "passed" "$(summary_count passed)" "$((NAMED_TOTAL + OTHER_TOTAL))"
+expect "PASS lines" "$(pass_lines)" "$((NAMED_TOTAL + OTHER_TOTAL))"
+expect_no_line "deselected"
+finish
+
+# An empty variable is the shell's spelling of "unset": same bytes as no filter.
+begin "empty variables are no filter"
+CBM_TEST_ONLY="" CBM_TEST_ONLY_FILE="" run empty "${NAMED_SUITE}" "${OTHER_SUITE}"
+expect "exit status" "${RC}" 0
+if ! cmp -s "${tmpdir}/unfiltered.out" "${OUT}"; then
+ echo " FAIL ${CASE}: output differs from the unfiltered run"
+ CASE_OK=0
+fi
+finish
+
+# ── CBM_TEST_ONLY: exactly the named test; the rest of its suite is deselected,
+# and a requested suite that no token names does not execute at all ──
+begin "CBM_TEST_ONLY runs exactly the named test"
+CBM_TEST_ONLY="${NAMED_SUITE}:da_last" run only "${NAMED_SUITE}" "${OTHER_SUITE}"
+expect "exit status" "${RC}" 0
+expect "passed" "$(summary_count passed)" 1
+expect "deselected" "$(summary_count deselected)" "$((NAMED_TOTAL - 1))"
+expect "PASS lines" "$(pass_lines)" 1
+expect_line " da_last "
+expect_no_line "=== ${OTHER_SUITE} ==="
+finish
+
+# ── Strict: a token that matches nothing fails the run and is named ──
+begin "unknown test token fails and is named"
+CBM_TEST_ONLY="${NAMED_SUITE}:da_last,${NAMED_SUITE}:no_such_test" run unknown-test "${NAMED_SUITE}"
+expect_nonzero_exit
+expect_line "${UNKNOWN_MESSAGE} ${NAMED_SUITE}:no_such_test"
+expect "passed" "$(summary_count passed)" 1
+finish
+
+begin "unknown suite token fails and is named"
+CBM_TEST_ONLY="no_such_suite:da_last" run unknown-suite "${NAMED_SUITE}"
+expect_nonzero_exit
+expect_line "${UNKNOWN_MESSAGE} no_such_suite:da_last"
+expect "PASS lines" "$(pass_lines)" 0
+finish
+
+# ── Argv keeps selecting suites; the filter narrows inside that selection. A
+# token for a suite this process was not asked to run is out of its scope, not
+# an error: the parallel harness gives every per-suite process the whole list ──
+begin "token outside the argv selection is not an error"
+CBM_TEST_ONLY="${NAMED_SUITE}:da_last,${OTHER_SUITE}:${OTHER_TEST}" run argv-scope "${NAMED_SUITE}"
+expect "exit status" "${RC}" 0
+expect "passed" "$(summary_count passed)" 1
+expect_no_line "=== ${OTHER_SUITE} ==="
+finish
+
+# ── CBM_TEST_ONLY_FILE: one token per line; blank lines, comments, CRLF ──
+selection="${tmpdir}/selection.txt"
+printf '%s\n' "# written by the test-impact selector" "" \
+ "${NAMED_SUITE}:da_last" " ${NAMED_SUITE}:da_clear # trailing comment" > "${selection}"
+printf '%s\r\n' "${NAMED_SUITE}:da_free" >> "${selection}"
+
+begin "CBM_TEST_ONLY_FILE runs exactly the listed tests"
+CBM_TEST_ONLY_FILE="${selection}" run file "${NAMED_SUITE}" "${OTHER_SUITE}"
+expect "exit status" "${RC}" 0
+expect "passed" "$(summary_count passed)" 3
+expect "deselected" "$(summary_count deselected)" "$((NAMED_TOTAL - 3))"
+expect "PASS lines" "$(pass_lines)" 3
+expect_no_line "=== ${OTHER_SUITE} ==="
+finish
+
+begin "CBM_TEST_ONLY and CBM_TEST_ONLY_FILE select their union"
+CBM_TEST_ONLY="${OTHER_SUITE}:${OTHER_TEST}" CBM_TEST_ONLY_FILE="${selection}" \
+ run union "${NAMED_SUITE}" "${OTHER_SUITE}"
+expect "exit status" "${RC}" 0
+expect "passed" "$(summary_count passed)" 4
+expect "deselected" "$(summary_count deselected)" "$((NAMED_TOTAL + OTHER_TOTAL - 4))"
+finish
+
+printf '%s\n' "${NAMED_SUITE}:da_last" "${NAMED_SUITE}:no_such_test" > "${tmpdir}/unknown.txt"
+begin "unknown test token in the file fails and is named"
+CBM_TEST_ONLY_FILE="${tmpdir}/unknown.txt" run file-unknown "${NAMED_SUITE}"
+expect_nonzero_exit
+expect_line "${UNKNOWN_MESSAGE} ${NAMED_SUITE}:no_such_test"
+finish
+
+# ── Fail closed: a selection that cannot be read, names nothing, or is not a
+# token must never fall back to running everything or nothing in silence ──
+begin "unreadable selection file fails before any test"
+CBM_TEST_ONLY_FILE="${tmpdir}/does-not-exist.txt" run file-missing "${NAMED_SUITE}"
+expect_nonzero_exit
+expect_line "does-not-exist.txt"
+expect "PASS lines" "$(pass_lines)" 0
+finish
+
+printf '%s\n' "# nothing selected" "" > "${tmpdir}/empty.txt"
+begin "selection file naming no test fails before any test"
+CBM_TEST_ONLY_FILE="${tmpdir}/empty.txt" run file-empty "${NAMED_SUITE}"
+expect_nonzero_exit
+expect "PASS lines" "$(pass_lines)" 0
+finish
+
+begin "malformed token fails before any test"
+CBM_TEST_ONLY="${NAMED_SUITE}" run malformed "${NAMED_SUITE}"
+expect_nonzero_exit
+expect_line "malformed test selection token: ${NAMED_SUITE}"
+expect "PASS lines" "$(pass_lines)" 0
+finish
+
+if [[ "${FAILURES}" -gt 0 ]]; then
+ echo "FAIL: ${FAILURES} per-test selection case(s) failed"
+ exit 1
+fi
+echo "PASS: per-test selection is exact, strict, and inert without a filter"
diff --git a/tests/test_main.c b/tests/test_main.c
index 5c3dcf067..0ad96434a 100644
--- a/tests/test_main.c
+++ b/tests/test_main.c
@@ -7,6 +7,7 @@
int tf_pass_count = 0;
int tf_fail_count = 0;
int tf_skip_count = 0;
+int tf_deselected_count = 0;
#include "test_framework.h"
#include "test_helpers.h"
@@ -27,6 +28,7 @@ int tf_skip_count = 0;
#include "ui/http_server.h" /* deleted-self executable probe */
#include "result_spill.h" /* pinned free disk: spill verdicts ignore the host disk */
#include
+#include
#include
#include
#include
@@ -821,11 +823,401 @@ static int tf_maybe_run_deleted_self_probe(int argc, char **argv) {
#endif
}
+/* ── Per-test selection: CBM_TEST_ONLY / CBM_TEST_ONLY_FILE ──────────
+ * The test-impact selector names tests as `suite:test` tokens: comma-separated
+ * in CBM_TEST_ONLY, one per line (blank lines and # comments allowed) in the
+ * file CBM_TEST_ONLY_FILE names. Both may be set; the selection is their union.
+ * Argv keeps selecting suites and the tokens narrow inside that selection, so
+ * the parallel harness can hand every per-suite process the same list.
+ *
+ * Tokens are strict. After the run, one that names a suite this build does not
+ * register, or a test its suite never ran, fails the run by name: a selection
+ * that quietly ran less than it named would be a green for work never done. A
+ * token for a registered suite that argv left out belongs to another process
+ * and is not this one to judge. */
+typedef enum {
+ TF_ONLY_UNSEEN = 0, /* no registered suite carries the suite name */
+ TF_ONLY_REGISTERED, /* the suite exists; argv did not ask this process for it */
+ TF_ONLY_ENTERED, /* the suite ran and no RUN_TEST carried the test name */
+ TF_ONLY_MATCHED,
+} tf_only_state_t;
+
+typedef struct {
+ char *suite; /* one allocation holding "suite\0test" */
+ const char *test;
+ tf_only_state_t state;
+} tf_only_token_t;
+
+static tf_only_token_t *tf_only_tokens = NULL;
+static size_t tf_only_count = 0;
+static size_t tf_only_cap = 0;
+static bool tf_only_active = false;
+/* Tokens are sorted by suite, so the running suite has one run [first, end). */
+static size_t tf_only_suite_first = 0;
+static size_t tf_only_suite_end = 0;
+
+const char *tf_current_suite = NULL;
+
+static bool tf_only_name_valid(const char *name, size_t length) {
+ for (size_t i = 0; i < length; i++) {
+ if (!isalnum((unsigned char)name[i]) && name[i] != '_') {
+ return false;
+ }
+ }
+ return length > 0;
+}
+
+/* Adds the token in text[0..length). Surrounding whitespace is not part of it
+ * (the CR of a CRLF file included), and an empty span is not a token. */
+static bool tf_only_add(const char *text, size_t length) {
+ while (length > 0 && isspace((unsigned char)text[0])) {
+ text++;
+ length--;
+ }
+ while (length > 0 && isspace((unsigned char)text[length - 1])) {
+ length--;
+ }
+ if (length == 0) {
+ return true;
+ }
+ /* No test or suite name comes near this; it also bounds the copy below. */
+ const char *colon = length < CBM_SZ_512 ? memchr(text, ':', length) : NULL;
+ size_t suite_length = colon ? (size_t)(colon - text) : 0;
+ if (!colon || !tf_only_name_valid(text, suite_length) ||
+ !tf_only_name_valid(colon + 1, length - suite_length - 1)) {
+ fprintf(stderr, "malformed test selection token: %.*s (expected suite:test)\n", (int)length,
+ text);
+ return false;
+ }
+ if (tf_only_count == tf_only_cap) {
+ size_t cap = tf_only_cap ? tf_only_cap * 2 : CBM_SZ_64;
+ tf_only_token_t *grown = realloc(tf_only_tokens, cap * sizeof(*grown));
+ if (!grown) {
+ fprintf(stderr, "Failed to allocate the test selection\n");
+ return false;
+ }
+ tf_only_tokens = grown;
+ tf_only_cap = cap;
+ }
+ char *copy = malloc(length + 1);
+ if (!copy) {
+ fprintf(stderr, "Failed to allocate the test selection\n");
+ return false;
+ }
+ memcpy(copy, text, length);
+ copy[suite_length] = '\0';
+ copy[length] = '\0';
+ tf_only_tokens[tf_only_count++] =
+ (tf_only_token_t){.suite = copy, .test = copy + suite_length + 1, .state = TF_ONLY_UNSEEN};
+ return true;
+}
+
+static bool tf_only_add_list(const char *list) {
+ for (const char *cursor = list;;) {
+ const char *comma = strchr(cursor, ',');
+ if (!tf_only_add(cursor, comma ? (size_t)(comma - cursor) : strlen(cursor))) {
+ return false;
+ }
+ if (!comma) {
+ return true;
+ }
+ cursor = comma + 1;
+ }
+}
+
+static bool tf_only_add_file(const char *path) {
+ FILE *file = cbm_fopen(path, "r");
+ if (!file) {
+ fprintf(stderr, "cannot read test selection file: %s\n", path);
+ return false;
+ }
+ char line[CBM_SZ_1K];
+ bool ok = true;
+ while (ok && fgets(line, sizeof(line), file)) {
+ size_t length = strlen(line);
+ if (length == sizeof(line) - 1 && line[length - 1] != '\n') {
+ /* The rest of the line would be read back as a token of its own. */
+ fprintf(stderr, "test selection file line too long: %s\n", path);
+ ok = false;
+ } else {
+ ok = tf_only_add(line, strcspn(line, "#\n"));
+ }
+ }
+ if (ok && ferror(file)) {
+ fprintf(stderr, "cannot read test selection file: %s\n", path);
+ ok = false;
+ }
+ (void)fclose(file);
+ return ok;
+}
+
+static int tf_only_compare(const void *left, const void *right) {
+ const tf_only_token_t *a = left;
+ const tf_only_token_t *b = right;
+ int by_suite = strcmp(a->suite, b->suite);
+ return by_suite != 0 ? by_suite : strcmp(a->test, b->test);
+}
+
+static void tf_only_free(void) {
+ for (size_t i = 0; i < tf_only_count; i++) {
+ free(tf_only_tokens[i].suite);
+ }
+ free(tf_only_tokens);
+ tf_only_tokens = NULL;
+ tf_only_count = 0;
+ tf_only_cap = 0;
+ tf_only_active = false;
+}
+
+/* Reads the selection once. False means the selection itself is unusable: the
+ * caller stops before any suite, because falling back to "everything" or to
+ * "nothing" would decide a gate by accident. An empty variable is unset, as it
+ * is in a shell; a selection that is set and names no test is an error, since
+ * a caller who wants every test leaves both variables unset. */
+static bool tf_only_init(void) {
+ const char *list = getenv("CBM_TEST_ONLY");
+ const char *path = getenv("CBM_TEST_ONLY_FILE");
+ bool has_list = list && list[0];
+ bool has_path = path && path[0];
+ if (!has_list && !has_path) {
+ return true;
+ }
+ if ((has_list && !tf_only_add_list(list)) || (has_path && !tf_only_add_file(path))) {
+ return false;
+ }
+ if (tf_only_count == 0) {
+ fprintf(stderr, "test selection names no test (CBM_TEST_ONLY, CBM_TEST_ONLY_FILE)\n");
+ return false;
+ }
+ qsort(tf_only_tokens, tf_only_count, sizeof(*tf_only_tokens), tf_only_compare);
+ tf_only_active = true;
+ return true;
+}
+
+/* Returns the first token of `suite` and stores one past its last. */
+static size_t tf_only_suite_run(const char *suite, size_t *end_out) {
+ size_t first = 0;
+ while (first < tf_only_count && strcmp(tf_only_tokens[first].suite, suite) != 0) {
+ first++;
+ }
+ size_t end = first;
+ while (end < tf_only_count && strcmp(tf_only_tokens[end].suite, suite) == 0) {
+ end++;
+ }
+ *end_out = end;
+ return first;
+}
+
+static void tf_only_raise(size_t first, size_t end, tf_only_state_t state) {
+ for (size_t i = first; i < end; i++) {
+ if (tf_only_tokens[i].state < state) {
+ tf_only_tokens[i].state = state;
+ }
+ }
+}
+
+bool tf_suite_enter(const char *suite) {
+ if (tf_current_suite) {
+ /* A suite run from inside another: its tests belong to the registered
+ * suite that runs it, which is the name a token and argv both use. */
+ return true;
+ }
+ if (tf_only_active) {
+ tf_only_suite_first = tf_only_suite_run(suite, &tf_only_suite_end);
+ if (tf_only_suite_first == tf_only_suite_end) {
+ return false; /* no token names it: its body, setup included, never runs */
+ }
+ tf_only_raise(tf_only_suite_first, tf_only_suite_end, TF_ONLY_ENTERED);
+ }
+ tf_current_suite = suite;
+ return true;
+}
+
+void tf_suite_leave(const char *suite) {
+ if (tf_current_suite && strcmp(tf_current_suite, suite) == 0) {
+ tf_current_suite = NULL;
+ tf_only_suite_first = 0;
+ tf_only_suite_end = 0;
+ }
+}
+
+bool tf_test_selected(const char *test) {
+ if (!tf_only_active) {
+ return true;
+ }
+ bool selected = false;
+ for (size_t i = tf_only_suite_first; i < tf_only_suite_end; i++) {
+ if (strcmp(tf_only_tokens[i].test, test) == 0) {
+ tf_only_tokens[i].state = TF_ONLY_MATCHED;
+ selected = true;
+ }
+ }
+ if (!selected) {
+ tf_deselected_count++;
+ }
+ return selected;
+}
+
+/* The strict half of the selection: every token this process was responsible
+ * for must have run a test. Each one that did not is a failure of the run. */
+static void tf_only_report_unmatched(void) {
+ for (size_t i = 0; i < tf_only_count; i++) {
+ const tf_only_token_t *token = &tf_only_tokens[i];
+ if (token->state != TF_ONLY_UNSEEN && token->state != TF_ONLY_ENTERED) {
+ continue;
+ }
+ if (i > 0 && tf_only_compare(token, &tf_only_tokens[i - 1]) == 0) {
+ continue; /* named twice, reported once */
+ }
+ fprintf(stderr, "selected test not compiled into this build or unknown: %s:%s\n",
+ token->suite, token->test);
+ tf_fail_count++;
+ }
+}
+
+#ifdef CBM_TEST_COVERAGE
+/* ── Per-test coverage profiles ──────────────────────────────────────
+ * Compiled only into the instrumented runner (make test-runner-cov: clang
+ * source coverage, no sanitizer). With CBM_TEST_COVERAGE_DIR set, every test
+ * leaves its own raw profiles under //:
+ * .parent.profraw what this process executed inside the test
+ * ..profraw one per child the test spawned. Children
+ * inherit LLVM_PROFILE_FILE, so a re-exec of this
+ * runner or a product binary names its own file
+ * _setup..profraw what ran between tests: suite setup, and the
+ * prologue and epilogue of the runner itself
+ * _setup.child..profraw children spawned between tests
+ * The first two are the coverage of the test; the others belong to no test. */
+void __llvm_profile_set_filename(const char *pattern);
+int __llvm_profile_write_file(void);
+void __llvm_profile_reset_counters(void);
+
+enum { TF_COVERAGE_PATH_CAP = CBM_SZ_4K + CBM_SZ_1K };
+static char tf_coverage_dir[CBM_SZ_4K]; /* canonical; empty = not a coverage run */
+/* The runtime keeps the pointer it is handed, not a copy: these outlive it. */
+static char tf_coverage_profile[TF_COVERAGE_PATH_CAP];
+static char tf_coverage_child_pattern[TF_COVERAGE_PATH_CAP];
+static unsigned int tf_coverage_setup_index = 0;
+
+/* Names the profile this process writes next and the one its children will. */
+static bool tf_coverage_route(const char *directory, const char *own, const char *child) {
+ int own_length =
+ snprintf(tf_coverage_profile, sizeof(tf_coverage_profile), "%s/%s.profraw", directory, own);
+ int child_length = snprintf(tf_coverage_child_pattern, sizeof(tf_coverage_child_pattern),
+ "%s/%s.%%p.profraw", directory, child);
+ if (own_length <= 0 || (size_t)own_length >= sizeof(tf_coverage_profile) || child_length <= 0 ||
+ (size_t)child_length >= sizeof(tf_coverage_child_pattern)) {
+ return false;
+ }
+ __llvm_profile_set_filename(tf_coverage_profile);
+ return cbm_setenv("LLVM_PROFILE_FILE", tf_coverage_child_pattern, 1) == 0;
+}
+
+/* Between tests the interval is suite setup, and so is any child it spawns. */
+static bool tf_coverage_route_setup(const char *suite_dir) {
+ char own[CBM_SZ_64];
+ (void)snprintf(own, sizeof(own), "_setup.%u", tf_coverage_setup_index);
+ return tf_coverage_route(suite_dir, own, "_setup.child");
+}
+
+/* Writes what this process executed since the last write to its current profile
+ * and restarts the counters, so every file holds exactly one interval. */
+static void tf_coverage_flush(void) {
+ (void)__llvm_profile_write_file();
+ __llvm_profile_reset_counters();
+}
+
+static bool tf_coverage_suite_dir(char *out, size_t cap) {
+ int length = snprintf(out, cap, "%s/%s", tf_coverage_dir,
+ tf_current_suite ? tf_current_suite : "_nosuite");
+ return length > 0 && (size_t)length < cap && cbm_mkdir_p(out, 0755);
+}
+
+/* A coverage run that loses a profile would report the test as executing
+ * nothing, and the impact map would trust that: fail the run instead. */
+static void tf_coverage_lost(const char *test) {
+ fprintf(stderr, "coverage profile could not be routed: %s:%s\n",
+ tf_current_suite ? tf_current_suite : "_nosuite", test);
+ tf_fail_count++;
+}
+
+void tf_coverage_test_begin(const char *test) {
+ char suite_dir[TF_COVERAGE_PATH_CAP];
+ char own[CBM_SZ_512];
+ if (!tf_coverage_dir[0]) {
+ return;
+ }
+ int own_length = snprintf(own, sizeof(own), "%s.parent", test);
+ if (own_length <= 0 || (size_t)own_length >= sizeof(own) ||
+ !tf_coverage_suite_dir(suite_dir, sizeof(suite_dir)) ||
+ !tf_coverage_route_setup(suite_dir)) {
+ tf_coverage_lost(test);
+ return;
+ }
+ tf_coverage_flush();
+ tf_coverage_setup_index++;
+ /* Named before the test runs, not after: a forked child that leaves through
+ * exit() writes to the name it inherited, which must not be the setup file. */
+ if (!tf_coverage_route(suite_dir, own, test)) {
+ tf_coverage_lost(test);
+ }
+}
+
+void tf_coverage_test_end(const char *test) {
+ char suite_dir[TF_COVERAGE_PATH_CAP];
+ char own[CBM_SZ_512];
+ if (!tf_coverage_dir[0]) {
+ return;
+ }
+ int own_length = snprintf(own, sizeof(own), "%s.parent", test);
+ if (own_length <= 0 || (size_t)own_length >= sizeof(own) ||
+ !tf_coverage_suite_dir(suite_dir, sizeof(suite_dir)) ||
+ !tf_coverage_route(suite_dir, own, test)) {
+ tf_coverage_lost(test);
+ return;
+ }
+ tf_coverage_flush();
+ /* The runtime writes once more at exit, to whatever name is current. Left
+ * on the file of this test, that write would replace the profile just
+ * taken with the teardown that follows it. */
+ if (!tf_coverage_route_setup(suite_dir)) {
+ tf_coverage_lost(test);
+ }
+}
+
+/* False when the run asked for coverage and cannot have it. Children run in
+ * other working directories, so they are handed an absolute name. */
+static bool tf_coverage_init(void) {
+ const char *dir = getenv("CBM_TEST_COVERAGE_DIR");
+ if (!dir || !dir[0]) {
+ return true;
+ }
+ if (!cbm_mkdir_p(dir, 0755) ||
+ !cbm_canonical_path(dir, tf_coverage_dir, sizeof(tf_coverage_dir))) {
+ tf_coverage_dir[0] = '\0';
+ fprintf(stderr, "cannot use coverage directory: %s\n", dir);
+ return false;
+ }
+ /* Routing sets the environment, which may move what `dir` points at. */
+ if (!tf_coverage_route(tf_coverage_dir, "_runner.%p", "_runner.child")) {
+ fprintf(stderr, "cannot route coverage profiles under: %s\n", tf_coverage_dir);
+ tf_coverage_dir[0] = '\0';
+ return false;
+ }
+ return true;
+}
+#endif /* CBM_TEST_COVERAGE */
+
static int g_suite_argc = 0;
static char **g_suite_argv = NULL;
static bool *g_suite_arg_matched = NULL;
static bool suite_requested(const char *name) {
+ /* Every registered suite passes through here, requested or not: this is
+ * where a selection token learns that its suite exists in this build. */
+ size_t token_end = 0;
+ size_t token_first = tf_only_suite_run(name, &token_end);
+ tf_only_raise(token_first, token_end, TF_ONLY_REGISTERED);
if (g_suite_argc <= 1) {
return true;
}
@@ -1173,6 +1565,16 @@ int main(int argc, char **argv) {
g_suite_argc = argc;
g_suite_argv = argv;
}
+ if (!g_list_only && !tf_only_init()) {
+ tf_only_free();
+ return 2;
+ }
+#ifdef CBM_TEST_COVERAGE
+ if (!g_list_only && !tf_coverage_init()) {
+ tf_only_free();
+ return 2;
+ }
+#endif
if (g_suite_argc > 1) {
g_suite_arg_matched = calloc((size_t)argc, sizeof(*g_suite_arg_matched));
if (!g_suite_arg_matched) {
@@ -1417,6 +1819,8 @@ int main(int argc, char **argv) {
if (g_suite_argc > 1 && !any_suite_matched) {
fprintf(stderr, "No matching test suites requested\n");
}
+ tf_only_report_unmatched();
+ tf_only_free();
free(g_suite_arg_matched);
g_suite_arg_matched = NULL;