diff --git a/.gitignore b/.gitignore index 3705501c2..1533c93e0 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,9 @@ graph-ui/dist/ # Generated by scripts/embed-frontend.sh into the source tree (the linker needs # it beside the other UI sources). Removed by scripts/clean.sh; never committed. src/ui/embedded_assets.c +# Raw coverage profiles. A coverage-instrumented binary (make test-runner-cov, +# cbm-cov) run without LLVM_PROFILE_FILE drops default.profraw in its cwd. +*.profraw # Generated reports BENCHMARK_REPORT.md diff --git a/Makefile.cbm b/Makefile.cbm index ab5c89db7..b59c5ed00 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -71,13 +71,29 @@ ifeq ($(TEST_SEAMS),1) TEST_SEAM_DEFINE := -DCBM_ENABLE_TEST_SEAMS=1 endif +# Per-test coverage for the test-impact map. TEST_COVERAGE=1 compiles clang +# source coverage into OUR code only -- the one-shot sources, the LSP unity +# object and the preprocessor -- in the test runner and in the product binary, +# and turns on the runner's per-test profile hooks (tests/test_framework.h). +# Vendored objects stay uninstrumented: a vendored change selects tests by rule, +# and every counter compiled in is written out again for every test. Like every +# other flavour it needs its own BUILD_DIR; the test-runner-cov and cbm-cov +# targets pin one. +TEST_COVERAGE ?= 0 +TEST_COVERAGE_FLAGS := +TEST_COVERAGE_DEFINE := +ifeq ($(TEST_COVERAGE),1) +TEST_COVERAGE_FLAGS := -fprofile-instr-generate -fcoverage-mapping +TEST_COVERAGE_DEFINE := -DCBM_TEST_COVERAGE=1 +endif + # Production flags (CFLAGS_EXTRA allows CI to inject -DCBM_VERSION) # CBM_BIND_TS_ALLOCATOR=1: bind the tree-sitter runtime to mimalloc (#424). Only # the prod build uses mimalloc (MI_OVERRIDE=1); the test build is CRT+ASan, where # binding would create an alloc/free mismatch, so the guard is prod-only. CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(TEST_SEAM_DEFINE) \ - $(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(CFLAGS_EXTRA) -CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS) + $(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(TEST_COVERAGE_FLAGS) $(CFLAGS_EXTRA) +CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS) $(TEST_COVERAGE_FLAGS) # Test flags: debug + sanitizers (override SANITIZE= to disable on Windows) SANITIZE = -fsanitize=address,undefined -fno-omit-frame-pointer @@ -102,8 +118,10 @@ INCREMENTAL_TEST_DEFINE = -DCBM_INCREMENTAL_TEST_API=1 COVERAGE_MARKER_TEST_DEFINE = -DCBM_COVERAGE_MARKER_TEST_API=1 CFLAGS_TEST = $(CFLAGS_COMMON) $(EDITOR_TEST_DEFINES) $(SANITIZED_DEFINE) \ $(KOTLIN_DEDUP_TEST_DEFINE) $(CALL_REFERENCE_LOOKUP_TEST_DEFINE) \ - $(INCREMENTAL_TEST_DEFINE) $(COVERAGE_MARKER_TEST_DEFINE) -g -O1 $(SANITIZE) -CXXFLAGS_TEST = $(CXXFLAGS_COMMON) $(SANITIZED_DEFINE) -g -O1 $(SANITIZE) $(CXX_STDLIB_FLAGS) + $(INCREMENTAL_TEST_DEFINE) $(COVERAGE_MARKER_TEST_DEFINE) $(TEST_COVERAGE_DEFINE) \ + -g -O1 $(SANITIZE) $(TEST_COVERAGE_FLAGS) +CXXFLAGS_TEST = $(CXXFLAGS_COMMON) $(SANITIZED_DEFINE) -g -O1 $(SANITIZE) $(TEST_COVERAGE_FLAGS) \ + $(CXX_STDLIB_FLAGS) # TSan (can't combine with ASan) TSAN_SANITIZE = -fsanitize=thread -fno-omit-frame-pointer @@ -851,7 +869,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o # ── Targets ────────────────────────────────────────────────────── -.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz +.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke test-runner-cov cbm-cov cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz $(BUILD_DIR): mkdir -p $(BUILD_DIR) @@ -890,7 +908,7 @@ $(BUILD_DIR)/ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR) $(CC) $(GRAMMAR_CFLAGS_TEST) -c -o $@ $< $(BUILD_DIR)/lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR) - $(CC) $(GRAMMAR_CFLAGS_TEST) $(SANITIZE) -c -o $@ $< + $(CC) $(GRAMMAR_CFLAGS_TEST) $(SANITIZE) $(TEST_COVERAGE_FLAGS) -c -o $@ $< $(BUILD_DIR)/preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR) $(CXX) $(CXXFLAGS_TEST) -w -I$(CBM_DIR)/vendored -c -o $@ $< @@ -1132,7 +1150,7 @@ $(BUILD_DIR)/prod_ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR) $(CC) $(GRAMMAR_CFLAGS) -c -o $@ $< $(BUILD_DIR)/prod_lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR) - $(CC) $(GRAMMAR_CFLAGS) -c -o $@ $< + $(CC) $(GRAMMAR_CFLAGS) $(TEST_COVERAGE_FLAGS) -c -o $@ $< $(BUILD_DIR)/prod_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR) $(CXX) $(CXXFLAGS_PROD) -w -I$(CBM_DIR)/vendored -c -o $@ $< @@ -1169,7 +1187,7 @@ MAIN_SRC = src/main.c # for trap debugging -- silently re-run the previously instrumented binary, # which during this bisect produced a "crashes without sanitizers too" result # that was pure artifact and sent the investigation down the wrong path. -BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE) +BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE)|TEST_COVERAGE=$(TEST_COVERAGE) .PHONY: build-config-check build-config-check: $(BUILD_DIR)/.build-config: build-config-check | $(BUILD_DIR) @@ -1365,6 +1383,30 @@ test-lsan: cd $(CURDIR) && ASAN_OPTIONS="detect_leaks=1:halt_on_error=1" \ ./build/lsan/test-runner $(LSAN_SUITES) +# ── Per-test coverage build (test-impact map) ─────────────────── +# The instrumented runner and product binary (TEST_COVERAGE=1 above), in their +# own build directory. Homebrew LLVM rather than Apple clang: the raw profile +# format is versioned with the compiler, so llvm-profdata and llvm-cov have to +# come from the toolchain that built the binary. +# LLVM_PROFILE_FILE=/dev/null CBM_TEST_COVERAGE_DIR= \ +# build/cov/test-runner [suite...] +# leaves one profile set per test (tests/test_main.c). The runner names every +# profile itself, but the profile runtime creates its default file before main +# runs: started without LLVM_PROFILE_FILE it leaves an empty default.profraw in +# the working directory. +# No C suite runs the product binary -- they re-exec the runner itself. The +# shell process regressions do, and they find the instrumented one the way they +# find any other: CBM_TEST_BINARY=build/cov/codebase-memory-mcp, with +# LLVM_PROFILE_FILE set by whoever collects the profiles. +COV_LLVM_BIN ?= /opt/homebrew/opt/llvm/bin +COV_BUILD_DIR ?= build/cov +COV_MAKE = $(MAKE) -f Makefile.cbm CC=$(COV_LLVM_BIN)/clang CXX=$(COV_LLVM_BIN)/clang++ \ + BUILD_DIR=$(COV_BUILD_DIR) SANITIZE= TEST_COVERAGE=1 +test-runner-cov: + $(COV_MAKE) $(COV_BUILD_DIR)/test-runner +cbm-cov: + $(COV_MAKE) $(COV_BUILD_DIR)/codebase-memory-mcp TEST_SEAMS=1 + # ── Coverage-guided fuzzing (libFuzzer) ───────────────────────── # One binary, three targets (tests/fuzz/cbm_fuzz.c): the per-file extractor, # the Cypher parser, the TOML/YAML/JSON config editors. ASan+UBSan as in the diff --git a/scripts/test.sh b/scripts/test.sh index eae756138..81483c115 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -381,6 +381,15 @@ BUILD_DIR="$BUILD_DIR" scripts/clean.sh # CBM_TEST_SEQUENTIAL=1 restores the single-process runner. make -j"$NPROC" -f Makefile.cbm "$BUILD_DIR/test-runner" ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"} assert_test_runner_build_config "$BUILD_DIR/test-runner" "$EXPECTED_SANITIZED" + +# Step 2b: per-test selection process regression. CBM_TEST_ONLY must run exactly +# the tests it names and fail on a token that matches nothing; a suite cannot +# assert that about the runner executing it. Runs against the runner just built +# and BEFORE the suites: it takes seconds, and a selection that silently drops +# tests is a property of the harness every later result depends on. +echo "=== Step 2b: per-test selection regression (CBM_TEST_ONLY) ===" +CBM_TEST_RUNNER="$ROOT/$BUILD_DIR/test-runner" bash "$ROOT/tests/test_harness_test_only.sh" + if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; then make -f Makefile.cbm test ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"} else diff --git a/tests/repro/repro_main.c b/tests/repro/repro_main.c index 7fc1cc20b..d164fce90 100644 --- a/tests/repro/repro_main.c +++ b/tests/repro/repro_main.c @@ -24,12 +24,29 @@ int tf_pass_count = 0; int tf_fail_count = 0; int tf_skip_count = 0; +int tf_deselected_count = 0; #include "test_framework.h" #include "repro_runner.h" #include "foundation/compat.h" /* cbm_setenv — #845 supervisor kill switch */ #include "daemon/bootstrap.h" /* refuse the developer's default daemon rendezvous */ +/* Per-test selection (CBM_TEST_ONLY) and per-test coverage belong to the gating + * runner, tests/test_main.c. This board selects whole suites with + * CBM_REPRO_ONLY, and every test of a selected suite runs. */ +bool tf_test_selected(const char *test) { + (void)test; + return true; +} +#ifdef CBM_TEST_COVERAGE +void tf_coverage_test_begin(const char *test) { + (void)test; +} +void tf_coverage_test_end(const char *test) { + (void)test; +} +#endif + /* Per-suite summary + filter. RUN_SUITE prints a one-line * "[SUITE] P passed, F failed" report (greppable for which suites still * have reds). The shared selector implementation lives with its gating tests in diff --git a/tests/test_framework.h b/tests/test_framework.h index 2af654b4d..ac9539a6d 100644 --- a/tests/test_framework.h +++ b/tests/test_framework.h @@ -6,6 +6,7 @@ * - Test suite grouping * - Pass/fail counting with summary * - Color output (when isatty) + * - Per-test selection (CBM_TEST_ONLY) and per-test coverage profiles * * Usage: * TEST(my_test) { @@ -26,6 +27,7 @@ #ifndef TEST_FRAMEWORK_H #define TEST_FRAMEWORK_H +#include #include #include #include @@ -37,6 +39,32 @@ extern int tf_pass_count; extern int tf_fail_count; extern int tf_skip_count; +extern int tf_deselected_count; + +/* ── Per-test selection (defined by the runner) ────────────────── */ + +/* CBM_TEST_ONLY / CBM_TEST_ONLY_FILE narrow a run to `suite:test` tokens (see + * test_main.c). tf_suite_enter decides whether a suite's body executes at all; + * inside it tf_test_selected decides each RUN_TEST and counts the tests it + * leaves out. Without a selection both are always true. */ +extern const char *tf_current_suite; +bool tf_suite_enter(const char *suite); +void tf_suite_leave(const char *suite); +bool tf_test_selected(const char *test); + +/* ── Per-test coverage (CBM_TEST_COVERAGE builds only) ─────────── */ + +/* A coverage-instrumented runner (make test-runner-cov) writes one profile per + * test under CBM_TEST_COVERAGE_DIR. No other build compiles any of it. */ +#ifdef CBM_TEST_COVERAGE +void tf_coverage_test_begin(const char *test); +void tf_coverage_test_end(const char *test); +#define TF_COVERAGE_TEST_BEGIN(test) tf_coverage_test_begin(test) +#define TF_COVERAGE_TEST_END(test) tf_coverage_test_end(test) +#else +#define TF_COVERAGE_TEST_BEGIN(test) ((void)0) +#define TF_COVERAGE_TEST_END(test) ((void)0) +#endif /* ── Color helpers ─────────────────────────────────────────────── */ @@ -69,10 +97,10 @@ static inline const char *tf_reset(void) { /* Hard failure with a message — for setup/environment failures that must NOT be * silently skipped (a test that cannot establish its preconditions has FAILED, * not "skipped"). Mirrors the ASSERT failure path (red FAIL + file:line). */ -#define FAIL(reason) \ - do { \ +#define FAIL(reason) \ + do { \ printf(" %sFAIL%s %s:%d: %s\n", tf_red(), tf_reset(), __FILE__, __LINE__, (reason)); \ - return 1; \ + return 1; \ } while (0) /* The ONLY tolerable skip: a test that is inherently platform-specific and does @@ -224,9 +252,14 @@ static inline const char *tf_reset(void) { #define RUN_TEST(name) \ do { \ + if (!tf_test_selected(#name)) { \ + break; \ + } \ printf(" %-55s", #name); \ fflush(stdout); \ + TF_COVERAGE_TEST_BEGIN(#name); \ int _result = test_##name(); \ + TF_COVERAGE_TEST_END(#name); \ if (_result == 0) { \ printf("%sPASS%s\n", tf_green(), tf_reset()); \ tf_pass_count++; \ @@ -243,22 +276,28 @@ static inline const char *tf_reset(void) { #define RUN_SUITE(name) \ do { \ + if (!tf_suite_enter(#name)) { \ + break; \ + } \ printf("\n%s=== %s ===%s\n", tf_dim(), #name, tf_reset()); \ suite_##name(); \ + tf_suite_leave(#name); \ } while (0) /* ── Summary ───────────────────────────────────────────────────── */ -#define TEST_SUMMARY() \ - do { \ - printf("\n────────────────────────────────────────────\n"); \ - printf(" %s%d passed%s", tf_green(), tf_pass_count, tf_reset()); \ - if (tf_fail_count > 0) \ - printf(", %s%d failed%s", tf_red(), tf_fail_count, tf_reset()); \ - if (tf_skip_count > 0) \ - printf(", %s%d skipped%s", tf_dim(), tf_skip_count, tf_reset()); \ - printf("\n────────────────────────────────────────────\n\n"); \ - return tf_fail_count > 0 ? 1 : 0; \ +#define TEST_SUMMARY() \ + do { \ + printf("\n────────────────────────────────────────────\n"); \ + printf(" %s%d passed%s", tf_green(), tf_pass_count, tf_reset()); \ + if (tf_fail_count > 0) \ + printf(", %s%d failed%s", tf_red(), tf_fail_count, tf_reset()); \ + if (tf_skip_count > 0) \ + printf(", %s%d skipped%s", tf_dim(), tf_skip_count, tf_reset()); \ + if (tf_deselected_count > 0) \ + printf(", %s%d deselected%s", tf_dim(), tf_deselected_count, tf_reset()); \ + printf("\n────────────────────────────────────────────\n\n"); \ + return tf_fail_count > 0 ? 1 : 0; \ } while (0) #endif /* TEST_FRAMEWORK_H */ diff --git a/tests/test_harness_test_only.sh b/tests/test_harness_test_only.sh new file mode 100644 index 000000000..04318308b --- /dev/null +++ b/tests/test_harness_test_only.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +# Per-test selection in the C test runner: CBM_TEST_ONLY / CBM_TEST_ONLY_FILE. +# +# The test-impact selector hands the runner `suite:test` tokens. Two properties +# make that safe to gate on, and both are process-level behaviour a suite cannot +# assert about the runner that is executing it: +# 1. exactly the named tests run: the other tests of a named suite are counted +# as deselected, and a suite no token names never executes its body; +# 2. a token that matches nothing is an ERROR. A selection that quietly ran +# fewer tests than it named would report green for work that never ran. +# +# Every case drives the built runner over two small in-memory suites, so the +# whole script is a dozen runner starts. Expected counts are read from the suite +# sources, never hard-coded, so adding a test to either suite cannot stale them. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# scripts/test.sh builds into $BUILD_DIR, which is not build/c on every leg; it +# passes the runner it built. The default keeps a bare manual invocation working. +RUNNER="${CBM_TEST_RUNNER:-${ROOT}/build/c/test-runner}" +if [[ ! -x "${RUNNER}" && -x "${RUNNER}.exe" ]]; then + RUNNER="${RUNNER}.exe" +fi +if [[ ! -x "${RUNNER}" ]]; then + echo "missing test runner: ${RUNNER}" >&2 + exit 2 +fi + +# A selection inherited from the caller would decide every case below. +unset CBM_TEST_ONLY CBM_TEST_ONLY_FILE + +NAMED_SUITE=dyn_array +OTHER_SUITE=str_intern +UNKNOWN_MESSAGE="selected test not compiled into this build or unknown:" + +run_test_count() { + awk '/^[[:space:]]*RUN_TEST\(/ { n++ } END { print n + 0 }' "$1" +} +first_run_test() { + awk -F'[()]' '/^[[:space:]]*RUN_TEST\(/ { print $2; exit }' "$1" +} +NAMED_TOTAL="$(run_test_count "${ROOT}/tests/test_${NAMED_SUITE}.c")" +OTHER_TOTAL="$(run_test_count "${ROOT}/tests/test_${OTHER_SUITE}.c")" +OTHER_TEST="$(first_run_test "${ROOT}/tests/test_${OTHER_SUITE}.c")" +if [[ "${NAMED_TOTAL}" -lt 3 || "${OTHER_TOTAL}" -lt 1 || -z "${OTHER_TEST}" ]]; then + echo "fixture suites changed shape: ${NAMED_SUITE}=${NAMED_TOTAL} ${OTHER_SUITE}=${OTHER_TOTAL}" >&2 + exit 2 +fi + +tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/cbm-test-only.XXXXXX")" +trap 'rm -rf "${tmpdir}"' EXIT + +FAILURES=0 +RC=0 +OUT="" + +# run [suite...] — runs the runner under the caller's VAR=VAL prefix and +# records its merged output and exit status. CR is stripped: the Windows CRT +# ends every stdout line with CRLF. +run() { + local name="$1" + shift + OUT="${tmpdir}/${name}.out" + set +e + "${RUNNER}" "$@" > "${OUT}.raw" 2>&1 + RC=$? + set -e + tr -d '\r' < "${OUT}.raw" > "${OUT}" +} + +# The summary line, e.g. " 1 passed, 25 deselected". +summary_count() { + awk -v label="$1" ' + /^ [0-9]+ passed/ { + n = 0 + count = split($0, part, ", ") + for (i = 1; i <= count; i++) { + gsub(/^ +/, "", part[i]) + if (split(part[i], field, " ") == 2 && field[2] == label) n = field[1] + } + found = 1 + } + END { print found ? n : "none" } + ' "${OUT}" +} +pass_lines() { + awk '/PASS$/ { n++ } END { print n + 0 }' "${OUT}" +} +has_line() { + grep -F -- "$1" "${OUT}" > /dev/null +} + +CASE="" +CASE_OK=1 +begin() { + CASE="$1" + CASE_OK=1 +} +expect() { + local what="$1" actual="$2" wanted="$3" + if [[ "${actual}" != "${wanted}" ]]; then + echo " FAIL ${CASE}: ${what} = ${actual}, expected ${wanted}" + CASE_OK=0 + fi +} +expect_nonzero_exit() { + if [[ "${RC}" -eq 0 ]]; then + echo " FAIL ${CASE}: runner exited 0, expected a nonzero exit" + CASE_OK=0 + fi +} +expect_line() { + if ! has_line "$1"; then + echo " FAIL ${CASE}: output lacks: $1" + CASE_OK=0 + fi +} +expect_no_line() { + if has_line "$1"; then + echo " FAIL ${CASE}: output unexpectedly has: $1" + CASE_OK=0 + fi +} +finish() { + if [[ "${CASE_OK}" -eq 1 ]]; then + echo " ok ${CASE}" + else + FAILURES=$((FAILURES + 1)) + echo " ---- ${CASE}: runner exit ${RC}, output tail ----" + tail -6 "${OUT}" | sed 's/^/ | /' + fi +} + +echo "per-test selection: ${RUNNER}" + +# ── No filter: every test of the requested suites runs, nothing is deselected ── +begin "no filter runs every test" +run unfiltered "${NAMED_SUITE}" "${OTHER_SUITE}" +expect "exit status" "${RC}" 0 +expect "passed" "$(summary_count passed)" "$((NAMED_TOTAL + OTHER_TOTAL))" +expect "PASS lines" "$(pass_lines)" "$((NAMED_TOTAL + OTHER_TOTAL))" +expect_no_line "deselected" +finish + +# An empty variable is the shell's spelling of "unset": same bytes as no filter. +begin "empty variables are no filter" +CBM_TEST_ONLY="" CBM_TEST_ONLY_FILE="" run empty "${NAMED_SUITE}" "${OTHER_SUITE}" +expect "exit status" "${RC}" 0 +if ! cmp -s "${tmpdir}/unfiltered.out" "${OUT}"; then + echo " FAIL ${CASE}: output differs from the unfiltered run" + CASE_OK=0 +fi +finish + +# ── CBM_TEST_ONLY: exactly the named test; the rest of its suite is deselected, +# and a requested suite that no token names does not execute at all ── +begin "CBM_TEST_ONLY runs exactly the named test" +CBM_TEST_ONLY="${NAMED_SUITE}:da_last" run only "${NAMED_SUITE}" "${OTHER_SUITE}" +expect "exit status" "${RC}" 0 +expect "passed" "$(summary_count passed)" 1 +expect "deselected" "$(summary_count deselected)" "$((NAMED_TOTAL - 1))" +expect "PASS lines" "$(pass_lines)" 1 +expect_line " da_last " +expect_no_line "=== ${OTHER_SUITE} ===" +finish + +# ── Strict: a token that matches nothing fails the run and is named ── +begin "unknown test token fails and is named" +CBM_TEST_ONLY="${NAMED_SUITE}:da_last,${NAMED_SUITE}:no_such_test" run unknown-test "${NAMED_SUITE}" +expect_nonzero_exit +expect_line "${UNKNOWN_MESSAGE} ${NAMED_SUITE}:no_such_test" +expect "passed" "$(summary_count passed)" 1 +finish + +begin "unknown suite token fails and is named" +CBM_TEST_ONLY="no_such_suite:da_last" run unknown-suite "${NAMED_SUITE}" +expect_nonzero_exit +expect_line "${UNKNOWN_MESSAGE} no_such_suite:da_last" +expect "PASS lines" "$(pass_lines)" 0 +finish + +# ── Argv keeps selecting suites; the filter narrows inside that selection. A +# token for a suite this process was not asked to run is out of its scope, not +# an error: the parallel harness gives every per-suite process the whole list ── +begin "token outside the argv selection is not an error" +CBM_TEST_ONLY="${NAMED_SUITE}:da_last,${OTHER_SUITE}:${OTHER_TEST}" run argv-scope "${NAMED_SUITE}" +expect "exit status" "${RC}" 0 +expect "passed" "$(summary_count passed)" 1 +expect_no_line "=== ${OTHER_SUITE} ===" +finish + +# ── CBM_TEST_ONLY_FILE: one token per line; blank lines, comments, CRLF ── +selection="${tmpdir}/selection.txt" +printf '%s\n' "# written by the test-impact selector" "" \ + "${NAMED_SUITE}:da_last" " ${NAMED_SUITE}:da_clear # trailing comment" > "${selection}" +printf '%s\r\n' "${NAMED_SUITE}:da_free" >> "${selection}" + +begin "CBM_TEST_ONLY_FILE runs exactly the listed tests" +CBM_TEST_ONLY_FILE="${selection}" run file "${NAMED_SUITE}" "${OTHER_SUITE}" +expect "exit status" "${RC}" 0 +expect "passed" "$(summary_count passed)" 3 +expect "deselected" "$(summary_count deselected)" "$((NAMED_TOTAL - 3))" +expect "PASS lines" "$(pass_lines)" 3 +expect_no_line "=== ${OTHER_SUITE} ===" +finish + +begin "CBM_TEST_ONLY and CBM_TEST_ONLY_FILE select their union" +CBM_TEST_ONLY="${OTHER_SUITE}:${OTHER_TEST}" CBM_TEST_ONLY_FILE="${selection}" \ + run union "${NAMED_SUITE}" "${OTHER_SUITE}" +expect "exit status" "${RC}" 0 +expect "passed" "$(summary_count passed)" 4 +expect "deselected" "$(summary_count deselected)" "$((NAMED_TOTAL + OTHER_TOTAL - 4))" +finish + +printf '%s\n' "${NAMED_SUITE}:da_last" "${NAMED_SUITE}:no_such_test" > "${tmpdir}/unknown.txt" +begin "unknown test token in the file fails and is named" +CBM_TEST_ONLY_FILE="${tmpdir}/unknown.txt" run file-unknown "${NAMED_SUITE}" +expect_nonzero_exit +expect_line "${UNKNOWN_MESSAGE} ${NAMED_SUITE}:no_such_test" +finish + +# ── Fail closed: a selection that cannot be read, names nothing, or is not a +# token must never fall back to running everything or nothing in silence ── +begin "unreadable selection file fails before any test" +CBM_TEST_ONLY_FILE="${tmpdir}/does-not-exist.txt" run file-missing "${NAMED_SUITE}" +expect_nonzero_exit +expect_line "does-not-exist.txt" +expect "PASS lines" "$(pass_lines)" 0 +finish + +printf '%s\n' "# nothing selected" "" > "${tmpdir}/empty.txt" +begin "selection file naming no test fails before any test" +CBM_TEST_ONLY_FILE="${tmpdir}/empty.txt" run file-empty "${NAMED_SUITE}" +expect_nonzero_exit +expect "PASS lines" "$(pass_lines)" 0 +finish + +begin "malformed token fails before any test" +CBM_TEST_ONLY="${NAMED_SUITE}" run malformed "${NAMED_SUITE}" +expect_nonzero_exit +expect_line "malformed test selection token: ${NAMED_SUITE}" +expect "PASS lines" "$(pass_lines)" 0 +finish + +if [[ "${FAILURES}" -gt 0 ]]; then + echo "FAIL: ${FAILURES} per-test selection case(s) failed" + exit 1 +fi +echo "PASS: per-test selection is exact, strict, and inert without a filter" diff --git a/tests/test_main.c b/tests/test_main.c index 5c3dcf067..0ad96434a 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -7,6 +7,7 @@ int tf_pass_count = 0; int tf_fail_count = 0; int tf_skip_count = 0; +int tf_deselected_count = 0; #include "test_framework.h" #include "test_helpers.h" @@ -27,6 +28,7 @@ int tf_skip_count = 0; #include "ui/http_server.h" /* deleted-self executable probe */ #include "result_spill.h" /* pinned free disk: spill verdicts ignore the host disk */ #include +#include #include #include #include @@ -821,11 +823,401 @@ static int tf_maybe_run_deleted_self_probe(int argc, char **argv) { #endif } +/* ── Per-test selection: CBM_TEST_ONLY / CBM_TEST_ONLY_FILE ────────── + * The test-impact selector names tests as `suite:test` tokens: comma-separated + * in CBM_TEST_ONLY, one per line (blank lines and # comments allowed) in the + * file CBM_TEST_ONLY_FILE names. Both may be set; the selection is their union. + * Argv keeps selecting suites and the tokens narrow inside that selection, so + * the parallel harness can hand every per-suite process the same list. + * + * Tokens are strict. After the run, one that names a suite this build does not + * register, or a test its suite never ran, fails the run by name: a selection + * that quietly ran less than it named would be a green for work never done. A + * token for a registered suite that argv left out belongs to another process + * and is not this one to judge. */ +typedef enum { + TF_ONLY_UNSEEN = 0, /* no registered suite carries the suite name */ + TF_ONLY_REGISTERED, /* the suite exists; argv did not ask this process for it */ + TF_ONLY_ENTERED, /* the suite ran and no RUN_TEST carried the test name */ + TF_ONLY_MATCHED, +} tf_only_state_t; + +typedef struct { + char *suite; /* one allocation holding "suite\0test" */ + const char *test; + tf_only_state_t state; +} tf_only_token_t; + +static tf_only_token_t *tf_only_tokens = NULL; +static size_t tf_only_count = 0; +static size_t tf_only_cap = 0; +static bool tf_only_active = false; +/* Tokens are sorted by suite, so the running suite has one run [first, end). */ +static size_t tf_only_suite_first = 0; +static size_t tf_only_suite_end = 0; + +const char *tf_current_suite = NULL; + +static bool tf_only_name_valid(const char *name, size_t length) { + for (size_t i = 0; i < length; i++) { + if (!isalnum((unsigned char)name[i]) && name[i] != '_') { + return false; + } + } + return length > 0; +} + +/* Adds the token in text[0..length). Surrounding whitespace is not part of it + * (the CR of a CRLF file included), and an empty span is not a token. */ +static bool tf_only_add(const char *text, size_t length) { + while (length > 0 && isspace((unsigned char)text[0])) { + text++; + length--; + } + while (length > 0 && isspace((unsigned char)text[length - 1])) { + length--; + } + if (length == 0) { + return true; + } + /* No test or suite name comes near this; it also bounds the copy below. */ + const char *colon = length < CBM_SZ_512 ? memchr(text, ':', length) : NULL; + size_t suite_length = colon ? (size_t)(colon - text) : 0; + if (!colon || !tf_only_name_valid(text, suite_length) || + !tf_only_name_valid(colon + 1, length - suite_length - 1)) { + fprintf(stderr, "malformed test selection token: %.*s (expected suite:test)\n", (int)length, + text); + return false; + } + if (tf_only_count == tf_only_cap) { + size_t cap = tf_only_cap ? tf_only_cap * 2 : CBM_SZ_64; + tf_only_token_t *grown = realloc(tf_only_tokens, cap * sizeof(*grown)); + if (!grown) { + fprintf(stderr, "Failed to allocate the test selection\n"); + return false; + } + tf_only_tokens = grown; + tf_only_cap = cap; + } + char *copy = malloc(length + 1); + if (!copy) { + fprintf(stderr, "Failed to allocate the test selection\n"); + return false; + } + memcpy(copy, text, length); + copy[suite_length] = '\0'; + copy[length] = '\0'; + tf_only_tokens[tf_only_count++] = + (tf_only_token_t){.suite = copy, .test = copy + suite_length + 1, .state = TF_ONLY_UNSEEN}; + return true; +} + +static bool tf_only_add_list(const char *list) { + for (const char *cursor = list;;) { + const char *comma = strchr(cursor, ','); + if (!tf_only_add(cursor, comma ? (size_t)(comma - cursor) : strlen(cursor))) { + return false; + } + if (!comma) { + return true; + } + cursor = comma + 1; + } +} + +static bool tf_only_add_file(const char *path) { + FILE *file = cbm_fopen(path, "r"); + if (!file) { + fprintf(stderr, "cannot read test selection file: %s\n", path); + return false; + } + char line[CBM_SZ_1K]; + bool ok = true; + while (ok && fgets(line, sizeof(line), file)) { + size_t length = strlen(line); + if (length == sizeof(line) - 1 && line[length - 1] != '\n') { + /* The rest of the line would be read back as a token of its own. */ + fprintf(stderr, "test selection file line too long: %s\n", path); + ok = false; + } else { + ok = tf_only_add(line, strcspn(line, "#\n")); + } + } + if (ok && ferror(file)) { + fprintf(stderr, "cannot read test selection file: %s\n", path); + ok = false; + } + (void)fclose(file); + return ok; +} + +static int tf_only_compare(const void *left, const void *right) { + const tf_only_token_t *a = left; + const tf_only_token_t *b = right; + int by_suite = strcmp(a->suite, b->suite); + return by_suite != 0 ? by_suite : strcmp(a->test, b->test); +} + +static void tf_only_free(void) { + for (size_t i = 0; i < tf_only_count; i++) { + free(tf_only_tokens[i].suite); + } + free(tf_only_tokens); + tf_only_tokens = NULL; + tf_only_count = 0; + tf_only_cap = 0; + tf_only_active = false; +} + +/* Reads the selection once. False means the selection itself is unusable: the + * caller stops before any suite, because falling back to "everything" or to + * "nothing" would decide a gate by accident. An empty variable is unset, as it + * is in a shell; a selection that is set and names no test is an error, since + * a caller who wants every test leaves both variables unset. */ +static bool tf_only_init(void) { + const char *list = getenv("CBM_TEST_ONLY"); + const char *path = getenv("CBM_TEST_ONLY_FILE"); + bool has_list = list && list[0]; + bool has_path = path && path[0]; + if (!has_list && !has_path) { + return true; + } + if ((has_list && !tf_only_add_list(list)) || (has_path && !tf_only_add_file(path))) { + return false; + } + if (tf_only_count == 0) { + fprintf(stderr, "test selection names no test (CBM_TEST_ONLY, CBM_TEST_ONLY_FILE)\n"); + return false; + } + qsort(tf_only_tokens, tf_only_count, sizeof(*tf_only_tokens), tf_only_compare); + tf_only_active = true; + return true; +} + +/* Returns the first token of `suite` and stores one past its last. */ +static size_t tf_only_suite_run(const char *suite, size_t *end_out) { + size_t first = 0; + while (first < tf_only_count && strcmp(tf_only_tokens[first].suite, suite) != 0) { + first++; + } + size_t end = first; + while (end < tf_only_count && strcmp(tf_only_tokens[end].suite, suite) == 0) { + end++; + } + *end_out = end; + return first; +} + +static void tf_only_raise(size_t first, size_t end, tf_only_state_t state) { + for (size_t i = first; i < end; i++) { + if (tf_only_tokens[i].state < state) { + tf_only_tokens[i].state = state; + } + } +} + +bool tf_suite_enter(const char *suite) { + if (tf_current_suite) { + /* A suite run from inside another: its tests belong to the registered + * suite that runs it, which is the name a token and argv both use. */ + return true; + } + if (tf_only_active) { + tf_only_suite_first = tf_only_suite_run(suite, &tf_only_suite_end); + if (tf_only_suite_first == tf_only_suite_end) { + return false; /* no token names it: its body, setup included, never runs */ + } + tf_only_raise(tf_only_suite_first, tf_only_suite_end, TF_ONLY_ENTERED); + } + tf_current_suite = suite; + return true; +} + +void tf_suite_leave(const char *suite) { + if (tf_current_suite && strcmp(tf_current_suite, suite) == 0) { + tf_current_suite = NULL; + tf_only_suite_first = 0; + tf_only_suite_end = 0; + } +} + +bool tf_test_selected(const char *test) { + if (!tf_only_active) { + return true; + } + bool selected = false; + for (size_t i = tf_only_suite_first; i < tf_only_suite_end; i++) { + if (strcmp(tf_only_tokens[i].test, test) == 0) { + tf_only_tokens[i].state = TF_ONLY_MATCHED; + selected = true; + } + } + if (!selected) { + tf_deselected_count++; + } + return selected; +} + +/* The strict half of the selection: every token this process was responsible + * for must have run a test. Each one that did not is a failure of the run. */ +static void tf_only_report_unmatched(void) { + for (size_t i = 0; i < tf_only_count; i++) { + const tf_only_token_t *token = &tf_only_tokens[i]; + if (token->state != TF_ONLY_UNSEEN && token->state != TF_ONLY_ENTERED) { + continue; + } + if (i > 0 && tf_only_compare(token, &tf_only_tokens[i - 1]) == 0) { + continue; /* named twice, reported once */ + } + fprintf(stderr, "selected test not compiled into this build or unknown: %s:%s\n", + token->suite, token->test); + tf_fail_count++; + } +} + +#ifdef CBM_TEST_COVERAGE +/* ── Per-test coverage profiles ────────────────────────────────────── + * Compiled only into the instrumented runner (make test-runner-cov: clang + * source coverage, no sanitizer). With CBM_TEST_COVERAGE_DIR set, every test + * leaves its own raw profiles under //: + * .parent.profraw what this process executed inside the test + * ..profraw one per child the test spawned. Children + * inherit LLVM_PROFILE_FILE, so a re-exec of this + * runner or a product binary names its own file + * _setup..profraw what ran between tests: suite setup, and the + * prologue and epilogue of the runner itself + * _setup.child..profraw children spawned between tests + * The first two are the coverage of the test; the others belong to no test. */ +void __llvm_profile_set_filename(const char *pattern); +int __llvm_profile_write_file(void); +void __llvm_profile_reset_counters(void); + +enum { TF_COVERAGE_PATH_CAP = CBM_SZ_4K + CBM_SZ_1K }; +static char tf_coverage_dir[CBM_SZ_4K]; /* canonical; empty = not a coverage run */ +/* The runtime keeps the pointer it is handed, not a copy: these outlive it. */ +static char tf_coverage_profile[TF_COVERAGE_PATH_CAP]; +static char tf_coverage_child_pattern[TF_COVERAGE_PATH_CAP]; +static unsigned int tf_coverage_setup_index = 0; + +/* Names the profile this process writes next and the one its children will. */ +static bool tf_coverage_route(const char *directory, const char *own, const char *child) { + int own_length = + snprintf(tf_coverage_profile, sizeof(tf_coverage_profile), "%s/%s.profraw", directory, own); + int child_length = snprintf(tf_coverage_child_pattern, sizeof(tf_coverage_child_pattern), + "%s/%s.%%p.profraw", directory, child); + if (own_length <= 0 || (size_t)own_length >= sizeof(tf_coverage_profile) || child_length <= 0 || + (size_t)child_length >= sizeof(tf_coverage_child_pattern)) { + return false; + } + __llvm_profile_set_filename(tf_coverage_profile); + return cbm_setenv("LLVM_PROFILE_FILE", tf_coverage_child_pattern, 1) == 0; +} + +/* Between tests the interval is suite setup, and so is any child it spawns. */ +static bool tf_coverage_route_setup(const char *suite_dir) { + char own[CBM_SZ_64]; + (void)snprintf(own, sizeof(own), "_setup.%u", tf_coverage_setup_index); + return tf_coverage_route(suite_dir, own, "_setup.child"); +} + +/* Writes what this process executed since the last write to its current profile + * and restarts the counters, so every file holds exactly one interval. */ +static void tf_coverage_flush(void) { + (void)__llvm_profile_write_file(); + __llvm_profile_reset_counters(); +} + +static bool tf_coverage_suite_dir(char *out, size_t cap) { + int length = snprintf(out, cap, "%s/%s", tf_coverage_dir, + tf_current_suite ? tf_current_suite : "_nosuite"); + return length > 0 && (size_t)length < cap && cbm_mkdir_p(out, 0755); +} + +/* A coverage run that loses a profile would report the test as executing + * nothing, and the impact map would trust that: fail the run instead. */ +static void tf_coverage_lost(const char *test) { + fprintf(stderr, "coverage profile could not be routed: %s:%s\n", + tf_current_suite ? tf_current_suite : "_nosuite", test); + tf_fail_count++; +} + +void tf_coverage_test_begin(const char *test) { + char suite_dir[TF_COVERAGE_PATH_CAP]; + char own[CBM_SZ_512]; + if (!tf_coverage_dir[0]) { + return; + } + int own_length = snprintf(own, sizeof(own), "%s.parent", test); + if (own_length <= 0 || (size_t)own_length >= sizeof(own) || + !tf_coverage_suite_dir(suite_dir, sizeof(suite_dir)) || + !tf_coverage_route_setup(suite_dir)) { + tf_coverage_lost(test); + return; + } + tf_coverage_flush(); + tf_coverage_setup_index++; + /* Named before the test runs, not after: a forked child that leaves through + * exit() writes to the name it inherited, which must not be the setup file. */ + if (!tf_coverage_route(suite_dir, own, test)) { + tf_coverage_lost(test); + } +} + +void tf_coverage_test_end(const char *test) { + char suite_dir[TF_COVERAGE_PATH_CAP]; + char own[CBM_SZ_512]; + if (!tf_coverage_dir[0]) { + return; + } + int own_length = snprintf(own, sizeof(own), "%s.parent", test); + if (own_length <= 0 || (size_t)own_length >= sizeof(own) || + !tf_coverage_suite_dir(suite_dir, sizeof(suite_dir)) || + !tf_coverage_route(suite_dir, own, test)) { + tf_coverage_lost(test); + return; + } + tf_coverage_flush(); + /* The runtime writes once more at exit, to whatever name is current. Left + * on the file of this test, that write would replace the profile just + * taken with the teardown that follows it. */ + if (!tf_coverage_route_setup(suite_dir)) { + tf_coverage_lost(test); + } +} + +/* False when the run asked for coverage and cannot have it. Children run in + * other working directories, so they are handed an absolute name. */ +static bool tf_coverage_init(void) { + const char *dir = getenv("CBM_TEST_COVERAGE_DIR"); + if (!dir || !dir[0]) { + return true; + } + if (!cbm_mkdir_p(dir, 0755) || + !cbm_canonical_path(dir, tf_coverage_dir, sizeof(tf_coverage_dir))) { + tf_coverage_dir[0] = '\0'; + fprintf(stderr, "cannot use coverage directory: %s\n", dir); + return false; + } + /* Routing sets the environment, which may move what `dir` points at. */ + if (!tf_coverage_route(tf_coverage_dir, "_runner.%p", "_runner.child")) { + fprintf(stderr, "cannot route coverage profiles under: %s\n", tf_coverage_dir); + tf_coverage_dir[0] = '\0'; + return false; + } + return true; +} +#endif /* CBM_TEST_COVERAGE */ + static int g_suite_argc = 0; static char **g_suite_argv = NULL; static bool *g_suite_arg_matched = NULL; static bool suite_requested(const char *name) { + /* Every registered suite passes through here, requested or not: this is + * where a selection token learns that its suite exists in this build. */ + size_t token_end = 0; + size_t token_first = tf_only_suite_run(name, &token_end); + tf_only_raise(token_first, token_end, TF_ONLY_REGISTERED); if (g_suite_argc <= 1) { return true; } @@ -1173,6 +1565,16 @@ int main(int argc, char **argv) { g_suite_argc = argc; g_suite_argv = argv; } + if (!g_list_only && !tf_only_init()) { + tf_only_free(); + return 2; + } +#ifdef CBM_TEST_COVERAGE + if (!g_list_only && !tf_coverage_init()) { + tf_only_free(); + return 2; + } +#endif if (g_suite_argc > 1) { g_suite_arg_matched = calloc((size_t)argc, sizeof(*g_suite_arg_matched)); if (!g_suite_arg_matched) { @@ -1417,6 +1819,8 @@ int main(int argc, char **argv) { if (g_suite_argc > 1 && !any_suite_matched) { fprintf(stderr, "No matching test suites requested\n"); } + tf_only_report_unmatched(); + tf_only_free(); free(g_suite_arg_matched); g_suite_arg_matched = NULL;