diff --git a/test-infrastructure/docker-compose.yml b/test-infrastructure/docker-compose.yml index 0b6acfb34..bf184f615 100644 --- a/test-infrastructure/docker-compose.yml +++ b/test-infrastructure/docker-compose.yml @@ -319,5 +319,13 @@ volumes: # native filesystem: object writes over the virtiofs bind mount are the # container legs' largest avoidable I/O cost, and the fixture cache makes # the network clone once-per-volume instead of once-per-container. + # The build volume is the ONLY per-run-mutable one: concurrent legs writing + # the same build dir clobber each other's objects and test-logs (a scheduler + # then fails reading a suite log that another run replaced). run.sh gives each + # run its own volume via CBM_CI_BUILD_VOLUME and removes it afterwards; the + # default name preserves the single-run fast path. ccache and the fixture + # cache stay SHARED on purpose — ccache is concurrency-safe and + # content-verified, and sharing them is what keeps isolated runs fast. cbm-build: + name: ${CBM_CI_BUILD_VOLUME:-cbm-build} cbm-fixture-cache: diff --git a/test-infrastructure/run.sh b/test-infrastructure/run.sh index 14a4a6af3..fedf986d6 100755 --- a/test-infrastructure/run.sh +++ b/test-infrastructure/run.sh @@ -56,6 +56,46 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +# ── Per-run isolation (concurrent agents / multiple worktrees) ─────────────── +# Container names are already unique per `compose run --rm`, but the BUILD +# volume was not: two legs running at once wrote the same /src/build, so one +# run's objects and test-logs replaced the other's — after which the parallel +# scheduler dies reading a suite log that another run had removed. Each run now +# gets its own build volume, keyed by a unique run id. ccache and the fixture +# cache stay SHARED on purpose: ccache is concurrency-safe and content-verified, +# and sharing them is what keeps an isolated run fast rather than cold. +# +# CBM_CI_RUN_ID= name/reuse a run (default: pid + epoch, unique) +# CBM_CI_SHARED_BUILD=1 opt back into the single shared `cbm-build` volume +# CBM_CI_KEEP=1 keep this run's build volume even on success +RUN_ID="${CBM_CI_RUN_ID:-$$-$(date +%s)}" +if [ "${CBM_CI_SHARED_BUILD:-0}" = "1" ]; then + CBM_CI_BUILD_VOLUME="cbm-build" +else + CBM_CI_BUILD_VOLUME="cbm-build-${RUN_ID}" +fi +export CBM_CI_BUILD_VOLUME + +# Tidy what this run generated. A FAILED run KEEPS its volume — those artifacts +# are the post-mortem — and prints how to inspect and drop it. Successful runs +# leave nothing behind. Shared-volume mode never removes anything. +ci_cleanup() { + local rc=$? + if [ "${CBM_CI_BUILD_VOLUME}" = "cbm-build" ] || [ "${CBM_CI_KEEP:-0}" = "1" ]; then + return $rc + fi + if [ $rc -eq 0 ]; then + docker volume rm -f "${CBM_CI_BUILD_VOLUME}" >/dev/null 2>&1 || true + else + echo "run.sh: kept ${CBM_CI_BUILD_VOLUME} for post-mortem (run failed)" >&2 + echo " inspect: docker run --rm -v ${CBM_CI_BUILD_VOLUME}:/b alpine ls -R /b" >&2 + echo " drop: docker volume rm ${CBM_CI_BUILD_VOLUME}" >&2 + fi + return $rc +} +trap ci_cleanup EXIT + COMPOSE="docker compose -f $ROOT/test-infrastructure/docker-compose.yml" usage() { diff --git a/test-infrastructure/vm/vm-run-tests.sh b/test-infrastructure/vm/vm-run-tests.sh index b87cce060..2bd76da4f 100644 --- a/test-infrastructure/vm/vm-run-tests.sh +++ b/test-infrastructure/vm/vm-run-tests.sh @@ -46,7 +46,25 @@ case "${1:-}" in esac RUNNER="${CBM_VM_RUNNER:-}" -LOG="${CBM_VM_TEST_LOG:-/tmp/win-test.log}" + +# Per-run identity. The VM holds ONE checkout (/c/cbm), so two concurrent runs +# shared a FIXED log path and the same build dir: the later run's output +# replaced the earlier one's, and -PruneStale could delete a live run's temp +# root out from under it. The run id namespaces the log and the build dir; the +# protected temp root is already unique per run. +CALLER_RUN_ID="${CBM_CI_RUN_ID:-}" +RUN_ID="${CBM_CI_RUN_ID:-$$-$(date +%s)}" +export CBM_CI_RUN_ID="$RUN_ID" +LOG="${CBM_VM_TEST_LOG:-/tmp/win-test-${RUN_ID}.log}" + +# A caller that sets CBM_CI_RUN_ID is declaring concurrency, so give that run +# its own BUILD_DIR; the default single-run path keeps the shared one and its +# incremental reuse (a per-run build dir on the VM costs a full rebuild). +if [ -n "$CALLER_RUN_ID" ]; then + BUILD_ARGS=("BUILD_DIR=build/vm-${RUN_ID}") +else + BUILD_ARGS=() +fi [ $# -ge 1 ] || { echo "vm-run-tests: missing arguments. Please consult --help." >&2; exit 2; } if [ "$1" = "--soak" ]; then @@ -130,13 +148,25 @@ if [ -n "$RUNNER" ]; then rc="${PIPESTATUS[0]}" fi elif [ "$1" = "--par" ]; then - scripts/test.sh CC=clang CXX=clang++ 2>&1 | tee "$LOG" + scripts/test.sh CC=clang CXX=clang++ \ + ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} 2>&1 | tee "$LOG" rc="${PIPESTATUS[0]}" else - scripts/test.sh --suites "$*" CC=clang CXX=clang++ 2>&1 | tee "$LOG" + scripts/test.sh --suites "$*" CC=clang CXX=clang++ \ + ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} 2>&1 | tee "$LOG" rc="${PIPESTATUS[0]}" fi +# Tidy this run's own build dir on success; keep it on failure as the +# post-mortem. Only ever touches a dir this run created (concurrent mode). +if [ -n "$CALLER_RUN_ID" ] && [ "${CBM_CI_KEEP:-0}" != "1" ]; then + if [ "${rc:-1}" -eq 0 ]; then + rm -rf "build/vm-${RUN_ID}" + else + echo "vm-run-tests: kept build/vm-${RUN_ID} and $LOG for post-mortem" >&2 + fi +fi + if ! grep -Eq '[0-9]+ passed' "$LOG"; then echo "GUARD: test runner produced no completion summary — the suites did" \ "not validly run; treating as failure (runner rc=$rc)" >&2