From 5444da4166ba15c9b31e04ae010bfa274fe7d5c7 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 6 Aug 2026 01:45:56 +0200 Subject: [PATCH] fix(ci): isolate local-CI runs per run id, and tidy what each run creates Concurrent local-CI legs clobbered each other. Container names were already unique per `compose run --rm`, but the mutable state was not: - Docker: `cbm-build` is ONE named volume shared by every service and every concurrent run, so two legs wrote the same /src/build - objects and test-logs included. The parallel scheduler then dies reading a suite log another run replaced ('cannot read suite log .../extraction.log'). - Windows VM: one checkout (/c/cbm) plus a FIXED log path (/tmp/win-test.log) and a shared build dir; -PruneStale could also delete a live run's temp root. Each run now carries a unique id (pid+epoch, overridable via CBM_CI_RUN_ID): - docker-compose: the build volume takes its name from CBM_CI_BUILD_VOLUME, defaulting to today's `cbm-build` so the single-run path is unchanged. - run.sh: derives the id, points the build volume at it, and removes that volume on success. CBM_CI_SHARED_BUILD=1 opts back into the shared volume. - vm-run-tests.sh: per-run log; and when a CALLER sets CBM_CI_RUN_ID (i.e. declares concurrency) a per-run BUILD_DIR too, removed on success. ccache and the fixture cache stay SHARED deliberately - ccache is concurrency-safe and content-verified, and sharing them is what keeps an isolated run fast instead of cold. Cleanup follows one rule: a run tidies what it created, EXCEPT when it failed - then the artifacts are the post-mortem and the run prints how to inspect and drop them. CBM_CI_KEEP=1 keeps them regardless. Verified: bash -n on both scripts; `docker compose config` valid, and with CBM_CI_BUILD_VOLUME set the volume resolves to the per-run name. Signed-off-by: Martin Vogel --- test-infrastructure/docker-compose.yml | 8 ++++++ test-infrastructure/run.sh | 40 ++++++++++++++++++++++++++ test-infrastructure/vm/vm-run-tests.sh | 36 +++++++++++++++++++++-- 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/test-infrastructure/docker-compose.yml b/test-infrastructure/docker-compose.yml index 0b6acfb34..bf184f615 100644 --- a/test-infrastructure/docker-compose.yml +++ b/test-infrastructure/docker-compose.yml @@ -319,5 +319,13 @@ volumes: # native filesystem: object writes over the virtiofs bind mount are the # container legs' largest avoidable I/O cost, and the fixture cache makes # the network clone once-per-volume instead of once-per-container. + # The build volume is the ONLY per-run-mutable one: concurrent legs writing + # the same build dir clobber each other's objects and test-logs (a scheduler + # then fails reading a suite log that another run replaced). run.sh gives each + # run its own volume via CBM_CI_BUILD_VOLUME and removes it afterwards; the + # default name preserves the single-run fast path. ccache and the fixture + # cache stay SHARED on purpose — ccache is concurrency-safe and + # content-verified, and sharing them is what keeps isolated runs fast. cbm-build: + name: ${CBM_CI_BUILD_VOLUME:-cbm-build} cbm-fixture-cache: diff --git a/test-infrastructure/run.sh b/test-infrastructure/run.sh index 14a4a6af3..fedf986d6 100755 --- a/test-infrastructure/run.sh +++ b/test-infrastructure/run.sh @@ -56,6 +56,46 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +# ── Per-run isolation (concurrent agents / multiple worktrees) ─────────────── +# Container names are already unique per `compose run --rm`, but the BUILD +# volume was not: two legs running at once wrote the same /src/build, so one +# run's objects and test-logs replaced the other's — after which the parallel +# scheduler dies reading a suite log that another run had removed. Each run now +# gets its own build volume, keyed by a unique run id. ccache and the fixture +# cache stay SHARED on purpose: ccache is concurrency-safe and content-verified, +# and sharing them is what keeps an isolated run fast rather than cold. +# +# CBM_CI_RUN_ID= name/reuse a run (default: pid + epoch, unique) +# CBM_CI_SHARED_BUILD=1 opt back into the single shared `cbm-build` volume +# CBM_CI_KEEP=1 keep this run's build volume even on success +RUN_ID="${CBM_CI_RUN_ID:-$$-$(date +%s)}" +if [ "${CBM_CI_SHARED_BUILD:-0}" = "1" ]; then + CBM_CI_BUILD_VOLUME="cbm-build" +else + CBM_CI_BUILD_VOLUME="cbm-build-${RUN_ID}" +fi +export CBM_CI_BUILD_VOLUME + +# Tidy what this run generated. A FAILED run KEEPS its volume — those artifacts +# are the post-mortem — and prints how to inspect and drop it. Successful runs +# leave nothing behind. Shared-volume mode never removes anything. +ci_cleanup() { + local rc=$? + if [ "${CBM_CI_BUILD_VOLUME}" = "cbm-build" ] || [ "${CBM_CI_KEEP:-0}" = "1" ]; then + return $rc + fi + if [ $rc -eq 0 ]; then + docker volume rm -f "${CBM_CI_BUILD_VOLUME}" >/dev/null 2>&1 || true + else + echo "run.sh: kept ${CBM_CI_BUILD_VOLUME} for post-mortem (run failed)" >&2 + echo " inspect: docker run --rm -v ${CBM_CI_BUILD_VOLUME}:/b alpine ls -R /b" >&2 + echo " drop: docker volume rm ${CBM_CI_BUILD_VOLUME}" >&2 + fi + return $rc +} +trap ci_cleanup EXIT + COMPOSE="docker compose -f $ROOT/test-infrastructure/docker-compose.yml" usage() { diff --git a/test-infrastructure/vm/vm-run-tests.sh b/test-infrastructure/vm/vm-run-tests.sh index b87cce060..2bd76da4f 100644 --- a/test-infrastructure/vm/vm-run-tests.sh +++ b/test-infrastructure/vm/vm-run-tests.sh @@ -46,7 +46,25 @@ case "${1:-}" in esac RUNNER="${CBM_VM_RUNNER:-}" -LOG="${CBM_VM_TEST_LOG:-/tmp/win-test.log}" + +# Per-run identity. The VM holds ONE checkout (/c/cbm), so two concurrent runs +# shared a FIXED log path and the same build dir: the later run's output +# replaced the earlier one's, and -PruneStale could delete a live run's temp +# root out from under it. The run id namespaces the log and the build dir; the +# protected temp root is already unique per run. +CALLER_RUN_ID="${CBM_CI_RUN_ID:-}" +RUN_ID="${CBM_CI_RUN_ID:-$$-$(date +%s)}" +export CBM_CI_RUN_ID="$RUN_ID" +LOG="${CBM_VM_TEST_LOG:-/tmp/win-test-${RUN_ID}.log}" + +# A caller that sets CBM_CI_RUN_ID is declaring concurrency, so give that run +# its own BUILD_DIR; the default single-run path keeps the shared one and its +# incremental reuse (a per-run build dir on the VM costs a full rebuild). +if [ -n "$CALLER_RUN_ID" ]; then + BUILD_ARGS=("BUILD_DIR=build/vm-${RUN_ID}") +else + BUILD_ARGS=() +fi [ $# -ge 1 ] || { echo "vm-run-tests: missing arguments. Please consult --help." >&2; exit 2; } if [ "$1" = "--soak" ]; then @@ -130,13 +148,25 @@ if [ -n "$RUNNER" ]; then rc="${PIPESTATUS[0]}" fi elif [ "$1" = "--par" ]; then - scripts/test.sh CC=clang CXX=clang++ 2>&1 | tee "$LOG" + scripts/test.sh CC=clang CXX=clang++ \ + ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} 2>&1 | tee "$LOG" rc="${PIPESTATUS[0]}" else - scripts/test.sh --suites "$*" CC=clang CXX=clang++ 2>&1 | tee "$LOG" + scripts/test.sh --suites "$*" CC=clang CXX=clang++ \ + ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} 2>&1 | tee "$LOG" rc="${PIPESTATUS[0]}" fi +# Tidy this run's own build dir on success; keep it on failure as the +# post-mortem. Only ever touches a dir this run created (concurrent mode). +if [ -n "$CALLER_RUN_ID" ] && [ "${CBM_CI_KEEP:-0}" != "1" ]; then + if [ "${rc:-1}" -eq 0 ]; then + rm -rf "build/vm-${RUN_ID}" + else + echo "vm-run-tests: kept build/vm-${RUN_ID} and $LOG for post-mortem" >&2 + fi +fi + if ! grep -Eq '[0-9]+ passed' "$LOG"; then echo "GUARD: test runner produced no completion summary — the suites did" \ "not validly run; treating as failure (runner rc=$rc)" >&2