fix(ci): isolate local-CI runs per run id, and tidy what each run creates

Concurrent local-CI legs clobbered each other. Container names were already
unique per `compose run --rm`, but the mutable state was not:

- Docker: `cbm-build` is ONE named volume shared by every service and every
  concurrent run, so two legs wrote the same /src/build - objects and
  test-logs included. The parallel scheduler then dies reading a suite log
  another run replaced ('cannot read suite log .../extraction.log').
- Windows VM: one checkout (/c/cbm) plus a FIXED log path (/tmp/win-test.log)
  and a shared build dir; -PruneStale could also delete a live run's temp root.

Each run now carries a unique id (pid+epoch, overridable via CBM_CI_RUN_ID):

- docker-compose: the build volume takes its name from CBM_CI_BUILD_VOLUME,
  defaulting to today's `cbm-build` so the single-run path is unchanged.
- run.sh: derives the id, points the build volume at it, and removes that
  volume on success. CBM_CI_SHARED_BUILD=1 opts back into the shared volume.
- vm-run-tests.sh: per-run log; and when a CALLER sets CBM_CI_RUN_ID (i.e.
  declares concurrency) a per-run BUILD_DIR too, removed on success.

ccache and the fixture cache stay SHARED deliberately - ccache is
concurrency-safe and content-verified, and sharing them is what keeps an
isolated run fast instead of cold.

Cleanup follows one rule: a run tidies what it created, EXCEPT when it failed -
then the artifacts are the post-mortem and the run prints how to inspect and
drop them. CBM_CI_KEEP=1 keeps them regardless.

Verified: bash -n on both scripts; `docker compose config` valid, and with
CBM_CI_BUILD_VOLUME set the volume resolves to the per-run name.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

test-infrastructure/docker-compose.yml

@@ -319,5 +319,13 @@ volumes:
   # native filesystem: object writes over the virtiofs bind mount are the
   # container legs' largest avoidable I/O cost, and the fixture cache makes
   # the network clone once-per-volume instead of once-per-container.
+  # The build volume is the ONLY per-run-mutable one: concurrent legs writing
+  # the same build dir clobber each other's objects and test-logs (a scheduler
+  # then fails reading a suite log that another run replaced). run.sh gives each
+  # run its own volume via CBM_CI_BUILD_VOLUME and removes it afterwards; the
+  # default name preserves the single-run fast path. ccache and the fixture
+  # cache stay SHARED on purpose — ccache is concurrency-safe and
+  # content-verified, and sharing them is what keeps isolated runs fast.
   cbm-build:
+    name: ${CBM_CI_BUILD_VOLUME:-cbm-build}
   cbm-fixture-cache:

test-infrastructure/run.sh

@@ -56,6 +56,46 @@
 set -euo pipefail
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+
+# ── Per-run isolation (concurrent agents / multiple worktrees) ───────────────
+# Container names are already unique per `compose run --rm`, but the BUILD
+# volume was not: two legs running at once wrote the same /src/build, so one
+# run's objects and test-logs replaced the other's — after which the parallel
+# scheduler dies reading a suite log that another run had removed. Each run now
+# gets its own build volume, keyed by a unique run id. ccache and the fixture
+# cache stay SHARED on purpose: ccache is concurrency-safe and content-verified,
+# and sharing them is what keeps an isolated run fast rather than cold.
+#
+#   CBM_CI_RUN_ID=<id>       name/reuse a run (default: pid + epoch, unique)
+#   CBM_CI_SHARED_BUILD=1    opt back into the single shared `cbm-build` volume
+#   CBM_CI_KEEP=1            keep this run's build volume even on success
+RUN_ID="${CBM_CI_RUN_ID:-$$-$(date +%s)}"
+if [ "${CBM_CI_SHARED_BUILD:-0}" = "1" ]; then
+    CBM_CI_BUILD_VOLUME="cbm-build"
+else
+    CBM_CI_BUILD_VOLUME="cbm-build-${RUN_ID}"
+fi
+export CBM_CI_BUILD_VOLUME
+
+# Tidy what this run generated. A FAILED run KEEPS its volume — those artifacts
+# are the post-mortem — and prints how to inspect and drop it. Successful runs
+# leave nothing behind. Shared-volume mode never removes anything.
+ci_cleanup() {
+    local rc=$?
+    if [ "${CBM_CI_BUILD_VOLUME}" = "cbm-build" ] || [ "${CBM_CI_KEEP:-0}" = "1" ]; then
+        return $rc
+    fi
+    if [ $rc -eq 0 ]; then
+        docker volume rm -f "${CBM_CI_BUILD_VOLUME}" >/dev/null 2>&1 || true
+    else
+        echo "run.sh: kept ${CBM_CI_BUILD_VOLUME} for post-mortem (run failed)" >&2
+        echo "        inspect: docker run --rm -v ${CBM_CI_BUILD_VOLUME}:/b alpine ls -R /b" >&2
+        echo "        drop:    docker volume rm ${CBM_CI_BUILD_VOLUME}" >&2
+    fi
+    return $rc
+}
+trap ci_cleanup EXIT
+
 COMPOSE="docker compose -f $ROOT/test-infrastructure/docker-compose.yml"
 
 usage() {

test-infrastructure/vm/vm-run-tests.sh

@@ -46,7 +46,25 @@ case "${1:-}" in
 esac
 
 RUNNER="${CBM_VM_RUNNER:-}"
-LOG="${CBM_VM_TEST_LOG:-/tmp/win-test.log}"
+
+# Per-run identity. The VM holds ONE checkout (/c/cbm), so two concurrent runs
+# shared a FIXED log path and the same build dir: the later run's output
+# replaced the earlier one's, and -PruneStale could delete a live run's temp
+# root out from under it. The run id namespaces the log and the build dir; the
+# protected temp root is already unique per run.
+CALLER_RUN_ID="${CBM_CI_RUN_ID:-}"
+RUN_ID="${CBM_CI_RUN_ID:-$$-$(date +%s)}"
+export CBM_CI_RUN_ID="$RUN_ID"
+LOG="${CBM_VM_TEST_LOG:-/tmp/win-test-${RUN_ID}.log}"
+
+# A caller that sets CBM_CI_RUN_ID is declaring concurrency, so give that run
+# its own BUILD_DIR; the default single-run path keeps the shared one and its
+# incremental reuse (a per-run build dir on the VM costs a full rebuild).
+if [ -n "$CALLER_RUN_ID" ]; then
+    BUILD_ARGS=("BUILD_DIR=build/vm-${RUN_ID}")
+else
+    BUILD_ARGS=()
+fi
 
 [ $# -ge 1 ] || { echo "vm-run-tests: missing arguments. Please consult --help." >&2; exit 2; }
 if [ "$1" = "--soak" ]; then
@@ -130,13 +148,25 @@ if [ -n "$RUNNER" ]; then
         rc="${PIPESTATUS[0]}"
     fi
 elif [ "$1" = "--par" ]; then
-    scripts/test.sh CC=clang CXX=clang++ 2>&1 | tee "$LOG"
+    scripts/test.sh CC=clang CXX=clang++ \
+        ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} 2>&1 | tee "$LOG"
     rc="${PIPESTATUS[0]}"
 else
-    scripts/test.sh --suites "$*" CC=clang CXX=clang++ 2>&1 | tee "$LOG"
+    scripts/test.sh --suites "$*" CC=clang CXX=clang++ \
+        ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} 2>&1 | tee "$LOG"
     rc="${PIPESTATUS[0]}"
 fi
 
+# Tidy this run's own build dir on success; keep it on failure as the
+# post-mortem. Only ever touches a dir this run created (concurrent mode).
+if [ -n "$CALLER_RUN_ID" ] && [ "${CBM_CI_KEEP:-0}" != "1" ]; then
+    if [ "${rc:-1}" -eq 0 ]; then
+        rm -rf "build/vm-${RUN_ID}"
+    else
+        echo "vm-run-tests: kept build/vm-${RUN_ID} and $LOG for post-mortem" >&2
+    fi
+fi
+
 if ! grep -Eq '[0-9]+ passed' "$LOG"; then
     echo "GUARD: test runner produced no completion summary — the suites did" \
          "not validly run; treating as failure (runner rc=$rc)" >&2