test-infra: overlap the local ladder, open the serial tail, keep container I/O off virtiofs

Local runs left most of the machine idle: the serial tail ran sixteen
suites one at a time on otherwise-idle cores, the three platform legs
were launched by hand (usually sequentially), and every container-leg
object write crossed the virtiofs bind mount.

- run-tests-parallel.sh tail scheduling in two phases: the FLEX suites
  (timing-shaped but free of the shared per-account daemon runtime
  namespace) run CBM_TAIL_JOBS-wide (default 2), then the EXCL group —
  daemon-family plus the suites that drive daemon one-shots or
  supervisor rendezvous — runs strictly sequentially on a machine
  exactly as quiet as the old fully-serial tail gave it. The wave was
  already fed longest-first by the shard dealing order, so the drain-out
  no longer ends on a heavy straggler.
- ladder.sh: one maintained entry point for the full local push gate
  with the legs overlapped — lint, the Linux container suite, and the
  Windows VM suite in the background, the macOS suite in the
  foreground, one verdict per leg, logs kept per leg. A missing
  prerequisite fails its leg loudly instead of silently skipping.
- win.sh test-par now runs through vm-run-tests.sh (--par mode): the
  full parallel harness under the CI-shaped protected temp root with
  complete output. It previously ran under the MSYS-shared /tmp and
  piped through `tail -25` — the same truncated-blindness class that
  hid 40 Windows failures from the `test` command.
- docker-compose: build artifacts and the incremental fixture cache
  move to named volumes on the container VM's native filesystem. Object
  writes over the virtiofs bind mount are the container legs' largest
  avoidable I/O cost, and the fixture cache now survives across
  container runs.
- test_mem(win): the first valid full-parallel VM run proved working-set
  trimming beats the re-touch mitigation (19 MB resident of a 256 MB
  double-touch at 18 parallel suites). The RSS probe now VirtualLocks a
  64 MB span — locked pages are exempt from trimming, making the
  measurement pressure-immune — with bounded touch-and-sample retries
  when the lock is unavailable. Red-to-green under the same 18-job load.
- cli: the portable install's staging error now appends the activation
  refusal note (predicate, SID, object) like the managed path already
  does — a bare "activation transaction I/O failed" on a CI-only
  failure is undiagnosable without it.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

scripts/run-tests-parallel.sh

@@ -186,10 +186,34 @@ run_one() {
 export -f run_one
 
 xargs -P "$JOBS" -I{} bash -c 'run_one "$@"' _ {} < "$PAR_FILE"
-# Serial tail: quiet machine for the deadline-sensitive suites.
+
+# Tail scheduling in two phases. The FLEX suites are timing-shaped but do
+# not rendezvous through the shared per-account daemon runtime namespace,
+# so a small fixed overlap (CBM_TAIL_JOBS, default 2) is safe and converts
+# idle cores into wall time — the old fully-serial tail ran them one at a
+# time on an idle machine. The EXCL group (daemon-family plus the suites
+# that drive daemon one-shots or supervisor rendezvous) then runs strictly
+# sequentially on a machine exactly as quiet as the old tail gave it.
+TAIL_EXCL="cli mcp index_supervisor daemon_application daemon_runtime \
+    daemon_frontend daemon_bootstrap daemon_ipc"
+is_tail_excl() {
+    case " $TAIL_EXCL " in *" $1 "*) return 0 ;; *) return 1 ;; esac
+}
+FLEX_FILE="$LOGDIR/suites-tail-flex.txt"
+EXCL_FILE="$LOGDIR/suites-tail-excl.txt"
+: > "$FLEX_FILE"
+: > "$EXCL_FILE"
+while IFS= read -r sname; do
+    if is_tail_excl "$sname"; then
+        echo "$sname" >> "$EXCL_FILE"
+    else
+        echo "$sname" >> "$FLEX_FILE"
+    fi
+done < "$SER_FILE"
+xargs -P "${CBM_TAIL_JOBS:-2}" -I{} bash -c 'run_one "$@"' _ {} < "$FLEX_FILE"
 while IFS= read -r sname; do
     run_one "$sname"
-done < "$SER_FILE"
+done < "$EXCL_FILE"
 
 # Machine-checkable manifest for CI's cross-shard completeness job: it
 # proves at runtime that the shards of one leg agree on N and on the full

src/cli/cli.c

@@ -10627,8 +10627,10 @@ int cbm_cmd_install(int argc, char **argv) {
         cli_binary_validator_t staged_validator = {{0}};
         if (stage_status != CBM_ACTIVATION_TRANSACTION_OK || !binary_transaction ||
             !cli_activation_transaction_expected_build(binary_transaction, &staged_validator)) {
-            (void)fprintf(stderr, "error: failed to stage install candidate: %s\n",
-                          cbm_activation_transaction_status_message(stage_status));
+            const char *stage_refusal = cbm_activation_transaction_refusal_note();
+            (void)fprintf(stderr, "error: failed to stage install candidate: %s%s%s\n",
+                          cbm_activation_transaction_status_message(stage_status),
+                          stage_refusal[0] ? ": " : "", stage_refusal);
             (void)cli_activation_transaction_abort(&binary_transaction);
             if (prepared_dir[0]) {
                 (void)cbm_rmdir(prepared_dir);

test-infrastructure/docker-compose.yml

@@ -20,6 +20,8 @@ services:
     platform: linux/arm64
     volumes:
       - ..:/src
+      - cbm-build:/src/build
+      - cbm-fixture-cache:/root/.cache/cbm-test-fixtures
       - cbm-ccache-arm64:/root/.ccache
     environment:
       CCACHE_DIR: /root/.ccache
@@ -33,6 +35,8 @@ services:
     platform: linux/amd64
     volumes:
       - ..:/src
+      - cbm-build:/src/build
+      - cbm-fixture-cache:/root/.cache/cbm-test-fixtures
       - cbm-ccache-amd64:/root/.ccache
     environment:
       CCACHE_DIR: /root/.ccache
@@ -50,6 +54,8 @@ services:
     platform: linux/arm64
     volumes:
       - ..:/src
+      - cbm-build:/src/build
+      - cbm-fixture-cache:/root/.cache/cbm-test-fixtures
       - cbm-ccache-arm64:/root/.ccache
     environment:
       CCACHE_DIR: /root/.ccache
@@ -71,6 +77,8 @@ services:
     platform: linux/amd64
     volumes:
       - ..:/src
+      - cbm-build:/src/build
+      - cbm-fixture-cache:/root/.cache/cbm-test-fixtures
       - cbm-ccache-amd64:/root/.ccache
     environment:
       CCACHE_DIR: /root/.ccache
@@ -236,3 +244,9 @@ services:
 volumes:
   cbm-ccache-arm64:
   cbm-ccache-amd64:
+  # Build artifacts and the incremental-suite fixture cache stay on the VM's
+  # native filesystem: object writes over the virtiofs bind mount are the
+  # container legs' largest avoidable I/O cost, and the fixture cache makes
+  # the network clone once-per-volume instead of once-per-container.
+  cbm-build:
+  cbm-fixture-cache:

test-infrastructure/ladder.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# ladder.sh — the full local push gate, with all platform legs OVERLAPPED.
+#
+# The three platforms are separate scheduling domains (macOS native, the
+# Colima container VM, the Windows UTM VM), so running their legs
+# sequentially leaves most of the machine idle most of the time. This
+# launches lint + the Linux container suite + the Windows VM suite in the
+# background, runs the macOS full suite in the foreground, then joins and
+# prints one verdict per leg. Each leg streams to its own log so a red leg
+# can be inspected without rerunning anything.
+#
+#   test-infrastructure/ladder.sh
+#
+# Prerequisites: Colima running (Linux/lint legs), the Windows VM
+# provisioned and reachable (vm/provision-windows.sh). A missing
+# prerequisite fails that leg loudly — never silently skips it.
+set -uo pipefail
+
+cd "$(dirname "$0")/.."
+LOGS="${CBM_LADDER_LOGS:-${TMPDIR:-/tmp}/cbm-ladder-$$}"
+mkdir -p "$LOGS"
+
+echo "=== local ladder: lint + linux + windows in background, mac in foreground ==="
+echo "=== logs: $LOGS ==="
+
+./test-infrastructure/run.sh lint > "$LOGS/lint.log" 2>&1 &
+LINT_PID=$!
+./test-infrastructure/run.sh test > "$LOGS/linux.log" 2>&1 &
+LINUX_PID=$!
+(./test-infrastructure/vm/win.sh sync &&
+    ./test-infrastructure/vm/win.sh test-par) > "$LOGS/windows.log" 2>&1 &
+WIN_PID=$!
+
+scripts/test.sh 2>&1 | tee "$LOGS/mac.log"
+MAC_RC=${PIPESTATUS[0]}
+
+wait "$LINT_PID"
+LINT_RC=$?
+wait "$LINUX_PID"
+LINUX_RC=$?
+wait "$WIN_PID"
+WIN_RC=$?
+
+echo ""
+echo "=== ladder verdicts ==="
+overall=0
+report() {
+    local leg="$1" rc="$2" log="$3"
+    if [ "$rc" -eq 0 ]; then
+        echo "  PASS $leg"
+    else
+        echo "  FAIL $leg (rc=$rc) — $log"
+        echo "  ── last 15 lines ──"
+        tail -15 "$log" | sed 's/^/    /'
+        overall=1
+    fi
+}
+report "macOS   " "$MAC_RC" "$LOGS/mac.log"
+report "lint    " "$LINT_RC" "$LOGS/lint.log"
+report "linux   " "$LINUX_RC" "$LOGS/linux.log"
+report "windows " "$WIN_RC" "$LOGS/windows.log"
+exit "$overall"

test-infrastructure/vm/vm-run-tests.sh

@@ -20,7 +20,7 @@ set -uo pipefail
 RUNNER="${CBM_VM_RUNNER:-build/c/test-runner}"
 LOG="${CBM_VM_TEST_LOG:-/tmp/win-test.log}"
 
-[ $# -ge 1 ] || { echo "usage: vm-run-tests.sh <suite...>" >&2; exit 2; }
+[ $# -ge 1 ] || { echo "usage: vm-run-tests.sh <suite...> | --par" >&2; exit 2; }
 [ -x "$RUNNER" ] || { echo "ERROR: runner '$RUNNER' missing — build first" >&2; exit 2; }
 
 # Stale roots from earlier runs are removed up front; the current root is kept
@@ -70,8 +70,16 @@ MSYS2_ARG_CONV_EXCL='*' icacls "${runner_dir_w}\\*" /reset /T /C /Q >/dev/null 2
 
 echo "=== vm-run-tests: runner=$RUNNER temp=$TEMP suites: $* ==="
 
-"$RUNNER" "$@" 2>&1 | tee "$LOG"
-rc="${PIPESTATUS[0]}"
+# --par runs the repo's full parallel harness (wave + tail scheduling,
+# per-shard union guard, manifest) under this same protected environment;
+# explicit suite names keep the direct single-runner invocation.
+if [ "$1" = "--par" ]; then
+    bash scripts/run-tests-parallel.sh "$RUNNER" 2>&1 | tee "$LOG"
+    rc="${PIPESTATUS[0]}"
+else
+    "$RUNNER" "$@" 2>&1 | tee "$LOG"
+    rc="${PIPESTATUS[0]}"
+fi
 
 if ! grep -Eq '[0-9]+ passed' "$LOG"; then
     echo "GUARD: test runner produced no completion summary — the suites did" \

test-infrastructure/vm/win.sh

@@ -191,8 +191,11 @@ pageheap)
     esac
     ;;
 test-par)
-    # Full-suite parallel run on all VM cores via the repo's parallel harness.
-    vm clangarm64 "cd /c/cbm && bash scripts/run-tests-parallel.sh build/c/test-runner 2>&1 | tail -25"
+    # Full-suite parallel run on all VM cores via the repo's parallel harness,
+    # under the CI-shaped protected temp root with FULL output (this leg once
+    # ran under the MSYS-shared /tmp and piped through `tail -25` — the same
+    # truncated-blindness class that hid 40 Windows failures from `test`).
+    vm clangarm64 "cd /c/cbm && CBM_VM_TEST_LOG=/tmp/win-test-par.log bash test-infrastructure/vm/vm-run-tests.sh --par"
     ;;
 *)
     echo "unknown command: $cmd (see header for usage)" >&2; exit 2

tests/test_mem.c

@@ -21,6 +21,11 @@
 #include <mimalloc.h>
 #ifndef _WIN32
 #include <sys/mman.h>
+#else
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
 #endif
 
 /* ASan detection — mimalloc MI_OVERRIDE=0 under ASan, mi_process_info
@@ -275,11 +280,32 @@ TEST(mem_rss_reflects_external_resident_memory) {
      * reporting a broken small counter instead of true resident memory — which
      * the Linux #else branch exercises directly against the undercount. */
     const size_t threshold = (size_t)32 * 1024 * 1024;
+    const size_t lock_span = (size_t)64 * 1024 * 1024;
     void *big = malloc(region);
     ASSERT_NOT_NULL(big);
     memset(big, 0x5A, region);
-    memset(big, 0x5B, region); /* re-touch right before the measurement */
-    size_t rss = cbm_mem_rss();
+    /* Trimming can evict even a just-touched region: at 18 parallel suites
+     * the VM kept 19 MB resident of a 256 MB double-touch, losing the
+     * re-touch race this test previously relied on. Locked pages are exempt
+     * from working-set trimming, so lock a span comfortably above the
+     * threshold and the measurement becomes pressure-immune. When the lock
+     * is unavailable (working-set quota policy), fall back to bounded
+     * touch-and-sample retries — those races are transient. */
+    HANDLE self_process = GetCurrentProcess();
+    bool locked = SetProcessWorkingSetSize(self_process, lock_span + (size_t)32 * 1024 * 1024,
+                                           (size_t)512 * 1024 * 1024) != 0 &&
+                  VirtualLock(big, lock_span) != 0;
+    size_t rss = 0;
+    for (int attempt = 0; attempt < 6; attempt++) {
+        memset(big, 0x5B + attempt, lock_span);
+        rss = cbm_mem_rss();
+        if (locked || rss >= threshold) {
+            break;
+        }
+    }
+    if (locked) {
+        (void)VirtualUnlock(big, lock_span);
+    }
     ASSERT_GTE(rss, threshold);
     free(big);
 #else