diff --git a/scripts/run-tests-parallel.sh b/scripts/run-tests-parallel.sh index 05a223722..271242cec 100644 --- a/scripts/run-tests-parallel.sh +++ b/scripts/run-tests-parallel.sh @@ -186,10 +186,34 @@ run_one() { export -f run_one xargs -P "$JOBS" -I{} bash -c 'run_one "$@"' _ {} < "$PAR_FILE" -# Serial tail: quiet machine for the deadline-sensitive suites. + +# Tail scheduling in two phases. The FLEX suites are timing-shaped but do +# not rendezvous through the shared per-account daemon runtime namespace, +# so a small fixed overlap (CBM_TAIL_JOBS, default 2) is safe and converts +# idle cores into wall time — the old fully-serial tail ran them one at a +# time on an idle machine. The EXCL group (daemon-family plus the suites +# that drive daemon one-shots or supervisor rendezvous) then runs strictly +# sequentially on a machine exactly as quiet as the old tail gave it. +TAIL_EXCL="cli mcp index_supervisor daemon_application daemon_runtime \ + daemon_frontend daemon_bootstrap daemon_ipc" +is_tail_excl() { + case " $TAIL_EXCL " in *" $1 "*) return 0 ;; *) return 1 ;; esac +} +FLEX_FILE="$LOGDIR/suites-tail-flex.txt" +EXCL_FILE="$LOGDIR/suites-tail-excl.txt" +: > "$FLEX_FILE" +: > "$EXCL_FILE" +while IFS= read -r sname; do + if is_tail_excl "$sname"; then + echo "$sname" >> "$EXCL_FILE" + else + echo "$sname" >> "$FLEX_FILE" + fi +done < "$SER_FILE" +xargs -P "${CBM_TAIL_JOBS:-2}" -I{} bash -c 'run_one "$@"' _ {} < "$FLEX_FILE" while IFS= read -r sname; do run_one "$sname" -done < "$SER_FILE" +done < "$EXCL_FILE" # Machine-checkable manifest for CI's cross-shard completeness job: it # proves at runtime that the shards of one leg agree on N and on the full diff --git a/src/cli/cli.c b/src/cli/cli.c index 04013a669..7162394ba 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -10627,8 +10627,10 @@ int cbm_cmd_install(int argc, char **argv) { cli_binary_validator_t staged_validator = {{0}}; if (stage_status != CBM_ACTIVATION_TRANSACTION_OK || !binary_transaction || !cli_activation_transaction_expected_build(binary_transaction, &staged_validator)) { - (void)fprintf(stderr, "error: failed to stage install candidate: %s\n", - cbm_activation_transaction_status_message(stage_status)); + const char *stage_refusal = cbm_activation_transaction_refusal_note(); + (void)fprintf(stderr, "error: failed to stage install candidate: %s%s%s\n", + cbm_activation_transaction_status_message(stage_status), + stage_refusal[0] ? ": " : "", stage_refusal); (void)cli_activation_transaction_abort(&binary_transaction); if (prepared_dir[0]) { (void)cbm_rmdir(prepared_dir); diff --git a/test-infrastructure/docker-compose.yml b/test-infrastructure/docker-compose.yml index 342ead06d..c8f6f0ccb 100644 --- a/test-infrastructure/docker-compose.yml +++ b/test-infrastructure/docker-compose.yml @@ -20,6 +20,8 @@ services: platform: linux/arm64 volumes: - ..:/src + - cbm-build:/src/build + - cbm-fixture-cache:/root/.cache/cbm-test-fixtures - cbm-ccache-arm64:/root/.ccache environment: CCACHE_DIR: /root/.ccache @@ -33,6 +35,8 @@ services: platform: linux/amd64 volumes: - ..:/src + - cbm-build:/src/build + - cbm-fixture-cache:/root/.cache/cbm-test-fixtures - cbm-ccache-amd64:/root/.ccache environment: CCACHE_DIR: /root/.ccache @@ -50,6 +54,8 @@ services: platform: linux/arm64 volumes: - ..:/src + - cbm-build:/src/build + - cbm-fixture-cache:/root/.cache/cbm-test-fixtures - cbm-ccache-arm64:/root/.ccache environment: CCACHE_DIR: /root/.ccache @@ -71,6 +77,8 @@ services: platform: linux/amd64 volumes: - ..:/src + - cbm-build:/src/build + - cbm-fixture-cache:/root/.cache/cbm-test-fixtures - cbm-ccache-amd64:/root/.ccache environment: CCACHE_DIR: /root/.ccache @@ -236,3 +244,9 @@ services: volumes: cbm-ccache-arm64: cbm-ccache-amd64: + # Build artifacts and the incremental-suite fixture cache stay on the VM's + # native filesystem: object writes over the virtiofs bind mount are the + # container legs' largest avoidable I/O cost, and the fixture cache makes + # the network clone once-per-volume instead of once-per-container. + cbm-build: + cbm-fixture-cache: diff --git a/test-infrastructure/ladder.sh b/test-infrastructure/ladder.sh new file mode 100644 index 000000000..14b533d71 --- /dev/null +++ b/test-infrastructure/ladder.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# ladder.sh — the full local push gate, with all platform legs OVERLAPPED. +# +# The three platforms are separate scheduling domains (macOS native, the +# Colima container VM, the Windows UTM VM), so running their legs +# sequentially leaves most of the machine idle most of the time. This +# launches lint + the Linux container suite + the Windows VM suite in the +# background, runs the macOS full suite in the foreground, then joins and +# prints one verdict per leg. Each leg streams to its own log so a red leg +# can be inspected without rerunning anything. +# +# test-infrastructure/ladder.sh +# +# Prerequisites: Colima running (Linux/lint legs), the Windows VM +# provisioned and reachable (vm/provision-windows.sh). A missing +# prerequisite fails that leg loudly — never silently skips it. +set -uo pipefail + +cd "$(dirname "$0")/.." +LOGS="${CBM_LADDER_LOGS:-${TMPDIR:-/tmp}/cbm-ladder-$$}" +mkdir -p "$LOGS" + +echo "=== local ladder: lint + linux + windows in background, mac in foreground ===" +echo "=== logs: $LOGS ===" + +./test-infrastructure/run.sh lint > "$LOGS/lint.log" 2>&1 & +LINT_PID=$! +./test-infrastructure/run.sh test > "$LOGS/linux.log" 2>&1 & +LINUX_PID=$! +(./test-infrastructure/vm/win.sh sync && + ./test-infrastructure/vm/win.sh test-par) > "$LOGS/windows.log" 2>&1 & +WIN_PID=$! + +scripts/test.sh 2>&1 | tee "$LOGS/mac.log" +MAC_RC=${PIPESTATUS[0]} + +wait "$LINT_PID" +LINT_RC=$? +wait "$LINUX_PID" +LINUX_RC=$? +wait "$WIN_PID" +WIN_RC=$? + +echo "" +echo "=== ladder verdicts ===" +overall=0 +report() { + local leg="$1" rc="$2" log="$3" + if [ "$rc" -eq 0 ]; then + echo " PASS $leg" + else + echo " FAIL $leg (rc=$rc) — $log" + echo " ── last 15 lines ──" + tail -15 "$log" | sed 's/^/ /' + overall=1 + fi +} +report "macOS " "$MAC_RC" "$LOGS/mac.log" +report "lint " "$LINT_RC" "$LOGS/lint.log" +report "linux " "$LINUX_RC" "$LOGS/linux.log" +report "windows " "$WIN_RC" "$LOGS/windows.log" +exit "$overall" diff --git a/test-infrastructure/vm/vm-run-tests.sh b/test-infrastructure/vm/vm-run-tests.sh index ab10b28ca..5a58fdb10 100644 --- a/test-infrastructure/vm/vm-run-tests.sh +++ b/test-infrastructure/vm/vm-run-tests.sh @@ -20,7 +20,7 @@ set -uo pipefail RUNNER="${CBM_VM_RUNNER:-build/c/test-runner}" LOG="${CBM_VM_TEST_LOG:-/tmp/win-test.log}" -[ $# -ge 1 ] || { echo "usage: vm-run-tests.sh " >&2; exit 2; } +[ $# -ge 1 ] || { echo "usage: vm-run-tests.sh | --par" >&2; exit 2; } [ -x "$RUNNER" ] || { echo "ERROR: runner '$RUNNER' missing — build first" >&2; exit 2; } # Stale roots from earlier runs are removed up front; the current root is kept @@ -70,8 +70,16 @@ MSYS2_ARG_CONV_EXCL='*' icacls "${runner_dir_w}\\*" /reset /T /C /Q >/dev/null 2 echo "=== vm-run-tests: runner=$RUNNER temp=$TEMP suites: $* ===" -"$RUNNER" "$@" 2>&1 | tee "$LOG" -rc="${PIPESTATUS[0]}" +# --par runs the repo's full parallel harness (wave + tail scheduling, +# per-shard union guard, manifest) under this same protected environment; +# explicit suite names keep the direct single-runner invocation. +if [ "$1" = "--par" ]; then + bash scripts/run-tests-parallel.sh "$RUNNER" 2>&1 | tee "$LOG" + rc="${PIPESTATUS[0]}" +else + "$RUNNER" "$@" 2>&1 | tee "$LOG" + rc="${PIPESTATUS[0]}" +fi if ! grep -Eq '[0-9]+ passed' "$LOG"; then echo "GUARD: test runner produced no completion summary — the suites did" \ diff --git a/test-infrastructure/vm/win.sh b/test-infrastructure/vm/win.sh index 592ed1bdc..5e499bea0 100755 --- a/test-infrastructure/vm/win.sh +++ b/test-infrastructure/vm/win.sh @@ -191,8 +191,11 @@ pageheap) esac ;; test-par) - # Full-suite parallel run on all VM cores via the repo's parallel harness. - vm clangarm64 "cd /c/cbm && bash scripts/run-tests-parallel.sh build/c/test-runner 2>&1 | tail -25" + # Full-suite parallel run on all VM cores via the repo's parallel harness, + # under the CI-shaped protected temp root with FULL output (this leg once + # ran under the MSYS-shared /tmp and piped through `tail -25` — the same + # truncated-blindness class that hid 40 Windows failures from `test`). + vm clangarm64 "cd /c/cbm && CBM_VM_TEST_LOG=/tmp/win-test-par.log bash test-infrastructure/vm/vm-run-tests.sh --par" ;; *) echo "unknown command: $cmd (see header for usage)" >&2; exit 2 diff --git a/tests/test_mem.c b/tests/test_mem.c index b94977c89..99469203a 100644 --- a/tests/test_mem.c +++ b/tests/test_mem.c @@ -21,6 +21,11 @@ #include #ifndef _WIN32 #include +#else +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include #endif /* ASan detection — mimalloc MI_OVERRIDE=0 under ASan, mi_process_info @@ -275,11 +280,32 @@ TEST(mem_rss_reflects_external_resident_memory) { * reporting a broken small counter instead of true resident memory — which * the Linux #else branch exercises directly against the undercount. */ const size_t threshold = (size_t)32 * 1024 * 1024; + const size_t lock_span = (size_t)64 * 1024 * 1024; void *big = malloc(region); ASSERT_NOT_NULL(big); memset(big, 0x5A, region); - memset(big, 0x5B, region); /* re-touch right before the measurement */ - size_t rss = cbm_mem_rss(); + /* Trimming can evict even a just-touched region: at 18 parallel suites + * the VM kept 19 MB resident of a 256 MB double-touch, losing the + * re-touch race this test previously relied on. Locked pages are exempt + * from working-set trimming, so lock a span comfortably above the + * threshold and the measurement becomes pressure-immune. When the lock + * is unavailable (working-set quota policy), fall back to bounded + * touch-and-sample retries — those races are transient. */ + HANDLE self_process = GetCurrentProcess(); + bool locked = SetProcessWorkingSetSize(self_process, lock_span + (size_t)32 * 1024 * 1024, + (size_t)512 * 1024 * 1024) != 0 && + VirtualLock(big, lock_span) != 0; + size_t rss = 0; + for (int attempt = 0; attempt < 6; attempt++) { + memset(big, 0x5B + attempt, lock_span); + rss = cbm_mem_rss(); + if (locked || rss >= threshold) { + break; + } + } + if (locked) { + (void)VirtualUnlock(big, lock_span); + } ASSERT_GTE(rss, threshold); free(big); #else