From 776bb119ac3f921804428c67694d39f792d78d1f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 5 Oct 2026 18:52:31 +0200 Subject: [PATCH] ci(test-impact): team artifact producer, PR shadow and the PR-CI narrowing switch Producer. test-impact-artifact.yml publishes, on every push to main, the commit's team artifact bundle: its frozen graph, built incrementally from the previous bundle, and its per-test coverage map, re-run only for the suites the push selects (scripts/ci/test-impact-publish.sh). A weekly run rebuilds both from scratch and fails when the incremental graph's topology differs from the full one. Bundles are Actions artifacts kept 30 days, never committed. Shadow. The report-only test-impact-shadow job also builds the checkout and runs `test-impact select` against the base side of the PR merge commit (scripts/ci/test-impact-select.sh). A bundle is used only when it comes from a successful main-branch producer run for exactly that commit (scripts/ci/test-impact-bundle.sh). The job stays continue-on-error and outside ci-ok. Narrowing switch, PR CI only. While the repository variable TEST_IMPACT_MODE is `gate`, a select-tests job turns the engine's answer into the tests the test-unix and test-windows legs run. Unset, nothing changes. It fails open by construction: - scripts/test-impact/selection.py is the only place an answer becomes a runner filter. Only a well-formed selected lane whose filter matches its own suite list narrows; run-all answers, a lane that selects nothing, unknown schemas or tokens, and any crash run everything. - `test` passes a selection only from a successful, narrowed select-tests run; ci-ok does not need select-tests. A broken selection costs minutes, never coverage. - scripts/run-tests-parallel.sh launches only the selected suites and keeps every guard on the selected list. A selected suite the build does not register fails the run, unless the engine saw its registration as conditional; then it is dropped with a notice, exactly as a full run of that build never runs it. Each runner judges its own tokens, so a selected test that does not exist fails its suite. - The shard manifest records the selection's digest, and verify-shard-union.sh requires every shard of a leg to agree on it. - Dry runs and releases never pass a selection (lane-wiring contract). Also fixed on the way: - publish crashed on any run-all push: a run-all lane carries "suites": null. - the git-facts hostile-diff probe re-ran the runner with the parent's CBM_TEST_ONLY_FILE still set; under a narrowed run naming its suite it re-ran the spawning test, which spawned again. The child now clears it (a new test fails without that). - the select summary crashed on the same null suites. Contracts: tests/test_test_impact_selection.sh (Step 0z8), tests/test_harness_selection.sh (Step 2c, the real runner through the real harness), and new cases in the lane-gate and lane-wiring contracts. Signed-off-by: Martin Vogel --- .github/workflows/_test.yml | 20 +++ .github/workflows/pr.yml | 66 +++++++++- .github/workflows/test-impact-artifact.yml | 85 +++++++++++++ scripts/ci/test-impact-bundle.sh | 62 ++++++++++ scripts/ci/test-impact-publish.sh | 114 ++++++++++++++++++ scripts/ci/test-impact-select.sh | 102 ++++++++++++++++ scripts/ci/verify-shard-union.sh | 16 ++- scripts/run-tests-parallel.sh | 60 ++++++++- scripts/test-impact/selection.py | 134 +++++++++++++++++++++ scripts/test.sh | 38 +++++- tests/test_harness_selection.sh | 103 ++++++++++++++++ tests/test_lane_gate_contract.sh | 11 ++ tests/test_lane_wiring_contract.sh | 37 +++++- tests/test_test_impact_git.c | 55 ++++++++- tests/test_test_impact_selection.sh | 92 ++++++++++++++ 15 files changed, 984 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/test-impact-artifact.yml create mode 100644 scripts/ci/test-impact-bundle.sh create mode 100644 scripts/ci/test-impact-publish.sh create mode 100644 scripts/ci/test-impact-select.sh create mode 100644 scripts/test-impact/selection.py create mode 100644 tests/test_harness_selection.sh create mode 100644 tests/test_test_impact_selection.sh diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index d250dd031..766db9d97 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -24,6 +24,10 @@ on: description: 'PR lane selection (scripts/ci/select-lanes.sh JSON list): only those legs and jobs run; "all" = every one (dry run, release)' type: string default: all + test_selection: + description: 'PR test selection (smart CI): the artifact the select-tests job uploaded; the test-unix and test-windows legs run only its tests. Empty = every test (dry run, release, and any PR the selection does not narrow)' + type: string + default: '' permissions: contents: read @@ -217,12 +221,20 @@ jobs: ccache-test-${{ matrix.os }}-${{ matrix.cc }}-${{ github.ref }}- ccache-test-${{ matrix.os }}-${{ matrix.cc }}- + - name: Fetch the test selection + if: ${{ inputs.test_selection != '' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ inputs.test_selection }} + path: ${{ runner.temp }}/test-selection + - name: Test run: scripts/test.sh CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} env: CBM_TEST_SHARD: ${{ matrix.shard }} CBM_TEST_LEG: ${{ matrix.os }}-${{ matrix.cc }} CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }} + CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && format('{0}/test-selection', runner.temp) || '' }} CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1500M @@ -539,6 +551,13 @@ jobs: ccache-test-${{ matrix.os }}-${{ matrix.msystem }}-${{ github.ref }}- ccache-test-${{ matrix.os }}-${{ matrix.msystem }}- + - name: Fetch the test selection + if: ${{ inputs.test_selection != '' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ inputs.test_selection }} + path: ${{ runner.temp }}/test-selection + - name: Test shell: msys2 {0} # AddressSanitizer is unavailable on native ARM64 Windows (LLVM ships no @@ -566,6 +585,7 @@ jobs: CBM_TEST_SHARD: ${{ matrix.shard }} CBM_TEST_LEG: ${{ matrix.os }}-${{ matrix.msystem }} CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }} + CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && format('{0}/test-selection', runner.temp) || '' }} CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1500M diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index da13aea51..7af2efa5e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -44,12 +44,58 @@ jobs: with: lanes: ${{ needs.changes.outputs.lanes }} + # ── Smart CI, PR CI only: run only the tests this change can reach. + # Active only while the repository variable TEST_IMPACT_MODE is `gate`; + # unset (shadow) every test runs and test-impact-shadow only reports. + # test-impact-select.sh --gate narrows only on a well-formed answer: a + # failure of this job, a run-all answer or a lane that selects nothing + # leaves `test` without a selection, and every test runs. ci-ok does not + # need this job: when it breaks, PRs cost more minutes, never coverage. + # Dry runs and releases never pass a selection. + select-tests: + needs: [changes] + if: ${{ vars.TEST_IMPACT_MODE == 'gate' }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + actions: read # the merge base's team-artifact bundle (test-impact-artifact.yml) + outputs: + selection: ${{ steps.select.outputs.selection }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The PR merge commit and its base parent: the change to select for. + fetch-depth: 2 + persist-credentials: false + - name: Build this checkout's engine + run: | + sudo apt-get update && sudo apt-get install -y zlib1g-dev + scripts/build.sh CC=gcc CXX=g++ + - name: Select the tests + id: select + env: + GH_TOKEN: ${{ github.token }} + run: scripts/ci/test-impact-select.sh --binary build/c/codebase-memory-mcp --out "$RUNNER_TEMP/test-impact" --gate "$RUNNER_TEMP/test-selection" + - name: Upload the selection + if: ${{ steps.select.outputs.selection == 'narrowed' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-selection + path: ${{ runner.temp }}/test-selection + if-no-files-found: error + overwrite: true + test: - needs: [changes, lint] + needs: [changes, lint, select-tests] if: ${{ !cancelled() && needs.lint.result == 'success' }} uses: ./.github/workflows/_test.yml with: lanes: ${{ needs.changes.outputs.lanes }} + # Smart CI: only a successful select-tests run that narrowed passes a + # selection; skipped (shadow), failed, run-all and nothing-selected + # answers all run every test. + test_selection: ${{ needs.select-tests.result == 'success' && needs.select-tests.outputs.selection == 'narrowed' && 'test-selection' || '' }} # Perf assertions are timing-sensitive on shared runners; they stay in # dry runs and releases where a flaky red does not block a merge. skip_perf: true @@ -227,8 +273,11 @@ jobs: # ci-ok's needs: it can never block or pass a merge. ── test-impact-shadow: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 45 continue-on-error: true + permissions: + contents: read + actions: read # the merge base's team-artifact bundle (test-impact-artifact.yml) steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -250,6 +299,17 @@ jobs: cp build/c/codebase-memory-mcp "$RUNNER_TEMP/cbm-bin/" - name: Predict the impacted suites run: scripts/ci/test-impact-shadow.sh --binary "$RUNNER_TEMP/cbm-bin/codebase-memory-mcp" --out "$RUNNER_TEMP/test-impact" + # The engine (test-impact select) is this checkout's own code: the + # released binary predates it, so the checkout is built either way. + - name: Build this checkout's engine + if: ${{ steps.release.outcome != 'failure' }} + run: | + sudo apt-get update && sudo apt-get install -y zlib1g-dev + scripts/build.sh CC=gcc CXX=g++ + - name: Select with the team artifact (engine shadow) + env: + GH_TOKEN: ${{ github.token }} + run: scripts/ci/test-impact-select.sh --binary build/c/codebase-memory-mcp --out "$RUNNER_TEMP/test-impact" - name: Upload the prediction if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -259,6 +319,8 @@ jobs: ${{ runner.temp }}/test-impact/prediction.json ${{ runner.temp }}/test-impact/selection.json ${{ runner.temp }}/test-impact/changed-files.txt + ${{ runner.temp }}/test-impact/selection-engine.json + ${{ runner.temp }}/test-impact/summary-engine.md if-no-files-found: warn # ── Docs-only and non-product PRs (tiers T0a, T0b) run the static contract diff --git a/.github/workflows/test-impact-artifact.yml b/.github/workflows/test-impact-artifact.yml new file mode 100644 index 000000000..9a13d9ad0 --- /dev/null +++ b/.github/workflows/test-impact-artifact.yml @@ -0,0 +1,85 @@ +# Team artifact for test selection (smart CI, report-only until the oracles +# and the shadow comparison are green). Every push to main publishes the +# commit's bundle — its frozen graph, built incrementally from the previous +# bundle, and its per-test coverage map, re-run only for the suites the push +# selects — as the Actions artifact test-impact-. A weekly run rebuilds +# both from scratch and fails when the incremental graph differs from the full +# one. PR selection fetches the bundle of its merge base +# (scripts/ci/test-impact-bundle.sh) and never narrows on anything else. +name: test-impact-artifact + +on: + push: + branches: [main] + schedule: + - cron: "23 3 * * 0" # weekly full refresh, Sunday 03:23 UTC + workflow_dispatch: + inputs: + mode: + description: incremental or full + type: choice + options: [incremental, full] + default: full + +permissions: + contents: read + actions: read # fetch the previous bundle + +concurrency: + # Bundles chain: each push builds on the previous one, so runs never overlap + # and are never cancelled half-way. + group: test-impact-artifact + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + + - name: Install build deps and LLVM 21 (coverage toolchain) + run: | + sudo apt-get update && sudo apt-get install -y zlib1g-dev + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-21 main" | sudo tee /etc/apt/sources.list.d/llvm-21.list + sudo apt-get update + sudo apt-get install -y clang-21 llvm-21 libclang-rt-21-dev + + - name: Build the production binary and the coverage runner + run: | + scripts/build.sh CC=gcc CXX=g++ + make -f Makefile.cbm test-runner-cov COV_LLVM_BIN=/usr/lib/llvm-21/bin -j"$(nproc)" + + - name: Publish the bundle + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + BEFORE: ${{ github.event.before }} + REQUESTED: ${{ inputs.mode }} + # The selection lane excludes @perf and PR CI skips perf, so their + # (slowest, longest under coverage) suites are neither mapped nor + # listed: the runner then neither lists nor runs them. + CBM_SKIP_PERF: "1" + run: | + mode=incremental + if [ "$EVENT" = schedule ] || [ "$REQUESTED" = full ]; then mode=full; fi + before="$(git rev-parse HEAD^1)" + if [ "$EVENT" = push ] && [ -n "$BEFORE" ] && [ "$BEFORE" != 0000000000000000000000000000000000000000 ]; then + before="$BEFORE" + fi + scripts/ci/test-impact-publish.sh --binary build/c/codebase-memory-mcp \ + --runner build/cov/test-runner --llvm-bin /usr/lib/llvm-21/bin \ + --out "$RUNNER_TEMP/bundle" --mode "$mode" --before "$before" + + - name: Upload the bundle + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-impact-${{ github.sha }} + path: ${{ runner.temp }}/bundle + include-hidden-files: true # the graph lives in .codebase-memory/ + retention-days: 30 + if-no-files-found: error diff --git a/scripts/ci/test-impact-bundle.sh b/scripts/ci/test-impact-bundle.sh new file mode 100644 index 000000000..bc83e01a0 --- /dev/null +++ b/scripts/ci/test-impact-bundle.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# test-impact-bundle.sh — fetch the team artifact bundle of one main commit. +# +# The bundle (codebase-memory-mcp test-impact publish; src/mcp/test_impact_artifact.h) +# is published by .github/workflows/test-impact-artifact.yml as the Actions +# artifact `test-impact-`. A selection may pass --artifact-verified ONLY +# for a bundle this script fetched: it accepts nothing but a SUCCESSFUL run of +# that workflow, in THIS repository, on the main branch, whose head is exactly +# the requested commit — the provenance the receipt itself can never assert. +# +# Usage: scripts/ci/test-impact-bundle.sh --fetch SHA DIR +# Download the bundle of SHA into DIR (created). Needs GH_TOKEN with +# actions:read; $GH overrides the gh command, $GITHUB_REPOSITORY the +# repository (OWNER/NAME). +# Exit: 0 = fetched and verified · 3 = no admissible bundle for SHA · +# 1 = the tooling broke · 2 = usage error. +set -euo pipefail + +GH="${GH:-gh}" +REPO="${GITHUB_REPOSITORY:-DeusData/codebase-memory-mcp}" +WORKFLOW="test-impact-artifact.yml" + +usage() { sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//'; } + +[ $# -eq 3 ] && [ "$1" = "--fetch" ] || { usage >&2; exit 2; } +SHA="$2" +DIR="$3" +case "$SHA" in + *[!0-9a-f]* | "") echo "test-impact-bundle: not a lowercase commit id: $SHA" >&2; exit 2 ;; +esac +[ ${#SHA} -eq 40 ] || [ ${#SHA} -eq 64 ] || { echo "test-impact-bundle: bad id length" >&2; exit 2; } + +# The newest successful main-branch run of the producer for exactly this head. +run=$("$GH" api -X GET "repos/$REPO/actions/workflows/$WORKFLOW/runs" \ + -f branch=main -f status=success -f head_sha="$SHA" -f per_page=20 \ + --jq '[.workflow_runs[] + | select(.head_sha == "'"$SHA"'" and .head_branch == "main" + and .repository.full_name == "'"$REPO"'" + and .head_repository.full_name == "'"$REPO"'" + and (.event == "push" or .event == "schedule" + or .event == "workflow_dispatch"))] + | sort_by(.run_started_at) | last | .id // empty') || exit 1 +if [ -z "$run" ]; then + echo "test-impact-bundle: no successful producer run for $SHA" >&2 + exit 3 +fi +mkdir -p "$DIR" +if ! "$GH" run download "$run" --repo "$REPO" --name "test-impact-$SHA" --dir "$DIR"; then + echo "test-impact-bundle: run $run has no artifact test-impact-$SHA" >&2 + exit 3 +fi +# The receipt must name the commit the run was for; the engine checks the rest. +if ! python3 - "$DIR/receipt.json" "$SHA" <<'PY' +import json, sys +receipt = json.load(open(sys.argv[1])) +sys.exit(0 if receipt.get("commit") == sys.argv[2] else 1) +PY +then + echo "test-impact-bundle: the receipt does not name $SHA" >&2 + exit 3 +fi +echo "test-impact-bundle: run $run, bundle of $SHA in $DIR" diff --git a/scripts/ci/test-impact-publish.sh b/scripts/ci/test-impact-publish.sh new file mode 100644 index 000000000..8e9dbeeb6 --- /dev/null +++ b/scripts/ci/test-impact-publish.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# test-impact-publish.sh — publish the team artifact bundle of the checked-out +# main commit (.github/workflows/test-impact-artifact.yml). +# +# Incremental (a push): the previous commit's bundle is the graph base and its +# coverage map the starting point; only the suites the push's own selection +# picks are re-run with coverage and merged in (scripts/test-impact/ +# coverage-merge.py). The oldest observation is carried forward, so an +# incremental map ages out unless a full refresh runs. +# Full (weekly, manual, or no usable previous bundle): every suite is re-run +# and the graph is built from scratch. A weekly run ALSO builds the graph +# incrementally from the previous bundle and fails when the two graphs' +# content digests differ: an incremental index must equal a full one. +# +# Usage: scripts/ci/test-impact-publish.sh --binary PATH --runner PATH --out DIR +# --mode incremental|full [--before SHA] [--llvm-bin DIR] [--platform LABEL] +# Exit: 0 = published · 1 = failed · 2 = usage error. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +BINARY="" RUNNER="" OUT="" MODE="" BEFORE="" LLVM_BIN="" PLATFORM="linux-x86_64-clang21" +while [ $# -gt 0 ]; do + case "$1" in + --binary) BINARY="$2"; shift 2 ;; + --runner) RUNNER="$2"; shift 2 ;; + --out) OUT="$2"; shift 2 ;; + --mode) MODE="$2"; shift 2 ;; + --before) BEFORE="$2"; shift 2 ;; + --llvm-bin) LLVM_BIN="$2"; shift 2 ;; + --platform) PLATFORM="$2"; shift 2 ;; + *) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2 ;; + esac +done +[ -n "$BINARY" ] && [ -n "$RUNNER" ] && [ -n "$OUT" ] || exit 2 +case "$MODE" in incremental | full) ;; *) exit 2 ;; esac + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +# A private daemon rendezvous: the socket path must fit sun_path. +export CBM_RUNTIME_DIR="$(mktemp -d /tmp/cbm-rt.XXXXXX)" +export CBM_CACHE_DIR="$WORK/cache" +mkdir -p "$WORK/engine" && chmod 700 "$WORK/engine" +HEAD_SHA="$(git -C "$ROOT" rev-parse HEAD)" +PREVIOUS="" + +if [ -n "$BEFORE" ]; then + if scripts/ci/test-impact-bundle.sh --fetch "$BEFORE" "$WORK/previous"; then + PREVIOUS="$WORK/previous" + fi +fi +if [ "$MODE" = incremental ] && [ -z "$PREVIOUS" ]; then + echo "test-impact-publish: no previous bundle for ${BEFORE:-?}: full refresh" + MODE=full +fi + +map() { # out [suite...] + local out="$1"; shift + local args=() + for suite in "$@"; do args+=(--suite "$suite"); done + python3 "$ROOT/scripts/test-impact/coverage-map.py" --runner "$RUNNER" --out "$out" \ + ${LLVM_BIN:+--llvm-bin "$LLVM_BIN"} ${args[@]+"${args[@]}"} +} + +OBSERVED="" +if [ "$MODE" = full ]; then + OBSERVED="$(date +%s)" + map "$WORK/coverage" +else + # The push's own selection decides what to re-run (whole suites). + "$BINARY" test-impact select --repo "$ROOT" --base "$BEFORE" --work "$WORK/engine" \ + --artifact "$PREVIOUS" --artifact-commit "$BEFORE" --artifact-verified \ + --platform "$PLATFORM" > "$WORK/selection.json" + python3 "$ROOT/scripts/test-impact/selection.py" suites "$WORK/selection.json" > "$WORK/suites.txt" + "$RUNNER" --list-suites > "$WORK/listed.txt" + if grep -qx '\*' "$WORK/suites.txt"; then + OBSERVED="$(date +%s)" + map "$WORK/coverage" + else + mapfile -t rerun < <(sort -u "$WORK/suites.txt") + map "$WORK/fresh" ${rerun[@]+"${rerun[@]}"} + python3 "$ROOT/scripts/test-impact/coverage-merge.py" --previous "$PREVIOUS/coverage" \ + --fresh "$WORK/fresh" --suites "$WORK/listed.txt" --commit "$HEAD_SHA" \ + --out "$WORK/coverage" + fi +fi + +publish() { # out [previous] + "$BINARY" test-impact publish --repo "$ROOT" --out "$1" --work "$WORK/engine" \ + --coverage "$WORK/coverage" ${OBSERVED:+--observed-at "$OBSERVED"} \ + --platform "$PLATFORM" ${2:+--previous "$2"} +} +if [ "$MODE" = full ]; then + publish "$OUT" + if [ -n "$PREVIOUS" ]; then + publish "$WORK/incremental" "$PREVIOUS" + field() { python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$1" "$2"; } + # Topology (what selection reads) must be equal; edge properties and the + # corpus-statistical edges are reported, not gated (recorded defects). + full_topology="$(field "$OUT/receipt.json" graph_topology_sha256)" + incr_topology="$(field "$WORK/incremental/receipt.json" graph_topology_sha256)" + if [ "$full_topology" != "$incr_topology" ]; then + echo "test-impact-publish: incremental topology differs from the full one ($incr_topology != $full_topology)" >&2 + exit 1 + fi + if [ "$(field "$OUT/receipt.json" graph_content_sha256)" != \ + "$(field "$WORK/incremental/receipt.json" graph_content_sha256)" ]; then + echo "::warning::incremental graph content (edge properties, similarity edges) differs from the full one; topology is equal" + fi + echo "test-impact-publish: incremental topology equals the full one ($full_topology)" + fi +else + publish "$OUT" "$PREVIOUS" +fi +echo "test-impact-publish: bundle of $HEAD_SHA in $OUT ($MODE)" diff --git a/scripts/ci/test-impact-select.sh b/scripts/ci/test-impact-select.sh new file mode 100644 index 000000000..39208683e --- /dev/null +++ b/scripts/ci/test-impact-select.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# test-impact-select.sh — the PR selection with the team artifact: what +# `codebase-memory-mcp test-impact select` runs for this change. +# +# Shadow (default): REPORT-ONLY; pr.yml's test-impact-shadow job runs it +# continue-on-error and ci-ok does not need it. +# --gate DIR (pr.yml's select-tests job, only when the repository variable +# TEST_IMPACT_MODE is `gate`): also turns the answer into what the test legs +# run (scripts/test-impact/selection.py) — DIR/test-only.txt and +# DIR/optional-suites.txt for a narrowed selection — and writes +# `selection=narrowed` or `selection=all` to $GITHUB_OUTPUT. Anything but a +# well-formed narrowed answer, a lane that selects nothing included, is `all`. +# +# The base is the base side of the PR merge commit (HEAD^1 by default). Its +# bundle is fetched with scripts/ci/test-impact-bundle.sh, which only accepts +# the main-branch producer's own run for exactly that commit; only then does +# the selection pass --artifact-verified. Without a bundle the engine builds +# the graph itself and admits no coverage: every reached suite runs whole. +# +# Usage: scripts/ci/test-impact-select.sh --binary PATH --out DIR [--base REV] +# [--platform LABEL] [--gate DIR] +# Writes DIR/selection-engine.json and DIR/summary-engine.md (appended +# to $GITHUB_STEP_SUMMARY when set). +# Exit: 0 = selection written · 1 = the tooling broke · 2 = usage error. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +BINARY="" OUT="" BASE="HEAD^1" PLATFORM="linux-x86_64-clang21" GATE="" +while [ $# -gt 0 ]; do + case "$1" in + --binary) BINARY="$2"; shift 2 ;; + --out) OUT="$2"; shift 2 ;; + --base) BASE="$2"; shift 2 ;; + --platform) PLATFORM="$2"; shift 2 ;; + --gate) GATE="$2"; shift 2 ;; + *) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2 ;; + esac +done +[ -n "$BINARY" ] && [ -n "$OUT" ] || exit 2 + +mkdir -p "$OUT" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +export CBM_RUNTIME_DIR="$(mktemp -d /tmp/cbm-rt.XXXXXX)" # sun_path must fit +export CBM_CACHE_DIR="$WORK/cache" +mkdir -p "$WORK/engine" && chmod 700 "$WORK/engine" +base="$(git -C "$ROOT" rev-parse "$BASE")" + +artifact=() +bundle_rc=0 +"$ROOT/scripts/ci/test-impact-bundle.sh" --fetch "$base" "$WORK/bundle" || bundle_rc=$? +if [ "$bundle_rc" -eq 0 ]; then + artifact=(--artifact "$WORK/bundle" --artifact-commit "$base" --artifact-verified) +elif [ "$bundle_rc" -ne 3 ]; then + # No bundle means no coverage: more tests run, never fewer. Say so. + echo "::warning::test-impact bundle fetch failed (exit $bundle_rc): selecting without coverage" +fi +started=$(date +%s) +"$BINARY" test-impact select --repo "$ROOT" --base "$base" --work "$WORK/engine" \ + --platform "$PLATFORM" ${artifact[@]+"${artifact[@]}"} > "$OUT/selection-engine.json" +elapsed=$(( $(date +%s) - started )) + +python3 - "$OUT/selection-engine.json" "$elapsed" "${#artifact[@]}" > "$OUT/summary-engine.md" <<'PY' +import json, sys +answer = json.load(open(sys.argv[1])) +coverage = answer.get("receipt", {}).get("coverage", {}) +lines = ["### test-impact engine", "", + f"- decision: `{answer.get('decision')}` in {sys.argv[2]} s", + f"- team artifact: {'fetched' if sys.argv[3] != '0' else 'none'}; coverage " + f"`{coverage.get('state')}` {coverage.get('rejection_reasons') or ''}"] +for lane in answer.get("lanes") or []: + suites = lane.get("suites") or [] + whole = [s["suite"] for s in suites if s.get("mode") == "whole"] + narrowed = {s["suite"]: len(s.get("tests") or []) for s in suites if s.get("mode") == "tests"} + lines.append(f"- lane `{lane.get('lane')}` ({lane.get('decision')}): {len(whole)} whole, " + f"{len(narrowed)} narrowed ({sum(narrowed.values())} tests)") +print("\n".join(lines)) +PY + +if [ -n "$GATE" ]; then + mkdir -p "$GATE" + selection=all + filter_rc=0 + python3 "$ROOT/scripts/test-impact/selection.py" filter "$OUT/selection-engine.json" \ + --lane unit --out "$GATE/test-only.txt" --optional-out "$GATE/optional-suites.txt" || + filter_rc=$? + if [ "$filter_rc" -eq 0 ]; then + selection=narrowed + echo "- PR CI runs $(wc -l < "$GATE/test-only.txt" | tr -d ' ') selection tokens" \ + >> "$OUT/summary-engine.md" + else + rm -f "$GATE/test-only.txt" "$GATE/optional-suites.txt" + echo "- PR CI runs every test (selection exit $filter_rc)" >> "$OUT/summary-engine.md" + fi + if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "selection=$selection" >> "$GITHUB_OUTPUT" + fi +fi +cat "$OUT/summary-engine.md" +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "$OUT/summary-engine.md" >> "$GITHUB_STEP_SUMMARY" +fi diff --git a/scripts/ci/verify-shard-union.sh b/scripts/ci/verify-shard-union.sh index b2c69d0ff..433d164bb 100755 --- a/scripts/ci/verify-shard-union.sh +++ b/scripts/ci/verify-shard-union.sh @@ -8,7 +8,9 @@ # For every leg it asserts: all shards agree on the shard count, indices form # exactly 1..n, every shard saw the same full suite list, and the UNION of the # shard slices equals that list — a rename/re-shard can never silently drop a -# suite (gate-quality loss) without failing here. +# suite (gate-quality loss) without failing here. A narrowed leg (PR CI test +# selection) records the selected list and its selection_sha256; all shards +# must agree on that too. # # No manifests at all is a failure — unless the PR's lane selection (--lanes, # the JSON list from scripts/ci/select-lanes.sh) holds no test leg, in which @@ -20,7 +22,7 @@ set -eu case "${1:-}" in -h | --help) - sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//' + sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//' exit 0 ;; esac @@ -78,6 +80,16 @@ for leg in $(grep -h '^leg=' $files | sort -u | sed 's/^leg=//'); do rc=1 continue fi + # A narrowed leg: every shard must have applied the same test selection + # (two selections can name the same suites and different tests). + sel_sha=$(for f in $leg_files; do + grep -h '^selection_sha256=' "$f" || echo "selection_sha256=none" + done | sort -u) + if [ "$(printf '%s\n' "$sel_sha" | wc -l)" -ne 1 ]; then + echo "FAIL: $leg shards applied different test selections" >&2 + rc=1 + continue + fi union_sha=$(for f in $leg_files; do sed -n '/^--- slice ---$/,$p' "$f" | tail -n +2 done | sort | sha256sum | awk '{print $1}') diff --git a/scripts/run-tests-parallel.sh b/scripts/run-tests-parallel.sh index d28829540..2607b5d7d 100644 --- a/scripts/run-tests-parallel.sh +++ b/scripts/run-tests-parallel.sh @@ -35,7 +35,10 @@ if [ -z "$JOBS" ]; then fi fi -LOGDIR="$(dirname "$RUNNER")/test-logs" +# CBM_TEST_LOG_DIR: a private log directory (tests/test_harness_selection.sh +# drives this harness before the real run and must not touch its logs). Keep +# it relative: the native Windows runner reads the selection file below it. +LOGDIR="${CBM_TEST_LOG_DIR:-$(dirname "$RUNNER")/test-logs}" rm -rf "$LOGDIR" mkdir -p "$LOGDIR" @@ -129,6 +132,54 @@ if [ "$NSUITES" -lt 1 ] || grep -qvE '^[a-z0-9_]+$' "$SUITES_FILE"; then exit 1 fi +# Test selection (smart CI; PR CI only, scripts/test-impact/selection.py). +# CBM_TEST_SELECTION_FILE lists `suite` / `suite:test` tokens. Only the suites +# it names are launched, and every launched runner judges its own tokens +# through CBM_TEST_ONLY_FILE, so a selected test the runner cannot find fails +# that suite. A selected suite this build does not register fails the run, +# unless CBM_TEST_SELECTION_OPTIONAL names it (registered only in some builds, +# e.g. the coverage runner): then it is dropped with a notice, exactly as a +# full run of this build never runs it. Everything below — sharding, the +# manifest, the union guard — then works on the selected list. Unset: the +# full list, unchanged. +SELECTION="none" +if [ -n "${CBM_TEST_SELECTION_FILE:-}" ]; then + SEL_TOKENS="$LOGDIR/selection-tokens.txt" + if ! tr -d '\r' < "$CBM_TEST_SELECTION_FILE" > "$SEL_TOKENS"; then + echo "FAIL: cannot read the test selection $CBM_TEST_SELECTION_FILE" >&2 + exit 1 + fi + if [ ! -s "$SEL_TOKENS" ] || grep -qvE '^[A-Za-z0-9_]+(:[A-Za-z0-9_]+)?$' "$SEL_TOKENS"; then + echo "FAIL: test selection empty or malformed: $CBM_TEST_SELECTION_FILE" >&2 + exit 1 + fi + cut -d: -f1 "$SEL_TOKENS" | sort -u > "$LOGDIR/selection-suites.txt" + if [ -n "${CBM_TEST_SELECTION_OPTIONAL:-}" ]; then + tr -d '\r' < "$CBM_TEST_SELECTION_OPTIONAL" | sort -u > "$LOGDIR/selection-optional.txt" + else + : > "$LOGDIR/selection-optional.txt" + fi + ABSENT=$(comm -23 "$LOGDIR/selection-suites.txt" <(sort -u "$SUITES_FILE")) + REQUIRED_ABSENT=$(comm -23 <(printf '%s\n' $ABSENT | sed '/^$/d' | sort -u) "$LOGDIR/selection-optional.txt") + if [ -n "$REQUIRED_ABSENT" ]; then + echo "FAIL: the test selection names suites this build does not register:" $REQUIRED_ABSENT >&2 + exit 1 + fi + [ -z "$ABSENT" ] || echo "selection: conditional suites not in this build, not run:" $ABSENT + # The runner's filter keeps only this build's suites (an unknown suite's + # token would fail every process); the suite list keeps list order. + awk 'NR == FNR { listed[$0] = 1; next } { split($0, p, ":") } p[1] in listed' \ + "$SUITES_FILE" "$SEL_TOKENS" > "$LOGDIR/selection.txt" + awk 'NR == FNR { selected[$0] = 1; next } $0 in selected' \ + "$LOGDIR/selection-suites.txt" "$SUITES_FILE" > "$SUITES_FILE.selected" + mv "$SUITES_FILE.selected" "$SUITES_FILE" + SELECTION="$(sort "$LOGDIR/selection.txt" | { sha256sum 2>/dev/null || shasum -a 256; } | awk '{print $1}')" + echo "selection: $(wc -l < "$SUITES_FILE" | tr -d ' ') of $NSUITES suites, $(wc -l < "$LOGDIR/selection.txt" | tr -d ' ') tokens ($SELECTION)" + NSUITES=$(wc -l < "$SUITES_FILE" | tr -d ' ') + export CBM_TEST_ONLY_FILE="$LOGDIR/selection.txt" + unset CBM_TEST_ONLY +fi + # CBM_TEST_SHARD="i/N" runs this invocation's deterministic slice of the # suite list so CI can spread one platform's suites across N runner jobs. # Unset (or "1/1") selects everything — the sharding path is inert unless a @@ -246,6 +297,7 @@ NSHARD=$(wc -l < "$SHARD_EXPECT" | tr -d ' ') echo "leg=${CBM_TEST_LEG:-local}" echo "shard=${SHARD_INDEX}/${SHARD_TOTAL}" echo "list_sha256=$(sort "$SUITES_FILE" | { sha256sum 2>/dev/null || shasum -a 256; } | awk '{print $1}')" + echo "selection_sha256=$SELECTION" echo "--- slice ---" cat "$SHARD_EXPECT" } > "$LOGDIR/shard-manifest.txt" @@ -344,7 +396,11 @@ for f in $(grep -v ' rc=0 ' "$RESULTS_FILE" | awk '{print $1}'); do done echo "────────────────────────────────────────────" -echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES suites, $JOBS jobs)" +if [ "$SELECTION" = none ]; then + echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES suites, $JOBS jobs)" +else + echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES selected suites, $JOBS jobs; selection $SELECTION)" +fi echo "────────────────────────────────────────────" if [ "$TOTAL_FAIL" -gt 0 ] || [ "$BAD_RC" -gt 0 ]; then diff --git a/scripts/test-impact/selection.py b/scripts/test-impact/selection.py new file mode 100644 index 000000000..f4a8ff08b --- /dev/null +++ b/scripts/test-impact/selection.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Turn a test-impact answer (`codebase-memory-mcp test-impact select`) into +what a CI consumer runs. + + selection.py suites ANSWER + One line per suite whose tests the answer selects, or a single `*` when a + lane runs everything. The team-artifact producer re-runs exactly these + suites with coverage (scripts/ci/test-impact-publish.sh). + + selection.py filter ANSWER --lane LANE --out FILE [--optional-out FILE2] + exit 0 narrowed: FILE holds the lane's CBM_TEST_ONLY_FILE lines, and + FILE2 the selected suites the engine saw registered only + conditionally (CONDITIONAL): a build may legitimately lack them + (scripts/run-tests-parallel.sh, CBM_TEST_SELECTION_OPTIONAL) + exit 11 the lane selects nothing: FILE is empty + anything else: run everything (FILE and FILE2 are removed) + +It fails safe by construction: a run-all answer, a lane it cannot find, an +unknown schema or filter encoding, a token that is not `suite` or +`suite:test`, and a filter that disagrees with the lane's own suite list all +mean "run everything", and so does any crash (exit 1). +""" +import argparse +import json +import os +import re +import sys + +SCHEMAS = {"cbm.test_impact.v0", "cbm.test_impact.v1"} +TOKEN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(:[A-Za-z_][A-Za-z0-9_]*)?$") +NARROWED, NOTHING, RUN_ALL = 0, 11, 10 + + +def suites(answer): + if answer.get("decision") == "run_all": + return ["*"] + out = [] + for lane in answer.get("lanes") or []: + if lane.get("decision") == "run_all" or lane.get("suites") is None: + return ["*"] + out += [suite["suite"] for suite in lane["suites"]] + return sorted(set(out)) + + +def run_all(why): + print(f"selection: run everything: {why}", file=sys.stderr) + return RUN_ALL + + +def lane_filter(answer, name): + """-> (exit code, lines)""" + if answer.get("schema") not in SCHEMAS: + return run_all(f"unknown schema {answer.get('schema')!r}"), [] + if answer.get("decision") == "run_all": + return run_all("the answer runs everything"), [] + lanes = [lane for lane in answer.get("lanes") or [] if lane.get("lane") == name] + if len(lanes) != 1: + return run_all(f"lane {name!r} appears {len(lanes)} times"), [] + lane = lanes[0] + decision = lane.get("decision") + if decision == "skip": + if lane.get("suites") or lane.get("runner_filter") is not None: + return run_all("a skipped lane that still lists suites or a filter"), [] + return NOTHING, [] + if decision != "selected" or not isinstance(lane.get("suites"), list): + return run_all(f"lane decision {decision!r}"), [] + flt = lane.get("runner_filter") + if not isinstance(flt, dict) or flt.get("encoding") != "cbm-test-only-lines" or \ + not isinstance(flt.get("value"), str): + return run_all("no cbm-test-only-lines filter"), [] + lines = [line for line in flt["value"].split("\n") if line] + bad = [line for line in lines if not TOKEN.match(line)] + if not lines or bad: + return run_all(f"filter tokens unusable ({bad[:3] or 'empty'})"), [] + expected = set() + for suite in lane["suites"]: + if suite.get("mode") == "whole": + expected.add(suite["suite"]) + elif suite.get("mode") == "tests" and suite.get("tests"): + expected |= {f"{suite['suite']}:{test['test']}" for test in suite["tests"]} + else: + return run_all(f"suite {suite.get('suite')!r} has mode {suite.get('mode')!r}"), [] + if set(lines) != expected: + return run_all("the filter disagrees with the lane's suite list"), [] + return NARROWED, lines + + +def conditional_suites(answer, name): + """The lane's selected suites whose registration the engine saw as conditional.""" + for lane in answer.get("lanes") or []: + if lane.get("lane") == name: + return sorted({suite["suite"] for suite in lane.get("suites") or [] + if "CONDITIONAL" in (suite.get("whole_reason") or [])}) + return [] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + s = sub.add_parser("suites") + s.add_argument("answer") + f = sub.add_parser("filter") + f.add_argument("answer") + f.add_argument("--lane", required=True) + f.add_argument("--out", required=True) + f.add_argument("--optional-out") + args = parser.parse_args() + if args.command == "suites": + with open(args.answer, encoding="utf-8") as handle: + print("\n".join(suites(json.load(handle)))) + return 0 + for stale in (args.out, args.optional_out): + if stale and os.path.exists(stale): + os.remove(stale) + try: + with open(args.answer, encoding="utf-8") as handle: + answer = json.load(handle) + except (OSError, ValueError) as error: + return run_all(f"unreadable answer ({error})") + if not isinstance(answer, dict): + return run_all("the answer is not an object") + code, lines = lane_filter(answer, args.lane) + if code in (NARROWED, NOTHING): + with open(args.out, "w", encoding="utf-8") as handle: + handle.write("".join(line + "\n" for line in lines)) + if args.optional_out: + optional = conditional_suites(answer, args.lane) if code == NARROWED else [] + with open(args.optional_out, "w", encoding="utf-8") as handle: + handle.write("".join(suite + "\n" for suite in optional)) + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test.sh b/scripts/test.sh index 91594e581..5774b659f 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -54,6 +54,11 @@ Environment: CBM_RUN_HANG_TEST=1 Opt-in C++ index-hang regression (#410, needs prod). CBM_NO_CCACHE=1 Disable the content-verified compiler cache. CBM_TEST_SHARD/_LEG Set by CI's sharded legs; leave unset locally. + CBM_TEST_SELECTION_DIR PR CI's test selection (smart CI): a directory with + test-only.txt (suite / suite:test lines) and + optional-suites.txt. Narrows the parallel suite step + only; contract steps and CBM_TEST_SEQUENTIAL=1 runs + keep running everything. Examples: scripts/test.sh # the full venue leg (what CI runs) @@ -375,6 +380,10 @@ bash "$ROOT/tests/test_coverage_map.sh" # selection a map that claims tests ran functions they never touched. echo "=== Step 0z7: incremental coverage map merge contract ===" bash "$ROOT/tests/test_coverage_merge.sh" +# Step 0z8: the one place a test-impact answer becomes what PR CI runs. Only a +# well-formed, self-consistent selection may narrow; everything else runs all. +echo "=== Step 0z8: test selection (answer -> runner filter) contract ===" +bash "$ROOT/tests/test_test_impact_selection.sh" if [ "$CONTRACTS_ONLY" -eq 1 ]; then echo "=== test.sh: contracts-only — every contract step passed ===" @@ -402,11 +411,38 @@ assert_test_runner_build_config "$BUILD_DIR/test-runner" "$EXPECTED_SANITIZED" # tests is a property of the harness every later result depends on. echo "=== Step 2b: per-test selection regression (CBM_TEST_ONLY) ===" CBM_TEST_RUNNER="$ROOT/$BUILD_DIR/test-runner" bash "$ROOT/tests/test_harness_test_only.sh" +# Step 2c: PR CI's test selection (smart CI) through the parallel harness — the +# selected suites only, unknown suites and tests fail, conditional ones drop. +echo "=== Step 2c: parallel-harness test selection regression ===" +CBM_TEST_RUNNER="$ROOT/$BUILD_DIR/test-runner" bash "$ROOT/tests/test_harness_selection.sh" + +# PR CI's test selection (smart CI, the select-tests job) narrows the parallel +# suite step and nothing else (scripts/run-tests-parallel.sh). A selection that +# was promised but is missing is a plumbing fault: fail, never run silently. +SELECTION_ENV=() +if [ -n "${CBM_TEST_SELECTION_DIR:-}" ]; then + selection_dir="$CBM_TEST_SELECTION_DIR" + if command -v cygpath > /dev/null 2>&1; then + selection_dir="$(cygpath -u "$selection_dir")" + fi + if [ ! -s "$selection_dir/test-only.txt" ]; then + echo "FAIL: CBM_TEST_SELECTION_DIR=$CBM_TEST_SELECTION_DIR holds no test-only.txt" >&2 + exit 1 + fi + SELECTION_ENV=(CBM_TEST_SELECTION_FILE="$selection_dir/test-only.txt") + if [ -f "$selection_dir/optional-suites.txt" ]; then + SELECTION_ENV+=(CBM_TEST_SELECTION_OPTIONAL="$selection_dir/optional-suites.txt") + fi + if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; then + echo "test.sh: CBM_TEST_SEQUENTIAL=1 runs every test; the selection narrows the parallel harness only" + fi +fi if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; then make -f Makefile.cbm test ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"} else - make -f Makefile.cbm test-par ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"} + env ${SELECTION_ENV[@]+"${SELECTION_ENV[@]}"} \ + make -f Makefile.cbm test-par ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"} fi # Step 4: C++ large-TU index-hang regression guard (#410). Runs the PROD binary diff --git a/tests/test_harness_selection.sh b/tests/test_harness_selection.sh new file mode 100644 index 000000000..bc8a7f9b7 --- /dev/null +++ b/tests/test_harness_selection.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# PR CI's test selection in the parallel harness (scripts/run-tests-parallel.sh, +# CBM_TEST_SELECTION_FILE / CBM_TEST_SELECTION_OPTIONAL), driven over the +# runner just built — the same binary and scheduler every leg runs: +# 1. only the selected suites are launched, each runner applies the tokens +# (a narrowed suite deselects its other tests), and the shard manifest +# records the selected list and the selection's digest; +# 2. a selected suite this build does not register FAILS the run, unless the +# selection marks it conditional, when it is dropped with a notice; +# 3. a malformed selection and a selected test the build does not have FAIL. +# A selection that quietly ran less than it named would report green for work +# that never ran. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RUNNER="${CBM_TEST_RUNNER:-${ROOT}/build/c/test-runner}" +if [[ ! -x "${RUNNER}" && -x "${RUNNER}.exe" ]]; then + RUNNER="${RUNNER}.exe" +fi +if [[ ! -x "${RUNNER}" ]]; then + echo "missing test runner: ${RUNNER}" >&2 + exit 2 +fi +# Relative paths, as the real run uses them: the native Windows runner reads +# the selection file the harness writes under its log directory. +cd "${ROOT}" +RUNNER_REL="${RUNNER#"${ROOT}"/}" +LOGS="$(dirname "${RUNNER_REL}")/test-logs-selection" + +# Anything inherited from the caller would decide the cases below. +unset CBM_TEST_ONLY CBM_TEST_ONLY_FILE CBM_TEST_SELECTION_FILE CBM_TEST_SELECTION_OPTIONAL +unset CBM_TEST_SHARD CBM_TEST_LEG + +NAMED_SUITE=dyn_array +OTHER_SUITE=str_intern +OTHER_TEST="$(awk -F'[()]' '/^[[:space:]]*RUN_TEST\(/ { print $2; exit }' "tests/test_${OTHER_SUITE}.c")" +OTHER_TOTAL="$(awk '/^[[:space:]]*RUN_TEST\(/ { n++ } END { print n + 0 }' "tests/test_${OTHER_SUITE}.c")" +if [[ -z "${OTHER_TEST}" || "${OTHER_TOTAL}" -lt 2 ]]; then + echo "fixture suite changed shape: ${OTHER_SUITE}=${OTHER_TOTAL}" >&2 + exit 2 +fi + +tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/cbm-selection.XXXXXX")" +trap 'rm -rf "${tmpdir}" "${LOGS}"' EXIT +FAILURES=0 +fail() { + echo "FAIL: $*" >&2 + FAILURES=$((FAILURES + 1)) +} + +# harness [optional suites] -> RC, OUT +harness() { + printf '%b' "$2" > "${tmpdir}/$1.sel" + local optional=() + if [[ -n "${3:-}" ]]; then + printf '%b' "$3" > "${tmpdir}/$1.opt" + optional=(CBM_TEST_SELECTION_OPTIONAL="${tmpdir}/$1.opt") + fi + RC=0 + OUT="$(env CBM_TEST_LOG_DIR="${LOGS}" CBM_TEST_SELECTION_FILE="${tmpdir}/$1.sel" \ + ${optional[@]+"${optional[@]}"} \ + bash scripts/run-tests-parallel.sh "${RUNNER_REL}" 2 2>&1 | tr -d '\r')" || RC=$? +} +ran_suites() { + awk '{ print $1 }' "${LOGS}/results.txt" | sort | tr '\n' ' ' +} + +# 1. Narrowed: one whole suite, one suite by a single test. +harness narrowed "${NAMED_SUITE}\n${OTHER_SUITE}:${OTHER_TEST}\n" +[[ "${RC}" -eq 0 ]] || fail "a valid selection failed (exit ${RC}): ${OUT}" +[[ "$(ran_suites)" == "${NAMED_SUITE} ${OTHER_SUITE} " ]] || + fail "launched suites: $(ran_suites)" +grep -q "deselected" "${LOGS}/${OTHER_SUITE}.log" || + fail "${OTHER_SUITE} ran unnarrowed: $(tail -3 "${LOGS}/${OTHER_SUITE}.log")" +grep -qE '^ *1 passed' "${LOGS}/${OTHER_SUITE}.log" || + fail "${OTHER_SUITE} did not run exactly its selected test" +[[ "$(sed -n '/^--- slice ---$/,$p' "${LOGS}/shard-manifest.txt" | tail -n +2 | sort | tr '\n' ' ')" == \ + "${NAMED_SUITE} ${OTHER_SUITE} " ]] || fail "the manifest slice is not the selected list" +grep -qE '^selection_sha256=[0-9a-f]{64}$' "${LOGS}/shard-manifest.txt" || + fail "the manifest does not record the selection digest" + +# 2. A selected suite this build does not register. +harness unknown_suite "${NAMED_SUITE}\nno_such_suite\n" +[[ "${RC}" -ne 0 && "${OUT}" == *"does not register"*"no_such_suite"* ]] || + fail "an unregistered selected suite did not fail the run (exit ${RC})" +harness optional_suite "${NAMED_SUITE}\nno_such_suite\n" "no_such_suite\n" +[[ "${RC}" -eq 0 && "${OUT}" == *"not in this build, not run: no_such_suite"* ]] || + fail "a conditional suite absent from this build was not dropped (exit ${RC}): ${OUT}" +[[ "$(ran_suites)" == "${NAMED_SUITE} " ]] || fail "optional case launched: $(ran_suites)" + +# 3. Malformed selections and unknown tests fail. +harness malformed "${NAMED_SUITE} extra\n" +[[ "${RC}" -ne 0 ]] || fail "a malformed selection was accepted" +harness empty "" +[[ "${RC}" -ne 0 ]] || fail "an empty selection was accepted" +harness unknown_test "${OTHER_SUITE}:no_such_test_in_this_build\n" +[[ "${RC}" -ne 0 ]] || fail "a selected test the build does not have did not fail the run" + +if [[ "${FAILURES}" -ne 0 ]]; then + echo "test selection harness: ${FAILURES} case(s) failed" >&2 + exit 1 +fi +echo "test selection harness: all cases passed" diff --git a/tests/test_lane_gate_contract.sh b/tests/test_lane_gate_contract.sh index a3eaf928c..da70d43e1 100755 --- a/tests/test_lane_gate_contract.sh +++ b/tests/test_lane_gate_contract.sh @@ -143,6 +143,7 @@ manifest() { # dir leg shard slice-suites... echo "leg=$leg" echo "shard=$shard" echo "list_sha256=$sha" + [ -z "${SEL:-}" ] || echo "selection_sha256=$SEL" echo "--- slice ---" printf '%s\n' "$@" } >"$dir/shard-manifest.txt" @@ -151,6 +152,13 @@ mkdir -p "$WORK/empty" manifest "$WORK/whole/m1" ubuntu-latest-gcc 1/1 a b c manifest "$WORK/lost/m1" ubuntu-latest-gcc 1/2 a manifest "$WORK/lost/m2" ubuntu-latest-gcc 2/2 b +# A narrowed leg (smart CI): the shards must agree on the selection applied. +SEL=s1 manifest "$WORK/selsame/m1" ubuntu-latest-gcc 1/2 a b +SEL=s1 manifest "$WORK/selsame/m2" ubuntu-latest-gcc 2/2 c +SEL=s1 manifest "$WORK/seldiff/m1" ubuntu-latest-gcc 1/2 a b +SEL=s2 manifest "$WORK/seldiff/m2" ubuntu-latest-gcc 2/2 c +SEL=s1 manifest "$WORK/selhalf/m1" ubuntu-latest-gcc 1/2 a b +manifest "$WORK/selhalf/m2" ubuntu-latest-gcc 2/2 c union "no manifests, no selection given (dry run / release)" 1 "$WORK/empty" union "no manifests, selection 'all'" 1 "$WORK/empty" --lanes all @@ -161,6 +169,9 @@ union "no manifests but the windows leg selected" 1 "$WORK/empty" --lanes '["lin union "a complete 1/1 leg" 0 "$WORK/whole" --lanes '["lint","security-static","unix-x86"]' union "a lost slice is still a gate-quality loss" 1 "$WORK/lost" --lanes '["lint","security-static","unix-x86"]' union "a malformed selection is a usage error" 2 "$WORK/empty" --lanes 'lint,unix-x86' +union "shards that applied one test selection" 0 "$WORK/selsame" --lanes '["lint","security-static","unix-x86"]' +union "shards that applied different test selections" 1 "$WORK/seldiff" --lanes '["lint","security-static","unix-x86"]' +union "a narrowed shard next to a full one" 1 "$WORK/selhalf" --lanes '["lint","security-static","unix-x86"]' if [ "$failures" -ne 0 ]; then echo "LANE GATE CONTRACT VIOLATED: $failures case(s)" >&2 diff --git a/tests/test_lane_wiring_contract.sh b/tests/test_lane_wiring_contract.sh index 31a2b00b8..bd5f26b30 100755 --- a/tests/test_lane_wiring_contract.sh +++ b/tests/test_lane_wiring_contract.sh @@ -14,6 +14,11 @@ # job: matrix lanes as a "lane":"" tag in the leg data (the tag # ci-ok reads back from the job name), single lanes by an `if:` that # names them. A lane nothing serves would be selected and never run. +# 4. the test selection (smart CI) is PR-only and fails open: _test.yml's +# test_selection defaults to empty and only test-unix / test-windows +# read it; dry runs and releases never pass it; pr.yml's select-tests +# runs only under vars.TEST_IMPACT_MODE == 'gate', ci-ok does not need it, +# and `test` passes a selection only from a successful narrowed run. # Text-level checks (no YAML library on every leg), like the other workflow # contracts. # @@ -60,9 +65,9 @@ for caller in ("dry-run.yml", "release.yml"): for job, body in jobs(text(caller)).items(): called = re.search(r"uses: \./\.github/workflows/(_\w+\.yml)", body) if called and called.group(1) in REUSABLE: - require(not re.search(r"(?m)^\s+(lanes|shard_profile):", body), - f"{caller} job {job} passes a lane selection to {called.group(1)}; " - f"dry runs and releases must run every lane") + require(not re.search(r"(?m)^\s+(lanes|shard_profile|test_selection):", body), + f"{caller} job {job} passes a lane or test selection to {called.group(1)}; " + f"dry runs and releases must run every lane and every test") m = re.search(r"(?ms)^ shard_profile:\n(.*?)(?=^ \S|^\S|\Z)", text("_test.yml")) require(m is not None and re.search(r"(?m)^ default: standard$", m.group(1)) is not None, "_test.yml must declare a shard_profile input defaulting to standard") @@ -116,6 +121,32 @@ test_wf = jobs(text("_test.yml")) require('--lanes "$LANES"' in test_wf.get("shard-completeness", ""), "shard-completeness must pass the selection to verify-shard-union.sh") +# 4. the test selection: PR-only, fails open. +m = re.search(r"(?ms)^ test_selection:\n(.*?)(?=^ \S|^\S|\Z)", text("_test.yml")) +require(m is not None and re.search(r"(?m)^ default: ''$", m.group(1)) is not None, + "_test.yml must declare a test_selection input defaulting to '' (every test)") +readers = sorted(job for job, body in test_wf.items() if "inputs.test_selection" in body) +require(readers == ["test-unix", "test-windows"], + f"only test-unix and test-windows may read test_selection, found {readers}") +for job in ("test-unix", "test-windows"): + require("CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && " in test_wf.get(job, ""), + f"_test.yml {job} must hand the selection to scripts/test.sh as CBM_TEST_SELECTION_DIR " + f"only when one was passed") +select = pr.get("select-tests", "") +require(re.search(r"(?m)^ if: \$\{\{ vars\.TEST_IMPACT_MODE == 'gate' \}\}$", select) is not None, + "pr.yml select-tests must run only when vars.TEST_IMPACT_MODE == 'gate'") +require("scripts/ci/test-impact-select.sh" in select and "--gate" in select, + "pr.yml select-tests must run scripts/ci/test-impact-select.sh --gate") +require("select-tests" not in needs, + "ci-ok must not need select-tests: a broken selection runs every test, never blocks") +test_job = pr.get("test", "") +require(re.search(r"(?m)^ needs: \[changes, lint, select-tests\]$", test_job) is not None + and "!cancelled()" in test_job, + "pr.yml test must need select-tests and still run when it is skipped or failed") +require("test_selection: ${{ needs.select-tests.result == 'success' && " + "needs.select-tests.outputs.selection == 'narrowed' && 'test-selection' || '' }}" in test_job, + "pr.yml test must pass a selection only from a successful, narrowed select-tests run") + # 3. every lane is served by a job. MATRIX = {"unix-x86": "_test.yml", "unix-arm64": "_test.yml", "unix-macos14": "_test.yml", "unix-macos-intel": "_test.yml", "windows": "_test.yml", "tsan-x86": "_test.yml", diff --git a/tests/test_test_impact_git.c b/tests/test_test_impact_git.c index 498a846cc..46c4ced08 100644 --- a/tests/test_test_impact_git.c +++ b/tests/test_test_impact_git.c @@ -60,8 +60,12 @@ int tf_maybe_run_git_facts_diff_probe(int argc, char **argv) { } if (argc == 2 && argv && strcmp(argv[1], "__cbm_git_facts_hostile_diff_probe") == 0) { /* Only this isolated child mutates its environment, before harness threads start. - * Re-enter the existing exact-diff test; the parent process remains untouched. */ + * Re-enter the existing exact-diff test; the parent process remains untouched. + * An inherited CBM_TEST_ONLY_FILE (a narrowed CI run) is cleared: the runner + * unions it with CBM_TEST_ONLY, and one naming this suite would re-run the + * spawning test, which spawns again. */ if (cbm_setenv("GIT_DIFF_OPTS", "--unified=999", 1) != 0 || + cbm_setenv("CBM_TEST_ONLY_FILE", "", 1) != 0 || cbm_setenv( "CBM_TEST_ONLY", "test_impact_git:test_git_facts_diff_uses_merge_base_zero_context_and_nul_metadata", @@ -945,6 +949,54 @@ TEST(test_git_facts_inherited_diff_options_cannot_override_zero_context) { PASS(); } +/* A narrowed CI run hands every runner process a CBM_TEST_ONLY_FILE. The + * hostile-diff child must still run exactly its own test: the runner unions + * the file with CBM_TEST_ONLY, so an inherited file would add its tests, and + * one naming this suite would make the child re-run the spawning test. */ +#define GF_OTHER_TEST "test_git_facts_rejects_malformed_options_and_expected_head_mismatch" +TEST(test_git_facts_hostile_probe_ignores_an_inherited_selection_file) { + const char *home = th_mktempdir("cbm-git-diff-only"); + ASSERT_NOT_NULL(home); + char log[GF_PATH]; + char only[GF_PATH]; + static const char token[] = "test_impact_git:" GF_OTHER_TEST "\n"; + bool prepared = gf_runner_binary && gf_path(log, home, "child.log") && + gf_path(only, home, "only.txt") && gf_write(only, token, sizeof(token) - 1); + const char *inherited = getenv("CBM_TEST_ONLY_FILE"); + char *saved = inherited ? cbm_strdup(inherited) : NULL; + bool set = prepared && (!inherited || saved) && cbm_setenv("CBM_TEST_ONLY_FILE", only, 1) == 0; + const char *argv[] = {gf_runner_binary, "__cbm_git_facts_hostile_diff_probe", NULL}; + cbm_proc_opts_t options = { + .bin = gf_runner_binary, .argv = argv, .log_file = log, .quiet_timeout_ms = 60000}; + cbm_proc_result_t result = {0}; + int launched = set ? cbm_subprocess_run(&options, &result) : -1; + bool restored = saved ? cbm_setenv("CBM_TEST_ONLY_FILE", saved, 1) == 0 + : cbm_unsetenv("CBM_TEST_ONLY_FILE") == 0; + free(saved); + char detail[16384] = ""; + FILE *file = launched == 0 ? cbm_fopen(log, "rb") : NULL; + if (file) { + size_t n = fread(detail, 1, sizeof(detail) - 1, file); + detail[n] = '\0'; + (void)fclose(file); + } + bool own_test_only = launched == 0 && result.outcome == CBM_PROC_CLEAN && + result.exit_code == 0 && + strstr(detail, "test_git_facts_diff_uses_merge_base_zero_context") && + !strstr(detail, GF_OTHER_TEST); + if (!own_test_only && launched == 0) { + fprintf(stderr, "Isolated hostile-diff child: %s\n", detail); + } + int cleanup = th_rmtree(home); + ASSERT_EQ(cleanup, 0); + ASSERT_TRUE(set); + ASSERT_TRUE(restored); + ASSERT_EQ(launched, 0); + ASSERT_TRUE(own_test_only); + PASS(); +} +#undef GF_OTHER_TEST + /* Independent D1 inventory tests. Reuse the native local-object gf_* fixture * helpers above; construct byte-only names in Git objects, never on disk. */ typedef struct { @@ -2340,6 +2392,7 @@ SUITE(test_impact_git) { RUN_TEST(test_git_facts_inventory_cancellation_clears_prefix_and_latches); RUN_TEST(test_git_facts_inventory_output_limit_has_no_prefix_and_latches); RUN_TEST(test_git_facts_inherited_diff_options_cannot_override_zero_context); + RUN_TEST(test_git_facts_hostile_probe_ignores_an_inherited_selection_file); RUN_TEST(test_git_facts_gitfile_redirection_cannot_certify_false_ancestry); RUN_TEST(test_git_facts_pins_actual_merge_base_and_both_refs); RUN_TEST(test_git_facts_ref_index_and_worktree_mutation_cannot_change_snapshot); diff --git a/tests/test_test_impact_selection.sh b/tests/test_test_impact_selection.sh new file mode 100644 index 000000000..dfdb616cd --- /dev/null +++ b/tests/test_test_impact_selection.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# test_test_impact_selection.sh — contract of scripts/test-impact/selection.py, +# the only place a test-impact answer becomes what CI runs. `filter` narrows +# (exit 0, the lane's CBM_TEST_ONLY_FILE lines) only for a well-formed +# selected lane whose filter matches its own suite list; a skipped lane runs +# nothing (exit 11); everything else — run-all answers, unknown schemas or +# encodings, bad tokens, a filter that disagrees with the suites, an +# unreadable answer — must run everything (any other exit, no file left). +# `suites` lists what the team-artifact producer re-runs, `*` for run-all. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SEL="$ROOT/scripts/test-impact/selection.py" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +fail() { echo "FAIL: $*" >&2; exit 1; } + +answer() { # name lane-json + printf '{"schema":"cbm.test_impact.v0","decision":"selected","lanes":[%s]}\n' "$2" > "$TMP/$1.json" +} +filter_exit() { # name -> exit code of `filter` + local rc=0 + python3 "$SEL" filter "$TMP/$1.json" --lane unit --out "$TMP/$1.only" 2> /dev/null || rc=$? + echo "$rc" +} + +# Narrowed: one whole suite, one suite by tests. +answer narrowed '{"lane":"unit","decision":"selected","suites":[ + {"suite":"alpha","mode":"whole","whole_reason":["STATIC"]}, + {"suite":"beta","mode":"tests","tests":[{"test":"b_one"},{"test":"b_two"}]}], + "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\nbeta:b_one\nbeta:b_two\n"}}' +[ "$(filter_exit narrowed)" = 0 ] || fail "a well-formed selection did not narrow" +[ "$(cat "$TMP/narrowed.only")" = "$(printf 'alpha\nbeta:b_one\nbeta:b_two')" ] || + fail "narrowed filter lines differ: $(cat "$TMP/narrowed.only")" +[ "$(python3 "$SEL" suites "$TMP/narrowed.json")" = "$(printf 'alpha\nbeta')" ] || + fail "suites of a narrowed answer" + +# --optional-out: the selected suites registered only conditionally. +answer conditional '{"lane":"unit","decision":"selected","suites":[ + {"suite":"alpha","mode":"whole","whole_reason":["CONDITIONAL","STATIC"]}, + {"suite":"gamma","mode":"whole","whole_reason":["STATIC"]}], + "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\ngamma\n"}}' +rc=0 +python3 "$SEL" filter "$TMP/conditional.json" --lane unit --out "$TMP/conditional.only" \ + --optional-out "$TMP/conditional.optional" 2> /dev/null || rc=$? +[ "$rc" = 0 ] || fail "a selection with a conditional suite did not narrow" +[ "$(cat "$TMP/conditional.optional")" = alpha ] || + fail "optional suites: $(cat "$TMP/conditional.optional")" +echo alpha > "$TMP/stale.optional" +answer stale '{"lane":"unit","decision":"run_all","suites":null,"runner_filter":null}' +rc=0 +python3 "$SEL" filter "$TMP/stale.json" --lane unit --out "$TMP/stale.only" \ + --optional-out "$TMP/stale.optional" 2> /dev/null || rc=$? +[ "$rc" != 0 ] && [ ! -e "$TMP/stale.optional" ] || fail "a run-all answer left an optional-suites file" + +# Skipped lane: nothing to run, an empty file. +answer skipped '{"lane":"unit","decision":"skip","suites":[],"runner_filter":null}' +[ "$(filter_exit skipped)" = 11 ] || fail "a skipped lane did not report nothing-selected" +[ -f "$TMP/skipped.only" ] && [ ! -s "$TMP/skipped.only" ] || fail "a skipped lane must leave an empty file" +[ -z "$(python3 "$SEL" suites "$TMP/skipped.json")" ] || fail "suites of a skipped lane" + +# Every one of these must run everything and leave no filter file. +printf '{"schema":"cbm.test_impact.v0","decision":"run_all","lanes":[{"lane":"unit","decision":"run_all","suites":null,"runner_filter":null}]}\n' \ + > "$TMP/run_all.json" +answer lane_run_all '{"lane":"unit","decision":"run_all","suites":null,"runner_filter":null}' +printf '{"schema":"cbm.test_impact.v9","decision":"selected","lanes":[]}\n' > "$TMP/schema.json" +answer other_lane '{"lane":"integration","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}], + "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n"}}' +answer encoding '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}], + "runner_filter":{"encoding":"csv","value":"alpha"}}' +answer no_filter '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}],"runner_filter":null}' +answer bad_token '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}], + "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n--all\n"}}' +answer disagrees '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"},{"suite":"beta","mode":"whole"}], + "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n"}}' +answer empty_tests '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"tests","tests":[]}], + "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n"}}' +answer skip_with_suites '{"lane":"unit","decision":"skip","suites":[{"suite":"alpha","mode":"whole"}],"runner_filter":null}' +printf '{"schema":"cbm.test_impact.v0","decision":"selected","lanes":[' > "$TMP/truncated.json" +for name in run_all lane_run_all schema other_lane encoding no_filter bad_token disagrees empty_tests \ + skip_with_suites truncated missing; do + rc="$(filter_exit "$name")" + case "$rc" in 0 | 11) fail "$name narrowed (exit $rc) instead of running everything" ;; esac + [ ! -e "$TMP/$name.only" ] || fail "$name left a filter file behind" +done +# A stale filter from an earlier run is removed, never reused. +echo "alpha" > "$TMP/run_all.only" +[ "$(filter_exit run_all)" != 0 ] && [ ! -e "$TMP/run_all.only" ] || fail "a stale filter file survived" + +[ "$(python3 "$SEL" suites "$TMP/run_all.json")" = "*" ] || fail "suites of a run-all answer" +[ "$(python3 "$SEL" suites "$TMP/lane_run_all.json")" = "*" ] || fail "suites of a run-all lane (suites: null)" +echo "test-impact selection: all cases passed"