diff --git a/scripts/ci/README.md b/scripts/ci/README.md index 0504f01d1..3e74b86d7 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -13,9 +13,9 @@ CI and the local infrastructure — both of which the venue-parity contract | `preflight-docker.sh` | Same idea for Colima/docker: prune runner-unlike residue, assert free space on the filesystem backing the docker data root (not the VM's `/`). Build cache + named volumes KEPT (the local analogue of actions/cache); `--deep` drops them. | `test-infrastructure/run.sh` | | `check-glibc-compat.sh` | Run a linux binary in debian:bullseye (glibc 2.31) — the portable binary must start on old glibc. | `_smoke.yml` portable legs | | `generate-sbom.py` | The release SPDX SBOM (vendored versions reviewable here, diffable by vendoring PRs — was inline YAML). | `release.yml` | -| `require-all-green.sh` | The aggregate gate: fail unless every needed job succeeded or legitimately skipped (was inline YAML). | `pr.yml ci-ok` | +| `require-all-green.sh` | The aggregate gate: fail unless every needed job succeeded or legitimately skipped (was inline YAML) — and, under lane selection, every SELECTED lane ran and succeeded (skipped/cancelled/missing = red; unselected lanes may skip). | `pr.yml ci-ok` | | `select-lanes.sh` | Decide which PR lanes a change set runs (tier, lanes, full, reasons) from its file list — the R1 rule set, fail-safe to every lane. Pinned by `tests/test_select_lanes.sh` and the September 2026 history replay (`scripts/test-impact/replay-selector.sh`). | `pr.yml changes` | -| `verify-shard-union.sh` | Prove sharded test legs lost nothing: shard count agreement, indices 1..n, identical suite lists, union of slices == full list (was inline YAML). | `_test.yml` shard-completeness | +| `verify-shard-union.sh` | Prove sharded test legs lost nothing: shard count agreement, indices 1..n, identical suite lists, union of slices == full list (was inline YAML). No manifests is green only when the lane selection holds no test leg. | `_test.yml` shard-completeness | | `prepare-release-candidates.sh` | Copy one linker output into stripped/unstripped candidates, finalize signatures, composition-check them without execution, and record their hashes. | `_build.yml`, local artifact smoke | | `stage-release-candidates.py` | Admit exactly eight candidate artifacts / sixteen byte-distinct binaries into the content-addressed VirusTotal scan set. | `_build.yml` | | `select-release-candidates.py` | Apply the reviewed tuple-local VT truth table, or the explicit dry-run stripped default, and atomically copy one content-bound binary per target. | `_build.yml` | diff --git a/scripts/ci/require-all-green.sh b/scripts/ci/require-all-green.sh index bb9c5a33f..027118287 100755 --- a/scripts/ci/require-all-green.sh +++ b/scripts/ci/require-all-green.sh @@ -1,31 +1,121 @@ #!/usr/bin/env bash # require-all-green.sh — the aggregate gate step: fail unless every needed job -# succeeded (or was legitimately skipped). +# succeeded (or was legitimately skipped) and every SELECTED lane ran green. # # Canonical CI step (called by pr.yml ci-ok with RESULTS = toJSON(needs)). # Lived inline in workflow YAML until 2026-07-26; the venue-parity contract -# forbids logic in workflow run-blocks. `skipped` counts as OK because gated -# jobs (e.g. pr-smoke on docs-only PRs) skip by design — a matrix rename can -# still never silently deadlock a merge, which is this gate's purpose. +# forbids logic in workflow run-blocks. `skipped` counts as OK at the needs +# level because gated jobs (e.g. pr-smoke on docs-only PRs) skip by design — a +# matrix rename can still never silently deadlock a merge, which is this +# gate's purpose. # -# Usage: RESULTS='' scripts/ci/require-all-green.sh +# Lane selection (LANES + JOBS, both set by pr.yml): a PR runs only the lanes +# scripts/ci/select-lanes.sh selected, and there `skipped` is no longer a free +# pass — a selected lane whose jobs were skipped, cancelled, never created or +# are anything but success is a silent gate loss, so it fails. Lanes nobody +# selected may be skipped or absent (but may not fail); memwaste is +# report-only and never gates. The selection itself must be trustworthy: a +# list of known lanes that holds lint or contracts (every tier runs one). +# +# Usage: RESULTS='' \ +# [LANES='' JOBS=] \ +# scripts/ci/require-all-green.sh +# JOBS: this run's jobs as JSON lines {"name","status","conclusion"} +# (gh api .../actions/runs//jobs?filter=latest --jq '.jobs[] | ...'). set -euo pipefail case "${1:-}" in -h | --help) - sed -n '2,11p' "$0" | sed 's/^# \{0,1\}//' + sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//' exit 0 ;; esac : "${RESULTS:?set RESULTS to the toJSON(needs) context}" +UNIVERSE="" +if [ -n "${LANES+set}" ]; then + UNIVERSE=$(bash "$(dirname "$0")/select-lanes.sh" --list-lanes) +fi + printf '%s' "$RESULTS" | python3 -c " -import json, sys +import json, os, re, sys + needs = json.load(sys.stdin) bad = {k: v['result'] for k, v in needs.items() if v['result'] not in ('success', 'skipped')} -if bad: - print('CI NOT OK:', bad) +problems = [f'needed job {k}: {v}' for k, v in sorted(bad.items())] + +if 'LANES' in os.environ: + universe = set(sys.argv[1].split()) + lanes = None + try: + lanes = json.loads(os.environ['LANES']) + except ValueError: + pass + if not isinstance(lanes, list) or not all(isinstance(l, str) for l in lanes): + problems.append('LANES is not a JSON list of lane names: ' + repr(os.environ['LANES'])) + lanes = [] + selected = set(lanes) + if selected - universe: + problems.append('unknown lane(s) selected: ' + ', '.join(sorted(selected - universe))) + if not selected & {'lint', 'contracts'}: + problems.append('selection has neither lint nor contracts -- refusing to trust it') + + # job name -> lane. Matrix jobs carry their lane among the values in + # parentheses; everything else is named by its (caller /) job id. + single = { + 'lint / lint': 'lint', 'lint / lint-mem': 'lint-mem', + 'security / security-static': 'security-static', + 'security / license-gate': 'license-gate', 'security / codeql-gate': 'codeql-gate', + 'test / test-diag': 'diag', 'test / test-msan': 'msan', + 'test / test-lsan-macos': 'lsan-macos', 'test / test-windows-guards': 'windows-guards', + 'test / shard-completeness': 'shard-completeness', 'contracts': 'contracts', + } + matrix = ('test / test-unix (', 'test / test-windows (', 'test / test-tsan (', 'pr-smoke (') + def lane_of(name): + if name in single: + return single[name] + if name.startswith('test / test-package-wrappers'): + return 'pkg-wrappers' + if name == 'memwaste' or name.startswith('memwaste / '): + return 'memwaste' + if name.startswith(matrix) and name.endswith(')'): + values = name[name.index('(') + 1:-1].split(', ') + hits = [v for v in values if v in universe] + return hits[0] if len(hits) == 1 else None + return None + + jobs_by_lane = {} + path = os.environ.get('JOBS', '') + if not path or not os.path.isfile(path): + problems.append('JOBS (the run\'s job list) is missing: cannot verify the selected lanes') + else: + for line in open(path, encoding='utf-8'): + if line.strip(): + job = json.loads(line) + lane = lane_of(job.get('name', '')) + if lane: + jobs_by_lane.setdefault(lane, []).append(job) + for lane in sorted(universe - {'memwaste'}): + jobs = jobs_by_lane.get(lane, []) + if lane in selected: + if not jobs: + problems.append(f'selected lane {lane}: no job ran') + for job in jobs: + state = job.get('conclusion') if job.get('status') == 'completed' else job.get('status') + if state != 'success': + problems.append(f'selected lane {lane}: {job[\"name\"]} is {state}') + else: + for job in jobs: + if job.get('status') == 'completed' and job.get('conclusion') not in ('success', 'skipped'): + problems.append(f'unselected lane {lane}: {job[\"name\"]} is {job[\"conclusion\"]}') + if not problems: + print('selected lanes green:', ', '.join(sorted(selected))) + +if problems: + print('CI NOT OK:') + for p in problems: + print(' ' + p) sys.exit(1) print('CI OK:', ', '.join(needs)) -" +" "$UNIVERSE" diff --git a/scripts/ci/verify-shard-union.sh b/scripts/ci/verify-shard-union.sh index 70b91ef4b..b2c69d0ff 100755 --- a/scripts/ci/verify-shard-union.sh +++ b/scripts/ci/verify-shard-union.sh @@ -10,22 +10,52 @@ # shard slices equals that list — a rename/re-shard can never silently drop a # suite (gate-quality loss) without failing here. # -# Usage: scripts/ci/verify-shard-union.sh +# No manifests at all is a failure — unless the PR's lane selection (--lanes, +# the JSON list from scripts/ci/select-lanes.sh) holds no test leg, in which +# case no leg ran by design. Without --lanes (or with "all": dry run, +# release) every leg is expected, exactly as before lane selection. +# +# Usage: scripts/ci/verify-shard-union.sh [--lanes |all] set -eu case "${1:-}" in -h | --help) - sed -n '2,13p' "$0" | sed 's/^# \{0,1\}//' + sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//' exit 0 ;; esac MANIFEST_DIR="${1:?usage: verify-shard-union.sh (see --help)}" +LANES=all +if [ "${2:-}" = "--lanes" ]; then + LANES="${3:-}" +elif [ -n "${2:-}" ]; then + echo "verify-shard-union.sh: unknown argument '$2'. Please consult --help." >&2 + exit 2 +fi +case "$LANES" in +all | \[*\]) ;; +*) + echo "verify-shard-union.sh: --lanes must be 'all' or a JSON list. Please consult --help." >&2 + exit 2 + ;; +esac -files=$(find "$MANIFEST_DIR" -name shard-manifest.txt | sort) +files="" +if [ -d "$MANIFEST_DIR" ]; then + files=$(find "$MANIFEST_DIR" -name shard-manifest.txt | sort) +fi if [ -z "$files" ]; then - echo "FAIL: no shard manifests were uploaded" >&2 - exit 1 + # The test legs are the sharded ones: unix-* and windows (matched with its + # quotes, so the windows-guards lane never reads as the windows leg). + case "$LANES" in + all | *\"unix-* | *\"windows\"*) + echo "FAIL: no shard manifests were uploaded" >&2 + exit 1 + ;; + esac + echo "OK: no test leg was selected ($LANES) — no manifests expected" + exit 0 fi rc=0 for leg in $(grep -h '^leg=' $files | sort -u | sed 's/^leg=//'); do diff --git a/scripts/test.sh b/scripts/test.sh index 9f00f6a9c..3089a7695 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -345,6 +345,8 @@ echo "=== Step 0z: PR lane selector contract ===" bash "$ROOT/tests/test_select_lanes.sh" echo "=== Step 0z2: lane selector history replay ===" bash "$ROOT/scripts/test-impact/replay-selector.sh" +echo "=== Step 0z3: lane-aware aggregate gates (ci-ok, shard union) ===" +bash "$ROOT/tests/test_lane_gate_contract.sh" # Verify compiler supports target arch verify_compiler "$CC" diff --git a/tests/test_lane_gate_contract.sh b/tests/test_lane_gate_contract.sh new file mode 100755 index 000000000..a3eaf928c --- /dev/null +++ b/tests/test_lane_gate_contract.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Lane-gate contract: the two aggregate gates under lane selection. +# +# With lane selection a PR runs a SUBSET of its jobs, and a subset is exactly +# where a silent gate loss hides: before selection a `skipped` job only ever +# meant "not needed for this PR", so ci-ok could wave every skip through. Now a +# selected lane whose jobs were skipped, cancelled or never created looks just +# as quiet. So: +# - scripts/ci/require-all-green.sh (ci-ok): given the selector's LANES and +# the run's JOBS, a SELECTED lane must have run and succeeded; an +# unselected lane may be skipped or absent; memwaste never gates. +# - scripts/ci/verify-shard-union.sh (shard-completeness): "no manifests" is +# green only when the selection holds no test leg at all; with legs (or +# with no selection given: dry run, release) it stays a failure. +# Both scripts are driven with synthetic inputs, never copied. +# +# Usage: tests/test_lane_gate_contract.sh [repo-root] + +set -uo pipefail + +ROOT="${1:-$(cd "$(dirname "$0")/.." && pwd)}" +GREEN_GATE="$ROOT/scripts/ci/require-all-green.sh" +UNION="$ROOT/scripts/ci/verify-shard-union.sh" +for f in "$GREEN_GATE" "$UNION"; do + if [ ! -f "$f" ]; then + echo "FAIL: $f not found" >&2 + exit 1 + fi +done + +WORK=$(mktemp -d "${TMPDIR:-/tmp}/cbm-lane-gate.XXXXXX") || exit 1 +trap 'rm -rf "$WORK"' EXIT +failures=0 + +# ── ci-ok ──────────────────────────────────────────────────────────────── +NEEDS_OK='{"changes":{"result":"success"},"security":{"result":"success"},"lint":{"result":"success"},"test":{"result":"success"},"pr-smoke":{"result":"skipped"},"contracts":{"result":"skipped"}}' +NEEDS_RED='{"changes":{"result":"success"},"security":{"result":"success"},"lint":{"result":"failure"},"test":{"result":"skipped"},"pr-smoke":{"result":"skipped"},"contracts":{"result":"skipped"}}' +T1='["lint","lint-mem","security-static","shard-completeness","unix-macos14","unix-x86","windows"]' + +job() { # name conclusion [status] + printf '{"name":"%s","status":"%s","conclusion":%s}\n' "$1" "${3:-completed}" \ + "$([ "$2" = null ] && echo null || echo "\"$2\"")" +} +t1_jobs() { # the jobs of a green T1 run; $1 = conclusion of the x86 leg + job "changes" success + job "security / security-static" success + job "security / license-gate" skipped + job "security / codeql-gate" skipped + job "lint / lint" success + job "lint / lint-mem" success + job "test / setup-matrix" success + job "test / test-unix (ubuntu-latest, gcc, g++, unix-x86, 1/1)" "$1" + job "test / test-unix (macos-14, cc, c++, unix-macos14, 1/1)" success + job "test / test-windows (windows-latest, CLANG64, x86_64, windows, 1/1)" success + job "test / test-windows-guards" skipped + job "test / test-package-wrappers" skipped + job "test / test-diag" skipped + job "test / test-msan" skipped + job "test / test-lsan-macos" skipped + job "test / test-tsan" skipped + job "test / shard-completeness" success + job "pr-smoke" skipped + job "contracts" skipped + job "memwaste" skipped + job "test-impact-shadow" failure + job "ci-ok" null in_progress +} + +gate() { # description want needs lanes jobs-file|- + local description="$1" want="$2" needs="$3" lanes="$4" jobs="$5" got + if [ "$lanes" = "-" ]; then + RESULTS="$needs" bash "$GREEN_GATE" >"$WORK/out" 2>&1 + else + RESULTS="$needs" LANES="$lanes" JOBS="$jobs" bash "$GREEN_GATE" >"$WORK/out" 2>&1 + fi + got=$? + if [ "$got" -ne "$want" ]; then + echo "FAIL: ci-ok: $description — exit $got, expected $want" >&2 + sed 's/^/ /' "$WORK/out" >&2 + failures=$((failures + 1)) + fi +} + +t1_jobs success >"$WORK/t1-green" +t1_jobs skipped >"$WORK/t1-x86-skipped" +t1_jobs cancelled >"$WORK/t1-x86-cancelled" +grep -v 'unix-x86' "$WORK/t1-green" >"$WORK/t1-x86-missing" +t1_jobs success | sed 's|"lint / lint","status":"completed","conclusion":"success"|"lint / lint","status":"completed","conclusion":"cancelled"|' >"$WORK/t1-lint-cancelled" +t1_jobs success | sed 's|"test / test-diag","status":"completed","conclusion":"skipped"|"test / test-diag","status":"completed","conclusion":"failure"|' >"$WORK/t1-unselected-failed" +t1_jobs success | sed 's|"lint / lint","status":"completed"|"lint / lint","status":"in_progress"|' >"$WORK/t1-lint-running" +{ t1_jobs success; job "memwaste / event-linux" failure; } >"$WORK/t1-memwaste-red" + +# Without a selection the gate is exactly the old needs-level gate. +gate "no selection, every stage green or skipped" 0 "$NEEDS_OK" - - +gate "no selection, a failed stage" 1 "$NEEDS_RED" - - +# The selection contract. +gate "selected lanes all succeeded; unselected skipped or absent" 0 "$NEEDS_OK" "$T1" "$WORK/t1-green" +gate "a SELECTED lane that was skipped is red" 1 "$NEEDS_OK" "$T1" "$WORK/t1-x86-skipped" +gate "a SELECTED lane that was cancelled is red" 1 "$NEEDS_OK" "$T1" "$WORK/t1-x86-cancelled" +gate "a SELECTED lane with no job at all is red" 1 "$NEEDS_OK" "$T1" "$WORK/t1-x86-missing" +gate "a cancelled selected single job is red" 1 "$NEEDS_OK" "$T1" "$WORK/t1-lint-cancelled" +gate "a selected job that has not completed is red" 1 "$NEEDS_OK" "$T1" "$WORK/t1-lint-running" +gate "an unselected lane that ran and FAILED is red" 1 "$NEEDS_OK" "$T1" "$WORK/t1-unselected-failed" +gate "memwaste is report-only even when selected" 0 "$NEEDS_OK" "${T1%]},\"memwaste\"]" "$WORK/t1-memwaste-red" +gate "the needs-level gate still applies under a selection" 1 "$NEEDS_RED" "$T1" "$WORK/t1-green" +# A selection ci-ok cannot trust is red, never "nothing to check". +gate "an empty selection is red (no floor lane)" 1 "$NEEDS_OK" '[]' "$WORK/t1-green" +gate "a selection without lint or contracts is red" 1 "$NEEDS_OK" '["security-static","unix-x86"]' "$WORK/t1-green" +gate "an unknown lane is red" 1 "$NEEDS_OK" '["lint","security-static","no-such-lane"]' "$WORK/t1-green" +gate "a malformed selection is red" 1 "$NEEDS_OK" 'lint,unix-x86' "$WORK/t1-green" +gate "a selection without the job list is red" 1 "$NEEDS_OK" "$T1" "$WORK/does-not-exist" + +# Matrix jobs carry their lane in the name; the neighbouring single job +# (test-windows-guards) must never be read as the windows leg. +{ + job "security / security-static" success + job "lint / lint" success + job "lint / lint-mem" success + job "test / test-windows-guards" success + job "pr-smoke (macos-14, smoke-mac)" success +} >"$WORK/guards-only" +gate "windows-guards is not the windows leg" 1 "$NEEDS_OK" '["lint","lint-mem","security-static","windows"]' "$WORK/guards-only" +gate "pr-smoke matrix name maps to its lane" 0 "$NEEDS_OK" '["lint","lint-mem","security-static","smoke-mac","windows-guards"]' "$WORK/guards-only" + +# ── shard-completeness ─────────────────────────────────────────────────── +union() { # description want dir [--lanes value] + local description="$1" want="$2" dir="$3" got + shift 3 + bash "$UNION" "$dir" "$@" >"$WORK/out" 2>&1 + got=$? + if [ "$got" -ne "$want" ]; then + echo "FAIL: shard union: $description — exit $got, expected $want" >&2 + sed 's/^/ /' "$WORK/out" >&2 + failures=$((failures + 1)) + fi +} +manifest() { # dir leg shard slice-suites... + local dir="$1" leg="$2" shard="$3" sha + shift 3 + mkdir -p "$dir" + sha=$(printf '%s\n' a b c | sort | { sha256sum 2>/dev/null || shasum -a 256; } | awk '{print $1}') + { + echo "leg=$leg" + echo "shard=$shard" + echo "list_sha256=$sha" + echo "--- slice ---" + printf '%s\n' "$@" + } >"$dir/shard-manifest.txt" +} +mkdir -p "$WORK/empty" +manifest "$WORK/whole/m1" ubuntu-latest-gcc 1/1 a b c +manifest "$WORK/lost/m1" ubuntu-latest-gcc 1/2 a +manifest "$WORK/lost/m2" ubuntu-latest-gcc 2/2 b + +union "no manifests, no selection given (dry run / release)" 1 "$WORK/empty" +union "no manifests, selection 'all'" 1 "$WORK/empty" --lanes all +union "no manifests and no test leg selected" 0 "$WORK/empty" --lanes '["lint","lint-mem","security-static","windows-guards"]' +union "no download dir at all and no test leg selected" 0 "$WORK/never-created" --lanes '["lint","lint-mem","security-static"]' +union "no manifests but a unix leg selected" 1 "$WORK/empty" --lanes '["lint","security-static","unix-x86"]' +union "no manifests but the windows leg selected" 1 "$WORK/empty" --lanes '["lint","security-static","windows"]' +union "a complete 1/1 leg" 0 "$WORK/whole" --lanes '["lint","security-static","unix-x86"]' +union "a lost slice is still a gate-quality loss" 1 "$WORK/lost" --lanes '["lint","security-static","unix-x86"]' +union "a malformed selection is a usage error" 2 "$WORK/empty" --lanes 'lint,unix-x86' + +if [ "$failures" -ne 0 ]; then + echo "LANE GATE CONTRACT VIOLATED: $failures case(s)" >&2 + exit 1 +fi +echo "lane-gate contract OK (ci-ok lane strictness + shard-union selection)"