mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
ci: run only the selected lanes on PRs; dry run and release unchanged
pr.yml's `changes` job now lists the changed files (renames with their old path too), runs scripts/ci/select-lanes.sh in a credential-less step and outputs tier, full, lanes and the pr-smoke matrix. security, lint and test pass the lanes to their reusable workflows, which take a `lanes` input defaulting to "all": dry-run.yml and release.yml pass none and run exactly as before (setup-matrix output for every broad/shard combination is byte-identical; the TSan matrix moved into setup-matrix with the same legs). Under a selection, setup-matrix keeps only the tagged legs of the selected lanes (an untagged leg is an error, never a silent drop) and each lane job has an `if:` on its lane. pr-smoke and memwaste run on their lanes instead of the old product regex. Docs-only PRs (T0a) get a `contracts` job running the new `scripts/test.sh --contracts-only` (Step 0 contracts, no build). ci-ok needs contracts, lists the run's jobs (actions: read) and passes LANES/JOBS to require-all-green.sh; shard-completeness passes --lanes. tests/test_lane_wiring_contract.sh (Step 0z4) pins it: dry run and release pass no lanes and the inputs default to all, pr.yml wires the selection through, and every lane is served by a tagged leg or an `if:` naming it. The smoke-fixture contract now checks the selector's decisions for the smoke-harness paths instead of the removed regex; select-lanes.sh joins the venue-parity interface probes. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
1 parent
fb558a6b13
commit
adec281d1a
9 files changed
+345
-57
No files matched your search
@@ -3,13 +3,19 @@
|
||||
name: Lint & Security
|
||||
|
||||
on:
|
||||
workflow_call: {}
|
||||
workflow_call:
|
||||
inputs:
|
||||
lanes:
|
||||
description: 'PR lane selection (scripts/ci/select-lanes.sh JSON list); "all" = every job (dry run, release)'
|
||||
type: string
|
||||
default: all
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'lint') }}
|
||||
runs-on: ubuntu-latest
|
||||
# 15 min proved too tight on a slow runner day and a timed-out lint reads
|
||||
# as "cancelled", which the release graph must treat as a hard stop; keep
|
||||
@@ -60,6 +66,7 @@ jobs:
|
||||
# provability (never suppressed — the NOLINT ban applies here too).
|
||||
# Vendored-tree diagnostics are path-filtered, mirroring .cppcheck.
|
||||
lint-mem:
|
||||
if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'lint-mem') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
|
||||
@@ -4,7 +4,12 @@
|
||||
name: Security Gate
|
||||
|
||||
on:
|
||||
workflow_call: {}
|
||||
workflow_call:
|
||||
inputs:
|
||||
lanes:
|
||||
description: 'PR lane selection (scripts/ci/select-lanes.sh JSON list); "all" = every job (dry run, release)'
|
||||
type: string
|
||||
default: all
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -23,6 +28,7 @@ jobs:
|
||||
run: scripts/security-vendored.sh
|
||||
|
||||
license-gate:
|
||||
if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'license-gate') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 240
|
||||
steps:
|
||||
@@ -39,6 +45,7 @@ jobs:
|
||||
run: scripts/audit-license-provenance.py
|
||||
|
||||
codeql-gate:
|
||||
if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'codeql-gate') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 240
|
||||
# Without security-events:read this job inherits the workflow's
|
||||
|
||||
+61
-11
@@ -16,6 +16,10 @@ on:
|
||||
description: 'Split each C-suite leg across parallel shard jobs (ubuntu x3, windows x2); off = one job per leg, byte-identical to the pre-shard topology'
|
||||
type: boolean
|
||||
default: false
|
||||
lanes:
|
||||
description: 'PR lane selection (scripts/ci/select-lanes.sh JSON list): only those legs and jobs run; "all" = every one (dry run, release)'
|
||||
type: string
|
||||
default: all
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -30,18 +34,25 @@ jobs:
|
||||
outputs:
|
||||
unix: ${{ steps.set.outputs.unix }}
|
||||
windows: ${{ steps.set.outputs.windows }}
|
||||
tsan: ${{ steps.set.outputs.tsan }}
|
||||
has_unix: ${{ steps.set.outputs.has_unix }}
|
||||
has_windows: ${{ steps.set.outputs.has_windows }}
|
||||
has_tsan: ${{ steps.set.outputs.has_tsan }}
|
||||
steps:
|
||||
- name: Compute matrices
|
||||
id: set
|
||||
env:
|
||||
BROAD: ${{ inputs.broad_platforms }}
|
||||
SHARDS: ${{ inputs.shard_suites }}
|
||||
LANES: ${{ inputs.lanes }}
|
||||
run: |
|
||||
# Every CORE leg is tagged with the lane (scripts/ci/select-lanes.sh)
|
||||
# it serves; the tag is what lane selection filters on below.
|
||||
CORE_UNIX='[
|
||||
{"os":"ubuntu-latest","cc":"gcc","cxx":"g++"},
|
||||
{"os":"ubuntu-24.04-arm","cc":"gcc","cxx":"g++"},
|
||||
{"os":"macos-14","cc":"cc","cxx":"c++"},
|
||||
{"os":"macos-15-intel","cc":"cc","cxx":"c++"}
|
||||
{"os":"ubuntu-latest","cc":"gcc","cxx":"g++","lane":"unix-x86"},
|
||||
{"os":"ubuntu-24.04-arm","cc":"gcc","cxx":"g++","lane":"unix-arm64"},
|
||||
{"os":"macos-14","cc":"cc","cxx":"c++","lane":"unix-macos14"},
|
||||
{"os":"macos-15-intel","cc":"cc","cxx":"c++","lane":"unix-macos-intel"}
|
||||
]'
|
||||
# Broad matrix legs are REQUIRED gates (no optional/continue-on-error
|
||||
# escape hatches): the release dry run must be all-green, every platform.
|
||||
@@ -59,7 +70,7 @@ jobs:
|
||||
# interception crashed (interception_win: unhandled instruction). With
|
||||
# the native ARM64 toolchain ASan instruments native ARM64 code, so it
|
||||
# is a real (non-optional) gate, not a tolerated emulated-flake.
|
||||
CORE_WIN='[{"os":"windows-latest","msystem":"CLANG64","pkg":"x86_64"}]'
|
||||
CORE_WIN='[{"os":"windows-latest","msystem":"CLANG64","pkg":"x86_64","lane":"windows"}]'
|
||||
BROAD_WIN='[{"os":"windows-2025","msystem":"CLANG64","pkg":"x86_64"},{"os":"windows-11-arm","msystem":"CLANGARM64","pkg":"aarch64"}]'
|
||||
if [ "$BROAD" = "true" ]; then
|
||||
UNIX=$(jq -cn --argjson a "$CORE_UNIX" --argjson b "$BROAD_UNIX" '$a + $b')
|
||||
@@ -68,6 +79,30 @@ jobs:
|
||||
UNIX=$(jq -cn --argjson a "$CORE_UNIX" '$a')
|
||||
WIN=$(jq -cn --argjson a "$CORE_WIN" '$a')
|
||||
fi
|
||||
# The ThreadSanitizer legs (test-tsan), tagged the same way.
|
||||
TSAN='[
|
||||
{"os":"ubuntu-latest","cc":"clang","cxx":"clang++","lane":"tsan-x86"},
|
||||
{"os":"ubuntu-24.04-arm","cc":"clang","cxx":"clang++","lane":"tsan-arm"},
|
||||
{"os":"macos-14","cc":"cc","cxx":"c++","lane":"tsan-mac"}
|
||||
]'
|
||||
# Lane selection. "all" (dry run, release) keeps every leg and drops
|
||||
# the tags, so those matrices -- and their job names -- stay exactly
|
||||
# as before lane selection. A PR keeps only its selected legs, and
|
||||
# the tag stays in the job name, where ci-ok reads it back. An
|
||||
# untagged leg could never be selected: that is a configuration
|
||||
# error here, never a leg silently dropped.
|
||||
if [ "$LANES" = "all" ]; then
|
||||
UNIX=$(jq -c 'map(del(.lane))' <<< "$UNIX")
|
||||
WIN=$(jq -c 'map(del(.lane))' <<< "$WIN")
|
||||
TSAN=$(jq -c 'map(del(.lane))' <<< "$TSAN")
|
||||
else
|
||||
for M in "$UNIX" "$WIN" "$TSAN"; do
|
||||
jq -e 'all(.[]; has("lane"))' <<< "$M" > /dev/null || { echo "::error::untagged test leg in $M"; exit 1; }
|
||||
done
|
||||
UNIX=$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$UNIX")
|
||||
WIN=$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$WIN")
|
||||
TSAN=$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$TSAN")
|
||||
fi
|
||||
# Suite sharding (opt-in via the shard_suites input): each platform's
|
||||
# C-suite run is split across N runner jobs. CBM_TEST_SHARD="i/N"
|
||||
# slices deterministically inside run-tests-parallel.sh; each
|
||||
@@ -89,9 +124,16 @@ jobs:
|
||||
fi
|
||||
echo "unix={\"include\":$UNIX}" >> "$GITHUB_OUTPUT"
|
||||
echo "windows={\"include\":$WIN}" >> "$GITHUB_OUTPUT"
|
||||
echo "tsan={\"include\":$TSAN}" >> "$GITHUB_OUTPUT"
|
||||
# GitHub rejects an empty matrix, so a job whose legs were all
|
||||
# deselected is skipped through these flags instead.
|
||||
echo "has_unix=$(jq 'length > 0' <<< "$UNIX")" >> "$GITHUB_OUTPUT"
|
||||
echo "has_windows=$(jq 'length > 0' <<< "$WIN")" >> "$GITHUB_OUTPUT"
|
||||
echo "has_tsan=$(jq 'length > 0' <<< "$TSAN")" >> "$GITHUB_OUTPUT"
|
||||
|
||||
test-unix:
|
||||
needs: setup-matrix
|
||||
if: ${{ needs.setup-matrix.outputs.has_unix == 'true' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJSON(needs.setup-matrix.outputs.unix) }}
|
||||
@@ -205,6 +247,7 @@ jobs:
|
||||
# inode/flock behavior and Windows handle/link behavior are each gating,
|
||||
# without adding Node/Go provisioning to every sanitizer/container leg.
|
||||
test-package-wrappers:
|
||||
if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'pkg-wrappers') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -245,6 +288,7 @@ jobs:
|
||||
# parallel suite — catches what an older compiler-rt can miss. Same
|
||||
# canonical scripts/test.sh wave as every other leg.
|
||||
test-diag:
|
||||
if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'diag') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
@@ -308,6 +352,7 @@ jobs:
|
||||
# list both venues now share. Zero use-of-uninitialized-value findings were
|
||||
# reported across the lane on either architecture.
|
||||
test-msan:
|
||||
if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'msan') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
@@ -332,6 +377,7 @@ jobs:
|
||||
# so this leg is the ordinary ASan suite built with that toolchain and run
|
||||
# with detect_leaks=1. Mirrors the local `make -f Makefile.cbm test-lsan`.
|
||||
test-lsan-macos:
|
||||
if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'lsan-macos') }}
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
@@ -370,13 +416,11 @@ jobs:
|
||||
key: ccache-lsan-${{ github.ref }}-${{ github.sha }}
|
||||
|
||||
test-tsan:
|
||||
needs: setup-matrix
|
||||
if: ${{ needs.setup-matrix.outputs.has_tsan == 'true' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { os: ubuntu-latest, cc: clang, cxx: clang++ }
|
||||
- { os: ubuntu-24.04-arm, cc: clang, cxx: clang++ }
|
||||
- { os: macos-14, cc: cc, cxx: c++ }
|
||||
matrix: ${{ fromJSON(needs.setup-matrix.outputs.tsan) }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
@@ -429,6 +473,7 @@ jobs:
|
||||
|
||||
test-windows:
|
||||
needs: setup-matrix
|
||||
if: ${{ needs.setup-matrix.outputs.has_windows == 'true' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJSON(needs.setup-matrix.outputs.windows) }}
|
||||
@@ -562,8 +607,12 @@ jobs:
|
||||
with:
|
||||
pattern: shard-manifest-*
|
||||
path: manifests
|
||||
# A PR with no test leg selected uploads no manifest; the script
|
||||
# accepts that only when the selection really holds no leg.
|
||||
- name: Verify cross-shard union per leg
|
||||
run: repo/scripts/ci/verify-shard-union.sh manifests
|
||||
env:
|
||||
LANES: ${{ inputs.lanes }}
|
||||
run: repo/scripts/ci/verify-shard-union.sh manifests --lanes "$LANES"
|
||||
|
||||
# Windows product-surface regression guards. Distinct from test-windows above
|
||||
# (the sanitizer C suite): these drive a real product binary + external UI pack
|
||||
@@ -574,6 +623,7 @@ jobs:
|
||||
# CLI arguments being mangled by the narrow-argv main() (#423/#20, fixed by the
|
||||
# wide-argv entrypoint that reads GetCommandLineW).
|
||||
test-windows-guards:
|
||||
if: ${{ inputs.lanes == 'all' || contains(fromJSON(inputs.lanes), 'windows-guards') }}
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
|
||||
+85
-30
@@ -1,7 +1,10 @@
|
||||
# PR validation: security gates + lint + full test suite. Builds, smoke and
|
||||
# soak stay maintainer-driven via the dry-run workflow_dispatch. Branch
|
||||
# PR validation: security gates + lint + the test legs this PR's changes need.
|
||||
# `changes` runs scripts/ci/select-lanes.sh on the PR's file list; every lane
|
||||
# job below runs only when selected (unknown paths, workflows, the harness,
|
||||
# Makefile.cbm and vendored code select everything). Builds, smoke and soak at
|
||||
# full breadth stay maintainer-driven via the dry-run workflow_dispatch. Branch
|
||||
# protection requires `dco` + `ci-ok` — a single stable summary context that
|
||||
# fails unless every PR stage succeeded.
|
||||
# fails unless every PR stage succeeded and every selected lane ran green.
|
||||
name: PR
|
||||
|
||||
on:
|
||||
@@ -21,6 +24,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
security:
|
||||
needs: [changes]
|
||||
uses: ./.github/workflows/_security.yml
|
||||
# The called job requests security-events:read to count code-scanning
|
||||
# alerts; a reusable workflow cannot request more than its caller grants,
|
||||
@@ -30,15 +34,22 @@ jobs:
|
||||
security-events: read
|
||||
actions: read
|
||||
secrets: inherit
|
||||
with:
|
||||
lanes: ${{ needs.changes.outputs.lanes }}
|
||||
|
||||
lint:
|
||||
needs: [changes]
|
||||
if: ${{ contains(fromJSON(needs.changes.outputs.lanes), 'lint') || contains(fromJSON(needs.changes.outputs.lanes), 'lint-mem') }}
|
||||
uses: ./.github/workflows/_lint.yml
|
||||
with:
|
||||
lanes: ${{ needs.changes.outputs.lanes }}
|
||||
|
||||
test:
|
||||
needs: [lint]
|
||||
needs: [changes, lint]
|
||||
if: ${{ !cancelled() && needs.lint.result == 'success' }}
|
||||
uses: ./.github/workflows/_test.yml
|
||||
with:
|
||||
lanes: ${{ needs.changes.outputs.lanes }}
|
||||
# Perf assertions are timing-sensitive on shared runners; they stay in
|
||||
# dry runs and releases where a flaky red does not block a merge.
|
||||
skip_perf: true
|
||||
@@ -47,16 +58,19 @@ jobs:
|
||||
# shard-completeness job). Dry runs and releases shard identically.
|
||||
shard_suites: true
|
||||
|
||||
# ── Which files changed? Gate the (heavier) build+smoke on product code so
|
||||
# docs / CI / test-only PRs stay fast. ──
|
||||
# ── Which lanes does this PR need? The changed files go through
|
||||
# scripts/ci/select-lanes.sh (tier, lanes, full, reasons); docs / CI /
|
||||
# test-only PRs stay fast, anything it cannot place runs everything. ──
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
product: ${{ steps.f.outputs.product }}
|
||||
tier: ${{ steps.select.outputs.tier }}
|
||||
full: ${{ steps.select.outputs.full }}
|
||||
lanes: ${{ steps.select.outputs.lanes }}
|
||||
smoke: ${{ steps.select.outputs.smoke }}
|
||||
steps:
|
||||
- name: Detect product-code changes
|
||||
id: f
|
||||
- name: List the changed files
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR: ${{ github.event.pull_request.number }}
|
||||
@@ -64,7 +78,9 @@ jobs:
|
||||
run: |
|
||||
# The full .diff endpoint rejects large-but-valid PRs at 20k lines.
|
||||
# The paginated files endpoint remains filename-only for this gate.
|
||||
if FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[].filename'); then
|
||||
# A rename lists its old path too, as the fallback's --no-renames
|
||||
# diff does: moving a file away is a change to where it was.
|
||||
if FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[] | .filename, (.previous_filename // empty)'); then
|
||||
echo "file list: pulls/files API"
|
||||
else
|
||||
# The files endpoint renders the diff server-side too, and answers
|
||||
@@ -81,31 +97,41 @@ jobs:
|
||||
"https://github.com/$REPO" "refs/pull/$PR/merge"
|
||||
FILES=$(git --git-dir="$CHANGES_GIT" diff --name-only --no-renames FETCH_HEAD^1 FETCH_HEAD)
|
||||
fi
|
||||
printf '%s\n' "$FILES"
|
||||
# A here-string, not a pipe: under this shell's pipefail, grep -q
|
||||
# exits on its first match, printf dies of SIGPIPE once the list
|
||||
# outgrows the pipe buffer, and the MATCH is then reported as a
|
||||
# failure -- product=false, smoke skipped, on the largest PRs.
|
||||
if grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)' <<<"$FILES"; then
|
||||
echo "product=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "product=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
printf '%s\n' "$FILES" > "$RUNNER_TEMP/changed-files.txt"
|
||||
cat "$RUNNER_TEMP/changed-files.txt"
|
||||
|
||||
# The selector is PR code: it runs in its own step, after the
|
||||
# token-bearing one, from a checkout that keeps no credentials.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Select lanes
|
||||
id: select
|
||||
run: |
|
||||
scripts/ci/select-lanes.sh --format github-output "$RUNNER_TEMP/changed-files.txt" > "$RUNNER_TEMP/selection.txt"
|
||||
cat "$RUNNER_TEMP/selection.txt"
|
||||
cat "$RUNNER_TEMP/selection.txt" >> "$GITHUB_OUTPUT"
|
||||
# pr-smoke's legs, each tagged with its lane (kept in the job name,
|
||||
# where ci-ok reads it back); the job gets the selected ones.
|
||||
SMOKE='[{"os":"ubuntu-latest","lane":"smoke-ubuntu"},{"os":"macos-14","lane":"smoke-mac"},{"os":"windows-latest","lane":"smoke-win"}]'
|
||||
LANES=$(sed -n 's/^lanes=//p' "$RUNNER_TEMP/selection.txt")
|
||||
echo "smoke={\"include\":$(jq -c --argjson sel "$LANES" 'map(select(.lane | IN($sel[])))' <<< "$SMOKE")}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# ── Light smoke: build the PRODUCTION binary and run the core smoke on the
|
||||
# three RELIABLE NATIVE platforms. Catches built-binary regressions at PR
|
||||
# time (e.g. the Windows CreateProcess argv-quoting class that previously
|
||||
# only surfaced in the release dry run). The full broad/emulated smoke stays
|
||||
# in the dry run. Only product-code PRs pay this; every leg gates.
|
||||
# in the dry run. Only the selected legs run (ubuntu from the extraction
|
||||
# tier up, all three for platform/packaging changes); every leg gates.
|
||||
# The maintained local wrappers add a race-free release fixture so the
|
||||
# download/checksum/install/update phases run here as well. ──
|
||||
pr-smoke:
|
||||
needs: [changes]
|
||||
if: ${{ !cancelled() && needs.changes.outputs.product == 'true' }}
|
||||
if: ${{ !cancelled() && needs.changes.result == 'success' && (contains(fromJSON(needs.changes.outputs.lanes), 'smoke-ubuntu') || contains(fromJSON(needs.changes.outputs.lanes), 'smoke-mac') || contains(fromJSON(needs.changes.outputs.lanes), 'smoke-win')) }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-14, windows-latest]
|
||||
matrix: ${{ fromJSON(needs.changes.outputs.smoke) }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
@@ -177,27 +203,56 @@ jobs:
|
||||
# ── Waste sanitizer, REPORT-ONLY: memory + CPU waste on this repository and
|
||||
# the per-function scaling lane (Linux). Deliberately NOT in ci-ok's needs:
|
||||
# it uploads evidence and prints the ratchet, it never blocks a merge. Only
|
||||
# product-code PRs pay for it. ──
|
||||
# PRs whose selection holds the memwaste lane (memory core, memwaste
|
||||
# tooling, full runs) pay for it. ──
|
||||
memwaste:
|
||||
needs: [changes]
|
||||
if: ${{ !cancelled() && needs.changes.outputs.product == 'true' }}
|
||||
if: ${{ !cancelled() && needs.changes.result == 'success' && contains(fromJSON(needs.changes.outputs.lanes), 'memwaste') }}
|
||||
uses: ./.github/workflows/_memwaste.yml
|
||||
with:
|
||||
mode: pr
|
||||
|
||||
# ── Docs-only PRs (tier T0a) select no test leg, so this runs the static
|
||||
# contract steps of the canonical test leg on their own (they police
|
||||
# docs surfaces too: language counts, version metadata). Every test leg
|
||||
# runs the same steps, so the job exists only when no leg does. ──
|
||||
contracts:
|
||||
needs: [changes]
|
||||
if: ${{ !cancelled() && needs.changes.result == 'success' && contains(fromJSON(needs.changes.outputs.lanes), 'contracts') }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- name: Contract steps only
|
||||
run: scripts/test.sh --contracts-only
|
||||
|
||||
ci-ok:
|
||||
# The one required context (besides dco) — fails unless every PR stage
|
||||
# succeeded, so matrix renames can never silently deadlock merges. `skipped`
|
||||
# is OK: pr-smoke skips on docs/CI/test-only PRs (changes.product == false).
|
||||
needs: [security, lint, test, changes, pr-smoke]
|
||||
# succeeded, so matrix renames can never silently deadlock merges, and
|
||||
# every lane the selector chose ran green: `skipped` is OK only for a lane
|
||||
# nobody selected (a selected lane that skipped is a silent gate loss).
|
||||
# memwaste and the test-impact shadow are report-only and not needed here.
|
||||
needs: [changes, security, lint, test, pr-smoke, contracts]
|
||||
if: ${{ always() }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
# actions:read lists this run's jobs, the lane-level truth behind `needs`.
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
# Needs a checkout (unlike before): the gate logic lives in the canonical
|
||||
# scripts/ci entry, not inline YAML (venue-parity contract).
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- name: All PR stages must have succeeded
|
||||
- name: List this run's jobs
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs?filter=latest&per_page=100" \
|
||||
--jq '.jobs[] | {name, status, conclusion}' > "$RUNNER_TEMP/jobs.jsonl"
|
||||
- name: All PR stages and every selected lane must have succeeded
|
||||
env:
|
||||
RESULTS: ${{ toJSON(needs) }}
|
||||
LANES: ${{ needs.changes.outputs.lanes }}
|
||||
JOBS: ${{ runner.temp }}/jobs.jsonl
|
||||
run: scripts/ci/require-all-green.sh
|
||||
+1
-1
@@ -17,7 +17,7 @@ codes) and rejects unknown flags with exit 2 + `Please consult --help.`
|
||||
|
||||
| leg | entry | what a run gives you |
|
||||
|---|---|---|
|
||||
| **test** | `test.sh` | DEFAULT = the venue leg: static contracts (Step 0a–0r) + CLEAN ASan+UBSan build + all suites via the parallel harness + prod-binary guards. `--suites a,b` = iteration mode (incremental rebuild, subset, seconds). `--tsan` = the ThreadSanitizer leg. CLANGARM64 gets CI's trap-UBSan flags automatically. |
|
||||
| **test** | `test.sh` | DEFAULT = the venue leg: static contracts (Step 0a–0r) + CLEAN ASan+UBSan build + all suites via the parallel harness + prod-binary guards. `--suites a,b` = iteration mode (incremental rebuild, subset, seconds). `--tsan` = the ThreadSanitizer leg. `--contracts-only` = the static contract steps alone (PR CI's docs-only job). CLANGARM64 gets CI's trap-UBSan flags automatically. |
|
||||
| **package wrappers** | `ci/test-package-wrappers.sh` | Go, npm, and PyPI runtime-set publication/lock suites on the current host. CI runs the same entry on Linux and Windows so both platform lock implementations gate. |
|
||||
| **build** | `build.sh` | CLEAN production runtime set (native executable + authenticated integration asset; `--with-ui` adds one content-addressed UI pack). ccache via `env.sh` makes repeats fast; `CCACHE_COMPILERCHECK=content` guarantees a hit is byte-identical to a cold compile — never stale. `--version`, `STATIC=1`, `BUILD_DIR=`. |
|
||||
| **lint** | `lint.sh` | clang-tidy + cppcheck + clang-format (+ no-skips policy). `--ci` = the CI gate set (no clang-tidy). Drives the same make targets as `make lint`/`lint-ci`. |
|
||||
|
||||
+20
-2
@@ -10,7 +10,7 @@ cd "$ROOT"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: scripts/test.sh [--suites LIST] [--arch ARCH] [VAR=VAL ...]
|
||||
Usage: scripts/test.sh [--suites LIST | --tsan | --contracts-only] [--arch ARCH] [VAR=VAL ...]
|
||||
|
||||
The canonical test entry: identical in local CI, PR CI, dry run and release.
|
||||
DEFAULT (no --suites) is exactly what CI runs: static contract checks
|
||||
@@ -29,6 +29,10 @@ Modes:
|
||||
--tsan ThreadSanitizer leg (data-race gate): builds and runs the
|
||||
widened TSan runner via make test-tsan — the same leg CI's
|
||||
tsan jobs and the compose test-tsan service run.
|
||||
--contracts-only
|
||||
Only the static contract steps (Step 0*) of the default leg:
|
||||
no compiler, no build, no suites. PR CI runs this for
|
||||
docs-only changes, where no test leg is selected.
|
||||
|
||||
Options:
|
||||
--arch ARCH Force target arch (arm64 | x86_64), e.g. under Rosetta.
|
||||
@@ -64,11 +68,13 @@ EOF
|
||||
# silently swallowed — agents must know exactly what a run will do.
|
||||
SUITES=""
|
||||
TSAN=0
|
||||
CONTRACTS_ONLY=0
|
||||
prev_arg=""
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
-h|--help) usage; exit 0 ;;
|
||||
--tsan) :;;
|
||||
--contracts-only) :;;
|
||||
--suites) :;; # next arg is the value, handled below
|
||||
--suites=*) SUITES="${arg#--suites=}" ;;
|
||||
--arch) :;; # next arg is the value, handled below
|
||||
@@ -96,6 +102,7 @@ done
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--tsan) TSAN=1 ;;
|
||||
--contracts-only) CONTRACTS_ONLY=1 ;;
|
||||
arm64|x86_64)
|
||||
if [[ "${prev_arg2:-}" == "--arch" ]]; then
|
||||
export CBM_ARCH="$arg"
|
||||
@@ -121,6 +128,10 @@ if [ "$TSAN" -eq 1 ] && [ -n "$SUITES" ]; then
|
||||
echo "test.sh: --tsan and --suites are separate modes (the TSan leg has its own suite set). Please consult --help." >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ "$CONTRACTS_ONLY" -eq 1 ] && { [ "$TSAN" -eq 1 ] || [ -n "$SUITES" ]; }; then
|
||||
echo "test.sh: --contracts-only is its own mode (no build, no suites). Please consult --help." >&2
|
||||
exit 2
|
||||
fi
|
||||
prev_arg=""
|
||||
|
||||
# Also support --arch=value
|
||||
@@ -152,7 +163,7 @@ for arg in "$@"; do
|
||||
CC=*|CXX=*) export "${arg}" ;;
|
||||
--arch|--arch=*) ;; # already handled
|
||||
arm64|x86_64) ;; # already handled
|
||||
--tsan) ;; # already handled
|
||||
--tsan|--contracts-only) ;; # already handled
|
||||
--suites|--suites=*) ;; # already handled (value skipped via prev_arg below)
|
||||
BUILD_DIR=*) BUILD_DIR="${arg#BUILD_DIR=}"; MAKE_ARGS+=("$arg") ;;
|
||||
SANITIZE=*)
|
||||
@@ -347,6 +358,13 @@ echo "=== Step 0z2: lane selector history replay ==="
|
||||
bash "$ROOT/scripts/test-impact/replay-selector.sh"
|
||||
echo "=== Step 0z3: lane-aware aggregate gates (ci-ok, shard union) ==="
|
||||
bash "$ROOT/tests/test_lane_gate_contract.sh"
|
||||
echo "=== Step 0z4: lane wiring (PRs select, dry run and release run all) ==="
|
||||
bash "$ROOT/tests/test_lane_wiring_contract.sh"
|
||||
|
||||
if [ "$CONTRACTS_ONLY" -eq 1 ]; then
|
||||
echo "=== test.sh: contracts-only — every contract step passed ==="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Verify compiler supports target arch
|
||||
verify_compiler "$CC"
|
||||
|
||||
Executable
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env bash
|
||||
# Lane-wiring contract: the workflows honour the lane selection, and only PRs
|
||||
# are selected at all.
|
||||
#
|
||||
# 1. dry-run.yml and release.yml pass no `lanes` (and no shard profile) to
|
||||
# _test.yml / _lint.yml / _security.yml, whose inputs default to "all":
|
||||
# those runs execute every job exactly as before lane selection.
|
||||
# 2. pr.yml feeds the selector's lanes to every reusable workflow, runs the
|
||||
# docs-only contracts job, and hands ci-ok both the selection and the
|
||||
# run's job list (actions: read) -- ci-ok needs `contracts` but never the
|
||||
# report-only memwaste / shadow jobs.
|
||||
# 3. every lane of scripts/ci/select-lanes.sh --list-lanes is served by a
|
||||
# job: matrix lanes as a "lane":"<name>" tag in the leg data (the tag
|
||||
# ci-ok reads back from the job name), single lanes by an `if:` that
|
||||
# names them. A lane nothing serves would be selected and never run.
|
||||
# Text-level checks (no YAML library on every leg), like the other workflow
|
||||
# contracts.
|
||||
#
|
||||
# Usage: tests/test_lane_wiring_contract.sh [repo-root]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${1:-$(cd "$(dirname "$0")/.." && pwd)}"
|
||||
LANES=$(bash "$ROOT/scripts/ci/select-lanes.sh" --list-lanes)
|
||||
|
||||
python3 - "$ROOT" "$LANES" <<'PY'
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
lanes = sys.argv[2].split()
|
||||
wf = root / ".github" / "workflows"
|
||||
failures = []
|
||||
|
||||
|
||||
def text(name):
|
||||
return (wf / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def jobs(source):
|
||||
"""job id -> body (the lines indented under ` <id>:`)."""
|
||||
out = {}
|
||||
for m in re.finditer(r"(?m)^ ([A-Za-z0-9_-]+):\s*$", source):
|
||||
rest = source[m.end():]
|
||||
end = re.search(r"(?m)^ [A-Za-z0-9_#-]", rest)
|
||||
out[m.group(1)] = rest[:end.start()] if end else rest
|
||||
return out
|
||||
|
||||
|
||||
def require(cond, message):
|
||||
if not cond:
|
||||
failures.append(message)
|
||||
|
||||
|
||||
# 1. dry run and release: no selection, so every job runs.
|
||||
REUSABLE = ("_test.yml", "_lint.yml", "_security.yml")
|
||||
for caller in ("dry-run.yml", "release.yml"):
|
||||
for job, body in jobs(text(caller)).items():
|
||||
called = re.search(r"uses: \./\.github/workflows/(_\w+\.yml)", body)
|
||||
if called and called.group(1) in REUSABLE:
|
||||
require(not re.search(r"(?m)^\s+(lanes|shard_profile):", body),
|
||||
f"{caller} job {job} passes a lane selection to {called.group(1)}; "
|
||||
f"dry runs and releases must run every lane")
|
||||
for callee in REUSABLE:
|
||||
src = text(callee)
|
||||
m = re.search(r"(?ms)^ lanes:\n(.*?)(?=^ \S|^\S|\Z)", src)
|
||||
require(m is not None, f"{callee} must declare a `lanes` input")
|
||||
if m:
|
||||
require(re.search(r"(?m)^ type: string$", m.group(1)) is not None,
|
||||
f"{callee} lanes input must be a string")
|
||||
require(re.search(r"(?m)^ default: all$", m.group(1)) is not None,
|
||||
f"{callee} lanes input must default to all")
|
||||
|
||||
# 2. pr.yml wiring.
|
||||
pr = jobs(text("pr.yml"))
|
||||
changes = pr.get("changes", "")
|
||||
require("scripts/ci/select-lanes.sh --format github-output" in changes,
|
||||
"pr.yml changes must run scripts/ci/select-lanes.sh --format github-output")
|
||||
for key in ("tier", "full", "lanes"):
|
||||
require(re.search(rf"(?m)^ {key}: \$\{{\{{ steps\.select\.outputs\.{key} \}}\}}$", changes),
|
||||
f"pr.yml changes must output {key}")
|
||||
for job in ("security", "lint", "test"):
|
||||
require("lanes: ${{ needs.changes.outputs.lanes }}" in pr.get(job, ""),
|
||||
f"pr.yml {job} must pass the selected lanes")
|
||||
require("scripts/test.sh --contracts-only" in pr.get("contracts", ""),
|
||||
"pr.yml contracts job must run scripts/test.sh --contracts-only")
|
||||
ci_ok = pr.get("ci-ok", "")
|
||||
needs = re.search(r"needs: \[([^\]]*)\]", ci_ok)
|
||||
needs = {n.strip() for n in needs.group(1).split(",")} if needs else set()
|
||||
require({"changes", "security", "lint", "test", "pr-smoke", "contracts"} <= needs,
|
||||
f"ci-ok must need every gating stage incl. contracts, has {sorted(needs)}")
|
||||
require(not needs & {"memwaste", "test-impact-shadow"},
|
||||
"ci-ok must not need the report-only memwaste / shadow jobs")
|
||||
for needle, why in (("LANES: ${{ needs.changes.outputs.lanes }}", "the selection"),
|
||||
("JOBS: ", "the run's job list"),
|
||||
("actions: read", "actions: read to list jobs"),
|
||||
("jobs?filter=latest", "the latest attempt of every job"),
|
||||
("scripts/ci/require-all-green.sh", "the canonical gate")):
|
||||
require(needle in ci_ok, f"ci-ok must use {why} ({needle})")
|
||||
test_wf = jobs(text("_test.yml"))
|
||||
require('--lanes "$LANES"' in test_wf.get("shard-completeness", ""),
|
||||
"shard-completeness must pass the selection to verify-shard-union.sh")
|
||||
|
||||
# 3. every lane is served by a job.
|
||||
MATRIX = {"unix-x86": "_test.yml", "unix-arm64": "_test.yml", "unix-macos14": "_test.yml",
|
||||
"unix-macos-intel": "_test.yml", "windows": "_test.yml", "tsan-x86": "_test.yml",
|
||||
"tsan-arm": "_test.yml", "tsan-mac": "_test.yml", "smoke-ubuntu": "pr.yml",
|
||||
"smoke-mac": "pr.yml", "smoke-win": "pr.yml"}
|
||||
SINGLE = {"lint": ("_lint.yml", "lint"), "lint-mem": ("_lint.yml", "lint-mem"),
|
||||
"license-gate": ("_security.yml", "license-gate"),
|
||||
"codeql-gate": ("_security.yml", "codeql-gate"),
|
||||
"pkg-wrappers": ("_test.yml", "test-package-wrappers"),
|
||||
"diag": ("_test.yml", "test-diag"), "msan": ("_test.yml", "test-msan"),
|
||||
"lsan-macos": ("_test.yml", "test-lsan-macos"),
|
||||
"windows-guards": ("_test.yml", "test-windows-guards"),
|
||||
"memwaste": ("pr.yml", "memwaste"), "contracts": ("pr.yml", "contracts")}
|
||||
ALWAYS = {"security-static": ("_security.yml", "security-static"),
|
||||
"shard-completeness": ("_test.yml", "shard-completeness")}
|
||||
for lane in lanes:
|
||||
if lane in MATRIX:
|
||||
require(f'"lane":"{lane}"' in text(MATRIX[lane]),
|
||||
f"lane {lane}: no leg in {MATRIX[lane]} is tagged with it")
|
||||
elif lane in SINGLE:
|
||||
name, job = SINGLE[lane]
|
||||
body = jobs(text(name)).get(job)
|
||||
cond = re.search(r"(?m)^ if: (.*)$", body or "")
|
||||
require(body is not None and cond is not None and f"'{lane}'" in cond.group(1),
|
||||
f"lane {lane}: {name} job {job} must exist and run only when selected")
|
||||
elif lane in ALWAYS:
|
||||
name, job = ALWAYS[lane]
|
||||
body = jobs(text(name)).get(job)
|
||||
require(body is not None, f"lane {lane}: {name} job {job} must exist")
|
||||
else:
|
||||
failures.append(f"lane {lane}: served by no job this contract knows")
|
||||
|
||||
if failures:
|
||||
print("LANE WIRING CONTRACT VIOLATED:")
|
||||
for failure in failures:
|
||||
print(f" {failure}")
|
||||
sys.exit(1)
|
||||
print(f"lane-wiring contract OK ({len(lanes)} lanes served; dry run and release unselected)")
|
||||
PY
|
||||
@@ -5,10 +5,22 @@ set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
python3 - "$ROOT" <<'PY'
|
||||
# PR smoke runs the legs scripts/ci/select-lanes.sh selects, so the paths the
|
||||
# release-fixture smoke depends on are checked against its real decisions.
|
||||
SMOKE_DECISIONS=""
|
||||
for changed_path in install.sh install.ps1 scripts/smoke-local.sh \
|
||||
scripts/smoke-fixture-server.py scripts/gen-third-party-notices.sh \
|
||||
test-infrastructure/vm/vm-smoke.sh test-infrastructure/vm/windows-user-path-guard.ps1; do
|
||||
decision=$(printf '%s\n' "$changed_path" | bash "$ROOT/scripts/ci/select-lanes.sh")
|
||||
SMOKE_DECISIONS="$SMOKE_DECISIONS$changed_path $decision
|
||||
"
|
||||
done
|
||||
|
||||
python3 - "$ROOT" "$SMOKE_DECISIONS" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
@@ -408,17 +420,11 @@ require(
|
||||
"Phase 14 must assert update leaves the binary byte-identical",
|
||||
)
|
||||
|
||||
for changed_path in (
|
||||
"install\\.(sh|ps1)",
|
||||
"scripts/smoke-local",
|
||||
"scripts/smoke-fixture-server",
|
||||
"scripts/gen-third-party-notices",
|
||||
"test-infrastructure/vm/vm-smoke",
|
||||
"windows-user-path-guard",
|
||||
):
|
||||
for line in sys.argv[2].splitlines():
|
||||
changed_path, decision = line.split(" ", 1)
|
||||
require(
|
||||
changed_path in pr_workflow,
|
||||
f"PR product-change detector must include {changed_path}",
|
||||
{"smoke-ubuntu", "smoke-mac", "smoke-win"} <= set(json.loads(decision)["lanes"]),
|
||||
f"a change to {changed_path} must select all three PR smoke legs",
|
||||
)
|
||||
require(
|
||||
"tests/test_smoke_fixture_contract.sh" in test_driver,
|
||||
|
||||
@@ -392,6 +392,7 @@ scripts/smoke-invariants.sh
|
||||
scripts/ci/preflight-docker.sh
|
||||
scripts/ci/require-all-green.sh
|
||||
scripts/ci/verify-shard-union.sh
|
||||
scripts/ci/select-lanes.sh
|
||||
scripts/ci/generate-sbom.py
|
||||
scripts/package-release.sh
|
||||
scripts/ci/smoke-artifact.sh
|
||||
@@ -436,6 +437,7 @@ test-infrastructure/vm/vm-smoke.sh
|
||||
scripts/smoke-invariants.sh
|
||||
scripts/fuzz.sh
|
||||
scripts/memwaste.sh
|
||||
scripts/ci/select-lanes.sh
|
||||
"
|
||||
for entry in $STRICT_ENTRIES; do
|
||||
out=$(cd "$ROOT" && bash "$entry" --definitely-not-a-flag 2>&1) && rc=0 || rc=$?
|
||||
|
||||
Reference in new issue
Block a user