diff --git a/.codebase-memory.json b/.codebase-memory.json index 448c14f6f..80f8b521f 100644 --- a/.codebase-memory.json +++ b/.codebase-memory.json @@ -14,12 +14,27 @@ }, { "id": "engine", - "paths": [".codebase-memory.json", "src/mcp/test_impact*", "src/store/store_impact.*"], + "paths": [ + ".codebase-memory.json", + "src/mcp/test_impact*", + "src/store/store_impact.*", + "scripts/test-impact/**", + "scripts/ci/test-impact-*" + ], "action": "run_all" }, { "id": "build", - "paths": ["scripts/**", "test-infrastructure/**", "pkg/**"], + "paths": [ + "scripts/build.sh", + "scripts/env.sh", + "scripts/test.sh", + "scripts/run-tests-parallel.sh", + "scripts/suite-failure-report.sh", + "scripts/ci/verify-shard-union.sh", + "scripts/embed-frontend.sh", + "/.gitattributes" + ], "action": "run_all" }, { @@ -35,13 +50,31 @@ }, { "id": "contracts", - "paths": ["tests/**/*.{sh,py,json,txt}"], + "paths": ["tests/**/*.{sh,py,json,txt,ps1}"], "action": "ignore" }, { "id": "notices", "paths": ["NOTICE*", "CHANGELOG*", "THIRD_PARTY*"], "action": "ignore" + }, + { + "id": "repo-data", + "paths": [ + "scripts/**", + "pkg/**", + "test-infrastructure/**", + "docs/**", + "**/*.md", + "graph-ui/**", + "tools/**", + "tests/fuzz/corpus/**", + "/{.clang-format,.clang-tidy,.cppcheck,.gitignore,.gitleaksignore,DCO,Formula}", + "/{flake.lock,flake.nix,glama.json,install.ps1,install.sh,server.json}" + ], + "action": "referencing_tests", + "key": "path", + "unreferenced": "ignore" } ], "unmapped_files": "run_all", diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 0a1f0c488..39baa516c 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -25,7 +25,7 @@ on: type: string default: all test_selection: - description: 'PR test selection (smart CI): the artifact the select-tests job uploaded; the test-unix and test-windows legs run only its tests. Empty = every test (dry run, release, and any PR the selection does not narrow)' + description: 'PR test selection (smart CI): the artifact the select-tests job uploaded; the Linux x86_64 test-unix leg runs only its tests, the arm64, macOS and Windows legs run every test. Empty = every test (dry run, release, and any PR the selection does not narrow)' type: string default: '' @@ -222,7 +222,11 @@ jobs: ccache-test-${{ matrix.os }}-${{ matrix.cc }}- - name: Fetch the test selection - if: ${{ inputs.test_selection != '' }} + # Only the Linux x86_64 leg narrows (user decision 2026-10-16): the + # coverage that justifies an omission comes from the Linux x86_64 + # producer, so arm64, macOS and Windows code under #ifdef has static + # evidence only and those legs run every test. + if: ${{ inputs.test_selection != '' && runner.os == 'Linux' && runner.arch == 'X64' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ inputs.test_selection }} @@ -234,7 +238,7 @@ jobs: CBM_TEST_SHARD: ${{ matrix.shard }} CBM_TEST_LEG: ${{ matrix.os }}-${{ matrix.cc }} CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }} - CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && format('{0}/test-selection', runner.temp) || '' }} + CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && runner.os == 'Linux' && runner.arch == 'X64' && format('{0}/test-selection', runner.temp) || '' }} CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1500M @@ -617,13 +621,6 @@ jobs: ccache-test-${{ matrix.os }}-${{ matrix.msystem }}-${{ github.ref }}- ccache-test-${{ matrix.os }}-${{ matrix.msystem }}- - - name: Fetch the test selection - if: ${{ inputs.test_selection != '' }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ inputs.test_selection }} - path: ${{ runner.temp }}/test-selection - - name: Test shell: msys2 {0} # AddressSanitizer is unavailable on native ARM64 Windows (LLVM ships no @@ -651,7 +648,8 @@ jobs: CBM_TEST_SHARD: ${{ matrix.shard }} CBM_TEST_LEG: ${{ matrix.os }}-${{ matrix.msystem }} CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }} - CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && format('{0}/test-selection', runner.temp) || '' }} + # Windows legs run every test: see the Linux x86_64-only fetch in test-unix. + CBM_TEST_SELECTION_DIR: '' CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1500M diff --git a/docs/SMART_CI.md b/docs/SMART_CI.md new file mode 100644 index 000000000..93d474d34 --- /dev/null +++ b/docs/SMART_CI.md @@ -0,0 +1,504 @@ +# Smart CI: pull-request test selection on GitHub Actions + +Smart CI runs, for each pull request, only the CI jobs and tests that the +change can reach. This page explains how this repository does it and how +another repository on GitHub Actions could do the same with the released +`codebase-memory-mcp` binary. + +Status: experimental. Every answer carries `"experimental": true`. The +selection narrows real test runs only while the repository variable +`TEST_IMPACT_MODE` is `gate`; otherwise it runs in report-only (shadow) mode. + +> **Read the scope section before you start.** The lane layer is a pattern +> any repository can copy. The test layer narrows only C/C++ code whose tests +> follow this repository's test-framework conventions (or declared +> equivalents), and only with a per-test coverage map that you must produce +> yourself. In any other repository the engine answers "run everything". That +> is safe, but it saves nothing. + +## The guarantee + +Smart CI never runs fewer tests unless it can show that the omitted tests +cannot be affected. When it is unsure, it runs everything: + +- a path that no rule knows, an empty or truncated file list, or a change to + the selection's own configuration, tooling or workflows; +- a shallow clone, a diff that cannot be read in full, or an invalid config; +- no coverage map, or a coverage map that is not admitted (wrong commit, more + than 7 days old, made for another platform or harness); +- a change that reaches the test runner's `main`; +- any failure of the selection job itself. + +Each of these makes the run cost more CI minutes. None of them loses coverage. + +## Two layers + +``` +changed files ──► Layer 1: lane selector ──► which CI jobs run + (scripts/ci/select-lanes.sh, your own rules) + │ + ▼ (Linux test legs only, in this repo) +merge base + PR ─► Layer 2: test-impact engine ─► which suites / tests run + (codebase-memory-mcp test-impact select + + team-artifact bundle from main) +``` + +| | Layer 1: lanes | Layer 2: tests | +|---|---|---| +| Question | Which jobs (lint, test legs, smoke, sanitizers) does this PR need? | Inside a test job, which suites and tests? | +| Input | The PR's changed-file list | Git history, the code graph, a per-test coverage map | +| Implementation | `scripts/ci/select-lanes.sh` (repository-specific rules) | The binary's `test-impact select` subcommand | +| Unknown input | Every lane runs | Every test runs (`run_all` with reasons) | +| Portable? | As a pattern: you write your own rules | C/C++ with supported test conventions only (see below) | + +## Can your repository use it? + +| Piece | General? | What you provide | +|---|---|---| +| Lane selection (Layer 1) | Pattern only | Your own path → category → lanes rules. `select-lanes.sh` encodes this repository's paths. | +| Engine (`test-impact select`) | Ships in the binary | Nothing to build. | +| Source languages it can narrow | **C family only**: `.c .h .cc .cpp .hpp .m .mm .cxx .hh` | Every other changed file must match a config rule (`ignore`, `referencing_tests`, ...). Otherwise the answer is `run_all` (`UNMAPPED_FILE`). | +| Test model | **C family only** | The `c-cbm` preset (this repository's `TEST`/`SUITE`/`RUN_TEST`/`RUN_SELECTED_SUITE` macros, see `tests/test_framework.h`), or custom `tests.conventions` records for equivalent C macros. | +| Coverage map | **Required to narrow at all** | Format-1 files (`functions.tsv`, `tests.tsv`, `meta.json`). This repository produces them with an LLVM-instrumented runner and `scripts/test-impact/coverage-map.py`, which are both specific to its harness. | +| Runner filter | Pattern | A test runner that can run a `suite` / `suite:test` list and fails when a listed test does not exist. | + +The other presets (`pytest`, `go`, `junit`, `gtest`, `rust`, `jest`) are +accepted switches, but the engine does not model them. An enabled `gtest` +blocks narrowing for C++ files. Custom `tests.conventions` are covered by the +model's unit tests. The engine's end-to-end tests use `c-cbm` only. Treat +custom conventions as unproven until you have shadow evidence. + +Without admitted coverage the engine marks every suite `whole`, and the lane +reports `run_all`. Static reachability alone never omits a suite. + +## Prerequisites + +1. **The binary, pinned.** Use the same release in the producer and the PR + jobs. Download that release's Linux archive and verify it against the + release's `checksums.txt`, as this repository's shadow job does + (`scripts/ci/test-impact-shadow.sh --fetch-release`): + + ```bash + CBM_VERSION=vX.Y.Z # a release that contains the features you use (see below) + asset=codebase-memory-mcp-linux-amd64.tar.gz + gh release download "$CBM_VERSION" --repo DeusData/codebase-memory-mcp \ + --pattern "$asset" --pattern checksums.txt --dir "$RUNNER_TEMP/cbm" + (cd "$RUNNER_TEMP/cbm" && grep " $asset\$" checksums.txt | sha256sum -c - && + tar -xzf "$asset" codebase-memory-mcp) + "$RUNNER_TEMP/cbm/codebase-memory-mcp" --version + ``` + + For a quick start, `install.sh` installs the latest release with the same + checks (HTTPS only, SHA-256 against `checksums.txt`, the archive's member + list, `--version`); `--skip-config` skips agent configuration, which a CI + runner does not need. It has no version flag, so it is not a pin. + + **Version note.** `v0.12.0` has `test-impact select`/`publish`. It does + not have `key: "path"`, `unreferenced`, `--vetoed-out` or `run_all_rules`; + they ship in the first release after `v0.12.0`. With `v0.12.0`, a config + that uses those keys is invalid, and every answer is `run_all`. Until that + release, build the binary from source (`make -f Makefile.cbm cbm`), as this + repository's producer and `select-tests` jobs do. + +2. **Full git history.** Check out with `fetch-depth: 0` in every job that + runs the engine. The engine verifies the history it diffs and refuses a + shallow clone (`GIT_UNAVAILABLE`, everything runs). + +3. **A runner that runs a named subset.** See "The test job" below. + +4. **A coverage producer on the default branch** that writes the format-1 map + for every test, child processes included. A test whose coverage is + uncertain must be marked `incomplete`. An incomplete test is always + selected. + +5. **`gh`, `python3`, and `GH_TOKEN` with `actions: read`** in the PR job, to + fetch the bundle. GitHub-hosted runners include `gh` and `python3`. + +## Recipe + +Copy these files from this repository and adapt the marked lines: + +| File | Adapt | +|---|---| +| `scripts/ci/test-impact-bundle.sh` | `WORKFLOW=` (producer file name) and the hard-coded `main` branch filter. | +| `scripts/ci/test-impact-select.sh` | `--lane unit` (your lane name) and the `PLATFORM` default (must equal the producer's). | +| `scripts/test-impact/selection.py` | Nothing, if your runner accepts `suite` / `suite:test` lines. | + +### (a) Producer: publish a bundle for every commit on main + +The bundle holds the commit's frozen graph, its coverage map and a +`receipt.json` that binds both by SHA-256. It is uploaded as the Actions +artifact `test-impact-`. `test-impact-bundle.sh` accepts a bundle only +from a successful `push`, `schedule` or `workflow_dispatch` run of the +producer workflow, in the same repository, on `main`, whose head is exactly +the requested commit. + +```yaml +# .github/workflows/test-impact-artifact.yml (the name test-impact-bundle.sh looks for) +name: test-impact-artifact +on: + push: + branches: [main] + schedule: + - cron: "23 3 * * 0" # weekly full refresh + workflow_dispatch: +permissions: + contents: read + actions: read # fetch the previous bundle (incremental mode) +concurrency: + group: test-impact-artifact # bundles chain: never overlap, never cancel + cancel-in-progress: false +jobs: + publish: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@ + with: + fetch-depth: 0 + persist-credentials: false + - name: Install codebase-memory-mcp + run: + - name: Per-test coverage map # YOURS: format 1, see coverage-map.py + run: | + echo "OBSERVED=$(date +%s)" >> "$GITHUB_ENV" # when the coverage run starts + ./ci/coverage-map.sh --out "$RUNNER_TEMP/coverage" + - name: Publish the bundle + run: | + export CBM_CACHE_DIR="$RUNNER_TEMP/ti-cache" + export CBM_RUNTIME_DIR="$(mktemp -d /tmp/cbm-rt.XXXXXX)" # short: sun_path + mkdir -p "$RUNNER_TEMP/ti-work" && chmod 700 "$RUNNER_TEMP/ti-work" + "$RUNNER_TEMP/cbm/codebase-memory-mcp" test-impact publish \ + --repo "$GITHUB_WORKSPACE" --out "$RUNNER_TEMP/bundle" \ + --work "$RUNNER_TEMP/ti-work" --coverage "$RUNNER_TEMP/coverage" \ + --observed-at "$OBSERVED" --platform linux-x86_64-clang21 + - uses: actions/upload-artifact@ + with: + name: test-impact-${{ github.sha }} + path: ${{ runner.temp }}/bundle + include-hidden-files: true # the graph lives in .codebase-memory/ + retention-days: 30 + if-no-files-found: error +``` + +This repository's producer (`.github/workflows/test-impact-artifact.yml` with +`scripts/ci/test-impact-publish.sh`) adds an incremental mode. A push re-runs +coverage only for the suites that its own selection picks and merges them into +the previous map (`coverage-merge.py`). The graph is built incrementally from +the previous bundle (`--previous`). The weekly run rebuilds everything and +fails if the incremental graph topology differs from the full one. The +incremental mode is an optimisation. A full coverage run on every push is +simpler and just as correct. + +The receipt keeps the time of the oldest coverage observation. An incremental +publish carries it forward unchanged. A map whose oldest observation is more +than 7 days old is refused (`TOO_OLD`), so a full refresh has to run more often +than that. + +### (b) PR job, shadow mode (report only) + +```yaml + test-impact-shadow: + runs-on: ubuntu-latest + continue-on-error: true # can never block or pass a merge + permissions: + contents: read + actions: read # the merge base's bundle + steps: + - uses: actions/checkout@ + with: + fetch-depth: 0 + persist-credentials: false + - name: Install codebase-memory-mcp + run: + - name: Select with the team artifact (shadow) + env: + GH_TOKEN: ${{ github.token }} + run: scripts/ci/test-impact-select.sh --binary "$RUNNER_TEMP/cbm/codebase-memory-mcp" --out "$RUNNER_TEMP/test-impact" + - uses: actions/upload-artifact@ + if: ${{ always() }} + with: + name: test-impact-prediction + path: | + ${{ runner.temp }}/test-impact/selection-engine.json + ${{ runner.temp }}/test-impact/summary-engine.md +``` + +Leave this job out of your aggregate required check. What +`test-impact-select.sh` does: + +1. Takes the base side of the PR merge commit (`HEAD^1`) and fetches that + commit's bundle. +2. If the producer skipped that commit (merge bursts: GitHub keeps one pending + run per concurrency group), it uses the nearest first-parent ancestor that + has a bundle, up to 20 commits back, and diffs against that ancestor. If the + PR reverts a change that main made after the ancestor, the ancestor's map + cannot speak for that file, so the script selects without coverage (the + revert guard). +3. Passes `--artifact-verified` only for a bundle that it fetched and checked + itself. Nothing inside a bundle can assert where the bundle came from. +4. Runs `test-impact select` and writes `selection-engine.json`. It also + writes a Markdown summary to the job's step summary. + +The engine reads `.codebase-memory.json` from the **merge base**, not from the +PR. A PR cannot loosen its own selection. A PR that changes the config runs +everything (`builtin:test-impact-config`). + +### (c) Gate job and the test job + +```yaml + select-tests: + if: ${{ vars.TEST_IMPACT_MODE == 'gate' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: read + outputs: + selection: ${{ steps.select.outputs.selection }} + steps: + - uses: actions/checkout@ + with: + fetch-depth: 0 + persist-credentials: false + - name: Install codebase-memory-mcp + run: + - id: select + env: + GH_TOKEN: ${{ github.token }} + run: scripts/ci/test-impact-select.sh --binary "$RUNNER_TEMP/cbm/codebase-memory-mcp" --out "$RUNNER_TEMP/test-impact" --gate "$RUNNER_TEMP/test-selection" + - if: ${{ steps.select.outputs.selection == 'narrowed' }} + uses: actions/upload-artifact@ + with: + name: test-selection + path: ${{ runner.temp }}/test-selection + if-no-files-found: error + + test: + needs: [select-tests] + if: ${{ !cancelled() }} # a skipped or failed select-tests must NOT skip the tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ + - if: ${{ needs.select-tests.result == 'success' && needs.select-tests.outputs.selection == 'narrowed' }} + uses: actions/download-artifact@ + with: + name: test-selection + path: ${{ runner.temp }}/test-selection + - run: ./ci/run-tests.sh # YOURS: narrows only if test-only.txt is present +``` + +With `--gate DIR`, the script runs `selection.py filter … --lane unit` and +writes `selection=narrowed` or `selection=all` to `$GITHUB_OUTPUT`. A narrowed +answer writes: + +- `DIR/test-only.txt`: one token per line, either `suite` (run the whole + suite) or `suite:test`; +- `DIR/optional-suites.txt`: selected suites that are registered only under a + preprocessor condition. A build may lack them. + +Everything else means `all`: a `run_all` answer, an unknown schema or filter +encoding, a filter that disagrees with the lane's suite list, a crash, and a +lane that selects nothing. The test job then gets no selection and runs every +test. + +**The test job's runner** has to keep the same contract as this repository's +runner (`scripts/test.sh` → `scripts/run-tests-parallel.sh` → +`CBM_TEST_ONLY_FILE`, read by `tests/test_main.c`): + +- launch only the suites that the tokens name, and only the named tests inside + a `suite:test` suite; +- **fail** when the selection names a suite the build does not register, + unless `optional-suites.txt` lists it; +- **fail** when a selected test cannot be found; +- **fail** when a selection was promised but the file is missing or empty. + +A selection that is silently ignored, or partly ignored, would turn a narrowing +into missed coverage. + +**A runner that starts itself again.** Tests often spawn the test binary as a +child (` __probe_x ...`) and its `main` hands over to a probe. A change +that reaches `main` reaches every test, so the engine runs everything +(`RUNNER_REACHED`). List the probes instead in a table in the runner's source, +one entry per line, as `tests/test_main.c` does: + +```c +TF_ROLE(TF_PHASE_SEAMED, "__cbm_sockprobe", tf_maybe_run_socket_probe), +TF_ROLE(TF_PHASE_WORKER, "--index-worker", tf_maybe_run_index_worker), +TF_ROLE(TF_PHASE_SPAWNER, "CBM_INDEX_SUPERVISOR", tf_maybe_run_index_worker), +``` + +The engine reads the table: a change that only a probe's handler reaches +selects the tests whose source names one of that handler's literals (the argv +the child gets, or a switch that lets production code start it). A reached +handler that no test names, and a change reaching `main` any other way, still +run everything; the engine note names the call chain into `main`. + +**Only the legs the coverage speaks for should narrow.** Here the coverage +comes from one Linux x86_64 clang-21 producer. Only the Linux x86_64 test leg +downloads the selection (`runner.os == 'Linux' && runner.arch == 'X64'` in +`_test.yml`). The arm64, macOS and Windows legs always run everything, +because code under architecture or platform `#ifdef`s has static evidence +only there. + +### (d) Flip the variable + +Shadow is the default: the variable is unset, `select-tests` is skipped, and +every test runs. When the evidence is in, set the repository variable: + +```bash +gh variable set TEST_IMPACT_MODE --body gate # gate +gh variable delete TEST_IMPACT_MODE # back to shadow +``` + +Do not add `select-tests` to your required checks or to the aggregate gate. +When it breaks, PRs cost more minutes. They do not lose coverage. + +### Optional: Layer 1 (lanes) + +`scripts/ci/select-lanes.sh` is this repository's lane rule set. A `changes` +job lists the PR's files (the `pulls/{n}/files` API, with a git fallback), +runs the selector, and writes `tier`, `full`, `lanes` (a JSON list) and +`reasons` to `$GITHUB_OUTPUT`. Each lane job then runs +`if: contains(fromJSON(needs.changes.outputs.lanes), '')`. A single +aggregate job (`ci-ok`, `scripts/ci/require-all-green.sh`) is the only required +check. It fails when a *selected* lane was skipped, cancelled or missing. + +For your own version, keep these properties: + +- An unknown path, an empty list, workflow and harness changes, and a list + truncated at the API's 3000-file cap all select every lane. +- The selector runs from a checkout without credentials, after the step that + holds the token. +- Replay your PR history through the selector and check that every real + failure had its catching lane selected. This repository's equivalent is + `scripts/test-impact/replay-selector.sh`. + +## Configuration: `.codebase-memory.json` → `test_impact` + +| Key | Values | Default | +|---|---|---| +| `version` | `1` (required) | — | +| `tests.presets` | `{"c-cbm": true, ...}`. Unknown names are an error. | `c-cbm` off, the six others on | +| `tests.conventions` | C-family macro records: `case`, `suite`, `registration`, `suite_registration` | none | +| `rules` | ordered list, see below | none (built-ins only) | +| `lanes` | list of `{name, suites, exclude_suites?, default: "run"\|"skip", narrow?}` | one lane `unit`: all suites except `@perf`, `narrow: true` | +| `unmapped_files` | `"run_all"` (the only accepted value) | `run_all` | +| `coverage.compatibility_paths` | repo-relative files that the coverage map depends on (harness, runner, build flags) | none | + +**Rules.** Each rule is `{id, paths, action, ...}`. `paths` are gitignore-style +globs with `{a,b}` alternatives. A pattern with a `/` is anchored at the +repository root. Without a `/`, it matches at any depth. `**` matches across +directories. Project rules come first, then the built-ins. For each changed +path, **the first matching rule wins**, so order matters. + +| `action` | Effect | +|---|---| +| `run_all` | Any match runs everything. The summary names the rule (`run_all_rules`). | +| `ignore` | The path selects nothing. | +| `referencing_tests` | Selects the tests whose source contains the path as text. The engine tries the path itself, then its parent directories down to the `key`, each with and without its first segment. `key: "path_segment:N"` stops at segment N (zero-based). `key: "path"` tries the exact path only. If no test names the path, everything runs, unless `unreferenced: "ignore"` is set. | +| `suites` | Runs the named suites whole. | +| `lanes` | Activates the named lanes (for lanes with `default: "skip"`). | + +Built-in rules, applied after yours: + +| id | paths | action | +|---|---|---| +| `builtin:test-impact-config` | `.codebase-memory.json` | `run_all` | +| `builtin:build` | `Makefile*`, `CMakeLists.txt`, `*.cmake`, `meson.build`, `BUILD*`, `WORKSPACE`, `package*.json`, `go.mod`, `go.sum`, `Cargo.*`, `pom.xml`, `build.gradle*`, `pyproject.toml`, `setup.*`, `requirements*.txt` | `run_all` | +| `builtin:ci` | `.github/**`, `.gitlab-ci.yml`, `.circleci/**`, `Jenkinsfile*`, `.buildkite/**`, `azure-pipelines*.yml` | `run_all` | +| `builtin:vendor` | `vendor*/**`, `third_party/**` | `run_all` | +| `builtin:conftest` | `conftest.py` | `run_all` | +| `builtin:documentation` | `docs/**`, `**/*.md`, `LICENSE*` | `ignore` | + +**Lanes.** A lane that you declare yourself does **not** narrow unless you set +`"narrow": true`. `signals` is accepted but has no effect yet. + +**This repository's file, explained:** + +| Rule | Why | +|---|---| +| `harness` → `run_all` | The test framework and the runner's `main` touch every suite. | +| `engine` → `run_all` | The selection's own code and config. A change to them could loosen what is selected. | +| `build` → `run_all` | Build and runner scripts: the file list cannot tell a flag change from a harmless one. `/.gitattributes` is anchored at the root. | +| `grammars` → `run_all` | Vendored parsers are not modelled per test. | +| `fixtures` → `referencing_tests`, `path_segment:2` | `tests/fixtures//...` selects the tests that mention the fixture, down to `tests/fixtures/`. | +| `contracts` → `ignore` | Shell, Python and data contract tests run in every leg's contract step, which is never narrowed. | +| `notices` → `ignore` | Release notices. | +| `repo-data` → `referencing_tests`, `path`, `unreferenced: ignore` | Scripts, docs and packaging. A C test that reads one of these files is selected. A file that no test names selects nothing. | + +`coverage.compatibility_paths` lists the harness and coverage tools. Changing +the bytes of any of them, or the `--platform` label, makes the existing coverage +map incompatible until the producer publishes a new one. + +## Rollout and evidence + +1. **Shadow.** Run (a) and (b). Leave `TEST_IMPACT_MODE` unset. +2. **Measure.** For each PR run, compare `selection-engine.json` with what + failed in that run's full test legs. + - A **miss** is a test that failed in the full run but was not selected. + One unexplained miss blocks the flip. + - **Savings** are the suites and tests that were not selected. + + Nothing in this repository computes misses for Layer 2 yet. Layer 1's + history replay (`replay-selector.sh`) is the model to follow. +3. **Gate.** Set `TEST_IMPACT_MODE=gate`. Keep the shadow job running, so that + the evidence keeps coming. + +**Reading the step summary** (`### test-impact engine`): + +| Line | Meaning | +|---|---| +| `decision: \`selected\`` / `\`run_all\`` / `\`nothing\``, then `run-all reasons [...]` | The top-level answer and why everything runs. `decided by rule(s) [...]` names the matched `run_all` rule ids. | +| `team artifact: fetched (ancestor …)` / `none`; `coverage \`admitted\`` / `\`rejected\` [reasons]` / `\`unavailable\`` | Whether a bundle was used, and whether its coverage was admitted. | +| `engine note: …` | The engine's first diagnostic line, such as the path that caused `UNMAPPED_FILE`, or the call chain into the runner's `main` for `RUNNER_REACHED`. | +| `lane \`unit\` (selected): N whole [REASON n, …], M narrowed (K tests)` | Per lane: suites that run whole (with a histogram of reasons), and suites narrowed to K tests. | +| `the runner-main veto (RUNNER_REACHED) discarded this selection:` | The change reached the runner's `main`, so everything runs. The lines below show what would have been selected otherwise. | +| `PR CI runs N selection tokens` / `PR CI runs every test (selection exit N)` | Gate mode only: what the test legs received. | + +Common run-all reasons: + +| Reason | Cause | +|---|---| +| `RULE_RUN_ALL` | A `run_all` rule matched (see `run_all_rules`). | +| `UNMAPPED_FILE` | A changed file matched no rule and is not C-family source. | +| `RULE_TARGET_UNKNOWN` | A `referencing_tests` path that no test names (without `unreferenced: "ignore"`). | +| `RUNNER_REACHED` | The change reaches the test runner's `main` (or a child role none of whose literals a test names). | +| `GIT_UNAVAILABLE` | No git, a shallow clone, or a diff that cannot be read in full. | +| `CONFIG_INVALID` / `POLICY_UNAVAILABLE` | The merge base's config or rules do not parse. | + +Reasons per suite: `ARTIFACT_REJECTED` (no admitted coverage), `SETUP_HIT` / +`SETUP_UNKNOWN` (the suite's between-test setup ran changed code, or its +coverage is incomplete), `INVENTORY_UNKNOWN` (the suite cannot be read with +certainty), `CONDITIONAL` (`#if`-registered), `STATIC` / `CHANGED` / `COVERAGE` +/ `RULE` (the evidence that selected it). + +## Failure modes + +Every row below costs CI minutes, not coverage. + +| What happens | Result | +|---|---| +| No bundle for the merge base | Nearest bundled ancestor, up to 20 first-parent commits back. Otherwise no coverage, and everything runs. | +| Bundle fetch fails (token, API) | Warning, no coverage, and everything runs. | +| The PR reverts a change that main made after the ancestor | Revert guard: no coverage, and everything runs. | +| Coverage older than 7 days, or a platform/compatibility mismatch | Map refused: every suite runs whole. | +| Shallow checkout | `GIT_UNAVAILABLE`: everything runs. | +| Engine deadline (default 1800 s) or crash | The job fails. The test job gets no selection, so everything runs. | +| `select-tests` skipped (shadow) or failed | Everything runs (`!cancelled()` on the test job). | +| The selection names a suite or test that the build lacks | The test run **fails**: a plumbing fault, reported, never skipped. | + +## Known limits + +- Narrowing covers C-family source and C test conventions only. Everything + else is decided by rules or runs everything. +- The coverage map format and its producer come from this repository's LLVM + toolchain and harness. You need your own equivalent. +- One producer platform. The selection narrows only the leg that matches the + producer's platform; every other leg runs everything. Narrowing more legs + needs a producer (and a bundle) per platform. +- The weekly full refresh and the 7-day age limit are the same length. Each + week, when the refresh runs late, there can be a short window in which maps + are refused and every test runs. +- In merge bursts the producer skips commits. The ancestor fallback covers up + to 20 of them. +- The answer schema is `cbm.test_impact.v0` and is marked experimental. It may + change between releases. Pin the binary. diff --git a/scripts/ci/select-lanes.sh b/scripts/ci/select-lanes.sh index 04abb1880..bd25e9761 100755 --- a/scripts/ci/select-lanes.sh +++ b/scripts/ci/select-lanes.sh @@ -123,20 +123,53 @@ BUILD = {"Makefile.cbm", "scripts/test.sh", "scripts/run-tests-parallel.sh", "scripts/env.sh", "scripts/build.sh", "scripts/clean.sh", "scripts/path-safety.sh", "scripts/test-runtime.sh", "scripts/msan.sh", "scripts/lint.sh", ".clang-format", ".clang-tidy", - "scripts/vendored-checksums.txt"} + "scripts/vendored-checksums.txt", "scripts/run-test-wave.py", + "scripts/embed-frontend.sh"} SMOKE = {"scripts/smoke-test.sh", "scripts/smoke-local.sh", "scripts/smoke-fixture-server.py", "scripts/smoke-invariants.sh", "test-infrastructure/vm/vm-smoke.sh", "test-infrastructure/vm/windows-user-path-guard.ps1", - "scripts/test-windows.ps1", "scripts/smoke-wrappers.sh"} + "scripts/test-windows.ps1", "scripts/smoke-wrappers.sh", + "scripts/security-fuzz.sh", "scripts/security-fuzz-random.sh", + "scripts/security-install.sh", "scripts/security-network.sh", + "scripts/security-strings.sh", "scripts/test_mcp_interactive.py"} LINT_CONFIG = {"scripts/memory-core-baseline.txt", "scripts/lint-memory-core.py", - "scripts/hooks/pre-commit"} + "scripts/hooks/pre-commit", ".cppcheck", "scripts/check-no-test-skips.sh", + "scripts/check-nolint-whitelist.sh", "scripts/lint-mem-gate.py", + "scripts/lint-mem-whitelist.txt", "scripts/lint-tidy-diff.sh"} +# Local tools and release-only jobs: no PR lane runs them. NON_GATING = {"scripts/package-release.sh", "scripts/benchmark-index.sh", "scripts/benchmark-search-graph.sh", "scripts/benchmark-streaming.py", - "scripts/soak-test.sh", ".cbmignore", ".node-version", "flake.nix"} -PKG_FILES = {"install.sh", "install.ps1", "server.json", "package.json", "manifest.json"} + "scripts/soak-test.sh", ".cbmignore", ".node-version", "flake.nix", + "flake.lock", "scripts/soak-legs.sh", "scripts/repro.sh", + "scripts/audit-grammar-security.sh", "scripts/check-dco.sh", + "scripts/check-lsp-originality.sh", "scripts/clone-bench-repos.sh", + "scripts/extract_nomic_vectors.py", "scripts/gen-pdf-unicode.py", + "scripts/gen-py-stdlib.py", "scripts/generate-lang-code.py", + "scripts/new-languages.json", "scripts/git-hooks/commit-msg", + "scripts/install-git-hooks.sh", "scripts/sanitizer-coverage-map.py", + "scripts/sanitizer-map.json", "scripts/test_mcp_rapid_init.py", + "scripts/vendor-grammar.sh"} +# The license-gate lane runs these (and only that lane). +LICENSE_TOOLS = {"scripts/license-gate.sh", "scripts/license-gate-check.py", + "scripts/license-gate-check-npm.py", "scripts/license-policy.json", + "scripts/ui_browser_licenses.py"} +# security-static runs these; it runs on every PR (ALWAYS). +SECURITY_TOOLS = {"scripts/security-audit.sh", "scripts/security-allowlist.txt", + "scripts/security-network-source-audit.py", "scripts/security-ui.sh"} +# Repository files C tests read: the test legs run (a docs-only tier runs no C +# suite, so test_cli's README/site contract would never see a README change). +READ_BY_TESTS = {"scripts/security-vendored.sh", ".codebase-memory.json", "README.md", + "docs/index.html", "docs/llms.txt"} +# The test legs' failure report (tests/test_failure_report_contract.sh). +TEST_REPORT = {"scripts/suite-failure-report.sh"} +# The release notices ship inside every archive. +PKG_SCRIPTS = {"scripts/gen-third-party-notices.sh", "scripts/gen-ui-licenses.py"} +PKG_FILES = {"install.sh", "install.ps1", "server.json", "package.json", "manifest.json", + "Formula"} DOC_SUFFIXES = (".md", ".png", ".svg", ".jpg", ".gif", ".txt") -DOC_FILES = {"LICENSE", "CODEOWNERS", ".gitignore", ".gitattributes"} +DOC_FILES = {"LICENSE", "CODEOWNERS", ".gitignore", ".gitattributes", "DCO", + ".gitleaksignore", "glama.json"} def category(path): @@ -169,6 +202,18 @@ def category(path): return "build" if path in SMOKE: return "smoke" + if path in LICENSE_TOOLS: + return "license-tools" + if path in SECURITY_TOOLS: + return "security-tools" + if path in READ_BY_TESTS: + return "test" + if path in TEST_REPORT: + return "test-contract" + if path in PKG_SCRIPTS: + return "pkg" + if path.startswith("scripts/") and path.endswith(".md"): + return "docs" if path.startswith("tests/fixtures/"): return "test" if path.startswith("tests/windows/"): @@ -213,7 +258,8 @@ MID = {"pipeline", "mcp", "store", "cypher", "discover", "git", "graph_buffer", PLATFORM = {"daemon", "cli", "watcher", "ui", "foundation", "smoke", "pkg", "test-win"} CORE = {"mem-core", "build", "vendored", "ci-gate", "other", "scripts-other"} LOWER = {"test", "test-contract", "fuzz", "ui-frontend", "ci-other", - "local-infra", "docs", "lint-cfg", "memwaste-tools"} + "local-infra", "docs", "lint-cfg", "memwaste-tools", "license-tools", + "security-tools"} def normalized(cat): @@ -240,7 +286,8 @@ def tier_of(cats): return "T2-extract" if cats & {"test", "test-contract", "fuzz"}: return "T1-tests" - if cats & {"ui-frontend", "ci-other", "local-infra", "lint-cfg", "memwaste-tools"}: + if cats & {"ui-frontend", "ci-other", "local-infra", "lint-cfg", "memwaste-tools", + "license-tools", "security-tools"}: return "T0b-nonproduct" return "T0a-docs" @@ -270,7 +317,7 @@ R1 = { # because a pkg path is never docs-only (it lifts the tier to T4). ADD_ONS = [ ({"pkg"}, {"pkg-wrappers", "smoke-ubuntu", "smoke-mac", "smoke-win"}), - ({"vendored", "pkg", "ci-gate", "build"}, {"license-gate"}), + ({"vendored", "pkg", "ci-gate", "build", "license-tools"}, {"license-gate"}), ({"ui-frontend"}, {"windows-guards", "smoke-ubuntu"}), ({"memwaste-tools", "mem-core"}, {"memwaste"}), ({"test-win"}, {"windows", "windows-guards"}), diff --git a/scripts/ci/test-impact-select.sh b/scripts/ci/test-impact-select.sh index 39208683e..175c1fdb6 100755 --- a/scripts/ci/test-impact-select.sh +++ b/scripts/ci/test-impact-select.sh @@ -15,7 +15,8 @@ # bundle is fetched with scripts/ci/test-impact-bundle.sh, which only accepts # the main-branch producer's own run for exactly that commit; only then does # the selection pass --artifact-verified. Without a bundle the engine builds -# the graph itself and admits no coverage: every reached suite runs whole. +# the graph itself and admits no coverage: every suite runs whole (the lane +# then reports run_all; the summary shows why each suite runs whole). # # Usage: scripts/ci/test-impact-select.sh --binary PATH --out DIR [--base REV] # [--platform LABEL] [--gate DIR] @@ -26,6 +27,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" BINARY="" OUT="" BASE="HEAD^1" PLATFORM="linux-x86_64-clang21" GATE="" +ANCESTOR_MAX=20 # first-parent commits before the merge base whose bundle may stand in while [ $# -gt 0 ]; do case "$1" in --binary) BINARY="$2"; shift 2 ;; @@ -48,32 +50,96 @@ base="$(git -C "$ROOT" rev-parse "$BASE")" artifact=() bundle_rc=0 +select_base="$base" # the commit the selection diffs against: the bundle's own +ancestor_note="" "$ROOT/scripts/ci/test-impact-bundle.sh" --fetch "$base" "$WORK/bundle" || bundle_rc=$? +if [ "$bundle_rc" -eq 3 ]; then + # The producer skips main commits in merge bursts (GitHub keeps one pending + # run per concurrency group). Use the nearest first-parent ancestor A with + # a bundle instead, and diff against A: A..HEAD carries main's own commits + # since A as well as the PR, so A's coverage map stays exact for its base. + for candidate in $(git -C "$ROOT" rev-list --first-parent --max-count="$((ANCESTOR_MAX + 1))" "$base" | tail -n +2); do + rm -rf "$WORK/bundle" + bundle_rc=0 + "$ROOT/scripts/ci/test-impact-bundle.sh" --fetch "$candidate" "$WORK/bundle" || bundle_rc=$? + if [ "$bundle_rc" -ne 3 ]; then + [ "$bundle_rc" -eq 0 ] && select_base="$candidate" + break + fi + done + if [ "$select_base" != "$base" ]; then + # Exact-revert guard: a file main changed after A that the PR changes + # back drops out of A..HEAD. Then A's map cannot speak for it: no coverage. + missing=$(comm -23 <(git -C "$ROOT" diff --no-renames --name-only "$base" HEAD | sort -u) \ + <(git -C "$ROOT" diff --no-renames --name-only "$select_base" HEAD | sort -u)) + distance=$(git -C "$ROOT" rev-list --first-parent --count "$select_base..$base") + if [ -n "$missing" ]; then + echo "::warning::test-impact: ancestor bundle $select_base unusable (the change reverts main's own change in: $(echo "$missing" | head -3 | tr '\n' ' ')): selecting without coverage" + ancestor_note="ancestor $select_base rejected by the revert guard" + select_base="$base" + bundle_rc=3 + else + ancestor_note="ancestor $select_base, $distance first-parent commits before the merge base" + fi + fi +fi if [ "$bundle_rc" -eq 0 ]; then - artifact=(--artifact "$WORK/bundle" --artifact-commit "$base" --artifact-verified) + artifact=(--artifact "$WORK/bundle" --artifact-commit "$select_base" --artifact-verified) elif [ "$bundle_rc" -ne 3 ]; then # No bundle means no coverage: more tests run, never fewer. Say so. echo "::warning::test-impact bundle fetch failed (exit $bundle_rc): selecting without coverage" fi started=$(date +%s) -"$BINARY" test-impact select --repo "$ROOT" --base "$base" --work "$WORK/engine" \ - --platform "$PLATFORM" ${artifact[@]+"${artifact[@]}"} > "$OUT/selection-engine.json" +rm -f "$OUT/selection-vetoed.json" +select_rc=0 +"$BINARY" test-impact select --repo "$ROOT" --base "$select_base" --work "$WORK/engine" \ + --platform "$PLATFORM" ${artifact[@]+"${artifact[@]}"} \ + --vetoed-out "$OUT/selection-vetoed.json" > "$OUT/selection-engine.json" \ + 2> "$WORK/select-stderr.txt" || select_rc=$? +cat "$WORK/select-stderr.txt" >&2 # the engine's diagnostic stays in the log too +[ "$select_rc" -eq 0 ] || exit 1 elapsed=$(( $(date +%s) - started )) -python3 - "$OUT/selection-engine.json" "$elapsed" "${#artifact[@]}" > "$OUT/summary-engine.md" <<'PY' -import json, sys +python3 - "$OUT/selection-engine.json" "$elapsed" "${#artifact[@]}" "$OUT/selection-vetoed.json" \ + "$ancestor_note" "$WORK/select-stderr.txt" > "$OUT/summary-engine.md" <<'PY' +import collections, json, os, sys answer = json.load(open(sys.argv[1])) coverage = answer.get("receipt", {}).get("coverage", {}) + + +def lane_lines(doc, prefix): + out = [] + for lane in doc.get("lanes") or []: + suites = lane.get("suites") or [] + whole = [s for s in suites if s.get("mode") == "whole"] + narrowed = {s["suite"]: len(s.get("tests") or []) for s in suites if s.get("mode") == "tests"} + why = collections.Counter(r for s in whole for r in (s.get("reasons") or [])) + hist = ", ".join(f"{name} {n}" for name, n in why.most_common()) + because = (lane.get("because") or {}).get("reasons") or [] + out.append(f"{prefix}lane `{lane.get('lane')}` ({lane.get('decision')}): {len(whole)} whole" + f"{f' [{hist}]' if hist else ''}, {len(narrowed)} narrowed " + f"({sum(narrowed.values())} tests)" + f"{f'; lane reasons {because}' if lane.get('decision') == 'run_all' and because else ''}") + return out + + +run_all_reasons = answer.get("run_all_reasons") or [] +run_all_rules = answer.get("run_all_rules") or [] +reasons_text = f"; run-all reasons {run_all_reasons}" if run_all_reasons else "" +if run_all_rules: + reasons_text += f"; decided by rule(s) {run_all_rules}" lines = ["### test-impact engine", "", - f"- decision: `{answer.get('decision')}` in {sys.argv[2]} s", - f"- team artifact: {'fetched' if sys.argv[3] != '0' else 'none'}; coverage " + f"- decision: `{answer.get('decision')}` in {sys.argv[2]} s{reasons_text}", + f"- team artifact: {'fetched' if sys.argv[3] != '0' else 'none'}" + f"{f' ({sys.argv[5]})' if sys.argv[5] else ''}; coverage " f"`{coverage.get('state')}` {coverage.get('rejection_reasons') or ''}"] -for lane in answer.get("lanes") or []: - suites = lane.get("suites") or [] - whole = [s["suite"] for s in suites if s.get("mode") == "whole"] - narrowed = {s["suite"]: len(s.get("tests") or []) for s in suites if s.get("mode") == "tests"} - lines.append(f"- lane `{lane.get('lane')}` ({lane.get('decision')}): {len(whole)} whole, " - f"{len(narrowed)} narrowed ({sum(narrowed.values())} tests)") +note = open(sys.argv[6]).read().strip() if os.path.exists(sys.argv[6]) else "" +if note: + lines.append(f"- engine note: `{note.splitlines()[0][:300]}`") +lines += lane_lines(answer, "- ") +if os.path.exists(sys.argv[4]): + lines.append("- the runner-main veto (RUNNER_REACHED) discarded this selection:") + lines += lane_lines(json.load(open(sys.argv[4])), " - ") print("\n".join(lines)) PY diff --git a/src/mcp/test_impact.h b/src/mcp/test_impact.h index 56cc012b9..e0363ead1 100644 --- a/src/mcp/test_impact.h +++ b/src/mcp/test_impact.h @@ -515,6 +515,11 @@ typedef struct { const char *const *lanes; int lane_count; int key_segment; /* zero-based path_segment:N; -1 for other actions */ + /* referencing_tests only. key_path: the key is the changed path itself (no + * parent directories). unreferenced_ignore: a changed path no test names + * selects nothing instead of running everything. */ + bool key_path; + bool unreferenced_ignore; bool builtin; } cbm_test_rule_t; diff --git a/src/mcp/test_impact_cli.c b/src/mcp/test_impact_cli.c index 4106d89db..74af90449 100644 --- a/src/mcp/test_impact_cli.c +++ b/src/mcp/test_impact_cli.c @@ -78,7 +78,7 @@ static int tic_usage(void) { " codebase-memory-mcp test-impact select --repo DIR --base REF\n" " [--config FILE] [--artifact BUNDLE --artifact-commit SHA\n" " --artifact-verified] [--platform LABEL] [--work DIR]\n" - " [--deadline-s N]\n"); + " [--deadline-s N] [--vetoed-out FILE]\n"); return 2; } @@ -125,6 +125,7 @@ static int tic_publish(int argc, char **argv) { static int tic_select(int argc, char **argv) { const char *repo = NULL, *base = NULL, *config = NULL, *artifact = NULL; const char *artifact_commit = NULL, *platform = NULL, *work = NULL, *deadline = NULL; + const char *vetoed_path = NULL; bool verified = false; const tic_option_t options[] = {{"--repo", &repo, NULL}, {"--base", &base, NULL}, @@ -134,7 +135,8 @@ static int tic_select(int argc, char **argv) { {"--artifact-verified", NULL, &verified}, {"--platform", &platform, NULL}, {"--work", &work, NULL}, - {"--deadline-s", &deadline, NULL}}; + {"--deadline-s", &deadline, NULL}, + {"--vetoed-out", &vetoed_path, NULL}}; if (!tic_parse(argc, argv, options, sizeof(options) / sizeof(options[0])) || !repo || !base) { return tic_usage(); } @@ -153,6 +155,10 @@ static int tic_select(int argc, char **argv) { .artifact_commit = artifact_commit, .artifact_verified = verified, .platform = platform}; + cbm_test_result_t *vetoed = NULL; + if (vetoed_path) { + request.vetoed_out = &vetoed; + } cbm_test_result_t *result = NULL; char diagnostic[512] = ""; cbm_test_impact_status_t status = @@ -162,12 +168,24 @@ static int tic_select(int argc, char **argv) { } if (status != CBM_TEST_IMPACT_OK) { (void)fprintf(stderr, "test-impact select: status %d\n", (int)status); + cbm_test_result_free(vetoed); return 1; } size_t length = 0; const char *json = cbm_test_result_json(result, &length); bool ok = json && fwrite(json, 1, length, stdout) == length && fputc('\n', stdout) != EOF; cbm_test_result_free(result); + /* The selection the runner-main veto discarded: a CI diagnostic, written + * only when the veto alone made the answer run-all. */ + if (ok && vetoed) { + const char *text = cbm_test_result_json(vetoed, &length); + FILE *f = text ? cbm_fopen(vetoed_path, "wb") : NULL; + ok = f && fwrite(text, 1, length, f) == length && fputc('\n', f) != EOF; + if (f && fclose(f) != 0) { + ok = false; + } + } + cbm_test_result_free(vetoed); return ok ? 0 : 1; } diff --git a/src/mcp/test_impact_engine.c b/src/mcp/test_impact_engine.c index 93a16f226..9d90b4e68 100644 --- a/src/mcp/test_impact_engine.c +++ b/src/mcp/test_impact_engine.c @@ -495,7 +495,9 @@ static void te_receipt(te_ctx_t *c, cbm_test_result_receipt_t *r, te_artifact_receipt(c, &r->coverage, te_oid); } -static cbm_test_impact_status_t te_answer(te_ctx_t *c, cbm_test_result_t **out) { +/* mask_runner: the answer without the RUNNER_REACHED fallback (the CI's view + * of what the veto discarded; see cbm_test_impact_request_t.vetoed_out). */ +static cbm_test_impact_status_t te_answer(te_ctx_t *c, cbm_test_result_t **out, bool mask_runner) { cbm_test_result_rule_match_t *matches = cbm_arena_calloc(&c->arena, (size_t)(c->match_count + 1) * sizeof(*matches)); if (!matches) { @@ -517,6 +519,23 @@ static cbm_test_impact_status_t te_answer(te_ctx_t *c, cbm_test_result_t **out) return CBM_TEST_IMPACT_OOM; } bool with_selection = c->selection && comparison == CBM_TEST_RESULT_COMPARISON_CHANGED; + const cbm_test_result_fallback_t *fallbacks = c->fallbacks; + size_t fallback_count = (size_t)c->fallback_count; + cbm_test_result_fallback_t *kept = NULL; + if (mask_runner) { + kept = cbm_arena_calloc(&c->arena, (fallback_count + 1) * sizeof(*kept)); + if (!kept) { + return CBM_TEST_IMPACT_OOM; + } + size_t n = 0; + for (size_t i = 0; i < fallback_count; i++) { + if (c->fallbacks[i] != CBM_TEST_RESULT_FALLBACK_RUNNER_REACHED) { + kept[n++] = c->fallbacks[i]; + } + } + fallbacks = kept; + fallback_count = n; + } cbm_test_result_input_t input = { .comparison = comparison, .selection = with_selection ? c->selection : NULL, @@ -527,8 +546,8 @@ static cbm_test_impact_status_t te_answer(te_ctx_t *c, cbm_test_result_t **out) .matched_rules = comparison == CBM_TEST_RESULT_COMPARISON_EMPTY ? NULL : matches, .matched_rule_count = comparison == CBM_TEST_RESULT_COMPARISON_EMPTY ? 0 : (size_t)c->match_count, - .fallbacks = c->fallbacks, - .fallback_count = (size_t)c->fallback_count, + .fallbacks = fallbacks, + .fallback_count = fallback_count, .warnings = c->warnings, .warning_count = (size_t)c->warning_count, .receipt = &receipt}; @@ -615,10 +634,25 @@ static void *te_job_main(void *arg) { .rq = job->request, .diagnostic = job->diagnostic, .diagnostic_len = job->diagnostic_len}; cbm_arena_init(&c.arena); te_steps(&c); - cbm_test_impact_status_t status = c.oom ? CBM_TEST_IMPACT_OOM : te_answer(&c, job->out); + cbm_test_impact_status_t status = c.oom ? CBM_TEST_IMPACT_OOM : te_answer(&c, job->out, false); + cbm_test_result_t **vetoed = job->request->vetoed_out; + if (vetoed) { + *vetoed = NULL; + bool runner_veto_only = + c.fallback_count == 1 && c.fallbacks[0] == CBM_TEST_RESULT_FALLBACK_RUNNER_REACHED; + if (status == CBM_TEST_IMPACT_OK && runner_veto_only && c.selection && + te_answer(&c, vetoed, true) != CBM_TEST_IMPACT_OK) { + cbm_test_result_free(*vetoed); /* a diagnostic only: never fails the answer */ + *vetoed = NULL; + } + } if (!c.oom && cbm_now_ms() > job->request->deadline_ms && status == CBM_TEST_IMPACT_OK) { cbm_test_result_free(*job->out); *job->out = NULL; + if (vetoed) { + cbm_test_result_free(*vetoed); + *vetoed = NULL; + } status = CBM_TEST_IMPACT_CANCELLED; } te_free(&c); diff --git a/src/mcp/test_impact_engine.h b/src/mcp/test_impact_engine.h index 6ce3e51fb..39a1b4d85 100644 --- a/src/mcp/test_impact_engine.h +++ b/src/mcp/test_impact_engine.h @@ -47,6 +47,11 @@ typedef struct { /* Platform/toolchain label of the leg the answer is for; part of the * coverage compatibility digest, so a map narrows only its own platform. */ const char *platform; + /* Optional, CI diagnostics only (never part of the MCP answer): when the + * one run-all reason is RUNNER_REACHED, *vetoed_out receives the answer + * with that veto masked — the selection the veto discarded. NULL + * otherwise. The caller frees it with cbm_test_result_free. */ + cbm_test_result_t **vetoed_out; } cbm_test_impact_request_t; typedef enum { diff --git a/src/mcp/test_impact_engine_reach.c b/src/mcp/test_impact_engine_reach.c index e54724aca..70e5affe4 100644 --- a/src/mcp/test_impact_engine_reach.c +++ b/src/mcp/test_impact_engine_reach.c @@ -14,13 +14,19 @@ #include "foundation/mem_core.h" #include "store/store_impact.h" +#include #include #include -/* The c-cbm runner's process-entry handlers: the runner re-executes itself - * with a role argument and these run product code in that child. They are - * reached, never expanded (their caller is the runner's main). */ -#define TE_RUNNER_ROLE_PREFIX "tf_maybe_run_" +/* The c-cbm runner's child-process roles: a test re-executes the runner with + * argv[1] = a role literal, and the runner's main dispatches it from its role + * table — `TF_ROLE(phase, "literal", handler)` entries in a runner file — by + * an exact argv[1] match, so a handler only ever runs in its own child. A + * handler is a sink (reached, never expanded into main): reaching it selects + * the tests that name its literal (te_role_lift). A handler the table does + * not name, or a TF_ROLE line that does not parse, is no sink: reaching it + * reaches main, which runs everything. */ +#define TE_ROLE_MACRO "TF_ROLE(" /* SPAWNS carries reach from a Process node, which enters the walk through * te_spawn_lift alone, and from a script of the repository (its Module / @@ -124,10 +130,28 @@ static int te_case_index(te_ctx_t *c, const char *file, const char *name) { return -1; } +static bool te_ids_has(const te_ids_t *a, int64_t id) { + for (int i = 0; a && i < a->count; i++) { + if (a->items[i] == id) { + return true; + } + } + return false; +} + /* Every line of a test file that contains `needle`: its enclosing - * definition is a fixture case or a seed. */ + * definition is a fixture case or a seed — unless it is one of `exclude` + * (a role handler naming its own literal is not a test that spawns it). */ +static bool te_scan_literal_ex(te_ctx_t *c, const char *needle, te_ids_t *seeds, bool *cases, + bool *hit, const te_ids_t *exclude); + static bool te_scan_literal(te_ctx_t *c, const char *needle, te_ids_t *seeds, bool *cases, bool *hit) { + return te_scan_literal_ex(c, needle, seeds, cases, hit, NULL); +} + +static bool te_scan_literal_ex(te_ctx_t *c, const char *needle, te_ids_t *seeds, bool *cases, + bool *hit, const te_ids_t *exclude) { int files = cbm_ti_source_file_count(c->source); size_t nlen = strlen(needle); for (int f = 0; f < files; f++) { @@ -153,7 +177,7 @@ static bool te_scan_literal(te_ctx_t *c, const char *needle, te_ids_t *seeds, bo "graph read failed") && false; } - if (!d) { + if (!d || te_ids_has(exclude, d->id)) { continue; } *hit = true; @@ -171,9 +195,11 @@ static bool te_scan_literal(te_ctx_t *c, const char *needle, te_ids_t *seeds, bo /* One fixture path: the path itself, then each parent down to the rule's * key segment; each with and without its first segment. The first key some * test file mentions decides. */ +#define TE_FIXTURE_KEY_MAX 4096 + static bool te_fixture(te_ctx_t *c, const char *path, int min_segments, te_ids_t *seeds, bool *cases, bool *hit) { - char key[4096]; + char key[TE_FIXTURE_KEY_MAX]; size_t len = strlen(path); if (len >= sizeof(key)) { return true; /* unresolved: run-all through the rule */ @@ -212,11 +238,22 @@ bool te_fixtures(te_ctx_t *c, int64_t **extra_seeds, int *extra_count, bool **fi const cbm_test_rule_t *rules = cbm_test_policy_rules(c->policy, &rule_count); for (int i = 0; i < c->fixture_count; i++) { int r = c->fixture_rules[i]; + const char *path = c->fixture_paths[i]; + int min_segments = rules[r].key_segment + 1; + if (rules[r].key_path) { + min_segments = 1; /* the path itself only: never a parent directory */ + for (const char *p = path; *p; p++) { + min_segments += *p == '/'; + } + } bool hit = false; - if (!te_fixture(c, c->fixture_paths[i], rules[r].key_segment + 1, &seeds, cases, &hit)) { + if (!te_fixture(c, path, min_segments, &seeds, cases, &hit)) { cbm_free(CBM_MEM_CLASS_EXTRACT, seeds.items); return false; } + if (!hit && rules[r].unreferenced_ignore && strlen(path) < TE_FIXTURE_KEY_MAX) { + continue; /* scanned, and no test names it: nothing can depend on it */ + } for (int m = 0; !hit && m < c->match_count; m++) { if (c->matches[m].rule_index == r) { c->matches[m].resolved = false; /* no test names it: run-all */ @@ -231,11 +268,153 @@ bool te_fixtures(te_ctx_t *c, int64_t **extra_seeds, int *extra_count, bool **fi /* ── The walk ────────────────────────────────────────────────────── */ +typedef struct { + const char *literal; /* what a test that starts the role's child names */ + int64_t id; /* the handler's function node; one handler may have several */ + bool reached; + bool scanned; + bool named; /* some test or helper names the literal */ +} te_role_t; + typedef struct { te_ids_t sinks; te_ids_t runner_main; + te_role_t *roles; + int role_count; + int role_cap; } te_sinks_t; +static const char *te_skip_space(const char *p, const char *end) { + while (p < end && (*p == ' ' || *p == '\t')) { + p++; + } + return p; +} + +/* One `TF_ROLE(phase, "literal", handler)` line. false: no entry here (the + * macro's own #define included); *bad: an entry that does not parse. */ +static bool te_role_parse(const char *text, size_t len, char *literal, size_t literal_cap, + char *handler, size_t handler_cap, bool *bad) { + *bad = false; + const char *end = text + len; + const char *p = te_skip_space(text, end); + size_t macro_len = sizeof(TE_ROLE_MACRO) - 1; + if (p >= end || *p == '#') { + return false; + } + const char *at = NULL; + for (const char *q = p; !at && q + macro_len <= end; q++) { + at = memcmp(q, TE_ROLE_MACRO, macro_len) == 0 ? q : NULL; + } + if (!at) { + return false; + } + *bad = true; + p = at + macro_len; + while (p < end && *p != ',') { + p++; /* the phase */ + } + p = te_skip_space(p + 1, end); + if (p >= end || *p != '"') { + return true; + } + const char *lit = ++p; + while (p < end && *p != '"' && *p != '\\') { + p++; + } + if (p >= end || *p != '"' || p == lit || (size_t)(p - lit) >= literal_cap) { + return true; + } + memcpy(literal, lit, (size_t)(p - lit)); + literal[p - lit] = '\0'; + p = te_skip_space(p + 1, end); + if (p >= end || *p != ',') { + return true; + } + p = te_skip_space(p + 1, end); + const char *name = p; + while (p < end && ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || + (*p >= '0' && *p <= '9') || *p == '_')) { + p++; + } + if (p == name || (size_t)(p - name) >= handler_cap) { + return true; + } + memcpy(handler, name, (size_t)(p - name)); + handler[p - name] = '\0'; + p = te_skip_space(p, end); + *bad = p >= end || *p != ')'; + return true; +} + +/* The handler's function node: the one Function of that name in a runner or + * test file. 0 when there is none or more than one (then it is no sink, and + * reaching it reaches main). */ +static bool te_role_handler(te_ctx_t *c, const char *name, int64_t *id) { + cbm_node_t *nodes = NULL; + int n = 0; + *id = 0; + if (cbm_store_find_nodes_by_name(c->store, "test-impact", name, &nodes, &n) != CBM_STORE_OK) { + return te_fallback(c, CBM_TEST_RESULT_FALLBACK_GRAPH_UNAVAILABLE, "graph read failed") && + false; + } + int found = 0; + int runners = 0; + const cbm_test_runner_suite_t *runner = cbm_test_model_runner_suites(c->model, &runners); + for (int i = 0; i < n; i++) { + bool in_tests = nodes[i].file_path && te_test_file(c, nodes[i].file_path); + for (int r = 0; !in_tests && nodes[i].file_path && r < runners; r++) { + in_tests = runner[r].file && strcmp(runner[r].file, nodes[i].file_path) == 0; + } + if (in_tests && nodes[i].label && strcmp(nodes[i].label, "Function") == 0) { + *id = nodes[i].id; + found++; + } + } + cbm_store_free_nodes(nodes, n); + if (found != 1) { + *id = 0; + } + return true; +} + +/* The role table of one runner file: each entry's handler becomes a sink. */ +static bool te_runner_roles(te_ctx_t *c, const char *file, te_sinks_t *s) { + int f = cbm_ti_source_find(c->source, file); + int lines = f >= 0 ? cbm_ti_source_line_count(c->source, f) : 0; + for (int line = 1; line <= lines; line++) { + size_t len = 0; + const char *text = cbm_ti_source_line(c->source, f, line, &len); + char literal[256]; + char handler[256]; + bool bad = false; + if (!text || + !te_role_parse(text, len, literal, sizeof(literal), handler, sizeof(handler), &bad) || + bad) { + continue; /* not an entry, or one that does not parse: no sink */ + } + int64_t id = 0; + if (!te_role_handler(c, handler, &id)) { + return false; + } + if (!id) { + continue; + } + if (!te_grow((void **)&s->roles, &s->role_cap, s->role_count, sizeof(*s->roles))) { + return te_oom(c); + } + const char *owned = cbm_arena_strdup(&c->arena, literal); + if (!owned) { + return te_oom(c); + } + s->roles[s->role_count++] = (te_role_t){.literal = owned, .id = id}; + if (!te_ids_push(c, &s->sinks, id)) { + return false; + } + } + return true; +} + static bool te_runner_sinks(te_ctx_t *c, te_sinks_t *s) { int n = 0; const cbm_test_runner_suite_t *runner = cbm_test_model_runner_suites(c->model, &n); @@ -253,19 +432,18 @@ static bool te_runner_sinks(te_ctx_t *c, te_sinks_t *s) { false; } for (int k = 0; k < count; k++) { - if (!te_is_function(&nodes[k]) || nodes[k].variant_span) { + if (!te_is_function(&nodes[k]) || nodes[k].variant_span || + strcmp(nodes[k].name, "main") != 0) { continue; } - bool main_fn = strcmp(nodes[k].name, "main") == 0; - bool role = strncmp(nodes[k].name, TE_RUNNER_ROLE_PREFIX, - sizeof(TE_RUNNER_ROLE_PREFIX) - 1) == 0; - if ((main_fn || role) && !te_ids_push(c, &s->sinks, nodes[k].id)) { - return false; - } - if (main_fn && !te_ids_push(c, &s->runner_main, nodes[k].id)) { + if (!te_ids_push(c, &s->sinks, nodes[k].id) || + !te_ids_push(c, &s->runner_main, nodes[k].id)) { return false; } } + if (!te_runner_roles(c, runner[i].file, s)) { + return false; + } } return true; } @@ -739,6 +917,186 @@ static bool te_spawn_lift(te_ctx_t *c, cbm_impact_walk_t *walk) { return ok; } +/* ── Child-process roles (TF_ROLE) ─────────────────────────────────── + * A reached role handler runs in the child its tests spawn by its literal. + * Mark the reached handlers, and every handler that calls a reached one (its + * child runs it in process); scan each role's literal in the test files — + * handlers and the runner's main excluded, so a handler naming its own + * literal is no spawner — selecting the tests that name it directly and + * walking on from helpers that do. A reached role no test names runs + * everything. Repeat until nothing new is reached. */ +static bool te_role_mark(te_ctx_t *c, cbm_impact_walk_t *walk, te_sinks_t *s, bool *grew) { + for (int i = 0; i < s->role_count; i++) { + if (!s->roles[i].reached && cbm_impact_walk_reached(walk, s->roles[i].id)) { + s->roles[i].reached = *grew = true; + } + } + for (bool chained = true; chained;) { + chained = false; + for (int i = 0; i < s->role_count; i++) { + if (!s->roles[i].reached) { + continue; + } + cbm_edge_t *edges = NULL; + int n = 0; + if (cbm_store_find_edges_by_target_type(c->store, s->roles[i].id, "CALLS", &edges, + &n) != CBM_STORE_OK) { + return te_fallback(c, CBM_TEST_RESULT_FALLBACK_GRAPH_UNAVAILABLE, + "graph read failed") && + false; + } + for (int e = 0; e < n; e++) { + for (int k = 0; k < s->role_count; k++) { + if (!s->roles[k].reached && s->roles[k].id == edges[e].source_id) { + s->roles[k].reached = chained = *grew = true; + } + } + } + cbm_store_free_edges(edges, n); + } + } + return true; +} + +/* A reached handler none of whose literals any test names: its spawners are + * unknown, so everything runs. One named literal names the handler's spawners + * (the index worker: "--index-worker" or the CBM_INDEX_SUPERVISOR switch). */ +static bool te_roles_named(te_ctx_t *c, const te_sinks_t *s) { + for (int i = 0; i < s->role_count; i++) { + const te_role_t *r = &s->roles[i]; + bool named = !r->reached; + for (int k = 0; !named && k < s->role_count; k++) { + named = s->roles[k].id == r->id && s->roles[k].named; + } + if (!named) { + char note[320]; + snprintf(note, sizeof(note), + "the change reaches the test-runner role %s, which no test names", r->literal); + return te_fallback(c, CBM_TEST_RESULT_FALLBACK_RUNNER_REACHED, note); + } + } + return true; +} + +static bool te_role_lift(te_ctx_t *c, cbm_impact_walk_t *walk, te_sinks_t *s, bool *cases) { + te_ids_t handlers = {0}; + bool ok = true; + for (int i = 0; ok && i < s->role_count; i++) { + ok = te_ids_push(c, &handlers, s->roles[i].id); + } + for (int i = 0; ok && i < s->runner_main.count; i++) { + ok = te_ids_push(c, &handlers, s->runner_main.items[i]); + } + for (bool grew = ok; ok && grew;) { + grew = false; + ok = te_role_mark(c, walk, s, &grew); + for (int i = 0; ok && i < s->role_count; i++) { + te_role_t *r = &s->roles[i]; + if (!r->reached || r->scanned) { + continue; + } + r->scanned = true; + te_ids_t seeds = {0}; + ok = te_scan_literal_ex(c, r->literal, &seeds, cases, &r->named, &handlers); + if (ok && seeds.count) { + ok = cbm_impact_walk_run(walk, seeds.items, seeds.count) == CBM_STORE_OK || + (te_fallback(c, CBM_TEST_RESULT_FALLBACK_ENGINE_SATURATED, + "the impact walk failed") && + false); + grew = true; + } + cbm_free(CBM_MEM_CLASS_EXTRACT, seeds.items); + } + } + cbm_free(CBM_MEM_CLASS_EXTRACT, handlers.items); + return ok && te_roles_named(c, s); +} + +/* The last two segments of a qualified name (`test_main.main`), or all of it. */ +static const char *te_short_qn(const char *qn) { + const char *last = strrchr(qn, '.'); + if (!last) { + return qn; + } + const char *start = qn; + for (const char *p = qn; p < last; p++) { + if (*p == '.') { + start = p + 1; + } + } + return start; +} + +/* prefix + the call chain the walk took from the changed code to `id`, in call + * order (`id` first: "test_main.main -> a.b -> ... -> changed.fn"), cut short + * with "..." when it does not fit. */ +static void te_call_chain(const cbm_impact_walk_t *walk, int64_t id, const char *prefix, char *out, + size_t cap) { + int used = snprintf(out, cap, "%s", prefix); + for (int step = 0; id && used >= 0 && (size_t)used < cap && step < 64; step++) { + const cbm_impact_hit_t *hit = cbm_impact_walk_hit(walk, id); + if (!hit) { + break; + } + const char *name = hit->qualified_name ? te_short_qn(hit->qualified_name) : "?"; + int n = snprintf(out + used, cap - (size_t)used, "%s%s", step ? " -> " : "", name); + if (n < 0 || (size_t)n >= cap - (size_t)used) { + if (cap > 4) { + memcpy(out + cap - 4, "...", 4); + } + return; + } + used += n; + id = hit->via_id; + } +} + +/* Whether the walk reaches the runner's main through code that can run, and + * through what: *way_in is main itself (a changed main, or a step that is no + * edge: a platform variant, a route), the reached node main has a walked edge + * to, or 0. An edge into a Field named like a suite main runs is no way in: + * the C extractor takes the macro argument of RUN_SELECTED_SUITE(mcp) for a + * use of some same-named data member (`char mcp[]` in an unrelated struct). */ +static bool te_main_reached(te_ctx_t *c, const cbm_impact_walk_t *walk, int64_t main_id, + int64_t *way_in) { + *way_in = 0; + const cbm_impact_hit_t *self = cbm_impact_walk_hit(walk, main_id); + if (!self) { + return true; + } + bool edge_step = false; + for (size_t t = 0; self->via_edge && t < sizeof(te_walk_types) / sizeof(te_walk_types[0]); + t++) { + edge_step = edge_step || strcmp(self->via_edge, te_walk_types[t]) == 0; + } + if (!edge_step) { + *way_in = main_id; + return true; + } + cbm_edge_t *edges = NULL; + int n = 0; + if (cbm_store_find_edges_by_source(c->store, main_id, &edges, &n) != CBM_STORE_OK) { + return te_fallback(c, CBM_TEST_RESULT_FALLBACK_GRAPH_UNAVAILABLE, "graph read failed") && + false; + } + for (int e = 0; !*way_in && e < n; e++) { + const cbm_impact_hit_t *to = cbm_impact_walk_hit(walk, edges[e].target_id); + bool walked = false; + for (size_t t = 0; + to && edges[e].type && t < sizeof(te_walk_types) / sizeof(te_walk_types[0]); t++) { + walked = walked || strcmp(edges[e].type, te_walk_types[t]) == 0; + } + const char *name = walked && to->qualified_name ? strrchr(to->qualified_name, '.') : NULL; + bool suite_arg = + name && to->label && strcmp(to->label, "Field") == 0 && te_is_runner_suite(c, name + 1); + if (walked && !suite_arg) { + *way_in = edges[e].target_id; + } + } + cbm_store_free_edges(edges, n); + return true; +} + bool te_reach(te_ctx_t *c) { int64_t *extra = NULL; int extra_count = 0; @@ -746,14 +1104,23 @@ bool te_reach(te_ctx_t *c) { te_sinks_t sinks = {0}; cbm_impact_walk_t *walk = NULL; bool ok = te_fixtures(c, &extra, &extra_count, &fixture_cases) && te_collect_sinks(c, &sinks) && - te_walk(c, &sinks, extra, extra_count, &walk) && te_parse_gaps(c, walk); - if (ok) { - for (int i = 0; i < sinks.runner_main.count; i++) { - if (cbm_impact_walk_reached(walk, sinks.runner_main.items[i])) { - ok = te_fallback(c, CBM_TEST_RESULT_FALLBACK_RUNNER_REACHED, - "the change reaches the test runner's main"); - break; - } + te_walk(c, &sinks, extra, extra_count, &walk) && te_parse_gaps(c, walk) && + te_role_lift(c, walk, &sinks, fixture_cases) && te_parse_gaps(c, walk); + for (int i = 0; ok && i < sinks.runner_main.count; i++) { + int64_t way_in = 0; + ok = te_main_reached(c, walk, sinks.runner_main.items[i], &way_in); + if (ok && way_in) { + const cbm_impact_hit_t *m = cbm_impact_walk_hit(walk, sinks.runner_main.items[i]); + const char *main_name = + m && m->qualified_name ? te_short_qn(m->qualified_name) : "main"; + bool direct = way_in == sinks.runner_main.items[i]; + char note[400]; + int used = + snprintf(note, sizeof(note), "the change reaches the test runner's main: %s%s", + direct ? "" : main_name, direct ? "" : " -> "); + te_call_chain(walk, way_in, "", note + used, sizeof(note) - (size_t)used); + ok = te_fallback(c, CBM_TEST_RESULT_FALLBACK_RUNNER_REACHED, note); + break; } } /* After the runner check: the runner re-executing itself is a spawn too, @@ -766,5 +1133,6 @@ bool te_reach(te_ctx_t *c) { cbm_free(CBM_MEM_CLASS_EXTRACT, extra); cbm_free(CBM_MEM_CLASS_EXTRACT, sinks.sinks.items); cbm_free(CBM_MEM_CLASS_EXTRACT, sinks.runner_main.items); + cbm_free(CBM_MEM_CLASS_EXTRACT, sinks.roles); return ok; } diff --git a/src/mcp/test_impact_policy.c b/src/mcp/test_impact_policy.c index 2b523908c..fc703bd70 100644 --- a/src/mcp/test_impact_policy.c +++ b/src/mcp/test_impact_policy.c @@ -587,6 +587,23 @@ static bool tp_key_segment(yyjson_val *value, int *segment) { return true; } +/* `unreferenced` of a referencing_tests rule: absent or "run_all" keeps the + * fail-safe default (a path no test names runs everything); "ignore" lets it + * select nothing — for data no test can depend on without naming it. */ +static bool tp_unreferenced(yyjson_val *value, bool *ignore) { + *ignore = false; + if (!value) + return true; + const char *text = tp_string(value, NULL); + if (!text) + return false; + if (strcmp(text, "ignore") == 0) { + *ignore = true; + return true; + } + return strcmp(text, "run_all") == 0; +} + static const char *const tp_build_paths[] = { "Makefile*", "CMakeLists.txt", "*.cmake", "meson.build", "BUILD*", "WORKSPACE", "package*.json", "go.mod", "go.sum", "Cargo.*", @@ -626,6 +643,7 @@ static bool tp_parse_rule(cbm_test_policy_t *policy, yyjson_val *value, int inde yyjson_val *lanes = tp_field(value, "lanes", &valid); yyjson_val *suites = tp_field(value, "suites", &valid); yyjson_val *key = tp_field(value, "key", &valid); + yyjson_val *unreferenced = tp_field(value, "unreferenced", &valid); const char *id_text = tp_string(id, NULL), *action_text = tp_string(action, NULL); if (!valid || !id_text || !action_text) return false; @@ -646,26 +664,33 @@ static bool tp_parse_rule(cbm_test_policy_t *policy, yyjson_val *value, int inde return false; if (strcmp(action_text, "run_all") == 0) { rule->action = CBM_TEST_RULE_RUN_ALL; - return !lanes && !suites && !key; + return !lanes && !suites && !key && !unreferenced; } if (strcmp(action_text, "ignore") == 0) { rule->action = CBM_TEST_RULE_IGNORE; - return !lanes && !suites && !key; + return !lanes && !suites && !key && !unreferenced; } if (strcmp(action_text, "lanes") == 0) { rule->action = CBM_TEST_RULE_LANES; - return !suites && !key && + return !suites && !key && !unreferenced && tp_string_array(policy, lanes, true, &rule->lanes, &rule->lane_count, NULL, false); } if (strcmp(action_text, "suites") == 0) { rule->action = CBM_TEST_RULE_SUITES; - return !lanes && !key && + return !lanes && !key && !unreferenced && tp_string_array(policy, suites, true, &rule->suites, &rule->suite_count, &matchers->suites, true); } if (strcmp(action_text, "referencing_tests") == 0) { rule->action = CBM_TEST_RULE_REFERENCING_TESTS; - return !lanes && !suites && tp_key_segment(key, &rule->key_segment); + if (lanes || suites || !tp_unreferenced(unreferenced, &rule->unreferenced_ignore)) + return false; + const char *key_text = tp_string(key, NULL); + if (key_text && strcmp(key_text, "path") == 0) { + rule->key_path = true; + return true; + } + return tp_key_segment(key, &rule->key_segment); } return false; } diff --git a/src/mcp/test_impact_result.c b/src/mcp/test_impact_result.c index 783aec9e8..fed0d51ec 100644 --- a/src/mcp/test_impact_result.c +++ b/src/mcp/test_impact_result.c @@ -1,5 +1,20 @@ #include "mcp/test_impact_result_internal.h" +/* The ids of the matched run_all rules, in policy order: which rule made a + * RULE_RUN_ALL answer run everything. Outside the decision hash. */ +static tir_json *tir_run_all_rules(tir_context *c) { + tir_json *array = tir_node(c, TIR_ARRAY); + for (size_t i = 0; i < c->rule_count; i++) { + if (!tir_step(c, 0, 1)) + return NULL; + const tir_rule *rule = &c->rules[i]; + if (rule->matched && rule->source->action == CBM_TEST_RULE_RUN_ALL && + !tir_add(c, array, NULL, tir_string(c, rule->id))) + return NULL; + } + return array; +} + static tir_json *tir_result_object(tir_context *c, cbm_test_result_t *result) { const char *decision = c->global ? "run_all" : (c->runnable ? "selected" : "nothing"); tir_json *o = tir_node(c, TIR_OBJECT); @@ -7,6 +22,7 @@ static tir_json *tir_result_object(tir_context *c, cbm_test_result_t *result) { tir_add(c, o, "experimental", tir_bool(c, true)); tir_add(c, o, "decision", tir_text(c, decision)); tir_add(c, o, "run_all_reasons", c->global_json); + tir_add(c, o, "run_all_rules", tir_run_all_rules(c)); tir_add(c, o, "lanes", c->lane_json); tir_add(c, o, "warnings", tir_warnings_json(c)); tir_add(c, o, "totals", tir_totals_json(c)); diff --git a/src/mcp/test_impact_result_project.c b/src/mcp/test_impact_result_project.c index 01cc334f9..0c62b3973 100644 --- a/src/mcp/test_impact_result_project.c +++ b/src/mcp/test_impact_result_project.c @@ -5,6 +5,8 @@ typedef struct { tir_json *suites; uint64_t reasons; size_t members; + size_t listed; /* member suites the plan lists (an omitted one is not) */ + size_t listed_whole; /* of those, the ones that run whole */ tir_lane_decision decision; } tir_lane_plan; @@ -38,6 +40,8 @@ static bool tir_member_suite(tir_context *c, size_t lane_index, tir_lane_plan *p if (c->global) return true; plan->reasons |= reasons; + plan->listed++; + plan->listed_whole += whole; if (!tir_charge(c, &c->suite_occurrences, 1, UINT64_MAX)) return false; if (whole) @@ -95,7 +99,10 @@ static bool tir_lane_plan_build(tir_context *c, size_t index, tir_lane_plan *pla else if (!plan->suites->as.children.first) plan->reasons = TIR_REASON(TIR_REASON_NO_SELECTED_TESTS); else { - plan->decision = lane->source->narrow ? TIR_SELECTED : TIR_ALL; + /* A lane that omits no member suite and runs every one whole runs + * everything (e.g. no admitted coverage): say so, whatever the reasons. */ + bool runs_everything = plan->listed == plan->members && plan->listed_whole == plan->listed; + plan->decision = lane->source->narrow && !runs_everything ? TIR_SELECTED : TIR_ALL; if (!tir_charge(c, &c->runnable, 1, UINT64_MAX)) return false; } diff --git a/tests/fixtures/select-lanes/cases.tsv b/tests/fixtures/select-lanes/cases.tsv index 15ce05df5..ff838416b 100644 --- a/tests/fixtures/select-lanes/cases.tsv +++ b/tests/fixtures/select-lanes/cases.tsv @@ -3,7 +3,9 @@ # Columns (TAB-separated): name, files (space-separated; "-" = empty list), # tier, full, lanes (space-separated, any order; "FULL" = every PR lane, # "FULL -a -b" = every PR lane except a and b). -docs-only README.md docs/BENCHMARK.md T0a-docs false contracts security-static +docs-only CONTRIBUTING.md docs/BENCHMARK.md T0a-docs false contracts security-static +readme-runs-the-test-legs README.md T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows +site-files-run-the-test-legs docs/index.html docs/llms.txt T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows issue-template-is-docs .github/ISSUE_TEMPLATE/bug.yml T0a-docs false contracts security-static local-infra test-infrastructure/run.sh T0b-nonproduct false contracts lint lint-mem security-static lint-config scripts/memory-core-baseline.txt T0b-nonproduct false contracts lint lint-mem security-static @@ -38,7 +40,17 @@ vendored-code vendored/sqlite3/sqlite3.c T5-core true FULL gate-workflow .github/workflows/codeql.yml T5-core true FULL unmatched-top-level-path weird/new.file T5-core true FULL unmatched-src-directory src/newthing/x.c T5-core true FULL -unmatched-script scripts/README.md T5-core true FULL +unmatched-script scripts/new-tool.sh T5-core true FULL +scripts-readme-is-docs scripts/README.md T0a-docs false contracts security-static +license-tool-runs-license-gate scripts/license-gate.sh T0b-nonproduct false contracts license-gate lint lint-mem security-static +security-tool-runs-security-static scripts/security-ui.sh T0b-nonproduct false contracts lint lint-mem security-static +local-tool-is-non-gating scripts/repro.sh T0b-nonproduct false contracts lint lint-mem security-static shard-completeness unix-x86 +lint-config-file .cppcheck T0b-nonproduct false contracts lint lint-mem security-static +script-c-tests-read scripts/security-vendored.sh T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows +engine-config-c-tests-read .codebase-memory.json T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows +test-failure-report scripts/suite-failure-report.sh T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows +smoke-security-script scripts/security-strings.sh T4-platform false FULL -msan -diag -memwaste +shipped-notices-are-pkg scripts/gen-third-party-notices.sh T4-platform false FULL -msan -diag -memwaste non-gate-workflow-forces-full .github/workflows/stale.yml T0b-nonproduct true FULL harness-main-forces-full tests/test_main.c T1-tests true FULL harness-framework-forces-full tests/test_framework.h T1-tests true FULL diff --git a/tests/test_lane_wiring_contract.sh b/tests/test_lane_wiring_contract.sh index bd5f26b30..d79579725 100755 --- a/tests/test_lane_wiring_contract.sh +++ b/tests/test_lane_wiring_contract.sh @@ -126,12 +126,17 @@ m = re.search(r"(?ms)^ test_selection:\n(.*?)(?=^ \S|^\S|\Z)", text("_ require(m is not None and re.search(r"(?m)^ default: ''$", m.group(1)) is not None, "_test.yml must declare a test_selection input defaulting to '' (every test)") readers = sorted(job for job, body in test_wf.items() if "inputs.test_selection" in body) -require(readers == ["test-unix", "test-windows"], - f"only test-unix and test-windows may read test_selection, found {readers}") -for job in ("test-unix", "test-windows"): - require("CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && " in test_wf.get(job, ""), - f"_test.yml {job} must hand the selection to scripts/test.sh as CBM_TEST_SELECTION_DIR " - f"only when one was passed") +# Only the Linux x86_64 leg narrows (user decision 2026-10-16): the coverage comes from +# the Linux x86_64 producer, so arm64, macOS and Windows legs run every test. +require(readers == ["test-unix"], f"only test-unix may read test_selection, found {readers}") +unix = test_wf.get("test-unix", "") +linux_x64 = "inputs.test_selection != '' && runner.os == 'Linux' && runner.arch == 'X64'" +require(f"if: ${{{{ {linux_x64} }}}}" in unix and + f"CBM_TEST_SELECTION_DIR: ${{{{ {linux_x64} && " in unix, + "_test.yml test-unix must fetch the selection and hand it to scripts/test.sh as " + "CBM_TEST_SELECTION_DIR only on Linux x86_64 and only when one was passed") +require("CBM_TEST_SELECTION_DIR: ''" in test_wf.get("test-windows", ""), + "_test.yml test-windows must run every test (CBM_TEST_SELECTION_DIR: '')") select = pr.get("select-tests", "") require(re.search(r"(?m)^ if: \$\{\{ vars\.TEST_IMPACT_MODE == 'gate' \}\}$", select) is not None, "pr.yml select-tests must run only when vars.TEST_IMPACT_MODE == 'gate'") diff --git a/tests/test_main.c b/tests/test_main.c index 5ff6bdb9f..d6e3c2cb2 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -1907,10 +1907,14 @@ extern void suite_dump_verify_io(void); * caches at thread teardown (pass_parallel.c). */ extern void cbm_kind_in_set_free_cache(void); -/* Native subprocess capture probe; implemented with its tests. */ -extern int tf_maybe_run_subprocess_stdout_probe(int argc, char **argv); +/* Runner roles and the runner-image bindings implemented with their tests. */ +extern int tf_role_subprocess_stdin_probe(int argc, char **argv); +extern int tf_role_subprocess_stdout_probe(int argc, char **argv); +extern void tf_subprocess_set_binary(const char *path); extern void tf_test_impact_runner_filter_set_binary(const char *path); -extern int tf_maybe_run_git_facts_diff_probe(int argc, char **argv); +extern int tf_role_git_facts_diff_probe(int argc, char **argv); +extern int tf_role_git_facts_hostile_diff_probe(int argc, char **argv); +extern void tf_test_impact_git_set_binary(const char *path); /* The runner's own image, for the tests that start or copy it again. argv[0] * is the caller's spelling, and a Windows parent that found the runner by @@ -1934,35 +1938,201 @@ const char *tf_runner_image(int argc, char **argv) { return argc > 0 && argv ? argv[0] : NULL; } +static int tf_role_env_values(int argc, char **argv) { + if (argc < 4 || argc % 2 != 0) { + return -1; + } + for (int i = 2; i < argc; i += 2) { + const char *value = getenv(argv[i]); + if (!value || strcmp(value, argv[i + 1]) != 0) { + return 61; + } + } + return 0; +} + +static int tf_role_env_isolation(int argc, char **argv) { + if (argc < 5) { + return -1; + } + const char *value = getenv(argv[2]); + if (!value || strcmp(value, argv[3]) != 0) { + return 61; + } + for (int i = 4; i < argc; i++) { + if (getenv(argv[i])) { + return 62; + } + } + return 0; +} + +/* Installation tests use this executable as a structurally real candidate. + * Mirror the production binary's minimal verification contract. */ +static int tf_role_version(int argc, char **argv) { + (void)argv; + if (argc != 2) { + return -1; + } + (void)puts("codebase-memory-mcp test-runner"); + return 0; +} + +static int tf_role_build_config(int argc, char **argv) { + (void)argv; + if (argc != 2) { + return -1; + } +#ifdef _WIN32 + if (_setmode(cbm_fileno(stdout), _O_BINARY) == -1) { + fprintf(stderr, "failed to set build-config stdout to binary mode\n"); + return 2; + } +#endif +#if defined(CBM_SANITIZED_BUILD) && CBM_SANITIZED_BUILD + const int sanitized = 1; +#else + const int sanitized = 0; +#endif +#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS + const int test_seams = 1; +#else + const int test_seams = 0; +#endif + (void)printf("sanitized=%d test_seams=%d\n", sanitized, test_seams); + return 0; +} + +/* #1830 userns smoke probe -- see the test that spawns it. + * + * The ancestor overflow uid is derived ONCE per process (pthread_once in + * src/daemon/ipc.c) from /proc/self/uid_map. That file is not immutable: + * unshare(CLONE_NEWUSER) is exactly what changes it, and pthread_once state + * survives fork(). A forked child therefore keeps the HOST answer and never + * re-derives inside its new namespace, so a fork-only smoke test refuses and + * cannot pass once anything earlier in the suite has primed the cache -- + * seven call sites do. Re-exec into this probe so the decision is made by a + * process that STARTED inside the namespace, which is the production shape + * the test means to cover. */ +static int tf_role_userns_secure_probe(int argc, char **argv) { +#if defined(__linux__) && defined(CBM_ENABLE_TEST_SEAMS) + if (argc == 3) { + /* _exit, not return: this process exists to answer ONE boolean. A + * return runs the atexit chain, and the runner is built with + * -fsanitize=address, so a future leak anywhere in the prologue would + * exit 23 and read as a security verdict on a test that has none. */ + _exit(cbm_daemon_ipc_private_directory_secure(argv[2]) ? 0 : 1); + } +#endif + (void)argc; + (void)argv; + return -1; +} + +/* The runner's subprocess roles: a test spawns this binary with argv[1] = the + * literal, and main hands over to the handler at the entry's phase. A handler + * returns the process's exit code, or -1 to let main continue. + * + * The test-impact engine reads this table (src/mcp/test_impact_engine_reach.c): + * a change only a role reaches runs the tests that name the role's literal, + * while one that reaches main itself runs every test. So every role main can + * enter by its argv[1] belongs here, and each entry stays on one line (phase, + * quoted literal, handler name) -- hence clang-format off. */ +typedef enum { + TF_PHASE_RAW, /* before anything else touches the process */ + TF_PHASE_GIT_CLEAN, /* after the caller's git environment is cleared */ + TF_PHASE_INSTALL, /* installation-candidate answers */ + TF_PHASE_SEAMED, /* after the spill and build-fingerprint pins */ + TF_PHASE_WORKER, /* after the fingerprint capture; the literal anywhere in argv */ + TF_PHASE_SPAWNER, /* never dispatched: one more literal a spawner names */ +} tf_phase_t; + +typedef struct { + tf_phase_t phase; + const char *literal; + int (*run)(int argc, char **argv); +} tf_role_t; + +#define TF_ROLE(phase, literal, handler) {phase, literal, handler} + +// clang-format off +static const tf_role_t tf_roles[] = { + TF_ROLE(TF_PHASE_RAW, "__cbm_env_values", tf_role_env_values), + TF_ROLE(TF_PHASE_RAW, "__cbm_env_isolation", tf_role_env_isolation), + TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_subprocess_stdin_probe", tf_role_subprocess_stdin_probe), + TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_subprocess_stdout_probe", tf_role_subprocess_stdout_probe), + TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_git_facts_diff_probe", tf_role_git_facts_diff_probe), + TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_git_facts_hostile_diff_probe", tf_role_git_facts_hostile_diff_probe), + TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_windows_memory_limit_probe", tf_maybe_run_windows_memory_limit_probe), + TF_ROLE(TF_PHASE_INSTALL, "--version", tf_role_version), + TF_ROLE(TF_PHASE_INSTALL, "--build-config", tf_role_build_config), + TF_ROLE(TF_PHASE_SEAMED, "--userns-secure-probe", tf_role_userns_secure_probe), + TF_ROLE(TF_PHASE_SEAMED, "__cbm_mcp_idxfailclosed_probe", tf_maybe_run_mcp_idxfailclosed_probe), + TF_ROLE(TF_PHASE_SEAMED, "__cbm_runtime_image_holder", tf_maybe_run_runtime_image_holder), + TF_ROLE(TF_PHASE_SEAMED, "__cbm_runtime_hello_client", tf_maybe_run_runtime_hello_client), + TF_ROLE(TF_PHASE_SEAMED, "__cbm_runtime_activation_client", tf_maybe_run_runtime_activation_client), + TF_ROLE(TF_PHASE_SEAMED, "__cbm_runtime_mapped_hello_client", tf_maybe_run_runtime_mapped_hello_client), + TF_ROLE(TF_PHASE_SEAMED, "__cbm_version_cohort_crash_holder", tf_maybe_run_version_cohort_crash_holder), + TF_ROLE(TF_PHASE_SEAMED, "__cbm_daemon_ipc_lock_probe", tf_maybe_run_daemon_ipc_lock_probe), + TF_ROLE(TF_PHASE_SEAMED, "__cbm_deleted_self_probe", tf_maybe_run_deleted_self_probe), + TF_ROLE(TF_PHASE_SEAMED, "__cbm_sockprobe", tf_maybe_run_socket_probe), + /* The supervised index worker (` cli --index-worker ...`): production + * code starts it only where a test re-enables the supervisor, which main + * pins off for every test process, so a spawner names one of the two. */ + TF_ROLE(TF_PHASE_WORKER, "--index-worker", tf_maybe_run_index_worker), + TF_ROLE(TF_PHASE_SPAWNER, "CBM_INDEX_SUPERVISOR", tf_maybe_run_index_worker), + /* The blocking-git probe: argv[0] (this runner copied as git) enters it, so + * main calls it directly; a spawner sets the marker it writes. */ + TF_ROLE(TF_PHASE_SPAWNER, "CBM_TEST_RUNTIME_BLOCKING_GIT_PID_FILE", tf_maybe_run_blocking_git_probe), +}; +// clang-format on + +/* Whether argv enters the role: argv[1] is its literal, or, for the index + * worker, any argument is (its own parser refuses a misplaced flag loudly). */ +static bool tf_role_named(tf_phase_t phase, int argc, char **argv, const char *literal) { + if (phase != TF_PHASE_WORKER) { + return strcmp(argv[1], literal) == 0; + } + for (int i = 1; i < argc; i++) { + if (argv[i] && strcmp(argv[i], literal) == 0) { + return true; + } + } + return false; +} + +/* Hands over to the role argv names, if this phase has it: its exit code, or + * -1 (no role here, or its handler let main continue). */ +static int tf_role_dispatch(tf_phase_t phase, int argc, char **argv) { + if (argc < 2 || !argv || !argv[1]) { + return -1; + } + for (size_t i = 0; i < sizeof(tf_roles) / sizeof(tf_roles[0]); i++) { + if (tf_roles[i].phase != phase || !tf_role_named(phase, argc, argv, tf_roles[i].literal)) { + continue; + } + int rc = tf_roles[i].run(argc, argv); + if (rc >= 0) { + return rc; + } + } + return -1; +} + const char *cbm_test_program_path; int main(int argc, char **argv) { cbm_test_program_path = tf_runner_image(argc, argv); - if (argc >= 4 && argc % 2 == 0 && strcmp(argv[1], "__cbm_env_values") == 0) { - for (int i = 2; i < argc; i += 2) { - const char *value = getenv(argv[i]); - if (!value || strcmp(value, argv[i + 1]) != 0) { - return 61; - } - } - return 0; - } - if (argc >= 5 && strcmp(argv[1], "__cbm_env_isolation") == 0) { - const char *value = getenv(argv[2]); - if (!value || strcmp(value, argv[3]) != 0) { - return 61; - } - for (int i = 4; i < argc; i++) { - if (getenv(argv[i])) { - return 62; - } - } - return 0; + int role_rc = tf_role_dispatch(TF_PHASE_RAW, argc, argv); + if (role_rc >= 0) { + return role_rc; } /* #2003: never let a caller's GIT_DIR/GIT_INDEX_FILE/... redirect fixture * git commands at the caller's real repository. */ th_clear_git_repo_env(); tf_test_impact_runner_filter_set_binary(tf_runner_image(argc, argv)); + tf_subprocess_set_binary(tf_runner_image(argc, argv)); + tf_test_impact_git_set_binary(tf_runner_image(argc, argv)); #ifdef CBM_TEST_COVERAGE tf_coverage_process_init(); #ifndef _WIN32 @@ -1972,47 +2142,19 @@ int main(int argc, char **argv) { } #endif #endif - int stdout_probe_rc = tf_maybe_run_subprocess_stdout_probe(argc, argv); - if (stdout_probe_rc >= 0) { - return stdout_probe_rc; - } - int git_facts_probe_rc = tf_maybe_run_git_facts_diff_probe(argc, argv); - if (git_facts_probe_rc >= 0) { - return git_facts_probe_rc; - } - int memory_limit_probe_rc = tf_maybe_run_windows_memory_limit_probe(argc, argv); - if (memory_limit_probe_rc >= 0) { - return memory_limit_probe_rc; + role_rc = tf_role_dispatch(TF_PHASE_GIT_CLEAN, argc, argv); + if (role_rc >= 0) { + return role_rc; } + /* argv[0] (this runner copied as git) enters the blocking-git probe, not + * argv[1], so main calls it here; tf_roles names its spawners. */ int blocking_git_rc = tf_maybe_run_blocking_git_probe(argc, argv); if (blocking_git_rc >= 0) { return blocking_git_rc; } - /* Installation tests use this executable as a structurally real candidate. - * Mirror the production binary's minimal verification contract. */ - if (argc == 2 && strcmp(argv[1], "--version") == 0) { - (void)puts("codebase-memory-mcp test-runner"); - return 0; - } - if (argc == 2 && strcmp(argv[1], "--build-config") == 0) { -#ifdef _WIN32 - if (_setmode(cbm_fileno(stdout), _O_BINARY) == -1) { - fprintf(stderr, "failed to set build-config stdout to binary mode\n"); - return 2; - } -#endif -#if defined(CBM_SANITIZED_BUILD) && CBM_SANITIZED_BUILD - const int sanitized = 1; -#else - const int sanitized = 0; -#endif -#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS - const int test_seams = 1; -#else - const int test_seams = 0; -#endif - (void)printf("sanitized=%d test_seams=%d\n", sanitized, test_seams); - return 0; + role_rc = tf_role_dispatch(TF_PHASE_INSTALL, argc, argv); + if (role_rc >= 0) { + return role_rc; } /* A test's verdict is a pure function of code, test, platform and seed -- * never of how full this machine's disk is. The spill store refuses to open @@ -2033,64 +2175,11 @@ int main(int argc, char **argv) { (void)cbm_setenv("CBM_TEST_BUILD_FINGERPRINT", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", 1); } - /* #1830 userns smoke probe -- see the test that spawns it. - * - * The ancestor overflow uid is derived ONCE per process (pthread_once in - * src/daemon/ipc.c) from /proc/self/uid_map. That file is not immutable: - * unshare(CLONE_NEWUSER) is exactly what changes it, and pthread_once state - * survives fork(). A forked child therefore keeps the HOST answer and never - * re-derives inside its new namespace, so a fork-only smoke test refuses and - * cannot pass once anything earlier in the suite has primed the cache -- - * seven call sites do. Re-exec into this probe so the decision is made by a - * process that STARTED inside the namespace, which is the production shape - * the test means to cover. */ -#if defined(__linux__) && defined(CBM_ENABLE_TEST_SEAMS) - if (argc == 3 && strcmp(argv[1], "--userns-secure-probe") == 0) { - /* _exit, not return: this process exists to answer ONE boolean. A - * return runs the atexit chain, and the runner is built with - * -fsanitize=address, so a future leak anywhere in the prologue would - * exit 23 and read as a security verdict on a test that has none. */ - _exit(cbm_daemon_ipc_private_directory_secure(argv[2]) ? 0 : 1); - } -#endif - int mcp_idxfailclosed_rc = tf_maybe_run_mcp_idxfailclosed_probe(argc, argv); - if (mcp_idxfailclosed_rc >= 0) { - return mcp_idxfailclosed_rc; - } - int runtime_image_rc = tf_maybe_run_runtime_image_holder(argc, argv); - if (runtime_image_rc >= 0) { - return runtime_image_rc; - } - int runtime_hello_rc = tf_maybe_run_runtime_hello_client(argc, argv); - if (runtime_hello_rc >= 0) { - return runtime_hello_rc; - } - int runtime_activation_rc = tf_maybe_run_runtime_activation_client(argc, argv); - if (runtime_activation_rc >= 0) { - return runtime_activation_rc; - } - int runtime_mapped_hello_rc = tf_maybe_run_runtime_mapped_hello_client(argc, argv); - if (runtime_mapped_hello_rc >= 0) { - return runtime_mapped_hello_rc; - } - int cohort_crash_rc = tf_maybe_run_version_cohort_crash_holder(argc, argv); - if (cohort_crash_rc >= 0) { - return cohort_crash_rc; - } - int daemon_ipc_probe_rc = tf_maybe_run_daemon_ipc_lock_probe(argc, argv); - if (daemon_ipc_probe_rc >= 0) { - return daemon_ipc_probe_rc; - } - int deleted_self_rc = tf_maybe_run_deleted_self_probe(argc, argv); - if (deleted_self_rc >= 0) { - return deleted_self_rc; - } - - /* #798 follow-up: if spawned as the socket-isolation probe, report whether an - * inheritable socket handle crossed into this child and exit before any suite. */ - int probe_rc = tf_maybe_run_socket_probe(argc, argv); - if (probe_rc >= 0) { - return probe_rc; + /* Roles that run after those pins: the #1830 userns probe, the daemon and + * runtime clients and holders, the #798 socket-isolation probe (tf_roles). */ + role_rc = tf_role_dispatch(TF_PHASE_SEAMED, argc, argv); + if (role_rc >= 0) { + return role_rc; } /* Capture once so the parent tests and any re-exec'd worker bind to this @@ -2100,9 +2189,9 @@ int main(int argc, char **argv) { /* #832: if spawned as a supervised index worker, do the real work and exit * before any suite runs (see tf_maybe_run_index_worker). */ - int worker_rc = tf_maybe_run_index_worker(argc, argv); - if (worker_rc >= 0) { - return worker_rc; + role_rc = tf_role_dispatch(TF_PHASE_WORKER, argc, argv); + if (role_rc >= 0) { + return role_rc; } /* #845 belt-and-suspenders: this binary EMBEDS cbm_mcp_handle_tool. The diff --git a/tests/test_select_lanes.sh b/tests/test_select_lanes.sh index 6cdc4bf84..a316630ce 100755 --- a/tests/test_select_lanes.sh +++ b/tests/test_select_lanes.sh @@ -161,7 +161,7 @@ if rc != 2 or "Please consult --help." not in err: # the cap, or a list that reaches the cap selects FULL (file_list_truncated); # the tier still classifies what was listed. A complete list decides as usual # (renames add their old path, so a complete list may exceed the count). -DOCS = ["README.md", "docs/a.md", "docs/b.md"] +DOCS = ["CONTRIBUTING.md", "docs/a.md", "docs/b.md"] CAP_LIST = [f"docs/page{i:04d}.md" for i in range(3000)] diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 74fab4555..c7f78b3d8 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -1784,18 +1784,23 @@ static void sp_stdout_payload(unsigned char bytes[SP_STDOUT_BYTES]) { bytes[SP_STDOUT_BYTES - 1] = 'Z'; /* final byte has no newline */ } -/* Called before normal harness setup. Normal entry remembers argv[0], whose - * storage lasts through main; the private exact argv grammar never runs suites. */ +/* Normal entry (test_main.c) remembers the runner image, whose storage lasts + * through main. */ +void tf_subprocess_set_binary(const char *path); +void tf_subprocess_set_binary(const char *path) { + sp_stdout_test_binary = path; +} + +/* Runner roles (test_main.c's role table), before normal harness setup; the + * private exact argv grammar never runs suites. */ static int sp_stdin_probe(int argc, char **argv); -const char *tf_runner_image(int argc, char **argv); /* test_main.c */ -int tf_maybe_run_subprocess_stdout_probe(int argc, char **argv); -int tf_maybe_run_subprocess_stdout_probe(int argc, char **argv) { - if (argc >= 2 && argv && strcmp(argv[1], "__cbm_subprocess_stdin_probe") == 0) - return sp_stdin_probe(argc, argv); - if (argc < 2 || !argv || strcmp(argv[1], "__cbm_subprocess_stdout_probe") != 0) { - sp_stdout_test_binary = tf_runner_image(argc, argv); - return -1; - } +int tf_role_subprocess_stdin_probe(int argc, char **argv); +int tf_role_subprocess_stdin_probe(int argc, char **argv) { + return sp_stdin_probe(argc, argv); +} + +int tf_role_subprocess_stdout_probe(int argc, char **argv); +int tf_role_subprocess_stdout_probe(int argc, char **argv) { if (argc != 4 || (strcmp(argv[2], "binary") != 0 && strcmp(argv[2], "text") != 0 && strcmp(argv[2], "empty") != 0)) return 91; diff --git a/tests/test_test_impact.c b/tests/test_test_impact.c index c88bdf909..0236535ea 100644 --- a/tests/test_test_impact.c +++ b/tests/test_test_impact.c @@ -2563,6 +2563,42 @@ TEST(test_policy_config_change_runs_all) { PASS(); } +TEST(test_policy_repository_rules_map_every_kind_of_path) { + /* This repository's own rules: a path no rule maps runs every test (unmapped_files), so + * each kind of tracked file has a rule, and only real build inputs run everything. */ + cbm_test_config_t *config = cbm_test_config_load(".codebase-memory.json", false); + ASSERT_NOT_NULL(config); + cbm_test_policy_t *policy = cbm_test_policy_new(config); + ASSERT_NOT_NULL(policy); + static const char *const cases[][2] = { + {"graph-ui/package.json", "repo-data"}, + {"graph-ui/src/main.tsx", "repo-data"}, + {"tools/tree-sitter-pascal/src/parser.c", "repo-data"}, + {"tests/fuzz/corpus/cypher/match.seed", "repo-data"}, + {"flake.nix", "repo-data"}, + {"DCO", "repo-data"}, + {"server.json", "repo-data"}, + {"scripts/security-ui.sh", "repo-data"}, + {"tests/test_windows_preflight_contract.ps1", "contracts"}, + {".gitattributes", "build"}, + {"scripts/build.sh", "build"}, + {"Makefile.cbm", "builtin:build"}, + {"tests/test_main.c", "harness"}, + {"src/mcp/test_impact_engine.c", "engine"}, + {"src/pipeline/pipeline.c", NULL}, + {"src/server.json", NULL}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + if (!policy_matches(policy, cases[i][0], cases[i][1])) { + printf(" %s: expected rule %s\n", cases[i][0], cases[i][1] ? cases[i][1] : "(none)"); + FAIL("repository rule mapping"); + } + } + cbm_test_policy_free(policy); + cbm_test_config_free(config); + PASS(); +} + TEST(test_policy_lanes_and_semantic_perf_membership) { cbm_test_policy_t *policy = policy_of( "{\"test_impact\":{\"version\":1," @@ -3748,6 +3784,7 @@ SUITE(test_impact) { RUN_TEST(test_policy_defaults_and_first_match_before_target_resolution); RUN_TEST(test_policy_coverage_compatibility_paths_are_strict); RUN_TEST(test_policy_config_change_runs_all); + RUN_TEST(test_policy_repository_rules_map_every_kind_of_path); RUN_TEST(test_policy_lanes_and_semantic_perf_membership); RUN_TEST(test_policy_globs_braces_escapes_and_directory_boundaries); RUN_TEST(test_policy_rejects_invalid_consumed_fields_without_changing_seed_loader); diff --git a/tests/test_test_impact_engine.c b/tests/test_test_impact_engine.c index ea50101da..502bda2f6 100644 --- a/tests/test_test_impact_engine.c +++ b/tests/test_test_impact_engine.c @@ -180,6 +180,9 @@ static void tie_close(tie_fixture_t *fx) { (void)th_rmtree(fx->home); } +/* The engine's diagnostic note of the last tie_answer_with call. */ +static char tie_last_note[512]; + /* The answer for the topic branch against main, as JSON (owned); with a * team artifact bundle when one is given. */ static char *tie_answer_with(tie_fixture_t *fx, const char *bundle, const char *commit, @@ -201,6 +204,7 @@ static char *tie_answer_with(tie_fixture_t *fx, const char *bundle, const char * if (diagnostic[0]) { printf(" engine note: %s\n", diagnostic); } + snprintf(tie_last_note, sizeof(tie_last_note), "%s", diagnostic); size_t len = 0; const char *json = cbm_test_result_json(out, &len); char *copy = json ? strdup(json) : NULL; @@ -212,6 +216,33 @@ static char *tie_answer(tie_fixture_t *fx) { return tie_answer_with(fx, NULL, NULL, false); } +/* The answer, plus in *vetoed the selection the runner veto discarded (NULL + * when the engine wrote none). */ +static char *tie_answer_vetoed(tie_fixture_t *fx, char **vetoed) { + cbm_test_result_t *discarded = NULL; + cbm_test_impact_request_t rq = {.repo_root = fx->repo, + .base_ref = "main", + .work_parent = fx->work, + .deadline_ms = cbm_now_ms() + 120000U, + .platform = "fixture", + .vetoed_out = &discarded}; + cbm_test_result_t *out = NULL; + char diagnostic[512]; + *vetoed = NULL; + if (cbm_test_impact_run(&rq, &out, diagnostic, sizeof(diagnostic)) != CBM_TEST_IMPACT_OK) { + printf(" engine: %s\n", diagnostic); + return NULL; + } + size_t len = 0; + const char *json = cbm_test_result_json(out, &len); + const char *masked = discarded ? cbm_test_result_json(discarded, &len) : NULL; + char *copy = json ? strdup(json) : NULL; + *vetoed = masked ? strdup(masked) : NULL; + cbm_test_result_free(out); + cbm_test_result_free(discarded); + return copy; +} + /* The "reasons" array of one suite's entry in the answer. */ static bool tie_suite_has(const char *json, const char *suite, const char *reason) { char key[128]; @@ -279,6 +310,7 @@ TEST(test_impact_engine_rules_unmapped_and_empty) { char *json = tie_answer(&fx); ASSERT_NOT_NULL(json); ASSERT_NOT_NULL(strstr(json, "\"decision\":\"nothing\"")); + ASSERT_NOT_NULL(strstr(json, "\"run_all_rules\":[]")); free(json); ASSERT_TRUE(tie_write(&fx, "Makefile", "all:\n\techo\n")); @@ -287,6 +319,8 @@ TEST(test_impact_engine_rules_unmapped_and_empty) { ASSERT_NOT_NULL(json); ASSERT_NOT_NULL(strstr(json, "\"decision\":\"run_all\"")); ASSERT_NOT_NULL(strstr(json, "RULE_RUN_ALL")); + /* The answer names the rule that ran everything (the repository rule wins). */ + ASSERT_NOT_NULL(strstr(json, "\"run_all_rules\":[\"build\"]")); free(json); const char *reset[] = {"reset", "--quiet", "--hard", "main", NULL}; @@ -296,6 +330,367 @@ TEST(test_impact_engine_rules_unmapped_and_empty) { json = tie_answer(&fx); ASSERT_NOT_NULL(json); ASSERT_NOT_NULL(strstr(json, "UNMAPPED_FILE")); + ASSERT_NOT_NULL(strstr(json, "\"run_all_rules\":[]")); + free(json); + tie_close(&fx); + PASS(); +} + +/* A referencing_tests rule keyed by the whole path, unreferenced: ignore. A + * changed data file a test names selects that test; one no test names selects + * nothing — instead of running everything (scripts, packaging and docs that no + * C test reads cannot change a C test's outcome). */ +static const char tie_config_named_data[] = + "{\"test_impact\":{\"version\":1,\"tests\":{\"presets\":{\"c-cbm\":true}}," + "\"rules\":[{\"id\":\"build\",\"paths\":[\"Makefile\"],\"action\":\"run_all\"}," + "{\"id\":\"data\",\"paths\":[\"data/**\"],\"action\":\"referencing_tests\"," + "\"key\":\"path\",\"unreferenced\":\"ignore\"}]}}\n"; + +static const char tie_beta_reads_data[] = "TEST(beta_reads_table) {\n" + " const char *path = \"data/named.txt\";\n" + " return path[0] == 'd' ? 0 : 1;\n" + "}\n" + "SUITE(beta) {\n" + " RUN_TEST(beta_reads_table);\n" + "}\n"; + +/* The answer's top-level decision: its keys are written sorted, so the + * document starts with it (a lane's own "decision" comes later). */ +static bool tie_top_decision(const char *json, const char *want) { + char prefix[64]; + snprintf(prefix, sizeof(prefix), "{\"decision\":\"%s\"", want); + return json && strncmp(json, prefix, strlen(prefix)) == 0; +} + +TEST(test_impact_engine_path_key_selects_naming_tests_or_ignores) { + tie_fixture_t fx; + const char *const files[] = {".codebase-memory.json", tie_config_named_data, + "tests/test_beta.c", tie_beta_reads_data, + "data/named.txt", "one\n", + "data/other.txt", "two\n"}; + ASSERT_TRUE(tie_open_files(&fx, files, 4)); + ASSERT_TRUE(tie_write(&fx, "data/other.txt", "two, changed\n")); + ASSERT_TRUE(tie_commit(&fx, "data no test names")); + char *json = tie_answer(&fx); + ASSERT_NOT_NULL(json); + if (tie_top_decision(json, "run_all")) { + printf(" %.900s\n", json); + } + ASSERT_FALSE(tie_top_decision(json, "run_all")); + ASSERT_NULL(strstr(json, "RULE_TARGET_UNKNOWN")); + free(json); + + const char *reset[] = {"reset", "--quiet", "--hard", "main", NULL}; + ASSERT_TRUE(tie_git(&fx, reset)); + ASSERT_TRUE(tie_write(&fx, "data/named.txt", "one, changed\n")); + ASSERT_TRUE(tie_commit(&fx, "data a test names")); + json = tie_answer(&fx); + ASSERT_NOT_NULL(json); + if (!tie_suite_has(json, "beta", "STATIC")) { + printf(" named: %.1400s\n", json); + } + ASSERT_FALSE(tie_top_decision(json, "run_all")); + ASSERT_TRUE(tie_suite_has(json, "beta", "STATIC")); + ASSERT_FALSE(tie_suite_has(json, "alpha", "STATIC")); + free(json); + tie_close(&fx); + PASS(); +} + +/* ── Child-process roles: the runner's TF_ROLE table ───────────────── + * The fixture runner dispatches its roles from a table by argv[1]; a change + * that reaches a role handler selects the tests that spawn the role, not + * every test (src/mcp/test_impact_engine_reach.c te_role_lift). */ +static const char tie_roles_main[] = + "#include \"test_framework.h\"\n" + "#include \n" + "int lib_value(void);\n" + "int lib_other(void);\n" + "int lib_third(void);\n" + "static int role_probe(int argc, char **argv) {\n" + " return lib_value();\n" + "}\n" + "static int role_inner(int argc, char **argv) {\n" + " return lib_third();\n" + "}\n" + "static int role_outer(int argc, char **argv) {\n" + " return role_inner(argc, argv);\n" + "}\n" + "static int role_orphan(int argc, char **argv) {\n" + " return lib_other();\n" + "}\n" + "typedef struct { const char *literal; int (*run)(int, char **); } tf_role_t;\n" + "#define TF_ROLE(phase, literal, handler) {literal, handler}\n" + "static const tf_role_t tf_roles[] = {\n" + " TF_ROLE(0, \"__fx_probe_role\", role_probe),\n" + " TF_ROLE(0, \"__fx_inner_role\", role_inner),\n" + " TF_ROLE(0, \"__fx_outer_role\", role_outer),\n" + " TF_ROLE(0, \"__fx_orphan_role\", role_orphan),\n" + "};\n" + "int main(int argc, char **argv) {\n" + " for (int i = 0; argc > 1 && i < 4; i++) {\n" + " if (strcmp(argv[1], tf_roles[i].literal) == 0) {\n" + " return tf_roles[i].run(argc, argv);\n" + " }\n" + " }\n" + " RUN_SELECTED_SUITE(alpha);\n" + " RUN_SELECTED_SUITE(beta);\n" + " RUN_SELECTED_SUITE(matrix);\n" + " return 0;\n" + "}\n"; + +/* beta spawns the probe role directly, the outer and inner roles directly, + * and the probe role once more through a helper. */ +static const char tie_roles_beta[] = "static int spawn_probe(void) {\n" + " const char *argv1 = \"__fx_probe_role\";\n" + " return argv1[0] == '_' ? 0 : 1;\n" + "}\n" + "TEST(beta_spawns_probe) {\n" + " const char *argv1 = \"__fx_probe_role\";\n" + " return argv1[0] == '_' ? 0 : 1;\n" + "}\n" + "TEST(beta_spawns_outer) {\n" + " const char *argv1 = \"__fx_outer_role\";\n" + " return argv1[0] == '_' ? 0 : 1;\n" + "}\n" + "TEST(beta_spawns_inner) {\n" + " const char *argv1 = \"__fx_inner_role\";\n" + " return argv1[0] == '_' ? 0 : 1;\n" + "}\n" + "TEST(beta_plain) {\n" + " return 0;\n" + "}\n" + "SUITE(beta) {\n" + " RUN_TEST(beta_spawns_probe);\n" + " RUN_TEST(beta_spawns_outer);\n" + " RUN_TEST(beta_spawns_inner);\n" + " RUN_TEST(beta_plain);\n" + "}\n"; + +/* matrix reaches the probe role only through a helper that names it. */ +static const char tie_roles_matrix[] = "int spawn_probe_helper(void) {\n" + " const char *argv1 = \"__fx_probe_role\";\n" + " return argv1[0] == '_' ? 0 : 1;\n" + "}\n" + "TEST(matrix_via_helper) {\n" + " return spawn_probe_helper();\n" + "}\n" + "SUITE(matrix) {\n" + " RUN_TEST(matrix_via_helper);\n" + "}\n"; + +static const char tie_roles_lib[] = "int lib_value(void) {\n" + " return 1;\n" + "}\n" + "int lib_other(void) {\n" + " return 2;\n" + "}\n" + "int lib_third(void) {\n" + " return 3;\n" + "}\n"; + +static bool tie_open_roles(tie_fixture_t *fx) { + const char *const files[] = {"tests/test_main.c", tie_roles_main, "tests/test_beta.c", + tie_roles_beta, "tests/test_matrix.c", tie_roles_matrix, + "src/lib.c", tie_roles_lib}; + return tie_open_files(fx, files, 4); +} + +/* A change reaching a role handler selects the tests that spawn the role + * (directly, or through a helper that names it) — not every test. */ +TEST(test_impact_engine_role_reach_selects_its_spawners) { + tie_fixture_t fx; + ASSERT_TRUE(tie_open_roles(&fx)); + ASSERT_TRUE(tie_write(&fx, "src/lib.c", + "int lib_value(void) {\n return 1 + 0;\n}\n" + "int lib_other(void) {\n return 2;\n}\n" + "int lib_third(void) {\n return 3;\n}\n")); + ASSERT_TRUE(tie_commit(&fx, "change lib_value")); + char *json = tie_answer(&fx); + ASSERT_NOT_NULL(json); + if (tie_top_decision(json, "run_all") || !tie_suite_has(json, "beta", "STATIC")) { + printf(" %.1600s\n", json); + } + ASSERT_FALSE(tie_top_decision(json, "run_all")); + ASSERT_TRUE(tie_suite_has(json, "alpha", "STATIC")); /* calls lib_value itself */ + ASSERT_TRUE(tie_suite_has(json, "beta", "STATIC")); /* spawns the probe role */ + ASSERT_TRUE(tie_suite_has(json, "matrix", "STATIC")); /* through its helper */ + free(json); + tie_close(&fx); + PASS(); +} + +/* A handler that calls another runs it in its own child: a change reaching + * the inner handler selects the outer role's spawners as well. */ +TEST(test_impact_engine_chained_roles_select_both_spawners) { + tie_fixture_t fx; + ASSERT_TRUE(tie_open_roles(&fx)); + ASSERT_TRUE(tie_write(&fx, "src/lib.c", + "int lib_value(void) {\n return 1;\n}\n" + "int lib_other(void) {\n return 2;\n}\n" + "int lib_third(void) {\n return 3 + 0;\n}\n")); + ASSERT_TRUE(tie_commit(&fx, "change lib_third")); + char *json = tie_answer(&fx); + ASSERT_NOT_NULL(json); + if (tie_top_decision(json, "run_all")) { + printf(" %.1600s\n", json); + } + ASSERT_FALSE(tie_top_decision(json, "run_all")); + ASSERT_TRUE(tie_suite_has(json, "beta", "STATIC")); + ASSERT_FALSE(tie_suite_has(json, "alpha", "STATIC")); + free(json); + tie_close(&fx); + PASS(); +} + +/* Fail safe: a reached role no test names runs everything, and so does a + * change reaching main by any other path. */ +TEST(test_impact_engine_unspawned_role_and_main_still_run_all) { + tie_fixture_t fx; + ASSERT_TRUE(tie_open_roles(&fx)); + ASSERT_TRUE(tie_write(&fx, "src/lib.c", + "int lib_value(void) {\n return 1;\n}\n" + "int lib_other(void) {\n return 2 + 0;\n}\n" + "int lib_third(void) {\n return 3;\n}\n")); + ASSERT_TRUE(tie_commit(&fx, "change lib_other")); + char *vetoed = NULL; + char *json = tie_answer_vetoed(&fx, &vetoed); + ASSERT_NOT_NULL(json); + ASSERT_TRUE(tie_top_decision(json, "run_all")); + ASSERT_NOT_NULL(strstr(json, "RUNNER_REACHED")); + /* The veto is the one reason: CI also gets the selection it discarded. */ + ASSERT_NOT_NULL(vetoed); + ASSERT_FALSE(tie_top_decision(vetoed, "run_all")); + ASSERT_NULL(strstr(vetoed, "RUNNER_REACHED")); + free(vetoed); + free(json); + + const char *reset[] = {"reset", "--quiet", "--hard", "main", NULL}; + ASSERT_TRUE(tie_git(&fx, reset)); + ASSERT_TRUE(tie_write(&fx, "tests/test_main.c", + "#include \"test_framework.h\"\n" + "int lib_value(void);\n" + "static int worker(int argc, char **argv) {\n" + " return argc > 9 ? lib_value() : -1;\n" + "}\n" + "int main(int argc, char **argv) {\n" + " if (worker(argc, argv) >= 0) {\n" + " return 0;\n" + " }\n" + " RUN_SELECTED_SUITE(alpha);\n" + " RUN_SELECTED_SUITE(beta);\n" + " RUN_SELECTED_SUITE(matrix);\n" + " return 0;\n" + "}\n")); + ASSERT_TRUE(tie_commit(&fx, "a role main calls directly")); + const char *base[] = {"branch", "--quiet", "-f", "main", "HEAD", NULL}; + ASSERT_TRUE(tie_git(&fx, base)); + ASSERT_TRUE(tie_write(&fx, "src/lib.c", + "int lib_value(void) {\n return 1 + 0;\n}\n" + "int lib_other(void) {\n return 2;\n}\n" + "int lib_third(void) {\n return 3;\n}\n")); + ASSERT_TRUE(tie_commit(&fx, "change lib_value")); + json = tie_answer(&fx); + ASSERT_NOT_NULL(json); + ASSERT_TRUE(tie_top_decision(json, "run_all")); + ASSERT_NOT_NULL(strstr(json, "RUNNER_REACHED")); + /* The note names the way in: main, the role it calls, the changed code. */ + ASSERT_NOT_NULL(strstr(tie_last_note, "test_main.main -> test_main.worker -> lib.lib_value")); + free(json); + tie_close(&fx); + PASS(); +} + +/* The role fixture with a second, never-dispatched literal for the orphan role + * and `beta` as given; the answer for a change that reaches only that role + * (NULL on a fixture failure). */ +static char *tie_answer_orphan_switch(tie_fixture_t *fx, const char *beta) { + const char *anchor = " TF_ROLE(0, \"__fx_orphan_role\", role_orphan),\n"; + const char *at = strstr(tie_roles_main, anchor); + size_t head = at ? (size_t)(at - tie_roles_main) + strlen(anchor) : 0; + char runner[4096]; + int n = snprintf(runner, sizeof(runner), + "%.*s TF_ROLE(9, \"__FX_ORPHAN_SWITCH\", role_orphan),\n%s", (int)head, + tie_roles_main, tie_roles_main + head); + const char *base[] = {"branch", "--quiet", "-f", "main", "HEAD", NULL}; + if (!at || n <= 0 || (size_t)n >= sizeof(runner) || !tie_open_roles(fx) || + !tie_write(fx, "tests/test_main.c", runner) || !tie_write(fx, "tests/test_beta.c", beta) || + !tie_commit(fx, "a second literal for the orphan role") || !tie_git(fx, base) || + !tie_write(fx, "src/lib.c", + "int lib_value(void) {\n return 1;\n}\n" + "int lib_other(void) {\n return 2 + 0;\n}\n" + "int lib_third(void) {\n return 3;\n}\n") || + !tie_commit(fx, "change lib_other")) { + return NULL; + } + char *json = tie_answer(fx); + if (json && (strstr(json, "RUNNER_REACHED") || !tie_suite_has(json, "beta", "STATIC"))) { + printf(" note: %s\n %.1600s\n", tie_last_note, json); + } + return json; +} + +/* A handler with two literals (the index worker: its argv flag and the switch + * that lets production code start it): a test naming either one is a spawner, + * and the unnamed one alone vetoes nothing. */ +TEST(test_impact_engine_role_with_two_literals_selects_both_spawners) { + tie_fixture_t fx; + char *json = tie_answer_orphan_switch(&fx, "TEST(beta_sets_switch) {\n" + " const char *name = \"__FX_ORPHAN_SWITCH\";\n" + " return name[0] == '_' ? 0 : 1;\n" + "}\n" + "TEST(beta_plain) {\n return 0;\n}\n" + "SUITE(beta) {\n" + " RUN_TEST(beta_sets_switch);\n" + " RUN_TEST(beta_plain);\n" + "}\n"); + ASSERT_NOT_NULL(json); + ASSERT_NULL(strstr(json, "RUNNER_REACHED")); + ASSERT_TRUE(tie_suite_has(json, "beta", "STATIC")); + free(json); + tie_close(&fx); + PASS(); +} + +/* The literal behind a #define (the blocking-git marker): the tests that use + * the macro are the spawners. */ +TEST(test_impact_engine_role_literal_behind_a_define_selects_its_users) { + tie_fixture_t fx; + char *json = tie_answer_orphan_switch(&fx, "#define BETA_SWITCH \"__FX_ORPHAN_SWITCH\"\n" + "TEST(beta_sets_switch) {\n" + " const char *name = BETA_SWITCH;\n" + " return name[0] == '_' ? 0 : 1;\n" + "}\n" + "TEST(beta_plain) {\n return 0;\n}\n" + "SUITE(beta) {\n" + " RUN_TEST(beta_sets_switch);\n" + " RUN_TEST(beta_plain);\n" + "}\n"); + ASSERT_NOT_NULL(json); + ASSERT_NULL(strstr(json, "RUNNER_REACHED")); + ASSERT_TRUE(tie_suite_has(json, "beta", "STATIC")); + free(json); + tie_close(&fx); + PASS(); +} + +/* A struct member named like a suite: the extractor can take the macro argument + * of RUN_SELECTED_SUITE(alpha) for a use of it. A changed member reaches main + * only through that edge into a Field: no veto. */ +TEST(test_impact_engine_field_call_is_no_way_into_main) { + tie_fixture_t fx; + ASSERT_TRUE( + tie_open_with(&fx, "tests/shapes.h", + "typedef struct {\n char alpha[8];\n int sides;\n} shape_t;\n")); + ASSERT_TRUE(tie_write(&fx, "tests/shapes.h", + "typedef struct {\n char alpha[16];\n int sides;\n} shape_t;\n")); + ASSERT_TRUE(tie_commit(&fx, "widen the member")); + char *json = tie_answer(&fx); + ASSERT_NOT_NULL(json); + if (strstr(json, "RUNNER_REACHED")) { + printf(" note: %s\n", tie_last_note); + } + ASSERT_NULL(strstr(json, "RUNNER_REACHED")); free(json); tie_close(&fx); PASS(); @@ -622,6 +1017,13 @@ SUITE(test_impact_engine) { RUN_TEST(test_impact_engine_reached_suites_carry_static); RUN_TEST(test_impact_engine_macro_registered_tests_select_their_suite); RUN_TEST(test_impact_engine_rules_unmapped_and_empty); + RUN_TEST(test_impact_engine_path_key_selects_naming_tests_or_ignores); + RUN_TEST(test_impact_engine_role_reach_selects_its_spawners); + RUN_TEST(test_impact_engine_chained_roles_select_both_spawners); + RUN_TEST(test_impact_engine_unspawned_role_and_main_still_run_all); + RUN_TEST(test_impact_engine_role_with_two_literals_selects_both_spawners); + RUN_TEST(test_impact_engine_role_literal_behind_a_define_selects_its_users); + RUN_TEST(test_impact_engine_field_call_is_no_way_into_main); RUN_TEST(test_impact_engine_parse_gaps_reach_what_they_name); RUN_TEST(test_impact_engine_admitted_coverage_narrows_only_when_verified); RUN_TEST(test_impact_engine_coverage_reaches_unmapped_code_through_mapped_callers); diff --git a/tests/test_test_impact_git.c b/tests/test_test_impact_git.c index 0964429b6..4f9fe1cc2 100644 --- a/tests/test_test_impact_git.c +++ b/tests/test_test_impact_git.c @@ -50,37 +50,46 @@ static bool gf_write(const char *path, const void *bytes, size_t length) { return ok; } -/* Native executable fixture for Git's optional diff/textconv command. Extra - * arguments added by Git are deliberately ignored. Normal harness entry keeps - * argv[0] for a private executable copy; neither mode changes the environment. */ -const char *tf_runner_image(int argc, char **argv); /* test_main.c */ -int tf_maybe_run_git_facts_diff_probe(int argc, char **argv); -int tf_maybe_run_git_facts_diff_probe(int argc, char **argv) { - if (argc >= 2 && argv && strcmp(argv[1], "__cbm_git_facts_diff_probe") == 0) { - return argc >= 3 && gf_write(argv[2], "invoked\n", 8) ? 0 : 96; +/* Normal harness entry (test_main.c) keeps the runner image for a private + * executable copy. */ +void tf_test_impact_git_set_binary(const char *path); +void tf_test_impact_git_set_binary(const char *path) { + gf_runner_binary = path; +} + +/* Runner role (test_main.c's role table): the native executable fixture for + * Git's optional diff/textconv command. Extra arguments added by Git are + * deliberately ignored; the environment is left unchanged. */ +int tf_role_git_facts_diff_probe(int argc, char **argv); +int tf_role_git_facts_diff_probe(int argc, char **argv) { + return argc >= 3 && gf_write(argv[2], "invoked\n", 8) ? 0 : 96; +} + +/* Runner role: only this isolated child mutates its environment, before + * harness threads start, and returns -1 so main re-enters the existing + * exact-diff test; the parent process remains untouched. An inherited + * CBM_TEST_ONLY_FILE (a narrowed CI run) is cleared: the runner unions it with + * CBM_TEST_ONLY, and one naming this suite would re-run the spawning test, + * which spawns again. An inherited CBM_TEST_COVERAGE_DIR (a coverage-map run) + * is cleared too: with it the child would start a coverage run of its own, + * which refuses the per-test LLVM_PROFILE_FILE it inherits. Without it the + * child's profile goes to that file, credited to the spawning test like any + * other child process. */ +int tf_role_git_facts_hostile_diff_probe(int argc, char **argv); +int tf_role_git_facts_hostile_diff_probe(int argc, char **argv) { + if (argc != 2) { + return -1; } - if (argc == 2 && argv && strcmp(argv[1], "__cbm_git_facts_hostile_diff_probe") == 0) { - /* Only this isolated child mutates its environment, before harness threads start. - * Re-enter the existing exact-diff test; the parent process remains untouched. - * An inherited CBM_TEST_ONLY_FILE (a narrowed CI run) is cleared: the runner - * unions it with CBM_TEST_ONLY, and one naming this suite would re-run the - * spawning test, which spawns again. An inherited CBM_TEST_COVERAGE_DIR (a - * coverage-map run) is cleared too: with it the child would start a coverage - * run of its own, which refuses the per-test LLVM_PROFILE_FILE it inherits. - * Without it the child's profile goes to that file, credited to the spawning - * test like any other child process. */ - if (cbm_setenv("GIT_DIFF_OPTS", "--unified=999", 1) != 0 || - cbm_setenv("CBM_TEST_ONLY_FILE", "", 1) != 0 || - cbm_unsetenv("CBM_TEST_COVERAGE_DIR") != 0 || - cbm_setenv( - "CBM_TEST_ONLY", - "test_impact_git:test_git_facts_diff_uses_merge_base_zero_context_and_nul_metadata", - 1) != 0) { - return 96; - } - argv[1] = "test_impact_git"; + if (cbm_setenv("GIT_DIFF_OPTS", "--unified=999", 1) != 0 || + cbm_setenv("CBM_TEST_ONLY_FILE", "", 1) != 0 || + cbm_unsetenv("CBM_TEST_COVERAGE_DIR") != 0 || + cbm_setenv( + "CBM_TEST_ONLY", + "test_impact_git:test_git_facts_diff_uses_merge_base_zero_context_and_nul_metadata", + 1) != 0) { + return 96; } - gf_runner_binary = tf_runner_image(argc, argv); + argv[1] = "test_impact_git"; return -1; } diff --git a/tests/test_test_impact_runner_filter.c b/tests/test_test_impact_runner_filter.c index 3907798f3..0f206cecb 100644 --- a/tests/test_test_impact_runner_filter.c +++ b/tests/test_test_impact_runner_filter.c @@ -57,6 +57,28 @@ TEST(test_impact_runner_filter_consumes_public_codec_bytes) { PASS(); } +TEST(test_impact_runner_filter_lane_running_every_suite_whole_is_run_all) { + /* A lane that lists every member suite whole runs everything: it says run_all + * rather than handing the runner a token list that happens to name it all. */ + const rf_spec_t sole = {.suite = "str_intern", .test = "rf_case", .whole = true, .sole = true}; + rf_fixture_t fixture; + bool prepared = rf_fixture_open(&fixture) && rf_prepare(&fixture, &sole); + cbm_test_result_status_t status = + prepared ? rf_build(&fixture, RF_JSON_CAP, false) : CBM_TEST_RESULT_INVALID; + bool parsed = status == CBM_TEST_RESULT_OK && rf_parse(&fixture); + yyjson_val *lane = parsed ? rf_lane(&fixture) : NULL; + bool run_all = lane && yyjson_equals_str(yyjson_obj_get(lane, "decision"), "run_all"); + if (!run_all) { + printf(" sole whole-suite lane: status=%d parsed=%d lane=%d\n", status, parsed, + lane != NULL); + } + bool removed = rf_fixture_close(&fixture); + ASSERT_TRUE(prepared); + ASSERT_TRUE(run_all); + ASSERT_TRUE(removed); + PASS(); +} + static bool rf_unknown_run(rf_fixture_t *fixture, const char *token, size_t length) { rf_run_t run = {0}; char diagnostic[640]; @@ -132,7 +154,7 @@ static bool rf_limit_control(void) { static bool rf_diagnostic_control(void) { char name[513]; rf_identifier(name, 512); - const rf_spec_t spec = {.suite = name, .test = "rf_case", .whole = true}; + const rf_spec_t spec = {.suite = name, .test = "rf_case", .whole = true, .sole = true}; rf_fixture_t fixture; bool prepared = rf_fixture_open(&fixture) && rf_prepare(&fixture, &spec); cbm_test_result_status_t status = @@ -165,6 +187,7 @@ TEST(test_impact_runner_filter_limits_and_diagnostic_domain) { SUITE(test_impact_runner_filter) { RUN_TEST(test_impact_runner_filter_consumes_public_codec_bytes); + RUN_TEST(test_impact_runner_filter_lane_running_every_suite_whole_is_run_all); RUN_TEST(test_impact_runner_filter_literal_width_boundaries); RUN_TEST(test_impact_runner_filter_limits_and_diagnostic_domain); } diff --git a/tests/test_test_impact_runner_filter.h b/tests/test_test_impact_runner_filter.h index 8bee58858..db9a0b2ff 100644 --- a/tests/test_test_impact_runner_filter.h +++ b/tests/test_test_impact_runner_filter.h @@ -12,6 +12,7 @@ typedef struct { const char *test; bool whole; bool mixed; + bool sole; /* no unselected rf_quiet suite: the spec's suites are the whole lane */ } rf_spec_t; typedef struct { diff --git a/tests/test_test_impact_runner_filter_fixture.c b/tests/test_test_impact_runner_filter_fixture.c index 16dbff2e2..ff822acbf 100644 --- a/tests/test_test_impact_runner_filter_fixture.c +++ b/tests/test_test_impact_runner_filter_fixture.c @@ -46,29 +46,46 @@ static bool rf_model(rf_fixture_t *fixture, const rf_spec_t *spec) { !cbm_test_model_add_source(fixture->model, "tests/rf_whole.c", other, sizeof(other) - 1)) { return false; } + /* A lane member the change does not reach keeps a whole-suite selection narrowed. */ + static const char quiet[] = "TEST(rf_quiet) {}\nSUITE(rf_quiet) { RUN_TEST(rf_quiet); }\n" + "void quiet_main(void) { RUN_SELECTED_SUITE(rf_quiet); }\n"; + if (!spec->sole && + !cbm_test_model_add_source(fixture->model, "tests/rf_quiet.c", quiet, sizeof(quiet) - 1)) { + return false; + } return cbm_test_model_finish(fixture->model) && cbm_test_model_complete(fixture->model); } static bool rf_selection(rf_fixture_t *fixture, const rf_spec_t *spec) { char rows[4096]; - int count = snprintf(rows, sizeof(rows), - "%s:*\tcomplete\t\t1\n%s:%s\tcomplete\t\t0\n" - "%s:rf_spare\tcomplete\t\t1\n%s", - spec->suite, spec->suite, spec->test, spec->suite, - spec->mixed ? "str_intern:*\tcomplete\t\t1\n" - "str_intern:rf_other\tcomplete\t\t1\n" - : ""); + int count = + snprintf(rows, sizeof(rows), + "%s:*\tcomplete\t\t1\n%s:%s\tcomplete\t\t0\n" + "%s:rf_spare\tcomplete\t\t1\n%s%s", + spec->suite, spec->suite, spec->test, spec->suite, + spec->mixed ? "str_intern:*\tcomplete\t\t1\n" + "str_intern:rf_other\tcomplete\t\t1\n" + : "", + spec->sole ? "" : "rf_quiet:*\tcomplete\t\t1\nrf_quiet:rf_quiet\tcomplete\t\t1\n"); static const char functions[] = "0\tsrc/changed.c\tchanged\n1\tsrc/quiet.c\tquiet\n"; if (count < 0 || (size_t)count >= sizeof(rows)) { return false; } fixture->coverage = cbm_coverage_map_parse(functions, sizeof(functions) - 1, rows, (size_t)count); - cbm_test_reach_t reach[] = { + cbm_test_reach_t reach[4] = { {.file = "tests/rf_subject.c", .test = spec->test, .mapped = true, .reached = true}, {.file = "tests/rf_subject.c", .test = "rf_spare", .mapped = true}, - {.file = "tests/rf_whole.c", .test = "rf_other", .mapped = true}, }; + size_t reach_count = 2; + if (spec->mixed) { + reach[reach_count++] = + (cbm_test_reach_t){.file = "tests/rf_whole.c", .test = "rf_other", .mapped = true}; + } + if (!spec->sole) { + reach[reach_count++] = + (cbm_test_reach_t){.file = "tests/rf_quiet.c", .test = "rf_quiet", .mapped = true}; + } cbm_test_suite_trigger_t trigger = {.suite = spec->mixed ? "str_intern" : spec->suite, .reached = true}; const int changed = 0; @@ -76,7 +93,7 @@ static bool rf_selection(rf_fixture_t *fixture, const rf_spec_t *spec) { .model = fixture->model, .coverage = fixture->coverage, .reach = reach, - .reach_count = spec->mixed ? 3 : 2, + .reach_count = reach_count, .suite_triggers = &trigger, .suite_trigger_count = spec->whole || spec->mixed ? 1 : 0, .changed_function_ids = &changed,