diff --git a/.codebase-memory.json b/.codebase-memory.json index 80f8b521f..7289ac032 100644 --- a/.codebase-memory.json +++ b/.codebase-memory.json @@ -30,6 +30,9 @@ "scripts/env.sh", "scripts/test.sh", "scripts/run-tests-parallel.sh", + "scripts/run-test-wave.py", + "scripts/path-safety.sh", + "scripts/clean.sh", "scripts/suite-failure-report.sh", "scripts/ci/verify-shard-union.sh", "scripts/embed-frontend.sh", diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 97e850a1b..815146253 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -86,6 +86,16 @@ jobs: path: ${{ runner.temp }}/test-selection if-no-files-found: error overwrite: true + # The selection a veto discarded (written only when one did): evidence + # for the vetoes, never read by a test leg. + - name: Upload the vetoed selection + if: ${{ !cancelled() && steps.select.outcome != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-selection-vetoed + path: ${{ runner.temp }}/test-impact/selection-vetoed.json + if-no-files-found: ignore + overwrite: true test: needs: [changes, lint, select-tests] diff --git a/docs/SMART_CI.md b/docs/SMART_CI.md index 93d474d34..9f6430b59 100644 --- a/docs/SMART_CI.md +++ b/docs/SMART_CI.md @@ -26,7 +26,10 @@ cannot be affected. When it is unsure, it runs everything: - a shallow clone, a diff that cannot be read in full, or an invalid config; - no coverage map, or a coverage map that is not admitted (wrong commit, more than 7 days old, made for another platform or harness); -- a change that reaches the test runner's `main`; +- a change that reaches the test runner's `main`, or a test-runner role + (below) that no test names; +- a role table entry the engine cannot read, or a macro naming a role's + literal that it cannot follow; - any failure of the selection job itself. Each of these makes the run cost more CI minutes. None of them loses coverage. @@ -37,7 +40,7 @@ Each of these makes the run cost more CI minutes. None of them loses coverage. changed files ──► Layer 1: lane selector ──► which CI jobs run (scripts/ci/select-lanes.sh, your own rules) │ - ▼ (Linux test legs only, in this repo) + ▼ (the Linux x86_64 test leg only, in this repo) merge base + PR ─► Layer 2: test-impact engine ─► which suites / tests run (codebase-memory-mcp test-impact select + team-artifact bundle from main) @@ -93,9 +96,9 @@ reports `run_all`. Static reachability alone never omits a suite. list, `--version`); `--skip-config` skips agent configuration, which a CI runner does not need. It has no version flag, so it is not a pin. - **Version note.** `v0.12.0` has `test-impact select`/`publish`. It does - not have `key: "path"`, `unreferenced`, `--vetoed-out` or `run_all_rules`; - they ship in the first release after `v0.12.0`. With `v0.12.0`, a config + **Version note.** `v0.12.0` and `v0.12.1` have `test-impact select`/`publish`. + They do not have `key: "path"`, `unreferenced`, `--vetoed-out` or + `run_all_rules`; those ship in a later release. With either version, a config that uses those keys is invalid, and every answer is `run_all`. Until that release, build the binary from source (`make -f Makefile.cbm cbm`), as this repository's producer and `select-tests` jobs do. @@ -239,8 +242,9 @@ Leave this job out of your aggregate required check. What 4. Runs `test-impact select` and writes `selection-engine.json`. It also writes a Markdown summary to the job's step summary. -The engine reads `.codebase-memory.json` from the **merge base**, not from the -PR. A PR cannot loosen its own selection. A PR that changes the config runs +The engine reads `.codebase-memory.json` from the **selection base** (the merge +base, or the ancestor whose bundle was admitted), not from the PR. A PR cannot +loosen its own selection. A PR that changes the config runs everything (`builtin:test-impact-config`). ### (c) Gate job and the test job @@ -328,16 +332,20 @@ TF_ROLE(TF_PHASE_SPAWNER, "CBM_INDEX_SUPERVISOR", tf_maybe_run_index_worker), The engine reads the table: a change that only a probe's handler reaches selects the tests whose source names one of that handler's literals (the argv -the child gets, or a switch that lets production code start it). A reached -handler that no test names, and a change reaching `main` any other way, still -run everything; the engine note names the call chain into `main`. +the child gets, or a switch that lets production code start it), also through +a `#define` that names the literal. The runner's own use of a literal names no +spawner. A reached handler that no test names, a table entry the engine cannot +read, and a change reaching `main` any other way still run everything; the +engine note names the call chain into `main`. **Only the legs the coverage speaks for should narrow.** Here the coverage comes from one Linux x86_64 clang-21 producer. Only the Linux x86_64 test leg downloads the selection (`runner.os == 'Linux' && runner.arch == 'X64'` in `_test.yml`). The arm64, macOS and Windows legs always run everything, because code under architecture or platform `#ifdef`s has static evidence -only there. +only there. That leg builds with gcc, not clang: code under a compiler `#if` +has static evidence only too, and `--platform` names the producer's platform, +so the engine's platform check admits the map by construction. ### (d) Flip the variable @@ -423,7 +431,7 @@ Built-in rules, applied after yours: | `fixtures` → `referencing_tests`, `path_segment:2` | `tests/fixtures//...` selects the tests that mention the fixture, down to `tests/fixtures/`. | | `contracts` → `ignore` | Shell, Python and data contract tests run in every leg's contract step, which is never narrowed. | | `notices` → `ignore` | Release notices. | -| `repo-data` → `referencing_tests`, `path`, `unreferenced: ignore` | Scripts, docs and packaging. A C test that reads one of these files is selected. A file that no test names selects nothing. | +| `repo-data` → `referencing_tests`, `path`, `unreferenced: ignore` | Scripts, docs and packaging. A C test whose source names one of these files (its path, as text) is selected. A file that no test names selects nothing. | `coverage.compatibility_paths` lists the harness and coverage tools. Changing the bytes of any of them, or the `--platform` label, makes the existing coverage diff --git a/scripts/ci/select-lanes.sh b/scripts/ci/select-lanes.sh index bd25e9761..aaabdb107 100755 --- a/scripts/ci/select-lanes.sh +++ b/scripts/ci/select-lanes.sh @@ -160,7 +160,7 @@ SECURITY_TOOLS = {"scripts/security-audit.sh", "scripts/security-allowlist.txt", # Repository files C tests read: the test legs run (a docs-only tier runs no C # suite, so test_cli's README/site contract would never see a README change). READ_BY_TESTS = {"scripts/security-vendored.sh", ".codebase-memory.json", "README.md", - "docs/index.html", "docs/llms.txt"} + "docs/index.html", "docs/llms.txt", "pkg/npm/README.md"} # The test legs' failure report (tests/test_failure_report_contract.sh). TEST_REPORT = {"scripts/suite-failure-report.sh"} # The release notices ship inside every archive. diff --git a/scripts/ci/test-impact-select.sh b/scripts/ci/test-impact-select.sh index 175c1fdb6..d891e627a 100755 --- a/scripts/ci/test-impact-select.sh +++ b/scripts/ci/test-impact-select.sh @@ -42,8 +42,9 @@ done mkdir -p "$OUT" WORK="$(mktemp -d)" -trap 'rm -rf "$WORK"' EXIT -export CBM_RUNTIME_DIR="$(mktemp -d /tmp/cbm-rt.XXXXXX)" # sun_path must fit +CBM_RUNTIME_DIR="$(mktemp -d /tmp/cbm-rt.XXXXXX)" # sun_path must fit +export CBM_RUNTIME_DIR +trap 'rm -rf "$WORK" "$CBM_RUNTIME_DIR"' EXIT export CBM_CACHE_DIR="$WORK/cache" mkdir -p "$WORK/engine" && chmod 700 "$WORK/engine" base="$(git -C "$ROOT" rev-parse "$BASE")" @@ -70,8 +71,13 @@ if [ "$bundle_rc" -eq 3 ]; then if [ "$select_base" != "$base" ]; then # Exact-revert guard: a file main changed after A that the PR changes # back drops out of A..HEAD. Then A's map cannot speak for it: no coverage. - missing=$(comm -23 <(git -C "$ROOT" diff --no-renames --name-only "$base" HEAD | sort -u) \ - <(git -C "$ROOT" diff --no-renames --name-only "$select_base" HEAD | sort -u)) + # A diff that fails proves nothing either: no coverage. + if git -C "$ROOT" diff --no-renames --name-only "$base" HEAD > "$WORK/pr-files.txt" && + git -C "$ROOT" diff --no-renames --name-only "$select_base" HEAD > "$WORK/ancestor-files.txt"; then + missing=$(comm -23 <(sort -u "$WORK/pr-files.txt") <(sort -u "$WORK/ancestor-files.txt")) + else + missing="(git diff failed)" + fi distance=$(git -C "$ROOT" rev-list --first-parent --count "$select_base..$base") if [ -n "$missing" ]; then echo "::warning::test-impact: ancestor bundle $select_base unusable (the change reverts main's own change in: $(echo "$missing" | head -3 | tr '\n' ' ')): selecting without coverage" diff --git a/src/mcp/test_impact_engine_reach.c b/src/mcp/test_impact_engine_reach.c index 70e5affe4..d8b4eb7c4 100644 --- a/src/mcp/test_impact_engine_reach.c +++ b/src/mcp/test_impact_engine_reach.c @@ -105,6 +105,18 @@ static bool te_is_runner_suite(te_ctx_t *c, const char *name) { return false; } +/* Whether `path` is a runner file: one whose main runs suites. */ +static bool te_runner_file(te_ctx_t *c, const char *path) { + int n = 0; + const cbm_test_runner_suite_t *runner = cbm_test_model_runner_suites(c->model, &n); + for (int i = 0; path && i < n; i++) { + if (runner[i].file && strcmp(runner[i].file, path) == 0) { + return true; + } + } + return false; +} + /* ── Fixtures (replay4 584-615) ──────────────────────────────────── */ /* Test files: the files that define a suite. */ @@ -139,19 +151,75 @@ static bool te_ids_has(const te_ids_t *a, int64_t id) { return false; } -/* Every line of a test file that contains `needle`: its enclosing - * definition is a fixture case or a seed — unless it is one of `exclude` - * (a role handler naming its own literal is not a test that spawns it). */ -static bool te_scan_literal_ex(te_ctx_t *c, const char *needle, te_ids_t *seeds, bool *cases, - bool *hit, const te_ids_t *exclude); - -static bool te_scan_literal(te_ctx_t *c, const char *needle, te_ids_t *seeds, bool *cases, - bool *hit) { - return te_scan_literal_ex(c, needle, seeds, cases, hit, NULL); +static const char *te_skip_space(const char *p, const char *end) { + while (p < end && (*p == ' ' || *p == '\t')) { + p++; + } + return p; } -static bool te_scan_literal_ex(te_ctx_t *c, const char *needle, te_ids_t *seeds, bool *cases, - bool *hit, const te_ids_t *exclude) { +static bool te_ident_char(char ch) { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || + ch == '_'; +} + +/* A `#define NAME ...` line: 1 with NAME copied out, 0 for any other line, + * -1 for a define whose name does not fit or does not parse. */ +static int te_define_name(const char *text, size_t len, char *name, size_t cap) { + static const char keyword[] = "define"; + size_t klen = sizeof(keyword) - 1; + const char *end = text + len; + const char *p = te_skip_space(text, end); + if (p >= end || *p != '#') { + return 0; + } + p = te_skip_space(p + 1, end); + if ((size_t)(end - p) <= klen || memcmp(p, keyword, klen) != 0 || + (p[klen] != ' ' && p[klen] != '\t')) { + return 0; + } + p = te_skip_space(p + klen, end); + const char *start = p; + while (p < end && te_ident_char(*p)) { + p++; + } + if (p == start || (size_t)(p - start) >= cap) { + return -1; + } + memcpy(name, start, (size_t)(p - start)); + name[p - start] = '\0'; + return 1; +} + +/* Whether `needle` occurs in the line; an identifier only as a whole word. */ +static bool te_line_names(const char *text, size_t len, const char *needle, size_t nlen, + bool ident) { + for (size_t i = 0; nlen && i + nlen <= len; i++) { + if (memcmp(text + i, needle, nlen) == 0 && + (!ident || ((i == 0 || !te_ident_char(text[i - 1])) && + (i + nlen == len || !te_ident_char(text[i + nlen]))))) { + return true; + } + } + return false; +} + +/* A macro that names a role literal through further macros: past this depth + * the tests that use it are unknown, and everything runs. */ +#define TE_DEFINE_DEPTH_MAX 4 + +/* Every line of a test file that contains `needle`: its enclosing + * definition is a fixture case or a seed — unless it is one of `exclude` + * (a role handler naming its own literal is not a test that spawns it). + * A `#define NAME "literal"` line names nobody: the code that uses NAME does, + * so the scan follows NAME (a whole word) instead, through macros that name + * macros. A define it cannot follow runs everything. In a role's scan + * (`runner_cases`: the walk's sinks) a runner file's helper — the runner's own + * use of a role's marker — still seeds the walk but names no spawner: there, + * only a test case does (a sink, found by its start line like every case). */ +static bool te_scan_text(te_ctx_t *c, const char *needle, bool ident, int depth, te_ids_t *seeds, + bool *cases, bool *hit, const te_ids_t *exclude, + const te_ids_t *runner_cases) { int files = cbm_ti_source_file_count(c->source); size_t nlen = strlen(needle); for (int f = 0; f < files; f++) { @@ -163,11 +231,26 @@ static bool te_scan_literal_ex(te_ctx_t *c, const char *needle, te_ids_t *seeds, for (int line = 1; line <= lines; line++) { size_t len = 0; const char *text = cbm_ti_source_line(c->source, f, line, &len); - bool found = false; - for (size_t i = 0; !found && nlen && i + nlen <= len; i++) { - found = memcmp(text + i, needle, nlen) == 0; + if (!text || !te_line_names(text, len, needle, nlen, ident)) { + continue; } - if (!found) { + char name[256]; + int define = te_define_name(text, len, name, sizeof(name)); + if (define > 0 && strcmp(name, needle) == 0) { + continue; /* the definition of the macro being followed */ + } + if (define < 0 || (define > 0 && depth >= TE_DEFINE_DEPTH_MAX)) { + char note[400]; + snprintf(note, sizeof(note), + "%s:%d defines a macro naming %s that the scan cannot follow", path, line, + needle); + return te_fallback(c, CBM_TEST_RESULT_FALLBACK_SELECTION_UNAVAILABLE, note); + } + if (define > 0) { + if (!te_scan_text(c, name, true, depth + 1, seeds, cases, hit, exclude, + runner_cases)) { + return false; + } continue; } bool ok = true; @@ -180,8 +263,10 @@ static bool te_scan_literal_ex(te_ctx_t *c, const char *needle, te_ids_t *seeds, if (!d || te_ids_has(exclude, d->id)) { continue; } - *hit = true; int ci = te_is_function(d) ? te_case_index(c, path, d->name) : -1; + if (!runner_cases || !te_runner_file(c, path) || te_ids_has(runner_cases, d->id)) { + *hit = true; + } if (ci >= 0) { cases[ci] = true; } else if (!te_ids_push(c, seeds, d->id)) { @@ -192,6 +277,18 @@ static bool te_scan_literal_ex(te_ctx_t *c, const char *needle, te_ids_t *seeds, return true; } +static bool te_scan_literal(te_ctx_t *c, const char *needle, te_ids_t *seeds, bool *cases, + bool *hit) { + return te_scan_text(c, needle, false, 0, seeds, cases, hit, NULL, NULL); +} + +/* The scan for a role's literal: `exclude` holds the handlers and main, + * `sinks` the walk's sinks (every test case among them). */ +static bool te_scan_literal_ex(te_ctx_t *c, const char *needle, te_ids_t *seeds, bool *cases, + bool *hit, const te_ids_t *exclude, const te_ids_t *sinks) { + return te_scan_text(c, needle, false, 0, seeds, cases, hit, exclude, sinks); +} + /* One fixture path: the path itself, then each parent down to the rule's * key segment; each with and without its first segment. The first key some * test file mentions decides. */ @@ -284,13 +381,6 @@ typedef struct { int role_cap; } te_sinks_t; -static const char *te_skip_space(const char *p, const char *end) { - while (p < end && (*p == ' ' || *p == '\t')) { - p++; - } - return p; -} - /* One `TF_ROLE(phase, "literal", handler)` line. false: no entry here (the * macro's own #define included); *bad: an entry that does not parse. */ static bool te_role_parse(const char *text, size_t len, char *literal, size_t literal_cap, @@ -303,7 +393,7 @@ static bool te_role_parse(const char *text, size_t len, char *literal, size_t li return false; } const char *at = NULL; - for (const char *q = p; !at && q + macro_len <= end; q++) { + for (const char *q = p; !at && (size_t)(end - q) >= macro_len; q++) { at = memcmp(q, TE_ROLE_MACRO, macro_len) == 0 ? q : NULL; } if (!at) { @@ -348,8 +438,7 @@ static bool te_role_parse(const char *text, size_t len, char *literal, size_t li } /* The handler's function node: the one Function of that name in a runner or - * test file. 0 when there is none or more than one (then it is no sink, and - * reaching it reaches main). */ + * test file. 0 when there is none or more than one. */ static bool te_role_handler(te_ctx_t *c, const char *name, int64_t *id) { cbm_node_t *nodes = NULL; int n = 0; @@ -378,7 +467,10 @@ static bool te_role_handler(te_ctx_t *c, const char *name, int64_t *id) { return true; } -/* The role table of one runner file: each entry's handler becomes a sink. */ +/* The role table of one runner file: each entry's handler becomes a sink. + * main dispatches through the table, so no edge leads from main to a handler: + * an entry the engine cannot read, or whose handler it cannot find, hides that + * role's spawners, and everything runs. */ static bool te_runner_roles(te_ctx_t *c, const char *file, te_sinks_t *s) { int f = cbm_ti_source_find(c->source, file); int lines = f >= 0 ? cbm_ti_source_line_count(c->source, f) : 0; @@ -389,16 +481,24 @@ static bool te_runner_roles(te_ctx_t *c, const char *file, te_sinks_t *s) { char handler[256]; bool bad = false; if (!text || - !te_role_parse(text, len, literal, sizeof(literal), handler, sizeof(handler), &bad) || - bad) { - continue; /* not an entry, or one that does not parse: no sink */ + !te_role_parse(text, len, literal, sizeof(literal), handler, sizeof(handler), &bad)) { + continue; /* not an entry */ + } + char note[400]; + if (bad) { + snprintf(note, sizeof(note), "%s:%d: a role table entry the engine cannot read", file, + line); + return te_fallback(c, CBM_TEST_RESULT_FALLBACK_SELECTION_UNAVAILABLE, note); } int64_t id = 0; if (!te_role_handler(c, handler, &id)) { return false; } if (!id) { - continue; + snprintf(note, sizeof(note), + "%s:%d: the role handler %s is no single function of the tests", file, line, + handler); + return te_fallback(c, CBM_TEST_RESULT_FALLBACK_SELECTION_UNAVAILABLE, note); } if (!te_grow((void **)&s->roles, &s->role_cap, s->role_count, sizeof(*s->roles))) { return te_oom(c); @@ -997,7 +1097,7 @@ static bool te_role_lift(te_ctx_t *c, cbm_impact_walk_t *walk, te_sinks_t *s, bo } r->scanned = true; te_ids_t seeds = {0}; - ok = te_scan_literal_ex(c, r->literal, &seeds, cases, &r->named, &handlers); + ok = te_scan_literal_ex(c, r->literal, &seeds, cases, &r->named, &handlers, &s->sinks); if (ok && seeds.count) { ok = cbm_impact_walk_run(walk, seeds.items, seeds.count) == CBM_STORE_OK || (te_fallback(c, CBM_TEST_RESULT_FALLBACK_ENGINE_SATURATED, @@ -1056,9 +1156,11 @@ static void te_call_chain(const cbm_impact_walk_t *walk, int64_t id, const char * edge: a platform variant, a route), the reached node main has a walked edge * to, or 0. An edge into a Field named like a suite main runs is no way in: * the C extractor takes the macro argument of RUN_SELECTED_SUITE(mcp) for a - * use of some same-named data member (`char mcp[]` in an unrelated struct). */ -static bool te_main_reached(te_ctx_t *c, const cbm_impact_walk_t *walk, int64_t main_id, - int64_t *way_in) { + * use of some same-named data member (`char mcp[]` in an unrelated struct). + * Nor is an edge into a sink (a role handler main calls, a suite, a test): + * the walk selects whoever runs it and never expands it. */ +static bool te_main_reached(te_ctx_t *c, const cbm_impact_walk_t *walk, const te_sinks_t *sinks, + int64_t main_id, int64_t *way_in) { *way_in = 0; const cbm_impact_hit_t *self = cbm_impact_walk_hit(walk, main_id); if (!self) { @@ -1089,7 +1191,7 @@ static bool te_main_reached(te_ctx_t *c, const cbm_impact_walk_t *walk, int64_t const char *name = walked && to->qualified_name ? strrchr(to->qualified_name, '.') : NULL; bool suite_arg = name && to->label && strcmp(to->label, "Field") == 0 && te_is_runner_suite(c, name + 1); - if (walked && !suite_arg) { + if (walked && !suite_arg && !te_ids_has(&sinks->sinks, edges[e].target_id)) { *way_in = edges[e].target_id; } } @@ -1097,6 +1199,19 @@ static bool te_main_reached(te_ctx_t *c, const cbm_impact_walk_t *walk, int64_t return true; } +/* Role lifts and parse gaps feed each other: a gap that a lift's walk names + * can reach a further handler, and a lift can reach a further gap. Both run + * until the walk stops growing (it only grows, so this ends). */ +static bool te_lift_fixpoint(te_ctx_t *c, cbm_impact_walk_t *walk, te_sinks_t *sinks, + bool *fixture_cases) { + bool ok = true; + for (int before = -1; ok && cbm_impact_walk_count(walk) != before;) { + before = cbm_impact_walk_count(walk); + ok = te_parse_gaps(c, walk) && te_role_lift(c, walk, sinks, fixture_cases); + } + return ok; +} + bool te_reach(te_ctx_t *c) { int64_t *extra = NULL; int extra_count = 0; @@ -1104,11 +1219,11 @@ bool te_reach(te_ctx_t *c) { te_sinks_t sinks = {0}; cbm_impact_walk_t *walk = NULL; bool ok = te_fixtures(c, &extra, &extra_count, &fixture_cases) && te_collect_sinks(c, &sinks) && - te_walk(c, &sinks, extra, extra_count, &walk) && te_parse_gaps(c, walk) && - te_role_lift(c, walk, &sinks, fixture_cases) && te_parse_gaps(c, walk); + te_walk(c, &sinks, extra, extra_count, &walk) && + te_lift_fixpoint(c, walk, &sinks, fixture_cases); for (int i = 0; ok && i < sinks.runner_main.count; i++) { int64_t way_in = 0; - ok = te_main_reached(c, walk, sinks.runner_main.items[i], &way_in); + ok = te_main_reached(c, walk, &sinks, sinks.runner_main.items[i], &way_in); if (ok && way_in) { const cbm_impact_hit_t *m = cbm_impact_walk_hit(walk, sinks.runner_main.items[i]); const char *main_name = @@ -1118,14 +1233,22 @@ bool te_reach(te_ctx_t *c) { int used = snprintf(note, sizeof(note), "the change reaches the test runner's main: %s%s", direct ? "" : main_name, direct ? "" : " -> "); + if (used < 0) { + note[0] = '\0'; + used = 0; + } else if ((size_t)used >= sizeof(note)) { + used = (int)sizeof(note) - 1; /* a truncated prefix: no room for the chain */ + } te_call_chain(walk, way_in, "", note + used, sizeof(note) - (size_t)used); ok = te_fallback(c, CBM_TEST_RESULT_FALLBACK_RUNNER_REACHED, note); break; } } /* After the runner check: the runner re-executing itself is a spawn too, - * and reaching its main through that is no change to the runner. */ - ok = ok && te_spawn_lift(c, walk); + * and reaching its main through that is no change to the runner. What the + * spawn lift reaches is lifted like the rest: a handler that starts the + * product is first reached here. */ + ok = ok && te_spawn_lift(c, walk) && te_lift_fixpoint(c, walk, &sinks, fixture_cases); ok = ok && te_reach_rows(c, walk, fixture_cases) && te_suite_triggers(c, walk) && te_rule_triggers(c) && te_header_triggers(c); c->static_complete = ok; diff --git a/tests/fixtures/select-lanes/cases.tsv b/tests/fixtures/select-lanes/cases.tsv index ff838416b..54a8404c2 100644 --- a/tests/fixtures/select-lanes/cases.tsv +++ b/tests/fixtures/select-lanes/cases.tsv @@ -6,6 +6,7 @@ docs-only CONTRIBUTING.md docs/BENCHMARK.md T0a-docs false contracts security-static readme-runs-the-test-legs README.md T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows site-files-run-the-test-legs docs/index.html docs/llms.txt T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows +npm-readme-runs-the-test-legs pkg/npm/README.md T1-tests false lint lint-mem security-static shard-completeness unix-macos14 unix-x86 windows issue-template-is-docs .github/ISSUE_TEMPLATE/bug.yml T0a-docs false contracts security-static local-infra test-infrastructure/run.sh T0b-nonproduct false contracts lint lint-mem security-static lint-config scripts/memory-core-baseline.txt T0b-nonproduct false contracts lint lint-mem security-static diff --git a/tests/test_test_impact_engine.c b/tests/test_test_impact_engine.c index 502bda2f6..96c5a0b24 100644 --- a/tests/test_test_impact_engine.c +++ b/tests/test_test_impact_engine.c @@ -674,6 +674,194 @@ TEST(test_impact_engine_role_literal_behind_a_define_selects_its_users) { PASS(); } +/* A macro that names that macro is followed too. A chain longer than the scan + * follows runs everything: the tests that use its last macro are unknown. */ +TEST(test_impact_engine_role_literal_behind_a_macro_chain) { + tie_fixture_t fx; + char *json = tie_answer_orphan_switch(&fx, "#define BETA_SWITCH \"__FX_ORPHAN_SWITCH\"\n" + "#define BETA_ALIAS BETA_SWITCH\n" + "TEST(beta_sets_switch) {\n" + " const char *name = BETA_ALIAS;\n" + " return name[0] == '_' ? 0 : 1;\n" + "}\n" + "TEST(beta_plain) {\n return 0;\n}\n" + "SUITE(beta) {\n" + " RUN_TEST(beta_sets_switch);\n" + " RUN_TEST(beta_plain);\n" + "}\n"); + ASSERT_NOT_NULL(json); + ASSERT_NULL(strstr(json, "RUNNER_REACHED")); + ASSERT_TRUE(tie_suite_has(json, "beta", "STATIC")); + free(json); + tie_close(&fx); + + json = tie_answer_orphan_switch(&fx, "#define BETA_M0 \"__FX_ORPHAN_SWITCH\"\n" + "#define BETA_M1 BETA_M0\n" + "#define BETA_M2 BETA_M1\n" + "#define BETA_M3 BETA_M2\n" + "#define BETA_M4 BETA_M3\n" + "#define BETA_M5 BETA_M4\n" + "TEST(beta_sets_switch) {\n" + " const char *name = BETA_M5;\n" + " return name[0] == '_' ? 0 : 1;\n" + "}\n" + "TEST(beta_plain) {\n return 0;\n}\n" + "SUITE(beta) {\n" + " RUN_TEST(beta_sets_switch);\n" + " RUN_TEST(beta_plain);\n" + "}\n"); + ASSERT_NOT_NULL(json); + ASSERT_TRUE(tie_top_decision(json, "run_all")); + ASSERT_NOT_NULL(strstr(json, "SELECTION_UNAVAILABLE")); + free(json); + tie_close(&fx); + PASS(); +} + +/* `src` with `insert` placed after `anchor` (which must occur in it) into + * `out`; false when it does not fit. */ +static bool tie_insert(char *out, size_t cap, const char *src, const char *anchor, + const char *insert) { + const char *at = strstr(src, anchor); + if (!at) { + return false; + } + size_t head = (size_t)(at - src) + strlen(anchor); + int n = snprintf(out, cap, "%.*s%s%s", (int)head, src, insert, src + head); + return n > 0 && (size_t)n < cap; +} + +static bool tie_runner_with(char *out, size_t cap, const char *anchor, const char *insert) { + return tie_insert(out, cap, tie_roles_main, anchor, insert); +} + +/* The role fixture's runner with a suite of its own (`body`, placed after the + * role table), which main runs as the real runner runs runner_isolation. */ +static bool tie_runner_with_suite(char *out, size_t cap, const char *body) { + char first[4096]; + return tie_runner_with(first, sizeof(first), "};\n", body) && + tie_insert(out, cap, first, " RUN_SELECTED_SUITE(matrix);\n", + " RUN_SELECTED_SUITE(runner_isolation);\n"); +} + +/* The role fixture with `runner` as its base, then a change to lib_value or + * lib_other; the answer (NULL on a fixture failure). */ +static char *tie_answer_runner(tie_fixture_t *fx, const char *runner, bool other) { + const char *base[] = {"branch", "--quiet", "-f", "main", "HEAD", NULL}; + if (!tie_open_roles(fx) || !tie_write(fx, "tests/test_main.c", runner) || + !tie_commit(fx, "the runner under test") || !tie_git(fx, base) || + !tie_write(fx, "src/lib.c", + other ? "int lib_value(void) {\n return 1;\n}\n" + "int lib_other(void) {\n return 2 + 0;\n}\n" + "int lib_third(void) {\n return 3;\n}\n" + : "int lib_value(void) {\n return 1 + 0;\n}\n" + "int lib_other(void) {\n return 2;\n}\n" + "int lib_third(void) {\n return 3;\n}\n") || + !tie_commit(fx, other ? "change lib_other" : "change lib_value")) { + return NULL; + } + char *json = tie_answer(fx); + if (json) { + printf(" note: %s\n", tie_last_note); + } + return json; +} + +/* main dispatches through the role table, so no edge leads from main to a + * handler: an entry the engine cannot read, or whose handler is no single + * function, hides that role's spawners, and every change runs everything. */ +TEST(test_impact_engine_unusable_role_entry_runs_all) { + const char *anchor = " TF_ROLE(0, \"__fx_orphan_role\", role_orphan),\n"; + const char *const entries[] = {" TF_ROLE(0, FX_GHOST_LITERAL, role_orphan),\n", + " TF_ROLE(0, \"__fx_ghost_role\", role_ghost),\n"}; + for (int k = 0; k < 2; k++) { + char runner[4096]; + ASSERT_TRUE(tie_runner_with(runner, sizeof(runner), anchor, entries[k])); + tie_fixture_t fx; + char *json = tie_answer_runner(&fx, runner, false); + ASSERT_NOT_NULL(json); + ASSERT_TRUE(tie_top_decision(json, "run_all")); + ASSERT_NOT_NULL(strstr(json, "SELECTION_UNAVAILABLE")); + free(json); + tie_close(&fx); + } + PASS(); +} + +/* A handler first reached by the spawn lift (it starts the product, whose + * main the change reaches) is lifted like any other: its literal is named by + * no test here, so everything runs. */ +TEST(test_impact_engine_role_reached_by_the_spawn_lift_is_lifted) { + static const char product[] = "int lib_other(void);\n" + "int main(void) {\n" + " return lib_other();\n" + "}\n"; + const char *call = " return lib_other();\n"; + const char *at = strstr(tie_roles_main, call); + ASSERT_NOT_NULL(at); + char runner[4096]; + int n = snprintf(runner, sizeof(runner), + "#include \n%.*s return system(\"./app --version\");\n%s", + (int)(at - tie_roles_main), tie_roles_main, at + strlen(call)); + ASSERT_TRUE(n > 0 && (size_t)n < sizeof(runner)); + const char *const files[] = { + "tests/test_main.c", runner, "tests/test_beta.c", tie_roles_beta, + "tests/test_matrix.c", tie_roles_matrix, "src/lib.c", tie_roles_lib, + "src/main.c", product}; + tie_fixture_t fx; + ASSERT_TRUE(tie_open_files(&fx, files, 5)); + ASSERT_TRUE(tie_write(&fx, "src/lib.c", + "int lib_value(void) {\n return 1;\n}\n" + "int lib_other(void) {\n return 2 + 0;\n}\n" + "int lib_third(void) {\n return 3;\n}\n")); + ASSERT_TRUE(tie_commit(&fx, "change lib_other")); + char *json = tie_answer(&fx); + ASSERT_NOT_NULL(json); + if (!strstr(json, "RUNNER_REACHED")) { + printf(" note: %s\n %.1600s\n", tie_last_note, json); + } + ASSERT_NOT_NULL(strstr(json, "RUNNER_REACHED")); + free(json); + tie_close(&fx); + PASS(); +} + +/* The runner's own use of a role's literal (its helper reading the marker) + * names no spawner, so a role only the runner names still runs everything; + * a test case in the runner file that names it is a spawner. */ +TEST(test_impact_engine_runner_helper_names_no_spawner) { + char runner[4096]; + ASSERT_TRUE( + tie_runner_with_suite(runner, sizeof(runner), + "static int runner_reads_marker(void) {\n" + " const char *s = \"__fx_orphan_role\";\n" + " return s[0] == '_' ? 0 : 1;\n" + "}\n" + "TEST(runner_plain) {\n return 0;\n}\n" + "SUITE(runner_isolation) {\n RUN_TEST(runner_plain);\n}\n")); + tie_fixture_t fx; + char *json = tie_answer_runner(&fx, runner, true); + ASSERT_NOT_NULL(json); + ASSERT_TRUE(tie_top_decision(json, "run_all")); + ASSERT_NOT_NULL(strstr(json, "RUNNER_REACHED")); + free(json); + tie_close(&fx); + + ASSERT_TRUE(tie_runner_with_suite(runner, sizeof(runner), + "TEST(runner_spawns_orphan) {\n" + " const char *s = \"__fx_orphan_role\";\n" + " return s[0] == '_' ? 0 : 1;\n" + "}\n" + "SUITE(runner_isolation) {\n" + " RUN_TEST(runner_spawns_orphan);\n}\n")); + json = tie_answer_runner(&fx, runner, true); + ASSERT_NOT_NULL(json); + ASSERT_NULL(strstr(json, "RUNNER_REACHED")); + free(json); + tie_close(&fx); + PASS(); +} + /* A struct member named like a suite: the extractor can take the macro argument * of RUN_SELECTED_SUITE(alpha) for a use of it. A changed member reaches main * only through that edge into a Field: no veto. */ @@ -696,6 +884,48 @@ TEST(test_impact_engine_field_call_is_no_way_into_main) { PASS(); } +/* main may call a role handler itself, as the runner's main calls the + * blocking-git probe. A reached handler is a sink whose spawners the role lift + * selects, so that call is no way into main. Here main is reached otherwise + * only through the edge into the suite-named member (above): no veto. */ +TEST(test_impact_engine_reached_role_handler_is_no_way_into_main) { + static const char shapes[] = + "typedef struct {\n char alpha[8];\n int sides;\n} shape_t;\n"; + const char *anchor = "int main(int argc, char **argv) {\n"; + const char *at = strstr(tie_roles_main, anchor); + ASSERT_NOT_NULL(at); + size_t head = (size_t)(at - tie_roles_main) + strlen(anchor); + char runner[4096]; + int n = snprintf(runner, sizeof(runner), + "%.*s if (argc > 9) {\n return role_probe(argc, argv);\n }\n%s", + (int)head, tie_roles_main, tie_roles_main + head); + ASSERT_TRUE(n > 0 && (size_t)n < sizeof(runner)); + const char *const files[] = {"tests/test_main.c", runner, + "tests/test_beta.c", tie_roles_beta, + "tests/test_matrix.c", tie_roles_matrix, + "src/lib.c", tie_roles_lib, + "tests/shapes.h", shapes}; + tie_fixture_t fx; + ASSERT_TRUE(tie_open_files(&fx, files, 5)); + ASSERT_TRUE(tie_write(&fx, "tests/shapes.h", + "typedef struct {\n char alpha[16];\n int sides;\n} shape_t;\n")); + ASSERT_TRUE(tie_write(&fx, "src/lib.c", + "int lib_value(void) {\n return 1 + 0;\n}\n" + "int lib_other(void) {\n return 2;\n}\n" + "int lib_third(void) {\n return 3;\n}\n")); + ASSERT_TRUE(tie_commit(&fx, "widen the member, change lib_value")); + char *json = tie_answer(&fx); + ASSERT_NOT_NULL(json); + if (strstr(json, "RUNNER_REACHED")) { + printf(" note: %s\n", tie_last_note); + } + ASSERT_NULL(strstr(json, "RUNNER_REACHED")); + ASSERT_TRUE(tie_suite_has(json, "beta", "STATIC")); /* spawns the probe role */ + free(json); + tie_close(&fx); + PASS(); +} + /* A parse gap hides the edges written in its lines: a helper whose broken * line calls the changed function has no CALLS edge to it. The gap's text * still names the function, so the helper is reached, and with it the test @@ -1023,7 +1253,12 @@ SUITE(test_impact_engine) { RUN_TEST(test_impact_engine_unspawned_role_and_main_still_run_all); RUN_TEST(test_impact_engine_role_with_two_literals_selects_both_spawners); RUN_TEST(test_impact_engine_role_literal_behind_a_define_selects_its_users); + RUN_TEST(test_impact_engine_role_literal_behind_a_macro_chain); + RUN_TEST(test_impact_engine_unusable_role_entry_runs_all); + RUN_TEST(test_impact_engine_role_reached_by_the_spawn_lift_is_lifted); + RUN_TEST(test_impact_engine_runner_helper_names_no_spawner); RUN_TEST(test_impact_engine_field_call_is_no_way_into_main); + RUN_TEST(test_impact_engine_reached_role_handler_is_no_way_into_main); RUN_TEST(test_impact_engine_parse_gaps_reach_what_they_name); RUN_TEST(test_impact_engine_admitted_coverage_narrows_only_when_verified); RUN_TEST(test_impact_engine_coverage_reaches_unmapped_code_through_mapped_callers);