From 6e70a8b019c8cffcde689d48551e4e44f5b673de Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 26 Jun 2026 00:09:23 +0200 Subject: [PATCH] test(repro): scaffold cumulative bug-reproduction suite + cross-platform board Stand up a separate, non-gating reproduce-first suite that holds one RED case per open bug issue (the redness is the deliverable + the regression guard): - tests/repro/ + repro_main.c -> `make test-repro` (its own runner with its own main + counters; deliberately EXCLUDED from ALL_TEST_SRCS so the gating `make test` / ci-ok required check stays green and PRs are not wedged) - repro_extraction.c: first reproduction, #554 -- a C++ out-of-line method's inner CALLS edge must attribute to the class-qualified Method QN, not the Module; ties the call's enclosing_func_qn to the method definition's own qualified_name so a class-qualifier drop (the live root cause) fails it - scripts/repro.sh: build+run the board; a build/link failure fails the job, while expected test redness is reported as the board state (job stays green) - .github/workflows/bug-repro.yml: workflow_dispatch (platform filter) + qa/** push; runs the board on linux x2 / macos x2 / windows so many bug vectors can be reproduced on many platforms at once Signed-off-by: Martin Vogel --- .github/workflows/bug-repro.yml | 82 +++++++++++++++++++++++++++++ Makefile.cbm | 25 ++++++++- scripts/repro.sh | 65 +++++++++++++++++++++++ tests/repro/repro_extraction.c | 93 +++++++++++++++++++++++++++++++++ tests/repro/repro_main.c | 45 ++++++++++++++++ 5 files changed, 308 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/bug-repro.yml create mode 100755 scripts/repro.sh create mode 100644 tests/repro/repro_extraction.c create mode 100644 tests/repro/repro_main.c diff --git a/.github/workflows/bug-repro.yml b/.github/workflows/bug-repro.yml new file mode 100644 index 000000000..e5b90653c --- /dev/null +++ b/.github/workflows/bug-repro.yml @@ -0,0 +1,82 @@ +# Bug-reproduction board — runs the cumulative reproduce-first suite (RED by +# design, one case per open bug) across every platform on a chosen branch. +# +# This is the "test many bug vectors on many platforms at once" harness. It is +# NON-GATING: dispatch-only, never a required check, so a red board never blocks +# a merge. Dispatch against a feature branch with: +# gh workflow run bug-repro.yml --ref -f platforms=all +name: Bug Repro Board + +on: + workflow_dispatch: + inputs: + platforms: + description: 'Which platforms to run the repro board on' + type: choice + options: ['all', 'linux', 'macos', 'windows'] + default: 'all' + # Iteration convenience: any push to a qa/** branch runs the board straight + # from that branch's own copy of this file (no main merge needed). Non-gating. + push: + branches: ['qa/**'] + +permissions: + contents: read + +jobs: + repro-unix: + if: ${{ github.event_name == 'push' || inputs.platforms == 'all' || inputs.platforms == 'linux' || inputs.platforms == 'macos' }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + group: linux + cc: gcc + cxx: g++ + - os: ubuntu-24.04-arm + group: linux + cc: gcc + cxx: g++ + - os: macos-14 + group: macos + cc: cc + cxx: c++ + - os: macos-15-intel + group: macos + cc: cc + cxx: c++ + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Install deps (Ubuntu) + if: startsWith(matrix.os, 'ubuntu') + run: sudo apt-get update && sudo apt-get install -y zlib1g-dev + + - name: Run bug-reproduction board + if: ${{ github.event_name == 'push' || inputs.platforms == 'all' || inputs.platforms == matrix.group }} + run: scripts/repro.sh CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} + + repro-windows: + if: ${{ github.event_name == 'push' || inputs.platforms == 'all' || inputs.platforms == 'windows' }} + runs-on: windows-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2 + with: + msystem: CLANG64 + path-type: inherit + install: >- + mingw-w64-clang-x86_64-clang + mingw-w64-clang-x86_64-compiler-rt + mingw-w64-clang-x86_64-zlib + make + git + + - name: Run bug-reproduction board + shell: msys2 {0} + run: scripts/repro.sh CC=clang CXX=clang++ diff --git a/Makefile.cbm b/Makefile.cbm index 2bcf7b4d7..1ad8b4861 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -389,7 +389,14 @@ TEST_SIMHASH_SRCS = tests/test_simhash.c TEST_STACK_OVERFLOW_SRCS = tests/test_stack_overflow.c -ALL_TEST_SRCS = $(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_PHP_LSP_SRCS) $(TEST_CS_LSP_SRCS) $(TEST_CS_LSP_BENCH_SRCS) $(TEST_SCOPE_SRCS) $(TEST_TYPE_REP_SRCS) $(TEST_PY_LSP_SRCS) $(TEST_PY_LSP_BENCH_SRCS) $(TEST_PY_LSP_STRESS_SRCS) $(TEST_PY_LSP_SCALE_SRCS) $(TEST_TS_LSP_SRCS) $(TEST_JAVA_LSP_SRCS) $(TEST_KOTLIN_LSP_SRCS) $(TEST_RUST_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_HTTPD_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_STACK_OVERFLOW_SRCS) $(TEST_INTEGRATION_SRCS) +# Cumulative BUG-REPRODUCTION suite (separate runner, NOT in ALL_TEST_SRCS). +# These cases are RED by design (one open bug each) — see tests/repro/repro_main.c. +# Kept out of the gating `make test` so `ci-ok` stays green; run via `make test-repro`. +TEST_REPRO_SRCS = \ + tests/repro/repro_main.c \ + tests/repro/repro_extraction.c + +ALL_TEST_SRCS =$(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_PHP_LSP_SRCS) $(TEST_CS_LSP_SRCS) $(TEST_CS_LSP_BENCH_SRCS) $(TEST_SCOPE_SRCS) $(TEST_TYPE_REP_SRCS) $(TEST_PY_LSP_SRCS) $(TEST_PY_LSP_BENCH_SRCS) $(TEST_PY_LSP_STRESS_SRCS) $(TEST_PY_LSP_SCALE_SRCS) $(TEST_TS_LSP_SRCS) $(TEST_JAVA_LSP_SRCS) $(TEST_KOTLIN_LSP_SRCS) $(TEST_RUST_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_HTTPD_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_STACK_OVERFLOW_SRCS) $(TEST_INTEGRATION_SRCS) # ── Build directories ──────────────────────────────────────────── @@ -413,7 +420,7 @@ PP_OBJ_TEST = $(BUILD_DIR)/preprocessor.o # ── Targets ────────────────────────────────────────────────────── -.PHONY: test test-foundation test-tsan cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security +.PHONY: test test-repro test-foundation test-tsan cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security $(BUILD_DIR): mkdir -p $(BUILD_DIR) @@ -505,6 +512,20 @@ $(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_ test: $(BUILD_DIR)/test-runner cd $(CURDIR) && $(BUILD_DIR)/test-runner +# ── Cumulative bug-reproduction runner (RED by design, non-gating) ── +# Mirrors test-runner's link line but uses repro_main.c (own main + counters) +# and TEST_REPRO_SRCS instead of ALL_TEST_SRCS. Exits non-zero while any bug is +# still reproduced (the expected state); bug-repro.yml surfaces it as a board. +$(BUILD_DIR)/test-repro-runner: $(TEST_REPRO_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) | $(BUILD_DIR) + $(CC) $(CFLAGS_TEST) -o $@ \ + $(TEST_REPRO_SRCS) $(PROD_SRCS) \ + $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \ + $(OBJS_VENDORED_TEST) \ + $(LDFLAGS_TEST) + +test-repro: $(BUILD_DIR)/test-repro-runner + cd $(CURDIR) && $(BUILD_DIR)/test-repro-runner + # ── TSan full test ─────────────────────────────────────────────── test-tsan: diff --git a/scripts/repro.sh b/scripts/repro.sh new file mode 100755 index 000000000..b877750e7 --- /dev/null +++ b/scripts/repro.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# repro.sh — Build + run the cumulative BUG-REPRODUCTION suite (test-repro). +# +# Unlike test.sh (the gating suite, must be GREEN), this suite is RED by design: +# every case reproduces an open bug. So we distinguish two outcomes: +# - BUILD/LINK failure → real breakage → exit non-zero (fail the CI job). +# - Test redness → EXPECTED → report the count, exit 0 (green board). +# +# Usage: scripts/repro.sh [CC=clang] [CXX=clang++] [--arch arm64|x86_64] +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +# --arch before sourcing env.sh (mirrors test.sh) +prev_arg="" +for arg in "$@"; do + case "$arg" in + arm64|x86_64) [[ "$prev_arg" == "--arch" ]] && export CBM_ARCH="$arg" ;; + --arch=*) export CBM_ARCH="${arg#--arch=}" ;; + esac + prev_arg="$arg" +done + +# shellcheck source=env.sh +source "$ROOT/scripts/env.sh" + +MAKE_ARGS="" +for arg in "$@"; do + case "$arg" in + CC=*|CXX=*) export "${arg?}" ;; + --arch|--arch=*|arm64|x86_64) ;; + *=*) MAKE_ARGS="$MAKE_ARGS $arg" ;; + esac +done + +print_env "repro.sh" +verify_compiler "$CC" + +OUT="$ROOT/repro-out.txt" +# test-repro both builds and runs the runner; tolerate its non-zero (red) exit. +set +e +$ARCH_PREFIX make -j"$NPROC" -f Makefile.cbm test-repro $MAKE_ARGS 2>&1 | tee "$OUT" +set -e + +# The runner prints a " passed[, failed]" summary line only if it actually +# ran. No summary line ⇒ the build/link failed ⇒ real breakage. +if ! grep -qE '[0-9]+ passed' "$OUT"; then + echo "::error::bug-repro runner did not execute — build or link failure" + exit 1 +fi + +reproduced=$(grep -oE '[0-9]+ failed' "$OUT" | head -1 | grep -oE '[0-9]+' || echo 0) +green=$(grep -oE '[0-9]+ passed' "$OUT" | head -1 | grep -oE '[0-9]+' || echo 0) + +{ + echo "## Bug-reproduction board — ${OS:-$(uname -s)} ${ARCH:-}" + echo "" + echo "- **${reproduced}** open bug(s) still reproduced (RED — expected)" + echo "- **${green}** case(s) PASSING — candidate-fixed → verify + close the issue + promote the guard to the gating suite" +} >> "${GITHUB_STEP_SUMMARY:-/dev/stderr}" + +echo "=== bug-repro board: ${reproduced} reproduced (RED), ${green} passing (candidate-fixed) ===" +# Green board: the suite ran. Redness is the data, not a job failure. +exit 0 diff --git a/tests/repro/repro_extraction.c b/tests/repro/repro_extraction.c new file mode 100644 index 000000000..99db6954d --- /dev/null +++ b/tests/repro/repro_extraction.c @@ -0,0 +1,93 @@ +/* + * repro_extraction.c — Reproduce-first cases for OPEN extraction-quality bugs. + * + * Each TEST() asserts the CORRECT behaviour and is RED until the bug is fixed. + * Keep one TEST() per issue; name it repro_issue_ and lead with a + * comment naming the issue, the root cause, and expected-vs-actual. + * + * Cluster (TIER A, in-process via cbm_extract_file): + * #554 — C++ out-of-line method CALLS source = Module, not enclosing Method + * (more added per wave: #495 #521 #382 #408 #523 #56 #333) + */ +#include "test_framework.h" +#include "cbm.h" + +/* Convenience: extract, return result (caller frees). Mirrors test_extraction.c. */ +static CBMFileResult *rx(const char *src, CBMLanguage lang, const char *proj, const char *path) { + return cbm_extract_file(src, (int)strlen(src), lang, proj, path, 0, NULL, NULL); +} + +/* Find the first definition matching label+name (either may be NULL = wildcard). */ +static CBMDefinition *find_def(CBMFileResult *r, const char *label, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + CBMDefinition *d = &r->defs.items[i]; + if (label && (!d->label || strcmp(d->label, label) != 0)) + continue; + if (name && (!d->name || strcmp(d->name, name) != 0)) + continue; + return d; + } + return NULL; +} + +/* ─────────────────────────────────────────────────────────────────── + * #554 — C++ out-of-line method definitions: the CALLS edge source falls + * back to the Module (file-level) instead of the enclosing Method. + * + * Root cause (#621 follow-up to #463/adc8304): for `void Foo::bar() { helper(); }` + * the inner call's `enclosing_func_qn` drops the CLASS qualifier — it resolves to + * the bare method name (e.g. "t.m.bar") instead of the method node's full + * class-qualified QN (e.g. "t.m.Foo.bar"). The pre-existing guard in + * test_extraction.c only checks `enclosing_func_qn != "t.m"` (module), which a + * buggy "t.m.bar" PASSES — so it never caught the class-qualifier drop. + * + * Strong reproduction: tie the call's enclosing_func_qn to the METHOD DEFINITION's + * own qualified_name (format-agnostic) AND require the class qualifier be present. + * Expected: enclosing_func_qn == def(bar).qualified_name, and that QN names "Foo". + * Actual (buggy): enclosing_func_qn loses "Foo" → mismatch → RED. + * ─────────────────────────────────────────────────────────────────── */ +TEST(repro_issue554_cpp_out_of_line_method_class_qualified) { + CBMFileResult *r = rx("struct Foo { void bar(); };\n" + "int helper(int x) { return x; }\n" + "void Foo::bar() { helper(1); }\n", + CBM_LANG_CPP, "t", "m.cpp"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + /* The out-of-line method definition: its qualified_name is the ground truth + * the inner CALLS edge must point at. */ + CBMDefinition *method = find_def(r, "Method", "bar"); + if (!method) + method = find_def(r, NULL, "bar"); /* tolerate label variance */ + ASSERT_NOT_NULL(method); + ASSERT_NOT_NULL(method->qualified_name); + + /* The method node must carry the class qualifier — either embedded in the QN + * or via parent_class. This is the heart of #554/#621. */ + int qn_has_class = strstr(method->qualified_name, "Foo") != NULL; + int parent_has_class = method->parent_class && strstr(method->parent_class, "Foo") != NULL; + ASSERT_TRUE(qn_has_class || parent_has_class); + + /* The helper() call inside Foo::bar must attribute to the method node, i.e. + * its enclosing_func_qn must EQUAL the method's qualified_name (class included), + * not the bare method name and not the module. */ + int saw_helper = 0; + for (int i = 0; i < r->calls.count; i++) { + if (strcmp(r->calls.items[i].callee_name, "helper") == 0) { + saw_helper = 1; + const char *enc = r->calls.items[i].enclosing_func_qn; + ASSERT_NOT_NULL(enc); + ASSERT_STR_EQ(enc, method->qualified_name); + ASSERT_TRUE(strstr(enc, "Foo") != NULL); /* class qualifier preserved */ + } + } + ASSERT_TRUE(saw_helper); + + cbm_free_result(r); + PASS(); +} + +/* ── Suite ──────────────────────────────────────────────────────── */ +SUITE(repro_extraction) { + RUN_TEST(repro_issue554_cpp_out_of_line_method_class_qualified); +} diff --git a/tests/repro/repro_main.c b/tests/repro/repro_main.c new file mode 100644 index 000000000..3d3c90c81 --- /dev/null +++ b/tests/repro/repro_main.c @@ -0,0 +1,45 @@ +/* + * repro_main.c — Entry point for the cumulative BUG-REPRODUCTION suite. + * + * This runner is SEPARATE from the gating `make test` (test-runner). It exists + * to hold reproduce-first cases for every OPEN bug issue. Each case asserts the + * CORRECT behaviour, so it is **RED until the bug is fixed** — the redness is the + * deliverable (proof the bug is real + the permanent regression guard). + * + * Because these cases are red by design, they MUST NOT live in `ALL_TEST_SRCS` + * (that would turn the PR gate `ci-ok` red and wedge every merge). They are built + * + run only via `make test-repro` and the `bug-repro.yml` workflow, neither of + * which gates branch protection. + * + * Exit status: non-zero when any reproduction is still RED (the expected state). + * The `bug-repro.yml` workflow treats that as the status board, not a hard fail. + * + * Adding a cluster: + * 1. create tests/repro/repro_.c exporting `void suite_repro_(void)` + * 2. add it to TEST_REPRO_SRCS in Makefile.cbm + * 3. forward-declare + RUN_SUITE it below + */ + +/* Global test counters (declared extern in test_framework.h) */ +int tf_pass_count = 0; +int tf_fail_count = 0; +int tf_skip_count = 0; + +#include "test_framework.h" + +/* ── Repro suites (one per bug cluster) ─────────────────────────── */ +extern void suite_repro_extraction(void); + +int main(void) { + printf("\n"); + printf("════════════════════════════════════════════════════════════\n"); + printf(" CUMULATIVE BUG-REPRODUCTION SUITE\n"); + printf(" RED rows are EXPECTED — each is an open bug reproduced.\n"); + printf(" A row that PASSES means that bug appears FIXED → flip it\n"); + printf(" into the gating suite and close the issue with the guard.\n"); + printf("════════════════════════════════════════════════════════════\n"); + + RUN_SUITE(repro_extraction); + + TEST_SUMMARY(); +}