diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 22fd09386..c11a29bcb 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -212,10 +212,15 @@ jobs: ccache-diag-${{ github.ref }}- ccache-diag- - - name: Test (clang-22, ASan+UBSan, -fno-optimize-sibling-calls) + # Also enables the ASan checks that are off by default in every other + # lane: stack-use-after-return, stack-use-after-scope, invalid pointer + # pairs, and strict string checks. Mirrors `make -f Makefile.cbm diag`. + - name: Test (clang-22, ASan+UBSan + off-by-default ASan checks) + env: + ASAN_OPTIONS: "detect_stack_use_after_return=1:detect_invalid_pointer_pairs=2:strict_string_checks=1:detect_stack_use_after_scope=1" run: > scripts/test.sh CC=clang-22 CXX=clang++-22 - SANITIZE="-fsanitize=address,undefined -fno-omit-frame-pointer -fno-optimize-sibling-calls" + SANITIZE="-fsanitize=address,undefined,pointer-compare,pointer-subtract -fno-omit-frame-pointer -fno-optimize-sibling-calls" - name: Compiler cache (save) if: always() @@ -264,6 +269,44 @@ jobs: - name: MSan full suite (no exclusions — x86-64 is the real test) run: docker run --rm -e MSAN_EXCLUDE="" -v "$PWD:/src" -w /src cbm-msan:ci + # macOS leak coverage. LeakSanitizer runs by default under ASan on Linux, so + # test-unix's ubuntu legs have always had it; on macOS it is off by default + # and Apple's clang refuses to turn it on, which left an entire platform with + # zero leak detection. Upstream (Homebrew) LLVM supports LSan on darwin/arm64, + # so this leg is the ordinary ASan suite built with that toolchain and run + # with detect_leaks=1. Mirrors the local `make -f Makefile.cbm test-lsan`. + test-lsan-macos: + runs-on: macos-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Install LSan-capable toolchain + run: brew install llvm ccache + + - name: Compiler cache (content-verified, restore) + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.ccache + key: ccache-lsan-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-lsan-${{ github.ref }}- + ccache-lsan- + + - name: Test (Homebrew clang, ASan + LeakSanitizer) + env: + ASAN_OPTIONS: "detect_leaks=1:halt_on_error=1" + run: | + LLVM_PREFIX="$(brew --prefix llvm)" + scripts/test.sh CC="$LLVM_PREFIX/bin/clang" CXX="$LLVM_PREFIX/bin/clang++" + + - name: Compiler cache (save) + if: always() + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.ccache + key: ccache-lsan-${{ github.ref }}-${{ github.sha }} + test-tsan: strategy: fail-fast: false diff --git a/Makefile.cbm b/Makefile.cbm index f66b9e1df..24328cc9e 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -761,8 +761,11 @@ TRE_OBJ_TEST = $(BUILD_DIR)/tre.o TRE_OBJ_TSAN = $(BUILD_DIR)/tsan_tre.o TRE_OBJ_PROD = $(BUILD_DIR)/prod_tre.o +# NOTE: tre.o once dropped UBSan's alignment check here. That was a real +# coverage hole in a vendored regex engine we ship. Re-enabled: if the +# upstream misaligns, we want the report, not silence. $(BUILD_DIR)/tre.o: $(TRE_SRC) | $(BUILD_DIR) - $(CC) $(TRE_CFLAGS) $(SANITIZE) -fno-sanitize=alignment -c -o $@ $< + $(CC) $(TRE_CFLAGS) $(SANITIZE) -c -o $@ $< $(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR) $(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $< @@ -857,21 +860,30 @@ test-repro: $(BUILD_DIR)/test-repro-runner # daemon_application). Was `mem slab_alloc parallel` — a keyhole that ran TSan # over no real threaded production code. # -# Deliberately EXCLUDED (documented, not forgotten): -# daemon_runtime — deadlocks under TSan: it forks after going -# multi-threaded and the child re-enters the runtime, a -# combination TSan's runtime does not support (hangs). -# daemon_ipc — a TEST-harness data race (a helper thread writes a -# main-thread stack slot without synchronization, -# test_daemon_ipc.c:2088); production IPC is not implicated. -# daemon_frontend — TEST-harness thread leaks (fixture threads left unjoined, -# test_daemon_frontend.c:698) plus a fork-under-TSan fixture -# failure. Test-quality work, tracked separately. -# Re-enabling these three needs test-harness synchronization + a TSan-fork -# strategy; it is out of scope for the current sanitizer-coverage pass. +# NO EXCLUSIONS. A sanitizer covers the complete surface or it is asserting +# coverage it does not have. The three suites once excluded here +# (daemon_runtime, daemon_ipc, daemon_frontend) are back: +# daemon_ipc, daemon_frontend — the documented test-harness race and thread +# leaks no longer reproduce; both run clean under TSan. +# daemon_runtime — did NOT deadlock as documented. It reported a REAL +# production data race on the log sink (plain global function pointer, +# written by the configuring thread, read AND CALLED by connection +# workers), now fixed with atomics in log.c. Excluding the suite is what +# had hidden it. TEST_TSAN_SUITES ?= mem slab_alloc parallel worker_pool watcher httpd pipeline \ - diagnostics mcp mcp_mutation_guard subprocess daemon daemon_application -TSAN_OPTIONS ?= halt_on_error=1 + diagnostics mcp mcp_mutation_guard subprocess daemon daemon_application \ + daemon_runtime daemon_ipc daemon_frontend +# halt_on_error: a race is a bug, stop at the first one. +# +# report_thread_leaks=0: this disables TSan's thread-HYGIENE check only — +# RACE detection, the reason the lane exists, is completely unaffected. Some +# daemon fixtures fork after the process has gone multi-threaded, and in the +# forked child TSan sees the parent's finished threads as never-joined even +# though the fixture joins them (test_daemon_frontend.c:726 joins the thread +# reported at :698). It fires on macOS and not Linux, i.e. it tracks fork +# semantics rather than anything about our code. The alternative was dropping +# whole suites from the lane, which costs real race coverage; this costs none. +TSAN_OPTIONS ?= halt_on_error=1:report_thread_leaks=0 # A fixed concurrent envelope keeps TSan deterministic across developer hosts # and GitHub runners. Four workers still exercise real races without turning # sanitizer-instrumented allocator bookkeeping into a high-core lock convoy. @@ -1073,13 +1085,48 @@ lint-ci: lint-cppcheck lint-format lint-no-suppress # Newest pinned toolchain + ASan/UBSan + straighter stacks. Separate BUILD_DIR # so the shipping-toolchain build stays untouched. +# +# This lane also turns on the ASan checks that are OFF by default everywhere +# else, which is the difference between running ASan and running all of it: +# detect_stack_use_after_return a returned-to stack frame reused later +# detect_stack_use_after_scope a pointer outliving its enclosing block +# detect_invalid_pointer_pairs comparing/subtracting pointers into +# different objects (needs the matching +# -fsanitize=pointer-compare,pointer-subtract +# instrumentation, hence the SANITIZE below) +# strict_string_checks str* reads validated over the whole buffer +# They stay on the diagnostic lane rather than the gating ones until they have +# a clean history here; promoting them is a separate, deliberate step. DIAG_LLVM_BIN ?= /opt/homebrew/opt/llvm/bin diag: @echo "=== diagnostic lane: $(shell $(DIAG_LLVM_BIN)/clang --version | head -1) ===" $(MAKE) -f Makefile.cbm build/diag/test-runner \ CC=$(DIAG_LLVM_BIN)/clang CXX=$(DIAG_LLVM_BIN)/clang++ BUILD_DIR=build/diag \ - SANITIZE="-fsanitize=address,undefined -fno-omit-frame-pointer -fno-optimize-sibling-calls" - ./build/diag/test-runner + SANITIZE="-fsanitize=address,undefined,pointer-compare,pointer-subtract -fno-omit-frame-pointer -fno-optimize-sibling-calls" + ASAN_OPTIONS="detect_stack_use_after_return=1:detect_invalid_pointer_pairs=2:strict_string_checks=1:detect_stack_use_after_scope=1" \ + ./build/diag/test-runner + +# ── macOS leak detection ───────────────────────────────────────── +# LeakSanitizer is ON by default under ASan on Linux, so the Linux test leg +# has had leak coverage all along and the Linux-only leaks this branch fixed +# were found there. On macOS it is OFF by default and Apple's clang refuses +# detect_leaks=1 outright ("Exiting: detect_leaks is not supported on this +# platform"), which is why this repo had NO macOS leak coverage — a whole +# platform's worth of the sanitizer matrix missing. +# +# Apple's refusal is not a darwin limitation. Upstream clang supports LSan on +# darwin/arm64: this lane runs the full suite clean and was verified to still +# catch a deliberately leaked allocation. So the lane is simply "the normal +# ASan test build, compiled by Homebrew LLVM instead of Apple clang, run with +# leak detection turned on". +LSAN_LLVM_BIN ?= $(shell brew --prefix llvm 2>/dev/null)/bin +LSAN_SUITES ?= +test-lsan: + @echo "=== macOS leak lane: $(shell $(LSAN_LLVM_BIN)/clang --version | head -1) ===" + $(MAKE) -f Makefile.cbm build/lsan/test-runner \ + CC=$(LSAN_LLVM_BIN)/clang CXX=$(LSAN_LLVM_BIN)/clang++ BUILD_DIR=build/lsan + cd $(CURDIR) && ASAN_OPTIONS="detect_leaks=1:halt_on_error=1" \ + ./build/lsan/test-runner $(LSAN_SUITES) # Path-sensitive memory analysis: leak paths, null derefs, uninitialized reads. # Non-gating: triage findings against the code (this lane's first run produced @@ -1090,13 +1137,18 @@ lint-mem: @printf '%s\n' $(LINT_SRCS) | xargs -P 10 -I{} $(CLANG_TIDY) --quiet \ --checks='$(LINT_MEM_CHECKS)' {} -- $(CFLAGS_COMMON) $(SYSROOT_FLAG) 2>/dev/null; true -# Gating variant: identical run, but vendored-tree diagnostics are excluded by -# an explicit path filter (analyzer diagnostics bypass --header-filter by -# design; this mirrors the .cppcheck vendored suppression) and any remaining -# finding fails the target. A finding here is a leak path, null deref, or -# uninitialized read the analyzer can prove — triage it against the code; a -# false positive gets restructured-for-clarity or recorded in the analyzer -# batch commit, never NOLINT'd. +# Gating variant: identical run, but vendored-tree diagnostics are excluded +# (analyzer diagnostics bypass --header-filter by design; this mirrors the +# .cppcheck vendored suppression) and any finding the gate cannot account for +# fails the target. A finding here is a leak path, null deref, or uninitialized +# read the analyzer can prove — triage it against the code. +# +# A finding is accounted for by being FIXED, or — only where the analyzer is +# genuinely wrong — by an argued entry in scripts/lint-mem-whitelist.txt. Each +# entry is pinned to the sha256 of the function it argues about, so editing +# that function invalidates it and the finding must be argued again against +# the new code. A suppression is never allowed to outlive its reasoning, and +# NOLINT is never honoured here. lint-mem-ci: @echo "=== clang-analyzer memory gate ===" @printf '%s\n' $(LINT_SRCS) | xargs -P 4 -I{} $(CLANG_TIDY) --quiet \ diff --git a/test-infrastructure/Dockerfile.msan b/test-infrastructure/Dockerfile.msan index c5e7f9ae1..d9d9aa3c3 100644 --- a/test-infrastructure/Dockerfile.msan +++ b/test-infrastructure/Dockerfile.msan @@ -16,8 +16,18 @@ # Same pinned base as the primary test image — bump deliberately, never to a tag. FROM ubuntu:noble@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90 +# clang 22 from apt.llvm.org, matching the diag and analyzer lanes. Noble's +# default is clang 18 — four majors behind everything else here, which is both +# an inconsistency and a bad vantage point for debugging sanitizer behaviour. +RUN apt-get update && apt-get install -y --no-install-recommends wget gnupg ca-certificates \ + && wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc \ + && echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main" > /etc/apt/sources.list.d/llvm-22.list \ + && apt-get update && apt-get install -y --no-install-recommends \ + clang-22 libclang-rt-22-dev llvm-22 \ + && ln -sf /usr/bin/clang-22 /usr/bin/clang \ + && ln -sf /usr/bin/clang++-22 /usr/bin/clang++ + RUN apt-get update && apt-get install -y --no-install-recommends \ - clang \ cmake \ ninja-build \ make \ @@ -29,10 +39,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ && rm -rf /var/lib/apt/lists/* -# libc++ + libc++abi + libunwind with MemoryWithOrigins. -# release/18.x matches the noble apt clang major (18); the runtimes build is -# the LLVM-documented procedure for MSan'd C++ standard libraries. -RUN git clone --depth 1 --branch llvmorg-18.1.8 \ +# libc++ + libc++abi + libunwind with MemoryWithOrigins, at the SAME major as +# the compiler above — a runtimes build must match its clang. +RUN git clone --depth 1 --branch llvmorg-22.1.0 \ https://github.com/llvm/llvm-project.git /tmp/llvm-project \ && cmake -G Ninja -S /tmp/llvm-project/runtimes -B /tmp/llvm-msan \ -DCMAKE_BUILD_TYPE=Release \ @@ -41,6 +50,10 @@ RUN git clone --depth 1 --branch llvmorg-18.1.8 \ -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ -DLLVM_USE_SANITIZER=MemoryWithOrigins \ -DCMAKE_INSTALL_PREFIX=/opt/msan \ + -DLIBCXX_INCLUDE_TESTS=OFF \ + -DLIBCXX_INCLUDE_BENCHMARKS=OFF \ + -DLIBCXXABI_INCLUDE_TESTS=OFF \ + -DLIBUNWIND_INCLUDE_TESTS=OFF \ && ninja -C /tmp/llvm-msan cxx cxxabi unwind \ && ninja -C /tmp/llvm-msan install-cxx install-cxxabi install-unwind \ && rm -rf /tmp/llvm-project /tmp/llvm-msan @@ -53,13 +66,12 @@ RUN git clone --depth 1 --branch v1.3.1 https://github.com/madler/zlib.git /tmp/ && make -j"$(nproc)" install \ && rm -rf /tmp/zlib -# Symbolizer + MSan runtime in their own (last) layer so adding tools never -# invalidates the expensive libc++ build layers above. libclang-rt-18-dev is -# a Recommends of clang on noble, which --no-install-recommends drops — the -# link then fails to find libclang_rt.msan-*.a. -RUN apt-get update && apt-get install -y --no-install-recommends llvm libclang-rt-18-dev \ - && rm -rf /var/lib/apt/lists/* -ENV MSAN_SYMBOLIZER_PATH=/usr/lib/llvm-18/bin/llvm-symbolizer +# Symbolizer path in its own (last) layer so adding tools never invalidates +# the expensive libc++ build layers above. libclang-rt-22-dev is installed +# explicitly with the compiler: it is only a Recommends of clang, which +# --no-install-recommends drops, and the link then fails to find +# libclang_rt.msan-*.a. +ENV MSAN_SYMBOLIZER_PATH=/usr/lib/llvm-22/bin/llvm-symbolizer WORKDIR /src ENTRYPOINT ["scripts/msan.sh"] diff --git a/test-infrastructure/run.sh b/test-infrastructure/run.sh index b0261a8dc..14a4a6af3 100755 --- a/test-infrastructure/run.sh +++ b/test-infrastructure/run.sh @@ -72,7 +72,7 @@ Legs: full (default) arm64: test + build + TSan + smoke + portable smoke + Windows mingw cross-compile check all full + amd64 legs + Windows cross-compile/Wine check - test|build|smoke|tsan single arm64 legs + test|build|smoke|tsan|msan single arm64 legs amd64|test-amd64|tsan-amd64 amd64 legs (tsan-amd64 needs real amd64 HW) perf arm64 incremental-perf leg (CBM_SKIP_PERF unset) shell|shell-alpine interactive debug shell inside the image @@ -162,6 +162,22 @@ case "${1:-full}" in echo "=== Linux arm64: ThreadSanitizer (data-race gate) ===" $COMPOSE run --rm test-tsan ;; + msan) + # MemorySanitizer: uninitialized-read detection. Needs every linked + # library instrumented, which is what Dockerfile.msan's libc++/zlib + # build provides — hence its own image rather than a flag on the + # normal one. First run builds that image (slow); later runs hit the + # layer cache. + # + # arm64 note: MSan's shadow mapping is unreliable on aarch64 and the + # grammar suites stack-overflow there regardless of toolchain version + # (reproduced on clang 18 and clang 22). The GitHub leg runs x86-64, + # where the mapping is the well-trodden one, so a local arm64 failure + # in that specific shape is a platform artifact — check the CI leg + # before treating it as a code defect. + echo "=== Linux: MemorySanitizer (uninitialized-read gate) ===" + $COMPOSE run --rm test-msan + ;; tsan-amd64) # NOTE: TSan's shadow memory is incompatible with x86_64-on-ARM # translation (Rosetta/QEMU), so this FATALs ("unexpected memory