diff --git a/.gitattributes b/.gitattributes index 365ab6abd..06a4e43d8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,7 @@ +# Shell scripts must stay LF: with git's Windows default (core.autocrlf=true) +# they check out as CRLF and bash inside the build containers rejects them. +*.sh text eol=lf + # Vendored tree-sitter grammars — machine-generated, hide from stats and diffs internal/cbm/vendored/grammars/**/parser.c linguist-generated=true diff=false internal/cbm/vendored/grammars/**/scanner.c linguist-generated=true diff=false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2694bbacd..5f81c708b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -350,6 +350,7 @@ jobs: VT_EXPECTED_SCAN_SET: binaries/scan-set.tsv VT_ASSOCIATIONS: binaries/associations.tsv VT_RESULTS_PATH: binaries/vt-results.tsv + VT_WITHHELD: binaries/virustotal-withheld.tsv MIN_ENGINES: 50 run: scripts/ci/check-virustotal.sh @@ -362,6 +363,7 @@ jobs: binaries/associations.tsv binaries/scan-set.tsv binaries/vt-results.tsv + binaries/virustotal-withheld.tsv if-no-files-found: warn - name: Publish durable public VirusTotal evidence diff --git a/Makefile.cbm b/Makefile.cbm index 5b90c388e..bf51a5a8c 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -729,7 +729,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o # ── Targets ────────────────────────────────────────────────────── -.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security +.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security $(BUILD_DIR): mkdir -p $(BUILD_DIR) @@ -1103,6 +1103,12 @@ lint-tidy: @echo "=== clang-tidy ===" @$(CLANG_TIDY) --quiet $(LINT_SRCS) -- $(CFLAGS_COMMON) $(SYSROOT_FLAG) +# clang-tidy, scoped to this commit's staged changes only (scripts/lint-tidy-diff.sh). +# The pre-commit hook runs this instead of the full lint-tidy sweep above, so a +# clean commit is not blocked by pre-existing findings elsewhere in the tree. +lint-tidy-diff: + @scripts/lint-tidy-diff.sh "$(CLANG_TIDY)" $(LINT_SRCS) -- $(CFLAGS_COMMON) $(SYSROOT_FLAG) + # cppcheck: complementary analysis (config in .cppcheck) lint-cppcheck: @echo "=== cppcheck ===" diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 3270bacc3..f9854b03e 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -3747,6 +3747,14 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec def.is_test = true; } + // A function/method defined in a file cbm treats as a test file is itself + // a test, regardless of its own name (tests/helpers/fixtures.c has none + // of the test_/_test naming conventions, but every function in it is + // still test code) (#1294). OR'd so the Rust attribute check above, which + // additionally catches #[test] functions embedded in an otherwise regular + // .rs file, is never downgraded by this. + def.is_test = def.is_test || ctx->result->is_test_file; + // Docstring def.docstring = extract_docstring(a, node, ctx->source, ctx->language); @@ -4728,6 +4736,10 @@ static void push_method_def(CBMExtractCtx *ctx, TSNode child, TSNode class_node, // MinHash fingerprint compute_fingerprint(ctx, &def, child); + // A method on a class defined in a test file (e.g. a JUnit/pytest + // TestFoo.test_bar) is itself a test, same as free functions (#1294). + def.is_test = def.is_test || ctx->result->is_test_file; + cbm_defs_push(&ctx->result->defs, a, def); } diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 3042f606e..f139ff61e 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -295,6 +295,19 @@ bool cbm_is_test_file(const char *rel_path, CBMLanguage lang) { } const char *base = path_basename(rel_path); + /* Directory-based, language-agnostic: a file under a conventional test + * directory is a test file regardless of its own basename (e.g. + * tests/helpers/fixtures.c, which no per-language suffix/prefix rule + * below would otherwise catch). Mirrors the directory set cbm_is_test_path + * (src/pipeline/pass_tests.c) already uses for TESTS-edge detection, so + * the extraction-time is_test_file flag agrees with it (#1294). */ + if (strstr(rel_path, "__tests__/") || strstr(rel_path, "/tests/") || + strstr(rel_path, "/test/") || strstr(rel_path, "/spec/") || + has_prefix(rel_path, "tests/") || has_prefix(rel_path, "test/") || + has_prefix(rel_path, "spec/") || has_prefix(rel_path, "__tests__/")) { + return true; + } + switch (lang) { case CBM_LANG_GO: return has_suffix(base, "_test.go"); diff --git a/scripts/ci/check-virustotal.sh b/scripts/ci/check-virustotal.sh index 8374e2395..9bee287bd 100755 --- a/scripts/ci/check-virustotal.sh +++ b/scripts/ci/check-virustotal.sh @@ -106,6 +106,7 @@ class ExpectedObject: size: int association_count: int association_kinds: frozenset[str] + withheld: bool @property def requires_microsoft(self) -> bool: @@ -188,7 +189,54 @@ def parse_versioned_tsv( return metadata, rows -def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]: +def load_withheld(raw_path: str) -> Dict[str, str]: + """Objects deliberately removed from the surface scan (VT_WITHHELD). + + exclude-rescanned-selected-objects.sh deletes the SELECTED executables from + the objects directory before the upload — their bytes were already scanned + as release candidates, identity is settled by sha256, and re-submitting + identical bytes re-rolls a probabilistic classifier (measured on v0.10.5: + verdicts flipped in both directions within the hour). The gate accepts a + withheld object only when this manifest vouches for it BY HASH; everything + else keeps the strict on-disk contract. An absent manifest keeps the + original behavior everywhere else this gate runs (candidate stage, + dry-run), where nothing is ever withheld.""" + path = pathlib.Path(raw_path).absolute() + if path.is_symlink() or not path.is_file(): + raise GateError(f"withheld manifest is not a regular file: {path}") + if path.stat().st_size > 1024 * 1024: + raise GateError(f"withheld manifest is unexpectedly large: {path}") + lines = path.read_text(encoding="utf-8").splitlines() + if not lines or lines[0] != "# cbm-virustotal-withheld-v1": + raise GateError("withheld manifest marker is missing") + cursor = 1 + metadata: Dict[str, str] = {} + while cursor < len(lines) and lines[cursor].startswith("# "): + key, separator, value = lines[cursor][2:].partition("=") + if not separator or not key or key in metadata or not value: + raise GateError(f"malformed withheld metadata: {lines[cursor]}") + metadata[key] = value + cursor += 1 + if metadata.get("reason") != "already-scanned-as-candidate": + raise GateError("withheld manifest does not state the accepted reason") + if cursor >= len(lines) or lines[cursor] != "sha256\tobject": + raise GateError("withheld manifest header is malformed") + withheld: Dict[str, str] = {} + for line in lines[cursor + 1 :]: + sha256, separator, name = line.partition("\t") + if not separator or SHA256_RE.fullmatch(sha256) is None or not name or "/" in name: + raise GateError(f"malformed withheld row: {line}") + if sha256 in withheld: + raise GateError(f"duplicate withheld sha256: {sha256}") + withheld[sha256] = name + if not withheld: + raise GateError("withheld manifest names no objects") + return withheld + + +def load_expected( + path: pathlib.Path, withheld: Dict[str, str] +) -> Tuple[List[ExpectedObject], int]: metadata, rows = parse_versioned_tsv( path, marker="cbm-release-scan-set-v2", @@ -230,15 +278,26 @@ def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]: ): raise GateError(f"unassociated object in expected set: {scan_path}") local_path = path.parent.joinpath(*pure.parts) - try: - mode = local_path.lstat().st_mode - except FileNotFoundError as error: - raise GateError(f"expected scan object is missing: {scan_path}") from error - if not stat.S_ISREG(mode): - raise GateError(f"expected scan object is not a regular file: {scan_path}") - actual_size = local_path.stat().st_size - if actual_size != size or sha256_file(local_path) != sha256: - raise GateError(f"expected scan object changed after extraction: {scan_path}") + is_withheld = sha256 in withheld + if is_withheld: + if withheld[sha256] != pure.parts[1]: + raise GateError( + f"withheld manifest names a different object for this hash: {scan_path}" + ) + if os.path.lexists(local_path): + raise GateError( + f"withheld object is still present in the scan directory: {scan_path}" + ) + else: + try: + mode = local_path.lstat().st_mode + except FileNotFoundError as error: + raise GateError(f"expected scan object is missing: {scan_path}") from error + if not stat.S_ISREG(mode): + raise GateError(f"expected scan object is not a regular file: {scan_path}") + actual_size = local_path.stat().st_size + if actual_size != size or sha256_file(local_path) != sha256: + raise GateError(f"expected scan object changed after extraction: {scan_path}") association_total += association_count objects.append( ExpectedObject( @@ -248,10 +307,17 @@ def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]: size=size, association_count=association_count, association_kinds=frozenset(kinds), + withheld=is_withheld, ) ) if association_total != associations: raise GateError("scan-set association counts do not match manifest metadata") + expected_hashes = {item.sha256 for item in objects} + spurious = sorted(set(withheld) - expected_hashes) + if spurious: + raise GateError(f"withheld manifest names hashes outside the expected scan set: {spurious}") + if all(item.withheld for item in objects): + raise GateError("every expected object is withheld; the surface scan would cover nothing") return objects, associations @@ -363,6 +429,7 @@ def parse_action_output( objects: Sequence[ExpectedObject], manifest: pathlib.Path, ) -> List[Submission]: + objects = [item for item in objects if not item.withheld] if not raw: raise GateError("VirusTotal action output is empty") aliases = output_aliases(objects, manifest) @@ -697,7 +764,10 @@ def main() -> None: raise GateError(f"unsafe pre-existing VT results path: {results_path}") results_path.unlink() - expected, associations = load_expected(expected_manifest) + withheld_path = os.environ.get("VT_WITHHELD", "") + withheld = load_withheld(withheld_path) if withheld_path else {} + + expected, associations = load_expected(expected_manifest, withheld) validate_associations( associations_manifest, objects=expected, @@ -708,6 +778,13 @@ def main() -> None: objects=expected, manifest=expected_manifest, ) + withheld_count = sum(1 for item in expected if item.withheld) + if withheld_count: + print( + f"=== {withheld_count} expected object(s) withheld from the surface scan " + f"(already scanned as candidates; identity settled by sha256, manifest: " + f"{withheld_path}) ===" + ) print( f"=== VirusTotal exact-set gate: {len(submissions)} distinct objects, " f"{associations} associations, >= {min_engines} decisive engines each ===" diff --git a/scripts/hooks/pre-commit b/scripts/hooks/pre-commit index bbd1ee71d..fa42d92ae 100755 --- a/scripts/hooks/pre-commit +++ b/scripts/hooks/pre-commit @@ -7,7 +7,11 @@ set -euo pipefail # git config core.hooksPath scripts/hooks echo "pre-commit: running all linters in parallel..." -make -j3 -f Makefile.cbm lint +# clang-tidy runs diff-scoped (lint-tidy-diff, see scripts/lint-tidy-diff.sh): +# the full lint-tidy sweep surfaces thousands of pre-existing findings that +# have nothing to do with this commit and would block every contributor who +# has clang-tidy on PATH. `make lint-tidy` remains available for a full audit. +make -j3 -f Makefile.cbm lint-ci lint-tidy-diff echo "pre-commit: security audit (source-level)..." scripts/security-audit.sh diff --git a/scripts/lint-tidy-diff.sh b/scripts/lint-tidy-diff.sh new file mode 100755 index 000000000..669c90e98 --- /dev/null +++ b/scripts/lint-tidy-diff.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# lint-tidy-diff.sh — clang-tidy scoped to the lines this commit changes. +# +# `make lint-tidy` (the full sweep) checks the entire LINT_SRCS tree, which +# currently surfaces ~5,100 pre-existing findings unrelated to any given +# commit (see Makefile.cbm's `lint-tidy` target). That is fine for a manual +# deep audit but makes the pre-commit hook unusable: it blocks every commit, +# including ones that touch files with zero findings on the touched lines. +# +# This script runs the same clang-tidy binary/config against the SAME staged +# .c files, but passes clang-tidy's own `-line-filter` so only lines the +# commit actually added/modified can produce a diagnostic. Pre-existing +# findings on untouched lines — in the same file or elsewhere — are not +# reported. +# +# Invoked by `make lint-tidy-diff` (scripts/hooks/pre-commit), which supplies +# the resolved clang-tidy binary, then LINT_SRCS, then `--`, then the +# compiler flags — the same argv shape as the `lint-tidy` target itself. +# +# Known limitation: only staged *.c files that are members of LINT_SRCS are +# checked (matching the full sweep's own scope: vendored code and tests/ are +# never clang-tidy'd). clang-tidy reports a header's diagnostics under +# whatever path it was #include-d with, which is not reliably the same +# string as its path in the working tree, so a commit that touches only a +# header is not covered here even if a .c file including it also changed; +# run `make lint-tidy` for a full pass in that case. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +if [ "$#" -lt 1 ]; then + echo "usage: lint-tidy-diff.sh [LINT_SRCS...] -- [CFLAGS...]" >&2 + exit 2 +fi + +CLANG_TIDY="$1" +shift + +LINT_SRCS=() +while [ "$#" -gt 0 ] && [ "$1" != "--" ]; do + LINT_SRCS+=("$1") + shift +done +[ "$#" -gt 0 ] && shift # drop the `--` separator +CFLAGS=("$@") + +# bash 3.2 (macOS default, see scripts/check-lsp-originality.sh) has neither +# `mapfile` nor associative arrays — read loop + linear membership check. +is_lint_src() { + local needle="$1" f + [ "${#LINT_SRCS[@]}" -eq 0 ] && return 1 + for f in "${LINT_SRCS[@]}"; do + [ "$f" = "$needle" ] && return 0 + done + return 1 +} + +# Staged .c files this commit actually changed (added/copied/modified/ +# renamed — a delete has no new lines to check) that are members of +# LINT_SRCS. +CHANGED=() +while IFS= read -r f; do + [ -n "$f" ] && is_lint_src "$f" && CHANGED+=("$f") +done < <(git diff --cached --name-only --diff-filter=ACMR -- '*.c') + +if [ "${#CHANGED[@]}" -eq 0 ]; then + echo "lint-tidy-diff: no staged changes to a linted .c file, skipping clang-tidy" + exit 0 +fi + +# For each changed file, turn its staged hunks into a clang-tidy line-filter +# range: `@@ -a,b +c,d @@` -> lines [c, c+d-1]. Zero-context (-U0) so a range +# never includes untouched, pre-existing lines. Pure-deletion hunks (d == 0) +# add no new lines and are skipped. +LINE_FILTER='[' +FILTER_SEP='' +TARGETS=() +for f in "${CHANGED[@]}"; do + [ -f "$f" ] || continue # renamed away / not present in the working tree + ranges='' + range_sep='' + while IFS= read -r hunk; do + if [[ "$hunk" =~ ^@@\ -[0-9]+(,[0-9]+)?\ \+([0-9]+)(,([0-9]+))?\ @@ ]]; then + new_start="${BASH_REMATCH[2]}" + new_count="${BASH_REMATCH[4]:-1}" + [ "$new_count" -eq 0 ] && continue + new_end=$((new_start + new_count - 1)) + ranges+="${range_sep}[${new_start},${new_end}]" + range_sep=',' + fi + done < <(git diff --cached -U0 --diff-filter=ACMR -- "$f" | grep -E '^@@ ') + [ -z "$ranges" ] && continue + + LINE_FILTER+="${FILTER_SEP}{\"name\":\"${f}\",\"lines\":[${ranges}]}" + FILTER_SEP=',' + TARGETS+=("$f") +done +LINE_FILTER+=']' + +if [ "${#TARGETS[@]}" -eq 0 ]; then + echo "lint-tidy-diff: staged .c changes are deletion-only, skipping clang-tidy" + exit 0 +fi + +echo "lint-tidy-diff: ${#TARGETS[@]} changed file(s), diff-scoped clang-tidy" +"$CLANG_TIDY" --quiet "-line-filter=${LINE_FILTER}" "${TARGETS[@]}" -- "${CFLAGS[@]}" diff --git a/scripts/security-fuzz-random.sh b/scripts/security-fuzz-random.sh index baa71e5ae..9b4903b4f 100755 --- a/scripts/security-fuzz-random.sh +++ b/scripts/security-fuzz-random.sh @@ -27,8 +27,11 @@ fi echo "=== Fuzz Testing ($DURATION seconds) ===" -FUZZ_TMPDIR=$(mktemp -d) -trap 'rm -rf "$FUZZ_TMPDIR"' EXIT +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +FUZZ_TMPDIR="$CBM_TEST_RUNTIME_ROOT" +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT CRASHES=0 ITERATIONS=0 diff --git a/scripts/security-fuzz.sh b/scripts/security-fuzz.sh index bdf1ca58d..2806989a9 100755 --- a/scripts/security-fuzz.sh +++ b/scripts/security-fuzz.sh @@ -22,11 +22,14 @@ PASS=0 TOTAL=0 # Temp directory for input files (avoids pipe/stdin issues with timeout) -FUZZ_TMPDIR=$(mktemp -d) -trap 'rm -rf "$FUZZ_TMPDIR"' EXIT +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +FUZZ_TMPDIR="$CBM_TEST_RUNTIME_ROOT" +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT FUZZ_HOME="$FUZZ_TMPDIR/home" -FUZZ_CACHE="$FUZZ_TMPDIR/cache" -mkdir -p "$FUZZ_HOME" "$FUZZ_CACHE" +mkdir "$FUZZ_HOME" +export HOME="$FUZZ_HOME" # Helper: send a payload to the MCP server and check it doesn't crash. # Uses temp file + perl alarm for portable timeout (works on macOS + Linux). @@ -49,11 +52,9 @@ test_payload() { # Run with 10s timeout: GNU timeout → perl alarm fallback local ec=0 if command -v timeout &>/dev/null; then - HOME="$FUZZ_HOME" CBM_CACHE_DIR="$FUZZ_CACHE" \ - timeout 10 "$BINARY" < "$tmpinput" > "$tmpoutput" 2>&1 || ec=$? + timeout 10 "$BINARY" < "$tmpinput" > "$tmpoutput" 2>&1 || ec=$? else - HOME="$FUZZ_HOME" CBM_CACHE_DIR="$FUZZ_CACHE" \ - perl -e 'alarm(10); exec @ARGV' -- "$BINARY" \ + perl -e 'alarm(10); exec @ARGV' -- "$BINARY" \ < "$tmpinput" > "$tmpoutput" 2>&1 || ec=$? fi @@ -88,8 +89,7 @@ test_invalid_index_interactive() { local tmpoutput="$FUZZ_TMPDIR/output_${TOTAL}.jsonl" local ec=0 - HOME="$FUZZ_HOME" CBM_CACHE_DIR="$FUZZ_CACHE" \ - python3 "$(dirname "$0")/test_mcp_interactive.py" "$BINARY" \ + python3 "$(dirname "$0")/test_mcp_interactive.py" "$BINARY" \ --scenario invalid-index --repo-path /nonexistent/path/abc123 \ --response-timeout 45 --exit-timeout 15 > "$tmpoutput" 2>&1 || ec=$? diff --git a/scripts/security-install.sh b/scripts/security-install.sh index 291d8d7f4..ac9d74fb9 100755 --- a/scripts/security-install.sh +++ b/scripts/security-install.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -# Layer 4: Install output audit — verifies install --dry-run writes only to expected paths. +# Layer 4: Install output audit — runs a sandboxed real install and audits its writes. # # Checks: # 1. All output file paths are in the expected set @@ -19,8 +19,11 @@ fi echo "=== Layer 4: Install Output Audit ===" -TMPDIR=$(mktemp -d) -trap 'rm -rf "$TMPDIR"' EXIT +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +TMPDIR="$CBM_TEST_RUNTIME_ROOT" +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT # Set HOME to tmpdir so install writes there instead of real home export HOME="$TMPDIR/home" diff --git a/scripts/security-network.sh b/scripts/security-network.sh index 60b64abdc..d7e83c1d9 100755 --- a/scripts/security-network.sh +++ b/scripts/security-network.sh @@ -30,8 +30,11 @@ if ! command -v strace &>/dev/null; then exit 0 fi -TMPDIR=$(mktemp -d) -trap 'rm -rf "$TMPDIR"' EXIT +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +TMPDIR="$CBM_TEST_RUNTIME_ROOT" +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT # Create a minimal test project mkdir -p "$TMPDIR/project/src" diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh new file mode 100644 index 000000000..cb69cadfb --- /dev/null +++ b/scripts/test-runtime.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +# Sourced by test harnesses. Host paths stay separate from paths exported to a +# native Windows product binary. +CBM_TEST_RUNTIME_ROOT="" +CBM_TEST_RUNTIME_DIR_HOST="" +CBM_TEST_CACHE_DIR_HOST="" +_CBM_TEST_RUNTIME_CREATED_ROOT="" +_CBM_TEST_RUNTIME_PRODUCT_RUNTIME="" +_CBM_TEST_RUNTIME_PRODUCT_CACHE="" + +_cbm_test_runtime_error() { + echo "test-runtime: $*" >&2 +} + +_cbm_test_runtime_discard_partial() { + local root="$1" parent="$2" name="${1##*/}" + [[ -n "$root" && "$root" != "$name" && "${root%/*}" == "$parent" && ! -L "$root" ]] || return 0 + case "$name" in cbmrt.?*|cbmrt-?*) rm -rf -- "$root" >/dev/null 2>&1 || true ;; esac +} + +cbm_test_runtime_init() { + [[ -z "$_CBM_TEST_RUNTIME_CREATED_ROOT" ]] || + { _cbm_test_runtime_error "runtime already initialized"; return 1; } + local platform windows=0 parent root="" runtime_host cache_host + local product_runtime product_cache helper_dir helper_native root_native + platform="$(uname -s)" || + { _cbm_test_runtime_error "cannot identify the host platform"; return 1; } + case "$platform" in + MINGW*|MSYS*|CYGWIN*) windows=1 ;; + Darwin) parent="/private/tmp" ;; + *) parent="/tmp" ;; + esac + if (( windows )); then + command -v cygpath >/dev/null 2>&1 || + { _cbm_test_runtime_error "cygpath is required on Windows"; return 1; } + if [[ -n "${CBM_CI_TEMP_ROOT:-}" ]]; then + parent="$(cygpath -u "$CBM_CI_TEMP_ROOT")" || + { _cbm_test_runtime_error "cannot convert CBM_CI_TEMP_ROOT"; return 1; } + [[ -d "$parent" && ! -L "$parent" ]] || + { _cbm_test_runtime_error "CBM_CI_TEMP_ROOT is not a real directory"; return 1; } + root="$(mktemp -d "${parent%/}/cbmrt.XXXXXX")" || + { _cbm_test_runtime_error "cannot create a private Windows CI root"; return 1; } + else + helper_dir="$(CDPATH= cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd)" || return 1 + helper_native="$(cygpath -w "$helper_dir/ci/new-protected-temp-root.ps1")" || + { _cbm_test_runtime_error "cannot convert protected-root helper"; return 1; } + root_native="$(MSYS2_ARG_CONV_EXCL='*' powershell.exe -NoProfile \ + -ExecutionPolicy Bypass -File "$helper_native" -Prefix 'cbmrt-')" || + { _cbm_test_runtime_error "protected Windows root creation failed"; return 1; } + root_native="${root_native//$'\r'/}" + [[ -n "$root_native" && "$root_native" != *$'\n'* ]] || + { _cbm_test_runtime_error "protected-root helper returned an invalid path"; return 1; } + root="$(cygpath -u "$root_native")" || + { _cbm_test_runtime_error "cannot convert protected Windows root"; return 1; } + parent="${root%/*}" + fi + else + root="$(mktemp -d "${parent%/}/cbmrt.XXXXXX")" || + { _cbm_test_runtime_error "cannot create private root under $parent"; return 1; } + chmod 700 "$root" || { + _cbm_test_runtime_discard_partial "$root" "$parent" + _cbm_test_runtime_error "cannot protect runtime root $root" + return 1 + } + fi + runtime_host="$root/runtime" + cache_host="$root/cache" + mkdir "$runtime_host" "$cache_host" || { + _cbm_test_runtime_discard_partial "$root" "$parent" + _cbm_test_runtime_error "cannot create runtime/cache directories" + return 1 + } + if (( ! windows )); then + chmod 700 "$runtime_host" "$cache_host" || { + _cbm_test_runtime_discard_partial "$root" "$parent" + _cbm_test_runtime_error "cannot protect runtime/cache directories" + return 1 + } + product_runtime="$runtime_host" + product_cache="$cache_host" + else + product_runtime="$(cygpath -m "$runtime_host")" || { + _cbm_test_runtime_discard_partial "$root" "$parent" + _cbm_test_runtime_error "cannot convert the runtime directory" + return 1 + } + product_cache="$(cygpath -m "$cache_host")" || { + _cbm_test_runtime_discard_partial "$root" "$parent" + _cbm_test_runtime_error "cannot convert the cache directory" + return 1 + } + fi + CBM_TEST_RUNTIME_ROOT="$root" + CBM_TEST_RUNTIME_DIR_HOST="$runtime_host" + CBM_TEST_CACHE_DIR_HOST="$cache_host" + _CBM_TEST_RUNTIME_CREATED_ROOT="$root" + _CBM_TEST_RUNTIME_PRODUCT_RUNTIME="$product_runtime" + _CBM_TEST_RUNTIME_PRODUCT_CACHE="$product_cache" + export CBM_RUNTIME_DIR="$product_runtime" CBM_CACHE_DIR="$product_cache" +} + +_cbm_test_runtime_daemon() { + CBM_RUNTIME_DIR="$_CBM_TEST_RUNTIME_PRODUCT_RUNTIME" \ + CBM_CACHE_DIR="$_CBM_TEST_RUNTIME_PRODUCT_CACHE" "$1" daemon "$2" +} + +cbm_test_runtime_cleanup() { + local binary="${1:-}" root="$_CBM_TEST_RUNTIME_CREATED_ROOT" + local name="${_CBM_TEST_RUNTIME_CREATED_ROOT##*/}" runtime_entry="" active=0 + [[ -n "$root" ]] || return 0 + case "$name" in + cbmrt.?*|cbmrt-?*) ;; + *) _cbm_test_runtime_error "refusing cleanup for unexpected basename: $name"; return 0 ;; + esac + if [[ "$root" != "$CBM_TEST_RUNTIME_ROOT" || -L "$root" ]]; then + _cbm_test_runtime_error "refusing cleanup for an unexpected runtime root: $root" + return 0 + fi + if [[ ! -e "$root" ]]; then + _CBM_TEST_RUNTIME_CREATED_ROOT="" + return 0 + fi + if [[ ! -d "$root" || "$CBM_TEST_RUNTIME_DIR_HOST" != "$root/runtime" || + "$CBM_TEST_CACHE_DIR_HOST" != "$root/cache" || + -L "$CBM_TEST_RUNTIME_DIR_HOST" || -L "$CBM_TEST_CACHE_DIR_HOST" ]]; then + _cbm_test_runtime_error "refusing cleanup after runtime path replacement: $root" + return 0 + fi + if [[ -d "$CBM_TEST_RUNTIME_DIR_HOST" ]] && + ! runtime_entry="$(find "$CBM_TEST_RUNTIME_DIR_HOST" -mindepth 1 -print -quit 2>/dev/null)"; then + _cbm_test_runtime_error "cannot inspect runtime state; leaving $root" + return 0 + fi + if [[ -n "$runtime_entry" ]]; then + [[ -x "$binary" ]] || { + _cbm_test_runtime_error "cannot check the private daemon; leaving $root" + return 0 + } + if _cbm_test_runtime_daemon "$binary" status >/dev/null 2>&1; then + active=1 + _cbm_test_runtime_daemon "$binary" stop >/dev/null 2>&1 || true + for _attempt in {1..50}; do + if ! _cbm_test_runtime_daemon "$binary" status >/dev/null 2>&1; then + active=0 + break + fi + sleep 0.1 + done + fi + fi + if (( active )); then + _cbm_test_runtime_error "private daemon is still active; leaving $root" + return 0 + fi + if rm -rf -- "$root"; then + _CBM_TEST_RUNTIME_CREATED_ROOT="" + else + _cbm_test_runtime_error "cleanup failed; leaving $root" + fi + return 0 +} diff --git a/scripts/test.sh b/scripts/test.sh index 2cce175ac..0ec489a59 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -14,7 +14,7 @@ Usage: scripts/test.sh [--suites LIST] [--arch ARCH] [VAR=VAL ...] The canonical test entry: identical in local CI, PR CI, dry run and release. DEFAULT (no --suites) is exactly what CI runs: static contract checks -(Step 0a-0s), a CLEAN sanitizer build, every suite via the parallel harness, +(Step 0a-0t), a CLEAN sanitizer build, every suite via the parallel harness, then the prod-binary regression guards (Steps 4-6). Modes: @@ -257,6 +257,9 @@ bash "$ROOT/tests/test_vt_candidate_selection_contract.sh" echo "=== Step 0s: release gate-chain ordering contract ===" bash "$ROOT/tests/test_release_gate_chain_contract.sh" +echo "=== Step 0t: test runtime isolation contract (#1691) ===" +bash "$ROOT/tests/test_runtime_isolation_contract.sh" + # Verify compiler supports target arch verify_compiler "$CC" diff --git a/src/cli/cli.c b/src/cli/cli.c index 1f850de8f..d981af266 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -12561,6 +12561,8 @@ static void cli_add_typed(yyjson_mut_doc *out, yyjson_mut_val *obj, const char * yyjson_mut_obj_add(obj, yyjson_mut_strcpy(out, key), vv); } +static bool cli_stdin_allowed_for_schema(const char *schema_str); + bool cbm_cli_args_from_stdin_allowed(const char *tool_name, bool stdin_is_tty) { /* An interactive run has nothing piped to read: consulting the terminal * would just sit waiting for the user to type JSON. Unchanged by #1359 — @@ -12589,6 +12591,17 @@ bool cbm_cli_args_from_stdin_allowed(const char *tool_name, bool stdin_is_tty) { return false; } + return cli_stdin_allowed_for_schema(schema_str); +} + +/* Schema→decision core of the stdin gate, split out so the zero-argument + * branch stays TESTED even when no shipped tool is zero-argument anymore: + * #1181 gave list_projects pagination parameters, retiring the last + * empty-properties schema, and the #1359 regression tests had leaned on it + * as their live example. The product contract is unchanged — a schema that + * declares no properties means stdin has nothing to carry and the read is + * pure deadlock; the tests now exercise this path directly. */ +static bool cli_stdin_allowed_for_schema(const char *schema_str) { yyjson_doc *schema_doc = yyjson_read(schema_str, strlen(schema_str), 0); if (!schema_doc) { /* The schemas are compile-time constants, so a parse failure means a @@ -12604,6 +12617,12 @@ bool cbm_cli_args_from_stdin_allowed(const char *tool_name, bool stdin_is_tty) { return accepts_arguments; } +#ifdef CBM_CLI_ENABLE_TEST_API +bool cbm_cli_stdin_allowed_for_schema_for_test(const char *schema_str) { + return cli_stdin_allowed_for_schema(schema_str); +} +#endif + char *cbm_cli_build_args_json(const char *tool_name, int argc, char **argv, char **err_out) { if (err_out) { *err_out = NULL; diff --git a/src/cypher/cypher.c b/src/cypher/cypher.c index fa3a3a14e..2b2efc110 100644 --- a/src/cypher/cypher.c +++ b/src/cypher/cypher.c @@ -3071,31 +3071,14 @@ static void scan_alternation_labels(cbm_store_t *store, const char *project, con free(copy); } -static void scan_pattern_nodes(cbm_store_t *store, const char *project, int max_rows, - cbm_node_pattern_t *first, cbm_node_t **out_nodes, int *out_count) { +static void scan_pattern_nodes(cbm_store_t *store, const char *project, cbm_node_pattern_t *first, + cbm_node_t **out_nodes, int *out_count) { if (first->label && strchr(first->label, '|')) { scan_alternation_labels(store, project, first->label, out_nodes, out_count); } else if (first->label) { cbm_store_find_nodes_by_label(store, project, first->label, out_nodes, out_count); } else { - cbm_search_params_t params = {.project = project, - .min_degree = CYP_FOUND_NONE, - .max_degree = CYP_FOUND_NONE, - .limit = max_rows * CYP_GROWTH_10}; - cbm_search_output_t sout = {0}; - cbm_store_search(store, ¶ms, &sout); - *out_count = sout.count; - *out_nodes = malloc(sout.count * sizeof(cbm_node_t)); - for (int i = 0; i < sout.count; i++) { - (*out_nodes)[i] = sout.results[i].node; - sout.results[i].node.name = NULL; - sout.results[i].node.project = NULL; - sout.results[i].node.label = NULL; - sout.results[i].node.qualified_name = NULL; - sout.results[i].node.file_path = NULL; - sout.results[i].node.properties_json = NULL; - } - cbm_store_search_free(&sout); + cbm_store_find_nodes(store, project, out_nodes, out_count); } /* Apply inline property filters — free rejected nodes' strings */ if (first->prop_count > 0) { @@ -4779,7 +4762,7 @@ static int expand_additional_patterns(cbm_store_t *store, cbm_query_t *q, const cbm_node_t *extra_nodes = NULL; int extra_count = 0; - scan_pattern_nodes(store, project, max_rows, &patn->nodes[0], &extra_nodes, &extra_count); + scan_pattern_nodes(store, project, &patn->nodes[0], &extra_nodes, &extra_count); int rc = 0; if (patn->rel_count == 0) { rc = cross_join_nodes(bindings, bind_count, extra_nodes, extra_count, nvar, opt); @@ -4831,7 +4814,7 @@ static int execute_single(cbm_store_t *store, cbm_query_t *q, const char *projec /* Step 1: Scan initial nodes */ cbm_node_t *scanned = NULL; int scan_count = 0; - scan_pattern_nodes(store, project, max_rows, &pat0->nodes[0], &scanned, &scan_count); + scan_pattern_nodes(store, project, &pat0->nodes[0], &scanned, &scan_count); /* Build initial bindings with early WHERE */ int bind_cap = scan_count > max_rows ? scan_count : (max_rows > 0 ? max_rows : SKIP_ONE); diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index 4ef1a238a..de9cd59a7 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -1344,12 +1344,18 @@ static bool private_log_base_name_valid(const char *base_name) { } static char *private_log_directory_path_copy(const char *directory_path) { -#ifdef __APPLE__ - /* Darwin exposes the trusted top-level aliases /tmp -> /private/tmp and - * /var -> /private/var. Resolve only those root-owned aliases before the +#if defined(__APPLE__) || defined(__FreeBSD__) + /* Resolve only those root-owned aliases before the * component-wise O_NOFOLLOW walk. Canonicalizing the complete caller path * would follow an attacker-controlled cache/log symlink and is forbidden. */ +#if defined(__APPLE__) + /* Darwin exposes the trusted top-level aliases /tmp -> /private/tmp and + * /var -> /private/var. */ static const char *const aliases[] = {"/tmp", "/var"}; +#else + /* FreeBSD additionally exposes /home -> /usr/home as a root-owned alias. */ + static const char *const aliases[] = {"/tmp", "/var", "/home"}; +#endif for (size_t index = 0; index < sizeof(aliases) / sizeof(aliases[0]); index++) { const char *alias = aliases[index]; size_t alias_length = strlen(alias); @@ -3365,6 +3371,8 @@ typedef BOOL(WINAPI *initialize_security_descriptor_fn)(PSECURITY_DESCRIPTOR, DW typedef BOOL(WINAPI *set_security_descriptor_dacl_fn)(PSECURITY_DESCRIPTOR, BOOL, PACL, BOOL); typedef BOOL(WINAPI *set_security_descriptor_owner_fn)(PSECURITY_DESCRIPTOR, PSID, BOOL); typedef BOOL(WINAPI *get_acl_information_fn)(PACL, LPVOID, DWORD, ACL_INFORMATION_CLASS); +typedef BOOL(WINAPI *get_security_descriptor_control_fn)(PSECURITY_DESCRIPTOR, + PSECURITY_DESCRIPTOR_CONTROL, LPDWORD); typedef BOOL(WINAPI *get_ace_fn)(PACL, DWORD, LPVOID *); typedef DWORD(WINAPI *get_security_info_fn)(HANDLE, SE_OBJECT_TYPE, SECURITY_INFORMATION, PSID *, PSID *, PACL *, PACL *, PSECURITY_DESCRIPTOR *); @@ -3393,6 +3401,7 @@ typedef struct { set_security_descriptor_dacl_fn set_security_descriptor_dacl; set_security_descriptor_owner_fn set_security_descriptor_owner; get_acl_information_fn get_acl_information; + get_security_descriptor_control_fn get_security_descriptor_control; get_ace_fn get_ace; get_security_info_fn get_security_info; set_security_info_fn set_security_info; @@ -3679,6 +3688,8 @@ static bool win_security_init(win_security_t *security) { "SetSecurityDescriptorOwner"); RESOLVE_ADVAPI_MEMBER(security, get_acl_information, get_acl_information_fn, "GetAclInformation"); + RESOLVE_ADVAPI_MEMBER(security, get_security_descriptor_control, + get_security_descriptor_control_fn, "GetSecurityDescriptorControl"); RESOLVE_ADVAPI_MEMBER(security, get_ace, get_ace_fn, "GetAce"); RESOLVE_ADVAPI_MEMBER(security, get_security_info, get_security_info_fn, "GetSecurityInfo"); RESOLVE_ADVAPI_MEMBER(security, set_security_info, set_security_info_fn, "SetSecurityInfo"); @@ -4170,6 +4181,187 @@ static bool win_file_security_secure(win_security_t *security, HANDLE file, win_file_acl_secure(security, file, mutation, ancestor); } +/* Is this object's DACL present but EMPTY (zero ACEs)? That denies everyone, + * including the owner, for anything the owner-rights path does not cover. + * + * It needs its own test because win_file_acl_secure() cannot detect it: that + * function scans ACEs for untrusted mutation grants, and a DACL with zero ACEs + * trivially has none, so damage reads as compliance. */ +/* The lock-directory ADOPTION predicate (private_win_owner_only_dacl in the + * foundation layer) demands a PROTECTED DACL whose single non-inherited ACE + * grants the CURRENT USER full access — strictly narrower than "no untrusted + * mutation rights", which also admits SYSTEM/Administrators ACEs. The + * already-correct fast path below must apply the CONSUMER'S predicate: a + * directory that merely passes the general secure() check but is not + * owner-only would skip the re-stamp and then strand every subsequent lock + * adoption (observed as 59 daemon-suite failures on a fresh runtime dir whose + * inherited DACL carried SYSTEM+Administrators). */ +static bool win_file_dacl_is_owner_only(win_security_t *security, HANDLE file) { + if (!security->get_security_descriptor_control) { + return false; + } + PSID owner = NULL; + PACL dacl = NULL; + PSECURITY_DESCRIPTOR descriptor = NULL; + if (security->get_security_info(file, SE_FILE_OBJECT, + OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION, &owner, + NULL, &dacl, NULL, &descriptor) != ERROR_SUCCESS) { + return false; + } + SECURITY_DESCRIPTOR_CONTROL control = 0; + DWORD revision = 0; + ACL_SIZE_INFORMATION information; + memset(&information, 0, sizeof(information)); + LPVOID opaque_ace = NULL; + bool valid = descriptor && owner && dacl && security->is_valid_sid(owner) && + security->equal_sid(owner, security->user_sid) && + security->get_security_descriptor_control(descriptor, &control, &revision) && + (control & SE_DACL_PRESENT) != 0 && (control & SE_DACL_PROTECTED) != 0 && + security->is_valid_acl(dacl) && + security->get_acl_information(dacl, &information, sizeof(information), + AclSizeInformation) && + information.AceCount == 1U && security->get_ace(dacl, 0, &opaque_ace) && + opaque_ace; + if (valid) { + ACCESS_ALLOWED_ACE *ace = (ACCESS_ALLOWED_ACE *)opaque_ace; + PSID ace_sid = (PSID)&ace->SidStart; + valid = ace->Header.AceType == ACCESS_ALLOWED_ACE_TYPE && + ace->Header.AceSize >= sizeof(ACCESS_ALLOWED_ACE) && + (ace->Header.AceFlags & (INHERITED_ACE | INHERIT_ONLY_ACE)) == 0 && + security->is_valid_sid(ace_sid) && + security->equal_sid(ace_sid, security->user_sid) && + (ace->Mask == FILE_ALL_ACCESS || ace->Mask == GENERIC_ALL); + } + if (descriptor) { + (void)LocalFree(descriptor); + } + return valid; +} + +static bool win_file_dacl_is_empty(win_security_t *security, HANDLE file) { + PACL dacl = NULL; + PSECURITY_DESCRIPTOR descriptor = NULL; + if (security->get_security_info(file, SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL, + &dacl, NULL, &descriptor) != ERROR_SUCCESS) { + return false; + } + ACL_SIZE_INFORMATION information; + memset(&information, 0, sizeof(information)); + bool empty = dacl && security->is_valid_acl(dacl) && + security->get_acl_information(dacl, &information, sizeof(information), + AclSizeInformation) && + information.AceCount == 0U; + if (descriptor) { + (void)LocalFree(descriptor); + } + return empty; +} + +/* Repair cache/runtime children left unusable by the pre-v0.10.3 DACL regime. + * + * Between v0.9.1-rc and v0.10.2 the runtime directory carried a PROTECTED DACL + * whose ACE was not inheritable. Windows therefore gave every file created + * inside it either an empty DACL or the token default (SYSTEM + TokenOwner + + * logon SID). Under an elevated token TokenOwner is BUILTIN\Administrators, so + * the interactive user ends up with no durable grant at all and the file is + * unreadable after the next logon — #1601, where takeown and icacls both fail + * non-elevated and the daemon can no longer open _config.db. + * + * #1531 fixed the cause forward-only in v0.10.3: the directory ACE is + * inheritable now, so newly created children are fine. Nothing repaired the + * children already damaged, which is why upgrading did not rescue anyone whose + * cache was written under the old regime. This is that repair. + * + * Deliberately bounded and conservative: + * - immediate children only, no recursion, capped; + * - regular files only; directories, reparse points and symlinks are skipped + * entirely rather than followed; + * - a child is touched ONLY when it is demonstrably damaged - an empty DACL, + * or an owner that is not the current user. A child that is merely unusual + * is left alone; + * - failures are counted and reported, never fatal. This runs inside daemon + * startup and must not be able to prevent it. + * + * Scope note: this only ever runs on cbm's own runtime/cache directory, which + * we created and own. It does not reach into user directories. */ +static void win_repair_runtime_children(win_security_t *security, const wchar_t *runtime_dir) { + enum { WIN_CHILD_REPAIR_MAX = 4096 }; + if (!security || !runtime_dir || !security->user_sid) { + return; + } + size_t dir_length = wcslen(runtime_dir); + if (dir_length == 0U || dir_length > 32000U) { + return; + } + wchar_t *pattern = calloc(dir_length + 3U, sizeof(wchar_t)); + if (!pattern) { + return; + } + (void)swprintf(pattern, dir_length + 3U, L"%ls\\*", runtime_dir); + WIN32_FIND_DATAW entry; + HANDLE search = FindFirstFileW(pattern, &entry); + free(pattern); + if (search == INVALID_HANDLE_VALUE) { + return; + } + unsigned examined = 0U; + unsigned repaired = 0U; + unsigned failed = 0U; + do { + if (wcscmp(entry.cFileName, L".") == 0 || wcscmp(entry.cFileName, L"..") == 0) { + continue; + } + if ((entry.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0U || + (entry.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0U) { + continue; + } + if (++examined > (unsigned)WIN_CHILD_REPAIR_MAX) { + break; + } + size_t name_length = wcslen(entry.cFileName); + size_t child_capacity = dir_length + name_length + 2U; + wchar_t *child_path = calloc(child_capacity, sizeof(wchar_t)); + if (!child_path) { + continue; + } + (void)swprintf(child_path, child_capacity, L"%ls\\%ls", runtime_dir, entry.cFileName); + HANDLE child = CreateFileW(child_path, READ_CONTROL | WRITE_DAC | WRITE_OWNER, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL); + free(child_path); + if (child == INVALID_HANDLE_VALUE) { + continue; + } + BY_HANDLE_FILE_INFORMATION child_info; + bool regular = GetFileInformationByHandle(child, &child_info) != 0 && + (child_info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0U && + (child_info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0U; + bool damaged = regular && (win_file_dacl_is_empty(security, child) || + !win_file_owner_secure(security, child, true)); + if (damaged) { + if (security->set_security_info( + child, SE_FILE_OBJECT, + (DWORD)OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION | + PROTECTED_DACL_SECURITY_INFORMATION, + security->user_sid, NULL, security->acl, NULL) == ERROR_SUCCESS) { + repaired++; + } else { + failed++; + } + } + (void)CloseHandle(child); + } while (FindNextFileW(search, &entry) != 0); + (void)FindClose(search); + if (repaired > 0U || failed > 0U) { + char repaired_text[16]; + char failed_text[16]; + (void)snprintf(repaired_text, sizeof(repaired_text), "%u", repaired); + (void)snprintf(failed_text, sizeof(failed_text), "%u", failed); + cbm_log_warn("daemon.runtime_child_acl_repaired", "repaired", repaired_text, "failed", + failed_text); + } +} + static bool win_runtime_directory_secure(const wchar_t *runtime_dir) { win_security_t security; if (!win_security_init(&security)) { @@ -4216,8 +4408,30 @@ static bool win_runtime_directory_secure(const wchar_t *runtime_dir) { * refused, and the final validation below still demands the exact user. */ bool owner_ok = owner_exact || (valid_handle && can_write_owner && win_file_owner_secure(&security, directory, false)); + /* Re-stamp only when the directory is not ALREADY correct. + * + * This used to fire on every process start, whether or not anything was + * wrong. Two costs, both observed in the field: + * + * - It rewrites the security descriptor of a directory that already has + * the right one, and Windows propagates that to children. #1601 counted + * ELEVEN "Security change" USN records against a single _config.db in + * one day, none of which changed anything. + * - Every rewrite is a window. #1620 loses an atomic publish to exactly + * this: MoveFileEx needs DELETE on the destination, and a concurrent + * re-protect of the parent is a chance to be refused for a state that is + * about to be correct again anyway. + * + * The repair is what matters, not the ritual. If the owner is already the + * exact current user and the DACL already passes the private-directory + * check, there is nothing to fix and the correct action is to leave it + * alone. When it IS wrong we still repair exactly as before. */ DWORD secure_result = ERROR_ACCESS_DENIED; - if (valid_handle && owner_ok) { + bool already_correct = + valid_handle && owner_exact && win_file_dacl_is_owner_only(&security, directory); + if (already_correct) { + secure_result = ERROR_SUCCESS; + } else if (valid_handle && owner_ok) { secure_result = security.set_security_info( directory, SE_FILE_OBJECT, (owner_exact ? 0U : (DWORD)OWNER_SECURITY_INFORMATION) | DACL_SECURITY_INFORMATION | @@ -4232,6 +4446,12 @@ static bool win_runtime_directory_secure(const wchar_t *runtime_dir) { bool final_private = secure_result == ERROR_SUCCESS && win_file_security_secure(&security, directory, true, win_private_mutation_rights(), false); + /* Repair damaged children only once the directory itself is known good. + * Repairing into a parent we have not secured would re-derive the same + * broken state on the next file created there. */ + if (final_private) { + win_repair_runtime_children(&security, runtime_dir); + } (void)CloseHandle(directory); win_security_destroy(&security); return valid_handle && owner_ok && final_private; diff --git a/src/daemon/runtime.c b/src/daemon/runtime.c index e62b1c7d4..2b1723fa2 100644 --- a/src/daemon/runtime.c +++ b/src/daemon/runtime.c @@ -50,10 +50,14 @@ void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force) { #include #include #include -#elif defined(__linux__) +#elif defined(__linux__) || defined(__FreeBSD__) #include #include #include +#if defined(__FreeBSD__) +#include +#include +#endif #endif enum { @@ -155,7 +159,7 @@ typedef struct { HANDLE file; BY_HANDLE_FILE_INFORMATION information; LARGE_INTEGER size; -#elif defined(__APPLE__) || defined(__linux__) +#elif defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) int fd; struct stat status; #endif @@ -510,7 +514,7 @@ static bool runtime_activation_response_decode( static uint64_t runtime_current_process_id(void) { #ifdef _WIN32 return (uint64_t)GetCurrentProcessId(); -#elif defined(__APPLE__) || defined(__linux__) +#elif defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) return (uint64_t)getpid(); #else return 0; @@ -524,7 +528,7 @@ static void runtime_process_image_reference_init(runtime_process_image_reference memset(reference, 0, sizeof(*reference)); #ifdef _WIN32 reference->file = INVALID_HANDLE_VALUE; -#elif defined(__APPLE__) || defined(__linux__) +#elif defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) reference->fd = -1; #endif } @@ -538,7 +542,7 @@ static bool runtime_process_image_reference_release(runtime_process_image_refere if (reference->file != INVALID_HANDLE_VALUE && !CloseHandle(reference->file)) { ok = false; } -#elif defined(__APPLE__) || defined(__linux__) +#elif defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) if (reference->fd >= 0 && close(reference->fd) != 0) { ok = false; } @@ -678,9 +682,9 @@ static bool runtime_mac_process_maps_file_executable(int process_id, const struc return false; } -#elif defined(__linux__) +#elif defined(__linux__) || defined(__FreeBSD__) -static bool runtime_linux_stat_same_image(const struct stat *first, const struct stat *second) { +static bool runtime_posix_stat_same_image(const struct stat *first, const struct stat *second) { return first && second && S_ISREG(first->st_mode) && S_ISREG(second->st_mode) && first->st_dev == second->st_dev && first->st_ino == second->st_ino && first->st_size == second->st_size && first->st_mtim.tv_sec == second->st_mtim.tv_sec && @@ -796,11 +800,11 @@ static bool runtime_process_image_reference_acquire( (!fingerprint || cbm_daemon_build_fingerprint_native_file((uintptr_t)image_fd, fingerprint)) && fstat(image_fd, &image_after) == 0 && - runtime_linux_stat_same_image(&image_before, &image_after); + runtime_posix_stat_same_image(&image_before, &image_after); int verify_fd = ok ? openat(process_fd, "exe", O_RDONLY | O_CLOEXEC) : -1; struct stat verify_status; ok = ok && verify_fd >= 0 && fstat(verify_fd, &verify_status) == 0 && - runtime_linux_stat_same_image(&image_after, &verify_status); + runtime_posix_stat_same_image(&image_after, &verify_status); if (verify_fd >= 0 && close(verify_fd) != 0) { ok = false; } @@ -814,6 +818,33 @@ static bool runtime_process_image_reference_acquire( } else if (image_fd >= 0) { (void)close(image_fd); } +#elif defined(__FreeBSD__) + if (process_id > INT_MAX) { + return false; + } + int pid = (int)process_id; + int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, pid}; + char path[PATH_MAX]; + size_t path_length = sizeof(path); + bool ok = sysctl(mib, 4, path, &path_length, NULL, 0) == 0 && path_length > 0; + if (ok) { + path[path_length < sizeof(path) ? path_length : sizeof(path) - 1] = '\0'; + } + int image_fd = ok ? open(path, O_RDONLY | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK) : -1; + struct stat image_before; + struct stat image_after; + ok = image_fd >= 0 && fstat(image_fd, &image_before) == 0 && S_ISREG(image_before.st_mode) && + (!fingerprint || + cbm_daemon_build_fingerprint_native_file((uintptr_t)image_fd, fingerprint)) && + fstat(image_fd, &image_after) == 0 && + runtime_posix_stat_same_image(&image_before, &image_after); + if (ok) { + reference->held = true; + reference->fd = image_fd; + reference->status = image_after; + } else if (image_fd >= 0) { + (void)close(image_fd); + } #else (void)process_id; bool ok = false; @@ -854,14 +885,14 @@ static bool runtime_process_image_reference_matches_process( runtime_mac_stat_same(&active->status, &peer.status); bool released = runtime_process_image_reference_release(&peer); return same && released; -#elif defined(__linux__) +#elif defined(__linux__) || defined(__FreeBSD__) runtime_process_image_reference_t peer; runtime_process_image_reference_init(&peer); bool same = runtime_process_image_reference_acquire(process_id, &peer, NULL); struct stat active_now; same = same && fstat(active->fd, &active_now) == 0 && - runtime_linux_stat_same_image(&active->status, &active_now) && - runtime_linux_stat_same_image(&active->status, &peer.status); + runtime_posix_stat_same_image(&active->status, &active_now) && + runtime_posix_stat_same_image(&active->status, &peer.status); bool released = runtime_process_image_reference_release(&peer); return same && released; #else diff --git a/src/discover/discover.c b/src/discover/discover.c index fbcd9bc8d..fe1e4e19d 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -689,7 +689,11 @@ static const char *file_skip_reason(const char *entry_name, const char *rel_path static CBMLanguage detect_file_language(const char *entry_name, const char *abs_path) { CBMLanguage lang = cbm_language_for_filename(entry_name); if (lang == CBM_LANG_COUNT) { - return CBM_LANG_COUNT; + /* Filename/extension detection failed: fall back to a conservative + * shebang probe so extensionless scripts get indexed (#1199). Filename + * detection stays authoritative — this runs only when it returns + * unknown. */ + return cbm_language_from_shebang(abs_path); } /* Special: .m files need content-based disambiguation */ const char *dot = strrchr(entry_name, '.'); diff --git a/src/discover/discover.h b/src/discover/discover.h index 87fc159d7..afb60fa36 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -50,6 +50,20 @@ CBMLanguage cbm_disambiguate_cls(const char *path); * On read failure, defaults to CBM_LANG_BITBAKE. */ CBMLanguage cbm_disambiguate_inc(const char *path); +/* Detect a supported script language from a file's shebang (#!...) first line. + * Conservative fallback used only when filename/extension detection is unknown + * (see detect_file_language); it never overrides extension or special-filename + * matches. Opens the file read-only and reads only a bounded first line. + * Recognizes the interpreter *basename* (not the parent path): + * python / python2 / python3 / dotted versions (python3.12) -> CBM_LANG_PYTHON + * sh / bash / dash / ksh / zsh -> CBM_LANG_BASH + * node / nodejs -> CBM_LANG_JAVASCRIPT + * ruby -> RUBY, perl -> PERL, php -> PHP, lua -> LUA + * Handles direct paths, "env ", "env -S ", and CRLF. + * Fails closed (returns CBM_LANG_COUNT) on read error, missing/malformed + * shebang, an embedded NUL in the first line, or an unknown interpreter. */ +CBMLanguage cbm_language_from_shebang(const char *path); + /* ── Gitignore pattern matching ──────────────────────────────────── */ typedef struct cbm_gitignore cbm_gitignore_t; diff --git a/src/discover/language.c b/src/discover/language.c index 4451b1b26..9854aa80c 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -945,6 +945,172 @@ const char *cbm_language_name(CBMLanguage lang) { return LANG_NAMES[lang] ? LANG_NAMES[lang] : "Unknown"; } +/* ── Shebang interpreter detection (extensionless scripts) ────────── */ + +/* Basename of an interpreter path: the segment after the last '/'. Shebangs + * are a POSIX convention, so only '/' is treated as a separator. */ +static const char *interp_basename(const char *path) { + const char *slash = strrchr(path, '/'); + return slash ? slash + SKIP_ONE : path; +} + +/* "python" optionally followed by an explicit numeric version (digits and dots + * only, e.g. "python3", "python3.12"). Bounded and explicit so arbitrary + * suffixes like "python-wrapper" are rejected. */ +static bool is_python_interp(const char *base) { + if (strncmp(base, "python", SLEN("python")) != 0) { + return false; + } + const char *version = base + SLEN("python"); + if (*version == '\0') { + return true; + } + + /* Each numeric component must contain at least one digit. */ + bool need_digit = true; + for (const char *v = version; *v; v++) { + if (isdigit((unsigned char)*v)) { + need_digit = false; + } else if (*v == '.' && !need_digit) { + need_digit = true; + } else { + return false; + } + } + return !need_digit; +} + +/* Map an interpreter basename to a language, or CBM_LANG_COUNT if unrecognized. + * Non-python interpreters are matched exactly (no prefix/suffix logic). */ +static CBMLanguage lang_for_interpreter(const char *base) { + if (is_python_interp(base)) { + return CBM_LANG_PYTHON; + } + static const struct { + const char *name; + CBMLanguage lang; + } INTERP_TABLE[] = { + {"sh", CBM_LANG_BASH}, {"bash", CBM_LANG_BASH}, {"dash", CBM_LANG_BASH}, + {"ksh", CBM_LANG_BASH}, {"zsh", CBM_LANG_BASH}, {"node", CBM_LANG_JAVASCRIPT}, + {"nodejs", CBM_LANG_JAVASCRIPT}, {"ruby", CBM_LANG_RUBY}, {"perl", CBM_LANG_PERL}, + {"php", CBM_LANG_PHP}, {"lua", CBM_LANG_LUA}, + }; + for (size_t i = 0; i < sizeof(INTERP_TABLE) / sizeof(INTERP_TABLE[0]); i++) { + if (strcmp(base, INTERP_TABLE[i].name) == 0) { + return INTERP_TABLE[i].lang; + } + } + return CBM_LANG_COUNT; +} + +/* Advance *cursor past leading blanks and return the next whitespace-delimited + * token (NUL-terminated in place), or NULL when the line is exhausted. */ +static char *shebang_next_token(char **cursor) { + char *p = *cursor; + while (*p == ' ' || *p == '\t') { + p++; + } + if (*p == '\0') { + *cursor = p; + return NULL; + } + char *start = p; + while (*p && *p != ' ' && *p != '\t') { + p++; + } + if (*p) { + *p = '\0'; + p++; + } + *cursor = p; + return start; +} + +CBMLanguage cbm_language_from_shebang(const char *path) { + if (!path) { + return CBM_LANG_COUNT; + } + + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return CBM_LANG_COUNT; /* fail closed on read error */ + } + + /* Read only a bounded first line. */ + char buf[CBM_SZ_256]; + size_t n = fread(buf, SKIP_ONE, sizeof(buf) - SKIP_ONE, f); + + /* Fail closed on any read error rather than parsing a partial buffer. */ + if (ferror(f)) { + (void)fclose(f); + return CBM_LANG_COUNT; + } + + /* If the bounded buffer filled without containing a newline, the first + * line may extend past our bound. Probe a single extra byte to tell an + * exact EOF (the whole file is <= 255 bytes) from a truncated longer + * line: any surviving byte -- including a newline just beyond the bound -- + * means the first line was cut off, so fail closed. A probe read error + * fails closed too. This keeps the read bounded (no unbounded line read + * or allocation). */ + bool have_newline = (memchr(buf, '\n', n) != NULL); + if (!have_newline && n == sizeof(buf) - SKIP_ONE) { + int probe = fgetc(f); + if (probe != EOF || ferror(f)) { + (void)fclose(f); + return CBM_LANG_COUNT; + } + } + (void)fclose(f); + + /* Must begin with "#!". */ + if (n < PAIR_LEN || buf[0] != '#' || buf[1] != '!') { + return CBM_LANG_COUNT; + } + + /* Isolate the first line; reject an embedded NUL before the newline. */ + size_t line_len = 0; + while (line_len < n && buf[line_len] != '\n') { + if (buf[line_len] == '\0') { + return CBM_LANG_COUNT; /* embedded NUL — treat as binary */ + } + line_len++; + } + /* Trim a trailing CR so CRLF first lines parse. */ + if (line_len > 0 && buf[line_len - SKIP_ONE] == '\r') { + line_len--; + } + buf[line_len] = '\0'; + + /* First token after "#!" is the interpreter (or env). */ + char *cursor = buf + PAIR_LEN; + char *interp = shebang_next_token(&cursor); + if (!interp) { + return CBM_LANG_COUNT; + } + const char *base = interp_basename(interp); + + /* "env [-S] [args...]": the real interpreter is the next token. + * Only the plain "env " and "env -S/--split-string [args]" + * shapes are supported. After the optional -S, the interpreter token must + * be a real command, so reject option tokens (leading '-') and NAME=value + * assignments (containing '=') -- e.g. "env PYTHON=/usr/bin/python + * python-wrapper", where env would treat the first token as an env-var + * setting rather than the program to run. */ + if (strcmp(base, "env") == 0) { + char *tok = shebang_next_token(&cursor); + if (tok && (strcmp(tok, "-S") == 0 || strcmp(tok, "--split-string") == 0)) { + tok = shebang_next_token(&cursor); + } + if (!tok || tok[0] == '-' || strchr(tok, '=') != NULL) { + return CBM_LANG_COUNT; + } + base = interp_basename(tok); + } + + return lang_for_interpreter(base); +} + /* ── .m file disambiguation ──────────────────────────────────────── */ /* Simple substring search helper */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 3250042dc..f7e22d001 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -606,8 +606,14 @@ static const tool_def_t TOOLS[] = { "offset parameter — raise limit or narrow with file_pattern / path_filter to see more." "\",\"default\":10,\"minimum\":1}},\"required\":[\"pattern\",\"project\"]}"}, - {"list_projects", "List projects", "List all indexed projects", - "{\"type\":\"object\",\"properties\":{}}"}, + {"list_projects", "List projects", "List indexed projects with deterministic pagination", + "{\"type\":\"object\",\"properties\":{\"offset\":{\"type\":\"integer\"," + "\"minimum\":0,\"default\":0}," + "\"limit\":{\"type\":\"integer\",\"minimum\":1,\"maximum\":100,\"default\":50}," + "\"include_details\":{\"type\":\"boolean\",\"default\":false," + "\"description\":\"Include branch, node/edge counts and database size. Slower.\"}," + "\"metadata_only\":{\"type\":\"boolean\",\"description\":\"Deprecated compatibility " + "alias for include_details=false.\"}}}"}, {"delete_project", "Delete project", "Delete a project from the index", "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" "\"project\"]}"}, @@ -2508,7 +2514,7 @@ static cbm_store_t *resolve_store_fallback_scan(const char *project) { * then append a JSON entry to arr. */ static void build_project_json_entry(yyjson_mut_doc *doc, yyjson_mut_val *arr, const char *dir_path, const char *name, size_t name_len, int64_t size_bytes, - bool metadata_only) { + bool include_details) { (void)name_len; char full_path[CBM_SZ_2K]; @@ -2527,7 +2533,7 @@ static void build_project_json_entry(yyjson_mut_doc *doc, yyjson_mut_val *arr, c int nodes = 0; int edges = 0; - if (!metadata_only) { + if (include_details) { nodes = cbm_store_count_nodes(pstore, project_name); edges = cbm_store_count_edges(pstore, project_name); } @@ -2548,7 +2554,7 @@ static void build_project_json_entry(yyjson_mut_doc *doc, yyjson_mut_val *arr, c * disambiguates same-repo projects). The 12-field git block — mostly * null for non-git roots — cost ~10KB across a full cache and is one * index_status call away for the project you actually care about. */ - if (!metadata_only && root_path_buf[0]) { + if (include_details && root_path_buf[0]) { cbm_git_context_t gctx = {0}; (void)cbm_git_context_resolve(root_path_buf, &gctx); if (gctx.is_git && gctx.branch) { @@ -2556,7 +2562,7 @@ static void build_project_json_entry(yyjson_mut_doc *doc, yyjson_mut_val *arr, c } cbm_git_context_free(&gctx); } - if (!metadata_only) { + if (include_details) { yyjson_mut_obj_add_int(doc, p, "nodes", nodes); yyjson_mut_obj_add_int(doc, p, "edges", edges); yyjson_mut_obj_add_int(doc, p, "size_bytes", size_bytes); @@ -2564,19 +2570,37 @@ static void build_project_json_entry(yyjson_mut_doc *doc, yyjson_mut_val *arr, c yyjson_mut_arr_add_val(arr, p); } +static int project_db_name_cmp(const void *a, const void *b) { + const char *const *sa = (const char *const *)a; + const char *const *sb = (const char *const *)b; + return strcmp(*sa, *sb); +} + /* list_projects: scan cache directory for .db files. * Each project is a single .db file — no central registry needed. */ static char *handle_list_projects(cbm_mcp_server_t *srv, const char *args) { (void)srv; - bool metadata_only = false; + int offset = cbm_mcp_get_int_arg(args, "offset", 0); + int limit = cbm_mcp_get_int_arg(args, "limit", 50); + bool include_details = cbm_mcp_get_bool_arg(args, "include_details"); yyjson_doc *args_doc = args ? yyjson_read(args, strlen(args), 0) : NULL; yyjson_val *args_root = args_doc ? yyjson_doc_get_root(args_doc) : NULL; yyjson_val *metadata_value = args_root && yyjson_is_obj(args_root) ? yyjson_obj_get(args_root, "metadata_only") : NULL; - metadata_only = metadata_value && yyjson_is_true(metadata_value); + if (metadata_value && yyjson_is_true(metadata_value)) { + include_details = false; + } if (args_doc) { yyjson_doc_free(args_doc); } + if (offset < 0) { + offset = 0; + } + if (limit < 1) { + limit = 1; + } else if (limit > 100) { + limit = 100; + } char dir_path[CBM_SZ_1K]; cache_dir(dir_path, sizeof(dir_path)); @@ -2598,6 +2622,10 @@ static char *handle_list_projects(cbm_mcp_server_t *srv, const char *args) { return cbm_mcp_text_result(msg, true); } + char **db_names = NULL; + size_t db_count = 0; + size_t db_cap = 0; + bool name_collection_failed = false; cbm_dirent_t *entry; while ((entry = cbm_readdir(d)) != NULL) { const char *name = entry->name; @@ -2605,17 +2633,65 @@ static char *handle_list_projects(cbm_mcp_server_t *srv, const char *args) { if (!is_project_db_file(name, len)) { continue; } + if (db_count == db_cap) { + size_t next_cap = db_cap ? db_cap * 2 : 64; + char **grown = realloc(db_names, next_cap * sizeof(*db_names)); + if (!grown) { + name_collection_failed = true; + break; + } + db_names = grown; + db_cap = next_cap; + } + db_names[db_count] = heap_strdup(name); + if (!db_names[db_count]) { + name_collection_failed = true; + break; + } + db_count++; + } + cbm_closedir(d); + + if (name_collection_failed) { + for (size_t i = 0; i < db_count; i++) { + free(db_names[i]); + } + free(db_names); + yyjson_mut_doc_free(doc); + return cbm_mcp_text_result( + "{\"error\":\"out of memory while collecting indexed projects\"}", true); + } + + if (db_count > 1) { + qsort(db_names, db_count, sizeof(*db_names), project_db_name_cmp); + } + size_t start = (size_t)offset < db_count ? (size_t)offset : db_count; + size_t end = start + (size_t)limit; + if (end > db_count) { + end = db_count; + } + for (size_t i = start; i < end; i++) { + const char *name = db_names[i]; + size_t len = strlen(name); char full_path[CBM_SZ_2K]; snprintf(full_path, sizeof(full_path), "%s/%s", dir_path, name); int64_t size_bytes = cbm_file_size(full_path); if (size_bytes < 0) { continue; } - build_project_json_entry(doc, arr, dir_path, name, len, size_bytes, metadata_only); + build_project_json_entry(doc, arr, dir_path, name, len, size_bytes, include_details); } - cbm_closedir(d); + for (size_t i = 0; i < db_count; i++) { + free(db_names[i]); + } + free(db_names); yyjson_mut_obj_add_val(doc, root, "projects", arr); + yyjson_mut_obj_add_uint(doc, root, "total", db_count); + yyjson_mut_obj_add_int(doc, root, "offset", offset); + yyjson_mut_obj_add_int(doc, root, "limit", limit); + yyjson_mut_obj_add_uint(doc, root, "returned", yyjson_mut_arr_size(arr)); + yyjson_mut_obj_add_bool(doc, root, "has_more", end < db_count); /* Guide user when no projects are indexed */ if (yyjson_mut_arr_size(arr) == 0) { @@ -3178,22 +3254,6 @@ static bool run_semantic_query_core(const char *args, cbm_store_t *store, const return type_error; } -/* Append the semantic_query vector-search results onto the doc. Returns - * true if semantic_query was provided as a non-array (type error — caller - * should surface to the user). */ -static bool run_semantic_query(yyjson_mut_doc *doc, yyjson_mut_val *root, const char *args, - cbm_store_t *store, const char *project, int limit) { - cbm_vector_result_t *vresults = NULL; - int vcount = 0; - bool type_error = - run_semantic_query_core(args, store, project, limit, &vresults, &vcount, NULL); - if (vcount > 0) { - emit_semantic_results(doc, root, vresults, vcount); - cbm_store_free_vector_results(vresults, vcount); - } - return type_error; -} - /* ── Tree output for search_graph ─────────────────────────────────── * Default response encoding: grouped tree rows (compact_out.h). The same * model is available as structured JSON via format:"json"; include_connected @@ -3265,6 +3325,27 @@ static int sg_parse_fields(const char *args, const char *out[], int max_out, yyj return n; } +/* Append a property as one compact-output cell. Compound values stay one + * column by using their compact JSON representation; the cell emitter quotes + * and escapes that representation as needed. */ +static void sg_toon_property_cell(cbm_sb_t *sb, yyjson_val *v) { + if (v && yyjson_is_str(v)) { + cbm_tree_cell_str(sb, yyjson_get_str(v), false); + } else if (v && yyjson_is_bool(v)) { + cbm_tree_cell_bool(sb, yyjson_get_bool(v), false); + } else if (v && yyjson_is_int(v)) { + cbm_tree_cell_int(sb, yyjson_get_int(v), false); + } else if (v && yyjson_is_real(v)) { + cbm_tree_cell_real(sb, yyjson_get_real(v), false); + } else if (v && !yyjson_is_null(v)) { + char *json = yyjson_val_write(v, 0, NULL); + cbm_tree_cell_str(sb, json ? json : "", false); + free(json); + } else { + cbm_tree_cell_str(sb, "", false); + } +} + /* Append one row's extra-field cells, pulled from the node's properties. */ static void sg_toon_extra_cells(cbm_sb_t *sb, const char *props_json, const char *const *fields, int nfields) { @@ -3273,17 +3354,7 @@ static void sg_toon_extra_cells(cbm_sb_t *sb, const char *props_json, const char yyjson_val *pr = pd ? yyjson_doc_get_root(pd) : NULL; for (int f = 0; f < nfields; f++) { yyjson_val *v = (pr && yyjson_is_obj(pr)) ? yyjson_obj_get(pr, fields[f]) : NULL; - if (v && yyjson_is_str(v)) { - cbm_tree_cell_str(sb, yyjson_get_str(v), false); - } else if (v && yyjson_is_bool(v)) { - cbm_tree_cell_bool(sb, yyjson_get_bool(v), false); - } else if (v && yyjson_is_int(v)) { - cbm_tree_cell_int(sb, yyjson_get_int(v), false); - } else if (v && yyjson_is_real(v)) { - cbm_tree_cell_real(sb, yyjson_get_real(v), false); - } else { - cbm_tree_cell_str(sb, "", false); - } + sg_toon_property_cell(sb, v); } if (pd) { yyjson_doc_free(pd); @@ -3415,17 +3486,7 @@ static void emit_search_results_tree(cbm_sb_t *sb, cbm_search_output_t *out, int yyjson_val *pr = pd ? yyjson_doc_get_root(pd) : NULL; for (int f = 0; f < nfields; f++) { yyjson_val *v = (pr && yyjson_is_obj(pr)) ? yyjson_obj_get(pr, fields[f]) : NULL; - if (v && yyjson_is_str(v)) { - cbm_tree_cell_str(sb, yyjson_get_str(v), false); - } else if (v && yyjson_is_int(v)) { - cbm_tree_cell_int(sb, yyjson_get_int(v), false); - } else if (v && yyjson_is_real(v)) { - cbm_tree_cell_real(sb, yyjson_get_real(v), false); - } else if (v && yyjson_is_bool(v)) { - cbm_tree_cell_bool(sb, yyjson_get_bool(v), false); - } else { - cbm_tree_cell_str(sb, "", false); /* emits "-" */ - } + sg_toon_property_cell(sb, v); } if (pd) { yyjson_doc_free(pd); @@ -3507,14 +3568,10 @@ static void emit_search_results_tree_json(yyjson_mut_doc *doc, yyjson_mut_val *r yyjson_val *pr = pd ? yyjson_doc_get_root(pd) : NULL; for (int f = 0; f < nfields; f++) { yyjson_val *v = (pr && yyjson_is_obj(pr)) ? yyjson_obj_get(pr, fields[f]) : NULL; - if (v && yyjson_is_str(v)) { - yyjson_mut_arr_add_strcpy(doc, row, yyjson_get_str(v)); - } else if (v && yyjson_is_int(v)) { - yyjson_mut_arr_add_int(doc, row, yyjson_get_int(v)); - } else if (v && yyjson_is_real(v)) { - yyjson_mut_arr_add_real(doc, row, yyjson_get_real(v)); - } else if (v && yyjson_is_bool(v)) { - yyjson_mut_arr_add_bool(doc, row, yyjson_get_bool(v)); + yyjson_mut_val *copy = + (v && !yyjson_is_null(v)) ? yyjson_val_mut_copy(doc, v) : NULL; + if (copy) { + yyjson_mut_arr_add_val(row, copy); } else { yyjson_mut_arr_add_null(doc, row); } @@ -3628,6 +3685,9 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { .min_degree = min_degree, .max_degree = max_degree, }; + bool has_structural_filters = label || name_pattern || qn_pattern || file_pattern || + relationship || exclude_entry_points || + min_degree != CBM_NOT_FOUND || max_degree != CBM_NOT_FOUND; if (!legacy_json) { const char *fields[SG_MAX_EXTRA_FIELDS]; @@ -3648,10 +3708,7 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { /* Semantic-only calls get semantic results only: the legacy * behavior also ran the UNFILTERED regex search and prepended * up to `limit` unrelated enriched nodes to the response. */ - bool has_filters = label || name_pattern || qn_pattern || file_pattern || - relationship || exclude_entry_points || - min_degree != CBM_NOT_FOUND || max_degree != CBM_NOT_FOUND; - bool semantic_only = sq_present && !has_filters; + bool semantic_only = sq_present && !has_structural_filters; cbm_sb_t sb; cbm_sb_init(&sb); @@ -3735,8 +3792,31 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { true); } + cbm_vector_result_t *vresults = NULL; + int vcount = 0; + bool sq_present = false; + bool sq_type_error = + run_semantic_query_core(args, store, project, limit, &vresults, &vcount, &sq_present); + if (sq_type_error) { + free(project); + free(label); + free(name_pattern); + free(qn_pattern); + free(file_pattern); + free(relationship); + return cbm_mcp_text_result( + "semantic_query must be an array of keyword strings, e.g. " + "[\"send\",\"pubsub\",\"publish\"] — not a single string. Split your query " + "into individual keywords; each is scored independently via per-keyword " + "min-cosine.", + true); + } + + bool semantic_only = sq_present && !has_structural_filters; cbm_search_output_t out = {0}; - cbm_store_search(store, ¶ms, &out); + if (!semantic_only) { + cbm_store_search(store, ¶ms, &out); + } yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = yyjson_mut_obj(doc); @@ -3775,23 +3855,9 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { } } - bool sq_type_error = run_semantic_query(doc, root, args, store, project, limit); - - if (sq_type_error) { - yyjson_mut_doc_free(doc); - cbm_store_search_free(&out); - free(project); - free(label); - free(name_pattern); - free(qn_pattern); - free(file_pattern); - free(relationship); - return cbm_mcp_text_result( - "semantic_query must be an array of keyword strings, e.g. " - "[\"send\",\"pubsub\",\"publish\"] — not a single string. Split your query " - "into individual keywords; each is scored independently via per-keyword " - "min-cosine.", - true); + if (vcount > 0) { + emit_semantic_results(doc, root, vresults, vcount); + cbm_store_free_vector_results(vresults, vcount); } char *json = yy_doc_to_str(doc); @@ -5741,14 +5807,23 @@ static yyjson_doc *resolve_trace_edge_types(const char *args, const char *mode, return NULL; } -/* Check if a file path looks like a test file. */ +/* Check if a file path looks like a test file. The substring checks below + * only catch a tests/ directory nested under another path component + * (".../tests/foo"); a project-root-relative path like "tests/repro/foo.c" + * has no leading slash before "tests" and fell through undetected, leaking + * whole test subtrees into query_graph/trace_path results with the default + * include_tests=false (#1294). */ static bool is_test_file(const char *path) { if (!path) { return false; } return strstr(path, "/test") != NULL || strstr(path, "test_") != NULL || strstr(path, "_test.") != NULL || strstr(path, "/tests/") != NULL || - strstr(path, "/spec/") != NULL || strstr(path, ".test.") != NULL; + strstr(path, "/spec/") != NULL || strstr(path, ".test.") != NULL || + strncmp(path, "tests/", SLEN("tests/")) == 0 || + strncmp(path, "test/", SLEN("test/")) == 0 || + strncmp(path, "spec/", SLEN("spec/")) == 0 || + strncmp(path, "__tests__/", SLEN("__tests__/")) == 0; } /* Convert BFS traversal results into a yyjson_mut array. */ @@ -8699,16 +8774,6 @@ static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) { /* ── search_code v2: graph-augmented code search ─────────────── */ -/* Strip non-ASCII bytes to guarantee valid UTF-8 JSON output */ -enum { ASCII_MAX = 127 }; -static void sanitize_ascii(char *s) { - for (unsigned char *p = (unsigned char *)s; *p; p++) { - if (*p > ASCII_MAX) { - *p = '?'; - } - } -} - /* Intermediate grep match */ typedef struct { char file[CBM_SZ_512]; @@ -8791,6 +8856,16 @@ bool cbm_search_code_file_pattern_can_prefilter(const char *file_pattern) { /* Build the grep/search command string based on scoped vs recursive mode. * On Windows, uses PowerShell Select-String with tab-delimited output. * On POSIX, uses grep with colon-delimited output. */ +/* Windows PowerShell 5.1 encodes stdout for a native-process pipe in the + * console OEM codepage, so any character the inherited CP cannot carry + * (Cyrillic under CP437/850, etc.) degrades to '?' before it ever reaches + * collect_grep_matches — and WHETHER it degrades depends on the console the + * server happened to inherit. Pin the pipe to UTF-8 inside every generated + * command so raw search content is codepage-independent. (The read side is + * already safe: Select-String decodes BOM-less UTF-8 via .NET StreamReader + * defaults.) */ +#define CBM_PS_UTF8_PRELUDE "[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; " + void cbm_search_code_build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bool scoped, const char *file_pattern, const char *tmpfile, const char *filelist, const char *root_path) { @@ -8801,7 +8876,8 @@ void cbm_search_code_build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bo if (cbm_search_code_file_pattern_can_prefilter(file_pattern)) { snprintf( cmd, cmd_sz, - "powershell -Command \"$pat = Get-Content -Encoding UTF8 -LiteralPath '%s'; " + "powershell -Command \"" CBM_PS_UTF8_PRELUDE + "$pat = Get-Content -Encoding UTF8 -LiteralPath '%s'; " "Get-Content -Encoding UTF8 -LiteralPath '%s'" " | Where-Object { $_ -like '%s' }" " | ForEach-Object { Select-String -LiteralPath $_ -Pattern $pat%s " @@ -8812,7 +8888,8 @@ void cbm_search_code_build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bo } else { snprintf( cmd, cmd_sz, - "powershell -Command \"$pat = Get-Content -Encoding UTF8 -LiteralPath '%s'; " + "powershell -Command \"" CBM_PS_UTF8_PRELUDE + "$pat = Get-Content -Encoding UTF8 -LiteralPath '%s'; " "Get-Content -Encoding UTF8 -LiteralPath '%s' | ForEach-Object { Select-String " "-LiteralPath $_ -Pattern $pat%s " "-ErrorAction SilentlyContinue }" @@ -8823,7 +8900,8 @@ void cbm_search_code_build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bo } else { snprintf( cmd, cmd_sz, - "powershell -Command \"$pat = Get-Content -Encoding UTF8 -LiteralPath '%s'; " + "powershell -Command \"" CBM_PS_UTF8_PRELUDE + "$pat = Get-Content -Encoding UTF8 -LiteralPath '%s'; " "Get-Content -Encoding UTF8 -LiteralPath '%s' | ForEach-Object { Select-String " "-LiteralPath $_ -Pattern $pat%s " "-ErrorAction SilentlyContinue }" @@ -8834,7 +8912,8 @@ void cbm_search_code_build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bo if (file_pattern) { snprintf( cmd, cmd_sz, - "powershell -Command \"Get-ChildItem -Recurse -Path '%s\\*' -Include '%s' -File " + "powershell -Command \"" CBM_PS_UTF8_PRELUDE + "Get-ChildItem -Recurse -Path '%s\\*' -Include '%s' -File " "-ErrorAction SilentlyContinue" " | Select-String -Pattern (Get-Content -Encoding UTF8 -LiteralPath '%s')%s " "-ErrorAction SilentlyContinue" @@ -8843,7 +8922,8 @@ void cbm_search_code_build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bo } else { snprintf( cmd, cmd_sz, - "powershell -Command \"Get-ChildItem -Recurse -Path '%s\\*' -File -ErrorAction " + "powershell -Command \"" CBM_PS_UTF8_PRELUDE + "Get-ChildItem -Recurse -Path '%s\\*' -File -ErrorAction " "SilentlyContinue" " | Select-String -Pattern (Get-Content -Encoding UTF8 -LiteralPath '%s')%s " "-ErrorAction SilentlyContinue" @@ -8949,8 +9029,11 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear } char *source = read_file_lines(abs_path, s, e); if (source) { - sanitize_ascii(source); - yyjson_mut_obj_add_strcpy(doc, item, "source", source); + char *safe_source = sanitize_utf8_lossy(source); + if (safe_source) { + yyjson_mut_obj_add_strcpy(doc, item, "source", safe_source); + free(safe_source); + } free(source); if (truncated) { yyjson_mut_obj_add_int(doc, item, "source_start", s); @@ -8965,8 +9048,11 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear } char *ctx = read_file_lines(abs_path, ctx_start, ctx_end); if (ctx) { - sanitize_ascii(ctx); - yyjson_mut_obj_add_strcpy(doc, item, "context", ctx); + char *safe_context = sanitize_utf8_lossy(ctx); + if (safe_context) { + yyjson_mut_obj_add_strcpy(doc, item, "context", safe_context); + free(safe_context); + } yyjson_mut_obj_add_int(doc, item, "context_start", ctx_start); free(ctx); } @@ -9138,16 +9224,17 @@ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_matc build_dedup_files_array(doc, sr, output_count, raw, raw_count)); } else { /* json-stringified tree: cols + column-ordered row arrays. FULL mode - * appends a per-row object cell with the (guarded, windowed) source — - * attach_result_source semantics unchanged. */ + * appends a per-row object cell with the (guarded, windowed) source; + * context requests append the corresponding context object. */ + bool attach_context = context_lines > 0 && mode != MODE_FULL; yyjson_mut_val *jcols = yyjson_mut_arr(doc); static const char *const sc_cols[] = {"qn", "label", "file", "lines", "matches", "in", "out"}; for (size_t ci = 0; ci < sizeof(sc_cols) / sizeof(sc_cols[0]); ci++) { yyjson_mut_arr_add_str(doc, jcols, sc_cols[ci]); } - if (mode == MODE_FULL) { - yyjson_mut_arr_add_str(doc, jcols, "source"); + if (mode == MODE_FULL || attach_context) { + yyjson_mut_arr_add_str(doc, jcols, mode == MODE_FULL ? "source" : "context"); } yyjson_mut_obj_add_val(doc, root_obj, "cols", jcols); @@ -9173,7 +9260,7 @@ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_matc yyjson_mut_arr_add_val(row, ml); yyjson_mut_arr_add_int(doc, row, r->in_degree); yyjson_mut_arr_add_int(doc, row, r->out_degree); - if (mode == MODE_FULL) { + if (mode == MODE_FULL || attach_context) { yyjson_mut_val *src = yyjson_mut_obj(doc); attach_result_source(doc, src, r, mode, context_lines, root_path); yyjson_mut_arr_add_val(row, src); @@ -9245,7 +9332,11 @@ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_matc char *json = yy_doc_to_str(doc); if (json) { - sanitize_ascii(json); + char *safe_json = sanitize_utf8_lossy(json); + if (safe_json) { + free(json); + json = safe_json; + } } yyjson_mut_doc_free(doc); @@ -9316,8 +9407,10 @@ static grep_match_t *collect_grep_matches(FILE *fp, const char *root_path, size_ safe_grow(gm, gm_count, gm_cap, PAIR_LEN); snprintf(gm[gm_count].file, sizeof(gm[0].file), "%s", file); gm[gm_count].line = (int)strtol(sep1 + SKIP_ONE, NULL, CBM_DECIMAL_BASE); - snprintf(gm[gm_count].content, sizeof(gm[0].content), "%s", sep2 + SKIP_ONE); - sanitize_ascii(gm[gm_count].content); + char *safe_content = sanitize_utf8_lossy(sep2 + SKIP_ONE); + snprintf(gm[gm_count].content, sizeof(gm[0].content), "%s", + safe_content ? safe_content : sep2 + SKIP_ONE); + free(safe_content); gm_count++; } diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 941a3ee56..189dd9f2d 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -635,6 +635,12 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, if (!target_node || source_node->id == target_node->id) { return 0; } + /* #725: suffix_match is language-agnostic and will attach a Python + * Store.commit() call to a JS function named commit (or a Bash main + * to a Python main). Drop that weak cross-language edge. */ + if (cbm_suppress_cross_language_suffix_match(lang, target_node->file_path, res.strategy)) { + return 0; + } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, imp_count, tsjs_drop_plain_call); return SKIP_ONE; diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index c3eb2aeec..0c91db09a 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2539,6 +2539,12 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB } atomic_fetch_add_explicit(&rc->time_ns_rc_target, extract_now_ns() - _rc_t0, memory_order_relaxed); + if (target_node && source_node->id != target_node->id && + cbm_suppress_cross_language_suffix_match(lang, target_node->file_path, res.strategy)) { + /* #725: same guard as pass_calls.c — do not emit a suffix_match + * CALLS edge across a language boundary. */ + continue; + } if (!target_node || source_node->id == target_node->id) { /* HTTP/ASYNC calls to an EXTERNAL client library (`requests.get(url)`) * resolve to an unindexed QN (target_node == NULL), but their edge diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 597c5e767..65ac75183 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -151,6 +151,7 @@ struct cbm_pipeline { char *project_name; cbm_git_context_t git_ctx; char *branch_qn; + cbm_index_mode_t requested_mode; cbm_index_mode_t mode; atomic_int cancelled_storage; atomic_int *cancelled; @@ -268,6 +269,7 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, p->project_name = cbm_project_name_from_path(repo_path); (void)cbm_git_context_resolve(repo_path, &p->git_ctx); p->branch_qn = cbm_git_context_branch_qn(p->project_name, &p->git_ctx); + p->requested_mode = mode; p->mode = mode; p->persistence = false; p->committed_nodes = -1; @@ -1371,7 +1373,7 @@ static int capture_existing_adr(cbm_pipeline_t *p, const char *db_path) { * metadata write has succeeded. */ static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *baseline_manifest, - int baseline_count) { + int baseline_count, bool force_full_on_mismatch) { char *db_path = resolve_db_path(p); if (!db_path) { return CBM_PIPELINE_FORCE_FULL_REINDEX; @@ -1393,7 +1395,7 @@ static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *file } cbm_log_info("pipeline.route", "path", "incremental_manifest"); int rc = cbm_pipeline_run_incremental(p, db_path, files, file_count, baseline_manifest, - baseline_count); + baseline_count, force_full_on_mismatch); /* Delete the existing generation ONLY when we are about to rebuild it. * On main this was guarded by an early `return rc` for the incremental * path; this function has no such early return, so the delete must be @@ -1442,19 +1444,20 @@ static int pipeline_mode_coverage_rank(cbm_index_mode_t mode) { * must never erase files that the cheaper discovery intentionally skips. The * exact-manifest pipeline therefore keeps the most comprehensive successfully * published mode and performs any changed rebuild at that coverage level. */ -static void promote_mode_to_existing_coverage(cbm_pipeline_t *p) { +static bool promote_mode_to_existing_coverage(cbm_pipeline_t *p) { if (!p || !p->project_name) { - return; + return false; } char *db_path = resolve_db_path(p); if (!db_path) { - return; + return false; } cbm_store_t *store = cbm_store_open_path_query(db_path); free(db_path); if (!store) { - return; + return false; } + bool promoted = false; cbm_coverage_meta_t meta = {0}; if (cbm_store_coverage_meta_get(store, p->project_name, &meta) == CBM_STORE_OK && meta.index_mode) { @@ -1470,31 +1473,12 @@ static void promote_mode_to_existing_coverage(cbm_pipeline_t *p) { cbm_log_info("pipeline.mode", "requested", pipeline_mode_name(p->mode), "effective", pipeline_mode_name(stored_mode), "reason", "preserve_existing_coverage"); p->mode = stored_mode; + promoted = true; } } cbm_store_coverage_meta_clear(&meta); cbm_store_close(store); -} - -int cbm_pipeline_refresh_artifact(cbm_pipeline_t *p, const char *db_path) { - if (!p || !db_path || !p->repo_path || !p->project_name) { - return 0; - } - bool existing = cbm_artifact_exists(p->repo_path); - if (!p->persistence && !existing) { - return 0; - } - int quality = p->persistence ? CBM_ARTIFACT_BEST : CBM_ARTIFACT_FAST; - int rc = cbm_artifact_export(db_path, p->repo_path, p->project_name, quality); - if (rc != 0) { - const char *err = cbm_artifact_export_last_error(); - if (p->persistence) { - cbm_log_error("pipeline.err", "phase", "artifact_export", "err", err ? err : "unknown"); - return rc; - } - cbm_log_warn("artifact.refresh_failed", "err", err ? err : "unknown"); - } - return 0; + return promoted; } /* Defined below, next to the other publication helpers. */ @@ -2014,8 +1998,8 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas cbm_pipeline_free_semantic_manifest(manifest, manifest_count); if (rc != 0) { /* db_path is this function's strdup (resolve_db_path); every return - * must release it -- refresh_artifact below only borrows it. LSan on - * the Linux leg caught exactly this pair of exits leaking. */ + * must release it. LSan on the Linux leg caught exactly this pair of + * exits leaking. */ free(db_path); return rc; } @@ -2028,12 +2012,8 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas free(p->saved_adr); p->saved_adr = NULL; - /* The SQLite generation is the commit point. Automatic refresh of an - * existing artifact is best-effort, but an explicitly requested artifact - * is caller-visible and must report an export failure. */ - int artifact_rc = cbm_pipeline_refresh_artifact(p, db_path); free(db_path); - return artifact_rc; + return 0; } /* Run githistory pass. */ @@ -2157,11 +2137,10 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, return rc; } -static int cbm_pipeline_run_staged(cbm_pipeline_t *p, bool *was_incremental) { +static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { if (!p) { return CBM_NOT_FOUND; } - *was_incremental = false; CBM_PROF_START(t_pipeline_total); struct timespec t0; @@ -2169,8 +2148,15 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p, bool *was_incremental) { cbm_path_alias_collection_t *path_aliases = NULL; cbm_file_hash_t *baseline_manifest = NULL; int baseline_count = 0; + char **requested_excluded_dirs = NULL; + int requested_excluded_count = 0; + cbm_ignored_file_t *requested_ignored_files = NULL; + int requested_ignored_count = 0; + int requested_ignored_total = 0; + bool restore_requested_discovery = false; - promote_mode_to_existing_coverage(p); + p->mode = p->requested_mode; + bool mode_promoted = promote_mode_to_existing_coverage(p); /* cbm_pipeline_new() may precede the actual run by an arbitrary interval. * Refresh once here, then use this exact snapshot for both Branch graph @@ -2193,7 +2179,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p, bool *was_incremental) { /* Phase 1: Discover files */ CBM_PROF_START(t_discover); cbm_discover_opts_t opts = { - .mode = p->mode, + .mode = p->requested_mode, .ignore_file = NULL, .max_file_size = 0, }; @@ -2227,26 +2213,62 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p, bool *was_incremental) { /* Snapshot every semantic input once before routing/extraction. The same * bytes drive exact no-op comparison and are checked against a fresh * rediscovery immediately before any replacement is published. */ - rc = cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files, file_count, - p->excluded_dirs, p->excluded_count, &p->git_ctx, - p->userconfig, &baseline_manifest, &baseline_count); + rc = mode_promoted + ? cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode, + &baseline_manifest, &baseline_count) + : cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files, + file_count, p->excluded_dirs, p->excluded_count, + &p->git_ctx, p->userconfig, &baseline_manifest, + &baseline_count); if (rc != 0) { rc = CBM_PIPELINE_ABORT_PRESERVE_DB; goto cleanup; } /* Check for existing DB → try incremental or delete for reindex */ - rc = try_incremental_or_delete_db(p, files, file_count, baseline_manifest, baseline_count); + rc = try_incremental_or_delete_db(p, files, file_count, baseline_manifest, baseline_count, + mode_promoted); if (rc == CBM_PIPELINE_ABORT_PRESERVE_DB || rc == CBM_PIPELINE_PERSIST_FAILED) { goto cleanup; } if (rc >= 0) { - *was_incremental = true; goto cleanup; } if (rc != CBM_PIPELINE_FORCE_FULL_REINDEX) { goto cleanup; } + + /* A changed downgrade rebuilds the complete graph at the stored effective + * mode. Keep the requested discovery lists to report the caller's scope. */ + if (mode_promoted) { + cbm_discover_free(files, file_count); + files = NULL; + file_count = 0; + + requested_excluded_dirs = p->excluded_dirs; + requested_excluded_count = p->excluded_count; + requested_ignored_files = p->ignored_files; + requested_ignored_count = p->ignored_count; + requested_ignored_total = p->ignored_total; + restore_requested_discovery = true; + + p->excluded_dirs = NULL; + p->excluded_count = 0; + p->ignored_files = NULL; + p->ignored_count = 0; + p->ignored_total = 0; + + opts.mode = p->mode; + rc = cbm_discover_ex2(p->repo_path, &opts, &files, &file_count, &p->excluded_dirs, + &p->excluded_count, &p->ignored_files, &p->ignored_count, + &p->ignored_total); + cbm_log_info("pipeline.rediscover", "requested_mode", pipeline_mode_name(p->requested_mode), + "effective_mode", pipeline_mode_name(p->mode), "files", itoa_buf(file_count)); + if (rc != 0 || check_cancel(p)) { + rc = CBM_NOT_FOUND; + goto cleanup; + } + } cbm_log_info("pipeline.route", "path", "full"); /* Phase 2: Create graph buffer and registry */ @@ -2296,6 +2318,15 @@ cleanup: cbm_registry_free(p->registry); p->registry = NULL; cbm_path_alias_collection_free(path_aliases); + if (restore_requested_discovery) { + cbm_discover_free_excluded(p->excluded_dirs, p->excluded_count); + cbm_discover_free_ignored(p->ignored_files, p->ignored_count); + p->excluded_dirs = requested_excluded_dirs; + p->excluded_count = requested_excluded_count; + p->ignored_files = requested_ignored_files; + p->ignored_count = requested_ignored_count; + p->ignored_total = requested_ignored_total; + } /* Clear and free user extension config */ cbm_set_user_lang_config(NULL); cbm_userconfig_free(p->userconfig); @@ -2458,7 +2489,7 @@ static int seal_staging_db(const char *staging_path) { return rc; } -static int export_after_publish(cbm_pipeline_t *p, const char *final_path, bool was_incremental) { +static int export_after_publish(cbm_pipeline_t *p, const char *final_path) { if (p->persistence) { CBM_PROF_START(t_art); int rc = cbm_artifact_export(final_path, p->repo_path, p->project_name, CBM_ARTIFACT_BEST); @@ -2469,7 +2500,7 @@ static int export_after_publish(cbm_pipeline_t *p, const char *final_path, bool } return rc; } - if (was_incremental && p->repo_path && cbm_artifact_exists(p->repo_path)) { + if (p->repo_path && cbm_artifact_exists(p->repo_path)) { (void)cbm_artifact_export(final_path, p->repo_path, p->project_name, CBM_ARTIFACT_FAST); } return 0; @@ -2511,8 +2542,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { free(final_path); return CBM_NOT_FOUND; } - bool was_incremental = false; - int rc = cbm_pipeline_run_staged(p, &was_incremental); + int rc = cbm_pipeline_run_staged(p); free(p->db_path); p->db_path = configured_db_path; @@ -2582,7 +2612,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { return CBM_PIPELINE_PERSIST_FAILED; } - rc = export_after_publish(p, final_path, was_incremental); + rc = export_after_publish(p, final_path); free(staging_path); free(final_path); return rc; diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index c01b07e02..5cde0c0f0 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -264,6 +264,14 @@ bool cbm_perl_suppress_generic_match(bool is_perl, bool is_method, const char *c * Pure; unit-tested in test_registry.c. */ bool cbm_tsjs_suppress_weak_method_match(bool is_tsjs, bool is_method, const char *strategy); +/* #725: drop a suffix_match CALLS edge when the caller language and the + * target file's language disagree. unique_name (candidates == 1) is #1572 + * and is left alone; same_module / import_map / lsp_* are kept. JS/TS/TSX + * are one family so a .ts helper calling a .tsx function is not dropped. + * Pure; unit-tested in test_registry.c. */ +bool cbm_suppress_cross_language_suffix_match(CBMLanguage caller_lang, const char *target_file_path, + const char *strategy); + /* Get the label of a qualified name, or NULL if not found. */ const char *cbm_registry_label_of(const cbm_registry_t *r, const char *qn); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 26cfbc16a..d3c5247f5 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -15,7 +15,6 @@ enum { INCR_RING_BUF = 4, INCR_RING_MASK = 3, INCR_TS_BUF = 24 }; #include "pipeline/pipeline.h" #include #include -#include "pipeline/artifact.h" #include "pipeline/lsp_surface.h" #include "pipeline/pass_lsp_cross.h" #include "sqlite3.h" @@ -1468,7 +1467,7 @@ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_file * generation boundary as full indexing. */ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *project, atomic_int *cancelled, const cbm_file_hash_t *manifest, - int manifest_count, const char *adr_content, const char *repo_path, + int manifest_count, const char *adr_content, const cbm_coverage_row_t *cov, int cov_count, const cbm_coverage_meta_t *meta_template, const cbm_lsp_surface_row_t *surface_rows, int surface_row_count) { @@ -1494,11 +1493,6 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p if (rc != 0) { return rc; } - - /* Auto-update artifact if one already exists (persistence was enabled previously) */ - if (repo_path && cbm_artifact_exists(repo_path)) { - cbm_artifact_export(db_path, repo_path, project, CBM_ARTIFACT_FAST); - } return 0; } @@ -2364,7 +2358,7 @@ out: int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *baseline_manifest, - int baseline_count) { + int baseline_count, bool force_full_on_mismatch) { struct timespec t0; cbm_clock_gettime(CLOCK_MONOTONIC, &t0); closure_plan_t closure_plan = {0}; @@ -2425,7 +2419,16 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil incr_test_set_last_route(CBM_INCREMENTAL_ROUTE_NOOP); #endif cbm_log_info("incremental.noop", "reason", "semantic_manifest_equal"); - return cbm_pipeline_refresh_artifact(p, db_path); + return 0; + } + if (force_full_on_mismatch) { + cbm_store_free_file_hashes(stored, stored_count); + cbm_store_close(store); +#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API + incr_test_set_last_route(CBM_INCREMENTAL_ROUTE_FORCED_FULL); +#endif + cbm_log_info("incremental.force_full", "reason", "mode_downgrade_changed"); + return CBM_PIPELINE_FORCE_FULL_REINDEX; } /* Manifest delta. Closure repair recomputes exactly the changed * files plus the recorded consumers of any changed SURFACE; every @@ -2850,9 +2853,9 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * re-parsed files have no codec output, and publishing a stale row * would satisfy a future closure plan with yesterday's surface; an * empty table just routes the next incremental to a full rebuild. */ - int persist_rc = dump_and_persist( - existing, db_path, project, cbm_pipeline_cancelled_ptr(p), manifest, manifest_count, - saved_adr, cbm_pipeline_repo_path(p), cov, cov_n, &coverage_meta, NULL, 0); + int persist_rc = + dump_and_persist(existing, db_path, project, cbm_pipeline_cancelled_ptr(p), manifest, + manifest_count, saved_adr, cov, cov_n, &coverage_meta, NULL, 0); cbm_pipeline_free_semantic_manifest(manifest, manifest_count); free(saved_adr); free(cov); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 50efd3ba5..754fa4ffa 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -657,7 +657,7 @@ int cbm_scan_project_env_urls_excluded(const char *root_path, cbm_env_binding_t * files, merges into disk DB. Returns 0 on success. */ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *baseline_manifest, - int baseline_count); + int baseline_count, bool force_full_on_mismatch); /* Exact semantic inputs for no-op/forced-full routing. The manifest contains * every discovered source plus repository controls actually consumed by @@ -752,11 +752,6 @@ int cbm_delta_patch(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf, i const cbm_delta_saved_edge_t *snapshot, int snapshot_count); /* discard helper shared with the delta executor (unlink stage + sidecars). */ void cbm_pipeline_discard_stage(const char *stage_path); -/* The SQLite generation is authoritative. An explicitly requested artifact is - * part of the caller-visible operation and its export error is returned; - * automatic refresh of an already-existing artifact remains best-effort. */ -int cbm_pipeline_refresh_artifact(cbm_pipeline_t *p, const char *db_path); - /* Hand the pipeline the per-file LSP-surface rows serialized at the * collect_all_defs seam (the only moment the result cache is alive). * Takes ownership; dump_and_persist_hashes writes them into the staging diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index cf1b3f008..dbb16d2d7 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -455,6 +455,49 @@ bool cbm_tsjs_suppress_weak_method_match(bool is_tsjs, bool is_method, const cha strcmp(strategy, "field_type_hint") == 0 || strcmp(strategy, "fuzzy") == 0; } +static bool js_ts_family(CBMLanguage lang) { + return lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX; +} + +static const char *path_basename(const char *path) { + if (!path || !path[0]) { + return path; + } + const char *slash = strrchr(path, '/'); +#ifdef _WIN32 + const char *bslash = strrchr(path, '\\'); + if (bslash && (!slash || bslash > slash)) { + slash = bslash; + } +#endif + return slash ? slash + 1 : path; +} + +bool cbm_suppress_cross_language_suffix_match(CBMLanguage caller_lang, const char *target_file_path, + const char *strategy) { + /* Two same-named symbols in different languages: suffix_match picks one + * winner by import-distance and attaches every bare-name call to it + * (#725, Bash/Python main, JS/Python commit). unique_name is the + * candidates==1 case (#1572) and is not this guard. */ + if (!strategy || strcmp(strategy, "suffix_match") != 0) { + return false; + } + if (caller_lang == CBM_LANG_COUNT || !target_file_path || !target_file_path[0]) { + return false; + } + CBMLanguage target_lang = cbm_language_for_filename(path_basename(target_file_path)); + if (target_lang == CBM_LANG_COUNT) { + return false; + } + if (caller_lang == target_lang) { + return false; + } + if (js_ts_family(caller_lang) && js_ts_family(target_lang)) { + return false; + } + return true; +} + /* ── Lifecycle ──────────────────────────────────────────────────── */ cbm_registry_t *cbm_registry_new(void) { diff --git a/src/store/store.c b/src/store/store.c index 4e2b12d62..aca9f7641 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -117,6 +117,7 @@ struct cbm_store { sqlite3_stmt *stmt_find_node_by_id; sqlite3_stmt *stmt_find_node_by_qn; sqlite3_stmt *stmt_find_node_by_qn_any; /* QN lookup without project filter */ + sqlite3_stmt *stmt_find_nodes; sqlite3_stmt *stmt_find_nodes_by_name; sqlite3_stmt *stmt_find_nodes_by_name_any; /* name lookup without project filter */ sqlite3_stmt *stmt_find_nodes_by_label; @@ -1140,6 +1141,7 @@ void cbm_store_close(cbm_store_t *s) { finalize_stmt(&s->stmt_find_node_by_id); finalize_stmt(&s->stmt_find_node_by_qn); finalize_stmt(&s->stmt_find_node_by_qn_any); + finalize_stmt(&s->stmt_find_nodes); finalize_stmt(&s->stmt_find_nodes_by_name); finalize_stmt(&s->stmt_find_nodes_by_name_any); finalize_stmt(&s->stmt_find_nodes_by_label); @@ -1978,7 +1980,9 @@ static int find_nodes_generic(cbm_store_t *s, sqlite3_stmt **slot, const char *s } bind_text(stmt, SKIP_ONE, project); - bind_text(stmt, ST_COL_2, val); + if (val) { + bind_text(stmt, ST_COL_2, val); + } int cap = ST_INIT_CAP_16; int n = 0; @@ -2006,6 +2010,19 @@ static int find_nodes_generic(cbm_store_t *s, sqlite3_stmt **slot, const char *s return CBM_STORE_OK; } +int cbm_store_find_nodes(cbm_store_t *s, const char *project, cbm_node_t **out, int *count) { + if (!s) { + *out = NULL; + *count = 0; + return CBM_STORE_ERR; + } + return find_nodes_generic(s, &s->stmt_find_nodes, + "SELECT id, project, label, name, qualified_name, file_path, " + "start_line, end_line, properties FROM nodes " + "WHERE project = ?1;", + project, NULL, out, count); +} + int cbm_store_find_nodes_by_name(cbm_store_t *s, const char *project, const char *name, cbm_node_t **out, int *count) { if (!s) { diff --git a/src/store/store.h b/src/store/store.h index 65f4971f8..8c2f76dd8 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -403,6 +403,9 @@ int cbm_store_find_node_by_qn(cbm_store_t *s, const char *project, const char *q /* Find node by qualified_name only (no project filter — QNs are globally unique). */ int cbm_store_find_node_by_qn_any(cbm_store_t *s, const char *qn, cbm_node_t *out); +/* Find all nodes in a project. Returns allocated array, caller frees. */ +int cbm_store_find_nodes(cbm_store_t *s, const char *project, cbm_node_t **out, int *count); + /* Find nodes by name (exact match). Returns allocated array, caller frees. */ int cbm_store_find_nodes_by_name(cbm_store_t *s, const char *project, const char *name, cbm_node_t **out, int *count); diff --git a/tests/test_cli.c b/tests/test_cli.c index 8eda0bb3f..95fff9d76 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -54,6 +54,7 @@ int cbm_cli_activation_abort_cleanup_probe_for_test(void); bool cbm_cli_activation_test_ops_installed(void); int cbm_cli_build_yaml_stdio_mcp_block_for_test(const char *binary_path, bool goose_schema, char *block, size_t block_size); +bool cbm_cli_stdin_allowed_for_schema_for_test(const char *schema_str); TEST(cli_progress_visibility_policy) { ASSERT_TRUE(cbm_cli_progress_enabled(true, false)); @@ -12453,22 +12454,22 @@ TEST(cli_print_tool_help_issue680) { * properties, so stdin could never have carried anything it accepts: the read * was pure deadlock. Gate the read on the tool actually declaring arguments. */ TEST(cli_zero_argument_tool_never_reads_stdin_issue1359) { - /* The reported hang: zero-argument tool + non-terminal stdin. */ - ASSERT_FALSE(cbm_cli_args_from_stdin_allowed("list_projects", false)); + /* #1359's deadlock class: a tool whose schema declares no properties must + * never read stdin (nothing it could carry; the read is pure deadlock). + * Since #1181 no SHIPPED tool is zero-argument anymore — list_projects, + * the original example, now declares pagination properties — so the + * zero-argument branch is exercised directly through the schema seam, + * and list_projects asserts its NEW truth. */ + ASSERT_FALSE( + cbm_cli_stdin_allowed_for_schema_for_test("{\"type\":\"object\",\"properties\":{}}")); + ASSERT_FALSE(cbm_cli_stdin_allowed_for_schema_for_test("{\"type\":\"object\"}")); + ASSERT_TRUE(cbm_cli_stdin_allowed_for_schema_for_test( + "{\"type\":\"object\",\"properties\":{\"p\":{\"type\":\"string\"}}}")); - /* The documented `echo '' | cli ` channel must survive. */ - ASSERT_TRUE(cbm_cli_args_from_stdin_allowed("index_status", false)); - ASSERT_TRUE(cbm_cli_args_from_stdin_allowed("search_graph", false)); - ASSERT_TRUE(cbm_cli_args_from_stdin_allowed("index_repository", false)); - - /* Interactive runs never read the terminal for arguments — unchanged. */ - ASSERT_FALSE(cbm_cli_args_from_stdin_allowed("index_status", true)); + /* list_projects now takes piped arguments (offset/limit/include_details); + * the TTY guard still refuses regardless of schema. */ + ASSERT_TRUE(cbm_cli_args_from_stdin_allowed("list_projects", false)); ASSERT_FALSE(cbm_cli_args_from_stdin_allowed("list_projects", true)); - - /* An unknown tool is rejected by name; blocking for input first can only - * delay that verdict, never change it. */ - ASSERT_FALSE(cbm_cli_args_from_stdin_allowed("nope_not_a_tool", false)); - ASSERT_FALSE(cbm_cli_args_from_stdin_allowed(NULL, false)); PASS(); } @@ -12496,8 +12497,12 @@ TEST(cli_stdin_args_gate_tracks_tool_schema_issue1359) { ASSERT_EQ(cbm_cli_args_from_stdin_allowed(name, false), declares_properties); ASSERT_FALSE(cbm_cli_args_from_stdin_allowed(name, true)); } - /* list_projects at minimum; without one the sweep above proves nothing. */ - ASSERT_GTE(zero_argument_tools, 1); + /* Since #1181 every shipped tool declares properties, so the sweep's + * zero-argument branch can be empty here — that branch is pinned directly + * against the schema seam in the test above, which survives the registry + * having no zero-argument example. The sweep still proves schema↔gate + * parity for every tool that exists. */ + (void)zero_argument_tools; PASS(); } diff --git a/tests/test_cypher.c b/tests/test_cypher.c index 1830d7286..4e9132cf9 100644 --- a/tests/test_cypher.c +++ b/tests/test_cypher.c @@ -1005,6 +1005,46 @@ TEST(cypher_exec_where_eq) { PASS(); } +/* #1196: max_rows limits projected results, not the source-node candidates + * considered before WHERE. The old unlabeled scan searched only + * 10 * max_rows nodes, so a valid match later in search order disappeared. */ +TEST(cypher_exec_unlabeled_where_beyond_result_limit_issue1196) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + ASSERT_EQ(cbm_store_upsert_project(s, "test", "/tmp/test"), CBM_STORE_OK); + + for (int i = 0; i < 11; i++) { + char name[32]; + char qn[64]; + snprintf(name, sizeof(name), "early_%02d", i); + snprintf(qn, sizeof(qn), "test.%s", name); + cbm_node_t distractor = {.project = "test", + .label = "Function", + .name = name, + .qualified_name = qn, + .file_path = "early.py"}; + ASSERT_GT(cbm_store_upsert_node(s, &distractor), 0); + } + + cbm_node_t late = {.project = "test", + .label = "Function", + .name = "zz_late_match", + .qualified_name = "test.zz_late_match", + .file_path = "late.py"}; + ASSERT_GT(cbm_store_upsert_node(s, &late), 0); + + cbm_cypher_result_t r = {0}; + int rc = cbm_cypher_execute( + s, "MATCH (n) WHERE n.name = \"zz_late_match\" RETURN n.name", "test", 1, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 1); + ASSERT_STR_EQ(r.rows[0][0], "zz_late_match"); + + cbm_cypher_result_free(&r); + cbm_store_close(s); + PASS(); +} + /* #874: coalesce(var.prop, literal) in WHERE — null-safe numeric filters * for audit queries over OPTIONAL graph properties. The parser rejected the * call outright ("unexpected operator"); RETURN-side coalesce already @@ -3897,6 +3937,7 @@ SUITE(cypher) { RUN_TEST(cypher_issue252_tointeger); RUN_TEST(cypher_issue305_count_star_alias); RUN_TEST(cypher_exec_where_eq); + RUN_TEST(cypher_exec_unlabeled_where_beyond_result_limit_issue1196); RUN_TEST(cypher_exec_varlength_path_semantics_issue797); RUN_TEST(cypher_exec_where_coalesce_issue874); RUN_TEST(cypher_exec_where_regex); diff --git a/tests/test_discover.c b/tests/test_discover.c index 00384a3da..7e0007b11 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -434,6 +434,47 @@ TEST(discover_bounded_count_fails_closed_after_deadline) { PASS(); } +/* The allocation-free bounded count must apply the same shebang fallback as + * full discovery (issue #1199): extensionless scripts count, extensionless + * plain text does not, and the limit boundary stays exact over that mix. */ +TEST(discover_bounded_count_matches_shebang_discovery) { + char *base = th_mktempdir("cbm_disc_bounded_sb"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "keep.c"), "int keep;\n"); + th_write_file(TH_PATH(base, "bin/build"), "#!/usr/bin/env python3\nprint(1)\n"); + th_write_file(TH_PATH(base, "bin/deploy"), "#!/bin/bash\necho hi\n"); + th_write_file(TH_PATH(base, "bin/notes"), "just some plain text\nno shebang here\n"); + + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + cbm_file_info_t *files = NULL; + int count = 0; + int full_rc = cbm_discover(base, &opts, &files, &count); + bool has_build = discover_has_rel_path(files, count, "bin/build"); + bool has_deploy = discover_has_rel_path(files, count, "bin/deploy"); + bool has_notes = discover_has_rel_path(files, count, "bin/notes"); + cbm_discover_free(files, count); + + int exact_count = -1; + cbm_discover_status_t exact = + cbm_discover_count_bounded(base, &opts, 3, cbm_now_ms() + 2000, &exact_count); + int limited_count = -1; + cbm_discover_status_t limited = + cbm_discover_count_bounded(base, &opts, 2, cbm_now_ms() + 2000, &limited_count); + + th_cleanup(base); + + ASSERT_EQ(full_rc, 0); + ASSERT_TRUE(has_build); + ASSERT_TRUE(has_deploy); + ASSERT_FALSE(has_notes); + ASSERT_EQ(count, 3); + ASSERT_EQ(exact, CBM_DISCOVER_OK); + ASSERT_EQ(exact_count, 3); + ASSERT_EQ(limited, CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT_EQ(limited_count, 2); + PASS(); +} + TEST(discover_skips_git_dir) { char *base = th_mktempdir("cbm_disc_git"); ASSERT(base != NULL); @@ -1395,6 +1436,239 @@ TEST(discover_many_nested_gitignores_do_not_exhaust_matcher_ownership) { PASS(); } +/* ── Shebang fallback for extensionless scripts (issue #1199) ────── */ + +/* Language detected for a discovered file by relative path, or CBM_LANG_COUNT + * if the file was not indexed. */ +static CBMLanguage discover_lang_of(const cbm_file_info_t *files, int count, const char *rel) { + for (int i = 0; i < count; i++) { + if (strcmp(files[i].rel_path, rel) == 0) { + return files[i].language; + } + } + return CBM_LANG_COUNT; +} + +/* Write raw bytes (may contain embedded NUL) to base/rel. Parent must exist. */ +static int write_bytes_file(const char *path, const void *data, size_t n) { + FILE *f = cbm_fopen(path, "wb"); + if (!f) { + return -1; + } + size_t w = fwrite(data, 1, n, f); + fclose(f); + return w == n ? 0 : -1; +} + +/* Discover a single file with the given first-line content and report the + * language it was classified as via *out_lang (CBM_LANG_COUNT if unindexed). + * Returns true only when temp-dir creation, the file write, and discovery all + * succeeded, so callers can assert setup success before asserting the language + * -- otherwise a negative expectation (CBM_LANG_COUNT) could pass vacuously + * when setup silently failed. */ +static bool shebang_probe(const char *prefix, const char *rel, const char *content, + CBMLanguage *out_lang) { + *out_lang = CBM_LANG_COUNT; + char *base = th_mktempdir(prefix); + if (!base) { + return false; + } + bool ok = false; + if (th_write_file(TH_PATH(base, rel), content) == 0) { + cbm_discover_opts_t opts = {0}; + cbm_file_info_t *files = NULL; + int count = 0; + if (cbm_discover(base, &opts, &files, &count) == 0) { + *out_lang = discover_lang_of(files, count, rel); + ok = true; + } + cbm_discover_free(files, count); + } + th_cleanup(base); + return ok; +} + +TEST(shebang_direct_python_path) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_dpy", "bin/run", "#!/usr/bin/python3\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +TEST(shebang_env_python) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_epy", "bin/run", "#!/usr/bin/env python3\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +TEST(shebang_env_split_string_python_args) { + CBMLanguage lang; + ASSERT( + shebang_probe("cbm_sb_spy", "bin/run", "#!/usr/bin/env -S python3 -u\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +TEST(shebang_versioned_python) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_vpy", "bin/run", "#!/usr/bin/python3.12\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +TEST(shebang_bash_direct) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_bash", "bin/run", "#!/bin/bash\necho hi\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_BASH); + PASS(); +} + +TEST(shebang_env_node_javascript) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_node", "bin/run", "#!/usr/bin/env node\nconsole.log(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_JAVASCRIPT); + PASS(); +} + +TEST(shebang_crlf_first_line) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_crlf", "bin/run", "#!/usr/bin/env python3\r\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +/* Recognized extension stays authoritative even with a conflicting shebang. */ +TEST(shebang_extension_authoritative) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_ext", "weird.py", "#!/bin/bash\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +/* Special filename stays authoritative even with a conflicting shebang. */ +TEST(shebang_special_filename_authoritative) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_mk", "Makefile", "#!/usr/bin/env python3\nall:\n\techo\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_MAKEFILE); + PASS(); +} + +TEST(shebang_plain_text_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_txt", "notes", "just some plain text\nno shebang here\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +TEST(shebang_malformed_unindexed) { + /* Missing '!' — not a shebang. */ + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_mal", "notes", "#/usr/bin/python3\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +TEST(shebang_unknown_interpreter_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_unk", "notes", "#!/usr/bin/frobnicate\ndata\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +/* Guard against arbitrary prefix matches (python-wrapper must NOT map). */ +TEST(shebang_prefix_not_matched_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_pw", "notes", "#!/usr/bin/python-wrapper\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +/* Numeric version components must be non-empty (reject trailing/doubled dots). */ +TEST(shebang_malformed_python_version_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_pdot", "notes", "#!/usr/bin/python3.\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + ASSERT(shebang_probe("cbm_sb_pddot", "notes", "#!/usr/bin/python3..12\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +/* env NAME=value assignment (with a slash in the value) must NOT be mistaken + * for an interpreter: env would set the var, not run "python" (issue #1199, + * Finding 2). */ +TEST(shebang_env_assignment_not_matched_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_asn", "notes", + "#!/usr/bin/env PYTHON=/usr/bin/python python-wrapper\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + /* Same shape behind -S must also stay unindexed. */ + ASSERT(shebang_probe("cbm_sb_asn2", "notes", + "#!/usr/bin/env -S PYTHON=/usr/bin/python python\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +/* Unsupported env option forms (a leading '-' token that is not -S/ + * --split-string) must stay unindexed rather than be treated as the + * interpreter (issue #1199, Finding 2). */ +TEST(shebang_env_unsupported_option_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_eopt", "notes", "#!/usr/bin/env -i python3\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + ASSERT(shebang_probe("cbm_sb_eopt2", "notes", "#!/usr/bin/env -S -i python3\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +TEST(shebang_embedded_nul_unindexed) { + char *base = th_mktempdir("cbm_sb_nul"); + ASSERT(base != NULL); + /* NUL embedded within the first line -> fail closed. */ + static const char bytes[] = "#!/usr/bin/py\0thon3\nprint(1)\n"; + ASSERT_EQ(write_bytes_file(TH_PATH(base, "run"), bytes, sizeof(bytes) - 1), 0); + + cbm_discover_opts_t opts = {0}; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), 0); + ASSERT_FALSE(discover_has_rel_path(files, count, "run")); + + cbm_discover_free(files, count); + th_cleanup(base); + PASS(); +} + +/* A first line longer than the bounded read window (255 bytes) with no + * newline inside that window must fail closed even when it begins with an + * otherwise-valid Python shebang: the interpreter token is truncated, so the + * file must remain unindexed (issue #1199, Finding 1). */ +TEST(shebang_oversized_first_line_unindexed) { + char *base = th_mktempdir("cbm_sb_long"); + ASSERT(base != NULL); + + /* "#!/usr/bin/python3 " + padding to push the newline past byte 255. */ + char content[512]; + int off = snprintf(content, sizeof(content), "#!/usr/bin/python3 "); + ASSERT(off > 0 && off < (int)sizeof(content)); + for (int i = off; i < 400; i++) { + content[i] = 'a'; + } + content[400] = '\n'; + content[401] = '\0'; + ASSERT_EQ(write_bytes_file(TH_PATH(base, "run"), content, 401), 0); + + cbm_discover_opts_t opts = {0}; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), 0); + ASSERT_FALSE(discover_has_rel_path(files, count, "run")); + + cbm_discover_free(files, count); + th_cleanup(base); + PASS(); +} + /* ── Suite ─────────────────────────────────────────────────────── */ SUITE(discover) { @@ -1467,11 +1741,32 @@ SUITE(discover) { RUN_TEST(pattern_stories); RUN_TEST(pattern_dts_full); + /* Shebang fallback for extensionless scripts (issue #1199) */ + RUN_TEST(shebang_direct_python_path); + RUN_TEST(shebang_env_python); + RUN_TEST(shebang_env_split_string_python_args); + RUN_TEST(shebang_versioned_python); + RUN_TEST(shebang_bash_direct); + RUN_TEST(shebang_env_node_javascript); + RUN_TEST(shebang_crlf_first_line); + RUN_TEST(shebang_extension_authoritative); + RUN_TEST(shebang_special_filename_authoritative); + RUN_TEST(shebang_plain_text_unindexed); + RUN_TEST(shebang_malformed_unindexed); + RUN_TEST(shebang_unknown_interpreter_unindexed); + RUN_TEST(shebang_prefix_not_matched_unindexed); + RUN_TEST(shebang_malformed_python_version_unindexed); + RUN_TEST(shebang_env_assignment_not_matched_unindexed); + RUN_TEST(shebang_env_unsupported_option_unindexed); + RUN_TEST(shebang_embedded_nul_unindexed); + RUN_TEST(shebang_oversized_first_line_unindexed); + /* Integration tests (cross-platform) */ RUN_TEST(discover_simple); RUN_TEST(discover_wide_sibling_fanout_exceeds_initial_walk_stack); RUN_TEST(discover_bounded_count_is_allocation_free_and_limit_exact); RUN_TEST(discover_bounded_count_fails_closed_after_deadline); + RUN_TEST(discover_bounded_count_matches_shebang_discovery); RUN_TEST(discover_skips_git_dir); RUN_TEST(discover_with_gitignore); RUN_TEST(discover_with_global_xdg_ignore); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index ccf5a1c6e..447e7c104 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4193,6 +4193,111 @@ TEST(extract_rust_test_attr_marks_is_test_issue855) { PASS(); } +/* Function.is_test must follow the file's test-directory membership, not just + * a test_/_test basename convention: a helper under tests/ with none of those + * naming conventions (tests/helpers/fixtures.c) was previously indexed as an + * ordinary, non-test Function — invisible to store.c's `is_test != 1` + * dead-code/search filters even though trace_path's own independent path + * check already treated the same file as a test (#1294). */ +TEST(extract_c_test_dir_marks_is_test_issue1294) { + const char *src = "void helper(void) {}\n"; + + /* Regression: a test_*-named file directly under tests/ must remain a + * test (this already worked before the fix). */ + CBMFileResult *r1 = extract(src, CBM_LANG_C, "t", "tests/test_pipeline.c"); + ASSERT_NOT_NULL(r1); + ASSERT_FALSE(r1->has_error); + ASSERT(has_def(r1, "Function", "helper")); + int reg = -1; + for (int i = 0; i < r1->defs.count; i++) { + if (strcmp(r1->defs.items[i].label, "Function") == 0 && r1->defs.items[i].name && + strcmp(r1->defs.items[i].name, "helper") == 0) { + reg = r1->defs.items[i].is_test ? 1 : 0; + } + } + ASSERT(reg == 1 && "test_*.c directly under tests/ is a test (regression)"); + cbm_free_result(r1); + + /* Positive: a file under tests/ that matches none of the test_/_test + * naming conventions must now ALSO be a test. */ + CBMFileResult *r2 = extract(src, CBM_LANG_C, "t", "tests/helpers/fixtures.c"); + ASSERT_NOT_NULL(r2); + ASSERT_FALSE(r2->has_error); + ASSERT(has_def(r2, "Function", "helper")); + int nonconv = -1; + for (int i = 0; i < r2->defs.count; i++) { + if (strcmp(r2->defs.items[i].label, "Function") == 0 && r2->defs.items[i].name && + strcmp(r2->defs.items[i].name, "helper") == 0) { + nonconv = r2->defs.items[i].is_test ? 1 : 0; + } + } + ASSERT(nonconv == 1 && "non-test_-named file under tests/ is now a test"); + cbm_free_result(r2); + + /* Negative: a file with no test/ directory or test_/_test naming in its + * path must never be flagged — this fix must not widen detection beyond + * the tests/ (and sibling) directory tree. */ + CBMFileResult *r3 = extract(src, CBM_LANG_C, "t", "src/pipeline/helper.c"); + ASSERT_NOT_NULL(r3); + ASSERT_FALSE(r3->has_error); + ASSERT(has_def(r3, "Function", "helper")); + int outside = -1; + for (int i = 0; i < r3->defs.count; i++) { + if (strcmp(r3->defs.items[i].label, "Function") == 0 && r3->defs.items[i].name && + strcmp(r3->defs.items[i].name, "helper") == 0) { + outside = r3->defs.items[i].is_test ? 1 : 0; + } + } + ASSERT(outside == 0 && "file outside tests/ is never a test"); + cbm_free_result(r3); + + PASS(); +} + +/* Same convergence for Method definitions (push_method_def), which take a + * separate code path from free functions: a class method on a class defined + * under tests/ with no test_/_test naming (e.g. tests/helpers/base.py) must + * be marked is_test too, and a method on the same class outside tests/ must + * not be (#1294). */ +TEST(extract_python_method_test_dir_marks_is_test_issue1294) { + const char *src = "class Foo:\n" + " def helper(self):\n" + " pass\n"; + + /* Python's LSP layer injects synthetic builtin stub Methods (str.upper, + * dict.get, ...) into defs.items alongside real ones (py_builtins.c), so + * matching must key on name, not just label="Method". */ + CBMFileResult *r1 = extract(src, CBM_LANG_PYTHON, "t", "tests/helpers/base.py"); + ASSERT_NOT_NULL(r1); + ASSERT_FALSE(r1->has_error); + ASSERT(has_def(r1, "Method", "helper")); + int in_tests = -1; + for (int i = 0; i < r1->defs.count; i++) { + if (strcmp(r1->defs.items[i].label, "Method") == 0 && r1->defs.items[i].name && + strcmp(r1->defs.items[i].name, "helper") == 0) { + in_tests = r1->defs.items[i].is_test ? 1 : 0; + } + } + ASSERT(in_tests == 1 && "method on a class under tests/helpers/ is a test"); + cbm_free_result(r1); + + CBMFileResult *r2 = extract(src, CBM_LANG_PYTHON, "t", "app/models/base.py"); + ASSERT_NOT_NULL(r2); + ASSERT_FALSE(r2->has_error); + ASSERT(has_def(r2, "Method", "helper")); + int outside_tests = -1; + for (int i = 0; i < r2->defs.count; i++) { + if (strcmp(r2->defs.items[i].label, "Method") == 0 && r2->defs.items[i].name && + strcmp(r2->defs.items[i].name, "helper") == 0) { + outside_tests = r2->defs.items[i].is_test ? 1 : 0; + } + } + ASSERT(outside_tests == 0 && "method on a class outside tests/ is never a test"); + cbm_free_result(r2); + + PASS(); +} + /* #1017: docstring truncation at MAX_COMMENT_LEN (500 bytes) can split a * multi-byte UTF-8 character, leaving an incomplete byte sequence. * Craft a Go comment whose 498th-500th bytes are a 3-byte CJK character @@ -5528,6 +5633,8 @@ SUITE(extraction) { RUN_TEST(extract_c_clean_file_no_recovery_duplicates_issue961); RUN_TEST(walk_defs_no_truncation_over_4096_issue668); RUN_TEST(extract_rust_test_attr_marks_is_test_issue855); + RUN_TEST(extract_c_test_dir_marks_is_test_issue1294); + RUN_TEST(extract_python_method_test_dir_marks_is_test_issue1294); RUN_TEST(docstring_utf8_truncation_boundary_issue1017); RUN_TEST(extract_ts_decorators_survive_interleaved_comment); diff --git a/tests/test_hook_conflict_notice.sh b/tests/test_hook_conflict_notice.sh index e44b0f327..35cd844d1 100644 --- a/tests/test_hook_conflict_notice.sh +++ b/tests/test_hook_conflict_notice.sh @@ -44,10 +44,13 @@ if ! LC_ALL=C grep -a -q -F 'CBM_TEST_HOOK_CLIENT_BUILD' "${BINARY}"; then exit 2 fi -tmpdir="$(mktemp -d)" +# shellcheck source=../scripts/test-runtime.sh +source "${ROOT}/scripts/test-runtime.sh" +cbm_test_runtime_init +tmpdir="${CBM_TEST_RUNTIME_ROOT}" cleanup() { CBM_CACHE_DIR="${tmpdir}/cache" "${BINARY}" daemon stop >/dev/null 2>&1 || true - rm -rf "${tmpdir}" + cbm_test_runtime_cleanup "${BINARY}" } trap cleanup EXIT diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 20f597d9c..7610929b5 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -1691,6 +1691,50 @@ static cbm_mcp_server_t *setup_snippet_server(char *tmp_dir, size_t tmp_sz); static void cleanup_snippet_dir(const char *tmp_dir); static char *extract_text_content(const char *mcp_result); +TEST(tool_search_graph_semantic_only_skips_structural_results_issue1295) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + const char *proj = "semantic-only"; + cbm_mcp_server_set_project(srv, proj); + cbm_store_upsert_project(st, proj, "/tmp/semantic-only"); + + cbm_node_t unrelated = {.project = proj, + .label = "Function", + .name = "unrelated_node", + .qualified_name = "semantic-only.unrelated_node", + .file_path = "unrelated.c", + .start_line = 1, + .end_line = 2}; + ASSERT_GT(cbm_store_upsert_node(st, &unrelated), 0); + + char *resp = cbm_mcp_handle_tool( + srv, "search_graph", + "{\"project\":\"semantic-only\",\"semantic_query\":[\"publish\"],\"limit\":5}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + free(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NULL(strstr(inner, "unrelated_node")); + free(inner); + + resp = cbm_mcp_handle_tool(srv, "search_graph", + "{\"project\":\"semantic-only\",\"semantic_query\":[\"publish\"]," + "\"format\":\"json\",\"limit\":5}"); + ASSERT_NOT_NULL(resp); + inner = extract_text_content(resp); + free(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "\"total\":0")); + ASSERT_NOT_NULL(strstr(inner, "\"count\":0")); + ASSERT_NOT_NULL(strstr(inner, "\"groups\":[]")); + ASSERT_NULL(strstr(inner, "unrelated_node")); + free(inner); + + cbm_mcp_server_free(srv); + PASS(); +} + /* callers_total/callees_total must count what the caller can enumerate: with * include_tests=false (default) test-file rows are hidden from the table, so * the totals must apply the same filter — a raw visited_count overstated the @@ -1761,6 +1805,77 @@ TEST(tool_trace_totals_respect_test_filter) { PASS(); } +/* Same as above, but the test-side caller lives directly under a + * project-root tests/ directory with no test_/_test naming convention + * (tests/repro/helper.c) rather than tests/test_x.c. is_test_file() in + * mcp.c matched a NESTED ".../tests/..." path but not a project-root- + * relative one, so this row leaked into results with the default + * include_tests=false (#1294, secondary bug). */ +TEST(tool_trace_totals_respect_test_filter_tests_root_subtree_issue1294) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + const char *proj = "totproj2"; + cbm_mcp_server_set_project(srv, proj); + cbm_store_upsert_project(st, proj, "/tmp/tot2"); + + cbm_node_t tgt = {.project = proj, + .label = "Function", + .name = "tgt2", + .qualified_name = "totproj2.a.tgt2", + .file_path = "a.c", + .start_line = 1, + .end_line = 5}; + int64_t tid = cbm_store_upsert_node(st, &tgt); + ASSERT_GT(tid, 0); + cbm_node_t prod = {.project = proj, + .label = "Function", + .name = "prod_caller2", + .qualified_name = "totproj2.a.prod_caller2", + .file_path = "a.c", + .start_line = 10, + .end_line = 15}; + int64_t pid = cbm_store_upsert_node(st, &prod); + ASSERT_GT(pid, 0); + cbm_node_t tst = {.project = proj, + .label = "Function", + .name = "repro_caller", + .qualified_name = "totproj2.t.repro_caller", + .file_path = "tests/repro/helper.c", + .start_line = 1, + .end_line = 5}; + int64_t xid = cbm_store_upsert_node(st, &tst); + ASSERT_GT(xid, 0); + cbm_edge_t e1 = {.project = proj, .source_id = pid, .target_id = tid, .type = "CALLS"}; + ASSERT_GT(cbm_store_insert_edge(st, &e1), 0); + cbm_edge_t e2 = {.project = proj, .source_id = xid, .target_id = tid, .type = "CALLS"}; + ASSERT_GT(cbm_store_insert_edge(st, &e2), 0); + + char *resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":92,\"method\":\"tools/call\",\"params\":{" + "\"name\":\"trace_call_path\",\"arguments\":{\"project\":\"totproj2\"," + "\"function_name\":\"tgt2\",\"direction\":\"inbound\"}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + free(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "callers_total: 1")); /* tests/repro/ row filtered by default */ + free(inner); + + resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":93,\"method\":\"tools/call\",\"params\":{" + "\"name\":\"trace_call_path\",\"arguments\":{\"project\":\"totproj2\"," + "\"function_name\":\"tgt2\",\"direction\":\"inbound\",\"include_tests\":true}}}"); + ASSERT_NOT_NULL(resp); + inner = extract_text_content(resp); + free(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "callers_total: 2")); /* both visible now */ + free(inner); + cbm_mcp_server_free(srv); + PASS(); +} + /* SCC condensation (get_architecture aspect "cycles"): a 3-function CALLS * cycle A->B->C->A must be reported as one circular dependency of size 3 with * all three members; a separate acyclic chain (D->E) must NOT appear. The @@ -2023,15 +2138,32 @@ TEST(tool_search_graph_includes_node_properties) { free(inner); free(resp); + /* List-valued fields are compact JSON in text output, rather than an + * empty placeholder. */ + resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":431,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_graph\"," + "\"arguments\":{\"project\":\"test-project\",\"label\":\"Function\"," + "\"name_pattern\":\"HandleRequest\",\"fields\":[\"base_classes\"]," + "\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "base_classes")); + ASSERT_NOT_NULL(strstr(inner, "HandlerBase")); + ASSERT_NOT_NULL(strstr(inner, "Audited")); + free(inner); + free(resp); + /* format:"json" = json-stringified tree: same grouped model, column- * ordered row arrays — never per-row key envelopes or property blobs. - * fields adds columns there too. */ + * fields adds columns there too and preserves compound JSON types. */ resp = cbm_mcp_server_handle( srv, "{\"jsonrpc\":\"2.0\",\"id\":44,\"method\":\"tools/call\"," "\"params\":{\"name\":\"search_graph\"," "\"arguments\":{\"project\":\"test-project\",\"label\":\"Function\"," "\"name_pattern\":\"HandleRequest\",\"format\":\"json\"," - "\"fields\":[\"signature\"],\"limit\":5}}}"); + "\"fields\":[\"signature\",\"base_classes\"],\"limit\":5}}}"); ASSERT_NOT_NULL(resp); inner = extract_text_content(resp); ASSERT_NOT_NULL(inner); @@ -2040,7 +2172,9 @@ TEST(tool_search_graph_includes_node_properties) { ASSERT_NOT_NULL(strstr(inner, "\"rows\"")); ASSERT_NOT_NULL(strstr(inner, "\"signature\"")); /* requested column */ ASSERT_NOT_NULL(strstr(inner, "func HandleRequest")); /* its value */ - ASSERT_NULL(strstr(inner, "is_exported")); /* blob never spills */ + ASSERT_NOT_NULL(strstr(inner, "\"base_classes\"")); + ASSERT_NOT_NULL(strstr(inner, "[\"HandlerBase\",\"Audited\"]")); + ASSERT_NULL(strstr(inner, "is_exported")); /* blob never spills */ free(inner); free(resp); @@ -2600,15 +2734,13 @@ TEST(tool_check_index_coverage_accepts_truncated_ignored_catalog_for_fresh_path_ ((int64_t)source_stat.st_mtimespec.tv_sec * (int64_t)CBM_NSEC_PER_SEC) + (int64_t)source_stat.st_mtimespec.tv_nsec; #elif defined(_WIN32) - int64_t source_mtime_ns = - (int64_t)source_stat.st_mtime * (int64_t)CBM_NSEC_PER_SEC; + int64_t source_mtime_ns = (int64_t)source_stat.st_mtime * (int64_t)CBM_NSEC_PER_SEC; #else - int64_t source_mtime_ns = - ((int64_t)source_stat.st_mtim.tv_sec * (int64_t)CBM_NSEC_PER_SEC) + - (int64_t)source_stat.st_mtim.tv_nsec; + int64_t source_mtime_ns = ((int64_t)source_stat.st_mtim.tv_sec * (int64_t)CBM_NSEC_PER_SEC) + + (int64_t)source_stat.st_mtim.tv_nsec; #endif - ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", - source_mtime_ns, source_stat.st_size), + ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", source_mtime_ns, + source_stat.st_size), CBM_STORE_OK); cbm_project_t project = {0}; ASSERT_EQ(cbm_store_get_project(store, "test-project", &project), CBM_STORE_OK); @@ -2622,8 +2754,7 @@ TEST(tool_check_index_coverage_accepts_truncated_ignored_catalog_for_fresh_path_ .coverage_version = 1, .hash_records_complete = true, }; - ASSERT_EQ(cbm_store_coverage_replace_ex(store, "test-project", NULL, 0, &meta), - CBM_STORE_OK); + ASSERT_EQ(cbm_store_coverage_replace_ex(store, "test-project", NULL, 0, &meta), CBM_STORE_OK); cbm_project_free_fields(&project); char *response = cbm_mcp_handle_tool( @@ -4231,6 +4362,250 @@ TEST(search_code_multi_word) { PASS(); } +/* Regression guard: search_code full results must preserve valid UTF-8 source. */ +static bool is_valid_json_response(const char *json); + +TEST(search_code_full_preserves_utf8_source) { + char tmp[512]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_srch_utf8_XXXXXX"); + ASSERT_TRUE(cbm_mkdtemp(tmp) != NULL); + + char project_dir[640]; + snprintf(project_dir, sizeof(project_dir), "%s/project", tmp); + ASSERT_EQ(cbm_mkdir(project_dir), 0); + char design_dir[768]; + snprintf(design_dir, sizeof(design_dir), "%s/design", project_dir); + ASSERT_EQ(cbm_mkdir(design_dir), 0); + + char source_path[768]; + snprintf(source_path, sizeof(source_path), "%s/design.md", design_dir); + FILE *fp = cbm_fopen(source_path, "wb"); + ASSERT_NOT_NULL(fp); + const char source[] = "# accounting-design\nРусский текст: бухгалтерский учет.\n"; + ASSERT_EQ(fwrite(source, 1, sizeof(source) - SKIP_ONE, fp), sizeof(source) - SKIP_ONE); + ASSERT_EQ(fclose(fp), 0); + + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + const char *project = "utf8-search"; + cbm_mcp_server_set_project(srv, project); + cbm_store_upsert_project(st, project, project_dir); + + cbm_node_t section = {.project = project, + .label = "Section", + .name = "accounting-design", + .qualified_name = "utf8-search.design.accounting-design", + .file_path = "design/design.md", + .start_line = 1, + .end_line = 2}; + ASSERT_GT(cbm_store_upsert_node(st, §ion), 0); + + char *resp = + cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":97,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_code\",\"arguments\":{" + "\"project\":\"utf8-search\",\"pattern\":\"accounting-design\"," + "\"file_pattern\":\"*.md\",\"path_filter\":\"^design/\"," + "\"mode\":\"full\",\"format\":\"json\",\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + + yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *rows = yyjson_obj_get(yyjson_doc_get_root(doc), "rows"); + ASSERT_NOT_NULL(rows); + ASSERT_TRUE(yyjson_arr_size(rows) > 0); + yyjson_val *row = yyjson_arr_get(rows, 0); + yyjson_val *source_obj = yyjson_arr_get(row, 7); + yyjson_val *source_val = yyjson_obj_get(source_obj, "source"); + ASSERT_NOT_NULL(source_val); + ASSERT_STR_EQ(yyjson_get_str(source_val), source); + yyjson_doc_free(doc); + + free(inner); + free(resp); + cbm_mcp_server_free(srv); + cbm_unlink(source_path); + cbm_rmdir(design_dir); + cbm_rmdir(project_dir); + cbm_rmdir(tmp); + PASS(); +} + +TEST(search_code_raw_match_preserves_utf8_content) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + + char raw_path[512]; + snprintf(raw_path, sizeof(raw_path), "%s/project/raw.md", tmp); + const char raw_source[] = "header\nraw-Русский content\n"; + FILE *fp = cbm_fopen(raw_path, "wb"); + ASSERT_NOT_NULL(fp); + ASSERT_EQ(fwrite(raw_source, 1, sizeof(raw_source) - SKIP_ONE, fp), + sizeof(raw_source) - SKIP_ONE); + ASSERT_EQ(fclose(fp), 0); + + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + cbm_node_t node = {.project = "test-project", + .label = "Section", + .name = "raw", + .qualified_name = "test-project.raw", + .file_path = "raw.md", + .start_line = 1, + .end_line = 1}; + ASSERT_GT(cbm_store_upsert_node(st, &node), 0); + + char *resp = + cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":98,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_code\",\"arguments\":{" + "\"project\":\"test-project\",\"pattern\":\"raw-\"," + "\"file_pattern\":\"*.md\",\"format\":\"json\",\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *raw_obj = yyjson_obj_get(yyjson_doc_get_root(doc), "raw_matches"); + ASSERT_NOT_NULL(raw_obj); + yyjson_val *raw_rows = yyjson_obj_get(raw_obj, "rows"); + ASSERT_NOT_NULL(raw_rows); + ASSERT_TRUE(yyjson_arr_size(raw_rows) > 0); + yyjson_val *raw_row = yyjson_arr_get(raw_rows, 0); + ASSERT_STR_EQ(yyjson_get_str(yyjson_arr_get(raw_row, 2)), "raw-Русский content"); + yyjson_doc_free(doc); + + free(inner); + free(resp); + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + +TEST(search_code_context_preserves_utf8_context) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + + char context_path[512]; + snprintf(context_path, sizeof(context_path), "%s/project/context.md", tmp); + const char context_source[] = "before-до\ncontext-needle\nпосле-после\n"; + FILE *fp = cbm_fopen(context_path, "wb"); + ASSERT_NOT_NULL(fp); + ASSERT_EQ(fwrite(context_source, 1, sizeof(context_source) - SKIP_ONE, fp), + sizeof(context_source) - SKIP_ONE); + ASSERT_EQ(fclose(fp), 0); + + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + cbm_node_t node = {.project = "test-project", + .label = "Section", + .name = "context", + .qualified_name = "test-project.context", + .file_path = "context.md", + .start_line = 1, + .end_line = 3}; + ASSERT_GT(cbm_store_upsert_node(st, &node), 0); + + char *resp = + cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":99,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_code\",\"arguments\":{" + "\"project\":\"test-project\",\"pattern\":\"context-needle\"," + "\"format\":\"json\",\"context\":1,\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *cols = yyjson_obj_get(root, "cols"); + yyjson_val *rows = yyjson_obj_get(root, "rows"); + ASSERT_NOT_NULL(cols); + ASSERT_NOT_NULL(rows); + ASSERT_TRUE(yyjson_arr_size(rows) > 0); + + size_t context_index = SIZE_MAX; + size_t col_count = yyjson_arr_size(cols); + for (size_t i = 0; i < col_count; i++) { + const char *col = yyjson_get_str(yyjson_arr_get(cols, i)); + if (col && strcmp(col, "context") == 0) { + context_index = i; + break; + } + } + ASSERT_TRUE(context_index != SIZE_MAX); + yyjson_val *context_obj = yyjson_arr_get(yyjson_arr_get(rows, 0), context_index); + ASSERT_NOT_NULL(context_obj); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(context_obj, "context")), context_source); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(context_obj, "context_start")), 1); + yyjson_doc_free(doc); + + free(inner); + free(resp); + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + +TEST(search_code_invalid_utf8_still_returns_valid_json) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + + char invalid_path[512]; + snprintf(invalid_path, sizeof(invalid_path), "%s/project/invalid.md", tmp); + static const unsigned char invalid_source[] = { + 'i', 'n', 'v', 'a', 'l', 'i', 'd', '-', 'n', 'e', 'e', 'd', 'l', + 'e', '\n', 'c', 'o', 'n', 't', 'e', 'x', 't', ' ', 0xFF, '\n', + }; + FILE *fp = cbm_fopen(invalid_path, "wb"); + ASSERT_NOT_NULL(fp); + ASSERT_EQ(fwrite(invalid_source, 1, sizeof(invalid_source), fp), sizeof(invalid_source)); + ASSERT_EQ(fclose(fp), 0); + + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + cbm_node_t node = {.project = "test-project", + .label = "Section", + .name = "invalid", + .qualified_name = "test-project.invalid", + .file_path = "invalid.md", + .start_line = 1, + .end_line = 2}; + ASSERT_GT(cbm_store_upsert_node(st, &node), 0); + + char *resp = + cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":100,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_code\",\"arguments\":{" + "\"project\":\"test-project\",\"pattern\":\"invalid-needle\"," + "\"mode\":\"full\",\"format\":\"json\",\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + ASSERT_TRUE(is_valid_json_response(inner)); + yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *rows = yyjson_obj_get(yyjson_doc_get_root(doc), "rows"); + ASSERT_NOT_NULL(rows); + ASSERT_TRUE(yyjson_arr_size(rows) > 0); + yyjson_val *source_obj = yyjson_arr_get(yyjson_arr_get(rows, 0), 7); + ASSERT_NOT_NULL(source_obj); + const char *safe_source = yyjson_get_str(yyjson_obj_get(source_obj, "source")); + ASSERT_NOT_NULL(safe_source); + ASSERT_NOT_NULL(strstr(safe_source, "\xEF\xBF\xBD")); + ASSERT_NULL(memchr(safe_source, 0xFF, strlen(safe_source))); + yyjson_doc_free(doc); + + free(inner); + free(resp); + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + /* Reproduce-first (#687): scoped content search over a repo whose ROOT PATH * contains a space. write_scoped_filelist emits "/" records that the * Unix pipeline pipes to grep via xargs. With plain `xargs` (newline-split) the @@ -4672,6 +5047,32 @@ TEST(search_code_windows_output_limit_fails_closed_and_cleans_scan) { PASS(); #else SKIP_PLATFORM("supervised Select-String output limit runs on Windows"); +/* Windows raw scans must pin the PowerShell pipe to UTF-8: PS 5.1 otherwise + * emits stdout in the console OEM codepage and non-ASCII content degrades to + * '?' depending on the inherited console CP (seen as test_mcp raw-Русский + * mojibake on CI). Every builder variant must carry the prelude. */ +TEST(search_code_windows_scan_pins_utf8_output) { +#ifdef _WIN32 + static const char prelude[] = + "powershell -Command \"[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; "; + char command[CBM_SZ_4K]; + struct { + bool scoped; + const char *file_pattern; + } cases[] = {{true, "*.go"}, /* scoped + prefilterable pattern */ + {true, "*x*z*.go"}, /* scoped + non-prefilterable pattern */ + {true, NULL}, /* scoped, no pattern */ + {false, "*.go"}, /* unscoped + pattern */ + {false, NULL}}; /* unscoped, no pattern */ + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + cbm_search_code_build_grep_cmd(command, sizeof(command), false, cases[i].scoped, + cases[i].file_pattern, "C:/tmp/pattern", "C:/tmp/filelist", + "C:/tmp/root"); + ASSERT_TRUE(strncmp(command, prelude, sizeof(prelude) - 1) == 0); + } + PASS(); +#else + SKIP_PLATFORM("PowerShell scan encoding applies on Windows"); #endif } @@ -7428,7 +7829,8 @@ static cbm_mcp_server_t *setup_snippet_server(char *tmp_dir, size_t tmp_sz) { n_hr.end_line = 5; n_hr.properties_json = "{\"signature\":\"func HandleRequest() error\"," "\"return_type\":\"error\"," - "\"is_exported\":true}"; + "\"is_exported\":true," + "\"base_classes\":[\"HandlerBase\",\"Audited\"]}"; int64_t id_hr = cbm_store_upsert_node(st, &n_hr); cbm_node_t n_po = {0}; @@ -8522,6 +8924,38 @@ TEST(tool_resolve_store_by_internal_name_issue704) { ASSERT_NULL(strstr(list, "ghost704")); /* 0-byte ghost filtered (RED before) */ free(list); + char *page = + cbm_mcp_server_handle(srv, "{\"jsonrpc\":\"2.0\",\"id\":11,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"list_projects\"," + "\"arguments\":{\"offset\":0,\"limit\":1}}}"); + ASSERT_NOT_NULL(page); + ASSERT_NOT_NULL(strstr(page, "\\\"total\\\":3")); + ASSERT_NOT_NULL(strstr(page, "\\\"limit\\\":1")); + ASSERT_NOT_NULL(strstr(page, "\\\"returned\\\":1")); + ASSERT_NOT_NULL(strstr(page, "\\\"has_more\\\":true")); + ASSERT_NOT_NULL(strstr(page, "alpha704")); + ASSERT_NULL(strstr(page, "beta704")); + ASSERT_NULL(strstr(page, "\\\"nodes\\\"")); /* details are opt-in */ + free(page); + + char *next_page = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":12,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"list_projects\"," + "\"arguments\":{\"offset\":1,\"limit\":1,\"metadata_only\":true}}}"); + ASSERT_NOT_NULL(next_page); + ASSERT_NOT_NULL(strstr(next_page, "beta704")); + ASSERT_NULL(strstr(next_page, "alpha704")); + ASSERT_NULL(strstr(next_page, "\\\"nodes\\\"")); /* compatibility alias stays lean */ + free(next_page); + + char *details = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":13,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"list_projects\"," + "\"arguments\":{\"offset\":0,\"limit\":1,\"include_details\":true}}}"); + ASSERT_NOT_NULL(details); + ASSERT_NOT_NULL(strstr(details, "\\\"nodes\\\"")); + free(details); + /* ── B: the drifted project resolves by its INTERNAL name ──────── */ char *q_beta = cbm_mcp_server_handle( srv, "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/call\"," @@ -10876,7 +11310,9 @@ SUITE(mcp) { RUN_TEST(tool_get_graph_schema_empty); RUN_TEST(tool_unknown_tool); RUN_TEST(tool_search_graph_basic); + RUN_TEST(tool_search_graph_semantic_only_skips_structural_results_issue1295); RUN_TEST(tool_trace_totals_respect_test_filter); + RUN_TEST(tool_trace_totals_respect_test_filter_tests_root_subtree_issue1294); RUN_TEST(tool_get_architecture_cycles_detects_scc); RUN_TEST(tool_get_code_snippet_clips_whole_file_node); RUN_TEST(tool_search_graph_includes_node_properties); @@ -10930,6 +11366,10 @@ SUITE(mcp) { RUN_TEST(tool_search_code_limit_declares_a_minimum_issue1511); RUN_TEST(tool_search_code_no_project); RUN_TEST(search_code_multi_word); + RUN_TEST(search_code_full_preserves_utf8_source); + RUN_TEST(search_code_raw_match_preserves_utf8_content); + RUN_TEST(search_code_context_preserves_utf8_context); + RUN_TEST(search_code_invalid_utf8_still_returns_valid_json); RUN_TEST(search_code_scoped_path_with_spaces_issue687); #ifdef _WIN32 RUN_TEST(search_code_scoped_path_with_cjk_root_issue903); @@ -10940,6 +11380,7 @@ SUITE(mcp) { RUN_TEST(search_code_windows_prefilter_precedes_content_scan); RUN_TEST(search_code_windows_cancel_cleans_supervised_scan); RUN_TEST(search_code_windows_output_limit_fails_closed_and_cleans_scan); + RUN_TEST(search_code_windows_scan_pins_utf8_output); RUN_TEST(search_code_invalid_regex_errors_issue283); RUN_TEST(search_code_literal_pipe_warns_issue282); RUN_TEST(search_code_reports_phase_timings_only_in_debug_mode); diff --git a/tests/test_parent_watchdog.sh b/tests/test_parent_watchdog.sh index 87f938d6e..512161a36 100755 --- a/tests/test_parent_watchdog.sh +++ b/tests/test_parent_watchdog.sh @@ -25,7 +25,10 @@ if [[ ! -x "${BINARY}" ]]; then exit 2 fi -tmpdir="$(mktemp -d)" +# shellcheck source=../scripts/test-runtime.sh +source "${ROOT}/scripts/test-runtime.sh" +cbm_test_runtime_init +tmpdir="${CBM_TEST_RUNTIME_ROOT}" wrapper_pid="" cleanup() { if [[ -s "${tmpdir}/child.pid" ]]; then @@ -34,7 +37,7 @@ cleanup() { [[ -n "${child_pid}" ]] && kill "${child_pid}" 2>/dev/null || true fi [[ -n "${wrapper_pid}" ]] && kill "${wrapper_pid}" 2>/dev/null || true - rm -rf "${tmpdir}" + cbm_test_runtime_cleanup "${BINARY}" } trap cleanup EXIT diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 61d920430..25878a8b3 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -17,6 +17,7 @@ #include "foundation/dump_verify.h" #include "foundation/sha256.h" #include "foundation/compat_fs.h" +#include "foundation/log.h" #include "foundation/win_utf8.h" // cbm_utf8_to_wide (Windows pipeline_test_set_mtime); no-op elsewhere #include "discover/userconfig.h" @@ -3314,6 +3315,116 @@ TEST(pipeline_exact_inputs_migrate_coverage_metadata_and_index_mode) { PASS(); } +typedef struct { + bool published; + int rename_calls; + int export_count; + int exports_before_publish; +} artifact_publish_observer_t; + +static artifact_publish_observer_t *g_artifact_publish_observer; + +static void observe_artifact_publish_log(const char *line) { + if (g_artifact_publish_observer && line && strstr(line, "msg=artifact.export")) { + g_artifact_publish_observer->export_count++; + if (!g_artifact_publish_observer->published) { + g_artifact_publish_observer->exports_before_publish++; + } + } +} + +static int observe_successful_publish_rename(const char *staging_path, const char *final_path, + void *arg) { + artifact_publish_observer_t *observer = (artifact_publish_observer_t *)arg; + observer->rename_calls++; + int rc = cbm_rename_replace(staging_path, final_path); + observer->published = rc == 0; + return rc; +} + +static int run_observing_artifact_publish(cbm_pipeline_t *pipeline, + artifact_publish_observer_t *observer) { + cbm_pipeline_set_rename_hook_for_tests(pipeline, observe_successful_publish_rename, observer); + CBMLogLevel previous_level = cbm_log_get_level(); + CBMLogFormat previous_format = cbm_log_get_format(); + g_artifact_publish_observer = observer; + cbm_log_set_level(CBM_LOG_INFO); + cbm_log_set_format(CBM_LOG_FORMAT_TEXT); + cbm_log_set_sink_ex(observe_artifact_publish_log, CBM_LOG_SINK_REPLACE); + int rc = cbm_pipeline_run(pipeline); + cbm_log_set_sink(NULL); + cbm_log_set_format(previous_format); + cbm_log_set_level(previous_level); + g_artifact_publish_observer = NULL; + return rc; +} + +static bool pipeline_reports_excluded_dir(cbm_pipeline_t *pipeline, const char *rel_path) { + char **excluded = NULL; + int excluded_count = 0; + cbm_pipeline_get_excluded(pipeline, &excluded, &excluded_count); + for (int i = 0; i < excluded_count; i++) { + if (excluded[i] && strcmp(excluded[i], rel_path) == 0) { + return true; + } + } + return false; +} + +typedef struct { + int rc; + cbm_incremental_route_t route; + bool tools_excluded; + artifact_publish_observer_t publish; +} observed_fast_run_t; + +static observed_fast_run_t run_observed_fast_pipeline(const char *repo_path, const char *db_path) { + observed_fast_run_t result = {.rc = CBM_NOT_FOUND}; + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo_path, db_path, CBM_MODE_FAST); + if (!pipeline) { + return result; + } + result.rc = run_observing_artifact_publish(pipeline, &result.publish); + result.route = cbm_pipeline_incremental_test_last_route(); + result.tools_excluded = pipeline_reports_excluded_dir(pipeline, "tools"); + cbm_pipeline_free(pipeline); + return result; +} + +typedef struct { + int rc; + int before_nodes; + int after_nodes; +} imported_generation_t; + +static imported_generation_t import_artifact_generation(const char *repo_path, + const char *import_path, + const char *project) { + imported_generation_t result = { + .rc = cbm_artifact_import(repo_path, import_path), + .before_nodes = -1, + .after_nodes = -1, + }; + if (result.rc == 0) { + observe_named_generation(import_path, project, "StoredBefore", "StoredAfter", + &result.before_nodes, &result.after_nodes); + } + return result; +} + +static bool stored_mode_is_full(const char *db_path, const char *project) { + cbm_store_t *store = cbm_store_open_path(db_path); + if (!store) { + return false; + } + cbm_coverage_meta_t meta = {0}; + bool full = cbm_store_coverage_meta_get(store, project, &meta) == CBM_STORE_OK && + meta.index_mode && strcmp(meta.index_mode, "full") == 0; + cbm_store_coverage_meta_clear(&meta); + cbm_store_close(store); + return full; +} + /* Once a repository contains a shared artifact, every subsequently published * full generation must refresh it, even when persistence was not explicitly * requested on that invocation. Otherwise the live DB advances while a clean @@ -3331,10 +3442,11 @@ TEST(pipeline_existing_artifact_refreshes_after_default_forced_full_reindex) { ASSERT_EQ(th_write_file(source_path, "def ArtifactGenerationBefore():\n return 1\n"), 0); cbm_pipeline_incremental_test_reset_faults(); + artifact_publish_observer_t baseline_observer = {0}; cbm_pipeline_t *baseline = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); ASSERT_NOT_NULL(baseline); cbm_pipeline_set_persistence(baseline, true); - ASSERT_EQ(cbm_pipeline_run(baseline), 0); + int baseline_rc = run_observing_artifact_publish(baseline, &baseline_observer); char project[256]; snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(baseline)); cbm_pipeline_free(baseline); @@ -3342,18 +3454,29 @@ TEST(pipeline_existing_artifact_refreshes_after_default_forced_full_reindex) { ASSERT_EQ(th_write_file(source_path, "def ArtifactGenerationAfter():\n return 2\n"), 0); cbm_pipeline_incremental_test_reset_faults(); + artifact_publish_observer_t reindex_observer = {0}; cbm_pipeline_t *default_reindex = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); ASSERT_NOT_NULL(default_reindex); - int reindex_rc = cbm_pipeline_run(default_reindex); + int reindex_rc = run_observing_artifact_publish(default_reindex, &reindex_observer); cbm_incremental_route_t reindex_route = cbm_pipeline_incremental_test_last_route(); cbm_pipeline_free(default_reindex); /* A derived artifact must not become an input that forces another rebuild, * and refreshing it must not switch the authoritative DB back to WAL. */ cbm_pipeline_incremental_test_reset_faults(); + artifact_publish_observer_t explicit_observer = {0}; + cbm_pipeline_t *explicit_noop = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(explicit_noop); + cbm_pipeline_set_persistence(explicit_noop, true); + int explicit_rc = run_observing_artifact_publish(explicit_noop, &explicit_observer); + cbm_incremental_route_t explicit_route = cbm_pipeline_incremental_test_last_route(); + cbm_pipeline_free(explicit_noop); + + cbm_pipeline_incremental_test_reset_faults(); + artifact_publish_observer_t observer = {0}; cbm_pipeline_t *unchanged = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); ASSERT_NOT_NULL(unchanged); - int unchanged_rc = cbm_pipeline_run(unchanged); + int unchanged_rc = run_observing_artifact_publish(unchanged, &observer); cbm_incremental_route_t unchanged_route = cbm_pipeline_incremental_test_last_route(); cbm_pipeline_free(unchanged); @@ -3397,10 +3520,25 @@ TEST(pipeline_existing_artifact_refreshes_after_default_forced_full_reindex) { cbm_pipeline_incremental_test_reset_faults(); th_rmtree(tmp); + ASSERT_EQ(baseline_rc, 0); + ASSERT_EQ(baseline_observer.rename_calls, 1); + ASSERT_EQ(baseline_observer.exports_before_publish, 0); + ASSERT_EQ(baseline_observer.export_count, 1); ASSERT_EQ(reindex_rc, 0); ASSERT_EQ(reindex_route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_EQ(reindex_observer.rename_calls, 1); + ASSERT_EQ(reindex_observer.exports_before_publish, 0); + ASSERT_EQ(reindex_observer.export_count, 1); + ASSERT_EQ(explicit_rc, 0); + ASSERT_EQ(explicit_route, CBM_INCREMENTAL_ROUTE_NOOP); + ASSERT_EQ(explicit_observer.rename_calls, 1); + ASSERT_EQ(explicit_observer.exports_before_publish, 0); + ASSERT_EQ(explicit_observer.export_count, 1); ASSERT_EQ(unchanged_rc, 0); ASSERT_EQ(unchanged_route, CBM_INCREMENTAL_ROUTE_NOOP); + ASSERT_EQ(observer.rename_calls, 1); + ASSERT_EQ(observer.exports_before_publish, 0); + ASSERT_EQ(observer.export_count, 1); ASSERT_TRUE(journal_ok); ASSERT_STR_EQ(journal_mode, "delete"); ASSERT_EQ(live_before, 0); @@ -5893,6 +6031,91 @@ TEST(pipeline_python_cross_module_call) { PASS(); } +/* #725: two same-named symbols across languages must not share CALLS edges. + * Python Store.commit is the real callee of save(); the JS Editor.commit + * function is a distinct binding and must have no inbound CALLS from Python. + * unique_name (candidates==1) is #1572 and is not this claim. */ +TEST(pipeline_cross_language_same_name_does_not_share_calls_issue725) { + const char *files[] = {"store.py", "app.py", "web/src/pages/Editor.js"}; + const char *contents[] = { + "class Store:\n" + " def commit(self):\n" + " return True\n", + + "from store import Store\n" + "\n" + "def save():\n" + " return Store().commit()\n", + + "export function commit() {\n" + " return 1;\n" + "}\n"}; + + if (setup_lang_repo(files, contents, 3) != 0) + FAIL("tmpdir"); + char db[512]; + snprintf(db, sizeof(db), "%s/test.db", g_lang_tmpdir); + + cbm_pipeline_t *p = cbm_pipeline_new(g_lang_tmpdir, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + + cbm_store_t *s = cbm_store_open_path(db); + ASSERT_NOT_NULL(s); + const char *proj = cbm_pipeline_project_name(p); + + cbm_node_t *commits = NULL; + int ncommit = 0; + cbm_store_find_nodes_by_name(s, proj, "commit", &commits, &ncommit); + ASSERT_GTE(ncommit, 2); + + int64_t js_id = 0; + int64_t py_id = 0; + for (int i = 0; i < ncommit; i++) { + if (commits[i].file_path && strstr(commits[i].file_path, "Editor.js")) + js_id = commits[i].id; + if (commits[i].file_path && strstr(commits[i].file_path, "store.py")) + py_id = commits[i].id; + } + ASSERT_TRUE(js_id != 0); + ASSERT_TRUE(py_id != 0); + + cbm_node_t *saves = NULL; + int nsave = 0; + cbm_store_find_nodes_by_name(s, proj, "save", &saves, &nsave); + ASSERT_GT(nsave, 0); + + cbm_edge_t *from_save = NULL; + int nfrom = 0; + cbm_store_find_edges_by_source_type(s, saves[0].id, "CALLS", &from_save, &nfrom); + bool save_calls_py = false; + bool save_calls_js = false; + for (int i = 0; i < nfrom; i++) { + if (from_save[i].target_id == py_id) + save_calls_py = true; + if (from_save[i].target_id == js_id) + save_calls_js = true; + } + ASSERT_TRUE(save_calls_py); + ASSERT_FALSE(save_calls_js); + + cbm_edge_t *into_js = NULL; + int njs = 0; + cbm_store_find_edges_by_target_type(s, js_id, "CALLS", &into_js, &njs); + ASSERT_EQ(njs, 0); + + if (from_save) + cbm_store_free_edges(from_save, nfrom); + if (into_js) + cbm_store_free_edges(into_js, njs); + cbm_store_free_nodes(commits, ncommit); + cbm_store_free_nodes(saves, nsave); + cbm_store_close(s); + cbm_pipeline_free(p); + teardown_lang_repo(); + PASS(); +} + TEST(pipeline_go_type_classification) { /* Port of TestGoTypeClassification */ const char *files[] = {"types.go"}; @@ -10411,6 +10634,135 @@ TEST(full_reindex_preserves_exact_long_db_path) { } #endif +TEST(incremental_downgrade_preserves_scope_and_artifact_across_change_noop_delete) { + char tmpdir[256]; + char artifact_tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_mode_scope_XXXXXX"); + snprintf(artifact_tmpdir, sizeof(artifact_tmpdir), "/tmp/cbm_mode_artifact_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmpdir)); + ASSERT_NOT_NULL(cbm_mkdtemp(artifact_tmpdir)); + + char dbpath[512]; + char cancelled_import_path[512]; + char retry_import_path[512]; + char deleted_import_path[512]; + snprintf(dbpath, sizeof(dbpath), "%s/test.db", tmpdir); + snprintf(cancelled_import_path, sizeof(cancelled_import_path), "%s/cancelled.db", + artifact_tmpdir); + snprintf(retry_import_path, sizeof(retry_import_path), "%s/retry.db", artifact_tmpdir); + snprintf(deleted_import_path, sizeof(deleted_import_path), "%s/deleted.db", artifact_tmpdir); + ASSERT_EQ(th_write_file(TH_PATH(tmpdir, "main.go"), "package main\n\nfunc main() {}\n"), 0); + ASSERT_TRUE(cbm_mkdir_p(TH_PATH(tmpdir, "tools"), 0755)); + ASSERT_EQ(th_write_file(TH_PATH(tmpdir, "tools/util.go"), + "package tools\n\nfunc StoredBefore() string { return \"old\" }\n"), + 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + cbm_pipeline_set_persistence(pipeline, true); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + ASSERT_NOT_NULL(project); + cbm_pipeline_free(pipeline); + ASSERT_TRUE(cbm_artifact_exists(tmpdir)); + + ASSERT_EQ(th_write_file(TH_PATH(tmpdir, "tools/util.go"), + "package tools\n\nfunc StoredAfter() string { return \"new\" }\n"), + 0); + + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_incremental_test_cancel_after_predump_once(); + observed_fast_run_t cancelled = run_observed_fast_pipeline(tmpdir, dbpath); + int cancelled_live_before; + int cancelled_live_after; + observe_named_generation(dbpath, project, "StoredBefore", "StoredAfter", &cancelled_live_before, + &cancelled_live_after); + imported_generation_t cancelled_artifact = + import_artifact_generation(tmpdir, cancelled_import_path, project); + + cbm_pipeline_incremental_test_reset_faults(); + observed_fast_run_t retry = run_observed_fast_pipeline(tmpdir, dbpath); + int retry_live_before; + int retry_live_after; + observe_named_generation(dbpath, project, "StoredBefore", "StoredAfter", &retry_live_before, + &retry_live_after); + bool retry_full_mode = stored_mode_is_full(dbpath, project); + imported_generation_t retry_artifact = + import_artifact_generation(tmpdir, retry_import_path, project); + + cbm_pipeline_incremental_test_reset_faults(); + observed_fast_run_t noop = run_observed_fast_pipeline(tmpdir, dbpath); + + ASSERT_EQ(cbm_unlink(TH_PATH(tmpdir, "tools/util.go")), 0); + cbm_pipeline_incremental_test_reset_faults(); + observed_fast_run_t deleted = run_observed_fast_pipeline(tmpdir, dbpath); + int deleted_live_before; + int deleted_live_after; + observe_named_generation(dbpath, project, "StoredBefore", "StoredAfter", &deleted_live_before, + &deleted_live_after); + bool delete_full_mode = stored_mode_is_full(dbpath, project); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + cbm_file_hash_t deleted_hash = {0}; + int deleted_hash_rc = cbm_store_get_file_hash(store, project, "tools/util.go", &deleted_hash); + cbm_store_clear_file_hash(&deleted_hash); + cbm_store_close(store); + imported_generation_t deleted_artifact = + import_artifact_generation(tmpdir, deleted_import_path, project); + + cbm_pipeline_incremental_test_reset_faults(); + free(project); + th_rmtree(artifact_tmpdir); + th_rmtree(tmpdir); + + ASSERT_EQ(cancelled.rc, CBM_PIPELINE_ABORT_PRESERVE_DB); + ASSERT_EQ(cancelled.route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_TRUE(cancelled.tools_excluded); + ASSERT_EQ(cancelled_live_before, 1); + ASSERT_EQ(cancelled_live_after, 0); + ASSERT_EQ(cancelled.publish.rename_calls, 0); + ASSERT_EQ(cancelled.publish.export_count, 0); + ASSERT_EQ(cancelled_artifact.rc, 0); + ASSERT_EQ(cancelled_artifact.before_nodes, 1); + ASSERT_EQ(cancelled_artifact.after_nodes, 0); + + ASSERT_EQ(retry.rc, 0); + ASSERT_EQ(retry.route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_TRUE(retry.tools_excluded); + ASSERT_TRUE(retry_full_mode); + ASSERT_EQ(retry_live_before, 0); + ASSERT_EQ(retry_live_after, 1); + ASSERT_EQ(retry.publish.rename_calls, 1); + ASSERT_EQ(retry.publish.exports_before_publish, 0); + ASSERT_EQ(retry.publish.export_count, 1); + ASSERT_EQ(retry_artifact.rc, 0); + ASSERT_EQ(retry_artifact.before_nodes, 0); + ASSERT_EQ(retry_artifact.after_nodes, 1); + + ASSERT_EQ(noop.rc, 0); + ASSERT_EQ(noop.route, CBM_INCREMENTAL_ROUTE_NOOP); + ASSERT_TRUE(noop.tools_excluded); + ASSERT_EQ(noop.publish.rename_calls, 1); + ASSERT_EQ(noop.publish.exports_before_publish, 0); + ASSERT_EQ(noop.publish.export_count, 1); + + ASSERT_EQ(deleted.rc, 0); + ASSERT_EQ(deleted.route, CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_TRUE(deleted.tools_excluded); + ASSERT_EQ(deleted.publish.rename_calls, 1); + ASSERT_EQ(deleted.publish.exports_before_publish, 0); + ASSERT_EQ(deleted.publish.export_count, 1); + ASSERT_EQ(deleted_hash_rc, CBM_STORE_NOT_FOUND); + ASSERT_TRUE(delete_full_mode); + ASSERT_EQ(deleted_live_before, 0); + ASSERT_EQ(deleted_live_after, 0); + ASSERT_EQ(deleted_artifact.rc, 0); + ASSERT_EQ(deleted_artifact.before_nodes, 0); + ASSERT_EQ(deleted_artifact.after_nodes, 0); + PASS(); +} + TEST(incremental_fast_preserves_mode_skipped_tools_dir) { /* Regression: 2026-04-13. A fast-mode reindex after a full-mode index * was silently destroying every file under FAST_SKIP_DIRS directories @@ -11668,6 +12020,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_go_cross_package_call); RUN_TEST(pipeline_swift_cross_package_import); RUN_TEST(pipeline_python_cross_module_call); + RUN_TEST(pipeline_cross_language_same_name_does_not_share_calls_issue725); RUN_TEST(pipeline_go_type_classification); RUN_TEST(pipeline_go_grouped_types); RUN_TEST(pipeline_kotlin_project); @@ -11901,6 +12254,7 @@ SUITE(pipeline) { * broad pipeline suite so RED/GREEN iterations exercise only this boundary; * the default all-suite run still executes it. */ SUITE(pipeline_semantic_manifest_repro) { + RUN_TEST(incremental_downgrade_preserves_scope_and_artifact_across_change_noop_delete); RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge); RUN_TEST(pipeline_parallel_manifest_is_byte_stable_above_threshold); RUN_TEST(pipeline_closure_repair_body_edit_converges_with_fresh_full); diff --git a/tests/test_registry.c b/tests/test_registry.c index 725a6f935..387bdc143 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -761,6 +761,34 @@ TEST(perl_suppress_keeps_high_confidence_and_genuine_calls) { PASS(); } +TEST(cross_language_suffix_match_drops_py_vs_js) { + /* #725: two same-named symbols in different languages. suffix_match is the + * strategy that collapses them; unique_name is #1572 and must stay. */ + ASSERT_TRUE(cbm_suppress_cross_language_suffix_match(CBM_LANG_PYTHON, "web/src/pages/Editor.js", + "suffix_match")); + ASSERT_TRUE(cbm_suppress_cross_language_suffix_match(CBM_LANG_JAVASCRIPT, "store.py", + "suffix_match")); + ASSERT_TRUE(cbm_suppress_cross_language_suffix_match(CBM_LANG_BASH, "cli/main.py", + "suffix_match")); + ASSERT_FALSE(cbm_suppress_cross_language_suffix_match(CBM_LANG_PYTHON, "store.py", + "suffix_match")); + ASSERT_FALSE(cbm_suppress_cross_language_suffix_match(CBM_LANG_PYTHON, "web/src/pages/Editor.js", + "unique_name")); + ASSERT_FALSE(cbm_suppress_cross_language_suffix_match(CBM_LANG_PYTHON, "web/src/pages/Editor.js", + "same_module")); + ASSERT_FALSE(cbm_suppress_cross_language_suffix_match(CBM_LANG_PYTHON, "web/src/pages/Editor.js", + "import_map")); + /* JS/TS/TSX are one family. */ + ASSERT_FALSE(cbm_suppress_cross_language_suffix_match(CBM_LANG_JAVASCRIPT, "lib/util.ts", + "suffix_match")); + ASSERT_FALSE(cbm_suppress_cross_language_suffix_match(CBM_LANG_TYPESCRIPT, "ui/Panel.tsx", + "suffix_match")); + ASSERT_FALSE(cbm_suppress_cross_language_suffix_match(CBM_LANG_PYTHON, NULL, "suffix_match")); + ASSERT_FALSE(cbm_suppress_cross_language_suffix_match(CBM_LANG_COUNT, "store.py", + "suffix_match")); + PASS(); +} + TEST(tsjs_suppress_drops_weak_method_matches) { /* #592/#606: a TS/JS member call whose receiver the LSP could not type, that * landed via a WEAK short-name strategy, is generic-resolver noise → drop. @@ -893,6 +921,7 @@ SUITE(registry) { RUN_TEST(perl_builtin_set_rejects_project_subs); RUN_TEST(perl_suppress_drops_weak_builtin_and_method_matches); RUN_TEST(perl_suppress_keeps_high_confidence_and_genuine_calls); + RUN_TEST(cross_language_suffix_match_drops_py_vs_js); RUN_TEST(tsjs_suppress_drops_weak_method_matches); RUN_TEST(tsjs_suppress_keeps_high_confidence_and_non_methods); } diff --git a/tests/test_runtime_isolation_contract.sh b/tests/test_runtime_isolation_contract.sh new file mode 100644 index 000000000..217230c7f --- /dev/null +++ b/tests/test_runtime_isolation_contract.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT +fail() { + echo "FAIL: $*" >&2 + exit 1 +} +normalize_path() { + local path="${1%$'\r'}" + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$path" 2>/dev/null && return 0 + fi + printf '%s\n' "${path//\\//}" +} +INSTALL_FIXTURE="$WORKDIR/install-fixture" +cat > "$INSTALL_FIXTURE" <<'EOF' +#!/usr/bin/env bash +[[ "$1 $2" == "daemon status" ]] && exit 1 +if [[ "$1 $2" == "install -y" ]]; then + printf '%s\t%s\t%s\n' "$HOME" "$CBM_CACHE_DIR" "$CBM_RUNTIME_DIR" > "$CBM_TEST_INSTALL_ENV_PROBE" +fi +exit 0 +EOF +chmod +x "$INSTALL_FIXTURE" +CONTROL="$INSTALL_FIXTURE" +CALLER_ROOT="$WORKDIR/caller-a" +mkdir -p "$CALLER_ROOT/runtime" "$CALLER_ROOT/cache" +touch "$CALLER_ROOT/sentinel" +( + export CBM_RUNTIME_DIR="$CALLER_ROOT/runtime" + export CBM_CACHE_DIR="$CALLER_ROOT/cache" + source "$ROOT/scripts/test-runtime.sh" + cbm_test_runtime_init + [[ "$CBM_RUNTIME_DIR" != "$CALLER_ROOT/runtime" ]] || fail "inherited runtime was reused" + [[ "$CBM_CACHE_DIR" != "$CALLER_ROOT/cache" ]] || fail "inherited cache was reused" + private_root="$CBM_TEST_RUNTIME_ROOT" + [[ -d "$private_root/runtime" && -d "$private_root/cache" ]] || fail "private directories missing" + cbm_test_runtime_cleanup "$CONTROL" + [[ ! -e "$private_root" ]] || fail "normal cleanup left its private root" +) +[[ -f "$CALLER_ROOT/sentinel" ]] || fail "cleanup touched the caller root" +for slot in 1 2; do + ( + source "$ROOT/scripts/test-runtime.sh" + cbm_test_runtime_init + printf '%s\n' "$CBM_TEST_RUNTIME_ROOT" > "$WORKDIR/root-$slot" + cbm_test_runtime_cleanup "$CONTROL" + ) & +done +wait +[[ "$(<"$WORKDIR/root-1")" != "$(<"$WORKDIR/root-2")" ]] || fail "parallel roots collided" +FAIL_FIXTURE="$WORKDIR/failure-fixture" +cat > "$FAIL_FIXTURE" <<'EOF' +#!/usr/bin/env bash +touch "$CBM_TEST_FAILURE_MARKER" +EOF +chmod +x "$FAIL_FIXTURE" +mkdir "$WORKDIR/windows-ci-root" +set +e +( + set -e + source "$ROOT/scripts/test-runtime.sh" + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) export CBM_CI_TEMP_ROOT="$(cygpath -m "$WORKDIR/windows-ci-root")" ;; + esac + mktemp() { return 1; } + cbm_test_runtime_init + CBM_TEST_FAILURE_MARKER="$WORKDIR/failure.marker" "$FAIL_FIXTURE" +) > "$WORKDIR/failure.out" 2>&1 +failure_rc=$? +set -e +[[ $failure_rc -ne 0 ]] || fail "simulated creation failure did not stop the harness" +[[ ! -e "$WORKDIR/failure.marker" ]] || fail "fixture ran after runtime creation failed" +mkdir "$WORKDIR/caller-home" "$WORKDIR/caller-cache" "$WORKDIR/caller-runtime" +HOME="$WORKDIR/caller-home" \ +CBM_CACHE_DIR="$WORKDIR/caller-cache" \ +CBM_RUNTIME_DIR="$WORKDIR/caller-runtime" \ +CBM_TEST_INSTALL_ENV_PROBE="$WORKDIR/install.env" \ + "$ROOT/scripts/security-install.sh" "$INSTALL_FIXTURE" > "$WORKDIR/install.out" 2>&1 +IFS=$'\t' read -r install_home install_cache install_runtime < "$WORKDIR/install.env" +install_home="$(normalize_path "$install_home")" +install_cache="$(normalize_path "$install_cache")" +install_runtime="$(normalize_path "$install_runtime")" +[[ "$install_home" != "$(normalize_path "$WORKDIR/caller-home")" ]] || fail "install reused caller HOME" +[[ "$install_cache" != "$(normalize_path "$WORKDIR/caller-cache")" ]] || fail "install reused caller cache" +[[ "$install_runtime" != "$(normalize_path "$WORKDIR/caller-runtime")" ]] || fail "install reused caller runtime" +install_root="${install_home%/*}" +[[ "$install_home" == "$install_root/home" && "$install_cache" == "$install_root/cache" && + "$install_runtime" == "$install_root/runtime" ]] || fail "install env did not share one private root" +ENTRY_POINTS=( + scripts/security-install.sh scripts/security-fuzz.sh scripts/security-fuzz-random.sh + scripts/security-network.sh tests/test_parent_watchdog.sh tests/test_worker_watchdog.sh + tests/test_worker_error_response.sh tests/test_hook_conflict_notice.sh +) +for entry in "${ENTRY_POINTS[@]}"; do + grep -q 'test-runtime.sh' "$ROOT/$entry" || fail "$entry does not source the helper" + grep -q 'cbm_test_runtime_init' "$ROOT/$entry" || fail "$entry does not initialize isolation" + grep -q 'cbm_test_runtime_cleanup' "$ROOT/$entry" || fail "$entry does not clean isolation" +done +echo "PASS: test harness runtimes are private, unique, fail-closed, and wired" diff --git a/tests/test_security_fuzz_harness.sh b/tests/test_security_fuzz_harness.sh index f3904a1eb..713107b1f 100644 --- a/tests/test_security_fuzz_harness.sh +++ b/tests/test_security_fuzz_harness.sh @@ -44,7 +44,7 @@ fi ENV_PROBE="$WORKDIR/environment-probe-mcp" cat > "$ENV_PROBE" <<'EOF' #!/usr/bin/env bash -printf '%s\t%s\n' "${HOME-}" "${CBM_CACHE_DIR-}" >> "$CBM_FUZZ_ENV_PROBE" +printf '%s\t%s\t%s\n' "${HOME-}" "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_FUZZ_ENV_PROBE" # Echo a JSON-RPC result for every request with a numeric id. This keeps the # fixture compatible with both the current fixed ids and a future per-case @@ -64,11 +64,13 @@ chmod +x "$ENV_PROBE" CALLER_HOME="$WORKDIR/caller-home" CALLER_CACHE="$WORKDIR/caller-cache" +CALLER_RUNTIME="$WORKDIR/caller-runtime" ENV_LOG="$WORKDIR/environment.log" -mkdir -p "$CALLER_HOME" "$CALLER_CACHE" +mkdir -p "$CALLER_HOME" "$CALLER_CACHE" "$CALLER_RUNTIME" if ! HOME="$CALLER_HOME" \ CBM_CACHE_DIR="$CALLER_CACHE" \ + CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ CBM_FUZZ_ENV_PROBE="$ENV_LOG" \ "$ROOT/scripts/security-fuzz.sh" "$ENV_PROBE" \ > "$WORKDIR/environment.out" 2>&1; then @@ -92,10 +94,12 @@ normalize_path() { CALLER_HOME_NORMALIZED=$(normalize_path "$CALLER_HOME") CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE") +CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME") -while IFS=$'\t' read -r child_home_raw child_cache_raw; do +while IFS=$'\t' read -r child_home_raw child_cache_raw child_runtime_raw; do child_home=$(normalize_path "$child_home_raw") child_cache=$(normalize_path "$child_cache_raw") + child_runtime=$(normalize_path "$child_runtime_raw") if [[ -z "$child_home" || "$child_home" == "$CALLER_HOME_NORMALIZED" ]]; then echo "FAIL: security-fuzz exposed the caller HOME to a fuzz target" exit 1 @@ -104,14 +108,21 @@ while IFS=$'\t' read -r child_home_raw child_cache_raw; do echo "FAIL: security-fuzz exposed the caller CBM_CACHE_DIR to a fuzz target" exit 1 fi + if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then + echo "FAIL: security-fuzz exposed the caller CBM_RUNTIME_DIR to a fuzz target" + exit 1 + fi child_home_parent=${child_home%/*} child_cache_parent=${child_cache%/*} + child_runtime_parent=${child_runtime%/*} if [[ "$child_home_parent" != "$child_cache_parent" || + "$child_home_parent" != "$child_runtime_parent" || "${child_home##*/}" != "home" || - "${child_cache##*/}" != "cache" ]]; then - echo "FAIL: fuzz HOME/cache were not isolated beneath one harness temp directory" + "${child_cache##*/}" != "cache" || + "${child_runtime##*/}" != "runtime" ]]; then + echo "FAIL: fuzz HOME/cache/runtime were not isolated beneath one harness temp directory" exit 1 fi done < "$ENV_LOG" -echo "PASS: security fuzz harness requires payload progress and isolates HOME/cache" +echo "PASS: security fuzz harness requires payload progress and isolates HOME/cache/runtime" diff --git a/tests/test_stack_overflow.c b/tests/test_stack_overflow.c index 6b2e8228c..9b646296c 100644 --- a/tests/test_stack_overflow.c +++ b/tests/test_stack_overflow.c @@ -16,6 +16,7 @@ #include #include #include +#include /* tree-sitter runtime allocator hooks (ts_runtime/src/alloc.h, TS_PUBLIC) and * mimalloc (vendored) — for the #424 allocator-binding regression test. */ diff --git a/tests/test_worker_error_response.sh b/tests/test_worker_error_response.sh index 5a28fb7da..33e2db115 100755 --- a/tests/test_worker_error_response.sh +++ b/tests/test_worker_error_response.sh @@ -35,11 +35,14 @@ if [[ ! "${BUILD_FINGERPRINT}" =~ ^[0-9a-f]{64}$ ]]; then exit 2 fi -tmpdir="$(mktemp -d)" +# shellcheck source=../scripts/test-runtime.sh +source "${ROOT}/scripts/test-runtime.sh" +cbm_test_runtime_init +tmpdir="${CBM_TEST_RUNTIME_ROOT}" cleanup() { CBM_CACHE_DIR="${tmpdir}/cache-supervisor" \ "${BINARY}" daemon stop >/dev/null 2>&1 || true - rm -rf "${tmpdir}" + cbm_test_runtime_cleanup "${BINARY}" } trap cleanup EXIT diff --git a/tests/test_worker_watchdog.sh b/tests/test_worker_watchdog.sh index 01ce360a6..7a52603a0 100755 --- a/tests/test_worker_watchdog.sh +++ b/tests/test_worker_watchdog.sh @@ -65,7 +65,10 @@ if [[ ! "${BUILD_FINGERPRINT}" =~ ^[0-9a-f]{64}$ ]]; then exit 2 fi -tmpdir="$(mktemp -d)" +# shellcheck source=../scripts/test-runtime.sh +source "${ROOT}/scripts/test-runtime.sh" +cbm_test_runtime_init +tmpdir="${CBM_TEST_RUNTIME_ROOT}" wrapper_pid="" cleanup() { if [[ -s "${tmpdir}/child.pid" ]]; then @@ -79,7 +82,7 @@ cleanup() { [[ -n "${descendant_pid}" ]] && kill -9 "${descendant_pid}" 2>/dev/null || true fi [[ -n "${wrapper_pid}" ]] && kill -9 "${wrapper_pid}" 2>/dev/null || true - rm -rf "${tmpdir}" + cbm_test_runtime_cleanup "${BINARY}" } trap cleanup EXIT