Merge remote-tracking branch 'origin/main' into distill/782-slab-aligned-pages

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel committed 2026-07-04 20:38:00 +02:00
commit c095a4a82e
9 files changed
+1311 -70

No files matched your search

+5 -1
View File
@@ -423,6 +423,10 @@ TEST_SECURITY_SRCS = tests/test_security.c
TEST_YAML_SRCS = tests/test_yaml.c
TEST_SEMANTIC_SRCS = tests/test_semantic.c
TEST_AST_PROFILE_SRCS = tests/test_ast_profile.c
TEST_SLAB_ALLOC_SRCS = tests/test_slab_alloc.c
TEST_SIMHASH_SRCS = tests/test_simhash.c
TEST_STACK_OVERFLOW_SRCS = tests/test_stack_overflow.c
@@ -487,7 +491,7 @@ TEST_REPRO_SRCS = \
tests/repro/repro_lsp_java_cs.c \
tests/repro/repro_lsp_kt_php_rust.c
ALL_TEST_SRCS =$(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_PHP_LSP_SRCS) $(TEST_CS_LSP_SRCS) $(TEST_CS_LSP_BENCH_SRCS) $(TEST_SCOPE_SRCS) $(TEST_TYPE_REP_SRCS) $(TEST_PY_LSP_SRCS) $(TEST_PY_LSP_BENCH_SRCS) $(TEST_PY_LSP_STRESS_SRCS) $(TEST_PY_LSP_SCALE_SRCS) $(TEST_TS_LSP_SRCS) $(TEST_JAVA_LSP_SRCS) $(TEST_KOTLIN_LSP_SRCS) $(TEST_RUST_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_HTTPD_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_STACK_OVERFLOW_SRCS) $(TEST_INTEGRATION_SRCS)
ALL_TEST_SRCS =$(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_PHP_LSP_SRCS) $(TEST_CS_LSP_SRCS) $(TEST_CS_LSP_BENCH_SRCS) $(TEST_SCOPE_SRCS) $(TEST_TYPE_REP_SRCS) $(TEST_PY_LSP_SRCS) $(TEST_PY_LSP_BENCH_SRCS) $(TEST_PY_LSP_STRESS_SRCS) $(TEST_PY_LSP_SCALE_SRCS) $(TEST_TS_LSP_SRCS) $(TEST_JAVA_LSP_SRCS) $(TEST_KOTLIN_LSP_SRCS) $(TEST_RUST_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_HTTPD_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SEMANTIC_SRCS) $(TEST_AST_PROFILE_SRCS) $(TEST_SLAB_ALLOC_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_STACK_OVERFLOW_SRCS) $(TEST_INTEGRATION_SRCS)
# ── Build directories ────────────────────────────────────────────
+232 -63
View File
@@ -40,15 +40,32 @@ enum {
#define PP_FIELD_HINT_CONF 0.85
enum { PP_CSHARP_M_PREFIX_LEN = 2 };
/* Source-retention caps for the parallel pipeline. The extract worker
* copies source bytes into result->arena so the fused cross-file LSP
* step in resolve_worker can run without re-reading from disk. Bound
* peak RSS with a per-file cap (skip retention for pathological huge
* generated files) and a total project-wide cap (skip when budget
* exhausted — cross-file LSP becomes a no-op for those late files,
* defs/calls already extracted are unaffected). */
#define PP_RETAIN_PER_FILE_MAX_BYTES (100LL * 1024 * 1024)
#define PP_RETAIN_TOTAL_BUDGET_BYTES (2LL * 1024 * 1024 * 1024)
/* Absolute source-retention ceilings for the parallel extract pipeline.
*
* The extract worker copies each file's source bytes into result->arena so
* the fused cross-file LSP step in resolve_worker can re-parse without
* re-opening the file. That retention is TRANSIENT (freed at run end) but it
* is a PEAK-RSS driver: every retained byte is resident at once across the
* extract→resolve handoff.
*
* A cap here is a FLOOR as much as a ceiling: whatever total we allow, we
* WILL hold that much resident at peak on a large repo. rust-analyzer bounds
* retained file *text* to a small fixed budget and re-reads source on a miss
* rather than scaling the retained set with host RAM — because the re-read is
* cheap relative to holding tens of GB resident. We follow the same model:
* - derive the total budget from the process memory budget (budget/8),
* - BUT clamp the RAM-derived DEFAULT to a small absolute ceiling (1 GiB)
* so a 512 GiB host does not retain 64 GiB of source it would re-read
* cheaply anyway, and keep the per-file cap modest (32 MiB) so one
* pathological generated blob cannot monopolise the budget.
* A file dropped from retention is NOT lost to cross-file resolution:
* resolve_worker re-reads it on demand, bounded and freed immediately
* (source_reread fallback). Both caps are env-overridable — CBM_RETAIN_TOTAL_MB
* / CBM_RETAIN_PER_FILE_MB raise or lower the auto-derived defaults directly
* (the hard ceilings bound only the RAM-derived default, never a deliberate
* operator/caller choice). */
#define PP_RETAIN_TOTAL_HARD_MAX_BYTES (1024ULL * 1024 * 1024) /* 1 GiB default ceiling */
#define PP_RETAIN_PER_FILE_HARD_MAX_BYTES (32ULL * 1024 * 1024) /* 32 MiB per file */
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */
@@ -73,6 +90,7 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
#include "simhash/minhash.h"
#include "semantic/ast_profile.h"
#include <errno.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
@@ -80,6 +98,95 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
#include <string.h>
#include <time.h>
/* Parse a positive MB-valued retention env knob (CBM_RETAIN_*_MB) into bytes.
* Follows the limits.c strtol convention: unset / unparseable / non-positive
* → return 0 so the caller keeps its derived default. */
static size_t cbm_retain_env_bytes(const char *name) {
const char *raw = getenv(name);
if (!raw || !raw[0]) {
return 0;
}
errno = 0;
char *end = NULL;
long v = strtol(raw, &end, 10);
if (errno != 0 || end == raw || *end != '\0' || v <= 0) {
return 0;
}
return (size_t)v * 1024 * 1024;
}
/* Auto-derived TOTAL retention budget: a fraction of the process memory
* budget, clamped to the absolute ceiling. The clamp bounds ONLY this
* RAM-derived default — a huge-RAM host must not retain tens of GB it would
* re-read cheaply. When the budget is unset (tests / no cgroup) fall back to
* the ceiling. */
static size_t cbm_parallel_extract_default_total_cap(void) {
size_t cap = cbm_mem_budget();
if (cap > 0) {
cap /= 8;
} else {
cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES;
}
if (cap > PP_RETAIN_TOTAL_HARD_MAX_BYTES) {
cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES;
}
return cap;
}
/* Auto-derived PER-FILE cap: modest absolute ceiling, never above the total. */
static size_t cbm_parallel_extract_default_per_file_cap(size_t total_cap) {
size_t cap = PP_RETAIN_PER_FILE_HARD_MAX_BYTES;
if (cap > total_cap) {
cap = total_cap;
}
return cap;
}
/* Resolve the effective retention options from (in precedence order):
* explicit caller opts > CBM_RETAIN_*_MB env knobs > RAM-derived defaults.
* The absolute hard ceilings apply only to the RAM-derived defaults; an
* operator/caller that sets a value explicitly is trusted. The only invariant
* enforced unconditionally is per-file ≤ total. */
static cbm_parallel_extract_opts_t cbm_parallel_extract_resolve_opts(
const cbm_parallel_extract_opts_t *opts) {
cbm_parallel_extract_opts_t resolved = {
.retain_sources = true,
.retain_sources_set = true,
.retain_total_budget_bytes = cbm_parallel_extract_default_total_cap(),
.retain_per_file_max_bytes = 0,
};
size_t env_total = cbm_retain_env_bytes("CBM_RETAIN_TOTAL_MB");
if (env_total > 0) {
resolved.retain_total_budget_bytes = env_total;
}
resolved.retain_per_file_max_bytes =
cbm_parallel_extract_default_per_file_cap(resolved.retain_total_budget_bytes);
size_t env_per_file = cbm_retain_env_bytes("CBM_RETAIN_PER_FILE_MB");
if (env_per_file > 0) {
resolved.retain_per_file_max_bytes = env_per_file;
}
if (opts) {
if (opts->retain_sources_set) {
resolved.retain_sources = opts->retain_sources;
}
if (opts->retain_total_budget_bytes > 0) {
resolved.retain_total_budget_bytes = opts->retain_total_budget_bytes;
}
if (opts->retain_per_file_max_bytes > 0) {
resolved.retain_per_file_max_bytes = opts->retain_per_file_max_bytes;
}
}
/* Correctness invariant: a single file can never exceed the total budget. */
if (resolved.retain_per_file_max_bytes > resolved.retain_total_budget_bytes) {
resolved.retain_per_file_max_bytes = resolved.retain_total_budget_bytes;
}
return resolved;
}
static uint64_t extract_now_ns(void) {
struct timespec ts;
cbm_clock_gettime(CLOCK_MONOTONIC, &ts);
@@ -569,7 +676,12 @@ typedef struct {
_Atomic int next_file_idx;
cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */
_Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */
bool retain_sources; /* copy source into result->arena for cross-file LSP */
size_t retain_total_budget_bytes; /* project-wide retention cap (peak-RSS bound) */
size_t retain_per_file_max_bytes; /* per-file retention cap */
_Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */
_Atomic int retain_cap_warned; /* WARN index.retain_capped emitted once per run */
/* Per-worker skip lists (separate allocation, indexed by worker_id — no hot-
* path lock). Merged into the pipeline in the sequential merge loop. */
@@ -768,29 +880,44 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
source_len, &ec->pkg_entries[worker_id]);
}
/* Retain source bytes in result->arena so the fused cross-file
* LSP step in resolve_worker can run without re-reading from
* disk. Capped per-file (PP_RETAIN_PER_FILE_MAX_BYTES) and
* globally (PP_RETAIN_TOTAL_BUDGET_BYTES) to bound peak RSS.
* Skipping retention just means cross-file LSP no-ops for this
* file — defs/calls already extracted are unaffected. */
if (source_len > 0 && (int64_t)source_len <= PP_RETAIN_PER_FILE_MAX_BYTES) {
int64_t prior = atomic_fetch_add_explicit(&ec->retained_bytes, (int64_t)source_len,
memory_order_relaxed);
if (prior + (int64_t)source_len <= PP_RETAIN_TOTAL_BUDGET_BYTES) {
char *copy = (char *)cbm_arena_alloc(&result->arena, (size_t)source_len + 1);
if (copy) {
memcpy(copy, source, (size_t)source_len);
copy[source_len] = '\0';
result->source = copy;
result->source_len = source_len;
/* Retain source bytes in result->arena so the fused cross-file LSP
* step in resolve_worker can re-parse without re-reading from disk.
* Bounded per-file (retain_per_file_max_bytes) and by a project-wide
* budget (retain_total_budget_bytes) to bound peak RSS — see the
* retention-cap comment at the top of this file. A file DROPPED here
* is NOT lost to cross-file resolution: resolve_worker re-reads it on
* demand (bounded, freed immediately). WARN once per run so the
* operator knows retention was capped (index.retain_capped). */
if (ec->retain_sources && source_len > 0) {
bool dropped = false;
if ((size_t)source_len > ec->retain_per_file_max_bytes) {
dropped = true; /* over the per-file cap */
} else {
int64_t prior = atomic_fetch_add_explicit(&ec->retained_bytes, (int64_t)source_len,
memory_order_relaxed);
if ((size_t)(prior + (int64_t)source_len) <= ec->retain_total_budget_bytes) {
char *copy = (char *)cbm_arena_alloc(&result->arena, (size_t)source_len + 1);
if (copy) {
memcpy(copy, source, (size_t)source_len);
copy[source_len] = '\0';
result->source = copy;
result->source_len = source_len;
} else {
/* Arena OOM — not a cap drop; re-read still covers
* cross-file resolution, so don't emit the WARN. */
atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len,
memory_order_relaxed);
}
} else {
atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len,
memory_order_relaxed);
dropped = true; /* project-wide budget exhausted */
}
} else {
atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len,
memory_order_relaxed);
}
if (dropped &&
atomic_exchange_explicit(&ec->retain_cap_warned, 1, memory_order_relaxed) == 0) {
cbm_log_warn("index.retain_capped", "path", fi->rel_path ? fi->rel_path : "?",
"bytes", itoa_log((int)source_len));
}
}
@@ -856,15 +983,24 @@ static void log_extract_mem_stats(int worker_count) {
}
}
int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
int worker_count) {
int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
int worker_count, const cbm_parallel_extract_opts_t *opts) {
cbm_parallel_extract_opts_t resolved_opts = cbm_parallel_extract_resolve_opts(opts);
if (file_count == 0) {
return 0;
}
cbm_log_info("parallel.extract.start", "files", itoa_log(file_count), "workers",
itoa_log(worker_count));
{
size_t mb = (size_t)CBM_SZ_1K * CBM_SZ_1K;
cbm_log_info("parallel.extract.retention", "retain_sources",
resolved_opts.retain_sources ? "true" : "false", "total_mb",
itoa_log((int)(resolved_opts.retain_total_budget_bytes / mb)), "per_file_mb",
itoa_log((int)(resolved_opts.retain_per_file_max_bytes / mb)));
}
/* Log per-worker memory budget */
if (cbm_mem_budget() > 0) {
@@ -885,7 +1021,10 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
/* Sub-phase: Sort files by descending size for tail-latency reduction */
CBM_PROF_START(t_sort);
file_sort_entry_t *sorted = malloc(file_count * sizeof(file_sort_entry_t));
file_sort_entry_t *sorted = malloc((size_t)file_count * sizeof(file_sort_entry_t));
if (!sorted) {
return CBM_NOT_FOUND;
}
for (int i = 0; i < file_count; i++) {
sorted[i].idx = i;
sorted[i].size = files[i].size;
@@ -903,11 +1042,16 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
memset(workers, 0, (size_t)worker_count * sizeof(extract_worker_state_t));
/* Per-worker manifest entry arrays (separate from cache-line-aligned worker state) */
cbm_pkg_entries_t *pkg_entries = calloc(worker_count, sizeof(cbm_pkg_entries_t));
cbm_pkg_entries_t *pkg_entries = calloc((size_t)worker_count, sizeof(cbm_pkg_entries_t));
if (!pkg_entries) {
cbm_aligned_free(workers);
free(sorted);
return CBM_NOT_FOUND;
}
/* Per-worker skip lists (separate allocation; merged into the pipeline in the
* sequential merge loop below). */
pp_err_list_t *err_lists = calloc(worker_count, sizeof(pp_err_list_t));
pp_err_list_t *err_lists = calloc((size_t)worker_count, sizeof(pp_err_list_t));
extract_ctx_t ec = {
.files = files,
@@ -922,16 +1066,20 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
.cancelled = ctx->cancelled,
.pkg_entries = pkg_entries,
.err_lists = err_lists,
.retain_sources = resolved_opts.retain_sources,
.retain_total_budget_bytes = resolved_opts.retain_total_budget_bytes,
.retain_per_file_max_bytes = resolved_opts.retain_per_file_max_bytes,
};
atomic_init(&ec.next_worker_id, 0);
atomic_init(&ec.next_file_idx, 0);
atomic_init(&ec.retained_bytes, 0);
atomic_init(&ec.retain_cap_warned, 0);
atomic_init(&ec.oversized_warned, 0);
/* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
CBM_PROF_START(t_dispatch);
cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
cbm_parallel_for(worker_count, extract_worker, &ec, opts);
cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false};
cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts);
CBM_PROF_END_N("parallel_extract", "3_dispatch_workers_parallel", t_dispatch, file_count);
/* Sub-phase: Merge all local gbufs into main gbuf (SEQUENTIAL, gbuf not thread-safe) */
@@ -982,6 +1130,13 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
return 0;
}
int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
int worker_count) {
return cbm_parallel_extract_ex(ctx, files, file_count, result_cache, shared_ids, worker_count,
NULL);
}
/* ── Phase 3B: Serial Registry Build ─────────────────────────────── */
/* Register one definition and create DEFINES + DEFINES_METHOD edges. Returns edge count. */
@@ -2466,18 +2621,31 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
* Runs BEFORE resolve_file_calls so its additions to
* result->resolved_calls are picked up by
* cbm_pipeline_find_lsp_resolution when calls become CALLS
* edges. Requires result->source to have been retained in
* result->arena during extract (PP_RETAIN_*); files over the
* cap or past the budget have result->source==NULL and are
* counted as skipped_no_source — defs/calls already in the
* extract are unaffected.
* edges. Prefers source bytes retained in result->arena during
* extract; when the low-RAM retention cap dropped this file
* (result->source==NULL) it FALLS BACK to a bounded per-file
* read from disk, freed immediately after the LSP call. This is
* the correctness guarantee: lowering the retention cap trades
* retained RAM for a bounded re-read, it NEVER drops a cross-file
* edge. Only a genuine read failure (deleted/unreadable/oversized)
* leaves source NULL and is counted as skipped_no_source; defs/calls
* already in the extract are unaffected either way.
*
* Slab reclaim afterward: the LSP re-parses via tree-sitter,
* which allocates through this worker's TLS slab. Reclaiming
* here keeps the slab high-water bounded as the resolve phase
* walks across thousands of files in a single worker thread. */
if (cross_lsp_eligible) {
if (result->source && result->source_len > 0) {
char *lsp_source_owned = NULL;
const char *lsp_source = result->source;
int lsp_source_len = result->source_len;
if ((!lsp_source || lsp_source_len <= 0) && rc->files[file_idx].path) {
/* Retention cap skipped this file — re-read on demand (bounded
* by read_file's cbm_max_file_bytes cap), freed below. */
lsp_source_owned = read_file(rc->files[file_idx].path, &lsp_source_len, NULL, NULL);
lsp_source = lsp_source_owned;
}
if (lsp_source && lsp_source_len > 0) {
const char *def_module = rc->def_modules ? rc->def_modules[file_idx] : module_qn;
uint64_t lsp_t0 = extract_now_ns();
@@ -2511,8 +2679,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
}
case CBM_LANG_PYTHON:
cbm_run_py_lsp_cross_with_registry(
&result->arena, result->source, result->source_len, def_module,
prebuilt, imp_keys, imp_vals, imp_count, result->cached_tree,
&result->arena, lsp_source, lsp_source_len, def_module, prebuilt,
imp_keys, imp_vals, imp_count, result->cached_tree,
&result->resolved_calls);
used_prebuilt = true;
break;
@@ -2520,16 +2688,15 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
case CBM_LANG_CPP:
case CBM_LANG_CUDA:
cbm_run_c_lsp_cross_with_registry(
&result->arena, result->source, result->source_len, def_module,
&result->arena, lsp_source, lsp_source_len, def_module,
(lang != CBM_LANG_C), prebuilt, imp_keys, imp_vals, imp_count,
result->cached_tree, &result->resolved_calls);
used_prebuilt = true;
break;
case CBM_LANG_CSHARP:
cbm_run_cs_lsp_cross_with_registry(&result->arena, result->source,
result->source_len, def_module, prebuilt,
imp_vals, imp_count, result->cached_tree,
&result->resolved_calls);
cbm_run_cs_lsp_cross_with_registry(
&result->arena, lsp_source, lsp_source_len, def_module, prebuilt,
imp_vals, imp_count, result->cached_tree, &result->resolved_calls);
used_prebuilt = true;
break;
case CBM_LANG_JAVASCRIPT:
@@ -2557,8 +2724,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
}
}
cbm_run_ts_lsp_cross_with_registry(
&result->arena, result->source, result->source_len, def_module, js, jsx,
dts, prebuilt, ts_defs, ts_def_count, imp_keys, imp_vals, imp_count,
&result->arena, lsp_source, lsp_source_len, def_module, js, jsx, dts,
prebuilt, ts_defs, ts_def_count, imp_keys, imp_vals, imp_count,
result->cached_tree, &result->resolved_calls);
free(ts_filtered);
used_prebuilt = true;
@@ -2590,16 +2757,17 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
lang == CBM_LANG_TSX) {
bool js, jsx, dts;
cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts);
cbm_pxc_run_one_ts(result, result->source, result->source_len, def_module,
cbm_pxc_run_one_ts(result, lsp_source, lsp_source_len, def_module,
file_defs, file_def_count, imp_keys, imp_vals, imp_count,
js, jsx, dts);
} else {
cbm_pxc_run_one(lang, result, result->source, result->source_len,
def_module, file_defs, file_def_count, imp_keys, imp_vals,
imp_count);
cbm_pxc_run_one(lang, result, lsp_source, lsp_source_len, def_module,
file_defs, file_def_count, imp_keys, imp_vals, imp_count);
}
}
free(filtered);
/* Free the on-demand re-read (no-op when source was retained). */
free_source(lsp_source_owned);
/* Contract: cbm_slab_reclaim() requires the thread parser to be
* destroyed first; otherwise its lexer holds slab pointers
* (lexer.included_ranges) that get freed underneath it, causing
@@ -2617,13 +2785,14 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
}
atomic_fetch_add_explicit(&rc->lsp_cross_processed, SKIP_ONE, memory_order_relaxed);
} else {
/* No retained source → the cross-file LSP refinement no-ops for
* this file. This is a bounded OPTIMIZATION skip, NOT a file
* failure: defs/calls were already extracted and are unaffected.
* Deliberately NOT recorded as a cbm_file_error — doing so would
* flood skipped[] with false positives (itself a false-guard
* bug). The "cross_lsp" phase string is reserved for Track C's
* real crash-attribution signal; leave it unwired here. */
/* Source unavailable even after the re-read fallback (file
* deleted / unreadable / oversized) → the cross-file LSP
* refinement no-ops for this file. This is a bounded skip, NOT
* a file failure: defs/calls were already extracted and are
* unaffected. Deliberately NOT recorded as a cbm_file_error —
* doing so would flood skipped[] with false positives (itself a
* false-guard bug). The "cross_lsp" phase string is reserved for
* Track C's real crash-attribution signal; leave it unwired. */
atomic_fetch_add_explicit(&rc->lsp_cross_skipped_no_source, SKIP_ONE,
memory_order_relaxed);
}
+15
View File
@@ -444,6 +444,21 @@ char *cbm_infra_qn(const char *project_name, const char *rel_path, const char *i
* Each worker creates nodes in a per-worker gbuf, then merges into ctx->gbuf.
* Caches CBMFileResult* in result_cache[file_idx] for reuse in Phase 3B/4.
* shared_ids provides globally unique node/edge IDs across workers. */
/* Source-retention tuning for cbm_parallel_extract_ex. Zero-valued byte caps
* mean "use the derived default" (RAM-fraction total, clamped to an absolute
* ceiling; modest per-file cap); CBM_RETAIN_TOTAL_MB / CBM_RETAIN_PER_FILE_MB
* override those. retain_sources_set=false keeps the default retain policy. */
typedef struct {
bool retain_sources;
bool retain_sources_set; /* false keeps the default retain_sources policy */
size_t retain_total_budget_bytes;
size_t retain_per_file_max_bytes;
} cbm_parallel_extract_opts_t;
int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
int worker_count, const cbm_parallel_extract_opts_t *opts);
int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
int worker_count);
+177
View File
@@ -0,0 +1,177 @@
/*
* test_ast_profile.c — Unit tests for ast_profile.c (serialization/deserialization).
*
* Covers: to_str, from_str round-trip, to_vector, edge cases.
*/
#include "test_framework.h"
#include <semantic/ast_profile.h>
#include <string.h>
/* ── Helper ──────────────────────────────────────────────────────── */
static cbm_ast_profile_t make_profile(void) {
cbm_ast_profile_t p;
memset(&p, 0, sizeof(p));
p.if_count = 5;
p.for_count = 3;
p.while_count = 1;
p.switch_count = 2;
p.try_count = 0;
p.return_count = 4;
p.max_nesting_depth = 3;
p.avg_nesting_depth_x10 = 15;
p.comparison_ops = 10;
p.arithmetic_ops = 8;
p.logical_ops = 2;
p.assignment_count = 7;
p.string_literals = 3;
p.number_literals = 5;
p.bool_literals = 1;
p.param_count = 4;
p.params_in_returns = 2;
p.params_in_conditions = 1;
p.variable_reassigns = 3;
p.unique_operators = 12;
p.unique_operands = 20;
p.total_operators = 45;
p.total_operands = 60;
p.body_lines = 30;
p.body_tokens = 200;
return p;
}
/* ── to_str + from_str round-trip ────────────────────────────────── */
TEST(ast_profile_roundtrip) {
cbm_ast_profile_t original = make_profile();
char buf[200];
cbm_ast_profile_to_str(&original, buf, sizeof(buf));
cbm_ast_profile_t decoded;
ASSERT_TRUE(cbm_ast_profile_from_str(buf, &decoded));
ASSERT_EQ(original.if_count, decoded.if_count);
ASSERT_EQ(original.for_count, decoded.for_count);
ASSERT_EQ(original.while_count, decoded.while_count);
ASSERT_EQ(original.switch_count, decoded.switch_count);
ASSERT_EQ(original.try_count, decoded.try_count);
ASSERT_EQ(original.return_count, decoded.return_count);
ASSERT_EQ(original.max_nesting_depth, decoded.max_nesting_depth);
ASSERT_EQ(original.avg_nesting_depth_x10, decoded.avg_nesting_depth_x10);
ASSERT_EQ(original.comparison_ops, decoded.comparison_ops);
ASSERT_EQ(original.arithmetic_ops, decoded.arithmetic_ops);
ASSERT_EQ(original.logical_ops, decoded.logical_ops);
ASSERT_EQ(original.assignment_count, decoded.assignment_count);
ASSERT_EQ(original.string_literals, decoded.string_literals);
ASSERT_EQ(original.number_literals, decoded.number_literals);
ASSERT_EQ(original.bool_literals, decoded.bool_literals);
ASSERT_EQ(original.param_count, decoded.param_count);
ASSERT_EQ(original.params_in_returns, decoded.params_in_returns);
ASSERT_EQ(original.params_in_conditions, decoded.params_in_conditions);
ASSERT_EQ(original.variable_reassigns, decoded.variable_reassigns);
ASSERT_EQ(original.unique_operators, decoded.unique_operators);
ASSERT_EQ(original.unique_operands, decoded.unique_operands);
ASSERT_EQ(original.total_operators, decoded.total_operators);
ASSERT_EQ(original.total_operands, decoded.total_operands);
ASSERT_EQ(original.body_lines, decoded.body_lines);
ASSERT_EQ(original.body_tokens, decoded.body_tokens);
PASS();
}
TEST(ast_profile_to_str_null) {
char buf[200];
cbm_ast_profile_to_str(NULL, buf, sizeof(buf)); /* should not crash */
PASS();
}
TEST(ast_profile_to_str_small_buf) {
cbm_ast_profile_t p = make_profile();
/* buf too small: should write empty string */
char buf[1] = {'X'};
cbm_ast_profile_to_str(&p, buf, 0);
/* 0-length buffer: function should handle gracefully */
PASS();
}
TEST(ast_profile_from_str_null) {
ASSERT_FALSE(cbm_ast_profile_from_str(NULL, NULL));
PASS();
}
TEST(ast_profile_from_str_invalid) {
cbm_ast_profile_t out;
ASSERT_FALSE(cbm_ast_profile_from_str("not,a,valid,string", &out));
ASSERT_FALSE(cbm_ast_profile_from_str("", &out));
PASS();
}
TEST(ast_profile_from_str_too_few_fields) {
cbm_ast_profile_t out;
/* Only 5 fields instead of 25 */
ASSERT_FALSE(cbm_ast_profile_from_str("1,2,3,4,5", &out));
PASS();
}
/* ── to_vector ───────────────────────────────────────────────────── */
TEST(ast_profile_to_vector_range) {
cbm_ast_profile_t p = make_profile();
float vec[25];
cbm_ast_profile_to_vector(&p, vec);
/* All values should be in [0, 1] range (normalized) */
for (int i = 0; i < 25; i++) {
ASSERT_GTE(vec[i], 0.0f);
ASSERT_LTE(vec[i], 1.0f);
}
PASS();
}
TEST(ast_profile_to_vector_zero) {
cbm_ast_profile_t p;
memset(&p, 0, sizeof(p));
float vec[25];
cbm_ast_profile_to_vector(&p, vec);
/* All zeros should produce all-zero vector */
for (int i = 0; i < 25; i++) {
ASSERT_FLOAT_EQ(vec[i], 0.0f, 0.001f);
}
PASS();
}
TEST(ast_profile_to_vector_null) {
float vec[25];
memset(vec, 0xFF, sizeof(vec));
cbm_ast_profile_to_vector(NULL, vec); /* should not crash or write */
PASS();
}
/* ── to_vector extremes (saturation) ─────────────────────────────── */
TEST(ast_profile_to_vector_saturates) {
cbm_ast_profile_t p;
memset(&p, 0, sizeof(p));
/* count / MAX_COUNT (100) → can exceed 1.0; body_tokens / MAX_TOKENS (2000) = 2.5 */
p.if_count = 500;
p.body_tokens = 5000;
float vec[25];
cbm_ast_profile_to_vector(&p, vec);
ASSERT_TRUE(vec[0] > 1.0f);
ASSERT_TRUE(vec[24] > 1.0f);
PASS();
}
/* ── Suite ───────────────────────────────────────────────────────── */
SUITE(ast_profile) {
RUN_TEST(ast_profile_roundtrip);
RUN_TEST(ast_profile_to_str_null);
RUN_TEST(ast_profile_to_str_small_buf);
RUN_TEST(ast_profile_from_str_null);
RUN_TEST(ast_profile_from_str_invalid);
RUN_TEST(ast_profile_from_str_too_few_fields);
RUN_TEST(ast_profile_to_vector_range);
RUN_TEST(ast_profile_to_vector_zero);
RUN_TEST(ast_profile_to_vector_null);
RUN_TEST(ast_profile_to_vector_saturates);
}
+6
View File
@@ -124,6 +124,9 @@ extern void suite_grammar_probe_e(void);
extern void suite_grammar_probe_f(void);
extern void suite_grammar_probe_g(void);
extern void suite_incremental(void);
extern void suite_semantic(void);
extern void suite_ast_profile(void);
extern void suite_slab_alloc(void);
extern void suite_simhash(void);
extern void suite_stack_overflow(void);
extern void suite_dump_verify(void);
@@ -253,6 +256,7 @@ int main(int argc, char **argv) {
RUN_SELECTED_SUITE(parallel);
/* mem + arena + slab integration */
RUN_SELECTED_SUITE(slab_alloc);
RUN_SELECTED_SUITE(mem);
/* UI (config, embedded assets, layout) */
@@ -268,6 +272,8 @@ int main(int argc, char **argv) {
RUN_SELECTED_SUITE(yaml);
/* SimHash / SIMILAR_TO */
RUN_SELECTED_SUITE(semantic);
RUN_SELECTED_SUITE(ast_profile);
RUN_SELECTED_SUITE(simhash);
/* Stack overflow regression (GitHub #199) */
+163
View File
@@ -684,6 +684,167 @@ static void teardown_mem_test_repo(void) {
}
}
static size_t count_retained_source_bytes(CBMFileResult **result_cache, int file_count,
int *retained_count) {
size_t retained_bytes = 0;
int count = 0;
for (int i = 0; i < file_count; i++) {
CBMFileResult *result = result_cache[i];
if (result && result->source) {
retained_bytes += (size_t)result->source_len;
count++;
}
}
if (retained_count) {
*retained_count = count;
}
return retained_bytes;
}
/* retain_sources=false disables source retention entirely: no result->source is
* kept, yet extraction still produces defs/nodes. Guards the low-RAM opt-out. */
TEST(parallel_extract_without_source_retention) {
if (setup_mem_test_repo() != 0) {
FAIL("tmpdir setup failed");
}
cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL};
cbm_file_info_t *files = NULL;
int file_count = 0;
if (cbm_discover(g_mem_tmpdir, &opts, &files, &file_count) != 0) {
teardown_mem_test_repo();
FAIL("discover failed");
}
cbm_gbuf_t *gbuf = cbm_gbuf_new("mem-test", g_mem_tmpdir);
cbm_registry_t *reg = cbm_registry_new();
atomic_int cancelled;
atomic_init(&cancelled, 0);
cbm_pipeline_ctx_t ctx = {
.project_name = "mem-test",
.repo_path = g_mem_tmpdir,
.gbuf = gbuf,
.registry = reg,
.cancelled = &cancelled,
};
_Atomic int64_t shared_ids;
atomic_init(&shared_ids, cbm_gbuf_next_id(gbuf));
CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *));
ASSERT_NOT_NULL(result_cache);
cbm_parallel_extract_opts_t extract_opts = {
.retain_sources = false,
.retain_sources_set = true,
.retain_total_budget_bytes = 0,
.retain_per_file_max_bytes = 0,
};
int rc = cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, 2,
&extract_opts);
ASSERT_EQ(rc, 0);
int defs_seen = 0;
for (int i = 0; i < file_count; i++) {
if (result_cache[i]) {
ASSERT_EQ(result_cache[i]->source, NULL);
defs_seen += result_cache[i]->defs.count;
}
}
ASSERT_GT(defs_seen, 0);
ASSERT_GT(cbm_gbuf_node_count(gbuf), 0);
for (int i = 0; i < file_count; i++) {
if (result_cache[i]) {
cbm_free_result(result_cache[i]);
}
}
free(result_cache);
cbm_registry_free(reg);
cbm_gbuf_free(gbuf);
cbm_discover_free(files, file_count);
teardown_mem_test_repo();
PASS();
}
/* Guard B (peak bound): a tiny total retention budget must actually bound the
* retained source bytes — retained_bytes <= budget — while extraction still
* produces defs/nodes. Over-budget files fall back to a bounded re-read during
* cross-file resolution (exercised in test_parallel.c), so the cap trades
* retained RAM, never correctness. */
TEST(parallel_extract_tiny_source_retention_budget) {
if (setup_mem_test_repo() != 0) {
FAIL("tmpdir setup failed");
}
cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL};
cbm_file_info_t *files = NULL;
int file_count = 0;
if (cbm_discover(g_mem_tmpdir, &opts, &files, &file_count) != 0) {
teardown_mem_test_repo();
FAIL("discover failed");
}
cbm_gbuf_t *gbuf = cbm_gbuf_new("mem-test", g_mem_tmpdir);
cbm_registry_t *reg = cbm_registry_new();
atomic_int cancelled;
atomic_init(&cancelled, 0);
cbm_pipeline_ctx_t ctx = {
.project_name = "mem-test",
.repo_path = g_mem_tmpdir,
.gbuf = gbuf,
.registry = reg,
.cancelled = &cancelled,
};
_Atomic int64_t shared_ids;
atomic_init(&shared_ids, cbm_gbuf_next_id(gbuf));
CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *));
ASSERT_NOT_NULL(result_cache);
const size_t retain_total_budget_bytes = 256;
cbm_parallel_extract_opts_t extract_opts = {
.retain_sources = true,
.retain_sources_set = true,
.retain_total_budget_bytes = retain_total_budget_bytes,
.retain_per_file_max_bytes = 100U * 1024U * 1024U,
};
int rc = cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, 2,
&extract_opts);
ASSERT_EQ(rc, 0);
int retained_count = 0;
size_t retained_bytes = count_retained_source_bytes(result_cache, file_count, &retained_count);
int defs_seen = 0;
for (int i = 0; i < file_count; i++) {
if (result_cache[i]) {
defs_seen += result_cache[i]->defs.count;
}
}
ASSERT_GT(defs_seen, 0);
ASSERT_GT(retained_count, 0);
ASSERT_LTE(retained_bytes, retain_total_budget_bytes);
ASSERT_GT(cbm_gbuf_node_count(gbuf), 0);
for (int i = 0; i < file_count; i++) {
if (result_cache[i]) {
cbm_free_result(result_cache[i]);
}
}
free(result_cache);
cbm_registry_free(reg);
cbm_gbuf_free(gbuf);
cbm_discover_free(files, file_count);
teardown_mem_test_repo();
PASS();
}
TEST(parallel_extract_with_slab) {
cbm_mem_init(0.5);
@@ -790,5 +951,7 @@ SUITE(mem) {
RUN_TEST(slab_reclaim_with_foreign_live_chunk_is_safe);
RUN_TEST(slab_destroy_with_foreign_live_chunk_is_safe);
/* Integration */
RUN_TEST(parallel_extract_without_source_retention);
RUN_TEST(parallel_extract_tiny_source_retention_budget);
RUN_TEST(parallel_extract_with_slab);
}
+142 -6
View File
@@ -112,8 +112,10 @@ static cbm_gbuf_t *run_sequential(const char *project, const char *repo_path,
/* ── Run parallel pipeline on files, returning gbuf ───────────────── */
static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_file_info_t *files,
int file_count, int worker_count) {
static cbm_gbuf_t *run_parallel_with_extract_opts(const char *project, const char *repo_path,
cbm_file_info_t *files, int file_count,
int worker_count,
const cbm_parallel_extract_opts_t *extract_opts) {
cbm_gbuf_t *gbuf = cbm_gbuf_new(project, repo_path);
cbm_registry_t *reg = cbm_registry_new();
atomic_int cancelled;
@@ -131,10 +133,15 @@ static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_
int64_t gbuf_next = cbm_gbuf_next_id(gbuf);
atomic_init(&shared_ids, gbuf_next);
CBMFileResult **result_cache = calloc(file_count, sizeof(CBMFileResult *));
CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *));
cbm_init();
cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count);
if (extract_opts) {
cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, worker_count,
extract_opts);
} else {
cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count);
}
cbm_gbuf_set_next_id(gbuf, atomic_load(&shared_ids));
cbm_build_registry_from_cache(&ctx, files, file_count, result_cache);
@@ -175,6 +182,12 @@ static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_
return gbuf;
}
static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_file_info_t *files,
int file_count, int worker_count) {
return run_parallel_with_extract_opts(project, repo_path, files, file_count, worker_count,
NULL);
}
/* ── Parity Tests ─────────────────────────────────────────────────── */
static cbm_gbuf_t *g_seq_gbuf = NULL;
@@ -741,8 +754,7 @@ TEST(parallel_lsp_tail_match_fallbacks_gated_to_jvm) {
int64_t nid = cbm_gbuf_upsert_node(tgbuf, "Method", "run", "proj.zeta.Helper.run",
"zeta/helper.py", 1, 3, NULL);
ASSERT_TRUE(nid != 0);
ASSERT_TRUE(cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", false) ==
NULL);
ASSERT_TRUE(cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", false) == NULL);
const cbm_gbuf_node_t *jvm_hit =
cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", true);
ASSERT_NOT_NULL(jvm_hit);
@@ -884,6 +896,129 @@ TEST(parallel_python_lsp_override_cross_file_emits_lsp_strategy_edges) {
PASS();
}
/* RED/GREEN A — the graph-quality guarantee behind the low-RAM retention cap.
*
* The fused cross-file LSP step re-parses each file's source to resolve calls
* whose receiver type lives in ANOTHER file (the per-file pass cannot). When
* the retention cap drops a file's source, that resolution MUST still happen
* via a bounded on-demand re-read; otherwise the cross-file CALLS edge is LOST.
*
* Fixture: a Java<->Kotlin pair with genuinely cross-language calls that only
* the cross-file LSP resolves — JavaCaller.call -> KotlinService.ping (Java ->
* Kotlin) and KotlinService.ping -> JavaService.pong (Kotlin -> Java). These
* carry the "lsp" strategy and do NOT exist without the cross-file source,
* unlike same-file or import-local Python calls which the per-file pass already
* resolves (so counting lsp edges on those cannot detect the fallback).
*
* Three scenarios asserted GREEN with the re-read fallback in place:
* 1. CONTROL — default retention: both cross-file edges present. Proves the
* fixture genuinely produces them (non-vacuity guard).
* 2. NO-RETAIN — retain_sources=false: nothing retained -> edges survive only
* via the re-read fallback.
* 3. OVER-CAP — per-file cap = 1 byte: every file dropped by the SIZE cap ->
* edges survive only via the re-read fallback.
* On main (no fallback) scenarios 2 and 3 LOSE both edges = RED; scenario 1
* stays present = the non-vacuity control. */
TEST(parallel_cross_file_reread_preserves_unretained_edges) {
char tmpdir[256];
snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_par_xf_reread_XXXXXX");
if (!cbm_mkdtemp(tmpdir)) {
FAIL("mkdtemp failed");
}
char jpath[512];
snprintf(jpath, sizeof(jpath), "%s/src/main/java/com/example/Example.java", tmpdir);
char jdir[512];
snprintf(jdir, sizeof(jdir), "%s/src/main/java/com/example", tmpdir);
cbm_mkdir_p(jdir, 0755);
FILE *jf = fopen(jpath, "w");
if (!jf) {
FAIL("fopen Example.java failed");
}
fprintf(jf, "package com.example;\n"
"\n"
"class JavaCaller {\n"
" String call(KotlinService kotlinService) {\n"
" return kotlinService.ping(new JavaService());\n"
" }\n"
"}\n"
"\n"
"class JavaService {\n"
" String pong() {\n"
" return \"pong\";\n"
" }\n"
"}\n");
fclose(jf);
char kpath[512];
snprintf(kpath, sizeof(kpath), "%s/src/main/kotlin/com/example/KotlinService.kt", tmpdir);
char kdir[512];
snprintf(kdir, sizeof(kdir), "%s/src/main/kotlin/com/example", tmpdir);
cbm_mkdir_p(kdir, 0755);
FILE *kf = fopen(kpath, "w");
if (!kf) {
FAIL("fopen KotlinService.kt failed");
}
fprintf(kf, "package com.example\n"
"\n"
"class KotlinService {\n"
" fun ping(javaService: JavaService): String {\n"
" return javaService.pong()\n"
" }\n"
"}\n");
fclose(kf);
cbm_file_info_t files[2] = {0};
files[0].path = jpath;
files[0].rel_path = (char *)"src/main/java/com/example/Example.java";
files[0].language = CBM_LANG_JAVA;
files[1].path = kpath;
files[1].rel_path = (char *)"src/main/kotlin/com/example/KotlinService.kt";
files[1].language = CBM_LANG_KOTLIN;
/* CONTROL (retained) + two drop scenarios that reach the cross-file edge
* only via the on-demand re-read: NO-RETAIN disables retention entirely;
* OVER-CAP sets a 1-byte per-file cap so every file is dropped by size. */
const cbm_parallel_extract_opts_t no_retain = {
.retain_sources = false,
.retain_sources_set = true,
};
const cbm_parallel_extract_opts_t over_cap = {
.retain_sources = true,
.retain_sources_set = true,
.retain_per_file_max_bytes = 1, /* 1 byte → every file dropped by the size cap */
};
const cbm_parallel_extract_opts_t *scenarios[3] = {NULL, &no_retain, &over_cap};
for (int s = 0; s < 3; s++) {
cbm_gbuf_t *gbuf = run_parallel_with_extract_opts("com", tmpdir, files, 2, 2, scenarios[s]);
ASSERT_NOT_NULL(gbuf);
const cbm_gbuf_edge_t *java_to_kotlin =
find_calls_edge_by_tails(gbuf, "JavaCaller.call", "KotlinService.ping");
const cbm_gbuf_edge_t *kotlin_to_java =
find_calls_edge_by_tails(gbuf, "KotlinService.ping", "JavaService.pong");
/* Both cross-file (Java↔Kotlin) CALLS edges must be present in EVERY
* scenario. In the drop scenarios (s=1,2) the caller's source is NOT
* retained, so these edges exist ONLY because resolve_worker re-reads
* the source on demand. Without that fallback (main) they are LOST. */
ASSERT_NOT_NULL(java_to_kotlin);
ASSERT_NOT_NULL(kotlin_to_java);
/* And they must come from the source-dependent cross-file LSP, not a
* source-free suffix heuristic — proving the re-read actually ran. */
ASSERT_NOT_NULL(java_to_kotlin->properties_json);
ASSERT_NOT_NULL(strstr(java_to_kotlin->properties_json, "\"strategy\":\"lsp"));
ASSERT_NOT_NULL(kotlin_to_java->properties_json);
ASSERT_NOT_NULL(strstr(kotlin_to_java->properties_json, "\"strategy\":\"lsp"));
cbm_gbuf_free(gbuf);
}
th_rmtree(tmpdir);
PASS();
}
/* issue #294: gRPC service-name extraction must (a) preserve the canonical
* proto service name (FooServiceClient → FooService, not Foo) and (b) only
* match real stub/client types — ordinary receiver vars must NOT produce
@@ -941,6 +1076,7 @@ SUITE(parallel) {
RUN_TEST(parallel_node_count);
RUN_TEST(parallel_python_lsp_override_emits_lsp_strategy_edges);
RUN_TEST(parallel_python_lsp_override_cross_file_emits_lsp_strategy_edges);
RUN_TEST(parallel_cross_file_reread_preserves_unretained_edges);
RUN_TEST(parallel_java_kotlin_lsp_override_cross_file_emits_lsp_strategy_edges);
RUN_TEST(parallel_lsp_tail_match_fallbacks_gated_to_jvm);
RUN_TEST(parallel_calls_parity);
+376
View File
@@ -0,0 +1,376 @@
/*
* test_semantic.c — Unit tests for semantic.c (pure functions).
*
* Covers: tokenize, cosine, normalize, vec_add_scaled, random_index,
* proximity, diffuse, corpus lifecycle, get_config.
*/
#include "test_framework.h"
#include <semantic/semantic.h>
#include <math.h>
#include <stdlib.h>
#include <string.h>
/* ── Tokenize ────────────────────────────────────────────────────── */
TEST(sem_tokenize_camel) {
char *tokens[32];
int n = cbm_sem_tokenize("parseUserInput", tokens, 32);
ASSERT_GTE(n, 3);
ASSERT_STR_EQ(tokens[0], "parse");
ASSERT_STR_EQ(tokens[1], "user");
ASSERT_STR_EQ(tokens[2], "input");
for (int i = 0; i < n; i++) free(tokens[i]);
PASS();
}
TEST(sem_tokenize_snake) {
char *tokens[32];
int n = cbm_sem_tokenize("handle_http_request", tokens, 32);
ASSERT_GTE(n, 3);
ASSERT_STR_EQ(tokens[0], "handle");
ASSERT_STR_EQ(tokens[1], "http");
ASSERT_STR_EQ(tokens[2], "request");
for (int i = 0; i < n; i++) free(tokens[i]);
PASS();
}
TEST(sem_tokenize_dot) {
char *tokens[32];
int n = cbm_sem_tokenize("net.http.client", tokens, 32);
ASSERT_GTE(n, 3);
ASSERT_STR_EQ(tokens[0], "net");
ASSERT_STR_EQ(tokens[1], "http");
ASSERT_STR_EQ(tokens[2], "client");
for (int i = 0; i < n; i++) free(tokens[i]);
PASS();
}
TEST(sem_tokenize_null) {
int n = cbm_sem_tokenize(NULL, NULL, 0);
ASSERT_EQ(n, 0);
PASS();
}
TEST(sem_tokenize_max_out) {
char *tokens[3];
int n = cbm_sem_tokenize("a_b_c_d_e_f_g", tokens, 3);
ASSERT_EQ(n, 3);
for (int i = 0; i < n; i++) free(tokens[i]);
PASS();
}
TEST(sem_tokenize_abbrev_expansion) {
char *tokens[32];
int n = cbm_sem_tokenize("getCtxErrMsg", tokens, 32);
/* get, ctx, context, err, error, msg, message */
ASSERT_GTE(n, 4);
bool has_ctx = false, has_context = false, has_err = false, has_error = false;
for (int i = 0; i < n; i++) {
if (strcmp(tokens[i], "ctx") == 0) has_ctx = true;
if (strcmp(tokens[i], "context") == 0) has_context = true;
if (strcmp(tokens[i], "err") == 0) has_err = true;
if (strcmp(tokens[i], "error") == 0) has_error = true;
}
ASSERT_TRUE(has_ctx && has_context && has_err && has_error);
for (int i = 0; i < n; i++) free(tokens[i]);
PASS();
}
/* ── Cosine similarity ───────────────────────────────────────────── */
static void fill_vec(cbm_sem_vec_t *v, float val) {
for (int i = 0; i < CBM_SEM_DIM; i++) v->v[i] = val;
}
TEST(sem_cosine_identical) {
cbm_sem_vec_t a, b;
fill_vec(&a, 0.5f);
fill_vec(&b, 0.5f);
float sim = cbm_sem_cosine(&a, &b);
ASSERT_FLOAT_EQ(sim, 1.0f, 0.001f);
PASS();
}
TEST(sem_cosine_orthogonal) {
cbm_sem_vec_t a, b;
memset(&a, 0, sizeof(a));
memset(&b, 0, sizeof(b));
a.v[0] = 1.0f;
b.v[1] = 1.0f;
float sim = cbm_sem_cosine(&a, &b);
ASSERT_FLOAT_EQ(sim, 0.0f, 0.001f);
PASS();
}
TEST(sem_cosine_zero_vector) {
cbm_sem_vec_t a, b;
memset(&a, 0, sizeof(a));
fill_vec(&b, 1.0f);
float sim = cbm_sem_cosine(&a, &b);
ASSERT_FLOAT_EQ(sim, 0.0f, 0.001f);
PASS();
}
TEST(sem_cosine_negative) {
cbm_sem_vec_t a, b;
memset(&a, 0, sizeof(a));
memset(&b, 0, sizeof(b));
a.v[0] = 1.0f;
b.v[0] = -1.0f;
float sim = cbm_sem_cosine(&a, &b);
ASSERT_FLOAT_EQ(sim, -1.0f, 0.001f);
PASS();
}
TEST(sem_cosine_null) {
ASSERT_FLOAT_EQ(cbm_sem_cosine(NULL, NULL), 0.0f, 0.001f);
PASS();
}
/* ── Normalize ───────────────────────────────────────────────────── */
TEST(sem_normalize_unit) {
cbm_sem_vec_t v;
memset(&v, 0, sizeof(v));
v.v[0] = 1.0f;
cbm_sem_normalize(&v);
ASSERT_FLOAT_EQ(cbm_sem_cosine(&v, &v), 1.0f, 0.001f);
PASS();
}
TEST(sem_normalize_scales) {
cbm_sem_vec_t v;
fill_vec(&v, 2.0f);
cbm_sem_normalize(&v);
float mag_sq = 0.0f;
for (int i = 0; i < CBM_SEM_DIM; i++) mag_sq += v.v[i] * v.v[i];
float mag = sqrtf(mag_sq);
ASSERT_FLOAT_EQ(mag, 1.0f, 0.01f);
PASS();
}
TEST(sem_normalize_zero) {
cbm_sem_vec_t v;
memset(&v, 0, sizeof(v));
cbm_sem_normalize(&v);
/* Should remain zero (no division by zero) */
PASS();
}
TEST(sem_normalize_null) {
cbm_sem_normalize(NULL); /* should not crash */
PASS();
}
/* ── Vec add scaled ──────────────────────────────────────────────── */
TEST(sem_vec_add_scaled_basic) {
cbm_sem_vec_t dst;
memset(&dst, 0, sizeof(dst));
cbm_sem_vec_t src;
fill_vec(&src, 1.0f);
cbm_sem_vec_add_scaled(&dst, &src, 0.5f);
ASSERT_FLOAT_EQ(dst.v[0], 0.5f, 0.001f);
ASSERT_FLOAT_EQ(dst.v[CBM_SEM_DIM - 1], 0.5f, 0.001f);
PASS();
}
TEST(sem_vec_add_scaled_null) {
cbm_sem_vec_t v;
fill_vec(&v, 1.0f);
cbm_sem_vec_add_scaled(NULL, &v, 1.0f); /* should not crash */
cbm_sem_vec_add_scaled(&v, NULL, 1.0f); /* should not crash */
PASS();
}
/* ── Random index ────────────────────────────────────────────────── */
TEST(sem_random_index_deterministic) {
cbm_sem_vec_t a, b;
cbm_sem_random_index("hello", &a);
cbm_sem_random_index("hello", &b);
ASSERT_FLOAT_EQ(cbm_sem_cosine(&a, &b), 1.0f, 0.001f);
PASS();
}
TEST(sem_random_index_different_tokens) {
cbm_sem_vec_t a, b;
cbm_sem_random_index("function", &a);
cbm_sem_random_index("variable", &b);
/* Different tokens should produce different vectors */
float sim = cbm_sem_cosine(&a, &b);
ASSERT_TRUE(sim < 1.0f - 1e-6f);
PASS();
}
TEST(sem_random_index_null) {
cbm_sem_vec_t v;
memset(&v, 0, sizeof(v));
cbm_sem_random_index(NULL, &v);
/* Should produce zero vector for NULL token */
for (int i = 0; i < CBM_SEM_DIM; i++) {
ASSERT_FLOAT_EQ(v.v[i], 0.0f, 0.001f);
}
PASS();
}
/* ── Proximity ───────────────────────────────────────────────────── */
TEST(sem_proximity_same_file) {
float p = cbm_sem_proximity("src/main.c", "src/main.c");
ASSERT_FLOAT_EQ(p, 1.1f, 0.01f); /* CBM_SEM_UNIT_POS + CBM_SEM_PROX_MAX_BOOST */
PASS();
}
TEST(sem_proximity_same_dir) {
/* Files sharing 1 of 2 directory components: ratio = 0.5 → 1.0 + 0.5*0.10 = 1.05 */
float p = cbm_sem_proximity("src/core/a.c", "src/io/b.c");
ASSERT_TRUE(p > 1.0f && p < 1.10f);
PASS();
}
TEST(sem_proximity_different_paths) {
float p = cbm_sem_proximity("src/foo/a.c", "tests/bar/b.c");
ASSERT_FLOAT_EQ(p, 1.0f, 0.01f);
PASS();
}
TEST(sem_proximity_null) {
ASSERT_FLOAT_EQ(cbm_sem_proximity(NULL, "foo.c"), 1.0f, 0.01f);
ASSERT_FLOAT_EQ(cbm_sem_proximity("foo.c", NULL), 1.0f, 0.01f);
PASS();
}
/* ── Diffuse ─────────────────────────────────────────────────────── */
TEST(sem_diffuse_zero_neighbors) {
cbm_sem_vec_t v;
fill_vec(&v, 0.5f);
cbm_sem_diffuse(&v, NULL, 0, 0.3f);
/* With zero neighbors, vector should be unchanged */
ASSERT_FLOAT_EQ(v.v[0], 0.5f, 0.001f);
PASS();
}
TEST(sem_diffuse_single_neighbor) {
cbm_sem_vec_t v;
memset(&v, 0, sizeof(v));
v.v[0] = 0.5f;
v.v[1] = 0.5f;
cbm_sem_normalize(&v); /* unit-length input */
cbm_sem_vec_t nb;
memset(&nb, 0, sizeof(nb));
nb.v[0] = 1.0f;
cbm_sem_normalize(&nb);
cbm_sem_diffuse(&v, &nb, 1, 0.3f);
/* After diffuse+normalize, result should still be unit-length */
float mag_sq = 0.0f;
for (int i = 0; i < CBM_SEM_DIM; i++) mag_sq += v.v[i] * v.v[i];
ASSERT_FLOAT_EQ(sqrtf(mag_sq), 1.0f, 0.01f);
/* Component 0 should be pulled toward neighbor's strong dim-0 */
ASSERT_TRUE(v.v[0] > 0.0f);
PASS();
}
/* ── Corpus lifecycle ────────────────────────────────────────────── */
TEST(sem_corpus_new_free) {
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
ASSERT_NOT_NULL(c);
ASSERT_EQ(cbm_sem_corpus_doc_count(c), 0);
ASSERT_EQ(cbm_sem_corpus_token_count(c), 0);
cbm_sem_corpus_free(c);
PASS();
}
TEST(sem_corpus_add_one_doc) {
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
ASSERT_NOT_NULL(c);
const char *tokens[] = {"parse", "user", "input"};
cbm_sem_corpus_add_doc(c, tokens, 3);
ASSERT_EQ(cbm_sem_corpus_doc_count(c), 1);
ASSERT_TRUE(cbm_sem_corpus_token_count(c) > 0);
cbm_sem_corpus_free(c);
PASS();
}
TEST(sem_corpus_idf) {
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
ASSERT_NOT_NULL(c);
const char *doc1[] = {"a", "b", "c"};
const char *doc2[] = {"a", "d", "e"};
cbm_sem_corpus_add_doc(c, doc1, 3);
cbm_sem_corpus_add_doc(c, doc2, 3);
/* IDF for "a" (appears in 2 docs): log(2/2) = log(1) = 0 */
float idf_a = cbm_sem_corpus_idf(c, "a");
ASSERT_TRUE(idf_a < 0.01f);
/* IDF for "b" (appears in 1 doc): log(2/1) > 0 */
float idf_b = cbm_sem_corpus_idf(c, "b");
ASSERT_TRUE(idf_b > 0.0f);
cbm_sem_corpus_free(c);
PASS();
}
TEST(sem_corpus_add_null_doc) {
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
ASSERT_NOT_NULL(c);
cbm_sem_corpus_add_doc(c, NULL, 0);
cbm_sem_corpus_add_doc(c, NULL, -1);
ASSERT_EQ(cbm_sem_corpus_doc_count(c), 0);
cbm_sem_corpus_free(c);
PASS();
}
TEST(sem_corpus_free_null) {
cbm_sem_corpus_free(NULL); /* should not crash */
PASS();
}
/* ── Config ──────────────────────────────────────────────────────── */
TEST(sem_get_config_defaults) {
cbm_sem_config_t cfg = cbm_sem_get_config();
ASSERT_TRUE(cfg.w_tfidf > 0.0f);
ASSERT_TRUE(cfg.w_ri > 0.0f);
ASSERT_TRUE(cfg.threshold > 0.0f);
ASSERT_TRUE(cfg.max_edges > 0);
PASS();
}
/* ── Suite ───────────────────────────────────────────────────────── */
SUITE(semantic) {
RUN_TEST(sem_tokenize_camel);
RUN_TEST(sem_tokenize_snake);
RUN_TEST(sem_tokenize_dot);
RUN_TEST(sem_tokenize_null);
RUN_TEST(sem_tokenize_max_out);
RUN_TEST(sem_tokenize_abbrev_expansion);
RUN_TEST(sem_cosine_identical);
RUN_TEST(sem_cosine_orthogonal);
RUN_TEST(sem_cosine_zero_vector);
RUN_TEST(sem_cosine_negative);
RUN_TEST(sem_cosine_null);
RUN_TEST(sem_normalize_unit);
RUN_TEST(sem_normalize_scales);
RUN_TEST(sem_normalize_zero);
RUN_TEST(sem_normalize_null);
RUN_TEST(sem_vec_add_scaled_basic);
RUN_TEST(sem_vec_add_scaled_null);
RUN_TEST(sem_random_index_deterministic);
RUN_TEST(sem_random_index_different_tokens);
RUN_TEST(sem_random_index_null);
RUN_TEST(sem_proximity_same_file);
RUN_TEST(sem_proximity_same_dir);
RUN_TEST(sem_proximity_different_paths);
RUN_TEST(sem_proximity_null);
RUN_TEST(sem_diffuse_zero_neighbors);
RUN_TEST(sem_diffuse_single_neighbor);
RUN_TEST(sem_corpus_new_free);
RUN_TEST(sem_corpus_add_one_doc);
RUN_TEST(sem_corpus_idf);
RUN_TEST(sem_corpus_add_null_doc);
RUN_TEST(sem_corpus_free_null);
RUN_TEST(sem_get_config_defaults);
}
+195
View File
@@ -0,0 +1,195 @@
/*
* test_slab_alloc.c — Unit tests for slab_alloc.c.
*
* Covers: slab malloc/free/calloc/realloc for ≤64B and >64B paths,
* realloc grow/shrink, zeroing, ownership, thread reset.
*/
#include "test_framework.h"
#include <foundation/slab_alloc.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
/* ── Constants from slab_alloc.c ─────────────────────────────────── */
enum { SLAB_CHUNK_SIZE = 64 };
/* ── Alloc → free (small, slab path) ─────────────────────────────── */
TEST(slab_malloc_free_small) {
void *p = cbm_slab_test_malloc(32);
ASSERT_NOT_NULL(p);
memset(p, 0xAA, 32);
cbm_slab_test_free(p);
PASS();
}
TEST(slab_malloc_free_large) {
void *p = cbm_slab_test_malloc(128);
ASSERT_NOT_NULL(p);
memset(p, 0xBB, 128);
cbm_slab_test_free(p);
PASS();
}
TEST(slab_malloc_zero_size) {
/* malloc(0) should return non-NULL on most platforms */
void *p = cbm_slab_test_malloc(0);
ASSERT_NOT_NULL(p);
cbm_slab_test_free(p);
PASS();
}
/* ── Calloc ──────────────────────────────────────────────────────── */
TEST(slab_calloc_zeros) {
void *p = cbm_slab_test_calloc(10, 8);
ASSERT_NOT_NULL(p);
unsigned char *b = p;
for (int i = 0; i < 80; i++) {
ASSERT_EQ(b[i], 0);
}
cbm_slab_test_free(p);
PASS();
}
TEST(slab_calloc_overflows) {
void *p = cbm_slab_test_calloc(SIZE_MAX, 2);
ASSERT_NULL(p);
PASS();
}
/* ── Realloc ─────────────────────────────────────────────────────── */
TEST(slab_realloc_null_is_malloc) {
void *p = cbm_slab_test_realloc(NULL, 64);
ASSERT_NOT_NULL(p);
cbm_slab_test_free(p);
PASS();
}
TEST(slab_realloc_shrink_to_zero) {
void *p = cbm_slab_test_malloc(32);
ASSERT_NOT_NULL(p);
void *q = cbm_slab_test_realloc(p, 0); /* frees and returns NULL */
ASSERT_NULL(q);
PASS();
}
TEST(slab_realloc_small_keep_same) {
void *p = cbm_slab_test_malloc(16);
ASSERT_NOT_NULL(p);
memset(p, 0xCC, 16);
/* Realloc within slab size: same pointer, content preserved */
void *q = cbm_slab_test_realloc(p, 32);
ASSERT_EQ(p, q);
unsigned char *b = q;
ASSERT_EQ(b[0], 0xCC);
cbm_slab_test_free(q);
PASS();
}
TEST(slab_realloc_promote_to_heap) {
void *p = cbm_slab_test_malloc(32);
ASSERT_NOT_NULL(p);
memset(p, 0xDD, 32);
/* Realloc above slab threshold: new ptr, content copied */
void *q = cbm_slab_test_realloc(p, 128);
ASSERT_NOT_NULL(q);
unsigned char *b = q;
ASSERT_EQ(b[0], 0xDD);
ASSERT_EQ(b[31], 0xDD);
cbm_slab_test_free(q);
PASS();
}
/* ── Multiple allocations (free list cycling) ────────────────────── */
TEST(slab_many_small_allocs) {
void *ptrs[64];
for (int i = 0; i < 64; i++) {
ptrs[i] = cbm_slab_test_malloc(SLAB_CHUNK_SIZE);
ASSERT_NOT_NULL(ptrs[i]);
memset(ptrs[i], (int)(i & 0xFF), SLAB_CHUNK_SIZE);
}
/* Free all: they go back to free list */
for (int i = 0; i < 64; i++) {
cbm_slab_test_free(ptrs[i]);
}
/* Re-allocate from free list: same pointer pool recycled */
void *again = cbm_slab_test_malloc(SLAB_CHUNK_SIZE);
ASSERT_NOT_NULL(again);
cbm_slab_test_free(again);
PASS();
}
/* ── Distinct pointer addresses ──────────────────────────────────── */
TEST(slab_distinct_allocations) {
void *a = cbm_slab_test_malloc(SLAB_CHUNK_SIZE);
void *b = cbm_slab_test_malloc(SLAB_CHUNK_SIZE);
void *c = cbm_slab_test_malloc(SLAB_CHUNK_SIZE);
ASSERT_NOT_NULL(a);
ASSERT_NOT_NULL(b);
ASSERT_NOT_NULL(c);
ASSERT_NEQ(a, b);
ASSERT_NEQ(b, c);
ASSERT_NEQ(a, c);
cbm_slab_test_free(a);
cbm_slab_test_free(b);
cbm_slab_test_free(c);
PASS();
}
/* ── Large allocation (heap path, >64B) ──────────────────────────── */
TEST(slab_large_alloc_free) {
void *p = cbm_slab_test_malloc(1024);
ASSERT_NOT_NULL(p);
cbm_slab_test_free(p);
PASS();
}
TEST(slab_large_calloc_zero) {
void *p = cbm_slab_test_calloc(64, 16);
ASSERT_NOT_NULL(p);
unsigned char *b = p;
ASSERT_EQ(b[0], 0);
ASSERT_EQ(b[1023], 0);
cbm_slab_test_free(p);
PASS();
}
/* ── Reset thread ────────────────────────────────────────────────── */
TEST(slab_reset_thread) {
/* Allocate then reset: should not crash */
void *p = cbm_slab_test_malloc(32);
ASSERT_NOT_NULL(p);
cbm_slab_test_free(p);
cbm_slab_reset_thread();
/* After reset, new allocs should still work */
void *q = cbm_slab_test_malloc(32);
ASSERT_NOT_NULL(q);
cbm_slab_test_free(q);
PASS();
}
/* ── Suite ───────────────────────────────────────────────────────── */
SUITE(slab_alloc) {
RUN_TEST(slab_malloc_free_small);
RUN_TEST(slab_malloc_free_large);
RUN_TEST(slab_malloc_zero_size);
RUN_TEST(slab_calloc_zeros);
RUN_TEST(slab_calloc_overflows);
RUN_TEST(slab_realloc_null_is_malloc);
RUN_TEST(slab_realloc_shrink_to_zero);
RUN_TEST(slab_realloc_small_keep_same);
RUN_TEST(slab_realloc_promote_to_heap);
RUN_TEST(slab_many_small_allocs);
RUN_TEST(slab_distinct_allocations);
RUN_TEST(slab_large_alloc_free);
RUN_TEST(slab_large_calloc_zero);
RUN_TEST(slab_reset_thread);
}