mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
Merge remote-tracking branch 'origin/main' into distill/782-slab-aligned-pages
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
commit
c095a4a82e
9 files changed
+1311
-70
No files matched your search
+5
-1
@@ -423,6 +423,10 @@ TEST_SECURITY_SRCS = tests/test_security.c
|
||||
|
||||
TEST_YAML_SRCS = tests/test_yaml.c
|
||||
|
||||
TEST_SEMANTIC_SRCS = tests/test_semantic.c
|
||||
TEST_AST_PROFILE_SRCS = tests/test_ast_profile.c
|
||||
TEST_SLAB_ALLOC_SRCS = tests/test_slab_alloc.c
|
||||
|
||||
TEST_SIMHASH_SRCS = tests/test_simhash.c
|
||||
|
||||
TEST_STACK_OVERFLOW_SRCS = tests/test_stack_overflow.c
|
||||
@@ -487,7 +491,7 @@ TEST_REPRO_SRCS = \
|
||||
tests/repro/repro_lsp_java_cs.c \
|
||||
tests/repro/repro_lsp_kt_php_rust.c
|
||||
|
||||
ALL_TEST_SRCS =$(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_PHP_LSP_SRCS) $(TEST_CS_LSP_SRCS) $(TEST_CS_LSP_BENCH_SRCS) $(TEST_SCOPE_SRCS) $(TEST_TYPE_REP_SRCS) $(TEST_PY_LSP_SRCS) $(TEST_PY_LSP_BENCH_SRCS) $(TEST_PY_LSP_STRESS_SRCS) $(TEST_PY_LSP_SCALE_SRCS) $(TEST_TS_LSP_SRCS) $(TEST_JAVA_LSP_SRCS) $(TEST_KOTLIN_LSP_SRCS) $(TEST_RUST_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_HTTPD_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_STACK_OVERFLOW_SRCS) $(TEST_INTEGRATION_SRCS)
|
||||
ALL_TEST_SRCS =$(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS) $(TEST_CYPHER_SRCS) $(TEST_MCP_SRCS) $(TEST_DISCOVER_SRCS) $(TEST_GRAPH_BUFFER_SRCS) $(TEST_PIPELINE_SRCS) $(TEST_WATCHER_SRCS) $(TEST_LZ4_SRCS) $(TEST_ZSTD_SRCS) $(TEST_ARTIFACT_SRCS) $(TEST_SQLITE_WRITER_SRCS) $(TEST_GO_LSP_SRCS) $(TEST_C_LSP_SRCS) $(TEST_PHP_LSP_SRCS) $(TEST_CS_LSP_SRCS) $(TEST_CS_LSP_BENCH_SRCS) $(TEST_SCOPE_SRCS) $(TEST_TYPE_REP_SRCS) $(TEST_PY_LSP_SRCS) $(TEST_PY_LSP_BENCH_SRCS) $(TEST_PY_LSP_STRESS_SRCS) $(TEST_PY_LSP_SCALE_SRCS) $(TEST_TS_LSP_SRCS) $(TEST_JAVA_LSP_SRCS) $(TEST_KOTLIN_LSP_SRCS) $(TEST_RUST_LSP_SRCS) $(TEST_TRACES_SRCS) $(TEST_CLI_SRCS) $(TEST_MEM_SRCS) $(TEST_UI_SRCS) $(TEST_HTTPD_SRCS) $(TEST_SECURITY_SRCS) $(TEST_YAML_SRCS) $(TEST_SEMANTIC_SRCS) $(TEST_AST_PROFILE_SRCS) $(TEST_SLAB_ALLOC_SRCS) $(TEST_SIMHASH_SRCS) $(TEST_STACK_OVERFLOW_SRCS) $(TEST_INTEGRATION_SRCS)
|
||||
|
||||
|
||||
# ── Build directories ────────────────────────────────────────────
|
||||
|
||||
+232
-63
@@ -40,15 +40,32 @@ enum {
|
||||
#define PP_FIELD_HINT_CONF 0.85
|
||||
enum { PP_CSHARP_M_PREFIX_LEN = 2 };
|
||||
|
||||
/* Source-retention caps for the parallel pipeline. The extract worker
|
||||
* copies source bytes into result->arena so the fused cross-file LSP
|
||||
* step in resolve_worker can run without re-reading from disk. Bound
|
||||
* peak RSS with a per-file cap (skip retention for pathological huge
|
||||
* generated files) and a total project-wide cap (skip when budget
|
||||
* exhausted — cross-file LSP becomes a no-op for those late files,
|
||||
* defs/calls already extracted are unaffected). */
|
||||
#define PP_RETAIN_PER_FILE_MAX_BYTES (100LL * 1024 * 1024)
|
||||
#define PP_RETAIN_TOTAL_BUDGET_BYTES (2LL * 1024 * 1024 * 1024)
|
||||
/* Absolute source-retention ceilings for the parallel extract pipeline.
|
||||
*
|
||||
* The extract worker copies each file's source bytes into result->arena so
|
||||
* the fused cross-file LSP step in resolve_worker can re-parse without
|
||||
* re-opening the file. That retention is TRANSIENT (freed at run end) but it
|
||||
* is a PEAK-RSS driver: every retained byte is resident at once across the
|
||||
* extract→resolve handoff.
|
||||
*
|
||||
* A cap here is a FLOOR as much as a ceiling: whatever total we allow, we
|
||||
* WILL hold that much resident at peak on a large repo. rust-analyzer bounds
|
||||
* retained file *text* to a small fixed budget and re-reads source on a miss
|
||||
* rather than scaling the retained set with host RAM — because the re-read is
|
||||
* cheap relative to holding tens of GB resident. We follow the same model:
|
||||
* - derive the total budget from the process memory budget (budget/8),
|
||||
* - BUT clamp the RAM-derived DEFAULT to a small absolute ceiling (1 GiB)
|
||||
* so a 512 GiB host does not retain 64 GiB of source it would re-read
|
||||
* cheaply anyway, and keep the per-file cap modest (32 MiB) so one
|
||||
* pathological generated blob cannot monopolise the budget.
|
||||
* A file dropped from retention is NOT lost to cross-file resolution:
|
||||
* resolve_worker re-reads it on demand, bounded and freed immediately
|
||||
* (source_reread fallback). Both caps are env-overridable — CBM_RETAIN_TOTAL_MB
|
||||
* / CBM_RETAIN_PER_FILE_MB raise or lower the auto-derived defaults directly
|
||||
* (the hard ceilings bound only the RAM-derived default, never a deliberate
|
||||
* operator/caller choice). */
|
||||
#define PP_RETAIN_TOTAL_HARD_MAX_BYTES (1024ULL * 1024 * 1024) /* 1 GiB default ceiling */
|
||||
#define PP_RETAIN_PER_FILE_HARD_MAX_BYTES (32ULL * 1024 * 1024) /* 32 MiB per file */
|
||||
#include "pipeline/pipeline.h"
|
||||
#include "pipeline/pipeline_internal.h"
|
||||
#include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */
|
||||
@@ -73,6 +90,7 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
|
||||
#include "simhash/minhash.h"
|
||||
#include "semantic/ast_profile.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
@@ -80,6 +98,95 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
/* Parse a positive MB-valued retention env knob (CBM_RETAIN_*_MB) into bytes.
|
||||
* Follows the limits.c strtol convention: unset / unparseable / non-positive
|
||||
* → return 0 so the caller keeps its derived default. */
|
||||
static size_t cbm_retain_env_bytes(const char *name) {
|
||||
const char *raw = getenv(name);
|
||||
if (!raw || !raw[0]) {
|
||||
return 0;
|
||||
}
|
||||
errno = 0;
|
||||
char *end = NULL;
|
||||
long v = strtol(raw, &end, 10);
|
||||
if (errno != 0 || end == raw || *end != '\0' || v <= 0) {
|
||||
return 0;
|
||||
}
|
||||
return (size_t)v * 1024 * 1024;
|
||||
}
|
||||
|
||||
/* Auto-derived TOTAL retention budget: a fraction of the process memory
|
||||
* budget, clamped to the absolute ceiling. The clamp bounds ONLY this
|
||||
* RAM-derived default — a huge-RAM host must not retain tens of GB it would
|
||||
* re-read cheaply. When the budget is unset (tests / no cgroup) fall back to
|
||||
* the ceiling. */
|
||||
static size_t cbm_parallel_extract_default_total_cap(void) {
|
||||
size_t cap = cbm_mem_budget();
|
||||
if (cap > 0) {
|
||||
cap /= 8;
|
||||
} else {
|
||||
cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES;
|
||||
}
|
||||
if (cap > PP_RETAIN_TOTAL_HARD_MAX_BYTES) {
|
||||
cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES;
|
||||
}
|
||||
return cap;
|
||||
}
|
||||
|
||||
/* Auto-derived PER-FILE cap: modest absolute ceiling, never above the total. */
|
||||
static size_t cbm_parallel_extract_default_per_file_cap(size_t total_cap) {
|
||||
size_t cap = PP_RETAIN_PER_FILE_HARD_MAX_BYTES;
|
||||
if (cap > total_cap) {
|
||||
cap = total_cap;
|
||||
}
|
||||
return cap;
|
||||
}
|
||||
|
||||
/* Resolve the effective retention options from (in precedence order):
|
||||
* explicit caller opts > CBM_RETAIN_*_MB env knobs > RAM-derived defaults.
|
||||
* The absolute hard ceilings apply only to the RAM-derived defaults; an
|
||||
* operator/caller that sets a value explicitly is trusted. The only invariant
|
||||
* enforced unconditionally is per-file ≤ total. */
|
||||
static cbm_parallel_extract_opts_t cbm_parallel_extract_resolve_opts(
|
||||
const cbm_parallel_extract_opts_t *opts) {
|
||||
cbm_parallel_extract_opts_t resolved = {
|
||||
.retain_sources = true,
|
||||
.retain_sources_set = true,
|
||||
.retain_total_budget_bytes = cbm_parallel_extract_default_total_cap(),
|
||||
.retain_per_file_max_bytes = 0,
|
||||
};
|
||||
|
||||
size_t env_total = cbm_retain_env_bytes("CBM_RETAIN_TOTAL_MB");
|
||||
if (env_total > 0) {
|
||||
resolved.retain_total_budget_bytes = env_total;
|
||||
}
|
||||
|
||||
resolved.retain_per_file_max_bytes =
|
||||
cbm_parallel_extract_default_per_file_cap(resolved.retain_total_budget_bytes);
|
||||
size_t env_per_file = cbm_retain_env_bytes("CBM_RETAIN_PER_FILE_MB");
|
||||
if (env_per_file > 0) {
|
||||
resolved.retain_per_file_max_bytes = env_per_file;
|
||||
}
|
||||
|
||||
if (opts) {
|
||||
if (opts->retain_sources_set) {
|
||||
resolved.retain_sources = opts->retain_sources;
|
||||
}
|
||||
if (opts->retain_total_budget_bytes > 0) {
|
||||
resolved.retain_total_budget_bytes = opts->retain_total_budget_bytes;
|
||||
}
|
||||
if (opts->retain_per_file_max_bytes > 0) {
|
||||
resolved.retain_per_file_max_bytes = opts->retain_per_file_max_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/* Correctness invariant: a single file can never exceed the total budget. */
|
||||
if (resolved.retain_per_file_max_bytes > resolved.retain_total_budget_bytes) {
|
||||
resolved.retain_per_file_max_bytes = resolved.retain_total_budget_bytes;
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
static uint64_t extract_now_ns(void) {
|
||||
struct timespec ts;
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
@@ -569,7 +676,12 @@ typedef struct {
|
||||
_Atomic int next_file_idx;
|
||||
|
||||
cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */
|
||||
_Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */
|
||||
|
||||
bool retain_sources; /* copy source into result->arena for cross-file LSP */
|
||||
size_t retain_total_budget_bytes; /* project-wide retention cap (peak-RSS bound) */
|
||||
size_t retain_per_file_max_bytes; /* per-file retention cap */
|
||||
_Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */
|
||||
_Atomic int retain_cap_warned; /* WARN index.retain_capped emitted once per run */
|
||||
|
||||
/* Per-worker skip lists (separate allocation, indexed by worker_id — no hot-
|
||||
* path lock). Merged into the pipeline in the sequential merge loop. */
|
||||
@@ -768,29 +880,44 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
|
||||
source_len, &ec->pkg_entries[worker_id]);
|
||||
}
|
||||
|
||||
/* Retain source bytes in result->arena so the fused cross-file
|
||||
* LSP step in resolve_worker can run without re-reading from
|
||||
* disk. Capped per-file (PP_RETAIN_PER_FILE_MAX_BYTES) and
|
||||
* globally (PP_RETAIN_TOTAL_BUDGET_BYTES) to bound peak RSS.
|
||||
* Skipping retention just means cross-file LSP no-ops for this
|
||||
* file — defs/calls already extracted are unaffected. */
|
||||
if (source_len > 0 && (int64_t)source_len <= PP_RETAIN_PER_FILE_MAX_BYTES) {
|
||||
int64_t prior = atomic_fetch_add_explicit(&ec->retained_bytes, (int64_t)source_len,
|
||||
memory_order_relaxed);
|
||||
if (prior + (int64_t)source_len <= PP_RETAIN_TOTAL_BUDGET_BYTES) {
|
||||
char *copy = (char *)cbm_arena_alloc(&result->arena, (size_t)source_len + 1);
|
||||
if (copy) {
|
||||
memcpy(copy, source, (size_t)source_len);
|
||||
copy[source_len] = '\0';
|
||||
result->source = copy;
|
||||
result->source_len = source_len;
|
||||
/* Retain source bytes in result->arena so the fused cross-file LSP
|
||||
* step in resolve_worker can re-parse without re-reading from disk.
|
||||
* Bounded per-file (retain_per_file_max_bytes) and by a project-wide
|
||||
* budget (retain_total_budget_bytes) to bound peak RSS — see the
|
||||
* retention-cap comment at the top of this file. A file DROPPED here
|
||||
* is NOT lost to cross-file resolution: resolve_worker re-reads it on
|
||||
* demand (bounded, freed immediately). WARN once per run so the
|
||||
* operator knows retention was capped (index.retain_capped). */
|
||||
if (ec->retain_sources && source_len > 0) {
|
||||
bool dropped = false;
|
||||
if ((size_t)source_len > ec->retain_per_file_max_bytes) {
|
||||
dropped = true; /* over the per-file cap */
|
||||
} else {
|
||||
int64_t prior = atomic_fetch_add_explicit(&ec->retained_bytes, (int64_t)source_len,
|
||||
memory_order_relaxed);
|
||||
if ((size_t)(prior + (int64_t)source_len) <= ec->retain_total_budget_bytes) {
|
||||
char *copy = (char *)cbm_arena_alloc(&result->arena, (size_t)source_len + 1);
|
||||
if (copy) {
|
||||
memcpy(copy, source, (size_t)source_len);
|
||||
copy[source_len] = '\0';
|
||||
result->source = copy;
|
||||
result->source_len = source_len;
|
||||
} else {
|
||||
/* Arena OOM — not a cap drop; re-read still covers
|
||||
* cross-file resolution, so don't emit the WARN. */
|
||||
atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
} else {
|
||||
atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len,
|
||||
memory_order_relaxed);
|
||||
dropped = true; /* project-wide budget exhausted */
|
||||
}
|
||||
} else {
|
||||
atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
if (dropped &&
|
||||
atomic_exchange_explicit(&ec->retain_cap_warned, 1, memory_order_relaxed) == 0) {
|
||||
cbm_log_warn("index.retain_capped", "path", fi->rel_path ? fi->rel_path : "?",
|
||||
"bytes", itoa_log((int)source_len));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -856,15 +983,24 @@ static void log_extract_mem_stats(int worker_count) {
|
||||
}
|
||||
}
|
||||
|
||||
int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
|
||||
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
|
||||
int worker_count) {
|
||||
int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
|
||||
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
|
||||
int worker_count, const cbm_parallel_extract_opts_t *opts) {
|
||||
cbm_parallel_extract_opts_t resolved_opts = cbm_parallel_extract_resolve_opts(opts);
|
||||
|
||||
if (file_count == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
cbm_log_info("parallel.extract.start", "files", itoa_log(file_count), "workers",
|
||||
itoa_log(worker_count));
|
||||
{
|
||||
size_t mb = (size_t)CBM_SZ_1K * CBM_SZ_1K;
|
||||
cbm_log_info("parallel.extract.retention", "retain_sources",
|
||||
resolved_opts.retain_sources ? "true" : "false", "total_mb",
|
||||
itoa_log((int)(resolved_opts.retain_total_budget_bytes / mb)), "per_file_mb",
|
||||
itoa_log((int)(resolved_opts.retain_per_file_max_bytes / mb)));
|
||||
}
|
||||
|
||||
/* Log per-worker memory budget */
|
||||
if (cbm_mem_budget() > 0) {
|
||||
@@ -885,7 +1021,10 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
|
||||
/* Sub-phase: Sort files by descending size for tail-latency reduction */
|
||||
CBM_PROF_START(t_sort);
|
||||
file_sort_entry_t *sorted = malloc(file_count * sizeof(file_sort_entry_t));
|
||||
file_sort_entry_t *sorted = malloc((size_t)file_count * sizeof(file_sort_entry_t));
|
||||
if (!sorted) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
sorted[i].idx = i;
|
||||
sorted[i].size = files[i].size;
|
||||
@@ -903,11 +1042,16 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
memset(workers, 0, (size_t)worker_count * sizeof(extract_worker_state_t));
|
||||
|
||||
/* Per-worker manifest entry arrays (separate from cache-line-aligned worker state) */
|
||||
cbm_pkg_entries_t *pkg_entries = calloc(worker_count, sizeof(cbm_pkg_entries_t));
|
||||
cbm_pkg_entries_t *pkg_entries = calloc((size_t)worker_count, sizeof(cbm_pkg_entries_t));
|
||||
if (!pkg_entries) {
|
||||
cbm_aligned_free(workers);
|
||||
free(sorted);
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
|
||||
/* Per-worker skip lists (separate allocation; merged into the pipeline in the
|
||||
* sequential merge loop below). */
|
||||
pp_err_list_t *err_lists = calloc(worker_count, sizeof(pp_err_list_t));
|
||||
pp_err_list_t *err_lists = calloc((size_t)worker_count, sizeof(pp_err_list_t));
|
||||
|
||||
extract_ctx_t ec = {
|
||||
.files = files,
|
||||
@@ -922,16 +1066,20 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
.cancelled = ctx->cancelled,
|
||||
.pkg_entries = pkg_entries,
|
||||
.err_lists = err_lists,
|
||||
.retain_sources = resolved_opts.retain_sources,
|
||||
.retain_total_budget_bytes = resolved_opts.retain_total_budget_bytes,
|
||||
.retain_per_file_max_bytes = resolved_opts.retain_per_file_max_bytes,
|
||||
};
|
||||
atomic_init(&ec.next_worker_id, 0);
|
||||
atomic_init(&ec.next_file_idx, 0);
|
||||
atomic_init(&ec.retained_bytes, 0);
|
||||
atomic_init(&ec.retain_cap_warned, 0);
|
||||
atomic_init(&ec.oversized_warned, 0);
|
||||
|
||||
/* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
|
||||
CBM_PROF_START(t_dispatch);
|
||||
cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
|
||||
cbm_parallel_for(worker_count, extract_worker, &ec, opts);
|
||||
cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false};
|
||||
cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts);
|
||||
CBM_PROF_END_N("parallel_extract", "3_dispatch_workers_parallel", t_dispatch, file_count);
|
||||
|
||||
/* Sub-phase: Merge all local gbufs into main gbuf (SEQUENTIAL, gbuf not thread-safe) */
|
||||
@@ -982,6 +1130,13 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
|
||||
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
|
||||
int worker_count) {
|
||||
return cbm_parallel_extract_ex(ctx, files, file_count, result_cache, shared_ids, worker_count,
|
||||
NULL);
|
||||
}
|
||||
|
||||
/* ── Phase 3B: Serial Registry Build ─────────────────────────────── */
|
||||
|
||||
/* Register one definition and create DEFINES + DEFINES_METHOD edges. Returns edge count. */
|
||||
@@ -2466,18 +2621,31 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
* Runs BEFORE resolve_file_calls so its additions to
|
||||
* result->resolved_calls are picked up by
|
||||
* cbm_pipeline_find_lsp_resolution when calls become CALLS
|
||||
* edges. Requires result->source to have been retained in
|
||||
* result->arena during extract (PP_RETAIN_*); files over the
|
||||
* cap or past the budget have result->source==NULL and are
|
||||
* counted as skipped_no_source — defs/calls already in the
|
||||
* extract are unaffected.
|
||||
* edges. Prefers source bytes retained in result->arena during
|
||||
* extract; when the low-RAM retention cap dropped this file
|
||||
* (result->source==NULL) it FALLS BACK to a bounded per-file
|
||||
* read from disk, freed immediately after the LSP call. This is
|
||||
* the correctness guarantee: lowering the retention cap trades
|
||||
* retained RAM for a bounded re-read, it NEVER drops a cross-file
|
||||
* edge. Only a genuine read failure (deleted/unreadable/oversized)
|
||||
* leaves source NULL and is counted as skipped_no_source; defs/calls
|
||||
* already in the extract are unaffected either way.
|
||||
*
|
||||
* Slab reclaim afterward: the LSP re-parses via tree-sitter,
|
||||
* which allocates through this worker's TLS slab. Reclaiming
|
||||
* here keeps the slab high-water bounded as the resolve phase
|
||||
* walks across thousands of files in a single worker thread. */
|
||||
if (cross_lsp_eligible) {
|
||||
if (result->source && result->source_len > 0) {
|
||||
char *lsp_source_owned = NULL;
|
||||
const char *lsp_source = result->source;
|
||||
int lsp_source_len = result->source_len;
|
||||
if ((!lsp_source || lsp_source_len <= 0) && rc->files[file_idx].path) {
|
||||
/* Retention cap skipped this file — re-read on demand (bounded
|
||||
* by read_file's cbm_max_file_bytes cap), freed below. */
|
||||
lsp_source_owned = read_file(rc->files[file_idx].path, &lsp_source_len, NULL, NULL);
|
||||
lsp_source = lsp_source_owned;
|
||||
}
|
||||
if (lsp_source && lsp_source_len > 0) {
|
||||
const char *def_module = rc->def_modules ? rc->def_modules[file_idx] : module_qn;
|
||||
|
||||
uint64_t lsp_t0 = extract_now_ns();
|
||||
@@ -2511,8 +2679,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
}
|
||||
case CBM_LANG_PYTHON:
|
||||
cbm_run_py_lsp_cross_with_registry(
|
||||
&result->arena, result->source, result->source_len, def_module,
|
||||
prebuilt, imp_keys, imp_vals, imp_count, result->cached_tree,
|
||||
&result->arena, lsp_source, lsp_source_len, def_module, prebuilt,
|
||||
imp_keys, imp_vals, imp_count, result->cached_tree,
|
||||
&result->resolved_calls);
|
||||
used_prebuilt = true;
|
||||
break;
|
||||
@@ -2520,16 +2688,15 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
case CBM_LANG_CPP:
|
||||
case CBM_LANG_CUDA:
|
||||
cbm_run_c_lsp_cross_with_registry(
|
||||
&result->arena, result->source, result->source_len, def_module,
|
||||
&result->arena, lsp_source, lsp_source_len, def_module,
|
||||
(lang != CBM_LANG_C), prebuilt, imp_keys, imp_vals, imp_count,
|
||||
result->cached_tree, &result->resolved_calls);
|
||||
used_prebuilt = true;
|
||||
break;
|
||||
case CBM_LANG_CSHARP:
|
||||
cbm_run_cs_lsp_cross_with_registry(&result->arena, result->source,
|
||||
result->source_len, def_module, prebuilt,
|
||||
imp_vals, imp_count, result->cached_tree,
|
||||
&result->resolved_calls);
|
||||
cbm_run_cs_lsp_cross_with_registry(
|
||||
&result->arena, lsp_source, lsp_source_len, def_module, prebuilt,
|
||||
imp_vals, imp_count, result->cached_tree, &result->resolved_calls);
|
||||
used_prebuilt = true;
|
||||
break;
|
||||
case CBM_LANG_JAVASCRIPT:
|
||||
@@ -2557,8 +2724,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
}
|
||||
}
|
||||
cbm_run_ts_lsp_cross_with_registry(
|
||||
&result->arena, result->source, result->source_len, def_module, js, jsx,
|
||||
dts, prebuilt, ts_defs, ts_def_count, imp_keys, imp_vals, imp_count,
|
||||
&result->arena, lsp_source, lsp_source_len, def_module, js, jsx, dts,
|
||||
prebuilt, ts_defs, ts_def_count, imp_keys, imp_vals, imp_count,
|
||||
result->cached_tree, &result->resolved_calls);
|
||||
free(ts_filtered);
|
||||
used_prebuilt = true;
|
||||
@@ -2590,16 +2757,17 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
lang == CBM_LANG_TSX) {
|
||||
bool js, jsx, dts;
|
||||
cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts);
|
||||
cbm_pxc_run_one_ts(result, result->source, result->source_len, def_module,
|
||||
cbm_pxc_run_one_ts(result, lsp_source, lsp_source_len, def_module,
|
||||
file_defs, file_def_count, imp_keys, imp_vals, imp_count,
|
||||
js, jsx, dts);
|
||||
} else {
|
||||
cbm_pxc_run_one(lang, result, result->source, result->source_len,
|
||||
def_module, file_defs, file_def_count, imp_keys, imp_vals,
|
||||
imp_count);
|
||||
cbm_pxc_run_one(lang, result, lsp_source, lsp_source_len, def_module,
|
||||
file_defs, file_def_count, imp_keys, imp_vals, imp_count);
|
||||
}
|
||||
}
|
||||
free(filtered);
|
||||
/* Free the on-demand re-read (no-op when source was retained). */
|
||||
free_source(lsp_source_owned);
|
||||
/* Contract: cbm_slab_reclaim() requires the thread parser to be
|
||||
* destroyed first; otherwise its lexer holds slab pointers
|
||||
* (lexer.included_ranges) that get freed underneath it, causing
|
||||
@@ -2617,13 +2785,14 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
}
|
||||
atomic_fetch_add_explicit(&rc->lsp_cross_processed, SKIP_ONE, memory_order_relaxed);
|
||||
} else {
|
||||
/* No retained source → the cross-file LSP refinement no-ops for
|
||||
* this file. This is a bounded OPTIMIZATION skip, NOT a file
|
||||
* failure: defs/calls were already extracted and are unaffected.
|
||||
* Deliberately NOT recorded as a cbm_file_error — doing so would
|
||||
* flood skipped[] with false positives (itself a false-guard
|
||||
* bug). The "cross_lsp" phase string is reserved for Track C's
|
||||
* real crash-attribution signal; leave it unwired here. */
|
||||
/* Source unavailable even after the re-read fallback (file
|
||||
* deleted / unreadable / oversized) → the cross-file LSP
|
||||
* refinement no-ops for this file. This is a bounded skip, NOT
|
||||
* a file failure: defs/calls were already extracted and are
|
||||
* unaffected. Deliberately NOT recorded as a cbm_file_error —
|
||||
* doing so would flood skipped[] with false positives (itself a
|
||||
* false-guard bug). The "cross_lsp" phase string is reserved for
|
||||
* Track C's real crash-attribution signal; leave it unwired. */
|
||||
atomic_fetch_add_explicit(&rc->lsp_cross_skipped_no_source, SKIP_ONE,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
|
||||
@@ -444,6 +444,21 @@ char *cbm_infra_qn(const char *project_name, const char *rel_path, const char *i
|
||||
* Each worker creates nodes in a per-worker gbuf, then merges into ctx->gbuf.
|
||||
* Caches CBMFileResult* in result_cache[file_idx] for reuse in Phase 3B/4.
|
||||
* shared_ids provides globally unique node/edge IDs across workers. */
|
||||
|
||||
/* Source-retention tuning for cbm_parallel_extract_ex. Zero-valued byte caps
|
||||
* mean "use the derived default" (RAM-fraction total, clamped to an absolute
|
||||
* ceiling; modest per-file cap); CBM_RETAIN_TOTAL_MB / CBM_RETAIN_PER_FILE_MB
|
||||
* override those. retain_sources_set=false keeps the default retain policy. */
|
||||
typedef struct {
|
||||
bool retain_sources;
|
||||
bool retain_sources_set; /* false keeps the default retain_sources policy */
|
||||
size_t retain_total_budget_bytes;
|
||||
size_t retain_per_file_max_bytes;
|
||||
} cbm_parallel_extract_opts_t;
|
||||
|
||||
int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
|
||||
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
|
||||
int worker_count, const cbm_parallel_extract_opts_t *opts);
|
||||
int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
|
||||
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
|
||||
int worker_count);
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
/*
|
||||
* test_ast_profile.c — Unit tests for ast_profile.c (serialization/deserialization).
|
||||
*
|
||||
* Covers: to_str, from_str round-trip, to_vector, edge cases.
|
||||
*/
|
||||
#include "test_framework.h"
|
||||
#include <semantic/ast_profile.h>
|
||||
|
||||
#include <string.h>
|
||||
|
||||
/* ── Helper ──────────────────────────────────────────────────────── */
|
||||
|
||||
static cbm_ast_profile_t make_profile(void) {
|
||||
cbm_ast_profile_t p;
|
||||
memset(&p, 0, sizeof(p));
|
||||
p.if_count = 5;
|
||||
p.for_count = 3;
|
||||
p.while_count = 1;
|
||||
p.switch_count = 2;
|
||||
p.try_count = 0;
|
||||
p.return_count = 4;
|
||||
p.max_nesting_depth = 3;
|
||||
p.avg_nesting_depth_x10 = 15;
|
||||
p.comparison_ops = 10;
|
||||
p.arithmetic_ops = 8;
|
||||
p.logical_ops = 2;
|
||||
p.assignment_count = 7;
|
||||
p.string_literals = 3;
|
||||
p.number_literals = 5;
|
||||
p.bool_literals = 1;
|
||||
p.param_count = 4;
|
||||
p.params_in_returns = 2;
|
||||
p.params_in_conditions = 1;
|
||||
p.variable_reassigns = 3;
|
||||
p.unique_operators = 12;
|
||||
p.unique_operands = 20;
|
||||
p.total_operators = 45;
|
||||
p.total_operands = 60;
|
||||
p.body_lines = 30;
|
||||
p.body_tokens = 200;
|
||||
return p;
|
||||
}
|
||||
|
||||
/* ── to_str + from_str round-trip ────────────────────────────────── */
|
||||
|
||||
TEST(ast_profile_roundtrip) {
|
||||
cbm_ast_profile_t original = make_profile();
|
||||
char buf[200];
|
||||
cbm_ast_profile_to_str(&original, buf, sizeof(buf));
|
||||
|
||||
cbm_ast_profile_t decoded;
|
||||
ASSERT_TRUE(cbm_ast_profile_from_str(buf, &decoded));
|
||||
|
||||
ASSERT_EQ(original.if_count, decoded.if_count);
|
||||
ASSERT_EQ(original.for_count, decoded.for_count);
|
||||
ASSERT_EQ(original.while_count, decoded.while_count);
|
||||
ASSERT_EQ(original.switch_count, decoded.switch_count);
|
||||
ASSERT_EQ(original.try_count, decoded.try_count);
|
||||
ASSERT_EQ(original.return_count, decoded.return_count);
|
||||
ASSERT_EQ(original.max_nesting_depth, decoded.max_nesting_depth);
|
||||
ASSERT_EQ(original.avg_nesting_depth_x10, decoded.avg_nesting_depth_x10);
|
||||
ASSERT_EQ(original.comparison_ops, decoded.comparison_ops);
|
||||
ASSERT_EQ(original.arithmetic_ops, decoded.arithmetic_ops);
|
||||
ASSERT_EQ(original.logical_ops, decoded.logical_ops);
|
||||
ASSERT_EQ(original.assignment_count, decoded.assignment_count);
|
||||
ASSERT_EQ(original.string_literals, decoded.string_literals);
|
||||
ASSERT_EQ(original.number_literals, decoded.number_literals);
|
||||
ASSERT_EQ(original.bool_literals, decoded.bool_literals);
|
||||
ASSERT_EQ(original.param_count, decoded.param_count);
|
||||
ASSERT_EQ(original.params_in_returns, decoded.params_in_returns);
|
||||
ASSERT_EQ(original.params_in_conditions, decoded.params_in_conditions);
|
||||
ASSERT_EQ(original.variable_reassigns, decoded.variable_reassigns);
|
||||
ASSERT_EQ(original.unique_operators, decoded.unique_operators);
|
||||
ASSERT_EQ(original.unique_operands, decoded.unique_operands);
|
||||
ASSERT_EQ(original.total_operators, decoded.total_operators);
|
||||
ASSERT_EQ(original.total_operands, decoded.total_operands);
|
||||
ASSERT_EQ(original.body_lines, decoded.body_lines);
|
||||
ASSERT_EQ(original.body_tokens, decoded.body_tokens);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(ast_profile_to_str_null) {
|
||||
char buf[200];
|
||||
cbm_ast_profile_to_str(NULL, buf, sizeof(buf)); /* should not crash */
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(ast_profile_to_str_small_buf) {
|
||||
cbm_ast_profile_t p = make_profile();
|
||||
/* buf too small: should write empty string */
|
||||
char buf[1] = {'X'};
|
||||
cbm_ast_profile_to_str(&p, buf, 0);
|
||||
/* 0-length buffer: function should handle gracefully */
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(ast_profile_from_str_null) {
|
||||
ASSERT_FALSE(cbm_ast_profile_from_str(NULL, NULL));
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(ast_profile_from_str_invalid) {
|
||||
cbm_ast_profile_t out;
|
||||
ASSERT_FALSE(cbm_ast_profile_from_str("not,a,valid,string", &out));
|
||||
ASSERT_FALSE(cbm_ast_profile_from_str("", &out));
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(ast_profile_from_str_too_few_fields) {
|
||||
cbm_ast_profile_t out;
|
||||
/* Only 5 fields instead of 25 */
|
||||
ASSERT_FALSE(cbm_ast_profile_from_str("1,2,3,4,5", &out));
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── to_vector ───────────────────────────────────────────────────── */
|
||||
|
||||
TEST(ast_profile_to_vector_range) {
|
||||
cbm_ast_profile_t p = make_profile();
|
||||
float vec[25];
|
||||
cbm_ast_profile_to_vector(&p, vec);
|
||||
/* All values should be in [0, 1] range (normalized) */
|
||||
for (int i = 0; i < 25; i++) {
|
||||
ASSERT_GTE(vec[i], 0.0f);
|
||||
ASSERT_LTE(vec[i], 1.0f);
|
||||
}
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(ast_profile_to_vector_zero) {
|
||||
cbm_ast_profile_t p;
|
||||
memset(&p, 0, sizeof(p));
|
||||
float vec[25];
|
||||
cbm_ast_profile_to_vector(&p, vec);
|
||||
/* All zeros should produce all-zero vector */
|
||||
for (int i = 0; i < 25; i++) {
|
||||
ASSERT_FLOAT_EQ(vec[i], 0.0f, 0.001f);
|
||||
}
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(ast_profile_to_vector_null) {
|
||||
float vec[25];
|
||||
memset(vec, 0xFF, sizeof(vec));
|
||||
cbm_ast_profile_to_vector(NULL, vec); /* should not crash or write */
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── to_vector extremes (saturation) ─────────────────────────────── */
|
||||
|
||||
TEST(ast_profile_to_vector_saturates) {
|
||||
cbm_ast_profile_t p;
|
||||
memset(&p, 0, sizeof(p));
|
||||
/* count / MAX_COUNT (100) → can exceed 1.0; body_tokens / MAX_TOKENS (2000) = 2.5 */
|
||||
p.if_count = 500;
|
||||
p.body_tokens = 5000;
|
||||
float vec[25];
|
||||
cbm_ast_profile_to_vector(&p, vec);
|
||||
ASSERT_TRUE(vec[0] > 1.0f);
|
||||
ASSERT_TRUE(vec[24] > 1.0f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Suite ───────────────────────────────────────────────────────── */
|
||||
|
||||
SUITE(ast_profile) {
|
||||
RUN_TEST(ast_profile_roundtrip);
|
||||
RUN_TEST(ast_profile_to_str_null);
|
||||
RUN_TEST(ast_profile_to_str_small_buf);
|
||||
RUN_TEST(ast_profile_from_str_null);
|
||||
RUN_TEST(ast_profile_from_str_invalid);
|
||||
RUN_TEST(ast_profile_from_str_too_few_fields);
|
||||
RUN_TEST(ast_profile_to_vector_range);
|
||||
RUN_TEST(ast_profile_to_vector_zero);
|
||||
RUN_TEST(ast_profile_to_vector_null);
|
||||
RUN_TEST(ast_profile_to_vector_saturates);
|
||||
}
|
||||
@@ -124,6 +124,9 @@ extern void suite_grammar_probe_e(void);
|
||||
extern void suite_grammar_probe_f(void);
|
||||
extern void suite_grammar_probe_g(void);
|
||||
extern void suite_incremental(void);
|
||||
extern void suite_semantic(void);
|
||||
extern void suite_ast_profile(void);
|
||||
extern void suite_slab_alloc(void);
|
||||
extern void suite_simhash(void);
|
||||
extern void suite_stack_overflow(void);
|
||||
extern void suite_dump_verify(void);
|
||||
@@ -253,6 +256,7 @@ int main(int argc, char **argv) {
|
||||
RUN_SELECTED_SUITE(parallel);
|
||||
|
||||
/* mem + arena + slab integration */
|
||||
RUN_SELECTED_SUITE(slab_alloc);
|
||||
RUN_SELECTED_SUITE(mem);
|
||||
|
||||
/* UI (config, embedded assets, layout) */
|
||||
@@ -268,6 +272,8 @@ int main(int argc, char **argv) {
|
||||
RUN_SELECTED_SUITE(yaml);
|
||||
|
||||
/* SimHash / SIMILAR_TO */
|
||||
RUN_SELECTED_SUITE(semantic);
|
||||
RUN_SELECTED_SUITE(ast_profile);
|
||||
RUN_SELECTED_SUITE(simhash);
|
||||
|
||||
/* Stack overflow regression (GitHub #199) */
|
||||
|
||||
@@ -684,6 +684,167 @@ static void teardown_mem_test_repo(void) {
|
||||
}
|
||||
}
|
||||
|
||||
static size_t count_retained_source_bytes(CBMFileResult **result_cache, int file_count,
|
||||
int *retained_count) {
|
||||
size_t retained_bytes = 0;
|
||||
int count = 0;
|
||||
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
CBMFileResult *result = result_cache[i];
|
||||
if (result && result->source) {
|
||||
retained_bytes += (size_t)result->source_len;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
if (retained_count) {
|
||||
*retained_count = count;
|
||||
}
|
||||
return retained_bytes;
|
||||
}
|
||||
|
||||
/* retain_sources=false disables source retention entirely: no result->source is
|
||||
* kept, yet extraction still produces defs/nodes. Guards the low-RAM opt-out. */
|
||||
TEST(parallel_extract_without_source_retention) {
|
||||
if (setup_mem_test_repo() != 0) {
|
||||
FAIL("tmpdir setup failed");
|
||||
}
|
||||
|
||||
cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL};
|
||||
cbm_file_info_t *files = NULL;
|
||||
int file_count = 0;
|
||||
if (cbm_discover(g_mem_tmpdir, &opts, &files, &file_count) != 0) {
|
||||
teardown_mem_test_repo();
|
||||
FAIL("discover failed");
|
||||
}
|
||||
|
||||
cbm_gbuf_t *gbuf = cbm_gbuf_new("mem-test", g_mem_tmpdir);
|
||||
cbm_registry_t *reg = cbm_registry_new();
|
||||
atomic_int cancelled;
|
||||
atomic_init(&cancelled, 0);
|
||||
|
||||
cbm_pipeline_ctx_t ctx = {
|
||||
.project_name = "mem-test",
|
||||
.repo_path = g_mem_tmpdir,
|
||||
.gbuf = gbuf,
|
||||
.registry = reg,
|
||||
.cancelled = &cancelled,
|
||||
};
|
||||
|
||||
_Atomic int64_t shared_ids;
|
||||
atomic_init(&shared_ids, cbm_gbuf_next_id(gbuf));
|
||||
|
||||
CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *));
|
||||
ASSERT_NOT_NULL(result_cache);
|
||||
|
||||
cbm_parallel_extract_opts_t extract_opts = {
|
||||
.retain_sources = false,
|
||||
.retain_sources_set = true,
|
||||
.retain_total_budget_bytes = 0,
|
||||
.retain_per_file_max_bytes = 0,
|
||||
};
|
||||
int rc = cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, 2,
|
||||
&extract_opts);
|
||||
ASSERT_EQ(rc, 0);
|
||||
|
||||
int defs_seen = 0;
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
if (result_cache[i]) {
|
||||
ASSERT_EQ(result_cache[i]->source, NULL);
|
||||
defs_seen += result_cache[i]->defs.count;
|
||||
}
|
||||
}
|
||||
ASSERT_GT(defs_seen, 0);
|
||||
ASSERT_GT(cbm_gbuf_node_count(gbuf), 0);
|
||||
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
if (result_cache[i]) {
|
||||
cbm_free_result(result_cache[i]);
|
||||
}
|
||||
}
|
||||
free(result_cache);
|
||||
cbm_registry_free(reg);
|
||||
cbm_gbuf_free(gbuf);
|
||||
cbm_discover_free(files, file_count);
|
||||
teardown_mem_test_repo();
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Guard B (peak bound): a tiny total retention budget must actually bound the
|
||||
* retained source bytes — retained_bytes <= budget — while extraction still
|
||||
* produces defs/nodes. Over-budget files fall back to a bounded re-read during
|
||||
* cross-file resolution (exercised in test_parallel.c), so the cap trades
|
||||
* retained RAM, never correctness. */
|
||||
TEST(parallel_extract_tiny_source_retention_budget) {
|
||||
if (setup_mem_test_repo() != 0) {
|
||||
FAIL("tmpdir setup failed");
|
||||
}
|
||||
|
||||
cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL};
|
||||
cbm_file_info_t *files = NULL;
|
||||
int file_count = 0;
|
||||
if (cbm_discover(g_mem_tmpdir, &opts, &files, &file_count) != 0) {
|
||||
teardown_mem_test_repo();
|
||||
FAIL("discover failed");
|
||||
}
|
||||
|
||||
cbm_gbuf_t *gbuf = cbm_gbuf_new("mem-test", g_mem_tmpdir);
|
||||
cbm_registry_t *reg = cbm_registry_new();
|
||||
atomic_int cancelled;
|
||||
atomic_init(&cancelled, 0);
|
||||
|
||||
cbm_pipeline_ctx_t ctx = {
|
||||
.project_name = "mem-test",
|
||||
.repo_path = g_mem_tmpdir,
|
||||
.gbuf = gbuf,
|
||||
.registry = reg,
|
||||
.cancelled = &cancelled,
|
||||
};
|
||||
|
||||
_Atomic int64_t shared_ids;
|
||||
atomic_init(&shared_ids, cbm_gbuf_next_id(gbuf));
|
||||
|
||||
CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *));
|
||||
ASSERT_NOT_NULL(result_cache);
|
||||
|
||||
const size_t retain_total_budget_bytes = 256;
|
||||
cbm_parallel_extract_opts_t extract_opts = {
|
||||
.retain_sources = true,
|
||||
.retain_sources_set = true,
|
||||
.retain_total_budget_bytes = retain_total_budget_bytes,
|
||||
.retain_per_file_max_bytes = 100U * 1024U * 1024U,
|
||||
};
|
||||
int rc = cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, 2,
|
||||
&extract_opts);
|
||||
ASSERT_EQ(rc, 0);
|
||||
|
||||
int retained_count = 0;
|
||||
size_t retained_bytes = count_retained_source_bytes(result_cache, file_count, &retained_count);
|
||||
int defs_seen = 0;
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
if (result_cache[i]) {
|
||||
defs_seen += result_cache[i]->defs.count;
|
||||
}
|
||||
}
|
||||
|
||||
ASSERT_GT(defs_seen, 0);
|
||||
ASSERT_GT(retained_count, 0);
|
||||
ASSERT_LTE(retained_bytes, retain_total_budget_bytes);
|
||||
ASSERT_GT(cbm_gbuf_node_count(gbuf), 0);
|
||||
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
if (result_cache[i]) {
|
||||
cbm_free_result(result_cache[i]);
|
||||
}
|
||||
}
|
||||
free(result_cache);
|
||||
cbm_registry_free(reg);
|
||||
cbm_gbuf_free(gbuf);
|
||||
cbm_discover_free(files, file_count);
|
||||
teardown_mem_test_repo();
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(parallel_extract_with_slab) {
|
||||
cbm_mem_init(0.5);
|
||||
|
||||
@@ -790,5 +951,7 @@ SUITE(mem) {
|
||||
RUN_TEST(slab_reclaim_with_foreign_live_chunk_is_safe);
|
||||
RUN_TEST(slab_destroy_with_foreign_live_chunk_is_safe);
|
||||
/* Integration */
|
||||
RUN_TEST(parallel_extract_without_source_retention);
|
||||
RUN_TEST(parallel_extract_tiny_source_retention_budget);
|
||||
RUN_TEST(parallel_extract_with_slab);
|
||||
}
|
||||
+142
-6
@@ -112,8 +112,10 @@ static cbm_gbuf_t *run_sequential(const char *project, const char *repo_path,
|
||||
|
||||
/* ── Run parallel pipeline on files, returning gbuf ───────────────── */
|
||||
|
||||
static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_file_info_t *files,
|
||||
int file_count, int worker_count) {
|
||||
static cbm_gbuf_t *run_parallel_with_extract_opts(const char *project, const char *repo_path,
|
||||
cbm_file_info_t *files, int file_count,
|
||||
int worker_count,
|
||||
const cbm_parallel_extract_opts_t *extract_opts) {
|
||||
cbm_gbuf_t *gbuf = cbm_gbuf_new(project, repo_path);
|
||||
cbm_registry_t *reg = cbm_registry_new();
|
||||
atomic_int cancelled;
|
||||
@@ -131,10 +133,15 @@ static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_
|
||||
int64_t gbuf_next = cbm_gbuf_next_id(gbuf);
|
||||
atomic_init(&shared_ids, gbuf_next);
|
||||
|
||||
CBMFileResult **result_cache = calloc(file_count, sizeof(CBMFileResult *));
|
||||
CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *));
|
||||
|
||||
cbm_init();
|
||||
cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count);
|
||||
if (extract_opts) {
|
||||
cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, worker_count,
|
||||
extract_opts);
|
||||
} else {
|
||||
cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count);
|
||||
}
|
||||
cbm_gbuf_set_next_id(gbuf, atomic_load(&shared_ids));
|
||||
|
||||
cbm_build_registry_from_cache(&ctx, files, file_count, result_cache);
|
||||
@@ -175,6 +182,12 @@ static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_
|
||||
return gbuf;
|
||||
}
|
||||
|
||||
static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_file_info_t *files,
|
||||
int file_count, int worker_count) {
|
||||
return run_parallel_with_extract_opts(project, repo_path, files, file_count, worker_count,
|
||||
NULL);
|
||||
}
|
||||
|
||||
/* ── Parity Tests ─────────────────────────────────────────────────── */
|
||||
|
||||
static cbm_gbuf_t *g_seq_gbuf = NULL;
|
||||
@@ -741,8 +754,7 @@ TEST(parallel_lsp_tail_match_fallbacks_gated_to_jvm) {
|
||||
int64_t nid = cbm_gbuf_upsert_node(tgbuf, "Method", "run", "proj.zeta.Helper.run",
|
||||
"zeta/helper.py", 1, 3, NULL);
|
||||
ASSERT_TRUE(nid != 0);
|
||||
ASSERT_TRUE(cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", false) ==
|
||||
NULL);
|
||||
ASSERT_TRUE(cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", false) == NULL);
|
||||
const cbm_gbuf_node_t *jvm_hit =
|
||||
cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", true);
|
||||
ASSERT_NOT_NULL(jvm_hit);
|
||||
@@ -884,6 +896,129 @@ TEST(parallel_python_lsp_override_cross_file_emits_lsp_strategy_edges) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* RED/GREEN A — the graph-quality guarantee behind the low-RAM retention cap.
|
||||
*
|
||||
* The fused cross-file LSP step re-parses each file's source to resolve calls
|
||||
* whose receiver type lives in ANOTHER file (the per-file pass cannot). When
|
||||
* the retention cap drops a file's source, that resolution MUST still happen
|
||||
* via a bounded on-demand re-read; otherwise the cross-file CALLS edge is LOST.
|
||||
*
|
||||
* Fixture: a Java<->Kotlin pair with genuinely cross-language calls that only
|
||||
* the cross-file LSP resolves — JavaCaller.call -> KotlinService.ping (Java ->
|
||||
* Kotlin) and KotlinService.ping -> JavaService.pong (Kotlin -> Java). These
|
||||
* carry the "lsp" strategy and do NOT exist without the cross-file source,
|
||||
* unlike same-file or import-local Python calls which the per-file pass already
|
||||
* resolves (so counting lsp edges on those cannot detect the fallback).
|
||||
*
|
||||
* Three scenarios asserted GREEN with the re-read fallback in place:
|
||||
* 1. CONTROL — default retention: both cross-file edges present. Proves the
|
||||
* fixture genuinely produces them (non-vacuity guard).
|
||||
* 2. NO-RETAIN — retain_sources=false: nothing retained -> edges survive only
|
||||
* via the re-read fallback.
|
||||
* 3. OVER-CAP — per-file cap = 1 byte: every file dropped by the SIZE cap ->
|
||||
* edges survive only via the re-read fallback.
|
||||
* On main (no fallback) scenarios 2 and 3 LOSE both edges = RED; scenario 1
|
||||
* stays present = the non-vacuity control. */
|
||||
TEST(parallel_cross_file_reread_preserves_unretained_edges) {
|
||||
char tmpdir[256];
|
||||
snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_par_xf_reread_XXXXXX");
|
||||
if (!cbm_mkdtemp(tmpdir)) {
|
||||
FAIL("mkdtemp failed");
|
||||
}
|
||||
|
||||
char jpath[512];
|
||||
snprintf(jpath, sizeof(jpath), "%s/src/main/java/com/example/Example.java", tmpdir);
|
||||
char jdir[512];
|
||||
snprintf(jdir, sizeof(jdir), "%s/src/main/java/com/example", tmpdir);
|
||||
cbm_mkdir_p(jdir, 0755);
|
||||
FILE *jf = fopen(jpath, "w");
|
||||
if (!jf) {
|
||||
FAIL("fopen Example.java failed");
|
||||
}
|
||||
fprintf(jf, "package com.example;\n"
|
||||
"\n"
|
||||
"class JavaCaller {\n"
|
||||
" String call(KotlinService kotlinService) {\n"
|
||||
" return kotlinService.ping(new JavaService());\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"class JavaService {\n"
|
||||
" String pong() {\n"
|
||||
" return \"pong\";\n"
|
||||
" }\n"
|
||||
"}\n");
|
||||
fclose(jf);
|
||||
|
||||
char kpath[512];
|
||||
snprintf(kpath, sizeof(kpath), "%s/src/main/kotlin/com/example/KotlinService.kt", tmpdir);
|
||||
char kdir[512];
|
||||
snprintf(kdir, sizeof(kdir), "%s/src/main/kotlin/com/example", tmpdir);
|
||||
cbm_mkdir_p(kdir, 0755);
|
||||
FILE *kf = fopen(kpath, "w");
|
||||
if (!kf) {
|
||||
FAIL("fopen KotlinService.kt failed");
|
||||
}
|
||||
fprintf(kf, "package com.example\n"
|
||||
"\n"
|
||||
"class KotlinService {\n"
|
||||
" fun ping(javaService: JavaService): String {\n"
|
||||
" return javaService.pong()\n"
|
||||
" }\n"
|
||||
"}\n");
|
||||
fclose(kf);
|
||||
|
||||
cbm_file_info_t files[2] = {0};
|
||||
files[0].path = jpath;
|
||||
files[0].rel_path = (char *)"src/main/java/com/example/Example.java";
|
||||
files[0].language = CBM_LANG_JAVA;
|
||||
files[1].path = kpath;
|
||||
files[1].rel_path = (char *)"src/main/kotlin/com/example/KotlinService.kt";
|
||||
files[1].language = CBM_LANG_KOTLIN;
|
||||
|
||||
/* CONTROL (retained) + two drop scenarios that reach the cross-file edge
|
||||
* only via the on-demand re-read: NO-RETAIN disables retention entirely;
|
||||
* OVER-CAP sets a 1-byte per-file cap so every file is dropped by size. */
|
||||
const cbm_parallel_extract_opts_t no_retain = {
|
||||
.retain_sources = false,
|
||||
.retain_sources_set = true,
|
||||
};
|
||||
const cbm_parallel_extract_opts_t over_cap = {
|
||||
.retain_sources = true,
|
||||
.retain_sources_set = true,
|
||||
.retain_per_file_max_bytes = 1, /* 1 byte → every file dropped by the size cap */
|
||||
};
|
||||
const cbm_parallel_extract_opts_t *scenarios[3] = {NULL, &no_retain, &over_cap};
|
||||
|
||||
for (int s = 0; s < 3; s++) {
|
||||
cbm_gbuf_t *gbuf = run_parallel_with_extract_opts("com", tmpdir, files, 2, 2, scenarios[s]);
|
||||
ASSERT_NOT_NULL(gbuf);
|
||||
|
||||
const cbm_gbuf_edge_t *java_to_kotlin =
|
||||
find_calls_edge_by_tails(gbuf, "JavaCaller.call", "KotlinService.ping");
|
||||
const cbm_gbuf_edge_t *kotlin_to_java =
|
||||
find_calls_edge_by_tails(gbuf, "KotlinService.ping", "JavaService.pong");
|
||||
|
||||
/* Both cross-file (Java↔Kotlin) CALLS edges must be present in EVERY
|
||||
* scenario. In the drop scenarios (s=1,2) the caller's source is NOT
|
||||
* retained, so these edges exist ONLY because resolve_worker re-reads
|
||||
* the source on demand. Without that fallback (main) they are LOST. */
|
||||
ASSERT_NOT_NULL(java_to_kotlin);
|
||||
ASSERT_NOT_NULL(kotlin_to_java);
|
||||
/* And they must come from the source-dependent cross-file LSP, not a
|
||||
* source-free suffix heuristic — proving the re-read actually ran. */
|
||||
ASSERT_NOT_NULL(java_to_kotlin->properties_json);
|
||||
ASSERT_NOT_NULL(strstr(java_to_kotlin->properties_json, "\"strategy\":\"lsp"));
|
||||
ASSERT_NOT_NULL(kotlin_to_java->properties_json);
|
||||
ASSERT_NOT_NULL(strstr(kotlin_to_java->properties_json, "\"strategy\":\"lsp"));
|
||||
|
||||
cbm_gbuf_free(gbuf);
|
||||
}
|
||||
|
||||
th_rmtree(tmpdir);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* issue #294: gRPC service-name extraction must (a) preserve the canonical
|
||||
* proto service name (FooServiceClient → FooService, not Foo) and (b) only
|
||||
* match real stub/client types — ordinary receiver vars must NOT produce
|
||||
@@ -941,6 +1076,7 @@ SUITE(parallel) {
|
||||
RUN_TEST(parallel_node_count);
|
||||
RUN_TEST(parallel_python_lsp_override_emits_lsp_strategy_edges);
|
||||
RUN_TEST(parallel_python_lsp_override_cross_file_emits_lsp_strategy_edges);
|
||||
RUN_TEST(parallel_cross_file_reread_preserves_unretained_edges);
|
||||
RUN_TEST(parallel_java_kotlin_lsp_override_cross_file_emits_lsp_strategy_edges);
|
||||
RUN_TEST(parallel_lsp_tail_match_fallbacks_gated_to_jvm);
|
||||
RUN_TEST(parallel_calls_parity);
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
/*
|
||||
* test_semantic.c — Unit tests for semantic.c (pure functions).
|
||||
*
|
||||
* Covers: tokenize, cosine, normalize, vec_add_scaled, random_index,
|
||||
* proximity, diffuse, corpus lifecycle, get_config.
|
||||
*/
|
||||
#include "test_framework.h"
|
||||
#include <semantic/semantic.h>
|
||||
|
||||
#include <math.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* ── Tokenize ────────────────────────────────────────────────────── */
|
||||
|
||||
TEST(sem_tokenize_camel) {
|
||||
char *tokens[32];
|
||||
int n = cbm_sem_tokenize("parseUserInput", tokens, 32);
|
||||
ASSERT_GTE(n, 3);
|
||||
ASSERT_STR_EQ(tokens[0], "parse");
|
||||
ASSERT_STR_EQ(tokens[1], "user");
|
||||
ASSERT_STR_EQ(tokens[2], "input");
|
||||
for (int i = 0; i < n; i++) free(tokens[i]);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_tokenize_snake) {
|
||||
char *tokens[32];
|
||||
int n = cbm_sem_tokenize("handle_http_request", tokens, 32);
|
||||
ASSERT_GTE(n, 3);
|
||||
ASSERT_STR_EQ(tokens[0], "handle");
|
||||
ASSERT_STR_EQ(tokens[1], "http");
|
||||
ASSERT_STR_EQ(tokens[2], "request");
|
||||
for (int i = 0; i < n; i++) free(tokens[i]);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_tokenize_dot) {
|
||||
char *tokens[32];
|
||||
int n = cbm_sem_tokenize("net.http.client", tokens, 32);
|
||||
ASSERT_GTE(n, 3);
|
||||
ASSERT_STR_EQ(tokens[0], "net");
|
||||
ASSERT_STR_EQ(tokens[1], "http");
|
||||
ASSERT_STR_EQ(tokens[2], "client");
|
||||
for (int i = 0; i < n; i++) free(tokens[i]);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_tokenize_null) {
|
||||
int n = cbm_sem_tokenize(NULL, NULL, 0);
|
||||
ASSERT_EQ(n, 0);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_tokenize_max_out) {
|
||||
char *tokens[3];
|
||||
int n = cbm_sem_tokenize("a_b_c_d_e_f_g", tokens, 3);
|
||||
ASSERT_EQ(n, 3);
|
||||
for (int i = 0; i < n; i++) free(tokens[i]);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_tokenize_abbrev_expansion) {
|
||||
char *tokens[32];
|
||||
int n = cbm_sem_tokenize("getCtxErrMsg", tokens, 32);
|
||||
/* get, ctx, context, err, error, msg, message */
|
||||
ASSERT_GTE(n, 4);
|
||||
bool has_ctx = false, has_context = false, has_err = false, has_error = false;
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (strcmp(tokens[i], "ctx") == 0) has_ctx = true;
|
||||
if (strcmp(tokens[i], "context") == 0) has_context = true;
|
||||
if (strcmp(tokens[i], "err") == 0) has_err = true;
|
||||
if (strcmp(tokens[i], "error") == 0) has_error = true;
|
||||
}
|
||||
ASSERT_TRUE(has_ctx && has_context && has_err && has_error);
|
||||
for (int i = 0; i < n; i++) free(tokens[i]);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Cosine similarity ───────────────────────────────────────────── */
|
||||
|
||||
static void fill_vec(cbm_sem_vec_t *v, float val) {
|
||||
for (int i = 0; i < CBM_SEM_DIM; i++) v->v[i] = val;
|
||||
}
|
||||
|
||||
TEST(sem_cosine_identical) {
|
||||
cbm_sem_vec_t a, b;
|
||||
fill_vec(&a, 0.5f);
|
||||
fill_vec(&b, 0.5f);
|
||||
float sim = cbm_sem_cosine(&a, &b);
|
||||
ASSERT_FLOAT_EQ(sim, 1.0f, 0.001f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_cosine_orthogonal) {
|
||||
cbm_sem_vec_t a, b;
|
||||
memset(&a, 0, sizeof(a));
|
||||
memset(&b, 0, sizeof(b));
|
||||
a.v[0] = 1.0f;
|
||||
b.v[1] = 1.0f;
|
||||
float sim = cbm_sem_cosine(&a, &b);
|
||||
ASSERT_FLOAT_EQ(sim, 0.0f, 0.001f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_cosine_zero_vector) {
|
||||
cbm_sem_vec_t a, b;
|
||||
memset(&a, 0, sizeof(a));
|
||||
fill_vec(&b, 1.0f);
|
||||
float sim = cbm_sem_cosine(&a, &b);
|
||||
ASSERT_FLOAT_EQ(sim, 0.0f, 0.001f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_cosine_negative) {
|
||||
cbm_sem_vec_t a, b;
|
||||
memset(&a, 0, sizeof(a));
|
||||
memset(&b, 0, sizeof(b));
|
||||
a.v[0] = 1.0f;
|
||||
b.v[0] = -1.0f;
|
||||
float sim = cbm_sem_cosine(&a, &b);
|
||||
ASSERT_FLOAT_EQ(sim, -1.0f, 0.001f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_cosine_null) {
|
||||
ASSERT_FLOAT_EQ(cbm_sem_cosine(NULL, NULL), 0.0f, 0.001f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Normalize ───────────────────────────────────────────────────── */
|
||||
|
||||
TEST(sem_normalize_unit) {
|
||||
cbm_sem_vec_t v;
|
||||
memset(&v, 0, sizeof(v));
|
||||
v.v[0] = 1.0f;
|
||||
cbm_sem_normalize(&v);
|
||||
ASSERT_FLOAT_EQ(cbm_sem_cosine(&v, &v), 1.0f, 0.001f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_normalize_scales) {
|
||||
cbm_sem_vec_t v;
|
||||
fill_vec(&v, 2.0f);
|
||||
cbm_sem_normalize(&v);
|
||||
float mag_sq = 0.0f;
|
||||
for (int i = 0; i < CBM_SEM_DIM; i++) mag_sq += v.v[i] * v.v[i];
|
||||
float mag = sqrtf(mag_sq);
|
||||
ASSERT_FLOAT_EQ(mag, 1.0f, 0.01f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_normalize_zero) {
|
||||
cbm_sem_vec_t v;
|
||||
memset(&v, 0, sizeof(v));
|
||||
cbm_sem_normalize(&v);
|
||||
/* Should remain zero (no division by zero) */
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_normalize_null) {
|
||||
cbm_sem_normalize(NULL); /* should not crash */
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Vec add scaled ──────────────────────────────────────────────── */
|
||||
|
||||
TEST(sem_vec_add_scaled_basic) {
|
||||
cbm_sem_vec_t dst;
|
||||
memset(&dst, 0, sizeof(dst));
|
||||
cbm_sem_vec_t src;
|
||||
fill_vec(&src, 1.0f);
|
||||
cbm_sem_vec_add_scaled(&dst, &src, 0.5f);
|
||||
ASSERT_FLOAT_EQ(dst.v[0], 0.5f, 0.001f);
|
||||
ASSERT_FLOAT_EQ(dst.v[CBM_SEM_DIM - 1], 0.5f, 0.001f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_vec_add_scaled_null) {
|
||||
cbm_sem_vec_t v;
|
||||
fill_vec(&v, 1.0f);
|
||||
cbm_sem_vec_add_scaled(NULL, &v, 1.0f); /* should not crash */
|
||||
cbm_sem_vec_add_scaled(&v, NULL, 1.0f); /* should not crash */
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Random index ────────────────────────────────────────────────── */
|
||||
|
||||
TEST(sem_random_index_deterministic) {
|
||||
cbm_sem_vec_t a, b;
|
||||
cbm_sem_random_index("hello", &a);
|
||||
cbm_sem_random_index("hello", &b);
|
||||
ASSERT_FLOAT_EQ(cbm_sem_cosine(&a, &b), 1.0f, 0.001f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_random_index_different_tokens) {
|
||||
cbm_sem_vec_t a, b;
|
||||
cbm_sem_random_index("function", &a);
|
||||
cbm_sem_random_index("variable", &b);
|
||||
/* Different tokens should produce different vectors */
|
||||
float sim = cbm_sem_cosine(&a, &b);
|
||||
ASSERT_TRUE(sim < 1.0f - 1e-6f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_random_index_null) {
|
||||
cbm_sem_vec_t v;
|
||||
memset(&v, 0, sizeof(v));
|
||||
cbm_sem_random_index(NULL, &v);
|
||||
/* Should produce zero vector for NULL token */
|
||||
for (int i = 0; i < CBM_SEM_DIM; i++) {
|
||||
ASSERT_FLOAT_EQ(v.v[i], 0.0f, 0.001f);
|
||||
}
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Proximity ───────────────────────────────────────────────────── */
|
||||
|
||||
TEST(sem_proximity_same_file) {
|
||||
float p = cbm_sem_proximity("src/main.c", "src/main.c");
|
||||
ASSERT_FLOAT_EQ(p, 1.1f, 0.01f); /* CBM_SEM_UNIT_POS + CBM_SEM_PROX_MAX_BOOST */
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_proximity_same_dir) {
|
||||
/* Files sharing 1 of 2 directory components: ratio = 0.5 → 1.0 + 0.5*0.10 = 1.05 */
|
||||
float p = cbm_sem_proximity("src/core/a.c", "src/io/b.c");
|
||||
ASSERT_TRUE(p > 1.0f && p < 1.10f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_proximity_different_paths) {
|
||||
float p = cbm_sem_proximity("src/foo/a.c", "tests/bar/b.c");
|
||||
ASSERT_FLOAT_EQ(p, 1.0f, 0.01f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_proximity_null) {
|
||||
ASSERT_FLOAT_EQ(cbm_sem_proximity(NULL, "foo.c"), 1.0f, 0.01f);
|
||||
ASSERT_FLOAT_EQ(cbm_sem_proximity("foo.c", NULL), 1.0f, 0.01f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Diffuse ─────────────────────────────────────────────────────── */
|
||||
|
||||
TEST(sem_diffuse_zero_neighbors) {
|
||||
cbm_sem_vec_t v;
|
||||
fill_vec(&v, 0.5f);
|
||||
cbm_sem_diffuse(&v, NULL, 0, 0.3f);
|
||||
/* With zero neighbors, vector should be unchanged */
|
||||
ASSERT_FLOAT_EQ(v.v[0], 0.5f, 0.001f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_diffuse_single_neighbor) {
|
||||
cbm_sem_vec_t v;
|
||||
memset(&v, 0, sizeof(v));
|
||||
v.v[0] = 0.5f;
|
||||
v.v[1] = 0.5f;
|
||||
cbm_sem_normalize(&v); /* unit-length input */
|
||||
cbm_sem_vec_t nb;
|
||||
memset(&nb, 0, sizeof(nb));
|
||||
nb.v[0] = 1.0f;
|
||||
cbm_sem_normalize(&nb);
|
||||
cbm_sem_diffuse(&v, &nb, 1, 0.3f);
|
||||
/* After diffuse+normalize, result should still be unit-length */
|
||||
float mag_sq = 0.0f;
|
||||
for (int i = 0; i < CBM_SEM_DIM; i++) mag_sq += v.v[i] * v.v[i];
|
||||
ASSERT_FLOAT_EQ(sqrtf(mag_sq), 1.0f, 0.01f);
|
||||
/* Component 0 should be pulled toward neighbor's strong dim-0 */
|
||||
ASSERT_TRUE(v.v[0] > 0.0f);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Corpus lifecycle ────────────────────────────────────────────── */
|
||||
|
||||
TEST(sem_corpus_new_free) {
|
||||
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
|
||||
ASSERT_NOT_NULL(c);
|
||||
ASSERT_EQ(cbm_sem_corpus_doc_count(c), 0);
|
||||
ASSERT_EQ(cbm_sem_corpus_token_count(c), 0);
|
||||
cbm_sem_corpus_free(c);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_corpus_add_one_doc) {
|
||||
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
|
||||
ASSERT_NOT_NULL(c);
|
||||
const char *tokens[] = {"parse", "user", "input"};
|
||||
cbm_sem_corpus_add_doc(c, tokens, 3);
|
||||
ASSERT_EQ(cbm_sem_corpus_doc_count(c), 1);
|
||||
ASSERT_TRUE(cbm_sem_corpus_token_count(c) > 0);
|
||||
cbm_sem_corpus_free(c);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_corpus_idf) {
|
||||
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
|
||||
ASSERT_NOT_NULL(c);
|
||||
const char *doc1[] = {"a", "b", "c"};
|
||||
const char *doc2[] = {"a", "d", "e"};
|
||||
cbm_sem_corpus_add_doc(c, doc1, 3);
|
||||
cbm_sem_corpus_add_doc(c, doc2, 3);
|
||||
/* IDF for "a" (appears in 2 docs): log(2/2) = log(1) = 0 */
|
||||
float idf_a = cbm_sem_corpus_idf(c, "a");
|
||||
ASSERT_TRUE(idf_a < 0.01f);
|
||||
/* IDF for "b" (appears in 1 doc): log(2/1) > 0 */
|
||||
float idf_b = cbm_sem_corpus_idf(c, "b");
|
||||
ASSERT_TRUE(idf_b > 0.0f);
|
||||
cbm_sem_corpus_free(c);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_corpus_add_null_doc) {
|
||||
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
|
||||
ASSERT_NOT_NULL(c);
|
||||
cbm_sem_corpus_add_doc(c, NULL, 0);
|
||||
cbm_sem_corpus_add_doc(c, NULL, -1);
|
||||
ASSERT_EQ(cbm_sem_corpus_doc_count(c), 0);
|
||||
cbm_sem_corpus_free(c);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(sem_corpus_free_null) {
|
||||
cbm_sem_corpus_free(NULL); /* should not crash */
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Config ──────────────────────────────────────────────────────── */
|
||||
|
||||
TEST(sem_get_config_defaults) {
|
||||
cbm_sem_config_t cfg = cbm_sem_get_config();
|
||||
ASSERT_TRUE(cfg.w_tfidf > 0.0f);
|
||||
ASSERT_TRUE(cfg.w_ri > 0.0f);
|
||||
ASSERT_TRUE(cfg.threshold > 0.0f);
|
||||
ASSERT_TRUE(cfg.max_edges > 0);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Suite ───────────────────────────────────────────────────────── */
|
||||
|
||||
SUITE(semantic) {
|
||||
RUN_TEST(sem_tokenize_camel);
|
||||
RUN_TEST(sem_tokenize_snake);
|
||||
RUN_TEST(sem_tokenize_dot);
|
||||
RUN_TEST(sem_tokenize_null);
|
||||
RUN_TEST(sem_tokenize_max_out);
|
||||
RUN_TEST(sem_tokenize_abbrev_expansion);
|
||||
RUN_TEST(sem_cosine_identical);
|
||||
RUN_TEST(sem_cosine_orthogonal);
|
||||
RUN_TEST(sem_cosine_zero_vector);
|
||||
RUN_TEST(sem_cosine_negative);
|
||||
RUN_TEST(sem_cosine_null);
|
||||
RUN_TEST(sem_normalize_unit);
|
||||
RUN_TEST(sem_normalize_scales);
|
||||
RUN_TEST(sem_normalize_zero);
|
||||
RUN_TEST(sem_normalize_null);
|
||||
RUN_TEST(sem_vec_add_scaled_basic);
|
||||
RUN_TEST(sem_vec_add_scaled_null);
|
||||
RUN_TEST(sem_random_index_deterministic);
|
||||
RUN_TEST(sem_random_index_different_tokens);
|
||||
RUN_TEST(sem_random_index_null);
|
||||
RUN_TEST(sem_proximity_same_file);
|
||||
RUN_TEST(sem_proximity_same_dir);
|
||||
RUN_TEST(sem_proximity_different_paths);
|
||||
RUN_TEST(sem_proximity_null);
|
||||
RUN_TEST(sem_diffuse_zero_neighbors);
|
||||
RUN_TEST(sem_diffuse_single_neighbor);
|
||||
RUN_TEST(sem_corpus_new_free);
|
||||
RUN_TEST(sem_corpus_add_one_doc);
|
||||
RUN_TEST(sem_corpus_idf);
|
||||
RUN_TEST(sem_corpus_add_null_doc);
|
||||
RUN_TEST(sem_corpus_free_null);
|
||||
RUN_TEST(sem_get_config_defaults);
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
* test_slab_alloc.c — Unit tests for slab_alloc.c.
|
||||
*
|
||||
* Covers: slab malloc/free/calloc/realloc for ≤64B and >64B paths,
|
||||
* realloc grow/shrink, zeroing, ownership, thread reset.
|
||||
*/
|
||||
#include "test_framework.h"
|
||||
#include <foundation/slab_alloc.h>
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* ── Constants from slab_alloc.c ─────────────────────────────────── */
|
||||
enum { SLAB_CHUNK_SIZE = 64 };
|
||||
|
||||
/* ── Alloc → free (small, slab path) ─────────────────────────────── */
|
||||
|
||||
TEST(slab_malloc_free_small) {
|
||||
void *p = cbm_slab_test_malloc(32);
|
||||
ASSERT_NOT_NULL(p);
|
||||
memset(p, 0xAA, 32);
|
||||
cbm_slab_test_free(p);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(slab_malloc_free_large) {
|
||||
void *p = cbm_slab_test_malloc(128);
|
||||
ASSERT_NOT_NULL(p);
|
||||
memset(p, 0xBB, 128);
|
||||
cbm_slab_test_free(p);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(slab_malloc_zero_size) {
|
||||
/* malloc(0) should return non-NULL on most platforms */
|
||||
void *p = cbm_slab_test_malloc(0);
|
||||
ASSERT_NOT_NULL(p);
|
||||
cbm_slab_test_free(p);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Calloc ──────────────────────────────────────────────────────── */
|
||||
|
||||
TEST(slab_calloc_zeros) {
|
||||
void *p = cbm_slab_test_calloc(10, 8);
|
||||
ASSERT_NOT_NULL(p);
|
||||
unsigned char *b = p;
|
||||
for (int i = 0; i < 80; i++) {
|
||||
ASSERT_EQ(b[i], 0);
|
||||
}
|
||||
cbm_slab_test_free(p);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(slab_calloc_overflows) {
|
||||
void *p = cbm_slab_test_calloc(SIZE_MAX, 2);
|
||||
ASSERT_NULL(p);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Realloc ─────────────────────────────────────────────────────── */
|
||||
|
||||
TEST(slab_realloc_null_is_malloc) {
|
||||
void *p = cbm_slab_test_realloc(NULL, 64);
|
||||
ASSERT_NOT_NULL(p);
|
||||
cbm_slab_test_free(p);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(slab_realloc_shrink_to_zero) {
|
||||
void *p = cbm_slab_test_malloc(32);
|
||||
ASSERT_NOT_NULL(p);
|
||||
void *q = cbm_slab_test_realloc(p, 0); /* frees and returns NULL */
|
||||
ASSERT_NULL(q);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(slab_realloc_small_keep_same) {
|
||||
void *p = cbm_slab_test_malloc(16);
|
||||
ASSERT_NOT_NULL(p);
|
||||
memset(p, 0xCC, 16);
|
||||
/* Realloc within slab size: same pointer, content preserved */
|
||||
void *q = cbm_slab_test_realloc(p, 32);
|
||||
ASSERT_EQ(p, q);
|
||||
unsigned char *b = q;
|
||||
ASSERT_EQ(b[0], 0xCC);
|
||||
cbm_slab_test_free(q);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(slab_realloc_promote_to_heap) {
|
||||
void *p = cbm_slab_test_malloc(32);
|
||||
ASSERT_NOT_NULL(p);
|
||||
memset(p, 0xDD, 32);
|
||||
/* Realloc above slab threshold: new ptr, content copied */
|
||||
void *q = cbm_slab_test_realloc(p, 128);
|
||||
ASSERT_NOT_NULL(q);
|
||||
unsigned char *b = q;
|
||||
ASSERT_EQ(b[0], 0xDD);
|
||||
ASSERT_EQ(b[31], 0xDD);
|
||||
cbm_slab_test_free(q);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Multiple allocations (free list cycling) ────────────────────── */
|
||||
|
||||
TEST(slab_many_small_allocs) {
|
||||
void *ptrs[64];
|
||||
for (int i = 0; i < 64; i++) {
|
||||
ptrs[i] = cbm_slab_test_malloc(SLAB_CHUNK_SIZE);
|
||||
ASSERT_NOT_NULL(ptrs[i]);
|
||||
memset(ptrs[i], (int)(i & 0xFF), SLAB_CHUNK_SIZE);
|
||||
}
|
||||
/* Free all: they go back to free list */
|
||||
for (int i = 0; i < 64; i++) {
|
||||
cbm_slab_test_free(ptrs[i]);
|
||||
}
|
||||
/* Re-allocate from free list: same pointer pool recycled */
|
||||
void *again = cbm_slab_test_malloc(SLAB_CHUNK_SIZE);
|
||||
ASSERT_NOT_NULL(again);
|
||||
cbm_slab_test_free(again);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Distinct pointer addresses ──────────────────────────────────── */
|
||||
|
||||
TEST(slab_distinct_allocations) {
|
||||
void *a = cbm_slab_test_malloc(SLAB_CHUNK_SIZE);
|
||||
void *b = cbm_slab_test_malloc(SLAB_CHUNK_SIZE);
|
||||
void *c = cbm_slab_test_malloc(SLAB_CHUNK_SIZE);
|
||||
ASSERT_NOT_NULL(a);
|
||||
ASSERT_NOT_NULL(b);
|
||||
ASSERT_NOT_NULL(c);
|
||||
ASSERT_NEQ(a, b);
|
||||
ASSERT_NEQ(b, c);
|
||||
ASSERT_NEQ(a, c);
|
||||
cbm_slab_test_free(a);
|
||||
cbm_slab_test_free(b);
|
||||
cbm_slab_test_free(c);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Large allocation (heap path, >64B) ──────────────────────────── */
|
||||
|
||||
TEST(slab_large_alloc_free) {
|
||||
void *p = cbm_slab_test_malloc(1024);
|
||||
ASSERT_NOT_NULL(p);
|
||||
cbm_slab_test_free(p);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(slab_large_calloc_zero) {
|
||||
void *p = cbm_slab_test_calloc(64, 16);
|
||||
ASSERT_NOT_NULL(p);
|
||||
unsigned char *b = p;
|
||||
ASSERT_EQ(b[0], 0);
|
||||
ASSERT_EQ(b[1023], 0);
|
||||
cbm_slab_test_free(p);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Reset thread ────────────────────────────────────────────────── */
|
||||
|
||||
TEST(slab_reset_thread) {
|
||||
/* Allocate then reset: should not crash */
|
||||
void *p = cbm_slab_test_malloc(32);
|
||||
ASSERT_NOT_NULL(p);
|
||||
cbm_slab_test_free(p);
|
||||
cbm_slab_reset_thread();
|
||||
/* After reset, new allocs should still work */
|
||||
void *q = cbm_slab_test_malloc(32);
|
||||
ASSERT_NOT_NULL(q);
|
||||
cbm_slab_test_free(q);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Suite ───────────────────────────────────────────────────────── */
|
||||
|
||||
SUITE(slab_alloc) {
|
||||
RUN_TEST(slab_malloc_free_small);
|
||||
RUN_TEST(slab_malloc_free_large);
|
||||
RUN_TEST(slab_malloc_zero_size);
|
||||
RUN_TEST(slab_calloc_zeros);
|
||||
RUN_TEST(slab_calloc_overflows);
|
||||
RUN_TEST(slab_realloc_null_is_malloc);
|
||||
RUN_TEST(slab_realloc_shrink_to_zero);
|
||||
RUN_TEST(slab_realloc_small_keep_same);
|
||||
RUN_TEST(slab_realloc_promote_to_heap);
|
||||
RUN_TEST(slab_many_small_allocs);
|
||||
RUN_TEST(slab_distinct_allocations);
|
||||
RUN_TEST(slab_large_alloc_free);
|
||||
RUN_TEST(slab_large_calloc_zero);
|
||||
RUN_TEST(slab_reset_thread);
|
||||
}
|
||||
Reference in new issue
Block a user