From 968d4a4be9f69a34a2654bbc6add8a4b3e50da4f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 7 Oct 2026 18:05:49 +0200 Subject: [PATCH] fix(semantic): TF-IDF compares vocabulary, not token positions The TF-IDF signal keyed each term by the token's POSITION in the function, so any two functions of similar length shared their first terms whatever their words: 0.20 of every SEMANTICALLY_RELATED score measured length, not vocabulary. Terms are now keyed by corpus token (cbm_sem_tfidf_terms: one term per distinct word, weight term frequency x idf, ascending token index, which the sparse cosine merges on). Measured on a blind judged sample: 222 function pairs from six held-out repositories, stratified by score band, read in three rounds. The share of admitted SEMANTICALLY_RELATED edges a reader judges related rises from 0.713 to 0.742, with about 19% fewer edges (fastapi 507 -> 426, django 64 -> 66). The same sample shows that no score threshold reaches 0.90 precision, so the documented "~95% at 0.75" did not hold before this change either. The harness that measured it ships with the fix, off by default: CBM_SEM_PAIR_SIGNALS= writes one line per scored pair (both qualified names, the score, every signal value, whether the pair was admitted, both locations); CBM_SEM_PAIR_SIGNALS_FLOOR also records pairs below the threshold, never admitting them. cbm_sem_combined_score is now cbm_sem_signal_values + cbm_sem_combine (same arithmetic, same order). CBM_SEMANTIC_INDEX_VERSION 3 -> 4: indexes built before score pairs differently. Tests: sem_tfidf_terms_compare_vocabulary_not_positions, sem_combine_weighs_signals, pipeline_semantic_pair_signals_never_change_the_graph; each is RED with its behavior reverted. Signed-off-by: Martin Vogel --- src/pipeline/pass_semantic_edges.c | 95 ++++++++++++++++------ src/pipeline/pipeline_internal.h | 9 ++- src/semantic/semantic.c | 122 ++++++++++++++++++++--------- src/semantic/semantic.h | 31 ++++++++ tests/test_pipeline.c | 87 ++++++++++++++++++++ tests/test_semantic.c | 107 +++++++++++++++++++++++++ 6 files changed, 390 insertions(+), 61 deletions(-) diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index 14597289e..a0bcb636d 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -71,6 +71,7 @@ typedef struct { int64_t target_id; float score; bool same_file; + bool below; /* under the threshold: recorded for CBM_SEM_PAIR_SIGNALS only, never admitted */ /* Canonical admission keys (determinism): func index of the discovering * side, its candidate rank, and the partner's func index. The sequential * admission pass replays pairs in (i, c) order so which pairs win the @@ -93,7 +94,7 @@ static void deferred_buf_init(deferred_edge_buf_t *buf) { } static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt, float score, - bool same_file, int i, int j, int c) { + bool same_file, bool below, int i, int j, int c) { if (buf->count >= buf->cap) { int nc = buf->cap < CBM_SZ_256 ? CBM_SZ_256 : buf->cap * GROW; deferred_edge_t *grown = @@ -108,6 +109,7 @@ static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt .target_id = tgt, .score = score, .same_file = same_file, + .below = below, .i = i, .j = j, .c = c}; @@ -724,19 +726,10 @@ static void vec_build_worker(int worker_id, void *ctx_ptr) { token_index[t] = cbm_sem_corpus_token_index(vc->corpus, tokens[t]); } - /* TF-IDF weights */ + /* TF-IDF terms, keyed by corpus token (cbm_sem_tfidf_terms). */ int *indices = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(int)); float *weights = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(float)); - int tfidf_len = 0; - for (int t = 0; t < tc; t++) { - float idf = token_index ? cbm_sem_corpus_idf_at(vc->corpus, token_index[t]) - : cbm_sem_corpus_idf(vc->corpus, tokens[t]); - if (idf > 0.0F) { - indices[tfidf_len] = t; - weights[tfidf_len] = idf; - tfidf_len++; - } - } + int tfidf_len = cbm_sem_tfidf_terms(vc->corpus, tokens, token_index, tc, indices, weights); vc->funcs[f].tfidf_indices = indices; vc->funcs[f].tfidf_weights = weights; vc->funcs[f].tfidf_len = tfidf_len; @@ -833,6 +826,7 @@ typedef struct { uint64_t *signatures; int *edge_counts; /* budget applied sequentially in phase6b (determinism) */ cbm_sem_config_t cfg; + float record_floor; /* kept from here up; under the threshold: signals dump only */ int func_count; /* LSH buckets (read-only during scoring) */ @@ -927,13 +921,13 @@ static void score_try_emit(score_ctx_t *sc, int i, int j, int c, deferred_edge_b return; } float score = cbm_sem_combined_score(&sc->funcs[i], &sc->funcs[j], &sc->cfg); - if (score < sc->cfg.threshold) { + if (score < sc->record_floor) { return; } bool same_file = sc->funcs[i].file_path && sc->funcs[j].file_path && strcmp(sc->funcs[i].file_path, sc->funcs[j].file_path) == 0; - deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file, i, j, - c); + deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file, + score < sc->cfg.threshold, i, j, c); } static void score_worker(int worker_id, void *ctx_ptr) { @@ -1138,8 +1132,31 @@ static int cmp_deferred_edge_canonical(const void *pa, const void *pb) { * and apply the per-node max_edges budget HERE — single-threaded — so the * admitted edge set is a pure function of the (canonically sorted) inputs, * independent of worker count and scheduling. */ +/* CBM_SEM_PAIR_SIGNALS=: one line per recorded pair (from the record + * floor up, CBM_SEM_PAIR_SIGNALS_FLOOR, default the threshold), in canonical + * order: both qualified names, the score, every signal value and whether the + * pair was admitted. The measuring harness for the semantic engine: judged + * samples are drawn from it, stratified by score. Off unless set. */ +static void pair_signals_line(FILE *f, const cbm_gbuf_t *gbuf, const cbm_sem_func_t *funcs, + const deferred_edge_t *de, bool admitted) { + cbm_sem_signals_t s; + cbm_sem_signal_values(&funcs[de->i], &funcs[de->j], &s); + const cbm_gbuf_node_t *a = cbm_gbuf_find_by_id(gbuf, de->source_id); + const cbm_gbuf_node_t *b = cbm_gbuf_find_by_id(gbuf, de->target_id); + (void)fprintf( + f, + "%s\t%s\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.4f\t%d\t%d" + "\t%s:%d-%d\t%s:%d-%d\n", + a && a->qualified_name ? a->qualified_name : "?", + b && b->qualified_name ? b->qualified_name : "?", (double)de->score, (double)s.tfidf, + (double)s.ri, (double)s.minhash, (double)s.api, (double)s.type, (double)s.decorator, + (double)s.struct_profile, (double)s.proximity, de->same_file ? 1 : 0, admitted ? 1 : 0, + a && a->file_path ? a->file_path : "?", a ? a->start_line : 0, a ? a->end_line : 0, + b && b->file_path ? b->file_path : "?", b ? b->start_line : 0, b ? b->end_line : 0); +} + static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_bufs, int worker_count, - int *edge_counts, int max_edges) { + int *edge_counts, int max_edges, const cbm_sem_func_t *funcs) { int total_pairs = 0; for (int w = 0; w < worker_count; w++) { total_pairs += worker_bufs[w].count; @@ -1175,20 +1192,34 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf * chase the near-threshold SEMANTICALLY_RELATED flicker: find the pair that * flips, then compare its score between runs. */ FILE *pair_dump = NULL; + FILE *signals = NULL; { char dump_path[CBM_SZ_1K]; if (cbm_safe_getenv("CBM_SEM_PAIR_DUMP", dump_path, sizeof(dump_path), NULL)) { pair_dump = cbm_fopen(dump_path, "w"); } + if (funcs && cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS", dump_path, sizeof(dump_path), NULL)) { + signals = cbm_fopen(dump_path, "w"); + } } int total_edges = 0; for (int e = 0; e < n; e++) { deferred_edge_t *de = &pairs[e]; + if (de->below) { + if (signals) { + pair_signals_line(signals, gbuf, funcs, de, false); + } + continue; + } if (pair_dump) { fprintf(pair_dump, "%d %d %.9g\n", de->i, de->j, (double)de->score); } - if (edge_counts[de->i] >= max_edges || edge_counts[de->j] >= max_edges) { + bool admitted = edge_counts[de->i] < max_edges && edge_counts[de->j] < max_edges; + if (signals) { + pair_signals_line(signals, gbuf, funcs, de, admitted); + } + if (!admitted) { continue; } char props[PROPS_BUF]; @@ -1202,6 +1233,9 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf if (pair_dump) { (void)fclose(pair_dump); } + if (signals) { + (void)fclose(signals); + } cbm_free(CBM_MEM_CLASS_SEMANTIC, pairs); return total_edges; } @@ -1542,13 +1576,14 @@ static void sem_corpus_dump(const cbm_sem_corpus_t *corpus, const char *suffix) /* Phase 6a: score candidate pairs in parallel and collect deferred edges. */ static void phase6a_score_candidates(cbm_sem_func_t *funcs, uint64_t *signatures, int *edge_counts, sem_bucket_t **band_buckets, cbm_sem_config_t cfg, - deferred_edge_buf_t *worker_bufs, int func_count, - int worker_count) { + float record_floor, deferred_edge_buf_t *worker_bufs, + int func_count, int worker_count) { score_ctx_t sc = { .funcs = funcs, .signatures = signatures, .edge_counts = edge_counts, .cfg = cfg, + .record_floor = record_floor, .func_count = func_count, .band_buckets = (void *)band_buckets, .worker_bufs = worker_bufs, @@ -1619,14 +1654,30 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t * deferred_buf_init(&worker_bufs[w]); } + /* The measuring harness may record pairs below the threshold (never + * admitted): CBM_SEM_PAIR_SIGNALS_FLOOR, only with CBM_SEM_PAIR_SIGNALS. */ + float record_floor = cfg.threshold; + { + char path[CBM_SZ_1K]; /* the dump path: a buffer too short reads as unset */ + char buf[CBM_SZ_64]; + if (cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS", path, sizeof(path), NULL) && + cbm_safe_getenv("CBM_SEM_PAIR_SIGNALS_FLOOR", buf, sizeof(buf), NULL)) { + float floor_value = strtof(buf, NULL); + if (floor_value > 0.0F && floor_value < cfg.threshold) { + record_floor = floor_value; + } + } + } + CBM_PROF_START(t_phase6a); sem_state_dump(funcs, func_count); - phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, worker_bufs, - func_count, worker_count); + phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, record_floor, + worker_bufs, func_count, worker_count); CBM_PROF_END_N("semantic_edges", "6a_score_parallel", t_phase6a, func_count); CBM_PROF_START(t_phase6b); - int total = phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges); + int total = + phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges, funcs); CBM_PROF_END_N("semantic_edges", "6b_edge_merge_seq", t_phase6b, total); cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 87b9c160b..b00dcd699 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1274,8 +1274,13 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr * sections and PDF pages hold MENTIONS edges into the code, ADR records * are nodes, and AsciiDoc files, PDFs and a Sphinx project's extra * source suffixes are indexed. An index built before has none of this - * for its unchanged documents, so it rebuilds once on upgrade. */ -enum { CBM_SEMANTIC_INDEX_VERSION = 6 }; + * for its unchanged documents, so it rebuilds once on upgrade. + * 7: SEMANTICALLY_RELATED changed: TF-IDF compares vocabulary, string + * properties are read with their escapes, admission is best-first and + * every edge carries "p"; doc_link_candidates holds doc section -> code + * candidates. An index built before has the old pairs and no + * candidates, so it rebuilds once on upgrade. */ +enum { CBM_SEMANTIC_INDEX_VERSION = 7 }; typedef struct { cbm_gbuf_t *gbuf; diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c index 2aa1df5a3..c28615d93 100644 --- a/src/semantic/semantic.c +++ b/src/semantic/semantic.c @@ -1619,6 +1619,47 @@ static float small_cosine(const float *a, const float *b, int dims) { return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom; } +static int cmp_int_asc(const void *pa, const void *pb) { + int a = *(const int *)pa; + int b = *(const int *)pb; + return (a > b) - (a < b); +} + +/* Terms are keyed by corpus token, so two functions share a term only when + * they share the word. (Until 2026-10 the key was the token's POSITION in the + * function: any two functions of similar length "shared" their first terms, + * and the TF-IDF signal measured length, not vocabulary.) */ +int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index, + int count, int *indices, float *weights) { + if (!corpus || !indices || !weights || count <= 0) { + return 0; + } + int known = 0; + for (int t = 0; t < count; t++) { + int idx = token_index ? token_index[t] : cbm_sem_corpus_token_index(corpus, tokens[t]); + if (idx >= 0 && cbm_sem_corpus_idf_at(corpus, idx) > 0.0F) { + indices[known++] = idx; + } + } + if (known > 1) { + qsort(indices, (size_t)known, sizeof(indices[0]), cmp_int_asc); + } + /* Each run of one index becomes one term; its weight is the idf added once + * per occurrence (equal indices have equal idf). */ + int terms = 0; + for (int k = 0; k < known; k++) { + float idf = cbm_sem_corpus_idf_at(corpus, indices[k]); + if (terms > 0 && indices[terms - SKIP_ONE] == indices[k]) { + weights[terms - SKIP_ONE] += idf; + } else { + indices[terms] = indices[k]; + weights[terms] = idf; + terms++; + } + } + return terms; +} + /* Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when * either side is empty or the magnitude product is below the epsilon guard. */ static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) { @@ -1651,49 +1692,47 @@ static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t * return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F; } -float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, - const cbm_sem_config_t *cfg) { - if (!a || !b || !cfg) { - return 0.0F; +void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b, + cbm_sem_signals_t *out) { + memset(out, 0, sizeof(*out)); + out->proximity = CBM_SEM_UNIT_POS; + if (!a || !b) { + return; } + if (a->has_minhash && b->has_minhash) { + double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, + (const cbm_minhash_t *)b->minhash); + out->minhash = (float)j; + out->near_copy = j >= CBM_MINHASH_JACCARD_THRESHOLD; + } + out->tfidf = sparse_tfidf_cosine(a, b); + out->ri = cbm_rsq_ip(&a->ri_code, &b->ri_code); /* signal 2: Random Indexing */ + out->api = cbm_rsq_ip(&a->api_code, &b->api_code); /* signal 4: API signatures */ + out->type = cbm_rsq_ip(&a->type_code, &b->type_code); /* signal 5: type signatures */ + out->decorator = cbm_rsq_ip(&a->deco_code, &b->deco_code); /* signal 7: decorators */ + /* signals 8+9+11: structural profile + data flow + Halstead */ + out->struct_profile = + small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS); + out->proximity = cbm_sem_proximity(a->file_path, b->file_path); /* signal 6: multiplier */ +} +float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg) { /* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold, * the pass_similarity pipeline already emitted a SIMILAR_TO edge for this pair. * Returning 0 here avoids flooding top-k with cross-service copy-paste boilerplate * (logging_middleware, shared push/pull handlers) that SIMILAR_TO already covers, * freeing the edge budget for true semantic leaps (vocabulary-bridged relations). */ - if (a->has_minhash && b->has_minhash) { - double early_j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, - (const cbm_minhash_t *)b->minhash); - if (early_j >= CBM_MINHASH_JACCARD_THRESHOLD) { - return 0.0F; - } + if (!s || !cfg || s->near_copy) { + return 0.0F; } - - float score = cfg->w_tfidf * sparse_tfidf_cosine(a, b); - - /* Signal 2: Random Indexing */ - score += cfg->w_ri * cbm_rsq_ip(&a->ri_code, &b->ri_code); - - /* Signal 3: MinHash Jaccard */ - if (a->has_minhash && b->has_minhash) { - double j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, - (const cbm_minhash_t *)b->minhash); - score += cfg->w_minhash * (float)j; - } - - /* Signal 4: API Signatures */ - score += cfg->w_api * cbm_rsq_ip(&a->api_code, &b->api_code); - - /* Signal 5: Type Signatures */ - score += cfg->w_type * cbm_rsq_ip(&a->type_code, &b->type_code); - - /* Signal 7: Decorator Pattern */ - score += cfg->w_decorator * cbm_rsq_ip(&a->deco_code, &b->deco_code); - - /* Signal 8+9+11: Structural profile + data flow + Halstead */ - float sp_score = small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS); - score += cfg->w_struct_profile * sp_score; + /* The terms in the historical order (float addition is not associative). */ + float score = cfg->w_tfidf * s->tfidf; + score += cfg->w_ri * s->ri; + score += cfg->w_minhash * s->minhash; + score += cfg->w_api * s->api; + score += cfg->w_type * s->type; + score += cfg->w_decorator * s->decorator; + score += cfg->w_struct_profile * s->struct_profile; /* Signal 6: Module proximity (multiplier, not additive). * Proximity returns [1.0, 1.10] — a same-file/same-dir boost for ranking. @@ -1701,17 +1740,26 @@ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, * cosine-similarity range. Without the clamp, a 0.95 base × 1.10 proximity * would emit a 1.045 score which violates the semantic of "similarity" and * breaks downstream consumers that expect a normalized value. */ - score *= cbm_sem_proximity(a->file_path, b->file_path); + score *= s->proximity; if (score > CBM_SEM_UNIT_POS) { score = CBM_SEM_UNIT_POS; } if (score < 0.0F) { score = 0.0F; } - return score; } +float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, + const cbm_sem_config_t *cfg) { + if (!a || !b || !cfg) { + return 0.0F; + } + cbm_sem_signals_t s; + cbm_sem_signal_values(a, b, &s); + return cbm_sem_combine(&s, cfg); +} + /* ── Graph diffusion ─────────────────────────────────────────────── */ void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, int neighbor_count, diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h index 1f99a54c0..34d738796 100644 --- a/src/semantic/semantic.h +++ b/src/semantic/semantic.h @@ -206,12 +206,43 @@ const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index, /* Free corpus. */ void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus); +/* The TF-IDF terms of one function: one entry per distinct corpus token whose + * idf is above 0, weighted term frequency x idf, in ascending token-index + * order (the order the TF-IDF cosine merges on). `token_index[t]` is the + * corpus index of tokens[t] (-1 when unknown); NULL looks each token up by + * name. `indices` and `weights` hold at least `count` entries. Returns the + * number of terms written. */ +int cbm_sem_tfidf_terms(const cbm_sem_corpus_t *corpus, char *const *tokens, const int *token_index, + int count, int *indices, float *weights); + /* ── Combined scoring ────────────────────────────────────────────── */ /* Compute combined similarity score between two functions. */ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, const cbm_sem_config_t *cfg); +/* The signal values cbm_sem_combined_score weighs, for measuring them one by + * one: each similarity in [0, 1] (minhash 0 when either side has none), + * proximity a multiplier in [1.0, 1.10]. `near_copy`: the pair is a SIMILAR_TO + * copy, which the combined score sets to 0. */ +typedef struct { + float tfidf; + float ri; + float minhash; + float api; + float type; + float decorator; + float struct_profile; + float proximity; + bool near_copy; +} cbm_sem_signals_t; + +void cbm_sem_signal_values(const cbm_sem_func_t *a, const cbm_sem_func_t *b, + cbm_sem_signals_t *out); + +/* cbm_sem_combined_score from the signal values (the same arithmetic). */ +float cbm_sem_combine(const cbm_sem_signals_t *s, const cbm_sem_config_t *cfg); + /* Module proximity multiplier based on file paths. */ float cbm_sem_proximity(const char *path_a, const char *path_b); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index a2861d054..ed7cb7099 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -18810,6 +18810,92 @@ TEST(pipeline_semantic_batched_matches_unbatched) { PASS(); } +/* CBM_SEM_PAIR_SIGNALS (+ _FLOOR) is the semantic engine's measuring + * harness: it records each scored pair with its signal values, from a floor + * below the threshold up. Recording must never change the graph: the same + * SEMANTICALLY_RELATED rows with and without it, one admitted line per edge, + * and every pair under the threshold recorded but never admitted. */ +TEST(pipeline_semantic_pair_signals_never_change_the_graph) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_semsig_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"}; + for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) { + char file[64]; + snprintf(file, sizeof(file), "%s_config.go", subjects[i]); + write_sem_family(tmp, file, subjects[i]); + } + char db_plain[512]; + char db_signals[512]; + char tsv[512]; + snprintf(db_plain, sizeof(db_plain), "%s/plain.db", tmp); + snprintf(db_signals, sizeof(db_signals), "%s/signals.db", tmp); + snprintf(tsv, sizeof(tsv), "%s/pairs.tsv", tmp); + + cbm_setenv("CBM_SEMANTIC_THRESHOLD", "0.3", 1); /* as above: admits the near-duplicates */ + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_plain, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + cbm_setenv("CBM_SEM_PAIR_SIGNALS", tsv, 1); + cbm_setenv("CBM_SEM_PAIR_SIGNALS_FLOOR", "0.05", 1); + p = cbm_pipeline_new(tmp, db_signals, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + int rc = cbm_pipeline_run(p); + cbm_unsetenv("CBM_SEM_PAIR_SIGNALS"); + cbm_unsetenv("CBM_SEM_PAIR_SIGNALS_FLOOR"); + cbm_unsetenv("CBM_SEMANTIC_THRESHOLD"); + cbm_pipeline_free(p); + ASSERT_EQ(rc, 0); + + cbm_store_t *sp = cbm_store_open_path(db_plain); + cbm_store_t *ss = cbm_store_open_path(db_signals); + ASSERT_NOT_NULL(sp); + ASSERT_NOT_NULL(ss); + int edges = 0; + ASSERT_TRUE( + sem_same_rows(cbm_store_get_db(sp), cbm_store_get_db(ss), + "SELECT s.qualified_name, t.qualified_name, e.properties FROM edges e " + "JOIN nodes s ON s.id = e.source_id JOIN nodes t ON t.id = e.target_id " + "WHERE e.type = 'SEMANTICALLY_RELATED' ORDER BY 1, 2", + &edges)); + ASSERT_GT(edges, 0); + cbm_store_close(sp); + cbm_store_close(ss); + + /* qa qb score tfidf ri minhash api type decorator struct proximity + * same_file admitted loc_a loc_b */ + FILE *f = cbm_fopen(tsv, "r"); + ASSERT_NOT_NULL(f); + char line[4096]; + int lines = 0; + int admitted = 0; + int below = 0; + while (fgets(line, sizeof(line), f)) { + char *field[16]; + int n = 0; + for (char *tok = strtok(line, "\t\n"); tok && n < 16; tok = strtok(NULL, "\t\n")) { + field[n++] = tok; + } + ASSERT_EQ(n, 15); + float score = strtof(field[2], NULL); + bool is_admitted = strcmp(field[12], "1") == 0; + if (score < 0.3F) { + below++; + ASSERT_FALSE(is_admitted); + } + admitted += is_admitted ? 1 : 0; + lines++; + } + fclose(f); + ASSERT_EQ(admitted, edges); + ASSERT_GT(below, 0); /* the floor recorded pairs under the threshold */ + ASSERT_GT(lines, admitted); + th_rmtree(tmp); + PASS(); +} + TEST(pipeline_semantic_edges_no_functions) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX"); @@ -19722,6 +19808,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body); RUN_TEST(pipeline_semantic_edges_no_functions); RUN_TEST(pipeline_semantic_batched_matches_unbatched); + RUN_TEST(pipeline_semantic_pair_signals_never_change_the_graph); RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153); RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153); diff --git a/tests/test_semantic.c b/tests/test_semantic.c index e56c02c23..87bc4dcec 100644 --- a/tests/test_semantic.c +++ b/tests/test_semantic.c @@ -355,6 +355,111 @@ TEST(sem_corpus_index_accessors_match_name_lookups) { PASS(); } +/* ── TF-IDF terms and the combined score ─────────────────────────── */ + +static int sem_terms_of(const cbm_sem_corpus_t *c, char **tokens, int n, cbm_sem_func_t *f, + int *indices, float *weights) { + memset(f, 0, sizeof(*f)); + f->tfidf_indices = indices; + f->tfidf_weights = weights; + f->tfidf_len = cbm_sem_tfidf_terms(c, tokens, NULL, n, indices, weights); + return f->tfidf_len; +} + +/* Two functions share a TF-IDF term only when they share the word. Until + * 2026-10 the term key was the token's POSITION: any two functions of equal + * length matched on every term, so the signal measured length (a held-out + * judged sample: admitted-edge precision 0.713 -> 0.742 with the fix). */ +TEST(sem_tfidf_terms_compare_vocabulary_not_positions) { + cbm_sem_corpus_t *c = cbm_sem_corpus_new(); + ASSERT_NOT_NULL(c); + static char *parse[] = {"parse", "header", "value"}; + static char *render[] = {"render", "widget", "frame"}; + static char *reparse[] = {"value", "parse", "header", "parse", "absent"}; + static char *other[] = {"queue", "drain"}; + cbm_sem_corpus_add_doc(c, (const char **)parse, 3); + cbm_sem_corpus_add_doc(c, (const char **)render, 3); + cbm_sem_corpus_add_doc(c, (const char **)reparse, 4); /* "absent" stays out */ + cbm_sem_corpus_add_doc(c, (const char **)other, 2); + cbm_sem_corpus_finalize(c); + + cbm_sem_func_t fa; + cbm_sem_func_t fb; + cbm_sem_func_t fc; + int ia[3]; + int ib[3]; + int ic[5]; + float wa[3]; + float wb[3]; + float wc[5]; + ASSERT_EQ(sem_terms_of(c, parse, 3, &fa, ia, wa), 3); + ASSERT_EQ(sem_terms_of(c, render, 3, &fb, ib, wb), 3); + /* A repeated word is one term, its weight twice the idf; an unknown word + * is no term; terms ascend by token index. */ + ASSERT_EQ(sem_terms_of(c, reparse, 5, &fc, ic, wc), 3); + ASSERT_TRUE(ic[0] < ic[1] && ic[1] < ic[2]); + int at_parse = ic[0] == cbm_sem_corpus_token_index(c, "parse") ? 0 + : ic[1] == cbm_sem_corpus_token_index(c, "parse") ? 1 + : 2; + ASSERT_EQ(ic[at_parse], cbm_sem_corpus_token_index(c, "parse")); + ASSERT_FLOAT_EQ(wc[at_parse], 2.0F * cbm_sem_corpus_idf(c, "parse"), 1e-6); + + cbm_sem_signals_t s; + cbm_sem_signal_values(&fa, &fb, &s); + ASSERT_FLOAT_EQ(s.tfidf, 0.0F, 1e-6); /* same length, no shared word */ + cbm_sem_signal_values(&fa, &fc, &s); + ASSERT_TRUE(s.tfidf > 0.9F); /* the same words, other order and length */ + cbm_sem_corpus_free(c); + PASS(); +} + +/* cbm_sem_combine is the weighted sum of the signals, times the proximity + * multiplier, clamped to [0, 1]; a SIMILAR_TO near copy scores 0. And + * cbm_sem_combined_score is exactly that, applied to cbm_sem_signal_values. */ +TEST(sem_combine_weighs_signals) { + cbm_sem_config_t cfg = cbm_sem_get_config(); + cbm_sem_signals_t s = {.tfidf = 0.5F, .ri = 0.25F, .proximity = 1.0F}; + ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F, 1e-6); + s.proximity = 1.10F; + ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), (cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F) * 1.10F, + 1e-6); + cbm_sem_signals_t all = {.tfidf = 1.0F, + .ri = 1.0F, + .minhash = 1.0F, + .api = 1.0F, + .type = 1.0F, + .decorator = 1.0F, + .struct_profile = 1.0F, + .proximity = 1.10F}; + ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 1.0F, 0.0); /* clamped */ + all.near_copy = true; + ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 0.0F, 0.0); + ASSERT_FLOAT_EQ(cbm_sem_combine(NULL, &cfg), 0.0F, 0.0); + + cbm_sem_corpus_t *c = cbm_sem_corpus_new(); + ASSERT_NOT_NULL(c); + static char *one[] = {"open", "socket", "port"}; + static char *two[] = {"open", "socket", "host", "retry"}; + cbm_sem_corpus_add_doc(c, (const char **)one, 3); + cbm_sem_corpus_add_doc(c, (const char **)two, 4); + cbm_sem_corpus_finalize(c); + cbm_sem_func_t fa; + cbm_sem_func_t fb; + int ia[3]; + int ib[4]; + float wa[3]; + float wb[4]; + sem_terms_of(c, one, 3, &fa, ia, wa); + sem_terms_of(c, two, 4, &fb, ib, wb); + fa.file_path = "net/dial.go"; + fb.file_path = "net/listen.go"; + cbm_sem_signals_t v; + cbm_sem_signal_values(&fa, &fb, &v); + ASSERT_FLOAT_EQ(cbm_sem_combined_score(&fa, &fb, &cfg), cbm_sem_combine(&v, &cfg), 0.0); + cbm_sem_corpus_free(c); + PASS(); +} + TEST(sem_corpus_add_null_doc) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); @@ -531,6 +636,8 @@ SUITE(semantic) { RUN_TEST(sem_corpus_add_one_doc); RUN_TEST(sem_corpus_idf); RUN_TEST(sem_corpus_index_accessors_match_name_lookups); + RUN_TEST(sem_tfidf_terms_compare_vocabulary_not_positions); + RUN_TEST(sem_combine_weighs_signals); RUN_TEST(sem_corpus_add_null_doc); RUN_TEST(sem_corpus_free_null); RUN_TEST(sem_get_config_defaults);