/* * test_semantic.c — Unit tests for semantic.c (pure functions). * * Covers: tokenize, cosine, normalize, vec_add_scaled, random_index, * proximity, diffuse, corpus lifecycle, get_config. */ #include "test_framework.h" #include "../src/foundation/compat.h" #include "../src/foundation/compat_thread.h" #include #include #include #include #include #include /* ── Tokenize ────────────────────────────────────────────────────── */ TEST(sem_tokenize_camel) { char *tokens[32]; int n = cbm_sem_tokenize("parseUserInput", tokens, 32); ASSERT_GTE(n, 3); ASSERT_STR_EQ(tokens[0], "parse"); ASSERT_STR_EQ(tokens[1], "user"); ASSERT_STR_EQ(tokens[2], "input"); for (int i = 0; i < n; i++) free(tokens[i]); PASS(); } TEST(sem_tokenize_snake) { char *tokens[32]; int n = cbm_sem_tokenize("handle_http_request", tokens, 32); ASSERT_GTE(n, 3); ASSERT_STR_EQ(tokens[0], "handle"); ASSERT_STR_EQ(tokens[1], "http"); ASSERT_STR_EQ(tokens[2], "request"); for (int i = 0; i < n; i++) free(tokens[i]); PASS(); } TEST(sem_tokenize_dot) { char *tokens[32]; int n = cbm_sem_tokenize("net.http.client", tokens, 32); ASSERT_GTE(n, 3); ASSERT_STR_EQ(tokens[0], "net"); ASSERT_STR_EQ(tokens[1], "http"); ASSERT_STR_EQ(tokens[2], "client"); for (int i = 0; i < n; i++) free(tokens[i]); PASS(); } TEST(sem_tokenize_null) { int n = cbm_sem_tokenize(NULL, NULL, 0); ASSERT_EQ(n, 0); PASS(); } TEST(sem_tokenize_max_out) { char *tokens[3]; int n = cbm_sem_tokenize("a_b_c_d_e_f_g", tokens, 3); ASSERT_EQ(n, 3); for (int i = 0; i < n; i++) free(tokens[i]); PASS(); } TEST(sem_tokenize_abbrev_expansion) { char *tokens[32]; int n = cbm_sem_tokenize("getCtxErrMsg", tokens, 32); /* get, ctx, context, err, error, msg, message */ ASSERT_GTE(n, 4); bool has_ctx = false, has_context = false, has_err = false, has_error = false; for (int i = 0; i < n; i++) { if (strcmp(tokens[i], "ctx") == 0) has_ctx = true; if (strcmp(tokens[i], "context") == 0) has_context = true; if (strcmp(tokens[i], "err") == 0) has_err = true; if (strcmp(tokens[i], "error") == 0) has_error = true; } ASSERT_TRUE(has_ctx && has_context && has_err && has_error); for (int i = 0; i < n; i++) free(tokens[i]); PASS(); } /* ── Cosine similarity ───────────────────────────────────────────── */ static void fill_vec(cbm_sem_vec_t *v, float val) { for (int i = 0; i < CBM_SEM_DIM; i++) v->v[i] = val; } TEST(sem_cosine_identical) { cbm_sem_vec_t a, b; fill_vec(&a, 0.5f); fill_vec(&b, 0.5f); float sim = cbm_sem_cosine(&a, &b); ASSERT_FLOAT_EQ(sim, 1.0f, 0.001f); PASS(); } TEST(sem_cosine_orthogonal) { cbm_sem_vec_t a, b; memset(&a, 0, sizeof(a)); memset(&b, 0, sizeof(b)); a.v[0] = 1.0f; b.v[1] = 1.0f; float sim = cbm_sem_cosine(&a, &b); ASSERT_FLOAT_EQ(sim, 0.0f, 0.001f); PASS(); } TEST(sem_cosine_zero_vector) { cbm_sem_vec_t a, b; memset(&a, 0, sizeof(a)); fill_vec(&b, 1.0f); float sim = cbm_sem_cosine(&a, &b); ASSERT_FLOAT_EQ(sim, 0.0f, 0.001f); PASS(); } TEST(sem_cosine_negative) { cbm_sem_vec_t a, b; memset(&a, 0, sizeof(a)); memset(&b, 0, sizeof(b)); a.v[0] = 1.0f; b.v[0] = -1.0f; float sim = cbm_sem_cosine(&a, &b); ASSERT_FLOAT_EQ(sim, -1.0f, 0.001f); PASS(); } TEST(sem_cosine_null) { ASSERT_FLOAT_EQ(cbm_sem_cosine(NULL, NULL), 0.0f, 0.001f); PASS(); } /* ── Normalize ───────────────────────────────────────────────────── */ TEST(sem_normalize_unit) { cbm_sem_vec_t v; memset(&v, 0, sizeof(v)); v.v[0] = 1.0f; cbm_sem_normalize(&v); ASSERT_FLOAT_EQ(cbm_sem_cosine(&v, &v), 1.0f, 0.001f); PASS(); } TEST(sem_normalize_scales) { cbm_sem_vec_t v; fill_vec(&v, 2.0f); cbm_sem_normalize(&v); float mag_sq = 0.0f; for (int i = 0; i < CBM_SEM_DIM; i++) mag_sq += v.v[i] * v.v[i]; float mag = sqrtf(mag_sq); ASSERT_FLOAT_EQ(mag, 1.0f, 0.01f); PASS(); } TEST(sem_normalize_zero) { cbm_sem_vec_t v; memset(&v, 0, sizeof(v)); cbm_sem_normalize(&v); /* Should remain zero (no division by zero) */ PASS(); } TEST(sem_normalize_null) { cbm_sem_normalize(NULL); /* should not crash */ PASS(); } /* ── Vec add scaled ──────────────────────────────────────────────── */ TEST(sem_vec_add_scaled_basic) { cbm_sem_vec_t dst; memset(&dst, 0, sizeof(dst)); cbm_sem_vec_t src; fill_vec(&src, 1.0f); cbm_sem_vec_add_scaled(&dst, &src, 0.5f); ASSERT_FLOAT_EQ(dst.v[0], 0.5f, 0.001f); ASSERT_FLOAT_EQ(dst.v[CBM_SEM_DIM - 1], 0.5f, 0.001f); PASS(); } TEST(sem_vec_add_scaled_null) { cbm_sem_vec_t v; fill_vec(&v, 1.0f); cbm_sem_vec_add_scaled(NULL, &v, 1.0f); /* should not crash */ cbm_sem_vec_add_scaled(&v, NULL, 1.0f); /* should not crash */ PASS(); } /* ── Random index ────────────────────────────────────────────────── */ TEST(sem_random_index_deterministic) { cbm_sem_vec_t a, b; cbm_sem_random_index("hello", &a); cbm_sem_random_index("hello", &b); ASSERT_FLOAT_EQ(cbm_sem_cosine(&a, &b), 1.0f, 0.001f); PASS(); } TEST(sem_random_index_different_tokens) { cbm_sem_vec_t a, b; cbm_sem_random_index("function", &a); cbm_sem_random_index("variable", &b); /* Different tokens should produce different vectors */ float sim = cbm_sem_cosine(&a, &b); ASSERT_TRUE(sim < 1.0f - 1e-6f); PASS(); } TEST(sem_random_index_null) { cbm_sem_vec_t v; memset(&v, 0, sizeof(v)); cbm_sem_random_index(NULL, &v); /* Should produce zero vector for NULL token */ for (int i = 0; i < CBM_SEM_DIM; i++) { ASSERT_FLOAT_EQ(v.v[i], 0.0f, 0.001f); } PASS(); } /* ── Proximity ───────────────────────────────────────────────────── */ TEST(sem_proximity_same_file) { float p = cbm_sem_proximity("src/main.c", "src/main.c"); ASSERT_FLOAT_EQ(p, 1.1f, 0.01f); /* CBM_SEM_UNIT_POS + CBM_SEM_PROX_MAX_BOOST */ PASS(); } TEST(sem_proximity_same_dir) { /* Files sharing 1 of 2 directory components: ratio = 0.5 → 1.0 + 0.5*0.10 = 1.05 */ float p = cbm_sem_proximity("src/core/a.c", "src/io/b.c"); ASSERT_TRUE(p > 1.0f && p < 1.10f); PASS(); } TEST(sem_proximity_different_paths) { float p = cbm_sem_proximity("src/foo/a.c", "tests/bar/b.c"); ASSERT_FLOAT_EQ(p, 1.0f, 0.01f); PASS(); } TEST(sem_proximity_null) { ASSERT_FLOAT_EQ(cbm_sem_proximity(NULL, "foo.c"), 1.0f, 0.01f); ASSERT_FLOAT_EQ(cbm_sem_proximity("foo.c", NULL), 1.0f, 0.01f); PASS(); } /* ── Diffuse ─────────────────────────────────────────────────────── */ TEST(sem_diffuse_zero_neighbors) { cbm_sem_vec_t v; fill_vec(&v, 0.5f); cbm_sem_diffuse(&v, NULL, 0, 0.3f); /* With zero neighbors, vector should be unchanged */ ASSERT_FLOAT_EQ(v.v[0], 0.5f, 0.001f); PASS(); } TEST(sem_diffuse_single_neighbor) { cbm_sem_vec_t v; memset(&v, 0, sizeof(v)); v.v[0] = 0.5f; v.v[1] = 0.5f; cbm_sem_normalize(&v); /* unit-length input */ cbm_sem_vec_t nb; memset(&nb, 0, sizeof(nb)); nb.v[0] = 1.0f; cbm_sem_normalize(&nb); cbm_sem_diffuse(&v, &nb, 1, 0.3f); /* After diffuse+normalize, result should still be unit-length */ float mag_sq = 0.0f; for (int i = 0; i < CBM_SEM_DIM; i++) mag_sq += v.v[i] * v.v[i]; ASSERT_FLOAT_EQ(sqrtf(mag_sq), 1.0f, 0.01f); /* Component 0 should be pulled toward neighbor's strong dim-0 */ ASSERT_TRUE(v.v[0] > 0.0f); PASS(); } /* ── Corpus lifecycle ────────────────────────────────────────────── */ TEST(sem_corpus_new_free) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); ASSERT_EQ(cbm_sem_corpus_doc_count(c), 0); ASSERT_EQ(cbm_sem_corpus_token_count(c), 0); cbm_sem_corpus_free(c); PASS(); } TEST(sem_corpus_add_one_doc) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); const char *tokens[] = {"parse", "user", "input"}; cbm_sem_corpus_add_doc(c, tokens, 3); ASSERT_EQ(cbm_sem_corpus_doc_count(c), 1); ASSERT_TRUE(cbm_sem_corpus_token_count(c) > 0); cbm_sem_corpus_free(c); PASS(); } TEST(sem_corpus_idf) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); const char *doc1[] = {"a", "b", "c"}; const char *doc2[] = {"a", "d", "e"}; cbm_sem_corpus_add_doc(c, doc1, 3); cbm_sem_corpus_add_doc(c, doc2, 3); /* IDF for "a" (appears in 2 docs): log(2/2) = log(1) = 0 */ float idf_a = cbm_sem_corpus_idf(c, "a"); ASSERT_TRUE(idf_a < 0.01f); /* IDF for "b" (appears in 1 doc): log(2/1) > 0 */ float idf_b = cbm_sem_corpus_idf(c, "b"); ASSERT_TRUE(idf_b > 0.0f); cbm_sem_corpus_free(c); PASS(); } /* The vector build resolves each token once and reads idf and vector by index * (pass_semantic_edges.c): the index accessors must give the same answers as * the name-keyed ones, before and after finalize, and agree on unknown tokens. */ TEST(sem_corpus_index_accessors_match_name_lookups) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); const char *doc1[] = {"alpha", "beta", "gamma"}; const char *doc2[] = {"alpha", "delta", "beta"}; const char *doc3[] = {"epsilon", "alpha"}; cbm_sem_corpus_add_doc(c, doc1, 3); cbm_sem_corpus_add_doc(c, doc2, 3); cbm_sem_corpus_add_doc(c, doc3, 2); static const char *const probe[] = {"alpha", "beta", "gamma", "delta", "epsilon", "absent"}; for (int round = 0; round < 2; round++) { for (size_t i = 0; i < sizeof(probe) / sizeof(probe[0]); i++) { int idx = cbm_sem_corpus_token_index(c, probe[i]); ASSERT_FLOAT_EQ(cbm_sem_corpus_idf_at(c, idx), cbm_sem_corpus_idf(c, probe[i]), 0.0); ASSERT_TRUE(cbm_sem_corpus_ri_vec_at(c, idx) == cbm_sem_corpus_ri_vec(c, probe[i])); } ASSERT_EQ(cbm_sem_corpus_token_index(c, "absent"), -1); ASSERT_NULL(cbm_sem_corpus_ri_vec_at(c, -1)); cbm_sem_corpus_finalize(c); /* second round: the finalized corpus */ } cbm_sem_corpus_free(c); PASS(); } /* ── TF-IDF terms and the combined score ─────────────────────────── */ static int sem_terms_of(const cbm_sem_corpus_t *c, char **tokens, int n, cbm_sem_func_t *f, int *indices, float *weights) { memset(f, 0, sizeof(*f)); f->tfidf_indices = indices; f->tfidf_weights = weights; f->tfidf_len = cbm_sem_tfidf_terms(c, tokens, NULL, n, indices, weights); return f->tfidf_len; } /* Two functions share a TF-IDF term only when they share the word. Until * 2026-10 the term key was the token's POSITION: any two functions of equal * length matched on every term, so the signal measured length (a held-out * judged sample: admitted-edge precision 0.713 -> 0.742 with the fix). */ TEST(sem_tfidf_terms_compare_vocabulary_not_positions) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); static char *parse[] = {"parse", "header", "value"}; static char *render[] = {"render", "widget", "frame"}; static char *reparse[] = {"value", "parse", "header", "parse", "absent"}; static char *other[] = {"queue", "drain"}; cbm_sem_corpus_add_doc(c, (const char **)parse, 3); cbm_sem_corpus_add_doc(c, (const char **)render, 3); cbm_sem_corpus_add_doc(c, (const char **)reparse, 4); /* "absent" stays out */ cbm_sem_corpus_add_doc(c, (const char **)other, 2); cbm_sem_corpus_finalize(c); cbm_sem_func_t fa; cbm_sem_func_t fb; cbm_sem_func_t fc; int ia[3]; int ib[3]; int ic[5]; float wa[3]; float wb[3]; float wc[5]; ASSERT_EQ(sem_terms_of(c, parse, 3, &fa, ia, wa), 3); ASSERT_EQ(sem_terms_of(c, render, 3, &fb, ib, wb), 3); /* A repeated word is one term, its weight twice the idf; an unknown word * is no term; terms ascend by token index. */ ASSERT_EQ(sem_terms_of(c, reparse, 5, &fc, ic, wc), 3); ASSERT_TRUE(ic[0] < ic[1] && ic[1] < ic[2]); int at_parse = ic[0] == cbm_sem_corpus_token_index(c, "parse") ? 0 : ic[1] == cbm_sem_corpus_token_index(c, "parse") ? 1 : 2; ASSERT_EQ(ic[at_parse], cbm_sem_corpus_token_index(c, "parse")); ASSERT_FLOAT_EQ(wc[at_parse], 2.0F * cbm_sem_corpus_idf(c, "parse"), 1e-6); cbm_sem_signals_t s; cbm_sem_signal_values(&fa, &fb, &s); ASSERT_FLOAT_EQ(s.tfidf, 0.0F, 1e-6); /* same length, no shared word */ cbm_sem_signal_values(&fa, &fc, &s); ASSERT_TRUE(s.tfidf > 0.9F); /* the same words, other order and length */ cbm_sem_corpus_free(c); PASS(); } /* cbm_sem_combine is the weighted sum of the signals, times the proximity * multiplier, clamped to [0, 1]; a SIMILAR_TO near copy scores 0. And * cbm_sem_combined_score is exactly that, applied to cbm_sem_signal_values. */ TEST(sem_combine_weighs_signals) { cbm_sem_config_t cfg = cbm_sem_get_config(); cbm_sem_signals_t s = {.tfidf = 0.5F, .ri = 0.25F, .proximity = 1.0F}; ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F, 1e-6); s.proximity = 1.10F; ASSERT_FLOAT_EQ(cbm_sem_combine(&s, &cfg), (cfg.w_tfidf * 0.5F + cfg.w_ri * 0.25F) * 1.10F, 1e-6); cbm_sem_signals_t all = {.tfidf = 1.0F, .ri = 1.0F, .minhash = 1.0F, .api = 1.0F, .type = 1.0F, .decorator = 1.0F, .struct_profile = 1.0F, .proximity = 1.10F}; ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 1.0F, 0.0); /* clamped */ all.near_copy = true; ASSERT_FLOAT_EQ(cbm_sem_combine(&all, &cfg), 0.0F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_combine(NULL, &cfg), 0.0F, 0.0); cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); static char *one[] = {"open", "socket", "port"}; static char *two[] = {"open", "socket", "host", "retry"}; cbm_sem_corpus_add_doc(c, (const char **)one, 3); cbm_sem_corpus_add_doc(c, (const char **)two, 4); cbm_sem_corpus_finalize(c); cbm_sem_func_t fa; cbm_sem_func_t fb; int ia[3]; int ib[4]; float wa[3]; float wb[4]; sem_terms_of(c, one, 3, &fa, ia, wa); sem_terms_of(c, two, 4, &fb, ib, wb); fa.file_path = "net/dial.go"; fb.file_path = "net/listen.go"; cbm_sem_signals_t v; cbm_sem_signal_values(&fa, &fb, &v); ASSERT_FLOAT_EQ(cbm_sem_combined_score(&fa, &fb, &cfg), cbm_sem_combine(&v, &cfg), 0.0); cbm_sem_corpus_free(c); PASS(); } /* Over budget, a function keeps its BEST partners, not its first ones (a * held-out judged sample: the pairs first-come admitted over budget were 0 of * 17 related). Equal scores keep the canonical order; an ineligible pair is * never admitted and takes no budget. */ TEST(sem_admit_best_first_keeps_the_best) { /* function 0 has four partners in canonical order, budget 2 */ const float scores[] = {0.80F, 0.90F, 0.95F, 0.99F, 0.90F}; const int fa[] = {0, 0, 0, 0, 5}; const int fb[] = {1, 2, 3, 4, 6}; const bool eligible[] = {true, true, true, false, true}; int counts[7] = {0}; bool admitted[5]; ASSERT_TRUE(cbm_sem_admit_best_first(scores, fa, fb, eligible, 5, 2, counts, admitted)); ASSERT_FALSE(admitted[0]); /* first-come would have kept 0.80 */ ASSERT_TRUE(admitted[1]); ASSERT_TRUE(admitted[2]); ASSERT_FALSE(admitted[3]); /* the best score, but recorded only */ ASSERT_TRUE(admitted[4]); ASSERT_EQ(counts[0], 2); ASSERT_EQ(counts[4], 0); /* ties: the earlier pair in canonical order wins the last slot */ const float tied[] = {0.85F, 0.85F}; const int ta[] = {0, 0}; const int tb[] = {1, 2}; const bool all[] = {true, true}; int tcounts[3] = {0}; bool tadmitted[2]; ASSERT_TRUE(cbm_sem_admit_best_first(tied, ta, tb, all, 2, 1, tcounts, tadmitted)); ASSERT_TRUE(tadmitted[0]); ASSERT_FALSE(tadmitted[1]); PASS(); } /* p by the stored (three-decimal) score: the judged bands, monotone. */ TEST(sem_calibrated_p_bands) { ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.750F), 0.626F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.772F), 0.626F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.773F), 0.760F, 0.0); /* the cut sits at the stored precision (0.773), not the raw quartile: * a stored "0.773" and a stored "0.772" never share a band */ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.7729F), 0.626F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.838F), 0.760F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(1.000F), 0.760F, 0.0); float last = 0.0F; for (int milli = 750; milli <= 1000; milli++) { float p = cbm_sem_calibrated_p((float)milli / 1000.0F); ASSERT_TRUE(p >= last); last = p; } PASS(); } static float docp(cbm_sem_doc_format_t fmt, cbm_sem_doc_kind_t kind, cbm_sem_doc_pos_t pos, float score) { return cbm_sem_doc_calibrated_p(fmt, kind, pos, score); } /* Doc -> code candidates, Markdown (sample 7): p by kind and position -- * functions outside the doc's home never stored, inside it from 0.30, for a * doc of the whole project (samples 4 and 7 pooled) from 0.30 too; local * extras 0.32 inside the home * only; whole files from 0.40; nothing below CBM_SEM_DOC_MIN_SCORE; every * curve monotone. A format without a judged curve stores nothing. */ TEST(sem_doc_calibrated_p_bands) { const cbm_sem_doc_format_t md = CBM_SEM_DOC_FMT_MARKDOWN; const cbm_sem_doc_kind_t fn = CBM_SEM_DOC_KIND_FUNCTION; const cbm_sem_doc_pos_t global = CBM_SEM_DOC_POS_GLOBAL; const cbm_sem_doc_pos_t local = CBM_SEM_DOC_POS_LOCAL; const cbm_sem_doc_pos_t outside = CBM_SEM_DOC_POS_OUTSIDE; ASSERT_FLOAT_EQ(docp(md, fn, local, 0.199F), 0.0F, 0.0); ASSERT_FLOAT_EQ(docp(md, fn, local, CBM_SEM_DOC_MIN_SCORE), 0.0F, 0.0); ASSERT_FLOAT_EQ(docp(md, fn, local, 0.30F), 0.325F, 0.0); ASSERT_FLOAT_EQ(docp(md, fn, local, 0.40F), 0.525F, 0.0); ASSERT_FLOAT_EQ(docp(md, fn, global, 0.299F), 0.0F, 0.0); ASSERT_FLOAT_EQ(docp(md, fn, global, 0.30F), 0.217F, 0.0); ASSERT_FLOAT_EQ(docp(md, fn, global, 0.40F), 0.509F, 0.0); ASSERT_FLOAT_EQ(docp(md, fn, outside, 0.90F), 0.0F, 0.0); ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.199F), 0.0F, 0.0); ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.20F), 0.317F, 0.0); ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_LOCAL, local, 0.90F), 0.317F, 0.0); ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FILE, outside, 0.399F), 0.0F, 0.0); ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FILE, outside, 0.40F), 0.30F, 0.0); ASSERT_FLOAT_EQ(docp(md, CBM_SEM_DOC_KIND_FOLDER, local, 0.90F), 0.0F, 0.0); static const cbm_sem_doc_kind_t KINDS[] = {CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_KIND_LOCAL, CBM_SEM_DOC_KIND_FILE}; for (size_t k = 0; k < sizeof(KINDS) / sizeof(KINDS[0]); k++) { for (int pos = 0; pos < CBM_SEM_DOC_POS_COUNT; pos++) { float last = 0.0F; for (int milli = 0; milli <= 1000; milli++) { float s = (float)milli / 1000.0F; float p = docp(md, KINDS[k], (cbm_sem_doc_pos_t)pos, s); ASSERT_TRUE(p >= last); last = p; ASSERT_TRUE(pos != CBM_SEM_DOC_POS_OUTSIDE || KINDS[k] == CBM_SEM_DOC_KIND_FILE || p == 0.0F); ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_OTHER, KINDS[k], (cbm_sem_doc_pos_t)pos, s), 0.0F, 0.0); ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_PDF, KINDS[k], (cbm_sem_doc_pos_t)pos, s), 0.0F, 0.0); } } } ASSERT_FLOAT_EQ(docp(CBM_SEM_DOC_FMT_COUNT, fn, global, 0.9F), 0.0F, 0.0); /* a home folder: README and index docs only, Markdown only */ ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/README.md"), 0.867F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/docs/index.mdx"), 0.867F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/readme"), 0.867F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/guide.md"), 0.0F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, "pkg/mail/README-old.md"), 0.0F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(CBM_SEM_DOC_FMT_RST, "pkg/mail/index.rst"), 0.0F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_folder_p(md, NULL), 0.0F, 0.0); /* reST (sample 6): 0.20-0.30 judged 0.125, not stored */ const cbm_sem_doc_format_t rst = CBM_SEM_DOC_FMT_RST; ASSERT_FLOAT_EQ( cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.25F), 0.0F, 0.0); ASSERT_FLOAT_EQ( cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.30F), 0.35F, 0.0); ASSERT_FLOAT_EQ( cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.40F), 0.625F, 0.0); /* AsciiDoc (sample 6): every band at or above the floor is stored */ const cbm_sem_doc_format_t adoc = CBM_SEM_DOC_FMT_ADOC; ASSERT_FLOAT_EQ( cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.199F), 0.0F, 0.0); ASSERT_FLOAT_EQ( cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.20F), 0.225F, 0.0); ASSERT_FLOAT_EQ( cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.35F), 0.25F, 0.0); ASSERT_FLOAT_EQ( cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, 0.90F), 0.50F, 0.0); for (int milli = 0; milli <= 1000; milli++) { float s = (float)milli / 1000.0F; ASSERT_TRUE( cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s) >= cbm_sem_doc_calibrated_p(rst, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s - 0.001F)); ASSERT_TRUE( cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s) >= cbm_sem_doc_calibrated_p(adoc, CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_POS_GLOBAL, s - 0.001F)); } PASS(); } /* A section's diagram family (DIAG-C): Mermaid and PlantUML structural kinds, * any other Mermaid or PlantUML kind is prose, any other format a graph. */ TEST(sem_diag_family_by_format_and_kind) { ASSERT_EQ(cbm_sem_diag_family("mermaid", "sequence"), CBM_SEM_DIAG_STRUCTURAL); ASSERT_EQ(cbm_sem_diag_family("mermaid", "class"), CBM_SEM_DIAG_STRUCTURAL); ASSERT_EQ(cbm_sem_diag_family("mermaid", "er"), CBM_SEM_DIAG_STRUCTURAL); ASSERT_EQ(cbm_sem_diag_family("mermaid", "c4_deployment"), CBM_SEM_DIAG_STRUCTURAL); ASSERT_EQ(cbm_sem_diag_family("mermaid", "architecture"), CBM_SEM_DIAG_STRUCTURAL); ASSERT_EQ(cbm_sem_diag_family("mermaid", "flowchart"), CBM_SEM_DIAG_PROSE); ASSERT_EQ(cbm_sem_diag_family("mermaid", "state"), CBM_SEM_DIAG_PROSE); ASSERT_EQ(cbm_sem_diag_family("mermaid", "usecase"), CBM_SEM_DIAG_PROSE); ASSERT_EQ(cbm_sem_diag_family("mermaid", NULL), CBM_SEM_DIAG_PROSE); ASSERT_EQ(cbm_sem_diag_family("plantuml", "usecase"), CBM_SEM_DIAG_STRUCTURAL); ASSERT_EQ(cbm_sem_diag_family("plantuml", "component"), CBM_SEM_DIAG_STRUCTURAL); ASSERT_EQ(cbm_sem_diag_family("plantuml", "c4"), CBM_SEM_DIAG_STRUCTURAL); ASSERT_EQ(cbm_sem_diag_family("plantuml", "activity"), CBM_SEM_DIAG_PROSE); ASSERT_EQ(cbm_sem_diag_family("plantuml", "er"), CBM_SEM_DIAG_PROSE); ASSERT_EQ(cbm_sem_diag_family("dot", "digraph"), CBM_SEM_DIAG_GRAPH); ASSERT_EQ(cbm_sem_diag_family("d2", "d2"), CBM_SEM_DIAG_GRAPH); ASSERT_EQ(cbm_sem_diag_family("", "sequence"), CBM_SEM_DIAG_NONE); ASSERT_EQ(cbm_sem_diag_family(NULL, NULL), CBM_SEM_DIAG_NONE); PASS(); } /* A Markdown section that embeds a diagram (DIAG-C): function candidates at * or inside the home and local extras carry the family's judged curve, from * the first band; outside functions, files, other formats, graph diagrams * and sections without a diagram keep cbm_sem_doc_calibrated_p. */ TEST(sem_doc_section_p_diagram_families) { const cbm_sem_doc_format_t md = CBM_SEM_DOC_FMT_MARKDOWN; const cbm_sem_doc_kind_t fn = CBM_SEM_DOC_KIND_FUNCTION; const cbm_sem_doc_kind_t lx = CBM_SEM_DOC_KIND_LOCAL; const cbm_sem_doc_pos_t global = CBM_SEM_DOC_POS_GLOBAL; const cbm_sem_doc_pos_t local = CBM_SEM_DOC_POS_LOCAL; const cbm_sem_doc_pos_t outside = CBM_SEM_DOC_POS_OUTSIDE; const cbm_sem_diag_family_t prose = CBM_SEM_DIAG_PROSE; const cbm_sem_diag_family_t st = CBM_SEM_DIAG_STRUCTURAL; ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, prose, fn, global, 0.199F), 0.0F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, prose, fn, global, 0.20F), 0.24F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, prose, fn, local, 0.30F), 0.28F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, prose, fn, global, 0.40F), 0.45F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, prose, lx, local, 0.20F), 0.275F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, st, fn, local, 0.25F), 0.32F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, st, fn, global, 0.35F), 0.464F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, st, fn, local, 0.90F), 0.567F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, st, lx, local, 0.50F), 0.778F, 0.0); ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, st, lx, local, 0.199F), 0.0F, 0.0); /* stored as two decimals: 0.275 -> 0.28, 0.464 -> 0.46, 0.567 -> 0.57 */ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(cbm_sem_doc_section_p(md, prose, lx, local, 0.5F)), 0.28, 1e-12); ASSERT_FLOAT_EQ(cbm_sem_p_2dp(cbm_sem_doc_section_p(md, st, fn, local, 0.35F)), 0.46, 1e-12); static const cbm_sem_doc_kind_t KINDS[] = {CBM_SEM_DOC_KIND_FUNCTION, CBM_SEM_DOC_KIND_LOCAL, CBM_SEM_DOC_KIND_FILE}; for (int fam = 0; fam < CBM_SEM_DIAG_COUNT; fam++) { for (size_t k = 0; k < sizeof(KINDS) / sizeof(KINDS[0]); k++) { for (int pos = 0; pos < CBM_SEM_DOC_POS_COUNT; pos++) { float last = 0.0F; for (int milli = 0; milli <= 1000; milli++) { float s = (float)milli / 1000.0F; cbm_sem_doc_pos_t ps = (cbm_sem_doc_pos_t)pos; float p = cbm_sem_doc_section_p(md, (cbm_sem_diag_family_t)fam, KINDS[k], ps, s); ASSERT_TRUE(p >= last); last = p; bool own = (fam == prose || fam == st) && ((KINDS[k] == fn && pos != outside) || (KINDS[k] == lx && pos == local)); if (!own) { ASSERT_FLOAT_EQ(p, cbm_sem_doc_calibrated_p(md, KINDS[k], ps, s), 0.0); } ASSERT_FLOAT_EQ( cbm_sem_doc_section_p(CBM_SEM_DOC_FMT_RST, (cbm_sem_diag_family_t)fam, KINDS[k], ps, s), cbm_sem_doc_calibrated_p(CBM_SEM_DOC_FMT_RST, KINDS[k], ps, s), 0.0); } } } } ASSERT_FLOAT_EQ(cbm_sem_doc_section_p(md, CBM_SEM_DIAG_COUNT, fn, global, 0.25F), 0.0F, 0.0); PASS(); } /* p is stored and shown at two decimals, half up, whatever the float * representation of the judged value (0.225F is 0.22499999...). */ TEST(sem_p_two_decimals_half_up) { ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.225F), 0.23, 1e-12); ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.325F), 0.33, 1e-12); /* float 0.32499998... */ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.525F), 0.53, 1e-12); /* float 0.52499997... */ ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.324F), 0.32, 1e-12); ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.317F), 0.32, 1e-12); ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.867F), 0.87, 1e-12); ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.625F), 0.63, 1e-12); ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.548F), 0.55, 1e-12); ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.36F), 0.36, 1e-12); ASSERT_FLOAT_EQ(cbm_sem_p_2dp(0.0F), 0.0, 1e-12); PASS(); } /* A section's format comes from its file's extension, case-insensitively; * a .txt section is reStructuredText (plain text has no sections). */ TEST(sem_doc_format_by_extension) { ASSERT_EQ(cbm_sem_doc_format("docs/guide.md"), CBM_SEM_DOC_FMT_MARKDOWN); ASSERT_EQ(cbm_sem_doc_format("README.MARKDOWN"), CBM_SEM_DOC_FMT_MARKDOWN); ASSERT_EQ(cbm_sem_doc_format("docs/index.rst"), CBM_SEM_DOC_FMT_RST); ASSERT_EQ(cbm_sem_doc_format("docs/topics/db.txt"), CBM_SEM_DOC_FMT_RST); ASSERT_EQ(cbm_sem_doc_format("manual/intro.adoc"), CBM_SEM_DOC_FMT_ADOC); ASSERT_EQ(cbm_sem_doc_format("paper.pdf"), CBM_SEM_DOC_FMT_PDF); ASSERT_EQ(cbm_sem_doc_format("src/main.go"), CBM_SEM_DOC_FMT_OTHER); ASSERT_EQ(cbm_sem_doc_format("v1.2/NOTES"), CBM_SEM_DOC_FMT_OTHER); ASSERT_EQ(cbm_sem_doc_format(NULL), CBM_SEM_DOC_FMT_OTHER); PASS(); } /* A properties string holding a quote is read whole, and escapes are not * glued to the next word (until 2026-10 the read stopped at the first quote * and kept "\n" raw, so "\nreturn" became the token "nreturn"). */ TEST(sem_json_str_honours_escapes) { static const char json[] = "{\"docstring\":\"Say \\\"hi\\\" to\\nthe user \\u00e9 now\",\"bt\":\"x\"}"; char buf[64]; ASSERT_NOT_NULL(cbm_sem_json_str(json, "docstring", buf, sizeof(buf))); ASSERT_STR_EQ(buf, "Say \"hi\" to the user now"); ASSERT_NOT_NULL(cbm_sem_json_str(json, "bt", buf, sizeof(buf))); ASSERT_STR_EQ(buf, "x"); ASSERT_NULL(cbm_sem_json_str(json, "absent", buf, sizeof(buf))); char small[6]; ASSERT_NOT_NULL(cbm_sem_json_str(json, "docstring", small, sizeof(small))); ASSERT_STR_EQ(small, "Say \""); /* cut at the buffer, still terminated */ PASS(); } TEST(sem_corpus_add_null_doc) { cbm_sem_corpus_t *c = cbm_sem_corpus_new(); ASSERT_NOT_NULL(c); cbm_sem_corpus_add_doc(c, NULL, 0); cbm_sem_corpus_add_doc(c, NULL, -1); ASSERT_EQ(cbm_sem_corpus_doc_count(c), 0); cbm_sem_corpus_free(c); PASS(); } TEST(sem_corpus_free_null) { cbm_sem_corpus_free(NULL); /* should not crash */ PASS(); } /* ── Config ──────────────────────────────────────────────────────── */ TEST(sem_get_config_defaults) { cbm_sem_config_t cfg = cbm_sem_get_config(); ASSERT_TRUE(cfg.w_tfidf > 0.0f); ASSERT_TRUE(cfg.w_ri > 0.0f); ASSERT_TRUE(cfg.threshold > 0.0f); ASSERT_TRUE(cfg.max_edges > 0); PASS(); } /* ── RaBitQ estimator quality (from-paper 4-bit quantization) ────── */ typedef struct { atomic_int *ready; atomic_int *start; float value; cbm_rsq_code_t code; } rotsq_thread_ctx_t; static void *rotsq_concurrent_first_encode(void *opaque) { rotsq_thread_ctx_t *ctx = opaque; float vec[CBM_RSQ_IN_DIM]; for (int i = 0; i < CBM_RSQ_IN_DIM; i++) { vec[i] = ctx->value + (float)i / (float)CBM_RSQ_IN_DIM; } atomic_fetch_add_explicit(ctx->ready, 1, memory_order_release); while (atomic_load_explicit(ctx->start, memory_order_acquire) == 0) { cbm_usleep(1000); } cbm_rsq_encode(vec, &ctx->code); return NULL; } /* The daemon can initialize semantic encoders from multiple request threads. * Run this first so ThreadSanitizer observes the one-time initialization. */ TEST(sem_rotsq_concurrent_first_encode) { atomic_int ready; atomic_int start; atomic_init(&ready, 0); atomic_init(&start, 0); rotsq_thread_ctx_t ctx[2] = { {.ready = &ready, .start = &start, .value = 0.25F}, {.ready = &ready, .start = &start, .value = -0.5F}, }; cbm_thread_t threads[2]; bool started0 = cbm_thread_create(&threads[0], 0, rotsq_concurrent_first_encode, &ctx[0]) == 0; bool started1 = cbm_thread_create(&threads[1], 0, rotsq_concurrent_first_encode, &ctx[1]) == 0; for (int spins = 0; started0 && started1 && spins < 5000 && atomic_load_explicit(&ready, memory_order_acquire) < 2; spins++) { cbm_usleep(1000); } bool both_ready = atomic_load_explicit(&ready, memory_order_acquire) == 2; atomic_store_explicit(&start, 1, memory_order_release); if (started0) { (void)cbm_thread_join(&threads[0]); } if (started1) { (void)cbm_thread_join(&threads[1]); } ASSERT_TRUE(started0); ASSERT_TRUE(started1); ASSERT_TRUE(both_ready); ASSERT_TRUE(ctx[0].code.scale > 0.0F); ASSERT_TRUE(ctx[1].code.scale > 0.0F); PASS(); } /* Deterministic pseudo-random unit vectors; validates that the quantized * inner-product estimator tracks the exact float IP within tight bounds. * These bounds gate the semantic pass's use of the codes: cosine scores are * thresholded at ~0.75, so the estimator error must be well under the * decision margin for typical pairs. */ TEST(sem_rotsq_ip_error_bounds) { enum { N = 64 }; static float vecs[N][CBM_RSQ_IN_DIM]; static cbm_rsq_code_t codes[N]; uint32_t state = 0xC0FFEEu; for (int i = 0; i < N; i++) { double norm = 0.0; for (int d = 0; d < CBM_RSQ_IN_DIM; d++) { /* xorshift32 → roughly uniform in [-1, 1] */ state ^= state << 13; state ^= state >> 17; state ^= state << 5; vecs[i][d] = ((float)(state & 0xFFFFFF) / (float)0x7FFFFF) - 1.0F; norm += (double)vecs[i][d] * (double)vecs[i][d]; } float inv = norm > 0.0 ? (float)(1.0 / sqrt(norm)) : 0.0F; for (int d = 0; d < CBM_RSQ_IN_DIM; d++) { vecs[i][d] *= inv; } cbm_rsq_encode(vecs[i], &codes[i]); } double max_err = 0.0; double sum_err = 0.0; int pairs = 0; for (int i = 0; i < N; i++) { for (int j = i; j < N; j++) { double exact = 0.0; for (int d = 0; d < CBM_RSQ_IN_DIM; d++) { exact += (double)vecs[i][d] * (double)vecs[j][d]; } double est = (double)cbm_rsq_ip(&codes[i], &codes[j]); double err = fabs(est - exact); sum_err += err; if (err > max_err) { max_err = err; } pairs++; } } double mean_err = sum_err / pairs; /* Self-IP of a unit vector must estimate ~1. */ double self_est = (double)cbm_rsq_ip(&codes[0], &codes[0]); ASSERT_TRUE(fabs(self_est - 1.0) < 0.05); /* 4-bit RaBitQ-style SQ after rotation: expect mean error well under 1% * of the unit scale and max under ~4% — comfortably inside the semantic * threshold's decision margin. */ ASSERT_TRUE(mean_err < 0.01); ASSERT_TRUE(max_err < 0.04); PASS(); } /* ── Suite ───────────────────────────────────────────────────────── */ SUITE(semantic) { RUN_TEST(sem_rotsq_concurrent_first_encode); RUN_TEST(sem_rotsq_ip_error_bounds); RUN_TEST(sem_tokenize_camel); RUN_TEST(sem_tokenize_snake); RUN_TEST(sem_tokenize_dot); RUN_TEST(sem_tokenize_null); RUN_TEST(sem_tokenize_max_out); RUN_TEST(sem_tokenize_abbrev_expansion); RUN_TEST(sem_cosine_identical); RUN_TEST(sem_cosine_orthogonal); RUN_TEST(sem_cosine_zero_vector); RUN_TEST(sem_cosine_negative); RUN_TEST(sem_cosine_null); RUN_TEST(sem_normalize_unit); RUN_TEST(sem_normalize_scales); RUN_TEST(sem_normalize_zero); RUN_TEST(sem_normalize_null); RUN_TEST(sem_vec_add_scaled_basic); RUN_TEST(sem_vec_add_scaled_null); RUN_TEST(sem_random_index_deterministic); RUN_TEST(sem_random_index_different_tokens); RUN_TEST(sem_random_index_null); RUN_TEST(sem_proximity_same_file); RUN_TEST(sem_proximity_same_dir); RUN_TEST(sem_proximity_different_paths); RUN_TEST(sem_proximity_null); RUN_TEST(sem_diffuse_zero_neighbors); RUN_TEST(sem_diffuse_single_neighbor); RUN_TEST(sem_corpus_new_free); RUN_TEST(sem_corpus_add_one_doc); RUN_TEST(sem_corpus_idf); RUN_TEST(sem_corpus_index_accessors_match_name_lookups); RUN_TEST(sem_tfidf_terms_compare_vocabulary_not_positions); RUN_TEST(sem_combine_weighs_signals); RUN_TEST(sem_admit_best_first_keeps_the_best); RUN_TEST(sem_calibrated_p_bands); RUN_TEST(sem_doc_calibrated_p_bands); RUN_TEST(sem_diag_family_by_format_and_kind); RUN_TEST(sem_doc_section_p_diagram_families); RUN_TEST(sem_doc_format_by_extension); RUN_TEST(sem_p_two_decimals_half_up); RUN_TEST(sem_json_str_honours_escapes); RUN_TEST(sem_corpus_add_null_doc); RUN_TEST(sem_corpus_free_null); RUN_TEST(sem_get_config_defaults); }