nprasad2077/codebase-memory-mcp
1
0 Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-11-04 10:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge origin/main into fix/snapshot-coherence

Browse Source 
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel committed 2026-10-28 04:07:48 +01:00
2 parents e897930388 1f934b6afb
commit bd144c2544
6 files changed
+589 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/cbm/extract_url_fold.c
+13 -1
View File
@@ -401,10 +401,22 @@ static bool is_interpolated_string(TSNode node) {
return false;
}
/* A Python string whose opening token is more than one quote -- a prefix
* (r"", b"", u"", an f"" with no field) or a triple quote -- cannot go down
* the literal path: that strips exactly one quote from each end, so the
* prefix and the extra quotes stayed in the URL and it never matched a route
* (#523). Folding reads the content pieces instead. JS/TS strings have no
* string_start, so this is Python only. */
static bool is_wrapped_string(TSNode node) {
TSNode start = ts_node_child(node, 0);
return !ts_node_is_null(start) && strcmp(ts_node_type(start), "string_start") == 0 &&
ts_node_end_byte(start) - ts_node_start_byte(start) > SKIP_ONE;
}
static bool is_foldable_arg(TSNode arg) {
const char *k = ts_node_type(arg);
if (strcmp(k, "string") == 0) {
return is_interpolated_string(arg);
return is_interpolated_string(arg) || is_wrapped_string(arg);
}
return strcmp(k, "template_string") == 0 || strcmp(k, "binary_expression") == 0 ||
strcmp(k, "binary_operator") == 0 || strcmp(k, "identifier") == 0 ||
src/mcp/mcp.c
+138 -21
View File
@@ -4024,6 +4024,7 @@ enum {
BM25_BIND_EXACT = 8,
BM25_BIND_KIND = 9,
BM25_BIND_NS = 10,
BM25_BIND_LABEL_WEIGHT = 11,
BM25_SQL_AUTO_LEN = -1,
/* Inner FTS5 candidate cap. SQLite can early-terminate a plain FTS5 query
* (no JOIN/WHERE on outer table) of the form:
@@ -4032,6 +4033,11 @@ enum {
* and then joining/filtering/re-ranking those, we bound all work to O(N) where
* N = BM25_INNER_LIMIT rather than the full match set size. */
BM25_INNER_LIMIT = 2000,
/* Longest label spliced into the MATCH pre-filter; real labels are short
* identifiers, so anything longer simply skips the pre-filter. */
BM25_LABEL_MAX = 64,
/* Room for "(<tokens>) AND label : \"<label>\"" around the token list. */
BM25_MATCH_BUF = BM25_QUERY_BUF + BM25_LABEL_MAX + 32,
};
/* Column weights for nodes_fts (name, qualified_name, label, file_path, body).
@@ -4050,8 +4056,28 @@ enum {
* Safe against a legacy four-column nodes_fts: FTS5's bm25() reads a weight
* only when an instance actually lands in that column (`nVal > ic`), so the
* fifth weight is simply never consulted on a table that has no fifth
* column. */
#define BM25_WEIGHTS "bm25(nodes_fts, 1.0, 1.0, 1.0, 1.0, 0.3)"
* column.
*
* The label weight is bound (?11): 1.0 normally, 0.0 when the MATCH is scoped
* to a label (#2386). Every candidate then carries that label, so a label hit
* cannot tell them apart, and a zero weight keeps the added `label : "..."`
* phrase from shifting BM25 order by row length. */
#define BM25_WEIGHTS "bm25(nodes_fts, 1.0, 1.0, ?11, 1.0, 0.3)"
static const double BM25_LABEL_WEIGHT = 1.0;
static const double BM25_LABEL_WEIGHT_SCOPED = 0.0;
/* Labels an UNLABELLED query never ranks: path-shaped File/Folder nodes, the
* Project node and Variables match almost any token through their qualified
* name and file path, and would crowd definitions out of the result. They are
* still in nodes_fts, so an explicit `label` that names one of them is served
* from its own label-scoped window like any other label (?7 is the caller's
* label, NULL when none was given). Excluding them there too returned 0 rows
* -- often with an exact "eq" total -- although such nodes match. Section and
* Module are NOT excluded (#518/#519): they carry the prose the body column
* indexes. Shared by the ranked and the count query, so the two cannot
* desynchronise. */
#define BM25_UNLABELLED_EXCLUSION \
"(?7 IS NOT NULL OR n.label NOT IN ('File','Folder','Variable','Project'))"
/* Module-local SQLITE_TRANSIENT wrapper to dodge performance-no-int-to-ptr.
* See the matching helper in src/store/store.c for the same pattern. */
@@ -4102,6 +4128,85 @@ static int bm25_build_match(const char *query, char *out, size_t out_size) {
return tokens;
}
/* Scope an FTS5 MATCH expression to one node label (#2386). The label filter
* must choose the BM25_INNER_LIMIT candidate window, not trim it afterwards:
* otherwise a label that is rare among a common token's matches falls outside
* the window and the search returns 0 rows although matching nodes exist.
* FTS5 matching is case-insensitive, so this is a superset pre-filter; the
* caller keeps the exact `n.label = ?` check. Only a plain identifier is
* spliced in, as a quoted FTS5 string; any other label leaves `tokens`
* unscoped, which is the pre-#2386 behaviour. Returns the label-column weight
* to bind for the resulting MATCH. */
static double bm25_scope_match_to_label(const char *tokens, const char *label, char *out,
size_t out_size) {
snprintf(out, out_size, "%s", tokens);
if (!label || !label[0]) {
return BM25_LABEL_WEIGHT;
}
size_t len = 0;
for (; label[len]; len++) {
char ch = label[len];
bool ident = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
(ch >= '0' && ch <= '9') || ch == '_';
if (!ident || len >= BM25_LABEL_MAX) {
return BM25_LABEL_WEIGHT;
}
}
char scoped[BM25_MATCH_BUF];
int n = snprintf(scoped, sizeof(scoped), "(%s) AND label : \"%s\"", tokens, label);
if (n <= 0 || (size_t)n >= sizeof(scoped) || (size_t)n >= out_size) {
return BM25_LABEL_WEIGHT;
}
memcpy(out, scoped, (size_t)n + 1);
return BM25_LABEL_WEIGHT_SCOPED;
}
/* A BM25 result is empty although nodes match when the label is spelled in a
* case no node carries: labels are case-sensitive, so `label=class` finds
* nothing while Class nodes match the query. Returns a hint (memory core,
* CBM_MEM_CLASS_OTHER) naming the label this project actually holds, or NULL
* when the label exists exactly (a true zero). The exact probe is an index
* seek on (project, label); the case-insensitive scan runs only when it
* misses. The caller's label is never echoed back, only the stored spelling. */
static char *bm25_label_zero_hint(sqlite3 *db, const char *project, const char *label) {
enum { HINT_BIND_PROJECT = 1, HINT_BIND_LABEL = 2, HINT_COL_LABEL = 0, HINT_PROBE_EXACT = 0 };
if (!db || !label || !label[0]) {
return NULL;
}
static const char *const probes[] = {
"SELECT label FROM nodes WHERE project = ?1 AND label = ?2 LIMIT 1",
"SELECT label FROM nodes WHERE project = ?1 AND label = ?2 COLLATE NOCASE LIMIT 1",
};
char found[CBM_SZ_256] = "";
for (size_t i = 0; i < sizeof(probes) / sizeof(probes[0]) && !found[0]; i++) {
sqlite3_stmt *st = NULL;
if (sqlite3_prepare_v2(db, probes[i], BM25_SQL_AUTO_LEN, &st, NULL) != SQLITE_OK) {
return NULL;
}
sqlite3_bind_text(st, HINT_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
sqlite3_bind_text(st, HINT_BIND_LABEL, label, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
if (sqlite3_step(st) == SQLITE_ROW) {
const char *stored = (const char *)sqlite3_column_text(st, HINT_COL_LABEL);
snprintf(found, sizeof(found), "%s", stored ? stored : "");
}
sqlite3_finalize(st);
if (found[0] && i == HINT_PROBE_EXACT) {
return NULL;
}
}
if (!found[0]) {
return cbm_mem_strdup(CBM_MEM_CLASS_OTHER,
"no node in this project carries this label; get_graph_schema "
"lists the labels this index holds");
}
char hint[CBM_SZ_512];
snprintf(hint, sizeof(hint),
"no node carries this label as spelled; labels are case-sensitive -- retry with "
"label=\"%s\"",
found);
return cbm_mem_strdup(CBM_MEM_CLASS_OTHER, hint);
}
static char *bm25_file_pattern_like(const char *file_pattern) {
if (!file_pattern) {
return NULL;
@@ -4153,7 +4258,8 @@ static void bm25_lines_str(char *out, size_t size, int start, int end) {
static void bm25_emit_metadata_tree(cbm_sb_t *sb, int total, int offset, int returned,
bool candidate_window_saturated, bool budget_hit,
bool floor_exceeded, size_t max_output_bytes) {
bool floor_exceeded, size_t max_output_bytes,
const char *zero_hint) {
bool has_more = total > offset + returned;
cbm_tree_scalar_int(sb, "total", total);
cbm_tree_scalar_str(sb, "total_relation", candidate_window_saturated ? "gte" : "eq");
@@ -4184,12 +4290,15 @@ static void bm25_emit_metadata_tree(cbm_sb_t *sb, int total, int offset, int ret
cbm_tree_scalar_str(sb, "hint",
"first identity row exceeds max_output_bytes; raise "
"max_output_tokens");
} else if (zero_hint) {
cbm_tree_scalar_str(sb, "hint", zero_hint);
}
}
static void bm25_emit_metadata_json(yyjson_mut_doc *doc, yyjson_mut_val *root, int total,
int offset, int returned, bool candidate_window_saturated,
bool budget_hit, bool floor_exceeded, size_t max_output_bytes) {
bool budget_hit, bool floor_exceeded, size_t max_output_bytes,
const char *zero_hint) {
bool has_more = total > offset + returned;
yyjson_mut_obj_add_int(doc, root, "total", total);
yyjson_mut_obj_add_str(doc, root, "total_relation", candidate_window_saturated ? "gte" : "eq");
@@ -4221,12 +4330,14 @@ static void bm25_emit_metadata_json(yyjson_mut_doc *doc, yyjson_mut_val *root, i
yyjson_mut_obj_add_str(doc, root, "hint",
"first identity row exceeds max_output_bytes; raise "
"max_output_tokens");
} else if (zero_hint) {
yyjson_mut_obj_add_strcpy(doc, root, "hint", zero_hint);
}
}
static char *bm25_render(const bm25_output_row_t *rows, int returned, int total, int offset,
bool candidate_window_saturated, bool tree_format, bool budget_hit,
bool floor_exceeded, size_t max_output_bytes) {
bool floor_exceeded, size_t max_output_bytes, const char *zero_hint) {
if (tree_format) {
cbm_sb_t sb;
cbm_sb_init(&sb);
@@ -4244,7 +4355,7 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
cbm_tree_row_end(&sb);
}
bm25_emit_metadata_tree(&sb, total, offset, returned, candidate_window_saturated,
budget_hit, floor_exceeded, max_output_bytes);
budget_hit, floor_exceeded, max_output_bytes, zero_hint);
return cbm_sb_finish(&sb);
}
@@ -4271,7 +4382,7 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
}
yyjson_mut_obj_add_val(doc, root, "rows", result_rows);
bm25_emit_metadata_json(doc, root, total, offset, returned, candidate_window_saturated,
budget_hit, floor_exceeded, max_output_bytes);
budget_hit, floor_exceeded, max_output_bytes, zero_hint);
char *json = yy_doc_to_str(doc);
yyjson_mut_doc_free(doc);
return json;
@@ -4288,11 +4399,16 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
if (!db) {
return NULL;
}
char fts_query[BM25_QUERY_BUF];
int tok_count = bm25_build_match(query, fts_query, sizeof(fts_query));
char fts_tokens[BM25_QUERY_BUF];
int tok_count = bm25_build_match(query, fts_tokens, sizeof(fts_tokens));
if (tok_count == 0) {
return NULL;
}
/* One MATCH string for the ranked query, the count and the saturation
* probe, so all three describe the same label-scoped window. */
char fts_query[BM25_MATCH_BUF];
double label_weight =
bm25_scope_match_to_label(fts_tokens, label, fts_query, sizeof(fts_query));
char *file_like = bm25_file_pattern_like(file_pattern);
/* BM25 ranked query using a two-step approach to enable FTS5 early termination.
@@ -4335,12 +4451,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
") fts "
"JOIN nodes n ON n.id = fts.rowid "
"WHERE n.project = ?2 "
/* Section and Module are NO LONGER excluded (#518/#519): they are the
* labels that carry prose — a Markdown section's body, a config file's
* description — so excluding them made the body column unreachable.
* This exclusion list is MIRRORED in the count query below; the two
* must be changed together or results desynchronise from counts. */
" AND n.label NOT IN ('File','Folder','Variable','Project') "
" AND " BM25_UNLABELLED_EXCLUSION " "
" AND (?6 IS NULL OR n.file_path LIKE ?6) "
/* The caller's label filter applies in query mode exactly as it does
* in the structural mode (2026-09-16 probe: `label=Class` was ignored
@@ -4390,6 +4501,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
} else {
sqlite3_bind_null(stmt, BM25_BIND_NS);
}
sqlite3_bind_double(stmt, BM25_BIND_LABEL_WEIGHT, label_weight);
/* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT.
* Uses the identical subquery structure so the FTS5 early-exit applies here too. */
@@ -4406,7 +4518,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
/* MIRRORS the ranked query's filter verbatim — same weights, same
* label exclusions. Changing one alone reports a total that does
* not describe the rows returned. */
" AND n.label NOT IN ('File','Folder','Variable','Project')"
" AND " BM25_UNLABELLED_EXCLUSION " "
" AND (?6 IS NULL OR n.file_path LIKE ?6)"
" AND (?7 IS NULL OR n.label = ?7)"
" AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
@@ -4423,6 +4535,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
sqlite3_bind_text(cs, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN,
MCP_SQLITE_TRANSIENT);
sqlite3_bind_int(cs, BM25_BIND_LIMIT, BM25_INNER_LIMIT);
sqlite3_bind_double(cs, BM25_BIND_LABEL_WEIGHT, label_weight);
if (file_like) {
sqlite3_bind_text(cs, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN,
MCP_SQLITE_TRANSIENT);
@@ -4455,9 +4568,10 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
/* The top-candidate window is a performance ceiling, not an exact-total
* boundary. Probe one candidate beyond it so a broad query never presents
* a window-local count as the complete match count. This is global to the
* FTS table; saturation is therefore conservatively reported even when
* later project/path filters might discard the hidden candidates. */
* a window-local count as the complete match count. The probe shares the
* label-scoped MATCH but is otherwise global to the FTS table; saturation
* is therefore conservatively reported even when later project/path
* filters might discard the hidden candidates. */
bool candidate_window_saturated = true;
{
const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
@@ -4495,13 +4609,15 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
if (!rows) {
return heap_strdup("out of memory");
}
/* Never a silent zero for a label the index spells differently. */
char *zero_hint = total == 0 ? bm25_label_zero_hint(db, project, label) : NULL;
int returned = row_count;
bool budget_hit = false;
char *payload = NULL;
for (;;) {
payload = bm25_render(rows, returned, total, offset, candidate_window_saturated,
tree_format, budget_hit, false, max_output_bytes);
tree_format, budget_hit, false, max_output_bytes, zero_hint);
if (!payload || strlen(payload) <= max_output_bytes || returned == 0) {
break;
}
@@ -4514,8 +4630,9 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
if (!payload || strlen(payload) > max_output_bytes || (row_count > 0 && returned == 0)) {
free(payload);
payload = bm25_render(rows, 0, total, offset, candidate_window_saturated, tree_format, true,
row_count > 0, max_output_bytes);
row_count > 0, max_output_bytes, zero_hint);
}
cbm_free(CBM_MEM_CLASS_OTHER, zero_hint);
bm25_output_rows_free(rows, row_count);
return payload;
}
src/pipeline/pipeline_incremental.c
+11
View File
@@ -3719,6 +3719,17 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
cbm_store_count_edges(staging, project));
cbm_log_info("delta.committed_counts", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
/* The staging file started as a byte-for-byte clone of the previous
* generation (cbm_delta_stage_clone above), so its `projects` row still
* carries whichever root_path/indexed_at the LAST full rebuild wrote.
* Every other write in this function patches nodes/edges/hashes/surfaces
* for the CURRENT run's repo_path, so the project row must be refreshed
* to match, or a later root_path-keyed read (get_code_snippet and
* friends) resolves files against a directory this run never touched. */
if (cbm_store_upsert_project(staging, project, cbm_pipeline_repo_path(p)) != CBM_STORE_OK) {
goto out;
}
{
int index_mode = cbm_pipeline_get_mode(p);
cbm_pipeline_generation_t generation = {
tests/test_extraction.c
+77
View File
@@ -6552,6 +6552,81 @@ TEST(extract_py_url_fold_no_fabrication_issue706) {
PASS();
}
/* Issue #523 follow-up: a Python string whose opening token is more than one
* quote -- a prefix (r"", u"", b"", Rb"", an f"" with no field) or a triple
* quote -- kept the prefix and the extra quotes in the URL (`f"http://..."`,
* `""http://...""`), so the HTTP_CALLS edge never matched a route. Each must
* yield the bare URL, positional or as `url=`. */
TEST(extract_py_prefixed_string_url_issue523) {
CBMFileResult *r = extract("import requests\n"
"def f_plain():\n"
" return requests.get(f\"http://svc/v2/health\")\n"
"def raw():\n"
" return requests.get(r\"http://svc/v2/health\")\n"
"def uni():\n"
" return requests.get(u\"http://svc/v2/health\")\n"
"def byt():\n"
" return requests.get(b\"http://svc/v2/health\")\n"
"def raw_bytes():\n"
" return requests.get(Rb\"http://svc/v2/health\")\n"
"def triple():\n"
" return requests.get(\"\"\"http://svc/v2/health\"\"\")\n"
"def triple_single():\n"
" return requests.get('''http://svc/v2/health''')\n"
"def f_triple():\n"
" return requests.get(f\"\"\"http://svc/v2/health\"\"\")\n"
"def kw():\n"
" return requests.get(url=r\"http://svc/v2/health\")\n",
CBM_LANG_PYTHON, "t", "client.py");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_STR_EQ(url_in(r, "requests.get", "f_plain"), "http://svc/v2/health");
ASSERT_STR_EQ(url_in(r, "requests.get", "raw"), "http://svc/v2/health");
ASSERT_STR_EQ(url_in(r, "requests.get", "uni"), "http://svc/v2/health");
ASSERT_STR_EQ(url_in(r, "requests.get", "byt"), "http://svc/v2/health");
ASSERT_STR_EQ(url_in(r, "requests.get", "raw_bytes"), "http://svc/v2/health");
ASSERT_STR_EQ(url_in(r, "requests.get", "triple"), "http://svc/v2/health");
ASSERT_STR_EQ(url_in(r, "requests.get", "triple_single"), "http://svc/v2/health");
ASSERT_STR_EQ(url_in(r, "requests.get", "f_triple"), "http://svc/v2/health");
ASSERT_STR_EQ(url_in(r, "requests.get", "kw"), "http://svc/v2/health");
/* A wrapped literal is still a fully known value, so it is exact. */
const CBMCall *raw = find_call_in_func(r, "requests.get", "raw");
ASSERT_NOT_NULL(raw);
ASSERT_GTE(raw->arg_count, 1);
ASSERT_NOT_NULL(raw->args[0].value);
ASSERT_STR_EQ(raw->args[0].value, "http://svc/v2/health");
cbm_free_result(r);
PASS();
}
/* Same issue, the controls: a plain one-quote literal keeps the literal path
* unchanged, and an interpolating f-string still folds its field to "{}". */
TEST(extract_py_prefixed_string_url_controls_issue523) {
CBMFileResult *r = extract("import requests\n"
"def plain():\n"
" return requests.get(\"http://svc/v2/health\")\n"
"def plain_single():\n"
" return requests.get('http://svc/v2/health')\n"
"def field(order_id):\n"
" return requests.get(f\"http://svc/v2/orders/{order_id}\")\n"
"def raw_field(order_id):\n"
" return requests.get(rf\"http://svc/v2/orders/{order_id}\")\n",
CBM_LANG_PYTHON, "t", "ctl.py");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT_STR_EQ(url_in(r, "requests.get", "plain"), "http://svc/v2/health");
ASSERT_STR_EQ(url_in(r, "requests.get", "plain_single"), "http://svc/v2/health");
ASSERT_STR_EQ(url_in(r, "requests.get", "field"), "http://svc/v2/orders/{}");
ASSERT_STR_EQ(url_in(r, "requests.get", "raw_field"), "http://svc/v2/orders/{}");
cbm_free_result(r);
PASS();
}
/* Issue #1147: a frontend that composes every endpoint from module constants
* -- a base constant, template literals over it, and an object-literal map of
* endpoints -- extracted none of its API calls. Each template / concatenation
@@ -13582,6 +13657,8 @@ SUITE(extraction) {
RUN_TEST(extract_ts_template_base_url_no_literal_path_issue2291);
RUN_TEST(extract_py_concat_const_url_issue706);
RUN_TEST(extract_py_url_fold_no_fabrication_issue706);
RUN_TEST(extract_py_prefixed_string_url_issue523);
RUN_TEST(extract_py_prefixed_string_url_controls_issue523);
RUN_TEST(extract_ts_const_template_url_issue1147);
RUN_TEST(extract_ts_url_fold_no_fabrication_issue1147);
RUN_TEST(extract_ts_folded_topics_keep_raw_identity);
tests/test_mcp.c
+279
View File
@@ -4512,6 +4512,281 @@ TEST(tool_search_graph_bm25_applies_label_filter) {
PASS();
}
/* #2386: the label filter must choose the BM25 candidate window, not trim it
* afterwards. 2001 Functions fill the window ahead of 3 Classes that rank
* strictly lower (longer file_path), so a post-window filter returns 0 Classes
* while reporting a saturated window. */
static char *bm25_label_window_search(cbm_mcp_server_t *srv, const char *label_json) {
char req[512];
snprintf(req, sizeof(req),
"{\"jsonrpc\":\"2.0\",\"id\":2386,\"method\":\"tools/call\","
"\"params\":{\"name\":\"search_graph\",\"arguments\":{"
"\"project\":\"bm25-label-window\",\"query\":\"windowneedle\","
"\"label\":%s,\"limit\":10,\"format\":\"json\"}}}",
label_json);
char *resp = cbm_mcp_server_handle(srv, req);
char *inner = resp ? extract_text_content(resp) : NULL;
free(resp);
return inner;
}
TEST(tool_search_graph_bm25_label_filter_selects_candidate_window) {
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
ASSERT_NOT_NULL(srv);
cbm_store_t *store = cbm_mcp_server_store(srv);
ASSERT_NOT_NULL(store);
const char *project = "bm25-label-window";
cbm_mcp_server_set_project(srv, project);
ASSERT_EQ(cbm_store_upsert_project(store, project, "/tmp/bm25-label-window"), CBM_STORE_OK);
for (int i = 0; i < 2001; i++) {
char qn[64];
snprintf(qn, sizeof(qn), "bm25-label-window.fn_%04d", i);
cbm_node_t node = {.project = project,
.label = "Function",
.name = "windowneedle",
.qualified_name = qn,
.file_path = "many.c",
.start_line = i + 1,
.end_line = i + 1};
ASSERT_GT(cbm_store_upsert_node(store, &node), 0);
}
for (int i = 0; i < 3; i++) {
char qn[64];
snprintf(qn, sizeof(qn), "bm25-label-window.cls_%d", i);
cbm_node_t node = {.project = project,
.label = "Class",
.name = "windowneedle",
.qualified_name = qn,
.file_path = "deeply/nested/package/layout/registry.c",
.start_line = i + 1,
.end_line = i + 1};
ASSERT_GT(cbm_store_upsert_node(store, &node), 0);
}
ASSERT_EQ(cbm_store_exec(store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');"),
CBM_STORE_OK);
ASSERT_EQ(cbm_store_exec(store, "INSERT INTO nodes_fts(rowid, name, qualified_name, label, "
"file_path) SELECT id, cbm_camel_split(name), qualified_name, "
"label, file_path FROM nodes;"),
CBM_STORE_OK);
/* The rare label comes back in full, and the window is not saturated. */
char *inner = bm25_label_window_search(srv, "\"Class\"");
ASSERT_NOT_NULL(inner);
ASSERT_NOT_NULL(strstr(inner, "\"bm25-label-window.cls_0\""));
ASSERT_NOT_NULL(strstr(inner, "\"bm25-label-window.cls_2\""));
ASSERT_NULL(strstr(inner, "\"Function\""));
ASSERT_NOT_NULL(strstr(inner, "\"total\":3"));
ASSERT_NOT_NULL(strstr(inner, "\"total_relation\":\"eq\""));
ASSERT_NULL(strstr(inner, "candidate_window_saturated"));
free(inner);
/* The common label still overflows its own window and says so. */
inner = bm25_label_window_search(srv, "\"Function\"");
ASSERT_NOT_NULL(inner);
ASSERT_NOT_NULL(strstr(inner, "\"total\":2000"));
ASSERT_NOT_NULL(strstr(inner, "\"candidate_window_saturated\":true"));
free(inner);
/* A label carrying FTS5 syntax is never spliced into MATCH: the query
* stays in BM25 mode and the exact label check matches nothing. */
inner = bm25_label_window_search(srv, "\"Class\\\" OR windowneedle\"");
ASSERT_NOT_NULL(inner);
ASSERT_NOT_NULL(strstr(inner, "\"search_mode\":\"bm25\""));
ASSERT_NOT_NULL(strstr(inner, "\"total\":0"));
free(inner);
cbm_mcp_server_free(srv);
PASS();
}
/* Scoping the MATCH to a label must not reorder that label's rows: a
* label-filtered query lists its rows in the same order as the unfiltered
* query does. The fixture's four `table` Methods differ in row length, which
* is exactly what a scored label phrase would re-weight. */
TEST(tool_search_graph_bm25_label_scope_keeps_rank_order) {
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
ASSERT_NOT_NULL(srv);
cbm_store_t *store = cbm_mcp_server_store(srv);
ASSERT_NOT_NULL(store);
cbm_mcp_server_set_project(srv, "bm25-find");
bm25_findability_fixture(store, "bm25-find");
char *all_resp = cbm_mcp_server_handle(
srv, "{\"jsonrpc\":\"2.0\",\"id\":2387,\"method\":\"tools/call\","
"\"params\":{\"name\":\"search_graph\",\"arguments\":{"
"\"project\":\"bm25-find\",\"query\":\"table\",\"limit\":20,\"format\":\"json\"}}}");
char *methods_resp = cbm_mcp_server_handle(
srv, "{\"jsonrpc\":\"2.0\",\"id\":2388,\"method\":\"tools/call\","
"\"params\":{\"name\":\"search_graph\",\"arguments\":{"
"\"project\":\"bm25-find\",\"query\":\"table\",\"label\":\"Method\","
"\"limit\":20,\"format\":\"json\"}}}");
ASSERT_NOT_NULL(all_resp);
ASSERT_NOT_NULL(methods_resp);
char *all = extract_text_content(all_resp);
char *methods = extract_text_content(methods_resp);
free(all_resp);
free(methods_resp);
ASSERT_NOT_NULL(all);
ASSERT_NOT_NULL(methods);
ASSERT_NOT_NULL(strstr(methods, "\"total\":4"));
static const char *const qns[] = {
"\"bm25-find.core.Table.Table.unquoted\"",
"\"bm25-find.core.Table.Table.describe\"",
"\"bm25-find.tests.SchemaTests.table_references_table_with_same_name\"",
"\"bm25-find.tests.SchemaTests.table_references_table_with_same_name_mysql\"",
};
enum { QN_COUNT = sizeof(qns) / sizeof(qns[0]) };
for (int i = 0; i < QN_COUNT; i++) {
for (int j = 0; j < QN_COUNT; j++) {
if (i == j) {
continue;
}
const char *ai = strstr(all, qns[i]);
const char *aj = strstr(all, qns[j]);
const char *mi = strstr(methods, qns[i]);
const char *mj = strstr(methods, qns[j]);
ASSERT_NOT_NULL(ai);
ASSERT_NOT_NULL(aj);
ASSERT_NOT_NULL(mi);
ASSERT_NOT_NULL(mj);
ASSERT_EQ(ai < aj, mi < mj);
}
}
free(all);
free(methods);
cbm_mcp_server_free(srv);
PASS();
}
/* An unlabelled BM25 query never ranks File, Folder, Variable or Project nodes
* (they match almost any token through their path), but an explicit label
* naming one of them must be served: those nodes are in nodes_fts, and
* excluding them there too answered `query=config label=Folder` with 0 rows
* and an exact "eq" total. One node per label shares the query tokens with a
* Function. */
static void bm25_unranked_labels_fixture(cbm_store_t *store, const char *project) {
cbm_store_upsert_project(store, project, "/tmp/bm25-noise");
struct {
const char *label, *name, *qn, *file;
} rows[] = {
{"Function", "loadconfig", "bm25-noise.app.loadconfig", "app/config.py"},
{"File", "__file__", "bm25-noise.app.config.py", "app/config.py"},
{"Folder", "config", "bm25-noise.app.config", "app/config"},
{"Variable", "CONFIG_PATH", "bm25-noise.app.config.CONFIG_PATH", "app/config.py"},
{"Project", "bm25-noise", "bm25-noise", ""},
};
for (size_t i = 0; i < sizeof(rows) / sizeof(rows[0]); i++) {
cbm_node_t node = {.project = project,
.label = rows[i].label,
.name = rows[i].name,
.qualified_name = rows[i].qn,
.file_path = rows[i].file,
.start_line = (int)i + 1,
.end_line = (int)i + 1};
cbm_store_upsert_node(store, &node);
}
cbm_store_exec(store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');");
cbm_store_exec(store, "INSERT INTO nodes_fts(rowid, name, qualified_name, label, "
"file_path) SELECT id, cbm_camel_split(name), qualified_name, "
"label, file_path FROM nodes;");
}
static char *bm25_noise_search(cbm_mcp_server_t *srv, const char *query, const char *label) {
char req[512];
char label_arg[128] = "";
if (label) {
snprintf(label_arg, sizeof(label_arg), ",\"label\":\"%s\"", label);
}
snprintf(req, sizeof(req),
"{\"jsonrpc\":\"2.0\",\"id\":2423,\"method\":\"tools/call\","
"\"params\":{\"name\":\"search_graph\",\"arguments\":{"
"\"project\":\"bm25-noise\",\"query\":\"%s\"%s,\"limit\":10,\"format\":\"json\"}}}",
query, label_arg);
char *resp = cbm_mcp_server_handle(srv, req);
char *inner = resp ? extract_text_content(resp) : NULL;
free(resp);
return inner;
}
TEST(tool_search_graph_bm25_serves_unranked_labels_when_named) {
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
ASSERT_NOT_NULL(srv);
cbm_store_t *store = cbm_mcp_server_store(srv);
ASSERT_NOT_NULL(store);
cbm_mcp_server_set_project(srv, "bm25-noise");
bm25_unranked_labels_fixture(store, "bm25-noise");
static const struct {
const char *label, *qn;
} named[] = {
{"File", "\"bm25-noise.app.config.py\",\"File\""},
{"Folder", "\"bm25-noise.app.config\",\"Folder\""},
{"Variable", "\"bm25-noise.app.config.CONFIG_PATH\",\"Variable\""},
{"Project", "\"bm25-noise\",\"Project\""},
};
for (size_t i = 0; i < sizeof(named) / sizeof(named[0]); i++) {
char *inner = bm25_noise_search(srv, "noise config", named[i].label);
ASSERT_NOT_NULL(inner);
ASSERT_NOT_NULL(strstr(inner, "\"search_mode\":\"bm25\""));
ASSERT_NOT_NULL(strstr(inner, named[i].qn));
ASSERT_NOT_NULL(strstr(inner, "\"total\":1,"));
ASSERT_NOT_NULL(strstr(inner, "\"total_relation\":\"eq\""));
ASSERT_NULL(strstr(inner, "\"hint\""));
free(inner);
}
/* Without a label the four stay out of the ranking: only the Function. */
char *inner = bm25_noise_search(srv, "noise config", NULL);
ASSERT_NOT_NULL(inner);
ASSERT_NOT_NULL(strstr(inner, "\"bm25-noise.app.loadconfig\",\"Function\""));
ASSERT_NOT_NULL(strstr(inner, "\"total\":1,"));
ASSERT_NULL(strstr(inner, "\"File\""));
ASSERT_NULL(strstr(inner, "\"Folder\""));
ASSERT_NULL(strstr(inner, "\"Variable\""));
ASSERT_NULL(strstr(inner, "\"Project\""));
free(inner);
cbm_mcp_server_free(srv);
PASS();
}
/* Labels are case-sensitive. A BM25 query whose label no node carries as
* spelled answers 0 rows like the structural path does, but never silently:
* the hint names the stored spelling, or get_graph_schema when the label is
* unknown. An existing label with no matching row is a true zero: no hint. */
TEST(tool_search_graph_bm25_zero_hint_names_the_label) {
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
ASSERT_NOT_NULL(srv);
cbm_store_t *store = cbm_mcp_server_store(srv);
ASSERT_NOT_NULL(store);
cbm_mcp_server_set_project(srv, "bm25-noise");
bm25_unranked_labels_fixture(store, "bm25-noise");
char *inner = bm25_noise_search(srv, "config", "folder");
ASSERT_NOT_NULL(inner);
ASSERT_NOT_NULL(strstr(inner, "\"total\":0,"));
ASSERT_NOT_NULL(strstr(inner, "\"hint\":"));
ASSERT_NOT_NULL(strstr(inner, "retry with label=\\\"Folder\\\""));
free(inner);
inner = bm25_noise_search(srv, "config", "Widget");
ASSERT_NOT_NULL(inner);
ASSERT_NOT_NULL(strstr(inner, "\"total\":0,"));
ASSERT_NOT_NULL(strstr(inner, "get_graph_schema"));
free(inner);
inner = bm25_noise_search(srv, "zzznomatch", "Function");
ASSERT_NOT_NULL(inner);
ASSERT_NOT_NULL(strstr(inner, "\"total\":0,"));
ASSERT_NULL(strstr(inner, "\"hint\""));
free(inner);
cbm_mcp_server_free(srv);
PASS();
}
/* The definition whose NAME is the query ranks first: the `Table` class above
* its own methods and above test methods that repeat "table" three times; the
* `get_object_or_404` function above the test methods that contain it. */
@@ -25549,6 +25824,10 @@ SUITE(mcp) {
RUN_TEST(tool_search_graph_query_honors_file_pattern_issue552);
RUN_TEST(tool_search_graph_bm25_reports_candidate_saturation);
RUN_TEST(tool_search_graph_bm25_applies_label_filter);
RUN_TEST(tool_search_graph_bm25_label_filter_selects_candidate_window);
RUN_TEST(tool_search_graph_bm25_label_scope_keeps_rank_order);
RUN_TEST(tool_search_graph_bm25_serves_unranked_labels_when_named);
RUN_TEST(tool_search_graph_bm25_zero_hint_names_the_label);
RUN_TEST(tool_search_graph_bm25_ranks_exact_name_first);
RUN_TEST(tool_search_graph_rejects_bm25_and_semantic_query_together);
RUN_TEST(tool_search_graph_semantic_ceiling_never_emits_unusable_continuation);
tests/test_pipeline.c
+71
View File
@@ -5921,6 +5921,76 @@ TEST(pipeline_closure_repair_removed_def_drops_dependent_edge) {
PASS();
}
/* #2276: the closure/delta route clones the previous generation's db file
* byte-for-byte and patches only nodes/edges/hashes/surfaces, so a reindex
* that keeps the same project name but a DIFFERENT repo_path (an artifact
* moved or reused from another checkout) leaves the `projects` row's
* root_path/indexed_at pinned to whichever directory the last FULL rebuild
* ran from, so every later root_path-keyed read then resolves files against
* a directory this run never touched. */
TEST(pipeline_incremental_root_path_refreshed_across_directories) {
char dir_a[256];
char dir_b[256];
snprintf(dir_a, sizeof(dir_a), "/tmp/cbm_rootpath_a_XXXXXX");
snprintf(dir_b, sizeof(dir_b), "/tmp/cbm_rootpath_b_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(dir_a));
ASSERT_NOT_NULL(cbm_mkdtemp(dir_b));
closure_probe_repo(dir_a);
closure_probe_repo(dir_b);
/* Body edit only, same exported names: the shape the closure route
* repairs without falling back to a full rebuild (mirrors
* pipeline_closure_repair_body_edit_converges_with_fresh_full above). */
write_temp_file(dir_b, "lib.ts",
"export function closureProbeHelper(x: number): string {\n"
" const doubled = x + x;\n"
" return String(doubled);\n"
"}\n");
char db[512];
snprintf(db, sizeof(db), "%s/rootpath.db", dir_a);
static const char *project = "rootpath-probe";
cbm_pipeline_t *first = cbm_pipeline_new(dir_a, db, CBM_MODE_FULL);
ASSERT_NOT_NULL(first);
ASSERT_TRUE(cbm_pipeline_set_project_name(first, project));
ASSERT_EQ(cbm_pipeline_run(first), 0);
cbm_pipeline_free(first);
/* Reindex the SAME project name from dir_b: a valid existing generation
* exists, so this routes through the incremental path, not a first-time
* full build. */
cbm_pipeline_t *second = cbm_pipeline_new(dir_b, db, CBM_MODE_FULL);
ASSERT_NOT_NULL(second);
ASSERT_TRUE(cbm_pipeline_set_project_name(second, project));
ASSERT_EQ(cbm_pipeline_run(second), 0);
cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();
cbm_pipeline_free(second);
/* The bug is specific to the closure/delta route; confirm we actually
* exercised it rather than a silent fall-back to a full rebuild, which
* would pass even with the fix reverted. */
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
/* cbm_pipeline_repo_path()/cbm_store_upsert_project() never call
* realpath(): the stored root_path is whatever was passed to
* cbm_pipeline_new(), verbatim, on both the full-rebuild path and this
* one. Compare against dir_b directly, not a canonicalized form, or
* this assertion only holds on a platform where /tmp is not itself a
* symlink (it resolves to /private/tmp on macOS). */
cbm_project_t info = {0};
cbm_store_t *store = cbm_store_open_path(db);
ASSERT_NOT_NULL(store);
ASSERT_EQ(cbm_store_get_project(store, project, &info), CBM_STORE_OK);
ASSERT_STR_EQ(info.root_path, dir_b);
free((char *)info.name);
free((char *)info.indexed_at);
free((char *)info.root_path);
cbm_store_close(store);
th_rmtree(dir_a);
th_rmtree(dir_b);
PASS();
}
TEST(pipeline_closure_repair_added_name_declines_to_full) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_closure_added_XXXXXX");
@@ -28651,6 +28721,7 @@ SUITE(pipeline_semantic_manifest_repro) {
RUN_TEST(pipeline_dbt_jinja_lineage);
RUN_TEST(pipeline_parallel_manifest_is_byte_stable_above_threshold);
RUN_TEST(pipeline_closure_repair_body_edit_converges_with_fresh_full);
RUN_TEST(pipeline_incremental_root_path_refreshed_across_directories);
RUN_TEST(pipeline_closure_repair_removed_def_drops_dependent_edge);
RUN_TEST(pipeline_closure_repair_added_name_declines_to_full);
RUN_TEST(pipeline_closure_repair_new_file_declines_to_full);