feat(index): persist coverage in separate table + queryable miss graph

Persist the parse-coverage signal outside the graph and make the misses
queryable both as a list and as a graph.

- store: new index_coverage table (project, rel_path, kind, detail) —
  coverage is metadata ABOUT the graph and never mixes into the graph
  tables; replace/get/free APIs with a deleted-file prune keyed off
  file_hashes (the live-file set)
- store: materialize a derived miss graph under the shadow project
  "<project>::coverage" (Project -> Folder chain -> File{kind,detail},
  CONTAINS_FOLDER/CONTAINS_FILE edges), rebuilt from the table inside
  the same transaction on every replace; invisible to list_projects
  (which scans the cache directory, not the projects table)
- pipeline: coverage is written in the same persist step as every other
  post-dump SQL write (hashes, ADR, FTS). Full runs replace the whole
  set; incremental runs merge rows surviving from files not re-extracted
  with this run's fresh entries, so a fixed file's flag clears and
  deleted files prune automatically
- mcp: new get_index_coverage tool (per-project parse_partial + skipped
  lists with the best-effort note); query_graph gains graph="coverage"
  to run the same cypher against the miss graph; the File-node property
  marker from the previous commit is dropped in favor of the separate
  table — the code graph gains no coverage rows
- wording: constructs in flagged ranges MAY be missing — tree-sitter
  error recovery still salvages some (verified: a function inside a
  flagged #ifdef region and even a broken def were still extracted)
- tests: store round-trip/prune/shadow-graph unit test; e2e asserts the
  response fields, the advertised coverage-graph query, code-graph
  purity, and that the flag clears after the file is fixed (incremental
  route)

Refs #963

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel committed 2026-07-08 17:47:59 +02:00
1 parent a4c17bd467
commit f9d7f4aa61
10 files changed
+667 -107

No files matched your search

-18
View File
@@ -769,24 +769,6 @@ const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn)
return cbm_ht_get(gb->node_by_qn, qn);
}
bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name,
const char *properties_json) {
if (!gb || !qualified_name || !properties_json) {
return false;
}
cbm_gbuf_node_t *node = cbm_ht_get(gb->node_by_qn, qualified_name);
if (!node) {
return false;
}
char *new_props = heap_strdup(properties_json);
if (!new_props) {
return false;
}
free(node->properties_json);
node->properties_json = new_props;
return true;
}
const cbm_gbuf_node_t *cbm_gbuf_find_by_id(const cbm_gbuf_t *gb, int64_t id) {
if (!gb || !gb->by_id || id < 0 || id >= gb->by_id_cap) {
return NULL;
-7
View File
@@ -73,13 +73,6 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name
const char *qualified_name, const char *file_path, int start_line,
int end_line, const char *properties_json);
/* Replace an existing node's properties_json by qualified name (copied).
* Bypasses the upsert survivor rule — use for post-hoc annotation of a node
* that is known to exist (e.g. parse-coverage markers on File nodes, #963).
* Returns true when the node was found and updated. */
bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name,
const char *properties_json);
/* Find a node by qualified name. Returns NULL if not found. */
const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn);
+136 -11
View File
@@ -319,11 +319,10 @@ static const tool_def_t TOOLS[] = {
"Requires target_projects param. Ensure target projects have fresh indexes first. "
"COVERAGE: the response reports files that were NOT fully indexed — 'skipped' (not "
"indexed at all: oversized/read/parse failures) and 'parse_partial' (indexed, but "
"constructs inside the listed line ranges could not be parsed and are absent from the "
"graph). parse_partial files also carry {parse_incomplete:true, error_ranges} on their "
"File node — query later via query_graph: MATCH (f:File) WHERE f.parse_incomplete = true "
"RETURN f. Both signals are best-effort: absence of a flag is NOT a completeness "
"guarantee; prefer grep inside flagged ranges.",
"constructs inside the listed line ranges could not be parsed and MAY be missing from "
"the graph). Query the persisted signal any time via the get_index_coverage tool or "
"structurally via query_graph(graph=\"coverage\"). Both signals are best-effort: absence "
"of a flag is NOT a completeness guarantee; prefer grep inside flagged ranges.",
"{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":"
"\"Path to the repository\"},"
"\"mode\":{\"type\":\"string\","
@@ -395,9 +394,20 @@ static const tool_def_t TOOLS[] = {
"no conditionally-guarded base case), param_count and max_access_depth (structure smells). "
"Find all hot-path candidates in one query, e.g. MATCH (f:Function) WHERE "
"f.transitive_loop_depth >= 3 OR f.linear_scan_in_loop >= 1 RETURN f.qualified_name, "
"f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC.",
"f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC. "
"COVERAGE GRAPH: pass graph=\"coverage\" to query the best-effort miss graph instead — "
"the file structure of files the indexer could NOT fully cover (Project → Folder → "
"File nodes with CONTAINS_FOLDER/CONTAINS_FILE edges; each File carries kind "
"(\"parse_partial\" = indexed but constructs in the flagged line ranges MAY be missing; "
"or a skip phase) and detail (the line ranges / reason)). Example: MATCH (f:File) WHERE "
"f.kind = \\\"parse_partial\\\" RETURN f.file_path, f.detail. Absence from this graph is "
"NOT a completeness guarantee.",
"{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\",\"description\":\"Cypher "
"query\"},\"project\":{\"type\":\"string\"},\"max_rows\":{\"type\":\"integer\","
"query\"},\"project\":{\"type\":\"string\"},"
"\"graph\":{\"type\":\"string\",\"enum\":[\"code\",\"coverage\"],\"default\":\"code\","
"\"description\":\"Which graph to query: the code knowledge graph (default) or the "
"indexing-coverage miss graph (files not fully indexed, as a file-structure graph).\"},"
"\"max_rows\":{\"type\":\"integer\","
"\"description\":"
"\"Optional row limit. Default: unlimited up to a 100k row "
"ceiling. No offset support — use search_graph for paginated browsing.\"}},"
@@ -489,6 +499,18 @@ static const tool_def_t TOOLS[] = {
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":["
"\"project\"]}"},
{"get_index_coverage", "Index coverage",
"Report which files the indexer could NOT fully cover (best-effort signal, #963): "
"'parse_partial' files WERE indexed but contain line ranges tree-sitter could not parse — "
"constructs there MAY be missing from the graph (some are still recovered); 'skipped' files "
"were not indexed at all (oversized/read/parse failure). Use this before trusting graph "
"completeness on a file: if a file is listed, ALSO grep it (especially the flagged ranges). "
"IMPORTANT: absence from this list is NOT a completeness guarantee — the signal only marks "
"what the indexer can detect. For structural queries over the misses use "
"query_graph(graph=\"coverage\").",
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":["
"\"project\"]}"},
{"detect_changes", "Detect changes", "Detect code changes and their impact",
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"},\"scope\":{\"type\":"
"\"string\"},\"depth\":{\"type\":\"integer\",\"default\":2},\"base_branch\":{\"type\":"
@@ -2065,10 +2087,21 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) {
cbm_store_t *store = resolve_store(srv, project);
int max_rows = cbm_mcp_get_int_arg(args, "max_rows", 0);
/* graph="coverage" (#963): run the SAME cypher against the derived
* miss-graph view (shadow project "<project>::coverage") instead of the
* code graph — file structure of not-fully-indexed files only. */
char *graph_arg = cbm_mcp_get_string_arg(args, "graph");
bool coverage_graph = graph_arg && strcmp(graph_arg, "coverage") == 0;
free(graph_arg);
if (!query) {
free(project);
return cbm_mcp_text_result("query is required", true);
}
if (coverage_graph && !project) {
free(query);
return cbm_mcp_text_result("project is required when graph=\"coverage\"", true);
}
if (!store) {
char *_err = build_project_list_error("project not found or not indexed");
char *_res = cbm_mcp_text_result(_err, true);
@@ -2085,8 +2118,15 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) {
return not_indexed;
}
char covproj[CBM_SZ_512];
const char *cypher_project = project;
if (coverage_graph) {
cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project);
cypher_project = covproj;
}
cbm_cypher_result_t result = {0};
int rc = cbm_cypher_execute(store, query, project, max_rows, &result);
int rc = cbm_cypher_execute(store, query, cypher_project, max_rows, &result);
if (rc < 0) {
char *err_msg = result.error ? result.error : "query execution failed";
@@ -2181,6 +2221,87 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) {
return result;
}
/* get_index_coverage (#963): report the best-effort indexing-coverage signal
* from the separate index_coverage table (coverage is metadata ABOUT the
* graph, stored outside it). Full per-project list, capped generously. */
enum { COVERAGE_FILE_CAP = 500 };
static char *handle_get_index_coverage(cbm_mcp_server_t *srv, const char *args) {
char *project = get_project_arg(args);
cbm_store_t *store = resolve_store(srv, project);
REQUIRE_STORE(store, project);
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
yyjson_mut_val *root = yyjson_mut_obj(doc);
yyjson_mut_doc_set_root(doc, root);
if (project) {
yyjson_mut_obj_add_str(doc, root, "project", project);
cbm_coverage_row_t *rows = NULL;
int count = 0;
(void)cbm_store_coverage_get(store, project, &rows, &count);
yyjson_mut_val *pp_files = yyjson_mut_arr(doc);
yyjson_mut_val *sk_files = yyjson_mut_arr(doc);
int pp_n = 0;
int sk_n = 0;
for (int i = 0; i < count; i++) {
bool partial = rows[i].kind && strcmp(rows[i].kind, "parse_partial") == 0;
if (partial) {
if (pp_n < COVERAGE_FILE_CAP) {
yyjson_mut_val *fe = yyjson_mut_obj(doc);
yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path);
yyjson_mut_obj_add_strcpy(doc, fe, "error_ranges",
rows[i].detail ? rows[i].detail : "");
yyjson_mut_arr_add_val(pp_files, fe);
}
pp_n++;
} else {
if (sk_n < COVERAGE_FILE_CAP) {
yyjson_mut_val *fe = yyjson_mut_obj(doc);
yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path);
yyjson_mut_obj_add_strcpy(doc, fe, "reason",
rows[i].detail ? rows[i].detail : "");
yyjson_mut_obj_add_strcpy(doc, fe, "phase", rows[i].kind ? rows[i].kind : "");
yyjson_mut_arr_add_val(sk_files, fe);
}
sk_n++;
}
}
cbm_store_free_coverage(rows, count);
yyjson_mut_val *pp = yyjson_mut_obj(doc);
yyjson_mut_obj_add_val(doc, pp, "files", pp_files);
yyjson_mut_obj_add_int(doc, pp, "count", pp_n);
yyjson_mut_obj_add_bool(doc, pp, "truncated", pp_n > COVERAGE_FILE_CAP);
yyjson_mut_obj_add_val(doc, root, "parse_partial", pp);
yyjson_mut_val *sk = yyjson_mut_obj(doc);
yyjson_mut_obj_add_val(doc, sk, "files", sk_files);
yyjson_mut_obj_add_int(doc, sk, "count", sk_n);
yyjson_mut_obj_add_bool(doc, sk, "truncated", sk_n > COVERAGE_FILE_CAP);
yyjson_mut_obj_add_val(doc, root, "skipped", sk);
yyjson_mut_obj_add_str(
doc, root, "note",
"Best-effort signal, not a completeness guarantee: parse_partial files WERE indexed, "
"but constructs inside the listed line ranges (1-based) MAY be missing from the graph "
"(tree-sitter error recovery still salvages some). skipped files were not indexed at "
"all. Prefer text search (grep) for flagged files/ranges. Files absent from this list "
"are NOT guaranteed to be fully indexed.");
} else {
yyjson_mut_obj_add_str(doc, root, "status", "no_project");
}
char *json = yy_doc_to_str(doc);
yyjson_mut_doc_free(doc);
free(project);
char *result = cbm_mcp_text_result(json, false);
free(json);
return result;
}
/* delete_project: just erase the .db file (and WAL/SHM). */
static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) {
char *name = get_project_arg(args);
@@ -3396,9 +3517,10 @@ static void add_parse_partial_summary(yyjson_mut_doc *doc, yyjson_mut_val *root,
yyjson_mut_obj_add_str(doc, pp, "note",
"Best-effort signal, not a completeness guarantee: these files WERE "
"indexed, but constructs inside the listed line ranges (1-based) could "
"not be parsed and are missing from the graph. Prefer text search "
"(grep) for those regions. Files absent from this list are NOT "
"guaranteed to be fully indexed.");
"not be parsed and MAY be missing from the graph (tree-sitter error "
"recovery still salvages some). Prefer text search (grep) for those "
"regions. Files absent from this list are NOT guaranteed to be fully "
"indexed. Query the persisted signal via get_index_coverage.");
yyjson_mut_obj_add_val(doc, root, "parse_partial", pp);
}
@@ -5856,6 +5978,9 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch
if (strcmp(tool_name, "query_graph") == 0) {
return handle_query_graph(srv, args_json);
}
if (strcmp(tool_name, "get_index_coverage") == 0) {
return handle_get_index_coverage(srv, args_json);
}
if (strcmp(tool_name, "index_status") == 0) {
return handle_index_status(srv, args_json);
}
+23 -39
View File
@@ -315,40 +315,6 @@ void cbm_pipeline_get_file_errors(const cbm_pipeline_t *p, cbm_file_error_t **ou
}
}
void cbm_pipeline_stamp_parse_partial(cbm_pipeline_t *p, cbm_gbuf_t *gbuf) {
if (!p || !gbuf) {
return;
}
int stamped = 0;
for (int i = 0; i < p->file_errors_count; i++) {
const cbm_file_error_t *e = &p->file_errors[i];
if (!e->path || !e->phase || strcmp(e->phase, "parse_partial") != 0) {
continue;
}
char *file_qn = cbm_pipeline_fqn_compute(p->project_name, e->path, "__file__");
if (!file_qn) {
continue;
}
const char *slash = strrchr(e->path, '/');
const char *basename = slash ? slash + SKIP_ONE : e->path;
const char *ext = strrchr(basename, '.');
/* Full replacement props (set_node_props does not merge): keep the
* "extension" key pass_structure wrote, add the coverage marker. The
* ranges string is digits/commas/dashes — JSON-safe unescaped. */
char props[CBM_SZ_2K];
snprintf(props, sizeof(props),
"{\"extension\":\"%s\",\"parse_incomplete\":true,\"error_ranges\":\"%s\"}",
ext ? ext : "", e->reason ? e->reason : "");
if (cbm_gbuf_set_node_props(gbuf, file_qn, props)) {
stamped++;
}
free(file_qn);
}
if (stamped > 0) {
cbm_log_info("index.parse_partial", "files", itoa_buf(stamped));
}
}
void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int *edges) {
if (nodes) {
*nodes = p ? p->committed_nodes : -1;
@@ -1148,6 +1114,29 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil
}
CBM_PROF_END_N("persist", "4_file_hashes", t_fh, file_count);
/* Coverage rows (#963): a full run's file_errors is the complete
* coverage truth for the project. The dump recreated the DB file, so
* the separate index_coverage table starts empty — write only when
* there is something to record (AFTER hashes, so the deleted-file
* prune inside replace sees the live file set). */
if (p->file_errors_count > 0) {
cbm_coverage_row_t *cov =
(cbm_coverage_row_t *)malloc((size_t)p->file_errors_count * sizeof(*cov));
if (cov) {
for (int i = 0; i < p->file_errors_count; i++) {
cov[i].rel_path = p->file_errors[i].path;
cov[i].kind = p->file_errors[i].phase;
cov[i].detail = p->file_errors[i].reason;
}
if (cbm_store_coverage_replace(hash_store, p->project_name, cov,
p->file_errors_count) != CBM_STORE_OK) {
cbm_log_error("pipeline.err", "phase", "persist_coverage", "project",
p->project_name);
}
free(cov);
}
}
/* FTS5 backfill: populate nodes_fts with camelCase-split names.
* Contentless FTS5 requires the special 'delete-all' command instead of
* DELETE FROM to wipe prior rows (there's no underlying content table).
@@ -1296,11 +1285,6 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
if (check_cancel(p)) {
return CBM_NOT_FOUND;
}
if (rc == 0) {
/* Extraction merged its per-file errors — stamp parse-partial File
* nodes (#963) before downstream passes and the dump. */
cbm_pipeline_stamp_parse_partial(p, p->gbuf);
}
return rc;
}
+52 -6
View File
@@ -631,7 +631,7 @@ static void run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_fil
static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *project,
cbm_file_info_t *files, int file_count,
const cbm_file_hash_t *mode_skipped, int mode_skipped_count,
const char *repo_path) {
const char *repo_path, const cbm_coverage_row_t *cov, int cov_count) {
struct timespec t;
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
@@ -651,6 +651,13 @@ static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *
if (hash_store) {
persist_hashes(hash_store, project, files, file_count, mode_skipped, mode_skipped_count);
/* Coverage rows (#963): re-write the merged set into the rebuilt DB
* (AFTER hashes, so the deleted-file prune sees the live file set). */
if (cov_count > 0 &&
cbm_store_coverage_replace(hash_store, project, cov, cov_count) != CBM_STORE_OK) {
cbm_log_error("incremental.err", "msg", "persist_coverage", "project", project);
}
/* FTS5 rebuild after incremental dump. The btree dump path bypasses
* any triggers that could have kept nodes_fts synchronized, so we
* rebuild from the nodes table here. See the full-dump path in
@@ -729,6 +736,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
cbm_store_free_file_hashes(stored, stored_count);
/* Coverage rows (#963): the dump below rebuilds the DB file, wiping the
* separate index_coverage table — capture the previous rows now (store
* still open) so entries for files NOT re-extracted this run survive. */
cbm_coverage_row_t *old_cov = NULL;
int old_cov_count = 0;
(void)cbm_store_coverage_get(store, project, &old_cov, &old_cov_count);
/* Build list of changed files */
cbm_file_info_t *changed_files =
(n_changed > 0) ? malloc((size_t)n_changed * sizeof(cbm_file_info_t)) : NULL;
@@ -762,6 +776,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
}
free(deleted);
free_mode_skipped(mode_skipped, mode_skipped_count);
cbm_store_free_coverage(old_cov, old_cov_count);
cbm_store_close(store);
return CBM_NOT_FOUND;
}
@@ -840,13 +855,42 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
}
run_extract_resolve(&ctx, changed_files, ci);
/* Stamp parse-partial File nodes (#963) for the changed slice. A changed
* file that parses cleanly now got fresh "{}" props above, so a stale
* marker from a previous run cannot survive the reindex. */
cbm_pipeline_stamp_parse_partial(p, existing);
cbm_pipeline_pass_k8s(&ctx, changed_files, ci);
run_postpasses(&ctx, changed_files, ci, project);
/* Coverage rows (#963): merge = previous rows for files NOT re-extracted
* this run + this run's fresh entries (changed files replace their old
* rows — a file that parses cleanly now simply contributes nothing, so
* its stale flag dies here). Rows for deleted files are pruned against
* file_hashes inside the replace. Borrowed strings: old_cov and the
* pipeline own them past the dump_and_persist call below. */
cbm_file_error_t *run_errs = NULL;
int run_err_count = 0;
cbm_pipeline_get_file_errors(p, &run_errs, &run_err_count);
cbm_coverage_row_t *cov = NULL;
int cov_n = 0;
if (old_cov_count + run_err_count > 0) {
cov = (cbm_coverage_row_t *)malloc((size_t)(old_cov_count + run_err_count) * sizeof(*cov));
}
if (cov) {
CBMHashTable *changed_set = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64);
for (int i = 0; i < ci; i++) {
cbm_ht_set(changed_set, changed_files[i].rel_path, &changed_files[i]);
}
for (int i = 0; i < old_cov_count; i++) {
if (old_cov[i].rel_path && !cbm_ht_get(changed_set, old_cov[i].rel_path)) {
cov[cov_n++] = old_cov[i];
}
}
cbm_ht_free(changed_set);
for (int i = 0; i < run_err_count; i++) {
cov[cov_n].rel_path = run_errs[i].path;
cov[cov_n].kind = run_errs[i].phase;
cov[cov_n].detail = run_errs[i].reason;
cov_n++;
}
}
free(changed_files);
cbm_registry_free(registry);
cbm_path_alias_collection_free(path_aliases);
@@ -871,7 +915,9 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
cbm_pipeline_set_committed_counts(p, cbm_gbuf_node_count(existing),
cbm_gbuf_edge_count(existing));
dump_and_persist(existing, db_path, project, files, file_count, mode_skipped,
mode_skipped_count, cbm_pipeline_repo_path(p));
mode_skipped_count, cbm_pipeline_repo_path(p), cov, cov_n);
free(cov);
cbm_store_free_coverage(old_cov, old_cov_count);
free_mode_skipped(mode_skipped, mode_skipped_count);
cbm_gbuf_free(existing);
-8
View File
@@ -520,14 +520,6 @@ void cbm_pipeline_create_route_nodes(cbm_gbuf_t *gb);
/* ── Pass function prototypes ────────────────────────────────────── */
/* Stamp File nodes for files recorded under phase "parse_partial" (#963):
* sets properties {"extension":...,"parse_incomplete":true,"error_ranges":...}
* on the (already-existing) File node so the coverage signal is queryable in
* the graph. Call after the extraction pass has merged its per-file errors —
* both the full path (pipeline.c) and the incremental path
* (pipeline_incremental.c) must call it. Implementation: pipeline.c. */
void cbm_pipeline_stamp_parse_partial(cbm_pipeline_t *p, cbm_gbuf_t *gbuf);
int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
int file_count);
+255
View File
@@ -269,6 +269,19 @@ static int init_schema(cbm_store_t *s) {
" source_hash TEXT NOT NULL,"
" created_at TEXT NOT NULL,"
" updated_at TEXT NOT NULL"
");"
/* Best-effort indexing-coverage signal (#963). One row per file the
* indexer could not fully cover: kind "parse_partial" (indexed, but the
* parse tree had ERROR/MISSING regions — detail = 1-based line ranges)
* or a skip phase ("read"/"extract"/"oversized" — detail = reason).
* Deliberately SEPARATE from the graph tables: coverage is metadata
* about the graph, not part of it. */
"CREATE TABLE IF NOT EXISTS index_coverage ("
" project TEXT NOT NULL,"
" rel_path TEXT NOT NULL,"
" kind TEXT NOT NULL,"
" detail TEXT DEFAULT '',"
" PRIMARY KEY (project, rel_path, kind)"
");";
int rc = exec_sql(s, ddl);
@@ -1833,6 +1846,248 @@ int cbm_store_delete_file_hashes(cbm_store_t *s, const char *project) {
return CBM_STORE_OK;
}
/* ── Index coverage (#963) ──────────────────────────────────────── */
void cbm_store_coverage_shadow_project(char *dst, size_t dstsz, const char *project) {
snprintf(dst, dstsz, "%s::coverage", project);
}
/* Minimal JSON string escape (quotes, backslashes, control chars → space). */
static void cov_json_escape(char *dst, size_t dstsz, const char *src) {
size_t o = 0;
for (const char *p = src; *p && o + 2 < dstsz; p++) {
unsigned char c = (unsigned char)*p;
if (c == '"' || c == '\\') {
dst[o++] = '\\';
dst[o++] = (char)c;
} else if (c < 0x20) {
dst[o++] = ' ';
} else {
dst[o++] = (char)c;
}
}
dst[o] = '\0';
}
/* Rebuild the derived miss-GRAPH view under the shadow project
* "<project>::coverage": ONLY the not-fully-indexed files, laid out as the
* file structure (Project → Folder chain → File), each File carrying
* {kind, detail}. Queryable via query_graph(graph="coverage") with zero
* cypher-engine changes; the real project's graph gains no rows. Derived
* data — rebuilt from the authoritative (post-prune) table contents. */
static void cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
char covproj[CBM_SZ_512];
cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project);
/* Wipe the previous view (edges first: no FK pragma guarantee). */
static const char *wipes[] = {"DELETE FROM edges WHERE project = ?1;",
"DELETE FROM nodes WHERE project = ?1;"};
for (int w = 0; w < 2; w++) {
sqlite3_stmt *del = NULL;
if (sqlite3_prepare_v2(s->db, wipes[w], CBM_NOT_FOUND, &del, NULL) == SQLITE_OK) {
bind_text(del, SKIP_ONE, covproj);
(void)sqlite3_step(del);
sqlite3_finalize(del);
}
}
cbm_coverage_row_t *rows = NULL;
int count = 0;
if (cbm_store_coverage_get(s, project, &rows, &count) != CBM_STORE_OK || count == 0) {
cbm_store_free_coverage(rows, count);
return;
}
/* nodes.project has an FK to projects(name) (enforced: foreign_keys=ON),
* so the shadow project needs its row. Invisible to list_projects, which
* scans the cache directory for .db files, not this table. */
if (cbm_store_upsert_project(s, covproj, "") != CBM_STORE_OK) {
cbm_store_free_coverage(rows, count);
return;
}
cbm_node_t root = {.project = covproj,
.label = "Project",
.name = project,
.qualified_name = covproj,
.properties_json = "{}"};
int64_t root_id = cbm_store_upsert_node(s, &root);
for (int i = 0; i < count; i++) {
const char *rel = rows[i].rel_path;
if (!rel || !rel[0] || root_id <= 0) {
continue;
}
char pathbuf[CBM_SZ_1K];
snprintf(pathbuf, sizeof(pathbuf), "%s", rel);
int64_t parent = root_id;
for (char *p = pathbuf; *p; p++) {
if (*p != '/') {
continue;
}
/* Truncate at this slash → pathbuf is the directory prefix; the
* upsert binds copies, so restore the slash right after. */
*p = '\0';
const char *seg = strrchr(pathbuf, '/');
cbm_node_t folder = {.project = covproj,
.label = "Folder",
.name = seg ? seg + 1 : pathbuf,
.qualified_name = pathbuf,
.file_path = pathbuf,
.properties_json = "{}"};
int64_t fid = cbm_store_upsert_node(s, &folder);
*p = '/';
if (fid > 0) {
cbm_edge_t e = {.project = covproj,
.source_id = parent,
.target_id = fid,
.type = "CONTAINS_FOLDER",
.properties_json = "{}"};
(void)cbm_store_insert_edge(s, &e);
parent = fid;
}
}
const char *base = strrchr(rel, '/');
char props[CBM_SZ_2K];
char detail_esc[CBM_SZ_1K];
cov_json_escape(detail_esc, sizeof(detail_esc), rows[i].detail ? rows[i].detail : "");
snprintf(props, sizeof(props), "{\"kind\":\"%s\",\"detail\":\"%s\"}",
rows[i].kind ? rows[i].kind : "", detail_esc);
cbm_node_t file = {.project = covproj,
.label = "File",
.name = base ? base + 1 : rel,
.qualified_name = rel,
.file_path = rel,
.properties_json = props};
int64_t file_id = cbm_store_upsert_node(s, &file);
if (file_id > 0) {
cbm_edge_t e = {.project = covproj,
.source_id = parent,
.target_id = file_id,
.type = "CONTAINS_FILE",
.properties_json = "{}"};
(void)cbm_store_insert_edge(s, &e);
}
}
cbm_store_free_coverage(rows, count);
}
int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows,
int count) {
if (!s || !s->db || !project) {
return CBM_STORE_ERR;
}
if (exec_sql(s, "BEGIN;") != CBM_STORE_OK) {
return CBM_STORE_ERR;
}
sqlite3_stmt *del = NULL;
if (sqlite3_prepare_v2(s->db, "DELETE FROM index_coverage WHERE project = ?1;", CBM_NOT_FOUND,
&del, NULL) != SQLITE_OK) {
store_set_error_sqlite(s, "coverage delete prepare");
(void)exec_sql(s, "ROLLBACK;");
return CBM_STORE_ERR;
}
bind_text(del, SKIP_ONE, project);
int rc = sqlite3_step(del);
sqlite3_finalize(del);
if (rc != SQLITE_DONE) {
store_set_error_sqlite(s, "coverage delete");
(void)exec_sql(s, "ROLLBACK;");
return CBM_STORE_ERR;
}
sqlite3_stmt *ins = NULL;
if (sqlite3_prepare_v2(
s->db,
"INSERT OR REPLACE INTO index_coverage (project, rel_path, kind, detail) "
"VALUES (?1, ?2, ?3, ?4);",
CBM_NOT_FOUND, &ins, NULL) != SQLITE_OK) {
store_set_error_sqlite(s, "coverage insert prepare");
(void)exec_sql(s, "ROLLBACK;");
return CBM_STORE_ERR;
}
for (int i = 0; i < count; i++) {
if (!rows[i].rel_path || !rows[i].kind) {
continue;
}
bind_text(ins, SKIP_ONE, project);
bind_text(ins, ST_COL_2, rows[i].rel_path);
bind_text(ins, ST_COL_3, rows[i].kind);
bind_text(ins, CBM_SZ_4, rows[i].detail ? rows[i].detail : "");
if (sqlite3_step(ins) != SQLITE_DONE) {
store_set_error_sqlite(s, "coverage insert");
sqlite3_finalize(ins);
(void)exec_sql(s, "ROLLBACK;");
return CBM_STORE_ERR;
}
sqlite3_reset(ins);
}
sqlite3_finalize(ins);
/* Prune rows for files no longer known to the index (deleted from the
* repo): file_hashes is the authoritative live-file set after persist. */
sqlite3_stmt *prune = NULL;
if (sqlite3_prepare_v2(s->db,
"DELETE FROM index_coverage WHERE project = ?1 AND rel_path NOT IN "
"(SELECT rel_path FROM file_hashes WHERE project = ?1);",
CBM_NOT_FOUND, &prune, NULL) == SQLITE_OK) {
bind_text(prune, SKIP_ONE, project);
(void)sqlite3_step(prune);
sqlite3_finalize(prune);
}
/* Rebuild the derived miss-graph view from the now-authoritative table
* contents (same transaction — the table and its view stay in step). */
cov_rebuild_shadow_graph(s, project);
return exec_sql(s, "COMMIT;");
}
int cbm_store_coverage_get(cbm_store_t *s, const char *project, cbm_coverage_row_t **out,
int *count) {
*out = NULL;
*count = 0;
if (!s || !s->db || !project) {
return CBM_STORE_ERR;
}
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(s->db,
"SELECT rel_path, kind, detail FROM index_coverage "
"WHERE project = ?1 ORDER BY rel_path, kind;",
CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
store_set_error_sqlite(s, "coverage get prepare");
return CBM_STORE_ERR;
}
bind_text(stmt, SKIP_ONE, project);
int cap = ST_INIT_CAP_16;
int n = 0;
cbm_coverage_row_t *arr = malloc(cap * sizeof(cbm_coverage_row_t));
while (sqlite3_step(stmt) == SQLITE_ROW) {
if (n >= cap) {
cap *= ST_GROWTH;
arr = safe_realloc(arr, cap * sizeof(cbm_coverage_row_t));
}
arr[n].rel_path = heap_strdup((const char *)sqlite3_column_text(stmt, 0));
arr[n].kind = heap_strdup((const char *)sqlite3_column_text(stmt, SKIP_ONE));
arr[n].detail = heap_strdup((const char *)sqlite3_column_text(stmt, CBM_SZ_2));
n++;
}
sqlite3_finalize(stmt);
*out = arr;
*count = n;
return CBM_STORE_OK;
}
void cbm_store_free_coverage(cbm_coverage_row_t *rows, int count) {
if (!rows) {
return;
}
for (int i = 0; i < count; i++) {
free((char *)rows[i].rel_path);
free((char *)rows[i].kind);
free((char *)rows[i].detail);
}
free(rows);
}
/* ── FindNodesByFileOverlap ─────────────────────────────────────── */
int cbm_store_find_nodes_by_file_overlap(cbm_store_t *s, const char *project, const char *file_path,
+34
View File
@@ -398,6 +398,40 @@ int cbm_store_delete_file_hash(cbm_store_t *s, const char *project, const char *
int cbm_store_delete_file_hashes(cbm_store_t *s, const char *project);
/* ── Index coverage (#963) ──────────────────────────────────────── */
/* One best-effort coverage row: a file the indexer could not fully cover.
* kind "parse_partial" = indexed but the parse tree had ERROR/MISSING regions
* (detail = 1-based line ranges "12-40,88-90"); skip kinds "read"/"extract"/
* "oversized" = not indexed at all (detail = reason). Stored in the separate
* index_coverage table — coverage is metadata ABOUT the graph, never mixed
* into the graph itself. */
typedef struct {
const char *rel_path;
const char *kind;
const char *detail;
} cbm_coverage_row_t;
/* Replace the project's coverage rows in one transaction, then prune rows for
* files absent from file_hashes (deleted from the repo). Call AFTER hashes
* were persisted for the run. */
int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows,
int count);
/* Fetch all coverage rows (ordered by rel_path). Caller frees via
* cbm_store_free_coverage. */
int cbm_store_coverage_get(cbm_store_t *s, const char *project, cbm_coverage_row_t **out,
int *count);
/* Name of the derived miss-graph shadow project ("<project>::coverage").
* cbm_store_coverage_replace materializes the coverage rows as a file-
* structure graph (Project → Folder → File{kind, detail}) under this project
* name — queryable via the normal cypher path without touching the real
* project's graph. */
void cbm_store_coverage_shadow_project(char *dst, size_t dstsz, const char *project);
void cbm_store_free_coverage(cbm_coverage_row_t *rows, int count);
/* ── Search ─────────────────────────────────────────────────────── */
int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_search_output_t *out);
+104 -18
View File
@@ -19,6 +19,11 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
/* Sleep before rewriting a fixture so its mtime_ns strictly increases and the
* incremental change classifier reliably sees the edit (10 ms). */
#define INCR_FIX_SLEEP_NS 10000000L
/* ── Local helpers ──────────────────────────────────────────────── */
@@ -358,35 +363,54 @@ TEST(index_parse_partial_reported) {
ASSERT_NOT_NULL(strstr(logtext, "split.c"));
free(logtext);
/* The File node carries the queryable marker. */
cbm_node_t *nodes = NULL;
int count = 0;
int rc = cbm_store_find_nodes_by_qn_suffix(store, lp.project, "__file__", &nodes, &count);
ASSERT_EQ(rc, CBM_STORE_OK);
/* The signal is persisted in the SEPARATE index_coverage table (never
* mixed into the graph tables) and queryable via get_index_coverage,
* exactly as the index_repository tool description advertises. */
cbm_coverage_row_t *rows = NULL;
int cov_count = 0;
ASSERT_EQ(cbm_store_coverage_get(store, lp.project, &rows, &cov_count), CBM_STORE_OK);
int marked = 0;
for (int i = 0; i < count; i++) {
if (nodes[i].file_path && strstr(nodes[i].file_path, "split.c")) {
ASSERT_NOT_NULL(nodes[i].properties_json);
ASSERT_NOT_NULL(strstr(nodes[i].properties_json, "\"parse_incomplete\":true"));
ASSERT_NOT_NULL(strstr(nodes[i].properties_json, "\"error_ranges\":\""));
for (int i = 0; i < cov_count; i++) {
if (rows[i].rel_path && strstr(rows[i].rel_path, "split.c")) {
ASSERT_NOT_NULL(rows[i].kind);
ASSERT_STR_EQ("parse_partial", rows[i].kind);
ASSERT_NOT_NULL(rows[i].detail);
ASSERT_GT((int)strlen(rows[i].detail), 0);
marked = 1;
}
}
cbm_store_free_nodes(nodes, count);
cbm_store_free_coverage(rows, cov_count);
ASSERT_TRUE(marked);
/* The marker is queryable exactly as the index_repository tool description
* advertises (agents are pointed at this query to find unindexed code). */
char qargs[900];
snprintf(qargs, sizeof(qargs),
"{\"project\":\"%s\",\"query\":\"MATCH (f:File) WHERE f.parse_incomplete = true "
"RETURN f.file_path, f.error_ranges\"}",
lp.project);
char *qresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs);
snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project);
char *qresp = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs);
ASSERT_NOT_NULL(qresp);
ASSERT_NOT_NULL(strstr(qresp, "split.c"));
ASSERT_NOT_NULL(strstr(qresp, "parse_partial"));
free(qresp);
/* The miss GRAPH: the same signal is queryable as a file-structure graph
* via query_graph(graph="coverage") — exactly the query the tool
* description advertises — and it lives under the shadow project, so the
* REAL code graph gained no coverage rows. */
snprintf(qargs, sizeof(qargs),
"{\"project\":\"%s\",\"graph\":\"coverage\",\"query\":\"MATCH (f:File) WHERE "
"f.kind = \\\"parse_partial\\\" RETURN f.file_path, f.detail\"}",
lp.project);
char *gresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs);
ASSERT_NOT_NULL(gresp);
ASSERT_NOT_NULL(strstr(gresp, "split.c"));
free(gresp);
snprintf(qargs, sizeof(qargs),
"{\"project\":\"%s\",\"query\":\"MATCH (f:File) WHERE f.kind = "
"\\\"parse_partial\\\" RETURN f.file_path\"}",
lp.project);
char *cresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs);
ASSERT_NOT_NULL(cresp);
ASSERT_NULL(strstr(cresp, "split.c")); /* code graph: no coverage rows */
free(cresp);
/* Clean neighbors still extract. */
int funcs = rh_count_label(store, lp.project, "Function");
ASSERT_GTE(funcs, 1);
@@ -397,8 +421,70 @@ TEST(index_parse_partial_reported) {
PASS();
}
/* INV(parse-partial-clears-on-fix, #963): the persisted coverage signal must
* stay FRESH — after the broken file is fixed and the project re-indexed
* (incremental route: the DB already exists), its parse_partial row is gone
* and get_index_coverage reports it no longer. A stale flag on a fixed file
* would make the whole signal untrustworthy. */
TEST(index_parse_partial_clears_on_fix) {
RProj lp;
memset(&lp, 0, sizeof(lp));
snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_resil_XXXXXX");
if (!cbm_mkdtemp(lp.tmpdir)) {
FAIL("mkdtemp failed");
}
rh_to_fwd_slashes(lp.tmpdir);
ri_write_text(lp.tmpdir, "flaky.py", "def ok():\n return 1\n\ndef broken(:\n pass\n");
ri_write_text(lp.tmpdir, "good.py", "def alpha():\n return 1\n");
char *resp = NULL;
cbm_store_t *store = ri_index_capture(&lp, &resp);
if (!resp) {
FAIL("no MCP response");
}
free(resp);
if (!store) {
FAIL("store did not open");
}
cbm_store_close(store);
/* Flagged after the first (full) index. */
char qargs[900];
snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project);
char *cov1 = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs);
ASSERT_NOT_NULL(cov1);
ASSERT_NOT_NULL(strstr(cov1, "flaky.py"));
free(cov1);
/* Fix the file; ensure a newer mtime so change detection can't miss it. */
struct timespec ts = {0, INCR_FIX_SLEEP_NS};
nanosleep(&ts, NULL);
ri_write_text(lp.tmpdir, "flaky.py", "def ok():\n return 1\n\ndef fixed():\n return 2\n");
/* Re-index WITHOUT deleting the DB → routes through the incremental path. */
char iargs[700];
snprintf(iargs, sizeof(iargs), "{\"repo_path\":\"%s\"}", lp.tmpdir);
char *resp2 = cbm_mcp_handle_tool(lp.srv, "index_repository", iargs);
ASSERT_NOT_NULL(resp2);
free(resp2);
char *cov2 = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs);
ASSERT_NOT_NULL(cov2);
ASSERT_NULL(strstr(cov2, "flaky.py"));
free(cov2);
store = cbm_store_open_path(lp.dbpath);
if (!store) {
FAIL("store did not reopen");
}
rh_cleanup(&lp, store);
PASS();
}
SUITE(index_resilience) {
RUN_TEST(index_oversized_file_reported);
RUN_TEST(index_clean_run_no_logfile);
RUN_TEST(index_parse_partial_reported);
RUN_TEST(index_parse_partial_clears_on_fix);
}
+63
View File
@@ -1593,7 +1593,70 @@ TEST(store_count_nodes_unknown_project) {
PASS();
}
/* ── Index coverage (#963) ──────────────────────────────────────── */
/* Round-trip + deleted-file prune + shadow miss-graph materialization +
* empty-replace wipe. The prune keys off file_hashes (the live-file set), so
* a row for a file with no hash row must not survive the replace. */
TEST(store_coverage_roundtrip_prune_shadow) {
cbm_store_t *s = cbm_store_open_memory();
cbm_store_upsert_project(s, "test", "/tmp/test");
cbm_store_upsert_file_hash(s, "test", "src/a.py", "", 1, 10);
cbm_coverage_row_t rows[] = {
{.rel_path = "src/a.py", .kind = "parse_partial", .detail = "4-7"},
{.rel_path = "gone.py", .kind = "oversized", .detail = "too big"},
};
ASSERT_EQ(cbm_store_coverage_replace(s, "test", rows, 2), CBM_STORE_OK);
cbm_coverage_row_t *got = NULL;
int n = 0;
ASSERT_EQ(cbm_store_coverage_get(s, "test", &got, &n), CBM_STORE_OK);
ASSERT_EQ(n, 1); /* gone.py pruned — no file_hashes row */
ASSERT_STR_EQ(got[0].rel_path, "src/a.py");
ASSERT_STR_EQ(got[0].kind, "parse_partial");
ASSERT_STR_EQ(got[0].detail, "4-7");
cbm_store_free_coverage(got, n);
/* Shadow miss-graph materialized under "test::coverage":
* Project → Folder(src) → File(a.py){kind,detail}. */
cbm_node_t *nodes = NULL;
int nc = 0;
ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "File", &nodes, &nc),
CBM_STORE_OK);
ASSERT_EQ(nc, 1);
ASSERT_STR_EQ(nodes[0].file_path, "src/a.py");
ASSERT_NOT_NULL(strstr(nodes[0].properties_json, "\"kind\":\"parse_partial\""));
ASSERT_NOT_NULL(strstr(nodes[0].properties_json, "\"detail\":\"4-7\""));
cbm_store_free_nodes(nodes, nc);
nodes = NULL;
nc = 0;
ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "Folder", &nodes, &nc),
CBM_STORE_OK);
ASSERT_EQ(nc, 1);
ASSERT_STR_EQ(nodes[0].name, "src");
cbm_store_free_nodes(nodes, nc);
/* Empty replace clears the table AND wipes the shadow graph. */
ASSERT_EQ(cbm_store_coverage_replace(s, "test", NULL, 0), CBM_STORE_OK);
got = NULL;
n = 0;
ASSERT_EQ(cbm_store_coverage_get(s, "test", &got, &n), CBM_STORE_OK);
ASSERT_EQ(n, 0);
free(got);
nodes = NULL;
nc = 0;
ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "File", &nodes, &nc),
CBM_STORE_OK);
ASSERT_EQ(nc, 0);
cbm_store_free_nodes(nodes, nc);
cbm_store_close(s);
PASS();
}
SUITE(store_nodes) {
RUN_TEST(store_coverage_roundtrip_prune_shadow);
RUN_TEST(store_open_memory);
RUN_TEST(store_close_null);
RUN_TEST(store_open_memory_twice);