diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 965cdd242..ca541e7d4 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -769,24 +769,6 @@ const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn) return cbm_ht_get(gb->node_by_qn, qn); } -bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name, - const char *properties_json) { - if (!gb || !qualified_name || !properties_json) { - return false; - } - cbm_gbuf_node_t *node = cbm_ht_get(gb->node_by_qn, qualified_name); - if (!node) { - return false; - } - char *new_props = heap_strdup(properties_json); - if (!new_props) { - return false; - } - free(node->properties_json); - node->properties_json = new_props; - return true; -} - const cbm_gbuf_node_t *cbm_gbuf_find_by_id(const cbm_gbuf_t *gb, int64_t id) { if (!gb || !gb->by_id || id < 0 || id >= gb->by_id_cap) { return NULL; diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index 8cea45413..6c8c6babe 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -73,13 +73,6 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name const char *qualified_name, const char *file_path, int start_line, int end_line, const char *properties_json); -/* Replace an existing node's properties_json by qualified name (copied). - * Bypasses the upsert survivor rule — use for post-hoc annotation of a node - * that is known to exist (e.g. parse-coverage markers on File nodes, #963). - * Returns true when the node was found and updated. */ -bool cbm_gbuf_set_node_props(cbm_gbuf_t *gb, const char *qualified_name, - const char *properties_json); - /* Find a node by qualified name. Returns NULL if not found. */ const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index c87fbdfe8..613a133b6 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -319,11 +319,10 @@ static const tool_def_t TOOLS[] = { "Requires target_projects param. Ensure target projects have fresh indexes first. " "COVERAGE: the response reports files that were NOT fully indexed — 'skipped' (not " "indexed at all: oversized/read/parse failures) and 'parse_partial' (indexed, but " - "constructs inside the listed line ranges could not be parsed and are absent from the " - "graph). parse_partial files also carry {parse_incomplete:true, error_ranges} on their " - "File node — query later via query_graph: MATCH (f:File) WHERE f.parse_incomplete = true " - "RETURN f. Both signals are best-effort: absence of a flag is NOT a completeness " - "guarantee; prefer grep inside flagged ranges.", + "constructs inside the listed line ranges could not be parsed and MAY be missing from " + "the graph). Query the persisted signal any time via the get_index_coverage tool or " + "structurally via query_graph(graph=\"coverage\"). Both signals are best-effort: absence " + "of a flag is NOT a completeness guarantee; prefer grep inside flagged ranges.", "{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":" "\"Path to the repository\"}," "\"mode\":{\"type\":\"string\"," @@ -395,9 +394,20 @@ static const tool_def_t TOOLS[] = { "no conditionally-guarded base case), param_count and max_access_depth (structure smells). " "Find all hot-path candidates in one query, e.g. MATCH (f:Function) WHERE " "f.transitive_loop_depth >= 3 OR f.linear_scan_in_loop >= 1 RETURN f.qualified_name, " - "f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC.", + "f.transitive_loop_depth, f.linear_scan_in_loop ORDER BY f.transitive_loop_depth DESC. " + "COVERAGE GRAPH: pass graph=\"coverage\" to query the best-effort miss graph instead — " + "the file structure of files the indexer could NOT fully cover (Project → Folder → " + "File nodes with CONTAINS_FOLDER/CONTAINS_FILE edges; each File carries kind " + "(\"parse_partial\" = indexed but constructs in the flagged line ranges MAY be missing; " + "or a skip phase) and detail (the line ranges / reason)). Example: MATCH (f:File) WHERE " + "f.kind = \\\"parse_partial\\\" RETURN f.file_path, f.detail. Absence from this graph is " + "NOT a completeness guarantee.", "{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\",\"description\":\"Cypher " - "query\"},\"project\":{\"type\":\"string\"},\"max_rows\":{\"type\":\"integer\"," + "query\"},\"project\":{\"type\":\"string\"}," + "\"graph\":{\"type\":\"string\",\"enum\":[\"code\",\"coverage\"],\"default\":\"code\"," + "\"description\":\"Which graph to query: the code knowledge graph (default) or the " + "indexing-coverage miss graph (files not fully indexed, as a file-structure graph).\"}," + "\"max_rows\":{\"type\":\"integer\"," "\"description\":" "\"Optional row limit. Default: unlimited up to a 100k row " "ceiling. No offset support — use search_graph for paginated browsing.\"}}," @@ -489,6 +499,18 @@ static const tool_def_t TOOLS[] = { "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" "\"project\"]}"}, + {"get_index_coverage", "Index coverage", + "Report which files the indexer could NOT fully cover (best-effort signal, #963): " + "'parse_partial' files WERE indexed but contain line ranges tree-sitter could not parse — " + "constructs there MAY be missing from the graph (some are still recovered); 'skipped' files " + "were not indexed at all (oversized/read/parse failure). Use this before trusting graph " + "completeness on a file: if a file is listed, ALSO grep it (especially the flagged ranges). " + "IMPORTANT: absence from this list is NOT a completeness guarantee — the signal only marks " + "what the indexer can detect. For structural queries over the misses use " + "query_graph(graph=\"coverage\").", + "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}},\"required\":[" + "\"project\"]}"}, + {"detect_changes", "Detect changes", "Detect code changes and their impact", "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"},\"scope\":{\"type\":" "\"string\"},\"depth\":{\"type\":\"integer\",\"default\":2},\"base_branch\":{\"type\":" @@ -2065,10 +2087,21 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { cbm_store_t *store = resolve_store(srv, project); int max_rows = cbm_mcp_get_int_arg(args, "max_rows", 0); + /* graph="coverage" (#963): run the SAME cypher against the derived + * miss-graph view (shadow project "::coverage") instead of the + * code graph — file structure of not-fully-indexed files only. */ + char *graph_arg = cbm_mcp_get_string_arg(args, "graph"); + bool coverage_graph = graph_arg && strcmp(graph_arg, "coverage") == 0; + free(graph_arg); + if (!query) { free(project); return cbm_mcp_text_result("query is required", true); } + if (coverage_graph && !project) { + free(query); + return cbm_mcp_text_result("project is required when graph=\"coverage\"", true); + } if (!store) { char *_err = build_project_list_error("project not found or not indexed"); char *_res = cbm_mcp_text_result(_err, true); @@ -2085,8 +2118,15 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { return not_indexed; } + char covproj[CBM_SZ_512]; + const char *cypher_project = project; + if (coverage_graph) { + cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project); + cypher_project = covproj; + } + cbm_cypher_result_t result = {0}; - int rc = cbm_cypher_execute(store, query, project, max_rows, &result); + int rc = cbm_cypher_execute(store, query, cypher_project, max_rows, &result); if (rc < 0) { char *err_msg = result.error ? result.error : "query execution failed"; @@ -2181,6 +2221,87 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { return result; } +/* get_index_coverage (#963): report the best-effort indexing-coverage signal + * from the separate index_coverage table (coverage is metadata ABOUT the + * graph, stored outside it). Full per-project list, capped generously. */ +enum { COVERAGE_FILE_CAP = 500 }; + +static char *handle_get_index_coverage(cbm_mcp_server_t *srv, const char *args) { + char *project = get_project_arg(args); + cbm_store_t *store = resolve_store(srv, project); + REQUIRE_STORE(store, project); + + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + + if (project) { + yyjson_mut_obj_add_str(doc, root, "project", project); + cbm_coverage_row_t *rows = NULL; + int count = 0; + (void)cbm_store_coverage_get(store, project, &rows, &count); + + yyjson_mut_val *pp_files = yyjson_mut_arr(doc); + yyjson_mut_val *sk_files = yyjson_mut_arr(doc); + int pp_n = 0; + int sk_n = 0; + for (int i = 0; i < count; i++) { + bool partial = rows[i].kind && strcmp(rows[i].kind, "parse_partial") == 0; + if (partial) { + if (pp_n < COVERAGE_FILE_CAP) { + yyjson_mut_val *fe = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path); + yyjson_mut_obj_add_strcpy(doc, fe, "error_ranges", + rows[i].detail ? rows[i].detail : ""); + yyjson_mut_arr_add_val(pp_files, fe); + } + pp_n++; + } else { + if (sk_n < COVERAGE_FILE_CAP) { + yyjson_mut_val *fe = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, fe, "path", rows[i].rel_path); + yyjson_mut_obj_add_strcpy(doc, fe, "reason", + rows[i].detail ? rows[i].detail : ""); + yyjson_mut_obj_add_strcpy(doc, fe, "phase", rows[i].kind ? rows[i].kind : ""); + yyjson_mut_arr_add_val(sk_files, fe); + } + sk_n++; + } + } + cbm_store_free_coverage(rows, count); + + yyjson_mut_val *pp = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, pp, "files", pp_files); + yyjson_mut_obj_add_int(doc, pp, "count", pp_n); + yyjson_mut_obj_add_bool(doc, pp, "truncated", pp_n > COVERAGE_FILE_CAP); + yyjson_mut_obj_add_val(doc, root, "parse_partial", pp); + + yyjson_mut_val *sk = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, sk, "files", sk_files); + yyjson_mut_obj_add_int(doc, sk, "count", sk_n); + yyjson_mut_obj_add_bool(doc, sk, "truncated", sk_n > COVERAGE_FILE_CAP); + yyjson_mut_obj_add_val(doc, root, "skipped", sk); + + yyjson_mut_obj_add_str( + doc, root, "note", + "Best-effort signal, not a completeness guarantee: parse_partial files WERE indexed, " + "but constructs inside the listed line ranges (1-based) MAY be missing from the graph " + "(tree-sitter error recovery still salvages some). skipped files were not indexed at " + "all. Prefer text search (grep) for flagged files/ranges. Files absent from this list " + "are NOT guaranteed to be fully indexed."); + } else { + yyjson_mut_obj_add_str(doc, root, "status", "no_project"); + } + + char *json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + free(project); + + char *result = cbm_mcp_text_result(json, false); + free(json); + return result; +} + /* delete_project: just erase the .db file (and WAL/SHM). */ static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) { char *name = get_project_arg(args); @@ -3396,9 +3517,10 @@ static void add_parse_partial_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, yyjson_mut_obj_add_str(doc, pp, "note", "Best-effort signal, not a completeness guarantee: these files WERE " "indexed, but constructs inside the listed line ranges (1-based) could " - "not be parsed and are missing from the graph. Prefer text search " - "(grep) for those regions. Files absent from this list are NOT " - "guaranteed to be fully indexed."); + "not be parsed and MAY be missing from the graph (tree-sitter error " + "recovery still salvages some). Prefer text search (grep) for those " + "regions. Files absent from this list are NOT guaranteed to be fully " + "indexed. Query the persisted signal via get_index_coverage."); yyjson_mut_obj_add_val(doc, root, "parse_partial", pp); } @@ -5856,6 +5978,9 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch if (strcmp(tool_name, "query_graph") == 0) { return handle_query_graph(srv, args_json); } + if (strcmp(tool_name, "get_index_coverage") == 0) { + return handle_get_index_coverage(srv, args_json); + } if (strcmp(tool_name, "index_status") == 0) { return handle_index_status(srv, args_json); } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index dbce7198b..f72b50e13 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -315,40 +315,6 @@ void cbm_pipeline_get_file_errors(const cbm_pipeline_t *p, cbm_file_error_t **ou } } -void cbm_pipeline_stamp_parse_partial(cbm_pipeline_t *p, cbm_gbuf_t *gbuf) { - if (!p || !gbuf) { - return; - } - int stamped = 0; - for (int i = 0; i < p->file_errors_count; i++) { - const cbm_file_error_t *e = &p->file_errors[i]; - if (!e->path || !e->phase || strcmp(e->phase, "parse_partial") != 0) { - continue; - } - char *file_qn = cbm_pipeline_fqn_compute(p->project_name, e->path, "__file__"); - if (!file_qn) { - continue; - } - const char *slash = strrchr(e->path, '/'); - const char *basename = slash ? slash + SKIP_ONE : e->path; - const char *ext = strrchr(basename, '.'); - /* Full replacement props (set_node_props does not merge): keep the - * "extension" key pass_structure wrote, add the coverage marker. The - * ranges string is digits/commas/dashes — JSON-safe unescaped. */ - char props[CBM_SZ_2K]; - snprintf(props, sizeof(props), - "{\"extension\":\"%s\",\"parse_incomplete\":true,\"error_ranges\":\"%s\"}", - ext ? ext : "", e->reason ? e->reason : ""); - if (cbm_gbuf_set_node_props(gbuf, file_qn, props)) { - stamped++; - } - free(file_qn); - } - if (stamped > 0) { - cbm_log_info("index.parse_partial", "files", itoa_buf(stamped)); - } -} - void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int *edges) { if (nodes) { *nodes = p ? p->committed_nodes : -1; @@ -1148,6 +1114,29 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil } CBM_PROF_END_N("persist", "4_file_hashes", t_fh, file_count); + /* Coverage rows (#963): a full run's file_errors is the complete + * coverage truth for the project. The dump recreated the DB file, so + * the separate index_coverage table starts empty — write only when + * there is something to record (AFTER hashes, so the deleted-file + * prune inside replace sees the live file set). */ + if (p->file_errors_count > 0) { + cbm_coverage_row_t *cov = + (cbm_coverage_row_t *)malloc((size_t)p->file_errors_count * sizeof(*cov)); + if (cov) { + for (int i = 0; i < p->file_errors_count; i++) { + cov[i].rel_path = p->file_errors[i].path; + cov[i].kind = p->file_errors[i].phase; + cov[i].detail = p->file_errors[i].reason; + } + if (cbm_store_coverage_replace(hash_store, p->project_name, cov, + p->file_errors_count) != CBM_STORE_OK) { + cbm_log_error("pipeline.err", "phase", "persist_coverage", "project", + p->project_name); + } + free(cov); + } + } + /* FTS5 backfill: populate nodes_fts with camelCase-split names. * Contentless FTS5 requires the special 'delete-all' command instead of * DELETE FROM to wipe prior rows (there's no underlying content table). @@ -1296,11 +1285,6 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, if (check_cancel(p)) { return CBM_NOT_FOUND; } - if (rc == 0) { - /* Extraction merged its per-file errors — stamp parse-partial File - * nodes (#963) before downstream passes and the dump. */ - cbm_pipeline_stamp_parse_partial(p, p->gbuf); - } return rc; } diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 4b5cedf8f..f2cc3fe79 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -631,7 +631,7 @@ static void run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_fil static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *project, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *mode_skipped, int mode_skipped_count, - const char *repo_path) { + const char *repo_path, const cbm_coverage_row_t *cov, int cov_count) { struct timespec t; cbm_clock_gettime(CLOCK_MONOTONIC, &t); @@ -651,6 +651,13 @@ static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char * if (hash_store) { persist_hashes(hash_store, project, files, file_count, mode_skipped, mode_skipped_count); + /* Coverage rows (#963): re-write the merged set into the rebuilt DB + * (AFTER hashes, so the deleted-file prune sees the live file set). */ + if (cov_count > 0 && + cbm_store_coverage_replace(hash_store, project, cov, cov_count) != CBM_STORE_OK) { + cbm_log_error("incremental.err", "msg", "persist_coverage", "project", project); + } + /* FTS5 rebuild after incremental dump. The btree dump path bypasses * any triggers that could have kept nodes_fts synchronized, so we * rebuild from the nodes table here. See the full-dump path in @@ -729,6 +736,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_store_free_file_hashes(stored, stored_count); + /* Coverage rows (#963): the dump below rebuilds the DB file, wiping the + * separate index_coverage table — capture the previous rows now (store + * still open) so entries for files NOT re-extracted this run survive. */ + cbm_coverage_row_t *old_cov = NULL; + int old_cov_count = 0; + (void)cbm_store_coverage_get(store, project, &old_cov, &old_cov_count); + /* Build list of changed files */ cbm_file_info_t *changed_files = (n_changed > 0) ? malloc((size_t)n_changed * sizeof(cbm_file_info_t)) : NULL; @@ -762,6 +776,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil } free(deleted); free_mode_skipped(mode_skipped, mode_skipped_count); + cbm_store_free_coverage(old_cov, old_cov_count); cbm_store_close(store); return CBM_NOT_FOUND; } @@ -840,13 +855,42 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil } run_extract_resolve(&ctx, changed_files, ci); - /* Stamp parse-partial File nodes (#963) for the changed slice. A changed - * file that parses cleanly now got fresh "{}" props above, so a stale - * marker from a previous run cannot survive the reindex. */ - cbm_pipeline_stamp_parse_partial(p, existing); cbm_pipeline_pass_k8s(&ctx, changed_files, ci); run_postpasses(&ctx, changed_files, ci, project); + /* Coverage rows (#963): merge = previous rows for files NOT re-extracted + * this run + this run's fresh entries (changed files replace their old + * rows — a file that parses cleanly now simply contributes nothing, so + * its stale flag dies here). Rows for deleted files are pruned against + * file_hashes inside the replace. Borrowed strings: old_cov and the + * pipeline own them past the dump_and_persist call below. */ + cbm_file_error_t *run_errs = NULL; + int run_err_count = 0; + cbm_pipeline_get_file_errors(p, &run_errs, &run_err_count); + cbm_coverage_row_t *cov = NULL; + int cov_n = 0; + if (old_cov_count + run_err_count > 0) { + cov = (cbm_coverage_row_t *)malloc((size_t)(old_cov_count + run_err_count) * sizeof(*cov)); + } + if (cov) { + CBMHashTable *changed_set = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64); + for (int i = 0; i < ci; i++) { + cbm_ht_set(changed_set, changed_files[i].rel_path, &changed_files[i]); + } + for (int i = 0; i < old_cov_count; i++) { + if (old_cov[i].rel_path && !cbm_ht_get(changed_set, old_cov[i].rel_path)) { + cov[cov_n++] = old_cov[i]; + } + } + cbm_ht_free(changed_set); + for (int i = 0; i < run_err_count; i++) { + cov[cov_n].rel_path = run_errs[i].path; + cov[cov_n].kind = run_errs[i].phase; + cov[cov_n].detail = run_errs[i].reason; + cov_n++; + } + } + free(changed_files); cbm_registry_free(registry); cbm_path_alias_collection_free(path_aliases); @@ -871,7 +915,9 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_pipeline_set_committed_counts(p, cbm_gbuf_node_count(existing), cbm_gbuf_edge_count(existing)); dump_and_persist(existing, db_path, project, files, file_count, mode_skipped, - mode_skipped_count, cbm_pipeline_repo_path(p)); + mode_skipped_count, cbm_pipeline_repo_path(p), cov, cov_n); + free(cov); + cbm_store_free_coverage(old_cov, old_cov_count); free_mode_skipped(mode_skipped, mode_skipped_count); cbm_gbuf_free(existing); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 1648bddfe..a3d806558 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -520,14 +520,6 @@ void cbm_pipeline_create_route_nodes(cbm_gbuf_t *gb); /* ── Pass function prototypes ────────────────────────────────────── */ -/* Stamp File nodes for files recorded under phase "parse_partial" (#963): - * sets properties {"extension":...,"parse_incomplete":true,"error_ranges":...} - * on the (already-existing) File node so the coverage signal is queryable in - * the graph. Call after the extraction pass has merged its per-file errors — - * both the full path (pipeline.c) and the incremental path - * (pipeline_incremental.c) must call it. Implementation: pipeline.c. */ -void cbm_pipeline_stamp_parse_partial(cbm_pipeline_t *p, cbm_gbuf_t *gbuf); - int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); diff --git a/src/store/store.c b/src/store/store.c index 33a9453c2..3e1735cac 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -269,6 +269,19 @@ static int init_schema(cbm_store_t *s) { " source_hash TEXT NOT NULL," " created_at TEXT NOT NULL," " updated_at TEXT NOT NULL" + ");" + /* Best-effort indexing-coverage signal (#963). One row per file the + * indexer could not fully cover: kind "parse_partial" (indexed, but the + * parse tree had ERROR/MISSING regions — detail = 1-based line ranges) + * or a skip phase ("read"/"extract"/"oversized" — detail = reason). + * Deliberately SEPARATE from the graph tables: coverage is metadata + * about the graph, not part of it. */ + "CREATE TABLE IF NOT EXISTS index_coverage (" + " project TEXT NOT NULL," + " rel_path TEXT NOT NULL," + " kind TEXT NOT NULL," + " detail TEXT DEFAULT ''," + " PRIMARY KEY (project, rel_path, kind)" ");"; int rc = exec_sql(s, ddl); @@ -1833,6 +1846,248 @@ int cbm_store_delete_file_hashes(cbm_store_t *s, const char *project) { return CBM_STORE_OK; } +/* ── Index coverage (#963) ──────────────────────────────────────── */ + +void cbm_store_coverage_shadow_project(char *dst, size_t dstsz, const char *project) { + snprintf(dst, dstsz, "%s::coverage", project); +} + +/* Minimal JSON string escape (quotes, backslashes, control chars → space). */ +static void cov_json_escape(char *dst, size_t dstsz, const char *src) { + size_t o = 0; + for (const char *p = src; *p && o + 2 < dstsz; p++) { + unsigned char c = (unsigned char)*p; + if (c == '"' || c == '\\') { + dst[o++] = '\\'; + dst[o++] = (char)c; + } else if (c < 0x20) { + dst[o++] = ' '; + } else { + dst[o++] = (char)c; + } + } + dst[o] = '\0'; +} + +/* Rebuild the derived miss-GRAPH view under the shadow project + * "::coverage": ONLY the not-fully-indexed files, laid out as the + * file structure (Project → Folder chain → File), each File carrying + * {kind, detail}. Queryable via query_graph(graph="coverage") with zero + * cypher-engine changes; the real project's graph gains no rows. Derived + * data — rebuilt from the authoritative (post-prune) table contents. */ +static void cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) { + char covproj[CBM_SZ_512]; + cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project); + + /* Wipe the previous view (edges first: no FK pragma guarantee). */ + static const char *wipes[] = {"DELETE FROM edges WHERE project = ?1;", + "DELETE FROM nodes WHERE project = ?1;"}; + for (int w = 0; w < 2; w++) { + sqlite3_stmt *del = NULL; + if (sqlite3_prepare_v2(s->db, wipes[w], CBM_NOT_FOUND, &del, NULL) == SQLITE_OK) { + bind_text(del, SKIP_ONE, covproj); + (void)sqlite3_step(del); + sqlite3_finalize(del); + } + } + + cbm_coverage_row_t *rows = NULL; + int count = 0; + if (cbm_store_coverage_get(s, project, &rows, &count) != CBM_STORE_OK || count == 0) { + cbm_store_free_coverage(rows, count); + return; + } + + /* nodes.project has an FK to projects(name) (enforced: foreign_keys=ON), + * so the shadow project needs its row. Invisible to list_projects, which + * scans the cache directory for .db files, not this table. */ + if (cbm_store_upsert_project(s, covproj, "") != CBM_STORE_OK) { + cbm_store_free_coverage(rows, count); + return; + } + + cbm_node_t root = {.project = covproj, + .label = "Project", + .name = project, + .qualified_name = covproj, + .properties_json = "{}"}; + int64_t root_id = cbm_store_upsert_node(s, &root); + + for (int i = 0; i < count; i++) { + const char *rel = rows[i].rel_path; + if (!rel || !rel[0] || root_id <= 0) { + continue; + } + char pathbuf[CBM_SZ_1K]; + snprintf(pathbuf, sizeof(pathbuf), "%s", rel); + + int64_t parent = root_id; + for (char *p = pathbuf; *p; p++) { + if (*p != '/') { + continue; + } + /* Truncate at this slash → pathbuf is the directory prefix; the + * upsert binds copies, so restore the slash right after. */ + *p = '\0'; + const char *seg = strrchr(pathbuf, '/'); + cbm_node_t folder = {.project = covproj, + .label = "Folder", + .name = seg ? seg + 1 : pathbuf, + .qualified_name = pathbuf, + .file_path = pathbuf, + .properties_json = "{}"}; + int64_t fid = cbm_store_upsert_node(s, &folder); + *p = '/'; + if (fid > 0) { + cbm_edge_t e = {.project = covproj, + .source_id = parent, + .target_id = fid, + .type = "CONTAINS_FOLDER", + .properties_json = "{}"}; + (void)cbm_store_insert_edge(s, &e); + parent = fid; + } + } + + const char *base = strrchr(rel, '/'); + char props[CBM_SZ_2K]; + char detail_esc[CBM_SZ_1K]; + cov_json_escape(detail_esc, sizeof(detail_esc), rows[i].detail ? rows[i].detail : ""); + snprintf(props, sizeof(props), "{\"kind\":\"%s\",\"detail\":\"%s\"}", + rows[i].kind ? rows[i].kind : "", detail_esc); + cbm_node_t file = {.project = covproj, + .label = "File", + .name = base ? base + 1 : rel, + .qualified_name = rel, + .file_path = rel, + .properties_json = props}; + int64_t file_id = cbm_store_upsert_node(s, &file); + if (file_id > 0) { + cbm_edge_t e = {.project = covproj, + .source_id = parent, + .target_id = file_id, + .type = "CONTAINS_FILE", + .properties_json = "{}"}; + (void)cbm_store_insert_edge(s, &e); + } + } + cbm_store_free_coverage(rows, count); +} + +int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows, + int count) { + if (!s || !s->db || !project) { + return CBM_STORE_ERR; + } + if (exec_sql(s, "BEGIN;") != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + sqlite3_stmt *del = NULL; + if (sqlite3_prepare_v2(s->db, "DELETE FROM index_coverage WHERE project = ?1;", CBM_NOT_FOUND, + &del, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage delete prepare"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + bind_text(del, SKIP_ONE, project); + int rc = sqlite3_step(del); + sqlite3_finalize(del); + if (rc != SQLITE_DONE) { + store_set_error_sqlite(s, "coverage delete"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + sqlite3_stmt *ins = NULL; + if (sqlite3_prepare_v2( + s->db, + "INSERT OR REPLACE INTO index_coverage (project, rel_path, kind, detail) " + "VALUES (?1, ?2, ?3, ?4);", + CBM_NOT_FOUND, &ins, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage insert prepare"); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + for (int i = 0; i < count; i++) { + if (!rows[i].rel_path || !rows[i].kind) { + continue; + } + bind_text(ins, SKIP_ONE, project); + bind_text(ins, ST_COL_2, rows[i].rel_path); + bind_text(ins, ST_COL_3, rows[i].kind); + bind_text(ins, CBM_SZ_4, rows[i].detail ? rows[i].detail : ""); + if (sqlite3_step(ins) != SQLITE_DONE) { + store_set_error_sqlite(s, "coverage insert"); + sqlite3_finalize(ins); + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } + sqlite3_reset(ins); + } + sqlite3_finalize(ins); + /* Prune rows for files no longer known to the index (deleted from the + * repo): file_hashes is the authoritative live-file set after persist. */ + sqlite3_stmt *prune = NULL; + if (sqlite3_prepare_v2(s->db, + "DELETE FROM index_coverage WHERE project = ?1 AND rel_path NOT IN " + "(SELECT rel_path FROM file_hashes WHERE project = ?1);", + CBM_NOT_FOUND, &prune, NULL) == SQLITE_OK) { + bind_text(prune, SKIP_ONE, project); + (void)sqlite3_step(prune); + sqlite3_finalize(prune); + } + /* Rebuild the derived miss-graph view from the now-authoritative table + * contents (same transaction — the table and its view stay in step). */ + cov_rebuild_shadow_graph(s, project); + return exec_sql(s, "COMMIT;"); +} + +int cbm_store_coverage_get(cbm_store_t *s, const char *project, cbm_coverage_row_t **out, + int *count) { + *out = NULL; + *count = 0; + if (!s || !s->db || !project) { + return CBM_STORE_ERR; + } + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, + "SELECT rel_path, kind, detail FROM index_coverage " + "WHERE project = ?1 ORDER BY rel_path, kind;", + CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage get prepare"); + return CBM_STORE_ERR; + } + bind_text(stmt, SKIP_ONE, project); + int cap = ST_INIT_CAP_16; + int n = 0; + cbm_coverage_row_t *arr = malloc(cap * sizeof(cbm_coverage_row_t)); + while (sqlite3_step(stmt) == SQLITE_ROW) { + if (n >= cap) { + cap *= ST_GROWTH; + arr = safe_realloc(arr, cap * sizeof(cbm_coverage_row_t)); + } + arr[n].rel_path = heap_strdup((const char *)sqlite3_column_text(stmt, 0)); + arr[n].kind = heap_strdup((const char *)sqlite3_column_text(stmt, SKIP_ONE)); + arr[n].detail = heap_strdup((const char *)sqlite3_column_text(stmt, CBM_SZ_2)); + n++; + } + sqlite3_finalize(stmt); + *out = arr; + *count = n; + return CBM_STORE_OK; +} + +void cbm_store_free_coverage(cbm_coverage_row_t *rows, int count) { + if (!rows) { + return; + } + for (int i = 0; i < count; i++) { + free((char *)rows[i].rel_path); + free((char *)rows[i].kind); + free((char *)rows[i].detail); + } + free(rows); +} + /* ── FindNodesByFileOverlap ─────────────────────────────────────── */ int cbm_store_find_nodes_by_file_overlap(cbm_store_t *s, const char *project, const char *file_path, diff --git a/src/store/store.h b/src/store/store.h index a4541e2df..e83be73e4 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -398,6 +398,40 @@ int cbm_store_delete_file_hash(cbm_store_t *s, const char *project, const char * int cbm_store_delete_file_hashes(cbm_store_t *s, const char *project); +/* ── Index coverage (#963) ──────────────────────────────────────── */ + +/* One best-effort coverage row: a file the indexer could not fully cover. + * kind "parse_partial" = indexed but the parse tree had ERROR/MISSING regions + * (detail = 1-based line ranges "12-40,88-90"); skip kinds "read"/"extract"/ + * "oversized" = not indexed at all (detail = reason). Stored in the separate + * index_coverage table — coverage is metadata ABOUT the graph, never mixed + * into the graph itself. */ +typedef struct { + const char *rel_path; + const char *kind; + const char *detail; +} cbm_coverage_row_t; + +/* Replace the project's coverage rows in one transaction, then prune rows for + * files absent from file_hashes (deleted from the repo). Call AFTER hashes + * were persisted for the run. */ +int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows, + int count); + +/* Fetch all coverage rows (ordered by rel_path). Caller frees via + * cbm_store_free_coverage. */ +int cbm_store_coverage_get(cbm_store_t *s, const char *project, cbm_coverage_row_t **out, + int *count); + +/* Name of the derived miss-graph shadow project ("::coverage"). + * cbm_store_coverage_replace materializes the coverage rows as a file- + * structure graph (Project → Folder → File{kind, detail}) under this project + * name — queryable via the normal cypher path without touching the real + * project's graph. */ +void cbm_store_coverage_shadow_project(char *dst, size_t dstsz, const char *project); + +void cbm_store_free_coverage(cbm_coverage_row_t *rows, int count); + /* ── Search ─────────────────────────────────────────────────────── */ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_search_output_t *out); diff --git a/tests/test_index_resilience.c b/tests/test_index_resilience.c index db8cc709f..a16c7fac1 100644 --- a/tests/test_index_resilience.c +++ b/tests/test_index_resilience.c @@ -19,6 +19,11 @@ #include #include #include +#include + +/* Sleep before rewriting a fixture so its mtime_ns strictly increases and the + * incremental change classifier reliably sees the edit (10 ms). */ +#define INCR_FIX_SLEEP_NS 10000000L /* ── Local helpers ──────────────────────────────────────────────── */ @@ -358,35 +363,54 @@ TEST(index_parse_partial_reported) { ASSERT_NOT_NULL(strstr(logtext, "split.c")); free(logtext); - /* The File node carries the queryable marker. */ - cbm_node_t *nodes = NULL; - int count = 0; - int rc = cbm_store_find_nodes_by_qn_suffix(store, lp.project, "__file__", &nodes, &count); - ASSERT_EQ(rc, CBM_STORE_OK); + /* The signal is persisted in the SEPARATE index_coverage table (never + * mixed into the graph tables) and queryable via get_index_coverage, + * exactly as the index_repository tool description advertises. */ + cbm_coverage_row_t *rows = NULL; + int cov_count = 0; + ASSERT_EQ(cbm_store_coverage_get(store, lp.project, &rows, &cov_count), CBM_STORE_OK); int marked = 0; - for (int i = 0; i < count; i++) { - if (nodes[i].file_path && strstr(nodes[i].file_path, "split.c")) { - ASSERT_NOT_NULL(nodes[i].properties_json); - ASSERT_NOT_NULL(strstr(nodes[i].properties_json, "\"parse_incomplete\":true")); - ASSERT_NOT_NULL(strstr(nodes[i].properties_json, "\"error_ranges\":\"")); + for (int i = 0; i < cov_count; i++) { + if (rows[i].rel_path && strstr(rows[i].rel_path, "split.c")) { + ASSERT_NOT_NULL(rows[i].kind); + ASSERT_STR_EQ("parse_partial", rows[i].kind); + ASSERT_NOT_NULL(rows[i].detail); + ASSERT_GT((int)strlen(rows[i].detail), 0); marked = 1; } } - cbm_store_free_nodes(nodes, count); + cbm_store_free_coverage(rows, cov_count); ASSERT_TRUE(marked); - /* The marker is queryable exactly as the index_repository tool description - * advertises (agents are pointed at this query to find unindexed code). */ char qargs[900]; - snprintf(qargs, sizeof(qargs), - "{\"project\":\"%s\",\"query\":\"MATCH (f:File) WHERE f.parse_incomplete = true " - "RETURN f.file_path, f.error_ranges\"}", - lp.project); - char *qresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs); + snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project); + char *qresp = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); ASSERT_NOT_NULL(qresp); ASSERT_NOT_NULL(strstr(qresp, "split.c")); + ASSERT_NOT_NULL(strstr(qresp, "parse_partial")); free(qresp); + /* The miss GRAPH: the same signal is queryable as a file-structure graph + * via query_graph(graph="coverage") — exactly the query the tool + * description advertises — and it lives under the shadow project, so the + * REAL code graph gained no coverage rows. */ + snprintf(qargs, sizeof(qargs), + "{\"project\":\"%s\",\"graph\":\"coverage\",\"query\":\"MATCH (f:File) WHERE " + "f.kind = \\\"parse_partial\\\" RETURN f.file_path, f.detail\"}", + lp.project); + char *gresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs); + ASSERT_NOT_NULL(gresp); + ASSERT_NOT_NULL(strstr(gresp, "split.c")); + free(gresp); + snprintf(qargs, sizeof(qargs), + "{\"project\":\"%s\",\"query\":\"MATCH (f:File) WHERE f.kind = " + "\\\"parse_partial\\\" RETURN f.file_path\"}", + lp.project); + char *cresp = cbm_mcp_handle_tool(lp.srv, "query_graph", qargs); + ASSERT_NOT_NULL(cresp); + ASSERT_NULL(strstr(cresp, "split.c")); /* code graph: no coverage rows */ + free(cresp); + /* Clean neighbors still extract. */ int funcs = rh_count_label(store, lp.project, "Function"); ASSERT_GTE(funcs, 1); @@ -397,8 +421,70 @@ TEST(index_parse_partial_reported) { PASS(); } +/* INV(parse-partial-clears-on-fix, #963): the persisted coverage signal must + * stay FRESH — after the broken file is fixed and the project re-indexed + * (incremental route: the DB already exists), its parse_partial row is gone + * and get_index_coverage reports it no longer. A stale flag on a fixed file + * would make the whole signal untrustworthy. */ +TEST(index_parse_partial_clears_on_fix) { + RProj lp; + memset(&lp, 0, sizeof(lp)); + snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_resil_XXXXXX"); + if (!cbm_mkdtemp(lp.tmpdir)) { + FAIL("mkdtemp failed"); + } + rh_to_fwd_slashes(lp.tmpdir); + + ri_write_text(lp.tmpdir, "flaky.py", "def ok():\n return 1\n\ndef broken(:\n pass\n"); + ri_write_text(lp.tmpdir, "good.py", "def alpha():\n return 1\n"); + + char *resp = NULL; + cbm_store_t *store = ri_index_capture(&lp, &resp); + if (!resp) { + FAIL("no MCP response"); + } + free(resp); + if (!store) { + FAIL("store did not open"); + } + cbm_store_close(store); + + /* Flagged after the first (full) index. */ + char qargs[900]; + snprintf(qargs, sizeof(qargs), "{\"project\":\"%s\"}", lp.project); + char *cov1 = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); + ASSERT_NOT_NULL(cov1); + ASSERT_NOT_NULL(strstr(cov1, "flaky.py")); + free(cov1); + + /* Fix the file; ensure a newer mtime so change detection can't miss it. */ + struct timespec ts = {0, INCR_FIX_SLEEP_NS}; + nanosleep(&ts, NULL); + ri_write_text(lp.tmpdir, "flaky.py", "def ok():\n return 1\n\ndef fixed():\n return 2\n"); + + /* Re-index WITHOUT deleting the DB → routes through the incremental path. */ + char iargs[700]; + snprintf(iargs, sizeof(iargs), "{\"repo_path\":\"%s\"}", lp.tmpdir); + char *resp2 = cbm_mcp_handle_tool(lp.srv, "index_repository", iargs); + ASSERT_NOT_NULL(resp2); + free(resp2); + + char *cov2 = cbm_mcp_handle_tool(lp.srv, "get_index_coverage", qargs); + ASSERT_NOT_NULL(cov2); + ASSERT_NULL(strstr(cov2, "flaky.py")); + free(cov2); + + store = cbm_store_open_path(lp.dbpath); + if (!store) { + FAIL("store did not reopen"); + } + rh_cleanup(&lp, store); + PASS(); +} + SUITE(index_resilience) { RUN_TEST(index_oversized_file_reported); RUN_TEST(index_clean_run_no_logfile); RUN_TEST(index_parse_partial_reported); + RUN_TEST(index_parse_partial_clears_on_fix); } diff --git a/tests/test_store_nodes.c b/tests/test_store_nodes.c index 5b30ef9f5..46a864841 100644 --- a/tests/test_store_nodes.c +++ b/tests/test_store_nodes.c @@ -1593,7 +1593,70 @@ TEST(store_count_nodes_unknown_project) { PASS(); } +/* ── Index coverage (#963) ──────────────────────────────────────── */ + +/* Round-trip + deleted-file prune + shadow miss-graph materialization + + * empty-replace wipe. The prune keys off file_hashes (the live-file set), so + * a row for a file with no hash row must not survive the replace. */ +TEST(store_coverage_roundtrip_prune_shadow) { + cbm_store_t *s = cbm_store_open_memory(); + cbm_store_upsert_project(s, "test", "/tmp/test"); + cbm_store_upsert_file_hash(s, "test", "src/a.py", "", 1, 10); + + cbm_coverage_row_t rows[] = { + {.rel_path = "src/a.py", .kind = "parse_partial", .detail = "4-7"}, + {.rel_path = "gone.py", .kind = "oversized", .detail = "too big"}, + }; + ASSERT_EQ(cbm_store_coverage_replace(s, "test", rows, 2), CBM_STORE_OK); + + cbm_coverage_row_t *got = NULL; + int n = 0; + ASSERT_EQ(cbm_store_coverage_get(s, "test", &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 1); /* gone.py pruned — no file_hashes row */ + ASSERT_STR_EQ(got[0].rel_path, "src/a.py"); + ASSERT_STR_EQ(got[0].kind, "parse_partial"); + ASSERT_STR_EQ(got[0].detail, "4-7"); + cbm_store_free_coverage(got, n); + + /* Shadow miss-graph materialized under "test::coverage": + * Project → Folder(src) → File(a.py){kind,detail}. */ + cbm_node_t *nodes = NULL; + int nc = 0; + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "File", &nodes, &nc), + CBM_STORE_OK); + ASSERT_EQ(nc, 1); + ASSERT_STR_EQ(nodes[0].file_path, "src/a.py"); + ASSERT_NOT_NULL(strstr(nodes[0].properties_json, "\"kind\":\"parse_partial\"")); + ASSERT_NOT_NULL(strstr(nodes[0].properties_json, "\"detail\":\"4-7\"")); + cbm_store_free_nodes(nodes, nc); + nodes = NULL; + nc = 0; + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "Folder", &nodes, &nc), + CBM_STORE_OK); + ASSERT_EQ(nc, 1); + ASSERT_STR_EQ(nodes[0].name, "src"); + cbm_store_free_nodes(nodes, nc); + + /* Empty replace clears the table AND wipes the shadow graph. */ + ASSERT_EQ(cbm_store_coverage_replace(s, "test", NULL, 0), CBM_STORE_OK); + got = NULL; + n = 0; + ASSERT_EQ(cbm_store_coverage_get(s, "test", &got, &n), CBM_STORE_OK); + ASSERT_EQ(n, 0); + free(got); + nodes = NULL; + nc = 0; + ASSERT_EQ(cbm_store_find_nodes_by_label(s, "test::coverage", "File", &nodes, &nc), + CBM_STORE_OK); + ASSERT_EQ(nc, 0); + cbm_store_free_nodes(nodes, nc); + + cbm_store_close(s); + PASS(); +} + SUITE(store_nodes) { + RUN_TEST(store_coverage_roundtrip_prune_shadow); RUN_TEST(store_open_memory); RUN_TEST(store_close_null); RUN_TEST(store_open_memory_twice);