mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
perf(store,ts): dedup coverage shadow-graph dirs; TS export re-export boundary
Two measured changes on the TypeScript corpus (81,397 files; v0.9.0 17.9 s, before 44.1 s, after 37.5 s): - cov_rebuild_shadow_graph upserted every directory segment for every failure row — 13,243 parse-partial baseline files under one tests/ subtree meant ~80k redundant node/edge round-trips, 9.1 s of a 9.2 s coverage_replace. An in-rebuild path->id map creates each directory once; identical graph (edges deduped by unique key before, absent now). Coverage block 9,162 -> 2,920 ms. Sub-block timings (publish.timing.coverage: del/rows/prune/meta/commit + row_count + detail_bytes) are kept — the caller-level number could not name the culprit. - JS/TS export_statement is an import CONTEXT only in its re-export forms (source field, or a bare specifier list without a declaration). The old is_export_of_declaration blacklist missed TS-only forms (ambient_declaration, function_signature, module_declaration), running declare-heavy subtrees (.d.ts, export namespace) behind inside_import: suppressed usages + per-identifier ancestor walks. Positive detection replaces the blacklist; +3,954 restored usage edges on the corpus, nodes identical. (Measured perf-neutral here — kept for correctness.) Suites green incl. store_nodes/edges/search, mcp, extraction, ts_lsp, complexity, and the 53-language calls-breadth contract. Refs #1669. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
1 parent
a4c0ffbce8
commit
a412642784
2 files changed
+116
-2
No files matched your search
@@ -1646,6 +1646,27 @@ static bool is_actual_import_boundary(CBMExtractCtx *ctx, TSNode node, const CBM
|
||||
char *name = ts_node_is_null(head) ? NULL : cbm_node_text(ctx->arena, head, ctx->source);
|
||||
|
||||
switch (ctx->language) {
|
||||
case CBM_LANG_JAVASCRIPT:
|
||||
case CBM_LANG_TYPESCRIPT:
|
||||
case CBM_LANG_TSX:
|
||||
if (strcmp(kind, "export_statement") == 0) {
|
||||
/* An export is an import CONTEXT only in its re-export forms:
|
||||
* `export ... from 'mod'` (source field) or a bare specifier list
|
||||
* `export { a, b }` with no declaration. An export OF a declaration
|
||||
* must not put the declaration's body behind inside_import — the
|
||||
* old kind-blacklist (is_export_of_declaration) missed the TS-only
|
||||
* forms (ambient_declaration, function_signature,
|
||||
* module_declaration), so declare-heavy code (.d.ts, baselines,
|
||||
* `export namespace`) ran whole subtrees as import context:
|
||||
* suppressed usages plus per-identifier ancestor walks. Positive
|
||||
* detection replaces the blacklist. */
|
||||
if (!ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("source")))) {
|
||||
return true;
|
||||
}
|
||||
return !ts_node_is_null(cbm_find_child_by_kind(node, "export_clause")) &&
|
||||
ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("declaration")));
|
||||
}
|
||||
return true; /* import_statement / import / require / extends: unchanged */
|
||||
case CBM_LANG_CSHARP:
|
||||
/* cs_import_types also lists namespace_declaration (so the import pass
|
||||
* can map namespace names) and using_statement (C#'s RAII block, a
|
||||
|
||||
+95
-2
@@ -2668,6 +2668,20 @@ static void cov_failure_fingerprint(const cbm_coverage_row_t *rows, int count,
|
||||
out[CBM_SHA256_HEX_LEN] = 0;
|
||||
}
|
||||
|
||||
static void cov_dir_ids_free_entry(const char *key, void *val, void *ud) {
|
||||
(void)ud;
|
||||
free((char *)key);
|
||||
free(val);
|
||||
}
|
||||
|
||||
static void cov_dir_ids_free(CBMHashTable *ht) {
|
||||
if (!ht) {
|
||||
return;
|
||||
}
|
||||
cbm_ht_foreach(ht, cov_dir_ids_free_entry, NULL);
|
||||
cbm_ht_free(ht);
|
||||
}
|
||||
|
||||
static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
char covproj[CBM_SZ_512];
|
||||
cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project);
|
||||
@@ -2765,6 +2779,12 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
|
||||
/* Directory nodes repeat massively across failure rows (13k baseline files
|
||||
* under one tests/ subtree meant ~80k redundant upsert/edge round-trips —
|
||||
* 9.1 s of a 9.2 s coverage_replace on the TypeScript corpus). Dedup them
|
||||
* in-memory for the rebuild: first sight of a directory creates its node
|
||||
* and containment edge; every later file under it is a hash hit. */
|
||||
CBMHashTable *dir_ids = cbm_ht_create(CBM_SZ_256);
|
||||
for (int i = 0; i < count; i++) {
|
||||
const char *rel = rows[i].rel_path;
|
||||
if (!rel || !rel[0]) {
|
||||
@@ -2787,6 +2807,12 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
/* Truncate at this slash → pathbuf is the directory prefix; the
|
||||
* upsert binds copies, so restore the slash right after. */
|
||||
*p = '\0';
|
||||
int64_t *cached = dir_ids ? (int64_t *)cbm_ht_get(dir_ids, pathbuf) : NULL;
|
||||
if (cached) {
|
||||
parent = *cached;
|
||||
*p = '/';
|
||||
continue;
|
||||
}
|
||||
const char *seg = strrchr(pathbuf, '/');
|
||||
cbm_node_t folder = {.project = covproj,
|
||||
.label = "Folder",
|
||||
@@ -2795,8 +2821,9 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
.file_path = pathbuf,
|
||||
.properties_json = "{}"};
|
||||
int64_t fid = cbm_store_upsert_node(s, &folder);
|
||||
*p = '/';
|
||||
if (fid <= 0) {
|
||||
*p = '/';
|
||||
cov_dir_ids_free(dir_ids);
|
||||
cbm_store_free_coverage(rows, count);
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
@@ -2806,9 +2833,24 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
.type = "CONTAINS_FOLDER",
|
||||
.properties_json = "{}"};
|
||||
if (cbm_store_insert_edge(s, &e) <= 0) {
|
||||
*p = '/';
|
||||
cov_dir_ids_free(dir_ids);
|
||||
cbm_store_free_coverage(rows, count);
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
if (dir_ids) {
|
||||
int64_t *idv = (int64_t *)malloc(sizeof(*idv));
|
||||
if (idv) {
|
||||
*idv = fid;
|
||||
char *kdup = strdup(pathbuf);
|
||||
if (kdup) {
|
||||
cbm_ht_set(dir_ids, kdup, idv);
|
||||
} else {
|
||||
free(idv);
|
||||
}
|
||||
}
|
||||
}
|
||||
*p = '/';
|
||||
parent = fid;
|
||||
}
|
||||
|
||||
@@ -2826,6 +2868,7 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
.properties_json = props};
|
||||
int64_t file_id = cbm_store_upsert_node(s, &file);
|
||||
if (file_id <= 0) {
|
||||
cov_dir_ids_free(dir_ids);
|
||||
cbm_store_free_coverage(rows, count);
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
@@ -2835,10 +2878,12 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
.type = "CONTAINS_FILE",
|
||||
.properties_json = "{}"};
|
||||
if (cbm_store_insert_edge(s, &e) <= 0) {
|
||||
cov_dir_ids_free(dir_ids);
|
||||
cbm_store_free_coverage(rows, count);
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
}
|
||||
cov_dir_ids_free(dir_ids);
|
||||
cbm_store_free_coverage(rows, count);
|
||||
{
|
||||
sqlite3_stmt *set = NULL;
|
||||
@@ -2862,6 +2907,15 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
if (exec_sql(s, "BEGIN;") != CBM_STORE_OK) {
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
/* Sub-block timings (publish.timing style): coverage_replace measured 9 s
|
||||
* on the TypeScript corpus and the caller-level block could not say WHY —
|
||||
* delete, 13k row inserts with large error_ranges payloads, the NOT-IN
|
||||
* prune, meta, and COMMIT are very different suspects. */
|
||||
struct timespec cov_t0;
|
||||
struct timespec cov_t1;
|
||||
long cov_ms[5] = {0, 0, 0, 0, 0};
|
||||
long long cov_detail_bytes = 0;
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t0);
|
||||
sqlite3_stmt *del = NULL;
|
||||
if (sqlite3_prepare_v2(s->db, "DELETE FROM index_coverage WHERE project = ?1;", CBM_NOT_FOUND,
|
||||
&del, NULL) != SQLITE_OK) {
|
||||
@@ -2877,6 +2931,10 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
(void)exec_sql(s, "ROLLBACK;");
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
|
||||
cov_ms[0] =
|
||||
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
|
||||
cov_t0 = cov_t1;
|
||||
sqlite3_stmt *ins = NULL;
|
||||
if (sqlite3_prepare_v2(
|
||||
s->db,
|
||||
@@ -2894,6 +2952,7 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
bind_text(ins, SKIP_ONE, project);
|
||||
bind_text(ins, ST_COL_2, rows[i].rel_path);
|
||||
bind_text(ins, ST_COL_3, rows[i].kind);
|
||||
cov_detail_bytes += rows[i].detail ? (long long)strlen(rows[i].detail) : 0;
|
||||
bind_text(ins, CBM_SZ_4, rows[i].detail ? rows[i].detail : "");
|
||||
if (sqlite3_step(ins) != SQLITE_DONE) {
|
||||
store_set_error_sqlite(s, "coverage insert");
|
||||
@@ -2904,6 +2963,10 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
sqlite3_reset(ins);
|
||||
}
|
||||
sqlite3_finalize(ins);
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
|
||||
cov_ms[1] =
|
||||
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
|
||||
cov_t0 = cov_t1;
|
||||
/* Prune FAILURE rows for files no longer known to the index (deleted from
|
||||
* the repo): file_hashes is the authoritative live-file set after
|
||||
* persist. By-design not_indexed_* rows are exempt — deliberately
|
||||
@@ -2927,6 +2990,10 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
(void)exec_sql(s, "ROLLBACK;");
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
|
||||
cov_ms[2] =
|
||||
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
|
||||
cov_t0 = cov_t1;
|
||||
|
||||
if (meta) {
|
||||
char recorded_at[CBM_SZ_64];
|
||||
@@ -3002,7 +3069,33 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
(void)exec_sql(s, "ROLLBACK;");
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
return exec_sql(s, "COMMIT;");
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
|
||||
cov_ms[3] =
|
||||
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
|
||||
cov_t0 = cov_t1;
|
||||
int commit_rc = exec_sql(s, "COMMIT;");
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
|
||||
cov_ms[4] =
|
||||
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
|
||||
{
|
||||
char b0[ST_BUF_64];
|
||||
char b1[ST_BUF_64];
|
||||
char b2[ST_BUF_64];
|
||||
char b3[ST_BUF_64];
|
||||
char b4[ST_BUF_64];
|
||||
char bn[ST_BUF_64];
|
||||
char bb[ST_BUF_64];
|
||||
snprintf(b0, sizeof(b0), "%ld", cov_ms[0]);
|
||||
snprintf(b1, sizeof(b1), "%ld", cov_ms[1]);
|
||||
snprintf(b2, sizeof(b2), "%ld", cov_ms[2]);
|
||||
snprintf(b3, sizeof(b3), "%ld", cov_ms[3]);
|
||||
snprintf(b4, sizeof(b4), "%ld", cov_ms[4]);
|
||||
snprintf(bn, sizeof(bn), "%d", count);
|
||||
snprintf(bb, sizeof(bb), "%lld", cov_detail_bytes);
|
||||
cbm_log_info("publish.timing.coverage", "del", b0, "rows", b1, "prune", b2, "meta", b3,
|
||||
"commit", b4, "row_count", bn, "detail_bytes", bb);
|
||||
}
|
||||
return commit_rc;
|
||||
}
|
||||
|
||||
int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows,
|
||||
|
||||
Reference in new issue
Block a user