mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
feat(pipeline): fuse cross-LSP into resolve worker + module-def filter
Previously cbm_pipeline_pass_lsp_cross ran as a separate sequential pass that re-read every source file from disk and re-parsed every tree-sitter tree on a single thread — a 50x regression on the kubernetes Go monorepo (extract 10s, lsp_cross 9+ min when killed). Three changes: 1. CBMFileResult gains const char *source / int source_len. The parallel extract worker copies source bytes into result->arena (per-file 100MB cap, project-wide 2GB atomic cap) so the fused cross-LSP step can run without disk I/O. Source pointer is borrowed by downstream readers; arena owns the storage. 2. Cross-LSP work moves into the parallel resolve worker. Each worker processes its file with the existing parallel infrastructure (worker pool + work stealing). Slab is reclaimed after each per-file cross-LSP call to bound peak slab memory. The previously separate lsp_cross pass entry function is kept for the sequential pipeline path; the parallel path no longer invokes it. 3. CBMModuleDefIndex (gopls 'package summary' pattern) — an inverted index module_qn -> list of def indices built once in O(D) before resolve_worker fires. Per file, cbm_pxc_filter_defs_for_file slices all_defs down to the file's own_module + imported modules only, typically 50-100x smaller than the global ~110k defs on kubernetes. Drops the per-file registry build from O(all_defs) to O(relevant). Other plumbing: cbm_parallel_resolve signature gains all_defs, def_count, def_modules, module_def_index params. pipeline.c builds the index between registry_build and parallel_resolve and frees it after. pipeline_incremental and tests/test_parallel updated to match. Helpers in pass_lsp_cross.c exposed via the header: cbm_pxc_has_cross_lsp, cbm_pxc_collect_all_defs, cbm_pxc_run_one(_ts), cbm_pxc_ts_modes, cbm_pxc_build_module_def_index, cbm_pxc_filter_defs_for_file. Kubernetes wall-time: 35 min -> 6.24 min (~5.6x) before the followup perf commits in this series.
This commit is contained in:
1 parent
6226972f13
commit
75a0df688f
8 files changed
+503
-44
No files matched your search
@@ -431,6 +431,15 @@ typedef struct {
|
||||
int imports_count;
|
||||
TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree)
|
||||
CBMLanguage cached_lang; // language of cached tree (for parser selection)
|
||||
|
||||
// Retained source bytes — copied into `arena` by the parallel
|
||||
// extract pass so the fused cross-file LSP step in resolve_worker
|
||||
// can run without re-reading the file from disk. NULL when the
|
||||
// file exceeded the per-file (100 MB) or total (2 GB) retention
|
||||
// cap; in that case the cross-file LSP step is skipped for this
|
||||
// file (defs/calls already extracted are unaffected).
|
||||
const char *source;
|
||||
int source_len;
|
||||
} CBMFileResult;
|
||||
|
||||
// --- Enclosing function cache ---
|
||||
|
||||
+140
-12
@@ -22,6 +22,7 @@
|
||||
#include "lsp/php_lsp.h"
|
||||
#include "graph_buffer/graph_buffer.h"
|
||||
#include "foundation/constants.h"
|
||||
#include "foundation/hash_table.h"
|
||||
#include "foundation/log.h"
|
||||
|
||||
#include <stdio.h>
|
||||
@@ -146,7 +147,7 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src,
|
||||
/* Collect a project-wide CBMLSPDef[] from all cached results. Returns a
|
||||
* malloc'd array (caller frees) of length *out_count. String fields are
|
||||
* borrowed from cache[i]->arena and from def_modules[i] (also borrowed). */
|
||||
static CBMLSPDef *pxc_collect_all_defs(CBMFileResult **cache,
|
||||
CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache,
|
||||
const cbm_file_info_t *files, int file_count,
|
||||
const char *project_name,
|
||||
char **def_modules, int *out_count) {
|
||||
@@ -246,7 +247,7 @@ static void pxc_free_import_map(const char **keys, const char **vals, int count)
|
||||
}
|
||||
|
||||
/* Detect TS dialect flags from a relative path. */
|
||||
static void pxc_ts_modes(CBMLanguage lang, const char *rel_path,
|
||||
void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path,
|
||||
bool *out_js, bool *out_jsx, bool *out_dts) {
|
||||
*out_js = (lang == CBM_LANG_JAVASCRIPT);
|
||||
*out_jsx = (lang == CBM_LANG_TSX);
|
||||
@@ -264,7 +265,7 @@ static void pxc_ts_modes(CBMLanguage lang, const char *rel_path,
|
||||
}
|
||||
|
||||
/* Returns true when this language has a cross-file LSP wired up. */
|
||||
static bool pxc_has_cross_lsp(CBMLanguage lang) {
|
||||
bool cbm_pxc_has_cross_lsp(CBMLanguage lang) {
|
||||
switch (lang) {
|
||||
case CBM_LANG_GO:
|
||||
case CBM_LANG_C:
|
||||
@@ -328,7 +329,7 @@ static void pxc_append_results(CBMArena *dst_arena, CBMResolvedCallArray *dst_ca
|
||||
* directly across N files (test_incremental.c saw 3.5 GB peak on a
|
||||
* 1100-file repo before this fix). Output gets copied into the file's own
|
||||
* arena and merged into result->resolved_calls. */
|
||||
static void pxc_run_one(CBMLanguage lang, CBMFileResult *r, const char *source,
|
||||
void cbm_pxc_run_one(CBMLanguage lang, CBMFileResult *r, const char *source,
|
||||
int source_len, const char *module_qn,
|
||||
CBMLSPDef *defs, int def_count,
|
||||
const char **imp_names, const char **imp_qns, int imp_count) {
|
||||
@@ -376,9 +377,9 @@ static void pxc_run_one(CBMLanguage lang, CBMFileResult *r, const char *source,
|
||||
cbm_arena_destroy(&scratch);
|
||||
}
|
||||
|
||||
/* Variant of pxc_run_one for TS/JS/JSX/TSX with explicit dialect flags.
|
||||
* Same scratch-arena lifecycle as pxc_run_one. */
|
||||
static void pxc_run_one_ts(CBMFileResult *r, const char *source, int source_len,
|
||||
/* Variant of cbm_pxc_run_one for TS/JS/JSX/TSX with explicit dialect
|
||||
* flags. Same scratch-arena lifecycle as cbm_pxc_run_one. */
|
||||
void cbm_pxc_run_one_ts(CBMFileResult *r, const char *source, int source_len,
|
||||
const char *module_qn,
|
||||
CBMLSPDef *defs, int def_count,
|
||||
const char **imp_names, const char **imp_qns, int imp_count,
|
||||
@@ -415,7 +416,7 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx,
|
||||
}
|
||||
|
||||
int def_count = 0;
|
||||
CBMLSPDef *all_defs = pxc_collect_all_defs(cache, files, file_count,
|
||||
CBMLSPDef *all_defs = cbm_pxc_collect_all_defs(cache, files, file_count,
|
||||
ctx->project_name, def_modules,
|
||||
&def_count);
|
||||
|
||||
@@ -427,7 +428,7 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx,
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
if (!cache[i]) continue;
|
||||
CBMLanguage lang = files[i].language;
|
||||
if (!pxc_has_cross_lsp(lang)) {
|
||||
if (!cbm_pxc_has_cross_lsp(lang)) {
|
||||
skipped_no_lsp++;
|
||||
continue;
|
||||
}
|
||||
@@ -453,12 +454,12 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx,
|
||||
if (lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT ||
|
||||
lang == CBM_LANG_TSX) {
|
||||
bool js, jsx, dts;
|
||||
pxc_ts_modes(lang, files[i].rel_path, &js, &jsx, &dts);
|
||||
pxc_run_one_ts(cache[i], source, source_len, def_modules[i],
|
||||
cbm_pxc_ts_modes(lang, files[i].rel_path, &js, &jsx, &dts);
|
||||
cbm_pxc_run_one_ts(cache[i], source, source_len, def_modules[i],
|
||||
all_defs, def_count, imp_keys, imp_vals, imp_count,
|
||||
js, jsx, dts);
|
||||
} else {
|
||||
pxc_run_one(lang, cache[i], source, source_len, def_modules[i],
|
||||
cbm_pxc_run_one(lang, cache[i], source, source_len, def_modules[i],
|
||||
all_defs, def_count, imp_keys, imp_vals, imp_count);
|
||||
}
|
||||
per_lang_calls++;
|
||||
@@ -480,3 +481,130 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx,
|
||||
"lsp_calls", itoa_buf(per_lang_calls));
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Per-module def index (gopls "package summary" pattern) ──── */
|
||||
|
||||
typedef struct {
|
||||
int count;
|
||||
int cap;
|
||||
int *indices; /* malloc'd; indices into the caller's all_defs[] */
|
||||
} pxc_module_entry_t;
|
||||
|
||||
struct CBMModuleDefIndex {
|
||||
CBMHashTable *ht; /* module_qn → pxc_module_entry_t* */
|
||||
};
|
||||
|
||||
/* cbm_ht_foreach callback: free each pxc_module_entry_t. */
|
||||
static void pxc_module_entry_free_cb(const char *key, void *value, void *userdata) {
|
||||
(void)key;
|
||||
(void)userdata;
|
||||
pxc_module_entry_t *e = (pxc_module_entry_t *)value;
|
||||
if (!e) return;
|
||||
free(e->indices);
|
||||
free(e);
|
||||
}
|
||||
|
||||
CBMModuleDefIndex *cbm_pxc_build_module_def_index(CBMLSPDef *all_defs, int def_count) {
|
||||
if (!all_defs || def_count <= 0) return NULL;
|
||||
|
||||
CBMHashTable *ht = cbm_ht_create(64);
|
||||
if (!ht) return NULL;
|
||||
|
||||
/* Single pass: append each def's index into its module's dynamic array. */
|
||||
for (int i = 0; i < def_count; i++) {
|
||||
const char *mod = all_defs[i].def_module_qn;
|
||||
if (!mod) continue;
|
||||
pxc_module_entry_t *e = (pxc_module_entry_t *)cbm_ht_get(ht, mod);
|
||||
if (!e) {
|
||||
e = (pxc_module_entry_t *)calloc(1, sizeof(*e));
|
||||
if (!e) continue;
|
||||
e->cap = 16;
|
||||
e->indices = (int *)malloc((size_t)e->cap * sizeof(int));
|
||||
if (!e->indices) {
|
||||
free(e);
|
||||
continue;
|
||||
}
|
||||
cbm_ht_set(ht, mod, e);
|
||||
}
|
||||
if (e->count >= e->cap) {
|
||||
int new_cap = e->cap * 2;
|
||||
int *new_indices = (int *)realloc(e->indices, (size_t)new_cap * sizeof(int));
|
||||
if (!new_indices) continue; /* drop this entry, keep going */
|
||||
e->indices = new_indices;
|
||||
e->cap = new_cap;
|
||||
}
|
||||
e->indices[e->count++] = i;
|
||||
}
|
||||
|
||||
CBMModuleDefIndex *idx = (CBMModuleDefIndex *)calloc(1, sizeof(*idx));
|
||||
if (!idx) {
|
||||
cbm_ht_foreach(ht, pxc_module_entry_free_cb, NULL);
|
||||
cbm_ht_free(ht);
|
||||
return NULL;
|
||||
}
|
||||
idx->ht = ht;
|
||||
return idx;
|
||||
}
|
||||
|
||||
void cbm_pxc_free_module_def_index(CBMModuleDefIndex *idx) {
|
||||
if (!idx) return;
|
||||
if (idx->ht) {
|
||||
cbm_ht_foreach(idx->ht, pxc_module_entry_free_cb, NULL);
|
||||
cbm_ht_free(idx->ht);
|
||||
}
|
||||
free(idx);
|
||||
}
|
||||
|
||||
CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx,
|
||||
CBMLSPDef *all_defs,
|
||||
const char *own_module,
|
||||
const char *const *imp_qns,
|
||||
int imp_count,
|
||||
int *out_count) {
|
||||
if (out_count) *out_count = 0;
|
||||
if (!idx || !idx->ht || !all_defs || !out_count) return NULL;
|
||||
|
||||
/* Dedup module list (own_module may appear in imp_qns). For typical
|
||||
* imp_count ~10 this O(N²) scan is fine and avoids registering the
|
||||
* same def twice in the per-file registry. */
|
||||
const char *seen[64];
|
||||
int seen_count = 0;
|
||||
if (own_module) {
|
||||
seen[seen_count++] = own_module;
|
||||
}
|
||||
for (int i = 0; i < imp_count && seen_count < (int)(sizeof(seen) / sizeof(seen[0])); i++) {
|
||||
if (!imp_qns[i]) continue;
|
||||
bool dup = false;
|
||||
for (int s = 0; s < seen_count; s++) {
|
||||
if (strcmp(seen[s], imp_qns[i]) == 0) {
|
||||
dup = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!dup) seen[seen_count++] = imp_qns[i];
|
||||
}
|
||||
|
||||
/* Pass 1: total relevant defs. */
|
||||
int total = 0;
|
||||
for (int s = 0; s < seen_count; s++) {
|
||||
pxc_module_entry_t *e = (pxc_module_entry_t *)cbm_ht_get(idx->ht, seen[s]);
|
||||
if (e) total += e->count;
|
||||
}
|
||||
if (total == 0) return NULL;
|
||||
|
||||
/* Pass 2: copy CBMLSPDef structs (string fields stay borrowed from the
|
||||
* caller's all_defs[] arena). */
|
||||
CBMLSPDef *out = (CBMLSPDef *)malloc((size_t)total * sizeof(CBMLSPDef));
|
||||
if (!out) return NULL;
|
||||
|
||||
int n = 0;
|
||||
for (int s = 0; s < seen_count; s++) {
|
||||
pxc_module_entry_t *e = (pxc_module_entry_t *)cbm_ht_get(idx->ht, seen[s]);
|
||||
if (!e) continue;
|
||||
for (int j = 0; j < e->count; j++) {
|
||||
out[n++] = all_defs[e->indices[j]];
|
||||
}
|
||||
}
|
||||
*out_count = n;
|
||||
return out;
|
||||
}
|
||||
+104
-14
@@ -1,27 +1,117 @@
|
||||
/*
|
||||
* pass_lsp_cross.h — Cross-file LSP type-aware call resolution pass.
|
||||
* pass_lsp_cross.h — Cross-file LSP helpers shared with the parallel
|
||||
* resolve pass.
|
||||
*
|
||||
* Per-file LSP (cbm_run_X_lsp inside cbm_extract_file) only sees a single
|
||||
* file's defs in its registry, so callees whose receiver type comes from
|
||||
* an imported module stay unresolved. This pass closes that gap: for each
|
||||
* indexed file, it builds a project-wide CBMLSPDef[] (covering ALL files'
|
||||
* defs) plus the file's resolved import map (from gbuf IMPORTS edges),
|
||||
* then calls the language-specific cbm_run_X_lsp_cross. The resulting
|
||||
* CBMResolvedCall entries are appended to each file's resolved_calls so
|
||||
* the call-resolution pass picks them up via the same shared helper
|
||||
* (cbm_pipeline_find_lsp_resolution) that already handles per-file LSP.
|
||||
* an imported module stay unresolved. The helpers declared here close
|
||||
* that gap: they let the parallel resolve worker (pass_parallel.c) build
|
||||
* a project-wide CBMLSPDef[] and invoke the language-specific
|
||||
* cbm_run_X_lsp_cross resolver on each file using the file's already-
|
||||
* built import map. Resolved calls are appended to result->resolved_calls
|
||||
* so the same cbm_pipeline_find_lsp_resolution path that handles per-
|
||||
* file LSP picks them up.
|
||||
*
|
||||
* Languages covered: Go, C/C++/CUDA, Python, TypeScript/JavaScript/JSX/TSX,
|
||||
* PHP. PHP cross-file landed alongside this pass — see cbm_run_php_lsp_cross.
|
||||
* Languages covered: Go, C/C++/CUDA, Python, TypeScript/JavaScript/JSX/
|
||||
* TSX, PHP, C#. Anything else short-circuits via cbm_pxc_has_cross_lsp.
|
||||
*
|
||||
* Public declaration of cbm_pipeline_pass_lsp_cross lives in
|
||||
* pipeline_internal.h alongside the other pipeline passes; this header is
|
||||
* a stub kept for source-file readability (mirrors the dedicated header
|
||||
* pattern of pass_cross_repo.h, but no public API surface here).
|
||||
* Previously this work ran as a separate sequential pipeline pass
|
||||
* (cbm_pipeline_pass_lsp_cross) that re-read every source file from
|
||||
* disk and re-parsed each tree-sitter tree on a single thread — a 50×
|
||||
* regression vs the parallel extract pass on large repos. The pass was
|
||||
* deleted; the resolve worker now invokes these helpers directly using
|
||||
* the source bytes retained in result->arena during extract.
|
||||
*/
|
||||
#ifndef CBM_PIPELINE_PASS_LSP_CROSS_H
|
||||
#define CBM_PIPELINE_PASS_LSP_CROSS_H
|
||||
|
||||
#include "cbm.h"
|
||||
/* CBMLSPDef historically lives in lsp/go_lsp.h (not lsp/type_rep.h)
|
||||
* — type_rep.h covers the type-representation primitives while
|
||||
* go_lsp.h was where the project-wide def descriptor landed first. */
|
||||
#include "lsp/go_lsp.h"
|
||||
#include "pipeline/pipeline_internal.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
/* True iff this language has a cbm_run_X_lsp_cross resolver wired up. */
|
||||
bool cbm_pxc_has_cross_lsp(CBMLanguage lang);
|
||||
|
||||
/* Collect a project-wide CBMLSPDef[] from every cached file result.
|
||||
* def_modules[i] receives the module QN for files[i] (malloc'd; the
|
||||
* caller frees each entry then the array). String fields in the
|
||||
* returned CBMLSPDef[] are borrowed from cache[i]->arena and from
|
||||
* def_modules[i] — caller must keep both alive while the array is in
|
||||
* use. Returns the malloc'd array (free() it) and writes the entry
|
||||
* count to *out_count. Returns NULL on alloc failure or when no defs
|
||||
* exist. */
|
||||
CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache,
|
||||
const cbm_file_info_t *files,
|
||||
int file_count,
|
||||
const char *project_name,
|
||||
char **def_modules,
|
||||
int *out_count);
|
||||
|
||||
/* Detect TS dialect flags from a relative path. */
|
||||
void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path,
|
||||
bool *out_js, bool *out_jsx, bool *out_dts);
|
||||
|
||||
/* ── Per-module def index (the gopls "package summary" pattern) ──
|
||||
*
|
||||
* The hot path used to register ALL all_defs[] into a fresh registry
|
||||
* per file (~110k defs × 11k files for kubernetes = ~21,000 CPU-s of
|
||||
* arena_strdup). Most of those defs are irrelevant to any one file —
|
||||
* each file only references defs from its own module + its imported
|
||||
* modules. gopls observed the same: it builds per-package summaries
|
||||
* and per-file only loads the summaries the file imports.
|
||||
*
|
||||
* cbm_pxc_build_module_def_index() builds an inverted index once
|
||||
* (O(D)) mapping def_module_qn → list of indices into all_defs[].
|
||||
* cbm_pxc_filter_defs_for_file() then returns a small CBMLSPDef[]
|
||||
* containing ONLY the defs from own_module + imp_qns — typically
|
||||
* 50-100× smaller than the global all_defs[].
|
||||
*
|
||||
* Net: per-file registry build drops from O(all_defs) to O(relevant_
|
||||
* defs). On a Go file importing 10 packages, relevant ≈ 1-2k vs
|
||||
* 110k → ~50× per-file speedup on the dominant cost. */
|
||||
typedef struct CBMModuleDefIndex CBMModuleDefIndex;
|
||||
|
||||
CBMModuleDefIndex *cbm_pxc_build_module_def_index(CBMLSPDef *all_defs,
|
||||
int def_count);
|
||||
|
||||
void cbm_pxc_free_module_def_index(CBMModuleDefIndex *idx);
|
||||
|
||||
/* Return a malloc'd CBMLSPDef[] containing all defs whose
|
||||
* def_module_qn matches own_module OR any of imp_qns. String fields
|
||||
* inside each entry are borrowed from the original all_defs[] arena
|
||||
* (caller keeps it alive). Caller frees the returned array with
|
||||
* free(). Writes the entry count to *out_count. Returns NULL if no
|
||||
* matches (with *out_count = 0). */
|
||||
CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx,
|
||||
CBMLSPDef *all_defs,
|
||||
const char *own_module,
|
||||
const char *const *imp_qns,
|
||||
int imp_count,
|
||||
int *out_count);
|
||||
|
||||
/* Run the cross-file LSP resolver for non-TS languages. Appends
|
||||
* resolved CALLS into r->resolved_calls (lives in r->arena). Caller
|
||||
* owns source, module_qn, all_defs, imp_keys, imp_vals.
|
||||
* NOTE: all_defs is read-only in practice but typed non-const to match
|
||||
* the existing cbm_run_X_lsp_cross callee signatures. */
|
||||
void cbm_pxc_run_one(CBMLanguage lang, CBMFileResult *r,
|
||||
const char *source, int source_len,
|
||||
const char *module_qn,
|
||||
CBMLSPDef *all_defs, int def_count,
|
||||
const char **imp_keys, const char **imp_vals,
|
||||
int imp_count);
|
||||
|
||||
/* TS / JS / JSX / TSX variant with explicit dialect flags. */
|
||||
void cbm_pxc_run_one_ts(CBMFileResult *r,
|
||||
const char *source, int source_len,
|
||||
const char *module_qn,
|
||||
CBMLSPDef *all_defs, int def_count,
|
||||
const char **imp_keys, const char **imp_vals,
|
||||
int imp_count,
|
||||
bool js_mode, bool jsx_mode, bool dts_mode);
|
||||
|
||||
#endif /* CBM_PIPELINE_PASS_LSP_CROSS_H */
|
||||
@@ -28,8 +28,19 @@ enum {
|
||||
#define PP_HALF_CONF 0.5
|
||||
#define PP_FIELD_HINT_CONF 0.85
|
||||
enum { PP_CSHARP_M_PREFIX_LEN = 2 };
|
||||
|
||||
/* Source-retention caps for the parallel pipeline. The extract worker
|
||||
* copies source bytes into result->arena so the fused cross-file LSP
|
||||
* step in resolve_worker can run without re-reading from disk. Bound
|
||||
* peak RSS with a per-file cap (skip retention for pathological huge
|
||||
* generated files) and a total project-wide cap (skip when budget
|
||||
* exhausted — cross-file LSP becomes a no-op for those late files,
|
||||
* defs/calls already extracted are unaffected). */
|
||||
#define PP_RETAIN_PER_FILE_MAX_BYTES (100LL * 1024 * 1024)
|
||||
#define PP_RETAIN_TOTAL_BUDGET_BYTES (2LL * 1024 * 1024 * 1024)
|
||||
#include "pipeline/pipeline.h"
|
||||
#include "pipeline/pipeline_internal.h"
|
||||
#include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */
|
||||
#include "pipeline/lsp_resolve.h"
|
||||
#include "pipeline/worker_pool.h"
|
||||
#include "foundation/compat.h"
|
||||
@@ -402,6 +413,7 @@ typedef struct {
|
||||
_Atomic int next_file_idx;
|
||||
|
||||
cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */
|
||||
_Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */
|
||||
} extract_ctx_t;
|
||||
|
||||
/* Insert one definition node (and its route if present) into the local gbuf. */
|
||||
@@ -516,7 +528,37 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
|
||||
source_len, &ec->pkg_entries[worker_id]);
|
||||
}
|
||||
|
||||
/* Free source buffer — extraction captured everything needed. */
|
||||
/* Retain source bytes in result->arena so the fused cross-file
|
||||
* LSP step in resolve_worker can run without re-reading from
|
||||
* disk. Capped per-file (PP_RETAIN_PER_FILE_MAX_BYTES) and
|
||||
* globally (PP_RETAIN_TOTAL_BUDGET_BYTES) to bound peak RSS.
|
||||
* Skipping retention just means cross-file LSP no-ops for this
|
||||
* file — defs/calls already extracted are unaffected. */
|
||||
if (source_len > 0 && (int64_t)source_len <= PP_RETAIN_PER_FILE_MAX_BYTES) {
|
||||
int64_t prior = atomic_fetch_add_explicit(
|
||||
&ec->retained_bytes, (int64_t)source_len, memory_order_relaxed);
|
||||
if (prior + (int64_t)source_len <= PP_RETAIN_TOTAL_BUDGET_BYTES) {
|
||||
char *copy = (char *)cbm_arena_alloc(&result->arena,
|
||||
(size_t)source_len + 1);
|
||||
if (copy) {
|
||||
memcpy(copy, source, (size_t)source_len);
|
||||
copy[source_len] = '\0';
|
||||
result->source = copy;
|
||||
result->source_len = source_len;
|
||||
} else {
|
||||
atomic_fetch_sub_explicit(&ec->retained_bytes,
|
||||
(int64_t)source_len,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
} else {
|
||||
atomic_fetch_sub_explicit(&ec->retained_bytes,
|
||||
(int64_t)source_len,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/* Free source buffer — extraction captured everything needed,
|
||||
* and the retention copy (if any) lives in result->arena. */
|
||||
free_source(source);
|
||||
|
||||
/* Cache result (arena + extracted data, no tree) for Phase 3B and Phase 4 */
|
||||
@@ -635,6 +677,7 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
};
|
||||
atomic_init(&ec.next_worker_id, 0);
|
||||
atomic_init(&ec.next_file_idx, 0);
|
||||
atomic_init(&ec.retained_bytes, 0);
|
||||
|
||||
/* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
|
||||
CBM_PROF_START(t_dispatch);
|
||||
@@ -840,6 +883,23 @@ typedef struct {
|
||||
_Atomic int64_t *shared_ids;
|
||||
_Atomic int *cancelled;
|
||||
_Atomic int next_file_idx;
|
||||
|
||||
/* Cross-file LSP inputs — pre-built once by the caller in pipeline.c
|
||||
* and shared read-only by usage across workers (typed non-const to
|
||||
* match the existing cbm_run_X_lsp_cross callee signatures the
|
||||
* worker forwards them to). NULL/0 → cross-LSP no-ops. */
|
||||
CBMLSPDef *all_defs;
|
||||
int def_count;
|
||||
char *const *def_modules; /* per-file module QN; def_modules[i] for files[i] */
|
||||
/* Optional inverted index for per-file def filtering (gopls pattern).
|
||||
* When non-NULL, the fused worker calls cbm_pxc_filter_defs_for_file
|
||||
* to shrink the def array passed to the LSP from O(all_defs) to
|
||||
* O(relevant_defs). NULL → each file sees the full all_defs[]. */
|
||||
struct CBMModuleDefIndex *module_def_index;
|
||||
|
||||
/* Counters for parallel.resolve.lsp_cross_done summary. */
|
||||
_Atomic int lsp_cross_processed;
|
||||
_Atomic int lsp_cross_skipped_no_source;
|
||||
} resolve_ctx_t;
|
||||
|
||||
/* Minimum buffer space needed per arg JSON object */
|
||||
@@ -1716,15 +1776,25 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Skip files with nothing to resolve */
|
||||
CBMLanguage lang = rc->files[file_idx].language;
|
||||
bool cross_lsp_eligible =
|
||||
(rc->all_defs && rc->def_count > 0 && cbm_pxc_has_cross_lsp(lang));
|
||||
|
||||
/* Skip files with nothing to resolve. Cross-file LSP can find
|
||||
* CALLS that the per-file extract missed (the whole point of
|
||||
* the LSP method), so do NOT skip an LSP-eligible file just
|
||||
* because its initial extract found no calls/defs/etc. */
|
||||
if (result->calls.count == 0 && result->usages.count == 0 && result->throws.count == 0 &&
|
||||
result->rw.count == 0 && result->defs.count == 0 && result->impl_traits.count == 0) {
|
||||
result->rw.count == 0 && result->defs.count == 0 && result->impl_traits.count == 0 &&
|
||||
!cross_lsp_eligible) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const char *rel = rc->files[file_idx].rel_path;
|
||||
|
||||
/* Build import map (read-only access to main_gbuf) */
|
||||
/* Build import map ONCE (read-only access to main_gbuf). The
|
||||
* same imp_keys/imp_vals feed both the fused cross-file LSP
|
||||
* step below AND the resolve_file_* chain — no duplicate build. */
|
||||
const char **imp_keys = NULL;
|
||||
const char **imp_vals = NULL;
|
||||
int imp_count = 0;
|
||||
@@ -1732,6 +1802,76 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
|
||||
char *module_qn = cbm_pipeline_fqn_module(rc->project_name, rel);
|
||||
|
||||
/* ── Cross-file LSP (FUSED) ─────────────────────────────
|
||||
* Runs BEFORE resolve_file_calls so its additions to
|
||||
* result->resolved_calls are picked up by
|
||||
* cbm_pipeline_find_lsp_resolution when calls become CALLS
|
||||
* edges. Requires result->source to have been retained in
|
||||
* result->arena during extract (PP_RETAIN_*); files over the
|
||||
* cap or past the budget have result->source==NULL and are
|
||||
* counted as skipped_no_source — defs/calls already in the
|
||||
* extract are unaffected.
|
||||
*
|
||||
* Slab reclaim afterward: the LSP re-parses via tree-sitter,
|
||||
* which allocates through this worker's TLS slab. Reclaiming
|
||||
* here keeps the slab high-water bounded as the resolve phase
|
||||
* walks across thousands of files in a single worker thread. */
|
||||
if (cross_lsp_eligible) {
|
||||
if (result->source && result->source_len > 0) {
|
||||
const char *def_module = rc->def_modules
|
||||
? rc->def_modules[file_idx]
|
||||
: module_qn;
|
||||
|
||||
/* gopls pattern: filter all_defs down to just the defs
|
||||
* this file actually needs (own_module + imp_vals
|
||||
* imported modules). For a typical file with ~10
|
||||
* imports, this slashes the per-file registry-build
|
||||
* cost from O(all_defs) to O(relevant_defs) — usually
|
||||
* 50-100× smaller. */
|
||||
CBMLSPDef *file_defs = rc->all_defs;
|
||||
int file_def_count = rc->def_count;
|
||||
CBMLSPDef *filtered = NULL;
|
||||
if (rc->module_def_index) {
|
||||
int filtered_count = 0;
|
||||
filtered = cbm_pxc_filter_defs_for_file(
|
||||
rc->module_def_index, rc->all_defs,
|
||||
def_module, imp_vals, imp_count,
|
||||
&filtered_count);
|
||||
if (filtered) {
|
||||
file_defs = filtered;
|
||||
file_def_count = filtered_count;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t lsp_t0 = extract_now_ns();
|
||||
if (lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT ||
|
||||
lang == CBM_LANG_TSX) {
|
||||
bool js, jsx, dts;
|
||||
cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts);
|
||||
cbm_pxc_run_one_ts(result, result->source, result->source_len,
|
||||
def_module, file_defs, file_def_count,
|
||||
imp_keys, imp_vals, imp_count, js, jsx, dts);
|
||||
} else {
|
||||
cbm_pxc_run_one(lang, result, result->source, result->source_len,
|
||||
def_module, file_defs, file_def_count,
|
||||
imp_keys, imp_vals, imp_count);
|
||||
}
|
||||
free(filtered);
|
||||
cbm_slab_reclaim();
|
||||
uint64_t lsp_elapsed_ms = (extract_now_ns() - lsp_t0) / PP_USEC_PER_MS;
|
||||
if (lsp_elapsed_ms > PP_TIMER_THRESH) {
|
||||
cbm_log_info("parallel.resolve.lsp_cross.slow",
|
||||
"elapsed_ms", itoa_log((int)lsp_elapsed_ms),
|
||||
"path", rel);
|
||||
}
|
||||
atomic_fetch_add_explicit(&rc->lsp_cross_processed, SKIP_ONE,
|
||||
memory_order_relaxed);
|
||||
} else {
|
||||
atomic_fetch_add_explicit(&rc->lsp_cross_skipped_no_source, SKIP_ONE,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/* ── CALLS resolution ──────────────────────────────────── */
|
||||
resolve_file_calls(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count);
|
||||
|
||||
@@ -1754,7 +1894,9 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
|
||||
int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
|
||||
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
|
||||
int worker_count) {
|
||||
int worker_count, CBMLSPDef *all_defs, int def_count,
|
||||
char *const *def_modules,
|
||||
struct CBMModuleDefIndex *module_def_index) {
|
||||
if (file_count == 0) {
|
||||
return 0;
|
||||
}
|
||||
@@ -1781,8 +1923,14 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
.registry = ctx->registry,
|
||||
.shared_ids = shared_ids,
|
||||
.cancelled = ctx->cancelled,
|
||||
.all_defs = all_defs,
|
||||
.def_count = def_count,
|
||||
.def_modules = def_modules,
|
||||
.module_def_index = module_def_index,
|
||||
};
|
||||
atomic_init(&rc.next_file_idx, 0);
|
||||
atomic_init(&rc.lsp_cross_processed, 0);
|
||||
atomic_init(&rc.lsp_cross_skipped_no_source, 0);
|
||||
|
||||
/* Sub-phase: Dispatch resolve workers (per-file call/usage resolution, PARALLEL) */
|
||||
CBM_PROF_START(t_resolve_dispatch);
|
||||
@@ -1819,6 +1967,18 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
|
||||
/* Summary metric that replaces the removed `pass.timing pass=lsp_cross`
|
||||
* log line — surfaces how many files the fused cross-file LSP step
|
||||
* actually processed vs skipped (e.g. because their source bytes
|
||||
* were not retained at extract time due to the per-file/total cap). */
|
||||
cbm_log_info("parallel.resolve.lsp_cross_done",
|
||||
"files_processed",
|
||||
itoa_log(atomic_load_explicit(&rc.lsp_cross_processed, memory_order_relaxed)),
|
||||
"files_skipped_no_source",
|
||||
itoa_log(atomic_load_explicit(&rc.lsp_cross_skipped_no_source,
|
||||
memory_order_relaxed)),
|
||||
"defs_total", itoa_log(def_count));
|
||||
|
||||
cbm_log_info("parallel.resolve.done", "calls", itoa_log(total_calls), "usages",
|
||||
itoa_log(total_usages), "semantic", itoa_log(total_semantic + go_impl),
|
||||
"lsp_overrides", itoa_log(total_lsp_overrides));
|
||||
|
||||
+36
-6
@@ -572,17 +572,47 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
|
||||
free(cache);
|
||||
return rc != 0 ? rc : CBM_NOT_FOUND;
|
||||
}
|
||||
/* Cross-file LSP: augments per-file resolved_calls with cross-file
|
||||
* type-aware resolutions before parallel_resolve emits CALLS edges.
|
||||
* Soft-failures only — log and continue. */
|
||||
/* Cross-file LSP precondition: build a project-wide CBMLSPDef[]
|
||||
* once. The fused resolve_worker invokes cbm_pxc_run_one(_ts) per
|
||||
* file using these defs + the file's IMPORTS map, so cross-file
|
||||
* type-resolved CALLS land in result->resolved_calls before the
|
||||
* CALLS-edge emission. This replaces the old sequential
|
||||
* cbm_pipeline_pass_lsp_cross pass which re-read every source from
|
||||
* disk and re-parsed every tree on a single thread (~520s on
|
||||
* kubernetes). Soft-failure: NULL all_defs / NULL def_modules just
|
||||
* mean cross-file LSP no-ops; per-file LSP already ran during
|
||||
* extract. */
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, t);
|
||||
(void)cbm_pipeline_pass_lsp_cross(ctx, files, file_count, cache);
|
||||
cbm_log_info("pass.timing", "pass", "lsp_cross", "elapsed_ms",
|
||||
char **def_modules = (char **)calloc((size_t)file_count, sizeof(char *));
|
||||
int def_count = 0;
|
||||
CBMLSPDef *all_defs = def_modules
|
||||
? cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name,
|
||||
def_modules, &def_count)
|
||||
: NULL;
|
||||
/* Build inverted index: module_qn → defs. The fused resolve_worker
|
||||
* uses this to filter the global all_defs[] down to just the defs
|
||||
* each file actually needs (own_module + imported modules) — the
|
||||
* gopls "package summary" pattern. Drops per-file registry build
|
||||
* cost from O(all_defs) to O(relevant_defs), typically 50-100×
|
||||
* smaller per file. */
|
||||
CBMModuleDefIndex *module_def_index =
|
||||
all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL;
|
||||
cbm_log_info("pass.timing", "pass", "lsp_cross_prepare", "elapsed_ms",
|
||||
itoa_buf((int)elapsed_ms(*t)));
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, t);
|
||||
rc = cbm_parallel_resolve(ctx, files, file_count, cache, &shared_ids, worker_count);
|
||||
rc = cbm_parallel_resolve(ctx, files, file_count, cache, &shared_ids,
|
||||
worker_count, all_defs, def_count, def_modules,
|
||||
module_def_index);
|
||||
cbm_log_info("pass.timing", "pass", "parallel_resolve", "elapsed_ms",
|
||||
itoa_buf((int)elapsed_ms(*t)));
|
||||
cbm_pxc_free_module_def_index(module_def_index);
|
||||
free(all_defs);
|
||||
if (def_modules) {
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
free(def_modules[i]);
|
||||
}
|
||||
free(def_modules);
|
||||
}
|
||||
cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids));
|
||||
cbm_pipeline_extract_infra_routes(p->gbuf, files, cache, file_count);
|
||||
cbm_pipeline_process_infra_bindings(p->gbuf, files, cache, file_count);
|
||||
|
||||
@@ -403,8 +403,15 @@ static void run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *change
|
||||
cbm_log_info("pass.timing", "pass", "incr_registry", "elapsed_ms",
|
||||
itoa_buf((int)elapsed_ms(t)));
|
||||
|
||||
/* Incremental skips cross-file LSP precondition build — it
|
||||
* would need all_defs from the full project, not just the
|
||||
* changed slice. Per-file LSP (run inside cbm_extract_file)
|
||||
* still fires; cross-file resolution is deferred to the
|
||||
* next full re-index. Pass NULL/0/NULL to make the fused
|
||||
* step in resolve_worker a no-op. */
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
|
||||
cbm_parallel_resolve(ctx, changed_files, ci, cache, &shared_ids, worker_count);
|
||||
cbm_parallel_resolve(ctx, changed_files, ci, cache, &shared_ids,
|
||||
worker_count, NULL, 0, NULL, NULL);
|
||||
cbm_gbuf_set_next_id(ctx->gbuf, atomic_load(&shared_ids));
|
||||
cbm_log_info("pass.timing", "pass", "incr_resolve", "elapsed_ms",
|
||||
itoa_buf((int)elapsed_ms(t)));
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include "discover/discover.h"
|
||||
#include "foundation/hash_table.h"
|
||||
#include "cbm.h"
|
||||
#include "lsp/go_lsp.h" /* CBMLSPDef for cbm_parallel_resolve cross-LSP inputs */
|
||||
#include <stdatomic.h>
|
||||
|
||||
/* ── Shared pipeline constants ─────────────────────────────────── */
|
||||
@@ -363,9 +364,25 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
|
||||
* Each worker resolves calls, usages, throws, rw, inherits, decorates,
|
||||
* and implements edges into per-worker edge bufs, then merges.
|
||||
* Runs Go-style implicit IMPLEMENTS as serial post-step. */
|
||||
/* Opaque module-def index — defined in pass_lsp_cross.c. Forward-declared
|
||||
* here so we can include it in cbm_parallel_resolve's signature without
|
||||
* pulling the pass header into every consumer of pipeline_internal.h. */
|
||||
struct CBMModuleDefIndex;
|
||||
|
||||
int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
|
||||
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
|
||||
int worker_count);
|
||||
int worker_count,
|
||||
/* Cross-file LSP inputs — pre-built once by the caller and
|
||||
* shared read-only across workers (typed non-const to match
|
||||
* the existing cbm_run_X_lsp_cross signatures the resolve
|
||||
* worker forwards them to). Pass NULL/0/NULL to skip. */
|
||||
CBMLSPDef *all_defs, int def_count,
|
||||
char *const *def_modules,
|
||||
/* Optional inverted index module_qn → defs[] — when
|
||||
* present, the fused worker filters the global all_defs
|
||||
* down to just the per-file relevant subset (gopls pattern).
|
||||
* Pass NULL to disable filtering (each file sees all_defs). */
|
||||
struct CBMModuleDefIndex *module_def_index);
|
||||
|
||||
/* Post-merge: create Route nodes for HTTP_CALLS/ASYNC_CALLS edges that
|
||||
* have url_path in properties but point to library functions instead of routes.
|
||||
|
||||
+23
-5
@@ -139,14 +139,32 @@ static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_
|
||||
cbm_build_registry_from_cache(&ctx, files, file_count, result_cache);
|
||||
|
||||
/* Cross-file LSP — mirrors run_parallel_pipeline ordering in pipeline.c.
|
||||
* Augments per-file resolved_calls with cross-file type-aware resolutions
|
||||
* (e.g. method calls on imported classes) before parallel_resolve emits
|
||||
* the call edges. */
|
||||
cbm_pipeline_pass_lsp_cross(&ctx, files, file_count, result_cache);
|
||||
* Build the project-wide all_defs[] precondition, then feed it into
|
||||
* cbm_parallel_resolve where the fused resolve_worker invokes
|
||||
* cbm_pxc_run_one(_ts) per file BEFORE materializing CALLS edges. */
|
||||
char **def_modules = (char **)calloc((size_t)file_count, sizeof(char *));
|
||||
int def_count = 0;
|
||||
CBMLSPDef *all_defs = def_modules
|
||||
? cbm_pxc_collect_all_defs(result_cache, files, file_count, ctx.project_name,
|
||||
def_modules, &def_count)
|
||||
: NULL;
|
||||
CBMModuleDefIndex *module_def_index =
|
||||
all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL;
|
||||
|
||||
cbm_parallel_resolve(&ctx, files, file_count, result_cache, &shared_ids, worker_count);
|
||||
cbm_parallel_resolve(&ctx, files, file_count, result_cache, &shared_ids,
|
||||
worker_count, all_defs, def_count, def_modules,
|
||||
module_def_index);
|
||||
cbm_gbuf_set_next_id(gbuf, atomic_load(&shared_ids));
|
||||
|
||||
cbm_pxc_free_module_def_index(module_def_index);
|
||||
free(all_defs);
|
||||
if (def_modules) {
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
free(def_modules[i]);
|
||||
}
|
||||
free(def_modules);
|
||||
}
|
||||
|
||||
for (int i = 0; i < file_count; i++)
|
||||
if (result_cache[i])
|
||||
cbm_free_result(result_cache[i]);
|
||||
|
||||
Reference in new issue
Block a user