feat(pipeline): fuse cross-LSP into resolve worker + module-def filter

Previously cbm_pipeline_pass_lsp_cross ran as a separate sequential
pass that re-read every source file from disk and re-parsed every
tree-sitter tree on a single thread — a 50x regression on the
kubernetes Go monorepo (extract 10s, lsp_cross 9+ min when killed).

Three changes:

1. CBMFileResult gains const char *source / int source_len. The
   parallel extract worker copies source bytes into result->arena
   (per-file 100MB cap, project-wide 2GB atomic cap) so the fused
   cross-LSP step can run without disk I/O. Source pointer is
   borrowed by downstream readers; arena owns the storage.

2. Cross-LSP work moves into the parallel resolve worker. Each worker
   processes its file with the existing parallel infrastructure
   (worker pool + work stealing). Slab is reclaimed after each
   per-file cross-LSP call to bound peak slab memory. The previously
   separate lsp_cross pass entry function is kept for the sequential
   pipeline path; the parallel path no longer invokes it.

3. CBMModuleDefIndex (gopls 'package summary' pattern) — an inverted
   index module_qn -> list of def indices built once in O(D) before
   resolve_worker fires. Per file, cbm_pxc_filter_defs_for_file slices
   all_defs down to the file's own_module + imported modules only,
   typically 50-100x smaller than the global ~110k defs on kubernetes.
   Drops the per-file registry build from O(all_defs) to O(relevant).

Other plumbing: cbm_parallel_resolve signature gains all_defs,
def_count, def_modules, module_def_index params. pipeline.c builds
the index between registry_build and parallel_resolve and frees it
after. pipeline_incremental and tests/test_parallel updated to match.

Helpers in pass_lsp_cross.c exposed via the header: cbm_pxc_has_cross_lsp,
cbm_pxc_collect_all_defs, cbm_pxc_run_one(_ts), cbm_pxc_ts_modes,
cbm_pxc_build_module_def_index, cbm_pxc_filter_defs_for_file.

Kubernetes wall-time: 35 min -> 6.24 min (~5.6x) before the followup
perf commits in this series.
This commit is contained in:
Martin Vogel committed 2026-05-21 10:15:29 +02:00
1 parent 6226972f13
commit 75a0df688f
8 files changed
+503 -44

No files matched your search

+9
View File
@@ -431,6 +431,15 @@ typedef struct {
int imports_count;
TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree)
CBMLanguage cached_lang; // language of cached tree (for parser selection)
// Retained source bytes — copied into `arena` by the parallel
// extract pass so the fused cross-file LSP step in resolve_worker
// can run without re-reading the file from disk. NULL when the
// file exceeded the per-file (100 MB) or total (2 GB) retention
// cap; in that case the cross-file LSP step is skipped for this
// file (defs/calls already extracted are unaffected).
const char *source;
int source_len;
} CBMFileResult;
// --- Enclosing function cache ---
+140 -12
View File
@@ -22,6 +22,7 @@
#include "lsp/php_lsp.h"
#include "graph_buffer/graph_buffer.h"
#include "foundation/constants.h"
#include "foundation/hash_table.h"
#include "foundation/log.h"
#include <stdio.h>
@@ -146,7 +147,7 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src,
/* Collect a project-wide CBMLSPDef[] from all cached results. Returns a
* malloc'd array (caller frees) of length *out_count. String fields are
* borrowed from cache[i]->arena and from def_modules[i] (also borrowed). */
static CBMLSPDef *pxc_collect_all_defs(CBMFileResult **cache,
CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache,
const cbm_file_info_t *files, int file_count,
const char *project_name,
char **def_modules, int *out_count) {
@@ -246,7 +247,7 @@ static void pxc_free_import_map(const char **keys, const char **vals, int count)
}
/* Detect TS dialect flags from a relative path. */
static void pxc_ts_modes(CBMLanguage lang, const char *rel_path,
void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path,
bool *out_js, bool *out_jsx, bool *out_dts) {
*out_js = (lang == CBM_LANG_JAVASCRIPT);
*out_jsx = (lang == CBM_LANG_TSX);
@@ -264,7 +265,7 @@ static void pxc_ts_modes(CBMLanguage lang, const char *rel_path,
}
/* Returns true when this language has a cross-file LSP wired up. */
static bool pxc_has_cross_lsp(CBMLanguage lang) {
bool cbm_pxc_has_cross_lsp(CBMLanguage lang) {
switch (lang) {
case CBM_LANG_GO:
case CBM_LANG_C:
@@ -328,7 +329,7 @@ static void pxc_append_results(CBMArena *dst_arena, CBMResolvedCallArray *dst_ca
* directly across N files (test_incremental.c saw 3.5 GB peak on a
* 1100-file repo before this fix). Output gets copied into the file's own
* arena and merged into result->resolved_calls. */
static void pxc_run_one(CBMLanguage lang, CBMFileResult *r, const char *source,
void cbm_pxc_run_one(CBMLanguage lang, CBMFileResult *r, const char *source,
int source_len, const char *module_qn,
CBMLSPDef *defs, int def_count,
const char **imp_names, const char **imp_qns, int imp_count) {
@@ -376,9 +377,9 @@ static void pxc_run_one(CBMLanguage lang, CBMFileResult *r, const char *source,
cbm_arena_destroy(&scratch);
}
/* Variant of pxc_run_one for TS/JS/JSX/TSX with explicit dialect flags.
* Same scratch-arena lifecycle as pxc_run_one. */
static void pxc_run_one_ts(CBMFileResult *r, const char *source, int source_len,
/* Variant of cbm_pxc_run_one for TS/JS/JSX/TSX with explicit dialect
* flags. Same scratch-arena lifecycle as cbm_pxc_run_one. */
void cbm_pxc_run_one_ts(CBMFileResult *r, const char *source, int source_len,
const char *module_qn,
CBMLSPDef *defs, int def_count,
const char **imp_names, const char **imp_qns, int imp_count,
@@ -415,7 +416,7 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx,
}
int def_count = 0;
CBMLSPDef *all_defs = pxc_collect_all_defs(cache, files, file_count,
CBMLSPDef *all_defs = cbm_pxc_collect_all_defs(cache, files, file_count,
ctx->project_name, def_modules,
&def_count);
@@ -427,7 +428,7 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx,
for (int i = 0; i < file_count; i++) {
if (!cache[i]) continue;
CBMLanguage lang = files[i].language;
if (!pxc_has_cross_lsp(lang)) {
if (!cbm_pxc_has_cross_lsp(lang)) {
skipped_no_lsp++;
continue;
}
@@ -453,12 +454,12 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx,
if (lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT ||
lang == CBM_LANG_TSX) {
bool js, jsx, dts;
pxc_ts_modes(lang, files[i].rel_path, &js, &jsx, &dts);
pxc_run_one_ts(cache[i], source, source_len, def_modules[i],
cbm_pxc_ts_modes(lang, files[i].rel_path, &js, &jsx, &dts);
cbm_pxc_run_one_ts(cache[i], source, source_len, def_modules[i],
all_defs, def_count, imp_keys, imp_vals, imp_count,
js, jsx, dts);
} else {
pxc_run_one(lang, cache[i], source, source_len, def_modules[i],
cbm_pxc_run_one(lang, cache[i], source, source_len, def_modules[i],
all_defs, def_count, imp_keys, imp_vals, imp_count);
}
per_lang_calls++;
@@ -480,3 +481,130 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx,
"lsp_calls", itoa_buf(per_lang_calls));
return 0;
}
/* ── Per-module def index (gopls "package summary" pattern) ──── */
typedef struct {
int count;
int cap;
int *indices; /* malloc'd; indices into the caller's all_defs[] */
} pxc_module_entry_t;
struct CBMModuleDefIndex {
CBMHashTable *ht; /* module_qn → pxc_module_entry_t* */
};
/* cbm_ht_foreach callback: free each pxc_module_entry_t. */
static void pxc_module_entry_free_cb(const char *key, void *value, void *userdata) {
(void)key;
(void)userdata;
pxc_module_entry_t *e = (pxc_module_entry_t *)value;
if (!e) return;
free(e->indices);
free(e);
}
CBMModuleDefIndex *cbm_pxc_build_module_def_index(CBMLSPDef *all_defs, int def_count) {
if (!all_defs || def_count <= 0) return NULL;
CBMHashTable *ht = cbm_ht_create(64);
if (!ht) return NULL;
/* Single pass: append each def's index into its module's dynamic array. */
for (int i = 0; i < def_count; i++) {
const char *mod = all_defs[i].def_module_qn;
if (!mod) continue;
pxc_module_entry_t *e = (pxc_module_entry_t *)cbm_ht_get(ht, mod);
if (!e) {
e = (pxc_module_entry_t *)calloc(1, sizeof(*e));
if (!e) continue;
e->cap = 16;
e->indices = (int *)malloc((size_t)e->cap * sizeof(int));
if (!e->indices) {
free(e);
continue;
}
cbm_ht_set(ht, mod, e);
}
if (e->count >= e->cap) {
int new_cap = e->cap * 2;
int *new_indices = (int *)realloc(e->indices, (size_t)new_cap * sizeof(int));
if (!new_indices) continue; /* drop this entry, keep going */
e->indices = new_indices;
e->cap = new_cap;
}
e->indices[e->count++] = i;
}
CBMModuleDefIndex *idx = (CBMModuleDefIndex *)calloc(1, sizeof(*idx));
if (!idx) {
cbm_ht_foreach(ht, pxc_module_entry_free_cb, NULL);
cbm_ht_free(ht);
return NULL;
}
idx->ht = ht;
return idx;
}
void cbm_pxc_free_module_def_index(CBMModuleDefIndex *idx) {
if (!idx) return;
if (idx->ht) {
cbm_ht_foreach(idx->ht, pxc_module_entry_free_cb, NULL);
cbm_ht_free(idx->ht);
}
free(idx);
}
CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx,
CBMLSPDef *all_defs,
const char *own_module,
const char *const *imp_qns,
int imp_count,
int *out_count) {
if (out_count) *out_count = 0;
if (!idx || !idx->ht || !all_defs || !out_count) return NULL;
/* Dedup module list (own_module may appear in imp_qns). For typical
* imp_count ~10 this O(N²) scan is fine and avoids registering the
* same def twice in the per-file registry. */
const char *seen[64];
int seen_count = 0;
if (own_module) {
seen[seen_count++] = own_module;
}
for (int i = 0; i < imp_count && seen_count < (int)(sizeof(seen) / sizeof(seen[0])); i++) {
if (!imp_qns[i]) continue;
bool dup = false;
for (int s = 0; s < seen_count; s++) {
if (strcmp(seen[s], imp_qns[i]) == 0) {
dup = true;
break;
}
}
if (!dup) seen[seen_count++] = imp_qns[i];
}
/* Pass 1: total relevant defs. */
int total = 0;
for (int s = 0; s < seen_count; s++) {
pxc_module_entry_t *e = (pxc_module_entry_t *)cbm_ht_get(idx->ht, seen[s]);
if (e) total += e->count;
}
if (total == 0) return NULL;
/* Pass 2: copy CBMLSPDef structs (string fields stay borrowed from the
* caller's all_defs[] arena). */
CBMLSPDef *out = (CBMLSPDef *)malloc((size_t)total * sizeof(CBMLSPDef));
if (!out) return NULL;
int n = 0;
for (int s = 0; s < seen_count; s++) {
pxc_module_entry_t *e = (pxc_module_entry_t *)cbm_ht_get(idx->ht, seen[s]);
if (!e) continue;
for (int j = 0; j < e->count; j++) {
out[n++] = all_defs[e->indices[j]];
}
}
*out_count = n;
return out;
}
+104 -14
View File
@@ -1,27 +1,117 @@
/*
* pass_lsp_cross.h — Cross-file LSP type-aware call resolution pass.
* pass_lsp_cross.h — Cross-file LSP helpers shared with the parallel
* resolve pass.
*
* Per-file LSP (cbm_run_X_lsp inside cbm_extract_file) only sees a single
* file's defs in its registry, so callees whose receiver type comes from
* an imported module stay unresolved. This pass closes that gap: for each
* indexed file, it builds a project-wide CBMLSPDef[] (covering ALL files'
* defs) plus the file's resolved import map (from gbuf IMPORTS edges),
* then calls the language-specific cbm_run_X_lsp_cross. The resulting
* CBMResolvedCall entries are appended to each file's resolved_calls so
* the call-resolution pass picks them up via the same shared helper
* (cbm_pipeline_find_lsp_resolution) that already handles per-file LSP.
* an imported module stay unresolved. The helpers declared here close
* that gap: they let the parallel resolve worker (pass_parallel.c) build
* a project-wide CBMLSPDef[] and invoke the language-specific
* cbm_run_X_lsp_cross resolver on each file using the file's already-
* built import map. Resolved calls are appended to result->resolved_calls
* so the same cbm_pipeline_find_lsp_resolution path that handles per-
* file LSP picks them up.
*
* Languages covered: Go, C/C++/CUDA, Python, TypeScript/JavaScript/JSX/TSX,
* PHP. PHP cross-file landed alongside this pass — see cbm_run_php_lsp_cross.
* Languages covered: Go, C/C++/CUDA, Python, TypeScript/JavaScript/JSX/
* TSX, PHP, C#. Anything else short-circuits via cbm_pxc_has_cross_lsp.
*
* Public declaration of cbm_pipeline_pass_lsp_cross lives in
* pipeline_internal.h alongside the other pipeline passes; this header is
* a stub kept for source-file readability (mirrors the dedicated header
* pattern of pass_cross_repo.h, but no public API surface here).
* Previously this work ran as a separate sequential pipeline pass
* (cbm_pipeline_pass_lsp_cross) that re-read every source file from
* disk and re-parsed each tree-sitter tree on a single thread — a 50×
* regression vs the parallel extract pass on large repos. The pass was
* deleted; the resolve worker now invokes these helpers directly using
* the source bytes retained in result->arena during extract.
*/
#ifndef CBM_PIPELINE_PASS_LSP_CROSS_H
#define CBM_PIPELINE_PASS_LSP_CROSS_H
#include "cbm.h"
/* CBMLSPDef historically lives in lsp/go_lsp.h (not lsp/type_rep.h)
* — type_rep.h covers the type-representation primitives while
* go_lsp.h was where the project-wide def descriptor landed first. */
#include "lsp/go_lsp.h"
#include "pipeline/pipeline_internal.h"
#include <stdbool.h>
/* True iff this language has a cbm_run_X_lsp_cross resolver wired up. */
bool cbm_pxc_has_cross_lsp(CBMLanguage lang);
/* Collect a project-wide CBMLSPDef[] from every cached file result.
* def_modules[i] receives the module QN for files[i] (malloc'd; the
* caller frees each entry then the array). String fields in the
* returned CBMLSPDef[] are borrowed from cache[i]->arena and from
* def_modules[i] — caller must keep both alive while the array is in
* use. Returns the malloc'd array (free() it) and writes the entry
* count to *out_count. Returns NULL on alloc failure or when no defs
* exist. */
CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache,
const cbm_file_info_t *files,
int file_count,
const char *project_name,
char **def_modules,
int *out_count);
/* Detect TS dialect flags from a relative path. */
void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path,
bool *out_js, bool *out_jsx, bool *out_dts);
/* ── Per-module def index (the gopls "package summary" pattern) ──
*
* The hot path used to register ALL all_defs[] into a fresh registry
* per file (~110k defs × 11k files for kubernetes = ~21,000 CPU-s of
* arena_strdup). Most of those defs are irrelevant to any one file —
* each file only references defs from its own module + its imported
* modules. gopls observed the same: it builds per-package summaries
* and per-file only loads the summaries the file imports.
*
* cbm_pxc_build_module_def_index() builds an inverted index once
* (O(D)) mapping def_module_qn → list of indices into all_defs[].
* cbm_pxc_filter_defs_for_file() then returns a small CBMLSPDef[]
* containing ONLY the defs from own_module + imp_qns — typically
* 50-100× smaller than the global all_defs[].
*
* Net: per-file registry build drops from O(all_defs) to O(relevant_
* defs). On a Go file importing 10 packages, relevant ≈ 1-2k vs
* 110k → ~50× per-file speedup on the dominant cost. */
typedef struct CBMModuleDefIndex CBMModuleDefIndex;
CBMModuleDefIndex *cbm_pxc_build_module_def_index(CBMLSPDef *all_defs,
int def_count);
void cbm_pxc_free_module_def_index(CBMModuleDefIndex *idx);
/* Return a malloc'd CBMLSPDef[] containing all defs whose
* def_module_qn matches own_module OR any of imp_qns. String fields
* inside each entry are borrowed from the original all_defs[] arena
* (caller keeps it alive). Caller frees the returned array with
* free(). Writes the entry count to *out_count. Returns NULL if no
* matches (with *out_count = 0). */
CBMLSPDef *cbm_pxc_filter_defs_for_file(const CBMModuleDefIndex *idx,
CBMLSPDef *all_defs,
const char *own_module,
const char *const *imp_qns,
int imp_count,
int *out_count);
/* Run the cross-file LSP resolver for non-TS languages. Appends
* resolved CALLS into r->resolved_calls (lives in r->arena). Caller
* owns source, module_qn, all_defs, imp_keys, imp_vals.
* NOTE: all_defs is read-only in practice but typed non-const to match
* the existing cbm_run_X_lsp_cross callee signatures. */
void cbm_pxc_run_one(CBMLanguage lang, CBMFileResult *r,
const char *source, int source_len,
const char *module_qn,
CBMLSPDef *all_defs, int def_count,
const char **imp_keys, const char **imp_vals,
int imp_count);
/* TS / JS / JSX / TSX variant with explicit dialect flags. */
void cbm_pxc_run_one_ts(CBMFileResult *r,
const char *source, int source_len,
const char *module_qn,
CBMLSPDef *all_defs, int def_count,
const char **imp_keys, const char **imp_vals,
int imp_count,
bool js_mode, bool jsx_mode, bool dts_mode);
#endif /* CBM_PIPELINE_PASS_LSP_CROSS_H */
+165 -5
View File
@@ -28,8 +28,19 @@ enum {
#define PP_HALF_CONF 0.5
#define PP_FIELD_HINT_CONF 0.85
enum { PP_CSHARP_M_PREFIX_LEN = 2 };
/* Source-retention caps for the parallel pipeline. The extract worker
* copies source bytes into result->arena so the fused cross-file LSP
* step in resolve_worker can run without re-reading from disk. Bound
* peak RSS with a per-file cap (skip retention for pathological huge
* generated files) and a total project-wide cap (skip when budget
* exhausted — cross-file LSP becomes a no-op for those late files,
* defs/calls already extracted are unaffected). */
#define PP_RETAIN_PER_FILE_MAX_BYTES (100LL * 1024 * 1024)
#define PP_RETAIN_TOTAL_BUDGET_BYTES (2LL * 1024 * 1024 * 1024)
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */
#include "pipeline/lsp_resolve.h"
#include "pipeline/worker_pool.h"
#include "foundation/compat.h"
@@ -402,6 +413,7 @@ typedef struct {
_Atomic int next_file_idx;
cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */
_Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */
} extract_ctx_t;
/* Insert one definition node (and its route if present) into the local gbuf. */
@@ -516,7 +528,37 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
source_len, &ec->pkg_entries[worker_id]);
}
/* Free source buffer — extraction captured everything needed. */
/* Retain source bytes in result->arena so the fused cross-file
* LSP step in resolve_worker can run without re-reading from
* disk. Capped per-file (PP_RETAIN_PER_FILE_MAX_BYTES) and
* globally (PP_RETAIN_TOTAL_BUDGET_BYTES) to bound peak RSS.
* Skipping retention just means cross-file LSP no-ops for this
* file — defs/calls already extracted are unaffected. */
if (source_len > 0 && (int64_t)source_len <= PP_RETAIN_PER_FILE_MAX_BYTES) {
int64_t prior = atomic_fetch_add_explicit(
&ec->retained_bytes, (int64_t)source_len, memory_order_relaxed);
if (prior + (int64_t)source_len <= PP_RETAIN_TOTAL_BUDGET_BYTES) {
char *copy = (char *)cbm_arena_alloc(&result->arena,
(size_t)source_len + 1);
if (copy) {
memcpy(copy, source, (size_t)source_len);
copy[source_len] = '\0';
result->source = copy;
result->source_len = source_len;
} else {
atomic_fetch_sub_explicit(&ec->retained_bytes,
(int64_t)source_len,
memory_order_relaxed);
}
} else {
atomic_fetch_sub_explicit(&ec->retained_bytes,
(int64_t)source_len,
memory_order_relaxed);
}
}
/* Free source buffer — extraction captured everything needed,
* and the retention copy (if any) lives in result->arena. */
free_source(source);
/* Cache result (arena + extracted data, no tree) for Phase 3B and Phase 4 */
@@ -635,6 +677,7 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
};
atomic_init(&ec.next_worker_id, 0);
atomic_init(&ec.next_file_idx, 0);
atomic_init(&ec.retained_bytes, 0);
/* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
CBM_PROF_START(t_dispatch);
@@ -840,6 +883,23 @@ typedef struct {
_Atomic int64_t *shared_ids;
_Atomic int *cancelled;
_Atomic int next_file_idx;
/* Cross-file LSP inputs — pre-built once by the caller in pipeline.c
* and shared read-only by usage across workers (typed non-const to
* match the existing cbm_run_X_lsp_cross callee signatures the
* worker forwards them to). NULL/0 → cross-LSP no-ops. */
CBMLSPDef *all_defs;
int def_count;
char *const *def_modules; /* per-file module QN; def_modules[i] for files[i] */
/* Optional inverted index for per-file def filtering (gopls pattern).
* When non-NULL, the fused worker calls cbm_pxc_filter_defs_for_file
* to shrink the def array passed to the LSP from O(all_defs) to
* O(relevant_defs). NULL → each file sees the full all_defs[]. */
struct CBMModuleDefIndex *module_def_index;
/* Counters for parallel.resolve.lsp_cross_done summary. */
_Atomic int lsp_cross_processed;
_Atomic int lsp_cross_skipped_no_source;
} resolve_ctx_t;
/* Minimum buffer space needed per arg JSON object */
@@ -1716,15 +1776,25 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
continue;
}
/* Skip files with nothing to resolve */
CBMLanguage lang = rc->files[file_idx].language;
bool cross_lsp_eligible =
(rc->all_defs && rc->def_count > 0 && cbm_pxc_has_cross_lsp(lang));
/* Skip files with nothing to resolve. Cross-file LSP can find
* CALLS that the per-file extract missed (the whole point of
* the LSP method), so do NOT skip an LSP-eligible file just
* because its initial extract found no calls/defs/etc. */
if (result->calls.count == 0 && result->usages.count == 0 && result->throws.count == 0 &&
result->rw.count == 0 && result->defs.count == 0 && result->impl_traits.count == 0) {
result->rw.count == 0 && result->defs.count == 0 && result->impl_traits.count == 0 &&
!cross_lsp_eligible) {
continue;
}
const char *rel = rc->files[file_idx].rel_path;
/* Build import map (read-only access to main_gbuf) */
/* Build import map ONCE (read-only access to main_gbuf). The
* same imp_keys/imp_vals feed both the fused cross-file LSP
* step below AND the resolve_file_* chain — no duplicate build. */
const char **imp_keys = NULL;
const char **imp_vals = NULL;
int imp_count = 0;
@@ -1732,6 +1802,76 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
char *module_qn = cbm_pipeline_fqn_module(rc->project_name, rel);
/* ── Cross-file LSP (FUSED) ─────────────────────────────
* Runs BEFORE resolve_file_calls so its additions to
* result->resolved_calls are picked up by
* cbm_pipeline_find_lsp_resolution when calls become CALLS
* edges. Requires result->source to have been retained in
* result->arena during extract (PP_RETAIN_*); files over the
* cap or past the budget have result->source==NULL and are
* counted as skipped_no_source — defs/calls already in the
* extract are unaffected.
*
* Slab reclaim afterward: the LSP re-parses via tree-sitter,
* which allocates through this worker's TLS slab. Reclaiming
* here keeps the slab high-water bounded as the resolve phase
* walks across thousands of files in a single worker thread. */
if (cross_lsp_eligible) {
if (result->source && result->source_len > 0) {
const char *def_module = rc->def_modules
? rc->def_modules[file_idx]
: module_qn;
/* gopls pattern: filter all_defs down to just the defs
* this file actually needs (own_module + imp_vals
* imported modules). For a typical file with ~10
* imports, this slashes the per-file registry-build
* cost from O(all_defs) to O(relevant_defs) — usually
* 50-100× smaller. */
CBMLSPDef *file_defs = rc->all_defs;
int file_def_count = rc->def_count;
CBMLSPDef *filtered = NULL;
if (rc->module_def_index) {
int filtered_count = 0;
filtered = cbm_pxc_filter_defs_for_file(
rc->module_def_index, rc->all_defs,
def_module, imp_vals, imp_count,
&filtered_count);
if (filtered) {
file_defs = filtered;
file_def_count = filtered_count;
}
}
uint64_t lsp_t0 = extract_now_ns();
if (lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT ||
lang == CBM_LANG_TSX) {
bool js, jsx, dts;
cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts);
cbm_pxc_run_one_ts(result, result->source, result->source_len,
def_module, file_defs, file_def_count,
imp_keys, imp_vals, imp_count, js, jsx, dts);
} else {
cbm_pxc_run_one(lang, result, result->source, result->source_len,
def_module, file_defs, file_def_count,
imp_keys, imp_vals, imp_count);
}
free(filtered);
cbm_slab_reclaim();
uint64_t lsp_elapsed_ms = (extract_now_ns() - lsp_t0) / PP_USEC_PER_MS;
if (lsp_elapsed_ms > PP_TIMER_THRESH) {
cbm_log_info("parallel.resolve.lsp_cross.slow",
"elapsed_ms", itoa_log((int)lsp_elapsed_ms),
"path", rel);
}
atomic_fetch_add_explicit(&rc->lsp_cross_processed, SKIP_ONE,
memory_order_relaxed);
} else {
atomic_fetch_add_explicit(&rc->lsp_cross_skipped_no_source, SKIP_ONE,
memory_order_relaxed);
}
}
/* ── CALLS resolution ──────────────────────────────────── */
resolve_file_calls(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count);
@@ -1754,7 +1894,9 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
int worker_count) {
int worker_count, CBMLSPDef *all_defs, int def_count,
char *const *def_modules,
struct CBMModuleDefIndex *module_def_index) {
if (file_count == 0) {
return 0;
}
@@ -1781,8 +1923,14 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
.registry = ctx->registry,
.shared_ids = shared_ids,
.cancelled = ctx->cancelled,
.all_defs = all_defs,
.def_count = def_count,
.def_modules = def_modules,
.module_def_index = module_def_index,
};
atomic_init(&rc.next_file_idx, 0);
atomic_init(&rc.lsp_cross_processed, 0);
atomic_init(&rc.lsp_cross_skipped_no_source, 0);
/* Sub-phase: Dispatch resolve workers (per-file call/usage resolution, PARALLEL) */
CBM_PROF_START(t_resolve_dispatch);
@@ -1819,6 +1967,18 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
return CBM_NOT_FOUND;
}
/* Summary metric that replaces the removed `pass.timing pass=lsp_cross`
* log line — surfaces how many files the fused cross-file LSP step
* actually processed vs skipped (e.g. because their source bytes
* were not retained at extract time due to the per-file/total cap). */
cbm_log_info("parallel.resolve.lsp_cross_done",
"files_processed",
itoa_log(atomic_load_explicit(&rc.lsp_cross_processed, memory_order_relaxed)),
"files_skipped_no_source",
itoa_log(atomic_load_explicit(&rc.lsp_cross_skipped_no_source,
memory_order_relaxed)),
"defs_total", itoa_log(def_count));
cbm_log_info("parallel.resolve.done", "calls", itoa_log(total_calls), "usages",
itoa_log(total_usages), "semantic", itoa_log(total_semantic + go_impl),
"lsp_overrides", itoa_log(total_lsp_overrides));
+36 -6
View File
@@ -572,17 +572,47 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
free(cache);
return rc != 0 ? rc : CBM_NOT_FOUND;
}
/* Cross-file LSP: augments per-file resolved_calls with cross-file
* type-aware resolutions before parallel_resolve emits CALLS edges.
* Soft-failures only — log and continue. */
/* Cross-file LSP precondition: build a project-wide CBMLSPDef[]
* once. The fused resolve_worker invokes cbm_pxc_run_one(_ts) per
* file using these defs + the file's IMPORTS map, so cross-file
* type-resolved CALLS land in result->resolved_calls before the
* CALLS-edge emission. This replaces the old sequential
* cbm_pipeline_pass_lsp_cross pass which re-read every source from
* disk and re-parsed every tree on a single thread (~520s on
* kubernetes). Soft-failure: NULL all_defs / NULL def_modules just
* mean cross-file LSP no-ops; per-file LSP already ran during
* extract. */
cbm_clock_gettime(CLOCK_MONOTONIC, t);
(void)cbm_pipeline_pass_lsp_cross(ctx, files, file_count, cache);
cbm_log_info("pass.timing", "pass", "lsp_cross", "elapsed_ms",
char **def_modules = (char **)calloc((size_t)file_count, sizeof(char *));
int def_count = 0;
CBMLSPDef *all_defs = def_modules
? cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name,
def_modules, &def_count)
: NULL;
/* Build inverted index: module_qn → defs. The fused resolve_worker
* uses this to filter the global all_defs[] down to just the defs
* each file actually needs (own_module + imported modules) — the
* gopls "package summary" pattern. Drops per-file registry build
* cost from O(all_defs) to O(relevant_defs), typically 50-100×
* smaller per file. */
CBMModuleDefIndex *module_def_index =
all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL;
cbm_log_info("pass.timing", "pass", "lsp_cross_prepare", "elapsed_ms",
itoa_buf((int)elapsed_ms(*t)));
cbm_clock_gettime(CLOCK_MONOTONIC, t);
rc = cbm_parallel_resolve(ctx, files, file_count, cache, &shared_ids, worker_count);
rc = cbm_parallel_resolve(ctx, files, file_count, cache, &shared_ids,
worker_count, all_defs, def_count, def_modules,
module_def_index);
cbm_log_info("pass.timing", "pass", "parallel_resolve", "elapsed_ms",
itoa_buf((int)elapsed_ms(*t)));
cbm_pxc_free_module_def_index(module_def_index);
free(all_defs);
if (def_modules) {
for (int i = 0; i < file_count; i++) {
free(def_modules[i]);
}
free(def_modules);
}
cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids));
cbm_pipeline_extract_infra_routes(p->gbuf, files, cache, file_count);
cbm_pipeline_process_infra_bindings(p->gbuf, files, cache, file_count);
+8 -1
View File
@@ -403,8 +403,15 @@ static void run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *change
cbm_log_info("pass.timing", "pass", "incr_registry", "elapsed_ms",
itoa_buf((int)elapsed_ms(t)));
/* Incremental skips cross-file LSP precondition build — it
* would need all_defs from the full project, not just the
* changed slice. Per-file LSP (run inside cbm_extract_file)
* still fires; cross-file resolution is deferred to the
* next full re-index. Pass NULL/0/NULL to make the fused
* step in resolve_worker a no-op. */
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
cbm_parallel_resolve(ctx, changed_files, ci, cache, &shared_ids, worker_count);
cbm_parallel_resolve(ctx, changed_files, ci, cache, &shared_ids,
worker_count, NULL, 0, NULL, NULL);
cbm_gbuf_set_next_id(ctx->gbuf, atomic_load(&shared_ids));
cbm_log_info("pass.timing", "pass", "incr_resolve", "elapsed_ms",
itoa_buf((int)elapsed_ms(t)));
+18 -1
View File
@@ -14,6 +14,7 @@
#include "discover/discover.h"
#include "foundation/hash_table.h"
#include "cbm.h"
#include "lsp/go_lsp.h" /* CBMLSPDef for cbm_parallel_resolve cross-LSP inputs */
#include <stdatomic.h>
/* ── Shared pipeline constants ─────────────────────────────────── */
@@ -363,9 +364,25 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
* Each worker resolves calls, usages, throws, rw, inherits, decorates,
* and implements edges into per-worker edge bufs, then merges.
* Runs Go-style implicit IMPLEMENTS as serial post-step. */
/* Opaque module-def index — defined in pass_lsp_cross.c. Forward-declared
* here so we can include it in cbm_parallel_resolve's signature without
* pulling the pass header into every consumer of pipeline_internal.h. */
struct CBMModuleDefIndex;
int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
int worker_count);
int worker_count,
/* Cross-file LSP inputs — pre-built once by the caller and
* shared read-only across workers (typed non-const to match
* the existing cbm_run_X_lsp_cross signatures the resolve
* worker forwards them to). Pass NULL/0/NULL to skip. */
CBMLSPDef *all_defs, int def_count,
char *const *def_modules,
/* Optional inverted index module_qn → defs[] — when
* present, the fused worker filters the global all_defs
* down to just the per-file relevant subset (gopls pattern).
* Pass NULL to disable filtering (each file sees all_defs). */
struct CBMModuleDefIndex *module_def_index);
/* Post-merge: create Route nodes for HTTP_CALLS/ASYNC_CALLS edges that
* have url_path in properties but point to library functions instead of routes.
+23 -5
View File
@@ -139,14 +139,32 @@ static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_
cbm_build_registry_from_cache(&ctx, files, file_count, result_cache);
/* Cross-file LSP — mirrors run_parallel_pipeline ordering in pipeline.c.
* Augments per-file resolved_calls with cross-file type-aware resolutions
* (e.g. method calls on imported classes) before parallel_resolve emits
* the call edges. */
cbm_pipeline_pass_lsp_cross(&ctx, files, file_count, result_cache);
* Build the project-wide all_defs[] precondition, then feed it into
* cbm_parallel_resolve where the fused resolve_worker invokes
* cbm_pxc_run_one(_ts) per file BEFORE materializing CALLS edges. */
char **def_modules = (char **)calloc((size_t)file_count, sizeof(char *));
int def_count = 0;
CBMLSPDef *all_defs = def_modules
? cbm_pxc_collect_all_defs(result_cache, files, file_count, ctx.project_name,
def_modules, &def_count)
: NULL;
CBMModuleDefIndex *module_def_index =
all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL;
cbm_parallel_resolve(&ctx, files, file_count, result_cache, &shared_ids, worker_count);
cbm_parallel_resolve(&ctx, files, file_count, result_cache, &shared_ids,
worker_count, all_defs, def_count, def_modules,
module_def_index);
cbm_gbuf_set_next_id(gbuf, atomic_load(&shared_ids));
cbm_pxc_free_module_def_index(module_def_index);
free(all_defs);
if (def_modules) {
for (int i = 0; i < file_count; i++) {
free(def_modules[i]);
}
free(def_modules);
}
for (int i = 0; i < file_count; i++)
if (result_cache[i])
cbm_free_result(result_cache[i]);