From 94189e69905a37b691cdbc1595e34df5ba42bb7a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 3 Aug 2026 17:44:52 +0200 Subject: [PATCH] feat(pipeline): serialize and publish per-file LSP surfaces Second piece of closure repair. At the collect_all_defs seam -- the only moment the per-file result cache is alive -- both drivers (parallel and sequential) now serialize each file's CBMLSPDef slice to canonical JSON, hash it, and hand the rows to the pipeline; cbm_pipeline_publish_generation writes them into the staging store next to the manifest, so surface data and graph always belong to the same generation. Canonical bytes are the point: every field is written in fixed order with an explicit null for absent strings (NULL and "" differ in the CBMLSPDef contract -- receiver_type NULL means "not a method"), so byte equality IS surface equality and the sha over the bytes is the early-cutoff key. Registry-only labels that pxc_map_label drops but the name registry serves (Field) are folded into the hash as a separate "reg" array, or renaming one would slip past the cutoff. Behaviour pinned in SUITE(pipeline): a fresh full index persists a versioned surface row per file; a BODY edit republishes the identical surface_sha; a SIGNATURE edit changes it. That pair of properties is what the routing layer will stand on. cbm_pxc_collect_all_defs gains an optional per-file prefix array -- the flat all_defs[] otherwise loses the file boundaries the serializer needs. CORRECTION to 6c22338's scope note: it claimed cbm_pipeline_publish_ generation was reachable only behind CBM_INCREMENTAL_TEST_API. Wrong -- dump_and_persist_hashes calls it on every production full index (pipeline.c:1863); the grep that "verified" test-only reachability had excluded pipeline.c itself. The predictable staging name WAS in the production publish path, which makes that fix a real production hardening, not a test-path cleanup. Signed-off-by: Martin Vogel --- Makefile.cbm | 1 + src/pipeline/lsp_surface.c | 269 +++++++++++++++++++++++++++++++ src/pipeline/lsp_surface.h | 43 +++++ src/pipeline/pass_lsp_cross.c | 31 +++- src/pipeline/pass_lsp_cross.h | 8 +- src/pipeline/pipeline.c | 54 ++++++- src/pipeline/pipeline_internal.h | 12 ++ tests/test_parallel.c | 8 +- tests/test_pipeline.c | 72 +++++++++ 9 files changed, 489 insertions(+), 9 deletions(-) create mode 100644 src/pipeline/lsp_surface.c create mode 100644 src/pipeline/lsp_surface.h diff --git a/Makefile.cbm b/Makefile.cbm index 47ac63aa8..e9c86b5f6 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -302,6 +302,7 @@ GRAPH_BUFFER_SRCS = src/graph_buffer/graph_buffer.c # Pipeline module (new) PIPELINE_SRCS = \ src/pipeline/fqn.c \ + src/pipeline/lsp_surface.c \ src/pipeline/path_alias.c \ src/pipeline/registry.c \ src/pipeline/pipeline.c \ diff --git a/src/pipeline/lsp_surface.c b/src/pipeline/lsp_surface.c new file mode 100644 index 000000000..92a3c67ca --- /dev/null +++ b/src/pipeline/lsp_surface.c @@ -0,0 +1,269 @@ +/* + * lsp_surface.c — per-file LSP-surface codec (closure-repair incremental). + * + * See lsp_surface.h for the contract. Two invariants carry the feature: + * + * 1. ROUND-TRIP FIDELITY: defs_from_json(build_json(defs)) must hand the + * per-language registrars the same values collect_all_defs would have + * built from a real parse — a lossy field here silently degrades + * cross-file resolution only on the incremental path, the exact class + * of divergence this feature exists to eliminate. + * + * 2. CANONICAL BYTES: every field is written, in fixed order, with an + * explicit JSON null for absent strings (NULL and "" are different + * values in the CBMLSPDef contract — receiver_type NULL means "not a + * method", and several registrars branch on that). Byte equality of the + * serialization therefore IS surface equality, and the sha over the + * bytes is the early-cutoff key: a body edit reserializes identically. + */ +#include "pipeline/lsp_surface.h" + +#include +#include + +#include "foundation/log.h" +#include "foundation/sha256.h" +#include "yyjson/yyjson.h" + +enum { SURFACE_CODEC_VERSION = 1 }; + +/* Labels the incremental name registry serves that pxc_map_label does NOT + * carry into the CBMLSPDef set. Their (name, qn, label) triple must still + * participate in the surface hash, or renaming one would slip past the + * early cutoff while stale references to it survive in dependent files. + * KEEP IN SYNC with pxc_map_label (pass_lsp_cross.c) and + * incr_label_is_registry_symbol (pipeline_incremental.c); the codec unit + * test cross-checks the three. */ +static bool surface_reg_only_label(const char *label) { + return label && strcmp(label, "Field") == 0; +} + +static void add_str_or_null(yyjson_mut_doc *doc, yyjson_mut_val *obj, const char *key, + const char *val) { + if (val) { + yyjson_mut_obj_add_str(doc, obj, key, val); + } else { + yyjson_mut_obj_add_null(doc, obj, key); + } +} + +static void add_str_array_or_null(yyjson_mut_doc *doc, yyjson_mut_val *obj, const char *key, + const char **items, int count_or_neg1_terminated) { + if (!items) { + yyjson_mut_obj_add_null(doc, obj, key); + return; + } + yyjson_mut_val *arr = yyjson_mut_arr(doc); + if (count_or_neg1_terminated >= 0) { + for (int i = 0; i < count_or_neg1_terminated; i++) { + yyjson_mut_arr_add_str(doc, arr, items[i] ? items[i] : "?"); + } + } else { + for (int i = 0; items[i]; i++) { + yyjson_mut_arr_add_str(doc, arr, items[i]); + } + } + yyjson_mut_obj_add_val(doc, obj, key, arr); +} + +/* Serialize one file's surface: its slice of all_defs plus the registry-only + * symbols from its raw extraction defs. Returns a malloc'd JSON string. */ +static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef *defs, + int def_count) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + if (!doc) { + return NULL; + } + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_obj_add_int(doc, root, "v", SURFACE_CODEC_VERSION); + + yyjson_mut_val *lsp = yyjson_mut_arr(doc); + for (int i = 0; i < def_count; i++) { + const CBMLSPDef *d = &defs[i]; + yyjson_mut_val *o = yyjson_mut_obj(doc); + add_str_or_null(doc, o, "qn", d->qualified_name); + add_str_or_null(doc, o, "sn", d->short_name); + add_str_or_null(doc, o, "lb", d->label); + add_str_or_null(doc, o, "rt", d->receiver_type); + add_str_or_null(doc, o, "dm", d->def_module_qn); + add_str_or_null(doc, o, "ret", d->return_types); + add_str_or_null(doc, o, "emb", d->embedded_types); + add_str_or_null(doc, o, "fd", d->field_defs); + add_str_or_null(doc, o, "mn", d->method_names_str); + add_str_array_or_null(doc, o, "spt", d->signature_param_types, d->signature_param_count); + yyjson_mut_obj_add_bool(doc, o, "ii", d->is_interface); + yyjson_mut_obj_add_int(doc, o, "lg", (int)d->lang); + add_str_or_null(doc, o, "ns", d->namespace_name); + add_str_or_null(doc, o, "tq", d->trait_qn); + yyjson_mut_obj_add_bool(doc, o, "ir", d->is_rust_impl_relation); + yyjson_mut_obj_add_bool(doc, o, "ab", d->is_abstract); + add_str_array_or_null(doc, o, "dec", d->decorators, -1); + yyjson_mut_arr_add_val(lsp, o); + } + yyjson_mut_obj_add_val(doc, root, "lsp", lsp); + + yyjson_mut_val *reg = yyjson_mut_arr(doc); + if (result) { + for (int i = 0; i < result->defs.count; i++) { + const CBMDefinition *d = &result->defs.items[i]; + if (!surface_reg_only_label(d->label) || !d->name || !d->qualified_name) { + continue; + } + yyjson_mut_val *o = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, o, "n", d->name); + yyjson_mut_obj_add_str(doc, o, "q", d->qualified_name); + yyjson_mut_obj_add_str(doc, o, "k", d->label); + yyjson_mut_arr_add_val(reg, o); + } + } + yyjson_mut_obj_add_val(doc, root, "reg", reg); + + char *json = yyjson_mut_write(doc, 0, NULL); + yyjson_mut_doc_free(doc); + return json; +} + +int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache, + const cbm_file_info_t *files, int file_count, + const CBMLSPDef *all_defs, const int *def_starts, + cbm_lsp_surface_row_t **out_rows, int *out_count) { + *out_rows = NULL; + *out_count = 0; + if (file_count <= 0) { + return 0; + } + cbm_lsp_surface_row_t *rows = calloc((size_t)file_count, sizeof(*rows)); + if (!rows) { + return -1; + } + int n = 0; + for (int i = 0; i < file_count; i++) { + if (!cache[i]) { + /* Never parsed this run (read/extract skip): no surface claim. + * The routing layer treats a missing row as "must full-rebuild + * before this file can be reasoned about", which is the correct + * fail-closed default for an unreadable file. */ + continue; + } + int start = def_starts ? def_starts[i] : 0; + int end = def_starts ? def_starts[i + 1] : 0; + char *json = + surface_file_to_json(cache[i], all_defs ? all_defs + start : NULL, end - start); + if (!json) { + cbm_store_free_lsp_surfaces(rows, n); + return -1; + } + char sha[CBM_SHA256_HEX_LEN + 1]; + cbm_sha256_hex(json, strlen(json), sha); + cbm_lsp_surface_row_t *r = &rows[n]; + r->project = strdup(project); + r->rel_path = strdup(files[i].rel_path); + r->surface_sha = strdup(sha); + r->defs_json = json; + r->ref_bloom = NULL; + r->ref_bloom_len = 0; + r->config_ctx = strdup(""); + if (!r->project || !r->rel_path || !r->surface_sha || !r->config_ctx) { + n++; + cbm_store_free_lsp_surfaces(rows, n); + return -1; + } + n++; + } + *out_rows = rows; + *out_count = n; + return 0; +} + +static const char *arena_str_or_null(CBMArena *arena, yyjson_val *v) { + if (!v || yyjson_is_null(v)) { + return NULL; + } + const char *s = yyjson_get_str(v); + return s ? cbm_arena_strdup(arena, s) : NULL; +} + +static const char **arena_str_array(CBMArena *arena, yyjson_val *arr, bool null_terminated, + int *out_count) { + *out_count = 0; + if (!arr || yyjson_is_null(arr) || !yyjson_is_arr(arr)) { + return NULL; + } + int count = (int)yyjson_arr_size(arr); + const char **items = + cbm_arena_alloc(arena, (size_t)(count + (null_terminated ? 1 : 0)) * sizeof(char *)); + if (!items) { + return NULL; + } + for (int i = 0; i < count; i++) { + const char *s = yyjson_get_str(yyjson_arr_get(arr, (size_t)i)); + items[i] = s ? cbm_arena_strdup(arena, s) : "?"; + } + if (null_terminated) { + items[count] = NULL; + } + *out_count = count; + return items; +} + +int cbm_lsp_surface_defs_from_json(CBMArena *arena, const char *defs_json, CBMLSPDef **out_defs) { + *out_defs = NULL; + if (!defs_json) { + return -1; + } + yyjson_doc *doc = yyjson_read(defs_json, strlen(defs_json), 0); + if (!doc) { + return -1; + } + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *ver = root ? yyjson_obj_get(root, "v") : NULL; + yyjson_val *lsp = root ? yyjson_obj_get(root, "lsp") : NULL; + if (!ver || yyjson_get_int(ver) != SURFACE_CODEC_VERSION || !lsp || !yyjson_is_arr(lsp)) { + yyjson_doc_free(doc); + return -1; + } + int count = (int)yyjson_arr_size(lsp); + if (count == 0) { + yyjson_doc_free(doc); + return 0; + } + CBMLSPDef *defs = cbm_arena_alloc(arena, (size_t)count * sizeof(CBMLSPDef)); + if (!defs) { + yyjson_doc_free(doc); + return -1; + } + memset(defs, 0, (size_t)count * sizeof(CBMLSPDef)); + for (int i = 0; i < count; i++) { + yyjson_val *o = yyjson_arr_get(lsp, (size_t)i); + CBMLSPDef *d = &defs[i]; + d->qualified_name = arena_str_or_null(arena, yyjson_obj_get(o, "qn")); + d->short_name = arena_str_or_null(arena, yyjson_obj_get(o, "sn")); + d->label = arena_str_or_null(arena, yyjson_obj_get(o, "lb")); + d->receiver_type = arena_str_or_null(arena, yyjson_obj_get(o, "rt")); + d->def_module_qn = arena_str_or_null(arena, yyjson_obj_get(o, "dm")); + d->return_types = arena_str_or_null(arena, yyjson_obj_get(o, "ret")); + d->embedded_types = arena_str_or_null(arena, yyjson_obj_get(o, "emb")); + d->field_defs = arena_str_or_null(arena, yyjson_obj_get(o, "fd")); + d->method_names_str = arena_str_or_null(arena, yyjson_obj_get(o, "mn")); + d->signature_param_types = + arena_str_array(arena, yyjson_obj_get(o, "spt"), false, &d->signature_param_count); + d->is_interface = yyjson_get_bool(yyjson_obj_get(o, "ii")); + d->lang = (CBMLanguage)yyjson_get_int(yyjson_obj_get(o, "lg")); + d->namespace_name = arena_str_or_null(arena, yyjson_obj_get(o, "ns")); + d->trait_qn = arena_str_or_null(arena, yyjson_obj_get(o, "tq")); + d->is_rust_impl_relation = yyjson_get_bool(yyjson_obj_get(o, "ir")); + d->is_abstract = yyjson_get_bool(yyjson_obj_get(o, "ab")); + int dec_count = 0; + d->decorators = arena_str_array(arena, yyjson_obj_get(o, "dec"), true, &dec_count); + if (!d->qualified_name || !d->short_name || !d->label) { + /* A def the writer could not have produced: qn/sn/label are + * unconditionally present in build_lsp_def output. Corrupt row. */ + yyjson_doc_free(doc); + return -1; + } + } + yyjson_doc_free(doc); + *out_defs = defs; + return count; +} diff --git a/src/pipeline/lsp_surface.h b/src/pipeline/lsp_surface.h new file mode 100644 index 000000000..a1d826158 --- /dev/null +++ b/src/pipeline/lsp_surface.h @@ -0,0 +1,43 @@ +/* + * lsp_surface.h — per-file LSP-surface codec (closure-repair incremental). + * + * A file's SURFACE is everything another file's resolution can consume from + * it: the CBMLSPDefs pass_lsp_cross registers into the cross registries, + * plus the registry-only symbols (labels the name registry serves that + * pxc_map_label drops — Field, today). Serialized to canonical JSON, hashed, + * and persisted per file (store: lsp_surface rows) at publication, so an + * incremental run can (a) detect that an edit left a file's surface + * unchanged — a body edit — and skip recomputing its dependents, and + * (b) rehydrate the cross registries for files it does not re-parse. + * + * The JSON is a versioned object {"v":1,"lsp":[...],"reg":[...]}. Field + * order and array order are fixed by the writer, so byte equality of the + * serialization is surface equality; the sha256 of these bytes is the + * stored surface_sha. Rows written by a different codec version fail the + * "v" check on load and route the run to a full rebuild. + */ +#ifndef CBM_PIPELINE_LSP_SURFACE_H +#define CBM_PIPELINE_LSP_SURFACE_H + +#include "pipeline/pass_lsp_cross.h" +#include "store/store.h" +#include "foundation/arena.h" + +/* Build one store row per file that produced surface content. `def_starts` + * is the (file_count + 1)-entry prefix array from cbm_pxc_collect_all_defs: + * all_defs[def_starts[i] .. def_starts[i+1]) are file i's defs. Rows carry + * heap strings; release with cbm_store_free_lsp_surfaces. Files with an + * empty surface still get a row (empty arrays hash too — "no defs" must be + * distinguishable from "no data"). Returns 0, or -1 on allocation failure. */ +int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache, + const cbm_file_info_t *files, int file_count, + const CBMLSPDef *all_defs, const int *def_starts, + cbm_lsp_surface_row_t **out_rows, int *out_count); + +/* Decode one row's defs_json back into the CBMLSPDef array pass_lsp_cross + * registration consumes. Strings are arena-allocated. Returns the def count, + * 0 for an empty surface, or -1 when the JSON is missing, malformed, or a + * different codec version — callers route that to a full rebuild. */ +int cbm_lsp_surface_defs_from_json(CBMArena *arena, const char *defs_json, CBMLSPDef **out_defs); + +#endif /* CBM_PIPELINE_LSP_SURFACE_H */ diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 30f9696d5..e6b4eea77 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -14,6 +14,8 @@ * multiple times if the pipeline gains a re-run path later. */ #include "pipeline/pass_lsp_cross.h" + +#include "pipeline/lsp_surface.h" #include "pipeline/pipeline_internal.h" #include "pipeline/lsp_resolve.h" #include "lsp/go_lsp.h" @@ -322,7 +324,7 @@ static int pxc_build_rust_impl_relation(CBMArena *arena, const CBMImplTrait *imp * borrowed from cache[i]->arena and from def_modules[i] (also borrowed). */ CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t *files, int file_count, const char *project_name, char **def_modules, - int *out_count) { + int *out_count, int *out_def_starts) { int total = 0; for (int i = 0; i < file_count; i++) { if (cache[i]) { @@ -334,15 +336,24 @@ CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t } if (total == 0) { *out_count = 0; + if (out_def_starts) { + memset(out_def_starts, 0, (size_t)(file_count + 1) * sizeof(int)); + } return NULL; } CBMLSPDef *defs = (CBMLSPDef *)calloc((size_t)total, sizeof(CBMLSPDef)); if (!defs) { *out_count = 0; + if (out_def_starts) { + memset(out_def_starts, 0, (size_t)(file_count + 1) * sizeof(int)); + } return NULL; } int idx = 0; for (int fi = 0; fi < file_count; fi++) { + if (out_def_starts) { + out_def_starts[fi] = idx; + } if (!cache[fi]) continue; if (!def_modules[fi]) { @@ -373,6 +384,9 @@ CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t } } } + if (out_def_starts) { + out_def_starts[file_count] = idx; + } *out_count = idx; return defs; } @@ -1186,8 +1200,21 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * } int def_count = 0; + int *def_starts = (int *)calloc((size_t)file_count + 1, sizeof(int)); CBMLSPDef *all_defs = cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name, - def_modules, &def_count); + def_modules, &def_count, def_starts); + /* Same seam as the parallel driver: serialize per-file surfaces while the + * result cache is alive. Failure only degrades to a full rebuild on the + * next incremental run. */ + if (ctx->pipeline && all_defs && def_starts) { + cbm_lsp_surface_row_t *surface_rows = NULL; + int surface_count = 0; + if (cbm_lsp_surface_build_rows(ctx->project_name, cache, files, file_count, all_defs, + def_starts, &surface_rows, &surface_count) == 0) { + cbm_pipeline_set_lsp_surfaces(ctx->pipeline, surface_rows, surface_count); + } + } + free(def_starts); /* Shared prepare (mirrors run_parallel_pipeline): inverted module-def * index + per-language shared registries, built ONCE for the whole pass. diff --git a/src/pipeline/pass_lsp_cross.h b/src/pipeline/pass_lsp_cross.h index c81eab1a9..11269ce62 100644 --- a/src/pipeline/pass_lsp_cross.h +++ b/src/pipeline/pass_lsp_cross.h @@ -49,10 +49,14 @@ bool cbm_pxc_has_cross_lsp(CBMLanguage lang); * def_modules[i] — caller must keep both alive while the array is in * use. Returns the malloc'd array (free() it) and writes the entry * count to *out_count. Returns NULL on alloc failure or when no defs - * exist. */ + * exist. out_def_starts (optional, file_count + 1 entries, caller-owned) + * receives per-file prefix offsets: file i's defs occupy + * [out_def_starts[i], out_def_starts[i+1]) — the LSP-surface serializer + * needs the per-file slices, which the flat array does not otherwise + * record. */ CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t *files, int file_count, const char *project_name, char **def_modules, - int *out_count); + int *out_count, int *out_def_starts); /* Detect TS dialect flags from a relative path. */ void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path, bool *out_js, bool *out_jsx, diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 71ed25a02..ecaa0af06 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -17,6 +17,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include "pipeline/pipeline.h" #include "pipeline/artifact.h" #include "pipeline/pipeline_internal.h" +#include "pipeline/lsp_surface.h" #include "pipeline/pass_lsp_cross.h" #include "pipeline/worker_pool.h" #include "graph_buffer/graph_buffer.h" @@ -192,6 +193,16 @@ struct cbm_pipeline { * can be restored after the rebuild. NULL when no ADR existed. Issue #516. */ char *saved_adr; + /* Per-file LSP surfaces serialized at the collect_all_defs seam (the only + * moment the result cache is alive), persisted by dump_and_persist_hashes + * so the closure-repair incremental route can early-cutoff on surface + * hashes and rehydrate cross registries without re-parsing. Heap rows, + * released with cbm_store_free_lsp_surfaces in cbm_pipeline_free. NULL + * when cross-LSP was disabled for the run — the incremental route then + * finds no rows and correctly falls back to a full rebuild. */ + cbm_lsp_surface_row_t *surface_rows; + int surface_row_count; + /* Deterministic test-only seam at the final publication boundary. Kept * per pipeline so concurrent test/process activity cannot cross-trigger. */ void (*before_publish_hook)(cbm_pipeline_t *, const char *, void *); @@ -312,6 +323,16 @@ bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) { return true; } +void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *rows, int count) { + if (!p) { + cbm_store_free_lsp_surfaces(rows, count); + return; + } + cbm_store_free_lsp_surfaces(p->surface_rows, p->surface_row_count); + p->surface_rows = rows; + p->surface_row_count = count; +} + void cbm_pipeline_free(cbm_pipeline_t *p) { if (!p) { return; @@ -339,6 +360,9 @@ void cbm_pipeline_free(cbm_pipeline_t *p) { free(p->saved_adr); /* freed here too: error paths can exit before the * restore in dump_and_persist_hashes runs. Issue #516. */ p->saved_adr = NULL; + cbm_store_free_lsp_surfaces(p->surface_rows, p->surface_row_count); + p->surface_rows = NULL; + p->surface_row_count = 0; cbm_git_context_free(&p->git_ctx); /* gbuf, store, registry freed during/after run */ /* Defensively free userconfig in case run() was never called or panicked */ @@ -1194,13 +1218,30 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, char **def_modules = NULL; int def_count = 0; CBMLSPDef *all_defs = NULL; + int *def_starts = NULL; if (run_cross_lsp) { def_modules = (char **)calloc((size_t)file_count, sizeof(char *)); + def_starts = (int *)calloc((size_t)file_count + 1, sizeof(int)); all_defs = def_modules ? cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name, - def_modules, &def_count) + def_modules, &def_count, def_starts) : NULL; } + /* Serialize per-file LSP surfaces NOW — the result cache dies with this + * pass, and the rows are what lets an incremental run detect body-only + * edits and rehydrate cross registries without re-parsing the world. + * Failure only degrades: no rows → the incremental route full-rebuilds. */ + if (ctx->pipeline && all_defs && def_starts) { + cbm_lsp_surface_row_t *surface_rows = NULL; + int surface_count = 0; + if (cbm_lsp_surface_build_rows(ctx->project_name, cache, files, file_count, all_defs, + def_starts, &surface_rows, &surface_count) == 0) { + cbm_pipeline_set_lsp_surfaces(ctx->pipeline, surface_rows, surface_count); + } else { + cbm_log_warn("lsp_surface.serialize_failed", "files", itoa_buf(file_count)); + } + } + free(def_starts); /* Build inverted index: module_qn → defs. The fused resolve_worker * uses this to filter the global all_defs[] down to just the defs * each file actually needs (own_module + imported modules) — the @@ -1644,6 +1685,15 @@ int cbm_pipeline_publish_generation(const cbm_pipeline_generation_t *generation) ok = ok && cbm_store_delete_file_hashes(store, generation->project) == CBM_STORE_OK && cbm_store_upsert_file_hash_batch(store, generation->manifest, generation->manifest_count) == CBM_STORE_OK; + /* LSP surfaces belong to the generation: written inside the same staging + * store, before the atomic rename, so graph and surface data can never + * publish separately. The delete guards the incremental path, whose + * staging DB starts as a copy of the previous generation. */ + if (ok) { + ok = cbm_store_delete_lsp_surfaces(store, generation->project) == CBM_STORE_OK && + cbm_store_upsert_lsp_surface_batch(store, generation->surface_rows, + generation->surface_row_count) == CBM_STORE_OK; + } if (ok && generation->adr_content) { ok = cbm_store_adr_store(store, generation->project, generation->adr_content) == CBM_STORE_OK; @@ -1810,6 +1860,8 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas .coverage_version = CBM_SEMANTIC_INDEX_VERSION, .hash_records_complete = true, }, + .surface_rows = p->surface_rows, + .surface_row_count = p->surface_row_count, }; free(db_dir); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 38818ea99..2a9add396 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -696,6 +696,12 @@ typedef struct { const cbm_coverage_row_t *coverage; int coverage_count; cbm_coverage_meta_t coverage_meta; + /* Per-file LSP surfaces for the generation being published (may be NULL: + * cross-LSP disabled, or a caller that has none). Written into the + * staging store alongside the manifest so surface data and graph always + * belong to the same generation. */ + const cbm_lsp_surface_row_t *surface_rows; + int surface_row_count; } cbm_pipeline_generation_t; /* Serialize and fully populate a sibling staging database, then atomically @@ -707,6 +713,12 @@ int cbm_pipeline_publish_generation(const cbm_pipeline_generation_t *generation) * automatic refresh of an already-existing artifact remains best-effort. */ int cbm_pipeline_refresh_artifact(cbm_pipeline_t *p, const char *db_path); +/* Hand the pipeline the per-file LSP-surface rows serialized at the + * collect_all_defs seam (the only moment the result cache is alive). + * Takes ownership; dump_and_persist_hashes writes them into the staging + * store and cbm_pipeline_free releases them. Passing NULL/0 clears. */ +void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *rows, int count); + /* Pipeline accessors for incremental use */ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p); diff --git a/tests/test_parallel.c b/tests/test_parallel.c index 6f7a2ecc1..c33d26690 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -323,10 +323,10 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator( * cbm_pxc_run_one(_ts) per file BEFORE materializing CALLS edges. */ char **def_modules = (char **)calloc((size_t)file_count, sizeof(char *)); int def_count = 0; - CBMLSPDef *all_defs = def_modules - ? cbm_pxc_collect_all_defs(result_cache, files, file_count, - ctx.project_name, def_modules, &def_count) - : NULL; + CBMLSPDef *all_defs = + def_modules ? cbm_pxc_collect_all_defs(result_cache, files, file_count, ctx.project_name, + def_modules, &def_count, NULL) + : NULL; CBMModuleDefIndex *module_def_index = all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL; diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index b9f44f113..b16cccb2b 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -11186,7 +11186,79 @@ TEST(pipeline_seq_ts_cross_uses_shared_registry) { PASS(); } +/* The closure-repair route lives and dies by two properties of the persisted + * per-file LSP surface: a BODY edit must leave the surface_sha unchanged (the + * early cutoff -- no dependent recomputation owed), while a SIGNATURE edit + * must change it (dependents owe re-resolution). Each run here is a fresh + * full index (DB removed in between) so what is compared is the published + * surface of the code as it stands, not any incremental behaviour. */ +TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_surface_persist_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char src_a[512]; + char db_path[512]; + snprintf(src_a, sizeof(src_a), "%s/a.py", tmp); + snprintf(db_path, sizeof(db_path), "%s/surface.db", tmp); + ASSERT_EQ(th_write_file(src_a, "def surface_probe(x):\n return 1\n"), 0); + + char sha_baseline[128] = {0}; + char json_probe[64] = {0}; + char project[256] = {0}; + + for (int round = 0; round < 3; round++) { + if (round == 1) { + /* Body edit: same def surface, different body. */ + ASSERT_EQ(th_write_file(src_a, "def surface_probe(x):\n return 2 + x\n"), 0); + } else if (round == 2) { + /* Signature edit: the surface other files consume changes. */ + ASSERT_EQ(th_write_file(src_a, "def surface_probe(x, y):\n return 1\n"), 0); + } + cbm_unlink(db_path); + cbm_remove_db_sidecars(db_path); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + if (round == 0) { + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p)); + } + cbm_pipeline_free(p); + + cbm_store_t *store = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(store); + cbm_lsp_surface_row_t *rows = NULL; + int row_count = 0; + ASSERT_EQ(cbm_store_get_lsp_surfaces(store, project, &rows, &row_count), CBM_STORE_OK); + cbm_store_close(store); + + const cbm_lsp_surface_row_t *row_a = NULL; + for (int i = 0; i < row_count; i++) { + if (strcmp(rows[i].rel_path, "a.py") == 0) { + row_a = &rows[i]; + } + } + ASSERT_NOT_NULL(row_a); + ASSERT_TRUE(row_a->surface_sha && row_a->surface_sha[0]); + if (round == 0) { + snprintf(sha_baseline, sizeof(sha_baseline), "%s", row_a->surface_sha); + /* The row is the versioned codec envelope with the def present. */ + snprintf(json_probe, sizeof(json_probe), "%.20s", row_a->defs_json); + ASSERT_TRUE(strstr(row_a->defs_json, "\"v\":1") != NULL); + ASSERT_TRUE(strstr(row_a->defs_json, "surface_probe") != NULL); + } else if (round == 1) { + ASSERT_STR_EQ(row_a->surface_sha, sha_baseline); + } else { + ASSERT_TRUE(strcmp(row_a->surface_sha, sha_baseline) != 0); + } + cbm_store_free_lsp_surfaces(rows, row_count); + } + (void)json_probe; + th_rmtree(tmp); + PASS(); +} + SUITE(pipeline) { + RUN_TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant); /* Index lock */ RUN_TEST(pipeline_lock_try_acquire); RUN_TEST(pipeline_lock_blocking);