mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
feat(pipeline): serialize and publish per-file LSP surfaces
Second piece of closure repair. At the collect_all_defs seam -- the only moment the per-file result cache is alive -- both drivers (parallel and sequential) now serialize each file's CBMLSPDef slice to canonical JSON, hash it, and hand the rows to the pipeline; cbm_pipeline_publish_generation writes them into the staging store next to the manifest, so surface data and graph always belong to the same generation. Canonical bytes are the point: every field is written in fixed order with an explicit null for absent strings (NULL and "" differ in the CBMLSPDef contract -- receiver_type NULL means "not a method"), so byte equality IS surface equality and the sha over the bytes is the early-cutoff key. Registry-only labels that pxc_map_label drops but the name registry serves (Field) are folded into the hash as a separate "reg" array, or renaming one would slip past the cutoff. Behaviour pinned in SUITE(pipeline): a fresh full index persists a versioned surface row per file; a BODY edit republishes the identical surface_sha; a SIGNATURE edit changes it. That pair of properties is what the routing layer will stand on. cbm_pxc_collect_all_defs gains an optional per-file prefix array -- the flat all_defs[] otherwise loses the file boundaries the serializer needs. CORRECTION to 6c22338's scope note: it claimed cbm_pipeline_publish_ generation was reachable only behind CBM_INCREMENTAL_TEST_API. Wrong -- dump_and_persist_hashes calls it on every production full index (pipeline.c:1863); the grep that "verified" test-only reachability had excluded pipeline.c itself. The predictable staging name WAS in the production publish path, which makes that fix a real production hardening, not a test-path cleanup. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
1 parent
8d3685bfca
commit
94189e6990
9 files changed
+489
-9
No files matched your search
@@ -302,6 +302,7 @@ GRAPH_BUFFER_SRCS = src/graph_buffer/graph_buffer.c
|
||||
# Pipeline module (new)
|
||||
PIPELINE_SRCS = \
|
||||
src/pipeline/fqn.c \
|
||||
src/pipeline/lsp_surface.c \
|
||||
src/pipeline/path_alias.c \
|
||||
src/pipeline/registry.c \
|
||||
src/pipeline/pipeline.c \
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
/*
|
||||
* lsp_surface.c — per-file LSP-surface codec (closure-repair incremental).
|
||||
*
|
||||
* See lsp_surface.h for the contract. Two invariants carry the feature:
|
||||
*
|
||||
* 1. ROUND-TRIP FIDELITY: defs_from_json(build_json(defs)) must hand the
|
||||
* per-language registrars the same values collect_all_defs would have
|
||||
* built from a real parse — a lossy field here silently degrades
|
||||
* cross-file resolution only on the incremental path, the exact class
|
||||
* of divergence this feature exists to eliminate.
|
||||
*
|
||||
* 2. CANONICAL BYTES: every field is written, in fixed order, with an
|
||||
* explicit JSON null for absent strings (NULL and "" are different
|
||||
* values in the CBMLSPDef contract — receiver_type NULL means "not a
|
||||
* method", and several registrars branch on that). Byte equality of the
|
||||
* serialization therefore IS surface equality, and the sha over the
|
||||
* bytes is the early-cutoff key: a body edit reserializes identically.
|
||||
*/
|
||||
#include "pipeline/lsp_surface.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/sha256.h"
|
||||
#include "yyjson/yyjson.h"
|
||||
|
||||
enum { SURFACE_CODEC_VERSION = 1 };
|
||||
|
||||
/* Labels the incremental name registry serves that pxc_map_label does NOT
|
||||
* carry into the CBMLSPDef set. Their (name, qn, label) triple must still
|
||||
* participate in the surface hash, or renaming one would slip past the
|
||||
* early cutoff while stale references to it survive in dependent files.
|
||||
* KEEP IN SYNC with pxc_map_label (pass_lsp_cross.c) and
|
||||
* incr_label_is_registry_symbol (pipeline_incremental.c); the codec unit
|
||||
* test cross-checks the three. */
|
||||
static bool surface_reg_only_label(const char *label) {
|
||||
return label && strcmp(label, "Field") == 0;
|
||||
}
|
||||
|
||||
static void add_str_or_null(yyjson_mut_doc *doc, yyjson_mut_val *obj, const char *key,
|
||||
const char *val) {
|
||||
if (val) {
|
||||
yyjson_mut_obj_add_str(doc, obj, key, val);
|
||||
} else {
|
||||
yyjson_mut_obj_add_null(doc, obj, key);
|
||||
}
|
||||
}
|
||||
|
||||
static void add_str_array_or_null(yyjson_mut_doc *doc, yyjson_mut_val *obj, const char *key,
|
||||
const char **items, int count_or_neg1_terminated) {
|
||||
if (!items) {
|
||||
yyjson_mut_obj_add_null(doc, obj, key);
|
||||
return;
|
||||
}
|
||||
yyjson_mut_val *arr = yyjson_mut_arr(doc);
|
||||
if (count_or_neg1_terminated >= 0) {
|
||||
for (int i = 0; i < count_or_neg1_terminated; i++) {
|
||||
yyjson_mut_arr_add_str(doc, arr, items[i] ? items[i] : "?");
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; items[i]; i++) {
|
||||
yyjson_mut_arr_add_str(doc, arr, items[i]);
|
||||
}
|
||||
}
|
||||
yyjson_mut_obj_add_val(doc, obj, key, arr);
|
||||
}
|
||||
|
||||
/* Serialize one file's surface: its slice of all_defs plus the registry-only
|
||||
* symbols from its raw extraction defs. Returns a malloc'd JSON string. */
|
||||
static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef *defs,
|
||||
int def_count) {
|
||||
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
|
||||
if (!doc) {
|
||||
return NULL;
|
||||
}
|
||||
yyjson_mut_val *root = yyjson_mut_obj(doc);
|
||||
yyjson_mut_doc_set_root(doc, root);
|
||||
yyjson_mut_obj_add_int(doc, root, "v", SURFACE_CODEC_VERSION);
|
||||
|
||||
yyjson_mut_val *lsp = yyjson_mut_arr(doc);
|
||||
for (int i = 0; i < def_count; i++) {
|
||||
const CBMLSPDef *d = &defs[i];
|
||||
yyjson_mut_val *o = yyjson_mut_obj(doc);
|
||||
add_str_or_null(doc, o, "qn", d->qualified_name);
|
||||
add_str_or_null(doc, o, "sn", d->short_name);
|
||||
add_str_or_null(doc, o, "lb", d->label);
|
||||
add_str_or_null(doc, o, "rt", d->receiver_type);
|
||||
add_str_or_null(doc, o, "dm", d->def_module_qn);
|
||||
add_str_or_null(doc, o, "ret", d->return_types);
|
||||
add_str_or_null(doc, o, "emb", d->embedded_types);
|
||||
add_str_or_null(doc, o, "fd", d->field_defs);
|
||||
add_str_or_null(doc, o, "mn", d->method_names_str);
|
||||
add_str_array_or_null(doc, o, "spt", d->signature_param_types, d->signature_param_count);
|
||||
yyjson_mut_obj_add_bool(doc, o, "ii", d->is_interface);
|
||||
yyjson_mut_obj_add_int(doc, o, "lg", (int)d->lang);
|
||||
add_str_or_null(doc, o, "ns", d->namespace_name);
|
||||
add_str_or_null(doc, o, "tq", d->trait_qn);
|
||||
yyjson_mut_obj_add_bool(doc, o, "ir", d->is_rust_impl_relation);
|
||||
yyjson_mut_obj_add_bool(doc, o, "ab", d->is_abstract);
|
||||
add_str_array_or_null(doc, o, "dec", d->decorators, -1);
|
||||
yyjson_mut_arr_add_val(lsp, o);
|
||||
}
|
||||
yyjson_mut_obj_add_val(doc, root, "lsp", lsp);
|
||||
|
||||
yyjson_mut_val *reg = yyjson_mut_arr(doc);
|
||||
if (result) {
|
||||
for (int i = 0; i < result->defs.count; i++) {
|
||||
const CBMDefinition *d = &result->defs.items[i];
|
||||
if (!surface_reg_only_label(d->label) || !d->name || !d->qualified_name) {
|
||||
continue;
|
||||
}
|
||||
yyjson_mut_val *o = yyjson_mut_obj(doc);
|
||||
yyjson_mut_obj_add_str(doc, o, "n", d->name);
|
||||
yyjson_mut_obj_add_str(doc, o, "q", d->qualified_name);
|
||||
yyjson_mut_obj_add_str(doc, o, "k", d->label);
|
||||
yyjson_mut_arr_add_val(reg, o);
|
||||
}
|
||||
}
|
||||
yyjson_mut_obj_add_val(doc, root, "reg", reg);
|
||||
|
||||
char *json = yyjson_mut_write(doc, 0, NULL);
|
||||
yyjson_mut_doc_free(doc);
|
||||
return json;
|
||||
}
|
||||
|
||||
int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache,
|
||||
const cbm_file_info_t *files, int file_count,
|
||||
const CBMLSPDef *all_defs, const int *def_starts,
|
||||
cbm_lsp_surface_row_t **out_rows, int *out_count) {
|
||||
*out_rows = NULL;
|
||||
*out_count = 0;
|
||||
if (file_count <= 0) {
|
||||
return 0;
|
||||
}
|
||||
cbm_lsp_surface_row_t *rows = calloc((size_t)file_count, sizeof(*rows));
|
||||
if (!rows) {
|
||||
return -1;
|
||||
}
|
||||
int n = 0;
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
if (!cache[i]) {
|
||||
/* Never parsed this run (read/extract skip): no surface claim.
|
||||
* The routing layer treats a missing row as "must full-rebuild
|
||||
* before this file can be reasoned about", which is the correct
|
||||
* fail-closed default for an unreadable file. */
|
||||
continue;
|
||||
}
|
||||
int start = def_starts ? def_starts[i] : 0;
|
||||
int end = def_starts ? def_starts[i + 1] : 0;
|
||||
char *json =
|
||||
surface_file_to_json(cache[i], all_defs ? all_defs + start : NULL, end - start);
|
||||
if (!json) {
|
||||
cbm_store_free_lsp_surfaces(rows, n);
|
||||
return -1;
|
||||
}
|
||||
char sha[CBM_SHA256_HEX_LEN + 1];
|
||||
cbm_sha256_hex(json, strlen(json), sha);
|
||||
cbm_lsp_surface_row_t *r = &rows[n];
|
||||
r->project = strdup(project);
|
||||
r->rel_path = strdup(files[i].rel_path);
|
||||
r->surface_sha = strdup(sha);
|
||||
r->defs_json = json;
|
||||
r->ref_bloom = NULL;
|
||||
r->ref_bloom_len = 0;
|
||||
r->config_ctx = strdup("");
|
||||
if (!r->project || !r->rel_path || !r->surface_sha || !r->config_ctx) {
|
||||
n++;
|
||||
cbm_store_free_lsp_surfaces(rows, n);
|
||||
return -1;
|
||||
}
|
||||
n++;
|
||||
}
|
||||
*out_rows = rows;
|
||||
*out_count = n;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const char *arena_str_or_null(CBMArena *arena, yyjson_val *v) {
|
||||
if (!v || yyjson_is_null(v)) {
|
||||
return NULL;
|
||||
}
|
||||
const char *s = yyjson_get_str(v);
|
||||
return s ? cbm_arena_strdup(arena, s) : NULL;
|
||||
}
|
||||
|
||||
static const char **arena_str_array(CBMArena *arena, yyjson_val *arr, bool null_terminated,
|
||||
int *out_count) {
|
||||
*out_count = 0;
|
||||
if (!arr || yyjson_is_null(arr) || !yyjson_is_arr(arr)) {
|
||||
return NULL;
|
||||
}
|
||||
int count = (int)yyjson_arr_size(arr);
|
||||
const char **items =
|
||||
cbm_arena_alloc(arena, (size_t)(count + (null_terminated ? 1 : 0)) * sizeof(char *));
|
||||
if (!items) {
|
||||
return NULL;
|
||||
}
|
||||
for (int i = 0; i < count; i++) {
|
||||
const char *s = yyjson_get_str(yyjson_arr_get(arr, (size_t)i));
|
||||
items[i] = s ? cbm_arena_strdup(arena, s) : "?";
|
||||
}
|
||||
if (null_terminated) {
|
||||
items[count] = NULL;
|
||||
}
|
||||
*out_count = count;
|
||||
return items;
|
||||
}
|
||||
|
||||
int cbm_lsp_surface_defs_from_json(CBMArena *arena, const char *defs_json, CBMLSPDef **out_defs) {
|
||||
*out_defs = NULL;
|
||||
if (!defs_json) {
|
||||
return -1;
|
||||
}
|
||||
yyjson_doc *doc = yyjson_read(defs_json, strlen(defs_json), 0);
|
||||
if (!doc) {
|
||||
return -1;
|
||||
}
|
||||
yyjson_val *root = yyjson_doc_get_root(doc);
|
||||
yyjson_val *ver = root ? yyjson_obj_get(root, "v") : NULL;
|
||||
yyjson_val *lsp = root ? yyjson_obj_get(root, "lsp") : NULL;
|
||||
if (!ver || yyjson_get_int(ver) != SURFACE_CODEC_VERSION || !lsp || !yyjson_is_arr(lsp)) {
|
||||
yyjson_doc_free(doc);
|
||||
return -1;
|
||||
}
|
||||
int count = (int)yyjson_arr_size(lsp);
|
||||
if (count == 0) {
|
||||
yyjson_doc_free(doc);
|
||||
return 0;
|
||||
}
|
||||
CBMLSPDef *defs = cbm_arena_alloc(arena, (size_t)count * sizeof(CBMLSPDef));
|
||||
if (!defs) {
|
||||
yyjson_doc_free(doc);
|
||||
return -1;
|
||||
}
|
||||
memset(defs, 0, (size_t)count * sizeof(CBMLSPDef));
|
||||
for (int i = 0; i < count; i++) {
|
||||
yyjson_val *o = yyjson_arr_get(lsp, (size_t)i);
|
||||
CBMLSPDef *d = &defs[i];
|
||||
d->qualified_name = arena_str_or_null(arena, yyjson_obj_get(o, "qn"));
|
||||
d->short_name = arena_str_or_null(arena, yyjson_obj_get(o, "sn"));
|
||||
d->label = arena_str_or_null(arena, yyjson_obj_get(o, "lb"));
|
||||
d->receiver_type = arena_str_or_null(arena, yyjson_obj_get(o, "rt"));
|
||||
d->def_module_qn = arena_str_or_null(arena, yyjson_obj_get(o, "dm"));
|
||||
d->return_types = arena_str_or_null(arena, yyjson_obj_get(o, "ret"));
|
||||
d->embedded_types = arena_str_or_null(arena, yyjson_obj_get(o, "emb"));
|
||||
d->field_defs = arena_str_or_null(arena, yyjson_obj_get(o, "fd"));
|
||||
d->method_names_str = arena_str_or_null(arena, yyjson_obj_get(o, "mn"));
|
||||
d->signature_param_types =
|
||||
arena_str_array(arena, yyjson_obj_get(o, "spt"), false, &d->signature_param_count);
|
||||
d->is_interface = yyjson_get_bool(yyjson_obj_get(o, "ii"));
|
||||
d->lang = (CBMLanguage)yyjson_get_int(yyjson_obj_get(o, "lg"));
|
||||
d->namespace_name = arena_str_or_null(arena, yyjson_obj_get(o, "ns"));
|
||||
d->trait_qn = arena_str_or_null(arena, yyjson_obj_get(o, "tq"));
|
||||
d->is_rust_impl_relation = yyjson_get_bool(yyjson_obj_get(o, "ir"));
|
||||
d->is_abstract = yyjson_get_bool(yyjson_obj_get(o, "ab"));
|
||||
int dec_count = 0;
|
||||
d->decorators = arena_str_array(arena, yyjson_obj_get(o, "dec"), true, &dec_count);
|
||||
if (!d->qualified_name || !d->short_name || !d->label) {
|
||||
/* A def the writer could not have produced: qn/sn/label are
|
||||
* unconditionally present in build_lsp_def output. Corrupt row. */
|
||||
yyjson_doc_free(doc);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
yyjson_doc_free(doc);
|
||||
*out_defs = defs;
|
||||
return count;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* lsp_surface.h — per-file LSP-surface codec (closure-repair incremental).
|
||||
*
|
||||
* A file's SURFACE is everything another file's resolution can consume from
|
||||
* it: the CBMLSPDefs pass_lsp_cross registers into the cross registries,
|
||||
* plus the registry-only symbols (labels the name registry serves that
|
||||
* pxc_map_label drops — Field, today). Serialized to canonical JSON, hashed,
|
||||
* and persisted per file (store: lsp_surface rows) at publication, so an
|
||||
* incremental run can (a) detect that an edit left a file's surface
|
||||
* unchanged — a body edit — and skip recomputing its dependents, and
|
||||
* (b) rehydrate the cross registries for files it does not re-parse.
|
||||
*
|
||||
* The JSON is a versioned object {"v":1,"lsp":[...],"reg":[...]}. Field
|
||||
* order and array order are fixed by the writer, so byte equality of the
|
||||
* serialization is surface equality; the sha256 of these bytes is the
|
||||
* stored surface_sha. Rows written by a different codec version fail the
|
||||
* "v" check on load and route the run to a full rebuild.
|
||||
*/
|
||||
#ifndef CBM_PIPELINE_LSP_SURFACE_H
|
||||
#define CBM_PIPELINE_LSP_SURFACE_H
|
||||
|
||||
#include "pipeline/pass_lsp_cross.h"
|
||||
#include "store/store.h"
|
||||
#include "foundation/arena.h"
|
||||
|
||||
/* Build one store row per file that produced surface content. `def_starts`
|
||||
* is the (file_count + 1)-entry prefix array from cbm_pxc_collect_all_defs:
|
||||
* all_defs[def_starts[i] .. def_starts[i+1]) are file i's defs. Rows carry
|
||||
* heap strings; release with cbm_store_free_lsp_surfaces. Files with an
|
||||
* empty surface still get a row (empty arrays hash too — "no defs" must be
|
||||
* distinguishable from "no data"). Returns 0, or -1 on allocation failure. */
|
||||
int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache,
|
||||
const cbm_file_info_t *files, int file_count,
|
||||
const CBMLSPDef *all_defs, const int *def_starts,
|
||||
cbm_lsp_surface_row_t **out_rows, int *out_count);
|
||||
|
||||
/* Decode one row's defs_json back into the CBMLSPDef array pass_lsp_cross
|
||||
* registration consumes. Strings are arena-allocated. Returns the def count,
|
||||
* 0 for an empty surface, or -1 when the JSON is missing, malformed, or a
|
||||
* different codec version — callers route that to a full rebuild. */
|
||||
int cbm_lsp_surface_defs_from_json(CBMArena *arena, const char *defs_json, CBMLSPDef **out_defs);
|
||||
|
||||
#endif /* CBM_PIPELINE_LSP_SURFACE_H */
|
||||
@@ -14,6 +14,8 @@
|
||||
* multiple times if the pipeline gains a re-run path later.
|
||||
*/
|
||||
#include "pipeline/pass_lsp_cross.h"
|
||||
|
||||
#include "pipeline/lsp_surface.h"
|
||||
#include "pipeline/pipeline_internal.h"
|
||||
#include "pipeline/lsp_resolve.h"
|
||||
#include "lsp/go_lsp.h"
|
||||
@@ -322,7 +324,7 @@ static int pxc_build_rust_impl_relation(CBMArena *arena, const CBMImplTrait *imp
|
||||
* borrowed from cache[i]->arena and from def_modules[i] (also borrowed). */
|
||||
CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t *files,
|
||||
int file_count, const char *project_name, char **def_modules,
|
||||
int *out_count) {
|
||||
int *out_count, int *out_def_starts) {
|
||||
int total = 0;
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
if (cache[i]) {
|
||||
@@ -334,15 +336,24 @@ CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t
|
||||
}
|
||||
if (total == 0) {
|
||||
*out_count = 0;
|
||||
if (out_def_starts) {
|
||||
memset(out_def_starts, 0, (size_t)(file_count + 1) * sizeof(int));
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
CBMLSPDef *defs = (CBMLSPDef *)calloc((size_t)total, sizeof(CBMLSPDef));
|
||||
if (!defs) {
|
||||
*out_count = 0;
|
||||
if (out_def_starts) {
|
||||
memset(out_def_starts, 0, (size_t)(file_count + 1) * sizeof(int));
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
int idx = 0;
|
||||
for (int fi = 0; fi < file_count; fi++) {
|
||||
if (out_def_starts) {
|
||||
out_def_starts[fi] = idx;
|
||||
}
|
||||
if (!cache[fi])
|
||||
continue;
|
||||
if (!def_modules[fi]) {
|
||||
@@ -373,6 +384,9 @@ CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t
|
||||
}
|
||||
}
|
||||
}
|
||||
if (out_def_starts) {
|
||||
out_def_starts[file_count] = idx;
|
||||
}
|
||||
*out_count = idx;
|
||||
return defs;
|
||||
}
|
||||
@@ -1186,8 +1200,21 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *
|
||||
}
|
||||
|
||||
int def_count = 0;
|
||||
int *def_starts = (int *)calloc((size_t)file_count + 1, sizeof(int));
|
||||
CBMLSPDef *all_defs = cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name,
|
||||
def_modules, &def_count);
|
||||
def_modules, &def_count, def_starts);
|
||||
/* Same seam as the parallel driver: serialize per-file surfaces while the
|
||||
* result cache is alive. Failure only degrades to a full rebuild on the
|
||||
* next incremental run. */
|
||||
if (ctx->pipeline && all_defs && def_starts) {
|
||||
cbm_lsp_surface_row_t *surface_rows = NULL;
|
||||
int surface_count = 0;
|
||||
if (cbm_lsp_surface_build_rows(ctx->project_name, cache, files, file_count, all_defs,
|
||||
def_starts, &surface_rows, &surface_count) == 0) {
|
||||
cbm_pipeline_set_lsp_surfaces(ctx->pipeline, surface_rows, surface_count);
|
||||
}
|
||||
}
|
||||
free(def_starts);
|
||||
|
||||
/* Shared prepare (mirrors run_parallel_pipeline): inverted module-def
|
||||
* index + per-language shared registries, built ONCE for the whole pass.
|
||||
|
||||
@@ -49,10 +49,14 @@ bool cbm_pxc_has_cross_lsp(CBMLanguage lang);
|
||||
* def_modules[i] — caller must keep both alive while the array is in
|
||||
* use. Returns the malloc'd array (free() it) and writes the entry
|
||||
* count to *out_count. Returns NULL on alloc failure or when no defs
|
||||
* exist. */
|
||||
* exist. out_def_starts (optional, file_count + 1 entries, caller-owned)
|
||||
* receives per-file prefix offsets: file i's defs occupy
|
||||
* [out_def_starts[i], out_def_starts[i+1]) — the LSP-surface serializer
|
||||
* needs the per-file slices, which the flat array does not otherwise
|
||||
* record. */
|
||||
CBMLSPDef *cbm_pxc_collect_all_defs(CBMFileResult **cache, const cbm_file_info_t *files,
|
||||
int file_count, const char *project_name, char **def_modules,
|
||||
int *out_count);
|
||||
int *out_count, int *out_def_starts);
|
||||
|
||||
/* Detect TS dialect flags from a relative path. */
|
||||
void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path, bool *out_js, bool *out_jsx,
|
||||
|
||||
+53
-1
@@ -17,6 +17,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 };
|
||||
#include "pipeline/pipeline.h"
|
||||
#include "pipeline/artifact.h"
|
||||
#include "pipeline/pipeline_internal.h"
|
||||
#include "pipeline/lsp_surface.h"
|
||||
#include "pipeline/pass_lsp_cross.h"
|
||||
#include "pipeline/worker_pool.h"
|
||||
#include "graph_buffer/graph_buffer.h"
|
||||
@@ -192,6 +193,16 @@ struct cbm_pipeline {
|
||||
* can be restored after the rebuild. NULL when no ADR existed. Issue #516. */
|
||||
char *saved_adr;
|
||||
|
||||
/* Per-file LSP surfaces serialized at the collect_all_defs seam (the only
|
||||
* moment the result cache is alive), persisted by dump_and_persist_hashes
|
||||
* so the closure-repair incremental route can early-cutoff on surface
|
||||
* hashes and rehydrate cross registries without re-parsing. Heap rows,
|
||||
* released with cbm_store_free_lsp_surfaces in cbm_pipeline_free. NULL
|
||||
* when cross-LSP was disabled for the run — the incremental route then
|
||||
* finds no rows and correctly falls back to a full rebuild. */
|
||||
cbm_lsp_surface_row_t *surface_rows;
|
||||
int surface_row_count;
|
||||
|
||||
/* Deterministic test-only seam at the final publication boundary. Kept
|
||||
* per pipeline so concurrent test/process activity cannot cross-trigger. */
|
||||
void (*before_publish_hook)(cbm_pipeline_t *, const char *, void *);
|
||||
@@ -312,6 +323,16 @@ bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) {
|
||||
return true;
|
||||
}
|
||||
|
||||
void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *rows, int count) {
|
||||
if (!p) {
|
||||
cbm_store_free_lsp_surfaces(rows, count);
|
||||
return;
|
||||
}
|
||||
cbm_store_free_lsp_surfaces(p->surface_rows, p->surface_row_count);
|
||||
p->surface_rows = rows;
|
||||
p->surface_row_count = count;
|
||||
}
|
||||
|
||||
void cbm_pipeline_free(cbm_pipeline_t *p) {
|
||||
if (!p) {
|
||||
return;
|
||||
@@ -339,6 +360,9 @@ void cbm_pipeline_free(cbm_pipeline_t *p) {
|
||||
free(p->saved_adr); /* freed here too: error paths can exit before the
|
||||
* restore in dump_and_persist_hashes runs. Issue #516. */
|
||||
p->saved_adr = NULL;
|
||||
cbm_store_free_lsp_surfaces(p->surface_rows, p->surface_row_count);
|
||||
p->surface_rows = NULL;
|
||||
p->surface_row_count = 0;
|
||||
cbm_git_context_free(&p->git_ctx);
|
||||
/* gbuf, store, registry freed during/after run */
|
||||
/* Defensively free userconfig in case run() was never called or panicked */
|
||||
@@ -1194,13 +1218,30 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
|
||||
char **def_modules = NULL;
|
||||
int def_count = 0;
|
||||
CBMLSPDef *all_defs = NULL;
|
||||
int *def_starts = NULL;
|
||||
if (run_cross_lsp) {
|
||||
def_modules = (char **)calloc((size_t)file_count, sizeof(char *));
|
||||
def_starts = (int *)calloc((size_t)file_count + 1, sizeof(int));
|
||||
all_defs = def_modules
|
||||
? cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name,
|
||||
def_modules, &def_count)
|
||||
def_modules, &def_count, def_starts)
|
||||
: NULL;
|
||||
}
|
||||
/* Serialize per-file LSP surfaces NOW — the result cache dies with this
|
||||
* pass, and the rows are what lets an incremental run detect body-only
|
||||
* edits and rehydrate cross registries without re-parsing the world.
|
||||
* Failure only degrades: no rows → the incremental route full-rebuilds. */
|
||||
if (ctx->pipeline && all_defs && def_starts) {
|
||||
cbm_lsp_surface_row_t *surface_rows = NULL;
|
||||
int surface_count = 0;
|
||||
if (cbm_lsp_surface_build_rows(ctx->project_name, cache, files, file_count, all_defs,
|
||||
def_starts, &surface_rows, &surface_count) == 0) {
|
||||
cbm_pipeline_set_lsp_surfaces(ctx->pipeline, surface_rows, surface_count);
|
||||
} else {
|
||||
cbm_log_warn("lsp_surface.serialize_failed", "files", itoa_buf(file_count));
|
||||
}
|
||||
}
|
||||
free(def_starts);
|
||||
/* Build inverted index: module_qn → defs. The fused resolve_worker
|
||||
* uses this to filter the global all_defs[] down to just the defs
|
||||
* each file actually needs (own_module + imported modules) — the
|
||||
@@ -1644,6 +1685,15 @@ int cbm_pipeline_publish_generation(const cbm_pipeline_generation_t *generation)
|
||||
ok = ok && cbm_store_delete_file_hashes(store, generation->project) == CBM_STORE_OK &&
|
||||
cbm_store_upsert_file_hash_batch(store, generation->manifest,
|
||||
generation->manifest_count) == CBM_STORE_OK;
|
||||
/* LSP surfaces belong to the generation: written inside the same staging
|
||||
* store, before the atomic rename, so graph and surface data can never
|
||||
* publish separately. The delete guards the incremental path, whose
|
||||
* staging DB starts as a copy of the previous generation. */
|
||||
if (ok) {
|
||||
ok = cbm_store_delete_lsp_surfaces(store, generation->project) == CBM_STORE_OK &&
|
||||
cbm_store_upsert_lsp_surface_batch(store, generation->surface_rows,
|
||||
generation->surface_row_count) == CBM_STORE_OK;
|
||||
}
|
||||
if (ok && generation->adr_content) {
|
||||
ok = cbm_store_adr_store(store, generation->project, generation->adr_content) ==
|
||||
CBM_STORE_OK;
|
||||
@@ -1810,6 +1860,8 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas
|
||||
.coverage_version = CBM_SEMANTIC_INDEX_VERSION,
|
||||
.hash_records_complete = true,
|
||||
},
|
||||
.surface_rows = p->surface_rows,
|
||||
.surface_row_count = p->surface_row_count,
|
||||
};
|
||||
|
||||
free(db_dir);
|
||||
|
||||
@@ -696,6 +696,12 @@ typedef struct {
|
||||
const cbm_coverage_row_t *coverage;
|
||||
int coverage_count;
|
||||
cbm_coverage_meta_t coverage_meta;
|
||||
/* Per-file LSP surfaces for the generation being published (may be NULL:
|
||||
* cross-LSP disabled, or a caller that has none). Written into the
|
||||
* staging store alongside the manifest so surface data and graph always
|
||||
* belong to the same generation. */
|
||||
const cbm_lsp_surface_row_t *surface_rows;
|
||||
int surface_row_count;
|
||||
} cbm_pipeline_generation_t;
|
||||
|
||||
/* Serialize and fully populate a sibling staging database, then atomically
|
||||
@@ -707,6 +713,12 @@ int cbm_pipeline_publish_generation(const cbm_pipeline_generation_t *generation)
|
||||
* automatic refresh of an already-existing artifact remains best-effort. */
|
||||
int cbm_pipeline_refresh_artifact(cbm_pipeline_t *p, const char *db_path);
|
||||
|
||||
/* Hand the pipeline the per-file LSP-surface rows serialized at the
|
||||
* collect_all_defs seam (the only moment the result cache is alive).
|
||||
* Takes ownership; dump_and_persist_hashes writes them into the staging
|
||||
* store and cbm_pipeline_free releases them. Passing NULL/0 clears. */
|
||||
void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *rows, int count);
|
||||
|
||||
/* Pipeline accessors for incremental use */
|
||||
const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p);
|
||||
atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p);
|
||||
|
||||
@@ -323,10 +323,10 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator(
|
||||
* cbm_pxc_run_one(_ts) per file BEFORE materializing CALLS edges. */
|
||||
char **def_modules = (char **)calloc((size_t)file_count, sizeof(char *));
|
||||
int def_count = 0;
|
||||
CBMLSPDef *all_defs = def_modules
|
||||
? cbm_pxc_collect_all_defs(result_cache, files, file_count,
|
||||
ctx.project_name, def_modules, &def_count)
|
||||
: NULL;
|
||||
CBMLSPDef *all_defs =
|
||||
def_modules ? cbm_pxc_collect_all_defs(result_cache, files, file_count, ctx.project_name,
|
||||
def_modules, &def_count, NULL)
|
||||
: NULL;
|
||||
CBMModuleDefIndex *module_def_index =
|
||||
all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL;
|
||||
|
||||
|
||||
@@ -11186,7 +11186,79 @@ TEST(pipeline_seq_ts_cross_uses_shared_registry) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* The closure-repair route lives and dies by two properties of the persisted
|
||||
* per-file LSP surface: a BODY edit must leave the surface_sha unchanged (the
|
||||
* early cutoff -- no dependent recomputation owed), while a SIGNATURE edit
|
||||
* must change it (dependents owe re-resolution). Each run here is a fresh
|
||||
* full index (DB removed in between) so what is compared is the published
|
||||
* surface of the code as it stands, not any incremental behaviour. */
|
||||
TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant) {
|
||||
char tmp[256];
|
||||
snprintf(tmp, sizeof(tmp), "/tmp/cbm_surface_persist_XXXXXX");
|
||||
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
|
||||
char src_a[512];
|
||||
char db_path[512];
|
||||
snprintf(src_a, sizeof(src_a), "%s/a.py", tmp);
|
||||
snprintf(db_path, sizeof(db_path), "%s/surface.db", tmp);
|
||||
ASSERT_EQ(th_write_file(src_a, "def surface_probe(x):\n return 1\n"), 0);
|
||||
|
||||
char sha_baseline[128] = {0};
|
||||
char json_probe[64] = {0};
|
||||
char project[256] = {0};
|
||||
|
||||
for (int round = 0; round < 3; round++) {
|
||||
if (round == 1) {
|
||||
/* Body edit: same def surface, different body. */
|
||||
ASSERT_EQ(th_write_file(src_a, "def surface_probe(x):\n return 2 + x\n"), 0);
|
||||
} else if (round == 2) {
|
||||
/* Signature edit: the surface other files consume changes. */
|
||||
ASSERT_EQ(th_write_file(src_a, "def surface_probe(x, y):\n return 1\n"), 0);
|
||||
}
|
||||
cbm_unlink(db_path);
|
||||
cbm_remove_db_sidecars(db_path);
|
||||
cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
|
||||
ASSERT_NOT_NULL(p);
|
||||
ASSERT_EQ(cbm_pipeline_run(p), 0);
|
||||
if (round == 0) {
|
||||
snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p));
|
||||
}
|
||||
cbm_pipeline_free(p);
|
||||
|
||||
cbm_store_t *store = cbm_store_open_path(db_path);
|
||||
ASSERT_NOT_NULL(store);
|
||||
cbm_lsp_surface_row_t *rows = NULL;
|
||||
int row_count = 0;
|
||||
ASSERT_EQ(cbm_store_get_lsp_surfaces(store, project, &rows, &row_count), CBM_STORE_OK);
|
||||
cbm_store_close(store);
|
||||
|
||||
const cbm_lsp_surface_row_t *row_a = NULL;
|
||||
for (int i = 0; i < row_count; i++) {
|
||||
if (strcmp(rows[i].rel_path, "a.py") == 0) {
|
||||
row_a = &rows[i];
|
||||
}
|
||||
}
|
||||
ASSERT_NOT_NULL(row_a);
|
||||
ASSERT_TRUE(row_a->surface_sha && row_a->surface_sha[0]);
|
||||
if (round == 0) {
|
||||
snprintf(sha_baseline, sizeof(sha_baseline), "%s", row_a->surface_sha);
|
||||
/* The row is the versioned codec envelope with the def present. */
|
||||
snprintf(json_probe, sizeof(json_probe), "%.20s", row_a->defs_json);
|
||||
ASSERT_TRUE(strstr(row_a->defs_json, "\"v\":1") != NULL);
|
||||
ASSERT_TRUE(strstr(row_a->defs_json, "surface_probe") != NULL);
|
||||
} else if (round == 1) {
|
||||
ASSERT_STR_EQ(row_a->surface_sha, sha_baseline);
|
||||
} else {
|
||||
ASSERT_TRUE(strcmp(row_a->surface_sha, sha_baseline) != 0);
|
||||
}
|
||||
cbm_store_free_lsp_surfaces(rows, row_count);
|
||||
}
|
||||
(void)json_probe;
|
||||
th_rmtree(tmp);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(pipeline) {
|
||||
RUN_TEST(pipeline_lsp_surface_persisted_and_body_edit_invariant);
|
||||
/* Index lock */
|
||||
RUN_TEST(pipeline_lock_try_acquire);
|
||||
RUN_TEST(pipeline_lock_blocking);
|
||||
|
||||
Reference in new issue
Block a user