mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 10:48:20 +00:00
AST-based route registration + remove prescan infrastructure
Route registration detection during AST extraction: callee QN matched against router frameworks (gin, chi, Express, Laravel, Ktor, etc.), handler reference captured as second_arg_name, Route + HANDLES edges created during resolution. Replaces disk-reading route discovery for non-decorator frameworks. Remove prescan infrastructure entirely (-350 net lines): - cbm_prescan_t, cbm_prescan_http_site_t, cbm_prescan_route_t, cbm_prescan_config_ref_t structs removed from pipeline_internal.h - prescan_http_sites, prescan_routes, prescan_config_refs, prescan_add_route removed from pass_parallel.c - prescan fast-paths removed from pass_httplinks.c and pass_configlink.c - prescan allocation/cleanup removed from pipeline.c All post-passes now work without prescan: - HTTP/async edges: service_patterns during resolution - Route registration: CBM_SVC_ROUTE_REG during resolution - Decorator routes: gbuf properties (no disk reads) - Config file refs: disk fallback (few files)
This commit is contained in:
1 parent
80680ea367
commit
b00e1f74c2
11 files changed
+258
-608
No files matched your search
@@ -111,6 +111,7 @@ typedef struct {
|
||||
const char *callee_name; // raw callee text ("pkg.Func", "foo")
|
||||
const char *enclosing_func_qn; // QN of enclosing function (or module QN)
|
||||
const char *first_string_arg; // first string literal argument (URL, topic, key) or NULL
|
||||
const char *second_arg_name; // second argument identifier (handler ref) or NULL
|
||||
} CBMCall;
|
||||
|
||||
typedef struct {
|
||||
|
||||
@@ -267,7 +267,7 @@ static void walk_calls(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec)
|
||||
if (callee && callee[0]) {
|
||||
// Skip keywords
|
||||
if (!cbm_is_keyword(callee, ctx->language)) {
|
||||
CBMCall call;
|
||||
CBMCall call = {0};
|
||||
call.callee_name = callee;
|
||||
call.enclosing_func_qn = cbm_enclosing_func_qn_cached(ctx, node);
|
||||
call.first_string_arg = NULL;
|
||||
@@ -504,6 +504,26 @@ void handle_calls(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec, Walk
|
||||
}
|
||||
}
|
||||
|
||||
/* Extract second argument name (handler ref for route registrations).
|
||||
* Pattern: router.GET("/path", handlerFunc) → second_arg_name = "handlerFunc"
|
||||
* Only extracted when first_string_arg looks like a path. */
|
||||
if (call.first_string_arg != NULL && call.first_string_arg[0] == '/' &&
|
||||
!ts_node_is_null(args)) {
|
||||
uint32_t nc2 = ts_node_named_child_count(args);
|
||||
for (uint32_t ai = 1; ai < nc2 && ai < 4 && !call.second_arg_name; ai++) {
|
||||
TSNode arg2 = ts_node_named_child(args, ai);
|
||||
const char *ak2 = ts_node_type(arg2);
|
||||
if (strcmp(ak2, "identifier") == 0) {
|
||||
call.second_arg_name = cbm_node_text(ctx->arena, arg2, ctx->source);
|
||||
} else if (strcmp(ak2, "member_expression") == 0 ||
|
||||
strcmp(ak2, "selector_expression") == 0 ||
|
||||
strcmp(ak2, "attribute") == 0 ||
|
||||
strcmp(ak2, "field_expression") == 0) {
|
||||
call.second_arg_name = cbm_node_text(ctx->arena, arg2, ctx->source);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cbm_calls_push(&ctx->result->calls, ctx->arena, call);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -282,13 +282,122 @@ static const lib_pattern_t config_libraries[] = {
|
||||
{NULL, CBM_SVC_NONE, NULL},
|
||||
};
|
||||
|
||||
/* ── HTTP method inference from function/method name suffix ───── */
|
||||
/* Route registration frameworks — callee resolves to one of these AND
|
||||
* has an HTTP method suffix → CBM_SVC_ROUTE_REG.
|
||||
* Distinguished from HTTP clients: "gin.GET" registers a handler,
|
||||
* "requests.get" makes an outbound HTTP call. */
|
||||
static const lib_pattern_t route_reg_libraries[] = {
|
||||
/* Go */
|
||||
{"gin-gonic/gin", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"gin.", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"go-chi/chi", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"chi.", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"gorilla/mux", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"labstack/echo", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"echo.", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"gofiber/fiber", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"fiber.", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"net/http.ServeMux", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"http.ServeMux", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"httprouter", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
/* JavaScript / TypeScript */
|
||||
{"express", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"fastify", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"koa-router", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"hono", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"hapi", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
/* Python (non-decorator, e.g., Flask add_url_rule) */
|
||||
{"flask", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"FastAPI", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"starlette", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
/* PHP */
|
||||
{"Laravel", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"Illuminate.Routing", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"Symfony.Routing", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
/* Kotlin */
|
||||
{"ktor.server", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"ktor.routing", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
/* Rust */
|
||||
{"actix-web", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"actix_web", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"axum", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"rocket", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
/* Java */
|
||||
{"Spring", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"jakarta.ws.rs", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
/* C# */
|
||||
{"Microsoft.AspNetCore", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"MapGet", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"MapPost", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
/* Ruby */
|
||||
{"ActionDispatch", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"Sinatra", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
/* Elixir */
|
||||
{"Phoenix.Router", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
/* Scala */
|
||||
{"akka.http.scaladsl.server", CBM_SVC_ROUTE_REG, NULL},
|
||||
{"play.api.routing", CBM_SVC_ROUTE_REG, NULL},
|
||||
|
||||
{NULL, CBM_SVC_NONE, NULL},
|
||||
};
|
||||
|
||||
/* Method suffix type (used by both route registration and HTTP client tables) */
|
||||
typedef struct {
|
||||
const char *suffix;
|
||||
const char *method;
|
||||
} method_suffix_t;
|
||||
|
||||
/* Route registration method suffixes — matched on callee name.
|
||||
* These are methods on router objects that register handlers. */
|
||||
static const method_suffix_t route_reg_suffixes[] = {
|
||||
/* HTTP method registrations */
|
||||
{".GET", "GET"},
|
||||
{".Get", "GET"},
|
||||
{".get", "GET"},
|
||||
{".POST", "POST"},
|
||||
{".Post", "POST"},
|
||||
{".post", "POST"},
|
||||
{".PUT", "PUT"},
|
||||
{".Put", "PUT"},
|
||||
{".put", "PUT"},
|
||||
{".DELETE", "DELETE"},
|
||||
{".Delete", "DELETE"},
|
||||
{".delete", "DELETE"},
|
||||
{".PATCH", "PATCH"},
|
||||
{".Patch", "PATCH"},
|
||||
{".patch", "PATCH"},
|
||||
/* Handle/HandleFunc (Go stdlib, gorilla) */
|
||||
{".Handle", "ANY"},
|
||||
{".HandleFunc", "ANY"},
|
||||
{".handle", "ANY"},
|
||||
/* Framework-specific route registration */
|
||||
{".Route", "ANY"},
|
||||
{".route", "ANY"},
|
||||
{"::get", "GET"},
|
||||
{"::post", "POST"},
|
||||
{"::put", "PUT"},
|
||||
{"::delete", "DELETE"},
|
||||
{"::patch", "PATCH"},
|
||||
/* Minimal API (C# ASP.NET) */
|
||||
{".MapGet", "GET"},
|
||||
{".MapPost", "POST"},
|
||||
{".MapPut", "PUT"},
|
||||
{".MapDelete", "DELETE"},
|
||||
{NULL, NULL},
|
||||
};
|
||||
|
||||
/* ── HTTP method inference from function/method name suffix ───── */
|
||||
|
||||
static const method_suffix_t method_suffixes[] = {
|
||||
{".get", "GET"}, {".Get", "GET"}, {".GET", "GET"},
|
||||
{".post", "POST"}, {".Post", "POST"}, {".POST", "POST"},
|
||||
@@ -331,7 +440,14 @@ cbm_svc_kind_t cbm_service_pattern_match(const char *resolved_qn) {
|
||||
return CBM_SVC_NONE;
|
||||
}
|
||||
|
||||
const lib_pattern_t *p = match_qn(resolved_qn, http_libraries);
|
||||
/* Route registration checked first — prevents gin/echo from matching
|
||||
* as HTTP clients (both have .get/.post suffixes). */
|
||||
const lib_pattern_t *p = match_qn(resolved_qn, route_reg_libraries);
|
||||
if (p) {
|
||||
return p->kind;
|
||||
}
|
||||
|
||||
p = match_qn(resolved_qn, http_libraries);
|
||||
if (p) {
|
||||
return p->kind;
|
||||
}
|
||||
@@ -363,6 +479,20 @@ const char *cbm_service_pattern_http_method(const char *callee_name) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char *cbm_service_pattern_route_method(const char *callee_name) {
|
||||
if (!callee_name) {
|
||||
return NULL;
|
||||
}
|
||||
size_t clen = strlen(callee_name);
|
||||
for (int i = 0; route_reg_suffixes[i].suffix != NULL; i++) {
|
||||
size_t slen = strlen(route_reg_suffixes[i].suffix);
|
||||
if (clen >= slen && strcmp(callee_name + clen - slen, route_reg_suffixes[i].suffix) == 0) {
|
||||
return route_reg_suffixes[i].method;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char *cbm_service_pattern_broker(const char *resolved_qn) {
|
||||
if (!resolved_qn) {
|
||||
return NULL;
|
||||
|
||||
@@ -13,10 +13,11 @@
|
||||
|
||||
/* Edge type returned by pattern match. */
|
||||
typedef enum {
|
||||
CBM_SVC_NONE = 0, /* Not a service pattern — use normal CALLS */
|
||||
CBM_SVC_HTTP = 1, /* Synchronous HTTP client call */
|
||||
CBM_SVC_ASYNC = 2, /* Async dispatch (message broker, task queue) */
|
||||
CBM_SVC_CONFIG = 3, /* Config/env accessor */
|
||||
CBM_SVC_NONE = 0, /* Not a service pattern — use normal CALLS */
|
||||
CBM_SVC_HTTP = 1, /* Synchronous HTTP client call */
|
||||
CBM_SVC_ASYNC = 2, /* Async dispatch (message broker, task queue) */
|
||||
CBM_SVC_CONFIG = 3, /* Config/env accessor */
|
||||
CBM_SVC_ROUTE_REG = 4, /* Route registration (router.GET, app.get, Route::post) */
|
||||
} cbm_svc_kind_t;
|
||||
|
||||
/* Initialize the pattern lookup tables. Call once at startup. Thread-safe after init. */
|
||||
@@ -32,6 +33,11 @@ cbm_svc_kind_t cbm_service_pattern_match(const char *resolved_qn);
|
||||
* Returns NULL if method cannot be inferred. */
|
||||
const char *cbm_service_pattern_http_method(const char *callee_name);
|
||||
|
||||
/* Get the HTTP method from a route registration callee name suffix
|
||||
* (e.g., "router.GET" → "GET", "app.post" → "POST").
|
||||
* Returns NULL if not a known route registration method. */
|
||||
const char *cbm_service_pattern_route_method(const char *callee_name);
|
||||
|
||||
/* Get the broker name for an async QN (e.g., "pubsub" from a Pub/Sub QN).
|
||||
* Returns NULL if not an async pattern. */
|
||||
const char *cbm_service_pattern_broker(const char *resolved_qn);
|
||||
|
||||
@@ -275,6 +275,46 @@ int cbm_pipeline_pass_calls(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
|
||||
/* Classify edge type by library in resolved QN */
|
||||
cbm_svc_kind_t svc = cbm_service_pattern_match(res.qualified_name);
|
||||
|
||||
if (svc == CBM_SVC_ROUTE_REG && call->first_string_arg != NULL &&
|
||||
call->first_string_arg[0] == '/') {
|
||||
/* Route registration: router.GET("/path", handler) → Route + HANDLES */
|
||||
const char *method = cbm_service_pattern_route_method(call->callee_name);
|
||||
char route_qn[CBM_ROUTE_QN_SIZE];
|
||||
snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", method ? method : "ANY",
|
||||
call->first_string_arg);
|
||||
char route_props[256];
|
||||
snprintf(route_props, sizeof(route_props), "{\"method\":\"%s\"}",
|
||||
method ? method : "ANY");
|
||||
int64_t route_id = cbm_gbuf_upsert_node(ctx->gbuf, "Route", call->first_string_arg,
|
||||
route_qn, "", 0, 0, route_props);
|
||||
|
||||
char props[512];
|
||||
snprintf(props, sizeof(props),
|
||||
"{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}",
|
||||
call->callee_name, call->first_string_arg);
|
||||
cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, route_id, "CALLS", props);
|
||||
|
||||
/* Resolve handler and create HANDLES edge */
|
||||
if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') {
|
||||
cbm_resolution_t hres =
|
||||
cbm_registry_resolve(ctx->registry, call->second_arg_name, module_qn,
|
||||
imp_keys, imp_vals, imp_count);
|
||||
if (hres.qualified_name != NULL && hres.qualified_name[0] != '\0') {
|
||||
const cbm_gbuf_node_t *handler =
|
||||
cbm_gbuf_find_by_qn(ctx->gbuf, hres.qualified_name);
|
||||
if (handler != NULL) {
|
||||
char hprops[256];
|
||||
snprintf(hprops, sizeof(hprops), "{\"handler\":\"%s\"}",
|
||||
hres.qualified_name);
|
||||
cbm_gbuf_insert_edge(ctx->gbuf, handler->id, route_id, "HANDLES",
|
||||
hprops);
|
||||
}
|
||||
}
|
||||
}
|
||||
resolved++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) {
|
||||
/* HTTP/async call — route through Route node for cross-service traversal.
|
||||
* Only create Route if string looks like a URL (HTTP) or topic name (async). */
|
||||
|
||||
@@ -353,134 +353,6 @@ static int strategy_dep_imports(cbm_gbuf_t *gb) {
|
||||
|
||||
/* ── Strategy 3: Config File Path → Code String Reference ───────── */
|
||||
|
||||
/* Prescan-based: config refs were extracted during extraction phase. */
|
||||
static int strategy_file_refs_prescan(cbm_gbuf_t *gb, const char *project,
|
||||
const cbm_prescan_t *prescan_cache, int prescan_count,
|
||||
CBMHashTable *prescan_path_map) {
|
||||
/* Collect config Module nodes for matching */
|
||||
const cbm_gbuf_node_t **modules = NULL;
|
||||
int mod_count = 0;
|
||||
if (cbm_gbuf_find_by_label(gb, "Module", &modules, &mod_count) != 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Build basename → Module and fullpath → Module maps */
|
||||
typedef struct {
|
||||
const char *key;
|
||||
int64_t node_id;
|
||||
} path_map_t;
|
||||
|
||||
path_map_t *base_map = calloc((size_t)mod_count, sizeof(path_map_t));
|
||||
path_map_t *full_map = calloc((size_t)mod_count, sizeof(path_map_t));
|
||||
int base_count = 0;
|
||||
int full_count = 0;
|
||||
|
||||
for (int i = 0; i < mod_count; i++) {
|
||||
if (cbm_has_config_extension(modules[i]->file_path)) {
|
||||
const char *slash = strrchr(modules[i]->file_path, '/');
|
||||
base_map[base_count].key = slash ? slash + 1 : modules[i]->file_path;
|
||||
base_map[base_count].node_id = modules[i]->id;
|
||||
base_count++;
|
||||
|
||||
full_map[full_count].key = modules[i]->file_path;
|
||||
full_map[full_count].node_id = modules[i]->id;
|
||||
full_count++;
|
||||
}
|
||||
}
|
||||
|
||||
if (base_count == 0) {
|
||||
free(base_map);
|
||||
free(full_map);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int edge_count = 0;
|
||||
|
||||
/* Iterate prescan results — each file's config refs were extracted during extraction */
|
||||
for (int fi = 0; fi < prescan_count; fi++) {
|
||||
const cbm_prescan_t *ps = &prescan_cache[fi];
|
||||
if (ps->config_ref_count == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Find the Module node for this file via prescan_path_map */
|
||||
/* We need to reverse-lookup: file_idx → rel_path → Module QN.
|
||||
* But we don't have the files array here. Instead, iterate modules
|
||||
* and check which ones have prescan data. */
|
||||
}
|
||||
|
||||
/* Alternative approach: iterate non-config modules and check their prescan */
|
||||
for (int i = 0; i < mod_count; i++) {
|
||||
if (cbm_has_config_extension(modules[i]->file_path)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Look up prescan data for this file */
|
||||
void *val = cbm_ht_get(prescan_path_map, modules[i]->file_path);
|
||||
if (!val) {
|
||||
continue;
|
||||
}
|
||||
int file_idx = (int)((intptr_t)val - 1);
|
||||
if (file_idx < 0 || file_idx >= prescan_count) {
|
||||
continue;
|
||||
}
|
||||
const cbm_prescan_t *ps = &prescan_cache[file_idx];
|
||||
if (ps->config_ref_count == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (int ri = 0; ri < ps->config_ref_count; ri++) {
|
||||
const char *ref_path = ps->config_refs[ri].ref_path;
|
||||
|
||||
int64_t target_id = 0;
|
||||
double confidence = 0.0;
|
||||
|
||||
/* Try full path match */
|
||||
for (int fi2 = 0; fi2 < full_count; fi2++) {
|
||||
if (strcmp(full_map[fi2].key, ref_path) == 0) {
|
||||
target_id = full_map[fi2].node_id;
|
||||
confidence = CONF_FILE_FULLPATH;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Try basename match */
|
||||
if (target_id == 0) {
|
||||
const char *ref_slash = strrchr(ref_path, '/');
|
||||
const char *ref_base = ref_slash ? ref_slash + 1 : ref_path;
|
||||
for (int bi = 0; bi < base_count; bi++) {
|
||||
if (strcmp(base_map[bi].key, ref_base) == 0) {
|
||||
target_id = base_map[bi].node_id;
|
||||
confidence = CONF_FILE_BASENAME;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (target_id != 0) {
|
||||
char *module_qn = cbm_pipeline_fqn_module(project, modules[i]->file_path);
|
||||
if (module_qn) {
|
||||
const cbm_gbuf_node_t *src_node = cbm_gbuf_find_by_qn(gb, module_qn);
|
||||
if (src_node) {
|
||||
char props[512];
|
||||
snprintf(props, sizeof(props),
|
||||
"{\"strategy\":\"file_reference\",\"confidence\":%.2f,\"ref_"
|
||||
"path\":\"%s\"}",
|
||||
confidence, ref_path);
|
||||
cbm_gbuf_insert_edge(gb, src_node->id, target_id, "CONFIGURES", props);
|
||||
edge_count++;
|
||||
}
|
||||
free(module_qn);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
free(base_map);
|
||||
free(full_map);
|
||||
return edge_count;
|
||||
}
|
||||
|
||||
/* Disk-based fallback: reads source from disk (sequential pipeline path). */
|
||||
// NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
|
||||
static int strategy_file_refs_disk(cbm_gbuf_t *gb, const char *project, const char *repo_path) {
|
||||
@@ -646,8 +518,6 @@ static int strategy_file_refs_disk(cbm_gbuf_t *gb, const char *project, const ch
|
||||
|
||||
int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx) {
|
||||
cbm_gbuf_t *gb = ctx->gbuf;
|
||||
const char *project = ctx->project_name;
|
||||
const char *repo_path = ctx->repo_path;
|
||||
/* Early exit: check if any config files exist in the project. */
|
||||
bool has_config = false;
|
||||
|
||||
@@ -694,11 +564,8 @@ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx) {
|
||||
cbm_log_info("configlinker.strategy", "name", "dep_import", "edges", buf2);
|
||||
|
||||
int ref_edges = 0;
|
||||
if (ctx->prescan_cache && ctx->prescan_path_map) {
|
||||
ref_edges = strategy_file_refs_prescan(gb, project, ctx->prescan_cache, ctx->prescan_count,
|
||||
ctx->prescan_path_map);
|
||||
} else if (repo_path) {
|
||||
ref_edges = strategy_file_refs_disk(gb, project, repo_path);
|
||||
if (ctx->repo_path) {
|
||||
ref_edges = strategy_file_refs_disk(gb, ctx->project_name, ctx->repo_path);
|
||||
}
|
||||
snprintf(buf3, sizeof(buf3), "%d", ref_edges);
|
||||
cbm_log_info("configlinker.strategy", "name", "file_ref", "edges", buf3);
|
||||
|
||||
@@ -1246,7 +1246,7 @@ int cbm_pipeline_pass_httplinks(cbm_pipeline_ctx_t *ctx) {
|
||||
worker_count = 1;
|
||||
}
|
||||
|
||||
/* ── Phase 1: Route collection from prescan or parallel discovery ── */
|
||||
/* ── Phase 1: Route collection from decorator properties + disk ── */
|
||||
|
||||
/* Need label_counts for fallback path */
|
||||
const char *route_labels[] = {"Function", "Method", "Module"};
|
||||
@@ -1263,27 +1263,11 @@ int cbm_pipeline_pass_httplinks(cbm_pipeline_ctx_t *ctx) {
|
||||
}
|
||||
int route_count = 0;
|
||||
|
||||
if (ctx->prescan_cache) {
|
||||
/* Fast path: routes were extracted during the extraction phase.
|
||||
* Zero disk reads, zero JSON parsing, zero node iteration. */
|
||||
for (int fi = 0; fi < ctx->prescan_count; fi++) {
|
||||
const cbm_prescan_t *ps = &ctx->prescan_cache[fi];
|
||||
for (int ri = 0; ri < ps->route_count && route_count < MAX_ROUTES; ri++) {
|
||||
const cbm_prescan_route_t *src = &ps->routes[ri];
|
||||
cbm_route_handler_t *dst = &routes[route_count];
|
||||
snprintf(dst->path, sizeof(dst->path), "%s", src->path);
|
||||
snprintf(dst->method, sizeof(dst->method), "%s", src->method);
|
||||
snprintf(dst->function_name, sizeof(dst->function_name), "%s", src->function_name);
|
||||
snprintf(dst->qualified_name, sizeof(dst->qualified_name), "%s",
|
||||
src->qualified_name);
|
||||
snprintf(dst->handler_ref, sizeof(dst->handler_ref), "%s", src->handler_ref);
|
||||
dst->resolved_handler_qn[0] = '\0';
|
||||
snprintf(dst->protocol, sizeof(dst->protocol), "%s", src->protocol);
|
||||
route_count++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
/* Fallback: parallel route discovery from disk (sequential pipeline path) */
|
||||
{
|
||||
/* Route discovery: decorator-based (from gbuf properties) + disk-based
|
||||
* (Go/Express/Laravel/Ktor source reading). AST-based route registration
|
||||
* detection (CBM_SVC_ROUTE_REG) creates Route+HANDLES during resolution,
|
||||
* so this path is primarily for decorator-discovered routes now. */
|
||||
int total_route_nodes = 0;
|
||||
for (int li = 0; li < 3; li++) {
|
||||
total_route_nodes += label_counts[li];
|
||||
@@ -1353,33 +1337,16 @@ int cbm_pipeline_pass_httplinks(cbm_pipeline_ctx_t *ctx) {
|
||||
/* ── Phase 4: Route nodes + HANDLES edges (serial) ────────── */
|
||||
int route_nodes = insert_route_nodes(ctx, routes, route_count);
|
||||
|
||||
/* ── Phase 5: Call site collection from prescan cache ────────── */
|
||||
/* During extraction, each file's source was scanned for HTTP/async
|
||||
* keywords and URL paths. Results are in ctx->prescan_cache[file_idx].
|
||||
* This eliminates 2M+ disk reads (the old approach read every function
|
||||
* from disk just to check for keywords, then discarded 99.9%). */
|
||||
/* ── Phase 5: Call site collection ──────────────────────────── */
|
||||
|
||||
cbm_http_call_site_t *sites = calloc(MAX_CALL_SITES, sizeof(cbm_http_call_site_t));
|
||||
int site_count = 0;
|
||||
|
||||
if (sites && ctx->prescan_cache) {
|
||||
for (int fi = 0; fi < ctx->prescan_count; fi++) {
|
||||
const cbm_prescan_t *ps = &ctx->prescan_cache[fi];
|
||||
for (int si = 0; si < ps->http_site_count && site_count < MAX_CALL_SITES; si++) {
|
||||
const cbm_prescan_http_site_t *src = &ps->http_sites[si];
|
||||
cbm_http_call_site_t *dst = &sites[site_count];
|
||||
snprintf(dst->path, sizeof(dst->path), "%s", src->path);
|
||||
snprintf(dst->method, sizeof(dst->method), "%s", src->method);
|
||||
snprintf(dst->source_name, sizeof(dst->source_name), "%s", src->source_name);
|
||||
snprintf(dst->source_qn, sizeof(dst->source_qn), "%s", src->source_qn);
|
||||
snprintf(dst->source_label, sizeof(dst->source_label), "%s", src->source_label);
|
||||
dst->is_async = src->is_async;
|
||||
site_count++;
|
||||
}
|
||||
}
|
||||
} else if (sites) {
|
||||
/* Fallback: no prescan (sequential pipeline path).
|
||||
* Use the old parallel disk-reading approach. */
|
||||
if (sites) {
|
||||
/* Call site collection: parallel disk-reading approach.
|
||||
* Note: HTTP_CALLS/ASYNC_CALLS edges are now also created during
|
||||
* resolution via service_patterns. This path provides additional
|
||||
* URL-based matching for call sites not caught by QN matching. */
|
||||
const char *site_labels[] = {"Function", "Method"};
|
||||
const cbm_gbuf_node_t **all_site_nodes = NULL;
|
||||
const char **all_site_labels = NULL;
|
||||
|
||||
+35
-320
@@ -351,317 +351,11 @@ typedef struct {
|
||||
_Atomic int next_worker_id;
|
||||
|
||||
CBMFileResult **result_cache;
|
||||
cbm_prescan_t *prescan_cache;
|
||||
_Atomic int64_t *shared_ids;
|
||||
_Atomic int *cancelled;
|
||||
_Atomic int next_file_idx;
|
||||
} extract_ctx_t;
|
||||
|
||||
/* ── Pre-scan: extract HTTP sites + config refs while source is in memory ── */
|
||||
|
||||
/* Extract lines [start_line..end_line] from source buffer as a malloc'd string. */
|
||||
static char *extract_lines(const char *source, int source_len, int start_line, int end_line) {
|
||||
if (start_line <= 0 || end_line <= 0 || end_line < start_line) {
|
||||
return NULL;
|
||||
}
|
||||
int line = 1;
|
||||
const char *p = source;
|
||||
const char *end = source + source_len;
|
||||
const char *range_start = NULL;
|
||||
const char *range_end = NULL;
|
||||
|
||||
while (p < end) {
|
||||
if (line == start_line) {
|
||||
range_start = p;
|
||||
}
|
||||
const char *nl = memchr(p, '\n', (size_t)(end - p));
|
||||
if (!nl) {
|
||||
nl = end;
|
||||
}
|
||||
if (line == end_line) {
|
||||
range_end = (nl < end) ? nl + 1 : end;
|
||||
break;
|
||||
}
|
||||
p = nl + 1;
|
||||
line++;
|
||||
}
|
||||
if (!range_start) {
|
||||
return NULL;
|
||||
}
|
||||
if (!range_end) {
|
||||
range_end = end;
|
||||
}
|
||||
size_t len = (size_t)(range_end - range_start);
|
||||
// NOLINTNEXTLINE(clang-analyzer-optin.taint.TaintedAlloc)
|
||||
char *out = malloc(len + 1);
|
||||
if (!out) {
|
||||
return NULL;
|
||||
}
|
||||
memcpy(out, range_start, len);
|
||||
out[len] = '\0';
|
||||
return out;
|
||||
}
|
||||
|
||||
/* Scan function bodies for HTTP/async keywords, extract URL paths. */
|
||||
static void prescan_http_sites(const char *source, int source_len, const CBMFileResult *result,
|
||||
cbm_prescan_t *ps) {
|
||||
for (int d = 0; d < result->defs.count; d++) {
|
||||
const CBMDefinition *def = &result->defs.items[d];
|
||||
if (!def->qualified_name || !def->name || !def->label) {
|
||||
continue;
|
||||
}
|
||||
if (strcmp(def->label, "Function") != 0 && strcmp(def->label, "Method") != 0) {
|
||||
continue;
|
||||
}
|
||||
if (def->start_line == 0 || def->end_line == 0) {
|
||||
continue;
|
||||
}
|
||||
/* Skip Python dunder methods */
|
||||
size_t nlen = strlen(def->name);
|
||||
if (nlen > 4 && def->name[0] == '_' && def->name[1] == '_' && def->name[nlen - 1] == '_' &&
|
||||
def->name[nlen - 2] == '_') {
|
||||
continue;
|
||||
}
|
||||
|
||||
char *func_src =
|
||||
extract_lines(source, source_len, (int)def->start_line, (int)def->end_line);
|
||||
if (!func_src) {
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Keyword pre-filter */
|
||||
bool has_http = false;
|
||||
for (int k = 0; k < cbm_http_client_keywords_count; k++) {
|
||||
if (strstr(func_src, cbm_http_client_keywords[k])) {
|
||||
has_http = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
bool has_async = false;
|
||||
for (int k = 0; k < cbm_async_dispatch_keywords_count; k++) {
|
||||
if (strstr(func_src, cbm_async_dispatch_keywords[k])) {
|
||||
has_async = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!has_http && !has_async) {
|
||||
free(func_src);
|
||||
continue;
|
||||
}
|
||||
|
||||
bool is_async = (has_async && !has_http) != 0;
|
||||
|
||||
/* Extract URL paths */
|
||||
char *paths[64];
|
||||
int path_count = cbm_extract_url_paths(func_src, paths, 64);
|
||||
|
||||
for (int p = 0; p < path_count; p++) {
|
||||
cbm_prescan_http_site_t *site =
|
||||
realloc(ps->http_sites,
|
||||
(size_t)(ps->http_site_count + 1) * sizeof(cbm_prescan_http_site_t));
|
||||
if (!site) {
|
||||
free(paths[p]);
|
||||
continue;
|
||||
}
|
||||
ps->http_sites = site;
|
||||
cbm_prescan_http_site_t *s = &ps->http_sites[ps->http_site_count];
|
||||
snprintf(s->path, sizeof(s->path), "%s", paths[p]);
|
||||
s->method[0] = '\0';
|
||||
snprintf(s->source_name, sizeof(s->source_name), "%s", def->name);
|
||||
snprintf(s->source_qn, sizeof(s->source_qn), "%s", def->qualified_name);
|
||||
snprintf(s->source_label, sizeof(s->source_label), "%s", def->label);
|
||||
s->is_async = is_async;
|
||||
ps->http_site_count++;
|
||||
free(paths[p]);
|
||||
}
|
||||
/* Free any remaining paths we couldn't store */
|
||||
for (int p = path_count; p < path_count; p++) {
|
||||
free(paths[p]);
|
||||
}
|
||||
free(func_src);
|
||||
}
|
||||
}
|
||||
|
||||
/* Helper: append a route to prescan results. */
|
||||
static void prescan_add_route(cbm_prescan_t *ps, const cbm_route_handler_t *rh) {
|
||||
cbm_prescan_route_t *routes =
|
||||
realloc(ps->routes, (size_t)(ps->route_count + 1) * sizeof(cbm_prescan_route_t));
|
||||
if (!routes) {
|
||||
return;
|
||||
}
|
||||
ps->routes = routes;
|
||||
cbm_prescan_route_t *r = &routes[ps->route_count];
|
||||
snprintf(r->path, sizeof(r->path), "%s", rh->path);
|
||||
snprintf(r->method, sizeof(r->method), "%s", rh->method);
|
||||
snprintf(r->function_name, sizeof(r->function_name), "%s", rh->function_name);
|
||||
snprintf(r->qualified_name, sizeof(r->qualified_name), "%s", rh->qualified_name);
|
||||
snprintf(r->handler_ref, sizeof(r->handler_ref), "%s", rh->handler_ref);
|
||||
snprintf(r->protocol, sizeof(r->protocol), "%s", rh->protocol);
|
||||
ps->route_count++;
|
||||
}
|
||||
|
||||
/* Check file extension for source-based route languages. */
|
||||
static bool is_route_source_lang(const char *path) {
|
||||
if (!path) {
|
||||
return false;
|
||||
}
|
||||
size_t len = strlen(path);
|
||||
/* Go */
|
||||
if (len > 3 && strcmp(path + len - 3, ".go") == 0) {
|
||||
return true;
|
||||
}
|
||||
/* JavaScript/TypeScript */
|
||||
if (len > 3 && (strcmp(path + len - 3, ".js") == 0 || strcmp(path + len - 3, ".ts") == 0)) {
|
||||
return true;
|
||||
}
|
||||
if (len > 4 && (strcmp(path + len - 4, ".jsx") == 0 || strcmp(path + len - 4, ".tsx") == 0)) {
|
||||
return true;
|
||||
}
|
||||
/* PHP */
|
||||
if (len > 4 && strcmp(path + len - 4, ".php") == 0) {
|
||||
return true;
|
||||
}
|
||||
/* Kotlin */
|
||||
if (len > 3 && strcmp(path + len - 3, ".kt") == 0) {
|
||||
return true;
|
||||
}
|
||||
if (len > 4 && strcmp(path + len - 4, ".kts") == 0) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Scan function definitions for HTTP routes (decorator + source-based). */
|
||||
static void prescan_routes(const char *source, int source_len, const CBMFileResult *result,
|
||||
const char *rel_path, cbm_prescan_t *ps) {
|
||||
bool is_route_lang = is_route_source_lang(rel_path);
|
||||
|
||||
for (int d = 0; d < result->defs.count; d++) {
|
||||
const CBMDefinition *def = &result->defs.items[d];
|
||||
if (!def->qualified_name || !def->name) {
|
||||
continue;
|
||||
}
|
||||
|
||||
cbm_route_handler_t routes[16];
|
||||
int total = 0;
|
||||
|
||||
/* Decorator-based routes (Python, Java) */
|
||||
if (def->decorators && def->decorators[0]) {
|
||||
int ndec = 0;
|
||||
while (def->decorators[ndec]) {
|
||||
ndec++;
|
||||
}
|
||||
int nr = cbm_extract_python_routes(def->name, def->qualified_name,
|
||||
(const char **)def->decorators, ndec, routes + total,
|
||||
16 - total);
|
||||
total += nr;
|
||||
nr = cbm_extract_java_routes(def->name, def->qualified_name,
|
||||
(const char **)def->decorators, ndec, routes + total,
|
||||
16 - total);
|
||||
total += nr;
|
||||
}
|
||||
|
||||
/* Source-based routes (Go, Express, Laravel, Ktor) — only for matching languages */
|
||||
if (is_route_lang && def->start_line > 0 && def->end_line > 0 && total < 16) {
|
||||
char *func_src =
|
||||
extract_lines(source, source_len, (int)def->start_line, (int)def->end_line);
|
||||
if (func_src) {
|
||||
int nr = cbm_extract_go_routes(def->name, def->qualified_name, func_src,
|
||||
routes + total, 16 - total);
|
||||
total += nr;
|
||||
nr = cbm_extract_express_routes(def->name, def->qualified_name, func_src,
|
||||
routes + total, 16 - total);
|
||||
total += nr;
|
||||
nr = cbm_extract_laravel_routes(def->name, def->qualified_name, func_src,
|
||||
routes + total, 16 - total);
|
||||
total += nr;
|
||||
nr = cbm_extract_ktor_routes(def->name, def->qualified_name, func_src,
|
||||
routes + total, 16 - total);
|
||||
total += nr;
|
||||
free(func_src);
|
||||
}
|
||||
}
|
||||
|
||||
for (int r = 0; r < total; r++) {
|
||||
prescan_add_route(ps, &routes[r]);
|
||||
}
|
||||
}
|
||||
|
||||
/* Module-level routes (PHP Laravel, JS/TS Express) */
|
||||
if (is_route_lang && source_len > 0) {
|
||||
size_t plen = strlen(rel_path);
|
||||
bool is_php = (plen > 4 && strcmp(rel_path + plen - 4, ".php") == 0) != 0;
|
||||
bool is_js = ((plen > 3 && (strcmp(rel_path + plen - 3, ".js") == 0 ||
|
||||
strcmp(rel_path + plen - 3, ".ts") == 0)) ||
|
||||
(plen > 4 && (strcmp(rel_path + plen - 4, ".jsx") == 0 ||
|
||||
strcmp(rel_path + plen - 4, ".tsx") == 0))) != 0;
|
||||
if (is_php || is_js) {
|
||||
/* Build a module name from rel_path */
|
||||
const char *slash = strrchr(rel_path, '/');
|
||||
const char *basename = slash ? slash + 1 : rel_path;
|
||||
cbm_route_handler_t mod_routes[16];
|
||||
int total = 0;
|
||||
if (is_php) {
|
||||
total += cbm_extract_laravel_routes(basename, "", source, mod_routes + total,
|
||||
16 - total);
|
||||
}
|
||||
if (is_js) {
|
||||
total += cbm_extract_express_routes(basename, "", source, mod_routes + total,
|
||||
16 - total);
|
||||
}
|
||||
for (int r = 0; r < total; r++) {
|
||||
prescan_add_route(ps, &mod_routes[r]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Scan full source for config file path references in string literals. */
|
||||
static void prescan_config_refs(const char *source, cbm_prescan_t *ps) {
|
||||
/* Quick reject: no config extension substring in source at all */
|
||||
static const char *exts[] = {".toml", ".yaml", ".yml", ".ini", ".json",
|
||||
".xml", ".conf", ".cfg", ".env", NULL};
|
||||
bool has_any = false;
|
||||
for (int i = 0; exts[i]; i++) {
|
||||
if (strstr(source, exts[i])) {
|
||||
has_any = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!has_any) {
|
||||
return;
|
||||
}
|
||||
|
||||
cbm_regex_t re;
|
||||
if (cbm_regcomp(&re, "[\"']([^\"']*\\.(toml|yaml|yml|ini|json|xml|conf|cfg|env))[\"']",
|
||||
CBM_REG_EXTENDED) != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
cbm_regmatch_t match[3];
|
||||
const char *cursor = source;
|
||||
while (cbm_regexec(&re, cursor, 3, match, 0) == 0) {
|
||||
int start = match[1].rm_so;
|
||||
int end = match[1].rm_eo;
|
||||
int ref_len = end - start;
|
||||
|
||||
if (ref_len > 0 && ref_len < 256) {
|
||||
cbm_prescan_config_ref_t *refs =
|
||||
realloc(ps->config_refs,
|
||||
(size_t)(ps->config_ref_count + 1) * sizeof(cbm_prescan_config_ref_t));
|
||||
if (refs) {
|
||||
ps->config_refs = refs;
|
||||
memcpy(refs[ps->config_ref_count].ref_path, cursor + start, (size_t)ref_len);
|
||||
refs[ps->config_ref_count].ref_path[ref_len] = '\0';
|
||||
ps->config_ref_count++;
|
||||
}
|
||||
}
|
||||
cursor += match[0].rm_eo;
|
||||
}
|
||||
cbm_regfree(&re);
|
||||
}
|
||||
|
||||
static void extract_worker(int worker_id, void *ctx_ptr) {
|
||||
extract_ctx_t *ec = ctx_ptr;
|
||||
extract_worker_state_t *ws = &ec->workers[worker_id];
|
||||
@@ -745,17 +439,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
|
||||
* are released before the slab is bulk-reclaimed. */
|
||||
cbm_free_tree(result);
|
||||
|
||||
/* Pre-scan source for HTTP call sites and config file references
|
||||
* while the buffer is still in memory. Eliminates 2M+ disk reads
|
||||
* in httplinks and 62K+ disk reads in configlink. */
|
||||
if (ec->prescan_cache) {
|
||||
cbm_prescan_t *ps = &ec->prescan_cache[file_idx];
|
||||
prescan_routes(source, source_len, result, fi->rel_path, ps);
|
||||
prescan_http_sites(source, source_len, result, ps);
|
||||
prescan_config_refs(source, ps);
|
||||
}
|
||||
|
||||
/* Free source buffer — prescan captured everything needed. */
|
||||
/* Free source buffer — extraction captured everything needed. */
|
||||
free_source(source);
|
||||
|
||||
/* Cache result (arena + extracted data, no tree) for Phase 3B and Phase 4 */
|
||||
@@ -840,7 +524,6 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *files, int fi
|
||||
.workers = workers,
|
||||
.max_workers = worker_count,
|
||||
.result_cache = result_cache,
|
||||
.prescan_cache = ctx->prescan_cache,
|
||||
.shared_ids = shared_ids,
|
||||
.cancelled = ctx->cancelled,
|
||||
};
|
||||
@@ -1003,10 +686,41 @@ typedef struct {
|
||||
* Extracted from resolve_worker to keep cognitive complexity under threshold. */
|
||||
static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source,
|
||||
const cbm_gbuf_node_t *target, const CBMCall *call,
|
||||
const cbm_resolution_t *res) {
|
||||
const cbm_resolution_t *res, const char *module_qn,
|
||||
const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf,
|
||||
const char **imp_keys, const char **imp_vals, int imp_count) {
|
||||
cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name);
|
||||
const char *arg = call->first_string_arg;
|
||||
|
||||
if (svc == CBM_SVC_ROUTE_REG && arg != NULL && arg[0] == '/') {
|
||||
const char *method = cbm_service_pattern_route_method(call->callee_name);
|
||||
char route_qn[CBM_ROUTE_QN_SIZE];
|
||||
snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", method ? method : "ANY", arg);
|
||||
char route_props[256];
|
||||
snprintf(route_props, sizeof(route_props), "{\"method\":\"%s\"}", method ? method : "ANY");
|
||||
int64_t route_id =
|
||||
cbm_gbuf_upsert_node(gbuf, "Route", arg, route_qn, "", 0, 0, route_props);
|
||||
char props[512];
|
||||
snprintf(props, sizeof(props),
|
||||
"{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}",
|
||||
call->callee_name, arg);
|
||||
cbm_gbuf_insert_edge(gbuf, source->id, route_id, "CALLS", props);
|
||||
if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') {
|
||||
cbm_resolution_t hres = cbm_registry_resolve(registry, call->second_arg_name, module_qn,
|
||||
imp_keys, imp_vals, imp_count);
|
||||
if (hres.qualified_name != NULL && hres.qualified_name[0] != '\0') {
|
||||
const cbm_gbuf_node_t *handler =
|
||||
cbm_gbuf_find_by_qn(main_gbuf, hres.qualified_name);
|
||||
if (handler != NULL) {
|
||||
char hprops[256];
|
||||
snprintf(hprops, sizeof(hprops), "{\"handler\":\"%s\"}", hres.qualified_name);
|
||||
cbm_gbuf_insert_edge(gbuf, handler->id, route_id, "HANDLES", hprops);
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
int has_url = (arg != NULL && arg[0] != '\0' && (arg[0] == '/' || strstr(arg, "://") != NULL));
|
||||
int has_topic = (arg != NULL && arg[0] != '\0' && svc == CBM_SVC_ASYNC && strlen(arg) > 2);
|
||||
|
||||
@@ -1128,7 +842,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
}
|
||||
|
||||
/* Classify and emit edge via helper (keeps resolve_worker complexity down) */
|
||||
emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res);
|
||||
emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res, module_qn,
|
||||
rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count);
|
||||
ws->calls_resolved++;
|
||||
}
|
||||
|
||||
|
||||
@@ -501,20 +501,6 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Allocate prescan cache: HTTP sites + config refs extracted during
|
||||
* extraction phase while source is in memory. Eliminates all disk
|
||||
* re-reads in httplinks (2M+ reads) and configlink (62K+ reads). */
|
||||
cbm_prescan_t *prescan_cache = calloc(file_count, sizeof(cbm_prescan_t));
|
||||
ctx.prescan_cache = prescan_cache;
|
||||
ctx.prescan_count = file_count;
|
||||
|
||||
/* Build path → file_idx map for prescan lookup by rel_path */
|
||||
CBMHashTable *prescan_map = cbm_ht_create(0);
|
||||
for (int i = 0; i < file_count; i++) {
|
||||
cbm_ht_set(prescan_map, files[i].rel_path, (void *)((intptr_t)i + 1));
|
||||
}
|
||||
ctx.prescan_path_map = prescan_map;
|
||||
|
||||
/* Phase 3A: Parallel extract + definition nodes */
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
|
||||
rc = cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count);
|
||||
@@ -784,21 +770,6 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
|
||||
itoa_buf((int)elapsed_ms(t)));
|
||||
}
|
||||
|
||||
/* Free prescan cache — no longer needed after httplinks + configlink */
|
||||
if (ctx.prescan_cache) {
|
||||
for (int i = 0; i < ctx.prescan_count; i++) {
|
||||
free(ctx.prescan_cache[i].http_sites);
|
||||
free(ctx.prescan_cache[i].config_refs);
|
||||
free(ctx.prescan_cache[i].routes);
|
||||
}
|
||||
free(ctx.prescan_cache);
|
||||
ctx.prescan_cache = NULL;
|
||||
}
|
||||
if (ctx.prescan_path_map) {
|
||||
cbm_ht_free(ctx.prescan_path_map);
|
||||
ctx.prescan_path_map = NULL;
|
||||
}
|
||||
|
||||
/* Direct dump: construct B-tree pages in C, fwrite() to .db file.
|
||||
* Zero SQLite library involvement — cbm_write_db() builds the binary
|
||||
* format directly from flat arrays. Atomic: writes .tmp then renames. */
|
||||
@@ -864,20 +835,6 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
|
||||
itoa_buf((int)elapsed_ms(t0)));
|
||||
|
||||
cleanup:
|
||||
/* Free prescan if not already freed */
|
||||
if (ctx.prescan_cache) {
|
||||
for (int i = 0; i < ctx.prescan_count; i++) {
|
||||
free(ctx.prescan_cache[i].http_sites);
|
||||
free(ctx.prescan_cache[i].config_refs);
|
||||
free(ctx.prescan_cache[i].routes);
|
||||
}
|
||||
free(ctx.prescan_cache);
|
||||
ctx.prescan_cache = NULL;
|
||||
}
|
||||
if (ctx.prescan_path_map) {
|
||||
cbm_ht_free(ctx.prescan_path_map);
|
||||
ctx.prescan_path_map = NULL;
|
||||
}
|
||||
cbm_discover_free(files, file_count);
|
||||
cbm_gbuf_free(p->gbuf);
|
||||
p->gbuf = NULL;
|
||||
|
||||
@@ -374,16 +374,15 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
|
||||
cbm_log_info("pass.timing", "pass", "incr_decorator_tags", "elapsed_ms",
|
||||
itoa_buf((int)elapsed_ms(t)));
|
||||
|
||||
/* Configlink: key_symbol + dep_imports are gbuf-only (safe).
|
||||
* file_refs needs prescan or disk — we set prescan=NULL so it uses
|
||||
* the disk fallback ONLY for config files (typically <10 files). */
|
||||
/* Configlink: key_symbol + dep_imports are gbuf-only.
|
||||
* file_refs uses disk fallback for config files (typically <10 files). */
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
|
||||
cbm_pipeline_pass_configlink(&ctx);
|
||||
cbm_log_info("pass.timing", "pass", "incr_configlink", "elapsed_ms",
|
||||
itoa_buf((int)elapsed_ms(t)));
|
||||
|
||||
/* Httplinks: route discovery uses disk fallback when prescan=NULL.
|
||||
* Only reads source for files with Route/Handler labels (~few files). */
|
||||
/* Httplinks: decorator-based routes from gbuf properties + disk fallback
|
||||
* for source-based routes. AST-based route registration already ran during resolve. */
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
|
||||
cbm_pipeline_pass_httplinks(&ctx);
|
||||
cbm_log_info("pass.timing", "pass", "incr_httplinks", "elapsed_ms",
|
||||
|
||||
@@ -28,45 +28,6 @@
|
||||
#define CBM_MS_PER_SEC 1000.0
|
||||
#define CBM_US_PER_SEC_F 1e6
|
||||
|
||||
/* ── Pre-scan results (extracted during parallel phase) ──────────── */
|
||||
|
||||
/* HTTP call site discovered during extraction (source already in memory).
|
||||
* Eliminates 2M+ disk reads in httplinks pass. */
|
||||
typedef struct {
|
||||
char path[256]; /* extracted URL path */
|
||||
char method[16]; /* HTTP method (empty if unknown) */
|
||||
char source_name[256]; /* function name */
|
||||
char source_qn[512]; /* function qualified name */
|
||||
char source_label[32]; /* "Function" or "Method" */
|
||||
bool is_async; /* async dispatch vs HTTP call */
|
||||
} cbm_prescan_http_site_t;
|
||||
|
||||
/* Config file reference found in source during extraction.
|
||||
* Eliminates 62K+ disk reads in configlink pass. */
|
||||
typedef struct {
|
||||
char ref_path[256]; /* referenced config file path (e.g. "config.yaml") */
|
||||
} cbm_prescan_config_ref_t;
|
||||
|
||||
/* HTTP route discovered during extraction. */
|
||||
typedef struct {
|
||||
char path[256];
|
||||
char method[16];
|
||||
char function_name[256];
|
||||
char qualified_name[512];
|
||||
char handler_ref[256];
|
||||
char protocol[8]; /* "ws", "sse", or "" */
|
||||
} cbm_prescan_route_t;
|
||||
|
||||
/* Per-file prescan results, parallel to result_cache[file_idx]. */
|
||||
typedef struct {
|
||||
cbm_prescan_http_site_t *http_sites;
|
||||
int http_site_count;
|
||||
cbm_prescan_config_ref_t *config_refs;
|
||||
int config_ref_count;
|
||||
cbm_prescan_route_t *routes;
|
||||
int route_count;
|
||||
} cbm_prescan_t;
|
||||
|
||||
/* ── Pipeline context (internal) ─────────────────────────────────── */
|
||||
|
||||
/* Shared context passed to each pass function.
|
||||
@@ -83,18 +44,6 @@ typedef struct {
|
||||
* and pass_calls/usages/semantic reuse cached results instead of re-extracting.
|
||||
* Indexed by file position in the files[] array. Owned by pipeline.c. */
|
||||
CBMFileResult **result_cache;
|
||||
|
||||
/* Pre-scan cache — indexed by file position, parallel to result_cache.
|
||||
* Populated during extraction phase while source is in memory.
|
||||
* Contains HTTP call sites and config file references extracted from source.
|
||||
* Eliminates disk re-reads in httplinks and configlink passes. */
|
||||
cbm_prescan_t *prescan_cache;
|
||||
int prescan_count;
|
||||
|
||||
/* File path → index hash map for prescan lookup by rel_path.
|
||||
* Allows httplinks (which iterates graph nodes, not files) to find
|
||||
* prescan data by node->file_path. */
|
||||
CBMHashTable *prescan_path_map;
|
||||
} cbm_pipeline_ctx_t;
|
||||
|
||||
/* Check cancellation. Returns non-zero if cancelled. */
|
||||
@@ -399,8 +348,7 @@ int cbm_pipeline_pass_httplinks(cbm_pipeline_ctx_t *ctx);
|
||||
/* Pre-dump pass: decorator tags enrichment (operates on gbuf). */
|
||||
int cbm_pipeline_pass_decorator_tags(cbm_gbuf_t *gbuf, const char *project);
|
||||
|
||||
/* Pre-dump pass: config ↔ code linking.
|
||||
* Uses prescan cache when available, falls back to disk reads. */
|
||||
/* Pre-dump pass: config ↔ code linking. */
|
||||
int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx);
|
||||
|
||||
/* ── Env URL scanner (pass_envscan.c) ────────────────────────────── */
|
||||
|
||||
Reference in new issue
Block a user