From b00e1f74c2c657a648f78da874b14600784b2baf Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 27 Mar 2026 18:29:19 +0100 Subject: [PATCH] AST-based route registration + remove prescan infrastructure Route registration detection during AST extraction: callee QN matched against router frameworks (gin, chi, Express, Laravel, Ktor, etc.), handler reference captured as second_arg_name, Route + HANDLES edges created during resolution. Replaces disk-reading route discovery for non-decorator frameworks. Remove prescan infrastructure entirely (-350 net lines): - cbm_prescan_t, cbm_prescan_http_site_t, cbm_prescan_route_t, cbm_prescan_config_ref_t structs removed from pipeline_internal.h - prescan_http_sites, prescan_routes, prescan_config_refs, prescan_add_route removed from pass_parallel.c - prescan fast-paths removed from pass_httplinks.c and pass_configlink.c - prescan allocation/cleanup removed from pipeline.c All post-passes now work without prescan: - HTTP/async edges: service_patterns during resolution - Route registration: CBM_SVC_ROUTE_REG during resolution - Decorator routes: gbuf properties (no disk reads) - Config file refs: disk fallback (few files) --- internal/cbm/cbm.h | 1 + internal/cbm/extract_calls.c | 22 +- internal/cbm/service_patterns.c | 134 ++++++++++- internal/cbm/service_patterns.h | 14 +- src/pipeline/pass_calls.c | 40 ++++ src/pipeline/pass_configlink.c | 137 +---------- src/pipeline/pass_httplinks.c | 57 +---- src/pipeline/pass_parallel.c | 355 +++------------------------- src/pipeline/pipeline.c | 43 ---- src/pipeline/pipeline_incremental.c | 9 +- src/pipeline/pipeline_internal.h | 54 +---- 11 files changed, 258 insertions(+), 608 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 4ba963aa0..0560cf665 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -111,6 +111,7 @@ typedef struct { const char *callee_name; // raw callee text ("pkg.Func", "foo") const char *enclosing_func_qn; // QN of enclosing function (or module QN) const char *first_string_arg; // first string literal argument (URL, topic, key) or NULL + const char *second_arg_name; // second argument identifier (handler ref) or NULL } CBMCall; typedef struct { diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index 3eeb20801..51d4204bb 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -267,7 +267,7 @@ static void walk_calls(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) if (callee && callee[0]) { // Skip keywords if (!cbm_is_keyword(callee, ctx->language)) { - CBMCall call; + CBMCall call = {0}; call.callee_name = callee; call.enclosing_func_qn = cbm_enclosing_func_qn_cached(ctx, node); call.first_string_arg = NULL; @@ -504,6 +504,26 @@ void handle_calls(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec, Walk } } + /* Extract second argument name (handler ref for route registrations). + * Pattern: router.GET("/path", handlerFunc) → second_arg_name = "handlerFunc" + * Only extracted when first_string_arg looks like a path. */ + if (call.first_string_arg != NULL && call.first_string_arg[0] == '/' && + !ts_node_is_null(args)) { + uint32_t nc2 = ts_node_named_child_count(args); + for (uint32_t ai = 1; ai < nc2 && ai < 4 && !call.second_arg_name; ai++) { + TSNode arg2 = ts_node_named_child(args, ai); + const char *ak2 = ts_node_type(arg2); + if (strcmp(ak2, "identifier") == 0) { + call.second_arg_name = cbm_node_text(ctx->arena, arg2, ctx->source); + } else if (strcmp(ak2, "member_expression") == 0 || + strcmp(ak2, "selector_expression") == 0 || + strcmp(ak2, "attribute") == 0 || + strcmp(ak2, "field_expression") == 0) { + call.second_arg_name = cbm_node_text(ctx->arena, arg2, ctx->source); + } + } + } + cbm_calls_push(&ctx->result->calls, ctx->arena, call); } } diff --git a/internal/cbm/service_patterns.c b/internal/cbm/service_patterns.c index 661df6656..649de9ef6 100644 --- a/internal/cbm/service_patterns.c +++ b/internal/cbm/service_patterns.c @@ -282,13 +282,122 @@ static const lib_pattern_t config_libraries[] = { {NULL, CBM_SVC_NONE, NULL}, }; -/* ── HTTP method inference from function/method name suffix ───── */ +/* Route registration frameworks — callee resolves to one of these AND + * has an HTTP method suffix → CBM_SVC_ROUTE_REG. + * Distinguished from HTTP clients: "gin.GET" registers a handler, + * "requests.get" makes an outbound HTTP call. */ +static const lib_pattern_t route_reg_libraries[] = { + /* Go */ + {"gin-gonic/gin", CBM_SVC_ROUTE_REG, NULL}, + {"gin.", CBM_SVC_ROUTE_REG, NULL}, + {"go-chi/chi", CBM_SVC_ROUTE_REG, NULL}, + {"chi.", CBM_SVC_ROUTE_REG, NULL}, + {"gorilla/mux", CBM_SVC_ROUTE_REG, NULL}, + {"labstack/echo", CBM_SVC_ROUTE_REG, NULL}, + {"echo.", CBM_SVC_ROUTE_REG, NULL}, + {"gofiber/fiber", CBM_SVC_ROUTE_REG, NULL}, + {"fiber.", CBM_SVC_ROUTE_REG, NULL}, + {"net/http.ServeMux", CBM_SVC_ROUTE_REG, NULL}, + {"http.ServeMux", CBM_SVC_ROUTE_REG, NULL}, + {"httprouter", CBM_SVC_ROUTE_REG, NULL}, + /* JavaScript / TypeScript */ + {"express", CBM_SVC_ROUTE_REG, NULL}, + {"fastify", CBM_SVC_ROUTE_REG, NULL}, + {"koa-router", CBM_SVC_ROUTE_REG, NULL}, + {"hono", CBM_SVC_ROUTE_REG, NULL}, + {"hapi", CBM_SVC_ROUTE_REG, NULL}, + + /* Python (non-decorator, e.g., Flask add_url_rule) */ + {"flask", CBM_SVC_ROUTE_REG, NULL}, + {"FastAPI", CBM_SVC_ROUTE_REG, NULL}, + {"starlette", CBM_SVC_ROUTE_REG, NULL}, + + /* PHP */ + {"Laravel", CBM_SVC_ROUTE_REG, NULL}, + {"Illuminate.Routing", CBM_SVC_ROUTE_REG, NULL}, + {"Symfony.Routing", CBM_SVC_ROUTE_REG, NULL}, + + /* Kotlin */ + {"ktor.server", CBM_SVC_ROUTE_REG, NULL}, + {"ktor.routing", CBM_SVC_ROUTE_REG, NULL}, + + /* Rust */ + {"actix-web", CBM_SVC_ROUTE_REG, NULL}, + {"actix_web", CBM_SVC_ROUTE_REG, NULL}, + {"axum", CBM_SVC_ROUTE_REG, NULL}, + {"rocket", CBM_SVC_ROUTE_REG, NULL}, + + /* Java */ + {"Spring", CBM_SVC_ROUTE_REG, NULL}, + {"jakarta.ws.rs", CBM_SVC_ROUTE_REG, NULL}, + + /* C# */ + {"Microsoft.AspNetCore", CBM_SVC_ROUTE_REG, NULL}, + {"MapGet", CBM_SVC_ROUTE_REG, NULL}, + {"MapPost", CBM_SVC_ROUTE_REG, NULL}, + + /* Ruby */ + {"ActionDispatch", CBM_SVC_ROUTE_REG, NULL}, + {"Sinatra", CBM_SVC_ROUTE_REG, NULL}, + + /* Elixir */ + {"Phoenix.Router", CBM_SVC_ROUTE_REG, NULL}, + + /* Scala */ + {"akka.http.scaladsl.server", CBM_SVC_ROUTE_REG, NULL}, + {"play.api.routing", CBM_SVC_ROUTE_REG, NULL}, + + {NULL, CBM_SVC_NONE, NULL}, +}; + +/* Method suffix type (used by both route registration and HTTP client tables) */ typedef struct { const char *suffix; const char *method; } method_suffix_t; +/* Route registration method suffixes — matched on callee name. + * These are methods on router objects that register handlers. */ +static const method_suffix_t route_reg_suffixes[] = { + /* HTTP method registrations */ + {".GET", "GET"}, + {".Get", "GET"}, + {".get", "GET"}, + {".POST", "POST"}, + {".Post", "POST"}, + {".post", "POST"}, + {".PUT", "PUT"}, + {".Put", "PUT"}, + {".put", "PUT"}, + {".DELETE", "DELETE"}, + {".Delete", "DELETE"}, + {".delete", "DELETE"}, + {".PATCH", "PATCH"}, + {".Patch", "PATCH"}, + {".patch", "PATCH"}, + /* Handle/HandleFunc (Go stdlib, gorilla) */ + {".Handle", "ANY"}, + {".HandleFunc", "ANY"}, + {".handle", "ANY"}, + /* Framework-specific route registration */ + {".Route", "ANY"}, + {".route", "ANY"}, + {"::get", "GET"}, + {"::post", "POST"}, + {"::put", "PUT"}, + {"::delete", "DELETE"}, + {"::patch", "PATCH"}, + /* Minimal API (C# ASP.NET) */ + {".MapGet", "GET"}, + {".MapPost", "POST"}, + {".MapPut", "PUT"}, + {".MapDelete", "DELETE"}, + {NULL, NULL}, +}; + +/* ── HTTP method inference from function/method name suffix ───── */ + static const method_suffix_t method_suffixes[] = { {".get", "GET"}, {".Get", "GET"}, {".GET", "GET"}, {".post", "POST"}, {".Post", "POST"}, {".POST", "POST"}, @@ -331,7 +440,14 @@ cbm_svc_kind_t cbm_service_pattern_match(const char *resolved_qn) { return CBM_SVC_NONE; } - const lib_pattern_t *p = match_qn(resolved_qn, http_libraries); + /* Route registration checked first — prevents gin/echo from matching + * as HTTP clients (both have .get/.post suffixes). */ + const lib_pattern_t *p = match_qn(resolved_qn, route_reg_libraries); + if (p) { + return p->kind; + } + + p = match_qn(resolved_qn, http_libraries); if (p) { return p->kind; } @@ -363,6 +479,20 @@ const char *cbm_service_pattern_http_method(const char *callee_name) { return NULL; } +const char *cbm_service_pattern_route_method(const char *callee_name) { + if (!callee_name) { + return NULL; + } + size_t clen = strlen(callee_name); + for (int i = 0; route_reg_suffixes[i].suffix != NULL; i++) { + size_t slen = strlen(route_reg_suffixes[i].suffix); + if (clen >= slen && strcmp(callee_name + clen - slen, route_reg_suffixes[i].suffix) == 0) { + return route_reg_suffixes[i].method; + } + } + return NULL; +} + const char *cbm_service_pattern_broker(const char *resolved_qn) { if (!resolved_qn) { return NULL; diff --git a/internal/cbm/service_patterns.h b/internal/cbm/service_patterns.h index adff9a72d..b526a8f7f 100644 --- a/internal/cbm/service_patterns.h +++ b/internal/cbm/service_patterns.h @@ -13,10 +13,11 @@ /* Edge type returned by pattern match. */ typedef enum { - CBM_SVC_NONE = 0, /* Not a service pattern — use normal CALLS */ - CBM_SVC_HTTP = 1, /* Synchronous HTTP client call */ - CBM_SVC_ASYNC = 2, /* Async dispatch (message broker, task queue) */ - CBM_SVC_CONFIG = 3, /* Config/env accessor */ + CBM_SVC_NONE = 0, /* Not a service pattern — use normal CALLS */ + CBM_SVC_HTTP = 1, /* Synchronous HTTP client call */ + CBM_SVC_ASYNC = 2, /* Async dispatch (message broker, task queue) */ + CBM_SVC_CONFIG = 3, /* Config/env accessor */ + CBM_SVC_ROUTE_REG = 4, /* Route registration (router.GET, app.get, Route::post) */ } cbm_svc_kind_t; /* Initialize the pattern lookup tables. Call once at startup. Thread-safe after init. */ @@ -32,6 +33,11 @@ cbm_svc_kind_t cbm_service_pattern_match(const char *resolved_qn); * Returns NULL if method cannot be inferred. */ const char *cbm_service_pattern_http_method(const char *callee_name); +/* Get the HTTP method from a route registration callee name suffix + * (e.g., "router.GET" → "GET", "app.post" → "POST"). + * Returns NULL if not a known route registration method. */ +const char *cbm_service_pattern_route_method(const char *callee_name); + /* Get the broker name for an async QN (e.g., "pubsub" from a Pub/Sub QN). * Returns NULL if not an async pattern. */ const char *cbm_service_pattern_broker(const char *resolved_qn); diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 9069de01d..9147df532 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -275,6 +275,46 @@ int cbm_pipeline_pass_calls(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file /* Classify edge type by library in resolved QN */ cbm_svc_kind_t svc = cbm_service_pattern_match(res.qualified_name); + if (svc == CBM_SVC_ROUTE_REG && call->first_string_arg != NULL && + call->first_string_arg[0] == '/') { + /* Route registration: router.GET("/path", handler) → Route + HANDLES */ + const char *method = cbm_service_pattern_route_method(call->callee_name); + char route_qn[CBM_ROUTE_QN_SIZE]; + snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", method ? method : "ANY", + call->first_string_arg); + char route_props[256]; + snprintf(route_props, sizeof(route_props), "{\"method\":\"%s\"}", + method ? method : "ANY"); + int64_t route_id = cbm_gbuf_upsert_node(ctx->gbuf, "Route", call->first_string_arg, + route_qn, "", 0, 0, route_props); + + char props[512]; + snprintf(props, sizeof(props), + "{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}", + call->callee_name, call->first_string_arg); + cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, route_id, "CALLS", props); + + /* Resolve handler and create HANDLES edge */ + if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') { + cbm_resolution_t hres = + cbm_registry_resolve(ctx->registry, call->second_arg_name, module_qn, + imp_keys, imp_vals, imp_count); + if (hres.qualified_name != NULL && hres.qualified_name[0] != '\0') { + const cbm_gbuf_node_t *handler = + cbm_gbuf_find_by_qn(ctx->gbuf, hres.qualified_name); + if (handler != NULL) { + char hprops[256]; + snprintf(hprops, sizeof(hprops), "{\"handler\":\"%s\"}", + hres.qualified_name); + cbm_gbuf_insert_edge(ctx->gbuf, handler->id, route_id, "HANDLES", + hprops); + } + } + } + resolved++; + continue; + } + if (svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) { /* HTTP/async call — route through Route node for cross-service traversal. * Only create Route if string looks like a URL (HTTP) or topic name (async). */ diff --git a/src/pipeline/pass_configlink.c b/src/pipeline/pass_configlink.c index d6bf94936..e384725ab 100644 --- a/src/pipeline/pass_configlink.c +++ b/src/pipeline/pass_configlink.c @@ -353,134 +353,6 @@ static int strategy_dep_imports(cbm_gbuf_t *gb) { /* ── Strategy 3: Config File Path → Code String Reference ───────── */ -/* Prescan-based: config refs were extracted during extraction phase. */ -static int strategy_file_refs_prescan(cbm_gbuf_t *gb, const char *project, - const cbm_prescan_t *prescan_cache, int prescan_count, - CBMHashTable *prescan_path_map) { - /* Collect config Module nodes for matching */ - const cbm_gbuf_node_t **modules = NULL; - int mod_count = 0; - if (cbm_gbuf_find_by_label(gb, "Module", &modules, &mod_count) != 0) { - return 0; - } - - /* Build basename → Module and fullpath → Module maps */ - typedef struct { - const char *key; - int64_t node_id; - } path_map_t; - - path_map_t *base_map = calloc((size_t)mod_count, sizeof(path_map_t)); - path_map_t *full_map = calloc((size_t)mod_count, sizeof(path_map_t)); - int base_count = 0; - int full_count = 0; - - for (int i = 0; i < mod_count; i++) { - if (cbm_has_config_extension(modules[i]->file_path)) { - const char *slash = strrchr(modules[i]->file_path, '/'); - base_map[base_count].key = slash ? slash + 1 : modules[i]->file_path; - base_map[base_count].node_id = modules[i]->id; - base_count++; - - full_map[full_count].key = modules[i]->file_path; - full_map[full_count].node_id = modules[i]->id; - full_count++; - } - } - - if (base_count == 0) { - free(base_map); - free(full_map); - return 0; - } - - int edge_count = 0; - - /* Iterate prescan results — each file's config refs were extracted during extraction */ - for (int fi = 0; fi < prescan_count; fi++) { - const cbm_prescan_t *ps = &prescan_cache[fi]; - if (ps->config_ref_count == 0) { - continue; - } - - /* Find the Module node for this file via prescan_path_map */ - /* We need to reverse-lookup: file_idx → rel_path → Module QN. - * But we don't have the files array here. Instead, iterate modules - * and check which ones have prescan data. */ - } - - /* Alternative approach: iterate non-config modules and check their prescan */ - for (int i = 0; i < mod_count; i++) { - if (cbm_has_config_extension(modules[i]->file_path)) { - continue; - } - - /* Look up prescan data for this file */ - void *val = cbm_ht_get(prescan_path_map, modules[i]->file_path); - if (!val) { - continue; - } - int file_idx = (int)((intptr_t)val - 1); - if (file_idx < 0 || file_idx >= prescan_count) { - continue; - } - const cbm_prescan_t *ps = &prescan_cache[file_idx]; - if (ps->config_ref_count == 0) { - continue; - } - - for (int ri = 0; ri < ps->config_ref_count; ri++) { - const char *ref_path = ps->config_refs[ri].ref_path; - - int64_t target_id = 0; - double confidence = 0.0; - - /* Try full path match */ - for (int fi2 = 0; fi2 < full_count; fi2++) { - if (strcmp(full_map[fi2].key, ref_path) == 0) { - target_id = full_map[fi2].node_id; - confidence = CONF_FILE_FULLPATH; - break; - } - } - - /* Try basename match */ - if (target_id == 0) { - const char *ref_slash = strrchr(ref_path, '/'); - const char *ref_base = ref_slash ? ref_slash + 1 : ref_path; - for (int bi = 0; bi < base_count; bi++) { - if (strcmp(base_map[bi].key, ref_base) == 0) { - target_id = base_map[bi].node_id; - confidence = CONF_FILE_BASENAME; - break; - } - } - } - - if (target_id != 0) { - char *module_qn = cbm_pipeline_fqn_module(project, modules[i]->file_path); - if (module_qn) { - const cbm_gbuf_node_t *src_node = cbm_gbuf_find_by_qn(gb, module_qn); - if (src_node) { - char props[512]; - snprintf(props, sizeof(props), - "{\"strategy\":\"file_reference\",\"confidence\":%.2f,\"ref_" - "path\":\"%s\"}", - confidence, ref_path); - cbm_gbuf_insert_edge(gb, src_node->id, target_id, "CONFIGURES", props); - edge_count++; - } - free(module_qn); - } - } - } - } - - free(base_map); - free(full_map); - return edge_count; -} - /* Disk-based fallback: reads source from disk (sequential pipeline path). */ // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) static int strategy_file_refs_disk(cbm_gbuf_t *gb, const char *project, const char *repo_path) { @@ -646,8 +518,6 @@ static int strategy_file_refs_disk(cbm_gbuf_t *gb, const char *project, const ch int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx) { cbm_gbuf_t *gb = ctx->gbuf; - const char *project = ctx->project_name; - const char *repo_path = ctx->repo_path; /* Early exit: check if any config files exist in the project. */ bool has_config = false; @@ -694,11 +564,8 @@ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx) { cbm_log_info("configlinker.strategy", "name", "dep_import", "edges", buf2); int ref_edges = 0; - if (ctx->prescan_cache && ctx->prescan_path_map) { - ref_edges = strategy_file_refs_prescan(gb, project, ctx->prescan_cache, ctx->prescan_count, - ctx->prescan_path_map); - } else if (repo_path) { - ref_edges = strategy_file_refs_disk(gb, project, repo_path); + if (ctx->repo_path) { + ref_edges = strategy_file_refs_disk(gb, ctx->project_name, ctx->repo_path); } snprintf(buf3, sizeof(buf3), "%d", ref_edges); cbm_log_info("configlinker.strategy", "name", "file_ref", "edges", buf3); diff --git a/src/pipeline/pass_httplinks.c b/src/pipeline/pass_httplinks.c index 7ecdda719..fd31ec806 100644 --- a/src/pipeline/pass_httplinks.c +++ b/src/pipeline/pass_httplinks.c @@ -1246,7 +1246,7 @@ int cbm_pipeline_pass_httplinks(cbm_pipeline_ctx_t *ctx) { worker_count = 1; } - /* ── Phase 1: Route collection from prescan or parallel discovery ── */ + /* ── Phase 1: Route collection from decorator properties + disk ── */ /* Need label_counts for fallback path */ const char *route_labels[] = {"Function", "Method", "Module"}; @@ -1263,27 +1263,11 @@ int cbm_pipeline_pass_httplinks(cbm_pipeline_ctx_t *ctx) { } int route_count = 0; - if (ctx->prescan_cache) { - /* Fast path: routes were extracted during the extraction phase. - * Zero disk reads, zero JSON parsing, zero node iteration. */ - for (int fi = 0; fi < ctx->prescan_count; fi++) { - const cbm_prescan_t *ps = &ctx->prescan_cache[fi]; - for (int ri = 0; ri < ps->route_count && route_count < MAX_ROUTES; ri++) { - const cbm_prescan_route_t *src = &ps->routes[ri]; - cbm_route_handler_t *dst = &routes[route_count]; - snprintf(dst->path, sizeof(dst->path), "%s", src->path); - snprintf(dst->method, sizeof(dst->method), "%s", src->method); - snprintf(dst->function_name, sizeof(dst->function_name), "%s", src->function_name); - snprintf(dst->qualified_name, sizeof(dst->qualified_name), "%s", - src->qualified_name); - snprintf(dst->handler_ref, sizeof(dst->handler_ref), "%s", src->handler_ref); - dst->resolved_handler_qn[0] = '\0'; - snprintf(dst->protocol, sizeof(dst->protocol), "%s", src->protocol); - route_count++; - } - } - } else { - /* Fallback: parallel route discovery from disk (sequential pipeline path) */ + { + /* Route discovery: decorator-based (from gbuf properties) + disk-based + * (Go/Express/Laravel/Ktor source reading). AST-based route registration + * detection (CBM_SVC_ROUTE_REG) creates Route+HANDLES during resolution, + * so this path is primarily for decorator-discovered routes now. */ int total_route_nodes = 0; for (int li = 0; li < 3; li++) { total_route_nodes += label_counts[li]; @@ -1353,33 +1337,16 @@ int cbm_pipeline_pass_httplinks(cbm_pipeline_ctx_t *ctx) { /* ── Phase 4: Route nodes + HANDLES edges (serial) ────────── */ int route_nodes = insert_route_nodes(ctx, routes, route_count); - /* ── Phase 5: Call site collection from prescan cache ────────── */ - /* During extraction, each file's source was scanned for HTTP/async - * keywords and URL paths. Results are in ctx->prescan_cache[file_idx]. - * This eliminates 2M+ disk reads (the old approach read every function - * from disk just to check for keywords, then discarded 99.9%). */ + /* ── Phase 5: Call site collection ──────────────────────────── */ cbm_http_call_site_t *sites = calloc(MAX_CALL_SITES, sizeof(cbm_http_call_site_t)); int site_count = 0; - if (sites && ctx->prescan_cache) { - for (int fi = 0; fi < ctx->prescan_count; fi++) { - const cbm_prescan_t *ps = &ctx->prescan_cache[fi]; - for (int si = 0; si < ps->http_site_count && site_count < MAX_CALL_SITES; si++) { - const cbm_prescan_http_site_t *src = &ps->http_sites[si]; - cbm_http_call_site_t *dst = &sites[site_count]; - snprintf(dst->path, sizeof(dst->path), "%s", src->path); - snprintf(dst->method, sizeof(dst->method), "%s", src->method); - snprintf(dst->source_name, sizeof(dst->source_name), "%s", src->source_name); - snprintf(dst->source_qn, sizeof(dst->source_qn), "%s", src->source_qn); - snprintf(dst->source_label, sizeof(dst->source_label), "%s", src->source_label); - dst->is_async = src->is_async; - site_count++; - } - } - } else if (sites) { - /* Fallback: no prescan (sequential pipeline path). - * Use the old parallel disk-reading approach. */ + if (sites) { + /* Call site collection: parallel disk-reading approach. + * Note: HTTP_CALLS/ASYNC_CALLS edges are now also created during + * resolution via service_patterns. This path provides additional + * URL-based matching for call sites not caught by QN matching. */ const char *site_labels[] = {"Function", "Method"}; const cbm_gbuf_node_t **all_site_nodes = NULL; const char **all_site_labels = NULL; diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 7c22a62ee..dcdd8573d 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -351,317 +351,11 @@ typedef struct { _Atomic int next_worker_id; CBMFileResult **result_cache; - cbm_prescan_t *prescan_cache; _Atomic int64_t *shared_ids; _Atomic int *cancelled; _Atomic int next_file_idx; } extract_ctx_t; -/* ── Pre-scan: extract HTTP sites + config refs while source is in memory ── */ - -/* Extract lines [start_line..end_line] from source buffer as a malloc'd string. */ -static char *extract_lines(const char *source, int source_len, int start_line, int end_line) { - if (start_line <= 0 || end_line <= 0 || end_line < start_line) { - return NULL; - } - int line = 1; - const char *p = source; - const char *end = source + source_len; - const char *range_start = NULL; - const char *range_end = NULL; - - while (p < end) { - if (line == start_line) { - range_start = p; - } - const char *nl = memchr(p, '\n', (size_t)(end - p)); - if (!nl) { - nl = end; - } - if (line == end_line) { - range_end = (nl < end) ? nl + 1 : end; - break; - } - p = nl + 1; - line++; - } - if (!range_start) { - return NULL; - } - if (!range_end) { - range_end = end; - } - size_t len = (size_t)(range_end - range_start); - // NOLINTNEXTLINE(clang-analyzer-optin.taint.TaintedAlloc) - char *out = malloc(len + 1); - if (!out) { - return NULL; - } - memcpy(out, range_start, len); - out[len] = '\0'; - return out; -} - -/* Scan function bodies for HTTP/async keywords, extract URL paths. */ -static void prescan_http_sites(const char *source, int source_len, const CBMFileResult *result, - cbm_prescan_t *ps) { - for (int d = 0; d < result->defs.count; d++) { - const CBMDefinition *def = &result->defs.items[d]; - if (!def->qualified_name || !def->name || !def->label) { - continue; - } - if (strcmp(def->label, "Function") != 0 && strcmp(def->label, "Method") != 0) { - continue; - } - if (def->start_line == 0 || def->end_line == 0) { - continue; - } - /* Skip Python dunder methods */ - size_t nlen = strlen(def->name); - if (nlen > 4 && def->name[0] == '_' && def->name[1] == '_' && def->name[nlen - 1] == '_' && - def->name[nlen - 2] == '_') { - continue; - } - - char *func_src = - extract_lines(source, source_len, (int)def->start_line, (int)def->end_line); - if (!func_src) { - continue; - } - - /* Keyword pre-filter */ - bool has_http = false; - for (int k = 0; k < cbm_http_client_keywords_count; k++) { - if (strstr(func_src, cbm_http_client_keywords[k])) { - has_http = true; - break; - } - } - bool has_async = false; - for (int k = 0; k < cbm_async_dispatch_keywords_count; k++) { - if (strstr(func_src, cbm_async_dispatch_keywords[k])) { - has_async = true; - break; - } - } - - if (!has_http && !has_async) { - free(func_src); - continue; - } - - bool is_async = (has_async && !has_http) != 0; - - /* Extract URL paths */ - char *paths[64]; - int path_count = cbm_extract_url_paths(func_src, paths, 64); - - for (int p = 0; p < path_count; p++) { - cbm_prescan_http_site_t *site = - realloc(ps->http_sites, - (size_t)(ps->http_site_count + 1) * sizeof(cbm_prescan_http_site_t)); - if (!site) { - free(paths[p]); - continue; - } - ps->http_sites = site; - cbm_prescan_http_site_t *s = &ps->http_sites[ps->http_site_count]; - snprintf(s->path, sizeof(s->path), "%s", paths[p]); - s->method[0] = '\0'; - snprintf(s->source_name, sizeof(s->source_name), "%s", def->name); - snprintf(s->source_qn, sizeof(s->source_qn), "%s", def->qualified_name); - snprintf(s->source_label, sizeof(s->source_label), "%s", def->label); - s->is_async = is_async; - ps->http_site_count++; - free(paths[p]); - } - /* Free any remaining paths we couldn't store */ - for (int p = path_count; p < path_count; p++) { - free(paths[p]); - } - free(func_src); - } -} - -/* Helper: append a route to prescan results. */ -static void prescan_add_route(cbm_prescan_t *ps, const cbm_route_handler_t *rh) { - cbm_prescan_route_t *routes = - realloc(ps->routes, (size_t)(ps->route_count + 1) * sizeof(cbm_prescan_route_t)); - if (!routes) { - return; - } - ps->routes = routes; - cbm_prescan_route_t *r = &routes[ps->route_count]; - snprintf(r->path, sizeof(r->path), "%s", rh->path); - snprintf(r->method, sizeof(r->method), "%s", rh->method); - snprintf(r->function_name, sizeof(r->function_name), "%s", rh->function_name); - snprintf(r->qualified_name, sizeof(r->qualified_name), "%s", rh->qualified_name); - snprintf(r->handler_ref, sizeof(r->handler_ref), "%s", rh->handler_ref); - snprintf(r->protocol, sizeof(r->protocol), "%s", rh->protocol); - ps->route_count++; -} - -/* Check file extension for source-based route languages. */ -static bool is_route_source_lang(const char *path) { - if (!path) { - return false; - } - size_t len = strlen(path); - /* Go */ - if (len > 3 && strcmp(path + len - 3, ".go") == 0) { - return true; - } - /* JavaScript/TypeScript */ - if (len > 3 && (strcmp(path + len - 3, ".js") == 0 || strcmp(path + len - 3, ".ts") == 0)) { - return true; - } - if (len > 4 && (strcmp(path + len - 4, ".jsx") == 0 || strcmp(path + len - 4, ".tsx") == 0)) { - return true; - } - /* PHP */ - if (len > 4 && strcmp(path + len - 4, ".php") == 0) { - return true; - } - /* Kotlin */ - if (len > 3 && strcmp(path + len - 3, ".kt") == 0) { - return true; - } - if (len > 4 && strcmp(path + len - 4, ".kts") == 0) { - return true; - } - return false; -} - -/* Scan function definitions for HTTP routes (decorator + source-based). */ -static void prescan_routes(const char *source, int source_len, const CBMFileResult *result, - const char *rel_path, cbm_prescan_t *ps) { - bool is_route_lang = is_route_source_lang(rel_path); - - for (int d = 0; d < result->defs.count; d++) { - const CBMDefinition *def = &result->defs.items[d]; - if (!def->qualified_name || !def->name) { - continue; - } - - cbm_route_handler_t routes[16]; - int total = 0; - - /* Decorator-based routes (Python, Java) */ - if (def->decorators && def->decorators[0]) { - int ndec = 0; - while (def->decorators[ndec]) { - ndec++; - } - int nr = cbm_extract_python_routes(def->name, def->qualified_name, - (const char **)def->decorators, ndec, routes + total, - 16 - total); - total += nr; - nr = cbm_extract_java_routes(def->name, def->qualified_name, - (const char **)def->decorators, ndec, routes + total, - 16 - total); - total += nr; - } - - /* Source-based routes (Go, Express, Laravel, Ktor) — only for matching languages */ - if (is_route_lang && def->start_line > 0 && def->end_line > 0 && total < 16) { - char *func_src = - extract_lines(source, source_len, (int)def->start_line, (int)def->end_line); - if (func_src) { - int nr = cbm_extract_go_routes(def->name, def->qualified_name, func_src, - routes + total, 16 - total); - total += nr; - nr = cbm_extract_express_routes(def->name, def->qualified_name, func_src, - routes + total, 16 - total); - total += nr; - nr = cbm_extract_laravel_routes(def->name, def->qualified_name, func_src, - routes + total, 16 - total); - total += nr; - nr = cbm_extract_ktor_routes(def->name, def->qualified_name, func_src, - routes + total, 16 - total); - total += nr; - free(func_src); - } - } - - for (int r = 0; r < total; r++) { - prescan_add_route(ps, &routes[r]); - } - } - - /* Module-level routes (PHP Laravel, JS/TS Express) */ - if (is_route_lang && source_len > 0) { - size_t plen = strlen(rel_path); - bool is_php = (plen > 4 && strcmp(rel_path + plen - 4, ".php") == 0) != 0; - bool is_js = ((plen > 3 && (strcmp(rel_path + plen - 3, ".js") == 0 || - strcmp(rel_path + plen - 3, ".ts") == 0)) || - (plen > 4 && (strcmp(rel_path + plen - 4, ".jsx") == 0 || - strcmp(rel_path + plen - 4, ".tsx") == 0))) != 0; - if (is_php || is_js) { - /* Build a module name from rel_path */ - const char *slash = strrchr(rel_path, '/'); - const char *basename = slash ? slash + 1 : rel_path; - cbm_route_handler_t mod_routes[16]; - int total = 0; - if (is_php) { - total += cbm_extract_laravel_routes(basename, "", source, mod_routes + total, - 16 - total); - } - if (is_js) { - total += cbm_extract_express_routes(basename, "", source, mod_routes + total, - 16 - total); - } - for (int r = 0; r < total; r++) { - prescan_add_route(ps, &mod_routes[r]); - } - } - } -} - -/* Scan full source for config file path references in string literals. */ -static void prescan_config_refs(const char *source, cbm_prescan_t *ps) { - /* Quick reject: no config extension substring in source at all */ - static const char *exts[] = {".toml", ".yaml", ".yml", ".ini", ".json", - ".xml", ".conf", ".cfg", ".env", NULL}; - bool has_any = false; - for (int i = 0; exts[i]; i++) { - if (strstr(source, exts[i])) { - has_any = true; - break; - } - } - if (!has_any) { - return; - } - - cbm_regex_t re; - if (cbm_regcomp(&re, "[\"']([^\"']*\\.(toml|yaml|yml|ini|json|xml|conf|cfg|env))[\"']", - CBM_REG_EXTENDED) != 0) { - return; - } - - cbm_regmatch_t match[3]; - const char *cursor = source; - while (cbm_regexec(&re, cursor, 3, match, 0) == 0) { - int start = match[1].rm_so; - int end = match[1].rm_eo; - int ref_len = end - start; - - if (ref_len > 0 && ref_len < 256) { - cbm_prescan_config_ref_t *refs = - realloc(ps->config_refs, - (size_t)(ps->config_ref_count + 1) * sizeof(cbm_prescan_config_ref_t)); - if (refs) { - ps->config_refs = refs; - memcpy(refs[ps->config_ref_count].ref_path, cursor + start, (size_t)ref_len); - refs[ps->config_ref_count].ref_path[ref_len] = '\0'; - ps->config_ref_count++; - } - } - cursor += match[0].rm_eo; - } - cbm_regfree(&re); -} - static void extract_worker(int worker_id, void *ctx_ptr) { extract_ctx_t *ec = ctx_ptr; extract_worker_state_t *ws = &ec->workers[worker_id]; @@ -745,17 +439,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) { * are released before the slab is bulk-reclaimed. */ cbm_free_tree(result); - /* Pre-scan source for HTTP call sites and config file references - * while the buffer is still in memory. Eliminates 2M+ disk reads - * in httplinks and 62K+ disk reads in configlink. */ - if (ec->prescan_cache) { - cbm_prescan_t *ps = &ec->prescan_cache[file_idx]; - prescan_routes(source, source_len, result, fi->rel_path, ps); - prescan_http_sites(source, source_len, result, ps); - prescan_config_refs(source, ps); - } - - /* Free source buffer — prescan captured everything needed. */ + /* Free source buffer — extraction captured everything needed. */ free_source(source); /* Cache result (arena + extracted data, no tree) for Phase 3B and Phase 4 */ @@ -840,7 +524,6 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *files, int fi .workers = workers, .max_workers = worker_count, .result_cache = result_cache, - .prescan_cache = ctx->prescan_cache, .shared_ids = shared_ids, .cancelled = ctx->cancelled, }; @@ -1003,10 +686,41 @@ typedef struct { * Extracted from resolve_worker to keep cognitive complexity under threshold. */ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target, const CBMCall *call, - const cbm_resolution_t *res) { + const cbm_resolution_t *res, const char *module_qn, + const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, + const char **imp_keys, const char **imp_vals, int imp_count) { cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); const char *arg = call->first_string_arg; + if (svc == CBM_SVC_ROUTE_REG && arg != NULL && arg[0] == '/') { + const char *method = cbm_service_pattern_route_method(call->callee_name); + char route_qn[CBM_ROUTE_QN_SIZE]; + snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", method ? method : "ANY", arg); + char route_props[256]; + snprintf(route_props, sizeof(route_props), "{\"method\":\"%s\"}", method ? method : "ANY"); + int64_t route_id = + cbm_gbuf_upsert_node(gbuf, "Route", arg, route_qn, "", 0, 0, route_props); + char props[512]; + snprintf(props, sizeof(props), + "{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}", + call->callee_name, arg); + cbm_gbuf_insert_edge(gbuf, source->id, route_id, "CALLS", props); + if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') { + cbm_resolution_t hres = cbm_registry_resolve(registry, call->second_arg_name, module_qn, + imp_keys, imp_vals, imp_count); + if (hres.qualified_name != NULL && hres.qualified_name[0] != '\0') { + const cbm_gbuf_node_t *handler = + cbm_gbuf_find_by_qn(main_gbuf, hres.qualified_name); + if (handler != NULL) { + char hprops[256]; + snprintf(hprops, sizeof(hprops), "{\"handler\":\"%s\"}", hres.qualified_name); + cbm_gbuf_insert_edge(gbuf, handler->id, route_id, "HANDLES", hprops); + } + } + } + return; + } + int has_url = (arg != NULL && arg[0] != '\0' && (arg[0] == '/' || strstr(arg, "://") != NULL)); int has_topic = (arg != NULL && arg[0] != '\0' && svc == CBM_SVC_ASYNC && strlen(arg) > 2); @@ -1128,7 +842,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { } /* Classify and emit edge via helper (keeps resolve_worker complexity down) */ - emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res); + emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res, module_qn, + rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count); ws->calls_resolved++; } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 0a0c206dc..57eb3f0df 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -501,20 +501,6 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { goto cleanup; } - /* Allocate prescan cache: HTTP sites + config refs extracted during - * extraction phase while source is in memory. Eliminates all disk - * re-reads in httplinks (2M+ reads) and configlink (62K+ reads). */ - cbm_prescan_t *prescan_cache = calloc(file_count, sizeof(cbm_prescan_t)); - ctx.prescan_cache = prescan_cache; - ctx.prescan_count = file_count; - - /* Build path → file_idx map for prescan lookup by rel_path */ - CBMHashTable *prescan_map = cbm_ht_create(0); - for (int i = 0; i < file_count; i++) { - cbm_ht_set(prescan_map, files[i].rel_path, (void *)((intptr_t)i + 1)); - } - ctx.prescan_path_map = prescan_map; - /* Phase 3A: Parallel extract + definition nodes */ cbm_clock_gettime(CLOCK_MONOTONIC, &t); rc = cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count); @@ -784,21 +770,6 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { itoa_buf((int)elapsed_ms(t))); } - /* Free prescan cache — no longer needed after httplinks + configlink */ - if (ctx.prescan_cache) { - for (int i = 0; i < ctx.prescan_count; i++) { - free(ctx.prescan_cache[i].http_sites); - free(ctx.prescan_cache[i].config_refs); - free(ctx.prescan_cache[i].routes); - } - free(ctx.prescan_cache); - ctx.prescan_cache = NULL; - } - if (ctx.prescan_path_map) { - cbm_ht_free(ctx.prescan_path_map); - ctx.prescan_path_map = NULL; - } - /* Direct dump: construct B-tree pages in C, fwrite() to .db file. * Zero SQLite library involvement — cbm_write_db() builds the binary * format directly from flat arrays. Atomic: writes .tmp then renames. */ @@ -864,20 +835,6 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { itoa_buf((int)elapsed_ms(t0))); cleanup: - /* Free prescan if not already freed */ - if (ctx.prescan_cache) { - for (int i = 0; i < ctx.prescan_count; i++) { - free(ctx.prescan_cache[i].http_sites); - free(ctx.prescan_cache[i].config_refs); - free(ctx.prescan_cache[i].routes); - } - free(ctx.prescan_cache); - ctx.prescan_cache = NULL; - } - if (ctx.prescan_path_map) { - cbm_ht_free(ctx.prescan_path_map); - ctx.prescan_path_map = NULL; - } cbm_discover_free(files, file_count); cbm_gbuf_free(p->gbuf); p->gbuf = NULL; diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 750b70e5b..3ba8bbdd0 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -374,16 +374,15 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_log_info("pass.timing", "pass", "incr_decorator_tags", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); - /* Configlink: key_symbol + dep_imports are gbuf-only (safe). - * file_refs needs prescan or disk — we set prescan=NULL so it uses - * the disk fallback ONLY for config files (typically <10 files). */ + /* Configlink: key_symbol + dep_imports are gbuf-only. + * file_refs uses disk fallback for config files (typically <10 files). */ cbm_clock_gettime(CLOCK_MONOTONIC, &t); cbm_pipeline_pass_configlink(&ctx); cbm_log_info("pass.timing", "pass", "incr_configlink", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); - /* Httplinks: route discovery uses disk fallback when prescan=NULL. - * Only reads source for files with Route/Handler labels (~few files). */ + /* Httplinks: decorator-based routes from gbuf properties + disk fallback + * for source-based routes. AST-based route registration already ran during resolve. */ cbm_clock_gettime(CLOCK_MONOTONIC, &t); cbm_pipeline_pass_httplinks(&ctx); cbm_log_info("pass.timing", "pass", "incr_httplinks", "elapsed_ms", diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index d0c36f89f..b7cd7ece0 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -28,45 +28,6 @@ #define CBM_MS_PER_SEC 1000.0 #define CBM_US_PER_SEC_F 1e6 -/* ── Pre-scan results (extracted during parallel phase) ──────────── */ - -/* HTTP call site discovered during extraction (source already in memory). - * Eliminates 2M+ disk reads in httplinks pass. */ -typedef struct { - char path[256]; /* extracted URL path */ - char method[16]; /* HTTP method (empty if unknown) */ - char source_name[256]; /* function name */ - char source_qn[512]; /* function qualified name */ - char source_label[32]; /* "Function" or "Method" */ - bool is_async; /* async dispatch vs HTTP call */ -} cbm_prescan_http_site_t; - -/* Config file reference found in source during extraction. - * Eliminates 62K+ disk reads in configlink pass. */ -typedef struct { - char ref_path[256]; /* referenced config file path (e.g. "config.yaml") */ -} cbm_prescan_config_ref_t; - -/* HTTP route discovered during extraction. */ -typedef struct { - char path[256]; - char method[16]; - char function_name[256]; - char qualified_name[512]; - char handler_ref[256]; - char protocol[8]; /* "ws", "sse", or "" */ -} cbm_prescan_route_t; - -/* Per-file prescan results, parallel to result_cache[file_idx]. */ -typedef struct { - cbm_prescan_http_site_t *http_sites; - int http_site_count; - cbm_prescan_config_ref_t *config_refs; - int config_ref_count; - cbm_prescan_route_t *routes; - int route_count; -} cbm_prescan_t; - /* ── Pipeline context (internal) ─────────────────────────────────── */ /* Shared context passed to each pass function. @@ -83,18 +44,6 @@ typedef struct { * and pass_calls/usages/semantic reuse cached results instead of re-extracting. * Indexed by file position in the files[] array. Owned by pipeline.c. */ CBMFileResult **result_cache; - - /* Pre-scan cache — indexed by file position, parallel to result_cache. - * Populated during extraction phase while source is in memory. - * Contains HTTP call sites and config file references extracted from source. - * Eliminates disk re-reads in httplinks and configlink passes. */ - cbm_prescan_t *prescan_cache; - int prescan_count; - - /* File path → index hash map for prescan lookup by rel_path. - * Allows httplinks (which iterates graph nodes, not files) to find - * prescan data by node->file_path. */ - CBMHashTable *prescan_path_map; } cbm_pipeline_ctx_t; /* Check cancellation. Returns non-zero if cancelled. */ @@ -399,8 +348,7 @@ int cbm_pipeline_pass_httplinks(cbm_pipeline_ctx_t *ctx); /* Pre-dump pass: decorator tags enrichment (operates on gbuf). */ int cbm_pipeline_pass_decorator_tags(cbm_gbuf_t *gbuf, const char *project); -/* Pre-dump pass: config ↔ code linking. - * Uses prescan cache when available, falls back to disk reads. */ +/* Pre-dump pass: config ↔ code linking. */ int cbm_pipeline_pass_configlink(cbm_pipeline_ctx_t *ctx); /* ── Env URL scanner (pass_envscan.c) ────────────────────────────── */