From 2083b7c6befeab0ec823e1a6b53a12a52142e7d9 Mon Sep 17 00:00:00 2001 From: Ben Fairless Date: Tue, 18 Aug 2026 16:06:29 +0800 Subject: [PATCH 001/217] fix(extraction): reduce A::B.new callee to the constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Ruby `Const.new` call is rewritten to name the constant rather than the method, so the call links the class the way every other language's `new T()` does — the constructor body lives in `initialize`, so naming `new` links nothing. That rewrite only fired for bare `constant` receivers. A scope-resolved receiver fell through it and kept the whole call expression as its callee: `Admin::User.new`. That resolves to nothing, and it cannot join the Ruby LSP's constructor rows, whose callee is the class QN. Treat `scope_resolution` receivers the same way. The new extraction row pins both spellings and fails without this change, printing the 'Admin::User.new' callee that is the bug's signature. Refs #1701 Assisted-by: OpenCode:anthropic/claude-opus-5 Signed-off-by: Ben Fairless --- internal/cbm/extract_calls.c | 6 ++++- tests/test_extraction.c | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index c93e46a44..085868bdb 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -1668,11 +1668,15 @@ static char *extract_callee_name(CBMArena *a, TSNode node, const char *source, C // method is `new`. The constructor body lives in `initialize`, so a callee // of "new" never resolves. Redirect to the receiver type name so the call // links to the class/constructor like every other language's `new T()`. + // Scope-resolved receivers (`Admin::User.new`) get the same treatment — + // the callee carries the full constant path ("Admin::User"), whose bare + // leaf ("User") is what resolution joins on. if (lang == CBM_LANG_RUBY) { TSNode m = ts_node_child_by_field_name(node, TS_FIELD("method")); TSNode recv = ts_node_child_by_field_name(node, TS_FIELD("receiver")); if (!ts_node_is_null(m) && !ts_node_is_null(recv) && - strcmp(ts_node_type(recv), "constant") == 0) { + (strcmp(ts_node_type(recv), "constant") == 0 || + strcmp(ts_node_type(recv), "scope_resolution") == 0)) { char *mt = cbm_node_text(a, m, source); if (mt && strcmp(mt, "new") == 0) { char *rt = cbm_node_text(a, recv, source); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 447e7c104..9f5109f13 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -619,6 +619,57 @@ TEST(ruby_module) { PASS(); } +/* A Ruby `Const.new` call is rewritten to name the CONSTANT, not the method: + * the constructor body lives in `initialize`, so the call should link the class + * like every other language's `new T()`. The rewrite fired only for bare + * `constant` receivers, so a scope-resolved receiver kept the whole call text + * as its callee ("Admin::User.new"), which resolves to nothing and cannot join + * the Ruby LSP's constructor rows, whose callee is the class QN. + * + * Both spellings must land on the bare constant path; a retained ".new" suffix + * is the bug's signature. */ +TEST(ruby_constructor_callee_is_constant_path) { + CBMFileResult *r = extract("module Admin\n class User\n end\nend\n" + "class Widget\nend\n" + "class Maker\n" + " def build\n" + " Admin::User.new\n" + " Widget.new\n" + " end\n" + "end\n", + CBM_LANG_RUBY, "t", "maker.rb"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + int scoped = 0, bare = 0, kept_new = 0; + for (int i = 0; i < r->calls.count; i++) { + const char *callee = r->calls.items[i].callee_name; + if (strcmp(callee, "Admin::User") == 0) + scoped = 1; + if (strcmp(callee, "Widget") == 0) + bare = 1; + /* Either failure mode: the method name survived on its own, or the + * whole `Recv.new` expression did. */ + size_t n = strlen(callee); + if (strcmp(callee, "new") == 0 || (n >= 4 && strcmp(callee + n - 4, ".new") == 0)) + kept_new = 1; + } + if (!scoped || !bare || kept_new) { + printf(" ruby callees (%d):\n", r->calls.count); + for (int i = 0; i < r->calls.count; i++) + printf(" '%s'\n", r->calls.items[i].callee_name); + } + /* Scope-resolved receiver reduces to the constant path; its bare leaf is + * what resolution joins on. */ + ASSERT(scoped); + /* Bare receiver: pre-existing behaviour, guarded against regression. */ + ASSERT(bare); + ASSERT_FALSE(kept_new); + + cbm_free_result(r); + PASS(); +} + /* --- C# --- */ TEST(csharp_class) { CBMFileResult *r = extract("namespace App { public class Service { public void Run() {} public " @@ -5396,6 +5447,7 @@ SUITE(extraction) { RUN_TEST(php_function); RUN_TEST(ruby_class); RUN_TEST(ruby_module); + RUN_TEST(ruby_constructor_callee_is_constant_path); RUN_TEST(csharp_class); RUN_TEST(csharp_interface); RUN_TEST(swift_class); From bd640caa918d5819f82c692a20005cdf83689778 Mon Sep 17 00:00:00 2001 From: Henry Hazan Date: Wed, 19 Aug 2026 01:41:32 +0300 Subject: [PATCH 002/217] fix(elixir): Phoenix channel extraction was unreachable in both branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Neither half of extract_channels_elixir could ever fire, so an Elixir project produced zero Channel nodes regardless of how much Phoenix.PubSub or Phoenix.Channel it used. The emit side read ts_node_child_by_field_name(call, "arguments"). tree-sitter-elixir defines no such field — its whole field set is key, left, operand, operator, quoted_start, quoted_end, right, target and value — so args was always null and elixir_emit_second_arg() returned early every time. extract_defs.c has always used a positional second-child fallback for this; the same fallback is now used here. The listener side dispatched on strcmp(kind, "def"). There is no `def` node type in the grammar: an Elixir definition is a `call` whose target is the macro name. elixir_process_function_def() was therefore dead code, and it also read `name` and `parameters` fields that do not exist. It now takes the def call, unwraps a `when` guard, and reads the head's name and parameters positionally, so guarded clauses like `def handle_in(e, p, s) when is_map(s)` are recognized too. Verified against a Phoenix application; the new test covers Phoenix.PubSub.broadcast/subscribe, a bare push/broadcast emit, and two handle_in clauses, one of them guarded. Signed-off-by: Henry Hazan --- internal/cbm/extract_channels.c | 85 ++++++++++++++++++++++++++++----- tests/test_extraction.c | 49 +++++++++++++++++++ 2 files changed, 123 insertions(+), 11 deletions(-) diff --git a/internal/cbm/extract_channels.c b/internal/cbm/extract_channels.c index c12d4ebeb..17ef8f802 100644 --- a/internal/cbm/extract_channels.c +++ b/internal/cbm/extract_channels.c @@ -901,8 +901,42 @@ static void elixir_emit_second_arg(CBMExtractCtx *ctx, TSNode call, TSNode args, } } -static void elixir_process_call(CBMExtractCtx *ctx, TSNode call) { +/* tree-sitter-elixir gives a call's arguments node NO field name — its whole + * field set is key, left, operand, operator, quoted_start, quoted_end, right, + * target and value — so the generic field lookup is always null here and every + * channel below was silently skipped. extract_defs.c has carried the same + * positional fallback for the definition side since Elixir was added. */ +static TSNode elixir_channel_call_args(TSNode call) { + TSNode args = ts_node_child_by_field_name(call, TS_FIELD("arguments")); + if (ts_node_is_null(args) && ts_node_child_count(call) > 1) { + args = ts_node_child(call, 1); + } + return args; +} + +/* The callee of an Elixir `call`. The grammar labels it `target`; child(0) is + * the same node and is what extract_defs.c reads, so accept either. */ +static TSNode elixir_channel_call_target(TSNode call) { TSNode target = ts_node_child_by_field_name(call, TS_FIELD("target")); + if (ts_node_is_null(target) && ts_node_child_count(call) > 0) { + target = ts_node_child(call, 0); + } + return target; +} + +/* Elixir def* macros. `def`/`defp` cover Phoenix channel callbacks; the macro + * forms are included because `handle_in` is generated by macros in some + * libraries. */ +static bool elixir_channel_is_def_macro(const char *macro) { + if (!macro) { + return false; + } + return strcmp(macro, "def") == 0 || strcmp(macro, "defp") == 0 || + strcmp(macro, "defmacro") == 0 || strcmp(macro, "defmacrop") == 0; +} + +static void elixir_process_call(CBMExtractCtx *ctx, TSNode call) { + TSNode target = elixir_channel_call_target(call); if (ts_node_is_null(target)) { return; } @@ -911,7 +945,7 @@ static void elixir_process_call(CBMExtractCtx *ctx, TSNode call) { return; } - TSNode args = ts_node_child_by_field_name(call, TS_FIELD("arguments")); + TSNode args = elixir_channel_call_args(call); if (strstr(target_text, "PubSub.broadcast") || strstr(target_text, "PubSub.local_broadcast")) { elixir_emit_second_arg(ctx, call, args, "phoenix_pubsub", CBM_CHANNEL_EMIT); @@ -927,10 +961,31 @@ static void elixir_process_call(CBMExtractCtx *ctx, TSNode call) { } } -/* Detect handle_in("event", payload, socket) — Phoenix Channel listener */ -static void elixir_process_function_def(CBMExtractCtx *ctx, TSNode func_def) { - TSNode name_node = ts_node_child_by_field_name(func_def, TS_FIELD("name")); - if (ts_node_is_null(name_node)) { +/* Detect handle_in("event", payload, socket) — Phoenix Channel listener. + * `def_call` is the `call` node of a def* macro. Elixir has no `def` node type + * and no `name`/`parameters` fields: a definition is a call whose first + * argument is the head, itself a call of name(params) — wrapped in a `when` + * binary_operator when the clause carries a guard. */ +static void elixir_process_function_def(CBMExtractCtx *ctx, TSNode def_call) { + TSNode args = elixir_channel_call_args(def_call); + if (ts_node_is_null(args) || ts_node_named_child_count(args) == 0) { + return; + } + TSNode head = ts_node_named_child(args, 0); + if (strcmp(ts_node_type(head), "binary_operator") == 0) { + TSNode op = ts_node_child_by_field_name(head, TS_FIELD("operator")); + TSNode left = ts_node_child_by_field_name(head, TS_FIELD("left")); + if (!ts_node_is_null(op) && !ts_node_is_null(left) && + ts_node_end_byte(op) - ts_node_start_byte(op) == 4 && + strncmp(ctx->source + ts_node_start_byte(op), "when", 4) == 0) { + head = left; + } + } + if (strcmp(ts_node_type(head), "call") != 0 || ts_node_child_count(head) == 0) { + return; + } + TSNode name_node = ts_node_child(head, 0); + if (ts_node_is_null(name_node) || strcmp(ts_node_type(name_node), "identifier") != 0) { return; } char *name = cbm_node_text(ctx->arena, name_node, ctx->source); @@ -938,7 +993,7 @@ static void elixir_process_function_def(CBMExtractCtx *ctx, TSNode func_def) { return; } /* First parameter is the event name pattern */ - TSNode params = ts_node_child_by_field_name(func_def, TS_FIELD("parameters")); + TSNode params = elixir_channel_call_args(head); if (ts_node_is_null(params)) { return; } @@ -952,7 +1007,7 @@ static void elixir_process_function_def(CBMExtractCtx *ctx, TSNode func_def) { val = literal_from_first_child(ctx, first_param); } if (val) { - push_channel(ctx, val, "phoenix_channel", CBM_CHANNEL_LISTEN, func_def); + push_channel(ctx, val, "phoenix_channel", CBM_CHANNEL_LISTEN, def_call); } } @@ -964,10 +1019,18 @@ static void extract_channels_elixir(CBMExtractCtx *ctx) { while (stack.count > 0) { TSNode node = ts_nstack_pop(&stack); const char *kind = ts_node_type(node); + /* Everything in Elixir is a `call`, including `def`. The old dispatch + * tested for a "def" node type the grammar does not have, so the + * handle_in listener branch was unreachable. */ if (strcmp(kind, "call") == 0) { - elixir_process_call(ctx, node); - } else if (strcmp(kind, "def") == 0) { - elixir_process_function_def(ctx, node); + TSNode target = elixir_channel_call_target(node); + char *macro = + ts_node_is_null(target) ? NULL : cbm_node_text(ctx->arena, target, ctx->source); + if (elixir_channel_is_def_macro(macro)) { + elixir_process_function_def(ctx, node); + } else { + elixir_process_call(ctx, node); + } } uint32_t count = ts_node_child_count(node); for (int i = (int)count - SKIP_ONE; i >= 0; i--) { diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 447e7c104..171f1f08f 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1005,6 +1005,54 @@ TEST(elixir_function) { PASS(); } +/* True when a channel with this name/transport/direction was extracted. */ +static int has_channel(CBMFileResult *r, const char *name, const char *transport, + CBMChannelDirection dir) { + for (int i = 0; i < r->channels.count; i++) { + const CBMChannel *c = &r->channels.items[i]; + if (c->channel_name && strcmp(c->channel_name, name) == 0 && c->transport && + strcmp(c->transport, transport) == 0 && c->direction == dir) + return 1; + } + return 0; +} + +/* Phoenix pub/sub and channel listeners. Both branches were unreachable: the + * emit side looked up an "arguments" FIELD that tree-sitter-elixir does not + * define, and the listener side dispatched on a "def" NODE TYPE that does not + * exist (Elixir defs are `call` nodes), so a Phoenix app produced no channels + * at all. */ +TEST(elixir_channels_pubsub_and_handle_in) { + CBMFileResult *r = + extract("defmodule Acme.Notifier do\n" + " def announce(p) do\n" + " Phoenix.PubSub.broadcast(Acme.PubSub, \"user:created\", p)\n" + " end\n" + " def watch, do: Phoenix.PubSub.subscribe(Acme.PubSub, \"user:updated\")\n" + "end\n" + "defmodule AcmeWeb.RoomChannel do\n" + " def handle_in(\"new_msg\", payload, socket) do\n" + " push(socket, \"pong\", %{})\n" + " {:noreply, socket}\n" + " end\n" + " def handle_in(\"ping\", _p, socket) when is_map(socket) do\n" + " {:noreply, socket}\n" + " end\n" + "end\n", + CBM_LANG_ELIXIR, "t", "chan.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_channel(r, "user:created", "phoenix_pubsub", CBM_CHANNEL_EMIT)); + ASSERT(has_channel(r, "user:updated", "phoenix_pubsub", CBM_CHANNEL_LISTEN)); + /* handle_in clauses are listeners — including the guarded one. */ + ASSERT(has_channel(r, "new_msg", "phoenix_channel", CBM_CHANNEL_LISTEN)); + ASSERT(has_channel(r, "ping", "phoenix_channel", CBM_CHANNEL_LISTEN)); + /* A bare push/broadcast inside the channel module is an emit. */ + ASSERT(has_channel(r, "pong", "phoenix_channel", CBM_CHANNEL_EMIT)); + cbm_free_result(r); + PASS(); +} + /* --- Haskell --- */ TEST(haskell_function) { CBMFileResult *r = extract("add :: Int -> Int -> Int\nadd x y = x + y\n\nmultiply :: Int -> " @@ -5432,6 +5480,7 @@ SUITE(extraction) { /* Functional */ RUN_TEST(elixir_function); + RUN_TEST(elixir_channels_pubsub_and_handle_in); RUN_TEST(haskell_function); RUN_TEST(ocaml_function); RUN_TEST(erlang_function); From c8f36262e18318f38387dd60a779eee5b59aa7fd Mon Sep 17 00:00:00 2001 From: Henry Hazan Date: Wed, 19 Aug 2026 02:02:54 +0300 Subject: [PATCH 003/217] fix(elixir): functions carry no complexity, fingerprint or line count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every Elixir Function shipped with complexity 0, cognitive 0, lines 0 and no MinHash fingerprint, so Elixir was invisible to complexity and bottleneck queries and never participated in SIMILAR_TO or duplicate detection — while the query_graph tool description states that "every Function and Method node carries queryable complexity properties". The generic path cannot be reused. cbm_compute_complexity() matches on node TYPE, but Elixir has no branching node types: `if`, `unless`, `case`, `cond`, `with`, `for`, `try` and `receive` are ordinary `call` nodes distinguished only by the identifier in callee position, and every case/cond/fn/rescue arm is a `stab_clause`. That is also why elixir_branch_types is `{"call"}` in lang_specs.c and has never been usable — it would score every function call as a decision. elixir_compute_complexity() is the Elixir-shaped equivalent, one walk with the same Campbell cognitive weighting as the generic version. It counts the branching construct itself, every stab_clause, the short-circuit operators (&&, ||, and, or) and `when` guards. Container and arm are both counted, which is the convention the generic tables already use: Go counts expression_switch_statement plus each expression_case, Python counts try_statement plus each except_clause. A `for` comprehension is the only real loop node; Enum.map/reduce are plain calls and counting them would make loop_depth meaningless. A straight-line single-clause function still scores 0, exactly like every other language's branch-free function. The fingerprint is taken from the do_block rather than the whole def call, so the head does not make every clause of a name look alike. Signed-off-by: Henry Hazan --- internal/cbm/extract_defs.c | 143 ++++++++++++++++++++++++++++++++++++ tests/test_extraction.c | 40 ++++++++++ 2 files changed, 183 insertions(+) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index f9854b03e..e2822d446 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -4985,6 +4985,134 @@ static TSNode elixir_call_args(TSNode node) { } // Handle Elixir def/defp/defmacro — extract function definition. +/* True when node `n` spans exactly the text `lit`. Avoids an arena allocation + * per node in the complexity walk, which visits every node of every def. */ +static bool elixir_node_text_is(TSNode n, const char *source, const char *lit) { + if (ts_node_is_null(n)) { + return false; + } + uint32_t s = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + size_t len = strlen(lit); + return (size_t)(e - s) == len && strncmp(source + s, lit, len) == 0; +} + +/* Elixir has no branching NODE TYPES. `if`, `unless`, `case`, `cond`, `with`, + * `for`, `try` and `receive` are ordinary `call` nodes told apart only by the + * identifier in callee position, and every case/cond/fn/rescue arm is a + * `stab_clause`. lang_specs.c cannot express that — the Elixir branch table is + * `{"call"}`, which would score every function call as a decision — and + * cbm_compute_complexity matches on node type, so the generic path is unusable + * and every Elixir function shipped complexity 0. + * + * Counted here: the branching construct itself, every `stab_clause`, the + * short-circuit operators, and `when` guards. Container AND arm are both + * counted, which is the convention the generic tables already use — Go counts + * `expression_switch_statement` plus each `expression_case`, Python counts + * `try_statement` plus each `except_clause`. Cognitive weighting is Campbell's, + * identical to the generic walk (+1 plus the current nesting depth). */ +static bool elixir_is_branch_call(TSNode call, const char *source) { + if (ts_node_child_count(call) == 0) { + return false; + } + TSNode callee = ts_node_child(call, 0); + if (ts_node_is_null(callee) || strcmp(ts_node_type(callee), "identifier") != 0) { + return false; + } + return elixir_node_text_is(callee, source, "if") || + elixir_node_text_is(callee, source, "unless") || + elixir_node_text_is(callee, source, "with") || + elixir_node_text_is(callee, source, "for") || + elixir_node_text_is(callee, source, "case") || + elixir_node_text_is(callee, source, "cond") || + elixir_node_text_is(callee, source, "try") || + elixir_node_text_is(callee, source, "receive"); +} + +/* A comprehension (`for x <- xs, do: ...`) is Elixir's only real loop node. + * Enum.map/reduce iterate too, but they are plain function calls; counting them + * would make loop_depth meaningless, so they are left out. */ +static bool elixir_is_loop_call(TSNode call, const char *source) { + if (ts_node_child_count(call) == 0) { + return false; + } + TSNode callee = ts_node_child(call, 0); + return !ts_node_is_null(callee) && strcmp(ts_node_type(callee), "identifier") == 0 && + elixir_node_text_is(callee, source, "for"); +} + +static bool elixir_is_branch_operator(TSNode node, const char *source) { + TSNode op = ts_node_child_by_field_name(node, TS_FIELD("operator")); + return elixir_node_text_is(op, source, "&&") || elixir_node_text_is(op, source, "||") || + elixir_node_text_is(op, source, "and") || elixir_node_text_is(op, source, "or") || + elixir_node_text_is(op, source, "when"); +} + +static void elixir_compute_complexity(TSNode node, const char *source, cbm_complexity_t *out) { + enum { ELIXIR_CX_STACK_CAP = 4096 }; + struct { + TSNode node; + int bdepth; + int ldepth; + int adepth; + } stack[ELIXIR_CX_STACK_CAP]; + memset(out, 0, sizeof(*out)); + int top = 0; + stack[top].node = node; + stack[top].bdepth = 0; + stack[top].ldepth = 0; + stack[top].adepth = 0; + top++; + while (top > 0) { + int cur = --top; + TSNode n = stack[cur].node; + int bdepth = stack[cur].bdepth; + int ldepth = stack[cur].ldepth; + int adepth = stack[cur].adepth; + const char *kind = ts_node_type(n); + bool is_branch = false; + if (strcmp(kind, "stab_clause") == 0) { + is_branch = true; + } else if (strcmp(kind, "call") == 0) { + is_branch = elixir_is_branch_call(n, source); + } else if (strcmp(kind, "binary_operator") == 0) { + is_branch = elixir_is_branch_operator(n, source); + } + int child_b = bdepth; + int child_l = ldepth; + /* `conn.assigns.user` nests as dot(dot(conn)), so consecutive `dot` + * nodes deepen the chain and anything else resets it. */ + int child_a = 0; + if (strcmp(kind, "dot") == 0) { + child_a = adepth + 1; + if (child_a > out->max_access_depth) { + out->max_access_depth = child_a; + } + } + if (is_branch) { + out->cyclomatic++; + out->cognitive += 1 + bdepth; + child_b = bdepth + 1; + } + if (strcmp(kind, "call") == 0 && elixir_is_loop_call(n, source)) { + out->loop_count++; + int d = ldepth + 1; + if (d > out->loop_depth) { + out->loop_depth = d; + } + child_l = d; + } + uint32_t nc = ts_node_child_count(n); + for (int i = (int)nc - 1; i >= 0 && top < ELIXIR_CX_STACK_CAP; i--) { + stack[top].node = ts_node_child(n, (uint32_t)i); + stack[top].bdepth = child_b; + stack[top].ldepth = child_l; + stack[top].adepth = child_a; + top++; + } + } +} + static void extract_elixir_func_def(CBMExtractCtx *ctx, TSNode node, const char *macro) { CBMArena *a = ctx->arena; TSNode args = elixir_call_args(node); @@ -5017,6 +5145,21 @@ static void extract_elixir_func_def(CBMExtractCtx *ctx, TSNode node, const char def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET; def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET; def.is_exported = (strcmp(macro, "def") == 0 || strcmp(macro, "defmacro") == 0); + def.lines = (int)(def.end_line - def.start_line + TS_LINE_OFFSET); + cbm_complexity_t cx; + elixir_compute_complexity(node, ctx->source, &cx); + def.complexity = cx.cyclomatic; + def.cognitive = cx.cognitive; + def.loop_count = cx.loop_count; + def.loop_depth = cx.loop_depth; + def.max_access_depth = cx.max_access_depth; + /* Fingerprint off the body, not the whole def call: including the head + * would make every clause of a name look alike. Elixir was the only + * language emitting Functions with no fingerprint at all, so it never + * appeared in similarity or duplicate results. Runs after `lines` — the + * AST profile reads it. */ + TSNode fp_body = cbm_find_child_by_kind(node, "do_block"); + compute_fingerprint(ctx, &def, ts_node_is_null(fp_body) ? node : fp_body); cbm_defs_push(&ctx->result->defs, a, def); } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 447e7c104..ea883bd29 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1005,6 +1005,45 @@ TEST(elixir_function) { PASS(); } +/* Elixir control flow is `call` nodes and `stab_clause` arms, not branch node + * TYPES, so the generic complexity walk scored every Elixir function 0 and the + * fingerprint was never computed at all — no similarity or duplicate detection. */ +TEST(elixir_complexity_and_fingerprint) { + CBMFileResult *r = extract("defmodule Cx do\n" + " def classify(x) do\n" + " case x do\n" + " 0 -> :zero\n" + " n when n > 0 -> :pos\n" + " _ -> :neg\n" + " end\n" + " end\n" + " def plain(a), do: a\n" + "end\n", + CBM_LANG_ELIXIR, "t", "cx.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + CBMDefinition *cls = NULL; + CBMDefinition *plain = NULL; + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].name, "classify") == 0) + cls = &r->defs.items[i]; + if (strcmp(r->defs.items[i].name, "plain") == 0) + plain = &r->defs.items[i]; + } + ASSERT_NOT_NULL(cls); + /* The `case` itself + its 3 arms + the second arm's own `when` guard. */ + ASSERT_EQ(5, cls->complexity); + ASSERT(cls->cognitive > 0); + ASSERT(cls->lines > 0); + /* A straight-line body has no decisions but still gets a line count — a + * language must not become nonzero everywhere just by being measured. */ + ASSERT_NOT_NULL(plain); + ASSERT_EQ(0, plain->complexity); + ASSERT(plain->lines > 0); + cbm_free_result(r); + PASS(); +} + /* --- Haskell --- */ TEST(haskell_function) { CBMFileResult *r = extract("add :: Int -> Int -> Int\nadd x y = x + y\n\nmultiply :: Int -> " @@ -5432,6 +5471,7 @@ SUITE(extraction) { /* Functional */ RUN_TEST(elixir_function); + RUN_TEST(elixir_complexity_and_fingerprint); RUN_TEST(haskell_function); RUN_TEST(ocaml_function); RUN_TEST(erlang_function); From 944b2293b44fee9328d2b54bd22f6c1b46e9a65f Mon Sep 17 00:00:00 2001 From: Ben Fairless Date: Wed, 19 Aug 2026 17:05:34 +0800 Subject: [PATCH 004/217] ci: retrigger after transient runner failures The previous run hit two transient infrastructure failures: - security/codeql-gate: gate poller timed out by ~4 seconds; CodeQL analysis itself completed successfully - test/test-unix (macos-15-intel): tests passed but the post-run artifact upload hit ENOTFOUND (network unavailability on the runner) No code changes. Assisted-by: Claude Code:claude-sonnet-4-6 Signed-off-by: Ben Fairless From ad786cd4661c98c16012d461b9443b3d7b3a197a Mon Sep 17 00:00:00 2001 From: Ben Fairless Date: Fri, 21 Aug 2026 10:05:00 +0800 Subject: [PATCH 005/217] fix(test): drop racy active_connections snapshot in connection-cap test daemon_runtime_connection_cap_covers_slow_hello_and_stopping_is_terminal failed on macos CI. The test's capacity_rejected condition sampled active_connections immediately after the overflow connect returned. On a loaded runner the server's HELLO timeout (5 s) can fire for the slow_hello raw connection during the overflow client_connect call (up to 2 s), dropping the count from 2 to 1 and falsifying the == 2 assertion. The capacity rejection message already proves the reason. Drop the point-in-time active_connections sample; keep active_clients == 1, which is stable because we still hold the accepted client open. Same class of race fixed in db91b88c for the forged-identity test. Assisted-by: Claude Code:claude-sonnet-4-6 Signed-off-by: Ben Fairless --- tests/test_daemon_runtime.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index 5f41d914e..34fda16d5 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -2677,10 +2677,15 @@ TEST(daemon_runtime_connection_cap_covers_slow_hello_and_stopping_is_terminal) { if (accepted) { overflow = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity, RUNTIME_TEST_TIMEOUT_MS, &overflow_result); + /* The "capacity" substring in the rejection message proves the reason. + * active_connections is NOT sampled here: on a loaded runner the + * server's HELLO timeout can fire for slow_hello between the raw + * connect and this read, making the count 1 instead of 2 — a + * scheduling race, not a product bug (see db91b88c for precedent). + * active_clients == 1 is safe because we still hold accepted. */ capacity_rejected = overflow == NULL && overflow_result.status == CBM_DAEMON_RUNTIME_CONNECT_REJECTED && strstr(overflow_result.message, "capacity") != NULL && - cbm_daemon_runtime_service_active_connections(fixture.service) == 2 && cbm_daemon_runtime_service_active_clients(fixture.service) == 1; cbm_daemon_ipc_connection_close(slow_hello); From 5ca234c2b24c27a8a484e744cc200b93e01f130b Mon Sep 17 00:00:00 2001 From: LeoLan Digital Date: Tue, 25 Aug 2026 15:15:56 +0200 Subject: [PATCH 006/217] fix: persist daemon watches across client disconnects Signed-off-by: LeoLan Digital Signed-off-by: LeoLan Digital Signed-off-by: LeoLan Digital --- src/daemon/application.c | 65 ++++++++++++++------ tests/test_daemon_application.c | 102 ++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 17 deletions(-) diff --git a/src/daemon/application.c b/src/daemon/application.c index 7e4a35be7..ce0cfdaf8 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -149,11 +149,11 @@ struct cbm_daemon_application_job { cbm_daemon_application_job_t *next; }; -/* A watcher-triggered physical job is owned by the exact live sessions that - * currently subscribe to its project/root watch. The callback waiting for the - * job is only a storage waiter; it is deliberately not an ownership - * subscription, so the worker is cancelled when the last matching session - * disconnects even while unrelated daemon sessions remain alive. */ +/* A watcher-triggered physical job is normally owned by the exact live + * sessions subscribed to its project/root watch. In permanent daemon mode the + * watcher itself additionally owns the physical job so background indexing can + * continue while no client session is connected. The callback remains a + * storage waiter in both modes and releases daemon ownership on completion. */ struct cbm_daemon_application_watch_job_subscription { cbm_daemon_application_session_t *session; cbm_daemon_application_job_t *job; @@ -424,7 +424,19 @@ static void application_release_session_watch_locked(cbm_daemon_application_sess if (watch->subscribers > 0) { watch->subscribers--; } - if (watch->subscribers == 0) { + + /* + * A permanent daemon owns the physical project watch independently of + * individual client sessions. Session disconnect releases only that + * session's logical subscription; the watcher remains available for + * background change detection while the daemon generation is alive. + * + * Non-permanent applications retain the historical session-scoped + * lifecycle and remove the physical watch after the final subscriber. + * Project deletion/pruning and daemon shutdown remove persistent watches + * through their dedicated lifecycle paths. + */ + if (!session->application->permanent && watch->subscribers == 0) { application_remove_watch_locked(session->application, watch); } } @@ -3375,10 +3387,13 @@ static int application_background_index(cbm_daemon_application_t *application, cbm_mutex_lock(&application->mutex); cbm_daemon_application_watch_t *watch = require_live_watch ? application_find_watch_locked(application, project_name) : NULL; - bool watch_live = !require_live_watch || - (watch && watch->subscribers > 0 && strcmp(watch->root, canonical_root) == 0); + bool watch_live = + !require_live_watch || + (watch && strcmp(watch->root, canonical_root) == 0 && + (watch->subscribers > 0 || application->permanent)); size_t watch_owner_count = 0; bool watch_subscriptions_ok = true; + bool watcher_owns_job = false; cbm_daemon_application_job_t *job = watch_live ? application_job_subscribe_locked(application, project_key, canonical_root, args, &subscribe_status) @@ -3386,16 +3401,28 @@ static int application_background_index(cbm_daemon_application_t *application, if (job && require_live_watch) { watch_subscriptions_ok = application_watch_job_subscribe_sessions_locked( application, watch, job, &watch_owner_count); - if (watch_subscriptions_ok && watch_owner_count > 0) { - job->watcher_waiters++; - } else if (!watch_subscriptions_ok) { + + if (!watch_subscriptions_ok) { subscribe_status = APPLICATION_JOB_SUBSCRIBE_ALLOCATION_FAILED; - } - /* application_job_subscribe_locked() lends the caller one ordinary - * subscriber. A watcher callback is only a storage waiter: exact live - * session subscriptions above own the physical work. */ - application_job_unsubscribe_locked(job); - if (!watch_subscriptions_ok || watch_owner_count == 0) { + application_job_unsubscribe_locked(job); + job = NULL; + } else if (application->permanent) { + /* + * The ordinary subscription acquired above is retained as daemon + * ownership. This allows the physical watcher to run indexing even + * while no client session currently owns the project watch. + */ + watcher_owns_job = true; + job->watcher_waiters++; + } else if (watch_owner_count > 0) { + /* + * Historical non-permanent behavior: exact live sessions own the + * physical work; the watcher callback itself is only a waiter. + */ + job->watcher_waiters++; + application_job_unsubscribe_locked(job); + } else { + application_job_unsubscribe_locked(job); job = NULL; } } @@ -3428,6 +3455,10 @@ static int application_background_index(cbm_daemon_application_t *application, if (job->watcher_waiters > 0) { job->watcher_waiters--; } + if (watcher_owns_job) { + application_job_unsubscribe_locked(job); + watcher_owns_job = false; + } } else { application_job_unsubscribe_locked(job); } diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index c639ace2f..b0b55e4f6 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -3621,6 +3621,107 @@ TEST(daemon_application_watcher_job_follows_exact_live_watch_owners) { PASS(); } + +TEST(daemon_application_permanent_watch_survives_disconnect_and_indexes_without_session) { + app_watch_race_fixture_t fixture; + bool fixture_ready = app_watch_race_fixture_init(&fixture, 47); + + if (fixture_ready) { + cbm_daemon_application_set_permanent(fixture.application, true); + } + + int watch_before_disconnect = + fixture_ready ? cbm_watcher_watch_count(fixture.watcher) : -1; + + if (fixture.session) { + fixture.callbacks.session_cancel(fixture.callbacks.context, fixture.session); + } + + int watch_after_cancel = + fixture_ready ? cbm_watcher_watch_count(fixture.watcher) : -1; + + if (fixture.session) { + fixture.callbacks.session_close(fixture.callbacks.context, fixture.session); + fixture.session = NULL; + } + + int watch_after_close = + fixture_ready ? cbm_watcher_watch_count(fixture.watcher) : -1; + + /* + * No client session remains at this point. A permanent daemon-owned + * physical watch must nevertheless be allowed to launch its background + * indexing job. + */ + app_watcher_index_thread_t request = { + .application = fixture.application, + .project = fixture.project, + .root = fixture.root, + .pause_before_subscribe = false, + .result = -1, + }; + atomic_init(&request.ready, false); + atomic_init(&request.proceed, true); + atomic_init(&request.done, false); + + cbm_thread_t thread; + bool thread_started = + fixture_ready && + watch_after_close == 1 && + cbm_thread_create(&thread, 0, app_watcher_index_thread, &request) == 0; + + bool worker_started = + thread_started && app_wait_for_atomic_int(&fixture.fake.starts, 1); + + int watch_during_job = + fixture_ready ? cbm_watcher_watch_count(fixture.watcher) : -1; + + /* + * The fake worker deliberately waits until the test releases it. + * Let the daemon-owned watcher job finish normally. + */ + if (worker_started) { + atomic_store(&fixture.fake.allow_completion, true); + } + + bool request_done = + thread_started && app_wait_for_atomic_bool(&request.done, true); + + bool thread_joined = + request_done && cbm_thread_join(&thread) == 0; + + int watch_after_job = + fixture_ready ? cbm_watcher_watch_count(fixture.watcher) : -1; + + int destroys_before_cleanup = + fixture_ready ? atomic_load(&fixture.fake.destroys) : -1; + + bool cleaned = app_watch_race_fixture_finish(&fixture); + + ASSERT_TRUE(fixture_ready); + ASSERT_EQ(watch_before_disconnect, 1); + + /* Permanent mode: disconnect must not unwatch the project. */ + ASSERT_EQ(watch_after_cancel, 1); + ASSERT_EQ(watch_after_close, 1); + + /* No session exists, yet the watcher still owns background indexing. */ + ASSERT_TRUE(thread_started); + ASSERT_TRUE(worker_started); + ASSERT_EQ(watch_during_job, 1); + + ASSERT_TRUE(request_done); + ASSERT_TRUE(thread_joined); + ASSERT_EQ(request.result, 0); + ASSERT_EQ(destroys_before_cleanup, 1); + + /* Completing the background job must not consume the persistent watch. */ + ASSERT_EQ(watch_after_job, 1); + + ASSERT_TRUE(cleaned); + PASS(); +} + TEST(daemon_application_late_watcher_session_owns_active_watcher_job) { app_watch_race_fixture_t fixture; bool fixture_ready = app_watch_race_fixture_init(&fixture, 45); @@ -5116,6 +5217,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_stale_watcher_callback_is_rejected_at_job_admission); RUN_TEST(daemon_application_final_cancel_drains_admitted_watcher_job); RUN_TEST(daemon_application_watcher_job_follows_exact_live_watch_owners); + RUN_TEST(daemon_application_permanent_watch_survives_disconnect_and_indexes_without_session); RUN_TEST(daemon_application_late_watcher_session_owns_active_watcher_job); RUN_TEST(daemon_application_serializes_adr_mutation_with_index_job); RUN_TEST(daemon_application_reserved_mutation_delays_worker_start); From 5e328814fc99ffe588daae4f0f1e8cf5066e084b Mon Sep 17 00:00:00 2001 From: Jumaga2015 Date: Tue, 1 Sep 2026 21:00:26 +0200 Subject: [PATCH 007/217] fix(store,ui): export paired free functions for FFI callers cbm_store_fetch_call_edges, cbm_leiden/cbm_louvain, and cbm_layout_to_json document caller-owned memory ("caller frees both" / "the caller frees it" / "Caller must free()") but exported no matching free function, unlike the rest of the API (cbm_store_find_nodes -> cbm_store_free_nodes, cbm_layout_compute -> cbm_layout_free, etc.). From C a bare free() works via static linking, but a caller across an FFI boundary (e.g. C# P/Invoke) has no way to reach the DLL's internal free() unless it is exported by name. Adds three paired free functions following the existing pattern: - cbm_store_free_call_edges(src, tgt) - cbm_leiden_free(result) - cbm_layout_free_json(json) Updates the one internal production caller (mcp.c SCC analysis) to use cbm_store_free_call_edges instead of a bare free(), and the existing unit tests for cbm_leiden/cbm_louvain/cbm_layout_to_json to use the new paired frees. Adds fetch_call_edges_basic, fetch_call_edges_truncated, and fetch_call_edges_empty_project to test_store_arch.c, covering cbm_store_fetch_call_edges / cbm_store_free_call_edges, which had no prior test coverage. Fixes #1762 Signed-off-by: Jumaga2015 --- src/mcp/mcp.c | 6 +-- src/store/store.c | 9 +++++ src/store/store.h | 6 +++ src/ui/layout3d.c | 4 ++ src/ui/layout3d.h | 3 ++ tests/test_store_arch.c | 81 +++++++++++++++++++++++++++++++++++++---- tests/test_ui.c | 4 +- 7 files changed, 100 insertions(+), 13 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index e908211b8..1d154bb28 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -5545,12 +5545,10 @@ static int arch_compute_cycles(cbm_store_t *store, const char *project, int64_t *scanned_edges = ecount; scc_graph_t g; if (!scc_build(src, tgt, ecount, &g)) { - free(src); - free(tgt); + cbm_store_free_call_edges(src, tgt); return CBM_STORE_OK; /* no edges = no cycles, not an error */ } - free(src); - free(tgt); + cbm_store_free_call_edges(src, tgt); int *comp = malloc((size_t)g.nverts * sizeof(int)); if (!comp) { diff --git a/src/store/store.c b/src/store/store.c index 03e42b0ed..e36c93b1b 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -3142,6 +3142,11 @@ int cbm_store_fetch_call_edges(cbm_store_t *s, const char *project, int max_edge return CBM_STORE_OK; } +void cbm_store_free_call_edges(int64_t *src, int64_t *tgt) { + free(src); + free(tgt); +} + int cbm_store_find_edges_by_source(cbm_store_t *s, int64_t source_id, cbm_edge_t **out, int *count) { bind_id_t b = {source_id}; @@ -8234,6 +8239,10 @@ int cbm_louvain(const int64_t *nodes, int node_count, const cbm_louvain_edge_t * return cbm_leiden(nodes, node_count, edges, edge_count, 1.0, out, out_count); } +void cbm_leiden_free(cbm_louvain_result_t *result) { + free(result); +} + /* ── Architecture: community clusters via Leiden ───────────────── */ enum { diff --git a/src/store/store.h b/src/store/store.h index 165356c31..7bd7e0bbe 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -574,6 +574,9 @@ int cbm_store_insert_edge_batch(cbm_store_t *s, const cbm_edge_t *edges, int cou int cbm_store_fetch_call_edges(cbm_store_t *s, const char *project, int max_edges, int64_t **out_src, int64_t **out_tgt, int *count, bool *truncated); +/* Free the (source_id, target_id) arrays returned by cbm_store_fetch_call_edges. */ +void cbm_store_free_call_edges(int64_t *src, int64_t *tgt); + /* Find edges by source node. */ int cbm_store_find_edges_by_source(cbm_store_t *s, int64_t source_id, cbm_edge_t **out, int *count); @@ -996,6 +999,9 @@ int cbm_leiden(const int64_t *nodes, int node_count, const cbm_louvain_edge_t *e int cbm_louvain(const int64_t *nodes, int node_count, const cbm_louvain_edge_t *edges, int edge_count, cbm_louvain_result_t **out, int *out_count); +/* Free the result array returned by cbm_leiden / cbm_louvain. */ +void cbm_leiden_free(cbm_louvain_result_t *result); + /* ── Memory management helpers ──────────────────────────────────── */ /* Free heap-allocated strings in a stack-allocated node (does NOT free the node itself). */ diff --git a/src/ui/layout3d.c b/src/ui/layout3d.c index 3176da2ff..8a5e32aea 100644 --- a/src/ui/layout3d.c +++ b/src/ui/layout3d.c @@ -867,3 +867,7 @@ char *cbm_layout_to_json(const cbm_layout_result_t *r) { } return json; } + +void cbm_layout_free_json(char *json) { + free(json); +} diff --git a/src/ui/layout3d.h b/src/ui/layout3d.h index 20303bf64..758f0be74 100644 --- a/src/ui/layout3d.h +++ b/src/ui/layout3d.h @@ -71,4 +71,7 @@ void cbm_layout_free(cbm_layout_result_t *result); /* Serialize layout result to JSON string. Caller must free(). */ char *cbm_layout_to_json(const cbm_layout_result_t *result); +/* Free the JSON string returned by cbm_layout_to_json. */ +void cbm_layout_free_json(char *json); + #endif /* CBM_UI_LAYOUT3D_H */ diff --git a/tests/test_store_arch.c b/tests/test_store_arch.c index 4e9f59068..aeb8b3d05 100644 --- a/tests/test_store_arch.c +++ b/tests/test_store_arch.c @@ -1189,6 +1189,68 @@ TEST(adr_validate_keys_empty) { PASS(); } +/* ── cbm_store_fetch_call_edges / cbm_store_free_call_edges tests ─ */ + +TEST(fetch_call_edges_basic) { + cbm_store_t *s = setup_arch_test_store(); + ASSERT_NOT_NULL(s); + + int64_t *src = NULL; + int64_t *tgt = NULL; + int count = 0; + bool truncated = true; + ASSERT_EQ(cbm_store_fetch_call_edges(s, "test", 100, &src, &tgt, &count, &truncated), + CBM_STORE_OK); + /* setup_arch_test_store() wires 5 CALLS edges among Function nodes. */ + ASSERT_EQ(count, 5); + ASSERT_FALSE(truncated); + ASSERT_NOT_NULL(src); + ASSERT_NOT_NULL(tgt); + + cbm_store_free_call_edges(src, tgt); + cbm_store_close(s); + PASS(); +} + +TEST(fetch_call_edges_truncated) { + cbm_store_t *s = setup_arch_test_store(); + ASSERT_NOT_NULL(s); + + int64_t *src = NULL; + int64_t *tgt = NULL; + int count = 0; + bool truncated = false; + /* Only 5 CALLS edges exist; capping at 2 must set truncated. */ + ASSERT_EQ(cbm_store_fetch_call_edges(s, "test", 2, &src, &tgt, &count, &truncated), + CBM_STORE_OK); + ASSERT_EQ(count, 2); + ASSERT_TRUE(truncated); + + cbm_store_free_call_edges(src, tgt); + cbm_store_close(s); + PASS(); +} + +TEST(fetch_call_edges_empty_project) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + cbm_store_upsert_project(s, "empty", "/tmp/empty"); + + int64_t *src = NULL; + int64_t *tgt = NULL; + int count = 0; + bool truncated = true; + ASSERT_EQ(cbm_store_fetch_call_edges(s, "empty", 100, &src, &tgt, &count, &truncated), + CBM_STORE_OK); + ASSERT_EQ(count, 0); + ASSERT_FALSE(truncated); + + /* cbm_store_free_call_edges(NULL, NULL) must not crash. */ + cbm_store_free_call_edges(src, tgt); + cbm_store_close(s); + PASS(); +} + /* ── Louvain tests ──────────────────────────────────────────────── */ TEST(louvain_basic) { @@ -1214,7 +1276,7 @@ TEST(louvain_basic) { /* Triangle and pair different */ ASSERT_TRUE(comm[1] != comm[4]); - free(result); + cbm_leiden_free(result); PASS(); } @@ -1223,7 +1285,7 @@ TEST(louvain_empty) { int count = 0; ASSERT_EQ(cbm_louvain(NULL, 0, NULL, 0, &result, &count), CBM_STORE_OK); ASSERT_EQ(count, 0); - free(result); + cbm_leiden_free(result); PASS(); } @@ -1234,7 +1296,7 @@ TEST(louvain_single_node) { ASSERT_EQ(cbm_louvain(nodes, 1, NULL, 0, &result, &count), CBM_STORE_OK); ASSERT_EQ(count, 1); ASSERT_EQ(result[0].node_id, 42); - free(result); + cbm_leiden_free(result); PASS(); } @@ -1297,7 +1359,7 @@ TEST(louvain_converges) { } ASSERT_TRUE(same_count >= 8); - free(result); + cbm_leiden_free(result); PASS(); } @@ -1431,7 +1493,7 @@ TEST(leiden_multilevel_collapses_noise) { } ASSERT_TRUE(same >= SZ - 1); } - free(result); + cbm_leiden_free(result); PASS(); } @@ -1463,8 +1525,8 @@ TEST(leiden_resolution_controls_granularity) { ASSERT_TRUE(n_hi > n_lo); ASSERT_TRUE(leiden_all_communities_connected(lo, N, edges, ne)); ASSERT_TRUE(leiden_all_communities_connected(hi, N, edges, ne)); - free(lo); - free(hi); + cbm_leiden_free(lo); + cbm_leiden_free(hi); PASS(); } @@ -1727,6 +1789,11 @@ SUITE(store_arch) { RUN_TEST(adr_splice_refuses_unterminated_fence); RUN_TEST(adr_validate_keys_empty); + /* Call-edge fetch / free */ + RUN_TEST(fetch_call_edges_basic); + RUN_TEST(fetch_call_edges_truncated); + RUN_TEST(fetch_call_edges_empty_project); + /* Louvain */ RUN_TEST(louvain_basic); RUN_TEST(louvain_empty); diff --git a/tests/test_ui.c b/tests/test_ui.c index a403dbbbe..c1975a4ca 100644 --- a/tests/test_ui.c +++ b/tests/test_ui.c @@ -548,7 +548,7 @@ TEST(layout_to_json) { ASSERT(strstr(json, "\"hello\"") != NULL); ASSERT(strstr(json, "\"Function\"") != NULL); - free(json); + cbm_layout_free_json(json); cbm_layout_free(r); cbm_store_close(store); PASS(); @@ -719,7 +719,7 @@ TEST(layout_dead_code_classification) { ASSERT_NOT_NULL(json); ASSERT(strstr(json, "\"status\":\"dead\"") != NULL); ASSERT(strstr(json, "\"in_calls\":2") != NULL); - free(json); + cbm_layout_free_json(json); cbm_layout_free(r); cbm_store_close(store); From 651ae3c2af1a22ec61a498b39350785aee480973 Mon Sep 17 00:00:00 2001 From: Ilya Brykau Date: Fri, 28 Aug 2026 17:26:26 +0200 Subject: [PATCH 008/217] fix(pipeline): suppress weak short-name matches for Go selector calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Go selector call x.foo() whose receiver the Go LSP cannot type falls through to the generic registry resolver, which binds it by bare short name to an arbitrary same-named project symbol. Stdlib calls are the worst case: f.Close() on an *os.File gets a CALLS edge to whatever project Close wins candidate ranking (measured on a real Go repo: confidence 0.11, 15 candidates; suffix_match + unique_name were 36% of all CALLS edges, and one 14-line stdlib-only function got 3 out of 3 false outbound edges). Extend the TS/JS receiver-aware guard (#592/#606) to Go: - extract_calls.c: flag Go call_expression with a selector_expression callee as is_method, mirroring the TS/JS member_expression flag. - registry.c: add cbm_go_suppress_weak_method_match. Unlike the TS/JS drop-list, field_type_hint is KEPT (Go struct fields carry declared types, so the hint is receiver-aware — lrp_go_s8_field_type_hint), and unique_name is dropped only when its confidence carries the import-unreachability penalty (the stdlib-hijack shape); an unpenalized lone candidate inside the caller's import closure never enters the field-type-hint upgrade and must survive. - pass_calls.c / pass_parallel.c: feed the Go gate next to the TS/JS one; the drop still defers to the emit path so service/route/HTTP edges stay main-identical. Reproduce-first: pipeline_go_receiver_suppresses_weak_method_edge is RED without the extractor flag (the f.Close -> project Close edge exists) and GREEN with it; typed same-package calls, bare local calls and import-qualified cross-package calls still resolve. The old extraction contract test used Go as the flag-exempt language — Python takes that role, and extract_go_selector_call_flags_is_method pins the new behavior. Signed-off-by: Ilya Brykau --- internal/cbm/extract_calls.c | 18 +++++++ src/pipeline/pass_calls.c | 12 ++++- src/pipeline/pass_parallel.c | 8 ++- src/pipeline/pipeline.h | 9 ++++ src/pipeline/registry.c | 27 +++++++++++ tests/test_extraction.c | 39 +++++++++++++-- tests/test_pipeline.c | 94 ++++++++++++++++++++++++++++++++++++ tests/test_registry.c | 42 ++++++++++++++++ 8 files changed, 242 insertions(+), 7 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index 6bb4a32bc..b7661d5e6 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -3619,6 +3619,24 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML } } } + // Go receiver-aware guard (same direction as the TS/JS flag above). + // Flag a selector call x.foo(). The Go AST cannot separate a method + // call on a value from a package-qualified call — but every selector + // call the Go LSP or the import/qualified registry strategies CAN + // place never reaches the weak short-name guards, so the flag only + // bites on unresolvable receivers (`f.Close()` on an os.File, + // `sha256.New()` behind an unindexed import), where a project-wide + // short-name match fabricates an edge to an unrelated project + // symbol sharing the name. Bare calls (helper()) keep + // is_method=false and resolve same-module/import paths as before. + if (ctx->language == CBM_LANG_GO && + strcmp(ts_node_type(node), "call_expression") == 0) { + TSNode gofn = ts_node_child_by_field_name(node, TS_FIELD("function")); + if (!ts_node_is_null(gofn) && + strcmp(ts_node_type(gofn), "selector_expression") == 0) { + call.is_method = true; + } + } TSNode args = ts_node_child_by_field_name(node, TS_FIELD("arguments")); // ObjectScript stores args under oref_method/method_args, not the diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index b25e9f592..db6a86178 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -623,12 +623,20 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, * language gated on only one resolver produces an edge on the sequential * path and not the parallel one (or vice versa), breaking MT determinism. * ArkTS belongs to the JS/TS family here (#1842); dropping it would - * reintroduce the #592/#606 false-edge class for .ets files. */ + * reintroduce the #592/#606 false-edge class for .ets files. + * + * Go (#1906) rides the same deferred-drop plumbing through its OWN + * predicate: its drop-list differs (field_type_hint is receiver-aware for + * Go, and unique_name drops only when import-unreachability-penalized), so + * it composes via cbm_go_suppress_weak_method_match instead of widening + * the shared gate. Same lockstep rule: mirror pass_parallel.c. */ bool suppress_weak_member = lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; bool drop_plain_call = - cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy); + cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy) || + cbm_go_suppress_weak_method_match(lang == CBM_LANG_GO, call->is_method, res.strategy, + res.confidence); /* Service-pattern HTTP/ASYNC calls to an EXTERNAL client library (e.g. * `requests.get("/api/orders/{id}")`) resolve to a QN containing the library diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 1ce766927..0a43a3f5e 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2488,12 +2488,16 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB * #606 direction. * * This language set MUST match the one in pass_calls.c exactly — see the - * note there. ArkTS belongs to the JS/TS family (#1842). */ + * note there. ArkTS belongs to the JS/TS family (#1842). Go (#1906) + * composes via its own predicate (different drop-list — see + * cbm_go_suppress_weak_method_match), mirrored in pass_calls.c. */ bool suppress_weak_member = lang == CBM_LANG_PYTHON || lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; bool drop_plain_call = - cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy); + cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy) || + cbm_go_suppress_weak_method_match(lang == CBM_LANG_GO, call->is_method, res.strategy, + res.confidence); /* Service-pattern HTTP/ASYNC client call (`requests.get(url)`): the * service signal lives in the callee_name. The registry can mis-resolve diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index d174bb6d6..6122f450d 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -280,6 +280,15 @@ bool cbm_perl_suppress_generic_match(bool is_perl, bool is_method, const char *c * Pure; unit-tested in test_registry.c. */ bool cbm_suppress_weak_member_match(bool enabled, bool is_method, const char *strategy); +/* Go analog of the TS/JS guard, same failure class: a selector call whose + * receiver the Go LSP could not type must not be bound by a receiver-blind + * short-name strategy. Drops suffix_match / fuzzy always, and unique_name only + * when its confidence is import-unreachability-penalized (the stdlib/vendor + * hijack shape). field_type_hint is deliberately NOT dropped for Go — struct + * fields carry declared types, so the hint is receiver-aware there. */ +bool cbm_go_suppress_weak_method_match(bool is_go, bool is_method, const char *strategy, + double confidence); + /* #725: drop a suffix_match CALLS edge when the caller language and the * target file's language disagree. unique_name (candidates == 1) is #1572 * and is left alone; same_module / import_map / lsp_* are kept. JS/TS/TSX diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index eefdb7596..10ee864bd 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -464,6 +464,33 @@ bool cbm_suppress_weak_member_match(bool enabled, bool is_method, const char *st strcmp(strategy, "field_type_hint") == 0 || strcmp(strategy, "fuzzy") == 0; } +bool cbm_go_suppress_weak_method_match(bool is_go, bool is_method, const char *strategy, + double confidence) { + if (!is_go || !is_method || !strategy || !strategy[0]) { + return false; + } + /* Go analog of the TS/JS guard above, same failure class: a selector call + * whose receiver the Go LSP could not type reaches the registry and a bare + * short-name strategy binds it to an arbitrary same-named project symbol + * (`f.Close()` on an os.File -> a project `Close`, suffix_match over 15 + * candidates). Unlike the TS/JS list, field_type_hint is KEPT: a Go struct + * field carries a declared type, so the parallel resolver's field-type + * hint is receiver-aware for Go (lrp_go_s8_field_type_hint), not a + * heuristic. */ + if (strcmp(strategy, "suffix_match") == 0 || strcmp(strategy, "fuzzy") == 0) { + return true; + } + /* unique_name is dropped only when PENALIZED: resolve_name_lookup scales + * CONF_UNIQUE_NAME by DEFAULT_CONFIDENCE exactly when the lone candidate + * is not reachable through the caller's imports — the stdlib/vendor + * hijack shape (`io.Copy` -> a project `Copy`). An unpenalized + * unique_name target sits inside the caller's import closure (or the + * file has no imports, e.g. a same-package call) and must be kept — + * dropping it kills genuinely-typed lone-candidate calls that never + * enter the field-type-hint upgrade (candidate_count == 1). */ + return strcmp(strategy, "unique_name") == 0 && confidence < CONF_UNIQUE_NAME; +} + static bool js_ts_family(CBMLanguage lang) { return lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 5b8d16f61..6f7d0c085 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4743,9 +4743,14 @@ TEST(extract_perl_method_call_flags_is_method) { /* Languages OUTSIDE the is_method flag set (only Perl and TS/JS/TSX set it) must * be unaffected: a Go method call never sets is_method. */ TEST(extract_flag_exempt_method_call_not_flagged_is_method) { - CBMFileResult *r = extract("package m\n" - "func run(o Obj) { o.Commit(); helper() }\n", - CBM_LANG_GO, "t", "x.go"); + /* Rust is flag-exempt: only Perl, Python, TS/JS and Go set is_method. + * Guards the blast radius of the receiver-aware flags for every other + * language. */ + CBMFileResult *r = extract("fn run(o: Obj) {\n" + " o.commit();\n" + " helper();\n" + "}\n", + CBM_LANG_RUST, "t", "x.rs"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); for (int i = 0; i < r->calls.count; i++) { @@ -4824,6 +4829,33 @@ TEST(extract_python_member_call_flags_is_method) { PASS(); } +TEST(extract_go_selector_call_flags_is_method) { + /* Go selector calls are flagged so the weak-match guard can fire when the + * Go LSP cannot type the receiver; bare calls stay unflagged. */ + CBMFileResult *r = extract("package m\n" + "func run(o Obj) { o.Commit(); helper() }\n", + CBM_LANG_GO, "t", "x.go"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + bool saw_selector = false; + bool saw_bare = false; + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *c = &r->calls.items[i]; + if (c->callee_name && strstr(c->callee_name, "Commit") != NULL) { + ASSERT_TRUE(c->is_method); + saw_selector = true; + } + if (c->callee_name && strcmp(c->callee_name, "helper") == 0) { + ASSERT_FALSE(c->is_method); + saw_bare = true; + } + } + ASSERT_TRUE(saw_selector); + ASSERT_TRUE(saw_bare); + cbm_free_result(r); + PASS(); +} + /* TS/JS/TSX receiver-aware flag (#592/#606; same intent as the Perl flag above). * A member call x.foo() with a non-this/super receiver is flagged is_method so * the resolver can suppress a weak short-name match (`re.test()` must not bind a @@ -6503,6 +6535,7 @@ SUITE(extraction) { RUN_TEST(extract_perl_method_call_flags_is_method); RUN_TEST(extract_flag_exempt_method_call_not_flagged_is_method); RUN_TEST(extract_python_member_call_flags_is_method); + RUN_TEST(extract_go_selector_call_flags_is_method); RUN_TEST(extract_ts_member_call_flags_is_method); RUN_TEST(extract_ts_this_super_receiver_not_flagged); RUN_TEST(extract_js_member_call_flags_is_method); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index d62a5572f..2c7f45732 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4693,6 +4693,99 @@ TEST(pipeline_python_receiver_suppresses_weak_method_edge) { PASS(); } +TEST(pipeline_go_receiver_suppresses_weak_method_edge) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_recv_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + /* go.mod makes project imports resolvable — real Go repos always have one, + * and import reachability (the unique_name penalty) depends on it. */ + write_temp_file(tmp, "go.mod", "module example.com/myapp\n\ngo 1.22\n"); + /* The lone project symbol named "Close" — a real method. */ + write_temp_file(tmp, "storage/storage.go", + "package storage\n" + "\n" + "type Storage struct{ open bool }\n" + "\n" + "func NewStorage() *Storage { return &Storage{open: true} }\n" + "\n" + "func (s *Storage) Close() {\n" + "\ts.open = false\n" + "}\n" + "\n" + "func Boot() {\n" + "\ts := NewStorage()\n" + "\ts.Close()\n" + "}\n"); + /* Cross-package control target: imported by hash.go, so the caller file + * has a non-empty import map (like any real Go file) and unreachable + * unique_name candidates get the import penalty. */ + write_temp_file(tmp, "util/util.go", + "package util\n" + "\n" + "func Tag() string { return \"t\" }\n"); + /* Stdlib receiver: `f.Close()` closes an *os.File, NOT the project method. + * The Go LSP cannot bind it to a project symbol → the registry would guess + * Close by short name (weak). This is the false edge to suppress — + * the exact shape that attached every file/rows/gzip Close in a real Go + * repo to one unrelated project method. */ + write_temp_file(tmp, "hash/hash.go", + "package hash\n" + "\n" + "import (\n" + "\t\"os\"\n" + "\n" + "\t\"example.com/myapp/util\"\n" + ")\n" + "\n" + "func FileLen(path string) int64 {\n" + "\tf, err := os.Open(path)\n" + "\tif err != nil {\n" + "\t\treturn 0\n" + "\t}\n" + "\tdefer f.Close()\n" + "\tst, err := f.Stat()\n" + "\tif err != nil {\n" + "\t\treturn 0\n" + "\t}\n" + "\treturn st.Size()\n" + "}\n" + "\n" + "func localHelper() int { return 1 }\n" + "\n" + "func CallsLocal() int { return localHelper() }\n" + "\n" + "func UsesUtil() string { return util.Tag() }\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/go_recv.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* (1) The false edge is suppressed (reproduce-first: RED before the fix). */ + ASSERT_FALSE(cross_file_call_exists(s, project, "FileLen", "Close")); + /* (2) The same-package typed-receiver call survives (LSP / same_module — + * both outside the weak drop-list). */ + ASSERT_TRUE(cross_file_call_exists(s, project, "Boot", "Close")); + /* (3) The bare local call survives (is_method stays false for bare calls). */ + ASSERT_TRUE(cross_file_call_exists(s, project, "CallsLocal", "localHelper")); + /* (4) The import-qualified cross-package call survives (import-aware + * strategies are outside the drop-list). */ + ASSERT_TRUE(cross_file_call_exists(s, project, "UsesUtil", "Tag")); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + /* Fixture for the #1928 cross-language reference-guard probes (sequential and * parallel twins). pad_files > 0 adds filler files to push the index over the * parallel-pipeline threshold, since USAGE/WRITES/READS have one resolver per @@ -13115,6 +13208,7 @@ SUITE(pipeline) { #endif RUN_TEST(pipeline_tsjs_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_python_receiver_suppresses_weak_method_edge); + RUN_TEST(pipeline_go_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c_parallel); RUN_TEST(pipeline_go_bare_ref_never_binds_field); diff --git a/tests/test_registry.c b/tests/test_registry.c index 3e1d604af..b58e98dde 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -901,6 +901,46 @@ TEST(dynamic_suppress_keeps_high_confidence_and_non_methods) { PASS(); } +TEST(go_suppress_drops_weak_selector_matches) { + /* Go selector call with an untyped receiver, landed via a receiver-blind + * short-name strategy → drop (same failure class as #592/#606). + * suffix_match/fuzzy drop at any confidence; unique_name drops only when + * import-unreachability-penalized (CONF_UNIQUE_NAME 0.75 * 0.5 = 0.375 — + * the `io.Copy` -> project `Copy` stdlib-hijack shape). */ + ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "suffix_match", 0.9)); + ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "suffix_match", 0.11)); + ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "fuzzy", 0.9)); + ASSERT_TRUE(cbm_go_suppress_weak_method_match(true, true, "unique_name", 0.375)); + PASS(); +} + +TEST(go_suppress_keeps_typed_and_import_aware_matches) { + /* Unpenalized unique_name = lone candidate inside the caller's import + * closure (or an import-free file, e.g. same-package) — a genuinely-typed + * lone-candidate call never enters the field-type-hint upgrade, so it must + * survive (lrp_go_s8_field_type_hint). */ + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "unique_name", 0.75)); + /* field_type_hint is receiver-aware for Go — struct fields carry declared + * types (lrp_go_s8_field_type_hint) — so it stays, unlike the TS/JS list. */ + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "field_type_hint", 0.85)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "same_module", 0.9)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "import_map", 0.95)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "import_map_suffix", 0.9)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "qualified_suffix", 0.9)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "callee_suffix", 0.5)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "service_pattern", 0.5)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "lsp_strategy_cross_file", 0.92)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "lsp_direct", 0.95)); + /* A bare call (is_method=false) is a free-function call → never suppressed. */ + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, false, "suffix_match", 0.11)); + /* Non-Go languages are never affected by this gate. */ + ASSERT_FALSE(cbm_go_suppress_weak_method_match(false, true, "suffix_match", 0.11)); + /* No match (NULL/empty strategy) → nothing to suppress. */ + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, NULL, 0.5)); + ASSERT_FALSE(cbm_go_suppress_weak_method_match(true, true, "", 0.5)); + PASS(); +} + /* ── Suite ─────────────────────────────────────────────────────── */ /* Method call THROUGH an imported symbol that is itself an indexed node @@ -997,4 +1037,6 @@ SUITE(registry) { RUN_TEST(go_bare_ref_never_binds_field); RUN_TEST(dynamic_suppress_drops_weak_method_matches); RUN_TEST(dynamic_suppress_keeps_high_confidence_and_non_methods); + RUN_TEST(go_suppress_drops_weak_selector_matches); + RUN_TEST(go_suppress_keeps_typed_and_import_aware_matches); } From dbba7573016e17f20fe5afaafe5cc2cddd98744e Mon Sep 17 00:00:00 2001 From: Ilya Brykau Date: Sat, 29 Aug 2026 18:04:16 +0200 Subject: [PATCH 009/217] fix(extract): receiver-qualify Go method QNs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Go method's QN was the flat package form (proj.pkg.method) — the receiver was ignored, so every same-name method in a package collided on one QN and the graph upsert kept exactly one node. Measured on a real Go repo: 20 Process/Name methods across 15 files kept 2 nodes; 9 different Task() methods fused into one chimera node carrying all nine bodies' call edges; 19 structs pointed DEFINES_METHOD at a single shared method node; a _test.go mock Close outranked the production Close in the dedupe tie-break. The upsert's own comment calls kind-disambiguated QNs 'the real cure'. Qualify the QN with the receiver type (proj.pkg.Recv.method), the same shape as Go interface members and the C++ out-of-line method path right below it in extract_func_def: - extract_defs.c: def.qualified_name = parent_class + name whenever the receiver type resolves; go_receiver_type_name becomes the shared cbm_go_receiver_type_name (exported via helpers.h) so both sides of the contract use one formula. - extract_unified.c (compute_func_qn): mirror branch for method_declaration, so method-body calls keep exact source attribution instead of degrading to File-node fallback (calls_find_source). - Consumers already agree: pxc_build_lsp_def passes the def QN and parent_class (receiver_type) verbatim into the Go LSP registries, and check_go_class_implements explicitly supports class-qualified method QNs (its path (b)). Side effect: resolve_same_module's exact module.name hash no longer matches concrete methods, which kills the conf-0.9 false edges where an interface-typed call bound to an unrelated same-package method. Fixes #1909 Signed-off-by: Ilya Brykau --- internal/cbm/extract_defs.c | 14 +++++++-- internal/cbm/extract_unified.c | 19 +++++++++++++ internal/cbm/helpers.h | 6 ++++ internal/cbm/lsp/go_lsp.c | 25 ++++++++++------ tests/test_extraction.c | 50 ++++++++++++++++++++++++++++++-- tests/test_parallel.c | 4 ++- tests/test_pipeline.c | 52 ++++++++++++++++++++++++++++++++++ 7 files changed, 156 insertions(+), 14 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 2e359af67..b298ec50c 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -3508,7 +3508,7 @@ static void set_def_complexity(CBMDefinition *def, TSNode body, const CBMLangSpe * Walks to the parameter_declaration's `type` field, unwrapping pointer_type * and generic_type, and returns the type_identifier text (e.g. "OrderService"). * Returns NULL if no type_identifier is found. */ -static char *go_receiver_type_name(CBMArena *a, TSNode recv, const char *source) { +char *cbm_go_receiver_type_name(CBMArena *a, TSNode recv, const char *source) { uint32_t nc = ts_node_child_count(recv); for (uint32_t i = 0; i < nc; i++) { TSNode child = ts_node_child(recv, i); @@ -3720,12 +3720,22 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec * (and downstream Go IMPLEMENTS/OVERRIDE) link the method to its owning * struct/type node. The parent QN must match the type's node QN, which * is computed the same way (cbm_fqn_compute on the type name). */ - char *recv_type = go_receiver_type_name(a, recv, ctx->source); + char *recv_type = cbm_go_receiver_type_name(a, recv, ctx->source); if (recv_type && recv_type[0]) { /* Must match the Go type node QN (directory-based module) so the * DEFINES_METHOD edge links the method to its owning type. */ def.parent_class = cbm_fqn_compute_source_lang(a, ctx->project, ctx->rel_path, recv_type, ctx->language); + /* Receiver-qualify the method QN (proj.pkg.Recv.method) — same + * shape as the C++ out-of-line path below and Go interface + * members. With the flat proj.pkg.method QN every same-name + * method in a package collided in the graph upsert: one body + * survived and the twins' call edges accreted onto it. The + * call-scope side (compute_func_qn in extract_unified.c) mirrors + * this formula, and go_lsp consumers read the def QN and + * parent_class (receiver_type) verbatim, so resolution joins + * stay exact. */ + def.qualified_name = cbm_arena_sprintf(a, "%s.%s", def.parent_class, name); } } diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index a5e314272..8502c37b8 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -899,6 +899,25 @@ static const char *compute_func_qn(CBMExtractCtx *ctx, TSNode node, const CBMLan } } + /* Go method `func (s *Storage) Close() {...}`: the def extractor records + * this as Method "proj.pkg.Storage.Close" (receiver-qualified, mirroring + * the C++ out-of-line rule above). The call-scope QN must match — a bare + * "proj.pkg.Close" names a node that no longer exists, so every call + * inside the method body would fall back to File-node attribution + * (calls_find_source). Same ONE-formula contract as the def side: + * cbm_go_receiver_type_name + cbm_fqn_compute_source_lang. */ + if (ctx->language == CBM_LANG_GO && strcmp(ts_node_type(node), "method_declaration") == 0) { + TSNode recv = ts_node_child_by_field_name(node, TS_FIELD("receiver")); + if (!ts_node_is_null(recv)) { + char *recv_type = cbm_go_receiver_type_name(ctx->arena, recv, ctx->source); + if (recv_type && recv_type[0]) { + const char *type_qn = cbm_fqn_compute_source_lang( + ctx->arena, ctx->project, ctx->rel_path, recv_type, ctx->language); + return cbm_arena_sprintf(ctx->arena, "%s.%s", type_qn, name); + } + } + } + /* Nix: a binding's own attrpath contributes scope (`a.b.fn = …`), and the def * extractor bakes it into the def QN. Compose it identically here — otherwise * an in-body call sources to a QN one or more segments short of the def, and diff --git a/internal/cbm/helpers.h b/internal/cbm/helpers.h index 4e8150b6f..29a858328 100644 --- a/internal/cbm/helpers.h +++ b/internal/cbm/helpers.h @@ -119,6 +119,12 @@ TSNode cbm_resolve_func_name(TSNode node, CBMLanguage lang); // def extractor — drift dropped the class qualifier from in-body calls (#554/#621). char *cbm_cpp_out_of_line_parent_class(CBMArena *a, TSNode node, const char *source); +/* Go: resolve a method_declaration's receiver parameter_list down to the bare + * receiver type_identifier (unwrapping pointer_type / generic_type). Shared by + * def extraction (extract_defs.c) and call-scope attribution + * (extract_unified.c) so the receiver-qualified method QN has ONE formula. */ +char *cbm_go_receiver_type_name(CBMArena *a, TSNode recv, const char *source); + // Find a child node by kind string. TSNode cbm_find_child_by_kind(TSNode parent, const char *kind); diff --git a/internal/cbm/lsp/go_lsp.c b/internal/cbm/lsp/go_lsp.c index 7efa273e1..812026291 100644 --- a/internal/cbm/lsp/go_lsp.c +++ b/internal/cbm/lsp/go_lsp.c @@ -1859,15 +1859,24 @@ static void process_function(GoLSPContext* ctx, TSNode func_node) { char* func_name = lsp_node_text(ctx, name_node); if (!func_name || !func_name[0]) return; - // Enclosing-function QN must be the BARE package.Func form (no receiver - // type segment). The textual call events (extract_unified.c) source calls - // as package_qn.func_name — methods included — and the defs pass creates - // the graph Method node under the same QN, so any other form breaks the - // caller-QN join in cbm_pipeline_find_lsp_resolution and the LSP-resolved - // call silently falls back to the registry short-name resolver. The - // receiver type still reaches the registry via the def's parent_class / - // method->receiver_type; it just does not appear in the caller QN. + // Enclosing-function QN must be EXACTLY the QN the textual call events + // (extract_unified.c compute_func_qn) and the defs pass produce, or the + // caller-QN join in cbm_pipeline_find_lsp_resolution breaks and the + // LSP-resolved call silently falls back to the registry short-name + // resolver. Since #1909 receiver-qualified Go method QNs, that shared + // form is package_qn.ReceiverType.method for methods (the ONE-formula + // contract: cbm_go_receiver_type_name) and package_qn.func for functions. ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->package_qn, func_name); + if (strcmp(ts_node_type(func_node), "method_declaration") == 0) { + TSNode recv = ts_node_child_by_field_name(func_node, "receiver", 8); + if (!ts_node_is_null(recv)) { + char *recv_type = cbm_go_receiver_type_name(ctx->arena, recv, ctx->source); + if (recv_type && recv_type[0]) { + ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.%s.%s", ctx->package_qn, + recv_type, func_name); + } + } + } // Push function scope CBMScope* saved_scope = ctx->current_scope; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 6f7d0c085..8f5fbf2d2 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4254,12 +4254,12 @@ TEST(extract_go_no_filename_in_module_qn) { ASSERT_NOT_NULL(conn); ASSERT_STR_EQ(conn->qualified_name, "proj.myapp.db.Conn"); - /* Go method nodes keep a FLAT QN (module + name) with a separate - * parent_class link to the receiver type — the QN must carry the + /* Go method nodes carry a receiver-qualified QN (module + receiver type + + * name) plus the parent_class link — the QN must carry the * directory-based module and NOT the `.conn.` filename segment. */ const CBMDefinition *query = find_def_by_name(r, "Query"); ASSERT_NOT_NULL(query); - ASSERT_STR_EQ(query->qualified_name, "proj.myapp.db.Query"); + ASSERT_STR_EQ(query->qualified_name, "proj.myapp.db.Conn.Query"); ASSERT_EQ(strstr(query->qualified_name, ".conn."), NULL); /* The method's parent_class must match the type node QN (for DEFINES_METHOD). */ ASSERT_NOT_NULL(query->parent_class); @@ -4856,6 +4856,49 @@ TEST(extract_go_selector_call_flags_is_method) { PASS(); } +TEST(extract_go_method_receiver_qualified_qn) { + /* A Go method QN carries the receiver type (proj.pkg.Recv.method), same + * shape as C++ out-of-line methods and Go interface members — so two + * same-name methods on different receivers no longer collide in the + * graph upsert. Free functions keep the flat package QN. */ + CBMFileResult *r = extract("package m\n" + "type Storage struct{}\n" + "type Cache struct{}\n" + "func (s *Storage) Close() {}\n" + "func (c Cache) Close() {}\n" + "func Shutdown() {}\n", + CBM_LANG_GO, "t", "x.go"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + bool saw_storage = false; + bool saw_cache = false; + bool saw_free = false; + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (!d->name || !d->qualified_name) { + continue; + } + if (strcmp(d->name, "Close") == 0 && strcmp(d->qualified_name, "t.Storage.Close") == 0) { + ASSERT_STR_EQ(d->label, "Method"); + ASSERT_STR_EQ(d->parent_class, "t.Storage"); + saw_storage = true; + } + if (strcmp(d->name, "Close") == 0 && strcmp(d->qualified_name, "t.Cache.Close") == 0) { + ASSERT_STR_EQ(d->parent_class, "t.Cache"); + saw_cache = true; + } + if (strcmp(d->name, "Shutdown") == 0) { + ASSERT_STR_EQ(d->qualified_name, "t.Shutdown"); + saw_free = true; + } + } + ASSERT_TRUE(saw_storage); + ASSERT_TRUE(saw_cache); + ASSERT_TRUE(saw_free); + cbm_free_result(r); + PASS(); +} + /* TS/JS/TSX receiver-aware flag (#592/#606; same intent as the Perl flag above). * A member call x.foo() with a non-this/super receiver is flagged is_method so * the resolver can suppress a weak short-name match (`re.test()` must not bind a @@ -6536,6 +6579,7 @@ SUITE(extraction) { RUN_TEST(extract_flag_exempt_method_call_not_flagged_is_method); RUN_TEST(extract_python_member_call_flags_is_method); RUN_TEST(extract_go_selector_call_flags_is_method); + RUN_TEST(extract_go_method_receiver_qualified_qn); RUN_TEST(extract_ts_member_call_flags_is_method); RUN_TEST(extract_ts_this_super_receiver_not_flagged); RUN_TEST(extract_js_member_call_flags_is_method); diff --git a/tests/test_parallel.c b/tests/test_parallel.c index 631446f9c..d3cd08417 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -3379,7 +3379,9 @@ TEST(parallel_go_cross_package_field_chain_resolves) { cbm_gbuf_t *gbuf = run_go_field_chain_sequential("go_field_fold", tmpdir, files, 2); ASSERT_NOT_NULL(gbuf); - const cbm_gbuf_edge_t *edge = find_call_edge_to_target_fragment(gbuf, "handler.PlaceOrder", ".service.PlaceOrder"); + /* Since #1909, Go method QNs are receiver-qualified on both endpoints. */ + const cbm_gbuf_edge_t *edge = find_call_edge_to_target_fragment( + gbuf, "OrderHandler.PlaceOrder", ".service.OrderService.PlaceOrder"); const bool found = edge != NULL; const bool dispatch = edge && edge->properties_json && diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 2c7f45732..96eac8714 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4693,6 +4693,57 @@ TEST(pipeline_python_receiver_suppresses_weak_method_edge) { PASS(); } +TEST(pipeline_go_method_caller_keeps_lsp_join) { + /* #1909: receiver-qualifying method QNs moves the def/textual-call QN to + * package.Type.method — go_lsp's enclosing-function QN must move with it + * (the ONE-formula contract), or every LSP resolution sourced from inside + * a method body loses its caller join and the edge dies. The callee name + * collides across receivers so no short-name fallback can mask the loss. */ + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_mcall_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_temp_file(tmp, "go.mod", "module example.com/fxmcall\n\ngo 1.22\n"); + write_temp_file(tmp, "svc/repo.go", + "package svc\n" + "\n" + "type Repo struct {\n" + "\tn int\n" + "}\n" + "\n" + "func (r *Repo) Install() int {\n" + "\treturn r.helper()\n" + "}\n"); + write_temp_file(tmp, "svc/helper.go", + "package svc\n" + "\n" + "func (r *Repo) helper() int {\n" + "\treturn r.n\n" + "}\n"); + write_temp_file(tmp, "other/other.go", + "package other\n" + "\n" + "type Decoy struct{}\n" + "\n" + "func (d *Decoy) helper() int { return 2 }\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/go_mcall.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + ASSERT_TRUE(cross_file_call_exists(s, project, "Install", "helper")); + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + TEST(pipeline_go_receiver_suppresses_weak_method_edge) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_recv_XXXXXX"); @@ -13208,6 +13259,7 @@ SUITE(pipeline) { #endif RUN_TEST(pipeline_tsjs_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_python_receiver_suppresses_weak_method_edge); + RUN_TEST(pipeline_go_method_caller_keeps_lsp_join); RUN_TEST(pipeline_go_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c_parallel); From adff62e7f20dd439129d17c5dc028c7069af92f2 Mon Sep 17 00:00:00 2001 From: Ilya Brykau Date: Sat, 29 Aug 2026 18:35:32 +0200 Subject: [PATCH 010/217] fix(extract): give each Go init() its own QN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Go allows any number of init() functions per package — even several in one file — and all of them run at start-up. On the flat QN (proj.pkg.init) they all collided and the graph upsert kept ONE node per package, silently dropping the rest: measured on a real Go repo, an event-source package registering each decoder in its own file's init() collapsed 21 init functions into one node (29 in source, 7 in the graph), erasing the whole registration pattern. Disambiguate with the #495 cfg-twin pattern: fold the file basename and line into the QN (proj.pkg.init#a.go:L5). Calling init explicitly is illegal in Go, so nothing ever joins on the plain QN; the call-scope side (compute_func_qn) mirrors the exact formula so init-body calls keep their source attribution instead of degrading to the calls_find_source File fallback. Reproduce-first: pipeline_go_multi_init_nodes_survive is RED without the def-side change (node count 1, expected 2) and GREEN with it; extract_go_multiple_init_disambiguated pins two same-file inits to distinct suffixed QNs. Fixes #1910 Signed-off-by: Ilya Brykau --- internal/cbm/extract_defs.c | 19 ++++++++++ internal/cbm/extract_unified.c | 16 +++++++++ internal/cbm/helpers.c | 36 +++++++++++++++++++ internal/cbm/helpers.h | 4 +++ tests/test_extraction.c | 63 ++++++++++++++++++++++++++++++++++ tests/test_pipeline.c | 50 +++++++++++++++++++++++++++ 6 files changed, 188 insertions(+) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index b298ec50c..af49ef485 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -3739,6 +3739,25 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec } } + /* Go allows any number of init() functions per package — even several in + * one file — and they all run at start-up. On the flat QN they all + * collided and the graph upsert kept ONE node per package, silently + * dropping the rest (#1910). Disambiguate with the #495 cfg-twin pattern: + * fold the file basename and the init's per-file ordinal into the QN. + * The ordinal (not the line) keeps the QN stable when code is edited + * above the function — the QN is node identity, and a line suffix would + * churn nodes on every unrelated insertion. Nothing ever joins on the + * plain QN — calling init explicitly is illegal in Go — and the + * call-scope side (compute_func_qn in extract_unified.c) computes the + * same cbm_go_init_ordinal so init-body calls keep their attribution. */ + if (ctx->language == CBM_LANG_GO && strcmp(def.label, "Function") == 0 && + strcmp(name, "init") == 0) { + const char *base = strrchr(ctx->rel_path, '/'); + base = base ? base + 1 : ctx->rel_path; + def.qualified_name = cbm_arena_sprintf(a, "%s#%s:%d", def.qualified_name, base, + cbm_go_init_ordinal(node, ctx->source)); + } + // C++/CUDA: out-of-line method definition (`Foo::bar` in a .cc/.cpp). The // class body in the header is declaration-only, so without this the // definition is recorded as a free Function. Promote it to a Method whose QN diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index 8502c37b8..1bcbd5753 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -918,6 +918,22 @@ static const char *compute_func_qn(CBMExtractCtx *ctx, TSNode node, const CBMLan } } + /* Go init(): the def extractor folds the file basename and the init's + * per-file ordinal into the QN so every init in a package survives the + * upsert (#1910, the #495 cfg-twin pattern; ordinal, not line, so the QN + * is stable under edits above the function). Mirror the exact formula + * here, or init-body calls fall back to File-node attribution + * (calls_find_source). */ + if (ctx->language == CBM_LANG_GO && strcmp(name, "init") == 0 && + strcmp(ts_node_type(node), "function_declaration") == 0) { + const char *base_qn = cbm_fqn_compute_source_lang(ctx->arena, ctx->project, ctx->rel_path, + name, ctx->language); + const char *base = strrchr(ctx->rel_path, '/'); + base = base ? base + 1 : ctx->rel_path; + return cbm_arena_sprintf(ctx->arena, "%s#%s:%d", base_qn, base, + cbm_go_init_ordinal(node, ctx->source)); + } + /* Nix: a binding's own attrpath contributes scope (`a.b.fn = …`), and the def * extractor bakes it into the def QN. Compose it identically here — otherwise * an in-body call sources to a QN one or more segments short of the def, and diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index e8a6eb8ec..5ac97f610 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -1734,6 +1734,42 @@ int cbm_classify_string(const char *str, int len) { * canonical parameter shape of server-side route paths * (`/things/${id}/x` -> "/things/{}/x"). Returns NULL when the node yields * no text or exceeds the route-sized buffer. */ +/* 1-based source-order ordinal of a top-level Go `init` function_declaration + * among the file's `init`s. Both the definition extractor and compute_func_qn + * fold it into the QN (`pkg.init#file.go:N`, #1910), so it must be computed + * from the tree on both sides with this one formula. Unlike a line number, + * the ordinal is stable under edits above the function — QNs are node + * identity, and a line-based suffix would churn nodes on every unrelated + * insertion (the #495 cfg suffix and the GoogleTest derivation are both + * edit-stable for the same reason). */ +int cbm_go_init_ordinal(TSNode fn_node, const char *source) { + TSNode root = fn_node; + for (TSNode p = ts_node_parent(root); !ts_node_is_null(p); p = ts_node_parent(p)) { + root = p; + } + int ordinal = 0; + uint32_t n = ts_node_named_child_count(root); + for (uint32_t i = 0; i < n; i++) { + TSNode child = ts_node_named_child(root, i); + if (strcmp(ts_node_type(child), "function_declaration") != 0) { + continue; + } + TSNode name = ts_node_child_by_field_name(child, TS_FIELD("name")); + if (ts_node_is_null(name)) { + continue; + } + uint32_t start = ts_node_start_byte(name); + if (ts_node_end_byte(name) - start != 4 || memcmp(source + start, "init", 4) != 0) { + continue; + } + ordinal++; + if (ts_node_eq(child, fn_node)) { + return ordinal; + } + } + return ordinal; +} + const char *cbm_template_string_text(CBMArena *a, TSNode node, const char *source) { enum { TPL_BUF = 512 }; char buf[TPL_BUF]; diff --git a/internal/cbm/helpers.h b/internal/cbm/helpers.h index 29a858328..e2f148ecf 100644 --- a/internal/cbm/helpers.h +++ b/internal/cbm/helpers.h @@ -197,6 +197,10 @@ bool cbm_is_loop_node_type(const char *kind); // Is this a module-level node? (not nested inside function/class body) bool cbm_is_module_level(TSNode node, CBMLanguage lang); +/* 1-based source-order ordinal of a Go `init` among its file's top-level + * `init` function_declarations (QN disambiguation, #1910). */ +int cbm_go_init_ordinal(TSNode fn_node, const char *source); + // Same check, but the node's PARENT is supplied directly — avoids the // O(n) ts_node_parent rescan. Use at call sites iterating a known // parent's children (the common case). `parent` is the parent of the diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 8f5fbf2d2..edecba47d 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4899,6 +4899,68 @@ TEST(extract_go_method_receiver_qualified_qn) { PASS(); } +TEST(extract_go_multiple_init_disambiguated) { + /* Go allows several init() per package (even per file); each gets a + * file+ordinal-suffixed QN (#495 cfg-twin pattern) so none is lost to the + * same-QN upsert. The ordinal — unlike a line number — must not move when + * code is inserted above the function: the QN is node identity, and a + * line-based suffix would churn nodes on every unrelated edit. */ + CBMFileResult *r = extract("package m\n" + "func init() { a() }\n" + "func init() { b() }\n" + "func a() {}\n" + "func b() {}\n", + CBM_LANG_GO, "t", "x.go"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + const char *first = NULL; + const char *second = NULL; + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (!d->name || strcmp(d->name, "init") != 0) { + continue; + } + if (!first) { + first = d->qualified_name; + } else { + second = d->qualified_name; + } + } + ASSERT_NOT_NULL(first); + ASSERT_NOT_NULL(second); + ASSERT_STR_EQ(first, "t.init#x.go:1"); + ASSERT_STR_EQ(second, "t.init#x.go:2"); + + /* Stability: an insertion ABOVE both inits must not change either QN. */ + CBMFileResult *r2 = extract("package m\n" + "import \"fmt\"\n" + "var shifted = fmt.Sprint(\"pad\")\n" + "func init() { a() }\n" + "func init() { b() }\n" + "func a() {}\n" + "func b() {}\n", + CBM_LANG_GO, "t", "x.go"); + ASSERT_NOT_NULL(r2); + ASSERT_FALSE(r2->has_error); + int seen = 0; + for (int i = 0; i < r2->defs.count; i++) { + const CBMDefinition *d = &r2->defs.items[i]; + if (!d->name || strcmp(d->name, "init") != 0) { + continue; + } + seen++; + if (seen == 1) { + ASSERT_STR_EQ(d->qualified_name, "t.init#x.go:1"); + } else { + ASSERT_STR_EQ(d->qualified_name, "t.init#x.go:2"); + } + } + ASSERT_EQ(seen, 2); + cbm_free_result(r2); + cbm_free_result(r); + PASS(); +} + /* TS/JS/TSX receiver-aware flag (#592/#606; same intent as the Perl flag above). * A member call x.foo() with a non-this/super receiver is flagged is_method so * the resolver can suppress a weak short-name match (`re.test()` must not bind a @@ -6580,6 +6642,7 @@ SUITE(extraction) { RUN_TEST(extract_python_member_call_flags_is_method); RUN_TEST(extract_go_selector_call_flags_is_method); RUN_TEST(extract_go_method_receiver_qualified_qn); + RUN_TEST(extract_go_multiple_init_disambiguated); RUN_TEST(extract_ts_member_call_flags_is_method); RUN_TEST(extract_ts_this_super_receiver_not_flagged); RUN_TEST(extract_js_member_call_flags_is_method); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 96eac8714..a5d5b545f 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4837,6 +4837,55 @@ TEST(pipeline_go_receiver_suppresses_weak_method_edge) { PASS(); } +TEST(pipeline_go_multi_init_nodes_survive) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_init_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + /* Two files, one package, one init() each — Go runs both at start-up. + * Pre-fix both collapsed onto one QN and the upsert kept one node + * (reproduce-first: RED asserts node count == 2). */ + write_temp_file(tmp, "go.mod", "module example.com/reg\n\ngo 1.22\n"); + write_temp_file(tmp, "reg/a.go", + "package reg\n" + "\n" + "var handlers = map[string]func(){}\n" + "\n" + "func init() { handlers[\"a\"] = handleA }\n" + "\n" + "func handleA() {}\n"); + write_temp_file(tmp, "reg/b.go", + "package reg\n" + "\n" + "func init() { handlers[\"b\"] = handleB }\n" + "\n" + "func handleB() {}\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/go_init.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* Both init functions survive as distinct nodes. */ + cbm_node_t *inits = NULL; + int ic = 0; + cbm_store_find_nodes_by_name(s, project, "init", &inits, &ic); + ASSERT_EQ(ic, 2); + cbm_store_free_nodes(inits, ic); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + /* Fixture for the #1928 cross-language reference-guard probes (sequential and * parallel twins). pad_files > 0 adds filler files to push the index over the * parallel-pipeline threshold, since USAGE/WRITES/READS have one resolver per @@ -13261,6 +13310,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_python_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_go_method_caller_keeps_lsp_join); RUN_TEST(pipeline_go_receiver_suppresses_weak_method_edge); + RUN_TEST(pipeline_go_multi_init_nodes_survive); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c_parallel); RUN_TEST(pipeline_go_bare_ref_never_binds_field); From ac46c17fc37f78ae518145e73d51b46ca3013fac Mon Sep 17 00:00:00 2001 From: Ilya Brykau Date: Sun, 30 Aug 2026 13:43:40 +0200 Subject: [PATCH 011/217] fix(pipeline): bind field_type_hint by owning QN segment, not substring try_field_type_hint accepted any candidate whose qualified name merely CONTAINED the hinted type name. A single-letter receiver (f.Close(), hint "F") matched almost every same-named candidate - the method name itself included - and rebound a call whose resolution had already failed to an arbitrary project symbol at confidence 0.85, presented as the one weak strategy the Go selector guard deliberately trusts. Require the candidate's owning dot-segment - the segment immediately before the method - to EQUAL the hinted type (or its I-prefixed interface form). The intended variable-named-after-its-type case keeps resolving (lrp_go_s8_field_type_hint stays green); the fabricated bindings lose their only anchor and fall back to the weak short-name strategies the guards already handle. Measured on a real ~1150-file Go repo (with the #1906/#1909/#1910 fixes applied): field_type_hint 1484 -> 356 CALLS edges, the single-character-receiver class 376 -> 0, Go->C/C++ targets 34 -> 0, and no other strategy absorbed the removed edges. 63 edges moved to the interface that declares the method instead of an arbitrary implementation. Fixes #1927 Signed-off-by: Ilya Brykau --- src/pipeline/pass_parallel.c | 26 +++++++- tests/test_pipeline.c | 120 +++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 1 deletion(-) diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 0a43a3f5e..6fffffea0 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2092,6 +2092,29 @@ static const cbm_gbuf_node_t *find_source_node(const cbm_gbuf_t *gbuf, const cha /* Field type hint resolution for obj.Method() with multiple candidates. * Strips C# field prefixes (_ / m_), capitalizes to get type name, and * checks if TypeName.Method or ITypeName.Method exists among candidates. */ +/* #1927: the hint may only bind a candidate whose OWNING dot-segment — the + * segment immediately before the method name — EQUALS the hinted type. The + * previous raw strstr() accepted the hinted name anywhere in the QN, so a + * single-letter receiver (`f.Close()` → hint "F") matched nearly every + * candidate ("F" ⊂ "…FileStore.Close", the method name included) and rebound + * a call whose resolution had already failed to an arbitrary project symbol + * at PP_FIELD_HINT_CONF. Segment equality keeps the intended + * variable-named-after-its-type case (`repo.Find()` → `Repo.Find`, + * lrp_go_s8_field_type_hint) and picks the candidate whose owner actually + * carries the name instead of whichever QN contains the letters. */ +static bool fth_owner_segment_is(const char *candidate_qn, const char *type_name) { + const char *last_dot = strrchr(candidate_qn, '.'); + if (!last_dot || last_dot == candidate_qn) { + return false; + } + const char *seg_start = last_dot; + while (seg_start > candidate_qn && seg_start[-1] != '.') { + seg_start--; + } + size_t seg_len = (size_t)(last_dot - seg_start); + return seg_len > 0 && strncmp(seg_start, type_name, seg_len) == 0 && type_name[seg_len] == '\0'; +} + static void try_field_type_hint(resolve_ctx_t *rc, cbm_resolution_t *res, const char *callee_name, int64_t source_id) { if (!res->qualified_name || res->candidate_count <= SKIP_ONE) { @@ -2131,7 +2154,8 @@ static void try_field_type_hint(resolve_ctx_t *rc, cbm_resolution_t *res, const int cand_count = 0; cbm_registry_find_by_name(rc->registry, method, &cands, &cand_count); for (int ci = 0; ci < cand_count; ci++) { - if (strstr(cands[ci], type_name) || strstr(cands[ci], iface_name)) { + if (fth_owner_segment_is(cands[ci], type_name) || + fth_owner_segment_is(cands[ci], iface_name)) { const cbm_gbuf_node_t *better = cbm_gbuf_find_by_qn(rc->main_gbuf, cands[ci]); if (better && better->id != source_id) { res->qualified_name = cands[ci]; diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index a5d5b545f..666b4674d 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4886,6 +4886,125 @@ TEST(pipeline_go_multi_init_nodes_survive) { PASS(); } +TEST(pipeline_go_parallel_field_hint_requires_owner_segment) { + /* #1927: try_field_type_hint (parallel resolver only) matched the hinted + * type name as a raw SUBSTRING of the candidate QN. A single-letter + * receiver — `f.Close()` on an *os.File — hints "F", and "F" is a + * substring of "…FileStore.Close", so a stdlib call was rebound to an + * arbitrary project method at PP_FIELD_HINT_CONF and presented as + * field_type_hint (the one weak strategy the Go selector guard trusts). + * The hint must fire only when the candidate's OWNING segment — the + * dot-segment immediately before the method — EQUALS the hinted type; + * otherwise the resolution stays a weak short-name match and the Go + * guard (#1906) drops it. >= 50 files forces pass_parallel.c, the only + * path that runs try_field_type_hint(). */ + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_go_fth_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "go.mod", "module example.com/fxhint\n\ngo 1.22\n"); + /* Two project methods named Close → the registry resolves f.Close() as an + * ambiguous short-name match (candidate_count 2), the shape the hint + * upgrades. FileStore's QN carries the capital F the buggy substring + * latches onto; Conn is the second candidate. */ + write_temp_file(tmp, "filestore/filestore.go", + "package filestore\n" + "\n" + "type FileStore struct{ open bool }\n" + "\n" + "func (fs *FileStore) Close() {\n" + "\tfs.open = false\n" + "}\n"); + write_temp_file(tmp, "conn/conn.go", + "package conn\n" + "\n" + "type Conn struct{ live bool }\n" + "\n" + "func (cn *Conn) Close() {\n" + "\tcn.live = false\n" + "}\n"); + /* Two project methods named Find → same ambiguity, but here the receiver + * variable IS named after its type (`finder`), the case the hint exists + * for: the owning segment of Finder.Find equals the hinted "Finder". */ + write_temp_file(tmp, "finder/finder.go", + "package finder\n" + "\n" + "type Finder struct{ n int }\n" + "\n" + "func NewFinder() *Finder { return &Finder{n: 1} }\n" + "\n" + "func (fi *Finder) Find(id int) int {\n" + "\treturn fi.n + id\n" + "}\n"); + write_temp_file(tmp, "locator/locator.go", + "package locator\n" + "\n" + "type Locator struct{ n int }\n" + "\n" + "func (lo *Locator) Find(id int) int {\n" + "\treturn lo.n - id\n" + "}\n"); + write_temp_file(tmp, "app/app.go", + "package app\n" + "\n" + "import (\n" + "\t\"os\"\n" + "\n" + "\txf \"example.com/fxhint/finder\"\n" + ")\n" + "\n" + "func Lookup(path string) int64 {\n" + "\tf, err := os.Open(path)\n" + "\tif err != nil {\n" + "\t\treturn 0\n" + "\t}\n" + "\tdefer f.Close()\n" + "\tst, err := f.Stat()\n" + "\tif err != nil {\n" + "\t\treturn 0\n" + "\t}\n" + "\treturn st.Size()\n" + "}\n" + "\n" + "func UseFinder(id int) int {\n" + "\tfinder := xf.NewFinder()\n" + "\treturn finder.Find(id)\n" + "}\n"); + for (int i = 0; i < 52; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "pad/filler%d.go", i); + snprintf(body, sizeof(body), "package pad\n\nfunc filler%d() int { return %d }\n", i, i); + write_temp_file(tmp, name, body); + } + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/go_fth.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* (1) Reproduce-first: RED before the fix. The substring hint rebinds the + * stdlib f.Close() to FileStore.Close ("F" ⊂ QN) at high confidence and + * the Go guard keeps it. With segment equality the upgrade is refused, + * the match stays suffix_match, and the Go guard drops it. */ + ASSERT_FALSE(cross_file_call_exists(s, project, "Lookup", "Close")); + /* (2) The intended hint case survives: `finder` names its type, so the + * owning segment of Finder.Find equals the hint. */ + ASSERT_TRUE(cross_file_call_exists(s, project, "UseFinder", "Find")); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + /* Fixture for the #1928 cross-language reference-guard probes (sequential and * parallel twins). pad_files > 0 adds filler files to push the index over the * parallel-pipeline threshold, since USAGE/WRITES/READS have one resolver per @@ -13310,6 +13429,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_python_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_go_method_caller_keeps_lsp_join); RUN_TEST(pipeline_go_receiver_suppresses_weak_method_edge); + RUN_TEST(pipeline_go_parallel_field_hint_requires_owner_segment); RUN_TEST(pipeline_go_multi_init_nodes_survive); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c); RUN_TEST(pipeline_go_rw_usage_never_cross_into_c_parallel); From 1e92269d7562d89a816271c4d99904ee53a94606 Mon Sep 17 00:00:00 2001 From: Mark Murawski Date: Wed, 2 Sep 2026 13:38:05 -0400 Subject: [PATCH 012/217] feat(workspace): add PATH_ALLOW_BROAD override for too-shallow roots CBM_WS_DENY_TOO_SHALLOW ("path is too broad to index as one root") has no override today: cbm_workspace_verdict_is_overridable() only lifts CBM_WS_DENY_SENSITIVE, and `allow-root --list` already documents shallow/absolute refusals as "always-refused" rather than configurable. That default is right -- a bare top-level tree like "/etc" or "/home" is almost always a mistake -- but it leaves no way to say "I mean this one specific broad root, on purpose." That's a real case: a person who deliberately wants one combined project spanning several sibling repos under a shared parent has no path forward today short of physically restructuring their checkout. PATH_ALLOW_BROAD names one exact canonical path that lifts a CBM_WS_DENY_TOO_SHALLOW verdict, and only that verdict: - exact match only, not a prefix -- naming one broad root must not quietly approve every root below it too, the same "/Users"-style breadth this depth rule exists to catch in the first place - process environment, not a recorded grant: it never appears in cbm_workspace_grant_list, and cbm_workspace_verdict_is_overridable() is untouched, so CBM_WS_DENY_ABSOLUTE and CBM_WS_DENY_SENSITIVE stay exactly as unliftable as before - read once per call site with getenv(), mirroring how CBM_ALLOWED_ROOT is already threaded into cbm_workspace_root_allowed() from each of its four callers, rather than read inside cbm_workspace_classify_root() itself -- that function is documented as a pure function of its arguments precisely so it stays host-independent and directly testable Also updates the refusal message to name the fix, the same way the sensitive-root refusal already names `allow-root --approve-sensitive`. Adds ws_allow_broad_root_lifts_too_shallow_for_an_exact_match_only alongside the existing too-shallow coverage, checking the exact-match requirement, the untouched sensitive/absolute paths, and that nothing is written to the grant store. Signed-off-by: Mark Murawski --- src/daemon/application.c | 3 ++- src/foundation/workspace.c | 28 +++++++++++++++++--- src/foundation/workspace.h | 11 +++++--- src/mcp/mcp.c | 6 +++-- src/ui/http_server.c | 3 ++- tests/test_workspace.c | 52 +++++++++++++++++++++++++++++++++----- 6 files changed, 86 insertions(+), 17 deletions(-) diff --git a/src/daemon/application.c b/src/daemon/application.c index 12543808e..ecb5095c7 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -437,7 +437,8 @@ static bool application_session_workspace_allowed(const cbm_daemon_application_s bool allowed = root && root[0] && cbm_workspace_root_allowed(root, cbm_workspace_home_dir(), cbm_workspace_cache_dir(), - cbm_mcp_server_allowed_root(session->mcp), boundary_error, + cbm_mcp_server_allowed_root(session->mcp), + getenv("PATH_ALLOW_BROAD"), boundary_error, sizeof(boundary_error)); if (!allowed) { cbm_log_warn("daemon.workspace.skipped", "operation", operation, "detail", diff --git a/src/foundation/workspace.c b/src/foundation/workspace.c index cb254f3d9..0daab3806 100644 --- a/src/foundation/workspace.c +++ b/src/foundation/workspace.c @@ -522,8 +522,8 @@ bool cbm_workspace_grant_add(const char *cache_dir, const char *home_dir, } bool cbm_workspace_root_allowed(const char *canonical_path, const char *home_dir, - const char *cache_dir, const char *configured_root, char *err, - size_t err_sz) { + const char *cache_dir, const char *configured_root, + const char *allow_broad_root, char *err, size_t err_sz) { if (err && err_sz) { err[0] = '\0'; } @@ -579,17 +579,37 @@ bool cbm_workspace_root_allowed(const char *canonical_path, const char *home_dir return true; } /* An explicit human approval recorded for exactly this path is the only thing - * that lifts a sensitive refusal. Absolute and shallow refusals cannot be - * lifted at all. */ + * that lifts a sensitive refusal. Absolute refusals cannot be lifted at all. */ if (verdict == CBM_WS_DENY_SENSITIVE && match.exact_sensitive) { return true; } + /* PATH_ALLOW_BROAD is a separate, narrower lever from the grant store above: + * an operator has named this exact root as one they have deliberately chosen + * to index whole, accepting that it may span multiple unrelated projects -- + * the tradeoff CBM_WS_DENY_TOO_SHALLOW normally refuses on their behalf. It + * must match canonical_path exactly, the same way an exact sensitive grant + * does above: a prefix match would let one broad allowance quietly cover + * every root below it too, which is the "/Users"-style breadth this rule + * exists to catch. It is environment, not a recorded grant, so it is not + * listed by cbm_workspace_grant_list and cannot lift CBM_WS_DENY_ABSOLUTE or + * CBM_WS_DENY_SENSITIVE. */ + if (verdict == CBM_WS_DENY_TOO_SHALLOW && allow_broad_root && allow_broad_root[0] && + ws_paths_equal(canonical_path, allow_broad_root)) { + return true; + } if (err) { if (cbm_workspace_verdict_is_overridable(verdict)) { snprintf(err, err_sz, "%s: %s. To index it anyway, run: codebase-memory-mcp allow-root " "--approve-sensitive %s", canonical_path, cbm_workspace_verdict_reason(verdict), canonical_path); + } else if (verdict == CBM_WS_DENY_TOO_SHALLOW) { + /* No allow-root command exists for this one -- the grant store is a + * human-approved list of real project roots, and this path is being + * refused for being the opposite of that. PATH_ALLOW_BROAD is the + * deliberate, narrower escape hatch: set it to this exact path. */ + snprintf(err, err_sz, "%s: %s. To index it anyway, set PATH_ALLOW_BROAD=%s", + canonical_path, cbm_workspace_verdict_reason(verdict), canonical_path); } else { snprintf(err, err_sz, "%s: %s", canonical_path, cbm_workspace_verdict_reason(verdict)); } diff --git a/src/foundation/workspace.h b/src/foundation/workspace.h index 276079155..4884dbdd0 100644 --- a/src/foundation/workspace.h +++ b/src/foundation/workspace.h @@ -93,10 +93,15 @@ bool cbm_workspace_grant_list(const char *cache_dir, char *out, size_t out_sz); /* The whole decision, used by every entry point that accepts a repo path. * canonical_path must already be canonicalized. configured_root is the legacy * CBM_ALLOWED_ROOT / session policy, treated as an additional grant, or NULL. - * On refusal, err receives a message that names the command which would fix it. */ + * allow_broad_root is PATH_ALLOW_BROAD (or NULL): an operator-set exact match + * for canonical_path that lifts a CBM_WS_DENY_TOO_SHALLOW verdict only. It is + * process environment, not a recorded grant, so it does not appear in + * cbm_workspace_grant_list and does not widen CBM_WS_DENY_ABSOLUTE or + * CBM_WS_DENY_SENSITIVE. On refusal, err receives a message that names the + * command which would fix it. */ bool cbm_workspace_root_allowed(const char *canonical_path, const char *home_dir, - const char *cache_dir, const char *configured_root, char *err, - size_t err_sz); + const char *cache_dir, const char *configured_root, + const char *allow_broad_root, char *err, size_t err_sz); /* The home and cache directories the policy should be evaluated against. Shared * so two callers cannot derive them differently and reach different verdicts for diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 73534404a..372cd521f 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -8747,7 +8747,8 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { char boundary_err[CBM_SZ_1K]; if (repo_path && repo_path[0] && !cbm_workspace_root_allowed(repo_path, cbm_workspace_home_dir(), cbm_workspace_cache_dir(), - allowed_root, boundary_err, sizeof(boundary_err))) { + allowed_root, getenv("PATH_ALLOW_BROAD"), boundary_err, + sizeof(boundary_err))) { free(mode_str); free(name_override); free(repo_path); @@ -12851,7 +12852,8 @@ static void maybe_auto_index(cbm_mcp_server_t *srv) { srv->allowed_root_policy_set ? srv->allowed_root : getenv("CBM_ALLOWED_ROOT"); char boundary_err[CBM_SZ_1K]; if (!cbm_workspace_root_allowed(srv->session_root, cbm_workspace_home_dir(), - cbm_workspace_cache_dir(), allowed_root, boundary_err, + cbm_workspace_cache_dir(), allowed_root, + getenv("PATH_ALLOW_BROAD"), boundary_err, sizeof(boundary_err))) { cbm_log_warn("autoindex.skip", "reason", "workspace_boundary", "detail", boundary_err); return; diff --git a/src/ui/http_server.c b/src/ui/http_server.c index b187438a5..c95c9d79e 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -1173,7 +1173,8 @@ static void handle_index_start(cbm_http_server_t *server, cbm_http_conn_t *c, } if (!cbm_workspace_root_allowed(canonical_root, cbm_workspace_home_dir(), cbm_workspace_cache_dir(), getenv("CBM_ALLOWED_ROOT"), - boundary_err, sizeof(boundary_err))) { + getenv("PATH_ALLOW_BROAD"), boundary_err, + sizeof(boundary_err))) { yyjson_doc_free(doc); char escaped[1024]; cbm_json_escape(escaped, (int)sizeof(escaped), boundary_err); diff --git a/tests/test_workspace.c b/tests/test_workspace.c index faf936e1b..e87958ea9 100644 --- a/tests/test_workspace.c +++ b/tests/test_workspace.c @@ -70,6 +70,45 @@ TEST(ws_posix_top_level_trees_are_too_shallow) { PASS(); } +/* PATH_ALLOW_BROAD is deliberately not part of the grant store's overridability: + * cbm_workspace_classify_root and cbm_workspace_verdict_is_overridable must stay + * exactly as strict as the test above already checked. Only the top-level + * decision, cbm_workspace_root_allowed, knows about it, and only as an exact + * match. */ +TEST(ws_allow_broad_root_lifts_too_shallow_for_an_exact_match_only) { + char cache[256]; + char *created = th_mktempdir("cbm_ws_allow_broad"); + ASSERT_NOT_NULL(created); + snprintf(cache, sizeof(cache), "%s", created); + + char err[1024]; + ASSERT_FALSE(cbm_workspace_root_allowed("/srv", "/home/dev", cache, NULL, NULL, err, + sizeof(err))); + ASSERT_NOT_NULL(strstr(err, "PATH_ALLOW_BROAD")); + + ASSERT_TRUE( + cbm_workspace_root_allowed("/srv", "/home/dev", cache, NULL, "/srv", err, sizeof(err))); + + /* A grant for the root does not extend to a sibling: PATH_ALLOW_BROAD names + * one path, not a prefix. */ + ASSERT_FALSE(cbm_workspace_root_allowed("/opt", "/home/dev", cache, NULL, "/srv", err, + sizeof(err))); + + /* Sensitive and absolute refusals are a different verdict and stay refused + * even when PATH_ALLOW_BROAD happens to name that exact path. */ + ASSERT_FALSE(cbm_workspace_root_allowed("/home/dev", "/home/dev", cache, NULL, "/home/dev", + err, sizeof(err))); + ASSERT_FALSE( + cbm_workspace_root_allowed("/", "/home/dev", cache, NULL, "/", err, sizeof(err))); + + /* Environment, not a grant: it never shows up in the recorded list. */ + char listed[64]; + ASSERT_FALSE(cbm_workspace_grant_list(cache, listed, sizeof(listed))); + + th_cleanup(cache); + PASS(); +} + /* Legitimately shallow project roots must survive: these are the false positives * a blanket depth rule would cause, which is why Windows counts drive-relative. */ TEST(ws_legitimate_shallow_roots_are_allowed) { @@ -170,10 +209,10 @@ TEST(ws_sensitive_root_explicit_approval_is_preserved) { snprintf(cache, sizeof(cache), "%s", created); char err[1024]; - ASSERT_FALSE(cbm_workspace_root_allowed(root, root, cache, NULL, err, sizeof(err))); + ASSERT_FALSE(cbm_workspace_root_allowed(root, root, cache, NULL, NULL, err, sizeof(err))); ASSERT_NOT_NULL(strstr(err, "--approve-sensitive")); ASSERT_TRUE(cbm_workspace_grant_add(cache, root, root, true, err, sizeof(err))); - ASSERT_TRUE(cbm_workspace_root_allowed(root, root, cache, NULL, err, sizeof(err))); + ASSERT_TRUE(cbm_workspace_root_allowed(root, root, cache, NULL, NULL, err, sizeof(err))); th_cleanup(root); th_cleanup(cache); @@ -193,10 +232,10 @@ TEST(ws_sensitive_approval_upgrades_existing_ordinary_exact_grant) { char err[1024]; ASSERT_TRUE(cbm_workspace_grant_add(cache, NULL, root, false, err, sizeof(err))); - ASSERT_FALSE(cbm_workspace_root_allowed(root, root, cache, NULL, err, sizeof(err))); + ASSERT_FALSE(cbm_workspace_root_allowed(root, root, cache, NULL, NULL, err, sizeof(err))); ASSERT_TRUE(cbm_workspace_grant_add(cache, root, root, true, err, sizeof(err))); - ASSERT_TRUE(cbm_workspace_root_allowed(root, root, cache, NULL, err, sizeof(err))); + ASSERT_TRUE(cbm_workspace_root_allowed(root, root, cache, NULL, NULL, err, sizeof(err))); /* A repeated explicit approval must recognize the marked exact grant rather * than append another exception. */ ASSERT_TRUE(cbm_workspace_grant_add(cache, root, root, true, err, sizeof(err))); @@ -228,11 +267,11 @@ TEST(ws_sensitive_approval_adds_exact_exception_under_ordinary_ancestor) { ASSERT_EQ(cbm_mkdir(sensitive), 0); char err[1024]; ASSERT_TRUE(cbm_workspace_grant_add(cache, NULL, ancestor, false, err, sizeof(err))); - ASSERT_FALSE(cbm_workspace_root_allowed(sensitive, sensitive, cache, NULL, err, sizeof(err))); + ASSERT_FALSE(cbm_workspace_root_allowed(sensitive, sensitive, cache, NULL, NULL, err, sizeof(err))); ASSERT_TRUE(cbm_workspace_grant_add(cache, sensitive, sensitive, true, err, sizeof(err))); ASSERT_TRUE( - cbm_workspace_root_allowed(sensitive, sensitive, cache, NULL, err, sizeof(err))); + cbm_workspace_root_allowed(sensitive, sensitive, cache, NULL, NULL, err, sizeof(err))); ASSERT_TRUE(cbm_workspace_grant_add(cache, sensitive, sensitive, true, err, sizeof(err))); char listed[4096]; @@ -385,6 +424,7 @@ SUITE(workspace) { RUN_TEST(ws_volume_roots_are_absolutely_denied); RUN_TEST(ws_non_absolute_paths_are_denied); RUN_TEST(ws_posix_top_level_trees_are_too_shallow); + RUN_TEST(ws_allow_broad_root_lifts_too_shallow_for_an_exact_match_only); RUN_TEST(ws_legitimate_shallow_roots_are_allowed); RUN_TEST(ws_home_itself_is_sensitive_but_subdirs_are_fine); RUN_TEST(ws_credential_directories_are_sensitive_at_any_depth); From eaf59046e5371eda24bfafc60743291b429e79d8 Mon Sep 17 00:00:00 2001 From: Yyunozor Date: Thu, 20 Aug 2026 01:31:29 +0200 Subject: [PATCH 013/217] fix(registry): stop external package imports binding to project homonyms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bare call whose name the calling file imports from a package outside the indexed tree was bound to whatever project symbol shared the simple name. An `import { eq, sql } from "drizzle-orm"` plus an unrelated local module exporting `eq`/`sql` produced two CALLS edges from the caller into that module, strategy unique_name — the same defect on the sequential and the fused-parallel resolver. The TS-LSP is not involved: it declines the external names, and the textual registry fallback fires anyway. The package materializes no node, so resolve_import_node returns NULL, no IMPORTS edge is written, and the per-file import map carries no key for the name. Strategies 1-2 miss and resolve_name_lookup binds the call by simple name. Confidence does not separate the cases: the fabricated edge lands at 0.75 whenever the file also imports anything from the target's module. Add cbm_suppress_external_import_shadow(), a pure predicate beside the existing perl/tsjs/cross-language guards and called at the same two emit sites. It drops the edge only when the callee is a bare identifier, the strategy is a project-wide guess (suffix_match / unique_name / field_type_hint / fuzzy), the file imports that exact local name from a non-relative specifier, and no import-map key binds it. Relative specifiers are excluded on purpose: they name a path inside the tree, so a missing IMPORTS edge there is an in-project resolution gap and the fallback can still be right. A name imported both relatively and from a package keeps its edge, independently of extraction order. Addresses the external-import sub-case of #1355. Signed-off-by: Yyunozor --- src/pipeline/pass_calls.c | 16 ++++- src/pipeline/pass_parallel.c | 8 +++ src/pipeline/pipeline.h | 10 +++ src/pipeline/registry.c | 82 +++++++++++++++++++++++ tests/test_pipeline.c | 124 +++++++++++++++++++++++++++++++++++ tests/test_registry.c | 89 +++++++++++++++++++++++++ 6 files changed, 326 insertions(+), 3 deletions(-) diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index c9744dda2..734e9b956 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -476,7 +476,8 @@ static const cbm_gbuf_node_t *calls_find_source(cbm_pipeline_ctx_t *ctx, const c /* Resolve one call and emit the appropriate edge. Returns 1 if resolved, 0 if not. */ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, - const CBMResolvedCallArray *lsp_calls, const char *rel, + const CBMResolvedCallArray *lsp_calls, + const CBMImportArray *file_imports, const char *rel, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, CBMLanguage lang) { const cbm_gbuf_node_t *source_node = calls_find_source(ctx, rel, call->enclosing_func_qn); @@ -668,6 +669,15 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, if (cbm_suppress_cross_language_suffix_match(lang, target_node->file_path, res.strategy)) { return 0; } + /* #1355: `import { eq } from "drizzle-orm"` binds `eq` to a package that is + * not in the indexed tree, so a project-wide same-name guess must not turn + * `eq(...)` into a CALLS edge to an unrelated project `eq`. Placed with the + * #725 guard, after the service-pattern bypasses above, so no HTTP/route + * edge can be lost to it. */ + if (cbm_suppress_external_import_shadow(call->callee_name, res.strategy, file_imports, imp_keys, + imp_count)) { + return 0; + } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, imp_count, drop_plain_call); return SKIP_ONE; @@ -827,8 +837,8 @@ int cbm_pipeline_pass_calls(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file continue; } total_calls++; - if (resolve_single_call(ctx, call, &result->resolved_calls, rel, module_qn, imp_keys, - imp_vals, imp_count, files[i].language)) { + if (resolve_single_call(ctx, call, &result->resolved_calls, &result->imports, rel, + module_qn, imp_keys, imp_vals, imp_count, files[i].language)) { resolved++; } else { unresolved++; diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index cdeafe9d8..76ed4e9a4 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2569,6 +2569,14 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB * CALLS edge across a language boundary. */ continue; } + if (target_node && source_node->id != target_node->id && + cbm_suppress_external_import_shadow(call->callee_name, res.strategy, &result->imports, + imp_keys, imp_count)) { + /* #1355: same guard as pass_calls.c — a bare call bound by an + * external package import must not become a CALLS edge to an + * unrelated project symbol of the same name. */ + continue; + } if (!target_node || source_node->id == target_node->id) { /* HTTP/ASYNC calls to an EXTERNAL client library (`requests.get(url)`) * resolve to an unindexed QN (target_node == NULL), but their edge diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index dfe81f726..f24cc3045 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -291,6 +291,16 @@ bool cbm_suppress_weak_member_match(bool enabled, bool is_method, const char *st * Pure; unit-tested in test_registry.c. */ bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_bound, const char *strategy); +/* #1355: drop a project-wide same-name guess (suffix_match / unique_name / + * field_type_hint / fuzzy) for a BARE call whose name the calling file binds to + * a NON-RELATIVE (package) import that resolved to nothing in the graph — + * `import { eq } from "drizzle-orm"` must not make `eq(...)` a CALLS edge to an + * unrelated project `eq`. A name the import map does bind, a relative + * specifier, a member/qualified callee, and every import-/receiver-aware + * strategy are all kept. Pure; unit-tested in test_registry.c. */ +bool cbm_suppress_external_import_shadow(const char *callee_name, const char *strategy, + const CBMImportArray *file_imports, + const char **import_map_keys, int import_map_count); /* #725: drop a suffix_match CALLS edge when the caller language and the * target file's language disagree. unique_name (candidates == 1) is #1572 diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 97c843355..ff78c4354 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -501,6 +501,88 @@ bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_b return weak_short_name_strategy(strategy); } +/* A module specifier that names a path inside the indexed tree ("./x", "../x", + * "/abs/x") rather than an external package. Package specifiers are everything + * else — "drizzle-orm", "rxjs/operators", "@scope/pkg". */ +static bool specifier_is_relative(const char *module_path) { + return module_path && (module_path[0] == '.' || module_path[0] == '/'); +} + +static bool import_map_binds(const char **import_map_keys, int import_map_count, + const char *local_name) { + if (!import_map_keys || import_map_count <= 0) { + return false; + } + for (int i = 0; i < import_map_count; i++) { + if (import_map_keys[i] && strcmp(import_map_keys[i], local_name) == 0) { + return true; + } + } + return false; +} + +/* #1355: a bare call whose name the calling file binds to an EXTERNAL package + * import must not fall back to a project-wide same-name guess. + * + * `import { eq } from "drizzle-orm"` binds `eq` in this file's scope. When the + * package is not part of the indexed tree it materializes no IMPORTS edge, so + * the import map has no entry, strategies 1-2 miss, and strategy 3/4 attach + * `eq(users.id, id)` to whatever project symbol happens to share the simple + * name — a CALLS edge into an unrelated local helper. The explicit import + * statement is positive evidence, taken from the caller's own source, that the + * identifier does NOT denote that symbol. + * + * Fires only when ALL of: + * - the callee is a BARE identifier: member (`x.foo()`) and package/namespace + * qualified callees are the receiver-aware guards' business, not this one; + * - the match came from a project-wide guess (suffix_match / unique_name; + * field_type_hint / fuzzy listed for the same defensive reason as the TS/JS + * guard) — every import-, receiver- or module-aware strategy is KEPT; + * - the file imports that exact local name from a NON-RELATIVE specifier; + * - no import-map key binds the name, i.e. that import resolved to nothing in + * the graph. A name the map does bind already had its chance at strategy 1 + * and is import-aware by construction. + * + * Relative specifiers are deliberately excluded: they name a path inside the + * indexed tree, so a missing IMPORTS edge is an in-project resolution gap and + * the same-name fallback can still be right. A name imported from both a + * relative and a package specifier keeps the edge — the relative binding wins + * the tie explicitly, so the outcome does not depend on import order. + * + * Pure + side-effect-free so the contract is unit-testable without a pipeline. */ +bool cbm_suppress_external_import_shadow(const char *callee_name, const char *strategy, + const CBMImportArray *file_imports, + const char **import_map_keys, int import_map_count) { + if (!callee_name || !callee_name[0] || !strategy || !strategy[0]) { + return false; + } + if (strcmp(strategy, "suffix_match") != 0 && strcmp(strategy, "unique_name") != 0 && + strcmp(strategy, "field_type_hint") != 0 && strcmp(strategy, "fuzzy") != 0) { + return false; + } + if (strchr(callee_name, '.') != NULL || strstr(callee_name, "::") != NULL) { + return false; + } + if (!file_imports || file_imports->count <= 0) { + return false; + } + if (import_map_binds(import_map_keys, import_map_count, callee_name)) { + return false; + } + bool external_binding = false; + for (int i = 0; i < file_imports->count; i++) { + const CBMImport *imp = &file_imports->items[i]; + if (!imp->local_name || !imp->module_path || strcmp(imp->local_name, callee_name) != 0) { + continue; + } + if (specifier_is_relative(imp->module_path)) { + return false; /* an in-tree binding for this name — never suppress */ + } + external_binding = true; + } + return external_binding; +} + static bool js_ts_family(CBMLanguage lang) { return lang == CBM_LANG_JAVASCRIPT || lang == CBM_LANG_TYPESCRIPT || lang == CBM_LANG_TSX || lang == CBM_LANG_ARKTS; diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index c5d3a9539..fd6d51855 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4976,6 +4976,129 @@ static int count_nodes_named(cbm_store_t *s, const char *project, const char *na * (axios.get, api.patch on a renamed-axios instance, supertest request(app).get). * The regex false edge must stay suppressed in parallel too. CBM_WORKERS forces * >1 worker so the parallel path is taken regardless of the host core count. */ +/* #1355: write the shared external-import-shadow fixture into `dir`. + * `pad_files` filler modules push the run over MIN_FILES_FOR_PARALLEL so the + * same tree can be indexed by both resolvers. */ +static void write_external_import_shadow_fixture(const char *dir, int pad_files) { + /* The lone project symbols named eq/sql — ordinary local helpers. */ + write_temp_file(dir, "src/text-utils.ts", + "export function eq(a: string, b: string): boolean {\n" + " return a.trim() === b.trim();\n" + "}\n" + "export function sql(chunk: string): string {\n" + " return chunk.replace(/\\s+/g, ' ');\n" + "}\n" + "export function normalize(s: string): string {\n" + " return s.trim().toLowerCase();\n" + "}\n"); + /* `eq` and `sql` come from an external package that is not in the tree, so + * the registry falls through to a project-wide same-name guess. These are + * the fabricated edges. `normalize` is imported relatively from the SAME + * file the guess would have picked — it must survive. */ + write_temp_file(dir, "src/queries.ts", + "import { eq, sql } from 'drizzle-orm';\n" + "import { normalize } from './text-utils';\n" + "export function buildQuery(id: string): unknown {\n" + " return [eq({ id }, id), sql('select 1'), normalize(id)];\n" + "}\n"); + /* No import of `eq` at all: nothing in the caller's source contradicts the + * guess, so the pre-existing fallback keeps its edge. */ + write_temp_file(dir, "src/no-import.ts", + "export function callsWithoutImport(a: string, b: string): boolean {\n" + " return eq(a, b);\n" + "}\n"); + for (int i = 0; i < pad_files; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "src/pad_%02d.ts", i); + snprintf(body, sizeof(body), "export function shadowPad%02d(): number { return %d; }\n", i, + i); + write_temp_file(dir, name, body); + } +} + +/* #1355: assert the guard's whole contract against one indexed store. */ +static int assert_external_import_shadow_contract(const char *dir, const char *db_name) { + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/%s", dir, db_name); + cbm_pipeline_t *p = cbm_pipeline_new(dir, db_path, CBM_MODE_FULL); + if (!p) { + return 1; + } + if (cbm_pipeline_run(p) != 0) { + cbm_pipeline_free(p); + return 2; + } + const char *project = cbm_pipeline_project_name(p); + cbm_store_t *s = cbm_store_open_path(db_path); + if (!s) { + cbm_pipeline_free(p); + return 3; + } + int rc = 0; + /* (1) the reported bug: an externally-imported name must not bind to the + * local homonym (RED before the fix, on both resolvers). */ + if (cross_file_call_exists(s, project, "buildQuery", "eq")) { + rc = 4; + } + if (rc == 0 && cross_file_call_exists(s, project, "buildQuery", "sql")) { + rc = 5; + } + /* (2) the relative import to the very same file still resolves. */ + if (rc == 0 && !cross_file_call_exists(s, project, "buildQuery", "normalize")) { + rc = 6; + } + /* (3) a bare call the caller never imports keeps its pre-existing edge. */ + if (rc == 0 && !cross_file_call_exists(s, project, "callsWithoutImport", "eq")) { + rc = 7; + } + cbm_store_close(s); + cbm_pipeline_free(p); + return rc; +} + +TEST(pipeline_external_import_shadow_not_bound_to_local_homonym_issue1355) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_ext_import_shadow_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + /* Enough files that CBM_WORKERS can take the fused-parallel path; the same + * tree is then indexed by each resolver in turn, because the guard lives at + * two independent emit sites (pass_calls.c and pass_parallel.c). */ + write_external_import_shadow_fixture(tmp, 50); + + char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + + cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1); + int sequential = assert_external_import_shadow_contract(tmp, "shadow-sequential.db"); + + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + cbm_setenv("CBM_WORKERS", "4", 1); + int parallel = assert_external_import_shadow_contract(tmp, "shadow-parallel.db"); + + if (saved_workers) { + cbm_setenv("CBM_WORKERS", saved_workers, 1); + free(saved_workers); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + if (saved_single) { + cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1); + free(saved_single); + } else { + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + } + th_rmtree(tmp); + + ASSERT_EQ(sequential, 0); + ASSERT_EQ(parallel, 0); + PASS(); +} + TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_tsjs_par_XXXXXX"); @@ -13324,6 +13447,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_python_receiver_parallel_suppresses_weak_method_edges); RUN_TEST(pipeline_python_bare_local_binding_suppresses_weak_edge); RUN_TEST(pipeline_python_bare_local_binding_parallel_suppresses_weak_edge); + RUN_TEST(pipeline_external_import_shadow_not_bound_to_local_homonym_issue1355); RUN_TEST(pipeline_parallel_python_cross_only_dunder_gets_synthetic_carrier); RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier); RUN_TEST(pipeline_arg_url_rejects_non_http_slash_arguments); diff --git a/tests/test_registry.c b/tests/test_registry.c index 247ca61ad..eb47679f3 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -970,6 +970,92 @@ TEST(weak_call_guards_share_one_drop_list) { } ASSERT_EQ(member, binding); } +TEST(external_import_shadow_drops_package_bound_bare_call) { + /* #1355: `import { eq, sql } from "drizzle-orm"` binds both names to a + * package outside the indexed tree, so no import-map key exists for them and + * a project-wide same-name guess is a fabricated edge. */ + CBMImport imports[] = { + {.local_name = "eq", .module_path = "drizzle-orm"}, + {.local_name = "sql", .module_path = "drizzle-orm"}, + {.local_name = "users", .module_path = "./schema"}, + }; + CBMImportArray arr = {.items = imports, .count = 3, .cap = 3}; + /* Only the relative import materialized an IMPORTS edge. */ + const char *keys[] = {"users"}; + + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, keys, 1)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("sql", "unique_name", &arr, keys, 1)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "suffix_match", &arr, keys, 1)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "field_type_hint", &arr, keys, 1)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "fuzzy", &arr, keys, 1)); + /* A file whose imports all failed to materialize is still covered: the + * evidence is the import statement, not the map. */ + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0)); + PASS(); +} + +TEST(external_import_shadow_keeps_everything_else) { + /* Negative space: every input that is NOT "bare call bound to a package + * import the graph could not resolve" must keep its edge. */ + CBMImport imports[] = { + {.local_name = "eq", .module_path = "drizzle-orm"}, + {.local_name = "normalize", .module_path = "./text-utils"}, + {.local_name = "getRootContainer", .module_path = "@repo/shared"}, + }; + CBMImportArray arr = {.items = imports, .count = 3, .cap = 3}; + /* The relative import and the workspace package both materialized. */ + const char *keys[] = {"normalize", "getRootContainer"}; + const int nkeys = 2; + + /* Import-/receiver-/module-aware strategies are never this guard's business. */ + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "import_map", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "import_map_suffix", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "same_module", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "qualified_suffix", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "lsp_ts_import", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "lsp_ts_method", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "service_pattern", &arr, keys, nkeys)); + /* A relative specifier names a path inside the tree — a missing IMPORTS edge + * there is an in-project gap, not an external binding. */ + ASSERT_FALSE( + cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, NULL, 0)); + /* A package the import map DOES bind resolved in-graph (workspace package). */ + ASSERT_FALSE( + cbm_suppress_external_import_shadow("getRootContainer", "unique_name", &arr, keys, nkeys)); + /* A name the file never imports (called with no import at all). */ + ASSERT_FALSE(cbm_suppress_external_import_shadow("helper", "unique_name", &arr, keys, nkeys)); + /* Member and package/namespace-qualified callees belong to the + * receiver-aware guards, not to this one. */ + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq.apply", "unique_name", &arr, keys, nkeys)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("eq::apply", "unique_name", &arr, keys, nkeys)); + /* Empty / absent inputs. */ + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", NULL, &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow(NULL, "unique_name", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("", "unique_name", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", NULL, keys, nkeys)); + PASS(); +} + +TEST(external_import_shadow_relative_binding_wins_the_tie) { + /* A name imported from BOTH a package and a relative path keeps its edge, + * and does so independently of the order the imports were extracted in — + * the relative binding is an explicit tie-break, not an emergent property + * of iteration order. */ + CBMImport pkg_first[] = { + {.local_name = "eq", .module_path = "drizzle-orm"}, + {.local_name = "eq", .module_path = "./text-utils"}, + }; + CBMImport rel_first[] = { + {.local_name = "eq", .module_path = "./text-utils"}, + {.local_name = "eq", .module_path = "drizzle-orm"}, + }; + CBMImportArray a = {.items = pkg_first, .count = 2, .cap = 2}; + CBMImportArray b = {.items = rel_first, .count = 2, .cap = 2}; + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &a, NULL, 0)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &b, NULL, 0)); PASS(); } @@ -1072,4 +1158,7 @@ SUITE(registry) { RUN_TEST(local_binding_suppress_drops_weak_shadowed_bare_calls); RUN_TEST(local_binding_suppress_keeps_unshadowed_and_strong_strategies); RUN_TEST(weak_call_guards_share_one_drop_list); + RUN_TEST(external_import_shadow_drops_package_bound_bare_call); + RUN_TEST(external_import_shadow_keeps_everything_else); + RUN_TEST(external_import_shadow_relative_binding_wins_the_tie); } From a8293c498ca60c97c4116e40f58dffb751fe22ab Mon Sep 17 00:00:00 2001 From: Yyunozor Date: Thu, 20 Aug 2026 20:41:26 +0200 Subject: [PATCH 014/217] fix: address review feedback on external-import-shadow guard - specifier_is_relative() now also recognizes Windows drive-letter (C:\ / C:/) and UNC (\\server\share) specifiers as in-tree, not external packages. pr-smoke runs this pipeline on Windows, and the previous POSIX-only check ('.' / '/') classified those specifiers as external, which could suppress a real edge on that platform only. Added external_import_shadow_windows_relative_specifier_kept (tests/test_registry.c): red before this fix, green after. - old_workers / old_single in the #1355 pipeline test are now const char*, matching getenv()'s read-only contract. - The pad_files literal (50) is now a named EXTERNAL_IMPORT_SHADOW_PARALLEL_PAD constant with a comment tying it to MIN_FILES_FOR_PARALLEL (a private #define in pipeline.c, not reachable for a static_assert from tests), so a future bump to that threshold can't silently drop the fixture back onto the sequential-only path. Signed-off-by: Yyunozor --- src/pipeline/registry.c | 26 +++++++++++++++++++++++--- tests/test_pipeline.c | 19 ++++++++++++++++--- tests/test_registry.c | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index ff78c4354..6c0cdb42a 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -31,6 +31,7 @@ enum { REG_MAX_CANDIDATES = 256 }; #include "foundation/dyn_array.h" #include "foundation/platform.h" +#include #include #include #include @@ -502,10 +503,29 @@ bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_b } /* A module specifier that names a path inside the indexed tree ("./x", "../x", - * "/abs/x") rather than an external package. Package specifiers are everything - * else — "drizzle-orm", "rxjs/operators", "@scope/pkg". */ + * "/abs/x", and on Windows "C:\x", "C:/x", "\\server\share\x") rather than an + * external package. Package specifiers are everything else — "drizzle-orm", + * "rxjs/operators", "@scope/pkg". Windows paths must count as in-tree here: + * pr-smoke runs this pipeline on Windows, and misclassifying a drive-letter or + * UNC specifier as "external" is exactly the false-external edge this guard + * exists to prevent (#1355). */ static bool specifier_is_relative(const char *module_path) { - return module_path && (module_path[0] == '.' || module_path[0] == '/'); + if (!module_path || !module_path[0]) { + return false; + } + if (module_path[0] == '.' || module_path[0] == '/') { + return true; + } + /* UNC share: \\server\share\... */ + if (module_path[0] == '\\' && module_path[1] == '\\') { + return true; + } + /* Drive-letter absolute path: C:\repo\... or C:/repo/... */ + if (isalpha((unsigned char)module_path[0]) && module_path[1] == ':' && + (module_path[2] == '\\' || module_path[2] == '/')) { + return true; + } + return false; } static bool import_map_binds(const char **import_map_keys, int import_map_count, diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index fd6d51855..863254fda 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4976,6 +4976,17 @@ static int count_nodes_named(cbm_store_t *s, const char *project, const char *na * (axios.get, api.patch on a renamed-axios instance, supertest request(app).get). * The regex false edge must stay suppressed in parallel too. CBM_WORKERS forces * >1 worker so the parallel path is taken regardless of the host core count. */ +/* #1355: padding count for write_external_import_shadow_fixture, deliberately + * well past MIN_FILES_FOR_PARALLEL (=50, a private #define in + * src/pipeline/pipeline.c — not exposed to tests, so this cannot be a + * static_assert against it) rather than sitting right at the threshold. The + * margin, not the exact value, is what the test depends on: a modest bump to + * the real threshold must not silently drop this fixture back onto the + * sequential-only path and leave the parallel resolver unexercised. Same + * pattern as ET_PARALLEL_PAD / CP_PARALLEL_PAD in test_edge_types_probe.c / + * test_convergence_probe.c. */ +enum { EXTERNAL_IMPORT_SHADOW_PARALLEL_PAD = 64 }; + /* #1355: write the shared external-import-shadow fixture into `dir`. * `pad_files` filler modules push the run over MIN_FILES_FOR_PARALLEL so the * same tree can be indexed by both resolvers. */ @@ -5066,11 +5077,13 @@ TEST(pipeline_external_import_shadow_not_bound_to_local_homonym_issue1355) { /* Enough files that CBM_WORKERS can take the fused-parallel path; the same * tree is then indexed by each resolver in turn, because the guard lives at * two independent emit sites (pass_calls.c and pass_parallel.c). */ - write_external_import_shadow_fixture(tmp, 50); + write_external_import_shadow_fixture(tmp, EXTERNAL_IMPORT_SHADOW_PARALLEL_PAD); - char *old_workers = getenv("CBM_WORKERS"); + /* getenv() returns a pointer into the process environment that must be + * treated as read-only; strdup() below only ever reads through it. */ + const char *old_workers = getenv("CBM_WORKERS"); char *saved_workers = old_workers ? strdup(old_workers) : NULL; - char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); char *saved_single = old_single ? strdup(old_single) : NULL; cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1); diff --git a/tests/test_registry.c b/tests/test_registry.c index eb47679f3..d4d7545aa 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -1059,6 +1059,39 @@ TEST(external_import_shadow_relative_binding_wins_the_tie) { PASS(); } +TEST(external_import_shadow_windows_relative_specifier_kept) { + /* #1355 follow-up: a Windows-style specifier — + * drive-letter absolute (`C:\...` / `C:/...`) or UNC share (`\\server\...`) + * — names a path inside the indexed tree exactly like a POSIX "./x" or + * "/x". pr-smoke runs this pipeline on Windows; before this fix these + * specifiers fell through to "external package" and the same-name guess + * they'd otherwise validate got suppressed on that platform only. */ + CBMImport imports[] = { + {.local_name = "eq", .module_path = "C:\\repo\\src\\text-utils.ts"}, + {.local_name = "sql", .module_path = "C:/repo/src/text-utils.ts"}, + {.local_name = "normalize", .module_path = "\\\\server\\share\\text-utils.ts"}, + }; + CBMImportArray arr = {.items = imports, .count = 3, .cap = 3}; + + /* None of these materialized an import-map key (as if resolution missed + * them, mirroring the reported scenario) — the specifier alone must still + * keep the edge because it is in-tree, not external. RED before the fix + * (specifier_is_relative saw '.' / '/' only, so all three were classified + * external and suppressed); GREEN after. */ + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("sql", "unique_name", &arr, NULL, 0)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, NULL, 0)); + /* A bare package specifier without any drive letter or leading separator + * (e.g. "drizzle-orm") must still be external — this predicate must not + * become "anything with a colon or backslash". */ + CBMImport pkg_only[] = { + {.local_name = "eq", .module_path = "drizzle-orm"}, + }; + CBMImportArray parr = {.items = pkg_only, .count = 1, .cap = 1}; + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &parr, NULL, 0)); + PASS(); +} + /* ── Suite ─────────────────────────────────────────────────────── */ /* Method call THROUGH an imported symbol that is itself an indexed node @@ -1161,4 +1194,5 @@ SUITE(registry) { RUN_TEST(external_import_shadow_drops_package_bound_bare_call); RUN_TEST(external_import_shadow_keeps_everything_else); RUN_TEST(external_import_shadow_relative_binding_wins_the_tie); + RUN_TEST(external_import_shadow_windows_relative_specifier_kept); } From 797a11a15eff6154428aeb268c0b55edb5647a7a Mon Sep 17 00:00:00 2001 From: Yyunozor Date: Fri, 4 Sep 2026 00:32:18 +0200 Subject: [PATCH 015/217] fix(registry): keep workspace-sibling imports out of the external-import guard A per-language blast-radius census on twelve public repositories found one regression class in the #1355 guard, and it is not per-language: it is per-topology. In a workspace monorepo a BARE specifier can still name code inside the indexed tree. `import { eq } from "drizzle-orm"` written inside the drizzle-orm repository is a sibling package, not a dependency. Measured on drizzle-team/drizzle-orm at b7862528, full index, both resolvers: the guard removed 2609 CALLS edges. Resolving each removed edge's specifier against the repository's own package manifests puts 1377 of them (52.8%) on the file the specifier actually names. 690 of those are strict: the specifier names a subpath, that subpath maps to one directory, and the target file is inside it. The other 687 come from the bare package root, where the check can only confirm the target is somewhere in the package, so read 690 as the defensible floor. Only 373 removed edges pointed at a third-party package, which is the defect #1355 reports. The same census on eleven other repositories (flask, scrapy, express, zustand, got, cobra, gin, chi, ripgrep, gson, jq) removed 12 CALLS edges in total: nine fabricated, one lost (a documentation example importing the repository's own published name), two weak links between a shell command and a manifest key. The regression is specific to trees that ship the package they import. Consult the pipeline package map before calling a specifier external. It is keyed by the `name` of every manifest found in the tree, so a workspace registers each of its own packages there, and a subpath specifier is walked back one slash at a time ("drizzle-orm/pg-core" -> "drizzle-orm"). A specifier the tree itself claims is now treated exactly like a relative one -- kept. The check is deliberately a NAME test rather than a resolution test. A workspace package usually points `main` at a build artifact ("./index.cjs") that is not checked in, so asking whether the entry file exists in the graph answers "no" for exactly the monorepos this has to protect. Whether the tree claims the name is decidable from the manifest alone. Passing NULL restores the previous specifier-shape-only contract, which is what the unit tests exercise. After the change the same census removes 391 edges on drizzle-orm, 375 of them CALLS, and every one resolves to a third-party package; of the other eleven repositories only got changes, recovering its self-referencing edge. Signed-off-by: Yyunozor --- src/pipeline/pass_calls.c | 2 +- src/pipeline/pass_parallel.c | 2 +- src/pipeline/pipeline.h | 13 ++-- src/pipeline/registry.c | 49 +++++++++++++- tests/test_pipeline.c | 111 +++++++++++++++++++++++++++++++ tests/test_registry.c | 125 ++++++++++++++++++++++++++--------- 6 files changed, 263 insertions(+), 39 deletions(-) diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 734e9b956..7fc6bc423 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -675,7 +675,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, * #725 guard, after the service-pattern bypasses above, so no HTTP/route * edge can be lost to it. */ if (cbm_suppress_external_import_shadow(call->callee_name, res.strategy, file_imports, imp_keys, - imp_count)) { + imp_count, cbm_pipeline_get_pkgmap())) { return 0; } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 76ed4e9a4..a57d594cf 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2571,7 +2571,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB } if (target_node && source_node->id != target_node->id && cbm_suppress_external_import_shadow(call->callee_name, res.strategy, &result->imports, - imp_keys, imp_count)) { + imp_keys, imp_count, cbm_pipeline_get_pkgmap())) { /* #1355: same guard as pass_calls.c — a bare call bound by an * external package import must not become a CALLS edge to an * unrelated project symbol of the same name. */ diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index f24cc3045..45ef74f84 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -19,8 +19,9 @@ #include #include -#include "discover/discover.h" /* cbm_ignored_file_t (#963) */ -#include "foundation/constants.h" /* CBM_SZ_512 */ +#include "discover/discover.h" /* cbm_ignored_file_t (#963) */ +#include "foundation/constants.h" /* CBM_SZ_512 */ +#include "foundation/hash_table.h" /* CBMHashTable (#1355 pkgmap lookup) */ /* Forward declarations */ typedef struct cbm_store cbm_store_t; @@ -297,10 +298,14 @@ bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_b * `import { eq } from "drizzle-orm"` must not make `eq(...)` a CALLS edge to an * unrelated project `eq`. A name the import map does bind, a relative * specifier, a member/qualified callee, and every import-/receiver-aware - * strategy are all kept. Pure; unit-tested in test_registry.c. */ + * strategy are all kept. `indexed_packages` is the pipeline package map: a + * specifier naming a package the tree itself declares (a workspace sibling) + * counts as in-tree and is kept too; NULL disables that check. + * Pure; unit-tested in test_registry.c. */ bool cbm_suppress_external_import_shadow(const char *callee_name, const char *strategy, const CBMImportArray *file_imports, - const char **import_map_keys, int import_map_count); + const char **import_map_keys, int import_map_count, + const CBMHashTable *indexed_packages); /* #725: drop a suffix_match CALLS edge when the caller language and the * target file's language disagree. unique_name (candidates == 1) is #1572 diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 6c0cdb42a..adbe2c713 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -528,6 +528,43 @@ static bool specifier_is_relative(const char *module_path) { return false; } +/* True when the specifier names a package that the indexed tree DECLARES — + * a workspace sibling, not a third-party dependency. `indexed_packages` is the + * pipeline's package map, keyed by the `name` of every manifest found in the + * tree (package.json, go.mod, Cargo.toml, ...), so a pnpm/npm/cargo workspace + * registers each of its own packages here. + * + * The key is the bare package name, so a subpath specifier is walked back one + * slash at a time: "drizzle-orm/pg-core" -> "drizzle-orm". Scoped names keep + * their leading segment ("@scope/pkg/sub" -> "@scope/pkg" -> "@scope"). + * + * Deliberately a NAME test, not a resolution test. A workspace package usually + * points `main`/`module` at a build artifact ("./index.cjs") that is not + * checked in, so asking whether the entry file exists in the graph answers + * "no" for exactly the monorepos this has to protect. Whether the tree claims + * the name is decidable from the manifest alone. */ +static bool specifier_names_indexed_package(const CBMHashTable *indexed_packages, + const char *module_path) { + if (!indexed_packages || !module_path || !module_path[0]) { + return false; + } + if (cbm_ht_has(indexed_packages, module_path)) { + return true; + } + char buf[CBM_SZ_512]; + if (strlen(module_path) >= sizeof(buf)) { + return false; + } + snprintf(buf, sizeof(buf), "%s", module_path); + for (char *slash = strrchr(buf, '/'); slash != NULL; slash = strrchr(buf, '/')) { + *slash = '\0'; + if (cbm_ht_has(indexed_packages, buf)) { + return true; + } + } + return false; +} + static bool import_map_binds(const char **import_map_keys, int import_map_count, const char *local_name) { if (!import_map_keys || import_map_count <= 0) { @@ -569,10 +606,17 @@ static bool import_map_binds(const char **import_map_keys, int import_map_count, * relative and a package specifier keeps the edge — the relative binding wins * the tie explicitly, so the outcome does not depend on import order. * + * `indexed_packages` extends that same reasoning to workspace monorepos, where + * a BARE specifier also names in-tree code: `import { eq } from "drizzle-orm"` + * inside the drizzle-orm repository is a sibling package, not a dependency. A + * specifier the tree's own manifests claim is treated exactly like a relative + * one. NULL disables the check and restores the specifier-shape-only contract. + * * Pure + side-effect-free so the contract is unit-testable without a pipeline. */ bool cbm_suppress_external_import_shadow(const char *callee_name, const char *strategy, const CBMImportArray *file_imports, - const char **import_map_keys, int import_map_count) { + const char **import_map_keys, int import_map_count, + const CBMHashTable *indexed_packages) { if (!callee_name || !callee_name[0] || !strategy || !strategy[0]) { return false; } @@ -595,7 +639,8 @@ bool cbm_suppress_external_import_shadow(const char *callee_name, const char *st if (!imp->local_name || !imp->module_path || strcmp(imp->local_name, callee_name) != 0) { continue; } - if (specifier_is_relative(imp->module_path)) { + if (specifier_is_relative(imp->module_path) || + specifier_names_indexed_package(indexed_packages, imp->module_path)) { return false; /* an in-tree binding for this name — never suppress */ } external_binding = true; diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 863254fda..f772acc69 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -5112,6 +5112,116 @@ TEST(pipeline_external_import_shadow_not_bound_to_local_homonym_issue1355) { PASS(); } +/* #1355 / #1732: the same tree, but the bare specifier now names a WORKSPACE + * package that the repository itself ships. `main` deliberately points at a + * build artifact that is not checked in — the shape every real monorepo has, + * and the reason an entry-file lookup cannot answer this question. */ +static void write_workspace_shadow_fixture(const char *dir, int pad_files) { + /* One directory level only: write_temp_file's mkdir is documented as + * "simple version, one level", and a deeper path fails silently — which + * makes the fixture look healthy while the file never lands. */ + write_temp_file(dir, "lib/package.json", + "{\n \"name\": \"@fixture/lib\",\n \"main\": \"./dist/index.js\"\n}\n"); + write_temp_file(dir, "lib/index.ts", + "export function wsEq(a: string, b: string): boolean {\n" + " return a === b;\n" + "}\n"); + write_temp_file(dir, "app/package.json", "{\n \"name\": \"@fixture/app\"\n}\n"); + /* The lone project symbol named wsSpec — an ordinary local helper that a + * project-wide guess would happily bind an external `wsSpec` to. */ + write_temp_file(dir, "app/spec-helpers.ts", + "export function wsSpec(label: string): string {\n" + " return label;\n" + "}\n"); + write_temp_file(dir, "app/query.ts", + "import { wsEq } from '@fixture/lib';\n" + "import { wsSpec } from 'vitest';\n" + "export function wsBuild(id: string): unknown {\n" + " return [wsEq(id, id), wsSpec('case')];\n" + "}\n"); + for (int i = 0; i < pad_files; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "app/pad_%02d.ts", i); + snprintf(body, sizeof(body), "export function wsPad%02d(): number { return %d; }\n", i, i); + write_temp_file(dir, name, body); + } +} + +static int assert_workspace_shadow_contract(const char *dir, const char *db_name) { + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/%s", dir, db_name); + cbm_pipeline_t *p = cbm_pipeline_new(dir, db_path, CBM_MODE_FULL); + if (!p) { + return 1; + } + if (cbm_pipeline_run(p) != 0) { + cbm_pipeline_free(p); + return 2; + } + const char *project = cbm_pipeline_project_name(p); + cbm_store_t *s = cbm_store_open_path(db_path); + if (!s) { + cbm_pipeline_free(p); + return 3; + } + int rc = 0; + /* (1) the workspace sibling: the tree declares @fixture/lib, so the bare + * specifier is in-project and the fallback keeps its chance. RED without + * the package-map check — this is the 1377-edge class measured on + * drizzle-team/drizzle-orm. */ + if (!cross_file_call_exists(s, project, "wsBuild", "wsEq")) { + rc = 4; + } + /* (2) a genuine third-party package in the very same file is still cut: + * the check must not disarm the guard wholesale. */ + if (rc == 0 && cross_file_call_exists(s, project, "wsBuild", "wsSpec")) { + rc = 5; + } + cbm_store_close(s); + cbm_pipeline_free(p); + return rc; +} + +TEST(pipeline_external_import_shadow_keeps_workspace_sibling_issue1355) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_ws_import_shadow_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_workspace_shadow_fixture(tmp, EXTERNAL_IMPORT_SHADOW_PARALLEL_PAD); + + const char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + + cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1); + int sequential = assert_workspace_shadow_contract(tmp, "ws-sequential.db"); + + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + cbm_setenv("CBM_WORKERS", "4", 1); + int parallel = assert_workspace_shadow_contract(tmp, "ws-parallel.db"); + + if (saved_workers) { + cbm_setenv("CBM_WORKERS", saved_workers, 1); + free(saved_workers); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + if (saved_single) { + cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1); + free(saved_single); + } else { + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + } + th_rmtree(tmp); + + ASSERT_EQ(sequential, 0); + ASSERT_EQ(parallel, 0); + PASS(); +} + TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_tsjs_par_XXXXXX"); @@ -13461,6 +13571,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_python_bare_local_binding_suppresses_weak_edge); RUN_TEST(pipeline_python_bare_local_binding_parallel_suppresses_weak_edge); RUN_TEST(pipeline_external_import_shadow_not_bound_to_local_homonym_issue1355); + RUN_TEST(pipeline_external_import_shadow_keeps_workspace_sibling_issue1355); RUN_TEST(pipeline_parallel_python_cross_only_dunder_gets_synthetic_carrier); RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier); RUN_TEST(pipeline_arg_url_rejects_non_http_slash_arguments); diff --git a/tests/test_registry.c b/tests/test_registry.c index d4d7545aa..d8494ce82 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -970,6 +970,9 @@ TEST(weak_call_guards_share_one_drop_list) { } ASSERT_EQ(member, binding); } + PASS(); +} + TEST(external_import_shadow_drops_package_bound_bare_call) { /* #1355: `import { eq, sql } from "drizzle-orm"` binds both names to a * package outside the indexed tree, so no import-map key exists for them and @@ -983,14 +986,14 @@ TEST(external_import_shadow_drops_package_bound_bare_call) { /* Only the relative import materialized an IMPORTS edge. */ const char *keys[] = {"users"}; - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, keys, 1)); - ASSERT_TRUE(cbm_suppress_external_import_shadow("sql", "unique_name", &arr, keys, 1)); - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "suffix_match", &arr, keys, 1)); - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "field_type_hint", &arr, keys, 1)); - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "fuzzy", &arr, keys, 1)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, keys, 1, NULL)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("sql", "unique_name", &arr, keys, 1, NULL)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "suffix_match", &arr, keys, 1, NULL)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "field_type_hint", &arr, keys, 1, NULL)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "fuzzy", &arr, keys, 1, NULL)); /* A file whose imports all failed to materialize is still covered: the * evidence is the import statement, not the map. */ - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, NULL)); PASS(); } @@ -1008,34 +1011,42 @@ TEST(external_import_shadow_keeps_everything_else) { const int nkeys = 2; /* Import-/receiver-/module-aware strategies are never this guard's business. */ - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "import_map", &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "import_map_suffix", &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "same_module", &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "qualified_suffix", &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "lsp_ts_import", &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "lsp_ts_method", &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "service_pattern", &arr, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "import_map", &arr, keys, nkeys, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("eq", "import_map_suffix", &arr, keys, nkeys, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "same_module", &arr, keys, nkeys, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("eq", "qualified_suffix", &arr, keys, nkeys, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("eq", "lsp_ts_import", &arr, keys, nkeys, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("eq", "lsp_ts_method", &arr, keys, nkeys, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("eq", "service_pattern", &arr, keys, nkeys, NULL)); /* A relative specifier names a path inside the tree — a missing IMPORTS edge * there is an in-project gap, not an external binding. */ ASSERT_FALSE( - cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, NULL, 0)); - /* A package the import map DOES bind resolved in-graph (workspace package). */ + cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, keys, nkeys, NULL)); ASSERT_FALSE( - cbm_suppress_external_import_shadow("getRootContainer", "unique_name", &arr, keys, nkeys)); + cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, NULL, 0, NULL)); + /* A package the import map DOES bind resolved in-graph (workspace package). */ + ASSERT_FALSE(cbm_suppress_external_import_shadow("getRootContainer", "unique_name", &arr, keys, + nkeys, NULL)); /* A name the file never imports (called with no import at all). */ - ASSERT_FALSE(cbm_suppress_external_import_shadow("helper", "unique_name", &arr, keys, nkeys)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("helper", "unique_name", &arr, keys, nkeys, NULL)); /* Member and package/namespace-qualified callees belong to the * receiver-aware guards, not to this one. */ - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq.apply", "unique_name", &arr, keys, nkeys)); ASSERT_FALSE( - cbm_suppress_external_import_shadow("eq::apply", "unique_name", &arr, keys, nkeys)); + cbm_suppress_external_import_shadow("eq.apply", "unique_name", &arr, keys, nkeys, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("eq::apply", "unique_name", &arr, keys, nkeys, NULL)); /* Empty / absent inputs. */ - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", NULL, &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "", &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow(NULL, "unique_name", &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("", "unique_name", &arr, keys, nkeys)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", NULL, keys, nkeys)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", NULL, &arr, keys, nkeys, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "", &arr, keys, nkeys, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow(NULL, "unique_name", &arr, keys, nkeys, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("", "unique_name", &arr, keys, nkeys, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", NULL, keys, nkeys, NULL)); PASS(); } @@ -1054,8 +1065,8 @@ TEST(external_import_shadow_relative_binding_wins_the_tie) { }; CBMImportArray a = {.items = pkg_first, .count = 2, .cap = 2}; CBMImportArray b = {.items = rel_first, .count = 2, .cap = 2}; - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &a, NULL, 0)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &b, NULL, 0)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &a, NULL, 0, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &b, NULL, 0, NULL)); PASS(); } @@ -1078,9 +1089,10 @@ TEST(external_import_shadow_windows_relative_specifier_kept) { * keep the edge because it is in-tree, not external. RED before the fix * (specifier_is_relative saw '.' / '/' only, so all three were classified * external and suppressed); GREEN after. */ - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("sql", "unique_name", &arr, NULL, 0)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, NULL, 0)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("sql", "unique_name", &arr, NULL, 0, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, NULL, 0, NULL)); /* A bare package specifier without any drive letter or leading separator * (e.g. "drizzle-orm") must still be external — this predicate must not * become "anything with a colon or backslash". */ @@ -1088,7 +1100,57 @@ TEST(external_import_shadow_windows_relative_specifier_kept) { {.local_name = "eq", .module_path = "drizzle-orm"}, }; CBMImportArray parr = {.items = pkg_only, .count = 1, .cap = 1}; - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &parr, NULL, 0)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &parr, NULL, 0, NULL)); + PASS(); +} + +TEST(external_import_shadow_workspace_sibling_kept) { + /* #1355 / #1732: inside a workspace monorepo a BARE specifier can still name + * in-tree code — `import { eq } from "drizzle-orm"` written inside the + * drizzle-orm repository is a sibling package, not a dependency. Measured on + * drizzle-team/drizzle-orm at b7862528: without this check the guard removed + * 2609 CALLS edges, 1377 of which pointed at exactly the file the specifier + * names. The package map is keyed by each in-tree manifest's `name`, so the + * tree's own claim decides. */ + CBMImport imports[] = { + {.local_name = "eq", .module_path = "drizzle-orm"}, + {.local_name = "pgTable", .module_path = "drizzle-orm/pg-core"}, + {.local_name = "hoisted", .module_path = "@scope/kit/sub"}, + {.local_name = "test", .module_path = "vitest"}, + }; + CBMImportArray arr = {.items = imports, .count = 4, .cap = 4}; + + CBMHashTable *pkgs = cbm_ht_create(8); + ASSERT_NOT_NULL(pkgs); + cbm_ht_set(pkgs, "drizzle-orm", (void *)"qn"); + cbm_ht_set(pkgs, "@scope/kit", (void *)"qn"); + + /* Exact name, subpath, and scoped subpath all resolve to an in-tree + * package — the same-name fallback keeps its chance. */ + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, pkgs)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("pgTable", "suffix_match", &arr, NULL, 0, pkgs)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("hoisted", "unique_name", &arr, NULL, 0, pkgs)); + /* A genuine third-party dependency is untouched by the new check: the tree + * declares no package called "vitest", so the #1355 removal still happens. */ + ASSERT_TRUE(cbm_suppress_external_import_shadow("test", "suffix_match", &arr, NULL, 0, pkgs)); + + /* A tree with no manifests at all (pkgmap NULL) keeps the pre-existing + * specifier-shape-only contract — every one of these is external again. */ + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, NULL)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("pgTable", "suffix_match", &arr, NULL, 0, NULL)); + + /* An unrelated package map must not accidentally match by prefix: "drizzle" + * is not "drizzle-orm", and the walk is by path segment, not by substring. */ + CBMHashTable *other = cbm_ht_create(8); + ASSERT_NOT_NULL(other); + cbm_ht_set(other, "drizzle", (void *)"qn"); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, other)); + + cbm_ht_free(other); + cbm_ht_free(pkgs); PASS(); } @@ -1195,4 +1257,5 @@ SUITE(registry) { RUN_TEST(external_import_shadow_keeps_everything_else); RUN_TEST(external_import_shadow_relative_binding_wins_the_tie); RUN_TEST(external_import_shadow_windows_relative_specifier_kept); + RUN_TEST(external_import_shadow_workspace_sibling_kept); } From 9f57edfa576072398728eda2c6fc2585d2ecb063 Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Mon, 7 Sep 2026 08:00:09 +0000 Subject: [PATCH 016/217] fix(pipeline): skip an unreadable subdirectory in the control-file walk instead of aborting the whole index semantic_manifest_walk_controls() returns CBM_NOT_FOUND the moment cbm_opendir() fails on any subdirectory, and that propagates all the way up through cbm_pipeline_build_semantic_manifest(), aborting the entire index_repository run. discover.c's walk_dir() already treats the same failure (a permission-denied directory it can't open) as "nothing to discover here" and moves on. The control walk has no reason to be stricter: it only looks for .gitignore/.cbmignore/package-manifest files, which a locked-out directory simply doesn't contribute. Skip it and keep walking, except at depth 0, where an inaccessible manifest root is still a real error (pipeline_semantic_manifest_rejects_non_directory_root pins that). Signed-off-by: Amir Fathi --- src/pipeline/pipeline_incremental.c | 5 +++- tests/test_pipeline.c | 42 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index bf71a1f84..aa7d04321 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -338,7 +338,10 @@ static int semantic_manifest_walk_controls(semantic_manifest_builder_t *builder, } cbm_dir_t *dir = cbm_opendir(abs_dir); if (!dir) { - return CBM_NOT_FOUND; + /* A nested directory we can't open has no control files to + * contribute; skip it like discover.c's walk_dir already does. + * The root call (depth 0) still fails closed. */ + return depth == 0 ? CBM_NOT_FOUND : 0; } int rc = 0; cbm_dirent_t *entry; diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 5608ee559..eae749bc1 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4123,6 +4123,45 @@ TEST(pipeline_semantic_manifest_rejects_non_directory_root) { PASS(); } +#ifndef _WIN32 +/* A nested directory the process cannot open (permission-denied) must not + * abort the whole manifest walk: it has no control files to contribute, so + * it is skipped, and control files in sibling directories still get found. + * Root-not-a-directory (above) is a different, still fail-closed case. */ +TEST(pipeline_semantic_manifest_skips_unreadable_subdirectory) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_manifest_locked_dir_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "readable.py", "def Readable():\n return 1\n"); + write_temp_file(tmp, "locked/secret.txt", "should never be opened\n"); + write_temp_file(tmp, "sibling/.gitignore", "*.log\n"); + + char locked_path[512]; + snprintf(locked_path, sizeof(locked_path), "%s/locked", tmp); + ASSERT_EQ(chmod(locked_path, 0), 0); + + cbm_file_hash_t *manifest = NULL; + int manifest_count = -1; + int rc = cbm_pipeline_build_semantic_manifest("manifest-skip-locked", tmp, NULL, 0, NULL, 0, + NULL, NULL, &manifest, &manifest_count); + + bool found_sibling_gitignore = false; + for (int i = 0; i < manifest_count; i++) { + if (manifest[i].rel_path && strcmp(manifest[i].rel_path, "sibling/.gitignore") == 0) { + found_sibling_gitignore = true; + } + } + + chmod(locked_path, 0755); + cbm_pipeline_free_semantic_manifest(manifest, manifest_count > 0 ? manifest_count : 0); + th_rmtree(tmp); + + ASSERT_EQ(rc, 0); + ASSERT_TRUE(found_sibling_gitignore); + PASS(); +} +#endif /* !_WIN32 */ + /* A fully validated staged graph must be able to recover from a definitely * non-SQLite destination without deleting evidence or overwriting an earlier * quarantine. The replacement happens only after the corrupt bytes are moved. */ @@ -13862,6 +13901,9 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_incremental_successful_publication_preserves_adr); RUN_TEST(pipeline_full_adr_capture_failure_preserves_previous_generation); RUN_TEST(pipeline_semantic_manifest_rejects_non_directory_root); +#ifndef _WIN32 + RUN_TEST(pipeline_semantic_manifest_skips_unreadable_subdirectory); +#endif RUN_TEST(pipeline_full_reindex_quarantines_corrupt_destination_without_overwrite); RUN_TEST(pipeline_full_reindex_replaces_legacy_schema_without_quarantine); #endif From 6b56d78e1bbb5715c9d0c715453244187bd299bc Mon Sep 17 00:00:00 2001 From: Sanskar gupta Date: Mon, 7 Sep 2026 13:04:34 +0000 Subject: [PATCH 017/217] fix(lang): track Solidity modifier invocations as decorators Signed-off-by: Sanskar gupta --- internal/cbm/lang_specs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index 3a5fad0d4..ed92bd674 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -1038,6 +1038,7 @@ static const char *solidity_field_types[] = {"state_variable_declaration", "stru static const char *solidity_call_types[] = {"call_expression", "call", "new_expression", NULL}; static const char *solidity_import_types[] = {"import_directive", "import", "using_directive", NULL}; +static const char *solidity_decorator_types[] = {"modifier_invocation", NULL}; static const char *solidity_branch_types[] = {"if_statement", "for_statement", "while_statement", "do_while_statement", "try_statement", NULL}; @@ -2115,7 +2116,7 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = { solidity_field_types, solidity_module_types, solidity_call_types, solidity_import_types, empty_types, solidity_branch_types, solidity_var_types, solidity_assign_types, solidity_throw_types, NULL, - empty_types, NULL, NULL, tree_sitter_solidity, NULL}, + solidity_decorator_types, NULL, NULL, tree_sitter_solidity, NULL}, // CBM_LANG_TYPST [CBM_LANG_TYPST] = {CBM_LANG_TYPST, typst_func_types, empty_types, empty_types, From b974adf6ed9b82d8669d1a65b1757816ce0d12df Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Mon, 7 Sep 2026 10:19:48 -0400 Subject: [PATCH 018/217] fix: grow the ts_lsp AST walk stacks so wide files keep every declaration Three walks in internal/cbm/lsp/ts_lsp.c held at most 256 AST nodes in a fixed array and stopped pushing when it filled. Nothing reported the stop, so the passes answered as if they had seen the whole file. In rebuild_signatures_from_ast and convert_signature_type_params the seeding loop alone can fill all 256 slots, because it pushes every top-level child of the file. A file with more than 256 top-level declarations therefore skipped every declaration past that point: each one kept whatever signature the first extraction pass had guessed, and a generic function past the cap kept NAMED instead of TYPE_PARAM, so calls through it stopped resolving. In infer_return_type_from_body the same cap made a wide function body answer "unknown return type" for a return the walk had never reached. The repo already carries the module that solves this: internal/cbm/extract_node_stack.h, written for GitHub issue #199 to replace fixed TSNode stack[] arrays that silently drop subtrees. Nine extract files use it; ts_lsp.c was never converted. This converts it. The header gains ts_nstack_init_arena, which takes an arena directly, because the LSP passes hold a TSLSPContext rather than a CBMExtractCtx. ts_nstack_init keeps its context argument, which is what makes handing over the wrong arena a type error, and now delegates to the new function. Traversal order changes from last-child-first to first-child-first, because ts_nstack_push_children reverses on push. That also drops an O(N^2) ts_node_child(n, i) loop for one cursor pass. For infer_return_type_from_body the new order matches the comment the function already carried: it finds the first return, where the old code found the last. New test tslsp_stress_declarations_past_stack_cap makes the same claim as tslsp_generic_identity_inference, 300 declarations deeper in the file. Without this change it fails with "MISSING resolved call: caller~.go -> callee~use"; with it the ts_lsp suite is 304 passed, 0 failed. Signed-off-by: Joshua Richter --- internal/cbm/extract_node_stack.h | 18 +++++--- internal/cbm/lsp/ts_lsp.c | 74 +++++++++++++++---------------- tests/test_ts_lsp.c | 30 +++++++++++++ 3 files changed, 78 insertions(+), 44 deletions(-) diff --git a/internal/cbm/extract_node_stack.h b/internal/cbm/extract_node_stack.h index 0013d53b1..443eb6e6a 100644 --- a/internal/cbm/extract_node_stack.h +++ b/internal/cbm/extract_node_stack.h @@ -34,6 +34,18 @@ typedef struct { CBMArena *scratch; } TSNodeStack; +/* Initialize a stack against an arena directly. The LSP passes hold a + * TSLSPContext, not a CBMExtractCtx, and every allocation in those files + * already comes from that one arena — see collect_children in ts_lsp.c. An + * extraction caller should still use ts_nstack_init below, whose context + * argument is what makes handing over the wrong arena a type error. */ +static inline void ts_nstack_init_arena(TSNodeStack *s, CBMArena *arena, int initial_cap) { + s->scratch = arena; + s->items = (TSNode *)cbm_arena_alloc(arena, (size_t)initial_cap * sizeof(TSNode)); + s->count = 0; + s->cap = s->items ? initial_cap : 0; +} + /* Initialize a stack with the given initial capacity, allocated from the * context's traversal scratch. Taking the context rather than an arena is * deliberate: it makes handing over ctx->arena, or a local alias of it, a type @@ -41,11 +53,7 @@ typedef struct { * scratch falls back to ctx->arena, which is the behaviour that shipped before * #1997, so no caller ever gets a NULL arena and silently loses nodes. */ static inline void ts_nstack_init(TSNodeStack *s, const CBMExtractCtx *ctx, int initial_cap) { - CBMArena *arena = ctx->scratch ? ctx->scratch : ctx->arena; - s->scratch = arena; - s->items = (TSNode *)cbm_arena_alloc(arena, (size_t)initial_cap * sizeof(TSNode)); - s->count = 0; - s->cap = s->items ? initial_cap : 0; + ts_nstack_init_arena(s, ctx->scratch ? ctx->scratch : ctx->arena, initial_cap); } /* Push a node onto the stack, growing 2x if needed. */ diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index fc8b8911f..2e7292416 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -27,6 +27,7 @@ */ #include "ts_lsp.h" +#include "extract_node_stack.h" #include #include #include @@ -4634,13 +4635,15 @@ static const CBMType *infer_return_type_from_body(TSLSPContext *ctx, TSNode body if (ts_node_is_null(body)) return cbm_type_unknown(); - // Iterative DFS using a small fixed-size stack to avoid C-stack blowup on big bodies. - enum { STACK_CAP = 256 }; - TSNode stack[STACK_CAP]; - int top = 0; - stack[top++] = body; - while (top > 0) { - TSNode n = stack[--top]; + // Iterative DFS to avoid C-stack blowup on big bodies. The stack grows, so a + // body wider than any fixed cap is still walked in full: a fixed 256 slots + // used to stop pushing without a word, and this function then answered + // "unknown return type" for a body whose return it had never reached. + TSNodeStack stack; + ts_nstack_init_arena(&stack, ctx->arena, 256); + ts_nstack_push(&stack, body); + while (stack.count > 0) { + TSNode n = ts_nstack_pop(&stack); if (ts_node_is_null(n)) continue; const char *k = ts_node_type(n); @@ -4662,10 +4665,7 @@ static const CBMType *infer_return_type_from_body(TSLSPContext *ctx, TSNode body strcmp(k, "arrow_function") == 0 || strcmp(k, "method_definition") == 0) continue; - uint32_t cnt = ts_node_child_count(n); - for (uint32_t i = 0; i < cnt && top < STACK_CAP; i++) { - stack[top++] = ts_node_child(n, i); - } + ts_nstack_push_children(&stack, n); } return cbm_type_unknown(); } @@ -4681,31 +4681,29 @@ static void rebuild_signatures_from_ast(TSLSPContext *ctx, TSNode root, CBMTypeR if (ts_node_is_null(root) || !reg || !ctx->module_qn) return; - enum { STACK_CAP = 256 }; - TSNode stack[STACK_CAP]; - int top = 0; - uint32_t nc = ts_node_child_count(root); - for (uint32_t i = 0; i < nc && top < STACK_CAP; i++) - stack[top++] = ts_node_child(root, i); + /* The stack grows. A fixed 256 slots used to be filled by the seeding loop + * alone on a file with that many top-level children, and every declaration + * past it was then skipped without a word — it kept whatever signature + * extract_defs had guessed. */ + TSNodeStack stack; + ts_nstack_init_arena(&stack, ctx->arena, 256); + ts_nstack_push_children(&stack, root); - while (top > 0) { - TSNode n = stack[--top]; + while (stack.count > 0) { + TSNode n = ts_nstack_pop(&stack); if (ts_node_is_null(n)) continue; const char *k = ts_node_type(n); // Recurse into export_statement and class bodies. if (strcmp(k, "export_statement") == 0 || strcmp(k, "class_body") == 0) { - uint32_t cnt = ts_node_child_count(n); - for (uint32_t i = 0; i < cnt && top < STACK_CAP; i++) { - stack[top++] = ts_node_child(n, i); - } + ts_nstack_push_children(&stack, n); continue; } if (strcmp(k, "class_declaration") == 0) { TSNode body = ts_node_child_by_field_name(n, "body", TS_LSP_FIELD_LEN("body")); - if (!ts_node_is_null(body) && top < STACK_CAP) - stack[top++] = body; + if (!ts_node_is_null(body)) + ts_nstack_push(&stack, body); continue; } @@ -4845,31 +4843,29 @@ static void convert_signature_type_params(TSLSPContext *ctx, TSNode root, CBMTyp return; // Walk: function_declaration, class_declaration { method_definition }, plus exported. - enum { STACK_CAP = 256 }; - TSNode stack[STACK_CAP]; - int top = 0; - uint32_t nc = ts_node_child_count(root); - for (uint32_t i = 0; i < nc && top < STACK_CAP; i++) - stack[top++] = ts_node_child(root, i); + /* The stack grows. A fixed 256 slots used to be filled by the seeding loop + * alone on a file with that many top-level children, and every declaration + * past it was then skipped without a word — it kept whatever signature + * extract_defs had guessed. */ + TSNodeStack stack; + ts_nstack_init_arena(&stack, ctx->arena, 256); + ts_nstack_push_children(&stack, root); - while (top > 0) { - TSNode n = stack[--top]; + while (stack.count > 0) { + TSNode n = ts_nstack_pop(&stack); if (ts_node_is_null(n)) continue; const char *k = ts_node_type(n); // Recurse into export_statement and class_body. if (strcmp(k, "export_statement") == 0 || strcmp(k, "class_body") == 0) { - uint32_t cnt = ts_node_child_count(n); - for (uint32_t i = 0; i < cnt && top < STACK_CAP; i++) { - stack[top++] = ts_node_child(n, i); - } + ts_nstack_push_children(&stack, n); continue; } if (strcmp(k, "class_declaration") == 0) { TSNode body = ts_node_child_by_field_name(n, "body", TS_LSP_FIELD_LEN("body")); - if (!ts_node_is_null(body) && top < STACK_CAP) - stack[top++] = body; + if (!ts_node_is_null(body)) + ts_nstack_push(&stack, body); continue; } diff --git a/tests/test_ts_lsp.c b/tests/test_ts_lsp.c index 52c3bbacb..49230da43 100644 --- a/tests/test_ts_lsp.c +++ b/tests/test_ts_lsp.c @@ -2817,6 +2817,35 @@ TEST(tslsp_stress_many_classes) { PASS(); } +TEST(tslsp_stress_declarations_past_stack_cap) { + /* 300 padding functions, then the generic pair that needs the AST walk. + * The walk used to hold 256 nodes at most, so every top-level declaration + * after the 256th was dropped without a word and kept whatever signature + * the first extraction pass had guessed. */ + enum { BUF_CAP = 64 * 1024 }; + char *buf = (char *)malloc(BUF_CAP); + if (!buf) + PASS(); + char *p = buf; + char *end = buf + BUF_CAP; + for (int i = 0; i < 300; i++) { + p += snprintf(p, (size_t)(end - p), "function pad%d(): void {}\n", i); + } + p += snprintf(p, (size_t)(end - p), "function identity(x: T): T { return x; }\n"); + p += snprintf(p, (size_t)(end - p), "class Box { use(): void {} }\n"); + p += snprintf(p, (size_t)(end - p), + "function go() { const b = identity(new Box()); b.use(); }\n"); + *p = '\0'; + + CBMFileResult *r = extract_ts(buf); + free(buf); + ASSERT_NOT_NULL(r); + /* Same claim as tslsp_generic_identity_inference, 300 declarations deeper. */ + ASSERT_GTE(require_resolved(r, ".go", "use"), 0); + cbm_free_result(r); + PASS(); +} + TEST(tslsp_stress_deep_inheritance) { /* Chain of 30 classes via extends, leaf method on root. */ char buf[16 * 1024]; @@ -4565,6 +4594,7 @@ SUITE(ts_lsp) { /* Stress tests */ RUN_TEST(tslsp_stress_many_classes); + RUN_TEST(tslsp_stress_declarations_past_stack_cap); RUN_TEST(tslsp_stress_deep_inheritance); RUN_TEST(tslsp_stress_long_method_chain); RUN_TEST(tslsp_stress_megafile_mixed); From ec56ea8f60f0058bd1cd0fffb38cc0153b99f895 Mon Sep 17 00:00:00 2001 From: Joshua Richter Date: Mon, 7 Sep 2026 12:28:01 -0400 Subject: [PATCH 019/217] test: cover the return-type walk past the old stack cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first commit converted three walks in ts_lsp.c but tested only two of them. infer_return_type_from_body was changed without a test of its own. tslsp_stress_return_past_stack_cap makes the same claim as the existing tslsp_return_inferred_local, with the return placed after 300 statements — more than the 256 slots the old fixed array held, so the walk used to stop before reaching it and answer "unknown return type". Without the fix the suite is 303 passed, 2 failed, the second failure being this test. With it, 305 passed. Signed-off-by: Joshua Richter --- tests/test_ts_lsp.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_ts_lsp.c b/tests/test_ts_lsp.c index 49230da43..ba0409e90 100644 --- a/tests/test_ts_lsp.c +++ b/tests/test_ts_lsp.c @@ -2846,6 +2846,35 @@ TEST(tslsp_stress_declarations_past_stack_cap) { PASS(); } +TEST(tslsp_stress_return_past_stack_cap) { + /* infer_return_type_from_body reads the return type out of a body that + * carries no written one. The return here sits after 300 statements, more + * than the old fixed cap held, so the walk used to stop before reaching it + * and answer "unknown return type". */ + enum { BUF_CAP = 64 * 1024 }; + char *buf = (char *)malloc(BUF_CAP); + if (!buf) + PASS(); + char *p = buf; + char *end = buf + BUF_CAP; + p += snprintf(p, (size_t)(end - p), "class Conn { ping(): void {} }\n"); + p += snprintf(p, (size_t)(end - p), "function makeConn() {\n"); + for (int i = 0; i < 300; i++) { + p += snprintf(p, (size_t)(end - p), " let a%d = 0;\n", i); + } + p += snprintf(p, (size_t)(end - p), " return new Conn();\n}\n"); + p += snprintf(p, (size_t)(end - p), "function go() { const c = makeConn(); c.ping(); }\n"); + *p = '\0'; + + CBMFileResult *r = extract_ts(buf); + free(buf); + ASSERT_NOT_NULL(r); + /* Same claim as tslsp_return_inferred_local, 300 statements deeper. */ + ASSERT_GTE(require_resolved(r, ".go", "ping"), 0); + cbm_free_result(r); + PASS(); +} + TEST(tslsp_stress_deep_inheritance) { /* Chain of 30 classes via extends, leaf method on root. */ char buf[16 * 1024]; @@ -4595,6 +4624,7 @@ SUITE(ts_lsp) { /* Stress tests */ RUN_TEST(tslsp_stress_many_classes); RUN_TEST(tslsp_stress_declarations_past_stack_cap); + RUN_TEST(tslsp_stress_return_past_stack_cap); RUN_TEST(tslsp_stress_deep_inheritance); RUN_TEST(tslsp_stress_long_method_chain); RUN_TEST(tslsp_stress_megafile_mixed); From 1891d448054ae8ff111d2797d91ec93bf3820907 Mon Sep 17 00:00:00 2001 From: Vladislav Rahmanov Date: Thu, 10 Sep 2026 17:50:01 +0300 Subject: [PATCH 020/217] feat(cli): grade index_repository outcomes with distinct exit codes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A freshness gate cannot read a tool's account of itself — it reads the process status. index_repository returned 0 for a partial index and the same 1 for both a missing repository and a pipeline that fell over inside an existing one, so no caller could tell those apart. The CLI now grades its own outcome: 0 indexed, quality at or above the thresholds 1 hard failure inside a reachable repository (unchanged meaning) 2 indexed but below a quality threshold 3 repo_path absent, unreadable, or not a directory The contract is opt-in: it applies only with CBM_GATE set to a non-empty value other than "0". Without it the CLI exits exactly as before, so a caller that never asked for grading never sees a new code. Grading happens in the CLI only. MCP responses are untouched, so agents and the soak test, which judge by JSON-RPC and tool-level isError, see exactly what they saw before. Thresholds are separate on purpose and overridable: CBM_GATE_MAX_UNUSABLE default 0 a file that did not parse at all CBM_GATE_MAX_PARTIAL_PCT default 10 share of partially parsed files Partial parsing marks constructs a grammar does not cover — a property of language support, not of index quality; this repository itself sits at 5.8%. A gate its own project cannot pass gets switched off. Two additive fields carry what the grading needs: files_indexed, parse_partial_pct the denominator the ratio needs and the share stated outright, so a gate can quote it in a refusal reason on failures: target_unavailable vs pipeline_failed Signed-off-by: Vladislav Rahmanov --- src/cli/cli.c | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/cli/cli.h | 42 +++++++++++++++++++++++++++++ src/main.c | 5 ++++ src/mcp/mcp.c | 32 ++++++++++++++++++++++ 4 files changed, 154 insertions(+) diff --git a/src/cli/cli.c b/src/cli/cli.c index f47c3031c..4ce60e9c4 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -157,6 +157,81 @@ int cbm_cli_exit_status_after_maintenance(int exit_status, bool maintenance_canc return maintenance_cancelled && exit_status == EXIT_SUCCESS ? EXIT_FAILURE : exit_status; } +/* One override, parsed strictly: a typo must not silently disable a gate, so + * anything that is not a whole number falls back to the documented default. */ +static long cli_gate_threshold(const char *name, long fallback) { + const char *raw = getenv(name); + if (!raw || !raw[0]) { + return fallback; + } + char *end = NULL; + errno = 0; + long value = strtol(raw, &end, 10); + /* Overflow clamps to LONG_MAX with the whole string consumed, which would + * read as a ceiling no run can exceed: the one typo that disables a gate. */ + if (errno == ERANGE || end == raw || *end != '\0') { + return fallback; + } + return value; +} + +/* The contract is opt-in. Unset, empty, or "0" leaves the historical 0/1 + * status untouched, so no existing caller sees a new code until it asks. */ +static bool cli_gate_enabled(void) { + const char *raw = getenv("CBM_GATE"); + return raw && raw[0] && strcmp(raw, "0") != 0; +} + +int cbm_cli_index_exit_status(const char *result, int base_status) { + if (!result || !cli_gate_enabled()) { + return base_status; + } + yyjson_doc *envelope = yyjson_read(result, strlen(result), 0); + if (!envelope) { + return base_status; + } + yyjson_val *root = yyjson_doc_get_root(envelope); + yyjson_val *content = yyjson_is_obj(root) ? yyjson_obj_get(root, "content") : NULL; + yyjson_val *first = yyjson_is_arr(content) ? yyjson_arr_get_first(content) : NULL; + const char *text = first ? yyjson_get_str(yyjson_obj_get(first, "text")) : NULL; + + int status = base_status; + yyjson_doc *payload = text ? yyjson_read(text, strlen(text), 0) : NULL; + yyjson_val *proot = payload ? yyjson_doc_get_root(payload) : NULL; + if (yyjson_is_obj(proot)) { + const char *state = yyjson_get_str(yyjson_obj_get(proot, "status")); + const char *reason = yyjson_get_str(yyjson_obj_get(proot, "reason")); + if (state && strcmp(state, "error") == 0) { + status = (reason && strcmp(reason, "target_unavailable") == 0) + ? CBM_CLI_EXIT_TARGET + : (base_status != CBM_CLI_EXIT_OK ? base_status : CBM_CLI_EXIT_FAILURE); + } else if (base_status == CBM_CLI_EXIT_OK) { + long unusable = (long)yyjson_get_int(yyjson_obj_get(proot, "parse_unusable_count")); + long partial = (long)yyjson_get_int(yyjson_obj_get(proot, "parse_partial_count")); + long files = (long)yyjson_get_int(yyjson_obj_get(proot, "files_indexed")); + long max_unusable = cli_gate_threshold("CBM_GATE_MAX_UNUSABLE", 0); + long max_partial_pct = cli_gate_threshold("CBM_GATE_MAX_PARTIAL_PCT", 10); + /* "degraded" is the pipeline's own verdict that the graph came out + * far smaller than the run expected — a quality failure by any + * reading, so it joins the two parse thresholds. */ + bool degraded = strcmp(state ? state : "", "degraded") == 0; + bool too_many_unusable = max_unusable >= 0 && unusable > max_unusable; + /* Integer arithmetic on purpose: a percentage compared through a + * double would make the threshold depend on rounding. */ + bool too_many_partial = + max_partial_pct >= 0 && files > 0 && partial * 100 > max_partial_pct * files; + if (degraded || too_many_unusable || too_many_partial) { + status = CBM_CLI_EXIT_QUALITY; + } + } + } + if (payload) { + yyjson_doc_free(payload); + } + yyjson_doc_free(envelope); + return status; +} + /* #1537. Two very different failures reached this one message: the cohort was * BUSY (real sessions are running — the reader can close them), or the * reservation failed outright (lock I/O, stale coordination state, permissions diff --git a/src/cli/cli.h b/src/cli/cli.h index 139221f0a..702dbc95b 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -57,6 +57,48 @@ bool cbm_cli_mcp_result_is_error(const char *result); * accepted cancellation into EXIT_FAILURE. */ int cbm_cli_exit_status_after_maintenance(int exit_status, bool maintenance_cancelled); +/* ── Index exit contract ────────────────────────────────────────── + * + * A freshness gate must not have to read anyone's account of itself: it reads + * the process status. `index_repository` therefore grades its own outcome. + * + * 0 indexed, quality at or above the thresholds + * 1 hard failure (pipeline fell over inside a reachable repository) + * 2 indexed, but below a quality threshold — a graph that exists and lies + * about how much of the tree it covers is worse than no graph + * 3 target unavailable: repo_path is absent, unreadable, or not a directory + * + * 1 keeps its historical meaning so existing `|| fail` scripts are unaffected; + * 2 and 3 are new and carve out cases that used to return 0 and 1. + * + * The contract is opt-in: it applies only when CBM_GATE is set to a non-empty + * value other than "0". Without it the CLI exits exactly as before, so a + * caller that never asked for grading never sees a new code. + * + * Thresholds, both overridable by environment: + * CBM_GATE_MAX_UNUSABLE (default 0) absolute count of unparsable files + * CBM_GATE_MAX_PARTIAL_PCT (default 10) parse_partial_count / files_indexed + * A threshold set to a negative value disables that check. + * + * The two numbers differ on purpose and must not be collapsed into one: a file + * that did not parse at all is a defect and gets no tolerance, while partial + * parsing marks constructs this grammar does not cover — a property of language + * support, not of index quality, and normal at a few percent in a large tree. + * A gate its own repository cannot pass gets switched off, which protects + * nothing. */ +enum { + CBM_CLI_EXIT_OK = 0, + CBM_CLI_EXIT_FAILURE = 1, + CBM_CLI_EXIT_QUALITY = 2, + CBM_CLI_EXIT_TARGET = 3, +}; + +/* Grade an index_repository result envelope. `base_status` is what the + * ordinary isError mapping already produced; it is preserved unless the + * payload justifies a more specific code. Unparsable payloads change + * nothing — silence is never upgraded into a verdict. */ +int cbm_cli_index_exit_status(const char *result, int base_status); + /* ── Self-update: version comparison ──────────────────────────── */ /* Compare two semver strings (e.g. "0.2.1" vs "0.2.0"). diff --git a/src/main.c b/src/main.c index 53d259b1c..de7adbc5c 100644 --- a/src/main.c +++ b/src/main.c @@ -1000,6 +1000,11 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo } else { exit_code = cli_print_mcp_result(result); } + /* One place for both presentations: a gate reading the status must get + * the same verdict whether or not the caller asked for --json. */ + if (tool_name && strcmp(tool_name, "index_repository") == 0) { + exit_code = cbm_cli_index_exit_status(result, exit_code); + } exit_code = cbm_cli_exit_status_after_maintenance(exit_code, maintenance_cancelled); if (cbm_index_worker_active()) { /* The supervisor protocol classifies the PROCESS, not the tool diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index c8978a79d..888c6c9d9 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -10316,6 +10316,28 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc * add_parse_partial_summary(doc, root, file_errors, file_error_count); add_parse_unusable_summary(doc, root, file_errors, file_error_count); } + /* Denominator for quality gates. parse_partial_count on its own cannot + * say whether sixty flagged files are a rounding error or half the + * repository; file_hashes holds exactly one row per indexed file. */ + if (store) { + cbm_file_hash_t *hashes = NULL; + int hash_count = 0; + if (cbm_store_get_file_hashes(store, project_name, &hashes, &hash_count) == CBM_STORE_OK) { + yyjson_mut_obj_add_int(doc, root, "files_indexed", hash_count); + /* The share, stated outright. A gate that only passes or fails + * cannot tell anyone HOW partial the parse was, and "the graph is + * fresh, but 6.2% of files parsed partially" is a usable hint + * where a bare verdict is not. Tenths of a percent are derived by + * integer division so the number never depends on rounding. */ + yyjson_mut_val *partial_val = yyjson_mut_obj_get(root, "parse_partial_count"); + if (partial_val && hash_count > 0) { + int partial = yyjson_mut_get_int(partial_val); + long tenths = (long)partial * 1000 / hash_count; + yyjson_mut_obj_add_real(doc, root, "parse_partial_pct", (double)tenths / 10.0); + } + cbm_store_free_file_hashes(hashes, hash_count); + } + } int nodes = 0; int edges = 0; bool degraded = false; @@ -11254,6 +11276,16 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { "previous index may have been rolled back."); } else { yyjson_mut_obj_add_str(doc, root, "status", "error"); + /* A repository that is not there at all is a different failure from a + * pipeline that fell over inside one that is — the hint below reads + * identically for both, so callers could not tell them apart. The + * split is deliberately about the ROOT: an unreadable subtree is a + * pipeline failure, not a missing target. */ + cbm_path_info_t target = {0}; + bool target_reachable = + cbm_path_info_utf8(repo_path, &target) == CBM_PATH_INFO_OK && target.is_directory; + yyjson_mut_obj_add_str(doc, root, "reason", + target_reachable ? "pipeline_failed" : "target_unavailable"); yyjson_mut_obj_add_str(doc, root, "hint", "Pipeline failed. Check repo_path exists and contains source files. " "Try mode='fast' for a quicker diagnostic run."); From b037b488be4620a5e92424e7e390ee625cd7e5af Mon Sep 17 00:00:00 2001 From: Vladislav Rahmanov Date: Mon, 7 Sep 2026 19:56:21 +0300 Subject: [PATCH 021/217] test(cli): pin the index_repository exit contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The contract landed measured but unpinned: six outcomes were checked by hand against the built binary and written into the pull request, and nothing in the suite would notice if a code moved. A verification that does not run again is a claim, not a test. Twelve cases over cbm_cli_index_exit_status, which grades a result envelope into a process exit code: clean run stays 0; an unusable file and a partial share above the ceiling both grade 2; "degraded" grades 2 on the status alone; an absent target grades 3 while a pipeline failure keeps 1 — the pair that shared code 1 before and gave the whole contract its reason. Three of them exist to fail if the grader stops working rather than to pass when it does: - the ceiling itself is exercised from both sides, 10 of 100 passing and 11 failing, because a test that only checks obvious excess stays green with the check switched off entirely; - the threshold overrides carry a positive control at the defaults, so "passed after raising the ceiling" cannot be confused with "no check ran"; - eight unreadable threshold values are pinned to fall back on the documented default, since a typo that silently disables a gate is the failure the strict parse was written to prevent. Silence is pinned too: a null, empty, unreadable, or content-less envelope preserves the base status in both directions. Inventing a quality verdict out of an unparsable payload would fail runs for the crime of an unexpected response shape. Envelopes are built with yyjson rather than hand-escaped, so the payload's quoting is the transport's and not the test's own. Signed-off-by: Vladislav Rahmanov --- tests/test_cli.c | 328 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 328 insertions(+) diff --git a/tests/test_cli.c b/tests/test_cli.c index 44259a63c..7d5a8f445 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -14886,6 +14886,319 @@ TEST(cli_update_only_names_an_installer_that_exists_issue1632) { PASS(); } +/* ── index_repository exit contract ────────────────────────────── + * + * cbm_cli_index_exit_status grades a result envelope into a process exit + * code, because a freshness gate reads process status rather than a tool's + * account of itself. The six outcomes were measured by hand against the + * built binary when the contract landed; measured once is not pinned, and + * a code that moves silently is exactly what the contract exists to stop. */ + +/* Build the envelope the CLI actually receives: the payload travels as a + * JSON string inside content[0].text. Escaping it by hand in every test + * would put the test's own escaping on trial instead of the grader. */ +static char *cli_index_envelope(const char *payload) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + if (!doc) { + return NULL; + } + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_val *content = yyjson_mut_arr(doc); + yyjson_mut_val *item = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, item, "text", payload); + yyjson_mut_arr_append(content, item); + yyjson_mut_obj_add_val(doc, root, "content", content); + char *json = yyjson_mut_write(doc, 0, NULL); + yyjson_mut_doc_free(doc); + return json; +} + +/* Grade one payload. Returns -1 only if the envelope could not be built, a + * value no exit code uses, so a setup failure cannot pass for a verdict. */ +static int cli_index_grade(const char *payload, int base_status) { + char *envelope = cli_index_envelope(payload); + if (!envelope) { + return -1; + } + int status = cbm_cli_index_exit_status(envelope, base_status); + free(envelope); + return status; +} + +typedef struct { + char *gate; + char *unusable; + char *partial_pct; +} cli_gate_env_t; + +/* A test that means to exercise the documented defaults must not inherit + * whatever the developer happens to have exported. The contract is opt-in, + * so the gate is switched on here; the one test about it being off unsets + * CBM_GATE itself. */ +static cli_gate_env_t cli_gate_env_clear(void) { + cli_gate_env_t saved = {save_test_env("CBM_GATE"), save_test_env("CBM_GATE_MAX_UNUSABLE"), + save_test_env("CBM_GATE_MAX_PARTIAL_PCT")}; + cbm_setenv("CBM_GATE", "1", 1); + cbm_unsetenv("CBM_GATE_MAX_UNUSABLE"); + cbm_unsetenv("CBM_GATE_MAX_PARTIAL_PCT"); + return saved; +} + +static void cli_gate_env_restore(cli_gate_env_t saved) { + restore_test_env("CBM_GATE", saved.gate); + restore_test_env("CBM_GATE_MAX_UNUSABLE", saved.unusable); + restore_test_env("CBM_GATE_MAX_PARTIAL_PCT", saved.partial_pct); +} + +/* Without CBM_GATE the CLI exits exactly as it always did: a payload that + * would grade 2 or 3 with the gate on keeps the base status. "0" and the + * empty string count as off, so an exported CBM_GATE=0 is not a surprise. */ +TEST(cli_index_exit_gate_off_keeps_base_status) { + cli_gate_env_t saved = cli_gate_env_clear(); + const char *below_quality = "{\"status\":\"ok\",\"files_indexed\":10," + "\"parse_partial_count\":5,\"parse_unusable_count\":1}"; + const char *missing_target = "{\"status\":\"error\",\"reason\":\"target_unavailable\"}"; + + /* Positive control: with the gate on both payloads DO change the status, + * so the unchanged codes below are the switch working, not a dead check. */ + int quality_on = cli_index_grade(below_quality, CBM_CLI_EXIT_OK); + int target_on = cli_index_grade(missing_target, CBM_CLI_EXIT_FAILURE); + + cbm_unsetenv("CBM_GATE"); + int quality_unset = cli_index_grade(below_quality, CBM_CLI_EXIT_OK); + int target_unset = cli_index_grade(missing_target, CBM_CLI_EXIT_FAILURE); + + cbm_setenv("CBM_GATE", "0", 1); + int quality_zero = cli_index_grade(below_quality, CBM_CLI_EXIT_OK); + + cbm_setenv("CBM_GATE", "", 1); + int quality_empty = cli_index_grade(below_quality, CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + + ASSERT_EQ(quality_on, CBM_CLI_EXIT_QUALITY); + ASSERT_EQ(target_on, CBM_CLI_EXIT_TARGET); + ASSERT_EQ(quality_unset, CBM_CLI_EXIT_OK); + ASSERT_EQ(target_unset, CBM_CLI_EXIT_FAILURE); + ASSERT_EQ(quality_zero, CBM_CLI_EXIT_OK); + ASSERT_EQ(quality_empty, CBM_CLI_EXIT_OK); + PASS(); +} + +/* A clean index keeps the code it always had: the contract adds verdicts, + * it does not make previously good runs start failing. */ +TEST(cli_index_exit_clean_run_stays_zero) { + cli_gate_env_t saved = cli_gate_env_clear(); + int status = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":4," + "\"parse_partial_count\":0,\"parse_unusable_count\":0}", + CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + ASSERT_EQ(status, CBM_CLI_EXIT_OK); + PASS(); +} + +/* A file that did not parse at all gets no tolerance — the default is zero, + * so one such file is already a quality failure. */ +TEST(cli_index_exit_unusable_file_is_a_quality_failure) { + cli_gate_env_t saved = cli_gate_env_clear(); + int status = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":5," + "\"parse_partial_count\":0,\"parse_unusable_count\":1}", + CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + ASSERT_EQ(status, CBM_CLI_EXIT_QUALITY); + PASS(); +} + +/* Partial parsing is graded as a share, not a count: 20 of 100 files is + * twice the default ceiling. */ +TEST(cli_index_exit_partial_above_threshold_is_a_quality_failure) { + cli_gate_env_t saved = cli_gate_env_clear(); + int status = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":100," + "\"parse_partial_count\":20,\"parse_unusable_count\":0}", + CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + ASSERT_EQ(status, CBM_CLI_EXIT_QUALITY); + PASS(); +} + +/* The boundary itself passes. The comparison is strict and integral on + * purpose: at exactly the documented ceiling a double could round either + * way, and a threshold whose verdict depends on rounding is not a + * threshold. 10 of 100 is the ceiling, and the ceiling is allowed. */ +TEST(cli_index_exit_partial_exactly_at_threshold_passes) { + cli_gate_env_t saved = cli_gate_env_clear(); + int at_ceiling = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":100," + "\"parse_partial_count\":10,\"parse_unusable_count\":0}", + CBM_CLI_EXIT_OK); + /* One file more is over it, which proves the case above is the boundary + * and not simply a check that never fires. */ + int over_ceiling = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":100," + "\"parse_partial_count\":11,\"parse_unusable_count\":0}", + CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + ASSERT_EQ(at_ceiling, CBM_CLI_EXIT_OK); + ASSERT_EQ(over_ceiling, CBM_CLI_EXIT_QUALITY); + PASS(); +} + +/* "degraded" is the pipeline's own verdict that the graph came out far + * smaller than the run expected. It carries no parse counts, so it has to + * be graded on the status alone. */ +TEST(cli_index_exit_degraded_status_is_a_quality_failure) { + cli_gate_env_t saved = cli_gate_env_clear(); + int status = cli_index_grade("{\"status\":\"degraded\",\"files_indexed\":40," + "\"parse_partial_count\":0,\"parse_unusable_count\":0}", + CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + ASSERT_EQ(status, CBM_CLI_EXIT_QUALITY); + PASS(); +} + +/* An absent target and a pipeline that fell over inside a reachable + * repository used to share code 1, which is the whole reason the contract + * was written: the caller could not tell "wrong path" from "broken run". */ +TEST(cli_index_exit_separates_missing_target_from_broken_run) { + cli_gate_env_t saved = cli_gate_env_clear(); + int target = cli_index_grade("{\"status\":\"error\",\"reason\":\"target_unavailable\"}", + CBM_CLI_EXIT_FAILURE); + int pipeline = cli_index_grade("{\"status\":\"error\",\"reason\":\"pipeline_failed\"}", + CBM_CLI_EXIT_FAILURE); + /* An error the mapping had not already flagged still has to land on a + * failure code rather than fall through as success. */ + int unflagged = cli_index_grade("{\"status\":\"error\",\"reason\":\"pipeline_failed\"}", + CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + ASSERT_EQ(target, CBM_CLI_EXIT_TARGET); + ASSERT_EQ(pipeline, CBM_CLI_EXIT_FAILURE); + ASSERT_EQ(unflagged, CBM_CLI_EXIT_FAILURE); + PASS(); +} + +/* Silence is never upgraded into a verdict. A payload the grader cannot + * read says nothing about index quality, and inventing a 2 from it would + * fail runs for the crime of an unexpected response shape. */ +TEST(cli_index_exit_never_upgrades_silence) { + cli_gate_env_t saved = cli_gate_env_clear(); + ASSERT_EQ(cbm_cli_index_exit_status(NULL, CBM_CLI_EXIT_OK), CBM_CLI_EXIT_OK); + ASSERT_EQ(cbm_cli_index_exit_status("", CBM_CLI_EXIT_OK), CBM_CLI_EXIT_OK); + ASSERT_EQ(cbm_cli_index_exit_status("not json at all", CBM_CLI_EXIT_OK), CBM_CLI_EXIT_OK); + /* A well-formed envelope carrying no content, and one whose text is not + * itself JSON — both are shapes a future response could take. */ + ASSERT_EQ(cbm_cli_index_exit_status("{\"content\":[]}", CBM_CLI_EXIT_OK), CBM_CLI_EXIT_OK); + ASSERT_EQ(cli_index_grade("plain text, not a payload", CBM_CLI_EXIT_OK), CBM_CLI_EXIT_OK); + /* And the same shapes must not erase a failure already established. */ + ASSERT_EQ(cbm_cli_index_exit_status(NULL, CBM_CLI_EXIT_FAILURE), CBM_CLI_EXIT_FAILURE); + cli_gate_env_restore(saved); + PASS(); +} + +/* Grading only ever makes a code more specific. A base failure survives a + * payload that looks perfectly healthy, because the transport already knew + * something the payload does not say. */ +TEST(cli_index_exit_does_not_downgrade_a_failing_base) { + cli_gate_env_t saved = cli_gate_env_clear(); + int status = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":9," + "\"parse_partial_count\":0,\"parse_unusable_count\":0}", + CBM_CLI_EXIT_FAILURE); + cli_gate_env_restore(saved); + ASSERT_EQ(status, CBM_CLI_EXIT_FAILURE); + PASS(); +} + +/* Both thresholds are overridable, and both are read at grading time. */ +TEST(cli_index_exit_thresholds_read_the_environment) { + cli_gate_env_t saved = cli_gate_env_clear(); + const char *twenty_percent = "{\"status\":\"ok\",\"files_indexed\":100," + "\"parse_partial_count\":20,\"parse_unusable_count\":0}"; + const char *three_unusable = "{\"status\":\"ok\",\"files_indexed\":50," + "\"parse_partial_count\":0,\"parse_unusable_count\":3}"; + + /* Positive control: at the defaults both payloads fail, so a pass below + * is the override working and not the check being absent. */ + int partial_default = cli_index_grade(twenty_percent, CBM_CLI_EXIT_OK); + int unusable_default = cli_index_grade(three_unusable, CBM_CLI_EXIT_OK); + + cbm_setenv("CBM_GATE_MAX_PARTIAL_PCT", "50", 1); + cbm_setenv("CBM_GATE_MAX_UNUSABLE", "5", 1); + int partial_raised = cli_index_grade(twenty_percent, CBM_CLI_EXIT_OK); + int unusable_raised = cli_index_grade(three_unusable, CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + + ASSERT_EQ(partial_default, CBM_CLI_EXIT_QUALITY); + ASSERT_EQ(unusable_default, CBM_CLI_EXIT_QUALITY); + ASSERT_EQ(partial_raised, CBM_CLI_EXIT_OK); + ASSERT_EQ(unusable_raised, CBM_CLI_EXIT_OK); + PASS(); +} + +/* A negative ceiling switches its own check off. This is the documented + * escape hatch for a repository whose grammars are known to be thin. */ +TEST(cli_index_exit_negative_threshold_disables_the_check) { + cli_gate_env_t saved = cli_gate_env_clear(); + cbm_setenv("CBM_GATE_MAX_PARTIAL_PCT", "-1", 1); + cbm_setenv("CBM_GATE_MAX_UNUSABLE", "-1", 1); + int status = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":100," + "\"parse_partial_count\":99,\"parse_unusable_count\":7}", + CBM_CLI_EXIT_OK); + /* Disabling the parse checks must not disable the pipeline's own + * verdict: "degraded" is not a threshold and has no off switch. */ + int degraded = cli_index_grade("{\"status\":\"degraded\",\"files_indexed\":10," + "\"parse_partial_count\":0,\"parse_unusable_count\":0}", + CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + ASSERT_EQ(status, CBM_CLI_EXIT_OK); + ASSERT_EQ(degraded, CBM_CLI_EXIT_QUALITY); + PASS(); +} + +/* A typo must not silently disable a gate. Anything that is not a whole + * number falls back to the documented default, so "1O" (letter O) fails the + * run it would have failed anyway instead of quietly waving it through. */ +TEST(cli_index_exit_unreadable_threshold_falls_back_to_default) { + cli_gate_env_t saved = cli_gate_env_clear(); + const char *twenty_percent = "{\"status\":\"ok\",\"files_indexed\":100," + "\"parse_partial_count\":20,\"parse_unusable_count\":0}"; + /* The last one overflows long: strtol clamps it to LONG_MAX with the + * whole string consumed, a ceiling no run could exceed. */ + const char *unreadable[] = { + "abc", "1O", "10pct", " 10", "10 ", "", "1e1", "10.0", "99999999999999999999999"}; + for (size_t i = 0; i < sizeof(unreadable) / sizeof(unreadable[0]); i++) { + cbm_setenv("CBM_GATE_MAX_PARTIAL_PCT", unreadable[i], 1); + int status = cli_index_grade(twenty_percent, CBM_CLI_EXIT_OK); + if (status != CBM_CLI_EXIT_QUALITY) { + printf(" unreadable threshold \"%s\" gave exit %d\n", unreadable[i], status); + } + ASSERT_EQ(status, CBM_CLI_EXIT_QUALITY); + } + /* A value that DOES read still takes effect, so the loop above is about + * unreadable text and not about the override being ignored outright. */ + cbm_setenv("CBM_GATE_MAX_PARTIAL_PCT", "50", 1); + int readable = cli_index_grade(twenty_percent, CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + ASSERT_EQ(readable, CBM_CLI_EXIT_OK); + PASS(); +} + +/* A share needs a denominator. Without files_indexed the percentage cannot + * be computed at all, and a run must not be failed on a number nobody + * could work out. */ +TEST(cli_index_exit_partial_without_denominator_is_not_graded) { + cli_gate_env_t saved = cli_gate_env_clear(); + int no_denominator = cli_index_grade("{\"status\":\"ok\",\"parse_partial_count\":20}", + CBM_CLI_EXIT_OK); + int zero_denominator = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":0," + "\"parse_partial_count\":20}", + CBM_CLI_EXIT_OK); + /* An unusable file is an absolute count and still grades without one. */ + int unusable_still_graded = cli_index_grade("{\"status\":\"ok\",\"parse_unusable_count\":1}", + CBM_CLI_EXIT_OK); + cli_gate_env_restore(saved); + ASSERT_EQ(no_denominator, CBM_CLI_EXIT_OK); + ASSERT_EQ(zero_denominator, CBM_CLI_EXIT_OK); + ASSERT_EQ(unusable_still_graded, CBM_CLI_EXIT_QUALITY); + PASS(); +} + SUITE(cli) { if (!th_secure_runtime_parent_new(g_cli_suite_runtime_parent, sizeof(g_cli_suite_runtime_parent), "cli-suite")) { @@ -15304,6 +15617,21 @@ SUITE(cli) { /* Stdin argument gate (#1359) */ RUN_TEST(cli_zero_argument_tool_never_reads_stdin_issue1359); RUN_TEST(cli_stdin_args_gate_tracks_tool_schema_issue1359); + + /* index_repository exit contract */ + RUN_TEST(cli_index_exit_gate_off_keeps_base_status); + RUN_TEST(cli_index_exit_clean_run_stays_zero); + RUN_TEST(cli_index_exit_unusable_file_is_a_quality_failure); + RUN_TEST(cli_index_exit_partial_above_threshold_is_a_quality_failure); + RUN_TEST(cli_index_exit_partial_exactly_at_threshold_passes); + RUN_TEST(cli_index_exit_degraded_status_is_a_quality_failure); + RUN_TEST(cli_index_exit_separates_missing_target_from_broken_run); + RUN_TEST(cli_index_exit_never_upgrades_silence); + RUN_TEST(cli_index_exit_does_not_downgrade_a_failing_base); + RUN_TEST(cli_index_exit_thresholds_read_the_environment); + RUN_TEST(cli_index_exit_negative_threshold_disables_the_check); + RUN_TEST(cli_index_exit_unreadable_threshold_falls_back_to_default); + RUN_TEST(cli_index_exit_partial_without_denominator_is_not_graded); cbm_cli_set_activation_runtime_parent_for_test(NULL); test_rmdir_r(g_cli_suite_runtime_parent); g_cli_suite_runtime_parent[0] = '\0'; From 80f8bda5670c8ac1956e2a8fac7221c40c12da47 Mon Sep 17 00:00:00 2001 From: Hubert Tarnacki Date: Wed, 9 Sep 2026 22:44:19 +0200 Subject: [PATCH 022/217] fix(discover): retain external source directories in fast mode `external` was listed in FAST_SKIP_DIRS, so fast and moderate discovery silently dropped every directory named `external`. In many codebases that name holds first-party code (external-facing APIs, adapters, integration layers) rather than vendored third-party sources, and the dedicated `vendor` / `third_party` / `3rdparty` entries already cover the vendored case. Remove `external` from the fast skip list so those sources are indexed in every mode. The shortened list is regrouped with category comments in the same style as ALWAYS_SKIP_DIRS; element order is unchanged. Signed-off-by: Hubert Tarnacki --- src/discover/discover.c | 24 +++++++++++++++++------- tests/test_discover.c | 7 +++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/discover/discover.c b/src/discover/discover.c index 5cadaed75..e42efa117 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -52,13 +52,23 @@ static const char *ALWAYS_SKIP_DIRS[] = { ".codebase-memory", ".qdrant_code_embeddings", ".tmp", "vendor", "vendored", NULL}; static const char *FAST_SKIP_DIRS[] = { - "generated", "gen", "auto-generated", "fixtures", "testdata", "test_data", - "__tests__", "__mocks__", "__snapshots__", "__fixtures__", "__test__", "docs", - "doc", "documentation", "examples", "example", "samples", "sample", - "assets", "static", "public", "media", "third_party", "thirdparty", - "3rdparty", "external", "migrations", "seeds", "e2e", "integration", - "locale", "locales", "i18n", "l10n", "scripts", "tools", - "hack", "bin", "build", "out", NULL}; + /* Generated */ + "generated", "gen", "auto-generated", + /* Tests */ + "fixtures", "testdata", "test_data", "__tests__", "__mocks__", "__snapshots__", "__fixtures__", + "__test__", + /* Docs and samples */ + "docs", "doc", "documentation", "examples", "example", "samples", "sample", + /* Assets */ + "assets", "static", "public", "media", + /* Third-party */ + "third_party", "thirdparty", "3rdparty", + /* Data and integration tests */ + "migrations", "seeds", "e2e", "integration", + /* Localization */ + "locale", "locales", "i18n", "l10n", + /* Tooling and build output */ + "scripts", "tools", "hack", "bin", "build", "out", NULL}; /* ── Ignored suffixes ───────────────────────────────── */ diff --git a/tests/test_discover.c b/tests/test_discover.c index 0f3dac7d2..d44e82bf3 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -153,6 +153,12 @@ TEST(skip_fast_e2e) { ASSERT_TRUE(cbm_should_skip_dir("e2e", CBM_MODE_FAST)); PASS(); } +TEST(no_skip_fast_external) { + ASSERT_FALSE(cbm_should_skip_dir("external", CBM_MODE_FAST)); + ASSERT_FALSE(cbm_should_skip_dir("external", CBM_MODE_MODERATE)); + ASSERT_FALSE(cbm_should_skip_dir("shop-external-api", CBM_MODE_FAST)); + PASS(); +} /* ── Suffix filters ────────────────────────────────────────────── */ @@ -1867,6 +1873,7 @@ SUITE(discover) { RUN_TEST(skip_fast_assets); RUN_TEST(skip_fast_3rdparty); RUN_TEST(skip_fast_e2e); + RUN_TEST(no_skip_fast_external); /* Suffix filters */ RUN_TEST(suffix_pyc); From 53d8bb406b63e8a3354ffa36e89856ec44b7cdf6 Mon Sep 17 00:00:00 2001 From: Benyamin Date: Fri, 21 Aug 2026 14:20:35 +0330 Subject: [PATCH 023/217] fix(codex): migrate skill to shared agents root Signed-off-by: Benyamin --- README.md | 7 +- scripts/smoke-test.sh | 102 +++++++++++++ src/cli/cli.c | 331 ++++++++++++++++++++++++++++++++++++++++- tests/test_cli.c | 337 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 761 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index c6ab67ee2..260f7c4d6 100644 --- a/README.md +++ b/README.md @@ -491,7 +491,7 @@ overwrite user-modified agents. | Agent | Activation | MCP config | Durable context / augmentation | |-------|------------|------------|--------------------------------| | Claude Code | Detected | `~/.claude.json` | Skill + three exact-tool graph agents; `SessionStart`, `SubagentStart`, non-blocking `PreToolUse` for `Grep`/`Glob`/`Bash`, and post-`Read` coverage | -| Codex CLI | Detected | `$CODEX_HOME/config.toml` | Managed `AGENTS.md` activation pointer, skill, three read-only agents; `SessionStart` + `SubagentStart` | +| Codex CLI | Detected | `$CODEX_HOME/config.toml` | Managed `AGENTS.md` activation pointer, `~/.agents/skills/codebase-memory/SKILL.md`, three read-only agents; `SessionStart` + `SubagentStart` | | Gemini CLI | Detected | `.gemini/settings.json` | `GEMINI.md`, three explicit read/graph-tool subagents; `BeforeTool`, `AfterTool` `read_file` coverage, and `SessionStart` | | Zed | Detected | platform `settings.json` (JSONC) | `AGENTS.md` + shared skill | | OpenCode | Detected | `$OPENCODE_CONFIG` or resolved global config | `AGENTS.md`, skill, three deny-by-default read-only agents; plugin adds grep/glob graph lookup, post-`read` coverage, first-tool-result session context, and post-compaction reinjection | @@ -540,6 +540,11 @@ For Codex, install keeps only a tiny managed activation pointer in global `$CODEX_HOME/AGENTS.md`; all detailed behavior lives in the installed `codebase-memory` skill. Fresh installs create the pointer, upgrades replace the legacy full managed block while preserving all user-owned bytes, and uninstall removes only the managed pointer. +Skill placement follows the cross-client user skill root and does not follow +`CODEX_HOME`. On upgrade, the installer migrates an installer-owned legacy +`$CODEX_HOME/skills/codebase-memory/SKILL.md` only after verifying the canonical +copy. Modified or foreign files are preserved; `--force` explicitly authorizes +replacement before migration. ### Sessions, compaction, and subagents diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index ca27edcc6..620e73c66 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -1313,6 +1313,108 @@ fi echo "" echo "=== Phase 8: agent config install E2E ===" +# Keep Codex skill placement independent from the broad multi-client fixture: +# Warp and OpenHands share the canonical root and could otherwise mask a Codex +# regression by creating the expected file themselves. +CODEX_ONLY_HOME=$(smoke_mktemp_dir) +CODEX_ONLY_CONFIG="$CODEX_ONLY_HOME/vendor-codex" +CODEX_ONLY_INSTALL="$CODEX_ONLY_HOME/bin" +CODEX_ONLY_SKILL="$CODEX_ONLY_HOME/.agents/skills/codebase-memory/SKILL.md" +CODEX_ONLY_LEGACY="$CODEX_ONLY_CONFIG/skills/codebase-memory/SKILL.md" +mkdir -p "$CODEX_ONLY_CONFIG" "$CODEX_ONLY_INSTALL" +if [[ "$BINARY" == *.exe ]]; then + CODEX_ONLY_BIN="$CODEX_ONLY_INSTALL/codebase-memory-mcp.exe" +else + CODEX_ONLY_BIN="$CODEX_ONLY_INSTALL/codebase-memory-mcp" +fi +CODEX_ONLY_LOG=$(smoke_mktemp_file) +run_codex_only() { + local executable="$1" + shift + HOME="$CODEX_ONLY_HOME" \ + XDG_CONFIG_HOME="$CODEX_ONLY_HOME/.config" \ + APPDATA="$CODEX_ONLY_HOME/AppData/Roaming" \ + LOCALAPPDATA="$CODEX_ONLY_HOME/AppData/Local" \ + CODEX_HOME="$CODEX_ONLY_CONFIG" \ + "$executable" "$@" > "$CODEX_ONLY_LOG" 2>&1 +} +if ! run_codex_only "$BINARY" install -y --clients=codex --dir="$CODEX_ONLY_INSTALL"; then + echo "FAIL 8-codex-1: isolated Codex install failed" + cat "$CODEX_ONLY_LOG" + exit 1 +fi +if ! grep -q '^name: codebase-memory$' "$CODEX_ONLY_SKILL" 2>/dev/null || + ! grep -q 'trace_path' "$CODEX_ONLY_SKILL" 2>/dev/null || + [ -e "$CODEX_ONLY_LEGACY" ]; then + echo "FAIL 8-codex-1: Codex skill was not installed exclusively at the canonical path" + cat "$CODEX_ONLY_LOG" + exit 1 +fi + +CODEX_ONLY_HASH=$(smoke_file_sha256 "$CODEX_ONLY_SKILL") +if ! run_codex_only "$CODEX_ONLY_BIN" install --plan --clients=codex \ + --dir="$CODEX_ONLY_INSTALL" || + [ "$(smoke_file_sha256 "$CODEX_ONLY_SKILL")" != "$CODEX_ONLY_HASH" ]; then + echo "FAIL 8-codex-2: Codex install plan mutated the canonical skill" + cat "$CODEX_ONLY_LOG" + exit 1 +fi +if ! run_codex_only "$CODEX_ONLY_BIN" install --dry-run -y --clients=codex \ + --dir="$CODEX_ONLY_INSTALL" || + [ "$(smoke_file_sha256 "$CODEX_ONLY_SKILL")" != "$CODEX_ONLY_HASH" ]; then + echo "FAIL 8-codex-2: Codex dry-run mutated the canonical skill" + cat "$CODEX_ONLY_LOG" + exit 1 +fi + +mkdir -p "$(dirname "$CODEX_ONLY_LEGACY")" +mv "$CODEX_ONLY_SKILL" "$CODEX_ONLY_LEGACY" +if ! run_codex_only "$CODEX_ONLY_BIN" install -y --clients=codex \ + --dir="$CODEX_ONLY_INSTALL"; then + echo "FAIL 8-codex-3: isolated Codex legacy migration failed" + cat "$CODEX_ONLY_LOG" + exit 1 +fi +if ! grep -q '^name: codebase-memory$' "$CODEX_ONLY_SKILL" 2>/dev/null || + [ -e "$CODEX_ONLY_LEGACY" ]; then + echo "FAIL 8-codex-3: Codex legacy skill was not migrated to the canonical path" + cat "$CODEX_ONLY_LOG" + exit 1 +fi + +rm -f "$CODEX_ONLY_SKILL" +mkdir -p "$(dirname "$CODEX_ONLY_LEGACY")" +printf '%s\n' '---' 'name: codebase-memory' '---' 'User-owned legacy skill.' \ + > "$CODEX_ONLY_LEGACY" +CODEX_ONLY_FOREIGN_HASH=$(smoke_file_sha256 "$CODEX_ONLY_LEGACY") +if ! run_codex_only "$CODEX_ONLY_BIN" install -y --clients=codex \ + --dir="$CODEX_ONLY_INSTALL" || + [ -e "$CODEX_ONLY_SKILL" ] || + [ "$(smoke_file_sha256 "$CODEX_ONLY_LEGACY")" != "$CODEX_ONLY_FOREIGN_HASH" ]; then + echo "FAIL 8-codex-4: non-force install did not preserve a foreign legacy skill" + cat "$CODEX_ONLY_LOG" + exit 1 +fi +if ! run_codex_only "$CODEX_ONLY_BIN" install --force -y --clients=codex \ + --dir="$CODEX_ONLY_INSTALL" || + ! grep -q '^name: codebase-memory$' "$CODEX_ONLY_SKILL" 2>/dev/null || + [ -e "$CODEX_ONLY_LEGACY" ]; then + echo "FAIL 8-codex-5: force did not converge a foreign legacy skill" + cat "$CODEX_ONLY_LOG" + exit 1 +fi +CODEX_ONLY_HASH=$(smoke_file_sha256 "$CODEX_ONLY_SKILL") +if ! run_codex_only "$CODEX_ONLY_BIN" install -y --clients=codex \ + --dir="$CODEX_ONLY_INSTALL" || + [ "$(smoke_file_sha256 "$CODEX_ONLY_SKILL")" != "$CODEX_ONLY_HASH" ]; then + echo "FAIL 8-codex-6: repeated Codex install was not idempotent" + cat "$CODEX_ONLY_LOG" + exit 1 +fi +echo "OK: Codex skill placement, previews, migration, force, and idempotence" +smoke_rmtree "$CODEX_ONLY_HOME" +rm -f "$CODEX_ONLY_LOG" + # Set up an isolated HOME. Directory-only agents get only the root required for # detection; CLI-detected agents use stubs below so install must create their # config parents from scratch. diff --git a/src/cli/cli.c b/src/cli/cli.c index 51fbafc02..8401179ab 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -8112,7 +8112,7 @@ static void print_detected_agents(const cbm_detected_agents_t *a, const char *ho * behavior (it is the same code path with mutations disabled). */ typedef struct { char agent[CLI_BUF_32]; - char kind[CLI_BUF_32]; /* mcp_config | instructions | skills | hook */ + char kind[CLI_BUF_32]; /* mcp_config | instructions | skill | cleanup_skill | hook */ char path[CLI_BUF_1K]; } cbm_plan_entry_t; @@ -8125,6 +8125,7 @@ typedef struct { static cbm_install_plan_t *g_install_plan = NULL; static int g_agent_install_errors = 0; static int g_agent_uninstall_errors = 0; +static char g_blocked_shared_skill_dir[CLI_BUF_1K]; /* Every agent configuration uninstall could not clean, kept for the closing * summary. A cleanup failure no longer stops executable and index removal @@ -8610,6 +8611,14 @@ static void install_agent_skill(const char *label, const char *skills_dir, bool if (written < 0 || (size_t)written >= sizeof(skill_path)) { return; } + if (g_blocked_shared_skill_dir[0] && + (cbm_json_mcp_paths_equal(skills_dir, g_blocked_shared_skill_dir) || + cbm_same_file(skills_dir, g_blocked_shared_skill_dir))) { + if (!g_install_plan) { + printf(" skill: %s (skipped to avoid a duplicate Codex skill)\n", skill_path); + } + return; + } if (g_install_plan) { plan_record(label, "skill", skill_path); return; @@ -8618,6 +8627,272 @@ static void install_agent_skill(const char *label, const char *skills_dir, bool printf(" skill: %s (%d installed)\n", skill_path, installed); } +typedef enum { + CBM_CODEX_SKILL_ABSENT, + CBM_CODEX_SKILL_MANAGED, + CBM_CODEX_SKILL_UNOWNED, + CBM_CODEX_SKILL_UNSAFE, +} cbm_codex_skill_state_t; + +typedef struct { + char canonical_dir[CLI_BUF_1K]; + char legacy_dir[CLI_BUF_1K]; + char canonical_file[CLI_BUF_1K]; + char legacy_file[CLI_BUF_1K]; + bool same_dir; +} cbm_codex_skill_paths_t; + +static bool cbm_codex_skill_paths_init(const char *home, const char *config_dir, + cbm_codex_skill_paths_t *paths) { + if (!home || !home[0] || !config_dir || !config_dir[0] || !paths) { + return false; + } + memset(paths, 0, sizeof(*paths)); + int canonical_dir = + snprintf(paths->canonical_dir, sizeof(paths->canonical_dir), "%s/.agents/skills", home); + int legacy_dir = + snprintf(paths->legacy_dir, sizeof(paths->legacy_dir), "%s/skills", config_dir); + int canonical_file = snprintf(paths->canonical_file, sizeof(paths->canonical_file), + "%s/codebase-memory/SKILL.md", paths->canonical_dir); + int legacy_file = snprintf(paths->legacy_file, sizeof(paths->legacy_file), + "%s/codebase-memory/SKILL.md", paths->legacy_dir); + if (canonical_dir < 0 || (size_t)canonical_dir >= sizeof(paths->canonical_dir) || + legacy_dir < 0 || (size_t)legacy_dir >= sizeof(paths->legacy_dir) || canonical_file < 0 || + (size_t)canonical_file >= sizeof(paths->canonical_file) || legacy_file < 0 || + (size_t)legacy_file >= sizeof(paths->legacy_file)) { + return false; + } + paths->same_dir = cbm_json_mcp_paths_equal(paths->canonical_dir, paths->legacy_dir) || + cbm_same_file(paths->canonical_dir, paths->legacy_dir); + return true; +} + +static cbm_codex_skill_state_t cbm_codex_skill_state(const char *skills_dir, + const char *skill_file) { + char skill_dir[CLI_BUF_1K]; + int written = snprintf(skill_dir, sizeof(skill_dir), "%s/codebase-memory", skills_dir); + if (written < 0 || (size_t)written >= sizeof(skill_dir)) { + return CBM_CODEX_SKILL_UNSAFE; + } + + struct stat state; + errno = 0; +#ifndef _WIN32 + int result = lstat(skill_dir, &state); +#else + int result = stat(skill_dir, &state); +#endif + if (result != 0) { + return errno == ENOENT ? CBM_CODEX_SKILL_ABSENT : CBM_CODEX_SKILL_UNSAFE; + } + if (!S_ISDIR(state.st_mode)) { + return CBM_CODEX_SKILL_UNSAFE; + } + + errno = 0; +#ifndef _WIN32 + result = lstat(skill_file, &state); +#else + result = stat(skill_file, &state); +#endif + if (result != 0) { + return errno == ENOENT ? CBM_CODEX_SKILL_ABSENT : CBM_CODEX_SKILL_UNSAFE; + } + if (!S_ISREG(state.st_mode)) { + return CBM_CODEX_SKILL_UNSAFE; + } + + int ownership = cbm_text_owned_document_status(skill_file, skill_content, NULL, 0); + if (ownership == CLI_OK) { + return CBM_CODEX_SKILL_MANAGED; + } + return ownership == CLI_TRUE ? CBM_CODEX_SKILL_UNOWNED : CBM_CODEX_SKILL_UNSAFE; +} + +static void cbm_warn_codex_skill_conflict(const cbm_codex_skill_paths_t *paths, + const char *reason) { + (void)fprintf(stderr, + "warning: Codex skill migration skipped: %s; documented_location=%s legacy=%s\n", + reason, paths->canonical_file, paths->legacy_file); +} + +static void cbm_block_shared_skill_install(const cbm_codex_skill_paths_t *paths) { + (void)snprintf(g_blocked_shared_skill_dir, sizeof(g_blocked_shared_skill_dir), "%s", + paths->canonical_dir); +} + +static void cbm_block_shared_skill_if_codex_conflicts(const char *home, const char *config_dir) { + cbm_codex_skill_paths_t paths; + if (!cbm_codex_skill_paths_init(home, config_dir, &paths) || paths.same_dir) { + return; + } + + cbm_codex_skill_state_t canonical = + cbm_codex_skill_state(paths.canonical_dir, paths.canonical_file); + cbm_codex_skill_state_t legacy = cbm_codex_skill_state(paths.legacy_dir, paths.legacy_file); + if (canonical == CBM_CODEX_SKILL_ABSENT && legacy != CBM_CODEX_SKILL_ABSENT) { + cbm_block_shared_skill_install(&paths); + } +} + +static bool cbm_remove_codex_managed_skill(const char *skills_dir, const char *skill_file, + bool dry_run, bool uninstalling, const char *operation) { + cbm_codex_skill_state_t state = cbm_codex_skill_state(skills_dir, skill_file); + if (state == CBM_CODEX_SKILL_ABSENT) { + return true; + } + if (state != CBM_CODEX_SKILL_MANAGED) { + if (state == CBM_CODEX_SKILL_UNOWNED) { + printf(" preserved non-owned Codex skill: %s\n", skill_file); + } else if (dry_run) { + printf(" preserved unsafe Codex skill path: %s\n", skill_file); + } else if (!dry_run) { + record_agent_config_error(uninstalling, "Codex CLI", operation, skill_file); + } + return state == CBM_CODEX_SKILL_UNOWNED; + } + if (dry_run) { + printf(" Codex skill: %s (would remove)\n", skill_file); + return true; + } + int removed = cbm_remove_skills(skills_dir, false); + if (removed == CBM_SKILL_COUNT && + cbm_codex_skill_state(skills_dir, skill_file) == CBM_CODEX_SKILL_ABSENT) { + return true; + } + record_agent_config_error(uninstalling, "Codex CLI", operation, skill_file); + return false; +} + +static void install_codex_skill(const char *home, const char *config_dir, bool force, + bool dry_run) { + cbm_codex_skill_paths_t paths; + if (!cbm_codex_skill_paths_init(home, config_dir, &paths)) { + if (!g_install_plan) { + record_agent_config_error(false, "Codex CLI", "skill_path", config_dir); + } + return; + } + + cbm_codex_skill_state_t canonical = + cbm_codex_skill_state(paths.canonical_dir, paths.canonical_file); + cbm_codex_skill_state_t legacy = + paths.same_dir ? canonical : cbm_codex_skill_state(paths.legacy_dir, paths.legacy_file); + + if (g_install_plan) { + if (canonical == CBM_CODEX_SKILL_ABSENT && + (paths.same_dir || legacy == CBM_CODEX_SKILL_ABSENT || + legacy == CBM_CODEX_SKILL_MANAGED)) { + plan_record("Codex CLI", "skill", paths.canonical_file); + } + if (!paths.same_dir && legacy == CBM_CODEX_SKILL_MANAGED && + (canonical == CBM_CODEX_SKILL_ABSENT || canonical == CBM_CODEX_SKILL_MANAGED)) { + plan_record("Codex CLI", "cleanup_skill", paths.legacy_file); + } + if (canonical == CBM_CODEX_SKILL_ABSENT && + (legacy == CBM_CODEX_SKILL_UNOWNED || legacy == CBM_CODEX_SKILL_UNSAFE)) { + cbm_block_shared_skill_install(&paths); + } + return; + } + + if (paths.same_dir) { + if (canonical == CBM_CODEX_SKILL_UNSAFE) { + record_agent_config_error(false, "Codex CLI", "skill_install", paths.canonical_file); + return; + } + if (!force && canonical == CBM_CODEX_SKILL_UNOWNED) { + printf(" preserved non-owned Codex skill: %s\n", paths.canonical_file); + return; + } + install_agent_skill("Codex CLI", paths.canonical_dir, force, dry_run); + if (!dry_run && cbm_codex_skill_state(paths.canonical_dir, paths.canonical_file) != + CBM_CODEX_SKILL_MANAGED) { + record_agent_config_error(false, "Codex CLI", "skill_install", paths.canonical_file); + } + return; + } + + if (canonical == CBM_CODEX_SKILL_UNSAFE) { + if (dry_run) { + cbm_warn_codex_skill_conflict(&paths, "the canonical skill path is unsafe"); + } else { + record_agent_config_error(false, "Codex CLI", "skill_install", paths.canonical_file); + } + return; + } + if (legacy == CBM_CODEX_SKILL_UNSAFE) { + if (canonical == CBM_CODEX_SKILL_ABSENT) { + cbm_block_shared_skill_install(&paths); + } + if (force && !dry_run) { + record_agent_config_error(false, "Codex CLI", "legacy_skill_migration", + paths.legacy_file); + } else { + cbm_warn_codex_skill_conflict(&paths, "the legacy skill path is unsafe"); + } + return; + } + + if (!force && legacy == CBM_CODEX_SKILL_UNOWNED) { + if (canonical == CBM_CODEX_SKILL_ABSENT) { + cbm_block_shared_skill_install(&paths); + } + cbm_warn_codex_skill_conflict(&paths, "the legacy skill is not installer-owned"); + return; + } + if (!force && canonical == CBM_CODEX_SKILL_UNOWNED) { + if (legacy == CBM_CODEX_SKILL_MANAGED) { + cbm_warn_codex_skill_conflict(&paths, "the canonical skill is not installer-owned"); + } else { + printf(" preserved non-owned Codex skill: %s\n", paths.canonical_file); + } + return; + } + + bool remove_legacy = legacy == CBM_CODEX_SKILL_MANAGED; + if (force && legacy == CBM_CODEX_SKILL_UNOWNED) { + remove_legacy = true; + if (dry_run) { + printf(" legacy Codex skill: %s (would replace for migration)\n", paths.legacy_file); + } else { + int replaced = cbm_install_skills(paths.legacy_dir, true, false); + printf(" legacy Codex skill: %s (%d updated for migration)\n", paths.legacy_file, + replaced); + if (cbm_codex_skill_state(paths.legacy_dir, paths.legacy_file) != + CBM_CODEX_SKILL_MANAGED) { + record_agent_config_error(false, "Codex CLI", "legacy_skill_migration", + paths.legacy_file); + return; + } + } + } + + bool install_canonical = force || canonical == CBM_CODEX_SKILL_ABSENT; + if (install_canonical) { + install_agent_skill("Codex CLI", paths.canonical_dir, force, dry_run); + } else { + printf(" skill: %s (0 installed)\n", paths.canonical_file); + } + + if (dry_run) { + if (remove_legacy) { + printf(" legacy Codex skill: %s (would remove after canonical install)\n", + paths.legacy_file); + } + return; + } + if (cbm_codex_skill_state(paths.canonical_dir, paths.canonical_file) != + CBM_CODEX_SKILL_MANAGED) { + record_agent_config_error(false, "Codex CLI", "skill_install", paths.canonical_file); + return; + } + if (remove_legacy && cbm_remove_codex_managed_skill(paths.legacy_dir, paths.legacy_file, false, + false, "legacy_skill_cleanup")) { + printf(" legacy Codex skill: %s (removed)\n", paths.legacy_file); + } +} + /* Derive tier siblings only from the exact shipped Verify basename. This keeps * vendor-specific suffixes such as .agent.md, .toml, and .json intact. */ static int cbm_tiered_profile_path(const char *verify_path, cbm_graph_tier_t tier, char *output, @@ -9502,12 +9777,10 @@ static void install_cli_agent_configs(const cbm_detected_agents_t *agents, const char config_dir[CLI_BUF_1K]; char cp[CLI_BUF_1K]; char ip[CLI_BUF_1K]; - char skills_dir[CLI_BUF_1K]; char ap[CLI_BUF_1K]; cbm_codex_config_dir(home, config_dir, sizeof(config_dir)); snprintf(cp, sizeof(cp), "%s/config.toml", config_dir); snprintf(ip, sizeof(ip), "%s/AGENTS.md", config_dir); - snprintf(skills_dir, sizeof(skills_dir), "%s/skills", config_dir); snprintf(ap, sizeof(ap), "%s/agents/codebase-memory.toml", config_dir); /* A broken hook config must not keep legacy full guidance active. */ bool pointer_installed = install_codex_activation_pointer(ip, dry_run); @@ -9528,6 +9801,7 @@ static void install_cli_agent_configs(const cbm_detected_agents_t *agents, const true, &preflight_failure) : CLI_ERR; if (preflight_result != CLI_OK) { + cbm_block_shared_skill_if_codex_conflicts(home, config_dir); if (!g_install_plan) { printf("Codex CLI:\n"); fflush(stdout); @@ -9548,7 +9822,7 @@ static void install_cli_agent_configs(const cbm_detected_agents_t *agents, const if (!g_install_plan) { report_codex_activation_pointer_install(ip, pointer_installed); } - install_agent_skill("Codex CLI", skills_dir, force, dry_run); + install_codex_skill(home, config_dir, force, dry_run); install_tiered_agent_profiles( (cbm_tiered_profile_set_t){ .label = "Codex CLI", @@ -10332,6 +10606,7 @@ static int cbm_install_agent_configs_in_scope(const char *home, const char *bina int cbm_install_agent_configs(const char *home, const char *binary_path, bool force, bool dry_run) { g_agent_install_errors = 0; + g_blocked_shared_skill_dir[0] = '\0'; cbm_detected_agents_t agents = cbm_detect_agents(home); if (g_client_selection && !cli_clients_apply_selection(g_client_selection, &agents)) { return CLI_ERR; @@ -10837,6 +11112,7 @@ static char *cbm_build_install_plan_json_options(const char *home, const char *b yyjson_mut_val *agent_files = yyjson_mut_arr(doc); yyjson_mut_val *prompt_files = yyjson_mut_arr(doc); yyjson_mut_val *hooks = yyjson_mut_arr(doc); + yyjson_mut_val *cleanup_actions = yyjson_mut_arr(doc); for (int i = 0; i < plan.count; i++) { cbm_plan_entry_t *e = &plan.items[i]; if (strcmp(e->kind, "mcp_config") == 0) { @@ -10855,6 +11131,14 @@ static char *cbm_build_install_plan_json_options(const char *home, const char *b } else if (strcmp(e->kind, "prompt") == 0) { yyjson_mut_arr_add_strcpy(doc, prompt_files, e->path); yyjson_mut_arr_add_strcpy(doc, instrs, e->path); + } else if (strcmp(e->kind, "cleanup_skill") == 0) { + yyjson_mut_val *cleanup = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, cleanup, "agent", e->agent); + yyjson_mut_obj_add_str(doc, cleanup, "kind", "skill"); + yyjson_mut_obj_add_str(doc, cleanup, "operation", + "remove_owned_legacy_copy_if_migrated"); + yyjson_mut_obj_add_strcpy(doc, cleanup, "path", e->path); + yyjson_mut_arr_add_val(cleanup_actions, cleanup); } else { yyjson_mut_arr_add_strcpy(doc, instrs, e->path); } @@ -10865,6 +11149,7 @@ static char *cbm_build_install_plan_json_options(const char *home, const char *b yyjson_mut_obj_add_val(doc, root, "agent_files_planned", agent_files); yyjson_mut_obj_add_val(doc, root, "prompt_files_planned", prompt_files); yyjson_mut_obj_add_val(doc, root, "hooks_planned", hooks); + yyjson_mut_obj_add_val(doc, root, "cleanup_actions_planned", cleanup_actions); yyjson_mut_obj_add_bool(doc, root, "writes_started", false); yyjson_mut_obj_add_bool(doc, root, "network_after_install", false); yyjson_mut_obj_add_str(doc, root, "next_safe_command", "codebase-memory-mcp install -y"); @@ -11568,6 +11853,40 @@ static void uninstall_agent_skill(const char *label, const char *skills_dir, boo printf(" %s skill: %d removed\n", label, removed); } +static void uninstall_codex_skills(const char *home, const char *config_dir, bool dry_run) { + cbm_codex_skill_paths_t paths; + if (!cbm_codex_skill_paths_init(home, config_dir, &paths)) { + if (!dry_run) { + record_agent_config_error(true, "Codex CLI", "skill_path", config_dir); + } + return; + } + + int removed = 0; + if (cbm_codex_skill_state(paths.canonical_dir, paths.canonical_file) == + CBM_CODEX_SKILL_MANAGED) { + if (cbm_remove_codex_managed_skill(paths.canonical_dir, paths.canonical_file, dry_run, true, + "skill_uninstall")) { + removed++; + } + } else { + (void)cbm_remove_codex_managed_skill(paths.canonical_dir, paths.canonical_file, dry_run, + true, "skill_uninstall"); + } + + if (!paths.same_dir && + cbm_codex_skill_state(paths.legacy_dir, paths.legacy_file) == CBM_CODEX_SKILL_MANAGED) { + if (cbm_remove_codex_managed_skill(paths.legacy_dir, paths.legacy_file, dry_run, true, + "legacy_skill_uninstall")) { + removed++; + } + } else if (!paths.same_dir) { + (void)cbm_remove_codex_managed_skill(paths.legacy_dir, paths.legacy_file, dry_run, true, + "legacy_skill_uninstall"); + } + printf(" Codex CLI skills: %d removed\n", removed); +} + static void uninstall_copilot_durable_context(const char *home, bool dry_run) { char config_dir[CLI_BUF_1K]; char hook_path[CLI_BUF_1K]; @@ -11989,13 +12308,11 @@ static void uninstall_cli_agents(const cbm_detected_agents_t *agents, const char char config_dir[CLI_BUF_1K]; char cp[CLI_BUF_1K]; char ip[CLI_BUF_1K]; - char skills_dir[CLI_BUF_1K]; char ap[CLI_BUF_1K]; char installed_binary[CLI_BUF_1K]; cbm_codex_config_dir(home, config_dir, sizeof(config_dir)); snprintf(cp, sizeof(cp), "%s/config.toml", config_dir); snprintf(ip, sizeof(ip), "%s/AGENTS.md", config_dir); - snprintf(skills_dir, sizeof(skills_dir), "%s/skills", config_dir); snprintf(ap, sizeof(ap), "%s/agents/codebase-memory.toml", config_dir); bool pointer_removed = uninstall_codex_activation_pointer(ip, dry_run); cbm_agent_installed_binary_path(home, installed_binary, sizeof(installed_binary)); @@ -12030,7 +12347,7 @@ static void uninstall_cli_agents(const cbm_detected_agents_t *agents, const char record_agent_config_error(true, "Codex CLI", "hook_uninstall", cp); } codex_toml_done: - uninstall_agent_skill("Codex CLI", skills_dir, dry_run); + uninstall_codex_skills(home, config_dir, dry_run); uninstall_tiered_agent_profiles( (cbm_tiered_profile_set_t){ .label = "Codex CLI", diff --git a/tests/test_cli.c b/tests/test_cli.c index 06ebb3841..21649fa96 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -6799,7 +6799,6 @@ TEST(cli_durable_profiles_follow_current_vendor_paths) { strstr(plan, "\"instruction_files_planned\""); const char *const planned[] = { "/.claude/agents/codebase-memory.md", - "/vendor-codex/skills/codebase-memory/SKILL.md", "/vendor-codex/agents/codebase-memory.toml", "/.cursor/skills/codebase-memory/SKILL.md", "/.cursor/agents/codebase-memory.md", @@ -6962,13 +6961,12 @@ TEST(cli_durable_profiles_follow_current_vendor_paths) { free(profile); const char *const skill_files[] = { - "/skills/codebase-memory/SKILL.md", "/.cursor/skills/codebase-memory/SKILL.md", "/.config/opencode/skills/codebase-memory/SKILL.md", "/.factory/skills/codebase-memory/SKILL.md", "/.agents/skills/codebase-memory/SKILL.md", }; - const char *const skill_roots[] = {codex_home, tmpdir, tmpdir, tmpdir, tmpdir}; + const char *const skill_roots[] = {tmpdir, tmpdir, tmpdir, tmpdir}; for (size_t i = 0U; files_ok && i < sizeof(skill_files) / sizeof(skill_files[0]); i++) { snprintf(path, sizeof(path), "%s%s", skill_roots[i], skill_files[i]); files_ok = test_file_contains_all(path, graph_terms, 3); @@ -9382,12 +9380,21 @@ TEST(cli_codex_respects_codex_home) { char *json = cbm_build_install_plan_json(tmpdir, "/usr/local/bin/codebase-memory-mcp"); char expected_config[640]; snprintf(expected_config, sizeof(expected_config), "%s/config.toml", codex_home); + char expected_skill[640]; + char legacy_skill[640]; + snprintf(expected_skill, sizeof(expected_skill), "%s/.agents/skills/codebase-memory/SKILL.md", + tmpdir); + snprintf(legacy_skill, sizeof(legacy_skill), "%s/skills/codebase-memory/SKILL.md", codex_home); yyjson_doc *plan_doc = json ? yyjson_read(json, strlen(json), 0) : NULL; yyjson_val *plan_root = plan_doc ? yyjson_doc_get_root(plan_doc) : NULL; bool plans_config = test_json_string_array_contains(plan_root, "config_files_planned", expected_config); bool plans_instructions = test_json_string_array_contains( plan_root, "instruction_files_planned", expected_instructions); + bool plans_canonical_skill = + test_json_string_array_contains(plan_root, "skill_files_planned", expected_skill); + bool plans_legacy_skill = + test_json_string_array_contains(plan_root, "skill_files_planned", legacy_skill); bool plans_cleanup = json && strstr(json, "remove_managed_block_if_present") != NULL; char *instructions_after = read_test_file_alloc(expected_instructions); bool plan_preserved_user_file = @@ -9401,9 +9408,10 @@ TEST(cli_codex_respects_codex_home) { if (!agents.codex) FAIL("Codex detection must honor CODEX_HOME"); - if (!plans_config || !plans_instructions || plans_cleanup || !plan_preserved_user_file) + if (!plans_config || !plans_instructions || !plans_canonical_skill || plans_legacy_skill || + plans_cleanup || !plan_preserved_user_file) FAIL("Codex plan must include the managed activation pointer under CODEX_HOME without " - "mutating existing user content"); + "mutating existing user content, and use HOME/.agents for the skill"); PASS(); } @@ -9529,7 +9537,6 @@ TEST(cli_codex_install_uses_global_activation_pointer_issue1689) { restore_test_env("PATH", saved_path); restore_test_env("CODEX_HOME", saved_codex); test_rmdir_r(tmpdir); - if (!fresh_pointer_installed || !other_surfaces_installed || !unowned_preserved || !managed_migrated_once || !marker_only_migrated || !malformed_preserved || !uninstall_preserved_foreign) @@ -9581,6 +9588,300 @@ TEST(cli_grok_respects_grok_home) { PASS(); } +static bool cli_test_file_equals(const char *path, const char *expected) { + char *data = read_test_file_alloc(path); + bool equal = data && strcmp(data, expected) == 0; + free(data); + return equal; +} + +TEST(cli_codex_skill_migrates_owned_legacy_to_agents_root) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-codex-skill-migrate-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + char codex_home[512]; + char legacy_root[640]; + char legacy_file[768]; + char sibling_file[768]; + char canonical_file[768]; + snprintf(codex_home, sizeof(codex_home), "%s/vendor-codex", tmpdir); + snprintf(legacy_root, sizeof(legacy_root), "%s/skills", codex_home); + snprintf(legacy_file, sizeof(legacy_file), "%s/codebase-memory/SKILL.md", legacy_root); + snprintf(sibling_file, sizeof(sibling_file), "%s/codebase-memory/notes.md", legacy_root); + snprintf(canonical_file, sizeof(canonical_file), "%s/.agents/skills/codebase-memory/SKILL.md", + tmpdir); + test_mkdirp(codex_home); + ASSERT_EQ(cbm_install_skills(legacy_root, false, false), CBM_SKILL_COUNT); + write_test_file(sibling_file, "user notes\n"); + + char *saved_home = save_test_env("HOME"); + char *saved_path = save_test_env("PATH"); + char *saved_codex = save_test_env("CODEX_HOME"); + cbm_setenv("HOME", tmpdir, 1); + cbm_setenv("PATH", tmpdir, 1); + cbm_setenv("CODEX_HOME", codex_home, 1); + + int dry_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, true); + struct stat state; + bool dry_safe = dry_rc == 0 && stat(canonical_file, &state) != 0 && + cli_test_file_equals(legacy_file, cbm_get_skills()[0].content); + char *plan = cbm_build_install_plan_json(tmpdir, "/opt/codebase-memory-mcp"); + bool plan_ok = plan && strstr(plan, canonical_file) && strstr(plan, legacy_file) && + strstr(plan, "cleanup_actions_planned") && + strstr(plan, "remove_owned_legacy_copy_if_migrated"); + bool plan_safe = stat(canonical_file, &state) != 0 && + cli_test_file_equals(legacy_file, cbm_get_skills()[0].content); + free(plan); + + int install_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + bool migrated = + install_rc == 0 && cli_test_file_equals(canonical_file, cbm_get_skills()[0].content) && + stat(legacy_file, &state) != 0 && cli_test_file_equals(sibling_file, "user notes\n"); + + restore_test_env("HOME", saved_home); + restore_test_env("PATH", saved_path); + restore_test_env("CODEX_HOME", saved_codex); + test_rmdir_r(tmpdir); + + if (!dry_safe || !plan_ok || !plan_safe || !migrated) + FAIL("Codex owned legacy skill migration must be previewable, destination-first, and " + "sibling-safe"); + PASS(); +} + +TEST(cli_codex_skill_preserves_foreign_legacy_until_force) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-codex-skill-foreign-legacy-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + const char *foreign = "---\nname: codebase-memory\n---\nUser-owned legacy skill.\n"; + char codex_home[512]; + char legacy_dir[768]; + char legacy_file[768]; + char sibling_file[768]; + char canonical_file[768]; + snprintf(codex_home, sizeof(codex_home), "%s/vendor-codex", tmpdir); + snprintf(legacy_dir, sizeof(legacy_dir), "%s/skills/codebase-memory", codex_home); + snprintf(legacy_file, sizeof(legacy_file), "%s/SKILL.md", legacy_dir); + snprintf(sibling_file, sizeof(sibling_file), "%s/user.txt", legacy_dir); + snprintf(canonical_file, sizeof(canonical_file), "%s/.agents/skills/codebase-memory/SKILL.md", + tmpdir); + test_mkdirp(legacy_dir); + write_test_file(legacy_file, foreign); + write_test_file(sibling_file, "keep\n"); + + char *saved_home = save_test_env("HOME"); + char *saved_path = save_test_env("PATH"); + char *saved_codex = save_test_env("CODEX_HOME"); + cbm_setenv("HOME", tmpdir, 1); + cbm_setenv("PATH", tmpdir, 1); + cbm_setenv("CODEX_HOME", codex_home, 1); + + int normal_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + struct stat state; + bool preserved = normal_rc == 0 && cli_test_file_equals(legacy_file, foreign) && + stat(canonical_file, &state) != 0; + int force_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", true, false); + bool forced = force_rc == 0 && + cli_test_file_equals(canonical_file, cbm_get_skills()[0].content) && + stat(legacy_file, &state) != 0 && cli_test_file_equals(sibling_file, "keep\n"); + + restore_test_env("HOME", saved_home); + restore_test_env("PATH", saved_path); + restore_test_env("CODEX_HOME", saved_codex); + test_rmdir_r(tmpdir); + if (!preserved || !forced) + FAIL("Codex must preserve foreign legacy content unless --force authorizes replacement"); + PASS(); +} + +TEST(cli_codex_skill_preserves_foreign_canonical_and_owned_legacy) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-codex-skill-foreign-canonical-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + const char *foreign = "---\nname: codebase-memory\n---\nUser-owned canonical skill.\n"; + char codex_home[512]; + char legacy_root[640]; + char legacy_file[768]; + char canonical_dir[768]; + char canonical_file[768]; + snprintf(codex_home, sizeof(codex_home), "%s/vendor-codex", tmpdir); + snprintf(legacy_root, sizeof(legacy_root), "%s/skills", codex_home); + snprintf(legacy_file, sizeof(legacy_file), "%s/codebase-memory/SKILL.md", legacy_root); + snprintf(canonical_dir, sizeof(canonical_dir), "%s/.agents/skills/codebase-memory", tmpdir); + snprintf(canonical_file, sizeof(canonical_file), "%s/SKILL.md", canonical_dir); + test_mkdirp(codex_home); + ASSERT_EQ(cbm_install_skills(legacy_root, false, false), CBM_SKILL_COUNT); + test_mkdirp(canonical_dir); + write_test_file(canonical_file, foreign); + + char *saved_home = save_test_env("HOME"); + char *saved_path = save_test_env("PATH"); + char *saved_codex = save_test_env("CODEX_HOME"); + cbm_setenv("HOME", tmpdir, 1); + cbm_setenv("PATH", tmpdir, 1); + cbm_setenv("CODEX_HOME", codex_home, 1); + + int normal_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + bool preserved = normal_rc == 0 && cli_test_file_equals(canonical_file, foreign) && + cli_test_file_equals(legacy_file, cbm_get_skills()[0].content); + int force_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", true, false); + struct stat state; + bool forced = force_rc == 0 && + cli_test_file_equals(canonical_file, cbm_get_skills()[0].content) && + stat(legacy_file, &state) != 0; + + restore_test_env("HOME", saved_home); + restore_test_env("PATH", saved_path); + restore_test_env("CODEX_HOME", saved_codex); + test_rmdir_r(tmpdir); + if (!preserved || !forced) + FAIL("Codex must preserve conflicting canonical bytes and its owned legacy copy until " + "--force"); + PASS(); +} + +TEST(cli_codex_skill_failed_destination_preserves_owned_legacy) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-codex-skill-failed-destination-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + char codex_home[512]; + char legacy_root[640]; + char legacy_file[768]; + char canonical_root[640]; + char blocked_path[768]; + snprintf(codex_home, sizeof(codex_home), "%s/vendor-codex", tmpdir); + snprintf(legacy_root, sizeof(legacy_root), "%s/skills", codex_home); + snprintf(legacy_file, sizeof(legacy_file), "%s/codebase-memory/SKILL.md", legacy_root); + snprintf(canonical_root, sizeof(canonical_root), "%s/.agents/skills", tmpdir); + snprintf(blocked_path, sizeof(blocked_path), "%s/codebase-memory", canonical_root); + test_mkdirp(codex_home); + ASSERT_EQ(cbm_install_skills(legacy_root, false, false), CBM_SKILL_COUNT); + test_mkdirp(canonical_root); + write_test_file(blocked_path, "not a directory\n"); + + char *saved_home = save_test_env("HOME"); + char *saved_path = save_test_env("PATH"); + char *saved_codex = save_test_env("CODEX_HOME"); + cbm_setenv("HOME", tmpdir, 1); + cbm_setenv("PATH", tmpdir, 1); + cbm_setenv("CODEX_HOME", codex_home, 1); + int rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + bool safe = rc != 0 && cli_test_file_equals(blocked_path, "not a directory\n") && + cli_test_file_equals(legacy_file, cbm_get_skills()[0].content); + + restore_test_env("HOME", saved_home); + restore_test_env("PATH", saved_path); + restore_test_env("CODEX_HOME", saved_codex); + test_rmdir_r(tmpdir); + if (!safe) + FAIL("Codex migration must retain the owned legacy copy when canonical publication fails"); + PASS(); +} + +TEST(cli_codex_skill_uninstall_cleans_both_owned_locations) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-codex-skill-uninstall-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + char codex_home[512]; + char canonical_root[640]; + char legacy_root[640]; + char canonical_file[768]; + char legacy_file[768]; + char binary_dir[640]; + char binary_path[768]; + snprintf(codex_home, sizeof(codex_home), "%s/.codex", tmpdir); + snprintf(canonical_root, sizeof(canonical_root), "%s/.agents/skills", tmpdir); + snprintf(legacy_root, sizeof(legacy_root), "%s/skills", codex_home); + snprintf(canonical_file, sizeof(canonical_file), "%s/codebase-memory/SKILL.md", canonical_root); + snprintf(legacy_file, sizeof(legacy_file), "%s/codebase-memory/SKILL.md", legacy_root); + snprintf(binary_dir, sizeof(binary_dir), "%s/.local/bin", tmpdir); +#ifdef _WIN32 + snprintf(binary_path, sizeof(binary_path), "%s/codebase-memory-mcp.exe", binary_dir); +#else + snprintf(binary_path, sizeof(binary_path), "%s/codebase-memory-mcp", binary_dir); +#endif + test_mkdirp(codex_home); + test_mkdirp(binary_dir); + write_test_file(binary_path, "installed binary\n"); + ASSERT_EQ(cbm_install_skills(canonical_root, false, false), CBM_SKILL_COUNT); + ASSERT_EQ(cbm_install_skills(legacy_root, false, false), CBM_SKILL_COUNT); + + char *saved_home = save_test_env("HOME"); + char *saved_path = save_test_env("PATH"); + char *saved_codex = save_test_env("CODEX_HOME"); + cbm_setenv("HOME", tmpdir, 1); + cbm_setenv("PATH", tmpdir, 1); + cbm_unsetenv("CODEX_HOME"); + char *argv[] = {"uninstall", "--yes"}; + int owned_rc = cli_test_cmd_uninstall(2, argv); + struct stat state; + bool owned_removed = + owned_rc == 0 && stat(canonical_file, &state) != 0 && stat(legacy_file, &state) != 0; + + const char *foreign = "---\nname: codebase-memory\n---\nUser-owned canonical skill.\n"; + write_test_file(binary_path, "installed binary\n"); + bool mixed_seeded = cbm_install_skills(canonical_root, false, false) == CBM_SKILL_COUNT && + cbm_install_skills(legacy_root, false, false) == CBM_SKILL_COUNT; + if (mixed_seeded) { + write_test_file(canonical_file, foreign); + } + int mixed_rc = mixed_seeded ? cli_test_cmd_uninstall(2, argv) : 1; + bool mixed_safe = mixed_rc == 0 && cli_test_file_equals(canonical_file, foreign) && + stat(legacy_file, &state) != 0; + + restore_test_env("HOME", saved_home); + restore_test_env("PATH", saved_path); + restore_test_env("CODEX_HOME", saved_codex); + test_rmdir_r(tmpdir); + if (!owned_removed || !mixed_safe) + FAIL("Codex uninstall must clean both owned roots while preserving foreign skill bytes"); + PASS(); +} + +TEST(cli_codex_skill_deduplicates_agents_codex_home) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-codex-skill-same-root-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + char codex_home[512]; + char skill_file[768]; + snprintf(codex_home, sizeof(codex_home), "%s/.agents", tmpdir); + snprintf(skill_file, sizeof(skill_file), "%s/skills/codebase-memory/SKILL.md", codex_home); + test_mkdirp(codex_home); + char *saved_home = save_test_env("HOME"); + char *saved_path = save_test_env("PATH"); + char *saved_codex = save_test_env("CODEX_HOME"); + cbm_setenv("HOME", tmpdir, 1); + cbm_setenv("PATH", tmpdir, 1); + cbm_setenv("CODEX_HOME", codex_home, 1); + + int install_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + char *plan = cbm_build_install_plan_json(tmpdir, "/opt/codebase-memory-mcp"); + bool deduplicated = install_rc == 0 && + cli_test_file_equals(skill_file, cbm_get_skills()[0].content) && plan && + !strstr(plan, "remove_owned_legacy_copy_if_migrated"); + free(plan); + + restore_test_env("HOME", saved_home); + restore_test_env("PATH", saved_path); + restore_test_env("CODEX_HOME", saved_codex); + test_rmdir_r(tmpdir); + if (!deduplicated) + FAIL("Codex must treat HOME/.agents and CODEX_HOME as one skill root when they coincide"); + PASS(); +} + TEST(cli_gemini_session_hook_uses_json_for_all_sources) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-gemini-session-XXXXXX"); @@ -11761,7 +12062,7 @@ TEST(cli_codex_migrates_to_single_hook_representation) { int uninstall_rc = cli_test_cmd_uninstall(1, uninstall_argv); char skill_path[768]; char agent_path[768]; - snprintf(skill_path, sizeof(skill_path), "%s/skills/codebase-memory/SKILL.md", codex_dir); + snprintf(skill_path, sizeof(skill_path), "%s/.agents/skills/codebase-memory/SKILL.md", tmpdir); snprintf(agent_path, sizeof(agent_path), "%s/agents/codebase-memory.toml", codex_dir); struct stat state; hooks = read_test_file_alloc(hooks_path); @@ -11908,9 +12209,16 @@ TEST(cli_codex_preflight_reports_heading_and_reason) { char codex_dir[512]; char config_path[640]; char agents_path[640]; + char legacy_root[640]; + char legacy_file[768]; + char canonical_file[768]; snprintf(codex_dir, sizeof(codex_dir), "%s/.codex", tmpdir); snprintf(config_path, sizeof(config_path), "%s/config.toml", codex_dir); snprintf(agents_path, sizeof(agents_path), "%s/AGENTS.md", codex_dir); + snprintf(legacy_root, sizeof(legacy_root), "%s/skills", codex_dir); + snprintf(legacy_file, sizeof(legacy_file), "%s/codebase-memory/SKILL.md", legacy_root); + snprintf(canonical_file, sizeof(canonical_file), "%s/.agents/skills/codebase-memory/SKILL.md", + tmpdir); if (test_mkdirp(codex_dir) != 0) { test_rmdir_r(tmpdir); FAIL("failed to create Codex preflight fixture directory"); @@ -11923,6 +12231,10 @@ TEST(cli_codex_preflight_reports_heading_and_reason) { test_rmdir_r(tmpdir); FAIL("failed to write Codex preflight fixture config"); } + if (cbm_install_skills(legacy_root, false, false) != CBM_SKILL_COUNT) { + test_rmdir_r(tmpdir); + FAIL("failed to seed owned legacy Codex skill"); + } char *saved_home = save_test_env("HOME"); char *saved_path = save_test_env("PATH"); @@ -11964,8 +12276,11 @@ TEST(cli_codex_preflight_reports_heading_and_reason) { } char *after = read_test_file_alloc(config_path); char *agents_after = read_test_file_alloc(agents_path); + struct stat state; bool unchanged = after && strcmp(after, ambiguous) == 0 && agents_after && - strcmp(agents_after, test_codex_activation_block) == 0; + strcmp(agents_after, test_codex_activation_block) == 0 && + stat(canonical_file, &state) != 0 && + cli_test_file_equals(legacy_file, cbm_get_skills()[0].content); bool diagnostic = strstr(output, "Codex CLI:\nerror: agent_config agent=Codex CLI op=hook_preflight path=") != NULL && @@ -16175,6 +16490,12 @@ SUITE(cli) { RUN_TEST(cli_codex_respects_codex_home); RUN_TEST(cli_grok_respects_grok_home); RUN_TEST(cli_codex_install_uses_global_activation_pointer_issue1689); + RUN_TEST(cli_codex_skill_migrates_owned_legacy_to_agents_root); + RUN_TEST(cli_codex_skill_preserves_foreign_legacy_until_force); + RUN_TEST(cli_codex_skill_preserves_foreign_canonical_and_owned_legacy); + RUN_TEST(cli_codex_skill_failed_destination_preserves_owned_legacy); + RUN_TEST(cli_codex_skill_uninstall_cleans_both_owned_locations); + RUN_TEST(cli_codex_skill_deduplicates_agents_codex_home); RUN_TEST(cli_gemini_session_hook_uses_json_for_all_sources); RUN_TEST(cli_gemini_installs_dedicated_graph_subagent); RUN_TEST(cli_antigravity_does_not_imply_gemini); From b961919223704bfb3e38adde55fab5651a2ca4e0 Mon Sep 17 00:00:00 2001 From: Benyamin Date: Fri, 21 Aug 2026 15:27:24 +0330 Subject: [PATCH 024/217] fix(cli): simplify Codex cleanup condition Signed-off-by: Benyamin --- src/cli/cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index 8401179ab..3aaadb01e 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -8746,7 +8746,7 @@ static bool cbm_remove_codex_managed_skill(const char *skills_dir, const char *s printf(" preserved non-owned Codex skill: %s\n", skill_file); } else if (dry_run) { printf(" preserved unsafe Codex skill path: %s\n", skill_file); - } else if (!dry_run) { + } else { record_agent_config_error(uninstalling, "Codex CLI", operation, skill_file); } return state == CBM_CODEX_SKILL_UNOWNED; From 7c88e64eb954e9400273d3ebb9f6f88f1aebc165 Mon Sep 17 00:00:00 2001 From: Benyamin Date: Tue, 15 Sep 2026 11:20:19 +0330 Subject: [PATCH 025/217] fix(codex): recognize released skill payloads Signed-off-by: Benyamin --- scripts/smoke-test.sh | 25 +++- src/cli/cli.c | 314 ++++++++++++++++++++++++++++++++++++------ src/cli/cli.h | 5 + tests/test_cli.c | 114 ++++++++++++++- 4 files changed, 413 insertions(+), 45 deletions(-) diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 620e73c66..198e81d33 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -1395,11 +1395,21 @@ if ! run_codex_only "$CODEX_ONLY_BIN" install -y --clients=codex \ cat "$CODEX_ONLY_LOG" exit 1 fi +mkdir -p "$CODEX_ONLY_HOME/.openhands" +if ! run_codex_only "$CODEX_ONLY_BIN" install -y --clients=codex,openhands \ + --dir="$CODEX_ONLY_INSTALL" || + ! grep -q '^name: codebase-memory$' "$CODEX_ONLY_SKILL" 2>/dev/null || + [ "$(smoke_file_sha256 "$CODEX_ONLY_LEGACY")" != "$CODEX_ONLY_FOREIGN_HASH" ]; then + echo "FAIL 8-codex-5: a foreign Codex legacy skill blocked a shared-client install" + cat "$CODEX_ONLY_LOG" + exit 1 +fi +rm -f "$CODEX_ONLY_SKILL" if ! run_codex_only "$CODEX_ONLY_BIN" install --force -y --clients=codex \ --dir="$CODEX_ONLY_INSTALL" || ! grep -q '^name: codebase-memory$' "$CODEX_ONLY_SKILL" 2>/dev/null || [ -e "$CODEX_ONLY_LEGACY" ]; then - echo "FAIL 8-codex-5: force did not converge a foreign legacy skill" + echo "FAIL 8-codex-6: force did not converge a foreign legacy skill" cat "$CODEX_ONLY_LOG" exit 1 fi @@ -1407,7 +1417,7 @@ CODEX_ONLY_HASH=$(smoke_file_sha256 "$CODEX_ONLY_SKILL") if ! run_codex_only "$CODEX_ONLY_BIN" install -y --clients=codex \ --dir="$CODEX_ONLY_INSTALL" || [ "$(smoke_file_sha256 "$CODEX_ONLY_SKILL")" != "$CODEX_ONLY_HASH" ]; then - echo "FAIL 8-codex-6: repeated Codex install was not idempotent" + echo "FAIL 8-codex-7: repeated Codex install was not idempotent" cat "$CODEX_ONLY_LOG" exit 1 fi @@ -1826,6 +1836,8 @@ fi CODEX_LIFECYCLE_HOME=$(smoke_mktemp_dir) CODEX_LIFECYCLE_ROOT="$CODEX_LIFECYCLE_HOME/.codex" CODEX_INSTRUCTIONS="$CODEX_LIFECYCLE_ROOT/AGENTS.md" +CODEX_LIFECYCLE_SKILL="$CODEX_LIFECYCLE_HOME/.agents/skills/codebase-memory/SKILL.md" +CODEX_LIFECYCLE_LEGACY_SKILL="$CODEX_LIFECYCLE_ROOT/skills/codebase-memory/SKILL.md" mkdir -p "$CODEX_LIFECYCLE_ROOT" CODEX_FRESH_LOG=$(smoke_mktemp_file) @@ -1839,7 +1851,8 @@ if [ ! -f "$CODEX_INSTRUCTIONS" ] || [ "$(smoke_file_sha256 "$CODEX_INSTRUCTIONS")" != \ "$(smoke_file_sha256 "$CODEX_POINTER_EXPECTED")" ] || ! grep -q '\[mcp_servers.codebase-memory-mcp\]' "$CODEX_LIFECYCLE_ROOT/config.toml" || - ! grep -q 'search_graph' "$CODEX_LIFECYCLE_ROOT/skills/codebase-memory/SKILL.md" || + ! grep -q 'search_graph' "$CODEX_LIFECYCLE_SKILL" || + [ -e "$CODEX_LIFECYCLE_LEGACY_SKILL" ] || [ ! -s "$CODEX_LIFECYCLE_ROOT/agents/codebase-memory-scout.toml" ] || [ ! -s "$CODEX_LIFECYCLE_ROOT/agents/codebase-memory.toml" ] || [ ! -s "$CODEX_LIFECYCLE_ROOT/agents/codebase-memory-auditor.toml" ] || @@ -1936,8 +1949,10 @@ if ! HOME="$CODEX_LIFECYCLE_HOME" \ fi if [ ! -f "$CODEX_INSTRUCTIONS" ] || [ "$(smoke_file_sha256 "$CODEX_INSTRUCTIONS")" != \ - "$(smoke_file_sha256 "$CODEX_EXPECTED_USER")" ]; then - echo "FAIL 8i: Codex uninstall did not remove only the activation pointer" + "$(smoke_file_sha256 "$CODEX_EXPECTED_USER")" ] || + [ -e "$CODEX_LIFECYCLE_SKILL" ] || + [ -e "$CODEX_LIFECYCLE_LEGACY_SKILL" ]; then + echo "FAIL 8i: Codex uninstall did not remove only installer-owned surfaces" exit 1 fi echo "OK 8i: isolated Codex activation-pointer lifecycle" diff --git a/src/cli/cli.c b/src/cli/cli.c index 3aaadb01e..dea1a26fb 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1421,6 +1421,232 @@ int cbm_replace_binary(const char *path, const unsigned char *data, int len, int /* Consolidated from 4 separate skills into 1 with progressive disclosure. * This embedded version is the single source of truth for the CLI installer. * Based on PR #81 by @gdilla — factual corrections applied. */ +static const char released_skill_content_v0_6_0[] = + "---\n" + "name: codebase-memory\n" + "description: Use the codebase knowledge graph for structural code queries. " + "Triggers on: explore the codebase, understand the architecture, what functions exist, " + "show me the structure, who calls this function, what does X call, trace the call chain, " + "find callers of, show dependencies, impact analysis, dead code, unused functions, " + "high fan-out, refactor candidates, code quality audit, graph query syntax, " + "Cypher query examples, edge types, how to use search_graph.\n" + "---\n" + "\n" + "# Codebase Memory — Knowledge Graph Tools\n" + "\n" + "Graph tools return precise structural results in ~500 tokens vs ~80K for grep.\n" + "\n" + "## Quick Decision Matrix\n" + "\n" + "| Question | Tool call |\n" + "|----------|----------|\n" + "| Who calls X? | `trace_path(direction=\"inbound\")` |\n" + "| What does X call? | `trace_path(direction=\"outbound\")` |\n" + "| Full call context | `trace_path(direction=\"both\")` |\n" + "| Find by name pattern | `search_graph(name_pattern=\"...\")` |\n" + "| Dead code | `search_graph(max_degree=0, exclude_entry_points=true)` |\n" + "| Cross-service edges | `query_graph` with Cypher |\n" + "| Impact of local changes | `detect_changes()` |\n" + "| Risk-classified trace | `trace_path(risk_labels=true)` |\n" + "| Text search | `search_code` or Grep |\n" + "\n" + "## Exploration Workflow\n" + "1. `list_projects` — check if project is indexed\n" + "2. `get_graph_schema` — understand node/edge types\n" + "3. `search_graph(label=\"Function\", name_pattern=\".*Pattern.*\")` — find code\n" + "4. `get_code_snippet(qualified_name=\"project.path.FuncName\")` — read source\n" + "\n" + "## Tracing Workflow\n" + "1. `search_graph(name_pattern=\".*FuncName.*\")` — discover exact name\n" + "2. `trace_path(function_name=\"FuncName\", direction=\"both\", depth=3)` — trace\n" + "3. `detect_changes()` — map git diff to affected symbols\n" + "\n" + "## Quality Analysis\n" + "- Dead code: `search_graph(max_degree=0, exclude_entry_points=true)`\n" + "- High fan-out: `search_graph(min_degree=10, relationship=\"CALLS\", " + "direction=\"outbound\")`\n" + "- High fan-in: `search_graph(min_degree=10, relationship=\"CALLS\", " + "direction=\"inbound\")`\n" + "\n" + "## 14 MCP Tools\n" + "`index_repository`, `index_status`, `list_projects`, `delete_project`,\n" + "`search_graph`, `search_code`, `trace_path`, `detect_changes`,\n" + "`query_graph`, `get_graph_schema`, `get_code_snippet`, `get_architecture`,\n" + "`manage_adr`, `ingest_traces`\n" + "\n" + "## Edge Types\n" + "CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD,\n" + "HANDLES, IMPLEMENTS, OVERRIDE, USAGE, FILE_CHANGES_WITH,\n" + "CONTAINS_FILE, CONTAINS_FOLDER, CONTAINS_PACKAGE\n" + "\n" + "## Cypher Examples (for query_graph)\n" + "```\n" + "MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, " + "r.confidence LIMIT 20\n" + "MATCH (f:Function) WHERE f.name =~ '.*Handler.*' RETURN f.name, f.file_path\n" + "MATCH (a)-[r:CALLS]->(b) WHERE a.name = 'main' RETURN b.name\n" + "```\n" + "\n" + "## Gotchas\n" + "1. `search_graph(relationship=\"HTTP_CALLS\")` filters nodes by degree — " + "use `query_graph` with Cypher to see actual edges.\n" + "2. `query_graph` has a 200-row cap — use `search_graph` with degree filters " + "for counting.\n" + "3. `trace_path` needs exact names — use `search_graph(name_pattern=...)` first.\n" + "4. `direction=\"outbound\"` misses cross-service callers — use " + "`direction=\"both\"`.\n" + "5. Results default to 10 per page — check `has_more` and use `offset`.\n"; + +#define CBM_RELEASED_SKILL_HEADER_UNQUOTED \ + "---\n" \ + "name: codebase-memory\n" \ + "description: Use the codebase knowledge graph for structural code queries. " \ + "Triggers on: explore the codebase, understand the architecture, what functions exist, " \ + "show me the structure, who calls this function, what does X call, trace the call chain, " \ + "find callers of, show dependencies, impact analysis, dead code, unused functions, " \ + "high fan-out, refactor candidates, code quality audit, graph query syntax, " \ + "Cypher query examples, edge types, how to use search_graph.\n" \ + "---\n" + +#define CBM_RELEASED_SKILL_HEADER_QUOTED \ + "---\n" \ + "name: codebase-memory\n" \ + "description: \"Use the codebase knowledge graph for structural code queries. " \ + "Triggers on: explore the codebase, understand the architecture, what functions exist, " \ + "show me the structure, who calls this function, what does X call, trace the call chain, " \ + "find callers of, show dependencies, impact analysis, dead code, unused functions, " \ + "high fan-out, refactor candidates, code quality audit, graph query syntax, " \ + "Cypher query examples, edge types, how to use search_graph.\"\n" \ + "---\n" + +#define CBM_RELEASED_SKILL_BODY_BEFORE_EDGE_TYPES \ + "\n" \ + "# Codebase Memory — Knowledge Graph Tools\n" \ + "\n" \ + "Graph tools return precise structural results in ~500 tokens vs ~80K for grep.\n" \ + "\n" \ + "## Quick Decision Matrix\n" \ + "\n" \ + "| Question | Tool call |\n" \ + "|----------|----------|\n" \ + "| Who calls X? | `trace_path(direction=\"inbound\")` |\n" \ + "| What does X call? | `trace_path(direction=\"outbound\")` |\n" \ + "| Full call context | `trace_path(direction=\"both\")` |\n" \ + "| Find by name pattern | `search_graph(name_pattern=\"...\")` |\n" \ + "| Dead code | `search_graph(max_degree=0, exclude_entry_points=true)` |\n" \ + "| Cross-service edges | `query_graph` with Cypher |\n" \ + "| Impact of local changes | `detect_changes()` |\n" \ + "| Risk-classified trace | `trace_path(risk_labels=true)` |\n" \ + "| Text search | `search_code` or Grep |\n" \ + "\n" \ + "## Exploration Workflow\n" \ + "1. `list_projects` — check if project is indexed\n" \ + "2. `get_graph_schema` — understand node/edge types\n" \ + "3. `search_graph(label=\"Function\", name_pattern=\".*Pattern.*\")` — find code\n" \ + "4. `get_code_snippet(qualified_name=\"project.path.FuncName\")` — read source\n" \ + "\n" \ + "## Tracing Workflow\n" \ + "1. `search_graph(name_pattern=\".*FuncName.*\")` — discover exact name\n" \ + "2. `trace_path(function_name=\"FuncName\", direction=\"both\", depth=3)` — trace\n" \ + "3. `detect_changes()` — map git diff to affected symbols\n" \ + "\n" \ + "## Evidence Tiers\n" \ + "- **Scout (Tier 1):** fast positive lookup with few graph calls and targeted source checks. " \ + "Treat results as provisional; never make absence, exhaustive, dead-code, or complete-impact " \ + "claims.\n" \ + "- **Verify (Tier 2, default):** task-directed searches, relevant trace directions, exact " \ + "snippets for material claims, and all relevant result pages.\n" \ + "- **Auditor (Tier 3):** bounded-scope full verification with a current graph generation, " \ + "complete relevant pagination, both call directions and broader relationships when material, " \ + "plus explicit unresolved limitations.\n" \ + "- **Every tier:** after candidate paths are known, call `check_index_coverage` once with " \ + "every " \ + "evidence path. For negative or exhaustive claims also include the relevant scopes. A clean " \ + "result means no recorded gap, not proof of completeness. For partial, skipped, excluded, " \ + "stale, pending, or unknown coverage, read/grep the reported ranges or scope before relying " \ + "on " \ + "the graph.\n" \ + "\n" \ + "## Sessions and Subagents\n" \ + "- At session start or after compaction, call `list_projects`/`index_status` before " \ + "structural exploration, then choose Scout, Verify, or Auditor for the task.\n" \ + "- Before delegating, query the graph and coverage in the parent. Pass the tier, exact " \ + "project, " \ + "generation/freshness, bounded scope, queries and pagination state, qualified symbols, " \ + "paths, " \ + "call-chain findings, coverage ranges/reasons, source fallback already performed, and " \ + "unresolved " \ + "questions to the child.\n" \ + "- Runtimes such as Hermes isolate child context: put those graph findings in the " \ + "`context` argument to `delegate_task`; do not assume the child inherits MCP access or " \ + "the parent's conversation.\n" \ + "- A child without MCP tools must not call or claim MCP access. It should work from the " \ + "supplied " \ + "evidence and use read/grep on exact source, especially every reported missed-coverage " \ + "range.\n" \ + "\n" \ + "## Quality Analysis\n" \ + "- Dead code: `search_graph(max_degree=0, exclude_entry_points=true)`\n" \ + "- High fan-out: `search_graph(min_degree=10, relationship=\"CALLS\", " \ + "direction=\"outbound\")`\n" \ + "- High fan-in: `search_graph(min_degree=10, relationship=\"CALLS\", " \ + "direction=\"inbound\")`\n" \ + "\n" \ + "## 15 MCP Tools\n" \ + "`index_repository`, `index_status`, `list_projects`, `delete_project`,\n" \ + "`search_graph`, `search_code`, `trace_path`, `detect_changes`,\n" \ + "`query_graph`, `get_graph_schema`, `get_code_snippet`, `get_architecture`,\n" \ + "`check_index_coverage`, `manage_adr`, `ingest_traces`\n" \ + "\n" \ + "## Edge Types\n" \ + "CALLS, HTTP_CALLS, ASYNC_CALLS, DATA_FLOWS, IMPORTS, DEFINES, DEFINES_METHOD,\n" + +#define CBM_RELEASED_SKILL_EDGE_TYPES_PRE_V0_10 \ + "HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CONFIGURES, FILE_CHANGES_WITH,\n" + +#define CBM_RELEASED_SKILL_EDGE_TYPES_V0_10 \ + "HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CALL_REFERENCE, CONFIGURES, FILE_CHANGES_WITH,\n" + +#define CBM_RELEASED_SKILL_BODY_AFTER_EDGE_TYPES \ + "SIMILAR_TO, SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER,\n" \ + "CONTAINS_PACKAGE\n" \ + "\n" \ + "## Cypher Examples (for query_graph)\n" \ + "```\n" \ + "MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, " \ + "r.confidence LIMIT 20\n" \ + "MATCH (f:Function) WHERE f.name =~ '.*Handler.*' RETURN f.name, f.file_path\n" \ + "MATCH (a)-[r:CALLS]->(b) WHERE a.name = 'main' RETURN b.name\n" \ + "```\n" \ + "\n" \ + "## Gotchas\n" \ + "1. `search_graph(relationship=\"HTTP_CALLS\")` filters nodes by degree — " \ + "use `query_graph` with Cypher to see actual edges.\n" \ + "2. `query_graph` has a 100k row ceiling — add a Cypher `LIMIT` for broad queries " \ + "or use `search_graph` pagination.\n" \ + "3. `trace_path` needs exact names — use `search_graph(name_pattern=...)` first.\n" \ + "4. `direction=\"outbound\"` misses cross-service callers — use " \ + "`direction=\"both\"`.\n" \ + "5. `search_graph` results default to 50 per page — check `has_more` and use `offset`.\n" + +/* Exact payload generations shipped before the current skill text. */ +static const char released_skill_content_v0_9_1[] = + CBM_RELEASED_SKILL_HEADER_UNQUOTED CBM_RELEASED_SKILL_BODY_BEFORE_EDGE_TYPES + CBM_RELEASED_SKILL_EDGE_TYPES_PRE_V0_10 CBM_RELEASED_SKILL_BODY_AFTER_EDGE_TYPES; +static const char released_skill_content_v0_10_0[] = + CBM_RELEASED_SKILL_HEADER_UNQUOTED CBM_RELEASED_SKILL_BODY_BEFORE_EDGE_TYPES + CBM_RELEASED_SKILL_EDGE_TYPES_V0_10 CBM_RELEASED_SKILL_BODY_AFTER_EDGE_TYPES; +static const char released_skill_content_v0_10_3[] = + CBM_RELEASED_SKILL_HEADER_QUOTED CBM_RELEASED_SKILL_BODY_BEFORE_EDGE_TYPES + CBM_RELEASED_SKILL_EDGE_TYPES_V0_10 CBM_RELEASED_SKILL_BODY_AFTER_EDGE_TYPES; + +#undef CBM_RELEASED_SKILL_BODY_AFTER_EDGE_TYPES +#undef CBM_RELEASED_SKILL_EDGE_TYPES_V0_10 +#undef CBM_RELEASED_SKILL_EDGE_TYPES_PRE_V0_10 +#undef CBM_RELEASED_SKILL_BODY_BEFORE_EDGE_TYPES +#undef CBM_RELEASED_SKILL_HEADER_QUOTED +#undef CBM_RELEASED_SKILL_HEADER_UNQUOTED + static const char skill_content[] = "---\n" "name: codebase-memory\n" @@ -1535,6 +1761,19 @@ static const char skill_content[] = "`direction=\"both\"`.\n" "5. `search_graph` results default to 50 per page — check `has_more` and use `offset`.\n"; +/* Byte-exact consolidated skill payloads shipped by each release range. */ +static const char *const released_skill_contents[] = { + /* 0.6.0 through v0.9.0 */ + released_skill_content_v0_6_0, + /* v0.9.1-rc.1 */ + released_skill_content_v0_9_1, + /* v0.10.0 through v0.10.2 */ + released_skill_content_v0_10_0, + /* v0.10.3 through v0.10.8 */ + released_skill_content_v0_10_3, +}; +enum { RELEASED_SKILL_CONTENT_COUNT = 4 }; + static const char codex_instructions_content[] = "For structural codebase exploration, use the installed `codebase-memory` skill.\n"; @@ -1555,6 +1794,16 @@ const cbm_skill_t *cbm_get_skills(void) { return skills; } +#ifdef CBM_CLI_ENABLE_TEST_API +size_t cbm_cli_released_skill_content_count(void) { + return RELEASED_SKILL_CONTENT_COUNT; +} + +const char *cbm_cli_released_skill_content_at(size_t index) { + return index < RELEASED_SKILL_CONTENT_COUNT ? released_skill_contents[index] : NULL; +} +#endif + const char *cbm_get_codex_instructions(void) { return codex_instructions_content; } @@ -8125,7 +8374,6 @@ typedef struct { static cbm_install_plan_t *g_install_plan = NULL; static int g_agent_install_errors = 0; static int g_agent_uninstall_errors = 0; -static char g_blocked_shared_skill_dir[CLI_BUF_1K]; /* Every agent configuration uninstall could not clean, kept for the closing * summary. A cleanup failure no longer stops executable and index removal @@ -8611,14 +8859,6 @@ static void install_agent_skill(const char *label, const char *skills_dir, bool if (written < 0 || (size_t)written >= sizeof(skill_path)) { return; } - if (g_blocked_shared_skill_dir[0] && - (cbm_json_mcp_paths_equal(skills_dir, g_blocked_shared_skill_dir) || - cbm_same_file(skills_dir, g_blocked_shared_skill_dir))) { - if (!g_install_plan) { - printf(" skill: %s (skipped to avoid a duplicate Codex skill)\n", skill_path); - } - return; - } if (g_install_plan) { plan_record(label, "skill", skill_path); return; @@ -8702,7 +8942,8 @@ static cbm_codex_skill_state_t cbm_codex_skill_state(const char *skills_dir, return CBM_CODEX_SKILL_UNSAFE; } - int ownership = cbm_text_owned_document_status(skill_file, skill_content, NULL, 0); + int ownership = cbm_text_owned_document_status( + skill_file, skill_content, released_skill_contents, RELEASED_SKILL_CONTENT_COUNT); if (ownership == CLI_OK) { return CBM_CODEX_SKILL_MANAGED; } @@ -8716,23 +8957,18 @@ static void cbm_warn_codex_skill_conflict(const cbm_codex_skill_paths_t *paths, reason, paths->canonical_file, paths->legacy_file); } -static void cbm_block_shared_skill_install(const cbm_codex_skill_paths_t *paths) { - (void)snprintf(g_blocked_shared_skill_dir, sizeof(g_blocked_shared_skill_dir), "%s", - paths->canonical_dir); -} - -static void cbm_block_shared_skill_if_codex_conflicts(const char *home, const char *config_dir) { - cbm_codex_skill_paths_t paths; - if (!cbm_codex_skill_paths_init(home, config_dir, &paths) || paths.same_dir) { - return; +static bool cbm_refresh_codex_managed_skill(const cbm_codex_skill_paths_t *paths, bool dry_run) { + if (dry_run) { + return true; } - - cbm_codex_skill_state_t canonical = - cbm_codex_skill_state(paths.canonical_dir, paths.canonical_file); - cbm_codex_skill_state_t legacy = cbm_codex_skill_state(paths.legacy_dir, paths.legacy_file); - if (canonical == CBM_CODEX_SKILL_ABSENT && legacy != CBM_CODEX_SKILL_ABSENT) { - cbm_block_shared_skill_install(&paths); + int result = + cbm_text_migrate_owned_document(paths->canonical_file, skill_content, + released_skill_contents, RELEASED_SKILL_CONTENT_COUNT); + if (result == CLI_OK) { + return true; } + record_agent_config_error(false, "Codex CLI", "skill_install", paths->canonical_file); + return false; } static bool cbm_remove_codex_managed_skill(const char *skills_dir, const char *skill_file, @@ -8755,9 +8991,15 @@ static bool cbm_remove_codex_managed_skill(const char *skills_dir, const char *s printf(" Codex skill: %s (would remove)\n", skill_file); return true; } - int removed = cbm_remove_skills(skills_dir, false); - if (removed == CBM_SKILL_COUNT && + int removed = cbm_text_remove_owned_document_any( + skill_file, skill_content, released_skill_contents, RELEASED_SKILL_CONTENT_COUNT); + if (removed == CLI_OK && cbm_codex_skill_state(skills_dir, skill_file) == CBM_CODEX_SKILL_ABSENT) { + char skill_dir[CLI_BUF_1K]; + int written = snprintf(skill_dir, sizeof(skill_dir), "%s/codebase-memory", skills_dir); + if (written > 0 && (size_t)written < sizeof(skill_dir)) { + (void)cbm_remove_empty_directory(skill_dir, false); + } return true; } record_agent_config_error(uninstalling, "Codex CLI", operation, skill_file); @@ -8789,10 +9031,6 @@ static void install_codex_skill(const char *home, const char *config_dir, bool f (canonical == CBM_CODEX_SKILL_ABSENT || canonical == CBM_CODEX_SKILL_MANAGED)) { plan_record("Codex CLI", "cleanup_skill", paths.legacy_file); } - if (canonical == CBM_CODEX_SKILL_ABSENT && - (legacy == CBM_CODEX_SKILL_UNOWNED || legacy == CBM_CODEX_SKILL_UNSAFE)) { - cbm_block_shared_skill_install(&paths); - } return; } @@ -8806,6 +9044,9 @@ static void install_codex_skill(const char *home, const char *config_dir, bool f return; } install_agent_skill("Codex CLI", paths.canonical_dir, force, dry_run); + if (!cbm_refresh_codex_managed_skill(&paths, dry_run)) { + return; + } if (!dry_run && cbm_codex_skill_state(paths.canonical_dir, paths.canonical_file) != CBM_CODEX_SKILL_MANAGED) { record_agent_config_error(false, "Codex CLI", "skill_install", paths.canonical_file); @@ -8822,9 +9063,6 @@ static void install_codex_skill(const char *home, const char *config_dir, bool f return; } if (legacy == CBM_CODEX_SKILL_UNSAFE) { - if (canonical == CBM_CODEX_SKILL_ABSENT) { - cbm_block_shared_skill_install(&paths); - } if (force && !dry_run) { record_agent_config_error(false, "Codex CLI", "legacy_skill_migration", paths.legacy_file); @@ -8835,9 +9073,6 @@ static void install_codex_skill(const char *home, const char *config_dir, bool f } if (!force && legacy == CBM_CODEX_SKILL_UNOWNED) { - if (canonical == CBM_CODEX_SKILL_ABSENT) { - cbm_block_shared_skill_install(&paths); - } cbm_warn_codex_skill_conflict(&paths, "the legacy skill is not installer-owned"); return; } @@ -8882,6 +9117,9 @@ static void install_codex_skill(const char *home, const char *config_dir, bool f } return; } + if (!cbm_refresh_codex_managed_skill(&paths, false)) { + return; + } if (cbm_codex_skill_state(paths.canonical_dir, paths.canonical_file) != CBM_CODEX_SKILL_MANAGED) { record_agent_config_error(false, "Codex CLI", "skill_install", paths.canonical_file); @@ -9801,7 +10039,6 @@ static void install_cli_agent_configs(const cbm_detected_agents_t *agents, const true, &preflight_failure) : CLI_ERR; if (preflight_result != CLI_OK) { - cbm_block_shared_skill_if_codex_conflicts(home, config_dir); if (!g_install_plan) { printf("Codex CLI:\n"); fflush(stdout); @@ -10606,7 +10843,6 @@ static int cbm_install_agent_configs_in_scope(const char *home, const char *bina int cbm_install_agent_configs(const char *home, const char *binary_path, bool force, bool dry_run) { g_agent_install_errors = 0; - g_blocked_shared_skill_dir[0] = '\0'; cbm_detected_agents_t agents = cbm_detect_agents(home); if (g_client_selection && !cli_clients_apply_selection(g_client_selection, &agents)) { return CLI_ERR; diff --git a/src/cli/cli.h b/src/cli/cli.h index 139221f0a..9f5c49aa2 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -108,6 +108,11 @@ typedef struct { /* Get the array of skill definitions. */ const cbm_skill_t *cbm_get_skills(void); +#ifdef CBM_CLI_ENABLE_TEST_API +size_t cbm_cli_released_skill_content_count(void); +const char *cbm_cli_released_skill_content_at(size_t index); +#endif + /* Install skills to skills_dir (e.g. ~/.claude/skills/). * If force is true, overwrite existing skills. * Returns count of skills written. */ diff --git a/tests/test_cli.c b/tests/test_cli.c index 21649fa96..3a48ebe2e 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -9430,7 +9430,7 @@ TEST(cli_codex_install_uses_global_activation_pointer_issue1689) { snprintf(codex_home, sizeof(codex_home), "%s/.codex", tmpdir); snprintf(agents_path, sizeof(agents_path), "%s/AGENTS.md", codex_home); snprintf(config_path, sizeof(config_path), "%s/config.toml", codex_home); - snprintf(skill_path, sizeof(skill_path), "%s/skills/codebase-memory/SKILL.md", codex_home); + snprintf(skill_path, sizeof(skill_path), "%s/.agents/skills/codebase-memory/SKILL.md", tmpdir); snprintf(profile_path, sizeof(profile_path), "%s/agents/codebase-memory.toml", codex_home); #ifdef _WIN32 snprintf(binary_path, sizeof(binary_path), "%s/.local/bin/codebase-memory-mcp.exe", tmpdir); @@ -9651,6 +9651,112 @@ TEST(cli_codex_skill_migrates_owned_legacy_to_agents_root) { PASS(); } +TEST(cli_codex_skill_migrates_every_released_legacy_payload) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-codex-skill-released-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + char codex_home[512]; + char legacy_dir[768]; + char legacy_file[768]; + char canonical_file[768]; + snprintf(codex_home, sizeof(codex_home), "%s/.codex", tmpdir); + snprintf(legacy_dir, sizeof(legacy_dir), "%s/skills/codebase-memory", codex_home); + snprintf(legacy_file, sizeof(legacy_file), "%s/SKILL.md", legacy_dir); + snprintf(canonical_file, sizeof(canonical_file), "%s/.agents/skills/codebase-memory/SKILL.md", + tmpdir); + test_mkdirp(codex_home); + + char *saved_home = save_test_env("HOME"); + char *saved_path = save_test_env("PATH"); + char *saved_codex = save_test_env("CODEX_HOME"); + cbm_setenv("HOME", tmpdir, 1); + cbm_setenv("PATH", tmpdir, 1); + cbm_setenv("CODEX_HOME", codex_home, 1); + cbm_cli_set_client_selection_for_testing("codex"); + + size_t released_count = cbm_cli_released_skill_content_count(); + bool migrated_all = released_count == 4U; + for (size_t i = 0U; migrated_all && i < released_count; i++) { + const char *released = cbm_cli_released_skill_content_at(i); + test_mkdirp(legacy_dir); + if (!released || write_test_file(legacy_file, released) != 0) { + migrated_all = false; + break; + } + (void)remove(canonical_file); + int rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + struct stat state; + migrated_all = rc == 0 && + cli_test_file_equals(canonical_file, cbm_get_skills()[0].content) && + stat(legacy_file, &state) != 0; + if (!migrated_all || write_test_file(canonical_file, released) != 0) { + migrated_all = false; + break; + } + rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + migrated_all = rc == 0 && cli_test_file_equals(canonical_file, cbm_get_skills()[0].content); + } + + cbm_cli_set_client_selection_for_testing(NULL); + restore_test_env("HOME", saved_home); + restore_test_env("PATH", saved_path); + restore_test_env("CODEX_HOME", saved_codex); + test_rmdir_r(tmpdir); + if (!migrated_all) + FAIL("Codex must migrate every released skill payload from legacy and canonical paths"); + PASS(); +} + +TEST(cli_codex_foreign_legacy_does_not_block_shared_clients) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-codex-skill-shared-client-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + const char *foreign = "---\nname: codebase-memory\n---\nUser-owned legacy skill.\n"; + char codex_home[512]; + char legacy_dir[768]; + char legacy_file[768]; + char openhands_dir[512]; + char canonical_file[768]; + snprintf(codex_home, sizeof(codex_home), "%s/.codex", tmpdir); + snprintf(legacy_dir, sizeof(legacy_dir), "%s/skills/codebase-memory", codex_home); + snprintf(legacy_file, sizeof(legacy_file), "%s/SKILL.md", legacy_dir); + snprintf(openhands_dir, sizeof(openhands_dir), "%s/.openhands", tmpdir); + snprintf(canonical_file, sizeof(canonical_file), "%s/.agents/skills/codebase-memory/SKILL.md", + tmpdir); + test_mkdirp(legacy_dir); + test_mkdirp(openhands_dir); + write_test_file(legacy_file, foreign); + + char *saved_home = save_test_env("HOME"); + char *saved_path = save_test_env("PATH"); + char *saved_codex = save_test_env("CODEX_HOME"); + cbm_setenv("HOME", tmpdir, 1); + cbm_setenv("PATH", tmpdir, 1); + cbm_setenv("CODEX_HOME", codex_home, 1); + + char *plan = cbm_build_install_plan_json(tmpdir, "/opt/codebase-memory-mcp"); + bool plan_keeps_shared_install = plan && strstr(plan, canonical_file) && + !strstr(plan, "remove_owned_legacy_copy_if_migrated"); + free(plan); + int install_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + int repeat_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); + bool installed = install_rc == 0 && repeat_rc == 0 && + cli_test_file_equals(legacy_file, foreign) && + cli_test_file_equals(canonical_file, cbm_get_skills()[0].content); + + restore_test_env("HOME", saved_home); + restore_test_env("PATH", saved_path); + restore_test_env("CODEX_HOME", saved_codex); + test_rmdir_r(tmpdir); + if (!plan_keeps_shared_install || !installed) + FAIL("A foreign Codex legacy skill must not suppress another client's shared skill"); + PASS(); +} + TEST(cli_codex_skill_preserves_foreign_legacy_until_force) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-codex-skill-foreign-legacy-XXXXXX"); @@ -9679,6 +9785,7 @@ TEST(cli_codex_skill_preserves_foreign_legacy_until_force) { cbm_setenv("HOME", tmpdir, 1); cbm_setenv("PATH", tmpdir, 1); cbm_setenv("CODEX_HOME", codex_home, 1); + cbm_cli_set_client_selection_for_testing("codex"); int normal_rc = cbm_install_agent_configs(tmpdir, "/opt/codebase-memory-mcp", false, false); struct stat state; @@ -9689,6 +9796,7 @@ TEST(cli_codex_skill_preserves_foreign_legacy_until_force) { cli_test_file_equals(canonical_file, cbm_get_skills()[0].content) && stat(legacy_file, &state) != 0 && cli_test_file_equals(sibling_file, "keep\n"); + cbm_cli_set_client_selection_for_testing(NULL); restore_test_env("HOME", saved_home); restore_test_env("PATH", saved_path); restore_test_env("CODEX_HOME", saved_codex); @@ -12242,6 +12350,7 @@ TEST(cli_codex_preflight_reports_heading_and_reason) { cbm_setenv("HOME", tmpdir, 1); cbm_setenv("PATH", tmpdir, 1); cbm_unsetenv("CODEX_HOME"); + cbm_cli_set_client_selection_for_testing("codex"); FILE *capture = tmpfile(); int saved_stdout = capture ? dup(STDOUT_FILENO) : -1; @@ -12288,6 +12397,7 @@ TEST(cli_codex_preflight_reports_heading_and_reason) { free(after); free(agents_after); + cbm_cli_set_client_selection_for_testing(NULL); restore_test_env("HOME", saved_home); restore_test_env("PATH", saved_path); restore_test_env("CODEX_HOME", saved_codex); @@ -16491,6 +16601,8 @@ SUITE(cli) { RUN_TEST(cli_grok_respects_grok_home); RUN_TEST(cli_codex_install_uses_global_activation_pointer_issue1689); RUN_TEST(cli_codex_skill_migrates_owned_legacy_to_agents_root); + RUN_TEST(cli_codex_skill_migrates_every_released_legacy_payload); + RUN_TEST(cli_codex_foreign_legacy_does_not_block_shared_clients); RUN_TEST(cli_codex_skill_preserves_foreign_legacy_until_force); RUN_TEST(cli_codex_skill_preserves_foreign_canonical_and_owned_legacy); RUN_TEST(cli_codex_skill_failed_destination_preserves_owned_legacy); From 7054e4ff8af76f4fc1fe89407874e9910d0e412d Mon Sep 17 00:00:00 2001 From: yasmoradi Date: Thu, 17 Sep 2026 22:57:02 +0200 Subject: [PATCH 026/217] fix(extraction): recover C# 14 extension block members The vendored tree-sitter-c-sharp has no extension production, so an extension block parses as a constructor_declaration named extension whose body holds the members as local_function_statement nodes. Nothing recurses into a constructor body, and has_error stays 0, so the members were dropped with no signal at all. Emit them against the enclosing type instead. A constructor carries its own type name, so this shape is only a genuine constructor inside a type literally called extension, which keeps its own test. Fixes #2071 Signed-off-by: yasmoradi --- internal/cbm/extract_defs.c | 64 +++++++++++++++++++++++++++++++++++++ tests/test_extraction.c | 39 ++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index b49147dc6..b00604753 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -5032,6 +5032,61 @@ static void extract_objc_impl_methods(CBMExtractCtx *ctx, TSNode impl_node, cons } } +/* True for a C# 14 extension block. The vendored tree-sitter-c-sharp has no + * `extension` production, so `extension(string text) { ... }` parses as a + * constructor_declaration named `extension` and nothing flags it: has_error + * stays 0. A constructor carries its own type's name, so this shape can only be + * a genuine constructor inside a type literally called `extension` (#2071). */ +static bool csharp_is_extension_block(TSNode node, TSNode class_node, const char *source) { + if (strcmp(ts_node_type(node), "constructor_declaration") != 0) { + return false; + } + TSNode name = resolve_method_name(node, CBM_LANG_CSHARP); + if (ts_node_is_null(name)) { + return false; + } + const size_t kw_len = sizeof("extension") - 1; + uint32_t len = ts_node_end_byte(name) - ts_node_start_byte(name); + if (len != kw_len || strncmp(source + ts_node_start_byte(name), "extension", kw_len) != 0) { + return false; + } + TSNode type_name = ts_node_child_by_field_name(class_node, TS_FIELD("name")); + if (!ts_node_is_null(type_name)) { + uint32_t tlen = ts_node_end_byte(type_name) - ts_node_start_byte(type_name); + if (tlen == kw_len && + strncmp(source + ts_node_start_byte(type_name), "extension", kw_len) == 0) { + return false; + } + } + return true; +} + +/* Lift the members of a misparsed C# 14 extension block onto the enclosing + * type. They sit in the block the grammar took for a constructor body, which + * nothing recurses into, so every one of them was dropped. Methods land there + * as local_function_statement; a property in the block parses as ERROR and is + * beyond recovery without a grammar that knows the construct (#2071). */ +static void extract_csharp_extension_members(CBMExtractCtx *ctx, TSNode ext_node, TSNode class_node, + const char *class_qn, const CBMLangSpec *spec) { + TSNode body = cbm_find_child_by_kind(ext_node, "block"); + if (ts_node_is_null(body)) { + return; + } + uint32_t count = ts_node_child_count(body); + for (uint32_t i = 0; i < count; i++) { + TSNode member = ts_node_child(body, i); + if (ts_node_is_null(member) || + strcmp(ts_node_type(member), "local_function_statement") != 0) { + continue; + } + TSNode name = resolve_method_name(member, ctx->language); + if (ts_node_is_null(name)) { + continue; + } + push_method_def(ctx, member, class_node, class_qn, spec, name); + } +} + // Extract methods inside a class body static void extract_class_methods(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn, const CBMLangSpec *spec) { @@ -5047,6 +5102,15 @@ static void extract_class_methods(CBMExtractCtx *ctx, TSNode class_node, const c continue; } + /* C# 14 extension block: emit the members it holds as methods of the + * enclosing type, rather than the block itself as a constructor named + * `extension` with its members buried in the body (#2071). */ + if (ctx->language == CBM_LANG_CSHARP && + csharp_is_extension_block(child, class_node, ctx->source)) { + extract_csharp_extension_members(ctx, child, class_node, class_qn, spec); + continue; + } + if (ctx->language == CBM_LANG_OBJC && strcmp(ts_node_type(child), "implementation_definition") == 0) { extract_objc_impl_methods(ctx, child, class_qn, spec); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 14d75bf8b..618561f74 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -772,6 +772,43 @@ TEST(csharp_interface) { PASS(); } +/* Issue #2071: a C# 14 extension block parses as a constructor_declaration + * named `extension` whose body holds the members, and has_error stays 0, so the + * members were dropped without any signal. They belong to the enclosing type. */ +TEST(csharp_extension_block_members) { + CBMFileResult *r = extract("namespace App;\n" + "public static class NumberExtensions\n" + "{\n" + " extension(IEnumerable numbers)\n" + " {\n" + " public int SumPositive() => 0;\n" + " public static int Zero() => 0;\n" + " }\n" + "}\n", + CBM_LANG_CSHARP, "t", "NumberExtensions.cs"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Class", "NumberExtensions")); + ASSERT(has_def(r, "Method", "SumPositive")); + ASSERT(has_def(r, "Method", "Zero")); + ASSERT_FALSE(has_def_any(r, "extension")); + cbm_free_result(r); + PASS(); +} + +/* The same shape is a genuine constructor inside a type actually called + * `extension`, and must still be emitted (#2071). */ +TEST(csharp_constructor_of_type_named_extension) { + CBMFileResult *r = + extract("namespace App { public class extension { public extension() { } } }", + CBM_LANG_CSHARP, "t", "extension.cs"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Method", "extension")); + cbm_free_result(r); + PASS(); +} + /* --- Swift --- */ TEST(swift_class) { CBMFileResult *r = extract("class Vehicle {\n var speed: Int = 0\n func accelerate() { " @@ -7580,6 +7617,8 @@ SUITE(extraction) { RUN_TEST(ruby_module); RUN_TEST(csharp_class); RUN_TEST(csharp_interface); + RUN_TEST(csharp_extension_block_members); + RUN_TEST(csharp_constructor_of_type_named_extension); RUN_TEST(swift_class); RUN_TEST(swift_protocol); RUN_TEST(kotlin_function); From 9b4d697a57a03f242164db956843a5313168005d Mon Sep 17 00:00:00 2001 From: lorenzozanee Date: Fri, 18 Sep 2026 14:22:37 +0800 Subject: [PATCH 027/217] fix(ui): preserve framework route entry points Signed-off-by: lorenzozanee --- src/ui/layout3d.c | 29 ++++++++++++++++++++++----- tests/test_ui.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/src/ui/layout3d.c b/src/ui/layout3d.c index 3176da2ff..111be4903 100644 --- a/src/ui/layout3d.c +++ b/src/ui/layout3d.c @@ -554,6 +554,7 @@ cbm_layout_result_t *cbm_layout_compute(cbm_store_t *store, const char *project, /* 3. Query edges — filter during fetch via binary search (O(e log n)) */ int *deg = calloc((size_t)n, sizeof(int)); + bool *framework_entry = calloc((size_t)n, sizeof(bool)); int mapped = 0; int edge_cap = CBM_SZ_256; cbm_edge_t *all_edges = malloc((size_t)edge_cap * sizeof(cbm_edge_t)); @@ -561,8 +562,9 @@ cbm_layout_result_t *cbm_layout_compute(cbm_store_t *store, const char *project, int *ed = malloc((size_t)edge_cap * sizeof(int)); cbm_schema_info_t schema; memset(&schema, 0, sizeof(schema)); - if (deg && all_edges && es && ed && - cbm_store_get_schema(store, project, &schema) == CBM_STORE_OK) { + bool framework_entries_valid = deg && framework_entry && all_edges && es && ed; + if (framework_entries_valid) { + if (cbm_store_get_schema(store, project, &schema) == CBM_STORE_OK) { for (int t = 0; t < schema.edge_type_count; t++) { cbm_edge_t *te = NULL; int tc = 0; @@ -571,6 +573,15 @@ cbm_layout_result_t *cbm_layout_compute(cbm_store_t *store, const char *project, for (int e = 0; e < tc; e++) { int si = find_node_index(id_map, n, te[e].source_id); int di = find_node_index(id_map, n, te[e].target_id); + if (si >= 0 && te[e].type && strcmp(te[e].type, "HANDLES") == 0) { + cbm_node_t route = {0}; + if (cbm_store_find_node_by_id(store, te[e].target_id, &route) == CBM_STORE_OK) { + if (route.label && strcmp(route.label, "Route") == 0) + framework_entry[si] = true; + cbm_node_free_fields(&route); + } else + framework_entries_valid = false; + } if (si >= 0 && di >= 0) { if (mapped >= edge_cap) { int nc = edge_cap * PAIR_LEN; @@ -578,6 +589,7 @@ cbm_layout_result_t *cbm_layout_compute(cbm_store_t *store, const char *project, int *ts = realloc(es, (size_t)nc * sizeof(int)); int *td = realloc(ed, (size_t)nc * sizeof(int)); if (!te2 || !ts || !td) { + framework_entries_valid = false; if (te2) all_edges = te2; if (ts) @@ -606,7 +618,11 @@ cbm_layout_result_t *cbm_layout_compute(cbm_store_t *store, const char *project, } } free(te); - } + } else + framework_entries_valid = false; + } + } else { + framework_entries_valid = false; } edges_done: cbm_store_schema_free(&schema); @@ -630,6 +646,7 @@ cbm_layout_result_t *cbm_layout_compute(cbm_store_t *store, const char *project, if (!result || !bodies) { free(bodies); free(deg); + free(framework_entry); free(es); free(ed); free(cdepth); @@ -735,11 +752,12 @@ cbm_layout_result_t *cbm_layout_compute(cbm_store_t *store, const char *project, status = "structural"; else if (testish) status = "test"; - else if (nf.is_entry || nf.is_route) + else if (nf.is_entry || nf.is_route || + (framework_entries_valid && framework_entry && framework_entry[i])) status = "entry"; else if (nf.is_exported) status = "exported"; - else if (ic == 0 && iu == 0) + else if (framework_entries_valid && ic == 0 && iu == 0) status = "dead"; else if (ic == 1) status = "single"; @@ -772,6 +790,7 @@ cbm_layout_result_t *cbm_layout_compute(cbm_store_t *store, const char *project, free(bodies); free(deg); + free(framework_entry); free(es); free(ed); free(cdepth); diff --git a/tests/test_ui.c b/tests/test_ui.c index a403dbbbe..e4f2129bb 100644 --- a/tests/test_ui.c +++ b/tests/test_ui.c @@ -627,6 +627,30 @@ TEST(layout_dead_code_classification) { .qualified_name = "dc::exportedfn", .file_path = "src/d.c", .properties_json = "{\"is_exported\":true}"}; + cbm_node_t framework = {.project = "dc", + .label = "Function", + .name = "frameworkfn", + .qualified_name = "dc::frameworkfn", + .file_path = "src/routes.c", + .properties_json = "{}"}; + cbm_node_t route = {.project = "dc", + .label = "Route", + .name = "GET /framework", + .qualified_name = "dc::route", + .file_path = "src/routes.c", + .properties_json = "{}"}; + cbm_node_t unrelated = {.project = "dc", + .label = "Class", + .name = "UnrelatedTarget", + .qualified_name = "dc::unrelated", + .file_path = "src/routes.c", + .properties_json = "{}"}; + cbm_node_t nonroute = {.project = "dc", + .label = "Function", + .name = "nonroutefn", + .qualified_name = "dc::nonroutefn", + .file_path = "src/routes.c", + .properties_json = "{}"}; cbm_node_t single = {.project = "dc", .label = "Function", .name = "calledonce", @@ -658,6 +682,14 @@ TEST(layout_dead_code_classification) { cbm_store_upsert_node(store, &tst); cbm_store_upsert_node(store, &tstpath); cbm_store_upsert_node(store, &exp); + int64_t id_framework = cbm_store_upsert_node(store, &framework); + int64_t id_route = cbm_store_upsert_node(store, &route); + int64_t id_unrelated = cbm_store_upsert_node(store, &unrelated); + int64_t id_nonroute = cbm_store_upsert_node(store, &nonroute); + ASSERT_GT(id_framework, 0); + ASSERT_GT(id_route, 0); + ASSERT_GT(id_unrelated, 0); + ASSERT_GT(id_nonroute, 0); int64_t id_single = cbm_store_upsert_node(store, &single); int64_t id_norm = cbm_store_upsert_node(store, &norm); int64_t id_caller = cbm_store_upsert_node(store, &caller); @@ -670,9 +702,15 @@ TEST(layout_dead_code_classification) { cbm_edge_t e2 = { .project = "dc", .source_id = id_caller, .target_id = id_norm, .type = "CALLS"}; cbm_edge_t e3 = {.project = "dc", .source_id = id_dead, .target_id = id_norm, .type = "CALLS"}; + cbm_edge_t framework_route = { + .project = "dc", .source_id = id_framework, .target_id = id_route, .type = "HANDLES"}; + cbm_edge_t framework_unrelated = { + .project = "dc", .source_id = id_nonroute, .target_id = id_unrelated, .type = "HANDLES"}; cbm_store_insert_edge(store, &e1); cbm_store_insert_edge(store, &e2); cbm_store_insert_edge(store, &e3); + ASSERT_GT(cbm_store_insert_edge(store, &framework_route), 0); + ASSERT_GT(cbm_store_insert_edge(store, &framework_unrelated), 0); cbm_layout_result_t *r = cbm_layout_compute(store, "dc", CBM_LAYOUT_OVERVIEW, NULL, 0, 100); ASSERT_NOT_NULL(r); @@ -700,6 +738,18 @@ TEST(layout_dead_code_classification) { ASSERT_NOT_NULL(ln); ASSERT_STR_EQ(ln->status, "exported"); + ln = find_layout_node(r, "frameworkfn"); + ASSERT_NOT_NULL(ln); + ASSERT_STR_EQ(ln->status, "entry"); + + ln = find_layout_node(r, "nonroutefn"); + ASSERT_NOT_NULL(ln); + ASSERT_STR_EQ(ln->status, "dead"); + + ln = find_layout_node(r, "GET /framework"); + ASSERT_NOT_NULL(ln); + ASSERT_STR_EQ(ln->status, "structural"); + ln = find_layout_node(r, "calledonce"); ASSERT_NOT_NULL(ln); ASSERT_STR_EQ(ln->status, "single"); From 138a62253d28b66650cf2d4ea25ba7c3b4f3a111 Mon Sep 17 00:00:00 2001 From: Yyunozor Date: Sat, 19 Sep 2026 23:46:59 +0200 Subject: [PATCH 028/217] fix(registry): resolve package-path imports against the project's own packages JVM package imports were read as external. A Kotlin or Java specifier is a dotted package path whether the package belongs to the indexed tree or to a dependency -- `import org.jetbrains.exposed.v1.tests.shared.assertEquals` and `import kotlin.test.assertEquals` have the same shape, and neither is "relative". The package map cannot separate them either: the manifests it reads for the JVM (pom.xml, build.gradle) name build artifacts, not packages. So every package-path specifier fell through to "external" and the guard cut calls into the project's own code. Measured on JetBrains/Exposed at 0e4d81a5, full index: the guard removed 426 CALLS edges, all of them suffix_match, all of them into the repository's own exposed-tests/.../shared/Assert.kt. Every callee was assertEquals, assertTrue or assertFalse -- names that file declares and that the test files import twice, once from the project package and once from kotlin.test. The double import is what leaves the import map without a key and hands the call to the project-wide guess. The evidence that separates the two is the tree's own `package` declarations, and the pipeline already collects them: the import passes build a namespace map keyed by the dot-normalized `package` / `namespace` / `use` of every indexed file, to resolve namespace imports. Publish that map for the duration of the run instead of freeing it at the end of the import pass, and consult it before calling a package-path specifier external. The trailing member segment is walked off one at a time, the same way the namespace-map lookup in cbm_pipeline_resolve_import_node does it. This keeps the effect the census called valuable. `java.net.URL` walks to "java.net" and then "java", neither of which any project file declares, so a bare `URL` bound to an in-tree URL class is still cut. Only specifiers the tree itself claims are kept. Passing NULL restores the previous contract, which is what the unit tests exercise. After the change Exposed carries 31195 CALLS edges, the same as the merge-base, with none removed and none added -- stable across 28 indexations of the repository (8 with this branch, 20 single-threaded). Total edge counts are not a usable invariant here: roughly one indexation in ten materializes a `.properties` resource file as a File node under Folder nodes instead of a Module with its Variables, on the merge-base binary as well, and the file it picks varies. That flip never touches a CALLS edge and disappears entirely under CBM_INDEX_SINGLE_THREAD=1. The walk that strips the trailing member segment has no floor, on purpose: a third-party package published under a prefix the tree declares reads as in-tree and the guard stands down, leaving exactly the edge the merge-base emits. A call-resolution suppressor has to fail on that side. Signed-off-by: Yyunozor --- src/pipeline/pass_calls.c | 3 +- src/pipeline/pass_definitions.c | 4 +- src/pipeline/pass_parallel.c | 9 +- src/pipeline/pipeline.c | 14 ++ src/pipeline/pipeline.h | 7 +- src/pipeline/pipeline_incremental.c | 6 + src/pipeline/pipeline_internal.h | 10 ++ src/pipeline/registry.c | 93 ++++++++++++- tests/test_pipeline.c | 127 +++++++++++++++++ tests/test_registry.c | 206 ++++++++++++++++++++++------ 10 files changed, 430 insertions(+), 49 deletions(-) diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 7fc6bc423..f0f9b499f 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -675,7 +675,8 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, * #725 guard, after the service-pattern bypasses above, so no HTTP/route * edge can be lost to it. */ if (cbm_suppress_external_import_shadow(call->callee_name, res.strategy, file_imports, imp_keys, - imp_count, cbm_pipeline_get_pkgmap())) { + imp_count, cbm_pipeline_get_pkgmap(), + cbm_pipeline_get_nsmap())) { return 0; } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 7ac98e9cd..eac9d0a96 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -844,7 +844,9 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t create_channel_edges_for_file(ctx, result, files[i].rel_path); cbm_pipeline_create_env_configures_for_file(ctx, result, files[i].rel_path); } - cbm_pipeline_namespace_map_free(namespace_map); + /* Publish instead of free — see the twin in pass_parallel.c (#1355). */ + cbm_pipeline_namespace_map_free(cbm_pipeline_get_nsmap()); + cbm_pipeline_set_nsmap(namespace_map); if (owns_local_cache) { for (int i = 0; i < file_count; i++) { if (local_cache[i]) { diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index a57d594cf..d7aa7bba9 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -1343,7 +1343,11 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t cbm_pipeline_create_env_configures_for_file(ctx, result, rel); } - cbm_pipeline_namespace_map_free(namespace_map); + /* Publish instead of free: the call pass needs the same declared-package + * set to tell an in-tree package path from a third-party one (#1355). + * The pipeline owns it from here and frees it in its cleanup. */ + cbm_pipeline_namespace_map_free(cbm_pipeline_get_nsmap()); + cbm_pipeline_set_nsmap(namespace_map); cbm_log_info("parallel.registry.done", "entries", itoa_log(reg_entries), "defines", itoa_log(defines_edges), "imports", itoa_log(imports_edges)); @@ -2571,7 +2575,8 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB } if (target_node && source_node->id != target_node->id && cbm_suppress_external_import_shadow(call->callee_name, res.strategy, &result->imports, - imp_keys, imp_count, cbm_pipeline_get_pkgmap())) { + imp_keys, imp_count, cbm_pipeline_get_pkgmap(), + cbm_pipeline_get_nsmap())) { /* #1355: same guard as pass_calls.c — a bare call bound by an * external package import must not become a CALLS edge to an * unrelated project symbol of the same name. */ diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 938437549..1f15263a4 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -229,6 +229,18 @@ void cbm_pipeline_set_pkgmap(CBMHashTable *map) { g_pkgmap = map; } +/* ── Global declared-package map (one active pipeline at a time) ── */ + +static CBMHashTable *g_nsmap = NULL; + +CBMHashTable *cbm_pipeline_get_nsmap(void) { + return g_nsmap; +} + +void cbm_pipeline_set_nsmap(CBMHashTable *map) { + g_nsmap = map; +} + bool cbm_pipeline_had_format_migration(const cbm_pipeline_t *p) { return p && p->format_migration; } @@ -2349,6 +2361,8 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cleanup: cbm_pkgmap_free(cbm_pipeline_get_pkgmap()); cbm_pipeline_set_pkgmap(NULL); + cbm_pipeline_namespace_map_free(cbm_pipeline_get_nsmap()); + cbm_pipeline_set_nsmap(NULL); cbm_discover_free(files, file_count); cbm_pipeline_free_semantic_manifest(baseline_manifest, baseline_count); cbm_gbuf_free(p->gbuf); diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 45ef74f84..81afe6b9a 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -301,11 +301,16 @@ bool cbm_suppress_weak_local_binding_call(bool enabled, bool callee_is_locally_b * strategy are all kept. `indexed_packages` is the pipeline package map: a * specifier naming a package the tree itself declares (a workspace sibling) * counts as in-tree and is kept too; NULL disables that check. + * `declared_packages` is the pipeline namespace map and does the same job for + * package-path specifiers, which have no in-tree/external shape of their own: + * `import org.example.util.assertThing` is kept when some indexed file + * declares `package org.example.util`; NULL disables that check. * Pure; unit-tested in test_registry.c. */ bool cbm_suppress_external_import_shadow(const char *callee_name, const char *strategy, const CBMImportArray *file_imports, const char **import_map_keys, int import_map_count, - const CBMHashTable *indexed_packages); + const CBMHashTable *indexed_packages, + const CBMHashTable *declared_packages); /* #725: drop a suffix_match CALLS edge when the caller language and the * target file's language disagree. unique_name (candidates == 1) is #1572 diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index bf71a1f84..6f147b25a 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1612,6 +1612,8 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project, * leaked one map per probed run (LSan, macOS CI). */ cbm_pkgmap_free(cbm_pipeline_get_pkgmap()); cbm_pipeline_set_pkgmap(NULL); + cbm_pipeline_namespace_map_free(cbm_pipeline_get_nsmap()); + cbm_pipeline_set_nsmap(NULL); cbm_path_alias_collection_free(aliases); cbm_gbuf_free(probe_gbuf); return rc; @@ -2183,6 +2185,8 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char } cbm_pkgmap_free(cbm_pipeline_get_pkgmap()); cbm_pipeline_set_pkgmap(NULL); + cbm_pipeline_namespace_map_free(cbm_pipeline_get_nsmap()); + cbm_pipeline_set_nsmap(NULL); if (phase_rc != 0) { cbm_log_error("delta.err", "phase", "extract_resolve", "rc", itoa_buf(phase_rc)); goto out; @@ -2762,6 +2766,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * full pipeline's ownership boundary on both success and failure. */ cbm_pkgmap_free(cbm_pipeline_get_pkgmap()); cbm_pipeline_set_pkgmap(NULL); + cbm_pipeline_namespace_map_free(cbm_pipeline_get_nsmap()); + cbm_pipeline_set_nsmap(NULL); if (phase_rc != 0) { cbm_log_error("incremental.err", "phase", "extract_resolve", "rc", itoa_buf(phase_rc)); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 45b0baf6b..a8e0d5616 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -172,6 +172,16 @@ static inline int cbm_pipeline_relpath_is_excluded(const char *rel_path, char *c CBMHashTable *cbm_pipeline_get_pkgmap(void); void cbm_pipeline_set_pkgmap(CBMHashTable *map); +/* Get the current pipeline's declared-package map (NULL if none): the + * namespace/package → File-QN map built by cbm_pipeline_namespace_map_build + * from every indexed file's own `package`/`namespace`/`using` declaration. + * The import passes build it to resolve namespace imports; it is published + * here so the call passes can ask whether a package-path specifier names a + * package the tree itself declares (#1355). Set before resolve starts, read + * only afterwards — same single-writer contract as the pkgmap above. */ +CBMHashTable *cbm_pipeline_get_nsmap(void); +void cbm_pipeline_set_nsmap(CBMHashTable *map); + /* Unified module resolver: relative → pkgmap → fqn_module fallback. * Handles bare specifiers via pkgmap lookup with prefix matching. * Caller must free() the returned string. */ diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index adbe2c713..f16c0ddb8 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -565,6 +565,84 @@ static bool specifier_names_indexed_package(const CBMHashTable *indexed_packages return false; } +/* True when the specifier is a package PATH the indexed tree itself DECLARES. + * `declared_packages` is the pipeline's namespace map, keyed by the + * dot-normalized `package` / `namespace` / `use` declaration of every indexed + * file (cbm_pipeline_namespace_map_build), so this is the JVM/CLR/PHP + * counterpart of the manifest-name test above: those languages have no + * specifier shape that separates in-tree from third-party, and their manifests + * (pom.xml, build.gradle) name artifacts, not packages. + * + * A package-path specifier names a member inside a package, so the trailing + * segment is walked off one at a time, exactly like the namespace-map lookup in + * cbm_pipeline_resolve_import_node: `org.jetbrains.exposed.v1.tests.shared + * .assertEquals` -> `org.jetbrains.exposed.v1.tests.shared`, which Exposed's + * own Assert.kt declares, so the call keeps its edge. `java.net.URL` walks to + * `java.net` then `java`, which no project file declares, so it stays external + * and the guard still cuts its bind to an in-tree `URL` class. + * + * Separator normalization mirrors the map builder's: PHP writes `\`, some + * grammars `::` or `/`. NULL disables the check. */ +static bool specifier_names_declared_package(const CBMHashTable *declared_packages, + const char *module_path) { + if (!declared_packages || !module_path || !module_path[0]) { + return false; + } + char buf[CBM_SZ_512]; + if (strlen(module_path) >= sizeof(buf)) { + return false; + } + snprintf(buf, sizeof(buf), "%s", module_path); + char *brace = strchr(buf, '{'); + if (brace) { + *brace = '\0'; /* grouped import — the prefix is the package */ + } + char *alias = strstr(buf, " as "); + if (alias) { + *alias = '\0'; + } + for (char *p = buf; *p; p++) { + if (*p == '\\' || *p == ':' || *p == '/') { + *p = '.'; + } + } + /* Collapse the empty segments "::" just produced, then strip glob/ + * separator noise off the tail. */ + for (;;) { + char *dd = strstr(buf, ".."); + if (!dd) { + break; + } + memmove(dd, dd + 1, strlen(dd + 1) + 1); + } + size_t len = strlen(buf); + while (len > 0 && (buf[len - 1] == '*' || buf[len - 1] == '.' || buf[len - 1] == ' ')) { + buf[--len] = '\0'; + } + if (buf[0] == '\0') { + return false; + } + if (cbm_ht_has(declared_packages, buf)) { + return true; + } + /* The walk has no floor, and that is a deliberate false-NEGATIVE: once the + * tree declares `com.example`, EVERY specifier under `com.example.**` reads + * as in-tree, including a third-party library that happens to publish under + * a declared prefix. Such a call keeps the same-name fallback it has on + * main, so the cost is a fabricated edge this guard could have removed — + * never a real edge lost. Stopping earlier would need a rule for where a + * package path ends and a type begins, and getting that wrong fails the + * other way. Pinned by external_import_shadow_declared_prefix_is_permissive + * in test_registry.c. */ + for (char *dot = strrchr(buf, '.'); dot != NULL; dot = strrchr(buf, '.')) { + *dot = '\0'; + if (cbm_ht_has(declared_packages, buf)) { + return true; + } + } + return false; +} + static bool import_map_binds(const char **import_map_keys, int import_map_count, const char *local_name) { if (!import_map_keys || import_map_count <= 0) { @@ -612,11 +690,21 @@ static bool import_map_binds(const char **import_map_keys, int import_map_count, * specifier the tree's own manifests claim is treated exactly like a relative * one. NULL disables the check and restores the specifier-shape-only contract. * + * `declared_packages` closes the same hole for the package-path languages, + * where NO specifier shape says in-tree: `import + * org.jetbrains.exposed.v1.tests.shared.assertEquals` is spelled exactly like + * `import kotlin.test.assertEquals`, and only the tree's own `package` + * declarations tell them apart. It is permissive by construction — any + * specifier UNDER a declared package prefix counts as in-tree, see the note on + * the walk — so it can leave a fabricated edge in place, never remove a real + * one. NULL disables the check. + * * Pure + side-effect-free so the contract is unit-testable without a pipeline. */ bool cbm_suppress_external_import_shadow(const char *callee_name, const char *strategy, const CBMImportArray *file_imports, const char **import_map_keys, int import_map_count, - const CBMHashTable *indexed_packages) { + const CBMHashTable *indexed_packages, + const CBMHashTable *declared_packages) { if (!callee_name || !callee_name[0] || !strategy || !strategy[0]) { return false; } @@ -640,7 +728,8 @@ bool cbm_suppress_external_import_shadow(const char *callee_name, const char *st continue; } if (specifier_is_relative(imp->module_path) || - specifier_names_indexed_package(indexed_packages, imp->module_path)) { + specifier_names_indexed_package(indexed_packages, imp->module_path) || + specifier_names_declared_package(declared_packages, imp->module_path)) { return false; /* an in-tree binding for this name — never suppress */ } external_binding = true; diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index f772acc69..36bb243a3 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -5222,6 +5222,132 @@ TEST(pipeline_external_import_shadow_keeps_workspace_sibling_issue1355) { PASS(); } +/* #1355: the same tree once more, in a package-path language. A Kotlin/Java + * specifier is a dotted package path whether the package is the project's own + * or a dependency, so specifier shape decides nothing and the package map only + * knows manifest artifact names. The tree's own `package` declarations are the + * evidence, and the pipeline already collects them (the namespace map the + * import passes build). `assertEquals` is imported twice — from the project's + * package and from kotlin.test — exactly as JetBrains/Exposed writes it, which + * is what leaves the import map without a key and hands the call to the + * project-wide guess. */ +static void write_declared_package_shadow_fixture(const char *dir, int pad_files) { + /* One directory level only — write_temp_file's mkdir does not recurse, and + * a deeper path fails silently. The package path lives in the `package` + * declaration, not in the directory layout, which is the whole point. */ + write_temp_file(dir, "util/Assert.kt", + "package com.example.util\n" + "\n" + "fun pkgEq(a: String, b: String): Boolean {\n" + " return a == b\n" + "}\n"); + /* The lone project symbol named pkgSpec, in a third package so the guess + * that reaches it is a project-wide one and not a same-module hit. */ + write_temp_file(dir, "spec/SpecHelpers.kt", + "package com.example.spec\n" + "\n" + "fun pkgSpec(label: String): Boolean {\n" + " return label.isNotEmpty()\n" + "}\n"); + write_temp_file(dir, "app/Query.kt", + "package com.example.app\n" + "\n" + "import com.example.util.pkgEq\n" + "import kotlin.test.pkgEq\n" + "import kotlin.test.pkgSpec\n" + "import org.junit.pkgSpec\n" + "\n" + "fun pkgBuild(id: String): Boolean {\n" + " return pkgEq(id, id) && pkgSpec(id)\n" + "}\n"); + for (int i = 0; i < pad_files; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "app/Pad%02d.kt", i); + snprintf(body, sizeof(body), + "package com.example.app\n\nfun pkgPad%02d(): Int {\n return %d\n}\n", i, i); + write_temp_file(dir, name, body); + } +} + +static int assert_declared_package_shadow_contract(const char *dir, const char *db_name) { + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/%s", dir, db_name); + cbm_pipeline_t *p = cbm_pipeline_new(dir, db_path, CBM_MODE_FULL); + if (!p) { + return 1; + } + if (cbm_pipeline_run(p) != 0) { + cbm_pipeline_free(p); + return 2; + } + const char *project = cbm_pipeline_project_name(p); + cbm_store_t *s = cbm_store_open_path(db_path); + if (!s) { + cbm_pipeline_free(p); + return 3; + } + int rc = 0; + /* The package the tree declares: `com.example.util` is in-project, so the + * call keeps its edge. RED without the declared-package check — this is the + * 426-edge class measured on JetBrains/Exposed. + * + * The matching negative — an undeclared package is still cut — is asserted + * where it can be reached on BOTH resolvers: directly on the predicate in + * test_registry.c (external_import_shadow_declared_package_kept, including + * `java.net.URL`), and end-to-end in the route fixture below. The + * sequential resolver builds its own import map straight from the + * extraction records, without the ambiguity check its parallel twin + * applies, so a doubly-imported Kotlin name resolves by `import_map` there + * and never reaches this guard at all — asserting the negative here would + * assert that quirk, not this contract. */ + if (!cross_file_call_exists(s, project, "pkgBuild", "pkgEq")) { + rc = 4; + } + cbm_store_close(s); + cbm_pipeline_free(p); + return rc; +} + +TEST(pipeline_external_import_shadow_keeps_declared_package_issue1355) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_pkg_import_shadow_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_declared_package_shadow_fixture(tmp, EXTERNAL_IMPORT_SHADOW_PARALLEL_PAD); + + const char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + + cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1); + int sequential = assert_declared_package_shadow_contract(tmp, "pkg-sequential.db"); + + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + cbm_setenv("CBM_WORKERS", "4", 1); + int parallel = assert_declared_package_shadow_contract(tmp, "pkg-parallel.db"); + + if (saved_workers) { + cbm_setenv("CBM_WORKERS", saved_workers, 1); + free(saved_workers); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + if (saved_single) { + cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1); + free(saved_single); + } else { + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + } + th_rmtree(tmp); + + ASSERT_EQ(sequential, 0); + ASSERT_EQ(parallel, 0); + PASS(); +} + TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_tsjs_par_XXXXXX"); @@ -13572,6 +13698,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_python_bare_local_binding_parallel_suppresses_weak_edge); RUN_TEST(pipeline_external_import_shadow_not_bound_to_local_homonym_issue1355); RUN_TEST(pipeline_external_import_shadow_keeps_workspace_sibling_issue1355); + RUN_TEST(pipeline_external_import_shadow_keeps_declared_package_issue1355); RUN_TEST(pipeline_parallel_python_cross_only_dunder_gets_synthetic_carrier); RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier); RUN_TEST(pipeline_arg_url_rejects_non_http_slash_arguments); diff --git a/tests/test_registry.c b/tests/test_registry.c index d8494ce82..d12048d83 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -986,14 +986,19 @@ TEST(external_import_shadow_drops_package_bound_bare_call) { /* Only the relative import materialized an IMPORTS edge. */ const char *keys[] = {"users"}; - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, keys, 1, NULL)); - ASSERT_TRUE(cbm_suppress_external_import_shadow("sql", "unique_name", &arr, keys, 1, NULL)); - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "suffix_match", &arr, keys, 1, NULL)); - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "field_type_hint", &arr, keys, 1, NULL)); - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "fuzzy", &arr, keys, 1, NULL)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("eq", "unique_name", &arr, keys, 1, NULL, NULL)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("sql", "unique_name", &arr, keys, 1, NULL, NULL)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("eq", "suffix_match", &arr, keys, 1, NULL, NULL)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("eq", "field_type_hint", &arr, keys, 1, NULL, NULL)); + ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "fuzzy", &arr, keys, 1, NULL, NULL)); /* A file whose imports all failed to materialize is still covered: the * evidence is the import statement, not the map. */ - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, NULL)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, NULL, NULL)); PASS(); } @@ -1011,42 +1016,47 @@ TEST(external_import_shadow_keeps_everything_else) { const int nkeys = 2; /* Import-/receiver-/module-aware strategies are never this guard's business. */ - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "import_map", &arr, keys, nkeys, NULL)); ASSERT_FALSE( - cbm_suppress_external_import_shadow("eq", "import_map_suffix", &arr, keys, nkeys, NULL)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "same_module", &arr, keys, nkeys, NULL)); + cbm_suppress_external_import_shadow("eq", "import_map", &arr, keys, nkeys, NULL, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "import_map_suffix", &arr, keys, nkeys, + NULL, NULL)); ASSERT_FALSE( - cbm_suppress_external_import_shadow("eq", "qualified_suffix", &arr, keys, nkeys, NULL)); + cbm_suppress_external_import_shadow("eq", "same_module", &arr, keys, nkeys, NULL, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "qualified_suffix", &arr, keys, nkeys, + NULL, NULL)); ASSERT_FALSE( - cbm_suppress_external_import_shadow("eq", "lsp_ts_import", &arr, keys, nkeys, NULL)); + cbm_suppress_external_import_shadow("eq", "lsp_ts_import", &arr, keys, nkeys, NULL, NULL)); ASSERT_FALSE( - cbm_suppress_external_import_shadow("eq", "lsp_ts_method", &arr, keys, nkeys, NULL)); - ASSERT_FALSE( - cbm_suppress_external_import_shadow("eq", "service_pattern", &arr, keys, nkeys, NULL)); + cbm_suppress_external_import_shadow("eq", "lsp_ts_method", &arr, keys, nkeys, NULL, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "service_pattern", &arr, keys, nkeys, + NULL, NULL)); /* A relative specifier names a path inside the tree — a missing IMPORTS edge * there is an in-project gap, not an external binding. */ + ASSERT_FALSE(cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, keys, nkeys, + NULL, NULL)); ASSERT_FALSE( - cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, keys, nkeys, NULL)); - ASSERT_FALSE( - cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, NULL, 0, NULL)); + cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, NULL, 0, NULL, NULL)); /* A package the import map DOES bind resolved in-graph (workspace package). */ ASSERT_FALSE(cbm_suppress_external_import_shadow("getRootContainer", "unique_name", &arr, keys, - nkeys, NULL)); + nkeys, NULL, NULL)); /* A name the file never imports (called with no import at all). */ - ASSERT_FALSE( - cbm_suppress_external_import_shadow("helper", "unique_name", &arr, keys, nkeys, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("helper", "unique_name", &arr, keys, nkeys, + NULL, NULL)); /* Member and package/namespace-qualified callees belong to the * receiver-aware guards, not to this one. */ - ASSERT_FALSE( - cbm_suppress_external_import_shadow("eq.apply", "unique_name", &arr, keys, nkeys, NULL)); - ASSERT_FALSE( - cbm_suppress_external_import_shadow("eq::apply", "unique_name", &arr, keys, nkeys, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq.apply", "unique_name", &arr, keys, nkeys, + NULL, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq::apply", "unique_name", &arr, keys, nkeys, + NULL, NULL)); /* Empty / absent inputs. */ - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", NULL, &arr, keys, nkeys, NULL)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "", &arr, keys, nkeys, NULL)); - ASSERT_FALSE(cbm_suppress_external_import_shadow(NULL, "unique_name", &arr, keys, nkeys, NULL)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("", "unique_name", &arr, keys, nkeys, NULL)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", NULL, keys, nkeys, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", NULL, &arr, keys, nkeys, NULL, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "", &arr, keys, nkeys, NULL, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow(NULL, "unique_name", &arr, keys, nkeys, NULL, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("", "unique_name", &arr, keys, nkeys, NULL, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("eq", "unique_name", NULL, keys, nkeys, NULL, NULL)); PASS(); } @@ -1065,8 +1075,8 @@ TEST(external_import_shadow_relative_binding_wins_the_tie) { }; CBMImportArray a = {.items = pkg_first, .count = 2, .cap = 2}; CBMImportArray b = {.items = rel_first, .count = 2, .cap = 2}; - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &a, NULL, 0, NULL)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &b, NULL, 0, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &a, NULL, 0, NULL, NULL)); + ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &b, NULL, 0, NULL, NULL)); PASS(); } @@ -1089,10 +1099,12 @@ TEST(external_import_shadow_windows_relative_specifier_kept) { * keep the edge because it is in-tree, not external. RED before the fix * (specifier_is_relative saw '.' / '/' only, so all three were classified * external and suppressed); GREEN after. */ - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, NULL)); - ASSERT_FALSE(cbm_suppress_external_import_shadow("sql", "unique_name", &arr, NULL, 0, NULL)); ASSERT_FALSE( - cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, NULL, 0, NULL)); + cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, NULL, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("sql", "unique_name", &arr, NULL, 0, NULL, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("normalize", "unique_name", &arr, NULL, 0, NULL, NULL)); /* A bare package specifier without any drive letter or leading separator * (e.g. "drizzle-orm") must still be external — this predicate must not * become "anything with a colon or backslash". */ @@ -1100,7 +1112,8 @@ TEST(external_import_shadow_windows_relative_specifier_kept) { {.local_name = "eq", .module_path = "drizzle-orm"}, }; CBMImportArray parr = {.items = pkg_only, .count = 1, .cap = 1}; - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &parr, NULL, 0, NULL)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("eq", "unique_name", &parr, NULL, 0, NULL, NULL)); PASS(); } @@ -1127,33 +1140,140 @@ TEST(external_import_shadow_workspace_sibling_kept) { /* Exact name, subpath, and scoped subpath all resolve to an in-tree * package — the same-name fallback keeps its chance. */ - ASSERT_FALSE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, pkgs)); ASSERT_FALSE( - cbm_suppress_external_import_shadow("pgTable", "suffix_match", &arr, NULL, 0, pkgs)); + cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, pkgs, NULL)); ASSERT_FALSE( - cbm_suppress_external_import_shadow("hoisted", "unique_name", &arr, NULL, 0, pkgs)); + cbm_suppress_external_import_shadow("pgTable", "suffix_match", &arr, NULL, 0, pkgs, NULL)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("hoisted", "unique_name", &arr, NULL, 0, pkgs, NULL)); /* A genuine third-party dependency is untouched by the new check: the tree * declares no package called "vitest", so the #1355 removal still happens. */ - ASSERT_TRUE(cbm_suppress_external_import_shadow("test", "suffix_match", &arr, NULL, 0, pkgs)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("test", "suffix_match", &arr, NULL, 0, pkgs, NULL)); /* A tree with no manifests at all (pkgmap NULL) keeps the pre-existing * specifier-shape-only contract — every one of these is external again. */ - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, NULL)); ASSERT_TRUE( - cbm_suppress_external_import_shadow("pgTable", "suffix_match", &arr, NULL, 0, NULL)); + cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, NULL, NULL)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("pgTable", "suffix_match", &arr, NULL, 0, NULL, NULL)); /* An unrelated package map must not accidentally match by prefix: "drizzle" * is not "drizzle-orm", and the walk is by path segment, not by substring. */ CBMHashTable *other = cbm_ht_create(8); ASSERT_NOT_NULL(other); cbm_ht_set(other, "drizzle", (void *)"qn"); - ASSERT_TRUE(cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, other)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("eq", "unique_name", &arr, NULL, 0, other, NULL)); cbm_ht_free(other); cbm_ht_free(pkgs); PASS(); } +TEST(external_import_shadow_declared_package_kept) { + /* #1355: package-path specifiers (Java, Kotlin, C#, PHP) carry NO shape + * that separates in-tree from third-party — `import + * org.jetbrains.exposed.v1.tests.shared.assertEquals` is spelled exactly + * like `import kotlin.test.assertEquals`, and neither is "relative". Only + * the tree's own `package` declarations tell them apart; the manifests the + * package map reads (pom.xml, build.gradle) name artifacts, not packages. + * Measured on JetBrains/Exposed at 0e4d81a4: without this check the guard + * removed 426 CALLS edges, every one of them a real in-project call into + * the repository's own Assert.kt. */ + CBMImport imports[] = { + {.local_name = "assertEquals", .module_path = "org.example.tests.shared.assertEquals"}, + {.local_name = "assertEquals", .module_path = "kotlin.test.assertEquals"}, + {.local_name = "Helper", .module_path = "org.example.util.Helper"}, + {.local_name = "URL", .module_path = "java.net.URL"}, + {.local_name = "Logger", .module_path = "App\\Support\\Logger"}, + }; + CBMImportArray arr = {.items = imports, .count = 5, .cap = 5}; + + CBMHashTable *nspkgs = cbm_ht_create(8); + ASSERT_NOT_NULL(nspkgs); + cbm_ht_set(nspkgs, "org.example.tests.shared", (void *)"qn"); + cbm_ht_set(nspkgs, "org.example.util", (void *)"qn"); + cbm_ht_set(nspkgs, "App.Support", (void *)"qn"); + + /* The specifier names a member of a package the tree declares, so the walk + * strips the member segment and finds it — the call keeps its edge. */ + ASSERT_FALSE(cbm_suppress_external_import_shadow("assertEquals", "suffix_match", &arr, NULL, 0, + NULL, nspkgs)); + ASSERT_FALSE( + cbm_suppress_external_import_shadow("Helper", "unique_name", &arr, NULL, 0, NULL, nspkgs)); + /* PHP writes its separator as a backslash; the key normalization has to + * match the namespace map's, which stores dots. */ + ASSERT_FALSE( + cbm_suppress_external_import_shadow("Logger", "unique_name", &arr, NULL, 0, NULL, nspkgs)); + /* Negative space, and the effect the census called valuable: `java.net.URL` + * walks to "java.net" then "java", neither declared by any project file, so + * a bare `URL` bound to an in-tree URL class is still cut. */ + ASSERT_TRUE( + cbm_suppress_external_import_shadow("URL", "suffix_match", &arr, NULL, 0, NULL, nspkgs)); + /* NULL restores the previous contract: without the declared-package set + * every one of these reads as external again. */ + ASSERT_TRUE(cbm_suppress_external_import_shadow("assertEquals", "suffix_match", &arr, NULL, 0, + NULL, NULL)); + ASSERT_TRUE( + cbm_suppress_external_import_shadow("Helper", "unique_name", &arr, NULL, 0, NULL, NULL)); + + /* The walk is by dotted segment, not by substring: a tree that declares + * "org.example.tests" must not claim "org.example.testsuite.helper". */ + CBMImport near[] = { + {.local_name = "near", .module_path = "org.example.testsuite.helper.near"}, + }; + CBMImportArray narr = {.items = near, .count = 1, .cap = 1}; + CBMHashTable *prefix_only = cbm_ht_create(8); + ASSERT_NOT_NULL(prefix_only); + cbm_ht_set(prefix_only, "org.example.tests", (void *)"qn"); + ASSERT_TRUE(cbm_suppress_external_import_shadow("near", "unique_name", &narr, NULL, 0, NULL, + prefix_only)); + + cbm_ht_free(prefix_only); + cbm_ht_free(nspkgs); + PASS(); +} + +TEST(external_import_shadow_declared_prefix_is_permissive) { + /* #1355, known limitation pinned deliberately. The declared-package walk + * has no floor: once the tree declares `com.example`, a third-party + * library publishing under `com.example.vendor` reads as in-tree and the + * guard stands down. The call then keeps exactly the edge it has on main, + * so this direction costs a fabricated edge the guard could have removed — + * it never removes a real one, which is the side a call-resolution + * suppressor has to fail on. + * + * This test exists so the behaviour cannot change silently: tightening the + * walk (stopping at the first type-shaped segment, say) must flip these + * two assertions, not slip through unnoticed. */ + CBMImport imports[] = { + {.local_name = "vendorCall", .module_path = "com.example.vendor.pkg.vendorCall"}, + }; + CBMImportArray arr = {.items = imports, .count = 1, .cap = 1}; + + CBMHashTable *declares_prefix = cbm_ht_create(8); + ASSERT_NOT_NULL(declares_prefix); + cbm_ht_set(declares_prefix, "com.example", (void *)"qn"); + /* The tree declares only the prefix, yet the whole subtree is treated as + * in-tree — the permissive direction. */ + ASSERT_FALSE(cbm_suppress_external_import_shadow("vendorCall", "unique_name", &arr, NULL, 0, + NULL, declares_prefix)); + + /* Change nothing but the declared package and the guard fires again, which + * is what proves the previous assertion came from the prefix walk and not + * from some other clause standing down. */ + CBMHashTable *declares_other = cbm_ht_create(8); + ASSERT_NOT_NULL(declares_other); + cbm_ht_set(declares_other, "com.other", (void *)"qn"); + ASSERT_TRUE(cbm_suppress_external_import_shadow("vendorCall", "unique_name", &arr, NULL, 0, + NULL, declares_other)); + + cbm_ht_free(declares_other); + cbm_ht_free(declares_prefix); + PASS(); +} + /* ── Suite ─────────────────────────────────────────────────────── */ /* Method call THROUGH an imported symbol that is itself an indexed node @@ -1258,4 +1378,6 @@ SUITE(registry) { RUN_TEST(external_import_shadow_relative_binding_wins_the_tie); RUN_TEST(external_import_shadow_windows_relative_specifier_kept); RUN_TEST(external_import_shadow_workspace_sibling_kept); + RUN_TEST(external_import_shadow_declared_package_kept); + RUN_TEST(external_import_shadow_declared_prefix_is_permissive); } From 81820262e61f6e7f11920a5c4596b4297e793cf8 Mon Sep 17 00:00:00 2001 From: Yyunozor Date: Sat, 19 Sep 2026 23:46:59 +0200 Subject: [PATCH 029/217] fix(pipeline): run the external-import guard after route classification The #1355 guard dropped the call before the emitters ran, so it did not only remove the fabricated CALLS edge it is meant to remove: it also skipped the route, HTTP, CONFIG and URL classification that happens inside emit_classified_edge / emit_service_edge. A PR that claims to touch nothing but weak same-name CALLS edges was silently costing Route nodes, and with them every HANDLES edge that pass_route_nodes.c later bridges onto those routes. The shape is not hypothetical. A route registration whose router is a dependency is exactly a bare call bound by a package specifier: the resolved QN carries the router library name, emit_classified_edge reads it as CBM_SVC_ROUTE_REG, and the first argument is path-shaped. On a 66-file TypeScript fixture main mints __route__ANY__/orders and the guard removed it, on both resolvers. The fix is the seam the #592/#606 member guard already uses, for the same reason and with the same comment: join the predicate to drop_plain_call and let the emitter decide. Only the plain-CALLS fall-through is suppressed, so the fabricated edge still goes and every service edge stays main-identical. The two resolvers keep identical gates, as the notes in both files require. The new pipeline test asserts both halves on both resolvers: the Route node survives, and a second bare call from the same external package in the same file is still cut, so the first assertion cannot pass by the guard having been disarmed. Signed-off-by: Yyunozor --- src/pipeline/pass_calls.c | 24 ++++---- src/pipeline/pass_parallel.c | 19 +++--- tests/test_pipeline.c | 109 +++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 21 deletions(-) diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index f0f9b499f..508db7310 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -634,10 +634,22 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, * flag is set only for Python — this gate MUST match pass_parallel.c's * exactly, for the same divergence reason noted above. */ bool suppress_weak_local_binding = lang == CBM_LANG_PYTHON; + /* #1355: `import { eq } from "drizzle-orm"` binds `eq` to a package that is + * not in the indexed tree, so a project-wide same-name guess must not turn + * `eq(...)` into a CALLS edge to an unrelated project `eq`. Joined to + * drop_plain_call for the reason spelled out above: dropping the call here + * would also skip route/HTTP/CONFIG classification, and a route + * registration reached through a static import (`import static + * spark.Spark.get` + `get("/x", handler)`) is exactly a bare call bound by + * a package specifier. Suppressing only the plain-CALLS fall-through keeps + * every Route node and service edge main-identical. */ bool drop_plain_call = cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy) || cbm_suppress_weak_local_binding_call(suppress_weak_local_binding, - call->callee_is_locally_bound, res.strategy); + call->callee_is_locally_bound, res.strategy) || + cbm_suppress_external_import_shadow(call->callee_name, res.strategy, file_imports, imp_keys, + imp_count, cbm_pipeline_get_pkgmap(), + cbm_pipeline_get_nsmap()); /* Service-pattern HTTP/ASYNC calls to an EXTERNAL client library (e.g. * `requests.get("/api/orders/{id}")`) resolve to a QN containing the library @@ -669,16 +681,6 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, if (cbm_suppress_cross_language_suffix_match(lang, target_node->file_path, res.strategy)) { return 0; } - /* #1355: `import { eq } from "drizzle-orm"` binds `eq` to a package that is - * not in the indexed tree, so a project-wide same-name guess must not turn - * `eq(...)` into a CALLS edge to an unrelated project `eq`. Placed with the - * #725 guard, after the service-pattern bypasses above, so no HTTP/route - * edge can be lost to it. */ - if (cbm_suppress_external_import_shadow(call->callee_name, res.strategy, file_imports, imp_keys, - imp_count, cbm_pipeline_get_pkgmap(), - cbm_pipeline_get_nsmap())) { - return 0; - } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, imp_count, drop_plain_call); return SKIP_ONE; diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index d7aa7bba9..7af3f5c7a 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2499,10 +2499,18 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB /* Bare-call local-binding suppression — see the note in pass_calls.c. * This gate MUST stay identical to the one there. */ bool suppress_weak_local_binding = lang == CBM_LANG_PYTHON; + /* #1355: same guard as pass_calls.c, and joined to drop_plain_call for + * the same reason — a `continue` here would also skip route/HTTP/CONFIG + * classification, and a static-import route registration is a bare call + * bound by a package specifier. Suppress only the plain-CALLS + * fall-through so every Route node stays main-identical. */ bool drop_plain_call = cbm_suppress_weak_member_match(suppress_weak_member, call->is_method, res.strategy) || cbm_suppress_weak_local_binding_call(suppress_weak_local_binding, - call->callee_is_locally_bound, res.strategy); + call->callee_is_locally_bound, res.strategy) || + cbm_suppress_external_import_shadow(call->callee_name, res.strategy, &result->imports, + imp_keys, imp_count, cbm_pipeline_get_pkgmap(), + cbm_pipeline_get_nsmap()); /* Service-pattern HTTP/ASYNC client call (`requests.get(url)`): the * service signal lives in the callee_name. The registry can mis-resolve @@ -2573,15 +2581,6 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB * CALLS edge across a language boundary. */ continue; } - if (target_node && source_node->id != target_node->id && - cbm_suppress_external_import_shadow(call->callee_name, res.strategy, &result->imports, - imp_keys, imp_count, cbm_pipeline_get_pkgmap(), - cbm_pipeline_get_nsmap())) { - /* #1355: same guard as pass_calls.c — a bare call bound by an - * external package import must not become a CALLS edge to an - * unrelated project symbol of the same name. */ - continue; - } if (!target_node || source_node->id == target_node->id) { /* HTTP/ASYNC calls to an EXTERNAL client library (`requests.get(url)`) * resolve to an unindexed QN (target_node == NULL), but their edge diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 36bb243a3..fa6e16fb3 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -5348,6 +5348,114 @@ TEST(pipeline_external_import_shadow_keeps_declared_package_issue1355) { PASS(); } +/* #1355: the guard must suppress ONLY the plain-CALLS fall-through, never the + * route/HTTP/CONFIG classification that runs inside the emitters. A bare call + * bound by a package import is exactly the shape a route registration takes + * when the router is a dependency, and dropping it before classification costs + * the Route node and every HANDLES edge downstream of it. Same reasoning, and + * the same drop_plain_call seam, as the #592/#606 member guard. */ +static void write_route_shadow_fixture(const char *dir, int pad_files) { + /* The project symbols the guess would bind to. The file name puts "express" + * in their qualified names, which is what makes the resolved QN classify as + * a route registration — the same substring match main uses on real trees. */ + write_temp_file(dir, "app/express-routes.ts", + "export function regGet(path: string, handler: unknown): string {\n" + " return path;\n" + "}\n" + "export function regSpec(label: string): string {\n" + " return label;\n" + "}\n"); + /* Both names come from a package outside the tree. `regGet` carries a + * path-shaped first argument, so it is a route registration; `regSpec` is + * an ordinary fabricated edge and must still go. */ + write_temp_file(dir, "app/server.ts", + "import { regGet, regSpec } from 'vendor-router';\n" + "export function boot(): string {\n" + " return regGet('/orders', () => 'ok') + regSpec('case');\n" + "}\n"); + for (int i = 0; i < pad_files; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "app/pad_%02d.ts", i); + snprintf(body, sizeof(body), "export function routePad%02d(): number { return %d; }\n", i, + i); + write_temp_file(dir, name, body); + } +} + +static int assert_route_shadow_contract(const char *dir, const char *db_name) { + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/%s", dir, db_name); + cbm_pipeline_t *p = cbm_pipeline_new(dir, db_path, CBM_MODE_FULL); + if (!p) { + return 1; + } + if (cbm_pipeline_run(p) != 0) { + cbm_pipeline_free(p); + return 2; + } + const char *project = cbm_pipeline_project_name(p); + cbm_store_t *s = cbm_store_open_path(db_path); + if (!s) { + cbm_pipeline_free(p); + return 3; + } + int rc = 0; + /* (1) the Route node survives the guard. RED when the guard returns before + * the emitter: main mints __route__ANY__/orders here and the published + * guard lost it, together with every HANDLES edge that hangs off it. */ + if (count_nodes_named(s, project, "/orders") < 1) { + rc = 4; + } + /* (2) the fabricated plain-CALLS edge in the same file is still removed, so + * (1) cannot pass by the guard having been disarmed. */ + if (rc == 0 && cross_file_call_exists(s, project, "boot", "regSpec")) { + rc = 5; + } + cbm_store_close(s); + cbm_pipeline_free(p); + return rc; +} + +TEST(pipeline_external_import_shadow_keeps_route_registration_issue1355) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_route_import_shadow_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_route_shadow_fixture(tmp, EXTERNAL_IMPORT_SHADOW_PARALLEL_PAD); + + const char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + + cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1); + int sequential = assert_route_shadow_contract(tmp, "route-sequential.db"); + + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + cbm_setenv("CBM_WORKERS", "4", 1); + int parallel = assert_route_shadow_contract(tmp, "route-parallel.db"); + + if (saved_workers) { + cbm_setenv("CBM_WORKERS", saved_workers, 1); + free(saved_workers); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + if (saved_single) { + cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1); + free(saved_single); + } else { + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + } + th_rmtree(tmp); + + ASSERT_EQ(sequential, 0); + ASSERT_EQ(parallel, 0); + PASS(); +} + TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_tsjs_par_XXXXXX"); @@ -13699,6 +13807,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_external_import_shadow_not_bound_to_local_homonym_issue1355); RUN_TEST(pipeline_external_import_shadow_keeps_workspace_sibling_issue1355); RUN_TEST(pipeline_external_import_shadow_keeps_declared_package_issue1355); + RUN_TEST(pipeline_external_import_shadow_keeps_route_registration_issue1355); RUN_TEST(pipeline_parallel_python_cross_only_dunder_gets_synthetic_carrier); RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier); RUN_TEST(pipeline_arg_url_rejects_non_http_slash_arguments); From 2a2be4a2e0e20d8102e3c3eaa0d97250891baf7a Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Wed, 2 Sep 2026 07:03:29 +0800 Subject: [PATCH 030/217] fix(ts): resolve inferred imported constructor receivers Signed-off-by: mikemikimike <13286568797@163.com> --- internal/cbm/lsp/ts_lsp.c | 17 +++++++++++++---- tests/repro/repro_lsp_ts.c | 26 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index fc8b8911f..ec0b80e0e 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -2176,10 +2176,19 @@ const CBMType *ts_eval_expr_type(TSLSPContext *ctx, TSNode node) { if (!ts_node_is_null(ctor)) { char *cname = node_text(ctx, ctor); if (cname) { - // Bare class name → qualify against module. - if (strchr(cname, '.') == NULL && ctx->module_qn) { - const char *qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->module_qn, cname); - result = cbm_type_named(ctx->arena, qn); + // Resolve lexical/import bindings before falling back to the + // current module. This preserves the imported class QN for + // inferred locals such as `const x = new ImportedClass()`. + if (strchr(cname, '.') == NULL) { + const CBMType *bound = type_of_identifier(ctx, cname); + if (bound && bound->kind == CBM_TYPE_NAMED && + bound->data.named.qualified_name) { + result = bound; + } else if (ctx->module_qn) { + const char *qn = + cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->module_qn, cname); + result = cbm_type_named(ctx->arena, qn); + } } else { result = cbm_type_named(ctx->arena, cname); } diff --git a/tests/repro/repro_lsp_ts.c b/tests/repro/repro_lsp_ts.c index bedcb33d8..c41bd8291 100644 --- a/tests/repro/repro_lsp_ts.c +++ b/tests/repro/repro_lsp_ts.c @@ -297,6 +297,23 @@ static const RFile kTsImport[] = { "function caller(v: number): number { return helper(v); }\n"}, }; +/* An imported class constructed through an inferred local variable must retain + * the imported class QN for subsequent member dispatch. The explicit type + * annotation path already covers this shape; this fixture isolates inference. + */ +static const RFile kTsInferredImportedMethod[] = { + {"store.ts", + "export class WidgetStore {\n" + " findById(id: string): string { return id; }\n" + "}\n"}, + {"service.ts", + "import { WidgetStore } from \"./store\";\n" + "function load(id: string): string {\n" + " const store = new WidgetStore();\n" + " return store.findById(id);\n" + "}\n"}, +}; + /* lsp_ts_jsx — JSX element whose tag is a module-local component * function (ts_lsp.c:2643-2647). TSX only (jsx_mode); the tag's first letter is * uppercase so it is NOT treated as an intrinsic HTML element; it resolves via @@ -370,6 +387,14 @@ TEST(repro_lsp_ts_import) { "lsp_ts_import"); } +TEST(repro_lsp_ts_inferred_import_receiver) { + return assert_lsp_strategy_files( + kTsInferredImportedMethod, + (int)(sizeof(kTsInferredImportedMethod) / + sizeof(kTsInferredImportedMethod[0])), + "lsp_ts_method"); +} + TEST(repro_lsp_ts_jsx) { return assert_lsp_strategy("app.tsx", kTsxJsx, "lsp_ts_jsx"); } @@ -403,6 +428,7 @@ SUITE(repro_lsp_ts) { RUN_TEST(repro_lsp_ts_method); RUN_TEST(repro_lsp_ts_namespace); RUN_TEST(repro_lsp_ts_import); + RUN_TEST(repro_lsp_ts_inferred_import_receiver); RUN_TEST(repro_lsp_ts_jsx); RUN_TEST(repro_lsp_ts_jsx_import); RUN_TEST(repro_lsp_ts_default); From aa4da14d23dc28df58b9881dc66d89c0d1c0d788 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Wed, 2 Sep 2026 08:58:42 +0800 Subject: [PATCH 031/217] test(ts): cover inferred stdlib constructor receivers Signed-off-by: mikemikimike <13286568797@163.com> --- tests/repro/repro_lsp_ts.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/repro/repro_lsp_ts.c b/tests/repro/repro_lsp_ts.c index c41bd8291..8c441af22 100644 --- a/tests/repro/repro_lsp_ts.c +++ b/tests/repro/repro_lsp_ts.c @@ -314,6 +314,14 @@ static const RFile kTsInferredImportedMethod[] = { "}\n"}, }; +/* A standard-library class constructed through an inferred local variable must + * retain the bare stdlib QN so subsequent member dispatch finds Map.get. */ +static const char kTsInferredStdlibMethod[] = + "function lookup(id: string): string {\n" + " const m = new Map();\n" + " return m.get(id);\n" + "}\n"; + /* lsp_ts_jsx — JSX element whose tag is a module-local component * function (ts_lsp.c:2643-2647). TSX only (jsx_mode); the tag's first letter is * uppercase so it is NOT treated as an intrinsic HTML element; it resolves via @@ -395,6 +403,11 @@ TEST(repro_lsp_ts_inferred_import_receiver) { "lsp_ts_method"); } +TEST(repro_lsp_ts_inferred_stdlib_receiver) { + return assert_lsp_strategy("main.ts", kTsInferredStdlibMethod, + "lsp_ts_method"); +} + TEST(repro_lsp_ts_jsx) { return assert_lsp_strategy("app.tsx", kTsxJsx, "lsp_ts_jsx"); } @@ -429,6 +442,7 @@ SUITE(repro_lsp_ts) { RUN_TEST(repro_lsp_ts_namespace); RUN_TEST(repro_lsp_ts_import); RUN_TEST(repro_lsp_ts_inferred_import_receiver); + RUN_TEST(repro_lsp_ts_inferred_stdlib_receiver); RUN_TEST(repro_lsp_ts_jsx); RUN_TEST(repro_lsp_ts_jsx_import); RUN_TEST(repro_lsp_ts_default); From 56de845327dce03cdd1269b8aa73c6a1ecdb5cfa Mon Sep 17 00:00:00 2001 From: lorenzozanee Date: Sun, 20 Sep 2026 16:35:23 +0800 Subject: [PATCH 032/217] fix(cli): support non-ASCII Windows agent paths Signed-off-by: lorenzozanee --- src/cli/cli.c | 9 +++------ tests/test_cli.c | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index 51fbafc02..7cd59e0bf 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -19,6 +19,7 @@ #include "daemon/runtime.h" #include "daemon/version_cohort.h" #include "foundation/compat.h" +#include "foundation/compat_fs.h" #include "foundation/platform.h" #include "foundation/constants.h" #include "foundation/log.h" @@ -2600,12 +2601,8 @@ int cbm_remove_zed_mcp_owned(const char *binary_path, const char *config_path) { /* ── Agent detection ──────────────────────────────────────────── */ static bool dir_exists(const char *path) { - struct stat st; -#ifndef _WIN32 - return lstat(path, &st) == 0 && S_ISDIR(st.st_mode); -#else - return stat(path, &st) == 0 && S_ISDIR(st.st_mode); -#endif + cbm_path_info_t info = {0}; + return cbm_path_info_utf8(path, &info) == CBM_PATH_INFO_OK && info.is_directory; } /* Resolve the Claude Code config dir. diff --git a/tests/test_cli.c b/tests/test_cli.c index 06ebb3841..b3c3c2f27 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -13074,6 +13074,24 @@ TEST(cli_detect_agents_finds_zed_in_roaming_appdata) { } #endif +#ifdef _WIN32 +TEST(cli_detect_agents_handles_non_ascii_windows_profile) { + char home[1024]; + snprintf(home, sizeof(home), "%s/cbm-用户-profile", cbm_tmpdir()); + test_rmdir_r(home); + char claude_dir[1024]; + snprintf(claude_dir, sizeof(claude_dir), "%s/.claude", home); + if (!test_mkdirp(claude_dir)) + FAIL("could not create non-ASCII Windows profile path"); + + cbm_detected_agents_t agents = cbm_detect_agents(home); + test_rmdir_r(home); + if (!agents.claude_code) + FAIL("Claude detection must support non-ASCII Windows profile paths"); + PASS(); +} +#endif + TEST(cli_detect_agents_finds_antigravity) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-detect-XXXXXX"); @@ -16258,6 +16276,7 @@ SUITE(cli) { #endif #ifdef _WIN32 RUN_TEST(cli_detect_agents_finds_zed_in_roaming_appdata); + RUN_TEST(cli_detect_agents_handles_non_ascii_windows_profile); #endif RUN_TEST(cli_detect_agents_finds_antigravity); RUN_TEST(cli_detect_agents_finds_kilocode); From 7036c4d5996253c05d720ec4d0b3008509b4cad4 Mon Sep 17 00:00:00 2001 From: Sanskar gupta Date: Sun, 20 Sep 2026 14:11:17 +0000 Subject: [PATCH 033/217] test: pin Solidity modifier_invocation decorator tracking (#2090) Signed-off-by: Sanskar gupta --- tests/test_extraction.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_extraction.c b/tests/test_extraction.c index cb01dcf14..a3610c207 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -3999,6 +3999,25 @@ TEST(extract_java_method_annotations_issue382) { PASS(); } +/* Issue #2090: Solidity modifier_invocation tracked as decorator. */ +TEST(solidity_modifier_decorator) { + CBMFileResult *r = extract("pragma solidity ^0.8.0;\n" + "contract Vault {\n" + " modifier onlyOwner() { _; }\n" + " function withdraw() public onlyOwner {\n" + " }\n" + "}\n", + CBM_LANG_SOLIDITY, "t", "Vault.sol"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + const CBMDefinition *m = find_def_by_name(r, "withdraw"); + ASSERT_NOT_NULL(m); + ASSERT(decorators_contain(m, "onlyOwner")); + cbm_free_result(r); + PASS(); +} + + /* ── ArkTS (HarmonyOS .ets) ─────────────────────────────────────── */ TEST(arkts_component_struct) { @@ -7445,6 +7464,7 @@ SUITE(extraction) { RUN_TEST(js_index_module_qn_not_collide_with_folder); RUN_TEST(python_regular_module_qn_unchanged); RUN_TEST(extract_java_method_annotations_issue382); + RUN_TEST(solidity_modifier_decorator); RUN_TEST(arkts_component_struct); RUN_TEST(arkts_exported_struct_decorators); RUN_TEST(arkts_member_decorators); From f9350caae0ccccf823d8cde85683a900b257f239 Mon Sep 17 00:00:00 2001 From: Sanskar gupta Date: Sun, 20 Sep 2026 14:52:06 +0000 Subject: [PATCH 034/217] fix: remove extra blank line Signed-off-by: Sanskar gupta --- tests/test_extraction.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_extraction.c b/tests/test_extraction.c index a3610c207..2bcb54210 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4017,7 +4017,6 @@ TEST(solidity_modifier_decorator) { PASS(); } - /* ── ArkTS (HarmonyOS .ets) ─────────────────────────────────────── */ TEST(arkts_component_struct) { From 9e793df87b51f07d3741c55ea6e27367ee1a7b7d Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Mon, 21 Sep 2026 11:48:06 -0500 Subject: [PATCH 035/217] fix(daemon): keep held runtime files fresh and exit when one is lost macOS com.apple.tmp_cleaner deletes /tmp files whose atime, mtime and ctime are all older than three days. The daemon's long-held lock files are empty and never written, so under a daemon that lives past three days the cleaner unlinks them while they are held. The next participant then creates and locks a fresh inode: forked index workers see the cohort daemon marker as free, read the live daemon as uncoordinated and exit 1 on every reindex, and a lost IPC lifetime lock or cohort lifetime file stops protecting the generation at all. A lost socket identity marker additionally makes listener close leave the socket pair behind. The host lifetime loop now runs a heartbeat every 60 s over every file the generation holds (cohort daemon claim and lease, participant guard, IPC lifetime reservation, identity marker). Each file is refreshed with futimens through its held descriptor, only after proving the path still names that descriptor, so the cleaner never selects it. If any file is already lost, the daemon logs daemon.lifetime_end reason=coordination_file_lost and stops cleanly; the next client starts a coordinated generation. Re-claiming in place would race that client. Windows needs no refresh (no age-based cleaner; held lock handles deny deletion), so its entry points only validate the handles. Closes #2178 Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Shane McCarron --- src/daemon/host.c | 57 ++++++++++++++++++- src/daemon/ipc.c | 57 +++++++++++++++++++ src/daemon/ipc.h | 13 +++++ src/daemon/runtime.c | 4 ++ src/daemon/runtime.h | 4 ++ src/daemon/version_cohort.c | 19 +++++++ src/daemon/version_cohort.h | 9 +++ src/foundation/private_file_lock.c | 32 +++++++++++ src/foundation/private_file_lock_internal.h | 7 +++ tests/test_daemon_ipc.c | 56 ++++++++++++++++-- tests/test_private_file_lock.c | 63 ++++++++++++++++++++- 11 files changed, 314 insertions(+), 7 deletions(-) diff --git a/src/daemon/host.c b/src/daemon/host.c index 3e509e207..856af3ee4 100644 --- a/src/daemon/host.c +++ b/src/daemon/host.c @@ -52,6 +52,9 @@ enum { HOST_HTTP_RETRY_INITIAL_MS = 1000, HOST_HTTP_RETRY_MAX_MS = 30000, HOST_WATCH_INTERVAL_MS = 5000, + /* Age-based temp cleaners need days of inactivity (macOS tmp_cleaner: 3); + * a minute bounds the blind window after a long sleep (#2178). */ + HOST_COORDINATION_TOUCH_MS = 60000, HOST_CONFLICT_LOG_CAP = 1024 * 1024, HOST_OPERATION_LOG_CAP = 5 * 1024 * 1024, HOST_PATH_CAP = 4096, @@ -868,10 +871,42 @@ static bool host_background_start(host_state_t *host) { return true; } +/* Coordination handles the host retains for the generation's whole lifetime. */ +typedef struct { + const cbm_daemon_ipc_endpoint_t *endpoint; + cbm_version_cohort_lease_t *cohort_lease; + cbm_version_cohort_daemon_claim_t *daemon_claim; + cbm_daemon_ipc_participant_guard_t *participant_guard; +} host_coordination_t; + +/* Refresh every long-held runtime file and name the first one found lost, or + * return NULL. A held lock whose path now names another inode coordinates + * nothing: forked index workers read the daemon as uncoordinated and a peer + * can start a second generation (#2178). */ +static const char *host_coordination_touch(const host_coordination_t *coordination, + cbm_daemon_runtime_service_t *service) { + if (!cbm_version_cohort_daemon_claim_touch(coordination->daemon_claim)) { + return "cohort_daemon_claim"; + } + if (!cbm_version_cohort_lease_touch(coordination->cohort_lease)) { + return "cohort_lease"; + } + if (!cbm_daemon_ipc_participant_guard_touch(coordination->endpoint, + coordination->participant_guard)) { + return "participant_guard"; + } + if (!cbm_daemon_runtime_service_touch_listener(service)) { + return "listener"; + } + return NULL; +} + static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service, - atomic_int *stop_requested, host_state_t *host, bool permanent) { + atomic_int *stop_requested, host_state_t *host, bool permanent, + const host_coordination_t *coordination) { uint64_t initial_deadline = cbm_now_ms() + HOST_INITIAL_CLIENT_TIMEOUT_MS; uint64_t stopping_deadline = 0; + uint64_t next_touch = 0; for (;;) { cbm_daemon_runtime_service_state_t state = cbm_daemon_runtime_service_state(service); if (state == CBM_DAEMON_RUNTIME_SERVICE_EXITED) { @@ -903,6 +938,17 @@ static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service, stop ? "stop_requested" : "initial_window_expired"); return cbm_daemon_runtime_service_stop(service, HOST_RUNTIME_SHUTDOWN_MS); } + /* Permanent generations too: exiting lets the next client start a + * coordinated generation, while re-claiming would race that client. */ + if (cbm_now_ms() >= next_touch) { + next_touch = cbm_now_ms() + HOST_COORDINATION_TOUCH_MS; + const char *lost = host_coordination_touch(coordination, service); + if (lost) { + cbm_log_warn("daemon.lifetime_end", "reason", "coordination_file_lost", "file", + lost); + return cbm_daemon_runtime_service_stop(service, HOST_RUNTIME_SHUTDOWN_MS); + } + } host_http_reconcile_at(host, cbm_now_ms(), false); /* Retire an ephemeral generation that lingered for cold-storm cohort * participants once they drain, or once its bounded linger elapses. @@ -1123,7 +1169,14 @@ int cbm_daemon_host_run(const cbm_daemon_host_config_t *config) { : "unavailable", "memory_budget_bytes", memory_budget, "physical_job_limit", physical_job_limit, "worker_memory_budget_bytes", worker_memory_budget); - if (!host_wait_for_lifetime(service, config->stop_requested, &host, config->permanent)) { + host_coordination_t coordination = { + .endpoint = config->endpoint, + .cohort_lease = cohort_lease, + .daemon_claim = daemon_claim, + .participant_guard = participant_guard, + }; + if (!host_wait_for_lifetime(service, config->stop_requested, &host, config->permanent, + &coordination)) { host_force_terminate("runtime"); } diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index a92befb08..53da43542 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -3220,6 +3220,46 @@ void cbm_daemon_ipc_listener_close(cbm_daemon_ipc_listener_t *listener) { free(listener); } +/* Touch through the held descriptor only after proving the path still names + * it: refreshing by path would keep a replacement inode fresh and hide the + * loss (#2178). */ +static bool posix_held_file_touch(int directory_fd, const char *base_name, int fd) { + return private_regular_file_snapshot(directory_fd, base_name, fd, 1, NULL) && + futimens(fd, NULL) == 0; +} + +static bool posix_held_lock_touch(int directory_fd, const process_lock_entry_t *entry, int fd) { + return entry && posix_held_file_touch(directory_fd, entry->lock_name, fd); +} + +bool cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener) { + if (!listener || listener->dir_fd < 0 || listener->owner_pid != getpid() || + !listener->lifetime_reservation) { + return false; + } + bool lifetime = + posix_held_lock_touch(listener->dir_fd, listener->lifetime_reservation->process_entry, + listener->lifetime_reservation->fd); + const cbm_daemon_ipc_participant_guard_t *guard = listener->participant_guard; + bool participant = + !guard || + posix_held_lock_touch(listener->dir_fd, guard->legacy_process_entry, guard->legacy_fd); + /* The marker is not held open; reopen it and require the published inode. */ + int marker_fd = openat(listener->dir_fd, listener->socket_identity_name, + O_RDONLY | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK); + struct stat marker_status; + bool marker = marker_fd >= 0 && + private_regular_file_snapshot(listener->dir_fd, listener->socket_identity_name, + marker_fd, 1, &marker_status) && + marker_status.st_dev == listener->identity_device && + marker_status.st_ino == listener->identity_inode && + futimens(marker_fd, NULL) == 0; + if (marker_fd >= 0) { + (void)close(marker_fd); + } + return lifetime && participant && marker; +} + int cbm_daemon_ipc_accept(cbm_daemon_ipc_listener_t *listener, uint32_t timeout_ms, cbm_daemon_ipc_connection_t **connection_out) { if (connection_out) { @@ -3603,6 +3643,12 @@ bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t return true; } +bool cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint, + cbm_daemon_ipc_participant_guard_t *guard) { + return endpoint && guard && guard->owner_pid == getpid() && + posix_held_lock_touch(endpoint->dir_fd, guard->legacy_process_entry, guard->legacy_fd); +} + int cbm_daemon_ipc_local_transition_try_acquire( const cbm_daemon_ipc_endpoint_t *endpoint, cbm_daemon_ipc_local_transition_t **transition_out) { if (transition_out) { @@ -6746,6 +6792,17 @@ bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t return true; } +/* Windows has no age-based cleaner of the private runtime directory, and its + * held handles deny deletion; the #2178 heartbeat has nothing to refresh. */ +bool cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener) { + return listener != NULL; +} + +bool cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint, + cbm_daemon_ipc_participant_guard_t *guard) { + return endpoint && guard; +} + int cbm_daemon_ipc_local_transition_try_acquire( const cbm_daemon_ipc_endpoint_t *endpoint, cbm_daemon_ipc_local_transition_t **transition_out) { if (transition_out) { diff --git a/src/daemon/ipc.h b/src/daemon/ipc.h index 545ecb357..ed214ea79 100644 --- a/src/daemon/ipc.h +++ b/src/daemon/ipc.h @@ -76,6 +76,15 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( cbm_daemon_ipc_lifetime_reservation_t **reservation_io); void cbm_daemon_ipc_listener_close(cbm_daemon_ipc_listener_t *listener); +/* Heartbeat for the daemon's long-lived runtime artifacts: refresh the held + * lifetime reservation, the listener's participant guard (if it owns one) and + * the published identity marker so an age-based temp cleaner never deletes + * them (#2178). Returns false once any of them no longer names the file this + * listener owns: a lost lifetime file lets a peer start a second generation, + * and a lost identity marker makes listener close leave the socket pair + * behind. Only the owning thread may call it while the listener is open. */ +bool cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener); + /* Create or validate one private current-user directory at an already * canonical local path. Ancestors are handle-validated without mutation: * POSIX permits only root/current-user owners and no group/other write (apart @@ -270,6 +279,10 @@ bool cbm_daemon_ipc_startup_lock_release(cbm_daemon_ipc_startup_lock_t **lock_io int cbm_daemon_ipc_participant_guard_try_join(const cbm_daemon_ipc_endpoint_t *endpoint, cbm_daemon_ipc_participant_guard_t **guard_out); bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t **guard_io); +/* Same heartbeat contract as cbm_daemon_ipc_listener_touch, for a guard the + * caller retains itself. */ +bool cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint, + cbm_daemon_ipc_participant_guard_t *guard); /* Standalone CLI work joins the legacy-compatible current participant group * without becoming a daemon client. Acquisition retains startup-v2 only for diff --git a/src/daemon/runtime.c b/src/daemon/runtime.c index efa41a020..7fb7d32a9 100644 --- a/src/daemon/runtime.c +++ b/src/daemon/runtime.c @@ -2578,6 +2578,10 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_ return count; } +bool cbm_daemon_runtime_service_touch_listener(cbm_daemon_runtime_service_t *service) { + return service && service->listener && cbm_daemon_ipc_listener_touch(service->listener); +} + void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service) { if (!service) { return; diff --git a/src/daemon/runtime.h b/src/daemon/runtime.h index 7a31dab61..304d7c898 100644 --- a/src/daemon/runtime.h +++ b/src/daemon/runtime.h @@ -336,6 +336,10 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_ * an unbounded idle hang. The host lifetime loop calls this every tick; it is a * no-op unless a last-committed-client linger is armed. */ void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service); +/* #2178 heartbeat over the listener's runtime artifacts; see + * cbm_daemon_ipc_listener_touch. Call only from the thread that stops and + * frees the service, which is the only thread that closes the listener. */ +bool cbm_daemon_runtime_service_touch_listener(cbm_daemon_runtime_service_t *service); size_t cbm_daemon_runtime_service_job_subscribers(cbm_daemon_runtime_service_t *service, const char *project_key); uint64_t cbm_daemon_runtime_service_client_process_id(cbm_daemon_runtime_service_t *service, diff --git a/src/daemon/version_cohort.c b/src/daemon/version_cohort.c index 3742a1330..82cc63b32 100644 --- a/src/daemon/version_cohort.c +++ b/src/daemon/version_cohort.c @@ -363,6 +363,25 @@ cbm_private_file_lock_status_t cbm_version_cohort_lease_release( return result; } +static bool version_cohort_lock_touch(cbm_private_file_lock_t *lock) { + return !lock || cbm_private_file_lock_touch(lock) == CBM_PRIVATE_FILE_LOCK_OK; +} + +bool cbm_version_cohort_lease_touch(cbm_version_cohort_lease_t *lease) { + if (!lease) { + return false; + } + /* Evaluate all three so one lost file does not leave the others to age. */ + bool lifetime = version_cohort_lock_touch(lease->lifetime); + bool admission = version_cohort_lock_touch(lease->admission); + bool maintenance = version_cohort_lock_touch(lease->maintenance); + return lifetime && admission && maintenance; +} + +bool cbm_version_cohort_daemon_claim_touch(cbm_version_cohort_daemon_claim_t *claim) { + return claim && claim->marker && version_cohort_lock_touch(claim->marker); +} + static cbm_version_cohort_status_t version_cohort_failed(cbm_version_cohort_lease_t *lease, cbm_version_cohort_status_t status, cbm_version_cohort_lease_t **lease_out) { diff --git a/src/daemon/version_cohort.h b/src/daemon/version_cohort.h index 51304cc95..ac79c8b86 100644 --- a/src/daemon/version_cohort.h +++ b/src/daemon/version_cohort.h @@ -145,6 +145,15 @@ cbm_version_cohort_daemon_presence_t cbm_version_cohort_daemon_presence_under_tr cbm_private_file_lock_status_t cbm_version_cohort_lease_release( cbm_version_cohort_lease_t **lease_io); +/* Heartbeat for a long-lived holder: refresh every lock file the lease or + * claim holds and report false once any held file was unlinked or replaced. + * An age-based temp cleaner deleting a held file silently ends coordination, + * because the next participant creates and locks a fresh inode (#2178). The + * only safe response to false is an orderly exit; re-claiming would race + * that participant. */ +bool cbm_version_cohort_lease_touch(cbm_version_cohort_lease_t *lease); +bool cbm_version_cohort_daemon_claim_touch(cbm_version_cohort_daemon_claim_t *claim); + /* Refuses teardown while a lease or retryable cleanup handle remains. */ cbm_private_file_lock_status_t cbm_version_cohort_manager_free( cbm_version_cohort_manager_t **manager_io); diff --git a/src/foundation/private_file_lock.c b/src/foundation/private_file_lock.c index 80df0d17c..a7d83172e 100644 --- a/src/foundation/private_file_lock.c +++ b/src/foundation/private_file_lock.c @@ -482,6 +482,24 @@ cbm_private_file_lock_status_t cbm_private_file_lock_payload_write(cbm_private_f return valid ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_IO; } +cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock_t *lock) { + if (!lock) { + return CBM_PRIVATE_FILE_LOCK_UNSAFE; + } + if (!cbm_private_file_lock_fork_guard_enter()) { + return CBM_PRIVATE_FILE_LOCK_IO; + } + /* An unlinked file keeps its inode alive for this handle with st_nlink 0, + * which the payload validity check already rejects. */ + bool linked = private_payload_fd_valid(lock, NULL); + bool touched = linked && futimens(lock->fd, NULL) == 0; + cbm_private_file_lock_fork_guard_leave(); + if (!linked) { + return CBM_PRIVATE_FILE_LOCK_UNSAFE; + } + return touched ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_IO; +} + cbm_private_file_lock_status_t cbm_private_file_lock_release(cbm_private_file_lock_t **lock_io) { if (!lock_io || !*lock_io) { return CBM_PRIVATE_FILE_LOCK_IO; @@ -1450,6 +1468,20 @@ cbm_private_file_lock_status_t cbm_private_file_lock_payload_write(cbm_private_f return valid ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_IO; } +cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock_t *lock) { + /* Windows has no age-based cleaner of the private lock directory; only + * confirm the held handle still names a linked file. */ + if (!lock) { + return CBM_PRIVATE_FILE_LOCK_UNSAFE; + } + if (!cbm_private_file_lock_fork_guard_enter()) { + return CBM_PRIVATE_FILE_LOCK_IO; + } + bool linked = private_win_payload_handle_valid(lock); + cbm_private_file_lock_fork_guard_leave(); + return linked ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_UNSAFE; +} + static bool private_win_release_unlock(cbm_private_file_lock_t *lock) { lock->test_unlock_attempts++; if (lock->test_fail_unlock_once) { diff --git a/src/foundation/private_file_lock_internal.h b/src/foundation/private_file_lock_internal.h index 8801d9b1a..b8ec34ac7 100644 --- a/src/foundation/private_file_lock_internal.h +++ b/src/foundation/private_file_lock_internal.h @@ -61,6 +61,13 @@ cbm_private_file_lock_status_t cbm_private_file_lock_payload_write(cbm_private_f const void *buffer, size_t length); +/* Long-lived holders call this periodically. It refreshes the held file's + * timestamps through the validated handle so age-based temp cleaners (macOS + * tmp_cleaner, systemd-tmpfiles) never select it, and returns UNSAFE once the + * handle no longer names a linked file: a peer opening the path would then + * lock a different inode and the lock coordinates nothing (#2178). */ +cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock_t *lock); + /* Forces the next successfully acquired native lock down the post-lock * validation cleanup path and injects pre-call release failures there. */ bool cbm_private_lock_directory_fail_post_acquire_cleanup_for_test( diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index b451f9bb3..d29a1e6c0 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -1004,8 +1004,8 @@ TEST(daemon_ipc_windows_sid_trust_accepts_local_admin_rejects_foreign_500) { (void)CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL, NULL, &builtin_needed); if (builtin_needed > 0) { builtin_admins = malloc(builtin_needed); - if (builtin_admins && - !CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL, builtin_admins, &builtin_needed)) { + if (builtin_admins && !CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL, builtin_admins, + &builtin_needed)) { free(builtin_admins); builtin_admins = NULL; } @@ -2063,6 +2063,54 @@ TEST(daemon_ipc_lifetime_reservation_survives_saturated_second_listen) { PASS(); } +/* #2178: an age-based temp cleaner unlinked the live daemon's lifetime lock + * and identity marker. The heartbeat must report either loss, so the daemon + * exits instead of serving under a lifetime file that coordinates nothing or + * leaving an unremovable socket pair behind at close. */ +TEST(daemon_ipc_listener_touch_detects_lost_runtime_files) { +#ifdef _WIN32 + SKIP_PLATFORM("unlink-while-held of runtime files is POSIX behavior"); +#else + static const char key[] = "2178a2178a2178a0"; + static const char *const lost_files[] = {"lifetime.lock", "sock.identity"}; + char parent[TEST_PATH_CAP] = {0}; + char runtime_dir[TEST_PATH_CAP] = {0}; + cbm_daemon_ipc_endpoint_t *endpoint = NULL; + bool started[2] = {false, false}; + bool touched_intact[2] = {false, false}; + bool unlinked[2] = {false, false}; + bool touched_after_loss[2] = {true, true}; + + if (ipc_test_parent_new(parent, "listener-touch")) { + endpoint = cbm_daemon_ipc_endpoint_new(key, parent); + } + if (endpoint) { + ipc_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint)); + } + for (size_t i = 0; endpoint && i < 2; i++) { + cbm_daemon_ipc_listener_t *listener = cbm_daemon_ipc_listen(endpoint); + started[i] = listener != NULL; + touched_intact[i] = started[i] && cbm_daemon_ipc_listener_touch(listener); + char path[TEST_PATH_CAP]; + int written = snprintf(path, sizeof(path), "%s/cbm-%s.%s", runtime_dir, key, lost_files[i]); + unlinked[i] = + touched_intact[i] && written > 0 && written < (int)sizeof(path) && unlink(path) == 0; + touched_after_loss[i] = unlinked[i] && cbm_daemon_ipc_listener_touch(listener); + cbm_daemon_ipc_listener_close(listener); + } + cbm_daemon_ipc_endpoint_free(endpoint); + ipc_test_remove_tree(runtime_dir, parent); + + for (size_t i = 0; i < 2; i++) { + ASSERT_TRUE(started[i]); + ASSERT_TRUE(touched_intact[i]); + ASSERT_TRUE(unlinked[i]); + ASSERT_FALSE(touched_after_loss[i]); + } + PASS(); +#endif +} + TEST(daemon_ipc_lifetime_reservation_transfers_without_unlock_window) { static const char key[] = "1029384756abcdef"; char parent[TEST_PATH_CAP] = {0}; @@ -5260,8 +5308,7 @@ TEST(daemon_ipc_posix_single_uid_userns_real_smoke_issue1830) { if (WEXITSTATUS(status) != 0 && WEXITSTATUS(status) != 1) { char unexpected[128]; (void)snprintf(unexpected, sizeof(unexpected), - "userns probe exited %d -- not a security verdict", - WEXITSTATUS(status)); + "userns probe exited %d -- not a security verdict", WEXITSTATUS(status)); FAIL(unexpected); } ASSERT_EQ(0, WEXITSTATUS(status)); @@ -5376,6 +5423,7 @@ SUITE(daemon_ipc) { RUN_TEST(daemon_ipc_rejects_uppercase_instance_key); RUN_TEST(daemon_ipc_no_spawn_probe_distinguishes_absent_active_and_busy); RUN_TEST(daemon_ipc_lifetime_reservation_survives_saturated_second_listen); + RUN_TEST(daemon_ipc_listener_touch_detects_lost_runtime_files); RUN_TEST(daemon_ipc_lifetime_reservation_transfers_without_unlock_window); RUN_TEST(daemon_ipc_local_frame_roundtrip); RUN_TEST(daemon_ipc_bounded_receive_rejects_oversize_before_payload); diff --git a/tests/test_private_file_lock.c b/tests/test_private_file_lock.c index 2760bf55b..b795f9176 100644 --- a/tests/test_private_file_lock.c +++ b/tests/test_private_file_lock.c @@ -166,7 +166,8 @@ static void private_lock_fixture_finish(private_lock_fixture_t *fixture) { "lock-attempt-cleanup.lock", "acl-directory.lock", "acl-file.lock", - "fork.lock"}; + "fork.lock", + "touch.lock"}; if (fixture->root[0]) { for (size_t index = 0; index < sizeof(files) / sizeof(files[0]); index++) { if (!private_lock_path(path, fixture, files[index])) { @@ -480,6 +481,65 @@ TEST(private_file_lock_consumed_close_error_never_retries_recycled_fd) { #endif } +/* #2178: macOS tmp_cleaner unlinks /tmp files whose atime, mtime and ctime are + * all older than three days, including empty lock files a daemon holds for its + * whole life. The next participant then creates and locks a fresh inode, so + * the old lock coordinates nothing. Touch must keep a held file young and + * report the loss once the path stops naming it. */ +TEST(private_file_lock_touch_refreshes_held_file_and_detects_unlink) { +#ifdef _WIN32 + SKIP_PLATFORM("age-based temp cleaning and unlink-while-held are POSIX behaviors"); +#else + private_lock_fixture_t fixture; + bool started = private_lock_fixture_start(&fixture); + char path[PRIVATE_LOCK_TEST_PATH_CAP]; + bool path_ok = started && private_lock_path(path, &fixture, "touch.lock"); + cbm_private_file_lock_t *held = NULL; + cbm_private_file_lock_t *successor = NULL; + cbm_private_file_lock_status_t acquired = + path_ok ? cbm_private_file_lock_try_acquire(fixture.directory, "touch.lock", + CBM_PRIVATE_FILE_LOCK_EX, &held) + : CBM_PRIVATE_FILE_LOCK_IO; + const time_t stale = 1000000000; + struct timespec stale_times[2] = {{.tv_sec = stale}, {.tv_sec = stale}}; + bool aged = acquired == CBM_PRIVATE_FILE_LOCK_OK && + futimens(cbm_private_file_lock_native_fd_for_test(held), stale_times) == 0; + cbm_private_file_lock_status_t linked_touch = + aged ? cbm_private_file_lock_touch(held) : CBM_PRIVATE_FILE_LOCK_IO; + struct stat refreshed = {0}; + bool refreshed_ok = linked_touch == CBM_PRIVATE_FILE_LOCK_OK && stat(path, &refreshed) == 0; + + bool unlinked = refreshed_ok && unlink(path) == 0; + cbm_private_file_lock_status_t unlinked_touch = + unlinked ? cbm_private_file_lock_touch(held) : CBM_PRIVATE_FILE_LOCK_IO; + /* The hazard itself: with the path gone, a peer's EX succeeds while the + * original holder still believes it is exclusive. */ + cbm_private_file_lock_status_t successor_status = + unlinked ? cbm_private_file_lock_try_acquire(fixture.directory, "touch.lock", + CBM_PRIVATE_FILE_LOCK_EX, &successor) + : CBM_PRIVATE_FILE_LOCK_IO; + if (successor) { + (void)cbm_private_file_lock_release(&successor); + } + if (held) { + (void)cbm_private_file_lock_release(&held); + } + private_lock_fixture_finish(&fixture); + + ASSERT_TRUE(started); + ASSERT_EQ(acquired, CBM_PRIVATE_FILE_LOCK_OK); + ASSERT_TRUE(aged); + ASSERT_EQ(linked_touch, CBM_PRIVATE_FILE_LOCK_OK); + ASSERT_TRUE(refreshed_ok); + ASSERT_TRUE(refreshed.st_atime > stale); + ASSERT_TRUE(refreshed.st_mtime > stale); + ASSERT_TRUE(unlinked); + ASSERT_EQ(unlinked_touch, CBM_PRIVATE_FILE_LOCK_UNSAFE); + ASSERT_EQ(successor_status, CBM_PRIVATE_FILE_LOCK_OK); + PASS(); +#endif +} + TEST(private_file_lock_post_acquire_failure_returns_cleanup_owner) { private_lock_fixture_t fixture; bool started = private_lock_fixture_start(&fixture); @@ -863,6 +923,7 @@ SUITE(private_file_lock) { RUN_TEST(private_file_lock_unlock_failure_retains_retryable_lock); RUN_TEST(private_file_lock_close_failure_retries_without_duplicate_unlock); RUN_TEST(private_file_lock_consumed_close_error_never_retries_recycled_fd); + RUN_TEST(private_file_lock_touch_refreshes_held_file_and_detects_unlink); RUN_TEST(private_file_lock_post_acquire_failure_returns_cleanup_owner); RUN_TEST(private_file_lock_windows_lock_attempt_failure_returns_cleanup_owner); RUN_TEST(private_file_lock_rejects_unsafe_entries_and_replaced_root); From 3cf2a6663f36a6a366806bc78e4cc0678e1f1b55 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Mon, 21 Sep 2026 13:02:07 -0500 Subject: [PATCH 036/217] fix(daemon): address QA round 1 - Separate loss from transient failure. Touch helpers now report OK / lost / transient (version_cohort: OK/UNSAFE/IO; ipc: 1/0/-1). Only a proven unlink or replacement stops the daemon. A refresh failure on a still-valid file (EROFS, EPERM, EMFILE on the marker reopen) logs daemon.coordination_touch_failed once, retries every tick, and logs daemon.coordination_touch_recovered when it clears. - Cohort and claim locks now prove the path still names the held inode (private_file_revalidate against the stored directory + name), not just st_nlink, so rename-away, hard-link+unlink and a replaced directory are caught like unlink. - The runtime service's self-owned participant guard is refreshed by the listener heartbeat too. - Tests: rename-away detection; cohort lease/claim touch (lease with only lifetime held is OK, each unlinked file reported independently); IPC listener touch now checks timestamps are actually refreshed and covers the participant guard and identity-marker replacement. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Shane McCarron --- src/daemon/host.c | 71 +++++++++---- src/daemon/ipc.c | 107 ++++++++++++-------- src/daemon/ipc.h | 26 +++-- src/daemon/runtime.c | 6 +- src/daemon/runtime.h | 10 +- src/daemon/version_cohort.c | 40 +++++--- src/daemon/version_cohort.h | 16 +-- src/foundation/private_file_lock.c | 13 ++- src/foundation/private_file_lock_internal.h | 9 +- tests/test_daemon_ipc.c | 59 +++++++---- tests/test_private_file_lock.c | 47 ++++++++- tests/test_version_cohort.c | 54 +++++++++- 12 files changed, 329 insertions(+), 129 deletions(-) diff --git a/src/daemon/host.c b/src/daemon/host.c index 856af3ee4..3dea464cf 100644 --- a/src/daemon/host.c +++ b/src/daemon/host.c @@ -879,26 +879,41 @@ typedef struct { cbm_daemon_ipc_participant_guard_t *participant_guard; } host_coordination_t; -/* Refresh every long-held runtime file and name the first one found lost, or - * return NULL. A held lock whose path now names another inode coordinates - * nothing: forked index workers read the daemon as uncoordinated and a peer - * can start a second generation (#2178). */ -static const char *host_coordination_touch(const host_coordination_t *coordination, - cbm_daemon_runtime_service_t *service) { - if (!cbm_version_cohort_daemon_claim_touch(coordination->daemon_claim)) { - return "cohort_daemon_claim"; +typedef struct { + const char *lost; /* first held file whose path no longer names it */ + const char *transient; /* first still-valid file that could not be refreshed */ +} host_touch_result_t; + +static void host_touch_note(host_touch_result_t *result, const char *file, int status) { + if (status == 0 && !result->lost) { + result->lost = file; + } else if (status < 0 && !result->transient) { + result->transient = file; } - if (!cbm_version_cohort_lease_touch(coordination->cohort_lease)) { - return "cohort_lease"; - } - if (!cbm_daemon_ipc_participant_guard_touch(coordination->endpoint, - coordination->participant_guard)) { - return "participant_guard"; - } - if (!cbm_daemon_runtime_service_touch_listener(service)) { - return "listener"; - } - return NULL; +} + +static int host_cohort_touch_status(cbm_version_cohort_status_t status) { + return status == CBM_VERSION_COHORT_OK ? 1 : (status == CBM_VERSION_COHORT_IO ? -1 : 0); +} + +/* Refresh every long-held runtime file. A held lock whose path now names + * another inode coordinates nothing: forked index workers read the daemon as + * uncoordinated and a peer can start a second generation (#2178). Every file + * is visited even after one fails so the rest keep being refreshed. */ +static host_touch_result_t host_coordination_touch(const host_coordination_t *coordination, + cbm_daemon_runtime_service_t *service) { + host_touch_result_t result = {0}; + host_touch_note(&result, "cohort_daemon_claim", + host_cohort_touch_status( + cbm_version_cohort_daemon_claim_touch(coordination->daemon_claim))); + host_touch_note( + &result, "cohort_lease", + host_cohort_touch_status(cbm_version_cohort_lease_touch(coordination->cohort_lease))); + host_touch_note(&result, "participant_guard", + cbm_daemon_ipc_participant_guard_touch(coordination->endpoint, + coordination->participant_guard)); + host_touch_note(&result, "listener", cbm_daemon_runtime_service_touch_listener(service)); + return result; } static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service, @@ -907,6 +922,7 @@ static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service, uint64_t initial_deadline = cbm_now_ms() + HOST_INITIAL_CLIENT_TIMEOUT_MS; uint64_t stopping_deadline = 0; uint64_t next_touch = 0; + bool touch_degraded = false; for (;;) { cbm_daemon_runtime_service_state_t state = cbm_daemon_runtime_service_state(service); if (state == CBM_DAEMON_RUNTIME_SERVICE_EXITED) { @@ -942,12 +958,23 @@ static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service, * coordinated generation, while re-claiming would race that client. */ if (cbm_now_ms() >= next_touch) { next_touch = cbm_now_ms() + HOST_COORDINATION_TOUCH_MS; - const char *lost = host_coordination_touch(coordination, service); - if (lost) { + host_touch_result_t touched = host_coordination_touch(coordination, service); + if (touched.lost) { cbm_log_warn("daemon.lifetime_end", "reason", "coordination_file_lost", "file", - lost); + touched.lost); return cbm_daemon_runtime_service_stop(service, HOST_RUNTIME_SHUTDOWN_MS); } + /* A refresh failure on a still-valid file (read-only remount, + * EMFILE) is not loss: keep serving, retry next tick, and log + * only transitions so a persistent condition cannot flood. */ + if ((touched.transient != NULL) != touch_degraded) { + touch_degraded = touched.transient != NULL; + if (touch_degraded) { + cbm_log_warn("daemon.coordination_touch_failed", "file", touched.transient); + } else { + cbm_log_info("daemon.coordination_touch_recovered"); + } + } } host_http_reconcile_at(host, cbm_now_ms(), false); /* Retire an ephemeral generation that lingered for cold-storm cohort diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index 53da43542..bedc5884d 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -3220,44 +3220,65 @@ void cbm_daemon_ipc_listener_close(cbm_daemon_ipc_listener_t *listener) { free(listener); } -/* Touch through the held descriptor only after proving the path still names - * it: refreshing by path would keep a replacement inode fresh and hide the - * loss (#2178). */ -static bool posix_held_file_touch(int directory_fd, const char *base_name, int fd) { - return private_regular_file_snapshot(directory_fd, base_name, fd, 1, NULL) && - futimens(fd, NULL) == 0; +/* Touch results: 1 refreshed, 0 lost (the path no longer names the owned + * file), -1 transient (a still-valid file could not be refreshed). Touch + * through the held descriptor only after proving the path still names it: + * refreshing by path would keep a replacement inode fresh and hide the loss + * (#2178). */ +static int posix_touch_fold(int result, int next) { + return result == 0 || next == 0 ? 0 : (result < 0 || next < 0 ? -1 : 1); } -static bool posix_held_lock_touch(int directory_fd, const process_lock_entry_t *entry, int fd) { - return entry && posix_held_file_touch(directory_fd, entry->lock_name, fd); -} - -bool cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener) { - if (!listener || listener->dir_fd < 0 || listener->owner_pid != getpid() || - !listener->lifetime_reservation) { - return false; +static int posix_held_file_touch(int directory_fd, const char *base_name, int fd) { + if (!private_regular_file_snapshot(directory_fd, base_name, fd, 1, NULL)) { + return 0; } - bool lifetime = - posix_held_lock_touch(listener->dir_fd, listener->lifetime_reservation->process_entry, - listener->lifetime_reservation->fd); - const cbm_daemon_ipc_participant_guard_t *guard = listener->participant_guard; - bool participant = - !guard || - posix_held_lock_touch(listener->dir_fd, guard->legacy_process_entry, guard->legacy_fd); - /* The marker is not held open; reopen it and require the published inode. */ + return futimens(fd, NULL) == 0 ? 1 : -1; +} + +static int posix_held_lock_touch(int directory_fd, const process_lock_entry_t *entry, int fd) { + return entry ? posix_held_file_touch(directory_fd, entry->lock_name, fd) : 0; +} + +static int posix_identity_marker_touch(const cbm_daemon_ipc_listener_t *listener) { + /* The marker is not held open; reopen it and require the published inode. + * Only ENOENT proves loss: EMFILE and friends say nothing about the file. */ int marker_fd = openat(listener->dir_fd, listener->socket_identity_name, O_RDONLY | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK); - struct stat marker_status; - bool marker = marker_fd >= 0 && - private_regular_file_snapshot(listener->dir_fd, listener->socket_identity_name, - marker_fd, 1, &marker_status) && - marker_status.st_dev == listener->identity_device && - marker_status.st_ino == listener->identity_inode && - futimens(marker_fd, NULL) == 0; - if (marker_fd >= 0) { - (void)close(marker_fd); + if (marker_fd < 0) { + return errno == ENOENT ? 0 : -1; } - return lifetime && participant && marker; + struct stat marker_status; + int result = 0; + if (private_regular_file_snapshot(listener->dir_fd, listener->socket_identity_name, marker_fd, + 1, &marker_status) && + marker_status.st_dev == listener->identity_device && + marker_status.st_ino == listener->identity_inode) { + result = futimens(marker_fd, NULL) == 0 ? 1 : -1; + } + (void)close(marker_fd); + return result; +} + +int cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener, + cbm_daemon_ipc_participant_guard_t *external_guard) { + if (!listener || listener->dir_fd < 0 || listener->owner_pid != getpid() || + !listener->lifetime_reservation) { + return 0; + } + int result = + posix_held_lock_touch(listener->dir_fd, listener->lifetime_reservation->process_entry, + listener->lifetime_reservation->fd); + const cbm_daemon_ipc_participant_guard_t *guards[] = {listener->participant_guard, + external_guard}; + for (size_t i = 0; i < sizeof(guards) / sizeof(guards[0]); i++) { + if (guards[i]) { + result = posix_touch_fold(result, posix_held_lock_touch(listener->dir_fd, + guards[i]->legacy_process_entry, + guards[i]->legacy_fd)); + } + } + return posix_touch_fold(result, posix_identity_marker_touch(listener)); } int cbm_daemon_ipc_accept(cbm_daemon_ipc_listener_t *listener, uint32_t timeout_ms, @@ -3643,10 +3664,12 @@ bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t return true; } -bool cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint, - cbm_daemon_ipc_participant_guard_t *guard) { - return endpoint && guard && guard->owner_pid == getpid() && - posix_held_lock_touch(endpoint->dir_fd, guard->legacy_process_entry, guard->legacy_fd); +int cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint, + cbm_daemon_ipc_participant_guard_t *guard) { + if (!endpoint || !guard || guard->owner_pid != getpid()) { + return 0; + } + return posix_held_lock_touch(endpoint->dir_fd, guard->legacy_process_entry, guard->legacy_fd); } int cbm_daemon_ipc_local_transition_try_acquire( @@ -6794,13 +6817,15 @@ bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t /* Windows has no age-based cleaner of the private runtime directory, and its * held handles deny deletion; the #2178 heartbeat has nothing to refresh. */ -bool cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener) { - return listener != NULL; +int cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener, + cbm_daemon_ipc_participant_guard_t *external_guard) { + (void)external_guard; + return listener ? 1 : 0; } -bool cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint, - cbm_daemon_ipc_participant_guard_t *guard) { - return endpoint && guard; +int cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint, + cbm_daemon_ipc_participant_guard_t *guard) { + return endpoint && guard ? 1 : 0; } int cbm_daemon_ipc_local_transition_try_acquire( diff --git a/src/daemon/ipc.h b/src/daemon/ipc.h index ed214ea79..9a1e7b044 100644 --- a/src/daemon/ipc.h +++ b/src/daemon/ipc.h @@ -77,13 +77,17 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( void cbm_daemon_ipc_listener_close(cbm_daemon_ipc_listener_t *listener); /* Heartbeat for the daemon's long-lived runtime artifacts: refresh the held - * lifetime reservation, the listener's participant guard (if it owns one) and - * the published identity marker so an age-based temp cleaner never deletes - * them (#2178). Returns false once any of them no longer names the file this - * listener owns: a lost lifetime file lets a peer start a second generation, - * and a lost identity marker makes listener close leave the socket pair - * behind. Only the owning thread may call it while the listener is open. */ -bool cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener); + * lifetime reservation, the listener's participant guard (if it owns one), + * external_guard (optional, a guard the caller retains beside this listener) + * and the published identity marker so an age-based temp cleaner never + * deletes them (#2178). Returns 1 when all were refreshed, 0 once any no + * longer names the file this listener owns (a lost lifetime file lets a peer + * start a second generation; a lost identity marker makes listener close + * leave the socket pair behind), and -1 when a still-valid file could not be + * refreshed (transient). Only the thread that closes the listener may call it + * while the listener is open. */ +int cbm_daemon_ipc_listener_touch(cbm_daemon_ipc_listener_t *listener, + cbm_daemon_ipc_participant_guard_t *external_guard); /* Create or validate one private current-user directory at an already * canonical local path. Ancestors are handle-validated without mutation: @@ -279,10 +283,10 @@ bool cbm_daemon_ipc_startup_lock_release(cbm_daemon_ipc_startup_lock_t **lock_io int cbm_daemon_ipc_participant_guard_try_join(const cbm_daemon_ipc_endpoint_t *endpoint, cbm_daemon_ipc_participant_guard_t **guard_out); bool cbm_daemon_ipc_participant_guard_release(cbm_daemon_ipc_participant_guard_t **guard_io); -/* Same heartbeat contract as cbm_daemon_ipc_listener_touch, for a guard the - * caller retains itself. */ -bool cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint, - cbm_daemon_ipc_participant_guard_t *guard); +/* Same heartbeat contract and 1/0/-1 result as cbm_daemon_ipc_listener_touch, + * for a guard the caller retains itself. */ +int cbm_daemon_ipc_participant_guard_touch(const cbm_daemon_ipc_endpoint_t *endpoint, + cbm_daemon_ipc_participant_guard_t *guard); /* Standalone CLI work joins the legacy-compatible current participant group * without becoming a daemon client. Acquisition retains startup-v2 only for diff --git a/src/daemon/runtime.c b/src/daemon/runtime.c index 7fb7d32a9..bce7118f4 100644 --- a/src/daemon/runtime.c +++ b/src/daemon/runtime.c @@ -2578,8 +2578,10 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_ return count; } -bool cbm_daemon_runtime_service_touch_listener(cbm_daemon_runtime_service_t *service) { - return service && service->listener && cbm_daemon_ipc_listener_touch(service->listener); +int cbm_daemon_runtime_service_touch_listener(cbm_daemon_runtime_service_t *service) { + return service && service->listener + ? cbm_daemon_ipc_listener_touch(service->listener, service->owned_participant_guard) + : 0; } void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service) { diff --git a/src/daemon/runtime.h b/src/daemon/runtime.h index 304d7c898..5bcd10a34 100644 --- a/src/daemon/runtime.h +++ b/src/daemon/runtime.h @@ -336,10 +336,12 @@ size_t cbm_daemon_runtime_service_active_connections(cbm_daemon_runtime_service_ * an unbounded idle hang. The host lifetime loop calls this every tick; it is a * no-op unless a last-committed-client linger is armed. */ void cbm_daemon_runtime_service_reconcile_lifetime(cbm_daemon_runtime_service_t *service); -/* #2178 heartbeat over the listener's runtime artifacts; see - * cbm_daemon_ipc_listener_touch. Call only from the thread that stops and - * frees the service, which is the only thread that closes the listener. */ -bool cbm_daemon_runtime_service_touch_listener(cbm_daemon_runtime_service_t *service); +/* #2178 heartbeat over the listener's runtime artifacts, plus the participant + * guard the service owns when it started itself; see + * cbm_daemon_ipc_listener_touch for the 1/0/-1 result. Call only from the + * thread that stops and frees the service, which is the only thread that + * closes the listener. */ +int cbm_daemon_runtime_service_touch_listener(cbm_daemon_runtime_service_t *service); size_t cbm_daemon_runtime_service_job_subscribers(cbm_daemon_runtime_service_t *service, const char *project_key); uint64_t cbm_daemon_runtime_service_client_process_id(cbm_daemon_runtime_service_t *service, diff --git a/src/daemon/version_cohort.c b/src/daemon/version_cohort.c index 82cc63b32..5acaa6d5e 100644 --- a/src/daemon/version_cohort.c +++ b/src/daemon/version_cohort.c @@ -363,23 +363,35 @@ cbm_private_file_lock_status_t cbm_version_cohort_lease_release( return result; } -static bool version_cohort_lock_touch(cbm_private_file_lock_t *lock) { - return !lock || cbm_private_file_lock_touch(lock) == CBM_PRIVATE_FILE_LOCK_OK; -} - -bool cbm_version_cohort_lease_touch(cbm_version_cohort_lease_t *lease) { - if (!lease) { - return false; +/* Folds one lock's touch into an accumulated result: a lost file (UNSAFE) + * outranks a transient refresh failure (IO), which outranks OK. A lease holds + * only lifetime after admission, so absent locks are skipped. */ +static cbm_version_cohort_status_t version_cohort_lock_touch(cbm_private_file_lock_t *lock, + cbm_version_cohort_status_t result) { + if (!lock || result == CBM_VERSION_COHORT_UNSAFE) { + return result; } - /* Evaluate all three so one lost file does not leave the others to age. */ - bool lifetime = version_cohort_lock_touch(lease->lifetime); - bool admission = version_cohort_lock_touch(lease->admission); - bool maintenance = version_cohort_lock_touch(lease->maintenance); - return lifetime && admission && maintenance; + cbm_private_file_lock_status_t status = cbm_private_file_lock_touch(lock); + if (status == CBM_PRIVATE_FILE_LOCK_UNSAFE) { + return CBM_VERSION_COHORT_UNSAFE; + } + return status == CBM_PRIVATE_FILE_LOCK_OK ? result : CBM_VERSION_COHORT_IO; } -bool cbm_version_cohort_daemon_claim_touch(cbm_version_cohort_daemon_claim_t *claim) { - return claim && claim->marker && version_cohort_lock_touch(claim->marker); +cbm_version_cohort_status_t cbm_version_cohort_lease_touch(cbm_version_cohort_lease_t *lease) { + if (!lease || !lease->lifetime) { + return CBM_VERSION_COHORT_UNSAFE; + } + cbm_version_cohort_status_t result = + version_cohort_lock_touch(lease->lifetime, CBM_VERSION_COHORT_OK); + result = version_cohort_lock_touch(lease->admission, result); + return version_cohort_lock_touch(lease->maintenance, result); +} + +cbm_version_cohort_status_t cbm_version_cohort_daemon_claim_touch( + cbm_version_cohort_daemon_claim_t *claim) { + return claim && claim->marker ? version_cohort_lock_touch(claim->marker, CBM_VERSION_COHORT_OK) + : CBM_VERSION_COHORT_UNSAFE; } static cbm_version_cohort_status_t version_cohort_failed(cbm_version_cohort_lease_t *lease, diff --git a/src/daemon/version_cohort.h b/src/daemon/version_cohort.h index ac79c8b86..0a6be9ce9 100644 --- a/src/daemon/version_cohort.h +++ b/src/daemon/version_cohort.h @@ -146,13 +146,15 @@ cbm_private_file_lock_status_t cbm_version_cohort_lease_release( cbm_version_cohort_lease_t **lease_io); /* Heartbeat for a long-lived holder: refresh every lock file the lease or - * claim holds and report false once any held file was unlinked or replaced. - * An age-based temp cleaner deleting a held file silently ends coordination, - * because the next participant creates and locks a fresh inode (#2178). The - * only safe response to false is an orderly exit; re-claiming would race - * that participant. */ -bool cbm_version_cohort_lease_touch(cbm_version_cohort_lease_t *lease); -bool cbm_version_cohort_daemon_claim_touch(cbm_version_cohort_daemon_claim_t *claim); + * claim holds. An age-based temp cleaner deleting a held file silently ends + * coordination, because the next participant creates and locks a fresh inode + * (#2178). Returns OK when all held files were refreshed, UNSAFE once any is + * lost (its path no longer names the held file), and IO when a still-valid + * file could not be refreshed. The only safe response to UNSAFE is an orderly + * exit; re-claiming would race that participant. IO is transient. */ +cbm_version_cohort_status_t cbm_version_cohort_lease_touch(cbm_version_cohort_lease_t *lease); +cbm_version_cohort_status_t cbm_version_cohort_daemon_claim_touch( + cbm_version_cohort_daemon_claim_t *claim); /* Refuses teardown while a lease or retryable cleanup handle remains. */ cbm_private_file_lock_status_t cbm_version_cohort_manager_free( diff --git a/src/foundation/private_file_lock.c b/src/foundation/private_file_lock.c index a7d83172e..a5fdf797d 100644 --- a/src/foundation/private_file_lock.c +++ b/src/foundation/private_file_lock.c @@ -38,6 +38,9 @@ struct cbm_private_file_lock { int fd; pid_t owner_pid; cbm_private_file_lock_mode_t mode; + /* Borrowed; the directory outlives every lock acquired through it. */ + const cbm_private_lock_directory_t *directory; + char base_name[NAME_MAX + 1]; struct cbm_private_file_lock *next_tracked; bool unlocked; bool test_fail_unlock_once; @@ -345,6 +348,8 @@ cbm_private_file_lock_status_t cbm_private_file_lock_try_acquire( lock->fd = fd; lock->owner_pid = getpid(); lock->mode = mode; + lock->directory = directory; + memcpy(lock->base_name, base_name, strlen(base_name) + 1); int operation = mode == CBM_PRIVATE_FILE_LOCK_SH ? LOCK_SH : LOCK_EX; if (private_flock_set(fd, operation | LOCK_NB) != 0) { @@ -489,9 +494,11 @@ cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock if (!cbm_private_file_lock_fork_guard_enter()) { return CBM_PRIVATE_FILE_LOCK_IO; } - /* An unlinked file keeps its inode alive for this handle with st_nlink 0, - * which the payload validity check already rejects. */ - bool linked = private_payload_fd_valid(lock, NULL); + /* Unlink leaves st_nlink 0 on the held inode, but a rename-away or a + * replaced directory keeps it linked elsewhere; only the path-vs-handle + * identity check proves the canonical path still names this lock. */ + bool linked = private_payload_fd_valid(lock, NULL) && lock->directory && + private_file_revalidate(lock->directory, lock->base_name, lock->fd, NULL); bool touched = linked && futimens(lock->fd, NULL) == 0; cbm_private_file_lock_fork_guard_leave(); if (!linked) { diff --git a/src/foundation/private_file_lock_internal.h b/src/foundation/private_file_lock_internal.h index b8ec34ac7..d39fe7d5f 100644 --- a/src/foundation/private_file_lock_internal.h +++ b/src/foundation/private_file_lock_internal.h @@ -63,9 +63,12 @@ cbm_private_file_lock_status_t cbm_private_file_lock_payload_write(cbm_private_f /* Long-lived holders call this periodically. It refreshes the held file's * timestamps through the validated handle so age-based temp cleaners (macOS - * tmp_cleaner, systemd-tmpfiles) never select it, and returns UNSAFE once the - * handle no longer names a linked file: a peer opening the path would then - * lock a different inode and the lock coordinates nothing (#2178). */ + * tmp_cleaner, systemd-tmpfiles) never select it. Returns UNSAFE once the + * lock's path in its directory no longer names the held file (unlinked, + * replaced, renamed away, or the directory itself replaced): a peer opening + * the path would then lock a different inode and the lock coordinates nothing + * (#2178). IO reports a refresh failure on a still-valid lock, which callers + * should treat as transient. The directory must outlive the lock. */ cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock_t *lock); /* Forces the next successfully acquired native lock down the post-lock diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index d29a1e6c0..8107a0ef5 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -2064,22 +2064,34 @@ TEST(daemon_ipc_lifetime_reservation_survives_saturated_second_listen) { } /* #2178: an age-based temp cleaner unlinked the live daemon's lifetime lock - * and identity marker. The heartbeat must report either loss, so the daemon - * exits instead of serving under a lifetime file that coordinates nothing or - * leaving an unremovable socket pair behind at close. */ + * and identity marker. The heartbeat must keep every listener-owned file + * young, and report loss (unlink, or replacement by a fresh file at the same + * name) so the daemon exits instead of serving under a lifetime file that + * coordinates nothing or leaving an unremovable socket pair behind at close. */ TEST(daemon_ipc_listener_touch_detects_lost_runtime_files) { #ifdef _WIN32 SKIP_PLATFORM("unlink-while-held of runtime files is POSIX behavior"); #else + enum { TOUCH_CASES = 4 }; static const char key[] = "2178a2178a2178a0"; - static const char *const lost_files[] = {"lifetime.lock", "sock.identity"}; + static const struct { + const char *suffix; + bool replace; + } cases[TOUCH_CASES] = { + {"lifetime.lock", false}, /* lifetime reservation */ + {"lock", false}, /* listener-owned participant guard */ + {"sock.identity", false}, /* identity marker, unlinked */ + {"sock.identity", true}, /* identity marker, replaced by a fresh inode */ + }; + const time_t stale = 1000000000; char parent[TEST_PATH_CAP] = {0}; char runtime_dir[TEST_PATH_CAP] = {0}; cbm_daemon_ipc_endpoint_t *endpoint = NULL; - bool started[2] = {false, false}; - bool touched_intact[2] = {false, false}; - bool unlinked[2] = {false, false}; - bool touched_after_loss[2] = {true, true}; + bool started[TOUCH_CASES] = {false}; + int touched_intact[TOUCH_CASES] = {0}; + bool refreshed[TOUCH_CASES] = {false}; + bool lost[TOUCH_CASES] = {false}; + int touched_after_loss[TOUCH_CASES] = {1, 1, 1, 1}; if (ipc_test_parent_new(parent, "listener-touch")) { endpoint = cbm_daemon_ipc_endpoint_new(key, parent); @@ -2087,25 +2099,36 @@ TEST(daemon_ipc_listener_touch_detects_lost_runtime_files) { if (endpoint) { ipc_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint)); } - for (size_t i = 0; endpoint && i < 2; i++) { + for (size_t i = 0; endpoint && i < TOUCH_CASES; i++) { cbm_daemon_ipc_listener_t *listener = cbm_daemon_ipc_listen(endpoint); started[i] = listener != NULL; - touched_intact[i] = started[i] && cbm_daemon_ipc_listener_touch(listener); char path[TEST_PATH_CAP]; - int written = snprintf(path, sizeof(path), "%s/cbm-%s.%s", runtime_dir, key, lost_files[i]); - unlinked[i] = - touched_intact[i] && written > 0 && written < (int)sizeof(path) && unlink(path) == 0; - touched_after_loss[i] = unlinked[i] && cbm_daemon_ipc_listener_touch(listener); + int written = + snprintf(path, sizeof(path), "%s/cbm-%s.%s", runtime_dir, key, cases[i].suffix); + bool path_ok = started[i] && written > 0 && written < (int)sizeof(path); + struct timespec stale_times[2] = {{.tv_sec = stale}, {.tv_sec = stale}}; + bool aged = path_ok && utimensat(AT_FDCWD, path, stale_times, AT_SYMLINK_NOFOLLOW) == 0; + touched_intact[i] = aged ? cbm_daemon_ipc_listener_touch(listener, NULL) : -2; + struct stat after; + refreshed[i] = touched_intact[i] == 1 && stat(path, &after) == 0 && + after.st_atime > stale && after.st_mtime > stale; + lost[i] = refreshed[i] && unlink(path) == 0; + if (lost[i] && cases[i].replace) { + int fd = open(path, O_RDWR | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600); + lost[i] = fd >= 0 && close(fd) == 0; + } + touched_after_loss[i] = lost[i] ? cbm_daemon_ipc_listener_touch(listener, NULL) : -2; cbm_daemon_ipc_listener_close(listener); } cbm_daemon_ipc_endpoint_free(endpoint); ipc_test_remove_tree(runtime_dir, parent); - for (size_t i = 0; i < 2; i++) { + for (size_t i = 0; i < TOUCH_CASES; i++) { ASSERT_TRUE(started[i]); - ASSERT_TRUE(touched_intact[i]); - ASSERT_TRUE(unlinked[i]); - ASSERT_FALSE(touched_after_loss[i]); + ASSERT_EQ(touched_intact[i], 1); + ASSERT_TRUE(refreshed[i]); + ASSERT_TRUE(lost[i]); + ASSERT_EQ(touched_after_loss[i], 0); } PASS(); #endif diff --git a/tests/test_private_file_lock.c b/tests/test_private_file_lock.c index b795f9176..988b73e71 100644 --- a/tests/test_private_file_lock.c +++ b/tests/test_private_file_lock.c @@ -167,7 +167,9 @@ static void private_lock_fixture_finish(private_lock_fixture_t *fixture) { "acl-directory.lock", "acl-file.lock", "fork.lock", - "touch.lock"}; + "touch.lock", + "touch-rename.lock", + "touch-renamed.lock"}; if (fixture->root[0]) { for (size_t index = 0; index < sizeof(files) / sizeof(files[0]); index++) { if (!private_lock_path(path, fixture, files[index])) { @@ -540,6 +542,48 @@ TEST(private_file_lock_touch_refreshes_held_file_and_detects_unlink) { #endif } +/* A rename-away keeps the held inode linked (st_nlink stays 1) while the + * canonical path no longer names it, so a peer would create and lock a fresh + * file there. Touch must compare the path with the handle, not just count + * links. */ +TEST(private_file_lock_touch_detects_rename_away) { +#ifdef _WIN32 + SKIP_PLATFORM("renaming a held lock file away is POSIX behavior"); +#else + private_lock_fixture_t fixture; + bool started = private_lock_fixture_start(&fixture); + char path[PRIVATE_LOCK_TEST_PATH_CAP]; + char moved[PRIVATE_LOCK_TEST_PATH_CAP]; + bool paths_ok = started && private_lock_path(path, &fixture, "touch-rename.lock") && + private_lock_path(moved, &fixture, "touch-renamed.lock"); + cbm_private_file_lock_t *held = NULL; + cbm_private_file_lock_status_t acquired = + paths_ok ? cbm_private_file_lock_try_acquire(fixture.directory, "touch-rename.lock", + CBM_PRIVATE_FILE_LOCK_EX, &held) + : CBM_PRIVATE_FILE_LOCK_IO; + cbm_private_file_lock_status_t intact_touch = acquired == CBM_PRIVATE_FILE_LOCK_OK + ? cbm_private_file_lock_touch(held) + : CBM_PRIVATE_FILE_LOCK_IO; + bool renamed = intact_touch == CBM_PRIVATE_FILE_LOCK_OK && rename(path, moved) == 0; + struct stat moved_status = {0}; + bool still_linked = renamed && stat(moved, &moved_status) == 0 && moved_status.st_nlink == 1; + cbm_private_file_lock_status_t renamed_touch = + renamed ? cbm_private_file_lock_touch(held) : CBM_PRIVATE_FILE_LOCK_IO; + if (held) { + (void)cbm_private_file_lock_release(&held); + } + private_lock_fixture_finish(&fixture); + + ASSERT_TRUE(started); + ASSERT_EQ(acquired, CBM_PRIVATE_FILE_LOCK_OK); + ASSERT_EQ(intact_touch, CBM_PRIVATE_FILE_LOCK_OK); + ASSERT_TRUE(renamed); + ASSERT_TRUE(still_linked); + ASSERT_EQ(renamed_touch, CBM_PRIVATE_FILE_LOCK_UNSAFE); + PASS(); +#endif +} + TEST(private_file_lock_post_acquire_failure_returns_cleanup_owner) { private_lock_fixture_t fixture; bool started = private_lock_fixture_start(&fixture); @@ -924,6 +968,7 @@ SUITE(private_file_lock) { RUN_TEST(private_file_lock_close_failure_retries_without_duplicate_unlock); RUN_TEST(private_file_lock_consumed_close_error_never_retries_recycled_fd); RUN_TEST(private_file_lock_touch_refreshes_held_file_and_detects_unlink); + RUN_TEST(private_file_lock_touch_detects_rename_away); RUN_TEST(private_file_lock_post_acquire_failure_returns_cleanup_owner); RUN_TEST(private_file_lock_windows_lock_attempt_failure_returns_cleanup_owner); RUN_TEST(private_file_lock_rejects_unsafe_entries_and_replaced_root); diff --git a/tests/test_version_cohort.c b/tests/test_version_cohort.c index 13e0f8065..f40e05c5e 100644 --- a/tests/test_version_cohort.c +++ b/tests/test_version_cohort.c @@ -330,9 +330,9 @@ TEST(version_cohort_conflict_is_retried_until_the_deadline) { /* An indefinite deadline never waits on a conflicting holder. */ retries_before = cbm_version_cohort_conflict_retries_for_testing(); - ASSERT_EQ(cbm_version_cohort_acquire(second, &requested, UINT64_MAX, &requested_lease, - &conflict), - CBM_VERSION_COHORT_CONFLICT); + ASSERT_EQ( + cbm_version_cohort_acquire(second, &requested, UINT64_MAX, &requested_lease, &conflict), + CBM_VERSION_COHORT_CONFLICT); ASSERT_NULL(requested_lease); ASSERT_EQ(cbm_version_cohort_conflict_retries_for_testing(), retries_before); @@ -669,6 +669,53 @@ TEST(version_cohort_does_not_repurpose_daemon_startup_lock_for_lifetime) { PASS(); } +/* #2178: a live daemon's lease holds only the lifetime file after admission + * (admission and maintenance are released), and its claim holds the marker. + * Touch must report both intact, then name each as lost once an age-based + * temp cleaner unlinks it, independently of the other. */ +TEST(version_cohort_touch_reports_each_lost_file) { +#ifdef _WIN32 + SKIP_PLATFORM("unlinking a held lock file is POSIX behavior"); +#else + version_cohort_fixture_t fixture; + ASSERT_TRUE(version_cohort_fixture_start(&fixture, "touch")); + cbm_version_cohort_manager_t *manager = cbm_version_cohort_manager_new(fixture.endpoint); + ASSERT_NOT_NULL(manager); + cbm_daemon_build_identity_t identity = version_cohort_identity("1.0.0", VERSION_COHORT_BUILD_A); + cbm_daemon_conflict_t conflict; + cbm_version_cohort_lease_t *lease = NULL; + cbm_version_cohort_daemon_claim_t *claim = NULL; + ASSERT_EQ( + cbm_version_cohort_acquire(manager, &identity, cbm_now_ms() + 5000, &lease, &conflict), + CBM_VERSION_COHORT_OK); + ASSERT_EQ(cbm_version_cohort_daemon_claim_acquire(manager, &claim), CBM_VERSION_COHORT_OK); + + ASSERT_EQ(cbm_version_cohort_lease_touch(lease), CBM_VERSION_COHORT_OK); + ASSERT_EQ(cbm_version_cohort_daemon_claim_touch(claim), CBM_VERSION_COHORT_OK); + + const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(fixture.endpoint); + char marker[VERSION_COHORT_TEST_PATH_CAP]; + char lifetime[VERSION_COHORT_TEST_PATH_CAP]; + ASSERT_TRUE(snprintf(marker, sizeof(marker), "%s/cbm-version-cohort-daemon-v1.lock", + runtime_dir) < (int)sizeof(marker)); + ASSERT_TRUE(snprintf(lifetime, sizeof(lifetime), "%s/cbm-version-cohort-lifetime-v1.lock", + runtime_dir) < (int)sizeof(lifetime)); + + ASSERT_EQ(unlink(marker), 0); + ASSERT_EQ(cbm_version_cohort_daemon_claim_touch(claim), CBM_VERSION_COHORT_UNSAFE); + ASSERT_EQ(cbm_version_cohort_lease_touch(lease), CBM_VERSION_COHORT_OK); + + ASSERT_EQ(unlink(lifetime), 0); + ASSERT_EQ(cbm_version_cohort_lease_touch(lease), CBM_VERSION_COHORT_UNSAFE); + + ASSERT_EQ(cbm_version_cohort_daemon_claim_release(&claim), CBM_PRIVATE_FILE_LOCK_OK); + version_cohort_release(&lease); + version_cohort_manager_close(&manager); + version_cohort_fixture_finish(&fixture); + PASS(); +#endif +} + TEST(version_cohort_distinguishes_coordinated_daemon_without_connecting) { version_cohort_fixture_t fixture; ASSERT_TRUE(version_cohort_fixture_start(&fixture, "daemon-marker")); @@ -1034,6 +1081,7 @@ SUITE(version_cohort) { RUN_TEST(version_cohort_mutation_waits_for_every_lifetime_participant); RUN_TEST(version_cohort_mutation_timeout_releases_all_guards); RUN_TEST(version_cohort_does_not_repurpose_daemon_startup_lock_for_lifetime); + RUN_TEST(version_cohort_touch_reports_each_lost_file); RUN_TEST(version_cohort_distinguishes_coordinated_daemon_without_connecting); RUN_TEST(version_cohort_transition_presence_is_authoritative_and_marker_checked); RUN_TEST(version_cohort_transition_shutdown_order_has_no_false_conflict); From 34d172c9038a9b7cee78e42bb7768b93fbb8ed70 Mon Sep 17 00:00:00 2001 From: News Date: Tue, 22 Sep 2026 22:33:51 +0900 Subject: [PATCH 037/217] fix(windows): pair aligned delete with its allocator Signed-off-by: News --- src/foundation/mem_override_win.c | 52 ++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/foundation/mem_override_win.c b/src/foundation/mem_override_win.c index e18b09d11..bb86a2e31 100644 --- a/src/foundation/mem_override_win.c +++ b/src/foundation/mem_override_win.c @@ -56,6 +56,7 @@ void *__real_malloc(size_t size); void *__real_calloc(size_t count, size_t size); void *__real_realloc(void *block, size_t size); void __real_free(void *block); +void __real__aligned_free(void *block); char *__real_strdup(const char *text); size_t __real__msize(void *block); /* Wrappers must call __real_* for anything that is itself wrapped: a plain @@ -155,9 +156,58 @@ void __wrap__aligned_free(void *block) { mi_free(block); return; } - __real_free(block); + /* A CRT aligned block has its own bookkeeping header. Plain free is not + * its matching deallocator. */ + __real__aligned_free(block); } +#if defined(CBM_MEM_GLOBAL_OVERRIDE) && CBM_MEM_GLOBAL_OVERRIDE +/* LLVM-MinGW's static C++ runtime calls _aligned_malloc through the linker's + * wrapper but its aligned operator delete can call the CRT import pointer + * __imp__aligned_free directly. --wrap=_aligned_free does not rewrite that + * import-pointer reference. Provide the six C++ ABI delete entry points so + * every aligned new/delete pair reaches the same owner-aware deallocator. */ +void cbm_cxx_aligned_delete(void *block) __asm__("_ZdlPvSt11align_val_t"); +void cbm_cxx_aligned_delete(void *block) { + __wrap__aligned_free(block); +} + +void cbm_cxx_aligned_delete_nothrow(void *block, + const void *tag) __asm__("_ZdlPvSt11align_val_tRKSt9nothrow_t"); +void cbm_cxx_aligned_delete_nothrow(void *block, const void *tag) { + (void)tag; + __wrap__aligned_free(block); +} + +void cbm_cxx_aligned_delete_sized(void *block, size_t size, + size_t alignment) __asm__("_ZdlPvySt11align_val_t"); +void cbm_cxx_aligned_delete_sized(void *block, size_t size, size_t alignment) { + (void)size; + (void)alignment; + __wrap__aligned_free(block); +} + +void cbm_cxx_aligned_array_delete(void *block) __asm__("_ZdaPvSt11align_val_t"); +void cbm_cxx_aligned_array_delete(void *block) { + __wrap__aligned_free(block); +} + +void cbm_cxx_aligned_array_delete_nothrow(void *block, const void *tag) __asm__( + "_ZdaPvSt11align_val_tRKSt9nothrow_t"); +void cbm_cxx_aligned_array_delete_nothrow(void *block, const void *tag) { + (void)tag; + __wrap__aligned_free(block); +} + +void cbm_cxx_aligned_array_delete_sized(void *block, size_t size, + size_t alignment) __asm__("_ZdaPvySt11align_val_t"); +void cbm_cxx_aligned_array_delete_sized(void *block, size_t size, size_t alignment) { + (void)size; + (void)alignment; + __wrap__aligned_free(block); +} +#endif + void *__wrap_realloc(void *block, size_t size) { if (!block) { void *fresh = mi_malloc(size); From 1133ba324656042f8c73578a788ebf5c77ec8172 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Tue, 22 Sep 2026 11:27:26 -0500 Subject: [PATCH 038/217] fix(daemon): address QA round 2 - Classify touch failures explicitly. A file is lost only on proof that the path no longer names it: ENOENT, st_nlink 0, another inode, a symlink or other non-regular entry, or a replaced lock directory. Any other syscall failure (EIO, ESTALE, ENOMEM, EACCES on reopen) is transient, so a flaky runtime volume no longer stops the daemon. The POSIX private-lock touch and the ipc lock/marker touches share this rule; the Windows touch reports a failed handle query as transient. - The identity marker is classified by path (fstatat, no follow) before it is reopened, so replacement by a symlink is now loss, not a transient reopen failure. - Tests: symlink replacement of the identity marker (lost); an unreadable marker is transient (-1) and recovers (1) once readable; the cohort touch test ages both held files and asserts they were refreshed. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Shane McCarron --- src/daemon/ipc.c | 47 ++++++++++++++++----- src/foundation/private_file_lock.c | 65 ++++++++++++++++++++++++------ tests/test_daemon_ipc.c | 47 +++++++++++++++++---- tests/test_version_cohort.c | 19 +++++++-- 4 files changed, 144 insertions(+), 34 deletions(-) diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index bedc5884d..a6259e1dd 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -3229,9 +3229,29 @@ static int posix_touch_fold(int result, int next) { return result == 0 || next == 0 ? 0 : (result < 0 || next < 0 ? -1 : 1); } +/* Whether base_name still names the regular file (device, inode): 1 yes, + * 0 proven not (absent, another inode, a symlink or other non-regular + * entry), -1 undetermined (any other lookup failure, e.g. EIO or ESTALE). */ +static int posix_path_names_inode(int directory_fd, const char *base_name, dev_t device, + ino_t inode) { + struct stat by_path; + if (fstatat(directory_fd, base_name, &by_path, AT_SYMLINK_NOFOLLOW) != 0) { + return errno == ENOENT ? 0 : -1; + } + return S_ISREG(by_path.st_mode) && by_path.st_dev == device && by_path.st_ino == inode ? 1 : 0; +} + static int posix_held_file_touch(int directory_fd, const char *base_name, int fd) { - if (!private_regular_file_snapshot(directory_fd, base_name, fd, 1, NULL)) { - return 0; + struct stat by_handle; + if (fstat(fd, &by_handle) != 0) { + return -1; + } + int identity = + by_handle.st_nlink == 0 + ? 0 + : posix_path_names_inode(directory_fd, base_name, by_handle.st_dev, by_handle.st_ino); + if (identity <= 0) { + return identity; } return futimens(fd, NULL) == 0 ? 1 : -1; } @@ -3241,20 +3261,27 @@ static int posix_held_lock_touch(int directory_fd, const process_lock_entry_t *e } static int posix_identity_marker_touch(const cbm_daemon_ipc_listener_t *listener) { - /* The marker is not held open; reopen it and require the published inode. - * Only ENOENT proves loss: EMFILE and friends say nothing about the file. */ + /* The marker is not held open. Classify by path first, so a symlink or + * other replacement is loss; only then reopen it to refresh, where a + * failure (EMFILE, EACCES) says nothing about the file and is transient. */ + int identity = posix_path_names_inode(listener->dir_fd, listener->socket_identity_name, + listener->identity_device, listener->identity_inode); + if (identity <= 0) { + return identity; + } int marker_fd = openat(listener->dir_fd, listener->socket_identity_name, O_RDONLY | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK); if (marker_fd < 0) { return errno == ENOENT ? 0 : -1; } struct stat marker_status; - int result = 0; - if (private_regular_file_snapshot(listener->dir_fd, listener->socket_identity_name, marker_fd, - 1, &marker_status) && - marker_status.st_dev == listener->identity_device && - marker_status.st_ino == listener->identity_inode) { - result = futimens(marker_fd, NULL) == 0 ? 1 : -1; + int result = -1; + if (fstat(marker_fd, &marker_status) == 0) { + /* Replaced between the lookup and the open. */ + result = marker_status.st_dev != listener->identity_device || + marker_status.st_ino != listener->identity_inode + ? 0 + : (futimens(marker_fd, NULL) == 0 ? 1 : -1); } (void)close(marker_fd); return result; diff --git a/src/foundation/private_file_lock.c b/src/foundation/private_file_lock.c index a5fdf797d..46111cc1a 100644 --- a/src/foundation/private_file_lock.c +++ b/src/foundation/private_file_lock.c @@ -487,6 +487,39 @@ cbm_private_file_lock_status_t cbm_private_file_lock_payload_write(cbm_private_f return valid ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_IO; } +/* Whether the lock's canonical path still names the held inode: 1 yes, 0 no + * (proven lost), -1 undetermined. Only proof counts as loss: unlink leaves + * st_nlink 0, a rename-away or replacement makes the path name another inode + * or nothing, and a replaced directory changes the directory's identity. Any + * other syscall failure (EIO, ESTALE, ENOMEM) says nothing about the file and + * is reported as undetermined so the caller retries instead of exiting. */ +static int private_held_identity(const cbm_private_file_lock_t *lock) { + const cbm_private_lock_directory_t *directory = lock->directory; + struct stat by_handle; + struct stat directory_by_path; + struct stat by_path; + if (fstat(lock->fd, &by_handle) != 0) { + return -1; + } + if (by_handle.st_nlink == 0) { + return 0; + } + if (lstat(directory->path, &directory_by_path) != 0) { + return errno == ENOENT || errno == ENOTDIR ? 0 : -1; + } + if (!S_ISDIR(directory_by_path.st_mode) || directory_by_path.st_dev != directory->device || + directory_by_path.st_ino != directory->inode) { + return 0; + } + if (fstatat(directory->fd, lock->base_name, &by_path, AT_SYMLINK_NOFOLLOW) != 0) { + return errno == ENOENT ? 0 : -1; + } + return S_ISREG(by_path.st_mode) && by_path.st_dev == by_handle.st_dev && + by_path.st_ino == by_handle.st_ino + ? 1 + : 0; +} + cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock_t *lock) { if (!lock) { return CBM_PRIVATE_FILE_LOCK_UNSAFE; @@ -494,17 +527,17 @@ cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock if (!cbm_private_file_lock_fork_guard_enter()) { return CBM_PRIVATE_FILE_LOCK_IO; } - /* Unlink leaves st_nlink 0 on the held inode, but a rename-away or a - * replaced directory keeps it linked elsewhere; only the path-vs-handle - * identity check proves the canonical path still names this lock. */ - bool linked = private_payload_fd_valid(lock, NULL) && lock->directory && - private_file_revalidate(lock->directory, lock->base_name, lock->fd, NULL); - bool touched = linked && futimens(lock->fd, NULL) == 0; - cbm_private_file_lock_fork_guard_leave(); - if (!linked) { - return CBM_PRIVATE_FILE_LOCK_UNSAFE; + cbm_private_file_lock_status_t status = CBM_PRIVATE_FILE_LOCK_UNSAFE; + if (lock->fd >= 0 && !lock->unlocked && lock->owner_pid == getpid() && lock->directory && + private_lock_is_tracked(lock)) { + int identity = private_held_identity(lock); + status = identity > 0 ? (futimens(lock->fd, NULL) == 0 ? CBM_PRIVATE_FILE_LOCK_OK + : CBM_PRIVATE_FILE_LOCK_IO) + : identity < 0 ? CBM_PRIVATE_FILE_LOCK_IO + : CBM_PRIVATE_FILE_LOCK_UNSAFE; } - return touched ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_IO; + cbm_private_file_lock_fork_guard_leave(); + return status; } cbm_private_file_lock_status_t cbm_private_file_lock_release(cbm_private_file_lock_t **lock_io) { @@ -1478,15 +1511,21 @@ cbm_private_file_lock_status_t cbm_private_file_lock_payload_write(cbm_private_f cbm_private_file_lock_status_t cbm_private_file_lock_touch(cbm_private_file_lock_t *lock) { /* Windows has no age-based cleaner of the private lock directory; only * confirm the held handle still names a linked file. */ - if (!lock) { + if (!lock || lock->handle == INVALID_HANDLE_VALUE || lock->unlocked) { return CBM_PRIVATE_FILE_LOCK_UNSAFE; } if (!cbm_private_file_lock_fork_guard_enter()) { return CBM_PRIVATE_FILE_LOCK_IO; } - bool linked = private_win_payload_handle_valid(lock); + BY_HANDLE_FILE_INFORMATION information; + bool queried = GetFileInformationByHandle(lock->handle, &information) != 0; cbm_private_file_lock_fork_guard_leave(); - return linked ? CBM_PRIVATE_FILE_LOCK_OK : CBM_PRIVATE_FILE_LOCK_UNSAFE; + /* A failed query says nothing about the file; only a proven unlink is loss. */ + if (!queried) { + return CBM_PRIVATE_FILE_LOCK_IO; + } + return information.nNumberOfLinks == 0 ? CBM_PRIVATE_FILE_LOCK_UNSAFE + : CBM_PRIVATE_FILE_LOCK_OK; } static bool private_win_release_unlock(cbm_private_file_lock_t *lock) { diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index 8107a0ef5..5fcf8c1fc 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -2072,16 +2072,18 @@ TEST(daemon_ipc_listener_touch_detects_lost_runtime_files) { #ifdef _WIN32 SKIP_PLATFORM("unlink-while-held of runtime files is POSIX behavior"); #else - enum { TOUCH_CASES = 4 }; + enum { TOUCH_CASES = 5 }; + enum { LOSS_UNLINK, LOSS_REPLACE_FILE, LOSS_REPLACE_SYMLINK }; static const char key[] = "2178a2178a2178a0"; static const struct { const char *suffix; - bool replace; + int loss; } cases[TOUCH_CASES] = { - {"lifetime.lock", false}, /* lifetime reservation */ - {"lock", false}, /* listener-owned participant guard */ - {"sock.identity", false}, /* identity marker, unlinked */ - {"sock.identity", true}, /* identity marker, replaced by a fresh inode */ + {"lifetime.lock", LOSS_UNLINK}, /* lifetime reservation */ + {"lock", LOSS_UNLINK}, /* listener-owned participant guard */ + {"sock.identity", LOSS_UNLINK}, /* identity marker, unlinked */ + {"sock.identity", LOSS_REPLACE_FILE}, /* ... replaced by a fresh inode */ + {"sock.identity", LOSS_REPLACE_SYMLINK} /* ... replaced by a symlink */, }; const time_t stale = 1000000000; char parent[TEST_PATH_CAP] = {0}; @@ -2091,7 +2093,9 @@ TEST(daemon_ipc_listener_touch_detects_lost_runtime_files) { int touched_intact[TOUCH_CASES] = {0}; bool refreshed[TOUCH_CASES] = {false}; bool lost[TOUCH_CASES] = {false}; - int touched_after_loss[TOUCH_CASES] = {1, 1, 1, 1}; + int touched_after_loss[TOUCH_CASES] = {1, 1, 1, 1, 1}; + int unreadable_touch = 0; + int readable_again_touch = 0; if (ipc_test_parent_new(parent, "listener-touch")) { endpoint = cbm_daemon_ipc_endpoint_new(key, parent); @@ -2099,6 +2103,19 @@ TEST(daemon_ipc_listener_touch_detects_lost_runtime_files) { if (endpoint) { ipc_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint)); } + /* A still-valid marker that merely cannot be reopened (EACCES) is a + * transient refresh failure, not loss: the daemon must keep serving. */ + if (endpoint) { + cbm_daemon_ipc_listener_t *listener = cbm_daemon_ipc_listen(endpoint); + char marker[TEST_PATH_CAP]; + int written = snprintf(marker, sizeof(marker), "%s/cbm-%s.sock.identity", runtime_dir, key); + if (listener && written > 0 && written < (int)sizeof(marker) && chmod(marker, 0) == 0) { + unreadable_touch = cbm_daemon_ipc_listener_touch(listener, NULL); + readable_again_touch = + chmod(marker, 0600) == 0 ? cbm_daemon_ipc_listener_touch(listener, NULL) : -2; + } + cbm_daemon_ipc_listener_close(listener); + } for (size_t i = 0; endpoint && i < TOUCH_CASES; i++) { cbm_daemon_ipc_listener_t *listener = cbm_daemon_ipc_listen(endpoint); started[i] = listener != NULL; @@ -2113,16 +2130,30 @@ TEST(daemon_ipc_listener_touch_detects_lost_runtime_files) { refreshed[i] = touched_intact[i] == 1 && stat(path, &after) == 0 && after.st_atime > stale && after.st_mtime > stale; lost[i] = refreshed[i] && unlink(path) == 0; - if (lost[i] && cases[i].replace) { + if (lost[i] && cases[i].loss == LOSS_REPLACE_FILE) { int fd = open(path, O_RDWR | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600); lost[i] = fd >= 0 && close(fd) == 0; + } else if (lost[i] && cases[i].loss == LOSS_REPLACE_SYMLINK) { + lost[i] = symlink("/dev/null", path) == 0; } touched_after_loss[i] = lost[i] ? cbm_daemon_ipc_listener_touch(listener, NULL) : -2; cbm_daemon_ipc_listener_close(listener); + /* A tampered marker makes close (correctly) keep the socket pair; clear + * this case's simulated leftovers so the next case can listen. */ + static const char *const leftovers[] = {"sock.identity", "sock", "anc"}; + for (size_t j = 0; j < sizeof(leftovers) / sizeof(leftovers[0]); j++) { + written = snprintf(path, sizeof(path), "%s/cbm-%s.%s", runtime_dir, key, leftovers[j]); + if (written > 0 && written < (int)sizeof(path)) { + (void)unlink(path); + } + } } cbm_daemon_ipc_endpoint_free(endpoint); ipc_test_remove_tree(runtime_dir, parent); + /* Root bypasses mode 0, so the reopen succeeds there and the touch is 1. */ + ASSERT_EQ(unreadable_touch, geteuid() == 0 ? 1 : -1); + ASSERT_EQ(readable_again_touch, 1); for (size_t i = 0; i < TOUCH_CASES; i++) { ASSERT_TRUE(started[i]); ASSERT_EQ(touched_intact[i], 1); diff --git a/tests/test_version_cohort.c b/tests/test_version_cohort.c index f40e05c5e..d5b6d4c5a 100644 --- a/tests/test_version_cohort.c +++ b/tests/test_version_cohort.c @@ -22,6 +22,8 @@ #endif #include #else +#include +#include #include #include #endif @@ -690,9 +692,6 @@ TEST(version_cohort_touch_reports_each_lost_file) { CBM_VERSION_COHORT_OK); ASSERT_EQ(cbm_version_cohort_daemon_claim_acquire(manager, &claim), CBM_VERSION_COHORT_OK); - ASSERT_EQ(cbm_version_cohort_lease_touch(lease), CBM_VERSION_COHORT_OK); - ASSERT_EQ(cbm_version_cohort_daemon_claim_touch(claim), CBM_VERSION_COHORT_OK); - const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(fixture.endpoint); char marker[VERSION_COHORT_TEST_PATH_CAP]; char lifetime[VERSION_COHORT_TEST_PATH_CAP]; @@ -701,6 +700,20 @@ TEST(version_cohort_touch_reports_each_lost_file) { ASSERT_TRUE(snprintf(lifetime, sizeof(lifetime), "%s/cbm-version-cohort-lifetime-v1.lock", runtime_dir) < (int)sizeof(lifetime)); + /* Age both held files the way a cleaner would see them, then prove the + * touch refreshed them through the held handles. */ + const time_t stale = 1000000000; + struct timespec stale_times[2] = {{.tv_sec = stale}, {.tv_sec = stale}}; + ASSERT_EQ(utimensat(AT_FDCWD, marker, stale_times, 0), 0); + ASSERT_EQ(utimensat(AT_FDCWD, lifetime, stale_times, 0), 0); + ASSERT_EQ(cbm_version_cohort_lease_touch(lease), CBM_VERSION_COHORT_OK); + ASSERT_EQ(cbm_version_cohort_daemon_claim_touch(claim), CBM_VERSION_COHORT_OK); + struct stat refreshed; + ASSERT_EQ(stat(marker, &refreshed), 0); + ASSERT_TRUE(refreshed.st_mtime > stale && refreshed.st_atime > stale); + ASSERT_EQ(stat(lifetime, &refreshed), 0); + ASSERT_TRUE(refreshed.st_mtime > stale && refreshed.st_atime > stale); + ASSERT_EQ(unlink(marker), 0); ASSERT_EQ(cbm_version_cohort_daemon_claim_touch(claim), CBM_VERSION_COHORT_UNSAFE); ASSERT_EQ(cbm_version_cohort_lease_touch(lease), CBM_VERSION_COHORT_OK); From 255463126650ab6deab3e62aab441b71479704f1 Mon Sep 17 00:00:00 2001 From: News Date: Wed, 23 Sep 2026 08:21:53 +0900 Subject: [PATCH 039/217] fix(windows): set UTF-8 environment values through wide CRT Signed-off-by: News --- src/foundation/compat.h | 8 +++----- tests/test_platform.c | 12 ++++++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/foundation/compat.h b/src/foundation/compat.h index 821d901f0..3080930f8 100644 --- a/src/foundation/compat.h +++ b/src/foundation/compat.h @@ -166,11 +166,9 @@ static inline int cbm_setenv(const char *name, const char *value, int overwrite) free(wide_value); return EINVAL; } - /* Keep the CRT's narrow environment useful for legacy getenv callers, - * then repair the process-wide Windows environment with the actual UTF-16 - * value. _putenv_s alone routes UTF-8 path bytes through the active ANSI - * code page, which corrupts non-ASCII cache roots inherited by children. */ - int status = _putenv_s(name, value); + /* _putenv_s can reject UTF-8 bytes outside the active ANSI code page. + * Narrow getenv returns bytes in that code page after this wide update. */ + int status = _wputenv_s(wide_name, wide_value); if (status == 0 && !SetEnvironmentVariableW(wide_name, wide_value)) { status = EINVAL; } diff --git a/tests/test_platform.c b/tests/test_platform.c index 5995a4b01..77d325d91 100644 --- a/tests/test_platform.c +++ b/tests/test_platform.c @@ -869,8 +869,8 @@ TEST(platform_env_long_refuses_what_it_cannot_read) { * setter must update that same wide environment; _putenv_s alone interprets * UTF-8 path bytes through the active ANSI code page. */ TEST(platform_setenv_preserves_utf8_in_wide_environment) { - static const char utf8[] = "C:/cbm-cache-\xce\x94-\xe6\x97\xa5\xe6\x9c\xac"; - static const wchar_t wide[] = L"C:/cbm-cache-\u0394-\u65e5\u672c"; + static const char utf8[] = "C:/cbm-cache-\xce\x94-\xe6\x97\xa5\xe6\x9c\xac-\xe4\xb8\x81"; + static const wchar_t wide[] = L"C:/cbm-cache-\u0394-\u65e5\u672c-\u4e01"; ASSERT_EQ(cbm_setenv("CBM_CACHE_DIR", utf8, 1), 0); wchar_t observed_wide[128]; ASSERT_EQ(GetEnvironmentVariableW(L"CBM_CACHE_DIR", observed_wide, 128), (DWORD)(wcslen(wide))); @@ -878,6 +878,14 @@ TEST(platform_setenv_preserves_utf8_in_wide_environment) { char observed_utf8[128]; ASSERT_NOT_NULL(cbm_safe_getenv("CBM_CACHE_DIR", observed_utf8, sizeof(observed_utf8), NULL)); ASSERT_STR_EQ(observed_utf8, utf8); + /* Narrow getenv must agree with the CRT's active ANSI code page. */ + char expected_narrow[128]; + ASSERT_GT(WideCharToMultiByte(CP_ACP, 0, wide, -1, expected_narrow, + (int)sizeof(expected_narrow), NULL, NULL), + 0); + const char *observed_narrow = getenv("CBM_CACHE_DIR"); + ASSERT_NOT_NULL(observed_narrow); + ASSERT_STR_EQ(observed_narrow, expected_narrow); (void)cbm_unsetenv("CBM_CACHE_DIR"); PASS(); } From f4c897110cb1f02edee37db7c8849e63894ed8a4 Mon Sep 17 00:00:00 2001 From: MopicMP Date: Wed, 23 Sep 2026 22:44:16 +0200 Subject: [PATCH 040/217] fix(java): resolve record accessor calls, and never bind a call to data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A record's accessors are implicit, so `record Point(double x, double y, double z)` declares no `x()` and extraction emitted no node for one. The call `p.x()` had nothing to resolve to, fell through to the short-name registry, and bound to the first `x` in the project — typically an unrelated class's private field. Measured on two indexed Java trees, that was 278 of 2444 and 2649 of 29509 CALLS edges. Extraction now emits one Method definition per record component, skipping any accessor the record declares itself (JLS 8.10.3), so the accessor has a node and the call resolves to it. And a Java call may no longer bind to a Variable or a Field. A Java call expression never names data, so such an edge is a spelling collision, not a call. The guard follows cbm_go_suppress_bare_field_ref — a pure, label-keyed predicate — and is called from both pass_calls.c and pass_parallel.c so the sequential and parallel resolvers stay identical. Java-gated: Kotlin properties hold function values and C has function pointers, so both legitimately call through data. Signed-off-by: MopicMP Co-authored-by: Claude Opus 5 --- internal/cbm/extract_defs.c | 80 ++++++++++++++++++++++++++ src/pipeline/pass_calls.c | 6 ++ src/pipeline/pass_parallel.c | 6 ++ src/pipeline/pipeline.h | 7 +++ src/pipeline/registry.c | 24 ++++++++ tests/test_call_reference_contract.c | 85 ++++++++++++++++++++++++++++ tests/test_registry.c | 22 +++++++ 7 files changed, 230 insertions(+) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 93da084be..fd051c482 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -204,6 +204,7 @@ static void extract_class_fields(CBMExtractCtx *ctx, TSNode class_node, const ch const CBMLangSpec *spec); static TSNode find_class_body(TSNode class_node, CBMLanguage lang); static void extract_enum_members(CBMExtractCtx *ctx, TSNode node, const char *class_qn); +static void extract_record_accessors(CBMExtractCtx *ctx, TSNode node, const char *class_qn); static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec); // --- Helpers --- @@ -4642,6 +4643,10 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec extract_enum_members(ctx, node, class_qn); } + if (ctx->language == CBM_LANG_JAVA && strcmp(kind, "record_declaration") == 0) { + extract_record_accessors(ctx, node, class_qn); + } + // Extract methods inside the class extract_class_methods(ctx, node, class_qn, spec); @@ -5529,6 +5534,81 @@ static void extract_enum_members(CBMExtractCtx *ctx, TSNode node, const char *cl } } +/* Whether the record body declares this accessor itself (JLS §8.10.3 lets a + * record override any of them), so the implicit one is not emitted twice. */ +static bool record_declares_accessor(CBMExtractCtx *ctx, TSNode node, const char *comp_name) { + TSNode body = find_class_body(node, ctx->language); + if (ts_node_is_null(body)) { + return false; + } + uint32_t mc = ts_node_named_child_count(body); + for (uint32_t mi = 0; mi < mc; mi++) { + TSNode member = ts_node_named_child(body, mi); + if (strcmp(ts_node_type(member), "method_declaration") != 0) { + continue; + } + TSNode mname = ts_node_child_by_field_name(member, TS_FIELD("name")); + if (ts_node_is_null(mname)) { + continue; + } + char *name = cbm_node_text(ctx->arena, mname, ctx->source); + if (!name || strcmp(name, comp_name) != 0) { + continue; + } + /* Same name, no parameters — that is the accessor. */ + TSNode params = ts_node_child_by_field_name(member, TS_FIELD("parameters")); + if (ts_node_is_null(params) || ts_node_named_child_count(params) == 0) { + return true; + } + } + return false; +} + +/* Java records: the compiler writes one accessor per component, so `p.x()` + * calls a method that appears nowhere in the source. Without a definition for + * it the call has nothing to resolve to and falls through to the project-wide + * name registry, which binds it to the first `x` in the tree — typically an + * unrelated class's private field. Emitting the accessors makes the record's + * own API visible, exactly as the language defines it (JLS §8.10.3). */ +static void extract_record_accessors(CBMExtractCtx *ctx, TSNode node, const char *class_qn) { + CBMArena *a = ctx->arena; + TSNode params = ts_node_child_by_field_name(node, TS_FIELD("parameters")); + if (ts_node_is_null(params)) { + return; + } + uint32_t pc = ts_node_named_child_count(params); + for (uint32_t pi = 0; pi < pc; pi++) { + TSNode comp = ts_node_named_child(params, pi); + if (strcmp(ts_node_type(comp), "formal_parameter") != 0) { + continue; + } + TSNode cname = ts_node_child_by_field_name(comp, TS_FIELD("name")); + if (ts_node_is_null(cname)) { + continue; + } + char *comp_name = cbm_node_text(a, cname, ctx->source); + if (!comp_name || !comp_name[0] || record_declares_accessor(ctx, node, comp_name)) { + continue; + } + CBMDefinition adef; + memset(&adef, 0, sizeof(adef)); + adef.name = comp_name; + adef.qualified_name = cbm_arena_sprintf(a, "%s.%s", class_qn, comp_name); + adef.label = "Method"; + adef.file_path = ctx->rel_path; + adef.start_line = ts_node_start_point(comp).row + TS_LINE_OFFSET; + adef.end_line = ts_node_end_point(comp).row + TS_LINE_OFFSET; + adef.lines = 1; + adef.parent_class = class_qn; + adef.is_exported = true; + TSNode ctype = ts_node_child_by_field_name(comp, TS_FIELD("type")); + if (!ts_node_is_null(ctype)) { + adef.return_type = cbm_node_text(a, ctype, ctx->source); + } + cbm_defs_push(&ctx->result->defs, a, adef); + } +} + /* Resolve the identifier node from a destructure pattern child. * pair_pattern → value field; shorthand/identifier → itself; others → first named child. */ static TSNode destructure_ident(TSNode pat_child) { diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 84958954c..c702f7efd 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -682,6 +682,12 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, if (cbm_suppress_cross_language_suffix_match(lang, target_node->file_path, res.strategy)) { return 0; } + /* A Java call never targets data — see cbm_java_suppress_call_to_data_member. + * The language gate lives here, as with the guards above, and MUST match + * pass_parallel.c exactly or the two resolvers diverge. */ + if (cbm_java_suppress_call_to_data_member(lang == CBM_LANG_JAVA, target_node->label)) { + return 0; + } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, imp_count, drop_plain_call); return SKIP_ONE; diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index e85ad015f..380679c13 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -3028,6 +3028,12 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB * CALLS edge across a language boundary. */ continue; } + if (target_node && source_node->id != target_node->id && + cbm_java_suppress_call_to_data_member(lang == CBM_LANG_JAVA, target_node->label)) { + /* Same guard as pass_calls.c — a Java call never targets a + * Variable or a Field. */ + continue; + } if (!target_node || source_node->id == target_node->id) { /* HTTP/ASYNC calls to an EXTERNAL client library (`requests.get(url)`) * resolve to an unindexed QN (target_node == NULL), but their edge diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index e5a672cb0..416c5b25a 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -344,6 +344,13 @@ bool cbm_suppress_cross_language_ref(CBMLanguage caller_lang, const char *target * unit-tested in test_registry.c. */ bool cbm_go_suppress_bare_field_ref(bool is_go, bool is_member_access, const char *target_label); +/* A Java call expression never names data: `p.x()` is a method invocation + * whatever `x` spells elsewhere in the project. Drops a CALLS bind whose + * target is a Variable or Field. Java only — Kotlin properties and C function + * pointers are callable names that are not methods. Pure; unit-tested in + * test_registry.c. */ +bool cbm_java_suppress_call_to_data_member(bool is_java, const char *target_label); + /* Get the label of a qualified name, or NULL if not found. */ const char *cbm_registry_label_of(const cbm_registry_t *r, const char *qn); diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index c99c72d8d..2a9fb19fb 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -811,6 +811,30 @@ bool cbm_go_suppress_bare_field_ref(bool is_go, bool is_member_access, const cha return strcmp(target_label, "Field") == 0; } +bool cbm_java_suppress_call_to_data_member(bool is_java, const char *target_label) { + /* A Java call expression never names data. `p.x()` is a method invocation + * whatever `x` spells elsewhere in the tree, so a CALLS edge into a + * Variable or a Field is a spelling collision and not a call. + * + * The bind is reached when the type-aware resolver finds no method to + * return: a record's accessors are implicit, so `record Point(double x, + * double y, double z)` declares no `x()` for java_lookup_method to find, + * and every `p.x()` falls through to the short-name registry — which + * matches the first `x` anywhere in the project, typically an unrelated + * class's private field. Measured on a 600-file Java tree (2026-09-22): + * 278 of 2444 CALLS edges landed on fields that way, and the callers of + * one four-field class absorbed every record read in the project. + * + * Java-gated, like the Go guard above, because the veto is only sound + * where no callable name can be anything but a method: Kotlin properties + * hold function values (`val f: () -> Unit; f()`) and C has function + * pointers, so both legitimately call through data. */ + if (!is_java || !target_label) { + return false; + } + return strcmp(target_label, "Variable") == 0 || strcmp(target_label, "Field") == 0; +} + /* ── Lifecycle ──────────────────────────────────────────────────── */ cbm_registry_t *cbm_registry_new(void) { diff --git a/tests/test_call_reference_contract.c b/tests/test_call_reference_contract.c index 3a7b3acd1..d7aa27939 100644 --- a/tests/test_call_reference_contract.c +++ b/tests/test_call_reference_contract.c @@ -35,6 +35,40 @@ static int crc_edge_count(cbm_store_t *store, const char *project, const char *t return matches; } +/* Edges of `type` out of `source_name` whose target's qualified name ends with + * `target_qn_suffix`. Two methods in one project can share a short name, and + * then only the qualified name tells them apart. */ +static int crc_edge_count_to_qn(cbm_store_t *store, const char *project, const char *type, + const char *source_name, const char *target_qn_suffix) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + if (cbm_store_find_edges_by_type(store, project, type, &edges, &edge_count) != CBM_STORE_OK) { + return -1; + } + int matches = 0; + size_t suffix_len = strlen(target_qn_suffix); + for (int i = 0; i < edge_count; i++) { + cbm_node_t source = {0}; + cbm_node_t target = {0}; + bool source_ok = + cbm_store_find_node_by_id(store, edges[i].source_id, &source) == CBM_STORE_OK; + bool target_ok = + cbm_store_find_node_by_id(store, edges[i].target_id, &target) == CBM_STORE_OK; + if (source_ok && target_ok && source.name && target.qualified_name && + strcmp(source.name, source_name) == 0) { + size_t qn_len = strlen(target.qualified_name); + if (qn_len >= suffix_len && + strcmp(target.qualified_name + qn_len - suffix_len, target_qn_suffix) == 0) { + matches++; + } + } + cbm_node_free_fields(&source); + cbm_node_free_fields(&target); + } + cbm_store_free_edges(edges, edge_count); + return matches; +} + static int crc_global_handler_edge_count(cbm_store_t *store, const char *project, const char *caller) { int references = @@ -423,6 +457,56 @@ TEST(call_reference_go_ambiguous_promoted_method_stays_usage) { PASS(); } +TEST(call_java_record_accessor_binds_the_record) { + /* A record's accessors are implicit, so `record Point(double x, ...)` + * declares no `x()` in its source. Extraction emits them, and `other.x()` + * binds to the record's own accessor. + * + * Two things must not happen instead. The call must not reach the + * project-wide short-name registry and land on the first `x` in the tree — + * here an unrelated class's private field. And the guard that refuses such + * data binds must not swallow a real method that merely shares the + * spelling: the builder's `x(int)`, declared beside its own `x` field, + * keeps its edge. */ + static const RFile files[] = { + {"Point.java", "package com.example;\n" + "public record Point(double x, double y, double z) {\n" + " public double first(Point other) {\n" + " return Maths.abs(other.x()) + new Builder().x(5).made();\n" + " }\n" + "}\n"}, + {"Spring.java", "package com.example;\n" + "public final class Spring {\n" + " private double x;\n" + " public void put(double v) { this.x = v; }\n" + "}\n"}, + {"Builder.java", "package com.example;\n" + "public final class Builder {\n" + " private int x;\n" + " public Builder x(int v) { this.x = v; return this; }\n" + " public int made() { return x; }\n" + "}\n"}, + {"Maths.java", "package com.example;\n" + "public final class Maths {\n" + " public static double abs(double v) { return v < 0 ? -v : v; }\n" + "}\n"}}; + RProj project; + cbm_store_t *store = rh_index_files(&project, files, 4); + ASSERT_NOT_NULL(store); + int fabricated_field = crc_edge_count(store, project.project, "CALLS", "first", "Field", "x"); + int fabricated_var = crc_edge_count(store, project.project, "CALLS", "first", "Variable", "x"); + int genuine = crc_edge_count(store, project.project, "CALLS", "first", "Method", "abs"); + int accessor = crc_edge_count_to_qn(store, project.project, "CALLS", "first", ".Point.x"); + int builder = crc_edge_count_to_qn(store, project.project, "CALLS", "first", ".Builder.x"); + rh_cleanup(&project, store); + ASSERT_EQ(fabricated_field, 0); + ASSERT_EQ(fabricated_var, 0); + ASSERT_EQ(genuine, 1); + ASSERT_EQ(accessor, 1); + ASSERT_EQ(builder, 1); + PASS(); +} + SUITE(call_reference_contract) { RUN_TEST(call_reference_typescript_direct_argument_is_exact); RUN_TEST(call_reference_kotlin_alias_argument_is_exact); @@ -439,4 +523,5 @@ SUITE(call_reference_contract) { RUN_TEST(call_reference_python_later_decorated_method_rebinding_stays_usage); RUN_TEST(call_reference_go_bound_method_argument_is_exact); RUN_TEST(call_reference_go_ambiguous_promoted_method_stays_usage); + RUN_TEST(call_java_record_accessor_binds_the_record); } diff --git a/tests/test_registry.c b/tests/test_registry.c index 5f751a899..126400da6 100644 --- a/tests/test_registry.c +++ b/tests/test_registry.c @@ -987,6 +987,27 @@ TEST(go_bare_ref_never_binds_field) { PASS(); } +TEST(java_call_never_binds_data_member) { + /* A Java call expression never names data, so a CALLS bind that landed on + * a Variable or a Field is a spelling collision. The shape that produces + * it: a record's accessors are implicit, `p.x()` finds no declared `x()`, + * and the short-name registry hands back the first `x` in the tree — an + * unrelated class's private field. */ + ASSERT_TRUE(cbm_java_suppress_call_to_data_member(true, "Field")); + ASSERT_TRUE(cbm_java_suppress_call_to_data_member(true, "Variable")); + /* Everything callable is kept, including the constructor's Class target. */ + ASSERT_FALSE(cbm_java_suppress_call_to_data_member(true, "Method")); + ASSERT_FALSE(cbm_java_suppress_call_to_data_member(true, "Function")); + ASSERT_FALSE(cbm_java_suppress_call_to_data_member(true, "Class")); + /* Other languages have callable data: a Kotlin property or a C function + * pointer is a name that is not a method and is called all the same. */ + ASSERT_FALSE(cbm_java_suppress_call_to_data_member(false, "Field")); + ASSERT_FALSE(cbm_java_suppress_call_to_data_member(false, "Variable")); + /* Degenerate input → nothing to judge. */ + ASSERT_FALSE(cbm_java_suppress_call_to_data_member(true, NULL)); + PASS(); +} + TEST(dynamic_suppress_drops_weak_method_matches) { /* #592/#606/#1276: a member call whose receiver the LSP could not type, that * landed via a WEAK short-name strategy, is generic-resolver noise → drop. @@ -1244,6 +1265,7 @@ SUITE(registry) { RUN_TEST(registry_tie_break_is_independent_of_registration_order); RUN_TEST(cross_language_ref_drops_go_vs_c); RUN_TEST(go_bare_ref_never_binds_field); + RUN_TEST(java_call_never_binds_data_member); RUN_TEST(dynamic_suppress_drops_weak_method_matches); RUN_TEST(dynamic_suppress_keeps_high_confidence_and_non_methods); RUN_TEST(python_builtin_member_table_matches_builtin_type_methods); From fee73ced23e4f0753523bc2157f1fbb853f51fdd Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 25 Sep 2026 14:09:27 -0500 Subject: [PATCH 041/217] test(daemon): cover host loop stop on coordination file loss Add cbm_daemon_host_wait_for_lifetime_for_test, which runs the production lifetime loop over caller-held coordination handles, and a runtime test that ages the claim marker, unlinks the cohort lifetime lock, and asserts the tick refreshes the marker, logs daemon.lifetime_end reason=coordination_file_lost file=cohort_lease, and stops the runtime. The seam runs an ephemeral generation, so a missed loss ends at the initial client window instead of hanging. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NbYKGdU7pqG6AEuMQuJxiF Signed-off-by: Shane McCarron --- src/daemon/host.c | 16 +++++ src/daemon/host_internal.h | 14 ++++ tests/test_daemon_runtime.c | 131 ++++++++++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+) diff --git a/src/daemon/host.c b/src/daemon/host.c index 3dea464cf..fb44acdd3 100644 --- a/src/daemon/host.c +++ b/src/daemon/host.c @@ -985,6 +985,22 @@ static bool host_wait_for_lifetime(cbm_daemon_runtime_service_t *service, } } +bool cbm_daemon_host_wait_for_lifetime_for_test(cbm_daemon_runtime_service_t *service, + const cbm_daemon_ipc_endpoint_t *endpoint, + cbm_version_cohort_lease_t *cohort_lease, + cbm_version_cohort_daemon_claim_t *daemon_claim, + cbm_daemon_ipc_participant_guard_t *guard) { + host_state_t host = {0}; + atomic_int stop_requested = 0; + host_coordination_t coordination = { + .endpoint = endpoint, + .cohort_lease = cohort_lease, + .daemon_claim = daemon_claim, + .participant_guard = guard, + }; + return host_wait_for_lifetime(service, &stop_requested, &host, false, &coordination); +} + static bool host_application_shutdown(host_state_t *host) { if (cbm_daemon_application_shutdown(host->application, HOST_APPLICATION_SHUTDOWN_MS)) { return true; diff --git a/src/daemon/host_internal.h b/src/daemon/host_internal.h index dd8856161..8573c0913 100644 --- a/src/daemon/host_internal.h +++ b/src/daemon/host_internal.h @@ -9,6 +9,10 @@ #include struct cbm_daemon_ipc_endpoint; +struct cbm_daemon_ipc_participant_guard; +struct cbm_daemon_runtime_service; +struct cbm_version_cohort_daemon_claim; +struct cbm_version_cohort_lease; typedef bool (*cbm_daemon_host_cleanup_release_for_test_fn)(void *context); @@ -68,4 +72,14 @@ bool cbm_daemon_host_http_reconcile_free_refusal_for_test( * the failure, the adapter must cancel so the final free succeeds. */ bool cbm_daemon_host_http_thread_create_failure_lifecycle_for_test(void); +/* Run the production lifetime loop over caller-held coordination handles with + * no HTTP host state and no stop request. The generation is ephemeral, so a + * loop that misses a loss still ends at the initial client window instead of + * hanging the test. Returns the loop's own result. */ +bool cbm_daemon_host_wait_for_lifetime_for_test( + struct cbm_daemon_runtime_service *service, const struct cbm_daemon_ipc_endpoint *endpoint, + struct cbm_version_cohort_lease *cohort_lease, + struct cbm_version_cohort_daemon_claim *daemon_claim, + struct cbm_daemon_ipc_participant_guard *guard); + #endif /* CBM_DAEMON_HOST_INTERNAL_H */ diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index 827a8cf34..5ca4c72dd 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -1262,6 +1262,19 @@ static bool runtime_test_fixture_start_failed(const char *tag, const char *stage static bool runtime_test_fixture_permanent = false; +typedef struct { + cbm_version_cohort_manager_t *manager; + cbm_version_cohort_lease_t *lease; + cbm_daemon_ipc_participant_guard_t *guard; + cbm_version_cohort_daemon_claim_t *claim; + bool held; +} runtime_test_coordination_t; + +/* When set, the fixture takes the host's coordination handles in the host's + * order before starting the service: cohort admission refuses once a + * generation is already serving. */ +static runtime_test_coordination_t *runtime_test_fixture_coordination = NULL; + static bool runtime_test_fixture_start_configured( runtime_test_fixture_t *fixture, const char *tag, const cbm_daemon_build_identity_t *identity, uint32_t max_clients, uint64_t lease_timeout_ms, @@ -1317,6 +1330,19 @@ static bool runtime_test_fixture_start_configured( if (application) { config.application = *application; } + runtime_test_coordination_t *coordination = runtime_test_fixture_coordination; + if (coordination) { + cbm_daemon_conflict_t conflict; + coordination->manager = cbm_version_cohort_manager_new(fixture->endpoint); + coordination->held = + coordination->manager && + cbm_version_cohort_acquire(coordination->manager, identity, cbm_now_ms() + 5000, + &coordination->lease, &conflict) == CBM_VERSION_COHORT_OK && + cbm_daemon_ipc_participant_guard_try_join(fixture->endpoint, &coordination->guard) == + 1 && + cbm_version_cohort_daemon_claim_acquire(coordination->manager, &coordination->claim) == + CBM_VERSION_COHORT_OK; + } fixture->service = cbm_daemon_runtime_service_start(&config); if (!fixture->service) { return runtime_test_fixture_start_failed(tag, "service-start", @@ -1370,6 +1396,110 @@ TEST(daemon_runtime_convenience_service_owns_participant_guard) { PASS(); } +#ifndef _WIN32 +/* The runtime's threads log concurrently while the loop stops the service, so + * the sink only sets flags. */ +static atomic_bool runtime_test_coordination_lost_logged; +static atomic_bool runtime_test_window_expired_logged; + +static void runtime_test_lifetime_log_sink(const char *line) { + /* Substrings, not key=value spelling, so text and JSON formats both match. */ + if (line && strstr(line, "daemon.lifetime_end") && strstr(line, "coordination_file_lost") && + strstr(line, "cohort_lease")) { + atomic_store(&runtime_test_coordination_lost_logged, true); + } + if (line && strstr(line, "initial_window_expired")) { + atomic_store(&runtime_test_window_expired_logged, true); + } +} +#endif + +/* #2178: the host lifetime loop must refresh every held coordination file on + * its tick and stop the generation once one is lost. The claim marker is aged + * and the cohort lifetime lock unlinked: the tick must refresh the marker + * through its held handle, name the lease (touched after the claim) as lost, + * and stop the runtime. */ +TEST(daemon_runtime_host_loop_stops_on_coordination_file_loss) { +#ifdef _WIN32 + SKIP_PLATFORM("unlinking a held lock file is POSIX behavior"); +#else + cbm_daemon_build_identity_t identity = + runtime_test_identity("2.4.0", runtime_test_self_build()); + identity.cache_fingerprint = RUNTIME_CACHE_A; /* cohort admission requires one */ + runtime_test_fixture_t fixture; + runtime_test_coordination_t coordination = {0}; + runtime_test_fixture_coordination = &coordination; + bool started = runtime_test_fixture_start(&fixture, "coordination-loss", &identity); + runtime_test_fixture_coordination = NULL; + bool held = coordination.held; + + char marker[RUNTIME_TEST_PATH_CAP]; + char lifetime[RUNTIME_TEST_PATH_CAP]; + int marker_written = snprintf(marker, sizeof(marker), "%s/cbm-version-cohort-daemon-v1.lock", + fixture.runtime_dir); + int lifetime_written = snprintf(lifetime, sizeof(lifetime), + "%s/cbm-version-cohort-lifetime-v1.lock", fixture.runtime_dir); + const time_t stale = 1000000000; + struct timespec stale_times[2] = {{.tv_sec = stale}, {.tv_sec = stale}}; + bool staged = held && marker_written > 0 && marker_written < (int)sizeof(marker) && + lifetime_written > 0 && lifetime_written < (int)sizeof(lifetime) && + utimensat(AT_FDCWD, marker, stale_times, 0) == 0 && unlink(lifetime) == 0; + + bool loop_ok = false; + if (staged) { + atomic_store(&runtime_test_coordination_lost_logged, false); + atomic_store(&runtime_test_window_expired_logged, false); + CBMLogLevel previous_log_level = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_INFO); + cbm_log_set_sink(runtime_test_lifetime_log_sink); + loop_ok = cbm_daemon_host_wait_for_lifetime_for_test( + fixture.service, fixture.endpoint, coordination.lease, coordination.claim, + coordination.guard); + cbm_log_set_sink(NULL); + cbm_log_set_level(previous_log_level); + } + cbm_daemon_runtime_service_state_t state = + started ? cbm_daemon_runtime_service_state(fixture.service) + : CBM_DAEMON_RUNTIME_SERVICE_RUNNING; + struct stat marker_status = {0}; + bool marker_refreshed = staged && stat(marker, &marker_status) == 0 && + marker_status.st_mtime > stale && marker_status.st_atime > stale; + + /* Host teardown order: service first, then guard, claim and lease. Every + * handle references the endpoint, which the fixture frees. */ + if (started && state != CBM_DAEMON_RUNTIME_SERVICE_EXITED) { + (void)cbm_daemon_runtime_service_stop(fixture.service, RUNTIME_TEST_TIMEOUT_MS); + } + if (fixture.service && cbm_daemon_runtime_service_free(fixture.service)) { + fixture.service = NULL; + } + (void)cbm_daemon_ipc_participant_guard_release(&coordination.guard); + while (coordination.claim && cbm_version_cohort_daemon_claim_release(&coordination.claim) != + CBM_PRIVATE_FILE_LOCK_OK) { + cbm_usleep(1000); + } + while (coordination.lease && + cbm_version_cohort_lease_release(&coordination.lease) != CBM_PRIVATE_FILE_LOCK_OK) { + cbm_usleep(1000); + } + while (coordination.manager && + cbm_version_cohort_manager_free(&coordination.manager) != CBM_PRIVATE_FILE_LOCK_OK) { + cbm_usleep(1000); + } + runtime_test_fixture_finish(&fixture); + + ASSERT_TRUE(started); + ASSERT_TRUE(held); + ASSERT_TRUE(staged); + ASSERT_TRUE(loop_ok); + ASSERT_EQ(state, CBM_DAEMON_RUNTIME_SERVICE_EXITED); + ASSERT_TRUE(marker_refreshed); + ASSERT_TRUE(atomic_load(&runtime_test_coordination_lost_logged)); + ASSERT_FALSE(atomic_load(&runtime_test_window_expired_logged)); + PASS(); +#endif +} + static bool runtime_test_fixture_start_application(runtime_test_fixture_t *fixture, const char *tag, const cbm_daemon_build_identity_t *identity, runtime_application_context_t *context) { @@ -5106,6 +5236,7 @@ SUITE(daemon_runtime) { RUN_TEST(daemon_runtime_process_fingerprint_never_hashes_replacement_path); #endif RUN_TEST(daemon_runtime_convenience_service_owns_participant_guard); + RUN_TEST(daemon_runtime_host_loop_stops_on_coordination_file_loss); RUN_TEST(daemon_runtime_rendezvous_layout_is_frozen_and_detailed_abi_independent); RUN_TEST(daemon_runtime_exact_hello_issues_connection_bound_identity); RUN_TEST(daemon_runtime_image_rejection_reaches_client_issue1383); From b15e7f91b95958555ac23a575b0bd1ebbfec7ddf Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 25 Sep 2026 14:09:27 -0500 Subject: [PATCH 042/217] style(tests): drop unrelated reflow hunks Restore main's formatting on pre-existing lines that a whole-file clang-format pass had reflowed, so the diff carries only the #2178 change. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NbYKGdU7pqG6AEuMQuJxiF Signed-off-by: Shane McCarron --- tests/test_daemon_ipc.c | 7 ++++--- tests/test_version_cohort.c | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index 5fcf8c1fc..5a4e0e386 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -1004,8 +1004,8 @@ TEST(daemon_ipc_windows_sid_trust_accepts_local_admin_rejects_foreign_500) { (void)CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL, NULL, &builtin_needed); if (builtin_needed > 0) { builtin_admins = malloc(builtin_needed); - if (builtin_admins && !CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL, builtin_admins, - &builtin_needed)) { + if (builtin_admins && + !CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL, builtin_admins, &builtin_needed)) { free(builtin_admins); builtin_admins = NULL; } @@ -5362,7 +5362,8 @@ TEST(daemon_ipc_posix_single_uid_userns_real_smoke_issue1830) { if (WEXITSTATUS(status) != 0 && WEXITSTATUS(status) != 1) { char unexpected[128]; (void)snprintf(unexpected, sizeof(unexpected), - "userns probe exited %d -- not a security verdict", WEXITSTATUS(status)); + "userns probe exited %d -- not a security verdict", + WEXITSTATUS(status)); FAIL(unexpected); } ASSERT_EQ(0, WEXITSTATUS(status)); diff --git a/tests/test_version_cohort.c b/tests/test_version_cohort.c index d5b6d4c5a..f48bd7225 100644 --- a/tests/test_version_cohort.c +++ b/tests/test_version_cohort.c @@ -332,9 +332,9 @@ TEST(version_cohort_conflict_is_retried_until_the_deadline) { /* An indefinite deadline never waits on a conflicting holder. */ retries_before = cbm_version_cohort_conflict_retries_for_testing(); - ASSERT_EQ( - cbm_version_cohort_acquire(second, &requested, UINT64_MAX, &requested_lease, &conflict), - CBM_VERSION_COHORT_CONFLICT); + ASSERT_EQ(cbm_version_cohort_acquire(second, &requested, UINT64_MAX, &requested_lease, + &conflict), + CBM_VERSION_COHORT_CONFLICT); ASSERT_NULL(requested_lease); ASSERT_EQ(cbm_version_cohort_conflict_retries_for_testing(), retries_before); From 8cd2ee5f421d0b7c06022091d59dbe71002de30d Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 25 Sep 2026 14:39:51 -0500 Subject: [PATCH 043/217] fix(daemon): address QA round 4 Reflow the host-loop seam call to clang-format and note that the fixture's convenience start joins its own participant guard, unlike the host's start_reserved. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NbYKGdU7pqG6AEuMQuJxiF Signed-off-by: Shane McCarron --- tests/test_daemon_runtime.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index 5ca4c72dd..5187bb7a4 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -1272,7 +1272,8 @@ typedef struct { /* When set, the fixture takes the host's coordination handles in the host's * order before starting the service: cohort admission refuses once a - * generation is already serving. */ + * generation is already serving. Unlike the host's start_reserved, the + * fixture's convenience start also joins a service-owned participant guard. */ static runtime_test_coordination_t *runtime_test_fixture_coordination = NULL; static bool runtime_test_fixture_start_configured( @@ -1452,9 +1453,9 @@ TEST(daemon_runtime_host_loop_stops_on_coordination_file_loss) { CBMLogLevel previous_log_level = cbm_log_get_level(); cbm_log_set_level(CBM_LOG_INFO); cbm_log_set_sink(runtime_test_lifetime_log_sink); - loop_ok = cbm_daemon_host_wait_for_lifetime_for_test( - fixture.service, fixture.endpoint, coordination.lease, coordination.claim, - coordination.guard); + loop_ok = cbm_daemon_host_wait_for_lifetime_for_test(fixture.service, fixture.endpoint, + coordination.lease, coordination.claim, + coordination.guard); cbm_log_set_sink(NULL); cbm_log_set_level(previous_log_level); } From 26e2d0cba6df7b1cb53e655d0b902bdb7debdbd4 Mon Sep 17 00:00:00 2001 From: News Date: Sat, 26 Sep 2026 20:30:02 +0900 Subject: [PATCH 044/217] fix(windows): read worker and CLI environment paths as UTF-8 Signed-off-by: News --- internal/cbm/cbm.c | 9 +++++++-- src/cli/cli.c | 4 +++- tests/test_extraction.c | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index b54fb54a7..f17d54757 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -25,6 +25,7 @@ #include "foundation/compat_fs.h" // cbm_fopen — crash-supervisor per-file marker write #include "foundation/hash_table.h" // CBMHashTable — crash-supervisor quarantine set #include "tree_sitter/api.h" // TSParser, TSNode, TSTree, TSInput, TSLanguage, TSPoint, TSParseOptions, TSParseState +#include "foundation/platform.h" #include "foundation/constants.h" #include "mimalloc.h" // mi_malloc/mi_calloc/mi_realloc/mi_free/mi_usable_size — bind 3rd-party allocators (#424) #if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR @@ -969,7 +970,9 @@ static int count_params_from_signature(const char *sig) { * unlucky in-flight file is never quarantined alone. The env var is set * solely by the supervisor during recovery — a no-op on normal runs. */ static void cbm_index_mark(const char *rel_path, char event) { - const char *mf = getenv("CBM_INDEX_MARKER_FILE"); + char marker_path[CBM_SZ_4K]; + const char *mf = + cbm_safe_getenv("CBM_INDEX_MARKER_FILE", marker_path, sizeof(marker_path), NULL); if (!mf || !mf[0] || !rel_path || !rel_path[0]) { return; } @@ -1006,7 +1009,9 @@ enum { CBM_QSET_UNINIT = 0, CBM_QSET_INITING = 1, CBM_QSET_INITED = 2 }; static atomic_int g_quarantine_state = CBM_QSET_UNINIT; static void cbm_quarantine_load(void) { - const char *qf = getenv("CBM_INDEX_QUARANTINE_FILE"); + char quarantine_path[CBM_SZ_4K]; + const char *qf = cbm_safe_getenv("CBM_INDEX_QUARANTINE_FILE", quarantine_path, + sizeof(quarantine_path), NULL); if (!qf || !qf[0]) { return; /* normal path: empty set */ } diff --git a/src/cli/cli.c b/src/cli/cli.c index b59a894a6..5de7b9db5 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -759,7 +759,9 @@ static bool cli_activation_production_context_init(cli_activation_production_con context->cleanup_ok = true; context->deadline_ms = cli_activation_deadline_after(CLI_ACTIVATION_DRAIN_TIMEOUT_MS); context->control_deadline_ms = cli_activation_deadline_after(CLI_ACTIVATION_CONTROL_TIMEOUT_MS); - const char *original_cache_environment = getenv("CBM_CACHE_DIR"); + char original_cache_buffer[CLI_BUF_4K]; + const char *original_cache_environment = cbm_safe_getenv("CBM_CACHE_DIR", original_cache_buffer, + sizeof(original_cache_buffer), NULL); context->original_cache_environment_present = original_cache_environment != NULL; if (context->original_cache_environment_present) { context->original_cache_environment = strdup(original_cache_environment); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 619b94045..806433b0b 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -7,6 +7,7 @@ */ #include "test_framework.h" #include "cbm.h" +#include "foundation/platform.h" #include "foundation/constants.h" /* CBM_SZ_* */ #include "preprocessor.h" /* cbm_export_macro_candidates (#1989) */ #include "../src/foundation/compat.h" /* cbm_clock_gettime (wide-flat scaling guard) */ @@ -8157,7 +8158,42 @@ TEST(extract_walk_truncated_when_a_node_budget_is_set) { PASS(); } +#ifdef _WIN32 +/* The supervisor sets this path through cbm_setenv. Exercise the actual + * journal writer, which must read it as UTF-8 before calling cbm_fopen. */ +TEST(extraction_marker_round_trips_non_ascii_environment_path) { + char dir[512] = "/tmp/cbm-marker-XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(dir)); + char marker[640]; + snprintf(marker, sizeof(marker), "%s/marker-\xce\x94-\xe4\xb8\x81.log", dir); + char saved[4096]; + const char *previous = cbm_safe_getenv("CBM_INDEX_MARKER_FILE", saved, sizeof(saved), NULL); + ASSERT_EQ(cbm_setenv("CBM_INDEX_MARKER_FILE", marker, 1), 0); + cbm_index_mark_start("synthetic.cpp"); + cbm_index_mark_done("synthetic.cpp"); + if (previous) { + (void)cbm_setenv("CBM_INDEX_MARKER_FILE", saved, 1); + } else { + (void)cbm_unsetenv("CBM_INDEX_MARKER_FILE"); + } + FILE *file = cbm_fopen(marker, "rb"); + char contents[128] = {0}; + if (file) { + (void)fread(contents, 1, sizeof(contents) - 1, file); + (void)fclose(file); + } + (void)cbm_unlink(marker); + (void)cbm_rmdir(dir); + ASSERT_NOT_NULL(file); + ASSERT_STR_EQ(contents, "S synthetic.cpp\nD synthetic.cpp\n"); + PASS(); +} +#endif + SUITE(extraction) { +#ifdef _WIN32 + RUN_TEST(extraction_marker_round_trips_non_ascii_environment_path); +#endif RUN_TEST(extract_compact_keeps_every_field_and_shrinks_the_arena); RUN_TEST(extract_compact_is_idempotent_and_survives_empty_results); RUN_TEST(extract_spill_round_trip_keeps_every_field); From 9dca8d2fede860605943ee8dbd57ba6b927ae41b Mon Sep 17 00:00:00 2001 From: News Date: Sat, 26 Sep 2026 20:38:47 +0900 Subject: [PATCH 045/217] fix(windows): limit allocator fix to CRT aligned blocks Signed-off-by: News --- src/foundation/mem_override_win.c | 47 ------------------------------- 1 file changed, 47 deletions(-) diff --git a/src/foundation/mem_override_win.c b/src/foundation/mem_override_win.c index bb86a2e31..a33b697d2 100644 --- a/src/foundation/mem_override_win.c +++ b/src/foundation/mem_override_win.c @@ -161,53 +161,6 @@ void __wrap__aligned_free(void *block) { __real__aligned_free(block); } -#if defined(CBM_MEM_GLOBAL_OVERRIDE) && CBM_MEM_GLOBAL_OVERRIDE -/* LLVM-MinGW's static C++ runtime calls _aligned_malloc through the linker's - * wrapper but its aligned operator delete can call the CRT import pointer - * __imp__aligned_free directly. --wrap=_aligned_free does not rewrite that - * import-pointer reference. Provide the six C++ ABI delete entry points so - * every aligned new/delete pair reaches the same owner-aware deallocator. */ -void cbm_cxx_aligned_delete(void *block) __asm__("_ZdlPvSt11align_val_t"); -void cbm_cxx_aligned_delete(void *block) { - __wrap__aligned_free(block); -} - -void cbm_cxx_aligned_delete_nothrow(void *block, - const void *tag) __asm__("_ZdlPvSt11align_val_tRKSt9nothrow_t"); -void cbm_cxx_aligned_delete_nothrow(void *block, const void *tag) { - (void)tag; - __wrap__aligned_free(block); -} - -void cbm_cxx_aligned_delete_sized(void *block, size_t size, - size_t alignment) __asm__("_ZdlPvySt11align_val_t"); -void cbm_cxx_aligned_delete_sized(void *block, size_t size, size_t alignment) { - (void)size; - (void)alignment; - __wrap__aligned_free(block); -} - -void cbm_cxx_aligned_array_delete(void *block) __asm__("_ZdaPvSt11align_val_t"); -void cbm_cxx_aligned_array_delete(void *block) { - __wrap__aligned_free(block); -} - -void cbm_cxx_aligned_array_delete_nothrow(void *block, const void *tag) __asm__( - "_ZdaPvSt11align_val_tRKSt9nothrow_t"); -void cbm_cxx_aligned_array_delete_nothrow(void *block, const void *tag) { - (void)tag; - __wrap__aligned_free(block); -} - -void cbm_cxx_aligned_array_delete_sized(void *block, size_t size, - size_t alignment) __asm__("_ZdaPvySt11align_val_t"); -void cbm_cxx_aligned_array_delete_sized(void *block, size_t size, size_t alignment) { - (void)size; - (void)alignment; - __wrap__aligned_free(block); -} -#endif - void *__wrap_realloc(void *block, size_t size) { if (!block) { void *fresh = mi_malloc(size); From 6f17caa7a763d9987ebcd7ed33063d47d34bf635 Mon Sep 17 00:00:00 2001 From: lorenzozanee Date: Sun, 27 Sep 2026 01:34:35 +0800 Subject: [PATCH 046/217] fix(mcp): report request timing and transport failures Signed-off-by: lorenzozanee --- src/daemon/application.c | 21 ++- src/foundation/log.c | 22 ++- src/foundation/log.h | 3 +- src/mcp/mcp.c | 302 +++++++++++++++++++++++++++++---------- src/ui/http_server.c | 25 +++- tests/test_log.c | 5 - tests/test_mcp.c | 165 +++++++++++++++++++-- 7 files changed, 439 insertions(+), 104 deletions(-) diff --git a/src/daemon/application.c b/src/daemon/application.c index 748b8bb75..94d865f4a 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -2144,13 +2144,19 @@ static void application_update_notice_inject(cbm_daemon_application_session_t *s char **response_io) { cbm_daemon_application_t *application = session->application; cbm_mutex_lock(&application->mutex); - if (session->background_eligible && !session->update_notice_delivered && - !session->pending_update_notice && application->update_thread_done && - application->update_notice[0] && - cbm_mcp_jsonrpc_response_prepend_notice(response_io, application->update_notice)) { + bool should_inject = session->background_eligible && !session->update_notice_delivered && + !session->pending_update_notice && application->update_thread_done && + application->update_notice[0]; + bool injected = should_inject && + cbm_mcp_jsonrpc_response_prepend_notice(response_io, + application->update_notice); + if (injected) { session->pending_update_notice = true; } cbm_mutex_unlock(&application->mutex); + if (should_inject && !injected) { + cbm_log_warn("daemon.update_notice.inject_failed", "status", "omitted"); + } } static bool application_watch_job_subscription_exists_locked( @@ -2556,6 +2562,13 @@ static cbm_daemon_runtime_application_status_t application_mcp_request( parsed_ok && parsed.has_id && parsed.method && strcmp(parsed.method, "tools/call") == 0; char *response = cbm_mcp_server_handle(session->mcp, message); free(message); + if (!response && (!parsed_ok || parsed.has_id)) { + cbm_log_error("daemon.mcp.response_missing", "status", "error"); + if (parsed_ok) { + cbm_jsonrpc_request_free(&parsed); + } + return CBM_DAEMON_RUNTIME_APPLICATION_HANDLER_ERROR; + } if (initialize_request && application_jsonrpc_success(response)) { cbm_mutex_lock(&session->application->mutex); session->pending_background_initialize = true; diff --git a/src/foundation/log.c b/src/foundation/log.c index da8531aab..111c09c1b 100644 --- a/src/foundation/log.c +++ b/src/foundation/log.c @@ -363,17 +363,29 @@ static void copy_path_without_query(const char *path, char *out, size_t outsz) { } void cbm_log_mcp_request(const char *method, const char *tool_name, bool is_error, - int64_t duration_us) { + int64_t duration_us, int64_t handler_duration_us, + int64_t response_serialization_us, bool cancelled, bool timed_out) { char duration_ms[CBM_SZ_32]; + char handler_duration_ms[CBM_SZ_32]; + char response_serialization_ms[CBM_SZ_32]; snprintf(duration_ms, sizeof(duration_ms), "%" PRId64, duration_us / 1000); + snprintf(handler_duration_ms, sizeof(handler_duration_ms), "%" PRId64, + handler_duration_us / 1000); + snprintf(response_serialization_ms, sizeof(response_serialization_ms), "%" PRId64, + response_serialization_us / 1000); + const char *status = timed_out ? "timeout" : (is_error ? "error" : "ok"); if (tool_name && tool_name[0] != '\0') { cbm_log(is_error ? CBM_LOG_WARN : CBM_LOG_INFO, "mcp.request", "protocol", "jsonrpc", - "method", method ? method : "", "tool", tool_name, "status", - is_error ? "error" : "ok", "duration_ms", duration_ms, NULL); + "method", method ? method : "", "tool", tool_name, "status", status, + "duration_ms", duration_ms, "handler_duration_ms", handler_duration_ms, + "response_serialization_ms", response_serialization_ms, "cancelled", + cancelled ? "true" : "false", "timed_out", timed_out ? "true" : "false", NULL); } else { cbm_log(is_error ? CBM_LOG_WARN : CBM_LOG_INFO, "mcp.request", "protocol", "jsonrpc", - "method", method ? method : "", "status", is_error ? "error" : "ok", "duration_ms", - duration_ms, NULL); + "method", method ? method : "", "status", status, "duration_ms", duration_ms, + "handler_duration_ms", handler_duration_ms, "response_serialization_ms", + response_serialization_ms, "cancelled", cancelled ? "true" : "false", "timed_out", + timed_out ? "true" : "false", NULL); } } diff --git a/src/foundation/log.h b/src/foundation/log.h index 88e0d25fc..ac4cfc3ca 100644 --- a/src/foundation/log.h +++ b/src/foundation/log.h @@ -95,7 +95,8 @@ void cbm_log_int(CBMLogLevel level, const char *msg, const char *key, int64_t va /* Operational event helpers. They deliberately avoid request bodies, headers, * arguments, and query strings. */ void cbm_log_mcp_request(const char *method, const char *tool_name, bool is_error, - int64_t duration_us); + int64_t duration_us, int64_t handler_duration_us, + int64_t response_serialization_us, bool cancelled, bool timed_out); void cbm_log_http_request(const char *component, const char *method, const char *path, int status, int64_t duration_ms, size_t request_bytes, size_t response_bytes); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index d6c7bd468..462bfba35 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -128,6 +128,7 @@ enum { /* JSON-RPC 2.0 standard error codes */ #define JSONRPC_PARSE_ERROR (-32700) +#define MCP_INPUT_CLOSED (-2) #define JSONRPC_METHOD_NOT_FOUND (-32601) #define JSONRPC_INVALID_PARAMS (-32602) #define JSONRPC_INTERNAL_ERROR (-32603) @@ -245,28 +246,45 @@ int cbm_jsonrpc_parse(const char *line, cbm_jsonrpc_request_t *out) { yyjson_val *v_id = yyjson_obj_get(root, "id"); yyjson_val *v_params = yyjson_obj_get(root, "params"); - if (!v_method || !yyjson_is_str(v_method)) { + if (!v_jsonrpc || !yyjson_is_str(v_jsonrpc) || + strcmp(yyjson_get_str(v_jsonrpc), "2.0") != 0 || !v_method || + !yyjson_is_str(v_method) || + (v_id && !yyjson_is_int(v_id) && !yyjson_is_str(v_id))) { yyjson_doc_free(doc); return CBM_NOT_FOUND; } - out->jsonrpc = - heap_strdup(v_jsonrpc && yyjson_is_str(v_jsonrpc) ? yyjson_get_str(v_jsonrpc) : "2.0"); + out->jsonrpc = heap_strdup(yyjson_get_str(v_jsonrpc)); out->method = heap_strdup(yyjson_get_str(v_method)); + if (!out->jsonrpc || !out->method) { + yyjson_doc_free(doc); + cbm_jsonrpc_request_free(out); + return CBM_NOT_FOUND; + } if (v_id) { out->has_id = true; if (yyjson_is_int(v_id)) { out->id = yyjson_get_int(v_id); - } else if (yyjson_is_str(v_id)) { + } else { /* JSON-RPC 2.0 §4 permits string ids (Claude Desktop uses them). * Preserve verbatim instead of coercing via strtol (issue #253). */ out->id_str = heap_strdup(yyjson_get_str(v_id)); + if (!out->id_str) { + yyjson_doc_free(doc); + cbm_jsonrpc_request_free(out); + return CBM_NOT_FOUND; + } } } if (v_params) { out->params_raw = yyjson_val_write(v_params, 0, NULL); + if (!out->params_raw) { + yyjson_doc_free(doc); + cbm_jsonrpc_request_free(out); + return CBM_NOT_FOUND; + } } yyjson_doc_free(doc); @@ -289,36 +307,62 @@ void cbm_jsonrpc_request_free(cbm_jsonrpc_request_t *r) { * ══════════════════════════════════════════════════════════════════ */ char *cbm_jsonrpc_format_response(const cbm_jsonrpc_response_t *resp) { + if (!resp) { + return NULL; + } yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + if (!doc) { + return NULL; + } yyjson_mut_val *root = yyjson_mut_obj(doc); + if (!root) { + yyjson_mut_doc_free(doc); + return NULL; + } yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_str(doc, root, "jsonrpc", "2.0"); + bool added = yyjson_mut_obj_add_str(doc, root, "jsonrpc", "2.0"); if (resp->id_str) { - yyjson_mut_obj_add_str(doc, root, "id", resp->id_str); + added = added && yyjson_mut_obj_add_str(doc, root, "id", resp->id_str); } else { - yyjson_mut_obj_add_int(doc, root, "id", resp->id); + added = added && yyjson_mut_obj_add_int(doc, root, "id", resp->id); + } + if (!added) { + yyjson_mut_doc_free(doc); + return NULL; } if (resp->error_json) { /* Parse the error JSON and embed */ yyjson_doc *err_doc = yyjson_read(resp->error_json, strlen(resp->error_json), 0); - if (err_doc) { - yyjson_mut_val *err_val = yyjson_val_mut_copy(doc, yyjson_doc_get_root(err_doc)); - yyjson_mut_obj_add_val(doc, root, "error", err_val); - yyjson_doc_free(err_doc); + if (!err_doc) { + yyjson_mut_doc_free(doc); + return NULL; + } + yyjson_mut_val *err_val = yyjson_val_mut_copy(doc, yyjson_doc_get_root(err_doc)); + bool added = err_val && yyjson_mut_obj_add_val(doc, root, "error", err_val); + yyjson_doc_free(err_doc); + if (!added) { + yyjson_mut_doc_free(doc); + return NULL; } } else if (resp->result_json) { /* Parse the result JSON and embed */ yyjson_doc *res_doc = yyjson_read(resp->result_json, strlen(resp->result_json), 0); - if (res_doc) { - yyjson_mut_val *res_val = yyjson_val_mut_copy(doc, yyjson_doc_get_root(res_doc)); - yyjson_mut_obj_add_val(doc, root, "result", res_val); - yyjson_doc_free(res_doc); + if (!res_doc) { + yyjson_mut_doc_free(doc); + return NULL; + } + yyjson_mut_val *res_val = yyjson_val_mut_copy(doc, yyjson_doc_get_root(res_doc)); + bool added = res_val && yyjson_mut_obj_add_val(doc, root, "result", res_val); + yyjson_doc_free(res_doc); + if (!added) { + yyjson_mut_doc_free(doc); + return NULL; } } else { - /* JSON-RPC 2.0 spec: response MUST contain "result" or "error" */ - yyjson_mut_obj_add_null(doc, root, "result"); + yyjson_mut_doc_free(doc); + return NULL; } char *out = yy_final_doc_to_str(doc); @@ -326,13 +370,44 @@ char *cbm_jsonrpc_format_response(const cbm_jsonrpc_response_t *resp) { return out; } +static char *mcp_format_response_or_error(const cbm_jsonrpc_response_t *resp, + bool *serialization_failed) { + if (serialization_failed) { + *serialization_failed = false; + } + char *formatted = cbm_jsonrpc_format_response(resp); + if (formatted) { + return formatted; + } + if (serialization_failed) { + *serialization_failed = true; + } + cbm_jsonrpc_response_t error_resp = { + .id = resp->id, + .id_str = resp->id_str, + .error_json = "{\"code\":-32603,\"message\":\"Failed to serialize the server response\"}", + }; + return cbm_jsonrpc_format_response(&error_resp); +} + char *cbm_jsonrpc_format_error(int64_t id, int code, const char *message) { yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + if (!doc) { + return NULL; + } yyjson_mut_val *root = yyjson_mut_obj(doc); + if (!root) { + yyjson_mut_doc_free(doc); + return NULL; + } yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_str(doc, root, "jsonrpc", "2.0"); - yyjson_mut_obj_add_int(doc, root, "id", id); + bool added = yyjson_mut_obj_add_str(doc, root, "jsonrpc", "2.0"); + added = added && yyjson_mut_obj_add_int(doc, root, "id", id); + if (!added) { + yyjson_mut_doc_free(doc); + return NULL; + } char *encoded_message = NULL; size_t encoded_message_len = 0; @@ -340,13 +415,21 @@ char *cbm_jsonrpc_format_error(int64_t id, int code, const char *message) { if (!cbm_output_encode_text(raw_message, strlen(raw_message), &encoded_message, &encoded_message_len)) { yyjson_mut_doc_free(doc); + free(encoded_message); return NULL; } yyjson_mut_val *err = yyjson_mut_obj(doc); - yyjson_mut_obj_add_int(doc, err, "code", code); - yyjson_mut_obj_add_str(doc, err, "message", encoded_message ? encoded_message : raw_message); - yyjson_mut_obj_add_val(doc, root, "error", err); + added = err && yyjson_mut_obj_add_int(doc, err, "code", code); + added = added && + yyjson_mut_obj_add_str(doc, err, "message", encoded_message ? encoded_message + : raw_message); + added = added && yyjson_mut_obj_add_val(doc, root, "error", err); + if (!added) { + yyjson_mut_doc_free(doc); + free(encoded_message); + return NULL; + } char *out = yy_final_doc_to_str(doc); yyjson_mut_doc_free(doc); @@ -2933,7 +3016,9 @@ static char *handle_list_projects(cbm_mcp_server_t *srv, const char *args) { free(records); return cbm_mcp_text_result("out of memory while listing projects", true); } - qsort(records, (size_t)record_count, sizeof(*records), project_record_compare); + if (record_count > 1) { + qsort(records, (size_t)record_count, sizeof(*records), project_record_compare); + } int limit = cbm_mcp_get_int_arg(args, "limit", 50); int offset = cbm_mcp_get_int_arg(args, "offset", 0); @@ -17796,6 +17881,11 @@ bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *not return true; } +static long long mcp_elapsed_us(struct timespec start, struct timespec end) { + return ((long long)(end.tv_sec - start.tv_sec) * MCP_S_TO_US) + + ((long long)(end.tv_nsec - start.tv_nsec) / MCP_MS_TO_US); +} + char *cbm_mcp_server_handle(cbm_mcp_server_t *srv, const char *line) { cbm_jsonrpc_request_t req = {0}; if (cbm_jsonrpc_parse(line, &req) < 0) { @@ -17826,7 +17916,11 @@ char *cbm_mcp_server_handle(cbm_mcp_server_t *srv, const char *line) { cbm_clock_gettime(CLOCK_MONOTONIC, &req_t0); char *result_json = NULL; char *request_error_json = NULL; - bool request_logged = false; + char *request_tool_name = NULL; + long long handler_duration_us = 0; + bool request_is_error = false; + bool request_cancelled = false; + bool request_timed_out = false; if (strcmp(req.method, "initialize") == 0) { result_json = cbm_mcp_initialize_response_for_profile(req.params_raw, srv->tool_profile); @@ -17853,6 +17947,7 @@ char *cbm_mcp_server_handle(cbm_mcp_server_t *srv, const char *line) { char *tool_name = req.params_raw ? cbm_mcp_get_tool_name(req.params_raw) : NULL; char *tool_args = req.params_raw ? cbm_mcp_get_arguments(req.params_raw) : heap_strdup("{}"); + request_tool_name = tool_name; srv->active_request_id = req.id; free(srv->active_request_id_str); srv->active_request_id_str = req.id_str ? heap_strdup(req.id_str) : NULL; @@ -17865,16 +17960,15 @@ char *cbm_mcp_server_handle(cbm_mcp_server_t *srv, const char *line) { srv->active_request_id_str = NULL; struct timespec t1; cbm_clock_gettime(CLOCK_MONOTONIC, &t1); - long long dur_us = ((long long)(t1.tv_sec - t0.tv_sec) * MCP_S_TO_US) + - ((long long)(t1.tv_nsec - t0.tv_nsec) / MCP_MS_TO_US); + long long dur_us = mcp_elapsed_us(t0, t1); bool is_err = (result_json != NULL) && (strstr(result_json, "\"isError\":true") != NULL); cbm_diag_record_query(dur_us, is_err); - long long request_dur_us = ((long long)(t1.tv_sec - req_t0.tv_sec) * MCP_S_TO_US) + - ((long long)(t1.tv_nsec - req_t0.tv_nsec) / MCP_MS_TO_US); - cbm_log_mcp_request(req.method, tool_name, is_err, request_dur_us); - request_logged = true; + handler_duration_us = mcp_elapsed_us(req_t0, t1); + request_is_error = is_err; + request_cancelled = mcp_request_cancelled(srv); + request_timed_out = result_json && + strstr(result_json, "\"code\":\"request_timeout\"") != NULL; - free(tool_name); free(tool_args); } else { /* Echo the original id (string or numeric, issue #253) on the error. */ @@ -17886,12 +17980,18 @@ char *cbm_mcp_server_handle(cbm_mcp_server_t *srv, const char *line) { .id_str = req.id_str, .error_json = err_obj, }; - char *err = cbm_jsonrpc_format_response(&err_resp); struct timespec t1; cbm_clock_gettime(CLOCK_MONOTONIC, &t1); - long long dur_us = ((long long)(t1.tv_sec - req_t0.tv_sec) * MCP_S_TO_US) + - ((long long)(t1.tv_nsec - req_t0.tv_nsec) / MCP_MS_TO_US); - cbm_log_mcp_request(req.method, NULL, true, dur_us); + long long handler_us = mcp_elapsed_us(req_t0, t1); + struct timespec serialize_t0; + cbm_clock_gettime(CLOCK_MONOTONIC, &serialize_t0); + char *err = mcp_format_response_or_error(&err_resp, NULL); + struct timespec serialize_t1; + cbm_clock_gettime(CLOCK_MONOTONIC, &serialize_t1); + long long serialization_us = mcp_elapsed_us(serialize_t0, serialize_t1); + long long duration_us = mcp_elapsed_us(req_t0, serialize_t1); + cbm_log_mcp_request(req.method, NULL, true, duration_us, handler_us, serialization_us, + false, false); cbm_mcp_server_request_scope_end(srv); cbm_jsonrpc_request_free(&req); return err; @@ -17903,32 +18003,43 @@ char *cbm_mcp_server_handle(cbm_mcp_server_t *srv, const char *line) { .id_str = req.id_str, .error_json = request_error_json, }; - char *err = cbm_jsonrpc_format_response(&err_resp); - struct timespec t1; - cbm_clock_gettime(CLOCK_MONOTONIC, &t1); - long long dur_us = ((long long)(t1.tv_sec - req_t0.tv_sec) * MCP_S_TO_US) + - ((long long)(t1.tv_nsec - req_t0.tv_nsec) / MCP_MS_TO_US); - cbm_log_mcp_request(req.method, NULL, true, dur_us); + struct timespec serialize_t0; + cbm_clock_gettime(CLOCK_MONOTONIC, &serialize_t0); + char *err = mcp_format_response_or_error(&err_resp, NULL); + struct timespec serialize_t1; + cbm_clock_gettime(CLOCK_MONOTONIC, &serialize_t1); + long long handler_us = mcp_elapsed_us(req_t0, serialize_t0); + long long serialization_us = mcp_elapsed_us(serialize_t0, serialize_t1); + long long duration_us = mcp_elapsed_us(req_t0, serialize_t1); + cbm_log_mcp_request(req.method, NULL, true, duration_us, handler_us, serialization_us, + false, false); free(request_error_json); cbm_mcp_server_request_scope_end(srv); cbm_jsonrpc_request_free(&req); return err; } - if (!request_logged) { - struct timespec t1; - cbm_clock_gettime(CLOCK_MONOTONIC, &t1); - long long dur_us = ((long long)(t1.tv_sec - req_t0.tv_sec) * MCP_S_TO_US) + - ((long long)(t1.tv_nsec - req_t0.tv_nsec) / MCP_MS_TO_US); - cbm_log_mcp_request(req.method, NULL, false, dur_us); - } - cbm_jsonrpc_response_t resp = { .id = req.id, .id_str = req.id_str, .result_json = result_json, }; - char *out = cbm_jsonrpc_format_response(&resp); + struct timespec serialize_t0; + cbm_clock_gettime(CLOCK_MONOTONIC, &serialize_t0); + bool serialization_failed = false; + char *out = mcp_format_response_or_error(&resp, &serialization_failed); + struct timespec serialize_t1; + cbm_clock_gettime(CLOCK_MONOTONIC, &serialize_t1); + long long serialization_us = mcp_elapsed_us(serialize_t0, serialize_t1); + long long duration_us = mcp_elapsed_us(req_t0, serialize_t1); + request_is_error = request_is_error || serialization_failed || out == NULL; + if (!request_tool_name) { + handler_duration_us = duration_us - serialization_us; + } + cbm_log_mcp_request(req.method, request_tool_name, request_is_error, duration_us, + handler_duration_us, serialization_us, request_cancelled, + request_timed_out); + free(request_tool_name); free(result_json); cbm_mcp_server_request_scope_end(srv); cbm_jsonrpc_request_free(&req); @@ -18123,8 +18234,7 @@ int cbm_mcp_read_message(FILE *in, char **message, bool *content_length_framed) } #ifndef _WIN32 -/* Unix 3-phase poll: non-blocking fd check, FILE* buffer peek, blocking poll. - * Returns: 1 = data ready, 0 = timeout (evicted idle stores), -1 = error/EOF. */ +/* Unix 3-phase poll: non-blocking fd check, FILE* buffer peek, blocking poll. */ static int poll_for_input_unix(cbm_mcp_server_t *srv, int fd, FILE *in) { struct pollfd pfd = {.fd = fd, .events = POLLIN}; int pr = poll(&pfd, SKIP_ONE, 0); /* Phase 1: non-blocking */ @@ -18139,26 +18249,28 @@ static int poll_for_input_unix(cbm_mcp_server_t *srv, int fd, FILE *in) { /* Phase 2: peek FILE* buffer */ int saved_flags = fcntl(fd, F_GETFL); if (saved_flags < 0) { - /* fcntl failed — fall through to a short blocking poll (see the Phase-3 - * note below on why the interval is bounded, not the full idle timeout) */ - pr = poll(&pfd, SKIP_ONE, MCP_TIMEOUT_MS); - if (pr < 0) { - return CBM_NOT_FOUND; - } - if (pr == 0) { - cbm_mcp_server_evict_idle(srv, STORE_IDLE_TIMEOUT_S); - return 0; - } - return SKIP_ONE; + return CBM_NOT_FOUND; } - (void)fcntl(fd, F_SETFL, saved_flags | O_NONBLOCK); + if (fcntl(fd, F_SETFL, saved_flags | O_NONBLOCK) < 0) { + return CBM_NOT_FOUND; + } + errno = 0; int c = fgetc(in); - (void)fcntl(fd, F_SETFL, saved_flags); + int read_errno = errno; + bool read_failed = ferror(in) != 0; + bool reached_eof = feof(in) != 0; + int restore_result = fcntl(fd, F_SETFL, saved_flags); + if (restore_result < 0) { + return CBM_NOT_FOUND; + } if (c == EOF) { - if (feof(in)) { - return CBM_NOT_FOUND; /* true EOF */ + if (reached_eof) { + return MCP_INPUT_CLOSED; + } + if (!read_failed || (read_errno != EAGAIN && read_errno != EWOULDBLOCK)) { + return CBM_NOT_FOUND; } clearerr(in); /* Phase 3: blocking poll, bounded to a SHORT interval (not the full idle @@ -18180,7 +18292,9 @@ static int poll_for_input_unix(cbm_mcp_server_t *srv, int fd, FILE *in) { return SKIP_ONE; } - (void)ungetc(c, in); + if (ungetc(c, in) == EOF) { + return CBM_NOT_FOUND; + } return SKIP_ONE; } #endif @@ -18221,7 +18335,8 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { HANDLE hStdin = (HANDLE)_get_osfhandle(fd); DWORD wr = WaitForSingleObject(hStdin, STORE_IDLE_TIMEOUT_S * MCP_TIMEOUT_MS); if (wr == WAIT_FAILED) { - break; + cbm_log_error("mcp.input.poll", "status", "error"); + return 1; } if (wr == WAIT_TIMEOUT) { cbm_mcp_server_evict_idle(srv, STORE_IDLE_TIMEOUT_S); @@ -18229,9 +18344,13 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { } #else int pr = poll_for_input_unix(srv, fd, in); - if (pr < 0) { + if (pr == MCP_INPUT_CLOSED) { break; } + if (pr < 0) { + cbm_log_error("mcp.input.poll", "status", "error"); + return 1; + } if (pr == 0) { continue; /* timeout — idle stores evicted */ } @@ -18239,21 +18358,46 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { char *message = NULL; bool content_length_framed = false; - if (cbm_mcp_read_message(in, &message, &content_length_framed) <= 0) { + int read_status = cbm_mcp_read_message(in, &message, &content_length_framed); + if (read_status == 0) { break; } + if (read_status < 0) { + cbm_log_error("mcp.request.read", "status", "error"); + return 1; + } char *resp = cbm_mcp_server_handle(srv, message); + bool notification = false; + if (!resp) { + cbm_jsonrpc_request_t request = {0}; + notification = cbm_jsonrpc_parse(message, &request) == 0 && !request.has_id; + cbm_jsonrpc_request_free(&request); + } free(message); - if (resp) { - if (content_length_framed) { - size_t response_len = strlen(resp); - (void)fprintf(out, "Content-Length: %zu\r\n\r\n%s", response_len, resp); - } else { - (void)fprintf(out, "%s\n", resp); + if (!resp) { + if (!notification) { + cbm_log_error("mcp.response.missing", "protocol", "jsonrpc", "status", "error"); + return 1; } - (void)fflush(out); - free(resp); + continue; + } + + int write_result; + if (content_length_framed) { + size_t response_len = strlen(resp); + write_result = fprintf(out, "Content-Length: %zu\r\n\r\n%s", response_len, resp); + } else { + write_result = fprintf(out, "%s\n", resp); + } + free(resp); + if (write_result < 0) { + cbm_log_error("mcp.response.write", "protocol", "jsonrpc", "stage", "write"); + return 1; + } + if (fflush(out) != 0) { + cbm_log_error("mcp.response.write", "protocol", "jsonrpc", "stage", "flush"); + return 1; } } diff --git a/src/ui/http_server.c b/src/ui/http_server.c index f47ac59be..b5a496d56 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -1803,7 +1803,30 @@ static void handle_rpc(cbm_http_conn_t *c, const cbm_http_req_t *req, cbm_mcp_se cbm_http_replyf(c, 200, g_cors_json, "%s", response); free(response); } else { - cbm_http_replyf(c, 204, g_cors, "%s", ""); + cbm_jsonrpc_request_t parsed = {0}; + bool parsed_ok = cbm_jsonrpc_parse(req->body, &parsed) == 0; + bool notification = parsed_ok && !parsed.has_id; + char *error_response = NULL; + if (parsed_ok && parsed.has_id) { + cbm_jsonrpc_response_t error = { + .id = parsed.id, + .id_str = parsed.id_str, + .error_json = "{\"code\":-32603,\"message\":\"Failed to produce a response\"}", + }; + error_response = cbm_jsonrpc_format_response(&error); + } + cbm_jsonrpc_request_free(&parsed); + if (notification) { + cbm_http_replyf(c, 204, g_cors, "%s", ""); + } else { + cbm_log_error("http.rpc.response_missing", "status", "error"); + if (error_response) { + cbm_http_replyf(c, 500, g_cors_json, "%s", error_response); + } else { + cbm_http_replyf(c, 500, g_cors, "%s", "Failed to produce a response"); + } + } + free(error_response); } } diff --git a/tests/test_log.c b/tests/test_log.c index 2406ee3b6..602672800 100644 --- a/tests/test_log.c +++ b/tests/test_log.c @@ -207,15 +207,10 @@ TEST(log_operational_helpers) { cbm_log_set_level(CBM_LOG_DEBUG); cbm_log_set_format(CBM_LOG_FORMAT_TEXT); capture_start(); - cbm_log_mcp_request("tools/call", "search_graph", false, 1250); cbm_log_http_request("graph_ui", "GET", "/api/layout", 200, 7, 0, 42); const char *output = capture_end(); cbm_log_set_level(CBM_LOG_INFO); - ASSERT(cbm_str_contains_raw(output, "msg=mcp.request")); - ASSERT(cbm_str_contains_raw(output, "protocol=jsonrpc")); - ASSERT(cbm_str_contains_raw(output, "method=tools/call")); - ASSERT(cbm_str_contains_raw(output, "tool=search_graph")); ASSERT(cbm_str_contains_raw(output, "msg=http.request")); ASSERT(cbm_str_contains_raw(output, "method=GET")); ASSERT(cbm_str_contains_raw(output, "path=/api/layout")); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index ffa9ea3a3..55ae389aa 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -1149,6 +1149,19 @@ TEST(jsonrpc_format_response) { PASS(); } +TEST(jsonrpc_format_response_rejects_missing_or_malformed_payload) { + cbm_jsonrpc_response_t resp = {.id = 2}; + ASSERT_NULL(cbm_jsonrpc_format_response(&resp)); + + resp.result_json = "{broken"; + ASSERT_NULL(cbm_jsonrpc_format_response(&resp)); + + resp.result_json = NULL; + resp.error_json = "{broken"; + ASSERT_NULL(cbm_jsonrpc_format_response(&resp)); + PASS(); +} + TEST(jsonrpc_format_error) { char *json = cbm_jsonrpc_format_error(5, -32600, "Invalid Request"); ASSERT_NOT_NULL(json); @@ -2225,6 +2238,10 @@ TEST(server_handle_logs_request_without_params) { ASSERT_NOT_NULL(strstr(mcp_log_buf, "protocol=jsonrpc")); ASSERT_NOT_NULL(strstr(mcp_log_buf, "method=tools/list")); ASSERT_NOT_NULL(strstr(mcp_log_buf, "status=ok")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "handler_duration_ms=")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "response_serialization_ms=")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "cancelled=false")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "timed_out=false")); ASSERT_NULL(strstr(mcp_log_buf, "token")); ASSERT_NULL(strstr(mcp_log_buf, "secret")); PASS(); @@ -10665,14 +10682,27 @@ TEST(search_code_cancel_cleans_supervised_scan) { }; cbm_mcp_server_set_command_test_hook(srv, mcp_search_command_hook_probe, &probe); - char *response = - cbm_mcp_handle_tool(srv, "search_code", - "{\"pattern\":\"HandleRequest\",\"project\":\"prefilter-search\"," - "\"file_pattern\":\"*.go\"}"); + mcp_log_buf[0] = '\0'; + CBMLogLevel previous_level = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_INFO); + cbm_log_set_sink_ex(mcp_capture_log, CBM_LOG_SINK_REPLACE); + char *response = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_code\",\"arguments\":{\"pattern\":" + "\"HandleRequest\",\"project\":\"prefilter-search\",\"file_pattern\":" + "\"*.go\"}}}"); + cbm_log_set_sink(NULL); + cbm_log_set_level(previous_level); ASSERT_NOT_NULL(response); ASSERT_TRUE(probe.cancel_accepted); ASSERT_NOT_NULL(strstr(response, "cancelled")); ASSERT_NOT_NULL(strstr(response, "\"isError\":true")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "msg=mcp.request")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "tool=search_code")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "cancelled=true")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "timed_out=false")); + ASSERT_NULL(strstr(mcp_log_buf, "HandleRequest")); + ASSERT_NULL(strstr(mcp_log_buf, "prefilter-search")); char logs[640]; snprintf(logs, sizeof(logs), "%s/logs", cache.path); @@ -10685,6 +10715,39 @@ TEST(search_code_cancel_cleans_supervised_scan) { PASS(); } +TEST(server_handle_logs_timeout_without_query) { + char tmp[512], src_path[768], vendor_path[768]; + cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path), + vendor_path, sizeof(vendor_path)); + ASSERT_NOT_NULL(srv); + cbm_mcp_server_set_search_scan_timeout_for_test(srv, 0, true); + + mcp_log_buf[0] = '\0'; + CBMLogLevel previous_level = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_INFO); + cbm_log_set_sink_ex(mcp_capture_log, CBM_LOG_SINK_REPLACE); + char *response = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_code\",\"arguments\":{\"pattern\":" + "\"HandleRequest\",\"project\":\"prefilter-search\"}}}"); + cbm_log_set_sink(NULL); + cbm_log_set_level(previous_level); + + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "status=timeout")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "timed_out=true")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "cancelled=false")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "handler_duration_ms=")); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "response_serialization_ms=")); + ASSERT_NULL(strstr(mcp_log_buf, "HandleRequest")); + ASSERT_NULL(strstr(mcp_log_buf, "prefilter-search")); + + free(response); + cbm_mcp_server_free(srv); + cleanup_prefilter_dir(tmp, src_path, vendor_path); + PASS(); +} + TEST(search_code_output_limit_fails_closed_and_cleans_scan) { mcp_search_cache_t cache; ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-search-limit")); @@ -16631,14 +16694,24 @@ TEST(jsonrpc_parse_empty_string) { } TEST(jsonrpc_parse_missing_jsonrpc_field) { - /* jsonrpc field absent — parser defaults to "2.0" if method present */ + /* JSON-RPC requests must declare their protocol version. */ const char *line = "{\"id\":1,\"method\":\"initialize\",\"params\":{}}"; cbm_jsonrpc_request_t req = {0}; int rc = cbm_jsonrpc_parse(line, &req); - ASSERT_EQ(rc, 0); - ASSERT_STR_EQ(req.jsonrpc, "2.0"); - ASSERT_STR_EQ(req.method, "initialize"); - ASSERT_TRUE(req.has_id); + ASSERT_EQ(rc, -1); + cbm_jsonrpc_request_free(&req); + PASS(); +} + +TEST(jsonrpc_parse_rejects_invalid_version_and_id) { + cbm_jsonrpc_request_t req = {0}; + ASSERT_EQ(cbm_jsonrpc_parse( + "{\"jsonrpc\":\"1.0\",\"id\":1,\"method\":\"ping\"}", &req), + -1); + cbm_jsonrpc_request_free(&req); + ASSERT_EQ(cbm_jsonrpc_parse( + "{\"jsonrpc\":\"2.0\",\"id\":null,\"method\":\"ping\"}", &req), + -1); cbm_jsonrpc_request_free(&req); PASS(); } @@ -16967,6 +17040,75 @@ TEST(mcp_server_run_rapid_messages) { fclose(in_fp); PASS(); } + +#ifdef __linux__ +TEST(mcp_server_run_reports_output_failure) { + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + const char message[] = "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/list\",\"params\":{}}\n"; + ASSERT_EQ(write(fds[1], message, sizeof(message) - 1), (ssize_t)(sizeof(message) - 1)); + close(fds[1]); + + FILE *in_fp = fdopen(fds[0], "r"); + FILE *out_fp = fopen("/dev/full", "w"); + ASSERT_NOT_NULL(in_fp); + ASSERT_NOT_NULL(out_fp); + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + + mcp_log_buf[0] = '\0'; + CBMLogLevel previous_level = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_ERROR); + cbm_log_set_sink_ex(mcp_capture_log, CBM_LOG_SINK_REPLACE); + int rc = cbm_mcp_server_run(srv, in_fp, out_fp); + cbm_log_set_sink(NULL); + cbm_log_set_level(previous_level); + bool output_error_logged = strstr(mcp_log_buf, "stage=write") != NULL || + strstr(mcp_log_buf, "stage=flush") != NULL; + + cbm_mcp_server_free(srv); + fclose(in_fp); + fclose(out_fp); + + ASSERT_TRUE(rc != 0); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "msg=mcp.response.write")); + ASSERT_TRUE(output_error_logged); + PASS(); +} + +TEST(mcp_server_run_reports_input_failure) { + FILE *in_fp = tmpfile(); + FILE *out_fp = tmpfile(); + ASSERT_NOT_NULL(in_fp); + ASSERT_NOT_NULL(out_fp); + ASSERT_TRUE(fputs("Content-Length: nope\r\n\r\n", in_fp) >= 0); + rewind(in_fp); + + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + mcp_log_buf[0] = '\0'; + CBMLogLevel previous_level = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_ERROR); + cbm_log_set_sink_ex(mcp_capture_log, CBM_LOG_SINK_REPLACE); + int rc = cbm_mcp_server_run(srv, in_fp, out_fp); + cbm_log_set_sink(NULL); + cbm_log_set_level(previous_level); + + cbm_mcp_server_free(srv); + fclose(in_fp); + fclose(out_fp); + ASSERT_TRUE(rc != 0); + ASSERT_NOT_NULL(strstr(mcp_log_buf, "msg=mcp.request.read")); + PASS(); +} +#else +TEST(mcp_server_run_reports_output_failure) { + SKIP_PLATFORM("/dev/full is Linux-specific"); +} +TEST(mcp_server_run_reports_input_failure) { + SKIP_PLATFORM("tmpfile poll behavior is Unix-specific"); +} +#endif #endif /* !_WIN32 */ /* Issue #235: passing an unrecognised project name to a tool crashed the @@ -20428,6 +20570,7 @@ SUITE(mcp) { /* JSON-RPC parsing — edge cases */ RUN_TEST(jsonrpc_parse_empty_string); RUN_TEST(jsonrpc_parse_missing_jsonrpc_field); + RUN_TEST(jsonrpc_parse_rejects_invalid_version_and_id); RUN_TEST(jsonrpc_parse_missing_method); RUN_TEST(jsonrpc_parse_string_id); RUN_TEST(jsonrpc_parse_no_params); @@ -20436,6 +20579,7 @@ SUITE(mcp) { /* JSON-RPC formatting */ RUN_TEST(jsonrpc_format_response); + RUN_TEST(jsonrpc_format_response_rejects_missing_or_malformed_payload); RUN_TEST(jsonrpc_format_error); /* MCP protocol helpers */ @@ -20639,6 +20783,7 @@ SUITE(mcp) { RUN_TEST(search_code_file_pattern_prefilter_boundaries); RUN_TEST(search_code_windows_scope_prefilter_removes_pipeline_filter); RUN_TEST(search_code_cancel_cleans_supervised_scan); + RUN_TEST(server_handle_logs_timeout_without_query); RUN_TEST(search_code_output_limit_fails_closed_and_cleans_scan); RUN_TEST(search_code_scan_deadline_fails_closed_and_resets); RUN_TEST(search_code_scan_deadline_override_is_per_server); @@ -20736,6 +20881,8 @@ SUITE(mcp) { /* Poll/getline FILE* buffering fix */ #ifndef _WIN32 RUN_TEST(mcp_server_run_rapid_messages); + RUN_TEST(mcp_server_run_reports_output_failure); + RUN_TEST(mcp_server_run_reports_input_failure); #endif /* Snippet resolution (port of snippet_test.go) */ From a5a836cf7269a10b8f7278a10727893a73e42895 Mon Sep 17 00:00:00 2001 From: Chad <4307099+BobbieBarker@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:26:22 -0700 Subject: [PATCH 047/217] refactor(elixir): read a def head through its guard from one helper `def name(args) when guard` does not parse as a call with a guard beside it. tree-sitter-elixir builds one `when` binary_operator for the whole head, and the `name(args)` call that names the function is that operator's left operand. Four walks read that same node, the def's first argument, and each asks a different question of it: the defs walk names the function from it; the unified walk opens the function's call scope from it; the usages walk bounds the head's bindings with it; the calls walk tells a definition head apart from an invocation by it. A private copy in one of them makes the rest disagree about what a guarded clause is. The disagreement is silent, and it mints an edge: a head one walk treats as a declaration and another does not is recorded as a call to the very function being defined. Two questions, so two exported entry points: cbm_elixir_def_head_unwrap_guard(first_arg) returns the head UNDER every guard, or the node unchanged when there is none. The three walks that need a name or a span take this. cbm_elixir_def_head_is(signature, node) is true for that head OR for any `when` operator above it. The calls walk takes this, because a guarded head reaches it by two routes and it sees only one node per visit: `def f(x) when g` arrives as the inner `f(x)` call, and `def f when g` has no inner call at all, so the `when` operator itself arrives. Recognising one route and not the other leaves the phantom in place for the other. Both test the operator TOKEN rather than the node kind. `def a + b` is a binary_operator head too, and peeling it would name the function after its own left parameter. This commit changes no extraction behaviour: nothing calls either helper yet. The walks are separate commits, each with its own measurement. Measured while writing the test, which corrects a claim worth not repeating: tree-sitter-elixir nests multiple guards RIGHT-associatively. `def f(x) when is_atom(x) when is_binary(x)` parses as `when(f(x), when(is_atom(x), is_binary(x)))`, so the head is the outer operator's left operand and one peel reaches it. The peel still runs to a fixed point (one extra token compare), so the helper's contract is "the head under every guard" even where one peel would do. In tests/test_extraction.c, the test elixir_def_head_is_covers_the_head_and_every_guard_above_it drives the helpers on the AST directly rather than through cbm_extract_file. It has to: the two routes into the calls walk are the pair a single-walk test cannot separate, because each walk only ever sees the node its own traversal reaches. It pins the unguarded case, both guarded shapes, the double guard, and that `def a + b` is returned unchanged with its left operand answering false. Signed-off-by: Chad <4307099+BobbieBarker@users.noreply.github.com> --- internal/cbm/helpers.c | 87 +++++++++++++++++++++ internal/cbm/helpers.h | 20 +++++ tests/test_extraction.c | 164 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 271 insertions(+) diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 50c75bdf2..6ddd8ceaf 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -1140,6 +1140,93 @@ const char *cbm_nix_qn_name(CBMArena *a, TSNode func_node, const char *source, c return scope ? cbm_arena_sprintf(a, "%s.%s", scope, name) : name; } +/* ── Elixir def-head guards ─────────────────────────────────── */ + +/* The guard sits on the head, not beside it: `def name(args) when guard` parses + * as a single `when` binary_operator whose left operand is `name(args)`. The + * operator field is an exact anonymous token, so the check is the token type + * rather than the node kind — `def a + b` is a binary_operator head too, and + * unwrapping it would name the function after its left parameter. */ +bool cbm_elixir_is_when_guard(TSNode node) { + if (ts_node_is_null(node) || strcmp(ts_node_type(node), "binary_operator") != 0) { + return false; + } + TSNode op = ts_node_child_by_field_name(node, TS_FIELD("operator")); + return !ts_node_is_null(op) && strcmp(ts_node_type(op), "when") == 0; +} + +/* One `when` level off a def's first argument, or the node unchanged. + * + * There is deliberately no named-child fallback for a missing `left`. An infix + * operator cannot reduce without a left operand, so tree-sitter-elixir never + * builds a `when` binary_operator that lacks one: measured over every guarded + * shape in the suite plus a corpus of truncated heads (`def when true`, + * `def f(x) when`, `def f(x) when when is_x(x)`, `def when() when when`), + * `left` was present on all 66 unwraps, ERROR-recovered parses included, and + * every one had the left operand as named child 0. Where the left operand was + * genuinely absent the grammar produced no `when` operator at all -- it read + * `when` as the head identifier -- so the unwrap is never entered. A + * named-child fallback would therefore be dead on a well-formed tree and, on + * the hypothetical malformed one, would hand back named child 0 without + * knowing it is the left operand rather than the guard expression: naming the + * function after its own guard. Returning the node unchanged degrades to the + * pre-guard behaviour instead, which is the safe direction. */ +static TSNode elixir_unwrap_one_guard(TSNode first_arg) { + if (!cbm_elixir_is_when_guard(first_arg)) { + return first_arg; + } + TSNode lhs = ts_node_child_by_field_name(first_arg, TS_FIELD("left")); + return ts_node_is_null(lhs) ? first_arg : lhs; +} + +/* The head under every guard a def's first argument carries, or first_arg + * unchanged when it carries none. Elixir admits more than one guard on a + * clause -- `def f(x) when is_atom(x) when is_binary(x)` -- and + * tree-sitter-elixir nests those RIGHT-associatively, measured on that exact + * source: `when(f(x), when(is_atom(x), is_binary(x)))`. The declared head is + * therefore the outer operator's left operand and one peel reaches it. The + * peel still runs to a fixed point: the extra iteration is one token compare, + * and it is what makes the helper's contract "the head under EVERY guard" + * rather than "the head under the first one", which is the property the three + * walks rely on. + * + * Left wrapped, the defs walk drops the clause, the unified walk opens no + * function scope (so every call in a guarded body sources to the FILE node), + * the calls walk stops recognising the head and emits it as an invocation of + * the very function being defined, and the usages walk swallows every + * identifier in the guard as part of the binding. Four walks read this same + * argument, which is why the unwrap lives here rather than in any one of + * them. */ +TSNode cbm_elixir_def_head_unwrap_guard(TSNode first_arg) { + for (;;) { + TSNode next = elixir_unwrap_one_guard(first_arg); + if (ts_node_eq(next, first_arg)) { + return first_arg; + } + first_arg = next; + } +} + +/* True when `node` is the head a def declares, or any of the `when` guard + * operators wrapping it. The calls walk needs the whole chain, not just its + * ends: each member reaches that walk by its own route -- `def f(x) when g` + * arrives as the inner `f(x)` call, `def f when g` has no inner call at all and + * the operator itself falls through to extract_callee_name's first-identifier + * last resort -- and a member it fails to recognise as a declaration is emitted + * as an invocation of the function being defined. */ +bool cbm_elixir_def_head_is(TSNode signature, TSNode node) { + for (;;) { + if (ts_node_eq(signature, node)) { + return true; + } + TSNode next = elixir_unwrap_one_guard(signature); + if (ts_node_eq(next, signature)) { + return false; + } + signature = next; + } +} + static const char *func_node_name(CBMArena *a, TSNode func_node, const char *source, CBMLanguage lang) { // Wolfram: set_delayed_top/set_top/set_delayed/set — LHS is apply(user_symbol("f"), ...) diff --git a/internal/cbm/helpers.h b/internal/cbm/helpers.h index 905de04a8..c18f27429 100644 --- a/internal/cbm/helpers.h +++ b/internal/cbm/helpers.h @@ -103,6 +103,26 @@ const char *cbm_nix_binding_scope_qn(CBMExtractCtx *ctx, TSNode node, const char // attrpath and an enclosing attrset compose into one qualified name. const char *cbm_nix_qn_name(CBMArena *a, TSNode func_node, const char *source, const char *name); +// ── Elixir def-head guards ── +// `def f(x) when g` parses its WHOLE head as a `when` binary_operator, so the +// `f(x)` call naming the function is the operator's left operand, and more than +// one guard (`when a when b`) nests those operators right-associatively, so the +// head stays the outermost operator's left operand in either shape. Four +// walks read that same first argument — the defs walk names the function from +// it, the unified walk opens the function's call scope from it, the calls walk +// tells a head apart from an invocation by it, and the usages walk excludes the +// head's identifiers from the usage set by it — so a private copy in one of +// them makes the rest disagree about what a guarded clause is. +// cbm_elixir_is_when_guard answers "is this node a guarded head"; the unwrap +// returns the head under every guard, or the node unchanged when it carries +// none; cbm_elixir_def_head_is answers whether a node is that head or one of +// the guard operators above it, which is the question the calls walk asks. +// All three test the operator token, so an operator definition (`def a + b`) is +// left alone rather than mis-read as a guard. +bool cbm_elixir_is_when_guard(TSNode node); +TSNode cbm_elixir_def_head_unwrap_guard(TSNode first_arg); +bool cbm_elixir_def_head_is(TSNode signature, TSNode node); + // Resolve a function/method definition node's NAME node across all ~130 grammars // (generic `name` field, arrow→declarator, C/C++ declarator chain, plus the many // per-language quirks: Fortran subroutine, SCSS mixin, SQL create_function, R, diff --git a/tests/test_extraction.c b/tests/test_extraction.c index afbdee26f..46f51feeb 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -16,6 +16,8 @@ #include "result_spill.h" #include "pipeline/pass_lsp_cross.h" #include "iris_export_xml.h" +#include "helpers.h" +#include "lang_specs.h" /* ── Helpers ───────────────────────────────────────────────────── */ @@ -1246,6 +1248,167 @@ TEST(elixir_function) { PASS(); } +/* ── Elixir def-head guard helpers (internal/cbm/helpers.c) ───────── + * + * `def name(args) when guard` parses its WHOLE head as one `when` + * binary_operator whose left operand is the `name(args)` call, and Elixir + * admits more than one guard on a clause, which tree-sitter-elixir nests + * right-associatively. Four walks read that same first argument and each asks a + * different question of it: the defs and call-scope walks want the head UNDER + * every guard, the usages walk wants it so that a parameter READ by the guard + * stays outside the binding, and the calls walk wants to know whether the node + * it is visiting IS that head or one of the guard operators above it. + * + * That last question is why cbm_elixir_def_head_is exists beside the unwrap, + * and it is exercised here on the AST rather than through cbm_extract_file + * because the two routes it has to answer for are indistinguishable from + * outside: `def f(x) when g` reaches the calls walk as the inner `f(x)` call, + * which the unwrap reaches, while `def f when g` has no inner call at all and + * reaches it as the `when` operator itself, which the unwrap does not. A test + * that only drives one walk sees whichever of the two that walk happens to hit. + * + * This file's helpers change no extraction behaviour on their own; the walks + * that consume them are separate changes. */ + +enum { ELIXIR_HEAD_PROBE_MAX = 16, ELIXIR_HEAD_STACK_MAX = 256 }; + +static bool head_probe_text_is(const char *source, TSNode node, const char *want) { + if (ts_node_is_null(node)) { + return false; + } + uint32_t start = ts_node_start_byte(node); + uint32_t end = ts_node_end_byte(node); + size_t len = strlen(want); + return end >= start && (size_t)(end - start) == len && memcmp(source + start, want, len) == 0; +} + +/* The first argument of every `def`/`defp` call under `root`, in source order. + * That argument is what all four walks read, so it is what the helpers take. */ +typedef struct { + TSNode signature[ELIXIR_HEAD_PROBE_MAX]; + int count; +} elixir_head_probe_t; + +static void collect_elixir_def_signatures(TSNode root, const char *source, + elixir_head_probe_t *out) { + TSNode stack[ELIXIR_HEAD_STACK_MAX]; + int top = 0; + stack[top++] = root; + while (top > 0) { + TSNode cur = stack[--top]; + uint32_t cc = ts_node_child_count(cur); + if (cc > 1 && strcmp(ts_node_type(cur), "call") == 0 && + (head_probe_text_is(source, ts_node_child(cur, 0), "def") || + head_probe_text_is(source, ts_node_child(cur, 0), "defp")) && + out->count < ELIXIR_HEAD_PROBE_MAX) { + TSNode args = ts_node_child(cur, 1); + if (!ts_node_is_null(args) && ts_node_child_count(args) > 0) { + out->signature[out->count++] = ts_node_child(args, 0); + } + } + for (int i = (int)cc - 1; i >= 0 && top < ELIXIR_HEAD_STACK_MAX; i--) { + stack[top++] = ts_node_child(cur, (uint32_t)i); + } + } +} + +TEST(elixir_def_head_is_covers_the_head_and_every_guard_above_it) { + const char *source = "defmodule Heads do\n" + " def plain(x), do: x\n" + " def guarded(x) when is_binary(x), do: x\n" + " def bare when true, do: :ok\n" + " def twice(x) when is_atom(x) when is_binary(x), do: x\n" + " def a + b, do: {a, b}\n" + "end\n"; + const TSLanguage *language = cbm_ts_language(CBM_LANG_ELIXIR); + ASSERT_NOT_NULL(language); + TSParser *parser = ts_parser_new(); + ASSERT_NOT_NULL(parser); + ASSERT_TRUE(ts_parser_set_language(parser, language)); + TSTree *tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)); + ASSERT_NOT_NULL(tree); + + elixir_head_probe_t probe; + probe.count = 0; + collect_elixir_def_signatures(ts_tree_root_node(tree), source, &probe); + ASSERT_EQ(5, probe.count); + + /* Unguarded: nothing to peel, and the head is the whole first argument. + * The name node inside it is NOT the head -- the calls walk compares the + * node it is visiting, and the `plain` identifier is one of those. */ + TSNode plain = probe.signature[0]; + ASSERT_FALSE(cbm_elixir_is_when_guard(plain)); + ASSERT_TRUE(ts_node_eq(cbm_elixir_def_head_unwrap_guard(plain), plain)); + ASSERT_TRUE(cbm_elixir_def_head_is(plain, plain)); + ASSERT_FALSE(cbm_elixir_def_head_is(plain, ts_node_child(plain, 0))); + + /* One guard: the first argument is the operator, the head is its left + * operand, and BOTH answer true -- the operator is the node the paren-less + * route hands the calls walk, the inner call is the node the peel route + * hands it. The guard expression answers false, or the walk would suppress + * a real call written inside a guard. */ + TSNode guarded = probe.signature[1]; + ASSERT_TRUE(cbm_elixir_is_when_guard(guarded)); + TSNode guarded_head = cbm_elixir_def_head_unwrap_guard(guarded); + ASSERT_FALSE(ts_node_eq(guarded_head, guarded)); + ASSERT_STR_EQ("call", ts_node_type(guarded_head)); + ASSERT_TRUE(head_probe_text_is(source, guarded_head, "guarded(x)")); + ASSERT_TRUE(cbm_elixir_def_head_is(guarded, guarded)); + ASSERT_TRUE(cbm_elixir_def_head_is(guarded, guarded_head)); + TSNode guard_expr = ts_node_child_by_field_name(guarded, TS_FIELD("right")); + ASSERT_FALSE(ts_node_is_null(guard_expr)); + ASSERT_TRUE(head_probe_text_is(source, guard_expr, "is_binary(x)")); + ASSERT_FALSE(cbm_elixir_def_head_is(guarded, guard_expr)); + + /* Paren-less: the head under the guard is a bare identifier, so there is no + * inner call for the peel route to produce. This is the shape that makes + * cbm_elixir_def_head_is necessary: the only node the calls walk ever sees + * for this clause is the operator. */ + TSNode bare = probe.signature[2]; + ASSERT_TRUE(cbm_elixir_is_when_guard(bare)); + TSNode bare_head = cbm_elixir_def_head_unwrap_guard(bare); + ASSERT_STR_EQ("identifier", ts_node_type(bare_head)); + ASSERT_TRUE(head_probe_text_is(source, bare_head, "bare")); + ASSERT_TRUE(cbm_elixir_def_head_is(bare, bare)); + ASSERT_TRUE(cbm_elixir_def_head_is(bare, bare_head)); + + /* Two guards on one clause. tree-sitter-elixir nests them + * RIGHT-associatively -- `when(twice(x), when(is_atom(x), is_binary(x)))`, + * which is what this asserts -- so the head is still the outer operator's + * left operand and the peel reaches it in one step. The repeated peel is + * therefore a fixed point rather than a chain walk on this grammar; the + * nodes that must answer true are the operator and the head. */ + TSNode twice = probe.signature[3]; + ASSERT_TRUE(cbm_elixir_is_when_guard(twice)); + TSNode second_guard = ts_node_child_by_field_name(twice, TS_FIELD("right")); + ASSERT_TRUE(cbm_elixir_is_when_guard(second_guard)); + TSNode twice_head = cbm_elixir_def_head_unwrap_guard(twice); + ASSERT_TRUE(head_probe_text_is(source, twice_head, "twice(x)")); + ASSERT_TRUE(cbm_elixir_def_head_is(twice, twice)); + ASSERT_TRUE(cbm_elixir_def_head_is(twice, twice_head)); + /* The second guard is a `when` operator too, and it is NOT on the head + * chain: a walk that treated any `when` above the head as a declaration + * would suppress the calls written inside it. */ + ASSERT_FALSE(cbm_elixir_def_head_is(twice, second_guard)); + + /* An operator definition's head is a binary_operator too, and unwrapping it + * would name the function after its own left parameter. Only the `when` + * token admits the peel, so `def a + b` is returned unchanged and its left + * operand is not a head. */ + TSNode op_def = probe.signature[4]; + ASSERT_STR_EQ("binary_operator", ts_node_type(op_def)); + ASSERT_FALSE(cbm_elixir_is_when_guard(op_def)); + ASSERT_TRUE(ts_node_eq(cbm_elixir_def_head_unwrap_guard(op_def), op_def)); + ASSERT_TRUE(cbm_elixir_def_head_is(op_def, op_def)); + TSNode op_lhs = ts_node_child_by_field_name(op_def, TS_FIELD("left")); + ASSERT_TRUE(head_probe_text_is(source, op_lhs, "a")); + ASSERT_FALSE(cbm_elixir_def_head_is(op_def, op_lhs)); + + ts_tree_delete(tree); + ts_parser_delete(parser); + PASS(); +} + /* tree-sitter-elixir gives a call's arguments node no field name, so the * generic `arguments` field lookup returns null and first_string_arg was never * populated for any Elixir call — Phoenix route paths, service URLs and config @@ -8487,6 +8650,7 @@ SUITE(extraction) { /* Functional */ RUN_TEST(elixir_function); + RUN_TEST(elixir_def_head_is_covers_the_head_and_every_guard_above_it); RUN_TEST(elixir_call_string_argument); RUN_TEST(haskell_function); RUN_TEST(ocaml_function); From 630f851526dfdd3ce5672e267598c0eeb5a2155a Mon Sep 17 00:00:00 2001 From: Chad <4307099+BobbieBarker@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:26:22 -0700 Subject: [PATCH 048/217] fix(elixir): extract a def head wrapped in a `when` guard extract_elixir_func_def reads the def's first argument directly and accepts only a `call` or an `identifier` there. A guard makes that argument a binary_operator, so the extractor found no name and returned. Every guarded clause was dropped, and a function whose clauses ALL carry guards never reached the graph at all. The loss is silent: a missing definition is not an error anywhere in the pipeline. Guards are ordinary Elixir. Measured by indexing one 970-file Elixir lib tree with the binary built from this commit's parent and with this one, and diffing the `nodes` and `edges` rows out of the two SQLite stores: Function nodes 20,882 -> 21,941 new qualified names 1,059 (functions with no node at all before) lost qualified names 0 CALLS edges 55,851 -> 57,129 USAGE edges 27,418 -> 28,223 Every one of those 1,059 is a function whose every clause carries a guard: a clause without one already minted the node they collided on. An agent asking "who calls this" about any of them got an empty answer that reads exactly like dead code. The unwrap goes in the shared helper. The calls, unified and usages walks read the same node, and a private copy in one of them makes the rest disagree about what a guarded clause is. Two regressions come with the new node. Neither is minted here: both are pre-existing phantoms that had nowhere to land while guarded clauses were dropped, and now have a node. 1. A phantom CALLS edge onto the function being defined. The calls walk decides a def head is a declaration by comparing the visited node against the def's first argument, and a guard makes that argument the `when` operator, so the inner `name(args)` call no longer compares equal. cbm.c flags self-recursion by line containment, so the phantom lands inside the function's own span: self_recursive Function nodes 129 -> 1,342 ...with no self-call form in span 24 -> 1,241 (18.6% -> 92.5%) (measured by reading each flagged node's own source span and looking for any non-declaration line that names it as a call, a capture or a pipe target.) 2. A phantom WRITES edge onto it, from a paren-less guarded head. `def f when g` has no inner call at all, so the `when` operator reaches the read/write extractor and its first identifier is recorded as an assignment target. Reproduced on `def bare_guarded when true, do: :ok`: one CBMReadWrite row, var_name "bare_guarded", is_write true, scoped to the file's Module node (the graph asserting that a module mutates a function). Corpus WRITES 11,220 -> 11,302 (+82). Both become observable with this commit, and the guarded call-scope commit later in this stack closes them, by different mechanisms. The CALLS phantom goes because that commit admits the `when` operator to the definition-container check, so a paren-less head is recognised as a declaration. The WRITES phantom goes because that commit opens a function scope for the clause: the read/write row is still recorded, but its enclosing_func_qn becomes the guarded clause itself rather than the enclosing module, so resolve_rw_edges finds src->id == tgt->id and drops the self-edge. They are stated here because a stack that lands partially must not hide them. tests/test_extraction.c: extract_elixir_guarded_def_head covers every guarded shape (one guard, two guards, a `defp`, a paren-less `def bare when true`) and both operator-definition forms that must NOT be unwrapped (`def a + b`, `def c - d when is_integer(c)`, neither of which is extracted, as before). It fails on the current tree for each of the guarded names. Depends on the shared def-head helper commit. Signed-off-by: Chad <4307099+BobbieBarker@users.noreply.github.com> --- internal/cbm/extract_defs.c | 10 ++++++++ tests/test_extraction.c | 49 +++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 0f376b960..cdd6b1c7f 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -5435,6 +5435,16 @@ static void extract_elixir_func_def(CBMExtractCtx *ctx, TSNode node, const char return; } + // `def name(args) when guard` parses the whole head as a `when` + // binary_operator, so the name lives on its left operand rather than + // directly under the call. Without unwrapping it, every guarded clause is + // dropped, and a function whose clauses ALL carry guards never appears in + // the graph at all -- silently, since a missing definition is not an error. + // The unwrap lives in helpers.c because it peels only `when`: an operator + // definition (`def a + b`) is a binary_operator head too, and unwrapping + // that one would name the function after its own left parameter. + first_arg = cbm_elixir_def_head_unwrap_guard(first_arg); + const char *fk = ts_node_type(first_arg); char *name = NULL; if (strcmp(fk, "call") == 0 && ts_node_child_count(first_arg) > 0) { diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 46f51feeb..f8b61d53c 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1409,6 +1409,54 @@ TEST(elixir_def_head_is_covers_the_head_and_every_guard_above_it) { PASS(); } +/* `def name(args) when guard` parses its whole head as a `when` + * binary_operator, so the name sits on the operator's left operand rather than + * directly under the call. The def extractor accepted only `call` and + * `identifier` there, so every guarded clause was dropped -- and a function + * whose clauses ALL carry guards never reached the graph at all, silently. + * Guards are ordinary Elixir: on one real 2.9k-function codebase this hid 216 + * of 270 all-guarded functions, and every call edge touching them. */ +TEST(extract_elixir_guarded_def_head) { + CBMFileResult *r = extract("defmodule Guarded do\n" + " def plain(x), do: x\n" + " def guarded(x) when is_binary(x), do: x\n" + " def guarded(x) when is_list(x), do: x\n" + " def multi(x, y) when is_binary(x) and is_integer(y), do: {x, y}\n" + " defp priv_guarded(x) when is_atom(x), do: x\n" + " def bare when true, do: :ok\n" + " def a + b, do: {a, b}\n" + " def c - d when is_integer(c), do: {c, d}\n" + " def multi_guard(x) when is_atom(x) when is_binary(x), do: x\n" + "end\n", + CBM_LANG_ELIXIR, "t", "guarded.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Function", "plain")); + ASSERT(has_def(r, "Function", "guarded")); + ASSERT(has_def(r, "Function", "multi")); + ASSERT(has_def(r, "Function", "priv_guarded")); + ASSERT(has_def(r, "Function", "bare")); + + /* Elixir admits more than one guard on a clause. tree-sitter-elixir nests + * those `when` operators right-associatively, so the declared head stays + * the outermost operator's left operand -- one node either way, and the + * peel must not stop short of it or return the guard. */ + ASSERT(has_def(r, "Function", "multi_guard")); + ASSERT_EQ(count_defs_named(r, "Function", "multi_guard"), 1); + + /* An operator definition is a binary_operator head too, but it is not a + * guard: unwrapping it names the function after its own LEFT PARAMETER. + * `def a + b` must not mint a function called `a`, nor `def c - d when ...` + * one called `c` -- the guard's left operand is itself the `c - d` + * operator, which names no function either. Neither operator form is + * extracted at all, which is what this grammar did before guards were + * handled; only the `when` token admits the unwrap. */ + ASSERT_EQ(count_defs_named(r, "Function", "a"), 0); + ASSERT_EQ(count_defs_named(r, "Function", "c"), 0); + cbm_free_result(r); + PASS(); +} + /* tree-sitter-elixir gives a call's arguments node no field name, so the * generic `arguments` field lookup returns null and first_string_arg was never * populated for any Elixir call — Phoenix route paths, service URLs and config @@ -8651,6 +8699,7 @@ SUITE(extraction) { /* Functional */ RUN_TEST(elixir_function); RUN_TEST(elixir_def_head_is_covers_the_head_and_every_guard_above_it); + RUN_TEST(extract_elixir_guarded_def_head); RUN_TEST(elixir_call_string_argument); RUN_TEST(haskell_function); RUN_TEST(ocaml_function); From 77d4beb7a8ad9a6b8486ee9196731dd46de69e9e Mon Sep 17 00:00:00 2001 From: Chad <4307099+BobbieBarker@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:26:22 -0700 Subject: [PATCH 049/217] fix(elixir): span a multi-clause function's node over all of its clauses Every clause of an Elixir function is its own `def` call, and an Elixir QN carries neither module nor arity, so all clauses compute one qualified name and collide on a single graph node. cbm_gbuf_upsert_node breaks a same-QN collision by keeping the largest start_line, so the survivor was whichever clause was written last, and get_code_snippet returned that clause as the whole function: def admin?(%__MODULE__{role: :admin}), do: true def admin?(%__MODULE__{}), do: false read back as `def admin?(%__MODULE__{}), do: false`, so an agent asking what the function does is told the predicate always returns false. One node per function is right, so the fix widens that node's span instead of splitting each clause onto its own node. Two bounds keep the widened span honest, and a clause folds only if it clears both: it must sit in the same module body as the previous one (an Elixir QN carries no module, so `Outer.run` and `Outer.Inner.run` compute the same name), and it must be adjacent to it (only the immediately preceding EXTRACTED definition is a candidate, so a def of another name ends the group). Measured by indexing one 970-file Elixir lib tree with the binary built from this commit's parent and with this one, and diffing the `nodes` and `edges` rows out of the two SQLite stores: 4,135 functions stop being reported as one line of source, and none starts, which is the get_code_snippet defect measured on this corpus. A widened span is only honest if nothing phantom lives inside it, which is why the head suppression in extract_calls.c lands with the fold rather than as a follow-up. A guarded clause head is a declaration of the name, never a call to it, and the calls walk did not agree: it suppressed a head only when the head node IS the def's first named argument, and `def f(x) when g` parses its whole head as a `when` binary_operator, so the inner `f(x)` was not that argument. It now asks cbm_elixir_def_head_is, the same helper the defs walk reads the head through. cbm.c flags self-recursion by line containment, so under the folded span that phantom would land inside the function's own node, and `recursive` is a queryable node property that seeds the cycle detection in pass_complexity. The widened span has a real cost. Reading each flagged node's own source span for any line that names it as a call, a capture or a pipe target, and counting a one-line clause's body after `, do:` as such a line: self_recursive Function nodes 129 -> 430 ...with no self-call form in span 6 -> 48 The lines responsible are the typespec heads a widened span now covers: `@spec f(t) :: u` puts the declared name in a `call` node and the unified walk reads it as code. That is a separate defect with its own fix later in this stack, and a fold cannot close it. Two things a reader of the graph will notice: Edge metadata moves on edges that SURVIVE. `edges` is UNIQUE on (source, target, type), so where the def-head phantom happened to be the dedup survivor for a pair, deleting it promotes another row and the surviving edge takes that row's `line`. Of the 119,696 edges that survive this commit, 1,054 do. The 2,436 that go are 2,104 CALLS and 332 USAGE, measured against this commit's parent, and none is added back. is_exported becomes the OR over the folded clauses. The arity-free QN puts a public entry point and its private accumulator on one node, and a name callable from outside must stay exported. Exactly 6 Function nodes change is_exported on this corpus, all false -> true and none the other way. Each is a `def` clause written above a `defp` clause of the same name, where the pre-patch last-clause-wins record reported a publicly callable name as unexported. Verified against the source in every one: the new value is correct. The typespec attributes put a declared name in a `call` node too and are deliberately NOT suppressed in the calls walk. A def head can be, because is_elixir_def_binding already treats a def's first argument as a binding, so the identifier handle_calls declines is declined by handle_usages too. Nothing treats a typespec subject as a binding, so declining one here would not remove its phantom, it would RELABEL it as a USAGE: a separate row under UNIQUE(source, target, type) that no longer collides with the real call it used to hide behind. extract_elixir_declaration_head_mints_no_reference_of_any_kind pins that as a relation rather than a count, because at extraction the relabel is invisible. Depends on the shared def-head helper and guarded def-heads commits. Signed-off-by: Chad <4307099+BobbieBarker@users.noreply.github.com> --- internal/cbm/extract_calls.c | 55 +++++- internal/cbm/extract_defs.c | 130 +++++++++++++- tests/test_extraction.c | 338 +++++++++++++++++++++++++++++++++++ 3 files changed, 515 insertions(+), 8 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index e1de896b8..e74801b47 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -708,6 +708,59 @@ static bool elixir_call_head_in(TSNode call, const char *source, const char *con call_node_text_in(ts_node_child(call, 0), source, heads); } +/* A definition's head declares a name; it is never a call to it. The head under + * a guard is not the def's first argument -- `def f(x) when g` parses its whole + * head as a `when` binary_operator -- so the plain node comparison below stopped + * recognising it, and the inner `f(x)` was recorded as a call to `f`. Under the + * widened span that phantom lands inside the function's own node, and cbm.c + * decides self-recursion by line containment, so an ordinary two-clause guarded + * function reported itself recursive. `recursive` is a queryable node property + * and seeds the cycle detection in pass_complexity, so that is a load-bearing + * signal, not a cosmetic one. The suppression therefore travels with the fold + * rather than following it. + * + * The head comes from cbm_elixir_def_head_is (helpers.c) rather than a private + * peel here, because three files read this same node and all three have to + * agree on which node is the head, or a node one of them is treating as a + * definition name is recorded by another as a reference to that very name: + * - extract_defs.c, extract_elixir_func_def, unwraps it to NAME the + * definition. It and this file disagreeing mints the phantom above. + * - extract_usages.c, is_elixir_def_binding, treats the whole first argument + * as a binding occurrence rather than a reference, which is what keeps a + * head suppressed here from re-emerging as a USAGE edge. It is deliberately + * WIDER than the unwrap -- it covers the guard expression too -- and that + * width is load-bearing: a suppression in one extractor of the unified walk + * only REMOVES a phantom if the others also decline the node. + * - extract_unified.c, compute_elixir_func_qn, resolves a def's QN for call + * scope tracking and does NOT unwrap, so a guarded clause pushes no + * function scope and its call edges are sourced from the File rather than + * from the enclosing Function. Pre-existing, independent of the two above + * (it decides edge SOURCE, not edge existence), and fixed separately. + * + * Suppressing the head does not only delete edges; it moves metadata on edges + * that SURVIVE, and that is worth stating because a caller reading an edge's + * `line` will see a different number than before. `edges` is UNIQUE on + * (source, target, type), so where the def-head phantom happened to be the + * dedup survivor for a pair, deleting it promotes some other row for that same + * pair and the surviving edge takes that row's `line`. + * + * Measured by indexing one 970-file Elixir lib tree with the binary built from + * this commit's parent and with this one, and diffing the `edges` rows out of + * the two SQLite stores keyed on (source QN, target QN, type): 2,436 edges go + * and NONE is added -- 2,104 CALLS and 332 USAGE. Of the 119,696 edges that + * survive, 1,054 record a different `line`. + * + * The typespec attributes -- `@spec f(t) :: u` and the @callback / @type family + * -- put the declared name in a `call` node too, and are deliberately NOT + * suppressed here. A def head can be, because extract_usages.c already treats a + * def's first argument as a binding. Nothing there treats a typespec subject as + * one, so declining it in this walk would not remove its phantom, it would + * RELABEL it: handle_usages reaches the bare identifier handle_calls just + * declined and mints a USAGE onto the same function, which under + * UNIQUE(source, target, type) is a separate row that no longer collides with + * the real call it used to hide behind, and which pass_importance then counts. + * A typespec phantom has to be removed before any extractor sees it, by + * skipping the whole subtree in the unified walk. */ static bool elixir_call_is_definition_role(TSNode node, const char *source) { static const char *const structural_heads[] = {"def", "defp", "defmacro", "defmodule", NULL}; static const char *const function_heads[] = {"def", "defp", "defmacro", NULL}; @@ -725,7 +778,7 @@ static bool elixir_call_is_definition_role(TSNode node, const char *source) { TSNode signature = ts_node_named_child_count(arguments) > 0 ? ts_node_named_child(arguments, 0) : arguments; - return ts_node_eq(signature, node); + return cbm_elixir_def_head_is(signature, node); } return false; } diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index cdd6b1c7f..839e482de 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -5422,8 +5422,110 @@ static TSNode elixir_call_args(TSNode node) { return args; } -// Handle Elixir def/defp/defmacro — extract function definition. -static void extract_elixir_func_def(CBMExtractCtx *ctx, TSNode node, const char *macro) { +// Fold one more clause of an Elixir function into the def already pushed for +// the previous clause. Every clause is its own `def` call, and an Elixir QN +// carries neither module nor arity, so all of them compute one qualified name +// and collide on a single graph node. cbm_gbuf_upsert_node breaks a same-QN +// collision by keeping the LARGEST start_line, so the survivor was the last +// clause in the file and get_code_snippet returned that clause as the whole +// function: `def admin?(%__MODULE__{role: :admin}), do: true` / +// `def admin?(%__MODULE__{}), do: false` read back as a predicate that always +// returns false. One node per function is right, so widen its span instead. +// +// Two bounds keep the widened span honest, and a clause folds only if it clears +// both: +// - Same module body. `scope` is the do_block the previous clause was written +// in, so a nested `defmodule` cannot merge with its parent: the QN carries +// no module, so `Outer.run` and `Outer.Inner.run` compute the same name and +// are array-adjacent when Inner is declared just above Outer's own clause. +// - Same group. Only the immediately preceding EXTRACTED definition is a +// candidate, so a def of another name between two same-named defs ends the +// group. That is a bound this code imposes, not a guarantee Elixir gives: +// the compiler only WARNS ("clauses with the same name and arity should be +// grouped together"), it warns per name AND arity while this QN is +// arity-free, and non-contiguous clauses do compile. Split clauses +// therefore keep separate nodes, which is the pre-existing behaviour. +// What the group bound does NOT exclude is a non-definition construct +// between two clauses: `@doc` / `@spec` are unary_operators, `use` / +// `alias` / `describe` are ordinary calls, and a `defimpl` block's inner +// defs are not extracted -- none of them push a def, so anything of that +// kind written between two clauses of one function ends up inside the +// widened span, and that is not rare. Measured by indexing one 970-file +// Elixir lib tree with the binary built from this commit's parent and with +// this one and diffing the `nodes` rows out of the two SQLite stores: 6,996 +// Function nodes widen their span, none is added, none is lost and none +// narrows. Reading each widened span for a module-body construct written at +// the module's own two-space indent, which no clause body reaches, 169 of +// the 6,996 cover such a line. Counting each node once per kind, 123 cover +// a typespec attribute (@spec/@type/@typep/@opaque/@callback/ +// @macrocallback), 100 an @doc or @typedoc, 46 a bare comment line, 14 an +// @impl, and none a `use`, `alias`, `import`, `require` or a module-level +// `quote`. Swallowed typespecs are the largest attribute kind, not an +// absent one. +// +// The widened span also admits a phantom call that a narrower span kept out, +// which is why the head suppression in extract_calls.c travels with this fold +// rather than after it. cbm.c flags self-recursion by finding the innermost +// Function whose [start_line, end_line] contains a recorded call whose short +// name matches the function's own, so any pre-existing phantom between the +// first and the last clause becomes a self-edge as soon as the span covers it. +// That cost is real and is measured rather than asserted away. Same corpus, +// reading each flagged node's own source span for any line that names it as a +// call, a capture or a pipe target, counting a one-line clause's body after +// `, do:` as such a line: self_recursive Function nodes go 129 -> 430, and the +// ones carrying no self-call form anywhere in their span go 6 -> 48. The lines +// responsible are the typespec heads the widened span now covers -- `@spec +// f(t) :: u` puts the declared name in a `call` node, and the unified walk +// reads it as code -- a separate defect with a separate fix, not something this +// fold can close. What the fold and the head suppression together remove is +// 2,436 edges, none of them added back: see elixir_call_is_definition_role in +// extract_calls.c. +// +// A clause whose macro differs (`def` foo/1 beside `defp` foo/2) still folds, +// because the arity-free QN already puts both on one node; is_exported is then +// the OR over the folded clauses, so a name any clause exports stays exported. +// That OR changes a recorded flag, which the node diff confirms and which is +// worth stating rather than leaving to be discovered: on the same corpus +// exactly 6 Function nodes change is_exported, all false -> true and none the +// other way. Each is a `def` clause written above a `defp` clause of the same +// name, where the pre-patch last-clause-wins record reported a publicly +// callable name as unexported. Verified against the source in every one: the +// new value is the correct one. +// Returns true when the clause folded. +static bool fold_elixir_clause(CBMExtractCtx *ctx, const char *qn, TSNode scope, TSNode node, + uint32_t end_line, bool is_exported) { + if (!qn || ctx->result->defs.count <= 0 || ts_node_is_null(scope)) { + return false; + } + TSNode here = ts_node_parent(node); + if (ts_node_is_null(here) || !ts_node_eq(here, scope)) { + return false; + } + CBMDefinition *prev = &ctx->result->defs.items[ctx->result->defs.count - 1]; + if (!prev->label || strcmp(prev->label, "Function") != 0 || !prev->qualified_name || + strcmp(prev->qualified_name, qn) != 0) { + return false; + } + /* Only end_line moves. prev->start_line is already the minimum, so this + * takes no start_line argument: a clause folds only into the def pushed + * for the immediately preceding extracted clause of the same do_block, and + * extract_elixir_call pushes a do_block's children onto its stack in + * reverse index order so they pop in source order. A `start_line < + * prev->start_line` guard here would be unreachable; if that traversal + * ever stops being source-ordered, this is the line that has to change + * with it. */ + if (end_line > prev->end_line) { + prev->end_line = end_line; + } + prev->is_exported = prev->is_exported || is_exported; + return true; +} + +// Handle Elixir def/defp/defmacro — extract function definition. `scope` is the +// module body the previous clause was extracted from, and is updated to this +// def's own module body; see fold_elixir_clause for what it bounds. +static void extract_elixir_func_def(CBMExtractCtx *ctx, TSNode node, const char *macro, + TSNode *scope) { CBMArena *a = ctx->arena; TSNode args = elixir_call_args(node); if (ts_node_is_null(args)) { @@ -5456,15 +5558,25 @@ static void extract_elixir_func_def(CBMExtractCtx *ctx, TSNode node, const char return; } + const char *qn = cbm_fqn_compute(a, ctx->project, ctx->rel_path, name); + uint32_t start_line = ts_node_start_point(node).row + TS_LINE_OFFSET; + uint32_t end_line = ts_node_end_point(node).row + TS_LINE_OFFSET; + bool is_exported = (strcmp(macro, "def") == 0 || strcmp(macro, "defmacro") == 0); + bool folded = fold_elixir_clause(ctx, qn, *scope, node, end_line, is_exported); + *scope = ts_node_parent(node); + if (folded) { + return; + } + CBMDefinition def; memset(&def, 0, sizeof(def)); def.name = name; - def.qualified_name = cbm_fqn_compute(a, ctx->project, ctx->rel_path, name); + def.qualified_name = qn; def.label = "Function"; def.file_path = ctx->rel_path; - def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET; - def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET; - def.is_exported = (strcmp(macro, "def") == 0 || strcmp(macro, "defmacro") == 0); + def.start_line = start_line; + def.end_line = end_line; + def.is_exported = is_exported; cbm_defs_push(&ctx->result->defs, a, def); } @@ -5501,6 +5613,10 @@ static TSNode emit_elixir_module_class(CBMExtractCtx *ctx, TSNode cur) { // without recursion between extract_elixir_call ↔ extract_elixir_module_def. static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) { (void)spec; + /* Module body the last extracted clause was written in; see + * fold_elixir_clause. Null until the first def, so nothing folds into a + * def left over from a previous top-level call node. */ + TSNode def_scope = {0}; TSNodeStack stack; ts_nstack_init(&stack, ctx, CBM_SZ_64); ts_nstack_push(&stack, node); @@ -5523,7 +5639,7 @@ static void extract_elixir_call(CBMExtractCtx *ctx, TSNode node, const CBMLangSp if (strcmp(macro, "def") == 0 || strcmp(macro, "defp") == 0 || strcmp(macro, "defmacro") == 0) { - extract_elixir_func_def(ctx, cur, macro); + extract_elixir_func_def(ctx, cur, macro, &def_scope); } else if (strcmp(macro, "defmodule") == 0) { TSNode do_block = emit_elixir_module_class(ctx, cur); if (!ts_node_is_null(do_block)) { diff --git a/tests/test_extraction.c b/tests/test_extraction.c index f8b61d53c..887457cde 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1457,6 +1457,338 @@ TEST(extract_elixir_guarded_def_head) { PASS(); } +/* Every clause of an Elixir function is its own `def` call, and an Elixir QN + * carries neither module nor arity, so all clauses compute one qualified name + * and collide on a single graph node. cbm_gbuf_upsert_node ranks a same-QN + * collision by LARGEST start_line, so the survivor was the last clause in the + * file and get_code_snippet returned that clause as the whole function: a + * two-clause `admin?` read back as `def admin?(%__MODULE__{}), do: false` -- + * the graph asserting a predicate that always returns false. The node has to + * span its own clauses, and ONLY its own: the fold is bounded to the + * immediately preceding definition and to the enclosing module body, so a def + * of another name ends the group and two same-named functions in two modules + * stay two nodes. */ +TEST(extract_elixir_clauses_fold_only_when_adjacent_in_one_module) { + CBMFileResult *r = extract("defmodule Roles do\n" /* 1 */ + " def admin?(%{role: :admin}), do: true\n" /* 2 */ + " def admin?(%{}), do: false\n" /* 3 */ + "\n" /* 4 */ + " def promote(user), do: user\n" /* 5 */ + "\n" /* 6 */ + " def admin?(_other), do: false\n" /* 7 */ + "end\n" /* 8 */ + "\n" /* 9 */ + "defmodule Guests do\n" /* 10 */ + " def admin?(:root), do: true\n" /* 11 */ + " def admin?(_any), do: false\n" /* 12 */ + "end\n", /* 13 */ + CBM_LANG_ELIXIR, "t", "roles.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + /* The two adjacent Roles clauses fold; the clause past `promote` is its own + * group; the Guests pair is a DIFFERENT function and stays its own node. */ + ASSERT_EQ(3, count_defs_named(r, "Function", "admin?")); + ASSERT_EQ(1, count_defs_named(r, "Function", "promote")); + + const CBMDefinition *admin[3] = {NULL, NULL, NULL}; + const CBMDefinition *promote = NULL; + int seen = 0; + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].label, "Function") != 0) { + continue; + } + if (strcmp(r->defs.items[i].name, "admin?") == 0 && seen < 3) { + admin[seen++] = &r->defs.items[i]; + } else if (strcmp(r->defs.items[i].name, "promote") == 0) { + promote = &r->defs.items[i]; + } + } + ASSERT_NOT_NULL(admin[0]); + ASSERT_NOT_NULL(admin[1]); + ASSERT_NOT_NULL(admin[2]); + ASSERT_NOT_NULL(promote); + + ASSERT_EQ(2, (int)admin[0]->start_line); + ASSERT_EQ(3, (int)admin[0]->end_line); + /* separated from lines 2-3 by `promote`, so it keeps its own span */ + ASSERT_EQ(7, (int)admin[1]->start_line); + ASSERT_EQ(7, (int)admin[1]->end_line); + /* Guests.admin? never reaches back over the module boundary */ + ASSERT_EQ(11, (int)admin[2]->start_line); + ASSERT_EQ(12, (int)admin[2]->end_line); + ASSERT_EQ(5, (int)promote->start_line); + ASSERT_EQ(5, (int)promote->end_line); + cbm_free_result(r); + PASS(); +} + +/* A nested module is the one place where the "def of another name ends the + * group" rule is not enough: an Elixir QN carries no module, so `Outer.run` and + * `Outer.Inner.run` compute the SAME qualified name, and when Inner sits + * directly above Outer's own clause the two are adjacent in the extracted defs. + * The fold is bounded to the module body as well, so they stay two nodes. */ +TEST(extract_elixir_nested_module_clauses_do_not_fold) { + CBMFileResult *r = extract("defmodule Outer do\n" /* 1 */ + " defmodule Inner do\n" /* 2 */ + " def run(a), do: a\n" /* 3 */ + " end\n" /* 4 */ + "\n" /* 5 */ + " def run(b), do: b + 1\n" /* 6 */ + "end\n", /* 7 */ + CBM_LANG_ELIXIR, "t", "nested.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_EQ(2, count_defs_named(r, "Function", "run")); + + const CBMDefinition *runs[2] = {NULL, NULL}; + int seen = 0; + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].label, "Function") == 0 && + strcmp(r->defs.items[i].name, "run") == 0 && seen < 2) { + runs[seen++] = &r->defs.items[i]; + } + } + ASSERT_NOT_NULL(runs[0]); + ASSERT_NOT_NULL(runs[1]); + ASSERT_EQ(3, (int)runs[0]->start_line); + ASSERT_EQ(3, (int)runs[0]->end_line); + ASSERT_EQ(6, (int)runs[1]->start_line); + ASSERT_EQ(6, (int)runs[1]->end_line); + cbm_free_result(r); + PASS(); +} + +/* Folding clauses has to decide is_exported for the merged node. A `def` clause + * and a `defp` clause of the same name AND arity are clauses of one function -- + * a private guard clause or a nil short-circuit written above the public head + * is ordinary Elixir -- so is_exported is the OR over the folded clauses: the + * name is callable from outside if any clause exports it, and a wholly private + * function stays unexported. */ +TEST(extract_elixir_folded_clause_is_exported_is_the_or) { + CBMFileResult *r = extract("defmodule Acc do\n" /* 1 */ + " def build(l), do: build(l)\n" /* 2 */ + " defp build([]), do: []\n" /* 3 */ + "\n" /* 4 */ + " defp trim(nil), do: nil\n" /* 5 */ + " defp trim(s), do: s\n" /* 6 */ + "end\n", /* 7 */ + CBM_LANG_ELIXIR, "t", "acc.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_EQ(1, count_defs_named(r, "Function", "build")); + ASSERT_EQ(1, count_defs_named(r, "Function", "trim")); + + const CBMDefinition *build = NULL; + const CBMDefinition *trim = NULL; + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].label, "Function") != 0) { + continue; + } + if (strcmp(r->defs.items[i].name, "build") == 0) { + build = &r->defs.items[i]; + } else if (strcmp(r->defs.items[i].name, "trim") == 0) { + trim = &r->defs.items[i]; + } + } + ASSERT_NOT_NULL(build); + ASSERT_NOT_NULL(trim); + ASSERT_EQ(2, (int)build->start_line); + ASSERT_EQ(3, (int)build->end_line); + ASSERT_TRUE(build->is_exported); + ASSERT_EQ(5, (int)trim->start_line); + ASSERT_EQ(6, (int)trim->end_line); + ASSERT_FALSE(trim->is_exported); + cbm_free_result(r); + + /* The other direction, and the only one the `prev->is_exported ||` disjunct + * decides. Above, the exporting clause is the FIRST one, so a plain + * `prev->is_exported = is_exported` assignment would also have to lose it + * on the later defp -- but a last-clause-wins assignment happens to be + * right whenever the def comes last. Here the defp comes first and the def + * second, so the surviving flag can only be true if the fold ORs rather + * than assigns. */ + CBMFileResult *o = extract("defmodule Ord do\n" /* 1 */ + " defp z(1), do: 1\n" /* 2 */ + " def z(2), do: 2\n" /* 3 */ + "end\n", /* 4 */ + CBM_LANG_ELIXIR, "t", "ord.ex"); + ASSERT_NOT_NULL(o); + ASSERT_FALSE(o->has_error); + ASSERT_EQ(1, count_defs_named(o, "Function", "z")); + const CBMDefinition *z = NULL; + for (int i = 0; i < o->defs.count; i++) { + if (strcmp(o->defs.items[i].label, "Function") == 0 && + strcmp(o->defs.items[i].name, "z") == 0) { + z = &o->defs.items[i]; + } + } + ASSERT_NOT_NULL(z); + ASSERT_EQ(2, (int)z->start_line); + ASSERT_EQ(3, (int)z->end_line); + ASSERT_TRUE(z->is_exported); + cbm_free_result(o); + PASS(); +} + +/* First definition with this name, for the Elixir clause-fold tests below. + * find_def() is declared much further down this file. */ +static const CBMDefinition *elixir_first_def(CBMFileResult *r, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].name && strcmp(r->defs.items[i].name, name) == 0) { + return &r->defs.items[i]; + } + } + return NULL; +} + +/* A guarded clause head is a DECLARATION of the name, never a call to it, and + * the call extractor has to agree with the def extractor about which node the + * head is. It did not: it suppressed a head only when the head node IS the + * def's first named argument, but `def f(x) when g` parses its whole head as a + * `when` binary_operator, so the inner `f(x)` was not that argument and was + * recorded as a call to `f`. Under the folded span that phantom lands inside + * the function's own node, and self-recursion is decided by line containment, + * so an ordinary two-clause guarded function reported itself recursive. + * Recursion is a load-bearing signal: `recursive` is a queryable node property + * and seeds the cycle detection in pass_complexity. */ +TEST(extract_elixir_guarded_clause_head_is_not_a_self_call) { + CBMFileResult *r = extract("defmodule Guarded do\n" /* 1 */ + " def f(x) when is_integer(x), do: x\n" /* 2 */ + " def f(_x), do: 0\n" /* 3 */ + "\n" /* 4 */ + " def solo(x) when is_binary(x), do: x\n" /* 5 */ + "end\n", /* 6 */ + CBM_LANG_ELIXIR, "t", "guard_call.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + /* No call to either name exists anywhere in this file. */ + ASSERT_EQ(0, count_calls_named(r, "f")); + ASSERT_EQ(0, count_calls_named(r, "solo")); + /* The guard predicate IS a real call and stays one. */ + ASSERT_EQ(1, count_calls_named(r, "is_integer")); + ASSERT_EQ(1, count_calls_named(r, "is_binary")); + + ASSERT_EQ(1, count_defs_named(r, "Function", "f")); + const CBMDefinition *f = elixir_first_def(r, "f"); + ASSERT_NOT_NULL(f); + ASSERT_EQ(2, (int)f->start_line); + ASSERT_EQ(3, (int)f->end_line); + ASSERT_FALSE(f->is_recursive); + + /* A single guarded clause was wrong even before the span widened: the + * phantom landed on its own one-line node. */ + const CBMDefinition *solo = elixir_first_def(r, "solo"); + ASSERT_NOT_NULL(solo); + ASSERT_FALSE(solo->is_recursive); + cbm_free_result(r); + PASS(); +} + +/* Every reference of any kind this file records for `name`, across all four + * extractors the unified walk runs over one subtree: a call site, a value + * usage, a read/write occurrence, a type reference. Suppressing a node in ONE + * of them does not remove a phantom reference, it moves it -- handle_calls + * declining a node leaves state->callee_expr unset, and handle_usages then + * reaches the bare identifier and mints a USAGE instead. USAGE and CALLS are + * separate rows in the edge table, so the relabelled phantom also escapes the + * UNIQUE(source,target,type) dedup that had been collapsing it onto a real + * call, and pass_importance counts an inbound reference that did not exist + * before. An assertion naming only r->calls cannot see any of that. */ +static int elixir_any_ref_count(CBMFileResult *r, const char *name) { + int count = count_calls_named(r, name); + for (int i = 0; i < r->usages.count; i++) { + if (r->usages.items[i].ref_name && strcmp(r->usages.items[i].ref_name, name) == 0) { + count++; + } + } + for (int i = 0; i < r->rw.count; i++) { + if (r->rw.items[i].var_name && strcmp(r->rw.items[i].var_name, name) == 0) { + count++; + } + } + for (int i = 0; i < r->type_refs.count; i++) { + if (r->type_refs.items[i].type_name && strcmp(r->type_refs.items[i].type_name, name) == 0) { + count++; + } + } + return count; +} + +/* The head suppression has to REMOVE the phantom, not relabel it. What makes + * that hold for a def head is not the calls walk alone: is_elixir_def_binding + * in extract_usages.c already treats everything inside a def's first argument + * as a binding occurrence rather than a reference, so the identifier + * handle_calls declines is declined by handle_usages too. That agreement is the + * invariant under test, and it is why the counts below are over every extractor + * rather than over r->calls. + * + * `spec_subject` pins the boundary of this patch, and pins it as a RELATION + * rather than a count, because the count cannot see the defect. A typespec + * subject is the same shape as a def head and is NOT suppressed here: nothing + * in extract_usages.c treats it as a binding, so declining it in handle_calls + * only relabels its phantom as a USAGE. At extraction that relabel is + * invisible -- one reference before, one after -- and it becomes a defect + * downstream, where CALLS and USAGE are separate rows under + * UNIQUE(source,target,type) and the relabelled phantom stops colliding with + * the real call it used to hide behind. So the assertion is that every + * reference recorded for a typespec subject is a CALL and none has become a + * usage, a read/write or a type reference. */ +TEST(extract_elixir_declaration_head_mints_no_reference_of_any_kind) { + CBMFileResult *r = extract("defmodule Heads do\n" /* 1 */ + " def guarded(x) when is_integer(x), do: x\n" /* 2 */ + " def guarded(_x), do: 0\n" /* 3 */ + "\n" /* 4 */ + " defp bare(x), do: x\n" /* 5 */ + "\n" /* 6 */ + " @spec spec_subject(integer) :: integer\n" /* 7 */ + " def spec_subject(n), do: n\n" /* 8 */ + "end\n", /* 9 */ + CBM_LANG_ELIXIR, "t", "heads.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + /* Nothing in this file references `guarded` or `bare`. Not as a call, not + * as a value, not as a read/write, not as a type. */ + ASSERT_EQ(0, elixir_any_ref_count(r, "guarded")); + ASSERT_EQ(0, elixir_any_ref_count(r, "bare")); + /* The guard predicate is a real call and survives as one. */ + ASSERT_EQ(1, count_calls_named(r, "is_integer")); + ASSERT_EQ(1, elixir_any_ref_count(r, "is_integer")); + /* Declared, not suppressed here; see the comment above. Every reference + * this file records for a typespec subject is a CALL -- none of them has + * been relabelled into a usage, a read/write or a type reference. */ + ASSERT_EQ(count_calls_named(r, "spec_subject"), + elixir_any_ref_count(r, "spec_subject")); + cbm_free_result(r); + PASS(); +} + +/* The other direction: suppressing declaration heads must not blind the + * detector to real recursion, and folding must not lose a self-call written in + * a clause that is no longer the surviving one. `walk` recurses from its + * guarded second clause; exactly one call to `walk` is recorded (the body one, + * not the two heads) and the folded node is recursive. */ +TEST(extract_elixir_real_recursion_survives_head_suppression) { + CBMFileResult *r = extract("defmodule Rec do\n" /* 1 */ + " def walk([]), do: []\n" /* 2 */ + " def walk([h | t]) when is_integer(h), do: [h | walk(t)]\n" /* 3 */ + "end\n", /* 4 */ + CBM_LANG_ELIXIR, "t", "rec.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT_EQ(1, count_calls_named(r, "walk")); + ASSERT_EQ(1, count_defs_named(r, "Function", "walk")); + const CBMDefinition *walk = elixir_first_def(r, "walk"); + ASSERT_NOT_NULL(walk); + ASSERT_EQ(2, (int)walk->start_line); + ASSERT_EQ(3, (int)walk->end_line); + ASSERT_TRUE(walk->is_recursive); + cbm_free_result(r); + PASS(); +} + /* tree-sitter-elixir gives a call's arguments node no field name, so the * generic `arguments` field lookup returns null and first_string_arg was never * populated for any Elixir call — Phoenix route paths, service URLs and config @@ -8700,6 +9032,12 @@ SUITE(extraction) { RUN_TEST(elixir_function); RUN_TEST(elixir_def_head_is_covers_the_head_and_every_guard_above_it); RUN_TEST(extract_elixir_guarded_def_head); + RUN_TEST(extract_elixir_clauses_fold_only_when_adjacent_in_one_module); + RUN_TEST(extract_elixir_nested_module_clauses_do_not_fold); + RUN_TEST(extract_elixir_folded_clause_is_exported_is_the_or); + RUN_TEST(extract_elixir_guarded_clause_head_is_not_a_self_call); + RUN_TEST(extract_elixir_declaration_head_mints_no_reference_of_any_kind); + RUN_TEST(extract_elixir_real_recursion_survives_head_suppression); RUN_TEST(elixir_call_string_argument); RUN_TEST(haskell_function); RUN_TEST(ocaml_function); From 198e211c3001e33538f67a1cadb623cbd550bf08 Mon Sep 17 00:00:00 2001 From: Chad <4307099+BobbieBarker@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:26:22 -0700 Subject: [PATCH 050/217] fix(elixir): open a function scope for a guarded clause body A guard wraps the whole head in a `when` binary_operator, and three walks that read a def's first argument were not expecting one. The three failures are independent, and none is an error at runtime: The unified walk resolves a def's QN to open the function's call scope, and compute_elixir_func_qn accepts only a `call` or an `identifier` there. Handed a binary_operator it returns NULL, no function scope is pushed, and every call in a guarded body is attributed to the File node. The function reports no outgoing edges at all, and each of its callees gains an in-edge from the file. So "what does this function call" answers empty for a guarded clause, and "who calls this" answers with a file. The calls walk tells a definition head apart from an invocation through cbm_elixir_def_head_is, which covers the head and the `when` operators above it. A paren-less guarded clause (`def bare_guarded when true, do: ...`) never reaches that check: call_node_is_definition_container only admitted a `call` node, and here the head is the operator itself. Elixir's call node types include binary_operator, so the operator reached extract_callee_name and, with no callee of its own, took that function's last resort, the first identifier child, minting a phantom CALLS edge onto the very function being defined. The usages walk asks whether an identifier sits inside the def's signature and treats everything that does as a binding site rather than a reference. A guard makes that signature the whole `when` operator, which spans the guard expression too, so a parameter read by the guard was classified as its own binding and emitted no USAGE row at all. A guard is an expression over parameters that `guarded(a, b)` has already bound, so every mention of one to its right is a read, exactly as the same mention in the body already is. Measured by indexing one 970-file Elixir lib tree with the binary built from the base, 5df8b044, and with this one, and diffing the `edges` rows out of the two SQLite stores: Counted as the cumulative effect of the first four commits against that base, because this commit's attribution change is only observable once guarded clauses have nodes to be attributed to: CALLS sourced from a Module node 11,061 -> 5,840 (-5,221) CALLS sourced from a Function node 42,659 -> 49,579 (+6,920) CALLS sourced from a File node 2,131 -> 1,269 (-862) Calls written inside a guarded body stop being attributed to the enclosing module and are attributed to the function that writes them. The File count falls rather than rises, because the guarded-def-heads commit earlier in this stack gives those functions a node to source from. Against this commit's own parent, 5,825 USAGE rows appear for parameters read inside a guard that had been classified as their own binding. Node counts are identical in the two stores: the change moves where an edge comes from and records reads that were dropped. This also closes the WRITES phantom disclosed in the guarded-def-heads commit, though the extraction-level row does not go away. `def bare_guarded when true` still records one CBMReadWrite with var_name "bare_guarded" and is_write true. Its scope changes: with a function scope now open, enclosing_func_qn is the guarded clause where it had been the enclosing module, so resolve_rw_edges finds src->id == tgt->id and drops the self-edge. Measured on that exact fixture: before, one row scoped to `t.store`; after, one row scoped to `t.store.bare_guarded`. Corpus WRITES at this commit is 12,098 against 11,220 on the base; the typespec commit at the end of this stack brings it to 11,207. The row is still wrong at extraction and would reappear as an edge the moment anything sourced it elsewhere; removing it means teaching the read/write extractor about `when`, which is a fourth walk and belongs in its own change. tests/test_extraction.c: two tests. extract_elixir_guarded_def_head_scope asserts call attribution by exact scope QN rather than by callee name, because a call the scope walk failed to place carries the FILE QN, and a bare count of the callee passes just as happily with every call homed on the file. The same test pins the file-node count at 0. extract_elixir_guarded_def_head_guard_usages asserts each name both within its scope and in total, so an unwrap that took the guard instead of the head (admitting the binding sites themselves) fails on the totals. Only a `when` operator is admitted to the definition-container check. The generic first-identifier last resort still overrides a deliberate NULL for every other Elixir binary_operator that extract_scripting_callee declines (`=`, `<-`, `->`, `\\`, `::`), so `def g(c \\ 2)` still emits a phantom `c` and `e = target(d)` a phantom `e`. That is pre-existing behaviour for those operators and is untouched here. Depends on the shared def-head helper and multi-clause span commits. Signed-off-by: Chad <4307099+BobbieBarker@users.noreply.github.com> --- internal/cbm/extract_calls.c | 47 +++++++++--- internal/cbm/extract_unified.c | 5 ++ internal/cbm/extract_usages.c | 5 ++ tests/test_extraction.c | 132 +++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 12 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index e74801b47..c4891c8e0 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -725,17 +725,24 @@ static bool elixir_call_head_in(TSNode call, const char *source, const char *con * definition name is recorded by another as a reference to that very name: * - extract_defs.c, extract_elixir_func_def, unwraps it to NAME the * definition. It and this file disagreeing mints the phantom above. - * - extract_usages.c, is_elixir_def_binding, treats the whole first argument - * as a binding occurrence rather than a reference, which is what keeps a - * head suppressed here from re-emerging as a USAGE edge. It is deliberately - * WIDER than the unwrap -- it covers the guard expression too -- and that - * width is load-bearing: a suppression in one extractor of the unified walk - * only REMOVES a phantom if the others also decline the node. - * - extract_unified.c, compute_elixir_func_qn, resolves a def's QN for call - * scope tracking and does NOT unwrap, so a guarded clause pushes no - * function scope and its call edges are sourced from the File rather than - * from the enclosing Function. Pre-existing, independent of the two above - * (it decides edge SOURCE, not edge existence), and fixed separately. + * - extract_usages.c, is_elixir_def_binding, treats an identifier inside that + * head as a binding occurrence rather than a reference, which is what keeps + * a head suppressed here from re-emerging as a USAGE edge: a suppression in + * one extractor of the unified walk only REMOVES a phantom if the others + * also decline the node. The two files ask different questions of the same + * helper -- this one whether the node IS the head or a `when` wrapper above + * it, that one whether the node sits INSIDE the head -- so there is one + * definition of the head and they cannot drift apart. That containment + * stops at the head rather than covering the whole `when` operator, which + * is deliberate in the other direction: only the parameters being BOUND are + * excluded, and a parameter READ in the guard is a reference, exactly as + * the same read in the body is. + * - extract_unified.c, compute_elixir_func_qn, resolves a def's QN to open + * the function's call scope and reads the head through the same helper. + * Handed the `when` operator instead it returns NULL, no scope is pushed, + * and every call in a guarded body is sourced from the File rather than + * from the enclosing Function. That failure is independent of the two above + * because it decides edge SOURCE, not edge existence. * * Suppressing the head does not only delete edges; it moves metadata on edges * that SURVIVE, and that is worth stating because a caller reading an edge's @@ -800,7 +807,23 @@ static bool call_node_is_definition_container(CBMLanguage lang, TSNode node, con if (lang == CBM_LANG_AGDA && strcmp(kind, "expr") == 0) { return agda_expr_is_definition_role(node); } - return lang == CBM_LANG_ELIXIR && strcmp(kind, "call") == 0 && + /* A guarded head is a `when` binary_operator, not a `call`, and Elixir's + * call node types include binary_operator -- so the head reaches this walk + * and must be able to answer that it is a definition. A paren-less clause + * (`def f when g`) has no inner call at all, so the operator node itself + * reaches extract_callee_name and, with no callee of its own, takes that + * function's last resort -- the first identifier child -- minting a phantom + * CALLS edge onto the very function being defined. + * + * That last resort overrides a deliberate NULL for every Elixir + * binary_operator extract_scripting_callee declines (`=`, `<-`, `->`, `\\`, + * `::`, `when`), so `def g(c \\ 2)` still emits a phantom `c` and + * `e = target(d)` a phantom `e`. That is the pre-existing behaviour for + * those operators and is untouched here: only a `when` operator is + * admitted, so an operator definition's own head (`def a + b`) stays an + * ordinary node, as it was before guards were handled at all. */ + return lang == CBM_LANG_ELIXIR && + (strcmp(kind, "call") == 0 || cbm_elixir_is_when_guard(node)) && elixir_call_is_definition_role(node, source); } diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index 086f501a0..68e0fbdcc 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -602,6 +602,11 @@ static const char *compute_elixir_func_qn(CBMExtractCtx *ctx, TSNode node) { if (ts_node_is_null(first_arg)) { return NULL; } + /* A guard wraps the whole head in a `when` operator, so the head naming the + * function is its left operand. Left wrapped, this returns NULL and the def + * opens NO function scope: every call in a guarded body then sources to the + * FILE node and the function itself reports no outgoing edges. */ + first_arg = cbm_elixir_def_head_unwrap_guard(first_arg); const char *fk = ts_node_type(first_arg); char *name = NULL; if (strcmp(fk, "call") == 0 && ts_node_child_count(first_arg) > 0) { diff --git a/internal/cbm/extract_usages.c b/internal/cbm/extract_usages.c index f6b391315..57594c665 100644 --- a/internal/cbm/extract_usages.c +++ b/internal/cbm/extract_usages.c @@ -617,6 +617,11 @@ static bool is_elixir_def_binding(CBMExtractCtx *ctx, TSNode node) { TSNode signature = ts_node_named_child_count(arguments) > 0 ? ts_node_named_child(arguments, 0) : arguments; + /* A guard makes that signature the whole `when` operator, which spans the + * guard expression as well as the head. Unwrapped to the head, only the + * parameters being BOUND are excluded here; a parameter READ inside the + * guard stays a usage, as the same read in the body already is. */ + signature = cbm_elixir_def_head_unwrap_guard(signature); return node_contains(head, node) || node_contains(signature, node); } return false; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 887457cde..1adca8bce 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -91,6 +91,47 @@ static int count_defs_named(CBMFileResult *r, const char *label, const char *nam return count; } +/* Count calls to `callee` whose enclosing scope is exactly `scope_qn`. A call the + * scope walk failed to place carries the FILE or module QN here, so a bare count + * of the callee passes just as happily with every call homed on the file. */ +static int count_calls_from(CBMFileResult *r, const char *callee, const char *scope_qn) { + int count = 0; + for (int i = 0; i < r->calls.count; i++) { + if (r->calls.items[i].callee_name && r->calls.items[i].enclosing_func_qn && + strcmp(r->calls.items[i].callee_name, callee) == 0 && + strcmp(r->calls.items[i].enclosing_func_qn, scope_qn) == 0) { + count++; + } + } + return count; +} + +/* Count usages of `ref` attributed to exactly `scope_qn`, and to any scope. A + * def head is a BINDING, not a usage, so the pair separates two different + * mistakes: too few rows means a read was swallowed by the binding, too many + * means a binding site was counted as a read. */ +static int count_usages_from(CBMFileResult *r, const char *ref, const char *scope_qn) { + int count = 0; + for (int i = 0; i < r->usages.count; i++) { + if (r->usages.items[i].ref_name && r->usages.items[i].enclosing_func_qn && + strcmp(r->usages.items[i].ref_name, ref) == 0 && + strcmp(r->usages.items[i].enclosing_func_qn, scope_qn) == 0) { + count++; + } + } + return count; +} + +static int count_usages_named(CBMFileResult *r, const char *ref) { + int count = 0; + for (int i = 0; i < r->usages.count; i++) { + if (r->usages.items[i].ref_name && strcmp(r->usages.items[i].ref_name, ref) == 0) { + count++; + } + } + return count; +} + static int count_calls_named(CBMFileResult *r, const char *callee) { int count = 0; for (int i = 0; i < r->calls.count; i++) { @@ -1457,6 +1498,95 @@ TEST(extract_elixir_guarded_def_head) { PASS(); } +/* A guard hides the head from the scope walk and from the calls walk. + * compute_elixir_func_qn accepts only a `call` or an `identifier` as a def's + * first argument, so a guarded clause resolved no QN and opened NO function + * scope: every call in its body was attributed to the FILE node, the function + * reported no outgoing edges, and its callees gained an in-edge from the file + * instead of from their caller. A paren-less guarded clause (`def f when g`) is + * worse still: it has no inner call node at all, so the guard operator itself + * falls through to extract_callee_name's first-identifier last resort and mints + * a phantom CALLS edge naming the function being defined. */ +TEST(extract_elixir_guarded_def_head_scope) { + CBMFileResult *r = extract("defmodule Guards do\n" + " def target(x), do: x\n" + " def other(x), do: x\n" + " def unguarded_caller(a), do: target(a)\n" + " def guarded_caller(a) when is_binary(a) do\n" + " target(a)\n" + " other(a)\n" + " end\n" + " def bare_guarded when true, do: target(:ok)\n" + "end\n", + CBM_LANG_ELIXIR, "t", "guard_scope.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + /* A paren-less guarded head is a definition, never a call to itself. */ + ASSERT_EQ(count_calls_named(r, "bare_guarded"), 0); + + /* Both calls in the guarded body source to the guarded function, and the + * paren-less clause's body call to that clause. */ + ASSERT_EQ(count_calls_from(r, "target", "t.guard_scope.guarded_caller"), 1); + ASSERT_EQ(count_calls_from(r, "other", "t.guard_scope.guarded_caller"), 1); + ASSERT_EQ(count_calls_from(r, "target", "t.guard_scope.bare_guarded"), 1); + + /* Control on the path the fix does not touch: an unguarded clause still + * sources its body call to itself. This is NOT a leak detector -- the walk + * expires scopes by tree depth (pop_expired_scopes), so a sibling clause + * always unwinds the previous one before pushing its own and "pushed but + * never popped" is not a state this walk can reach. */ + ASSERT_EQ(count_calls_from(r, "target", "t.guard_scope.unguarded_caller"), 1); + + /* Nothing falls back to the file node. */ + ASSERT_EQ(count_calls_from(r, "target", "t.guard_scope"), 0); + ASSERT_EQ(count_calls_from(r, "other", "t.guard_scope"), 0); + + cbm_free_result(r); + PASS(); +} + +/* The half the calls and scope assertions above cannot see: the usages walk + * asks whether an identifier sits inside the def's SIGNATURE, and treats + * everything that does as part of the binding rather than as a reference. A + * guard makes that signature the whole `when` operator, which spans the guard + * expression too -- so `a` read by `is_binary(a)` was classified as its own + * binding and emitted no USAGE row at all. A guard is an expression, not a + * pattern: the parameters are bound by `guarded(a, b)` and every mention of one + * to its right is a read, exactly as the same mention in the body already is. + * Unwrapping the signature to the head restores those rows without admitting + * the binding sites themselves, which is why each name is asserted both within + * its scope and in total. */ +TEST(extract_elixir_guarded_def_head_guard_usages) { + CBMFileResult *r = extract("defmodule GuardUsage do\n" + " def plain(x), do: x\n" + " def guarded(a, b) when is_binary(a) and byte_size(a) > 3 do\n" + " b\n" + " end\n" + "end\n", + CBM_LANG_ELIXIR, "t", "guard_usage.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + /* Both guard reads of the parameter are usages of the guarded clause. */ + ASSERT_EQ(count_usages_from(r, "a", "t.guard_usage.guarded"), 2); + /* ...and nothing else claims `a`: the `a` in `guarded(a, b)` binds it. */ + ASSERT_EQ(count_usages_named(r, "a"), 2); + + /* The body read is unchanged, and the head's `b` is still a binding -- + * an unwrap that took the guard instead of the head would add both head + * parameters here. */ + ASSERT_EQ(count_usages_from(r, "b", "t.guard_usage.guarded"), 1); + ASSERT_EQ(count_usages_named(r, "b"), 1); + + /* Control: an unguarded clause was never affected either way. */ + ASSERT_EQ(count_usages_from(r, "x", "t.guard_usage.plain"), 1); + ASSERT_EQ(count_usages_named(r, "x"), 1); + + cbm_free_result(r); + PASS(); +} + /* Every clause of an Elixir function is its own `def` call, and an Elixir QN * carries neither module nor arity, so all clauses compute one qualified name * and collide on a single graph node. cbm_gbuf_upsert_node ranks a same-QN @@ -9039,6 +9169,8 @@ SUITE(extraction) { RUN_TEST(extract_elixir_declaration_head_mints_no_reference_of_any_kind); RUN_TEST(extract_elixir_real_recursion_survives_head_suppression); RUN_TEST(elixir_call_string_argument); + RUN_TEST(extract_elixir_guarded_def_head_scope); + RUN_TEST(extract_elixir_guarded_def_head_guard_usages); RUN_TEST(haskell_function); RUN_TEST(ocaml_function); RUN_TEST(erlang_function); From 4d21c3bbd34b420c8987e8588b88b5fc5446116f Mon Sep 17 00:00:00 2001 From: Chad <4307099+BobbieBarker@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:26:22 -0700 Subject: [PATCH 051/217] fix(elixir): stop walking @spec and @type subtrees as code An Elixir module attribute parses as `unary_operator(@, call(, args))`, so the typespec family reaches the unified walk in exactly the shape real code has. `@spec foo(t) :: t` puts `foo(t)` in the same `call` node an invocation would occupy, and `@type entry :: String.t()` does the same to a type name. The walk then reads a declaration as code and mints reference edges from it, sourced at the enclosing Module. Measured on a fixture of three typespec lines (`@type state :: map()`, `@spec fetch(opts) :: state`, `@spec load(User.t()) :: :ok`) beside `def state`, `def opts` and a remote `MyApp.User.t/0`: seven edges, every one phantom and every one landing on a Function. CALLS onto each specified function; CALLS + USAGE + WRITES onto `state`, the WRITES asserting a mutation that does not exist; and USAGE onto `opts` and onto `MyApp.User.t/0`, because a bare or remote TYPE name resolves to the same-named FUNCTION. On a codebase whose convention is @spec on every public function, every specced function carries an inbound edge from its own module, so fan_in never reaches 0 and "which exported functions does nothing call" is unanswerable. Measured by indexing one 970-file Elixir lib tree before and after: every specced function loses the inbound reference its own typespec minted, and no node is added or removed. This is one of two mechanisms. It removes the phantom a typespec line mints; the def-head phantom is removed by the calls walk recognising a `when`-wrapped head as a declaration, earlier in this stack. A function carrying both an @spec and a `when` clause needs both, which is why this commit is ordered after that one and its test can assert 0 references onto such a function rather than pinning a remaining 1. The skip covers the whole subtree, and two costs come with that. Both were measured by indexing a fixture repo (a declaring file per attribute head plus the file each one reaches) with the pristine binary and with this one: A typespec's type references are not indexed at all. That loses nothing correct: a type declaration mints no node, so such a reference resolves onto nothing or onto a same-named FUNCTION, which is itself the pollution. `@spec build(MyApp.User.t())` where that module's `t` is a `@type` produced no edge in either build; the ones that did resolve landed on a same-named Function, cross-file included. @callback and @macrocallback lose a real relationship. The declared name is a function name, so it resolved cross-file onto the functions implementing the behaviour, and nothing replaces that edge: `@behaviour MyBehaviour` mints none of its own. It goes anyway because it is only name resolution: on a fixture with one @callback and three same-named `handle_it/1` defs, the single edge landed on the one module that implements nothing, leaving both real implementors at fan_in 0. Recording that relationship correctly means minting it from `@behaviour`, which is a separate change. A typespec head a widened span covers was also being read as a self-call, which is the other half of the cost the multi-clause span commit disclosed. Same 970-file tree, same method it used (each flagged node's own source span, counting a one-line clause's body after `, do:` as a line that names the function): self_recursive Function nodes 430 -> 389 ...with no self-call form in span 48 -> 8 The base is 129 and 6. All 8 that remain are the phantom a local variable sharing the function's name mints (`defp slug(value) do slug = ...`), which nothing in this stack touches: 6 are on the base as well, and the other 2 are guarded clauses that only have a node of their own to be flagged on because of the earlier commits here. `unquote(...)` inside a typespec does execute at compile time, so `@type t :: unquote(build_type())` loses its genuine `build_type` edge too. Every other attribute VALUE is ordinary compile-time code (`@timeout Application.compile_env(:app, :timeout)` really does call compile_env), so those subtrees are deliberately walked exactly as before, and their attribute NAME still mints one phantom onto a same-named Function. That half of the defect class needs its own fix. tests/test_extraction.c: extract_elixir_typespec_attribute_is_not_code exercises all six heads in the skip list, so deleting any one entry breaks it. It pins the two accepted costs as decisions, and it carries a control name (`without_spec`) that reads 0 on the pristine build too, so an instrument that scored a definition as a reference could not make the other zeroes vacuous. extract_elixir_declaration_head_mints_no_reference_of_any_kind tightens from a relation to 0: with both mechanisms present, a function with an @spec and a guard carries no reference of any kind. Depends on the guarded call-scope commit, for the ordering above. It touches no file the earlier commits in this stack touch except extract_unified.c, and none of their lines. Signed-off-by: Chad <4307099+BobbieBarker@users.noreply.github.com> --- internal/cbm/extract_defs.c | 15 +++- internal/cbm/extract_unified.c | 110 ++++++++++++++++++++++++++- tests/test_extraction.c | 135 ++++++++++++++++++++++++++++----- 3 files changed, 235 insertions(+), 25 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 839e482de..2faf10856 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -5477,9 +5477,18 @@ static TSNode elixir_call_args(TSNode node) { // responsible are the typespec heads the widened span now covers -- `@spec // f(t) :: u` puts the declared name in a `call` node, and the unified walk // reads it as code -- a separate defect with a separate fix, not something this -// fold can close. What the fold and the head suppression together remove is -// 2,436 edges, none of them added back: see elixir_call_is_definition_role in -// extract_calls.c. +// fold can close. +// +// Skipping the typespec subtrees, later in this stack, closes that half: the +// same method then reports 389 self_recursive Function nodes and 8 with no +// self-call form, against 129 and 6 on the base. All 8 are the pre-existing +// phantom a local variable sharing the function's name mints (`defp slug(value) +// do slug = ...`), 6 of them already present on the base; the other 2 are +// guarded clauses that only have a node of their own to be flagged on because +// of this stack. +// +// What the fold and the head suppression together remove is 2,436 edges, none +// of them added back: see elixir_call_is_definition_role in extract_calls.c. // // A clause whose macro differs (`def` foo/1 beside `defp` foo/2) still folds, // because the arity-free QN already puts both on one node; is_exported is then diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index 68e0fbdcc..9467d82e6 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -2041,6 +2041,111 @@ static bool is_unified_trivia_node(TSNode node) { return ts_node_is_extra(node); } +static bool unified_node_text_equals(const CBMExtractCtx *ctx, TSNode node, const char *expected) { + if (ts_node_is_null(node) || !ctx->source || !expected) { + return false; + } + uint32_t start = ts_node_start_byte(node); + uint32_t end = ts_node_end_byte(node); + size_t len = strlen(expected); + return end >= start && (size_t)(end - start) == len && + memcmp(ctx->source + start, expected, len) == 0; +} + +/* An Elixir module attribute parses as `unary_operator(@, call(, args))`, + * so the typespec family reaches the unified walk in the shape real code has: + * `@spec foo(t) :: t` puts `foo(t)` in the same `call` node an invocation would + * occupy, and `@type entry :: String.t()` does the same to a type name. The + * walk then reads a declaration as code and mints reference edges from it. + * + * Measured on a fixture of three typespec lines (`@type state :: map()`, `@spec + * fetch(opts) :: state`, `@spec load(User.t()) :: :ok`) beside `def state`, + * `def opts` and a remote `MyApp.User.t/0`: seven edges, every one phantom and + * every one landing on a Function. CALLS onto each specified function; + * CALLS + USAGE + WRITES onto `state`, the WRITES asserting a mutation that + * does not exist; and USAGE onto `opts` and onto `MyApp.User.t/0`, because a + * bare or remote TYPE name resolves to the same-named FUNCTION. On a codebase + * whose convention is @spec on every public function, fan_in therefore never + * reaches 0 and "which exported functions nothing calls" is unanswerable. + * + * The whole subtree is skipped, not just the specified name. What that costs is + * not uniform across the six heads, and both halves were measured by indexing + * one fixture repo -- a declaring file per head plus the file each one reaches + * -- with the pristine binary and with this one: + * + * spec, type, typep, opaque. A type reference resolves onto a FUNCTION or + * onto nothing, never onto the type it names, because a type declaration + * mints no node. `@spec build(MyApp.User.t())` where that module's `t` is a + * `@type` produced no edge in either build, and `@type ext :: Ecto.Schema.t()` + * naming a module outside the repo produced none either. The references that + * do resolve land on a same-named Function, cross-file included -- `USAGE + * lib/decl_type.ex -> Function type_target @lib/remote_type.ex`, and the + * @typep and @opaque equivalents. A Function standing in for a Type is the + * pollution rather than a record of it, so nothing correct is lost here. + * + * callback, macrocallback. The declared name IS a function name, so it + * resolves onto an implementation elsewhere in the corpus: pristine yields + * `CALLS lib/decl_callback.ex -> Function callback_fun @lib/impl_callback.ex` + * and the @macrocallback equivalent, both cross-file, and this build yields + * neither. That edge does point at a real Behaviour-to-Impl relationship, and + * nothing replaces it -- `@behaviour MyBehaviour` mints no edge of its own, + * so after this change nothing in the graph links a behaviour to its + * implementations. It is deleted anyway because it is name resolution and not + * a behaviour model: on a fixture with one @callback and three same-named + * `handle_it/1` defs, the single edge landed on the one module that + * implements nothing, leaving both real implementors at fan_in 0. Recording + * that relationship correctly means minting it from `@behaviour`, which is a + * separate change and not a reason to keep an arbitrary edge. + * + * The other accepted cost: `unquote(...)` inside a typespec does execute at + * compile time, so `@type t :: unquote(build_type())` loses its genuine + * `build_type` call edge along with the phantoms. + * + * Only this family is skipped. Every other attribute VALUE is ordinary + * compile-time code -- `@timeout Application.compile_env(:app, :timeout)` + * really does call compile_env -- so those subtrees are walked exactly as + * before, and their attribute NAME is still emitted as a callee: `@behaviour + * GenServer`, `@timeout 5_000` and `@doc "x"` each still mint one phantom CALLS + * onto a same-named Function. That half of the defect class is untouched here + * and needs its own fix. + * + * This subtree skip is one of two mechanisms, and neither subsumes the other. + * It removes the phantom a typespec LINE mints; the def-head phantom is removed + * separately, by the calls walk recognising a `when`-wrapped head as a + * declaration (elixir_call_is_definition_role, extract_calls.c). A function + * carrying both an @spec and a `when` clause needs both: with only this one it + * keeps the def-head reference, with only the other it keeps the @spec one. + * `extract_elixir_typespec_attribute_is_not_code` pins that by asserting 0 + * references onto such a function, so removing either mechanism fails it. + * + * Measured over forge-symphony-graph/lib (970 .ex files) by classifying the + * source line every CALLS/USAGE/WRITES edge records: a build without this skip + * anchors 4,565 CALLS edges on a typespec-attribute line, and this build + * anchors 0 -- with no USAGE or WRITES appearing in their place, which is what + * a relabel rather than a removal would look like. Total USAGE edges fall from + * 25,393 to 23,044 over the same corpus; they do not rise. */ +static bool is_elixir_typespec_attribute(const CBMExtractCtx *ctx, TSNode node) { + static const char *const typespec_heads[] = { + "spec", "callback", "macrocallback", "type", "typep", "opaque", NULL}; + if (ctx->language != CBM_LANG_ELIXIR || strcmp(ts_node_type(node), "unary_operator") != 0 || + !unified_node_text_equals(ctx, ts_node_child_by_field_name(node, TS_FIELD("operator")), + "@")) { + return false; + } + TSNode operand = ts_node_child_by_field_name(node, TS_FIELD("operand")); + if (ts_node_is_null(operand) || strcmp(ts_node_type(operand), "call") != 0 || + ts_node_child_count(operand) == 0) { + return false; + } + TSNode head = ts_node_child(operand, 0); + for (const char *const *name = typespec_heads; *name; name++) { + if (unified_node_text_equals(ctx, head, *name)) { + return true; + } + } + return false; +} + // JS/TS `export_statement` appears in import_node_types so re-exports // (`export { X } from './m'`) are treated as an import boundary. But it also // wraps exported *declarations* (`export function f(cfg: Config) {}`), and @@ -2631,7 +2736,8 @@ void cbm_extract_unified(CBMExtractCtx *ctx) { break; } bool trivia = is_unified_trivia_node(node); - if (!trivia) { + bool typespec = is_elixir_typespec_attribute(ctx, node); + if (!trivia && !typespec) { /* Trivia consumes no semantic state. Scope expiry may be deferred * until the next code-bearing node; pop restores the displaced * tuple and push applies the new frame's effect, both O(1) -- @@ -2661,7 +2767,7 @@ void cbm_extract_unified(CBMExtractCtx *ctx) { * asking the cursor to construct a child iterator for every one of * hundreds of thousands of flat comment siblings. Structured extras * still descend normally. */ - if ((!trivia || ts_node_child_count(node) > 0) && + if (!typespec && (!trivia || ts_node_child_count(node) > 0) && ts_tree_cursor_goto_first_child(&cursor)) { depth++; continue; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 1adca8bce..3f6544853 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1847,24 +1847,22 @@ static int elixir_any_ref_count(CBMFileResult *r, const char *name) { } /* The head suppression has to REMOVE the phantom, not relabel it. What makes - * that hold for a def head is not the calls walk alone: is_elixir_def_binding - * in extract_usages.c already treats everything inside a def's first argument - * as a binding occurrence rather than a reference, so the identifier - * handle_calls declines is declined by handle_usages too. That agreement is the - * invariant under test, and it is why the counts below are over every extractor - * rather than over r->calls. + * that hold for a def head is not the calls walk alone: is_elixir_def_binding in + * extract_usages.c already treats everything inside a def's first argument as + * a binding occurrence rather than a reference, so the identifier handle_calls + * declines is declined by handle_usages too. That agreement is the invariant + * under test, and it is why the counts below are over every extractor rather + * than over r->calls. * - * `spec_subject` pins the boundary of this patch, and pins it as a RELATION - * rather than a count, because the count cannot see the defect. A typespec - * subject is the same shape as a def head and is NOT suppressed here: nothing - * in extract_usages.c treats it as a binding, so declining it in handle_calls - * only relabels its phantom as a USAGE. At extraction that relabel is - * invisible -- one reference before, one after -- and it becomes a defect - * downstream, where CALLS and USAGE are separate rows under - * UNIQUE(source,target,type) and the relabelled phantom stops colliding with - * the real call it used to hide behind. So the assertion is that every - * reference recorded for a typespec subject is a CALL and none has become a - * usage, a read/write or a type reference. */ + * `spec_subject` pins the boundary between the two mechanisms. A typespec + * subject is the same shape as a def head, and suppressing it in the calls walk + * would only relabel its phantom as a USAGE, because nothing treats it as a + * binding. It is removed instead by skipping the whole `@spec` subtree before + * any extractor sees it (is_elixir_typespec_attribute, extract_unified.c), so + * the assertion is both that no reference of any kind survives AND that what + * does survive is never a relabelling: the equality holds at 0 == 0 here, and + * would hold at 1 == 1 on a build with the subtree skip removed but the calls + * suppression left in place. */ TEST(extract_elixir_declaration_head_mints_no_reference_of_any_kind) { CBMFileResult *r = extract("defmodule Heads do\n" /* 1 */ " def guarded(x) when is_integer(x), do: x\n" /* 2 */ @@ -1886,11 +1884,19 @@ TEST(extract_elixir_declaration_head_mints_no_reference_of_any_kind) { /* The guard predicate is a real call and survives as one. */ ASSERT_EQ(1, count_calls_named(r, "is_integer")); ASSERT_EQ(1, elixir_any_ref_count(r, "is_integer")); - /* Declared, not suppressed here; see the comment above. Every reference - * this file records for a typespec subject is a CALL -- none of them has - * been relabelled into a usage, a read/write or a type reference. */ + /* Neither the @spec line nor the def head it precedes leaves a reference, + * and nothing has been relabelled into a usage, a read/write or a type + * reference. */ + ASSERT_EQ(0, count_calls_named(r, "spec_subject")); ASSERT_EQ(count_calls_named(r, "spec_subject"), elixir_any_ref_count(r, "spec_subject")); + + ASSERT_EQ(1, count_defs_named(r, "Function", "guarded")); + const CBMDefinition *guarded = elixir_first_def(r, "guarded"); + ASSERT_NOT_NULL(guarded); + ASSERT_EQ(2, (int)guarded->start_line); + ASSERT_EQ(3, (int)guarded->end_line); + ASSERT_FALSE(guarded->is_recursive); cbm_free_result(r); PASS(); } @@ -1919,6 +1925,94 @@ TEST(extract_elixir_real_recursion_survives_head_suppression) { PASS(); } +/* An Elixir module attribute parses as `unary_operator(@, call(, args))`, + * so a typespec line reaches the unified walk as ordinary code and every symbol + * it names collects a phantom inbound edge from the enclosing Module -- a CALLS + * onto the specified function, and for `@type` a USAGE plus a WRITES asserting + * a mutation that does not exist. On a codebase whose convention is @spec on + * every public function, fan_in then never reaches 0 and "which exported + * functions nothing calls" is unanswerable. + * + * All six heads in the skip list are exercised, so deleting any one entry from + * typespec_heads[] breaks this test. + * + * Two of the groups below pin a decision rather than a repair, and both are + * argued in is_elixir_typespec_attribute's comment: + * + * type_refs[] -- a typespec's type references are not indexed at all. A type + * declaration mints no node, so such a reference resolves onto nothing or + * onto a same-named FUNCTION, which is the pollution rather than a record of + * it. Restoring any of them must break this test. + * + * behaviour_contract_names[] -- accepted cost. A @callback / @macrocallback + * name is a function name, so before this change it resolved cross-file onto + * the functions implementing the behaviour. Nothing replaces that edge. + * + * The attribute NAME phantom outside the typespec family is deliberately still + * there: `@timeout` keeps its one reference, and its VALUE is ordinary + * compile-time code that still runs. `guarded` is pinned at 0 rather than 1 + * because the guarded def head no longer mints a phantom either -- the calls + * walk now recognises a `when`-wrapped head as a declaration. */ +TEST(extract_elixir_typespec_attribute_is_not_code) { + /* Names a typespec declares, plus the six attribute heads themselves. + * Every one carries a reference on the pristine build. */ + static const char *const declared[] = {"with_spec", "entry", "secret", "handle", + "spec", "type", "typep", "opaque", + "callback", "macrocallback", NULL}; + /* Accepted cost: these used to resolve onto the behaviour's implementors. */ + static const char *const behaviour_contract_names[] = {"handle_it", "expand_it", NULL}; + /* Pinned decision: type references are not indexed -- remote, bare-local + * and builtin alike. */ + static const char *const type_refs[] = {"MyApp.User.t", "String.t", "MyApp.Vault.key", + "Macro.t", "t", "key", + "term", "integer", "reference", + "atom", NULL}; + CBMFileResult *r = extract("defmodule Specced do\n" + " @timeout Application.compile_env(:app, :timeout)\n" + " @type entry :: String.t()\n" + " @typep secret :: MyApp.Vault.key()\n" + " @opaque handle :: reference()\n" + " @callback handle_it(term) :: :ok\n" + " @macrocallback expand_it(term) :: Macro.t()\n" + " @spec with_spec(MyApp.User.t()) :: integer\n" + " def with_spec(n), do: n\n" + " def without_spec(n), do: n\n" + " @spec guarded(atom) :: :ok\n" + " def guarded(x) when is_atom(x), do: :ok\n" + "end\n", + CBM_LANG_ELIXIR, "t", "specced.ex"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + /* The definitions themselves are a separate pass and are unaffected. */ + ASSERT(has_def(r, "Function", "with_spec")); + ASSERT(has_def(r, "Function", "without_spec")); + ASSERT(has_def(r, "Function", "guarded")); + /* Control on the instrument, not a pin: `without_spec` carries no typespec + * and is never called, so it reads 0 on the pristine build too. It is here + * to show elixir_any_ref_count does not score a definition as a reference, + * which is what would make every 0 below vacuous. It cannot fail first and + * cannot catch a regression. */ + ASSERT_EQ(0, elixir_any_ref_count(r, "without_spec")); + for (const char *const *name = declared; *name; name++) { + ASSERT_EQ(0, elixir_any_ref_count(r, *name)); + } + for (const char *const *name = behaviour_contract_names; *name; name++) { + ASSERT_EQ(0, elixir_any_ref_count(r, *name)); + } + for (const char *const *name = type_refs; *name; name++) { + ASSERT_EQ(0, elixir_any_ref_count(r, *name)); + } + /* A non-typespec attribute is untouched: its value really is compile-time + * code, and its name still mints the one phantom this fix does not close. */ + ASSERT_EQ(1, count_calls_named(r, "Application.compile_env")); + ASSERT_EQ(1, elixir_any_ref_count(r, "timeout")); + /* Both phantoms onto `guarded` are gone: the @spec one to the subtree skip, + * the def-head one to the calls walk recognising a guarded head. */ + ASSERT_EQ(0, elixir_any_ref_count(r, "guarded")); + cbm_free_result(r); + PASS(); +} + /* tree-sitter-elixir gives a call's arguments node no field name, so the * generic `arguments` field lookup returns null and first_string_arg was never * populated for any Elixir call — Phoenix route paths, service URLs and config @@ -9168,6 +9262,7 @@ SUITE(extraction) { RUN_TEST(extract_elixir_guarded_clause_head_is_not_a_self_call); RUN_TEST(extract_elixir_declaration_head_mints_no_reference_of_any_kind); RUN_TEST(extract_elixir_real_recursion_survives_head_suppression); + RUN_TEST(extract_elixir_typespec_attribute_is_not_code); RUN_TEST(elixir_call_string_argument); RUN_TEST(extract_elixir_guarded_def_head_scope); RUN_TEST(extract_elixir_guarded_def_head_guard_usages); From 3db66f22ad7d5c435559869a34b6e14b1e54793b Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Mon, 28 Sep 2026 08:59:32 -0500 Subject: [PATCH 052/217] fix(pipeline): free trees and compact results in sequential definitions pass The sequential route (50 files or fewer, or CBM_WORKERS=1) cached each file's raw extraction result with its tree-sitter tree and uncompacted working arena for the rest of the run. Free the tree, compact the result and release the working arena exactly as the parallel worker already does; the cross-file LSP pass re-parses these files, as it always has on the parallel route. A 40-file repo of large Python files peaks at 246 MB instead of 962 MB; the graph is byte-identical. Signed-off-by: Nelson Spence --- src/pipeline/pass_definitions.c | 7 ++++ tests/test_parallel.c | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 7c7ed6884..b4d37fccf 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -770,6 +770,7 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t } free(local_cache); } + cbm_work_arena_release(); return CBM_NOT_FOUND; } @@ -868,6 +869,11 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t total_calls += result->calls.count; if (local_cache) { + /* The cache lives through every later pass: hold the result as the + * parallel extract does (pass_parallel.c), without its tree (the + * cross-file LSP re-parses) and compacted. */ + cbm_free_tree(result); + cbm_result_compact(result); local_cache[i] = result; } else { /* Cache unavailable: imports for this file can still only @@ -880,6 +886,7 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t cbm_free_result(result); } } + cbm_work_arena_release(); /* the working arena compaction kept between files */ /* Phase 2: now that all extraction results are cached and Module * nodes for every file are in the graph, walk the cache again to diff --git a/tests/test_parallel.c b/tests/test_parallel.c index 85df7d55d..0a285d21f 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -616,6 +616,65 @@ TEST(parallel_spill_mode_builds_the_same_graph) { PASS(); } +/* ── Sequential cache: held like the parallel one ─────────────────── */ + +/* The parallel extract caches each result without its parse tree and + * compacted (pass_parallel.c). The sequential definitions pass cached the raw + * result, tree and working arena included, and that cache lives through every + * later pass: about 5x the memory per file, for every file of the repo. */ +TEST(sequential_definitions_cache_drops_trees_and_compacts) { + if (ensure_parity_setup() != 0) + FAIL("setup failed"); + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + cbm_file_info_t *files = NULL; + int file_count = 0; + ASSERT_EQ(cbm_discover(g_par_tmpdir, &opts, &files, &file_count), 0); + ASSERT_GT(file_count, 0); + + cbm_gbuf_t *gbuf = cbm_gbuf_new("par-test", g_par_tmpdir); + cbm_registry_t *reg = cbm_registry_new(); + CBMFileResult **cache = (CBMFileResult **)calloc((size_t)file_count, sizeof(CBMFileResult *)); + ASSERT_NOT_NULL(cache); + atomic_int cancelled; + atomic_init(&cancelled, 0); + cbm_pipeline_ctx_t ctx = { + .project_name = "par-test", + .repo_path = g_par_tmpdir, + .gbuf = gbuf, + .registry = reg, + .cancelled = &cancelled, + .result_cache = cache, + }; + + cbm_init(); + cbm_work_arena_release(); /* start from a thread that keeps nothing */ + ASSERT_EQ(cbm_pipeline_pass_definitions(&ctx, files, file_count), 0); + ASSERT_FALSE(cbm_work_arena_keeping()); /* ...and end the pass that way */ + + int cached = 0; + for (int i = 0; i < file_count; i++) { + const CBMFileResult *r = cache[i]; + if (!r) { + continue; + } + cached++; + ASSERT_NULL(r->cached_tree); + /* One exact block: capacity == bytes used, no dead headroom. */ + ASSERT_EQ(r->arena.nblocks, 1); + ASSERT_EQ(cbm_arena_capacity(&r->arena), cbm_arena_total(&r->arena)); + } + ASSERT_EQ(cached, file_count); + + for (int i = 0; i < file_count; i++) { + cbm_free_result(cache[i]); + } + free(cache); + cbm_registry_free(reg); + cbm_gbuf_free(gbuf); + cbm_discover_free(files, file_count); + PASS(); +} + /* ── Empty file list ──────────────────────────────────────────────── */ TEST(parallel_empty_files) { @@ -4437,6 +4496,7 @@ SUITE(parallel) { RUN_TEST(parallel_semantic_fixture_expected_counts); RUN_TEST(parallel_total_edges); RUN_TEST(parallel_spill_mode_builds_the_same_graph); + RUN_TEST(sequential_definitions_cache_drops_trees_and_compacts); RUN_TEST(parallel_empty_files); RUN_TEST(parallel_args_json_no_overflow); From e36930742a63efd2bbca491996ba8e37bfa5822c Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Mon, 28 Sep 2026 08:59:32 -0500 Subject: [PATCH 053/217] perf(py-lsp): O(1) type membership in the shared Python cross-registry build The shared tier-2 registry registered Python defs one at a time without a mid-build finalize, so every Method's receiver probe scanned all registered types linearly: O(methods x types), single-threaded in lsp_cross_prepare. Keep the per-def registration order and back only the existence probe with a cbm_ht set of type QNs, as the Go and Rust builders do; the registry is identical by construction. On the Python 3.14 stdlib + site-packages the builder drops from 67-75 s to 0.20-0.25 s, with a byte-identical graph. Signed-off-by: Nelson Spence --- internal/cbm/lsp/py_lsp.c | 52 ++++++++++++++++++++--- tests/test_py_lsp.c | 86 +++++++++++++++++++++++++++++++++++++++ tests/test_py_lsp_scale.c | 66 ++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 5 deletions(-) diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 8a0396115..961e35f15 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -13,6 +13,7 @@ #include "py_lsp.h" #include "../cbm.h" #include "../helpers.h" +#include "../../../src/foundation/hash_table.h" #include "tree_sitter/api.h" #include #include @@ -4973,10 +4974,36 @@ static const char **py_split_pipe(CBMArena *arena, const char *text) { return out; } +/* Qualified names of reg->types[0..noted), for a caller that registers into a + * registry it does not finalize until the end (cbm_py_build_cross_registry): + * that registry's own lookup scans every type. qns == NULL probes the + * registry instead. */ +typedef struct { + CBMHashTable *qns; + int noted; +} PyKnownTypes; + +/* Note the types added since the last call. Run after every add_type, so an + * add that failed silently cannot desync set and registry. If the set cannot + * grow it is dropped and probes fall back to the registry. */ +static void py_note_known_types(PyKnownTypes *known, const CBMTypeRegistry *reg) { + for (; known && known->qns && known->noted < reg->type_count; known->noted++) { + const char *qn = reg->types[known->noted].qualified_name; + if (!qn || cbm_ht_has(known->qns, qn)) + continue; + cbm_ht_set(known->qns, qn, (void *)(uintptr_t)(known->noted + 1)); + if (!cbm_ht_has(known->qns, qn)) { + cbm_ht_free(known->qns); + known->qns = NULL; + } + } +} + /* Build a registry from CBMLSPDef[] supplied by the caller — covers both - * the source file's own defs and cross-file referenced defs. */ + * the source file's own defs and cross-file referenced defs. known is NULL + * except for the shared Tier-2 build. */ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg, - CBMLSPDef *defs, int def_count) { + CBMLSPDef *defs, int def_count, PyKnownTypes *known) { /* Pass 1: types only — the method pass probes the registry per Method def * (receiver auto-registration), which is a LINEAR scan pre-finalize: * O(methods x types) per file (same quadratic as php_register_lsp_defs; @@ -4999,6 +5026,7 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe rt.method_names = py_split_pipe(arena, d->method_names_str); } cbm_registry_add_type(reg, rt); + py_note_known_types(known, reg); } } /* idx_arena == NULL skips the mid-build finalize: the tier-2 builder @@ -5046,13 +5074,17 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe if (strcmp(d->label, "Method") == 0 && d->receiver_type && d->receiver_type[0]) { rf.receiver_type = d->receiver_type; /* borrowed */ - if (!cbm_registry_lookup_type(reg, rf.receiver_type)) { + bool recv_known = known && known->qns + ? cbm_ht_has(known->qns, rf.receiver_type) + : cbm_registry_lookup_type(reg, rf.receiver_type) != NULL; + if (!recv_known) { CBMRegisteredType auto_t; memset(&auto_t, 0, sizeof(auto_t)); auto_t.qualified_name = rf.receiver_type; const char *dot = strrchr(d->receiver_type, '.'); auto_t.short_name = dot ? dot + 1 : rf.receiver_type; /* borrowed substring */ cbm_registry_add_type(reg, auto_t); + py_note_known_types(known, reg); } } cbm_registry_add_func(reg, rf); @@ -5092,7 +5124,7 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len, /* Index allocations go to a per-call scratch arena (see php_lsp_cross). */ CBMArena idx_arena; cbm_arena_init(&idx_arena); - py_register_lsp_defs(arena, &idx_arena, ®, defs, def_count); + py_register_lsp_defs(arena, &idx_arena, ®, defs, def_count, NULL); py_mark_ambiguous_callable_bindings(®); /* Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize" @@ -5126,14 +5158,24 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i cbm_registry_init(reg, arena); cbm_python_stdlib_register(reg, arena); + /* The registry is not finalized until the end, so probing it for each + * Method's receiver scanned every type: O(methods x types), single-threaded + * (~54 s of lsp_cross_prepare on a large Python tree). Probe a set of + * the registered type QNs instead; registration order is unchanged. Sized + * for the stdlib seed, not def_count (the mixed all-language count). NULL + * on allocation failure: the registry probe runs as before. */ + PyKnownTypes known = {cbm_ht_create((uint32_t)reg->type_count), 0}; + py_note_known_types(&known, reg); + /* Filter to Python defs only — defs[] is mixed-language all_defs. */ for (int i = 0; i < def_count; i++) { CBMLSPDef *d = &defs[i]; if (d->lang != CBM_LANG_PYTHON) continue; /* Reuse the existing register fn on a single-def slice (n=1 inline). */ - py_register_lsp_defs(arena, NULL, reg, d, 1); + py_register_lsp_defs(arena, NULL, reg, d, 1, &known); } + cbm_ht_free(known.qns); py_mark_ambiguous_callable_bindings(reg); cbm_registry_finalize(reg); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index 3aaadee51..798c80a7b 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -637,6 +637,91 @@ TEST(pylsp_fused_self_attr_chain_via_overlay) { PASS(); } +/* Characterization of the shared Tier-2 build order. Defs register one at a + * time in defs[] order, so a Method seen before its Class auto-registers a + * placeholder receiver type that stays ahead of the Class entry, and a Method + * whose receiver is already known adds no type. Any change to how the build + * probes for known receivers must keep this exact sequence. */ +TEST(pylsp_cross_registry_interleaved_order) { + CBMArena arena; + cbm_arena_init(&arena); + CBMTypeRegistry *base = cbm_py_build_cross_registry(&arena, NULL, 0); + ASSERT_NOT_NULL(base); + int tb = base->type_count, fb = base->func_count; + + CBMLSPDef defs[] = { + {.qualified_name = "t.m.A.m", + .short_name = "m", + .label = "Method", + .receiver_type = "t.m.A", + .def_module_qn = "t.m", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "t.m.A", + .short_name = "A", + .label = "Class", + .def_module_qn = "t.m", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "t.m.B", + .short_name = "B", + .label = "Class", + .def_module_qn = "t.m", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "t.m.B.f", + .short_name = "f", + .label = "Method", + .receiver_type = "t.m.B", + .def_module_qn = "t.m", + .lang = CBM_LANG_PYTHON}, + {.qualified_name = "t.m.C.g", + .short_name = "g", + .label = "Method", + .receiver_type = "t.m.C", + .def_module_qn = "t.m", + .lang = CBM_LANG_PYTHON}, + /* Receiver auto-registered by C.g just above: no second placeholder. */ + {.qualified_name = "t.m.C.h", + .short_name = "h", + .label = "Method", + .receiver_type = "t.m.C", + .def_module_qn = "t.m", + .lang = CBM_LANG_PYTHON}, + /* Non-Python defs are skipped by the builder. */ + {.qualified_name = "t.m.D", + .short_name = "D", + .label = "Class", + .def_module_qn = "t.m", + .lang = CBM_LANG_GO}, + }; + CBMTypeRegistry *reg = + cbm_py_build_cross_registry(&arena, defs, (int)(sizeof(defs) / sizeof(defs[0]))); + ASSERT_NOT_NULL(reg); + + /* Types: placeholder A (from A.m), Class A, Class B, placeholder C. */ + ASSERT_EQ(reg->type_count, tb + 4); + ASSERT_TRUE(reg->types[tb].qualified_name == defs[0].receiver_type); + ASSERT_STR_EQ(reg->types[tb].short_name, "A"); + ASSERT_TRUE(reg->types[tb + 1].qualified_name == defs[1].qualified_name); + ASSERT_TRUE(reg->types[tb + 2].qualified_name == defs[2].qualified_name); + ASSERT_TRUE(reg->types[tb + 3].qualified_name == defs[4].receiver_type); + ASSERT_STR_EQ(reg->types[tb + 3].short_name, "C"); + + /* Funcs keep defs[] order. */ + ASSERT_EQ(reg->func_count, fb + 4); + ASSERT_STR_EQ(reg->funcs[fb].qualified_name, "t.m.A.m"); + ASSERT_STR_EQ(reg->funcs[fb + 1].qualified_name, "t.m.B.f"); + ASSERT_STR_EQ(reg->funcs[fb + 2].qualified_name, "t.m.C.g"); + ASSERT_STR_EQ(reg->funcs[fb + 2].receiver_type, "t.m.C"); + ASSERT_STR_EQ(reg->funcs[fb + 3].qualified_name, "t.m.C.h"); + + const CBMRegisteredType *a = cbm_registry_lookup_type(reg, "t.m.A"); + ASSERT_NOT_NULL(a); + ASSERT_STR_EQ(a->qualified_name, "t.m.A"); + ASSERT_NULL(cbm_registry_lookup_type(reg, "t.m.D")); + + cbm_arena_destroy(&arena); + PASS(); +} + /* Issue #228: a class/static method invoked directly on a CROSS-FILE imported * class name — ActionRecordX.build_from_text(...) — produced no CALLS edge, so * the method showed in/out degree 0 and was flagged as dead code. Distinct from @@ -2194,6 +2279,7 @@ SUITE(py_lsp) { /* Phase 9 — cross-file + batch */ RUN_TEST(pylsp_crossfile_method_dispatch); RUN_TEST(pylsp_fused_self_attr_chain_via_overlay); + RUN_TEST(pylsp_cross_registry_interleaved_order); RUN_TEST(pylsp_crossfile_classmethod_on_class_issue228); RUN_TEST(pylsp_crossfile_inheritance); RUN_TEST(pylsp_batch_two_files); diff --git a/tests/test_py_lsp_scale.c b/tests/test_py_lsp_scale.c index f20e76275..eee1575fd 100644 --- a/tests/test_py_lsp_scale.c +++ b/tests/test_py_lsp_scale.c @@ -111,6 +111,72 @@ TEST(pylsp_scale_linear_growth) { PASS(); } +/* Time cbm_py_build_cross_registry over n classes, each followed by two + * methods (extraction order): every method probes the registry for its + * receiver while the registry is still being built. */ +static double measure_cross_registry(int n_classes, int *out_types) { + CBMArena arena; + cbm_arena_init(&arena); + int n_defs = n_classes * 3; + CBMLSPDef *defs = (CBMLSPDef *)cbm_arena_calloc(&arena, (size_t)n_defs * sizeof(*defs)); + if (!defs) { + cbm_arena_destroy(&arena); + return -1.0; + } + for (int i = 0; i < n_classes; i++) { + CBMLSPDef *cls = &defs[i * 3]; + cls->qualified_name = cbm_arena_sprintf(&arena, "scale.mod.Cls%d", i); + cls->short_name = cls->qualified_name + strlen("scale.mod."); + cls->label = "Class"; + cls->def_module_qn = "scale.mod"; + cls->lang = CBM_LANG_PYTHON; + for (int j = 1; j <= 2; j++) { + CBMLSPDef *m = &defs[i * 3 + j]; + m->qualified_name = cbm_arena_sprintf(&arena, "%s.m%d", cls->qualified_name, j); + m->short_name = j == 1 ? "m1" : "m2"; + m->label = "Method"; + m->receiver_type = cls->qualified_name; + m->def_module_qn = "scale.mod"; + m->lang = CBM_LANG_PYTHON; + } + } + struct timespec t0, t1; + clock_gettime(CLOCK_MONOTONIC, &t0); + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, n_defs); + clock_gettime(CLOCK_MONOTONIC, &t1); + if (out_types) + *out_types = reg ? reg->type_count : -1; + cbm_arena_destroy(&arena); + return elapsed_ms(t0, t1); +} + +/* The shared build registers defs one at a time into an unfinalized registry; + * a receiver probe that scans every type registered so far makes the build + * O(methods x types). 16x input: linear ~16x time, quadratic well over 100x. */ +TEST(pylsp_scale_cross_registry_build_linear) { + int ty_small = 0, ty_large = 0; + /* Best of three for the small size: it is short enough for host noise. */ + double t_small = measure_cross_registry(1000, &ty_small); + for (int k = 0; k < 2; k++) { + double t = measure_cross_registry(1000, NULL); + if (t >= 0.0 && t < t_small) + t_small = t; + } + double t_large = measure_cross_registry(16000, &ty_large); + printf(" cross registry: 1000 classes=%.1fms (types=%d) 16000 classes=%.1fms " + "(types=%d)\n", + t_small, ty_small, t_large, ty_large); + ASSERT(t_small >= 0.0 && t_large >= 0.0); + ASSERT_EQ(ty_large - ty_small, 15000); + if (t_small > 0.5) { + double ratio = t_large / t_small; + printf(" cross registry ratio 16000/1000: %.1fx (linear ~16x)\n", ratio); + ASSERT(ratio < 48.0); + } + PASS(); +} + SUITE(py_lsp_scale) { RUN_TEST(pylsp_scale_linear_growth); + RUN_TEST(pylsp_scale_cross_registry_build_linear); } From ffa0b320047f4db99dc53f09e228c94703436ae1 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Mon, 28 Sep 2026 08:59:32 -0500 Subject: [PATCH 054/217] fix(py-lsp): walk each lambda body at most once per call chain py_emit_call_for re-walked a called lambda's body without tracking lambdas already being walked, so a lambda that reaches itself was re-walked once per nesting level up to the walk-depth cap, doubling with two calls per body. mpmath's calculus/extrapolation.py grows memory at about 1.9 GB/s until it is killed. Mark the registry entry as expanding during its re-walk and skip re-entry. The mpmath file now indexes in 1.1 s at about 80 MB peak RSS; the graph of the Python 3.14 stdlib + site-packages is byte-identical. Signed-off-by: Nelson Spence --- internal/cbm/lsp/py_lsp.c | 27 ++++++++++++++------ internal/cbm/lsp/py_lsp.h | 1 + tests/test_py_lsp.c | 52 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 8a0396115..039dec586 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -165,7 +165,7 @@ static void py_register_instance_field(PyLSPContext *ctx, const char *class_qn, static void py_bind_for_target(PyLSPContext *ctx, TSNode left, const CBMType *elem_type); static void py_register_lambda(PyLSPContext *ctx, const char *name, TSNode lambda_node); static void py_register_dict_literal(PyLSPContext *ctx, const char *name, TSNode dict_node); -static TSNode py_lookup_lambda(PyLSPContext *ctx, const char *name); +static int py_lookup_lambda(PyLSPContext *ctx, const char *name); static const char *py_lookup_dict_dispatch(PyLSPContext *ctx, const char *var, const char *key); /* ── Scope mutation wrappers ───────────────────────────────────────────── @@ -690,19 +690,21 @@ static void py_register_lambda(PyLSPContext *ctx, const char *name, TSNode lambd } ctx->lambdas[ctx->lambda_count].name = cbm_arena_strdup(ctx->arena, name); ctx->lambdas[ctx->lambda_count].lambda_node = lambda_node; + ctx->lambdas[ctx->lambda_count].expanding = false; ctx->lambda_count++; } -static TSNode py_lookup_lambda(PyLSPContext *ctx, const char *name) { - TSNode null_node = {0}; +/* Registry index of the lambda bound to `name`, or -1. An index, not a + * pointer: the registry array is replaced when it grows. */ +static int py_lookup_lambda(PyLSPContext *ctx, const char *name) { if (!ctx || !name) - return null_node; + return -1; for (int i = 0; i < ctx->lambda_count; i++) { if (ctx->lambdas[i].name && strcmp(ctx->lambdas[i].name, name) == 0) { - return ctx->lambdas[i].lambda_node; + return i; } } - return null_node; + return -1; } /* Helper for stripping quotes from a tree-sitter `string` node text. */ @@ -2519,11 +2521,18 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { // Walk the lambda body for any nested call sites with the params // bound to the call's arg types. Resolved calls get caller_qn // set to a synthetic child of the enclosing function. - TSNode lambda_node = py_lookup_lambda(ctx, fname); - if (!ts_node_is_null(lambda_node)) { + int lambda_idx = py_lookup_lambda(ctx, fname); + if (lambda_idx >= 0) { + TSNode lambda_node = ctx->lambdas[lambda_idx].lambda_node; TSNode params = ts_node_child_by_field_name(lambda_node, "parameters", 10); TSNode body = ts_node_child_by_field_name(lambda_node, "body", 4); TSNode args = ts_node_child_by_field_name(call_node, "arguments", 9); + // A lambda that reaches itself again (directly or through another + // lambda) would re-walk its body once per nesting level up to the + // walk-depth cap, and two such calls in one body double the work + // per level. Walk each body at most once per chain. + if (ctx->lambdas[lambda_idx].expanding) + return; CBMScope *saved = ctx->current_scope; ctx->current_scope = py_scope_push_checked(ctx); // Bind each lambda param to the call-site arg's type. @@ -2556,7 +2565,9 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) { ctx->enclosing_func_qn = cbm_arena_sprintf(ctx->arena, "%s.", prev_func ? prev_func : ctx->module_qn); if (!ts_node_is_null(body)) { + ctx->lambdas[lambda_idx].expanding = true; py_resolve_calls_in(ctx, body); + ctx->lambdas[lambda_idx].expanding = false; } ctx->enclosing_func_qn = prev_func; py_scope_restore(ctx, saved); diff --git a/internal/cbm/lsp/py_lsp.h b/internal/cbm/lsp/py_lsp.h index 939025100..6d1a3fb27 100644 --- a/internal/cbm/lsp/py_lsp.h +++ b/internal/cbm/lsp/py_lsp.h @@ -13,6 +13,7 @@ typedef struct { const char *name; // bound name, e.g. "fn" TSNode lambda_node; // the lambda AST node + bool expanding; // body is being re-walked for a call site right now } CBMLambdaEntry; // Function-as-dict-value entry. When `funcs = {"a": foo, "b": bar}` is diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index 3aaadee51..14ce8adb3 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -1931,6 +1931,56 @@ TEST(pylsp_lambda_rewalk_dedupes_dunder_occurrence) { PASS(); } +/* A lambda body is also re-walked for calls made inside another lambda's + * re-walk, so a lambda that reaches itself again (directly or through another + * lambda) was re-walked once per nesting level until the walk-depth cap. With + * two such calls in one body the work doubled per level, and a two-line + * `fib = lambda n: fib(n - 1) + fib(n - 2)` drove extraction to many GB. A + * lambda already being re-walked must not be re-entered, so each helper() + * site below resolves once. The recursion here is linear, so without the + * guard this stays small and fails the count instead of exhausting memory. */ +TEST(pylsp_recursive_lambda_rewalk_is_bounded) { + CBMFileResult *r = extract_py("def helper():\n" + " return 0\n" + "count = lambda n: helper() if n == 0 else count(n - 1)\n" + "even = lambda n: helper() if n == 0 else odd(n - 1)\n" + "odd = lambda n: even(n - 1)\n" + "count(3)\n" + "even(4)\n"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error || r->parse_incomplete); + + int helper_resolutions = 0; + for (int i = 0; i < r->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &r->resolved_calls.items[i]; + if (rc->callee_qn && strstr(rc->callee_qn, ".helper")) + helper_resolutions++; + } + ASSERT_EQ(helper_resolutions, 2); + + cbm_free_result(r); + PASS(); +} + +/* The re-walk guard is released after each walk, so a later call to the same + * lambda still binds its own argument type: both A.m and B.m resolve. */ +TEST(pylsp_lambda_rewalk_guard_released_between_calls) { + CBMFileResult *r = extract_py("class A:\n" + " def m(self):\n" + " return 1\n" + "class B:\n" + " def m(self):\n" + " return 2\n" + "call_m = lambda x: x.m()\n" + "call_m(A())\n" + "call_m(B())\n"); + ASSERT_NOT_NULL(r); + ASSERT_GTE(require_resolved(r, "", ".A.m"), 0); + ASSERT_GTE(require_resolved(r, "", ".B.m"), 0); + cbm_free_result(r); + PASS(); +} + /* Parser-backed Python calls have the same occurrence-identity requirement as * synthetic dunders. Distinct typed receivers can expose the same method leaf * in one caller; each CBMCall must join the semantic record for its own site. */ @@ -2243,6 +2293,8 @@ SUITE(py_lsp) { RUN_TEST(pylsp_round5_super_init); RUN_TEST(pylsp_dunder_same_leaf_occurrences_join_by_exact_site); RUN_TEST(pylsp_lambda_rewalk_dedupes_dunder_occurrence); + RUN_TEST(pylsp_recursive_lambda_rewalk_is_bounded); + RUN_TEST(pylsp_lambda_rewalk_guard_released_between_calls); RUN_TEST(pylsp_ordinary_same_leaf_calls_join_by_exact_site); /* Round 6 — generators, dataclasses, properties */ RUN_TEST(pylsp_round6_generator_yields_iterable); From fd1b88374c0c21892c991c29909d22ed499c20fa Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Mon, 28 Sep 2026 08:59:32 -0500 Subject: [PATCH 055/217] fix(discover): keep make/cargo dep-info .d files out of the D grammar .d mapped to D unconditionally, so make-style dependency files written by cargo, rustc, gcc/clang -MD and CMake were parsed with the D grammar and walked at superlinear cost. Add cbm_disambiguate_d, a 4 KB first-line sniff in the shape of the existing .frm/.cls/.m disambiguators, that returns CBM_LANG_COUNT only for a make rule whose targets look like paths and keeps D on any doubt. A 200-file cargo dep-info corpus goes from 108.9 s to 1.27 s to index; 0 of 12,062 real D files are misclassified. Signed-off-by: Nelson Spence --- src/discover/discover.c | 5 +++ src/discover/discover.h | 6 +++ src/discover/language.c | 81 +++++++++++++++++++++++++++++++++++++++++ tests/test_discover.c | 48 ++++++++++++++++++++++++ tests/test_language.c | 51 ++++++++++++++++++++++++++ 5 files changed, 191 insertions(+) diff --git a/src/discover/discover.c b/src/discover/discover.c index 0e5ae5e4f..f3518abb8 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -803,6 +803,11 @@ static CBMLanguage detect_file_language(const char *entry_name, const char *abs_ if (dot && strcmp(dot, ".res") == 0) { lang = cbm_disambiguate_res(abs_path); } + /* Special: .d is shared by D and make-style dep-info (cargo, gcc -MD). + * Probe only while .d maps to D, so a user override to another language wins. */ + if (dot && strcmp(dot, ".d") == 0 && lang == CBM_LANG_DLANG) { + lang = cbm_disambiguate_d(abs_path); + } /* Special: ObjectScript Studio Export XML () is * detected by content; otherwise .xml stays XML. */ if (lang == CBM_LANG_XML) { diff --git a/src/discover/discover.h b/src/discover/discover.h index 89ccca4f3..24a08d5f4 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -57,6 +57,12 @@ CBMLanguage cbm_disambiguate_frm(const char *path); * CBM_LANG_RESCRIPT. On read failure, defaults to CBM_LANG_RESCRIPT. */ CBMLanguage cbm_disambiguate_res(const char *path); +/* Disambiguate .d files by reading first 4KB of content. + * Returns CBM_LANG_COUNT (not source) for a make-style dependency file written + * by rustc/cargo, gcc/clang -MD or CMake ("target ...: prereq ..."), otherwise + * CBM_LANG_DLANG. On read failure or any doubt, defaults to CBM_LANG_DLANG. */ +CBMLanguage cbm_disambiguate_d(const char *path); + /* Disambiguate .inc files by reading first 4KB of content. * Returns CBM_LANG_OBJECTSCRIPT_ROUTINE if it looks like an ObjectScript * include (a "ROUTINE " header), otherwise CBM_LANG_BITBAKE. diff --git a/src/discover/language.c b/src/discover/language.c index dcc41dafd..66de4f4df 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -1330,6 +1330,87 @@ CBMLanguage cbm_disambiguate_res(const char *path) { return memchr(buf, '\0', n) ? CBM_LANG_COUNT : CBM_LANG_RESCRIPT; } +/* A make rule's targets end at a ':' followed by whitespace or the end of the + * file, which skips drive-letter colons ("C:/", "C:\"). */ +static bool is_rule_colon(const char *p, bool whole_file) { + char next = p[SKIP_ONE]; + return *p == ':' && (next == ' ' || next == '\t' || next == '\r' || next == '\n' || + (next == '\0' && whole_file)); +} + +/* True if the line at p is a make rule "target...: prereq..." whose targets all + * look like paths (contain '/', '\' or '.'); a backslash-escaped space stays + * inside its target. D source fails this: "public:", "@safe:", "extern(C):" + * and "import a.b : c" all have a non-path word or D punctuation before the + * colon. */ +static bool is_dep_rule_line(const char *p, bool whole_file) { + bool in_target = false; + bool path_like = false; + bool any_target = false; + for (; *p && *p != '\n'; p++) { + if (is_rule_colon(p, whole_file)) { + return in_target ? path_like : any_target; + } + if (*p == ' ' || *p == '\t' || *p == '\r') { + if (in_target && !path_like) { + return false; + } + in_target = false; + path_like = false; + continue; + } + if (strchr("(){};=\"',", *p)) { + return false; + } + in_target = true; + any_target = true; + path_like = path_like || *p == '/' || *p == '\\' || *p == '.'; + if (*p == '\\' && p[SKIP_ONE] == ' ') { + p++; /* escaped space: part of this target */ + } + } + return false; +} + +/* Disambiguate .d files: shared by D source and make-style dependency files + * written by rustc/cargo (target//deps/-.d), gcc/clang -MD + * and CMake (.o.d). Those are "target: prereq ..." rules, often tens of + * KB on one line, which the D grammar parses slowly into nothing but a module + * node, so they are reported as unsupported (CBM_LANG_COUNT). Defaults to D on + * any doubt (preserves existing behaviour). */ +CBMLanguage cbm_disambiguate_d(const char *path) { + if (!path) { + return CBM_LANG_DLANG; + } + + FILE *f = cbm_fopen(path, "r"); + if (!f) { + return CBM_LANG_DLANG; + } + + char buf[CBM_SZ_4K + SKIP_ONE]; + size_t n = fread(buf, SKIP_ONE, CBM_SZ_4K, f); + buf[n] = '\0'; + (void)fclose(f); + /* A colon at the end of buf only ends the rule if nothing was cut off. */ + bool whole_file = n < CBM_SZ_4K && strlen(buf) == n; + + const char *p = buf; + if (strncmp(p, "\xEF\xBB\xBF", SLEN("\xEF\xBB\xBF")) == 0) { + p += SLEN("\xEF\xBB\xBF"); + } + while (*p && isspace((unsigned char)*p)) { + p++; + } + /* A shebang or a leading D comment (including the DUB single-file + * "/+ dub.sdl:" recipe) is D; dep-info never starts that way. */ + if (*p == '#' || + (p[0] == '/' && (p[SKIP_ONE] == '/' || p[SKIP_ONE] == '*' || p[SKIP_ONE] == '+'))) { + return CBM_LANG_DLANG; + } + return is_dep_rule_line(p, whole_file) ? CBM_LANG_COUNT : CBM_LANG_DLANG; +} + /* Disambiguate .cls files: shared by InterSystems ObjectScript UDL, Salesforce * Apex and Visual Basic 6 class modules (#721). ObjectScript class files begin * with a line of the form "Class ..."; VB6 class modules diff --git a/tests/test_discover.c b/tests/test_discover.c index 5e9bdd8af..033eb10c9 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -6,6 +6,7 @@ #include "test_framework.h" #include "test_helpers.h" #include "discover/discover.h" +#include "discover/userconfig.h" #include "foundation/platform.h" typedef struct { @@ -1990,6 +1991,49 @@ TEST(shebang_oversized_first_line_unindexed) { PASS(); } +/* ── .d: D source vs make/cargo dep-info ───────────────────────── */ + +/* rustc/cargo dep-info (and gcc -MD, CMake .o.d) files share .d with D source + * but are Makefile rules the D grammar parses slowly into nothing useful, so + * they must not be indexed as D. */ +static const char CARGO_DEP_INFO[] = "/home/u/proj/t/debug/deps/foo-abc123.d: src/lib.rs src/a.rs\n" + "\n" + "src/lib.rs:\n" + "src/a.rs:\n"; + +TEST(discover_d_dep_info_not_indexed_as_dlang) { + CBMLanguage dep; + CBMLanguage src; + ASSERT(shebang_probe("cbm_disc_dotd", "t/debug/deps/foo-abc123.d", CARGO_DEP_INFO, &dep)); + ASSERT(shebang_probe("cbm_disc_dotd", "app.d", "module app;\nimport std.stdio : writeln;\n", + &src)); + ASSERT_EQ(dep, CBM_LANG_COUNT); + ASSERT_EQ(src, CBM_LANG_DLANG); + PASS(); +} + +/* The probe runs only while .d maps to D, so a user override to another + * language wins. */ +TEST(discover_d_user_override_skips_dep_info_probe) { + char *cfg_dir = th_mktempdir("cbm_disc_dotd_uc"); + ASSERT(cfg_dir != NULL); + ASSERT_EQ(th_write_file(TH_PATH(cfg_dir, ".codebase-memory.json"), + "{\"extra_extensions\":{\".d\":\"makefile\"}}"), + 0); + cbm_userconfig_t *cfg = cbm_userconfig_load(cfg_dir); + th_cleanup(cfg_dir); + ASSERT_NOT_NULL(cfg); + + cbm_set_user_lang_config(cfg); + CBMLanguage lang; + bool ok = shebang_probe("cbm_disc_dotd_uc", "foo-abc123.d", CARGO_DEP_INFO, &lang); + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(cfg); + ASSERT(ok); + ASSERT_EQ(lang, CBM_LANG_MAKEFILE); + PASS(); +} + /* ── Suite ─────────────────────────────────────────────────────── */ SUITE(discover) { @@ -2082,6 +2126,10 @@ SUITE(discover) { RUN_TEST(shebang_embedded_nul_unindexed); RUN_TEST(shebang_oversized_first_line_unindexed); + /* .d: D source vs make/cargo dep-info */ + RUN_TEST(discover_d_dep_info_not_indexed_as_dlang); + RUN_TEST(discover_d_user_override_skips_dep_info_probe); + /* Integration tests (cross-platform) */ RUN_TEST(discover_simple); RUN_TEST(discover_wide_sibling_fanout_exceeds_initial_walk_stack); diff --git a/tests/test_language.c b/tests/test_language.c index dc2bcf7d3..d9d38429f 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -835,6 +835,55 @@ TEST(lang_cfc_default_on_read_fail) { PASS(); } +/* ── .d: D source vs make/cargo dep-info ───────────────────────── */ + +/* Write content to a temp .d file and return cbm_disambiguate_d() for it. A + * setup failure returns -1 so neither a D nor a COUNT expectation can pass + * vacuously. */ +static int disambiguate_d_content(const char *content) { + char path[256]; + snprintf(path, sizeof(path), "%s/test_lang_dotd.d", cbm_tmpdir()); + if (!write_probe_file(path, content)) { + return -1; + } + CBMLanguage lang = cbm_disambiguate_d(path); + remove(path); + return (int)lang; +} + +TEST(lang_d_dep_info_unsupported) { + /* cargo (absolute target), gcc -MD continuation, CMake object rule, Windows + * drive letter (not the rule colon), escaped space (one target, not "app"), + * and the GNU make manual's "foo.o foo.d : ..." (two targets, spaced colon). */ + ASSERT_EQ(disambiguate_d_content("/home/u/t/debug/deps/foo-abc123.d: src/lib.rs src/a.rs\n" + "\nsrc/lib.rs:\n"), + CBM_LANG_COUNT); + ASSERT_EQ(disambiguate_d_content("foo.o: foo.c /usr/include/stdio.h \\\n" + " /usr/include/features.h\n"), + CBM_LANG_COUNT); + ASSERT_EQ(disambiguate_d_content("CMakeFiles/x.dir/a.cpp.o: \\\n /src/a.cpp\n"), + CBM_LANG_COUNT); + ASSERT_EQ(disambiguate_d_content("C:/b/foo.o: C:/s/foo.c \\\r\n C:/s/foo.h\r\n"), + CBM_LANG_COUNT); + ASSERT_EQ(disambiguate_d_content("out/my\\ app: main.c\n"), CBM_LANG_COUNT); + ASSERT_EQ(disambiguate_d_content("foo.o foo.d : foo.c defs.h\n"), CBM_LANG_COUNT); + PASS(); +} + +TEST(lang_d_source_stays_dlang) { + ASSERT_EQ(disambiguate_d_content("module a;\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("import std.stdio : writeln;\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("@safe:\nvoid f() {}\n"), CBM_LANG_DLANG); + /* A leading comment is D, even DUB's "/+ dub.sdl:" single-file recipe. */ + ASSERT_EQ(disambiguate_d_content("/+ dub.sdl:\n name \"hello\"\n+/\nvoid main() {}\n"), + CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("enum E : ubyte {\n a,\n}\n"), CBM_LANG_DLANG); + /* Default on doubt: empty or unreadable. */ + ASSERT_EQ(disambiguate_d_content(""), CBM_LANG_DLANG); + ASSERT_EQ(cbm_disambiguate_d("/tmp/nonexistent_file_12345.d"), CBM_LANG_DLANG); + PASS(); +} + /* --- New languages (auto-generated) --- */ TEST(lang_ext_solidity) { ASSERT_EQ(cbm_language_for_extension(".sol"), CBM_LANG_SOLIDITY); @@ -1488,6 +1537,8 @@ SUITE(language) { RUN_TEST(lang_frm_form_stays_form); RUN_TEST(lang_res_binary_resource_unsupported); RUN_TEST(lang_res_rescript_stays_rescript); + RUN_TEST(lang_d_dep_info_unsupported); + RUN_TEST(lang_d_source_stays_dlang); /* Go test ports */ /* New languages */ From 3f53a308f4034f4d280130fbcc275f545d7bf05e Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Mon, 28 Sep 2026 08:59:32 -0500 Subject: [PATCH 056/217] perf(python): climb the walk cursor for scope-directive and default checks For nearly every Python identifier, lexical_ancestor_kind and python_default_value_reference climbed toward the root with ts_node_parent, which re-descends from the root on every step, so a deep expression was cubic. In error-free trees, climb the walk's occurrence cursor instead and take a single parent step for global/nonlocal names, which the grammar makes direct children; trees with errors keep the original climb. sympy's polys/numberfields/resolvent_lookup.py drops from about 8.7 s to 54 ms in the definitions pass, ordinary Python extraction is about 25% faster, and the graph is byte-identical. Signed-off-by: Nelson Spence --- internal/cbm/extract_usages.c | 73 +++++++++++++++---- tests/repro/repro_lexical_binding_precision.c | 54 ++++++++++++++ tests/test_extraction.c | 44 +++++++++++ 3 files changed, 155 insertions(+), 16 deletions(-) diff --git a/internal/cbm/extract_usages.c b/internal/cbm/extract_usages.c index bdad1c8c7..50f717872 100644 --- a/internal/cbm/extract_usages.c +++ b/internal/cbm/extract_usages.c @@ -1779,9 +1779,25 @@ static uint32_t active_lexical_scope_id(const WalkState *state) { static uint32_t lexical_ancestor_of_kind(const WalkState *state, uint32_t start_id, bool want_function, bool want_block); -static bool python_default_value_reference(TSNode node) { - for (TSNode parent = ts_node_parent(node); !ts_node_is_null(parent); - parent = ts_node_parent(parent)) { +/* One step up: on the occurrence cursor when there is one, else with + * ts_node_parent, which descends from the ROOT, so a climb costs O(depth^2). */ +static TSNode usage_parent_step(TSTreeCursor *cursor, TSNode current) { + if (cursor) { + return ts_tree_cursor_goto_parent(cursor) ? ts_tree_cursor_current_node(cursor) + : (TSNode){0}; + } + usage_slow_parent_fallback_test_note(); + return ts_node_parent(current); +} + +/* Asked for every Python usage, and a deep expression puts O(depth) parents + * above each one. A tree without errors climbs them on the walk cursor, like + * python_scope_directive_kind; a tree with errors keeps ts_node_parent. */ +static bool python_default_value_reference(CBMExtractCtx *ctx, WalkState *state, TSNode node) { + TSTreeCursor *cursor = + ts_node_has_error(ctx->root) ? NULL : reset_occurrence_cursor(state, node); + for (TSNode parent = usage_parent_step(cursor, node); !ts_node_is_null(parent); + parent = usage_parent_step(cursor, parent)) { const char *kind = ts_node_type(parent); /* A nested executable scope inside the default owns its own lookups; * only a direct default expression is evaluated in the declaring @@ -1802,10 +1818,10 @@ static bool python_default_value_reference(TSNode node) { return false; } -static uint32_t usage_lexical_scope_id_for_node(CBMExtractCtx *ctx, const WalkState *state, - TSNode node) { +static uint32_t usage_lexical_scope_id_for_node(CBMExtractCtx *ctx, WalkState *state, TSNode node) { uint32_t scope_id = active_lexical_scope_id(state); - if (!ctx || ctx->language != CBM_LANG_PYTHON || !python_default_value_reference(node)) { + if (!ctx || ctx->language != CBM_LANG_PYTHON || + !python_default_value_reference(ctx, state, node)) { return scope_id; } uint32_t function_id = lexical_ancestor_of_kind(state, scope_id, true, false); @@ -1881,8 +1897,8 @@ static bool ensure_lexical_binding_capacity(WalkState *state) { } static bool lexical_ancestor_kind(TSNode node, const char *kind) { - for (TSNode parent = ts_node_parent(node); !ts_node_is_null(parent); - parent = ts_node_parent(parent)) { + for (TSNode parent = usage_parent_step(NULL, node); !ts_node_is_null(parent); + parent = usage_parent_step(NULL, parent)) { if (strcmp(ts_node_type(parent), kind) == 0) { return true; } @@ -1890,6 +1906,34 @@ static bool lexical_ancestor_kind(TSNode node, const char *kind) { return false; } +/* Is `node` a name listed by a Python global/nonlocal statement? The grammar + * makes those names direct children of the statement, so in a tree without + * errors one parent step on the walk cursor answers it. The full climb above + * steps with ts_node_parent, which descends from the ROOT each time: O(depth^2) + * per identifier, asked for every Python identifier, so one deep arithmetic + * expression went cubic. Error recovery can put a name under an ERROR child of + * the statement, so a tree with errors keeps the full climb. */ +static CBMPythonDirectiveKind python_scope_directive_kind(CBMExtractCtx *ctx, WalkState *state, + TSNode node) { + if (ts_node_has_error(ctx->root)) { + if (lexical_ancestor_kind(node, "global_statement")) { + return CBM_PYTHON_DIRECTIVE_GLOBAL; + } + return lexical_ancestor_kind(node, "nonlocal_statement") ? CBM_PYTHON_DIRECTIVE_NONLOCAL + : 0; + } + TSNode parent; + const char *field; + if (!occurrence_parent(reset_occurrence_cursor(state, node), node, &parent, &field)) { + return 0; + } + const char *kind = ts_node_type(parent); + if (strcmp(kind, "global_statement") == 0) { + return CBM_PYTHON_DIRECTIVE_GLOBAL; + } + return strcmp(kind, "nonlocal_statement") == 0 ? CBM_PYTHON_DIRECTIVE_NONLOCAL : 0; +} + static TSNode declared_function_name_owner(TSNode node, const CBMLangSpec *spec) { for (TSNode parent = ts_node_parent(node); !ts_node_is_null(parent); parent = ts_node_parent(parent)) { @@ -2236,13 +2280,11 @@ static void record_lexical_binding(CBMExtractCtx *ctx, WalkState *state, TSNode TSNode function_declaration = declared_function_name_owner(node, spec); TSNode class_declaration = declared_class_name_owner(node, spec); - if (ctx->language == CBM_LANG_PYTHON && (lexical_ancestor_kind(node, "global_statement") || - lexical_ancestor_kind(node, "nonlocal_statement"))) { + CBMPythonDirectiveKind scope_directive = + ctx->language == CBM_LANG_PYTHON ? python_scope_directive_kind(ctx, state, node) : 0; + if (scope_directive) { uint32_t function_id = lexical_ancestor_of_kind(state, current_id, true, false); - CBMPythonDirectiveKind directive = lexical_ancestor_kind(node, "global_statement") - ? CBM_PYTHON_DIRECTIVE_GLOBAL - : CBM_PYTHON_DIRECTIVE_NONLOCAL; - record_python_directive(state, function_id, name, directive); + record_python_directive(state, function_id, name, scope_directive); return; } @@ -2709,8 +2751,7 @@ void handle_usages(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec, Wal } bool python_scope_directive = - ctx->language == CBM_LANG_PYTHON && (lexical_ancestor_kind(node, "global_statement") || - lexical_ancestor_kind(node, "nonlocal_statement")); + ctx->language == CBM_LANG_PYTHON && python_scope_directive_kind(ctx, state, node) != 0; if (python_scope_directive || is_binding_occurrence(ctx, node, spec, state)) { char *binding_name = reference_name(ctx, node); if (binding_name && binding_name[0] && !cbm_is_keyword(binding_name, ctx->language)) { diff --git a/tests/repro/repro_lexical_binding_precision.c b/tests/repro/repro_lexical_binding_precision.c index 6c014c2af..7550cb547 100644 --- a/tests/repro/repro_lexical_binding_precision.c +++ b/tests/repro/repro_lexical_binding_precision.c @@ -963,6 +963,59 @@ TEST(repro_python_global_assignment_is_module_binding) { PASS(); } +/* A global/nonlocal name is a directive, not a read (each name of `global a, + * b`), and the rebinding lands where it says: the module from that point on, + * or the enclosing function. Source 1 is an error tree, whose directive and + * default-value checks take the ts_node_parent fallback. */ +TEST(repro_python_scope_directives_bind_the_named_scope) { + static const char *const sources[] = { + "def handler():\n pass\n" + "def outer():\n handler = None\n" + " def inner():\n nonlocal handler\n handler = outer\n" + "def early():\n accept_early(handler)\n" + "def caller():\n def nested():\n global handler, other\n" + " handler = outer\n accept_outer(handler)\n" + "def sibling():\n accept_sibling(handler)\n", + "def handler():\n pass\n" + "def make(handler=handler):\n return handler\n" + "def caller():\n global handler other\n handler = make\n make(handler)\n", + }; + static const struct { + int source; + const char *marker, *name; + int count, blocked, shadow; + } sites[] = { + {0, "global handler", "handler", 0, 0, 0}, + {0, ", other", "other", 0, 0, 0}, + {0, "nonlocal handler", "handler", 0, 0, 0}, + {0, "accept_early(handler)", "handler", 1, 0, 0}, + {0, "accept_outer(handler)", "handler", 1, 1, 0}, + {0, "accept_sibling(handler)", "handler", 1, 1, 0}, + {1, "global handler", "handler", 0, 0, 0}, + {1, "=handler", "handler", 1, 0, 0}, + {1, "make(handler)", "handler", 1, 1, 0}, + }; + int failures = 0; + for (int i = 0; i < (int)(sizeof(sites) / sizeof(sites[0])); i++) { + const char *source = sources[sites[i].source]; + uint32_t start = lb_identifier_offset(source, sites[i].marker, sites[i].name); + ASSERT_NEQ(start, UINT32_MAX); + CBMFileResult *result = lb_extract(source, CBM_LANG_PYTHON, "directives.py"); + ASSERT_NOT_NULL(result); + ASSERT_EQ(result->parse_incomplete, sites[i].source); + LBBindingSite got = lb_binding_site(result, NULL, sites[i].name, start); + cbm_free_result(result); + if (got.count != sites[i].count || got.blocked_count != sites[i].blocked || + got.local_shadow_count != sites[i].shadow) { + printf(" %s: count %d blocked %d shadow %d\n", sites[i].marker, got.count, + got.blocked_count, got.local_shadow_count); + failures++; + } + } + ASSERT_EQ(failures, 0); + PASS(); +} + /* Module assignments participate in name lookup too. A later function must * not raw-name-fallback through a module variable to an older same-named * callable declaration. */ @@ -1055,6 +1108,7 @@ SUITE(repro_lexical_binding_precision) { RUN_TEST(repro_objectscript_udl_set_target_binds_method_local); RUN_TEST(repro_python_default_expression_uses_enclosing_namespace); RUN_TEST(repro_python_global_assignment_is_module_binding); + RUN_TEST(repro_python_scope_directives_bind_the_named_scope); RUN_TEST(repro_python_module_assignment_blocks_callable_fallback); RUN_TEST(repro_go_function_literal_parameter_does_not_leak); } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index acf5b2319..afc9f5536 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -7258,6 +7258,49 @@ TEST(extract_rescript_let_bindings_use_the_walk_cursor) { ASSERT_EQ(big_fallbacks, 0); PASS(); } + +/* Every Python usage asks whether it is a global/nonlocal name and whether it + * sits in a parameter default. Both climbed with ts_node_parent, which + * descends from the ROOT: O(d^2) for a usage d deep, so one deep expression + * went cubic (2.2 s / 17 s / >90 s at 500/1000/2000 terms). Count every such + * root-descending step at N and 2N terms. */ +TEST(extract_python_deep_default_parent_climbs_are_linear) { + enum { SMALL = 200 }; + uint64_t slow_parents[2]; + for (int round = 0; round < 2; round++) { + /* def f(x=x + x + ... + x): return x -- a binary_operator chain. */ + int terms = SMALL << round; + size_t capacity = 32U + (size_t)terms * 4U; + char *source = malloc(capacity); + ASSERT_NOT_NULL(source); + int length = snprintf(source, capacity, "def f(x=x"); + for (int i = 1; i < terms; i++) { + length += snprintf(source + length, capacity - (size_t)length, " + x"); + } + length += snprintf(source + length, capacity - (size_t)length, "):\n return x\n"); + cbm_usage_field_lookup_test_reset(); + CBMFileResult *result = + cbm_extract_file(source, length, CBM_LANG_PYTHON, "proj", "deep.py", 0, NULL, NULL); + free(source); + ASSERT_NOT_NULL(result); + slow_parents[round] = cbm_usage_slow_parent_fallback_test_count(); + /* Anti-vacuous: every default x climbed its chain to the parameter and + * reads the enclosing namespace; only `return x` is shadowed by it. */ + int defaults = 0; + for (int i = 0; i < result->usages.count; i++) { + const CBMUsage *usage = &result->usages.items[i]; + defaults += usage->ref_name && strcmp(usage->ref_name, "x") == 0 && + !usage->semantic_reference_local_shadow; + } + cbm_free_result(result); + ASSERT_EQ(defaults, terms); + } + fprintf(stderr, " [python-deep-default] slow_parents(%d)=%llu slow_parents(%d)=%llu\n", SMALL, + (unsigned long long)slow_parents[0], 2 * SMALL, (unsigned long long)slow_parents[1]); + /* Twice the terms may cost about twice the steps; the cubic climb took 4x. */ + ASSERT_LTE(slow_parents[1], slow_parents[0] * 3U + 64U); + PASS(); +} #endif /* =================================================================== @@ -8605,6 +8648,7 @@ SUITE(extraction) { RUN_TEST(extract_wide_flat_reference_fields_are_linear); RUN_TEST(extract_csharp_argument_values_use_the_walk_cursor); RUN_TEST(extract_rescript_let_bindings_use_the_walk_cursor); + RUN_TEST(extract_python_deep_default_parent_climbs_are_linear); #endif /* Perl call-graph noise (#459 follow-up) */ From 2c71746fa52f4945f7e2edcbd92e5ded91bff395 Mon Sep 17 00:00:00 2001 From: Pcristin Date: Thu, 24 Sep 2026 14:40:36 +0000 Subject: [PATCH 057/217] fix(pipeline): expose unresolved call coverage Signed-off-by: Pcristin --- src/mcp/mcp.c | 143 +++++++++++++++++++++++++--- src/pipeline/pipeline.c | 127 +++++++++++++++++++++++- src/pipeline/pipeline_incremental.c | 50 +++++++++- src/pipeline/pipeline_internal.h | 7 +- src/store/store.c | 10 ++ src/store/store.h | 11 ++- tests/test_mcp.c | 115 ++++++++++++++++++++-- tests/test_pipeline.c | 93 ++++++++++++++++++ 8 files changed, 524 insertions(+), 32 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 78b8dccf9..f3e700aea 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -64,6 +64,7 @@ enum { #include "watcher/watcher.h" #include "foundation/mem.h" #include "foundation/mem_core.h" +#include "foundation/hash_table.h" #include "foundation/diagnostics.h" #include "foundation/platform.h" #include "foundation/compat.h" @@ -6090,11 +6091,13 @@ static void add_coverage_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_s yyjson_mut_val *pp_files = yyjson_mut_arr(doc); yyjson_mut_val *pu_files = yyjson_mut_arr(doc); + yyjson_mut_val *unresolved_files = yyjson_mut_arr(doc); yyjson_mut_val *sk_files = yyjson_mut_arr(doc); yyjson_mut_val *ni_dirs = yyjson_mut_arr(doc); yyjson_mut_val *ni_files = yyjson_mut_arr(doc); int pp_n = 0; int pu_n = 0; + int unresolved_n = 0; int sk_n = 0; int ni_dir_n = 0; int ni_file_n = 0; @@ -6124,6 +6127,11 @@ static void add_coverage_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_s yyjson_mut_arr_add_val(pu_files, fe); } pu_n++; + } else if (strcmp(kind, "unresolved_calls") == 0) { + if (unresolved_n < sample_limit) { + yyjson_mut_arr_add_strcpy(doc, unresolved_files, rows[i].rel_path); + } + unresolved_n++; } else if (strcmp(kind, "not_indexed_dir") == 0) { if (ni_dir_n < sample_limit) { yyjson_mut_arr_add_strcpy(doc, ni_dirs, rows[i].rel_path); @@ -6181,6 +6189,12 @@ static void add_coverage_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_s yyjson_mut_obj_add_bool(doc, pu, "truncated", pu_n > COVERAGE_FILE_CAP); yyjson_mut_obj_add_val(doc, root, "parse_unusable", pu); + yyjson_mut_val *unresolved = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, unresolved, "files", unresolved_files); + yyjson_mut_obj_add_int(doc, unresolved, "count", unresolved_n); + yyjson_mut_obj_add_bool(doc, unresolved, "truncated", unresolved_n > sample_limit); + yyjson_mut_obj_add_val(doc, root, "unresolved_calls", unresolved); + yyjson_mut_val *sk = yyjson_mut_obj(doc); yyjson_mut_obj_add_val(doc, sk, "files", sk_files); yyjson_mut_obj_add_int(doc, sk, "count", sk_n); @@ -6218,7 +6232,7 @@ static void add_coverage_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_s } yyjson_mut_obj_add_val(doc, root, "not_indexed", ni); - if (sample_limit > 0 && (pp_n > 0 || sk_n > 0)) { + if (sample_limit > 0 && (pp_n > 0 || sk_n > 0 || unresolved_n > 0)) { yyjson_mut_obj_add_str( doc, root, "coverage_note", "Best-effort signal, not a completeness guarantee: parse_partial files WERE indexed, " @@ -6226,7 +6240,9 @@ static void add_coverage_report(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_s "(tree-sitter error recovery still salvages some). skipped files were not indexed at " "all. Prefer text search (grep) for flagged files/ranges. Files absent from this list " "are NOT guaranteed to be fully indexed. (not_indexed entries are a separate, " - "BY-DESIGN class — deliberate ignore rules, not failures.)"); + "BY-DESIGN class — deliberate ignore rules, not failures.) " + "Unresolved callsites need source verification because CALLS totals may be " + "incomplete."); } } @@ -6467,6 +6483,9 @@ static const char *coverage_status(const cbm_coverage_row_t *rows, int count, if (pass == 0 && strcmp(kind, "parse_partial") == 0) { return "partial"; } + if (pass == 0 && strcmp(kind, "unresolved_calls") == 0) { + return "unresolved_calls"; + } if (pass == 1 && strncmp(kind, "not_indexed", 11) == 0) { return "excluded"; } @@ -6491,6 +6510,9 @@ static const char *coverage_recommended_action(const char *status, const char *f if (strcmp(status, "partial") == 0) { return "read_ranges_and_verify_scope"; } + if (strcmp(status, "unresolved_calls") == 0) { + return "read_source_and_verify_calls"; + } if (strcmp(status, "unusable") == 0) { /* The ranges cover nearly the whole file, so sending a reader to them * is the same as sending them to the file. Say the useful thing. */ @@ -9350,6 +9372,102 @@ static char *handle_trace_call_path(cbm_mcp_server_t *srv, const char *args) { bool optional_fields_omitted = false; char *json = NULL; + bool trace_includes_calls = false; + for (int i = 0; i < edge_type_count; i++) { + if (strcmp(edge_types[i], "CALLS") == 0) { + trace_includes_calls = true; + break; + } + } + bool unresolved_out = false; + bool unresolved_in = false; + if (trace_includes_calls) { + cbm_coverage_meta_t coverage_meta = {0}; + int meta_rc = cbm_store_coverage_meta_get(store, project, &coverage_meta); + if (meta_rc == CBM_STORE_OK) { + if (coverage_meta.coverage_version < CBM_UNRESOLVED_CALL_COVERAGE_VERSION || + !coverage_meta.recording_status || + strcmp(coverage_meta.recording_status, "unavailable") == 0) { + unresolved_out = do_outbound; + unresolved_in = do_inbound; + } + } else { + unresolved_out = do_outbound; + unresolved_in = do_inbound; + } + cbm_store_coverage_meta_clear(&coverage_meta); + if ((do_outbound && !unresolved_out) || (do_inbound && !unresolved_in)) { + CBMHashTable *callers = do_outbound ? cbm_ht_create(0) : NULL; + CBMHashTable *leaves = do_inbound ? cbm_ht_create(0) : NULL; + if ((do_outbound && !callers) || (do_inbound && !leaves)) { + unresolved_out = do_outbound; + unresolved_in = do_inbound; + } else { + bool key_set_failed = false; + for (int i = 0; i < node_count; i++) { + if (callers && nodes[i].qualified_name) { + cbm_ht_set(callers, nodes[i].qualified_name, (void *)1); + key_set_failed |= !cbm_ht_has(callers, nodes[i].qualified_name); + } + if (leaves && nodes[i].name) { + cbm_ht_set(leaves, nodes[i].name, (void *)1); + key_set_failed |= !cbm_ht_has(leaves, nodes[i].name); + } + } + for (int i = 0; callers && i < tr_out.visited_count; i++) { + const cbm_node_t *node = &tr_out.visited[i].node; + if ((include_tests || !is_test_file(node->file_path)) && node->qualified_name) { + cbm_ht_set(callers, node->qualified_name, (void *)1); + key_set_failed |= !cbm_ht_has(callers, node->qualified_name); + } + } + for (int i = 0; leaves && i < tr_in.visited_count; i++) { + const cbm_node_t *node = &tr_in.visited[i].node; + if ((include_tests || !is_test_file(node->file_path)) && node->name) { + cbm_ht_set(leaves, node->name, (void *)1); + key_set_failed |= !cbm_ht_has(leaves, node->name); + } + } + cbm_coverage_row_t *rows = NULL; + int row_count = 0; + if (key_set_failed || cbm_store_coverage_get_unresolved_calls( + store, project, &rows, &row_count) != CBM_STORE_OK) { + unresolved_out = do_outbound; + unresolved_in = do_inbound; + } else { + for (int i = 0; i < row_count && ((!unresolved_out && do_outbound) || + (!unresolved_in && do_inbound)); + i++) { + yyjson_doc *detail = yyjson_read(rows[i].detail, strlen(rows[i].detail), 0); + yyjson_val *sites = detail ? yyjson_doc_get_root(detail) : NULL; + if (!yyjson_is_arr(sites)) { + unresolved_out = do_outbound; + unresolved_in = do_inbound; + } else { + size_t idx, max; + yyjson_val *site; + yyjson_arr_foreach(sites, idx, max, site) { + const char *caller = yyjson_get_str(yyjson_obj_get(site, "caller")); + const char *leaf = yyjson_get_str(yyjson_obj_get(site, "leaf")); + if (callers && caller && cbm_ht_has(callers, caller)) + unresolved_out = true; + if (leaves && leaf && cbm_ht_has(leaves, leaf)) + unresolved_in = true; + } + } + if (detail) + yyjson_doc_free(detail); + } + } + cbm_store_free_coverage(rows, row_count); + } + cbm_ht_free(callers); + cbm_ht_free(leaves); + } + } + const char *out_relation = unresolved_out ? "unknown" : (tr_out.truncated ? "gte" : "eq"); + const char *in_relation = unresolved_in ? "unknown" : (tr_in.truncated ? "gte" : "eq"); + render_trace_output:; int rows_left = row_target; out_len = requested_out_len < rows_left ? requested_out_len : rows_left; @@ -9438,7 +9556,7 @@ render_trace_output:; bool flat_trace = render_risk || render_data_flow; if (do_outbound) { cbm_tree_scalar_int(&sb, "callees_total", out_total); - cbm_tree_scalar_str(&sb, "callees_total_relation", tr_out.truncated ? "gte" : "eq"); + cbm_tree_scalar_str(&sb, "callees_total_relation", out_relation); if (flat_trace) { bfs_to_toon_table(&sb, "callees", &view_out, render_risk, include_tests, render_data_flow, render_evidence, &out_edge_ctx); @@ -9449,7 +9567,7 @@ render_trace_output:; } if (do_inbound) { cbm_tree_scalar_int(&sb, "callers_total", in_total); - cbm_tree_scalar_str(&sb, "callers_total_relation", tr_in.truncated ? "gte" : "eq"); + cbm_tree_scalar_str(&sb, "callers_total_relation", in_relation); if (flat_trace) { bfs_to_toon_table(&sb, "callers", &view_in, render_risk, include_tests, render_data_flow, render_evidence, &in_edge_ctx); @@ -9515,8 +9633,7 @@ render_trace_output:; } if (do_outbound) { yyjson_mut_obj_add_int(doc, root, "callees_total", out_total); - yyjson_mut_obj_add_str(doc, root, "callees_total_relation", - tr_out.truncated ? "gte" : "eq"); + yyjson_mut_obj_add_str(doc, root, "callees_total_relation", out_relation); yyjson_mut_obj_add_val( doc, root, "callees", bfs_to_tree_json(doc, &view_out, risk_labels && emit_optional_fields, include_tests, @@ -9525,8 +9642,7 @@ render_trace_output:; } if (do_inbound) { yyjson_mut_obj_add_int(doc, root, "callers_total", in_total); - yyjson_mut_obj_add_str(doc, root, "callers_total_relation", - tr_in.truncated ? "gte" : "eq"); + yyjson_mut_obj_add_str(doc, root, "callers_total_relation", in_relation); yyjson_mut_obj_add_val( doc, root, "callers", bfs_to_tree_json(doc, &view_in, risk_labels && emit_optional_fields, include_tests, @@ -9584,13 +9700,11 @@ render_trace_output:; cbm_sb_init(&floor); if (do_outbound) { cbm_tree_scalar_int(&floor, "callees_total", out_total); - cbm_tree_scalar_str(&floor, "callees_total_relation", - tr_out.truncated ? "gte" : "eq"); + cbm_tree_scalar_str(&floor, "callees_total_relation", out_relation); } if (do_inbound) { cbm_tree_scalar_int(&floor, "callers_total", in_total); - cbm_tree_scalar_str(&floor, "callers_total_relation", - tr_in.truncated ? "gte" : "eq"); + cbm_tree_scalar_str(&floor, "callers_total_relation", in_relation); } cbm_tree_scalar_bool(&floor, "has_more", floor_has_more); if (floor_has_more) { @@ -9615,12 +9729,11 @@ render_trace_output:; if (do_outbound) { yyjson_mut_obj_add_int(floor_doc, floor, "callees_total", out_total); yyjson_mut_obj_add_str(floor_doc, floor, "callees_total_relation", - tr_out.truncated ? "gte" : "eq"); + out_relation); } if (do_inbound) { yyjson_mut_obj_add_int(floor_doc, floor, "callers_total", in_total); - yyjson_mut_obj_add_str(floor_doc, floor, "callers_total_relation", - tr_in.truncated ? "gte" : "eq"); + yyjson_mut_obj_add_str(floor_doc, floor, "callers_total_relation", in_relation); } yyjson_mut_obj_add_bool(floor_doc, floor, "has_more", floor_has_more); if (floor_has_more) { diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index c808cb1b4..9d12c55d9 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -20,6 +20,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include "pipeline/artifact.h" #include "pipeline/pipeline_internal.h" #include "pipeline/lsp_surface.h" +#include "pipeline/lsp_resolve.h" #include "pipeline/pass_lsp_cross.h" #include "pipeline/pass_ensemble_routing.h" #include "pipeline/worker_pool.h" @@ -42,6 +43,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include "foundation/mem_core.h" #include "result_spill.h" #include "foundation/secure_random.h" +#include "yyjson/yyjson.h" #include #include @@ -260,6 +262,11 @@ struct cbm_pipeline { cbm_lsp_surface_row_t *surface_rows; int surface_row_count; + /* Per-file unresolved invocation diagnostics, published as coverage rows. */ + cbm_coverage_row_t *unresolved_rows; + int unresolved_count; + bool unresolved_capture_failed; + /* Deterministic test-only seam at the final publication boundary. Kept * per pipeline so concurrent test/process activity cannot cross-trigger. */ void (*before_publish_hook)(cbm_pipeline_t *, const char *, void *); @@ -431,6 +438,96 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row p->surface_row_count = count; } +static void *unresolved_json_alloc(void *ctx, size_t size) { + (void)ctx; + return cbm_alloc(CBM_MEM_CLASS_DUMP, size); +} + +static void *unresolved_json_realloc(void *ctx, void *ptr, size_t old_size, size_t size) { + (void)ctx; + (void)old_size; + return cbm_realloc(CBM_MEM_CLASS_DUMP, ptr, size); +} + +static void unresolved_json_free(void *ctx, void *ptr) { + (void)ctx; + cbm_free(CBM_MEM_CLASS_DUMP, ptr); +} + +void cbm_pipeline_record_unresolved_calls(cbm_pipeline_t *p, const char *rel_path, + const CBMFileResult *result) { + if (!p || !rel_path || !result || result->resolved_calls.count == 0 || + p->unresolved_capture_failed) { + return; + } + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + if (!doc) { + p->unresolved_capture_failed = true; + return; + } + yyjson_mut_val *sites = yyjson_mut_arr(doc); + yyjson_mut_doc_set_root(doc, sites); + int count = 0; + for (int i = 0; i < result->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &result->resolved_calls.items[i]; + if (rc->kind != CBM_RESOLVED_INVOCATION || !rc->strategy || + strcmp(rc->strategy, "lsp_unresolved") != 0 || !rc->caller_qn || !rc->callee_qn) { + continue; + } + const char *leaf = strrchr(rc->callee_qn, '.'); + leaf = leaf ? leaf + 1 : rc->callee_qn; + yyjson_mut_val *site = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, site, "caller", rc->caller_qn); + yyjson_mut_obj_add_strcpy(doc, site, "leaf", leaf); + yyjson_mut_obj_add_uint(doc, site, "start_byte", rc->site_start_byte); + yyjson_mut_obj_add_uint(doc, site, "end_byte", rc->site_end_byte); + yyjson_mut_obj_add_strcpy(doc, site, "reason", rc->reason ? rc->reason : "unresolved"); + yyjson_mut_arr_add_val(sites, site); + count++; + } + if (count > 0) { + yyjson_alc allocator = {.malloc = unresolved_json_alloc, + .realloc = unresolved_json_realloc, + .free = unresolved_json_free}; + char *detail = yyjson_mut_write_opts(doc, 0, &allocator, NULL, NULL); + char *path = cbm_mem_strdup(CBM_MEM_CLASS_DUMP, rel_path); + if (!detail || !path) { + p->unresolved_capture_failed = true; + cbm_free(CBM_MEM_CLASS_DUMP, detail); + cbm_free(CBM_MEM_CLASS_DUMP, path); + } else { + cbm_coverage_row_t *rows = + cbm_realloc(CBM_MEM_CLASS_DUMP, p->unresolved_rows, + (size_t)(p->unresolved_count + 1) * sizeof(*rows)); + if (!rows) { + p->unresolved_capture_failed = true; + cbm_free(CBM_MEM_CLASS_DUMP, detail); + cbm_free(CBM_MEM_CLASS_DUMP, path); + } else { + p->unresolved_rows = rows; + p->unresolved_rows[p->unresolved_count++] = (cbm_coverage_row_t){ + .rel_path = path, .kind = "unresolved_calls", .detail = detail}; + } + } + } + yyjson_mut_doc_free(doc); +} + +void cbm_pipeline_mark_unresolved_capture_failed(cbm_pipeline_t *p) { + if (p) + p->unresolved_capture_failed = true; +} + +void cbm_pipeline_get_unresolved_calls(cbm_pipeline_t *p, cbm_coverage_row_t **rows, int *count, + bool *complete) { + if (rows) + *rows = p ? p->unresolved_rows : NULL; + if (count) + *count = p ? p->unresolved_count : 0; + if (complete) + *complete = p && !p->unresolved_capture_failed; +} + void cbm_pipeline_free(cbm_pipeline_t *p) { if (!p) { return; @@ -461,6 +558,11 @@ void cbm_pipeline_free(cbm_pipeline_t *p) { cbm_store_free_lsp_surfaces(p->surface_rows, p->surface_row_count); p->surface_rows = NULL; p->surface_row_count = 0; + for (int i = 0; i < p->unresolved_count; i++) { + cbm_free(CBM_MEM_CLASS_DUMP, (char *)p->unresolved_rows[i].rel_path); + cbm_free(CBM_MEM_CLASS_DUMP, (char *)p->unresolved_rows[i].detail); + } + cbm_free(CBM_MEM_CLASS_DUMP, p->unresolved_rows); cbm_git_context_free(&p->git_ctx); /* gbuf, store, registry freed during/after run */ /* Defensively free userconfig in case run() was never called or panicked */ @@ -1469,6 +1571,9 @@ static int run_sequential_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, if (seq_cache) { for (int i = 0; i < file_count; i++) { if (seq_cache[i]) { + if (rc == 0) { + cbm_pipeline_record_unresolved_calls(p, files[i].rel_path, seq_cache[i]); + } cbm_free_result(seq_cache[i]); } } @@ -1694,6 +1799,18 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, cbm_clock_gettime(CLOCK_MONOTONIC, t); rc = cbm_parallel_resolve(ctx, files, file_count, cache, &shared_ids, worker_count, all_defs, def_count, def_modules, module_def_index, &cross_registries); + if (rc == 0) { + for (int i = 0; i < file_count; i++) { + bool loaded = false; + CBMFileResult *result = cbm_pipeline_result_acquire(ctx, cache, i, NULL, &loaded); + if (result) { + cbm_pipeline_record_unresolved_calls(p, files[i].rel_path, result); + } else if (ctx->spill && cbm_result_spill_has(ctx->spill, i)) { + cbm_pipeline_mark_unresolved_capture_failed(p); + } + cbm_pipeline_result_release(result, loaded); + } + } cbm_log_info("pass.timing", "pass", "parallel_resolve", "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); pipeline_phase_mark("parallel_resolve"); @@ -2497,14 +2614,18 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas return CBM_PIPELINE_ABORT_PRESERVE_DB; } - int cov_total = p->file_errors_count + p->excluded_count + p->ignored_count; + int cov_total = + p->file_errors_count + p->excluded_count + p->ignored_count + p->unresolved_count; cbm_coverage_row_t *cov = NULL; int cov_count = 0; - bool coverage_rows_available = cov_total == 0; + bool coverage_rows_available = cov_total == 0 && !p->unresolved_capture_failed; if (cov_total > 0) { cov = malloc((size_t)cov_total * sizeof(*cov)); if (cov) { - coverage_rows_available = true; + coverage_rows_available = !p->unresolved_capture_failed; + for (int i = 0; i < p->unresolved_count; i++) { + cov[cov_count++] = p->unresolved_rows[i]; + } for (int i = 0; i < p->file_errors_count; i++) { cov[cov_count++] = (cbm_coverage_row_t){.rel_path = p->file_errors[i].path, .kind = p->file_errors[i].phase, diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 154518658..f0953764e 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -30,6 +30,7 @@ enum { INCR_RING_BUF = 4, INCR_RING_MASK = 3, INCR_TS_BUF = 24 }; #include "foundation/compat_thread.h" #include "foundation/platform.h" #include "foundation/sha256.h" +#include "result_spill.h" #include #include @@ -1351,6 +1352,19 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed rc = cbm_parallel_resolve(ctx, changed_files, ci, cache, &shared_ids, worker_count, all_defs, all_def_count, closure ? closure->def_modules : NULL, module_def_index, registries_arg); + if (rc == 0) { + for (int i = 0; i < ci; i++) { + bool loaded = false; + CBMFileResult *result = cbm_pipeline_result_acquire(ctx, cache, i, NULL, &loaded); + if (result) { + cbm_pipeline_record_unresolved_calls(ctx->pipeline, changed_files[i].rel_path, + result); + } else if (ctx->spill && cbm_result_spill_has(ctx->spill, i)) { + cbm_pipeline_mark_unresolved_capture_failed(ctx->pipeline); + } + cbm_pipeline_result_release(result, loaded); + } + } if (module_def_index) { cbm_pxc_free_module_def_index(module_def_index); } @@ -1401,6 +1415,14 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed if (rc == 0) { rc = cbm_pipeline_pass_semantic(ctx, changed_files, ci); } + if (rc == 0 && cache) { + for (int i = 0; i < ci; i++) { + if (cache[i]) { + cbm_pipeline_record_unresolved_calls(ctx->pipeline, changed_files[i].rel_path, + cache[i]); + } + } + } if (rc == 0) { rc = cbm_pipeline_check_cancel(ctx); } @@ -2237,11 +2259,17 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char int run_ignored_count = 0; int run_ignored_total = 0; cbm_pipeline_get_ignored(p, &run_ignored, &run_ignored_count, &run_ignored_total); - int cov_cap = old_cov_count + run_err_count + run_excluded_count + run_ignored_count; - bool coverage_rows_available = cov_cap == 0; + cbm_coverage_row_t *run_unresolved = NULL; + int run_unresolved_count = 0; + bool unresolved_complete = false; + cbm_pipeline_get_unresolved_calls(p, &run_unresolved, &run_unresolved_count, + &unresolved_complete); + int cov_cap = old_cov_count + run_err_count + run_excluded_count + run_ignored_count + + run_unresolved_count; + bool coverage_rows_available = cov_cap == 0 && unresolved_complete; if (cov_cap > 0) { cov = (cbm_coverage_row_t *)malloc((size_t)cov_cap * sizeof(*cov)); - coverage_rows_available = cov != NULL; + coverage_rows_available = cov != NULL && unresolved_complete; } if (cov) { for (int i = 0; i < old_cov_count; i++) { @@ -2257,6 +2285,9 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char cov[cov_n].detail = run_errs[i].reason; cov_n++; } + for (int i = 0; i < run_unresolved_count; i++) { + cov[cov_n++] = run_unresolved[i]; + } for (int i = 0; i < run_excluded_count; i++) { cov[cov_n].rel_path = run_excluded[i]; cov[cov_n].kind = "not_indexed_dir"; @@ -2810,11 +2841,17 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_pipeline_get_ignored(p, &run_ignored, &run_ignored_count, &run_ignored_total); cbm_coverage_row_t *cov = NULL; int cov_n = 0; - int cov_cap = old_cov_count + run_err_count + run_excluded_count + run_ignored_count; + cbm_coverage_row_t *run_unresolved = NULL; + int run_unresolved_count = 0; + bool unresolved_complete = false; + cbm_pipeline_get_unresolved_calls(p, &run_unresolved, &run_unresolved_count, + &unresolved_complete); + int cov_cap = old_cov_count + run_err_count + run_excluded_count + run_ignored_count + + run_unresolved_count; if (cov_cap > 0) { cov = (cbm_coverage_row_t *)malloc((size_t)cov_cap * sizeof(*cov)); } - bool coverage_rows_available = cov_cap == 0 || cov != NULL; + bool coverage_rows_available = (cov_cap == 0 || cov != NULL) && unresolved_complete; if (cov) { CBMHashTable *changed_set = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64); for (int i = 0; i < ci; i++) { @@ -2834,6 +2871,9 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cov[cov_n].detail = run_errs[i].reason; cov_n++; } + for (int i = 0; i < run_unresolved_count; i++) { + cov[cov_n++] = run_unresolved[i]; + } for (int i = 0; i < run_excluded_count; i++) { cov[cov_n].rel_path = run_excluded[i]; cov[cov_n].kind = "not_indexed_dir"; diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index e7782adef..182241eca 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -789,7 +789,7 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr /* Compatibility contract persisted in coverage metadata. Increment when a * graph/manifest semantic change makes prior exact-input indexes unsafe. */ -enum { CBM_SEMANTIC_INDEX_VERSION = 3 }; +enum { CBM_SEMANTIC_INDEX_VERSION = CBM_UNRESOLVED_CALL_COVERAGE_VERSION }; typedef struct { cbm_gbuf_t *gbuf; @@ -874,6 +874,11 @@ void cbm_pipeline_discard_stage(const char *stage_path); * Takes ownership; dump_and_persist_hashes writes them into the staging * store and cbm_pipeline_free releases them. Passing NULL/0 clears. */ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *rows, int count); +void cbm_pipeline_record_unresolved_calls(cbm_pipeline_t *p, const char *rel_path, + const CBMFileResult *result); +void cbm_pipeline_mark_unresolved_capture_failed(cbm_pipeline_t *p); +void cbm_pipeline_get_unresolved_calls(cbm_pipeline_t *p, cbm_coverage_row_t **rows, int *count, + bool *complete); /* Pipeline accessors for incremental use */ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); diff --git a/src/store/store.c b/src/store/store.c index 558f974d2..5b1936dba 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -329,6 +329,8 @@ static int init_schema(cbm_store_t *s) { " detail TEXT DEFAULT ''," " PRIMARY KEY (project, rel_path, kind)" ");" + "CREATE INDEX IF NOT EXISTS idx_index_coverage_project_kind " + "ON index_coverage(project, kind);" /* One row per completed coverage persistence attempt. Kept separate * from projects so existing graph/artifact schema stays compatible and * a missing row unambiguously means coverage metadata is unavailable. */ @@ -4215,6 +4217,14 @@ int cbm_store_coverage_get_scope(cbm_store_t *s, const char *project, const char return coverage_query_rows(s, project, scope, sql, out, count); } +int cbm_store_coverage_get_unresolved_calls(cbm_store_t *s, const char *project, + cbm_coverage_row_t **out, int *count) { + static const char sql[] = "SELECT rel_path, kind, detail FROM index_coverage " + "WHERE project = ?1 AND kind = ?2 " + "ORDER BY rel_path;"; + return coverage_query_rows(s, project, "unresolved_calls", sql, out, count); +} + void cbm_store_coverage_meta_clear(cbm_coverage_meta_t *meta) { if (!meta) { return; diff --git a/src/store/store.h b/src/store/store.h index b6ce844f7..82489255e 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -639,7 +639,9 @@ int cbm_store_delete_file_hashes(cbm_store_t *s, const char *project); /* One best-effort coverage row: a file the indexer could not fully cover. * kind "parse_partial" = indexed but the parse tree had ERROR/MISSING regions * (detail = 1-based line ranges "12-40,88-90"); skip kinds "read"/"extract"/ - * "oversized" = not indexed at all (detail = reason). Stored in the separate + * "oversized" = not indexed at all (detail = reason). "unresolved_calls" is + * indexed source whose invocation targets were not resolved (detail = JSON + * array of caller, leaf, source byte span, and reason). Stored in the separate * index_coverage table — coverage is metadata ABOUT the graph, never mixed * into the graph itself. */ typedef struct { @@ -648,6 +650,10 @@ typedef struct { const char *detail; } cbm_coverage_row_t; +/* Older generations have no unresolved-call records, so their exact trace + * totals cannot be trusted after this signal becomes part of coverage. */ +enum { CBM_UNRESOLVED_CALL_COVERAGE_VERSION = 4 }; + /* Metadata describing how completely one index run recorded the best-effort * coverage signal. `recording_status` is "complete", "truncated", or * "unavailable"; it is deliberately separate from hash_records_complete. @@ -690,6 +696,9 @@ int cbm_store_coverage_get_path(cbm_store_t *s, const char *project, const char * ancestor that covers the scope. Prefix matching is segment-boundary safe. */ int cbm_store_coverage_get_scope(cbm_store_t *s, const char *project, const char *scope, cbm_coverage_row_t **out, int *count); +/* Fetch only unresolved invocation diagnostics for trace exactness checks. */ +int cbm_store_coverage_get_unresolved_calls(cbm_store_t *s, const char *project, + cbm_coverage_row_t **out, int *count); /* Fetch/free the metadata paired with the current coverage row set. */ int cbm_store_coverage_meta_get(cbm_store_t *s, const char *project, cbm_coverage_meta_t *out); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 60649f505..02cb9db01 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -5971,9 +5971,9 @@ TEST(tool_check_index_coverage_freshness_uses_indexer_mtime_source_issue1714) { ASSERT_EQ(cbm_path_info_utf8(source_path, &info), 0); /* The hash exactly as the indexer writes it: same source, ns precision. */ - ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", info.mtime_ns, - info.size), - CBM_STORE_OK); + ASSERT_EQ( + cbm_store_upsert_file_hash(store, "test-project", "main.go", "", info.mtime_ns, info.size), + CBM_STORE_OK); /* format=json, like every other coverage test here: the DEFAULT response is * the compact table, in which a field name never appears. Keeping the @@ -5990,11 +5990,11 @@ TEST(tool_check_index_coverage_freshness_uses_indexer_mtime_source_issue1714) { /* A hash stored at seconds precision — what a stat-based reader previously * compared against — must NOT match an unchanged file: the comparison must * stay nanosecond-exact, or part of mtime resolution is silently dropped. */ - int64_t seconds_mtime_ns = (info.mtime_ns / (int64_t)CBM_NSEC_PER_SEC) * - (int64_t)CBM_NSEC_PER_SEC; + int64_t seconds_mtime_ns = + (info.mtime_ns / (int64_t)CBM_NSEC_PER_SEC) * (int64_t)CBM_NSEC_PER_SEC; if (seconds_mtime_ns != info.mtime_ns) { - ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", - seconds_mtime_ns, info.size), + ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", seconds_mtime_ns, + info.size), CBM_STORE_OK); response = cbm_mcp_handle_tool(srv, "check_index_coverage", "{\"project\":\"test-project\",\"paths\":[\"main.go\"]," @@ -6638,6 +6638,13 @@ TEST(tool_trace_budget_never_slices_identifiers) { const char *project = "trace-byte-budget"; cbm_mcp_server_set_project(srv, project); ASSERT_EQ(cbm_store_upsert_project(store, project, "/tmp/trace-byte-budget"), CBM_STORE_OK); + cbm_coverage_meta_t coverage_meta = {.generation = "fixture", + .index_mode = "full", + .recorded_at = "2026-09-24T00:00:00Z", + .recording_status = "complete", + .coverage_version = CBM_UNRESOLVED_CALL_COVERAGE_VERSION, + .hash_records_complete = true}; + ASSERT_EQ(cbm_store_coverage_replace_ex(store, project, NULL, 0, &coverage_meta), CBM_STORE_OK); cbm_node_t hub = {.project = project, .label = "Function", @@ -6983,6 +6990,13 @@ TEST(tool_trace_reports_engine_saturation_as_lower_bound) { const char *project = "trace-engine-cap"; cbm_mcp_server_set_project(srv, project); ASSERT_EQ(cbm_store_upsert_project(store, project, "/tmp/trace-engine-cap"), CBM_STORE_OK); + cbm_coverage_meta_t coverage_meta = {.generation = "fixture", + .index_mode = "full", + .recorded_at = "2026-09-24T00:00:00Z", + .recording_status = "complete", + .coverage_version = CBM_UNRESOLVED_CALL_COVERAGE_VERSION, + .hash_records_complete = true}; + ASSERT_EQ(cbm_store_coverage_replace_ex(store, project, NULL, 0, &coverage_meta), CBM_STORE_OK); cbm_node_t hub = {.project = project, .label = "Function", .name = "hub", @@ -7412,6 +7426,92 @@ TEST(tool_trace_path_evidence_is_opt_in_and_class_mapped) { PASS(); } +TEST(tool_trace_path_marks_unresolved_call_totals_unknown) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + cbm_store_t *st = cbm_mcp_server_store(srv); + const char *proj = "unresolved-trace"; + cbm_mcp_server_set_project(srv, proj); + cbm_store_upsert_project(st, proj, "/tmp/unresolved-trace"); + cbm_node_t caller = {.project = proj, + .label = "Function", + .name = "run", + .qualified_name = "unresolved-trace.service.run", + .file_path = "service.js", + .start_line = 1, + .end_line = 4}; + cbm_node_t callee = {.project = proj, + .label = "Function", + .name = "buscar", + .qualified_name = "unresolved-trace.client.buscar", + .file_path = "client.js", + .start_line = 1, + .end_line = 2}; + ASSERT_GT(cbm_store_upsert_node(st, &caller), 0); + ASSERT_GT(cbm_store_upsert_node(st, &callee), 0); + ASSERT_EQ(cbm_store_upsert_file_hash(st, proj, "service.js", "fixture", 0, 0), CBM_STORE_OK); + char *out = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":0,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_path\",\"arguments\":{\"project\":" + "\"unresolved-trace\",\"function_name\":\"run\",\"direction\":\"outbound\"," + "\"format\":\"json\"}}}"); + char *txt = extract_text_content(out); + ASSERT_NOT_NULL(strstr(txt, "\"callees_total_relation\":\"unknown\"")); + free(txt); + free(out); + cbm_coverage_row_t row = { + .rel_path = "service.js", + .kind = "unresolved_calls", + .detail = "[{\"caller\":\"unresolved-trace.service.run\",\"leaf\":\"buscar\"," + "\"start_byte\":42,\"end_byte\":59,\"reason\":\"method_not_in_registry\"}]"}; + cbm_coverage_meta_t meta = {.generation = "fixture", + .index_mode = "full", + .recorded_at = "2026-09-24T00:00:00Z", + .recording_status = "complete", + .coverage_version = CBM_UNRESOLVED_CALL_COVERAGE_VERSION, + .hash_records_complete = true}; + ASSERT_EQ(cbm_store_coverage_replace_ex(st, proj, &row, 1, &meta), CBM_STORE_OK); + + out = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_path\",\"arguments\":{\"project\":" + "\"unresolved-trace\",\"function_name\":\"run\",\"direction\":\"outbound\"," + "\"format\":\"json\"}}}"); + txt = extract_text_content(out); + ASSERT_NOT_NULL(strstr(txt, "\"callees_total_relation\":\"unknown\"")); + free(txt); + free(out); + out = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_path\",\"arguments\":{\"project\":" + "\"unresolved-trace\",\"function_name\":\"buscar\",\"direction\":\"inbound\"," + "\"format\":\"json\"}}}"); + txt = extract_text_content(out); + ASSERT_NOT_NULL(strstr(txt, "\"callers_total_relation\":\"unknown\"")); + free(txt); + free(out); + out = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":3,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"check_index_coverage\",\"arguments\":{\"project\":" + "\"unresolved-trace\",\"paths\":[\"service.js\"],\"diagnostics\":\"full\"," + "\"format\":\"json\"}}}"); + txt = extract_text_content(out); + ASSERT_NOT_NULL(strstr(txt, "\"status\":\"unresolved_calls\"")); + ASSERT_NOT_NULL(strstr(txt, "method_not_in_registry")); + free(txt); + free(out); + out = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":4,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_path\",\"arguments\":{\"project\":" + "\"unresolved-trace\",\"function_name\":\"run\",\"direction\":\"outbound\"," + "\"edge_types\":[\"IMPORTS\"],\"format\":\"json\"}}}"); + txt = extract_text_content(out); + ASSERT_NOT_NULL(strstr(txt, "\"callees_total_relation\":\"eq\"")); + free(txt); + free(out); + cbm_mcp_server_free(srv); + PASS(); +} + /* Edge-data lookup must follow the traversal direction and shortest-path hop. * The induced edge set contains a same-hop cross edge whose source is the row * node and sorts before its real inbound predecessor. An arbitrary incident- @@ -20640,6 +20740,7 @@ SUITE(mcp) { RUN_TEST(tool_trace_call_path_prefers_definition); RUN_TEST(trace_evidence_strategy_class_vocabulary_is_closed); RUN_TEST(tool_trace_path_evidence_is_opt_in_and_class_mapped); + RUN_TEST(tool_trace_path_marks_unresolved_call_totals_unknown); RUN_TEST(tool_trace_path_evidence_columns_match_header_issue1542); RUN_TEST(tool_trace_path_unreadable_confidence_reports_not_recorded); RUN_TEST(tool_trace_path_edge_details_use_canonical_predecessor); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 568740f17..9a072cd83 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -765,6 +765,98 @@ TEST(pipeline_adr_survives_full_reindex) { PASS(); } +TEST(pipeline_records_unresolved_injected_call_sites) { + char tmp[256] = "/tmp/cbm_unresolved_calls_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char db_path[512], path[512]; + snprintf(db_path, sizeof(db_path), "%s/test.db", tmp); + snprintf(path, sizeof(path), "%s/cliente.js", tmp); + FILE *f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fputs("export function crearCliente() { function buscar(id) { return id; } " + "return { buscar }; }\n", + f); + fclose(f); + snprintf(path, sizeof(path), "%s/servicio.js", tmp); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fputs("export function crearServicio({ cliente }) {\n" + " function procesar(id) { return cliente.buscar(id); }\n" + " return { procesar };\n}\n", + f); + fclose(f); + + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + cbm_store_t *st = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(st); + cbm_coverage_row_t *rows = NULL; + int count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(st, project, "servicio.js", &rows, &count), CBM_STORE_OK); + bool found = false; + for (int i = 0; i < count; i++) { + if (strcmp(rows[i].kind, "unresolved_calls") == 0 && strstr(rows[i].detail, "buscar") && + strstr(rows[i].detail, "method_not_in_registry")) { + found = true; + } + } + ASSERT_TRUE(found); + cbm_store_free_coverage(rows, count); + cbm_store_close(st); + + /* Reindexing another file must retain this file's diagnostic. */ + snprintf(path, sizeof(path), "%s/cliente.js", tmp); + f = fopen(path, "a"); + ASSERT_NOT_NULL(f); + fputs("\n// unrelated edit\n", f); + fclose(f); + p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + st = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(st); + rows = NULL; + count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(st, project, "servicio.js", &rows, &count), CBM_STORE_OK); + found = false; + for (int i = 0; i < count; i++) { + found |= strcmp(rows[i].kind, "unresolved_calls") == 0; + } + ASSERT_TRUE(found); + cbm_store_free_coverage(rows, count); + cbm_store_close(st); + + /* The edited file must lose its old diagnostic on the next generation. */ + snprintf(path, sizeof(path), "%s/servicio.js", tmp); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fputs( + "export function crearServicio({ cliente }) { return { procesar(id) { return id; } }; }\n", + f); + fclose(f); + p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + st = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(st); + rows = NULL; + count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(st, project, "servicio.js", &rows, &count), CBM_STORE_OK); + for (int i = 0; i < count; i++) { + ASSERT_FALSE(strcmp(rows[i].kind, "unresolved_calls") == 0); + } + cbm_store_free_coverage(rows, count); + cbm_store_close(st); + rm_rf(tmp); + PASS(); +} + TEST(pipeline_structure_edges) { if (setup_test_repo() != 0) { FAIL("failed to create temp dir"); @@ -15846,6 +15938,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_committed_counts_match_persisted); RUN_TEST(pipeline_adr_survives_full_reindex); RUN_TEST(pipeline_export_error_snapshot_on_artifact_failure); + RUN_TEST(pipeline_records_unresolved_injected_call_sites); RUN_TEST(pipeline_structure_edges); RUN_TEST(pipeline_branch_root_structure); RUN_TEST(pipeline_project_name_derived); From 3f7f4d78b572f6f2497763e150584b108d51e950 Mon Sep 17 00:00:00 2001 From: Pcristin Date: Mon, 28 Sep 2026 14:57:07 +0000 Subject: [PATCH 058/217] fix(pipeline): reconcile unresolved call coverage Signed-off-by: Pcristin --- internal/cbm/cbm.h | 4 + src/mcp/mcp.c | 252 +++++++++++++++--------- src/pipeline/pass_calls.c | 23 ++- src/pipeline/pass_parallel.c | 83 +++++--- src/pipeline/pipeline.c | 285 ++++++++++++++++++++++------ src/pipeline/pipeline_incremental.c | 13 -- src/store/store.h | 6 +- tests/test_mcp.c | 75 +++++++- tests/test_pipeline.c | 105 ++++++++++ 9 files changed, 641 insertions(+), 205 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 1b5d3f3ef..9b49a3fe4 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -294,6 +294,10 @@ typedef struct { // (self.compiler.apply_converters()). An object the // class owns, not a parameter: read by the weak-member // guard's unique-name exemption. Default false. + /* Ephemeral coverage evidence, set only while resolving this file. Capture + * it before releasing a spilled result; never retain registry pointers. */ + bool coverage_calls_emitted; + const char *coverage_candidate_qn; } CBMCall; typedef struct { diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index f3e700aea..a8acb0b29 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -9046,6 +9046,147 @@ static int clamp_mcp_depth(int depth, const char *tool) { return depth; } +typedef struct { + const cbm_node_t *roots; + int root_count; + const cbm_traverse_result_t *outbound; + const cbm_traverse_result_t *inbound; + bool include_tests; + bool do_outbound; + bool do_inbound; +} trace_coverage_scope_t; + +typedef struct { + CBMHashTable *callers; + CBMHashTable *targets; + bool unresolved_out; + bool unresolved_in; +} trace_coverage_match_t; + +static bool trace_coverage_add_node(CBMHashTable *table, const cbm_node_t *node) { + if (!table || !node->qualified_name) { + return true; + } + cbm_ht_set(table, node->qualified_name, (void *)node); + return cbm_ht_has(table, node->qualified_name); +} + +static bool trace_coverage_add_visited(CBMHashTable *table, const cbm_traverse_result_t *tr, + bool include_tests) { + if (!table) { + return true; + } + for (int i = 0; i < tr->visited_count; i++) { + const cbm_node_t *node = &tr->visited[i].node; + if ((include_tests || !is_test_file(node->file_path)) && + !trace_coverage_add_node(table, node)) { + return false; + } + } + return true; +} + +static bool trace_coverage_build_sets(trace_coverage_match_t *match, + const trace_coverage_scope_t *scope) { + for (int i = 0; i < scope->root_count; i++) { + if (!trace_coverage_add_node(match->callers, &scope->roots[i]) || + !trace_coverage_add_node(match->targets, &scope->roots[i])) { + return false; + } + } + return trace_coverage_add_visited(match->callers, scope->outbound, scope->include_tests) && + trace_coverage_add_visited(match->targets, scope->inbound, scope->include_tests); +} + +static void trace_coverage_match_site(trace_coverage_match_t *match, const char *rel_path, + yyjson_val *site) { + const char *caller = yyjson_get_str(yyjson_obj_get(site, "caller")); + const char *candidate = yyjson_get_str(yyjson_obj_get(site, "candidate")); + const cbm_node_t *caller_node = + match->callers && caller ? cbm_ht_get(match->callers, caller) : NULL; + int line = yyjson_get_int(yyjson_obj_get(site, "line")); + /* The producer joins the exact byte span to the extractor's innermost + * caller. Check its file and source range too; never match a containing + * factory merely because its range overlaps. */ + if (caller_node && caller_node->file_path && strcmp(caller_node->file_path, rel_path) == 0 && + (line <= 0 || (line >= caller_node->start_line && line <= caller_node->end_line))) { + match->unresolved_out = true; + } + /* A short-name collision is not evidence that the resolver considered + * this traced target. */ + if (match->targets && candidate && cbm_ht_has(match->targets, candidate)) { + match->unresolved_in = true; + } +} + +static void trace_coverage_match_row(trace_coverage_match_t *match, const cbm_coverage_row_t *row, + const trace_coverage_scope_t *scope) { + yyjson_doc *detail = yyjson_read(row->detail, strlen(row->detail), 0); + yyjson_val *sites = detail ? yyjson_doc_get_root(detail) : NULL; + if (!yyjson_is_arr(sites)) { + match->unresolved_out = scope->do_outbound; + match->unresolved_in = scope->do_inbound; + } else { + size_t idx; + size_t max; + yyjson_val *site; + yyjson_arr_foreach(sites, idx, max, site) { + trace_coverage_match_site(match, row->rel_path, site); + } + } + if (detail) { + yyjson_doc_free(detail); + } +} + +static bool trace_coverage_scan_rows(cbm_store_t *store, const char *project, + trace_coverage_match_t *match, + const trace_coverage_scope_t *scope) { + cbm_coverage_row_t *rows = NULL; + int row_count = 0; + if (cbm_store_coverage_get_unresolved_calls(store, project, &rows, &row_count) != + CBM_STORE_OK) { + return false; + } + for (int i = 0; i < row_count; i++) { + if ((!scope->do_outbound || match->unresolved_out) && + (!scope->do_inbound || match->unresolved_in)) { + break; + } + trace_coverage_match_row(match, &rows[i], scope); + } + cbm_store_free_coverage(rows, row_count); + return true; +} + +static bool trace_coverage_meta_available(cbm_store_t *store, const char *project) { + cbm_coverage_meta_t meta = {0}; + int rc = cbm_store_coverage_meta_get(store, project, &meta); + bool available = rc == CBM_STORE_OK && + meta.coverage_version >= CBM_UNRESOLVED_CALL_COVERAGE_VERSION && + meta.recording_status && strcmp(meta.recording_status, "unavailable") != 0; + cbm_store_coverage_meta_clear(&meta); + return available; +} + +static void trace_call_coverage(cbm_store_t *store, const char *project, + const trace_coverage_scope_t *scope, bool includes_calls, + bool *unresolved_out, bool *unresolved_in) { + if (!includes_calls) { + return; + } + trace_coverage_match_t match = {.callers = scope->do_outbound ? cbm_ht_create(0) : NULL, + .targets = scope->do_inbound ? cbm_ht_create(0) : NULL}; + bool ok = trace_coverage_meta_available(store, project) && + (!scope->do_outbound || match.callers) && (!scope->do_inbound || match.targets) && + trace_coverage_build_sets(&match, scope) && + trace_coverage_scan_rows(store, project, &match, scope); + *unresolved_out = ok ? match.unresolved_out : scope->do_outbound; + *unresolved_in = ok ? match.unresolved_in : scope->do_inbound; + cbm_ht_free(match.callers); + cbm_ht_free(match.targets); +} + static char *handle_trace_call_path(cbm_mcp_server_t *srv, const char *args) { char *func_name = cbm_mcp_get_string_arg(args, "function_name"); char *project = get_project_arg(args); @@ -9381,90 +9522,15 @@ static char *handle_trace_call_path(cbm_mcp_server_t *srv, const char *args) { } bool unresolved_out = false; bool unresolved_in = false; - if (trace_includes_calls) { - cbm_coverage_meta_t coverage_meta = {0}; - int meta_rc = cbm_store_coverage_meta_get(store, project, &coverage_meta); - if (meta_rc == CBM_STORE_OK) { - if (coverage_meta.coverage_version < CBM_UNRESOLVED_CALL_COVERAGE_VERSION || - !coverage_meta.recording_status || - strcmp(coverage_meta.recording_status, "unavailable") == 0) { - unresolved_out = do_outbound; - unresolved_in = do_inbound; - } - } else { - unresolved_out = do_outbound; - unresolved_in = do_inbound; - } - cbm_store_coverage_meta_clear(&coverage_meta); - if ((do_outbound && !unresolved_out) || (do_inbound && !unresolved_in)) { - CBMHashTable *callers = do_outbound ? cbm_ht_create(0) : NULL; - CBMHashTable *leaves = do_inbound ? cbm_ht_create(0) : NULL; - if ((do_outbound && !callers) || (do_inbound && !leaves)) { - unresolved_out = do_outbound; - unresolved_in = do_inbound; - } else { - bool key_set_failed = false; - for (int i = 0; i < node_count; i++) { - if (callers && nodes[i].qualified_name) { - cbm_ht_set(callers, nodes[i].qualified_name, (void *)1); - key_set_failed |= !cbm_ht_has(callers, nodes[i].qualified_name); - } - if (leaves && nodes[i].name) { - cbm_ht_set(leaves, nodes[i].name, (void *)1); - key_set_failed |= !cbm_ht_has(leaves, nodes[i].name); - } - } - for (int i = 0; callers && i < tr_out.visited_count; i++) { - const cbm_node_t *node = &tr_out.visited[i].node; - if ((include_tests || !is_test_file(node->file_path)) && node->qualified_name) { - cbm_ht_set(callers, node->qualified_name, (void *)1); - key_set_failed |= !cbm_ht_has(callers, node->qualified_name); - } - } - for (int i = 0; leaves && i < tr_in.visited_count; i++) { - const cbm_node_t *node = &tr_in.visited[i].node; - if ((include_tests || !is_test_file(node->file_path)) && node->name) { - cbm_ht_set(leaves, node->name, (void *)1); - key_set_failed |= !cbm_ht_has(leaves, node->name); - } - } - cbm_coverage_row_t *rows = NULL; - int row_count = 0; - if (key_set_failed || cbm_store_coverage_get_unresolved_calls( - store, project, &rows, &row_count) != CBM_STORE_OK) { - unresolved_out = do_outbound; - unresolved_in = do_inbound; - } else { - for (int i = 0; i < row_count && ((!unresolved_out && do_outbound) || - (!unresolved_in && do_inbound)); - i++) { - yyjson_doc *detail = yyjson_read(rows[i].detail, strlen(rows[i].detail), 0); - yyjson_val *sites = detail ? yyjson_doc_get_root(detail) : NULL; - if (!yyjson_is_arr(sites)) { - unresolved_out = do_outbound; - unresolved_in = do_inbound; - } else { - size_t idx, max; - yyjson_val *site; - yyjson_arr_foreach(sites, idx, max, site) { - const char *caller = yyjson_get_str(yyjson_obj_get(site, "caller")); - const char *leaf = yyjson_get_str(yyjson_obj_get(site, "leaf")); - if (callers && caller && cbm_ht_has(callers, caller)) - unresolved_out = true; - if (leaves && leaf && cbm_ht_has(leaves, leaf)) - unresolved_in = true; - } - } - if (detail) - yyjson_doc_free(detail); - } - } - cbm_store_free_coverage(rows, row_count); - } - cbm_ht_free(callers); - cbm_ht_free(leaves); - } - } + trace_coverage_scope_t coverage_scope = {.roots = nodes, + .root_count = node_count, + .outbound = &tr_out, + .inbound = &tr_in, + .include_tests = include_tests, + .do_outbound = do_outbound, + .do_inbound = do_inbound}; + trace_call_coverage(store, project, &coverage_scope, trace_includes_calls, &unresolved_out, + &unresolved_in); const char *out_relation = unresolved_out ? "unknown" : (tr_out.truncated ? "gte" : "eq"); const char *in_relation = unresolved_in ? "unknown" : (tr_in.truncated ? "gte" : "eq"); @@ -10215,11 +10281,12 @@ static bool is_parse_unusable(const cbm_file_error_t *e) { return e->phase && strcmp(e->phase, "parse_unusable") == 0; } -/* Either coverage phase. Both mean the file WAS indexed, so both must stay out +/* All coverage phases mean the file WAS indexed, so they must stay out * of skipped[] — a reader who sees a file there believes it is absent from the * graph entirely. */ -static bool is_parse_coverage(const cbm_file_error_t *e) { - return is_parse_partial(e) || is_parse_unusable(e); +static bool is_indexed_coverage(const cbm_file_error_t *e) { + return is_parse_partial(e) || is_parse_unusable(e) || + (e->phase && strcmp(e->phase, "unresolved_calls") == 0); } /* Attach a summary of per-file skips (Stage 2 / Track B). Always emits a @@ -10229,17 +10296,20 @@ static bool is_parse_coverage(const cbm_file_error_t *e) { * and, if a per-run logfile was written, "logfile": "". * The run status stays "indexed" — a skipped file is the expected handled * outcome, not a failure. errs[] is borrowed (copied into doc) and may contain - * parse_partial and parse_unusable entries, which are filtered out here (both - * reported separately by add_parse_partial_summary). */ + * parse_partial, parse_unusable, and unresolved_calls entries. These describe + * indexed files and are counted separately from skips. */ static void add_skipped_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_file_error_t *errs, int count, const char *logfile) { int skips = 0; + int unresolved = 0; for (int i = 0; i < count; i++) { - if (!is_parse_coverage(&errs[i])) { + unresolved += errs[i].phase && strcmp(errs[i].phase, "unresolved_calls") == 0; + if (!is_indexed_coverage(&errs[i])) { skips++; } } yyjson_mut_obj_add_int(doc, root, "skipped_count", skips); + yyjson_mut_obj_add_int(doc, root, "unresolved_calls_count", unresolved); if (logfile && logfile[0]) { yyjson_mut_obj_add_strcpy(doc, root, "logfile", logfile); } @@ -10250,7 +10320,7 @@ static void add_skipped_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, yyjson_mut_val *files = yyjson_mut_arr(doc); int shown = 0; for (int i = 0; i < count && shown < INDEX_SKIPPED_FILE_CAP; i++) { - if (is_parse_coverage(&errs[i])) { + if (is_indexed_coverage(&errs[i])) { continue; } yyjson_mut_val *fe = yyjson_mut_obj(doc); diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 84958954c..e8257f880 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -205,7 +205,7 @@ static void free_import_map(const char **keys, const char **vals, int count) { } /* Handle a route registration call: create Route node + HANDLES edge. */ -static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *call, +static void handle_route_registration(cbm_pipeline_ctx_t *ctx, CBMCall *call, const cbm_gbuf_node_t *source_node, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count) { const char *method = cbm_service_pattern_route_method(call->callee_name); @@ -225,7 +225,9 @@ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *ca snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}", esc_cn, esc_fa); - cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, route_id, "CALLS", props); + if (cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, route_id, "CALLS", props) > 0) { + call->coverage_calls_emitted = true; + } if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') { cbm_resolution_t hres = cbm_registry_resolve(ctx->registry, call->second_arg_name, module_qn, imp_keys, imp_vals, imp_count); @@ -284,7 +286,7 @@ static int64_t create_svc_route_node(cbm_pipeline_ctx_t *ctx, const char *url, c * this, so data_flow mode had no argument expressions to surface for small * (< 50 file) repos that take the sequential path (#514). Mirrors the parallel * path's append_args_json shape so both pipelines agree. */ -static void calls_append_args(char *props, size_t cap, const CBMCall *call) { +static void calls_append_args(char *props, size_t cap, CBMCall *call) { if (!call || call->arg_count <= 0) { return; } @@ -331,7 +333,7 @@ static void calls_append_args(char *props, size_t cap, const CBMCall *call) { } static void calls_emit_edge(cbm_gbuf_t *gbuf, int64_t src, int64_t tgt, const char *type, - char *props, size_t cap, const CBMCall *call) { + char *props, size_t cap, CBMCall *call) { if (call && call->start_line > 0 && strcmp(type, "CALLS") == 0) { size_t len = strlen(props); if (len >= SKIP_ONE && props[len - SKIP_ONE] == '}' && len + CBM_SZ_32 < cap) { @@ -342,10 +344,13 @@ static void calls_emit_edge(cbm_gbuf_t *gbuf, int64_t src, int64_t tgt, const ch if (call && strcmp(type, "CALLS") == 0) { calls_append_args(props, cap, call); } - cbm_gbuf_insert_edge(gbuf, src, tgt, type, props); + if (cbm_gbuf_insert_edge(gbuf, src, tgt, type, props) > 0 && call && + strcmp(type, "CALLS") == 0) { + call->coverage_calls_emitted = true; + } } -static void emit_http_async_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, +static void emit_http_async_edge(cbm_pipeline_ctx_t *ctx, CBMCall *call, const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target, const cbm_resolution_t *res, cbm_svc_kind_t svc, bool suppress_plain_calls) { @@ -414,7 +419,7 @@ static void emit_http_async_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, * match, #592/#606), the route/HTTP/ASYNC/CONFIG service classifications below * still run — only the plain CALLS fall-through is skipped, so a fabricated * project edge is dropped while every service edge stays main-identical. */ -static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, +static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, CBMCall *call, const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target, const cbm_resolution_t *res, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, @@ -504,7 +509,8 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, res.qualified_name = target_node->qualified_name; res.confidence = lsp->confidence; res.strategy = lsp->strategy; - res.candidate_count = 1; + res.candidate_count = CBM_ALLOC_ONE; + call->coverage_candidate_qn = target_node->qualified_name; emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, imp_count, false); return SKIP_ONE; @@ -682,6 +688,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, if (cbm_suppress_cross_language_suffix_match(lang, target_node->file_path, res.strategy)) { return 0; } + call->coverage_candidate_qn = target_node->qualified_name; emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, imp_count, drop_plain_call); return SKIP_ONE; diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index e85ad015f..e081c3c3a 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -1916,7 +1916,7 @@ static int format_call_arg(char *buf, size_t bufsize, const CBMCallArg *a, const return snprintf(buf, bufsize, "{\"i\":%d,\"e\":\"%s\"}", a->index, esc_e); } -static size_t append_args_json(char *buf, size_t bufsize, size_t pos, const CBMCall *call) { +static size_t append_args_json(char *buf, size_t bufsize, size_t pos, CBMCall *call) { if (call->arg_count == 0 || pos >= bufsize - PP_ARGS_MARGIN) { return pos; } @@ -1972,7 +1972,7 @@ static bool is_route_path_shaped(const char *val) { return val && val[0] == '/' && !cbm_service_pattern_is_comment_text(val); } -static const char *find_route_path_in_args(const CBMCall *call, const char **out_handler) { +static const char *find_route_path_in_args(CBMCall *call, const char **out_handler) { *out_handler = NULL; /* 1. First string arg starting with / */ if (is_route_path_shaped(call->first_string_arg)) { @@ -2012,7 +2012,7 @@ static const char *find_route_path_in_args(const CBMCall *call, const char **out /* Build props JSON, append args, close brace, emit edge. */ static void finalize_and_emit(cbm_gbuf_t *gbuf, int64_t src_id, int64_t tgt_id, - const char *edge_type, char *props, int n, const CBMCall *call) { + const char *edge_type, char *props, int n, CBMCall *call) { if (n > 0 && (size_t)n < CBM_SZ_2K - PP_ESC_SPACE) { size_t pos = append_args_json(props, CBM_SZ_2K, (size_t)n, call); if (call->start_line > 0 && strcmp(edge_type, "CALLS") == 0 && @@ -2027,7 +2027,10 @@ static void finalize_and_emit(cbm_gbuf_t *gbuf, int64_t src_id, int64_t tgt_id, props[pos + SKIP_ONE] = '\0'; } } - cbm_gbuf_insert_edge(gbuf, src_id, tgt_id, edge_type, props); + if (cbm_gbuf_insert_edge(gbuf, src_id, tgt_id, edge_type, props) > 0 && + strcmp(edge_type, "CALLS") == 0) { + call->coverage_calls_emitted = true; + } } /* Build Route node QN and properties for HTTP/async service edges. */ @@ -2057,7 +2060,7 @@ static int64_t build_service_route(cbm_gbuf_t *gbuf, const char *arg, const char /* Emit HTTP_CALLS or ASYNC_CALLS edge via Route node. */ static void emit_http_async_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const CBMCall *call, const cbm_resolution_t *res, + CBMCall *call, const cbm_resolution_t *res, cbm_svc_kind_t svc, const char *arg) { const char *edge_type = (svc == CBM_SVC_HTTP) ? "HTTP_CALLS" : "ASYNC_CALLS"; const char *method = @@ -2084,7 +2087,7 @@ static void emit_http_async_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t /* Emit CONFIGURES edge. */ static void emit_config_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const cbm_gbuf_node_t *target, const CBMCall *call, + const cbm_gbuf_node_t *target, CBMCall *call, const cbm_resolution_t *res, const char *arg) { /* emit_service_edge may be reached with target==NULL on the HTTP/ASYNC * external-client bypass (#523); a CONFIGURES edge needs a real target, so @@ -2104,7 +2107,7 @@ static void emit_config_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, /* Emit normal CALLS edge. */ static void emit_normal_calls_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const cbm_gbuf_node_t *target, const CBMCall *call, + const cbm_gbuf_node_t *target, CBMCall *call, const cbm_resolution_t *res) { /* A CALLS edge needs a real target; the HTTP/ASYNC external-client bypass * (#523) can reach emit_service_edge with target==NULL, so guard the deref. */ @@ -2123,11 +2126,11 @@ static void emit_normal_calls_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sour /* Classify a resolved call by library identity and emit the appropriate edge. */ /* Create Route node + CALLS + HANDLES edges for a route registration call. */ -static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const CBMCall *call, const char *route_path, - const char *handler_ref, const char *module_qn, - const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, - const char **ik, const char **iv, int ic) { +static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMCall *call, + const char *route_path, const char *handler_ref, + const char *module_qn, const cbm_registry_t *registry, + const cbm_gbuf_t *main_gbuf, const char **ik, const char **iv, + int ic) { const char *method = cbm_service_pattern_route_method(call->callee_name); char rqn[CBM_ROUTE_QN_SIZE]; char cpath[CBM_SZ_256]; @@ -2144,7 +2147,9 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}", esc_cn, esc_rp); - cbm_gbuf_insert_edge(gbuf, source->id, rid, "CALLS", props); + if (cbm_gbuf_insert_edge(gbuf, source->id, rid, "CALLS", props) > 0) { + call->coverage_calls_emitted = true; + } if (handler_ref && handler_ref[0] != '\0') { cbm_resolution_t hres = cbm_registry_resolve(registry, handler_ref, module_qn, ik, iv, ic); if (hres.qualified_name && hres.qualified_name[0] != '\0') { @@ -2212,8 +2217,7 @@ static bool normalize_url_arg(const char *url, char *norm, int norm_sz) { } /* Detect API paths in call arguments and create HTTP_CALLS edges. */ -static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const CBMCall *call) { +static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMCall *call) { for (int ai = 0; ai < call->arg_count; ai++) { const CBMCallArg *ca = &call->args[ai]; /* A slash-prefixed raw expression is not a URL string. In JS/TS this @@ -2318,7 +2322,7 @@ bool extract_grpc_service_method(const char *callee, char *service, size_t srv_s } /* Emit GRPC_CALLS edge via gRPC Route node. */ -static void emit_grpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call, +static void emit_grpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMCall *call, const cbm_resolution_t *res) { char service[CBM_SZ_256]; char method[CBM_SZ_256]; @@ -2355,7 +2359,7 @@ static void emit_grpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, cons } /* Emit GRAPHQL_CALLS edge. Extract operation from first string arg if available. */ -static void emit_graphql_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call, +static void emit_graphql_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMCall *call, const cbm_resolution_t *res) { const char *op = call->first_string_arg; if (!op || !op[0]) { @@ -2392,7 +2396,7 @@ static void emit_graphql_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, c } /* Emit TRPC_CALLS edge. Extract procedure path from callee chain. */ -static void emit_trpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call, +static void emit_trpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMCall *call, const cbm_resolution_t *res) { /* tRPC calls: trpc.user.getById.query() → extract "user.getById" */ const char *callee = call->callee_name; @@ -2435,7 +2439,7 @@ static void emit_trpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, cons * a verb-suffix HTTP client (api.patch('/x')), broker, or route registration * keeps its edge; only the fabricated project CALLS edge is dropped. */ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const cbm_gbuf_node_t *target, const CBMCall *call, + const cbm_gbuf_node_t *target, CBMCall *call, const cbm_resolution_t *res, const char *module_qn, const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, const char **imp_keys, const char **imp_vals, int imp_count, @@ -2692,7 +2696,7 @@ static bool lsp_idx_insert_leaf(CBMHashTable *index, CBMResolvedCall *candidate, return inserted; } -static const CBMResolvedCall *lsp_idx_lookup(const CBMHashTable *index, const CBMCall *call, +static const CBMResolvedCall *lsp_idx_lookup(const CBMHashTable *index, CBMCall *call, bool exact_site, bool *key_built, bool *ambiguous) { if (key_built) { *key_built = false; @@ -3049,6 +3053,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB } continue; } + call->coverage_candidate_qn = target_node->qualified_name; _rc_t0 = extract_now_ns(); emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, @@ -3449,6 +3454,28 @@ static bool pp_has_pending_lsp_site(const CBMFileResult *result) { return false; } +static CBMFileResult *resolve_load_result(resolve_ctx_t *rc, resolve_worker_state_t *ws, + int file_idx) { + if (ws->loaded) { + cbm_free_result(ws->loaded); + ws->loaded = NULL; + } + CBMFileResult *result = rc->result_cache[file_idx]; + if (!result && rc->pctx && rc->pctx->spill && cbm_result_spill_has(rc->pctx->spill, file_idx)) { + result = cbm_result_spill_load(rc->pctx->spill, file_idx); + ws->loaded = result; + if (!result) { + cbm_pipeline_mark_unresolved_capture_failed(rc->pctx->pipeline); + } + } + return result; +} + +static void resolve_capture_coverage(const resolve_ctx_t *rc, const char *rel, + const CBMFileResult *result) { + cbm_pipeline_record_unresolved_calls(rc->pctx ? rc->pctx->pipeline : NULL, rel, result); +} + static void resolve_worker(int worker_id, void *ctx_ptr) { resolve_ctx_t *rc = ctx_ptr; resolve_worker_state_t *ws = &rc->workers[worker_id]; @@ -3482,16 +3509,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { uint64_t _loop_t0 = extract_now_ns(); - if (ws->loaded) { - cbm_free_result(ws->loaded); - ws->loaded = NULL; - } - CBMFileResult *result = rc->result_cache[file_idx]; - if (!result && rc->pctx && rc->pctx->spill && - cbm_result_spill_has(rc->pctx->spill, file_idx)) { - result = cbm_result_spill_load(rc->pctx->spill, file_idx); - ws->loaded = result; - } + CBMFileResult *result = resolve_load_result(rc, ws, file_idx); if (!result) { atomic_fetch_add_explicit(&rc->time_ns_total_loop, extract_now_ns() - _loop_t0, memory_order_relaxed); @@ -3547,6 +3565,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { if (result->calls.count == 0 && result->usages.count == 0 && result->throws.count == 0 && result->rw.count == 0 && result->defs.count == 0 && result->impl_traits.count == 0 && !cross_lsp_eligible) { + resolve_capture_coverage(rc, rel, result); continue; } @@ -3695,6 +3714,10 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { atomic_fetch_add_explicit(&rc->time_ns_semantic, extract_now_ns() - _ph_t0, memory_order_relaxed); + /* The cross-file additions and actual emitted-site evidence live in + * this result, including when it was loaded from the spill store. */ + resolve_capture_coverage(rc, rel, result); + cbm_registry_reach_cache_end(); cbm_registry_import_map_cache_end(); cbm_registry_resolve_cache_end(); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 9d12c55d9..3e6952490 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -42,6 +42,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include "foundation/mem.h" #include "foundation/mem_core.h" #include "result_spill.h" +#include "cbm.h" #include "foundation/secure_random.h" #include "yyjson/yyjson.h" @@ -265,7 +266,9 @@ struct cbm_pipeline { /* Per-file unresolved invocation diagnostics, published as coverage rows. */ cbm_coverage_row_t *unresolved_rows; int unresolved_count; - bool unresolved_capture_failed; + int unresolved_capacity; + atomic_bool unresolved_capture_failed; + cbm_mutex_t unresolved_mutex; /* one append per file, after parallel resolution */ /* Deterministic test-only seam at the final publication boundary. Kept * per pipeline so concurrent test/process activity cannot cross-trigger. */ @@ -360,6 +363,8 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, p->committed_nodes = -1; p->committed_edges = -1; atomic_init(&p->cancelled_storage, 0); + atomic_init(&p->unresolved_capture_failed, false); + cbm_mutex_init(&p->unresolved_mutex); p->cancelled = &p->cancelled_storage; return p; @@ -454,63 +459,236 @@ static void unresolved_json_free(void *ctx, void *ptr) { cbm_free(CBM_MEM_CLASS_DUMP, ptr); } -void cbm_pipeline_record_unresolved_calls(cbm_pipeline_t *p, const char *rel_path, - const CBMFileResult *result) { - if (!p || !rel_path || !result || result->resolved_calls.count == 0 || - p->unresolved_capture_failed) { - return; +enum { UNRESOLVED_KEY_OVERHEAD = 128, UNRESOLVED_INITIAL_ROWS = 64, UNRESOLVED_ROW_GROWTH = 2 }; + +static char *unresolved_site_key(const char *caller, const char *leaf, uint32_t start, uint32_t end, + CBMSourceOrigin origin) { + size_t cn = strlen(caller); + size_t ln = strlen(leaf); + size_t cap = cn + ln + UNRESOLVED_KEY_OVERHEAD; + char *key = cbm_alloc(CBM_MEM_CLASS_DUMP, cap); + if (key) { + (void)snprintf(key, cap, "%zu:%s|%zu:%s|%u:%u:%u", cn, caller, ln, leaf, start, end, + (unsigned)origin); } - yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - if (!doc) { - p->unresolved_capture_failed = true; - return; + return key; +} + +typedef struct { + const CBMCall *call; +} unresolved_carrier_t; + +typedef struct { + yyjson_mut_doc *doc; + yyjson_mut_val *sites; + CBMHashTable *carriers; + CBMHashTable *seen; +} unresolved_capture_t; + +static void unresolved_carrier_free(const char *key, void *value, void *ctx) { + (void)ctx; + cbm_free(CBM_MEM_CLASS_DUMP, (void *)key); + cbm_free(CBM_MEM_CLASS_DUMP, value); +} + +static void unresolved_key_free(const char *key, void *value, void *ctx) { + (void)value; + (void)ctx; + cbm_free(CBM_MEM_CLASS_DUMP, (void *)key); +} + +static void unresolved_capture_free(unresolved_capture_t *capture) { + if (capture->carriers) { + cbm_ht_foreach(capture->carriers, unresolved_carrier_free, NULL); + cbm_ht_free(capture->carriers); } - yyjson_mut_val *sites = yyjson_mut_arr(doc); - yyjson_mut_doc_set_root(doc, sites); - int count = 0; + if (capture->seen) { + cbm_ht_foreach(capture->seen, unresolved_key_free, NULL); + cbm_ht_free(capture->seen); + } + if (capture->doc) { + yyjson_mut_doc_free(capture->doc); + } +} + +static bool unresolved_call_is_missing(const CBMResolvedCall *rc) { + return rc->kind == CBM_RESOLVED_INVOCATION && rc->strategy && + strcmp(rc->strategy, "lsp_unresolved") == 0 && rc->caller_qn && rc->callee_qn; +} + +static bool unresolved_has_missing_calls(const CBMFileResult *result) { + for (int i = 0; i < result->resolved_calls.count; i++) { + if (unresolved_call_is_missing(&result->resolved_calls.items[i])) { + return true; + } + } + return false; +} + +static bool unresolved_add_carrier(CBMHashTable *carriers, const CBMCall *call) { + if (!call->callee_name || call->site_end_byte <= call->site_start_byte) { + return true; + } + char *key = + unresolved_site_key("", cbm_lsp_bare_segment(call->callee_name), call->site_start_byte, + call->site_end_byte, call->source_origin); + if (!key) { + return false; + } + if (cbm_ht_has(carriers, key)) { + /* Ambiguous extractor occurrences must never erase a diagnostic. */ + unresolved_carrier_t *carrier = cbm_ht_get(carriers, key); + carrier->call = NULL; + cbm_free(CBM_MEM_CLASS_DUMP, key); + return true; + } + unresolved_carrier_t *carrier = cbm_alloc(CBM_MEM_CLASS_DUMP, sizeof(*carrier)); + if (!carrier) { + cbm_free(CBM_MEM_CLASS_DUMP, key); + return false; + } + carrier->call = call; + cbm_ht_set(carriers, key, carrier); + if (!cbm_ht_has(carriers, key)) { + cbm_free(CBM_MEM_CLASS_DUMP, key); + cbm_free(CBM_MEM_CLASS_DUMP, carrier); + return false; + } + return true; +} + +static bool unresolved_build_carriers(CBMHashTable *carriers, const CBMFileResult *result) { + /* Caller names in the TS LSP walk can denote an outer factory. The + * extractor's exact occurrence is the authoritative enclosing function. + * Index by span AND leaf: two receiver calls on one line remain distinct. */ + for (int i = 0; i < result->calls.count; i++) { + if (!unresolved_add_carrier(carriers, &result->calls.items[i])) { + return false; + } + } + return true; +} + +static bool unresolved_append_site(unresolved_capture_t *capture, const CBMResolvedCall *rc, + const CBMCall *call, const char *caller, const char *leaf) { + yyjson_mut_doc *doc = capture->doc; + yyjson_mut_val *site = yyjson_mut_obj(doc); + bool ok = + site && yyjson_mut_obj_add_strcpy(doc, site, "caller", caller) && + yyjson_mut_obj_add_strcpy(doc, site, "leaf", leaf) && + yyjson_mut_obj_add_uint(doc, site, "start_byte", rc->site_start_byte) && + yyjson_mut_obj_add_uint(doc, site, "end_byte", rc->site_end_byte) && + yyjson_mut_obj_add_uint(doc, site, "source_origin", rc->source_origin) && + yyjson_mut_obj_add_strcpy(doc, site, "reason", rc->reason ? rc->reason : "unresolved"); + if (ok && call && call->start_line > 0) { + ok = yyjson_mut_obj_add_int(doc, site, "line", call->start_line); + } + if (ok && call && call->coverage_candidate_qn) { + ok = yyjson_mut_obj_add_strcpy(doc, site, "candidate", call->coverage_candidate_qn); + } + return ok && yyjson_mut_arr_add_val(capture->sites, site); +} + +static bool unresolved_collect_site(unresolved_capture_t *capture, const CBMResolvedCall *rc) { + const char *leaf = cbm_lsp_bare_segment(rc->callee_qn); + char *lookup = + unresolved_site_key("", leaf, rc->site_start_byte, rc->site_end_byte, rc->source_origin); + if (!lookup) { + return false; + } + const unresolved_carrier_t *carrier = cbm_ht_get(capture->carriers, lookup); + const CBMCall *call = carrier ? carrier->call : NULL; + cbm_free(CBM_MEM_CLASS_DUMP, lookup); + /* This exact occurrence emitted a real CALLS edge, possibly via the + * registry after LSP failure. Never suppress a different occurrence. */ + if (call && call->coverage_calls_emitted) { + return true; + } + const char *caller = call && call->enclosing_func_qn ? call->enclosing_func_qn : rc->caller_qn; + char *key = unresolved_site_key(caller, leaf, rc->site_start_byte, rc->site_end_byte, + rc->source_origin); + if (!key) { + return false; + } + if (cbm_ht_has(capture->seen, key)) { + cbm_free(CBM_MEM_CLASS_DUMP, key); + return true; + } + cbm_ht_set(capture->seen, key, key); + if (!cbm_ht_has(capture->seen, key)) { + cbm_free(CBM_MEM_CLASS_DUMP, key); + return false; + } + return unresolved_append_site(capture, rc, call, caller, leaf); +} + +static bool unresolved_collect_sites(unresolved_capture_t *capture, const CBMFileResult *result) { for (int i = 0; i < result->resolved_calls.count; i++) { const CBMResolvedCall *rc = &result->resolved_calls.items[i]; - if (rc->kind != CBM_RESOLVED_INVOCATION || !rc->strategy || - strcmp(rc->strategy, "lsp_unresolved") != 0 || !rc->caller_qn || !rc->callee_qn) { - continue; + if (unresolved_call_is_missing(rc) && !unresolved_collect_site(capture, rc)) { + return false; } - const char *leaf = strrchr(rc->callee_qn, '.'); - leaf = leaf ? leaf + 1 : rc->callee_qn; - yyjson_mut_val *site = yyjson_mut_obj(doc); - yyjson_mut_obj_add_strcpy(doc, site, "caller", rc->caller_qn); - yyjson_mut_obj_add_strcpy(doc, site, "leaf", leaf); - yyjson_mut_obj_add_uint(doc, site, "start_byte", rc->site_start_byte); - yyjson_mut_obj_add_uint(doc, site, "end_byte", rc->site_end_byte); - yyjson_mut_obj_add_strcpy(doc, site, "reason", rc->reason ? rc->reason : "unresolved"); - yyjson_mut_arr_add_val(sites, site); - count++; } - if (count > 0) { - yyjson_alc allocator = {.malloc = unresolved_json_alloc, - .realloc = unresolved_json_realloc, - .free = unresolved_json_free}; - char *detail = yyjson_mut_write_opts(doc, 0, &allocator, NULL, NULL); + return true; +} + +/* Caller holds unresolved_mutex. Ownership transfers only on success. */ +static bool unresolved_append_row(cbm_pipeline_t *p, const char *path, const char *detail) { + if (!detail || !path || p->unresolved_capture_failed) { + return false; + } + if (p->unresolved_count == p->unresolved_capacity) { + int capacity = p->unresolved_capacity ? p->unresolved_capacity * UNRESOLVED_ROW_GROWTH + : UNRESOLVED_INITIAL_ROWS; + cbm_coverage_row_t *rows = + cbm_realloc(CBM_MEM_CLASS_DUMP, p->unresolved_rows, (size_t)capacity * sizeof(*rows)); + if (!rows) { + return false; + } + p->unresolved_rows = rows; + p->unresolved_capacity = capacity; + } + p->unresolved_rows[p->unresolved_count++] = + (cbm_coverage_row_t){.rel_path = path, .kind = "unresolved_calls", .detail = detail}; + return true; +} + +void cbm_pipeline_record_unresolved_calls(cbm_pipeline_t *p, const char *rel_path, + const CBMFileResult *result) { + if (!p || !rel_path || !result || p->unresolved_capture_failed) { + return; + } + if (!unresolved_has_missing_calls(result)) { + return; + } + yyjson_alc allocator = {.malloc = unresolved_json_alloc, + .realloc = unresolved_json_realloc, + .free = unresolved_json_free}; + unresolved_capture_t capture = {.doc = yyjson_mut_doc_new(&allocator), + .carriers = cbm_ht_create(0), + .seen = cbm_ht_create(0)}; + capture.sites = capture.doc ? yyjson_mut_arr(capture.doc) : NULL; + bool ok = capture.doc && capture.carriers && capture.seen && capture.sites; + if (capture.doc) { + yyjson_mut_doc_set_root(capture.doc, capture.sites); + } + ok = ok && unresolved_build_carriers(capture.carriers, result) && + unresolved_collect_sites(&capture, result); + if (ok && yyjson_mut_arr_size(capture.sites) > 0) { + char *detail = yyjson_mut_write_opts(capture.doc, 0, &allocator, NULL, NULL); char *path = cbm_mem_strdup(CBM_MEM_CLASS_DUMP, rel_path); - if (!detail || !path) { - p->unresolved_capture_failed = true; + cbm_mutex_lock(&p->unresolved_mutex); + ok = unresolved_append_row(p, path, detail); + cbm_mutex_unlock(&p->unresolved_mutex); + if (!ok) { cbm_free(CBM_MEM_CLASS_DUMP, detail); cbm_free(CBM_MEM_CLASS_DUMP, path); - } else { - cbm_coverage_row_t *rows = - cbm_realloc(CBM_MEM_CLASS_DUMP, p->unresolved_rows, - (size_t)(p->unresolved_count + 1) * sizeof(*rows)); - if (!rows) { - p->unresolved_capture_failed = true; - cbm_free(CBM_MEM_CLASS_DUMP, detail); - cbm_free(CBM_MEM_CLASS_DUMP, path); - } else { - p->unresolved_rows = rows; - p->unresolved_rows[p->unresolved_count++] = (cbm_coverage_row_t){ - .rel_path = path, .kind = "unresolved_calls", .detail = detail}; - } } } - yyjson_mut_doc_free(doc); + if (!ok) { + p->unresolved_capture_failed = true; + } + unresolved_capture_free(&capture); } void cbm_pipeline_mark_unresolved_capture_failed(cbm_pipeline_t *p) { @@ -563,6 +741,7 @@ void cbm_pipeline_free(cbm_pipeline_t *p) { cbm_free(CBM_MEM_CLASS_DUMP, (char *)p->unresolved_rows[i].detail); } cbm_free(CBM_MEM_CLASS_DUMP, p->unresolved_rows); + cbm_mutex_destroy(&p->unresolved_mutex); cbm_git_context_free(&p->git_ctx); /* gbuf, store, registry freed during/after run */ /* Defensively free userconfig in case run() was never called or panicked */ @@ -1799,18 +1978,6 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, cbm_clock_gettime(CLOCK_MONOTONIC, t); rc = cbm_parallel_resolve(ctx, files, file_count, cache, &shared_ids, worker_count, all_defs, def_count, def_modules, module_def_index, &cross_registries); - if (rc == 0) { - for (int i = 0; i < file_count; i++) { - bool loaded = false; - CBMFileResult *result = cbm_pipeline_result_acquire(ctx, cache, i, NULL, &loaded); - if (result) { - cbm_pipeline_record_unresolved_calls(p, files[i].rel_path, result); - } else if (ctx->spill && cbm_result_spill_has(ctx->spill, i)) { - cbm_pipeline_mark_unresolved_capture_failed(p); - } - cbm_pipeline_result_release(result, loaded); - } - } cbm_log_info("pass.timing", "pass", "parallel_resolve", "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); pipeline_phase_mark("parallel_resolve"); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index f0953764e..9d34ff241 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1352,19 +1352,6 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed rc = cbm_parallel_resolve(ctx, changed_files, ci, cache, &shared_ids, worker_count, all_defs, all_def_count, closure ? closure->def_modules : NULL, module_def_index, registries_arg); - if (rc == 0) { - for (int i = 0; i < ci; i++) { - bool loaded = false; - CBMFileResult *result = cbm_pipeline_result_acquire(ctx, cache, i, NULL, &loaded); - if (result) { - cbm_pipeline_record_unresolved_calls(ctx->pipeline, changed_files[i].rel_path, - result); - } else if (ctx->spill && cbm_result_spill_has(ctx->spill, i)) { - cbm_pipeline_mark_unresolved_capture_failed(ctx->pipeline); - } - cbm_pipeline_result_release(result, loaded); - } - } if (module_def_index) { cbm_pxc_free_module_def_index(module_def_index); } diff --git a/src/store/store.h b/src/store/store.h index 82489255e..2163eed09 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -650,9 +650,9 @@ typedef struct { const char *detail; } cbm_coverage_row_t; -/* Older generations have no unresolved-call records, so their exact trace - * totals cannot be trusted after this signal becomes part of coverage. */ -enum { CBM_UNRESOLVED_CALL_COVERAGE_VERSION = 4 }; +/* Older generations lack precise caller attribution and resolver candidates, + * so their exact trace totals cannot be trusted until the project is reindexed. */ +enum { CBM_UNRESOLVED_CALL_COVERAGE_VERSION = 5 }; /* Metadata describing how completely one index run recorded the best-effort * coverage signal. `recording_status` is "complete", "truncated", or diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 02cb9db01..2f8b41eb2 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -7462,7 +7462,9 @@ TEST(tool_trace_path_marks_unresolved_call_totals_unknown) { .rel_path = "service.js", .kind = "unresolved_calls", .detail = "[{\"caller\":\"unresolved-trace.service.run\",\"leaf\":\"buscar\"," - "\"start_byte\":42,\"end_byte\":59,\"reason\":\"method_not_in_registry\"}]"}; + "\"start_byte\":42,\"end_byte\":59,\"line\":3," + "\"candidate\":\"unresolved-trace.client.buscar\"," + "\"reason\":\"method_not_in_registry\"}]"}; cbm_coverage_meta_t meta = {.generation = "fixture", .index_mode = "full", .recorded_at = "2026-09-24T00:00:00Z", @@ -7489,6 +7491,20 @@ TEST(tool_trace_path_marks_unresolved_call_totals_unknown) { ASSERT_NOT_NULL(strstr(txt, "\"callers_total_relation\":\"unknown\"")); free(txt); free(out); + /* The same leaf in an unrelated module was never a resolver candidate. */ + cbm_node_t unrelated = callee; + unrelated.qualified_name = "unresolved-trace.other.buscar"; + unrelated.file_path = "other.js"; + ASSERT_GT(cbm_store_upsert_node(st, &unrelated), 0); + out = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":20,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_path\",\"arguments\":{\"project\":" + "\"unresolved-trace\",\"function_name\":\"unresolved-trace.other.buscar\"," + "\"direction\":\"inbound\",\"format\":\"json\"}}}"); + txt = extract_text_content(out); + ASSERT_NOT_NULL(strstr(txt, "\"callers_total_relation\":\"eq\"")); + free(txt); + free(out); out = cbm_mcp_server_handle( srv, "{\"jsonrpc\":\"2.0\",\"id\":3,\"method\":\"tools/call\"," "\"params\":{\"name\":\"check_index_coverage\",\"arguments\":{\"project\":" @@ -7512,6 +7528,62 @@ TEST(tool_trace_path_marks_unresolved_call_totals_unknown) { PASS(); } +TEST(tool_index_unresolved_nested_functions_are_indexed) { + char tmp[256] = "/tmp/cbm-mcp-unresolved-XXXXXX"; + char cache[256] = "/tmp/cbm-mcp-unresolved-cache-XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + ASSERT_NOT_NULL(cbm_mkdtemp(cache)); + const char *saved = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved ? strdup(saved) : NULL; + cbm_setenv("CBM_CACHE_DIR", cache, 1); + char path[512]; + snprintf(path, sizeof(path), "%s/service.js", tmp); + FILE *f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fputs("export function factory({ client }) {\n" + " function inner(id) {\n" + " return client.buscar(id);\n" + " }\n return { inner };\n}\n", + f); + fclose(f); + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + char args[1024]; + snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"mode\":\"full\"}", tmp); + char *out = cbm_mcp_handle_tool(srv, "index_repository", args); + ASSERT_NOT_NULL(out); + ASSERT(response_contains_json_fragment(out, "\"status\":\"indexed\"")); + ASSERT(response_contains_json_fragment(out, "\"skipped_count\":0")); + free(out); + char *project = cbm_project_name_from_path(tmp); + ASSERT_NOT_NULL(project); + snprintf(args, sizeof(args), + "{\"project\":\"%s\",\"function_name\":\"inner\"," + "\"direction\":\"outbound\",\"format\":\"json\"}", + project); + out = cbm_mcp_handle_tool(srv, "trace_path", args); + ASSERT_NOT_NULL(out); + ASSERT(response_contains_json_fragment(out, "\"callees_total_relation\":\"unknown\"")); + free(out); + snprintf(args, sizeof(args), + "{\"project\":\"%s\",\"function_name\":\"factory\"," + "\"direction\":\"outbound\",\"format\":\"json\"}", + project); + out = cbm_mcp_handle_tool(srv, "trace_path", args); + ASSERT_NOT_NULL(out); + ASSERT(response_contains_json_fragment(out, "\"callees_total_relation\":\"eq\"")); + free(out); + cbm_mcp_server_free(srv); + cleanup_project_db(cache, project); + restore_cache_dir(saved_copy); + free(saved_copy); + free(project); + remove(path); + cbm_rmdir(tmp); + cbm_rmdir(cache); + PASS(); +} + /* Edge-data lookup must follow the traversal direction and shortest-path hop. * The induced edge set contains a same-hop cross edge whose source is the row * node and sorts before its real inbound predecessor. An arbitrary incident- @@ -20741,6 +20813,7 @@ SUITE(mcp) { RUN_TEST(trace_evidence_strategy_class_vocabulary_is_closed); RUN_TEST(tool_trace_path_evidence_is_opt_in_and_class_mapped); RUN_TEST(tool_trace_path_marks_unresolved_call_totals_unknown); + RUN_TEST(tool_index_unresolved_nested_functions_are_indexed); RUN_TEST(tool_trace_path_evidence_columns_match_header_issue1542); RUN_TEST(tool_trace_path_unreadable_confidence_reports_not_recorded); RUN_TEST(tool_trace_path_edge_details_use_canonical_predecessor); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 9a072cd83..fd8ddd127 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -765,6 +765,108 @@ TEST(pipeline_adr_survives_full_reindex) { PASS(); } +static int pipeline_unresolved_review_case(int padding, bool spill) { + char tmp[256] = "/tmp/cbm_unresolved_review_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char db[512], path[512]; + const char *names[] = {"cliente.js", "servicio.js", "ayudante.js", + "directo.js", "missing.js", "route.js"}; + const char *sources[] = { + "export function crearCliente() { function buscar(id) { return id; } return { buscar }; " + "}\n", + "export function crearServicio({ cliente }) {\n" + " function procesar(id) {\n" + " cliente.buscar(id); return cliente.buscar(id + 1);\n" + " }\n return { procesar };\n}\n", + "export function ayudante(id) { return id; }\n", + "import { ayudante } from './ayudante.js';\n" + "export function usarDirecto(id) { ayudante(id); return ayudante(id + 1); }\n", + "import { absent } from './ayudante.js';\n" + "export function missing(id) { return absent(id); }\n", + "export function setup({ app }) { app.get('/x', handler); }\n" + "function handler() { return 1; }\n"}; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + snprintf(path, sizeof(path), "%s/%s", tmp, names[i]); + FILE *f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fputs(sources[i], f); + fclose(f); + } + for (int i = 0; i < padding; i++) { + snprintf(path, sizeof(path), "%s/pad%d.js", tmp, i); + FILE *f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "export function pad%d() { return %d; }\n", i, i); + fclose(f); + } + snprintf(db, sizeof(db), "%s/index.db", tmp); + if (spill) + cbm_setenv("CBM_MEM_SPILL", "1", 1); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + int rc = cbm_pipeline_run(p); + if (spill) + cbm_unsetenv("CBM_MEM_SPILL"); + ASSERT_EQ(rc, 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + cbm_store_t *st = cbm_store_open_path(db); + ASSERT_NOT_NULL(st); + cbm_coverage_row_t *rows = NULL; + int count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(st, project, "directo.js", &rows, &count), CBM_STORE_OK); + for (int i = 0; i < count; i++) + ASSERT_FALSE(strcmp(rows[i].kind, "unresolved_calls") == 0); + cbm_store_free_coverage(rows, count); + rows = NULL; + count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(st, project, "route.js", &rows, &count), CBM_STORE_OK); + for (int i = 0; i < count; i++) + ASSERT_FALSE(strcmp(rows[i].kind, "unresolved_calls") == 0); + cbm_store_free_coverage(rows, count); + rows = NULL; + count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(st, project, "servicio.js", &rows, &count), CBM_STORE_OK); + ASSERT_EQ(count, 1); + yyjson_doc *doc = yyjson_read(rows[0].detail, strlen(rows[0].detail), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *sites = yyjson_doc_get_root(doc); + ASSERT_EQ(yyjson_arr_size(sites), 2); + for (size_t i = 0; i < 2; i++) { + yyjson_val *site = yyjson_arr_get(sites, i); + const char *caller = yyjson_get_str(yyjson_obj_get(site, "caller")); + ASSERT_NOT_NULL(caller); + ASSERT_NOT_NULL(strstr(caller, ".procesar")); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(site, "line")), 3); + ASSERT_NOT_NULL(yyjson_get_str(yyjson_obj_get(site, "candidate"))); + } + yyjson_doc_free(doc); + cbm_store_free_coverage(rows, count); + rows = NULL; + count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(st, project, "missing.js", &rows, &count), CBM_STORE_OK); + ASSERT_EQ(count, 1); + doc = yyjson_read(rows[0].detail, strlen(rows[0].detail), 0); + ASSERT_NOT_NULL(doc); + ASSERT_EQ(yyjson_arr_size(yyjson_doc_get_root(doc)), 1); + yyjson_doc_free(doc); + cbm_store_free_coverage(rows, count); + cbm_store_close(st); + rm_rf(tmp); + PASS(); +} + +TEST(pipeline_unresolved_review_sequential) { + return pipeline_unresolved_review_case(0, false); +} +TEST(pipeline_unresolved_review_parallel) { + return pipeline_unresolved_review_case(55, false); +} +TEST(pipeline_unresolved_review_spill) { + return pipeline_unresolved_review_case(55, true); +} + TEST(pipeline_records_unresolved_injected_call_sites) { char tmp[256] = "/tmp/cbm_unresolved_calls_XXXXXX"; ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); @@ -15939,6 +16041,9 @@ SUITE(pipeline) { RUN_TEST(pipeline_adr_survives_full_reindex); RUN_TEST(pipeline_export_error_snapshot_on_artifact_failure); RUN_TEST(pipeline_records_unresolved_injected_call_sites); + RUN_TEST(pipeline_unresolved_review_sequential); + RUN_TEST(pipeline_unresolved_review_parallel); + RUN_TEST(pipeline_unresolved_review_spill); RUN_TEST(pipeline_structure_edges); RUN_TEST(pipeline_branch_root_structure); RUN_TEST(pipeline_project_name_derived); From 8d80be7bdef118ac4430464a0f99715680340673 Mon Sep 17 00:00:00 2001 From: Pcristin Date: Tue, 29 Sep 2026 09:09:57 +0000 Subject: [PATCH 059/217] fix(mcp): limit unresolved outbound totals to project symbols Signed-off-by: Pcristin --- src/mcp/mcp.c | 31 +++++++++++++++-- tests/test_mcp.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index a8acb0b29..cf2b276cb 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -9057,6 +9057,8 @@ typedef struct { } trace_coverage_scope_t; typedef struct { + cbm_store_t *store; + const char *project; CBMHashTable *callers; CBMHashTable *targets; bool unresolved_out; @@ -9098,9 +9100,31 @@ static bool trace_coverage_build_sets(trace_coverage_match_t *match, trace_coverage_add_visited(match->targets, scope->inbound, scope->include_tests); } +static bool trace_coverage_may_target_project(const trace_coverage_match_t *match, + const char *name) { + if (!name || !name[0]) { + return true; + } + cbm_node_t *nodes = NULL; + int count = 0; + int rc = cbm_store_find_nodes_by_name(match->store, match->project, name, &nodes, &count); + /* An unsuccessful lookup cannot prove that the callee is outside the + * project. Free the allocated result even when it contains zero rows. */ + bool possible = rc != CBM_STORE_OK; + for (int i = 0; i < count && !possible; i++) { + const char *label = nodes[i].label; + possible = label && strcmp(label, "File") != 0 && strcmp(label, "Folder") != 0 && + strcmp(label, "Project") != 0 && strcmp(label, "Module") != 0 && + strcmp(label, "Package") != 0 && strcmp(label, "Section") != 0; + } + cbm_store_free_nodes(nodes, count); + return possible; +} + static void trace_coverage_match_site(trace_coverage_match_t *match, const char *rel_path, yyjson_val *site) { const char *caller = yyjson_get_str(yyjson_obj_get(site, "caller")); + const char *leaf = yyjson_get_str(yyjson_obj_get(site, "leaf")); const char *candidate = yyjson_get_str(yyjson_obj_get(site, "candidate")); const cbm_node_t *caller_node = match->callers && caller ? cbm_ht_get(match->callers, caller) : NULL; @@ -9109,7 +9133,8 @@ static void trace_coverage_match_site(trace_coverage_match_t *match, const char * caller. Check its file and source range too; never match a containing * factory merely because its range overlaps. */ if (caller_node && caller_node->file_path && strcmp(caller_node->file_path, rel_path) == 0 && - (line <= 0 || (line >= caller_node->start_line && line <= caller_node->end_line))) { + (line <= 0 || (line >= caller_node->start_line && line <= caller_node->end_line)) && + trace_coverage_may_target_project(match, leaf)) { match->unresolved_out = true; } /* A short-name collision is not evidence that the resolver considered @@ -9175,7 +9200,9 @@ static void trace_call_coverage(cbm_store_t *store, const char *project, if (!includes_calls) { return; } - trace_coverage_match_t match = {.callers = scope->do_outbound ? cbm_ht_create(0) : NULL, + trace_coverage_match_t match = {.store = store, + .project = project, + .callers = scope->do_outbound ? cbm_ht_create(0) : NULL, .targets = scope->do_inbound ? cbm_ht_create(0) : NULL}; bool ok = trace_coverage_meta_available(store, project) && (!scope->do_outbound || match.callers) && (!scope->do_inbound || match.targets) && diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 2f8b41eb2..f91fd6897 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -7528,6 +7528,90 @@ TEST(tool_trace_path_marks_unresolved_call_totals_unknown) { PASS(); } +TEST(tool_trace_path_outbound_requires_project_symbol) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + cbm_store_t *st = cbm_mcp_server_store(srv); + const char *proj = "project-symbol-trace"; + cbm_mcp_server_set_project(srv, proj); + ASSERT_EQ(cbm_store_upsert_project(st, proj, "/tmp/project-symbol-trace"), CBM_STORE_OK); + cbm_node_t caller = {.project = proj, + .label = "Function", + .name = "run", + .qualified_name = "project-symbol-trace.service.run", + .file_path = "service.js", + .start_line = 1, + .end_line = 4}; + cbm_node_t local = {.project = proj, + .label = "Function", + .name = "buscar", + .qualified_name = "project-symbol-trace.client.buscar", + .file_path = "client.js"}; + cbm_node_t constructor = {.project = proj, + .label = "Class", + .name = "Cliente", + .qualified_name = "project-symbol-trace.client.Cliente", + .file_path = "client.js"}; + cbm_node_t foreign = {.project = "other-project", + .label = "Function", + .name = "foreignOnly", + .qualified_name = "other-project.client.foreignOnly", + .file_path = "client.js"}; + ASSERT_GT(cbm_store_upsert_node(st, &caller), 0); + ASSERT_GT(cbm_store_upsert_node(st, &local), 0); + ASSERT_GT(cbm_store_upsert_node(st, &constructor), 0); + ASSERT_EQ(cbm_store_upsert_project(st, foreign.project, "/tmp/other-project"), CBM_STORE_OK); + ASSERT_GT(cbm_store_upsert_node(st, &foreign), 0); + const char *containers[] = {"File", "Folder", "Project", "Module", "Package", "Section"}; + for (size_t i = 0; i < sizeof(containers) / sizeof(containers[0]); i++) { + char name[64], qn[128]; + snprintf(name, sizeof(name), "external%s", containers[i]); + snprintf(qn, sizeof(qn), "%s.%s", proj, name); + cbm_node_t container = { + .project = proj, .label = containers[i], .name = name, .qualified_name = qn}; + ASSERT_GT(cbm_store_upsert_node(st, &container), 0); + } + ASSERT_EQ(cbm_store_upsert_file_hash(st, proj, "service.js", "fixture", 0, 0), CBM_STORE_OK); + cbm_coverage_meta_t meta = {.generation = "fixture", + .index_mode = "full", + .recorded_at = "2026-09-29T00:00:00Z", + .recording_status = "complete", + .coverage_version = CBM_UNRESOLVED_CALL_COVERAGE_VERSION, + .hash_records_complete = true}; + const struct { + const char *leaf; + const char *relation; + } cases[] = {{"printf", "eq"}, {"foreignOnly", "eq"}, {"Buscar", "eq"}, + {"externalFile", "eq"}, {"externalFolder", "eq"}, {"externalProject", "eq"}, + {"externalModule", "eq"}, {"externalPackage", "eq"}, {"externalSection", "eq"}, + {"buscar", "unknown"}, {"Cliente", "unknown"}}; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + char detail[512]; + snprintf(detail, sizeof(detail), + "[{\"caller\":\"project-symbol-trace.service.run\",\"leaf\":\"%s\"," + "\"start_byte\":42,\"end_byte\":59,\"line\":3," + "\"reason\":\"method_not_in_registry\"}]", + cases[i].leaf); + cbm_coverage_row_t row = { + .rel_path = "service.js", .kind = "unresolved_calls", .detail = detail}; + ASSERT_EQ(cbm_store_coverage_replace_ex(st, proj, &row, 1, &meta), CBM_STORE_OK); + char *out = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_path\",\"arguments\":{\"project\":" + "\"project-symbol-trace\",\"function_name\":\"run\"," + "\"direction\":\"outbound\",\"format\":\"json\"}}}"); + char *txt = extract_text_content(out); + char expected[80]; + snprintf(expected, sizeof(expected), "\"callees_total_relation\":\"%s\"", + cases[i].relation); + ASSERT_NOT_NULL(strstr(txt, expected)); + ASSERT_NOT_NULL(strstr(txt, "\"callees_total\":0")); + free(txt); + free(out); + } + cbm_mcp_server_free(srv); + PASS(); +} + TEST(tool_index_unresolved_nested_functions_are_indexed) { char tmp[256] = "/tmp/cbm-mcp-unresolved-XXXXXX"; char cache[256] = "/tmp/cbm-mcp-unresolved-cache-XXXXXX"; @@ -7540,7 +7624,8 @@ TEST(tool_index_unresolved_nested_functions_are_indexed) { snprintf(path, sizeof(path), "%s/service.js", tmp); FILE *f = fopen(path, "w"); ASSERT_NOT_NULL(f); - fputs("export function factory({ client }) {\n" + fputs("export class Client { buscar(id) { return id; } }\n" + "export function factory({ client }) {\n" " function inner(id) {\n" " return client.buscar(id);\n" " }\n return { inner };\n}\n", @@ -20813,6 +20898,7 @@ SUITE(mcp) { RUN_TEST(trace_evidence_strategy_class_vocabulary_is_closed); RUN_TEST(tool_trace_path_evidence_is_opt_in_and_class_mapped); RUN_TEST(tool_trace_path_marks_unresolved_call_totals_unknown); + RUN_TEST(tool_trace_path_outbound_requires_project_symbol); RUN_TEST(tool_index_unresolved_nested_functions_are_indexed); RUN_TEST(tool_trace_path_evidence_columns_match_header_issue1542); RUN_TEST(tool_trace_path_unreadable_confidence_reports_not_recorded); From 2479724a35d252786b8dbbeb9f1a74d86c569620 Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Tue, 29 Sep 2026 09:25:46 +0000 Subject: [PATCH 060/217] fix(pipeline): refresh the projects row's root_path on the delta route run_closure_delta clones the previous generation's db file byte for byte and patches only nodes, edges, hashes and lsp surfaces, so a reindex that keeps the same project name but a different repo_path leaves root_path and indexed_at pinned to whichever directory the last full rebuild ran from. Every later root_path-keyed read then resolves files against a directory this run never touched. cbm_store_upsert_project already exists for exactly this purpose and is called on the full-rebuild path via cbm_gbuf_dump_to_sqlite; the delta route reaches cbm_pipeline_publish_staged directly and never calls it. Call it once on the staging store before publishing. Fixes #2276 Signed-off-by: Amir Fathi --- src/pipeline/pipeline_incremental.c | 11 +++++ tests/test_pipeline.c | 68 +++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 154518658..1574c31b9 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -2319,6 +2319,17 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char cbm_store_count_edges(staging, project)); cbm_log_info("delta.committed_counts", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + /* The staging file started as a byte-for-byte clone of the previous + * generation (cbm_delta_stage_clone above), so its `projects` row still + * carries whichever root_path/indexed_at the LAST full rebuild wrote. + * Every other write in this function patches nodes/edges/hashes/surfaces + * for the CURRENT run's repo_path, so the project row must be refreshed + * to match, or a later root_path-keyed read (get_code_snippet and + * friends) resolves files against a directory this run never touched. */ + if (cbm_store_upsert_project(staging, project, cbm_pipeline_repo_path(p)) != CBM_STORE_OK) { + goto out; + } + { int index_mode = cbm_pipeline_get_mode(p); cbm_pipeline_generation_t generation = { diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 568740f17..6ccde01e3 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3369,6 +3369,73 @@ TEST(pipeline_closure_repair_removed_def_drops_dependent_edge) { PASS(); } +/* #2276: the closure/delta route clones the previous generation's db file + * byte-for-byte and patches only nodes/edges/hashes/surfaces, so a reindex + * that keeps the same project name but a DIFFERENT repo_path (an artifact + * moved or reused from another checkout) leaves the `projects` row's + * root_path/indexed_at pinned to whichever directory the last FULL rebuild + * ran from, so every later root_path-keyed read then resolves files against + * a directory this run never touched. */ +TEST(pipeline_incremental_root_path_refreshed_across_directories) { + char dir_a[256]; + char dir_b[256]; + snprintf(dir_a, sizeof(dir_a), "/tmp/cbm_rootpath_a_XXXXXX"); + snprintf(dir_b, sizeof(dir_b), "/tmp/cbm_rootpath_b_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(dir_a)); + ASSERT_NOT_NULL(cbm_mkdtemp(dir_b)); + closure_probe_repo(dir_a); + closure_probe_repo(dir_b); + /* Body edit only, same exported names: the shape the closure route + * repairs without falling back to a full rebuild (mirrors + * pipeline_closure_repair_body_edit_converges_with_fresh_full above). */ + write_temp_file(dir_b, "lib.ts", + "export function closureProbeHelper(x: number): string {\n" + " const doubled = x + x;\n" + " return String(doubled);\n" + "}\n"); + + char db[512]; + snprintf(db, sizeof(db), "%s/rootpath.db", dir_a); + static const char *project = "rootpath-probe"; + + cbm_pipeline_t *first = cbm_pipeline_new(dir_a, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(first); + ASSERT_TRUE(cbm_pipeline_set_project_name(first, project)); + ASSERT_EQ(cbm_pipeline_run(first), 0); + cbm_pipeline_free(first); + + /* Reindex the SAME project name from dir_b: a valid existing generation + * exists, so this routes through the incremental path, not a first-time + * full build. */ + cbm_pipeline_t *second = cbm_pipeline_new(dir_b, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(second); + ASSERT_TRUE(cbm_pipeline_set_project_name(second, project)); + ASSERT_EQ(cbm_pipeline_run(second), 0); + cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route(); + cbm_pipeline_free(second); + /* The bug is specific to the closure/delta route; confirm we actually + * exercised it rather than a silent fall-back to a full rebuild, which + * would pass even with the fix reverted. */ + ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + + char canon_b[PATH_MAX]; + ASSERT_NOT_NULL(realpath(dir_b, canon_b)); + + cbm_project_t info = {0}; + cbm_store_t *store = cbm_store_open_path(db); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_get_project(store, project, &info), CBM_STORE_OK); + ASSERT_STR_EQ(info.root_path, canon_b); + free((char *)info.name); + free((char *)info.indexed_at); + free((char *)info.root_path); + cbm_store_close(store); + + th_rmtree(dir_a); + th_rmtree(dir_b); + PASS(); +} + TEST(pipeline_closure_repair_added_name_declines_to_full) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_closure_added_XXXXXX"); @@ -16182,6 +16249,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_dbt_jinja_lineage); RUN_TEST(pipeline_parallel_manifest_is_byte_stable_above_threshold); RUN_TEST(pipeline_closure_repair_body_edit_converges_with_fresh_full); + RUN_TEST(pipeline_incremental_root_path_refreshed_across_directories); RUN_TEST(pipeline_closure_repair_removed_def_drops_dependent_edge); RUN_TEST(pipeline_closure_repair_added_name_declines_to_full); RUN_TEST(pipeline_closure_repair_new_file_declines_to_full); From 4f7f8acca2aba292bd3f51a73dfa7288e11880c4 Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Wed, 30 Sep 2026 05:04:25 +0000 Subject: [PATCH 061/217] fix(test): compare root_path against dir_b, not a realpath()'d form cbm_pipeline_repo_path()/cbm_store_upsert_project() never call realpath(); root_path is stored verbatim on both the full-rebuild and delta routes. The test's realpath(dir_b) comparison only held because /tmp isn't a symlink on Linux; it resolves to /private/tmp on macOS, where the assertion failed. Signed-off-by: Amir Fathi --- tests/test_pipeline.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 6ccde01e3..2ac38d2d7 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3418,14 +3418,17 @@ TEST(pipeline_incremental_root_path_refreshed_across_directories) { * would pass even with the fix reverted. */ ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); - char canon_b[PATH_MAX]; - ASSERT_NOT_NULL(realpath(dir_b, canon_b)); - + /* cbm_pipeline_repo_path()/cbm_store_upsert_project() never call + * realpath(): the stored root_path is whatever was passed to + * cbm_pipeline_new(), verbatim, on both the full-rebuild path and this + * one. Compare against dir_b directly, not a canonicalized form, or + * this assertion only holds on a platform where /tmp is not itself a + * symlink (it resolves to /private/tmp on macOS). */ cbm_project_t info = {0}; cbm_store_t *store = cbm_store_open_path(db); ASSERT_NOT_NULL(store); ASSERT_EQ(cbm_store_get_project(store, project, &info), CBM_STORE_OK); - ASSERT_STR_EQ(info.root_path, canon_b); + ASSERT_STR_EQ(info.root_path, dir_b); free((char *)info.name); free((char *)info.indexed_at); free((char *)info.root_path); From b2a0a2cde52b8fc9237dbddfb347b8a058747032 Mon Sep 17 00:00:00 2001 From: Brian McNaboe Date: Wed, 30 Sep 2026 11:31:19 -0400 Subject: [PATCH 062/217] docs(cli): list daemon in --help The daemon subcommand was reachable but absent from the usage text. The description matches the hint daemon status already prints: a warm daemon lets CLI calls and hooks skip the per-command daemon start-up. It does not claim to remove a hook process's own image hash. Signed-off-by: Brian McNaboe --- src/main.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main.c b/src/main.c index fd3370c04..800898cd2 100644 --- a/src/main.c +++ b/src/main.c @@ -1110,6 +1110,9 @@ static void print_help(void) { printf(" codebase-memory-mcp uninstall [-y|-n] [--dry-run]\n"); printf(" codebase-memory-mcp update [-y|-n]\n"); printf(" codebase-memory-mcp config \n"); + printf(" codebase-memory-mcp daemon \n"); + printf(" start keeps a daemon warm so CLI calls and\n"); + printf(" hooks skip the per-command daemon start-up\n"); printf(" codebase-memory-mcp --version Print version\n"); printf(" codebase-memory-mcp --help Print this help\n"); printf("\nCLI output options:\n"); From 48f25595ca147a65515403b03ed4e15e673e634b Mon Sep 17 00:00:00 2001 From: Kris <2040188027@qq.com> Date: Wed, 30 Sep 2026 23:34:36 +0800 Subject: [PATCH 063/217] fix(cli): read PATH at its real length and probe Windows shim dirs (#221) Two independent reasons `install` missed opencode on Windows 11: 1. find_in_path() copied PATH into a fixed 4 KB stack buffer through cbm_safe_getenv(), and that helper REFUSES a value it cannot fit instead of truncating it. On a machine whose PATH exceeds 4096 characters the whole PATH was therefore skipped before a single directory was probed. 2. The fallback list had no Windows-shaped entries, and it probed only the bare name, so npm's %APPDATA%\npm, mise's %LOCALAPPDATA%\mise\shims and scoop's %USERPROFILE%\scoop\shims could never match a .cmd shim. - Add cbm_env_dup() to platform.c: the allocating sibling of cbm_safe_getenv, memory-core owned, wide-read on Windows so a non-ASCII PATH is not mangled by the ANSI code page. cli.c no longer needs in the middle of the file, nor any raw allocation. - find_in_path() now exits once, so the PATH copy is freed on every path instead of leaking on each success. - Keep the existing pipx (~/.local/bin) and cargo (~/.cargo/bin) fallbacks, built from the home_dir argument rather than cbm_get_home_dir(), and add the npm/mise/scoop locations. PATHEXT is probed for every fallback directory, not only for PATH entries. - Bound the fallback loop by NUM_PATHS, not NUM_RETRIES. - Hoist the PATHEXT list into one file-scope constant and add braces to every one-line statement. Regression guards in tests/repro/repro_issue221.c: a PATH longer than 4096 characters whose LAST entry holds the fixture, and the Windows fallback shims driven through APPDATA / LOCALAPPDATA / the home_dir argument. Both are red against the previous code and green with this change. The PATH case is not Windows-specific: the same 4 KB cap hid entries on macOS and Linux. No raw allocator call site is added under src/, so the memory-core ratchet is untouched; clang-format and the no-skips policy both pass locally. Signed-off-by: Kris <2040188027@qq.com> --- src/cli/cli.c | 115 ++++++++++++++----- src/foundation/platform.c | 71 ++++++++++++ src/foundation/platform.h | 13 +++ tests/repro/repro_issue221.c | 208 +++++++++++++++++++++++++++++++++++ 4 files changed, 379 insertions(+), 28 deletions(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index 6e33759e4..d8e91e7a0 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -22,6 +22,7 @@ #include "foundation/platform.h" #include "foundation/constants.h" #include "foundation/log.h" +#include "foundation/mem_core.h" #include "foundation/sha256.h" #include "cli/client_adapter.h" #include "mcp/mcp.h" // cbm_mcp_tool_input_schema — CLI flag parser + per-tool --help @@ -1076,36 +1077,50 @@ static bool is_executable(const char *path) { #endif } +/* Windows ships CLIs as extension-bearing shims -- .cmd from npm, .ps1 from + * mise, .exe from scoop -- so a bare-name probe never matches there. One + * definition serves both the PATH scan and the fallback-directory probe (#221). */ +#ifdef _WIN32 +static const char *const WIN_EXEC_EXTS[] = {".exe", ".cmd", ".bat", ".ps1", NULL}; +#endif + /* Search for an executable named `name` in the PATH environment variable. - * Returns the full path in `out` (max out_sz) if found, else empty string. */ + * Returns the full path in `out` (max out_sz) if found, else empty string. + * + * The PATH copy is sized to the REAL value, not to a fixed 4 KB stack buffer. + * That buffer was the first root cause of #221: a Windows dev machine's PATH + * routinely runs past 4096 characters, and cbm_safe_getenv() REFUSES a value it + * cannot fit rather than truncating it -- so find_in_path() returned false + * before looking at a single directory, and opencode (installed through + * mise/npm/scoop and present on PATH) read as "not installed". The same 4 KB + * cap hid entries on macOS/Linux too; the fix is not Windows-specific. */ static bool find_in_path(const char *name, char *out, size_t out_sz) { - char path_copy[CLI_BUF_4K]; - if (!cbm_safe_getenv("PATH", path_copy, sizeof(path_copy), NULL)) { + char *path_copy = cbm_env_dup("PATH"); + if (!path_copy) { return false; } - char *saveptr; + + /* One exit. PATH is a full live copy of the environment, the scan has + * several ways to stop, and the previous shape returned straight out of the + * loop -- which is exactly what leaked a PATH-sized allocation per call. */ + bool found = false; + char *saveptr = NULL; char *dir = strtok_r(path_copy, PATH_DELIM, &saveptr); - while (dir) { + while (dir != NULL && !found) { snprintf(out, out_sz, "%s/%s", dir, name); - if (is_executable(out)) { - return true; - } + found = is_executable(out); #ifdef _WIN32 - /* On Windows executables carry an extension (PATHEXT). A CLI like - * opencode is often installed as a .cmd / .ps1 / .exe shim (e.g. via - * mise or npm), so the bare-name probe above misses it (#221). Try the - * common executable extensions before moving to the next PATH entry. */ - static const char *const win_exts[] = {".exe", ".cmd", ".bat", ".ps1", NULL}; - for (int i = 0; win_exts[i]; i++) { - snprintf(out, out_sz, "%s/%s%s", dir, name, win_exts[i]); - if (is_executable(out)) { - return true; - } + /* PATHEXT variants for THIS entry, before moving to the next one. */ + for (int i = 0; !found && WIN_EXEC_EXTS[i] != NULL; i++) { + snprintf(out, out_sz, "%s/%s%s", dir, name, WIN_EXEC_EXTS[i]); + found = is_executable(out); } #endif dir = strtok_r(NULL, PATH_DELIM, &saveptr); } - return false; + + cbm_free(CBM_MEM_CLASS_OTHER, path_copy); + return found; } const char *cbm_find_cli(const char *name, const char *home_dir) { @@ -1119,22 +1134,66 @@ const char *cbm_find_cli(const char *name, const char *home_dir) { if (!home_dir || !home_dir[0]) { return ""; } + /* PATH-independent fallbacks, tried in order. Every home-derived entry is + * built from the `home_dir` ARGUMENT and not from cbm_get_home_dir(): a + * dry-run scan or a test passes a synthetic home and must not be shown the + * host's agents. `/` separators are fine on Windows -- the wide-safe file + * layer accepts them, and the rest of this file already composes paths so. */ enum { NUM_PATHS = 5 }; char paths[NUM_PATHS][CLI_BUF_512]; - snprintf(paths[0], sizeof(paths[0]), "/usr/local/bin/%s", name); - snprintf(paths[1], sizeof(paths[1]), "%s/.npm/bin/%s", home_dir, name); - snprintf(paths[2], sizeof(paths[2]), "%s/.local/bin/%s", home_dir, name); - snprintf(paths[3], sizeof(paths[3]), "%s/.cargo/bin/%s", home_dir, name); -#ifdef __APPLE__ - snprintf(paths[4], sizeof(paths[4]), "/opt/homebrew/bin/%s", name); + memset(paths, 0, sizeof(paths)); + int n = 0; +#ifdef _WIN32 + /* Locations that were already probed on Windows before #221 and are real + * there: pipx installs into ~/.local/bin and cargo into ~/.cargo/bin, so + * replacing the list rather than extending it would make pipx- and + * cargo-installed agents disappear. */ + snprintf(paths[n++], sizeof(paths[0]), "%s/.local/bin/%s", home_dir, name); + snprintf(paths[n++], sizeof(paths[0]), "%s/.cargo/bin/%s", home_dir, name); + /* Windows-shaped locations (issue #221, second root cause): npm puts its + * global bin under %APPDATA%\npm, and mise and scoop expose shims under + * %LOCALAPPDATA%\mise\shims and %USERPROFILE%\scoop\shims. Each is read + * through its own environment seam, so the list stays testable. */ + const char *appdata = cbm_app_config_dir(); + if (appdata) { + snprintf(paths[n++], sizeof(paths[0]), "%s/npm/%s", appdata, name); + } + const char *localapp = cbm_app_local_dir(); + if (localapp) { + snprintf(paths[n++], sizeof(paths[0]), "%s/mise/shims/%s", localapp, name); + } + snprintf(paths[n++], sizeof(paths[0]), "%s/scoop/shims/%s", home_dir, name); #else - paths[4][0] = '\0'; + snprintf(paths[n++], sizeof(paths[0]), "/usr/local/bin/%s", name); + snprintf(paths[n++], sizeof(paths[0]), "%s/.npm/bin/%s", home_dir, name); + snprintf(paths[n++], sizeof(paths[0]), "%s/.local/bin/%s", home_dir, name); + snprintf(paths[n++], sizeof(paths[0]), "%s/.cargo/bin/%s", home_dir, name); +#ifdef __APPLE__ + snprintf(paths[n++], sizeof(paths[0]), "/opt/homebrew/bin/%s", name); #endif - for (int i = 0; i < NUM_RETRIES; i++) { - if (paths[i][0] && is_executable(paths[i])) { +#endif + /* NUM_PATHS, not NUM_RETRIES: both are 5 today, so the old bound only + * looked right by coincidence -- what the loop means is the list length. */ + for (int i = 0; i < NUM_PATHS; i++) { + if (!paths[i][0]) { + continue; + } + if (is_executable(paths[i])) { snprintf(buf, sizeof(buf), "%s", paths[i]); return buf; } +#ifdef _WIN32 + /* The npm/mise/scoop shims carry an extension, so every fallback + * directory needs the same PATHEXT probe the PATH scan got. */ + for (int e = 0; WIN_EXEC_EXTS[e] != NULL; e++) { + char extpath[CLI_BUF_512]; + snprintf(extpath, sizeof(extpath), "%s%s", paths[i], WIN_EXEC_EXTS[e]); + if (is_executable(extpath)) { + snprintf(buf, sizeof(buf), "%s", extpath); + return buf; + } + } +#endif } return ""; } diff --git a/src/foundation/platform.c b/src/foundation/platform.c index cf9a0f469..410536030 100644 --- a/src/foundation/platform.c +++ b/src/foundation/platform.c @@ -7,6 +7,7 @@ #include "foundation/compat.h" #include "foundation/constants.h" +#include "foundation/mem_core.h" #include "foundation/platform_internal.h" #include #include @@ -437,6 +438,76 @@ const char *cbm_safe_getenv(const char *name, char *buf, size_t buf_sz, const ch return NULL; } +/* See platform.h. The long-value sibling of cbm_safe_getenv: there is no caller + * buffer to overrun, so a PATH past 4 KB is read in full instead of being + * refused (#221). Both the wide staging buffer and the UTF-8 result are + * allocated through the memory core, which is also the only allocator the + * caller may release them with -- keeping the memory-core ratchet honest means + * this file adds no raw allocation site to pay for the new read. */ +char *cbm_env_dup(const char *name) { + if (!name || !name[0]) { + return NULL; + } +#ifdef _WIN32 + /* Same wide route as cbm_safe_getenv: _environ holds ANSI-code-page bytes, + * so a non-ASCII PATH would arrive mojibake'd or with '?' substituted -- + * invalid in a Windows path and fatal to every file API downstream. */ + wchar_t wname[CBM_SZ_256]; + int wn = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, name, -1, wname, CBM_SZ_256); + if (wn <= 0) { + return NULL; + } + SetLastError(ERROR_SUCCESS); + DWORD needed = GetEnvironmentVariableW(wname, NULL, 0U); + DWORD lookup_error = GetLastError(); + if (needed == 0U) { + if (lookup_error != ERROR_ENVVAR_NOT_FOUND) { + /* Present but empty. Distinct from unset, and callers are entitled + * to tell the two apart, so return "" and not NULL. */ + return cbm_mem_strdup(CBM_MEM_CLASS_OTHER, ""); + } + return NULL; + } + wchar_t *wvalue = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)needed * sizeof(*wvalue)); + if (!wvalue) { + return NULL; + } + SetLastError(ERROR_SUCCESS); + DWORD got = GetEnvironmentVariableW(wname, wvalue, needed); + DWORD read_error = GetLastError(); + if (got >= needed || (got == 0U && read_error != ERROR_SUCCESS)) { + cbm_free(CBM_MEM_CLASS_OTHER, wvalue); + return NULL; + } + int utf8_size = + WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wvalue, -1, NULL, 0, NULL, NULL); + char *utf8 = utf8_size > 0 ? cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)utf8_size) : NULL; + int converted = utf8 ? WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wvalue, -1, utf8, + utf8_size, NULL, NULL) + : 0; + if (!utf8 || converted != utf8_size) { + cbm_free(CBM_MEM_CLASS_OTHER, utf8); + cbm_free(CBM_MEM_CLASS_OTHER, wvalue); + return NULL; + } + cbm_free(CBM_MEM_CLASS_OTHER, wvalue); + return utf8; +#else + /* Same environ walk as cbm_safe_getenv: getenv() returns a pointer another + * thread can invalidate with setenv(), so the value is copied out here. */ + char **env = CBM_ENVIRON; + if (env) { + size_t name_len = strlen(name); + for (; *env; env++) { + if (strncmp(*env, name, name_len) == 0 && (*env)[name_len] == '=') { + return cbm_mem_strdup(CBM_MEM_CLASS_OTHER, *env + name_len + SKIP_ONE); + } + } + } + return NULL; +#endif +} + /* See platform.h. The shape here is the one src/main.c:1104 already uses for * --port=: an end pointer says where the read stopped, errno catches a number * too large, and *end == '\0' catches anything left over. */ diff --git a/src/foundation/platform.h b/src/foundation/platform.h index 43738f621..52e01dc46 100644 --- a/src/foundation/platform.h +++ b/src/foundation/platform.h @@ -138,6 +138,19 @@ int cbm_default_worker_count(bool initial); * Returns NULL when the variable is unset and fallback is NULL. */ const char *cbm_safe_getenv(const char *name, char *buf, size_t buf_sz, const char *fallback); +/* Allocating sibling of cbm_safe_getenv, for a value that must not be capped at + * a caller-chosen size. A Windows PATH routinely runs past the 4096-byte buffer + * cbm_safe_getenv's callers pass, and the copy helper REPORTS that as failure + * rather than truncating -- so the whole PATH was skipped and every directory + * past the cut became invisible (#221). + * + * Returns a UTF-8 copy owned by the memory core: release it with + * cbm_free(CBM_MEM_CLASS_OTHER, ptr), never free(). NULL when the variable is + * unset or cannot be read; a variable that exists but is empty yields "". + * On Windows the value is read with the wide API and converted, the same route + * cbm_safe_getenv takes, so a non-ASCII value survives intact. */ +char *cbm_env_dup(const char *name); + /* Read an environment variable as a whole number. * * Answers true only when the variable is set, is not empty, and reads cleanly diff --git a/tests/repro/repro_issue221.c b/tests/repro/repro_issue221.c index cb4d27fd4..2baf2705d 100644 --- a/tests/repro/repro_issue221.c +++ b/tests/repro/repro_issue221.c @@ -152,7 +152,215 @@ TEST(repro_issue221_opencode_pathext_lookup) { PASS(); } +/* ── #221 regression: PATH longer than the old 4 KB copy ───────────────── */ + +/* Snapshot/restore one environment variable. The suite swaps PATH and the + * Windows profile roots, and a leaked swap would poison every later suite. */ +static char *repro221_save_env(const char *name) { + const char *value = getenv(name); + return value ? strdup(value) : NULL; +} + +static void repro221_restore_env(const char *name, char *saved) { + if (saved) { + cbm_setenv(name, saved, 1); + free(saved); + } else { + cbm_unsetenv(name); + } +} + +/* + * repro_issue221_long_path_beyond_four_kb + * + * find_in_path() used to copy PATH into a fixed 4096-byte stack buffer through + * cbm_safe_getenv(). That helper does not truncate -- it REFUSES a value it + * cannot fit -- so on a machine whose PATH exceeds 4096 characters the whole + * PATH was skipped and every agent on it read as "not installed". That is the + * layout the #221 reporter had: mise, npm and scoop each append entries. + * + * The fixture directory is the LAST PATH entry, so any truncation of the copy + * hides exactly the directory under test and nothing else. This case is not + * Windows-specific -- the same 4 KB cap hid entries on macOS and Linux. + * + * RED before the fix (empty result), GREEN after it. + */ +TEST(repro_issue221_long_path_beyond_four_kb) { + const char *created = th_mktempdir("repro221long"); + if (!created) { + FAIL("th_mktempdir failed"); + } + char tmpdir[512]; + snprintf(tmpdir, sizeof(tmpdir), "%s", created); + +#ifdef _WIN32 + const char *fixture_name = "cbm-repro221-long.cmd"; + const char *fixture_body = "@echo off\r\nrem fake long-PATH agent\r\n"; + const char path_delim = ';'; +#else + const char *fixture_name = "cbm-repro221-long"; + const char *fixture_body = "#!/bin/sh\n# fake long-PATH agent\n"; + const char path_delim = ':'; +#endif + + char fixture_path[1024]; + snprintf(fixture_path, sizeof(fixture_path), "%s/%s", tmpdir, fixture_name); + if (th_write_file(fixture_path, fixture_body) != 0) { + th_cleanup(tmpdir); + FAIL("could not write the long-PATH fixture"); + } + th_make_executable(fixture_path); + + /* + * Filler entries name directories that do not exist: the scan has to walk + * past every one of them to reach the fixture, which is appended after the + * filler so that it always sits beyond the 4096-byte mark. + */ + enum { FILLER_TARGET = 4600, PATH_CAP = FILLER_TARGET + 1024 }; + char long_path[PATH_CAP]; + size_t used = 0U; + while (used < (size_t)FILLER_TARGET) { + int written = snprintf(long_path + used, sizeof(long_path) - used, + "/nonexistent-repro221-filler%05zu%c", used, path_delim); + if (written <= 0 || (size_t)written >= sizeof(long_path) - used) { + th_cleanup(tmpdir); + FAIL("could not build the long PATH filler"); + } + used += (size_t)written; + } + ASSERT_GT(used, 4096); + int tail = snprintf(long_path + used, sizeof(long_path) - used, "%s", tmpdir); + if (tail <= 0 || (size_t)tail >= sizeof(long_path) - used) { + th_cleanup(tmpdir); + FAIL("could not append the fixture directory to the long PATH"); + } + + char *saved_path = repro221_save_env("PATH"); + cbm_setenv("PATH", long_path, 1); + + /* A home that does not exist keeps the PATH-independent fallbacks out of + * the way: the fixture's directory is the only place that can satisfy it. */ + const char *result = cbm_find_cli("cbm-repro221-long", "/nonexistent-repro221-home"); + + repro221_restore_env("PATH", saved_path); + + ASSERT_NOT_NULL(result); + ASSERT(result[0] != '\0'); + ASSERT(strncmp(result, tmpdir, strlen(tmpdir)) == 0); + ASSERT(strstr(result, "cbm-repro221-long") != NULL); + + th_cleanup(tmpdir); + PASS(); +} + +/* ── #221 regression: the Windows fallback shim locations ───────────────── */ + +/* + * repro_issue221_windows_fallback_shims + * + * An agent the PATH scan cannot see must still be found where Windows actually + * puts CLIs: %APPDATA%\npm (npm's global bin), %LOCALAPPDATA%\mise\shims, + * %USERPROFILE%\scoop\shims, plus the pipx and cargo homes (~\.local\bin, + * ~\.cargo\bin) that were already probed before #221 passed and must not be + * dropped by the fix. + * + * Each location is reached through its own seam -- APPDATA / LOCALAPPDATA, or + * the home_dir argument -- so this never reads the developer's real profile. + * Every shim carries a .cmd extension, which is why the fallback probe needs + * the same PATHEXT treatment the PATH scan got. + * + * Windows-only: the directory layout it asserts is the one Windows installers + * produce. */ +TEST(repro_issue221_windows_fallback_shims) { +#ifndef _WIN32 + SKIP_PLATFORM("Windows-only: npm/mise/scoop shim fallback layout"); +#else + const char *created = th_mktempdir("repro221fb"); + if (!created) { + FAIL("th_mktempdir failed"); + } + char root[512]; + snprintf(root, sizeof(root), "%s", created); + + char *saved_appdata = repro221_save_env("APPDATA"); + char *saved_localappdata = repro221_save_env("LOCALAPPDATA"); + char *saved_userprofile = repro221_save_env("USERPROFILE"); + char *saved_path = repro221_save_env("PATH"); + + /* APPDATA/LOCALAPPDATA/USERPROFILE all point at the fixture root. The + * scoop, pipx and cargo entries are built from the home_dir ARGUMENT, so + * USERPROFILE here mirrors what a real profile would supply. */ + cbm_setenv("APPDATA", root, 1); + cbm_setenv("LOCALAPPDATA", root, 1); + cbm_setenv("USERPROFILE", root, 1); + cbm_setenv("PATH", "/nonexistent-repro221-path", 1); + + /* One shim at a time, so the returned path proves which entry matched + * rather than merely that something did. Each case removes its shim before + * the next one runs, so the earlier entries cannot shadow the later ones. */ + const struct { + const char *subdir; + const char *label; + } cases[] = { + {".local/bin", "pipx home"}, {".cargo/bin", "cargo home"}, {"npm", "APPDATA npm"}, + {"mise/shims", "LOCALAPPDATA mise"}, {"scoop/shims", "scoop home"}, + }; + const char *shim_name = "cbm-repro221-fb.cmd"; + const size_t case_count = sizeof(cases) / sizeof(cases[0]); + + for (size_t i = 0; i < case_count; i++) { + char dir[512]; + snprintf(dir, sizeof(dir), "%s/%s", root, cases[i].subdir); + if (th_mkdir_p(dir) != 0) { + repro221_restore_env("PATH", saved_path); + repro221_restore_env("USERPROFILE", saved_userprofile); + repro221_restore_env("LOCALAPPDATA", saved_localappdata); + repro221_restore_env("APPDATA", saved_appdata); + th_cleanup(root); + FAIL("could not create the fallback fixture directory"); + } + char shim[1024]; + snprintf(shim, sizeof(shim), "%s/%s", dir, shim_name); + if (th_write_file(shim, "@echo off\r\nrem fake fallback shim\r\n") != 0) { + repro221_restore_env("PATH", saved_path); + repro221_restore_env("USERPROFILE", saved_userprofile); + repro221_restore_env("LOCALAPPDATA", saved_localappdata); + repro221_restore_env("APPDATA", saved_appdata); + th_cleanup(root); + FAIL("could not write the fallback shim"); + } + + const char *result = cbm_find_cli("cbm-repro221-fb", root); + char expected_prefix[512]; + snprintf(expected_prefix, sizeof(expected_prefix), "%s/%s/", root, cases[i].subdir); + bool matched = result != NULL && result[0] != '\0' && + strncmp(result, expected_prefix, strlen(expected_prefix)) == 0; + (void)remove(shim); + + if (!matched) { + printf(" fallback case %zu (%s) did not resolve in %s\n", i, cases[i].label, dir); + repro221_restore_env("PATH", saved_path); + repro221_restore_env("USERPROFILE", saved_userprofile); + repro221_restore_env("LOCALAPPDATA", saved_localappdata); + repro221_restore_env("APPDATA", saved_appdata); + th_cleanup(root); + FAIL("a Windows fallback shim location was not probed"); + } + } + + repro221_restore_env("PATH", saved_path); + repro221_restore_env("USERPROFILE", saved_userprofile); + repro221_restore_env("LOCALAPPDATA", saved_localappdata); + repro221_restore_env("APPDATA", saved_appdata); + + th_cleanup(root); + PASS(); +#endif +} + /* ── Suite ──────────────────────────────────────────────────────────────── */ SUITE(repro_issue221) { RUN_TEST(repro_issue221_opencode_pathext_lookup); + RUN_TEST(repro_issue221_long_path_beyond_four_kb); + RUN_TEST(repro_issue221_windows_fallback_shims); } From 42971f2c023db2ec4e1a0fa011f983dec9d67fb0 Mon Sep 17 00:00:00 2001 From: BumaldaOverTheWater94 <83429948+BumaldaOverTheWater94@users.noreply.github.com> Date: Fri, 25 Sep 2026 01:22:18 +0000 Subject: [PATCH 064/217] fix(cli): trust root-owned per-user /home entries during activation walk Managed Linux hosts keep /home a real directory and point /home/ at another tree (e.g. /local/home/). The O_NOFOLLOW walk rejected that entry and install failed with 'activation transaction I/O failed'. Resolve /home/ only when /home is root-owned and not group/world writable, the entry is a root-owned symlink, and it resolves to a directory owned by root or the current user. Arbitrary user-owned symlinks are still rejected. Fixes #2306 Co-authored-by: Claude Signed-off-by: BumaldaOverTheWater94 <83429948+BumaldaOverTheWater94@users.noreply.github.com> --- src/cli/activation_transaction.c | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/cli/activation_transaction.c b/src/cli/activation_transaction.c index a56052775..fb7b5983a 100644 --- a/src/cli/activation_transaction.c +++ b/src/cli/activation_transaction.c @@ -889,6 +889,53 @@ static bool activation_posix_acl_empty(int descriptor) { return cbm_macos_extended_acl_fd_is_empty(descriptor); } +#ifdef __linux__ +/* Managed Linux hosts often keep /home a real directory and point the per-user + * entry elsewhere (/home/alice -> /local/home/alice). Trust that entry only + * when root owns it inside a /home nobody else can write, and it resolves to a + * directory owned by root or the current user. Returns NULL when not + * applicable; the O_NOFOLLOW walk then rejects the path as before. */ +static char *activation_linux_home_entry_path(const char *directory) { + static const char home_prefix[] = "/home/"; + size_t prefix_length = sizeof(home_prefix) - 1U; + if (strncmp(directory, home_prefix, prefix_length) != 0) { + return NULL; + } + size_t name_length = strcspn(directory + prefix_length, "/"); + char entry[sizeof(home_prefix) + 256U]; + if (name_length == 0 || name_length >= 256U) { + return NULL; + } + memcpy(entry, directory, prefix_length + name_length); + entry[prefix_length + name_length] = '\0'; + + struct stat home_status; + struct stat entry_status; + struct stat resolved_status; + char resolved[4096]; + if (lstat("/home", &home_status) != 0 || !S_ISDIR(home_status.st_mode) || + home_status.st_uid != 0 || (home_status.st_mode & 0022) != 0 || + lstat(entry, &entry_status) != 0 || !S_ISLNK(entry_status.st_mode) || + entry_status.st_uid != 0 || !realpath(entry, resolved) || + lstat(resolved, &resolved_status) != 0 || !S_ISDIR(resolved_status.st_mode) || + (resolved_status.st_uid != 0 && resolved_status.st_uid != geteuid())) { + return NULL; + } + const char *rest = directory + prefix_length + name_length; + size_t needed = strlen(resolved) + strlen(rest) + 1U; + char *mapped = malloc(needed); + if (!mapped) { + return NULL; + } + int written = snprintf(mapped, needed, "%s%s", resolved, rest); + if (written <= 0 || (size_t)written >= needed) { + free(mapped); + return NULL; + } + return mapped; +} +#endif + static char *activation_posix_walk_path(const char *directory) { #if defined(__APPLE__) || defined(__linux__) /* macOS and immutable Linux layouts can expose writable trees through @@ -935,6 +982,12 @@ static char *activation_posix_walk_path(const char *directory) { } return mapped; } +#endif +#ifdef __linux__ + char *home_entry = activation_linux_home_entry_path(directory); + if (home_entry) { + return home_entry; + } #endif return activation_string_copy(directory); } From 2fbf3fb977c02c5fed5b2b8b6796902a270bc1c5 Mon Sep 17 00:00:00 2001 From: Glen Ko Date: Mon, 28 Sep 2026 21:24:43 +0000 Subject: [PATCH 065/217] fix(cli): fold per-user /home entry into the alias loop, add test seam Address review on #2309: - Fold the /home/ handling into the existing #2175 alias loop via a shared activation_alias_map() that builds the mapped path in a stack buffer and returns activation_string_copy(); raw allocator sites in activation_transaction.c drop from 80 to 78 (baseline lowered). - Comment the deliberate difference: the per-user entry may resolve to a directory owned by the current account, the whole-alias case may not. - Add cbm_activation_transaction_set_home_root_for_testing() (test seams only) and a Linux test that fails without the per-user branch. Co-authored-by: Claude Signed-off-by: Glen Ko --- scripts/memory-core-baseline.txt | 2 +- src/cli/activation_transaction.c | 122 ++++++++++++++-------------- src/cli/activation_transaction.h | 5 ++ tests/test_activation_transaction.c | 90 ++++++++++++++++++++ 4 files changed, 159 insertions(+), 60 deletions(-) diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index 138ef0f77..5bbae344a 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -14,7 +14,7 @@ internal/cbm/lsp/py_lsp.c 2 internal/cbm/macro_table.c 1 internal/cbm/service_patterns.c 2 internal/cbm/sqlite_writer.c 74 -src/cli/activation_transaction.c 80 +src/cli/activation_transaction.c 78 src/cli/agent_clients.c 73 src/cli/agent_profiles.c 7 src/cli/cli.c 170 diff --git a/src/cli/activation_transaction.c b/src/cli/activation_transaction.c index fb7b5983a..f526b2c9c 100644 --- a/src/cli/activation_transaction.c +++ b/src/cli/activation_transaction.c @@ -890,49 +890,44 @@ static bool activation_posix_acl_empty(int descriptor) { } #ifdef __linux__ -/* Managed Linux hosts often keep /home a real directory and point the per-user - * entry elsewhere (/home/alice -> /local/home/alice). Trust that entry only - * when root owns it inside a /home nobody else can write, and it resolves to a - * directory owned by root or the current user. Returns NULL when not - * applicable; the O_NOFOLLOW walk then rejects the path as before. */ -static char *activation_linux_home_entry_path(const char *directory) { - static const char home_prefix[] = "/home/"; - size_t prefix_length = sizeof(home_prefix) - 1U; - if (strncmp(directory, home_prefix, prefix_length) != 0) { - return NULL; - } - size_t name_length = strcspn(directory + prefix_length, "/"); - char entry[sizeof(home_prefix) + 256U]; - if (name_length == 0 || name_length >= 256U) { - return NULL; - } - memcpy(entry, directory, prefix_length + name_length); - entry[prefix_length + name_length] = '\0'; +/* Root of the per-user home tree and the account trusted to own it. Root and + * "/home" in production; the test seam substitutes a fixture tree and the test + * account, because the real layout needs root-owned entries to reproduce. */ +static const char *g_activation_home_root = "/home"; +static uid_t g_activation_home_owner = 0; - struct stat home_status; - struct stat entry_status; - struct stat resolved_status; +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_activation_transaction_set_home_root_for_testing(const char *home_root, + unsigned long owner_uid) { + g_activation_home_root = home_root ? home_root : "/home"; + g_activation_home_owner = home_root ? (uid_t)owner_uid : 0; +} +#endif +#endif + +#if defined(__APPLE__) || defined(__linux__) +/* Map a trusted symlink `link` (already lstat'd) onto its resolved target and + * append `rest`. The link and the target must both belong to `owner`; with + * accept_own_target the target may instead belong to the current account. + * Returns NULL when either check fails or the mapped path does not fit. */ +static char *activation_alias_map(const char *link, const struct stat *link_status, uid_t owner, + bool accept_own_target, const char *rest) { char resolved[4096]; - if (lstat("/home", &home_status) != 0 || !S_ISDIR(home_status.st_mode) || - home_status.st_uid != 0 || (home_status.st_mode & 0022) != 0 || - lstat(entry, &entry_status) != 0 || !S_ISLNK(entry_status.st_mode) || - entry_status.st_uid != 0 || !realpath(entry, resolved) || - lstat(resolved, &resolved_status) != 0 || !S_ISDIR(resolved_status.st_mode) || - (resolved_status.st_uid != 0 && resolved_status.st_uid != geteuid())) { + struct stat resolved_status; + if (link_status->st_uid != owner || !realpath(link, resolved) || + lstat(resolved, &resolved_status) != 0 || !S_ISDIR(resolved_status.st_mode)) { return NULL; } - const char *rest = directory + prefix_length + name_length; - size_t needed = strlen(resolved) + strlen(rest) + 1U; - char *mapped = malloc(needed); - if (!mapped) { + if (resolved_status.st_uid != owner && + !(accept_own_target && resolved_status.st_uid == geteuid())) { return NULL; } - int written = snprintf(mapped, needed, "%s%s", resolved, rest); - if (written <= 0 || (size_t)written >= needed) { - free(mapped); + char mapped[4096]; + int written = snprintf(mapped, sizeof(mapped), "%s%s", resolved, rest); + if (written <= 0 || (size_t)written >= sizeof(mapped)) { return NULL; } - return mapped; + return activation_string_copy(mapped); } #endif @@ -943,7 +938,7 @@ static char *activation_posix_walk_path(const char *directory) { * Resolve only these trusted system aliases; arbitrary user symlinks must * still fail the O_NOFOLLOW walk below. */ #ifdef __linux__ - static const char *const aliases[] = {"/tmp", "/var", "/home"}; + const char *const aliases[] = {"/tmp", "/var", g_activation_home_root}; #else static const char *const aliases[] = {"/tmp", "/var"}; #endif @@ -954,39 +949,48 @@ static char *activation_posix_walk_path(const char *directory) { (directory[alias_length] != '\0' && directory[alias_length] != '/')) { continue; } + uid_t owner = 0; +#ifdef __linux__ + bool is_home = alias == g_activation_home_root; + if (is_home) { + owner = g_activation_home_owner; + } +#endif struct stat alias_status; - char resolved[4096]; if (lstat(alias, &alias_status) != 0) { continue; } - if (!S_ISLNK(alias_status.st_mode)) { + if (S_ISLNK(alias_status.st_mode)) { + return activation_alias_map(alias, &alias_status, owner, false, + directory + alias_length); + } +#ifdef __linux__ + /* Managed Linux hosts often keep /home a real directory and point the + * per-user entry elsewhere (/home/alice -> /local/home/alice, #2306). + * Trust that entry only inside a root-owned /home nobody else can + * write, and only when root owns the entry itself. Unlike the whole- + * alias case above, the target may belong to the current account: + * that is the normal shape of a relocated home, and an account can + * already write its own home, so it gains nothing it did not have. */ + const char *name = directory + alias_length; + size_t name_length = *name == '/' ? strcspn(name + 1, "/") : 0; + if (!is_home || !S_ISDIR(alias_status.st_mode) || alias_status.st_uid != owner || + (alias_status.st_mode & 0022) != 0 || name_length == 0) { continue; } - if (alias_status.st_uid != 0 || !realpath(alias, resolved)) { + char entry[4096]; + size_t entry_length = alias_length + 1U + name_length; + if (entry_length >= sizeof(entry)) { return NULL; } - struct stat resolved_status; - if (lstat(resolved, &resolved_status) != 0 || !S_ISDIR(resolved_status.st_mode) || - resolved_status.st_uid != 0) { - return NULL; + memcpy(entry, directory, entry_length); + entry[entry_length] = '\0'; + struct stat entry_status; + if (lstat(entry, &entry_status) == 0 && S_ISLNK(entry_status.st_mode)) { + return activation_alias_map(entry, &entry_status, owner, true, + directory + entry_length); } - size_t needed = strlen(resolved) + strlen(directory + alias_length) + 1U; - char *mapped = malloc(needed); - if (!mapped) { - return NULL; - } - int written = snprintf(mapped, needed, "%s%s", resolved, directory + alias_length); - if (written <= 0 || (size_t)written >= needed) { - free(mapped); - return NULL; - } - return mapped; - } #endif -#ifdef __linux__ - char *home_entry = activation_linux_home_entry_path(directory); - if (home_entry) { - return home_entry; } #endif return activation_string_copy(directory); diff --git a/src/cli/activation_transaction.h b/src/cli/activation_transaction.h index c0dd89bfc..591f655fb 100644 --- a/src/cli/activation_transaction.h +++ b/src/cli/activation_transaction.h @@ -100,6 +100,11 @@ const char *cbm_activation_transaction_refusal_note(void); #ifdef CBM_ENABLE_TEST_SEAMS void cbm_activation_transaction_note_refusal_for_testing(const char *predicate, unsigned long os_error); +/* Linux only: stand home_root in for /home and owner_uid in for root when + * deciding whether /home or a /home/ entry is a trusted alias. NULL + * restores the production values. Inert elsewhere. */ +void cbm_activation_transaction_set_home_root_for_testing(const char *home_root, + unsigned long owner_uid); #endif #endif /* CBM_ACTIVATION_TRANSACTION_H */ diff --git a/tests/test_activation_transaction.c b/tests/test_activation_transaction.c index b984145f5..28f185f56 100644 --- a/tests/test_activation_transaction.c +++ b/tests/test_activation_transaction.c @@ -764,6 +764,95 @@ TEST(activation_transaction_rejects_symlink_candidate_target_and_parent) { PASS(); } +/* #2306: managed Linux hosts keep /home a real directory and point the + * per-user entry elsewhere (/home/alice -> /local/home/alice). Root owns that + * entry, so it is trusted like the #2175 /home alias; the seam stands a + * fixture tree in for /home and the test account in for root. Every status is + * captured before the seam is reset so a failure cannot leak into the + * following tests. */ +TEST(activation_transaction_follows_trusted_per_user_home_entry) { +#if defined(__linux__) && defined(CBM_ENABLE_TEST_SEAMS) + char root[ACTIVATION_TEST_PATH_CAP]; + char home[ACTIVATION_TEST_PATH_CAP]; + char entry[ACTIVATION_TEST_PATH_CAP]; + char relocated[ACTIVATION_TEST_PATH_CAP]; + char relocated_bin[ACTIVATION_TEST_PATH_CAP]; + char relocated_target[ACTIVATION_TEST_PATH_CAP]; + char target[ACTIVATION_TEST_PATH_CAP]; + ASSERT_TRUE(activation_test_fixture(root)); + ASSERT_TRUE(activation_test_path(home, root, "home")); + ASSERT_TRUE(activation_test_path(entry, home, "alice")); + ASSERT_TRUE(activation_test_path(relocated, root, "local-alice")); + ASSERT_TRUE(activation_test_path(relocated_bin, relocated, "bin")); + ASSERT_TRUE(activation_test_path(relocated_target, relocated_bin, "cbm")); + ASSERT_TRUE(activation_test_path(target, entry, "bin/cbm")); + ASSERT_TRUE(cbm_mkdir_p(home, 0755)); + ASSERT_EQ(chmod(home, 0755), 0); + ASSERT_TRUE(cbm_mkdir_p(relocated_bin, 0700)); + ASSERT_EQ(symlink(relocated, entry), 0); + unsigned long self = (unsigned long)geteuid(); + + /* Without the seam the entry is an ordinary user symlink: refused. */ + cbm_activation_transaction_t *transaction = NULL; + cbm_activation_transaction_status_t untrusted_status = cbm_activation_transaction_stage_bytes( + target, "candidate", strlen("candidate"), &transaction); + bool untrusted_created = transaction != NULL; + (void)cbm_activation_transaction_close(&transaction); + + /* The home root and the entry must belong to the trusted owner. */ + cbm_activation_transaction_set_home_root_for_testing(home, self + 1U); + cbm_activation_transaction_status_t foreign_owner_status = + cbm_activation_transaction_stage_bytes(target, "candidate", strlen("candidate"), + &transaction); + bool foreign_owner_created = transaction != NULL; + (void)cbm_activation_transaction_close(&transaction); + + /* A home root others can write could have its entry swapped: refused. */ + cbm_activation_transaction_set_home_root_for_testing(home, self); + (void)chmod(home, 0775); + cbm_activation_transaction_status_t writable_home_status = + cbm_activation_transaction_stage_bytes(target, "candidate", strlen("candidate"), + &transaction); + bool writable_home_created = transaction != NULL; + (void)cbm_activation_transaction_close(&transaction); + (void)chmod(home, 0755); + + /* Trusted root, trusted entry: staged and committed into the target. */ + cbm_activation_transaction_status_t trusted_status = cbm_activation_transaction_stage_bytes( + target, "candidate", strlen("candidate"), &transaction); + cbm_activation_transaction_status_t commit_status = + transaction ? cbm_activation_transaction_commit(transaction, NULL, NULL) + : CBM_ACTIVATION_TRANSACTION_INVALID_STATE; + cbm_activation_transaction_status_t finalize_status = + transaction ? cbm_activation_transaction_finalize(transaction) + : CBM_ACTIVATION_TRANSACTION_INVALID_STATE; + cbm_activation_transaction_status_t close_status = + cbm_activation_transaction_close(&transaction); + cbm_activation_transaction_set_home_root_for_testing(NULL, 0); + + char contents[ACTIVATION_TEST_CONTENT_CAP] = ""; + bool published = activation_test_read(relocated_target, contents); + int unlink_status = unlink(entry); + int cleanup_status = th_rmtree(root); + + ASSERT_EQ(untrusted_status, CBM_ACTIVATION_TRANSACTION_IO); + ASSERT_FALSE(untrusted_created); + ASSERT_EQ(foreign_owner_status, CBM_ACTIVATION_TRANSACTION_IO); + ASSERT_FALSE(foreign_owner_created); + ASSERT_EQ(writable_home_status, CBM_ACTIVATION_TRANSACTION_IO); + ASSERT_FALSE(writable_home_created); + ASSERT_EQ(trusted_status, CBM_ACTIVATION_TRANSACTION_OK); + ASSERT_EQ(commit_status, CBM_ACTIVATION_TRANSACTION_OK); + ASSERT_EQ(finalize_status, CBM_ACTIVATION_TRANSACTION_OK); + ASSERT_EQ(close_status, CBM_ACTIVATION_TRANSACTION_OK); + ASSERT_TRUE(published); + ASSERT_STR_EQ(contents, "candidate"); + ASSERT_EQ(unlink_status, 0); + ASSERT_EQ(cleanup_status, 0); +#endif + PASS(); +} + TEST(activation_transaction_fails_closed_if_target_directory_is_replaced) { char root[ACTIVATION_TEST_PATH_CAP]; char active[ACTIVATION_TEST_PATH_CAP]; @@ -1089,6 +1178,7 @@ SUITE(activation_transaction) { RUN_TEST(activation_transaction_admits_group_writable_ancestor); RUN_TEST(activation_transaction_rejects_windows_callback_allow_directory_ace); RUN_TEST(activation_transaction_rejects_symlink_candidate_target_and_parent); + RUN_TEST(activation_transaction_follows_trusted_per_user_home_entry); RUN_TEST(activation_transaction_fails_closed_if_target_directory_is_replaced); RUN_TEST(activation_transaction_does_not_replace_target_created_at_publish_boundary); RUN_TEST(activation_transaction_rejects_macos_mutating_extended_acl); From 9140e7dfbe2e4942428e017cab8efef1f9de7f84 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Thu, 1 Oct 2026 12:55:02 -0500 Subject: [PATCH 066/217] test(pipeline): dep-info .d files leave no File, Module or Folder nodes Run the full pipeline over a cargo dep-info file alone under its own directory, and a gcc -MD file beside real D source. The dep-info files get no File or Module node, their directory chain gets no Folder node, and the D file keeps its File, Module and Folder. With the .d probe disabled the test fails with three Folder nodes under the dep-info directory. Signed-off-by: Nelson Spence --- tests/test_pipeline.c | 66 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 03a5cad70..3b84a5d74 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -631,6 +631,71 @@ TEST(pipeline_structure_nodes) { PASS(); } +/* Number of `label` nodes whose file_path starts with `prefix`, or -1. */ +static int count_nodes_with_path_prefix(cbm_store_t *s, const char *project, const char *label, + const char *prefix) { + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_label(s, project, label, &nodes, &count) != CBM_STORE_OK) { + return -1; + } + int hits = 0; + for (int i = 0; i < count; i++) { + if (nodes[i].file_path && strncmp(nodes[i].file_path, prefix, strlen(prefix)) == 0) { + hits++; + } + } + cbm_store_free_nodes(nodes, count); + return hits; +} + +/* Make-style dep-info .d files are dropped at discovery, so the graph holds no + * File or Module node for them and no Folder for a directory that held only + * dep-info. Real D source beside a dep-info file keeps its nodes and folder. */ +TEST(pipeline_d_dep_info_leaves_no_nodes) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dotd_graph_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + write_temp_file(repo, "cargo-target/debug/deps/app-0123abcd.d", + "/r/cargo-target/debug/deps/app-0123abcd.d: src/main.rs\n\nsrc/main.rs:\n"); + write_temp_file(repo, "src/app.d", "module app;\n\nvoid main() {}\n"); + write_temp_file(repo, "src/app.o.d", "src/app.o: src/app.d\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/test.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + const char *project = cbm_pipeline_project_name(p); + + int dep_folders = count_nodes_with_path_prefix(s, project, "Folder", "cargo-target"); + int dep_files = count_nodes_with_path_prefix(s, project, "File", "cargo-target"); + int dep_modules = count_nodes_with_path_prefix(s, project, "Module", "cargo-target"); + int obj_files = count_nodes_with_path_prefix(s, project, "File", "src/app.o.d"); + int obj_modules = count_nodes_with_path_prefix(s, project, "Module", "src/app.o.d"); + int src_folders = count_nodes_with_path_prefix(s, project, "Folder", "src"); + int src_files = count_nodes_with_path_prefix(s, project, "File", "src/app.d"); + int src_modules = count_nodes_with_path_prefix(s, project, "Module", "src/app.d"); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + + ASSERT_EQ(dep_folders, 0); + ASSERT_EQ(dep_files, 0); + ASSERT_EQ(dep_modules, 0); + ASSERT_EQ(obj_files, 0); + ASSERT_EQ(obj_modules, 0); + ASSERT_EQ(src_folders, 1); + ASSERT_EQ(src_files, 1); + ASSERT_EQ(src_modules, 1); + PASS(); +} + /* Issue #516: an ADR stored via manage_adr (project_summaries) must survive a * full re-index. A full re-index deletes the DB and rebuilds it from the graph * buffer, which writes an empty project_summaries table; the fix captures the @@ -16218,6 +16283,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_sql_dump_graph_matches_the_full_parse_issue1735); RUN_TEST(pipeline_spill_resolves_namespace_imports_like_memory); RUN_TEST(pipeline_structure_nodes); + RUN_TEST(pipeline_d_dep_info_leaves_no_nodes); RUN_TEST(pipeline_committed_counts_match_persisted); RUN_TEST(pipeline_adr_survives_full_reindex); RUN_TEST(pipeline_export_error_snapshot_on_artifact_failure); From ceb729d8a36b9a02778d86d0b209c7decbe58dd5 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Thu, 1 Oct 2026 14:09:50 -0500 Subject: [PATCH 067/217] fix(discover): keep D whose first colon follows an attribute or comment is_dep_rule_line took any '.' or '/' before the rule colon as a path, so valid D such as "@1.0:" (a float UDA) and "public/**/:" (a comment between tokens) was dropped as dep-info. A target holding '@' or a comment opener ("/*", "/+", "//") is now D. Over 12,062 real D files and 4,494 real files under deps/ dirs, no classification changes. A dep-info file whose target path contains '@' now stays D, as it does on main. Signed-off-by: Nelson Spence --- src/discover/language.c | 9 +++++++-- tests/test_language.c | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/discover/language.c b/src/discover/language.c index 66de4f4df..0d6d3faca 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -1342,7 +1342,9 @@ static bool is_rule_colon(const char *p, bool whole_file) { * look like paths (contain '/', '\' or '.'); a backslash-escaped space stays * inside its target. D source fails this: "public:", "@safe:", "extern(C):" * and "import a.b : c" all have a non-path word or D punctuation before the - * colon. */ + * colon. An attribute ('@', as in "@1.0:") or a comment between tokens + * ("public/+ +/:") is D even with a '.' or '/' before the colon, so a target + * holding either is never dep-info. */ static bool is_dep_rule_line(const char *p, bool whole_file) { bool in_target = false; bool path_like = false; @@ -1359,7 +1361,10 @@ static bool is_dep_rule_line(const char *p, bool whole_file) { path_like = false; continue; } - if (strchr("(){};=\"',", *p)) { + if (strchr("(){};=\"',@", *p)) { + return false; + } + if (*p == '/' && (p[SKIP_ONE] == '*' || p[SKIP_ONE] == '+' || p[SKIP_ONE] == '/')) { return false; } in_target = true; diff --git a/tests/test_language.c b/tests/test_language.c index d9d38429f..d93eb0ecb 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -878,6 +878,14 @@ TEST(lang_d_source_stays_dlang) { ASSERT_EQ(disambiguate_d_content("/+ dub.sdl:\n name \"hello\"\n+/\nvoid main() {}\n"), CBM_LANG_DLANG); ASSERT_EQ(disambiguate_d_content("enum E : ubyte {\n a,\n}\n"), CBM_LANG_DLANG); + /* A float UDA ("@ FloatLiteral") has a '.' before its colon, alone or glued + * to a keyword. */ + ASSERT_EQ(disambiguate_d_content("@1.0:\nvoid f() {}\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("nothrow@1.0:\nvoid f() {}\n"), CBM_LANG_DLANG); + /* A comment between tokens puts a '/' before the colon. */ + ASSERT_EQ(disambiguate_d_content("public/**/:\nvoid f() {}\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("public/+ +/:\nvoid f() {}\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("public//note: x\nvoid f() {}\n"), CBM_LANG_DLANG); /* Default on doubt: empty or unreadable. */ ASSERT_EQ(disambiguate_d_content(""), CBM_LANG_DLANG); ASSERT_EQ(cbm_disambiguate_d("/tmp/nonexistent_file_12345.d"), CBM_LANG_DLANG); From 006f2a991c6a8e0d141750d53959150147f876ad Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Thu, 1 Oct 2026 14:40:06 -0500 Subject: [PATCH 068/217] fix(discover): split dep-rule targets on every D whitespace is_dep_rule_line split targets only on space, tab and CR, so valid D such as "import\fstd.stdio : writeln;" kept the keyword and the dotted module name in one path-like target and was dropped as dep-info. Vertical tab, form feed and the UTF-8 separators U+2028/U+2029 now split targets too. Over 12,062 real D files and 4,494 real files under deps/ dirs, no classification changes. Signed-off-by: Nelson Spence --- src/discover/language.c | 18 +++++++++++++++++- tests/test_language.c | 10 ++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/discover/language.c b/src/discover/language.c index 0d6d3faca..c0f6f1894 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -1338,6 +1338,20 @@ static bool is_rule_colon(const char *p, bool whole_file) { (next == '\0' && whole_file)); } +/* Bytes of D whitespace at p, or 0: space, tab, vertical tab, form feed, CR + * and the UTF-8 line and paragraph separators U+2028/U+2029, all of which split + * D tokens. '\n' ends the line and is the caller's. */ +static size_t d_space_len(const char *p) { + if (*p == ' ' || *p == '\t' || *p == '\v' || *p == '\f' || *p == '\r') { + return SKIP_ONE; + } + if (strncmp(p, "\xE2\x80\xA8", SLEN("\xE2\x80\xA8")) == 0 || + strncmp(p, "\xE2\x80\xA9", SLEN("\xE2\x80\xA9")) == 0) { + return SLEN("\xE2\x80\xA8"); + } + return 0; +} + /* True if the line at p is a make rule "target...: prereq..." whose targets all * look like paths (contain '/', '\' or '.'); a backslash-escaped space stays * inside its target. D source fails this: "public:", "@safe:", "extern(C):" @@ -1353,12 +1367,14 @@ static bool is_dep_rule_line(const char *p, bool whole_file) { if (is_rule_colon(p, whole_file)) { return in_target ? path_like : any_target; } - if (*p == ' ' || *p == '\t' || *p == '\r') { + size_t space = d_space_len(p); + if (space) { if (in_target && !path_like) { return false; } in_target = false; path_like = false; + p += space - SKIP_ONE; continue; } if (strchr("(){};=\"',@", *p)) { diff --git a/tests/test_language.c b/tests/test_language.c index d93eb0ecb..9b3bdc0a9 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -886,6 +886,16 @@ TEST(lang_d_source_stays_dlang) { ASSERT_EQ(disambiguate_d_content("public/**/:\nvoid f() {}\n"), CBM_LANG_DLANG); ASSERT_EQ(disambiguate_d_content("public/+ +/:\nvoid f() {}\n"), CBM_LANG_DLANG); ASSERT_EQ(disambiguate_d_content("public//note: x\nvoid f() {}\n"), CBM_LANG_DLANG); + /* Every D whitespace splits tokens, not only space, tab and CR: form feed, + * vertical tab, U+2028 and U+2029. */ + ASSERT_EQ(disambiguate_d_content("import\fstd.stdio : writeln;\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("import\vstd.stdio : writeln;\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("import\xE2\x80\xA8" + "std.stdio : writeln;\n"), + CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("import\xE2\x80\xA9" + "std.stdio : writeln;\n"), + CBM_LANG_DLANG); /* Default on doubt: empty or unreadable. */ ASSERT_EQ(disambiguate_d_content(""), CBM_LANG_DLANG); ASSERT_EQ(cbm_disambiguate_d("/tmp/nonexistent_file_12345.d"), CBM_LANG_DLANG); From 2443e92bece270d1207c715962707faee23387c3 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:21:46 -0700 Subject: [PATCH 069/217] fix: preserve CPU headroom during background indexing Fixes #1084 Signed-off-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> --- src/daemon/application.c | 11 ++-- src/mcp/mcp.c | 6 ++- src/pipeline/pipeline.c | 35 +++++++++++-- src/pipeline/pipeline.h | 12 +++++ src/pipeline/pipeline_incremental.c | 6 +-- tests/test_daemon_application.c | 6 +++ tests/test_mcp.c | 68 ++++++++++++++++++++++++ tests/test_pipeline.c | 80 +++++++++++++++++++++++++++++ 8 files changed, 210 insertions(+), 14 deletions(-) diff --git a/src/daemon/application.c b/src/daemon/application.c index 22c62029b..91b5937ec 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -2199,10 +2199,10 @@ static char *application_auto_index_args(cbm_daemon_application_t *application, return NULL; } yyjson_mut_doc_set_root(document, root); - char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) && - application_index_args_add_policy(application, document, root) - ? yyjson_mut_write(document, 0, NULL) - : NULL; + bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) && + application_index_args_add_policy(application, document, root) && + yyjson_mut_obj_add_bool(document, root, "_background", true); + char *args = encoded ? yyjson_mut_write(document, 0, NULL) : NULL; yyjson_mut_doc_free(document); return args; } @@ -3894,7 +3894,8 @@ static int application_background_index(cbm_daemon_application_t *application, } yyjson_mut_doc_set_root(document, root); bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) && - application_index_args_add_policy(application, document, root); + application_index_args_add_policy(application, document, root) && + yyjson_mut_obj_add_bool(document, root, "_background", true); char *default_project = cbm_project_name_from_path(canonical_root); bool custom_project = project_name[0] && (!default_project || strcmp(default_project, project_name) != 0); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index c877d5110..0d15ac017 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11109,7 +11109,8 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) || - !cbm_mcp_index_policy_add_to_args(doc, root, &policy)) { + !cbm_mcp_index_policy_add_to_args(doc, root, &policy) || + !yyjson_mut_obj_add_bool(doc, root, "_background", true)) { yyjson_mut_doc_free(doc); return NULL; } @@ -11498,6 +11499,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); char *mode_str = cbm_mcp_get_string_arg(args, "mode"); char *name_override = cbm_mcp_get_string_arg(args, "name"); + bool background = cbm_mcp_get_bool_arg(args, "_background"); cbm_normalize_path_sep(repo_path); if (!repo_path) { @@ -11660,6 +11662,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(repo_path); return cbm_mcp_text_result("failed to create pipeline", true); } + cbm_pipeline_set_background(p, background); if (name_override && name_override[0] && !cbm_pipeline_set_project_name(p, name_override)) { cbm_pipeline_free(p); mcp_project_mutation_end(srv, mutation_project); @@ -18106,6 +18109,7 @@ static void *autoindex_thread(void *arg) { cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed"); return NULL; } + cbm_pipeline_set_background(p, true); /* Block until any concurrent pipeline finishes */ cbm_pipeline_lock(); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index ba41b877c..f2aeb47be 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -71,6 +71,18 @@ static inline void *intptr_to_ptr(intptr_t v) { * Atomic spinlock: 0 = free, 1 = locked. */ static atomic_int g_pipeline_busy = 0; +#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API +static atomic_int g_last_worker_count = 0; + +void cbm_pipeline_worker_count_test_reset(void) { + atomic_store(&g_last_worker_count, 0); +} + +int cbm_pipeline_worker_count_test_last(void) { + return atomic_load(&g_last_worker_count); +} +#endif + #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API static atomic_bool g_persist_test_fail_after_stage_dump = false; static atomic_bool g_persist_test_cancel_after_predump = false; @@ -192,6 +204,7 @@ struct cbm_pipeline { bool persistence; /* write .codebase-memory/graph.db.zst after indexing */ cbm_index_resource_policy_t resource_policy; cbm_index_resource_violation_t resource_violation; + bool background; /* reserve CPU headroom when no user is waiting */ /* Snapshot of the artifact export failure of THIS run (set only by * export_after_publish failure, zeroed at run start, cleared on success). @@ -400,6 +413,12 @@ const char *cbm_pipeline_export_error(const cbm_pipeline_t *p) { return p ? p->export_error : ""; } +void cbm_pipeline_set_background(cbm_pipeline_t *p, bool background) { + if (p) { + p->background = background; + } +} + bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) { if (!p || !name || !name[0]) { return false; @@ -613,12 +632,18 @@ void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges) * crasher; a parallel re-run would race the marker. Honour that override * everywhere the worker count drives the parallel/sequential decision, so the * whole extraction phase collapses to the deterministic sequential path. */ -static int effective_worker_count(bool initial) { +int cbm_pipeline_worker_count(const cbm_pipeline_t *p) { const char *st = getenv("CBM_INDEX_SINGLE_THREAD"); + int workers; if (st && st[0] == '1') { - return 1; + workers = 1; + } else { + workers = cbm_default_worker_count(!p || !p->background); } - return cbm_default_worker_count(initial); +#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API + atomic_store(&g_last_worker_count, workers); +#endif + return workers; } /* Resolve the DB path for this pipeline. Caller must free(). */ @@ -2590,7 +2615,7 @@ static int run_githistory(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { gh_compute_arg_t gh_arg = {.repo_path = ctx->repo_path, .result = &gh_result}; if (p->mode != CBM_MODE_FAST) { - if (effective_worker_count(true) > SKIP_ONE) { + if (cbm_pipeline_worker_count(p) > SKIP_ONE) { if (cbm_thread_create(&gh_thread, 0, gh_compute_thread_fn, &gh_arg) == 0) { gh_threaded = true; } @@ -2690,7 +2715,7 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, return CBM_NOT_FOUND; } - int worker_count = effective_worker_count(true); + int worker_count = cbm_pipeline_worker_count(p); CBM_PROF_START(t_extract_total); int rc = (worker_count > SKIP_ONE && file_count > MIN_FILES_FOR_PARALLEL) ? run_parallel_pipeline(p, ctx, files, file_count, worker_count, &t) diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index c1d571ecd..e52513673 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -59,6 +59,18 @@ void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resourc /* Copy the exact discovery violation from the most recent run. */ void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, cbm_index_resource_violation_t *violation); +/* Mark work that runs without a waiting user. Background pipelines reserve + * CPU headroom; foreground pipelines retain the initial-index all-core policy. */ +void cbm_pipeline_set_background(cbm_pipeline_t *p, bool background); + +/* Resolve the worker policy for this pipeline, including environment and + * crash-recovery overrides. */ +int cbm_pipeline_worker_count(const cbm_pipeline_t *p); + +#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API +void cbm_pipeline_worker_count_test_reset(void); +int cbm_pipeline_worker_count_test_last(void); +#endif /* Snapshot of the artifact export failure of the last cbm_pipeline_run, or "" * when the run succeeded / did not reach post-publish export. Used to diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 154518658..4e7d61dc3 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1212,7 +1212,7 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed * a full build takes, which is what makes its output converge. */ #define MIN_FILES_FOR_PARALLEL_INCR 50 - int worker_count = cbm_default_worker_count(true); + int worker_count = cbm_pipeline_worker_count(ctx->pipeline); bool use_parallel = closure != NULL || (worker_count > SKIP_ONE && ci > MIN_FILES_FOR_PARALLEL_INCR); @@ -1590,7 +1590,7 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project, _Atomic int64_t probe_ids; atomic_init(&probe_ids, cbm_gbuf_next_id(probe_gbuf)); rc = cbm_parallel_extract(&probe_ctx, probe_files, probe_count, cache, &probe_ids, - cbm_default_worker_count(true)); + cbm_pipeline_worker_count(p)); } if (rc == 0) { char **def_modules = (char **)calloc((size_t)probe_count, sizeof(char *)); @@ -2061,7 +2061,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char elig[elig_count++] = row; } } - int workers = cbm_default_worker_count(true); + int workers = cbm_pipeline_worker_count(p); if (workers < 1) { workers = 1; } diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 7f4c9cc28..50abe5a8c 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1375,6 +1375,7 @@ typedef struct { * itself instead, which is the edge a reader on another thread needs. */ atomic_bool args_captured[APP_FAKE_MAX_ATTEMPTS]; size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS]; + bool background_requests[APP_FAKE_MAX_ATTEMPTS]; } app_fake_worker_context_t; typedef struct { @@ -1442,6 +1443,8 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem args_json ? args_json : ""); atomic_store(&context->args_captured[worker->attempt], true); context->memory_budgets[worker->attempt] = memory_budget_bytes; + context->background_requests[worker->attempt] = + cbm_mcp_get_bool_arg(args_json, "_background"); if (marker_file) { (void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s", marker_file); @@ -2202,6 +2205,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { ASSERT_TRUE(first_initialized); ASSERT_TRUE(first_owned); ASSERT_TRUE(auto_policy_propagated); + ASSERT_TRUE(fake.background_requests[0]); ASSERT_TRUE(second_initialized); ASSERT_TRUE(coalesced); ASSERT_TRUE(restricted_disconnect_kept_job); @@ -3634,6 +3638,7 @@ TEST(daemon_application_request_cancel_preserves_persistent_watch_and_session) { ASSERT_TRUE(thread_started); ASSERT_TRUE(subscribed); ASSERT_TRUE(worker_started); + ASSERT_FALSE(fixture.fake.background_requests[0]); ASSERT_TRUE(request_returned); ASSERT_TRUE(thread_joined); ASSERT_EQ(request.status, CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED); @@ -3931,6 +3936,7 @@ TEST(daemon_application_final_cancel_drains_admitted_watcher_job) { ASSERT_TRUE(fixture_ready); ASSERT_TRUE(thread_started); ASSERT_TRUE(worker_started); + ASSERT_TRUE(fixture.fake.background_requests[0]); ASSERT_TRUE(job_active); ASSERT_EQ(watches_after_cancel, 0); ASSERT_TRUE(thread_joined); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 22af972af..34563e0c9 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -20006,6 +20006,73 @@ TEST(autoindex_limit_guards_git_root_issue713) { PASS(); } +TEST(mcp_auto_index_in_process_uses_background_worker_policy) { + char cache[256]; + char repo[512]; + (void)snprintf(cache, sizeof(cache), "%s/cbm-autoindex-workers-XXXXXX", cbm_tmpdir()); + bool cache_ready = cbm_mkdtemp(cache) != NULL; + (void)snprintf(repo, sizeof(repo), "%s/repo", cache); + char source[640]; + (void)snprintf(source, sizeof(source), "%s/main.py", repo); + bool repo_ready = cache_ready && th_mkdir_p(repo) == 0 && + th_write_file(source, "def background_index():\n return True\n") == 0; + + mcp_test_env_backup_t environment[] = { + {.name = "CBM_CACHE_DIR"}, + {.name = "CBM_WORKERS"}, + {.name = "CBM_INDEX_SINGLE_THREAD"}, + }; + bool environment_saved = true; + for (size_t i = 0; i < sizeof(environment) / sizeof(environment[0]); i++) { + const char *value = getenv(environment[i].name); + environment[i].present = value != NULL; + environment[i].value = value ? strdup(value) : NULL; + environment_saved = environment_saved && (!value || environment[i].value); + } + bool environment_ready = + environment_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 && + cbm_unsetenv("CBM_WORKERS") == 0 && cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") == 0; + + char old_cwd[CBM_SZ_4K] = {0}; + bool cwd_ready = repo_ready && environment_ready && cbm_getcwd(old_cwd, sizeof(old_cwd)) && + cbm_chdir(repo) == 0; + cbm_config_t *config = cwd_ready ? cbm_config_open(cache) : NULL; + bool config_ready = config && cbm_config_set(config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(config, CBM_CONFIG_AUTO_WATCH, "false") == 0; + cbm_pipeline_worker_count_test_reset(); + cbm_mcp_server_t *server = config_ready ? cbm_mcp_server_new(NULL) : NULL; + char *response = NULL; + if (server) { + cbm_mcp_server_set_config(server, config); + response = cbm_mcp_server_handle( + server, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}"); + cbm_mcp_server_free(server); /* joins the automatic index thread */ + } + int selected_workers = cbm_pipeline_worker_count_test_last(); + bool server_ready = server != NULL; + bool response_ready = response != NULL; + + free(response); + cbm_config_close(config); + if (cwd_ready) { + (void)cbm_chdir(old_cwd); + } + mcp_test_restore_env(environment, sizeof(environment) / sizeof(environment[0])); + bool cleaned = !cache_ready || th_rmtree(cache) == 0; + + ASSERT_TRUE(cache_ready); + ASSERT_TRUE(repo_ready); + ASSERT_TRUE(environment_saved); + ASSERT_TRUE(environment_ready); + ASSERT_TRUE(cwd_ready); + ASSERT_TRUE(config_ready); + ASSERT_TRUE(server_ready); + ASSERT_TRUE(response_ready); + ASSERT_EQ(selected_workers, cbm_default_worker_count(false)); + ASSERT_TRUE(cleaned); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * #853 — auto_watch=false must ALSO gate the SUPERVISED fresh-index * watcher registration (keystone × #849 merge interaction) @@ -21361,6 +21428,7 @@ SUITE(mcp) { RUN_TEST(autoindex_limit_guards_non_git_root_issue713); RUN_TEST(autoindex_limit_admits_non_git_root_under_limit_issue713); RUN_TEST(autoindex_limit_guards_git_root_issue713); + RUN_TEST(mcp_auto_index_in_process_uses_background_worker_policy); } /* Kept separate so daemon-coordination regressions can be iterated without diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 56be6b88e..32f4b1a4c 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -117,6 +117,84 @@ TEST(pipeline_create_free) { PASS(); } +TEST(pipeline_background_worker_policy_preserves_overrides) { + const char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + cbm_unsetenv("CBM_WORKERS"); + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + + cbm_pipeline_t *foreground = cbm_pipeline_new("/some/path", NULL, CBM_MODE_FULL); + cbm_pipeline_t *background = cbm_pipeline_new("/some/path", NULL, CBM_MODE_FULL); + ASSERT_NOT_NULL(foreground); + ASSERT_NOT_NULL(background); + cbm_pipeline_set_background(background, true); + + int foreground_default = cbm_pipeline_worker_count(foreground); + int background_default = cbm_pipeline_worker_count(background); + bool defaults_match = foreground_default == cbm_default_worker_count(true) && + background_default == cbm_default_worker_count(false) && + foreground_default >= 1 && background_default >= 1; + + cbm_setenv("CBM_WORKERS", "3", 1); + bool env_override = cbm_pipeline_worker_count(foreground) == 3 && + cbm_pipeline_worker_count(background) == 3; + + cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1); + bool recovery_override = cbm_pipeline_worker_count(foreground) == 1 && + cbm_pipeline_worker_count(background) == 1; + + cbm_pipeline_free(foreground); + cbm_pipeline_free(background); + saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS"); + saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1) + : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + free(saved_workers); + free(saved_single); + + ASSERT_TRUE(defaults_match); + ASSERT_TRUE(env_override); + ASSERT_TRUE(recovery_override); + PASS(); +} + +TEST(pipeline_background_policy_preserves_index_results) { + ASSERT_EQ(setup_test_repo(), 0); + char foreground_db[512]; + char background_db[512]; + (void)snprintf(foreground_db, sizeof(foreground_db), "%s/foreground.db", g_tmpdir); + (void)snprintf(background_db, sizeof(background_db), "%s/background.db", g_tmpdir); + + cbm_pipeline_t *foreground = + cbm_pipeline_new(g_tmpdir, foreground_db, CBM_MODE_FULL); + cbm_pipeline_t *background = + cbm_pipeline_new(g_tmpdir, background_db, CBM_MODE_FULL); + ASSERT_NOT_NULL(foreground); + ASSERT_NOT_NULL(background); + cbm_pipeline_set_background(background, true); + + int foreground_rc = cbm_pipeline_run(foreground); + int background_rc = cbm_pipeline_run(background); + int foreground_nodes = -1; + int foreground_edges = -1; + int background_nodes = -1; + int background_edges = -1; + cbm_pipeline_get_committed_counts(foreground, &foreground_nodes, &foreground_edges); + cbm_pipeline_get_committed_counts(background, &background_nodes, &background_edges); + + cbm_pipeline_free(foreground); + cbm_pipeline_free(background); + teardown_test_repo(); + + ASSERT_EQ(foreground_rc, 0); + ASSERT_EQ(background_rc, 0); + ASSERT_GT(foreground_nodes, 0); + ASSERT_EQ(background_nodes, foreground_nodes); + ASSERT_EQ(background_edges, foreground_edges); + PASS(); +} + TEST(pipeline_null_repo) { cbm_pipeline_t *p = cbm_pipeline_new(NULL, NULL, CBM_MODE_FULL); ASSERT_NULL(p); @@ -16284,6 +16362,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_lock_release_allows_contender); /* Lifecycle */ RUN_TEST(pipeline_create_free); + RUN_TEST(pipeline_background_worker_policy_preserves_overrides); + RUN_TEST(pipeline_background_policy_preserves_index_results); RUN_TEST(pipeline_null_repo); RUN_TEST(pipeline_free_null); RUN_TEST(pipeline_cancel); From 9bfd5cbe609e24f2ee77dd05f48aefed2199bfba Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:56:56 -0700 Subject: [PATCH 070/217] fix: route the remaining all-core sites and fix the review findings Addresses the four blockers from review: - Capture the expected worker count while CBM_WORKERS is still unset in mcp_auto_index_in_process_uses_background_worker_policy, so the assertion no longer reads the lane override restored just before it. - Give pipeline_background_policy_preserves_index_results a fixture past MIN_FILES_FOR_PARALLEL and drop any inherited CBM_WORKERS / CBM_INDEX_SINGLE_THREAD pin, so it really compares an all-core index against a headroom index instead of two sequential runs. - Route the two remaining all-core sites through cbm_pipeline_worker_count: the LSP surface pass and incremental manifest hashing. The pipeline is plumbed into cbm_pipeline_build_semantic_manifest for that. - Rename the hidden key _background to _cbm_background, matching the existing _cbm_index_policy convention, across every producer and consumer. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Du315CaKLufAYPEdEwcocq Signed-off-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> --- src/daemon/application.c | 4 +- src/mcp/mcp.c | 4 +- src/pipeline/lsp_surface.c | 3 +- src/pipeline/pipeline.c | 2 +- src/pipeline/pipeline_incremental.c | 22 ++++++---- src/pipeline/pipeline_internal.h | 6 +-- tests/test_daemon_application.c | 2 +- tests/test_mcp.c | 5 ++- tests/test_pipeline.c | 64 ++++++++++++++++++++++------- 9 files changed, 78 insertions(+), 34 deletions(-) diff --git a/src/daemon/application.c b/src/daemon/application.c index 91b5937ec..d1f78841b 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -2201,7 +2201,7 @@ static char *application_auto_index_args(cbm_daemon_application_t *application, yyjson_mut_doc_set_root(document, root); bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) && application_index_args_add_policy(application, document, root) && - yyjson_mut_obj_add_bool(document, root, "_background", true); + yyjson_mut_obj_add_bool(document, root, "_cbm_background", true); char *args = encoded ? yyjson_mut_write(document, 0, NULL) : NULL; yyjson_mut_doc_free(document); return args; @@ -3895,7 +3895,7 @@ static int application_background_index(cbm_daemon_application_t *application, yyjson_mut_doc_set_root(document, root); bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) && application_index_args_add_policy(application, document, root) && - yyjson_mut_obj_add_bool(document, root, "_background", true); + yyjson_mut_obj_add_bool(document, root, "_cbm_background", true); char *default_project = cbm_project_name_from_path(canonical_root); bool custom_project = project_name[0] && (!default_project || strcmp(default_project, project_name) != 0); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 0d15ac017..cd7b5c711 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11110,7 +11110,7 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p yyjson_mut_doc_set_root(doc, root); if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) || !cbm_mcp_index_policy_add_to_args(doc, root, &policy) || - !yyjson_mut_obj_add_bool(doc, root, "_background", true)) { + !yyjson_mut_obj_add_bool(doc, root, "_cbm_background", true)) { yyjson_mut_doc_free(doc); return NULL; } @@ -11499,7 +11499,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); char *mode_str = cbm_mcp_get_string_arg(args, "mode"); char *name_override = cbm_mcp_get_string_arg(args, "name"); - bool background = cbm_mcp_get_bool_arg(args, "_background"); + bool background = cbm_mcp_get_bool_arg(args, "_cbm_background"); cbm_normalize_path_sep(repo_path); if (!repo_path) { diff --git a/src/pipeline/lsp_surface.c b/src/pipeline/lsp_surface.c index 4c3cde336..95ed7f63b 100644 --- a/src/pipeline/lsp_surface.c +++ b/src/pipeline/lsp_surface.c @@ -231,8 +231,9 @@ int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *projec .rows = rows, }; atomic_init(&job.failed, false); + int workers = cbm_pipeline_worker_count(ctx ? ctx->pipeline : NULL); cbm_parallel_for(file_count, surface_row_one, &job, - (cbm_parallel_for_opts_t){.max_workers = 0, .force_pthreads = false}); + (cbm_parallel_for_opts_t){.max_workers = workers, .force_pthreads = false}); if (atomic_load_explicit(&job.failed, memory_order_relaxed)) { cbm_store_free_lsp_surfaces(rows, file_count); /* untouched rows are all NULL */ return -1; diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index f2aeb47be..6f6655c5f 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -2810,7 +2810,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { rc = mode_promoted ? cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &baseline_manifest, &baseline_count) - : cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files, + : cbm_pipeline_build_semantic_manifest(p, p->project_name, p->repo_path, files, file_count, p->excluded_dirs, p->excluded_count, &p->git_ctx, p->userconfig, &baseline_manifest, &baseline_count); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 4e7d61dc3..8c348c3f7 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -413,9 +413,9 @@ static void *manifest_hash_worker(void *arg) { enum { MANIFEST_PARALLEL_MIN_FILES = 64 }; -int cbm_pipeline_build_semantic_manifest(const char *project, const char *repo_path, - const cbm_file_info_t *files, int file_count, - char **excluded_dirs, int excluded_count, +int cbm_pipeline_build_semantic_manifest(const cbm_pipeline_t *p, const char *project, + const char *repo_path, const cbm_file_info_t *files, + int file_count, char **excluded_dirs, int excluded_count, const cbm_git_context_t *git_ctx, const cbm_userconfig_t *userconfig, cbm_file_hash_t **out, int *out_count) { @@ -456,7 +456,7 @@ int cbm_pipeline_build_semantic_manifest(const char *project, const char *repo_p } struct timespec t_hash; cbm_clock_gettime(CLOCK_MONOTONIC, &t_hash); - int hash_workers = cbm_default_worker_count(true); + int hash_workers = cbm_pipeline_worker_count(p); if (rc == 0 && file_count >= MANIFEST_PARALLEL_MIN_FILES && hash_workers > SKIP_ONE) { char (*shas)[CBM_SHA256_HEX_LEN + 1] = malloc((size_t)file_count * sizeof(*shas)); int64_t *mtimes = calloc((size_t)file_count, sizeof(int64_t)); @@ -634,9 +634,9 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr &fresh_ignored_total); } if (rc == 0) { - rc = cbm_pipeline_build_semantic_manifest(project, repo_path, fresh_files, fresh_file_count, - fresh_excluded, fresh_excluded_count, - &fresh_git_ctx, fresh_userconfig, out, out_count); + rc = cbm_pipeline_build_semantic_manifest( + p, project, repo_path, fresh_files, fresh_file_count, fresh_excluded, + fresh_excluded_count, &fresh_git_ctx, fresh_userconfig, out, out_count); } cbm_set_user_lang_config(previous_userconfig); cbm_git_context_free(&fresh_git_ctx); @@ -1602,8 +1602,12 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project, if (def_modules && def_starts) { defs = cbm_pxc_collect_all_defs(NULL, &probe_arena, cache, probe_files, probe_count, project, def_modules, &def_count, def_starts); - rc = cbm_lsp_surface_build_rows(NULL, project, cache, probe_files, probe_count, defs, - def_starts, out_rows, out_count); + /* Extract left pipeline NULL so file errors are not recorded twice. + * Surface rows only borrow it for the background worker policy. */ + probe_ctx.pipeline = p; + rc = cbm_lsp_surface_build_rows(&probe_ctx, project, cache, probe_files, probe_count, + defs, def_starts, out_rows, out_count); + probe_ctx.pipeline = NULL; } else { rc = -1; } diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 7f266eae9..c2a87e0bf 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -808,9 +808,9 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil #define CBM_SEMANTIC_INPUT_GLOBAL_CONFIG CBM_SEMANTIC_INPUT_PREFIX "global-extension-config-v1" #define CBM_SEMANTIC_INPUT_PROJECT_CONFIG CBM_SEMANTIC_INPUT_PREFIX "project-extension-config-v1" -int cbm_pipeline_build_semantic_manifest(const char *project, const char *repo_path, - const cbm_file_info_t *files, int file_count, - char **excluded_dirs, int excluded_count, +int cbm_pipeline_build_semantic_manifest(const cbm_pipeline_t *p, const char *project, + const char *repo_path, const cbm_file_info_t *files, + int file_count, char **excluded_dirs, int excluded_count, const cbm_git_context_t *git_ctx, const cbm_userconfig_t *userconfig, cbm_file_hash_t **out, int *out_count); diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 50abe5a8c..0bec333a8 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1444,7 +1444,7 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem atomic_store(&context->args_captured[worker->attempt], true); context->memory_budgets[worker->attempt] = memory_budget_bytes; context->background_requests[worker->attempt] = - cbm_mcp_get_bool_arg(args_json, "_background"); + cbm_mcp_get_bool_arg(args_json, "_cbm_background"); if (marker_file) { (void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s", marker_file); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 34563e0c9..d334c4774 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -20057,6 +20057,9 @@ TEST(mcp_auto_index_in_process_uses_background_worker_policy) { if (cwd_ready) { (void)cbm_chdir(old_cwd); } + /* Capture while CBM_WORKERS is still unset. Restoring first makes the + * expected count follow the lane override instead of the headroom policy. */ + int expected = cbm_default_worker_count(false); mcp_test_restore_env(environment, sizeof(environment) / sizeof(environment[0])); bool cleaned = !cache_ready || th_rmtree(cache) == 0; @@ -20068,7 +20071,7 @@ TEST(mcp_auto_index_in_process_uses_background_worker_policy) { ASSERT_TRUE(config_ready); ASSERT_TRUE(server_ready); ASSERT_TRUE(response_ready); - ASSERT_EQ(selected_workers, cbm_default_worker_count(false)); + ASSERT_EQ(selected_workers, expected); ASSERT_TRUE(cleaned); PASS(); } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 32f4b1a4c..c541db5e2 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -161,32 +161,68 @@ TEST(pipeline_background_worker_policy_preserves_overrides) { TEST(pipeline_background_policy_preserves_index_results) { ASSERT_EQ(setup_test_repo(), 0); + /* 64 fillers plus setup_test_repo's three files exceed MIN_FILES_FOR_PARALLEL + * (50). Below that threshold both policies take the sequential path, so the + * comparison stays green even with the background policy removed. */ + for (int i = 0; i < 64; i++) { + char name[32]; + char body[96]; + (void)snprintf(name, sizeof(name), "pad%02d.go", i); + (void)snprintf(body, sizeof(body), "package main\n\nfunc Pad%02d() int {\n\treturn %d\n}\n", + i, i); + if (th_write_file(TH_PATH(g_tmpdir, name), body) != 0) { + teardown_test_repo(); + FAIL("failed to pad fixture past MIN_FILES_FOR_PARALLEL"); + } + } + + /* An inherited CBM_WORKERS pin makes both runs use the same count. Drop it + * so this compares an all-core index against a headroom index. */ + const char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + if ((old_workers && !saved_workers) || (old_single && !saved_single)) { + free(saved_workers); + free(saved_single); + teardown_test_repo(); + FAIL("failed to save worker env"); + } + cbm_unsetenv("CBM_WORKERS"); + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + char foreground_db[512]; char background_db[512]; (void)snprintf(foreground_db, sizeof(foreground_db), "%s/foreground.db", g_tmpdir); (void)snprintf(background_db, sizeof(background_db), "%s/background.db", g_tmpdir); - cbm_pipeline_t *foreground = - cbm_pipeline_new(g_tmpdir, foreground_db, CBM_MODE_FULL); - cbm_pipeline_t *background = - cbm_pipeline_new(g_tmpdir, background_db, CBM_MODE_FULL); - ASSERT_NOT_NULL(foreground); - ASSERT_NOT_NULL(background); - cbm_pipeline_set_background(background, true); - - int foreground_rc = cbm_pipeline_run(foreground); - int background_rc = cbm_pipeline_run(background); + cbm_pipeline_t *foreground = cbm_pipeline_new(g_tmpdir, foreground_db, CBM_MODE_FULL); + cbm_pipeline_t *background = cbm_pipeline_new(g_tmpdir, background_db, CBM_MODE_FULL); + int foreground_rc = -1; + int background_rc = -1; int foreground_nodes = -1; int foreground_edges = -1; int background_nodes = -1; int background_edges = -1; - cbm_pipeline_get_committed_counts(foreground, &foreground_nodes, &foreground_edges); - cbm_pipeline_get_committed_counts(background, &background_nodes, &background_edges); + bool created = foreground && background; + if (created) { + cbm_pipeline_set_background(background, true); + foreground_rc = cbm_pipeline_run(foreground); + background_rc = cbm_pipeline_run(background); + cbm_pipeline_get_committed_counts(foreground, &foreground_nodes, &foreground_edges); + cbm_pipeline_get_committed_counts(background, &background_nodes, &background_edges); + } cbm_pipeline_free(foreground); cbm_pipeline_free(background); teardown_test_repo(); + saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS"); + saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1) + : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + free(saved_workers); + free(saved_single); + ASSERT_TRUE(created); ASSERT_EQ(foreground_rc, 0); ASSERT_EQ(background_rc, 0); ASSERT_GT(foreground_nodes, 0); @@ -5458,8 +5494,8 @@ TEST(pipeline_semantic_manifest_rejects_non_directory_root) { cbm_file_hash_t *manifest = NULL; int manifest_count = -1; - int rc = cbm_pipeline_build_semantic_manifest("manifest-fail-closed", root_path, NULL, 0, NULL, - 0, NULL, NULL, &manifest, &manifest_count); + int rc = cbm_pipeline_build_semantic_manifest(NULL, "manifest-fail-closed", root_path, NULL, 0, + NULL, 0, NULL, NULL, &manifest, &manifest_count); cbm_pipeline_free_semantic_manifest(manifest, manifest_count > 0 ? manifest_count : 0); th_rmtree(tmp); From 26eece514fef17bef35cafb23b16c647c3358aaf Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:55:29 -0700 Subject: [PATCH 071/217] fix(mcp): keep full width for a project's first index Signed-off-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> --- src/mcp/mcp.c | 23 +++++++- src/pipeline/pipeline.c | 2 +- tests/test_mcp.c | 128 +++++++++++++++++++++++++++++++++------- 3 files changed, 129 insertions(+), 24 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index cd7b5c711..31efa9420 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11370,6 +11370,22 @@ static bool index_root_owner_resolve(const char *repo_path, char **owner_out, ch return false; } +/* Background CPU headroom is for a refresh of a graph that already serves. + * A project's first index keeps every core, even when it was started + * automatically: the caller is waiting on a graph that does not exist yet. + * An empty or unreadable file is not a committed index. Both sites that + * honour automatic indexing — handle_index_repository's _cbm_background + * flag and the in-process autoindex_thread — go through this so they + * cannot drift. */ +static bool project_has_committed_index(const char *project) { + if (!project || !project[0]) { + return false; + } + char db_path[CBM_SZ_1K]; + project_db_path(project, db_path, sizeof(db_path)); + return db_path[0] && project_db_is_servable(project, db_path); +} + /* The three heap strings handle_index_repository owns from * cbm_mcp_get_string_arg / resolved_repo_path_from_project_arg. One release * point keeps the dozen early-return paths in step; free(NULL) is a no-op, so @@ -11662,7 +11678,6 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(repo_path); return cbm_mcp_text_result("failed to create pipeline", true); } - cbm_pipeline_set_background(p, background); if (name_override && name_override[0] && !cbm_pipeline_set_project_name(p, name_override)) { cbm_pipeline_free(p); mcp_project_mutation_end(srv, mutation_project); @@ -11672,6 +11687,10 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { return cbm_mcp_text_result("invalid project name", true); } free(name_override); + /* Headroom follows the final project name, decided before artifact + * bootstrap can create the first local copy of this index. */ + cbm_pipeline_set_background(p, background && + project_has_committed_index(cbm_pipeline_project_name(p))); cbm_pipeline_set_persistence(p, persistence); cbm_pipeline_set_resource_policy(p, &resource_policy); @@ -18109,7 +18128,7 @@ static void *autoindex_thread(void *arg) { cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed"); return NULL; } - cbm_pipeline_set_background(p, true); + cbm_pipeline_set_background(p, project_has_committed_index(cbm_pipeline_project_name(p))); /* Block until any concurrent pipeline finishes */ cbm_pipeline_lock(); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 6f6655c5f..be881942e 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -204,7 +204,7 @@ struct cbm_pipeline { bool persistence; /* write .codebase-memory/graph.db.zst after indexing */ cbm_index_resource_policy_t resource_policy; cbm_index_resource_violation_t resource_violation; - bool background; /* reserve CPU headroom when no user is waiting */ + bool background; /* reserve CPU headroom when no user is waiting */ /* Snapshot of the artifact export failure of THIS run (set only by * export_after_publish failure, zeroed at run start, cleared on success). diff --git a/tests/test_mcp.c b/tests/test_mcp.c index d334c4774..f42f0dc6a 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -20006,10 +20006,43 @@ TEST(autoindex_limit_guards_git_root_issue713) { PASS(); } -TEST(mcp_auto_index_in_process_uses_background_worker_policy) { +/* A servable project row is a committed index. user_version stays 0, so a + * later run cannot take the incremental no-op and skip recording workers. */ +static bool mcp_publish_committed_index(const char *root) { + char *project = root ? cbm_project_name_from_path(root) : NULL; + cbm_store_t *store = project ? cbm_store_open(project) : NULL; + bool published = store && cbm_store_upsert_project(store, project, root) == CBM_STORE_OK; + cbm_store_close(store); + free(project); + return published; +} + +#ifdef CBM_ENABLE_TEST_SEAMS +typedef struct { + const char *root; + bool published; +} mcp_committed_seed_t; + +/* maybe_auto_index returns before the thread when .db already + * exists. Publish the committed index from the count hook, which runs after + * that check and before autoindex_thread chooses a width. */ +static void mcp_autoindex_publish_committed(void *context) { + mcp_committed_seed_t *seed = context; + if (seed) { + seed->published = mcp_publish_committed_index(seed->root); + } +} +#endif + +/* First index, including one started automatically, uses every core. Headroom + * applies only once a servable project database already exists. + * automatic: in-process auto-index (initialize → autoindex_thread). + * Otherwise index_repository with _cbm_background, the daemon/watcher path. */ +static int mcp_assert_inprocess_worker_policy(bool committed, bool automatic) { char cache[256]; char repo[512]; - (void)snprintf(cache, sizeof(cache), "%s/cbm-autoindex-workers-XXXXXX", cbm_tmpdir()); + (void)snprintf(cache, sizeof(cache), "%s/cbm-worker-policy-%d%d-XXXXXX", cbm_tmpdir(), + committed ? 1 : 0, automatic ? 1 : 0); bool cache_ready = cbm_mkdtemp(cache) != NULL; (void)snprintf(repo, sizeof(repo), "%s/repo", cache); char source[640]; @@ -20029,24 +20062,57 @@ TEST(mcp_auto_index_in_process_uses_background_worker_policy) { environment[i].value = value ? strdup(value) : NULL; environment_saved = environment_saved && (!value || environment[i].value); } - bool environment_ready = - environment_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 && - cbm_unsetenv("CBM_WORKERS") == 0 && cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") == 0; + bool environment_ready = environment_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 && + cbm_unsetenv("CBM_WORKERS") == 0 && + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") == 0; + bool seeded = !committed; char old_cwd[CBM_SZ_4K] = {0}; - bool cwd_ready = repo_ready && environment_ready && cbm_getcwd(old_cwd, sizeof(old_cwd)) && - cbm_chdir(repo) == 0; - cbm_config_t *config = cwd_ready ? cbm_config_open(cache) : NULL; - bool config_ready = config && cbm_config_set(config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && - cbm_config_set(config, CBM_CONFIG_AUTO_WATCH, "false") == 0; - cbm_pipeline_worker_count_test_reset(); - cbm_mcp_server_t *server = config_ready ? cbm_mcp_server_new(NULL) : NULL; + bool cwd_ready = true; + cbm_config_t *config = NULL; + bool config_ready = true; + cbm_mcp_server_t *server = NULL; char *response = NULL; - if (server) { - cbm_mcp_server_set_config(server, config); - response = cbm_mcp_server_handle( - server, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}"); - cbm_mcp_server_free(server); /* joins the automatic index thread */ + if (automatic) { + cwd_ready = repo_ready && environment_ready && cbm_getcwd(old_cwd, sizeof(old_cwd)) && + cbm_chdir(repo) == 0; + config = cwd_ready ? cbm_config_open(cache) : NULL; + config_ready = config && cbm_config_set(config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(config, CBM_CONFIG_AUTO_WATCH, "false") == 0; + cbm_pipeline_worker_count_test_reset(); + server = config_ready ? cbm_mcp_server_new(NULL) : NULL; + if (server) { + cbm_mcp_server_set_config(server, config); +#ifdef CBM_ENABLE_TEST_SEAMS + mcp_committed_seed_t seed = {.root = repo, .published = !committed}; + if (committed) { + cbm_mcp_server_set_auto_index_count_test_hook( + server, mcp_autoindex_publish_committed, &seed); + } +#endif + response = cbm_mcp_server_handle( + server, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}"); +#ifdef CBM_ENABLE_TEST_SEAMS + seeded = seed.published; +#endif + cbm_mcp_server_free(server); /* joins the automatic index thread */ + } + } else { + if (committed && repo_ready && environment_ready) { + seeded = mcp_publish_committed_index(repo); + } + cbm_pipeline_worker_count_test_reset(); + server = repo_ready && environment_ready && seeded ? cbm_mcp_server_new(NULL) : NULL; + if (server) { + char repo_json[512]; + char args[CBM_SZ_1K]; + (void)snprintf(repo_json, sizeof(repo_json), "%s", repo); + cbm_normalize_path_sep(repo_json); + (void)snprintf(args, sizeof(args), "{\"_cbm_background\":true,\"repo_path\":\"%s\"}", + repo_json); + response = cbm_mcp_handle_tool(server, "index_repository", args); + cbm_mcp_server_free(server); + } } int selected_workers = cbm_pipeline_worker_count_test_last(); bool server_ready = server != NULL; @@ -20054,12 +20120,12 @@ TEST(mcp_auto_index_in_process_uses_background_worker_policy) { free(response); cbm_config_close(config); - if (cwd_ready) { + if (automatic && cwd_ready) { (void)cbm_chdir(old_cwd); } /* Capture while CBM_WORKERS is still unset. Restoring first makes the - * expected count follow the lane override instead of the headroom policy. */ - int expected = cbm_default_worker_count(false); + * expected count follow the lane override instead of this policy. */ + int expected = cbm_default_worker_count(!committed); mcp_test_restore_env(environment, sizeof(environment) / sizeof(environment[0])); bool cleaned = !cache_ready || th_rmtree(cache) == 0; @@ -20069,6 +20135,7 @@ TEST(mcp_auto_index_in_process_uses_background_worker_policy) { ASSERT_TRUE(environment_ready); ASSERT_TRUE(cwd_ready); ASSERT_TRUE(config_ready); + ASSERT_TRUE(seeded); ASSERT_TRUE(server_ready); ASSERT_TRUE(response_ready); ASSERT_EQ(selected_workers, expected); @@ -20076,6 +20143,22 @@ TEST(mcp_auto_index_in_process_uses_background_worker_policy) { PASS(); } +TEST(mcp_auto_index_in_process_fresh_project_uses_full_width) { + return mcp_assert_inprocess_worker_policy(false, true); +} + +TEST(mcp_auto_index_in_process_existing_index_keeps_headroom) { + return mcp_assert_inprocess_worker_policy(true, true); +} + +TEST(mcp_background_flag_fresh_project_uses_full_width) { + return mcp_assert_inprocess_worker_policy(false, false); +} + +TEST(mcp_background_flag_existing_index_keeps_headroom) { + return mcp_assert_inprocess_worker_policy(true, false); +} + /* ══════════════════════════════════════════════════════════════════ * #853 — auto_watch=false must ALSO gate the SUPERVISED fresh-index * watcher registration (keystone × #849 merge interaction) @@ -21431,7 +21514,10 @@ SUITE(mcp) { RUN_TEST(autoindex_limit_guards_non_git_root_issue713); RUN_TEST(autoindex_limit_admits_non_git_root_under_limit_issue713); RUN_TEST(autoindex_limit_guards_git_root_issue713); - RUN_TEST(mcp_auto_index_in_process_uses_background_worker_policy); + RUN_TEST(mcp_auto_index_in_process_fresh_project_uses_full_width); + RUN_TEST(mcp_auto_index_in_process_existing_index_keeps_headroom); + RUN_TEST(mcp_background_flag_fresh_project_uses_full_width); + RUN_TEST(mcp_background_flag_existing_index_keeps_headroom); } /* Kept separate so daemon-coordination regressions can be iterated without From 2a83fe50e4f67df01be1770c3e103ed91b52f539 Mon Sep 17 00:00:00 2001 From: lorenzozanee Date: Sat, 3 Oct 2026 23:20:36 +0800 Subject: [PATCH 072/217] test(mcp): cover project discovery after store switches Signed-off-by: lorenzozanee --- tests/test_mcp.c | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 22af972af..af108e7f3 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -5452,7 +5452,7 @@ TEST(tool_query_graph_prefix_directory_recovers_rows_beyond_raw_estimate) { PASS(); } -TEST(tool_list_projects_tree_uses_one_stable_header_and_keeps_json_direct) { +TEST(tool_list_projects_stays_complete_across_store_switches_issue1115) { char cache[256]; snprintf(cache, sizeof(cache), "%s/cbm-list-lean-XXXXXX", cbm_tmpdir()); ASSERT_NOT_NULL(cbm_mkdtemp(cache)); @@ -5509,6 +5509,38 @@ TEST(tool_list_projects_tree_uses_one_stable_header_and_keeps_json_direct) { bool tree_not_duplicated = tree_response && !strstr(tree_response, "structuredContent"); bool page_truthful = page && strstr(page, "projects: 5") && strstr(page, "returned: 5") && strstr(page, "has_more: true") && strstr(page, "next_offset: 5"); + bool projects_resolve_after_switches = true; + for (int i = PROJECTS - 1; i >= 0; i--) { + char project[32]; + char args[96]; + snprintf(project, sizeof(project), "lean-project-%02d", i); + snprintf(args, sizeof(args), "{\"project\":\"%s\"}", project); + + char *status_response = cbm_mcp_handle_tool(srv, "index_status", args); + char *status = extract_text_content(status_response); + if (!status || !strstr(status, "ready")) { + projects_resolve_after_switches = false; + } + free(status); + free(status_response); + + snprintf(args, sizeof(args), "{\"project\":\"%s\",\"name_pattern\":\"entry\"}", + project); + char *search_response = cbm_mcp_handle_tool(srv, "search_graph", args); + char *search = extract_text_content(search_response); + if (!search || !strstr(search, "entry")) { + projects_resolve_after_switches = false; + } + free(search); + free(search_response); + } + char *post_switch_response = cbm_mcp_handle_tool(srv, "list_projects", "{\"limit\":50}"); + char *post_switch = extract_text_content(post_switch_response); + bool list_complete_after_switches = + post_switch && strstr(post_switch, "total: 12") && strstr(post_switch, "lean-project-00") && + strstr(post_switch, "lean-project-11"); + free(post_switch); + free(post_switch_response); bool json_direct = false; bool json_structured = json_response && strstr(json_response, "structuredContent"); yyjson_doc *doc = json ? yyjson_read(json, strlen(json), 0) : NULL; @@ -5545,6 +5577,8 @@ TEST(tool_list_projects_tree_uses_one_stable_header_and_keeps_json_direct) { ASSERT_TRUE(tree_is_leaner); ASSERT_TRUE(tree_not_duplicated); ASSERT_TRUE(page_truthful); + ASSERT_TRUE(projects_resolve_after_switches); + ASSERT_TRUE(list_complete_after_switches); ASSERT_TRUE(json_direct); ASSERT_TRUE(json_structured); PASS(); @@ -21142,7 +21176,7 @@ SUITE(mcp) { RUN_TEST(tool_query_graph_budget_bounds_first_row_and_json_escaping); RUN_TEST(tool_query_graph_prefix_directory_is_lossless_and_json_stays_direct); RUN_TEST(tool_query_graph_prefix_directory_recovers_rows_beyond_raw_estimate); - RUN_TEST(tool_list_projects_tree_uses_one_stable_header_and_keeps_json_direct); + RUN_TEST(tool_list_projects_stays_complete_across_store_switches_issue1115); RUN_TEST(tool_list_projects_preserves_root_beyond_one_kib); RUN_TEST(tool_index_status_no_project); RUN_TEST(tool_check_index_coverage_finds_path_beyond_status_cap); From 21c224fba56fe1b6b55cb13d003f5ab7f9199264 Mon Sep 17 00:00:00 2001 From: lorenzozanee Date: Tue, 6 Oct 2026 07:59:14 +0800 Subject: [PATCH 073/217] fix(pipeline): resolve constants in template URLs Signed-off-by: lorenzozanee --- internal/cbm/extract_calls.c | 15 ++++++++------- internal/cbm/extract_unified.c | 4 ++-- internal/cbm/helpers.c | 34 ++++++++++++++++++++++++++------- internal/cbm/helpers.h | 9 ++++----- tests/test_pipeline.c | 35 ++++++++++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 21 deletions(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index f14efb766..1ec05de2a 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -2192,9 +2192,9 @@ static void extract_call_args(CBMExtractCtx *ctx, TSNode args, CBMCall *call) { if (is_string_like(ak) && ca->expr) { ca->value = strip_quotes(ctx->arena, ca->expr); } else if (strcmp(ak, "template_string") == 0) { - /* Flattened {} form so downstream url-arg detection joins the - * canonical server route shape (issue #1006/#1009). */ - ca->value = cbm_template_string_text(ctx->arena, arg_node, ctx->source); + /* Resolve known constants; dynamic substitutions keep the route placeholder. */ + ca->value = cbm_template_string_text(ctx->arena, arg_node, ctx->source, + &ctx->string_constants); } else if (strcmp(ak, "identifier") == 0 && ca->expr) { ca->value = lookup_string_constant(ctx, ca->expr); } else if (strcmp(ak, "object") == 0) { @@ -2259,7 +2259,8 @@ static bool is_queue_topic_field(const char *key) { static const char *extract_string_value(CBMExtractCtx *ctx, TSNode val_node) { const char *vk = ts_node_type(val_node); if (strcmp(vk, "template_string") == 0) { - return cbm_template_string_text(ctx->arena, val_node, ctx->source); + return cbm_template_string_text(ctx->arena, val_node, ctx->source, + &ctx->string_constants); } if (is_string_like(vk)) { char *text = cbm_node_text(ctx->arena, val_node, ctx->source); @@ -2434,10 +2435,10 @@ static TSNode swift_unwrap_url_constructor(CBMExtractCtx *ctx, TSNode arg) { } static const char *extract_positional_url(CBMExtractCtx *ctx, TSNode arg, const char *ak) { - /* JS/TS template literals: `/things/${id}` normalizes to "/things/{}" so the - * client URL joins the server route's canonical placeholder (issue #1006). */ + /* Resolve constants while keeping dynamic route parts in canonical form. */ if (strcmp(ak, "template_string") == 0) { - const char *flat = cbm_template_string_text(ctx->arena, arg, ctx->source); + const char *flat = cbm_template_string_text(ctx->arena, arg, ctx->source, + &ctx->string_constants); if (flat) { return strip_and_validate_string_arg(ctx->arena, (char *)flat); } diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index 086f501a0..cce2100b7 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -1212,7 +1212,7 @@ static void handle_string_constants(CBMExtractCtx *ctx, TSNode node, const WalkS const char *value_kind = ts_node_type(value_node); const char *flat_value = NULL; if (strcmp(value_kind, "template_string") == 0) { - flat_value = cbm_template_string_text(ctx->arena, value_node, ctx->source); + flat_value = cbm_template_string_text(ctx->arena, value_node, ctx->source, NULL); if (!flat_value) { return; } @@ -1263,7 +1263,7 @@ static void handle_string_refs(CBMExtractCtx *ctx, TSNode node, const WalkState * template strings become string_refs with the canonical placeholder shape * shared with server route paths (issue #1006). */ if (strcmp(kind, "template_string") == 0) { - const char *flat = cbm_template_string_text(ctx->arena, node, ctx->source); + const char *flat = cbm_template_string_text(ctx->arena, node, ctx->source, NULL); if (!flat) { return; } diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 4c4d340f0..86c54c4e9 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -1803,13 +1803,10 @@ int cbm_classify_string(const char *str, int len) { return NOT_FOUND; } -/* Flatten a JS/TS `template_string` node into plain text (issue #1006). - * String fragments are kept verbatim; each ${...} substitution becomes the - * "{}" placeholder so client URLs built from template literals share the - * canonical parameter shape of server-side route paths - * (`/things/${id}/x` -> "/things/{}/x"). Returns NULL when the node yields - * no text or exceeds the route-sized buffer. */ -const char *cbm_template_string_text(CBMArena *a, TSNode node, const char *source) { +/* Flatten a JS/TS template string, resolving known string constants and + * replacing other substitutions with the route placeholder. */ +const char *cbm_template_string_text(CBMArena *a, TSNode node, const char *source, + const CBMStringConstantMap *string_constants) { enum { TPL_BUF = 512 }; char buf[TPL_BUF]; size_t pos = 0; @@ -1829,6 +1826,29 @@ const char *cbm_template_string_text(CBMArena *a, TSNode node, const char *sourc memcpy(buf + pos, frag, fl); pos += fl; } else if (strcmp(k, "template_substitution") == 0) { + const char *resolved = NULL; + TSNode expr = ts_node_named_child(c, 0); + if (string_constants && !ts_node_is_null(expr) && + strcmp(ts_node_type(expr), "identifier") == 0) { + char *name = cbm_node_text(a, expr, source); + for (int j = 0; name && j < string_constants->count; j++) { + if (!string_constants->is_url_builder[j] && string_constants->names[j] && + string_constants->values[j] && + strcmp(string_constants->names[j], name) == 0) { + resolved = string_constants->values[j]; + break; + } + } + } + if (resolved) { + size_t rl = strlen(resolved); + if (pos + rl >= TPL_BUF) { + return NULL; + } + memcpy(buf + pos, resolved, rl); + pos += rl; + continue; + } if (pos + PAIR_LEN >= TPL_BUF) { return NULL; } diff --git a/internal/cbm/helpers.h b/internal/cbm/helpers.h index 00e3d30dc..16c132d10 100644 --- a/internal/cbm/helpers.h +++ b/internal/cbm/helpers.h @@ -253,10 +253,9 @@ char *cbm_fqn_compute_source_lang(CBMArena *a, const char *project, const char * // Folder QN: project.dir_parts char *cbm_fqn_folder(CBMArena *a, const char *project, const char *rel_dir); -/* Flatten a JS/TS `template_string` node into plain text: string fragments are - * kept verbatim and each ${...} substitution becomes the "{}" placeholder, so - * client-side URLs built from template literals share the canonical parameter - * shape that server-side route paths already use. NULL when empty/oversized. */ -const char *cbm_template_string_text(CBMArena *a, TSNode node, const char *source); +/* Flatten JS/TS template strings for URL extraction. Known string constants + * resolve; other substitutions become "{}". NULL when empty/oversized. */ +const char *cbm_template_string_text(CBMArena *a, TSNode node, const char *source, + const CBMStringConstantMap *string_constants); #endif // CBM_HELPERS_H diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index ed9b0dd3c..7c8d61cbe 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8158,6 +8158,40 @@ TEST(pipeline_native_fetch_classified_as_http_calls) { PASS(); } +TEST(pipeline_native_fetch_template_base_url_makes_route) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_fetch_template_base_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "src/api.ts", + "const BASE_URL = \"http://payments:8080\";\n" + "export async function createPayment(body: unknown) {\n" + " return fetch(`${BASE_URL}/v1/payments`, { method: 'POST', body });\n" + "}\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/fetch_template_base.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + int http_call_count = cbm_store_count_edges_by_type(s, project, "HTTP_CALLS"); + int matching_route_count = + http_calls_to_route(s, project, "createPayment", "/v1/payments"); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + ASSERT_EQ(http_call_count, 1); + ASSERT_EQ(matching_route_count, 1); + PASS(); +} + /* #1892: Swift produced no Route node and no HTTP_CALLS edge, because the * Swift grammar has no "arguments" field and the generic lookup therefore read * no call arguments at all. Alamofire/URLSession were already in the service @@ -17671,6 +17705,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_ts_config_object_url_http_calls_issue2235); RUN_TEST(pipeline_arg_url_rejects_document_file_paths); RUN_TEST(pipeline_native_fetch_classified_as_http_calls); + RUN_TEST(pipeline_native_fetch_template_base_url_makes_route); RUN_TEST(pipeline_swift_nested_url_makes_route_issue1892); RUN_TEST(pipeline_swift_http_call_makes_route_issue1892); RUN_TEST(pipeline_native_fetch_parallel_classified_as_http_calls); From e632d4214bf5ca100ddbd27f303eb013ce9c105e Mon Sep 17 00:00:00 2001 From: Pcristin Date: Wed, 7 Oct 2026 11:00:47 +0000 Subject: [PATCH 074/217] fix(pipeline): initialize frozen coverage state Signed-off-by: Pcristin --- src/pipeline/pipeline.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 60f3c799f..be3a1af70 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -635,6 +635,8 @@ static cbm_pipeline_frozen_status_t frozen_allocate_pipeline( cbm_pipeline_t *p = calloc(1, sizeof(*p)); if (!p) return CBM_PIPELINE_FROZEN_OOM; + atomic_init(&p->unresolved_capture_failed, false); + cbm_mutex_init(&p->unresolved_mutex); owner->pipeline = p; p->frozen = owner; p->repo_path = strdup(in->source_root); From d4273ae901da4834f550d798b5d19d291b4bc29d Mon Sep 17 00:00:00 2001 From: Pcristin Date: Wed, 7 Oct 2026 21:49:42 +0000 Subject: [PATCH 075/217] fix(pipeline): bound unresolved call evidence Signed-off-by: Pcristin --- internal/cbm/cbm.h | 4 - src/foundation/mem_core.c | 2 +- src/foundation/mem_core.h | 1 + src/mcp/mcp.c | 147 ++++++++---- src/pipeline/pass_calls.c | 341 +++++++++++++++++----------- src/pipeline/pass_parallel.c | 162 +++++++------ src/pipeline/pipeline.c | 285 ++++++++++++++++------- src/pipeline/pipeline_incremental.c | 61 +++-- src/pipeline/pipeline_internal.h | 11 +- src/store/store.c | 278 +++++++++++++++++------ src/store/store.h | 14 +- src/store/store_content_digest.c | 38 +++- src/store/store_graph_digest.c | 140 ++++++++---- src/store/store_graph_digest.h | 3 +- tests/test_mcp.c | 136 +++++++++++ tests/test_pipeline.c | 136 +++++++++++ tests/test_store_bulk.c | 256 +++++++++++++++++++++ tests/test_store_graph_digest.c | 90 +++++++- 18 files changed, 1630 insertions(+), 475 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 6d16708ec..5d723c118 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -332,10 +332,6 @@ typedef struct { // (self.compiler.apply_converters()). An object the // class owns, not a parameter: read by the weak-member // guard's unique-name exemption. Default false. - /* Ephemeral coverage evidence, set only while resolving this file. Capture - * it before releasing a spilled result; never retain registry pointers. */ - bool coverage_calls_emitted; - const char *coverage_candidate_qn; } CBMCall; // What an import statement names, when the syntax says so. Only PHP's diff --git a/src/foundation/mem_core.c b/src/foundation/mem_core.c index 121d2696d..426dec44c 100644 --- a/src/foundation/mem_core.c +++ b/src/foundation/mem_core.c @@ -67,7 +67,7 @@ static mem_class_stats_t g_classes[CBM_MEM_CLASS_COUNT]; static const char *const g_class_names[CBM_MEM_CLASS_COUNT] = { "other", "gbuf_node", "gbuf_edge", "gbuf_string", "gbuf_index", "extract", "arena", - "ts_tree", "semantic", "dump", "store", "hash_table", "dyn_array", + "ts_tree", "semantic", "dump", "store", "hash_table", "dyn_array", "resolve", }; const char *cbm_mem_class_name(cbm_mem_class_t cls) { diff --git a/src/foundation/mem_core.h b/src/foundation/mem_core.h index 1514c3c68..6b5cb5cb3 100644 --- a/src/foundation/mem_core.h +++ b/src/foundation/mem_core.h @@ -66,6 +66,7 @@ typedef enum { CBM_MEM_CLASS_STORE, /* SQLite (bound mem methods) + store batches and row buffers */ CBM_MEM_CLASS_HASH_TABLE, /* CBMHashTable buckets/entries not claimed by an owner class */ CBM_MEM_CLASS_DYN_ARRAY, /* CBM_DYN_ARRAY item storage (every cbm_da_* user) */ + CBM_MEM_CLASS_RESOLVE, /* call resolution evidence and coverage capture */ CBM_MEM_CLASS_COUNT } cbm_mem_class_t; diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index efb217e95..9fb41d37c 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -6594,6 +6594,43 @@ static void coverage_add_row_json(yyjson_mut_doc *doc, yyjson_mut_val *array, yyjson_mut_arr_add_val(array, item); } +enum { COVERAGE_EXCLUDED_PREFIX_LENGTH = 11 }; + +enum { + COVERAGE_STATUS_PARSE, + COVERAGE_STATUS_EXCLUDED, + COVERAGE_STATUS_SKIPPED, + COVERAGE_STATUS_UNRESOLVED, + COVERAGE_STATUS_PASSES +}; + +static const char *coverage_row_status(const char *kind, int pass) { + kind = kind ? kind : ""; + if (pass == COVERAGE_STATUS_PARSE) { + if (strcmp(kind, "parse_unusable") == 0) { + return "unusable"; + } + if (strcmp(kind, "parse_partial") == 0) { + return "partial"; + } + } + if (pass == COVERAGE_STATUS_EXCLUDED && + strncmp(kind, "not_indexed", COVERAGE_EXCLUDED_PREFIX_LENGTH) == 0) { + return "excluded"; + } + if (pass == COVERAGE_STATUS_SKIPPED && kind[0] && strcmp(kind, "unresolved_calls") != 0) { + return "skipped"; + } + if (pass == COVERAGE_STATUS_UNRESOLVED && strcmp(kind, "unresolved_calls") == 0) { + return "unresolved_calls"; + } + return NULL; +} + +static bool coverage_unresolved_complete(bool have_meta, const cbm_coverage_meta_t *meta) { + return have_meta && meta->unresolved_calls_complete; +} + static const char *coverage_status(const cbm_coverage_row_t *rows, int count, const char *requested_path, const char *recording_status, bool generation_matches, bool lookup_ok, @@ -6608,29 +6645,14 @@ static const char *coverage_status(const cbm_coverage_row_t *rows, int count, break; } } - for (int pass = 0; pass < 3; pass++) { + for (int pass = 0; pass < COVERAGE_STATUS_PASSES; pass++) { for (int i = 0; i < count; i++) { if (exact && (!rows[i].rel_path || strcmp(rows[i].rel_path, requested_path) != 0)) { continue; } - const char *kind = rows[i].kind ? rows[i].kind : ""; - /* "parse_unusable" must be named here. Without its own case it - * falls through to the catch-all below and reports "skipped", - * which is wrong in the way that matters: the file WAS indexed. */ - if (pass == 0 && strcmp(kind, "parse_unusable") == 0) { - return "unusable"; - } - if (pass == 0 && strcmp(kind, "parse_partial") == 0) { - return "partial"; - } - if (pass == 0 && strcmp(kind, "unresolved_calls") == 0) { - return "unresolved_calls"; - } - if (pass == 1 && strncmp(kind, "not_indexed", 11) == 0) { - return "excluded"; - } - if (pass == 2 && kind[0]) { - return "skipped"; + const char *status = coverage_row_status(rows[i].kind, pass); + if (status) { + return status; } } } @@ -6736,6 +6758,8 @@ static char *handle_check_index_coverage(cbm_mcp_server_t *srv, const char *args have_meta ? meta.ignored_files_total : 0); yyjson_mut_obj_add_bool(doc, meta_obj, "hash_records_complete", have_meta && meta.hash_records_complete); + yyjson_mut_obj_add_bool(doc, meta_obj, "unresolved_calls_complete", + coverage_unresolved_complete(have_meta, &meta)); yyjson_mut_obj_add_int(doc, meta_obj, "coverage_version", have_meta ? meta.coverage_version : 0); yyjson_mut_obj_add_bool(doc, meta_obj, "generation_matches", generation_matches); @@ -9265,7 +9289,6 @@ static void trace_coverage_match_site(trace_coverage_match_t *match, const char yyjson_val *site) { const char *caller = yyjson_get_str(yyjson_obj_get(site, "caller")); const char *leaf = yyjson_get_str(yyjson_obj_get(site, "leaf")); - const char *candidate = yyjson_get_str(yyjson_obj_get(site, "candidate")); const cbm_node_t *caller_node = match->callers && caller ? cbm_ht_get(match->callers, caller) : NULL; int line = yyjson_get_int(yyjson_obj_get(site, "line")); @@ -9277,25 +9300,22 @@ static void trace_coverage_match_site(trace_coverage_match_t *match, const char trace_coverage_may_target_project(match, leaf)) { match->unresolved_out = true; } - /* A short-name collision is not evidence that the resolver considered - * this traced target. */ - if (match->targets && candidate && cbm_ht_has(match->targets, candidate)) { - match->unresolved_in = true; - } } -static void trace_coverage_match_row(trace_coverage_match_t *match, const cbm_coverage_row_t *row, - const trace_coverage_scope_t *scope) { +static void trace_coverage_match_row(trace_coverage_match_t *match, const cbm_coverage_row_t *row) { yyjson_doc *detail = yyjson_read(row->detail, strlen(row->detail), 0); yyjson_val *sites = detail ? yyjson_doc_get_root(detail) : NULL; if (!yyjson_is_arr(sites)) { - match->unresolved_out = scope->do_outbound; - match->unresolved_in = scope->do_inbound; + match->unresolved_out = true; } else { size_t idx; size_t max; yyjson_val *site; yyjson_arr_foreach(sites, idx, max, site) { + if (!yyjson_is_obj(site) || yyjson_get_bool(yyjson_obj_get(site, "truncated"))) { + match->unresolved_out = true; + break; + } trace_coverage_match_site(match, row->rel_path, site); } } @@ -9304,24 +9324,61 @@ static void trace_coverage_match_row(trace_coverage_match_t *match, const cbm_co } } -static bool trace_coverage_scan_rows(cbm_store_t *store, const char *project, - trace_coverage_match_t *match, - const trace_coverage_scope_t *scope) { +typedef struct { + trace_coverage_match_t *match; + CBMHashTable *files; + bool ok; +} trace_coverage_query_t; + +static void trace_coverage_query_file(const char *key, void *value, void *userdata) { + (void)key; + const cbm_node_t *caller = value; + trace_coverage_query_t *query = userdata; + const char *path = caller->file_path; + if (!query->ok || query->match->unresolved_out || !path || cbm_ht_has(query->files, path)) { + return; + } + cbm_ht_set(query->files, path, value); + if (!cbm_ht_has(query->files, path)) { + query->ok = false; + return; + } cbm_coverage_row_t *rows = NULL; - int row_count = 0; - if (cbm_store_coverage_get_unresolved_calls(store, project, &rows, &row_count) != - CBM_STORE_OK) { - return false; + int count = 0; + query->ok = cbm_store_coverage_get_unresolved_path(query->match->store, query->match->project, + path, &rows, &count) == CBM_STORE_OK; + for (int i = 0; query->ok && i < count; i++) { + trace_coverage_match_row(query->match, &rows[i]); } - for (int i = 0; i < row_count; i++) { - if ((!scope->do_outbound || match->unresolved_out) && - (!scope->do_inbound || match->unresolved_in)) { - break; + cbm_store_free_coverage(rows, count); +} + +static void trace_coverage_query_candidate(const char *key, void *value, void *userdata) { + (void)value; + trace_coverage_query_t *query = userdata; + if (!query->ok || query->match->unresolved_in) { + return; + } + query->ok = + cbm_store_coverage_has_unresolved_candidate(query->match->store, query->match->project, key, + &query->match->unresolved_in) == CBM_STORE_OK; +} + +static bool trace_coverage_query_evidence(trace_coverage_match_t *match, + const trace_coverage_scope_t *scope) { + trace_coverage_query_t query = {.match = match, .ok = true}; + if (scope->do_outbound) { + query.files = cbm_ht_create(0); + if (!query.files) { + return false; } - trace_coverage_match_row(match, &rows[i], scope); + cbm_ht_foreach(match->callers, trace_coverage_query_file, &query); + cbm_ht_free(query.files); } - cbm_store_free_coverage(rows, row_count); - return true; + if (scope->do_inbound) { + cbm_ht_foreach(match->targets, trace_coverage_query_candidate, &query); + } + return query.ok; } static bool trace_coverage_meta_available(cbm_store_t *store, const char *project) { @@ -9329,7 +9386,7 @@ static bool trace_coverage_meta_available(cbm_store_t *store, const char *projec int rc = cbm_store_coverage_meta_get(store, project, &meta); bool available = rc == CBM_STORE_OK && meta.coverage_version >= CBM_UNRESOLVED_CALL_COVERAGE_VERSION && - meta.recording_status && strcmp(meta.recording_status, "unavailable") != 0; + meta.unresolved_calls_complete; cbm_store_coverage_meta_clear(&meta); return available; } @@ -9347,7 +9404,7 @@ static void trace_call_coverage(cbm_store_t *store, const char *project, bool ok = trace_coverage_meta_available(store, project) && (!scope->do_outbound || match.callers) && (!scope->do_inbound || match.targets) && trace_coverage_build_sets(&match, scope) && - trace_coverage_scan_rows(store, project, &match, scope); + trace_coverage_query_evidence(&match, scope); *unresolved_out = ok ? match.unresolved_out : scope->do_outbound; *unresolved_in = ok ? match.unresolved_in : scope->do_inbound; cbm_ht_free(match.callers); diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 3359841b1..e9089b190 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -11,7 +11,13 @@ */ #include "foundation/constants.h" -enum { PC_RING = 4, PC_RING_MASK = 3, PC_SIG_SCAN = 15, PC_REGEX_GRP = 2 }; +enum { + PC_RING = 4, + PC_RING_MASK = 3, + PC_SIG_SCAN = 15, + PC_REGEX_GRP = 2, + PC_SINGLE_CANDIDATE_COUNT = 1 +}; /* Confidence for a service-pattern HTTP/ASYNC edge emitted when registry * resolution is empty (external, unindexed client library) — see #523. */ #define PC_SVC_PATTERN_CONF 0.5 @@ -382,7 +388,7 @@ bool cbm_pipeline_http_client_call_url(const cbm_gbuf_t *gbuf, const char *proje /* Handle a route registration call: create Route node + HANDLES edge. * route_mount is the framework mount of the registering file ("/api" for a * Laravel 11+ `withRouting(api: ...)` file, #1146) or "". */ -static void handle_route_registration(cbm_pipeline_ctx_t *ctx, CBMCall *call, +static bool handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source_node, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, const char *route_mount) { @@ -406,9 +412,7 @@ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, CBMCall *call, snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}", esc_cn, esc_fa); - if (cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, route_id, "CALLS", props) > 0) { - call->coverage_calls_emitted = true; - } + bool emitted = cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, route_id, "CALLS", props) > 0; if (call->second_arg_name != NULL && call->second_arg_name[0] != '\0') { cbm_resolution_t hres = cbm_registry_resolve(ctx->registry, call->second_arg_name, module_qn, imp_keys, imp_vals, imp_count); @@ -424,6 +428,7 @@ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, CBMCall *call, } } } + return emitted; } /* Emit an HTTP/async route edge for a service call. */ @@ -467,7 +472,7 @@ static int64_t create_svc_route_node(cbm_pipeline_ctx_t *ctx, const char *url, c * this, so data_flow mode had no argument expressions to surface for small * (< 50 file) repos that take the sequential path (#514). Mirrors the parallel * path's append_args_json shape so both pipelines agree. */ -static void calls_append_args(char *props, size_t cap, CBMCall *call) { +static void calls_append_args(char *props, size_t cap, const CBMCall *call) { if (!call || call->arg_count <= 0) { return; } @@ -513,8 +518,8 @@ static void calls_append_args(char *props, size_t cap, CBMCall *call) { } } -static void calls_emit_edge(cbm_gbuf_t *gbuf, int64_t src, int64_t tgt, const char *type, - char *props, size_t cap, CBMCall *call) { +static bool calls_emit_edge(cbm_gbuf_t *gbuf, int64_t src, int64_t tgt, const char *type, + char *props, size_t cap, const CBMCall *call) { if (call && call->start_line > 0 && strcmp(type, "CALLS") == 0) { size_t len = strlen(props); if (len >= SKIP_ONE && props[len - SKIP_ONE] == '}' && len + CBM_SZ_32 < cap) { @@ -525,13 +530,11 @@ static void calls_emit_edge(cbm_gbuf_t *gbuf, int64_t src, int64_t tgt, const ch if (call && strcmp(type, "CALLS") == 0) { calls_append_args(props, cap, call); } - if (cbm_gbuf_insert_edge(gbuf, src, tgt, type, props) > 0 && call && - strcmp(type, "CALLS") == 0) { - call->coverage_calls_emitted = true; - } + return cbm_gbuf_insert_edge(gbuf, src, tgt, type, props) > 0 && call && + strcmp(type, "CALLS") == 0; } -static void emit_http_async_edge(cbm_pipeline_ctx_t *ctx, CBMCall *call, +static bool emit_http_async_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target, const cbm_resolution_t *res, cbm_svc_kind_t svc, bool suppress_plain_calls) { @@ -552,7 +555,7 @@ static void emit_http_async_edge(cbm_pipeline_ctx_t *ctx, CBMCall *call, * target->id into a null dereference (clang-analyzer traced exactly * that), and with no callee node there is nothing to emit anyway. */ if (suppress_plain_calls || !target) { - return; + return false; } char esc_callee[CBM_SZ_256]; cbm_json_escape(esc_callee, sizeof(esc_callee), call->callee_name); @@ -561,8 +564,8 @@ static void emit_http_async_edge(cbm_pipeline_ctx_t *ctx, CBMCall *call, "{\"callee\":\"%s\",\"confidence\":%.2f,\"strategy\":\"%s\",\"candidates\":%d}", esc_callee, res->confidence, res->strategy ? res->strategy : "unknown", res->candidate_count); - calls_emit_edge(ctx->gbuf, source->id, target->id, "CALLS", props, sizeof(props), call); - return; + return calls_emit_edge(ctx->gbuf, source->id, target->id, "CALLS", props, sizeof(props), + call); } const char *edge_type = (svc == CBM_SVC_HTTP) ? "HTTP_CALLS" : "ASYNC_CALLS"; const char *method = @@ -593,6 +596,7 @@ static void emit_http_async_edge(cbm_pipeline_ctx_t *ctx, CBMCall *call, props[n + 1] = '\0'; } calls_emit_edge(ctx->gbuf, source->id, route_id, edge_type, props, sizeof(props), call); + return false; } /* Classify a resolved call and emit the appropriate edge. */ @@ -600,20 +604,18 @@ static void emit_http_async_edge(cbm_pipeline_ctx_t *ctx, CBMCall *call, * match, #592/#606), the route/HTTP/ASYNC/CONFIG service classifications below * still run — only the plain CALLS fall-through is skipped, so a fabricated * project edge is dropped while every service edge stays main-identical. */ -static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, CBMCall *call, +static bool emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target, const cbm_resolution_t *res, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, bool suppress_plain_calls, const char *route_mount) { cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); if (svc == CBM_SVC_ROUTE_REG && call->first_string_arg && call->first_string_arg[0] == '/') { - handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, imp_count, - route_mount); - return; + return handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, + imp_count, route_mount); } if (svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) { - emit_http_async_edge(ctx, call, source, target, res, svc, suppress_plain_calls); - return; + return emit_http_async_edge(ctx, call, source, target, res, svc, suppress_plain_calls); } if (svc == CBM_SVC_CONFIG) { char esc_c[CBM_SZ_256]; @@ -625,10 +627,10 @@ static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, CBMCall *call, esc_c, esc_k, res->confidence); calls_emit_edge(ctx->gbuf, source->id, target->id, "CONFIGURES", props, sizeof(props), call); - return; + return false; } if (suppress_plain_calls) { - return; /* weak TS/JS member-call match with an unresolved receiver (#606) */ + return false; /* weak TS/JS member-call match with an unresolved receiver (#606) */ } char esc_c2[CBM_SZ_256]; cbm_json_escape(esc_c2, sizeof(esc_c2), call->callee_name); @@ -637,7 +639,7 @@ static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, CBMCall *call, "{\"callee\":\"%s\",\"confidence\":%.2f,\"strategy\":\"%s\",\"candidates\":%d}", esc_c2, res->confidence, res->strategy ? res->strategy : "unknown", res->candidate_count); - calls_emit_edge(ctx->gbuf, source->id, target->id, "CALLS", props, sizeof(props), call); + return calls_emit_edge(ctx->gbuf, source->id, target->id, "CALLS", props, sizeof(props), call); } /* Find source node for a call: enclosing function or file node. */ @@ -659,11 +661,152 @@ static const cbm_gbuf_node_t *calls_find_source(cbm_pipeline_ctx_t *ctx, const c return src; } +static CBMCallEvidence *calls_evidence_at(CBMCallEvidence *evidence, int index, + CBMCallEvidence *fallback) { + if (evidence) { + return &evidence[index]; + } + return fallback; +} + +static CBMCallEvidence *calls_evidence_new(cbm_pipeline_t *pipeline, const CBMFileResult *result) { + CBMCallEvidence *evidence = + cbm_calloc(CBM_MEM_CLASS_RESOLVE, (size_t)result->calls.count * sizeof(*evidence)); + if (!evidence) { + cbm_pipeline_mark_unresolved_capture_failed(pipeline); + } + return evidence; +} + +/* Report whether a service spelling handled this call, independently of + * whether its classified edge is CALLS and therefore covers the occurrence. */ +static bool calls_emit_service_call(cbm_pipeline_ctx_t *ctx, const CBMCall *call, + const cbm_gbuf_node_t *source_node, cbm_svc_kind_t svc, + const cbm_resolution_t *resolved, CBMCallEvidence *evidence) { + if (svc != CBM_SVC_HTTP && svc != CBM_SVC_ASYNC) { + return false; + } + const char *url = call->first_string_arg; + if (!url || !url[0]) { + return false; + } + bool has_url_or_topic = url[0] == '/' || strstr(url, "://") != NULL || + (svc == CBM_SVC_ASYNC && strlen(url) > PAIR_LEN); + if (!has_url_or_topic) { + return false; + } + cbm_resolution_t fallback = {.qualified_name = call->callee_name, + .confidence = PC_SVC_PATTERN_CONF, + .strategy = "service_pattern", + .candidate_count = 0}; + const cbm_resolution_t *res = resolved ? resolved : &fallback; + evidence->calls_emitted |= emit_http_async_edge(ctx, call, source_node, NULL, res, svc, false); + return true; +} + +/* Route registration counts as handled even when its edge is not CALLS. */ +static bool calls_emit_route_call(cbm_pipeline_ctx_t *ctx, const CBMCall *call, + const cbm_gbuf_node_t *source_node, const char *module_qn, + const char **imp_keys, const char **imp_vals, int imp_count, + const char *route_mount, CBMCallEvidence *evidence) { + if (cbm_service_pattern_route_method(call->callee_name) == NULL || !call->first_string_arg || + call->first_string_arg[0] != '/') { + return false; + } + evidence->calls_emitted |= handle_route_registration( + ctx, call, source_node, module_qn, imp_keys, imp_vals, imp_count, route_mount); + return true; +} + +/* An external callee can still register a route or address an HTTP/async service. */ +static int calls_emit_unresolved_service(cbm_pipeline_ctx_t *ctx, const CBMCall *call, + const cbm_gbuf_node_t *source_node, const char *module_qn, + const char **imp_keys, const char **imp_vals, + int imp_count, const char *route_mount, + CBMCallEvidence *evidence) { + if (calls_emit_route_call(ctx, call, source_node, module_qn, imp_keys, imp_vals, imp_count, + route_mount, evidence)) { + return SKIP_ONE; + } + cbm_svc_kind_t svc = cbm_service_pattern_match(call->callee_name); + if (svc == CBM_SVC_NONE && cbm_service_pattern_is_global_fetch(call->callee_name)) { + svc = CBM_SVC_HTTP; + } + return calls_emit_service_call(ctx, call, source_node, svc, NULL, evidence); +} + +/* Apply the field-call policy after a registry target has been found. */ +static const cbm_gbuf_node_t *calls_field_target(cbm_pipeline_ctx_t *ctx, const CBMCall *call, + const CBMResolvedCallArray *lsp_calls, + CBMLanguage lang, const char *rel, + const cbm_gbuf_node_t *source_node, + const cbm_gbuf_node_t *target_node, + bool arrow_bound, cbm_resolution_t *res) { + switch (cbm_call_onto_field_policy(lang, call->callee_name, target_node->label, + target_node->file_path)) { + case CBM_FIELD_CALL_DROP: + return NULL; + case CBM_FIELD_CALL_BY_OWNER: + /* An arrow call was already bound this way by registry resolution. */ + if (!arrow_bound) { + target_node = cbm_pipeline_c_member_call_resolve( + lsp_calls, NULL, ctx->gbuf, ctx->registry, ctx->project_name, + call->enclosing_func_qn, call->callee_name, cbm_c_member_rule_file(lang, rel), res); + } + if (!target_node || source_node->id == target_node->id) { + return NULL; + } + break; + case CBM_FIELD_CALL_KEEP: + break; + } + return target_node; +} + +static bool calls_emit_lsp_target(cbm_pipeline_ctx_t *ctx, const CBMCall *call, + const CBMResolvedCall *lsp, bool allow_tail, CBMLanguage lang, + const CBMImportArray *imports, const cbm_gbuf_node_t *source_node, + const char *module_qn, const char **imp_keys, + const char **imp_vals, int imp_count, const char *route_mount, + CBMCallEvidence *evidence) { + if (!lsp) { + return false; + } + bool exact_external_target = + call->requires_lsp_resolution && cbm_pipeline_kotlin_external_target(lang, lsp->callee_qn); + const cbm_gbuf_node_t *target_node = + exact_external_target ? cbm_pipeline_lsp_target_node_strict(ctx->gbuf, ctx->project_name, + lsp->callee_qn, allow_tail) + : cbm_pipeline_lsp_target_node(ctx->gbuf, ctx->project_name, + lsp->callee_qn, allow_tail); + if (!target_node || source_node->id == target_node->id) { + return false; + } + cbm_resolution_t res = {0}; + /* Keep the canonical graph QN, including project prefixes added by fallback. */ + res.qualified_name = target_node->qualified_name; + res.confidence = lsp->confidence; + res.strategy = lsp->strategy; + res.candidate_count = PC_SINGLE_CANDIDATE_COUNT; + evidence->candidate_qn = target_node->qualified_name; + /* Synthetic builtins can still denote a spawn; keep the parallel pass policy. */ + cbm_pipeline_spawn_t spawn; + if (cbm_pipeline_spawn_site(ctx->gbuf, lang, call, imports, &res, &spawn)) { + cbm_pipeline_emit_spawn(ctx->gbuf, source_node, call, &spawn); + return true; + } + evidence->calls_emitted |= + emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, + imp_vals, imp_count, false, route_mount); + return true; +} + /* Resolve one call and emit the appropriate edge. Returns 1 if resolved, 0 if not. */ -static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBMFileResult *result, - const char *rel, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count, CBMLanguage lang, - const CBMImportArray *imports, const char *route_mount) { +static int resolve_single_call(cbm_pipeline_ctx_t *ctx, const CBMCall *call, + const CBMFileResult *result, const char *rel, const char *module_qn, + const char **imp_keys, const char **imp_vals, int imp_count, + CBMLanguage lang, const CBMImportArray *imports, + const char *route_mount, CBMCallEvidence *evidence) { const CBMResolvedCallArray *lsp_calls = &result->resolved_calls; const cbm_gbuf_node_t *source_node = calls_find_source(ctx, rel, call->enclosing_func_qn); if (!source_node) { @@ -675,36 +818,9 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM bool allow_tail = cbm_pipeline_lsp_allow_tail_match(lang); const CBMResolvedCall *lsp = cbm_pipeline_find_lsp_resolution_in_graph( lsp_calls, call, allow_tail, ctx->gbuf, ctx->project_name); - if (lsp) { - bool exact_external_target = call->requires_lsp_resolution && - cbm_pipeline_kotlin_external_target(lang, lsp->callee_qn); - const cbm_gbuf_node_t *target_node = - exact_external_target ? cbm_pipeline_lsp_target_node_strict( - ctx->gbuf, ctx->project_name, lsp->callee_qn, allow_tail) - : cbm_pipeline_lsp_target_node(ctx->gbuf, ctx->project_name, - lsp->callee_qn, allow_tail); - if (target_node && source_node->id != target_node->id) { - cbm_resolution_t res = {0}; - /* Use the gbuf node's QN so downstream edge props show the canonical - * project-qualified form even when fallback prefixed the project. */ - res.qualified_name = target_node->qualified_name; - res.confidence = lsp->confidence; - res.strategy = lsp->strategy; - res.candidate_count = CBM_ALLOC_ONE; - call->coverage_candidate_qn = target_node->qualified_name; - /* An LSP answer on a synthetic builtin (``) is - * not project code: a spawn spelling still spawns. The parallel - * pass keeps the LSP answer in `res` and asks the same question. - * MUST match pass_parallel.c. */ - cbm_pipeline_spawn_t spawn; - if (cbm_pipeline_spawn_site(ctx->gbuf, lang, call, imports, &res, &spawn)) { - cbm_pipeline_emit_spawn(ctx->gbuf, source_node, call, &spawn); - return SKIP_ONE; - } - emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, - imp_vals, imp_count, false, route_mount); - return SKIP_ONE; - } + if (calls_emit_lsp_target(ctx, call, lsp, allow_tail, lang, imports, source_node, module_qn, + imp_keys, imp_vals, imp_count, route_mount, evidence)) { + return SKIP_ONE; } /* Synthetic semantic candidates (currently implicit C++ operators) are @@ -740,23 +856,13 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM .confidence = PC_SVC_PATTERN_CONF, .strategy = "http_client_instance", .candidate_count = 0}; - emit_http_async_edge(ctx, &routed, source_node, NULL, &svc_res, CBM_SVC_HTTP, false); + evidence->calls_emitted |= + emit_http_async_edge(ctx, &routed, source_node, NULL, &svc_res, CBM_SVC_HTTP, false); return SKIP_ONE; } - cbm_svc_kind_t csvc = cbm_service_pattern_match(call->callee_name); - if (csvc == CBM_SVC_HTTP || csvc == CBM_SVC_ASYNC) { - const char *cu = call->first_string_arg; - bool chas_url = cu && cu[0] != '\0' && - (cu[0] == '/' || strstr(cu, "://") != NULL || - (csvc == CBM_SVC_ASYNC && strlen(cu) > PAIR_LEN)); - if (chas_url) { - cbm_resolution_t svc_res = {.qualified_name = call->callee_name, - .confidence = PC_SVC_PATTERN_CONF, - .strategy = "service_pattern", - .candidate_count = 0}; - emit_http_async_edge(ctx, call, source_node, NULL, &svc_res, csvc, false); - return SKIP_ONE; - } + if (calls_emit_service_call(ctx, call, source_node, + cbm_service_pattern_match(call->callee_name), NULL, evidence)) { + return SKIP_ONE; } /* #2053: a Rust call the LSP placed on an EXTERNAL symbol (std's @@ -819,31 +925,8 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM * apps. Classify by callee suffix + path-shaped first arg, exactly * like the parallel path's callee_suffix fallback; without this the * sequential path minted zero Route nodes for such files. */ - if (cbm_service_pattern_route_method(call->callee_name) != NULL && call->first_string_arg && - call->first_string_arg[0] == '/') { - handle_route_registration(ctx, call, source_node, module_qn, imp_keys, imp_vals, - imp_count, route_mount); - return SKIP_ONE; - } - cbm_svc_kind_t esvc = cbm_service_pattern_match(call->callee_name); - if (esvc == CBM_SVC_NONE && cbm_service_pattern_is_global_fetch(call->callee_name)) { - esvc = CBM_SVC_HTTP; - } - if (esvc == CBM_SVC_HTTP || esvc == CBM_SVC_ASYNC) { - const char *u = call->first_string_arg; - bool has_url_or_topic = u && u[0] != '\0' && - (u[0] == '/' || strstr(u, "://") != NULL || - (esvc == CBM_SVC_ASYNC && strlen(u) > PAIR_LEN)); - if (has_url_or_topic) { - cbm_resolution_t svc_res = {.qualified_name = call->callee_name, - .confidence = PC_SVC_PATTERN_CONF, - .strategy = "service_pattern", - .candidate_count = 0}; - emit_http_async_edge(ctx, call, source_node, NULL, &svc_res, esvc, false); - return SKIP_ONE; - } - } - return 0; + return calls_emit_unresolved_service(ctx, call, source_node, module_qn, imp_keys, imp_vals, + imp_count, route_mount, evidence); } /* Perl call-graph noise guard (#476). Perl has no LSP resolver, so the @@ -920,16 +1003,9 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM * the missing target must NOT drop the call — otherwise no HTTP_CALLS edge * is written and cross-repo matching finds nothing (#523). Emit directly * when the call carries a URL/topic first argument. */ - cbm_svc_kind_t svc = cbm_service_pattern_match(res.qualified_name); - if (svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) { - const char *u = call->first_string_arg; - bool has_url_or_topic = u && u[0] != '\0' && - (u[0] == '/' || strstr(u, "://") != NULL || - (svc == CBM_SVC_ASYNC && strlen(u) > PAIR_LEN)); - if (has_url_or_topic) { - emit_http_async_edge(ctx, call, source_node, NULL, &res, svc, false); - return SKIP_ONE; - } + if (calls_emit_service_call(ctx, call, source_node, + cbm_service_pattern_match(res.qualified_name), &res, evidence)) { + return SKIP_ONE; } const cbm_gbuf_node_t *target_node = cbm_gbuf_find_by_qn(ctx->gbuf, res.qualified_name); @@ -939,24 +1015,10 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM /* A call that resolved onto a struct Field: refused across languages and * for a bare C call; a C/C++ member call takes the Field its object's type * names instead of the one the member name happened to find. */ - switch (cbm_call_onto_field_policy(lang, call->callee_name, target_node->label, - target_node->file_path)) { - case CBM_FIELD_CALL_DROP: + target_node = calls_field_target(ctx, call, lsp_calls, lang, rel, source_node, target_node, + arrow_bound, &res); + if (!target_node) { return 0; - case CBM_FIELD_CALL_BY_OWNER: - /* An arrow call was already bound this way above. */ - if (!arrow_bound) { - target_node = cbm_pipeline_c_member_call_resolve( - lsp_calls, NULL, ctx->gbuf, ctx->registry, ctx->project_name, - call->enclosing_func_qn, call->callee_name, cbm_c_member_rule_file(lang, rel), - &res); - } - if (!target_node || source_node->id == target_node->id) { - return 0; - } - break; - case CBM_FIELD_CALL_KEEP: - break; } /* #725: suffix_match is language-agnostic and will attach a Python * Store.commit() call to a JS function named commit (or a Bash main @@ -969,17 +1031,13 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM * unresolved-callee fallback above mints, and nothing else. Dropping * the whole call lost every GET registration in a mixed-language repo * while POST (no `post` to collide with) survived. */ - if (cbm_service_pattern_route_method(call->callee_name) != NULL && call->first_string_arg && - call->first_string_arg[0] == '/') { - handle_route_registration(ctx, call, source_node, module_qn, imp_keys, imp_vals, - imp_count, route_mount); - return SKIP_ONE; - } - return 0; + return calls_emit_route_call(ctx, call, source_node, module_qn, imp_keys, imp_vals, + imp_count, route_mount, evidence); } - call->coverage_candidate_qn = target_node->qualified_name; - emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, - imp_count, drop_plain_call, route_mount); + evidence->candidate_qn = target_node->qualified_name; + evidence->calls_emitted |= + emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, + imp_vals, imp_count, drop_plain_call, route_mount); return SKIP_ONE; } @@ -1130,12 +1188,15 @@ int cbm_pipeline_pass_calls(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file } if (result->calls.count == 0) { + cbm_pipeline_record_unresolved_calls(ctx->pipeline, rel, result, NULL); if (result_owned) { cbm_free_result(result); } continue; } + CBMCallEvidence *evidence = calls_evidence_new(ctx->pipeline, result); + /* Build import map for this file */ const char **imp_keys = NULL; const char **imp_vals = NULL; @@ -1154,19 +1215,23 @@ int cbm_pipeline_pass_calls(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file /* Resolve each call */ for (int c = 0; c < result->calls.count; c++) { - CBMCall *call = &result->calls.items[c]; + const CBMCall *call = &result->calls.items[c]; if (!call->callee_name) { continue; } + CBMCallEvidence fallback = {0}; total_calls++; if (resolve_single_call(ctx, call, result, rel, module_qn, imp_keys, imp_vals, - imp_count, files[i].language, &result->imports, route_mount)) { + imp_count, files[i].language, &result->imports, route_mount, + calls_evidence_at(evidence, c, &fallback))) { resolved++; } else { unresolved++; } } + cbm_pipeline_record_unresolved_calls(ctx->pipeline, rel, result, evidence); + cbm_free(CBM_MEM_CLASS_RESOLVE, evidence); free(module_qn); free_import_map(imp_keys, imp_vals, imp_count); if (result_owned) { diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index b34db716c..05cd8e4cd 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -2109,7 +2109,7 @@ static int format_call_arg(char *buf, size_t bufsize, const CBMCallArg *a, const return snprintf(buf, bufsize, "{\"i\":%d,\"e\":\"%s\"}", a->index, esc_e); } -static size_t append_args_json(char *buf, size_t bufsize, size_t pos, CBMCall *call) { +static size_t append_args_json(char *buf, size_t bufsize, size_t pos, const CBMCall *call) { if (call->arg_count == 0 || pos >= bufsize - PP_ARGS_MARGIN) { return pos; } @@ -2165,7 +2165,7 @@ static bool is_route_path_shaped(const char *val) { return val && val[0] == '/' && !cbm_service_pattern_is_comment_text(val); } -static const char *find_route_path_in_args(CBMCall *call, const char **out_handler) { +static const char *find_route_path_in_args(const CBMCall *call, const char **out_handler) { *out_handler = NULL; /* 1. First string arg starting with / */ if (is_route_path_shaped(call->first_string_arg)) { @@ -2204,8 +2204,8 @@ static const char *find_route_path_in_args(CBMCall *call, const char **out_handl } /* Build props JSON, append args, close brace, emit edge. */ -static void finalize_and_emit(cbm_gbuf_t *gbuf, int64_t src_id, int64_t tgt_id, - const char *edge_type, char *props, int n, CBMCall *call) { +static bool finalize_and_emit(cbm_gbuf_t *gbuf, int64_t src_id, int64_t tgt_id, + const char *edge_type, char *props, int n, const CBMCall *call) { if (n > 0 && (size_t)n < CBM_SZ_2K - PP_ESC_SPACE) { size_t pos = append_args_json(props, CBM_SZ_2K, (size_t)n, call); if (call->start_line > 0 && strcmp(edge_type, "CALLS") == 0 && @@ -2220,10 +2220,8 @@ static void finalize_and_emit(cbm_gbuf_t *gbuf, int64_t src_id, int64_t tgt_id, props[pos + SKIP_ONE] = '\0'; } } - if (cbm_gbuf_insert_edge(gbuf, src_id, tgt_id, edge_type, props) > 0 && - strcmp(edge_type, "CALLS") == 0) { - call->coverage_calls_emitted = true; - } + return cbm_gbuf_insert_edge(gbuf, src_id, tgt_id, edge_type, props) > 0 && + strcmp(edge_type, "CALLS") == 0; } /* Build Route node QN and properties for HTTP/async service edges. */ @@ -2253,7 +2251,7 @@ static int64_t build_service_route(cbm_gbuf_t *gbuf, const char *arg, const char /* Emit HTTP_CALLS or ASYNC_CALLS edge via Route node. */ static void emit_http_async_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - CBMCall *call, const cbm_resolution_t *res, + const CBMCall *call, const cbm_resolution_t *res, cbm_svc_kind_t svc, const char *arg) { const char *edge_type = (svc == CBM_SVC_HTTP) ? "HTTP_CALLS" : "ASYNC_CALLS"; const char *method = @@ -2280,7 +2278,7 @@ static void emit_http_async_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t /* Emit CONFIGURES edge. */ static void emit_config_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const cbm_gbuf_node_t *target, CBMCall *call, + const cbm_gbuf_node_t *target, const CBMCall *call, const cbm_resolution_t *res, const char *arg) { /* emit_service_edge may be reached with target==NULL on the HTTP/ASYNC * external-client bypass (#523); a CONFIGURES edge needs a real target, so @@ -2299,13 +2297,13 @@ static void emit_config_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, } /* Emit normal CALLS edge. */ -static void emit_normal_calls_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const cbm_gbuf_node_t *target, CBMCall *call, +static bool emit_normal_calls_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, + const cbm_gbuf_node_t *target, const CBMCall *call, const cbm_resolution_t *res) { /* A CALLS edge needs a real target; the HTTP/ASYNC external-client bypass * (#523) can reach emit_service_edge with target==NULL, so guard the deref. */ if (!target) { - return; + return false; } char esc_c[CBM_SZ_256]; cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); @@ -2314,18 +2312,19 @@ static void emit_normal_calls_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sour "{\"callee\":\"%s\",\"confidence\":%.2f,\"strategy\":\"%s\",\"candidates\":%d", esc_c, res->confidence, res->strategy ? res->strategy : "unknown", res->candidate_count); - finalize_and_emit(gbuf, source->id, target->id, "CALLS", props, n, call); + return finalize_and_emit(gbuf, source->id, target->id, "CALLS", props, n, call); } /* Classify a resolved call by library identity and emit the appropriate edge. */ /* Create Route node + CALLS + HANDLES edges for a route registration call. * route_mount is the framework mount of the registering file ("/api" for a * Laravel 11+ `withRouting(api: ...)` file, #1146) or "". */ -static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMCall *call, - const char *route_path, const char *handler_ref, - const char *module_qn, const cbm_registry_t *registry, - const cbm_gbuf_t *main_gbuf, const char **ik, const char **iv, - int ic, const char *route_mount) { +static bool emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, + const CBMCall *call, const char *route_path, + const char *handler_ref, const char *module_qn, + const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, + const char **ik, const char **iv, int ic, + const char *route_mount) { const char *method = cbm_service_pattern_route_method(call->callee_name); char mounted[CBM_SZ_256]; route_path = cbm_laravel_mount_route(route_mount, route_path, mounted, sizeof(mounted)); @@ -2344,9 +2343,7 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}", esc_cn, esc_rp); - if (cbm_gbuf_insert_edge(gbuf, source->id, rid, "CALLS", props) > 0) { - call->coverage_calls_emitted = true; - } + bool emitted = cbm_gbuf_insert_edge(gbuf, source->id, rid, "CALLS", props) > 0; if (handler_ref && handler_ref[0] != '\0') { cbm_resolution_t hres = cbm_registry_resolve(registry, handler_ref, module_qn, ik, iv, ic); if (hres.qualified_name && hres.qualified_name[0] != '\0') { @@ -2361,6 +2358,7 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou } } } + return emitted; } /* Reject regex metacharacters, spaces, double-slashes in URL candidates. */ @@ -2414,7 +2412,8 @@ static bool normalize_url_arg(const char *url, char *norm, int norm_sz) { } /* Detect API paths in call arguments and create HTTP_CALLS edges. */ -static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMCall *call) { +static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, + const CBMCall *call) { for (int ai = 0; ai < call->arg_count; ai++) { const CBMCallArg *ca = &call->args[ai]; /* A slash-prefixed raw expression is not a URL string. In JS/TS this @@ -2519,7 +2518,7 @@ bool extract_grpc_service_method(const char *callee, char *service, size_t srv_s } /* Emit GRPC_CALLS edge via gRPC Route node. */ -static void emit_grpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMCall *call, +static void emit_grpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call, const cbm_resolution_t *res) { char service[CBM_SZ_256]; char method[CBM_SZ_256]; @@ -2596,7 +2595,7 @@ static void emit_graphql_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, c } /* Emit TRPC_CALLS edge. Extract procedure path from callee chain. */ -static void emit_trpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMCall *call, +static void emit_trpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call, const cbm_resolution_t *res) { /* tRPC calls: trpc.user.getById.query() → extract "user.getById" */ const char *callee = call->callee_name; @@ -2638,8 +2637,8 @@ static void emit_trpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMC * and the HTTP/ASYNC/gRPC/GraphQL/tRPC/CONFIG/route branches are unaffected, so * a verb-suffix HTTP client (api.patch('/x')), broker, or route registration * keeps its edge; only the fabricated project CALLS edge is dropped. */ -static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, - const cbm_gbuf_node_t *target, CBMCall *call, +static bool emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, + const cbm_gbuf_node_t *target, const CBMCall *call, const cbm_resolution_t *res, const char *module_qn, const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, const char **imp_keys, const char **imp_vals, int imp_count, @@ -2669,10 +2668,9 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const char *handler_ref = NULL; const char *route_path = find_route_path_in_args(call, &handler_ref); if (route_path) { - emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, - registry, main_gbuf, imp_keys, imp_vals, imp_count, - route_mount); - return; + return emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, + registry, main_gbuf, imp_keys, imp_vals, imp_count, + route_mount); } /* No path found — fall through to normal CALLS edge */ } @@ -2682,6 +2680,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, /* Set when this call already has its HTTP_CALLS edge from the service * patterns (typed with the verb its callee names). */ bool http_edge_emitted = false; + bool calls_emitted = false; if ((svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) && (has_url || has_topic)) { emit_http_async_service_edge(gbuf, source, call, res, svc, arg); @@ -2695,7 +2694,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, } else if (svc == CBM_SVC_CONFIG) { emit_config_edge(gbuf, source, target, call, res, arg); } else if (!suppress_plain_calls) { - emit_normal_calls_edge(gbuf, source, target, call, res); + calls_emitted = emit_normal_calls_edge(gbuf, source, target, call, res); } /* The arg-URL heuristic is for calls no service pattern knows (a local @@ -2705,6 +2704,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, if (!http_edge_emitted) { detect_url_in_args(gbuf, source, call); } + return calls_emitted; } /* The #725 guard refuses a suffix_match binding across a language boundary, @@ -2717,24 +2717,26 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, * CALLS edge to the refused target and no URL-argument scan. Dropping the * whole call lost every GET registration in a mixed-language repo while POST * (no `post` to collide with) survived. Mirrors pass_calls.c. */ -static void emit_xlang_refused_route(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, CBMCall *call, - const char *module_qn, const cbm_registry_t *registry, - const cbm_gbuf_t *main_gbuf, const char **imp_keys, - const char **imp_vals, int imp_count, +static bool emit_xlang_refused_route(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, + const CBMCall *call, const char *module_qn, + const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, + const char **imp_keys, const char **imp_vals, int imp_count, const char *route_mount) { if (cbm_service_pattern_route_method(call->callee_name) == NULL) { - return; + return false; } cbm_svc_kind_t svc = cbm_service_pattern_match(call->callee_name); if (svc != CBM_SVC_NONE && svc != CBM_SVC_ROUTE_REG) { - return; + return false; } const char *handler_ref = NULL; const char *route_path = find_route_path_in_args(call, &handler_ref); if (route_path) { - emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, registry, - main_gbuf, imp_keys, imp_vals, imp_count, route_mount); + return emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, + registry, main_gbuf, imp_keys, imp_vals, imp_count, + route_mount); } + return false; } /* Find the source node for an edge: enclosing function or file node. */ @@ -2937,7 +2939,7 @@ static bool lsp_idx_insert_leaf(CBMHashTable *index, CBMResolvedCall *candidate, return inserted; } -static const CBMResolvedCall *lsp_idx_lookup(const CBMHashTable *index, CBMCall *call, +static const CBMResolvedCall *lsp_idx_lookup(const CBMHashTable *index, const CBMCall *call, bool exact_site, bool *key_built, bool *ambiguous) { if (key_built) { *key_built = false; @@ -2981,10 +2983,31 @@ static const CBMResolvedCall *lsp_idx_lookup(const CBMHashTable *index, CBMCall : NULL; } +static CBMCallEvidence *resolve_evidence_at(CBMCallEvidence *evidence, int index, + CBMCallEvidence *fallback) { + if (evidence) { + return &evidence[index]; + } + return fallback; +} + +static CBMCallEvidence *resolve_evidence_new(const resolve_ctx_t *rc, const CBMFileResult *result) { + if (result->calls.count == 0) { + return NULL; + } + CBMCallEvidence *evidence = + cbm_calloc(CBM_MEM_CLASS_RESOLVE, (size_t)result->calls.count * sizeof(*evidence)); + if (!evidence) { + cbm_pipeline_mark_unresolved_capture_failed(rc->pctx ? rc->pctx->pipeline : NULL); + } + return evidence; +} + /* Resolve calls for one file and emit CALLS/HTTP_CALLS/ASYNC_CALLS edges. */ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFileResult *result, const char *rel, const char *module_qn, const char **imp_keys, - const char **imp_vals, int imp_count, CBMLanguage lang) { + const char **imp_vals, int imp_count, CBMLanguage lang, + CBMCallEvidence *evidence) { /* Framework mount of this file's routes (Laravel 11+ withRouting, #1146). */ char route_mount[CBM_SZ_128]; cbm_laravel_file_route_mount(rc->repo_path, rel, lang, result, route_mount, @@ -3060,7 +3083,9 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB bool field_index_built = false; bool field_index_ready = false; for (int c = 0; c < result->calls.count; c++) { - CBMCall *call = &result->calls.items[c]; + const CBMCall *call = &result->calls.items[c]; + CBMCallEvidence fallback = {0}; + CBMCallEvidence *site_evidence = resolve_evidence_at(evidence, c, &fallback); if (!call->callee_name) { continue; } @@ -3284,9 +3309,9 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB cbm_resolution_t svc_res = {.qualified_name = call->callee_name, .confidence = PP_HALF_CONF, .strategy = "service_pattern"}; - emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &svc_res, - module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, false, route_mount); + site_evidence->calls_emitted |= emit_service_edge( + ws->local_edge_buf, source_node, source_node, call, &svc_res, module_qn, + rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, false, route_mount); continue; } } @@ -3311,9 +3336,9 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB * route path (`cache.get(key)`) would be a self-loop; that holds * for #2053's LSP-external Rust calls too. Suppress it always; * route/HTTP/service classification is unchanged. */ - emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &fake_res, - module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, true, route_mount); + site_evidence->calls_emitted |= emit_service_edge( + ws->local_edge_buf, source_node, source_node, call, &fake_res, module_qn, + rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, true, route_mount); } else if (cbm_service_pattern_is_global_fetch(call->callee_name)) { /* Native `fetch()` (#856): only the global API once resolution * has failed to find a local/imported `fetch`. Call the low-level @@ -3373,8 +3398,9 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB /* #725: same guard as pass_calls.c — do not emit a suffix_match * CALLS edge across a language boundary. A route registration * behind the refused binding still gets its Route. */ - emit_xlang_refused_route(ws->local_edge_buf, source_node, call, module_qn, rc->registry, - rc->main_gbuf, imp_keys, imp_vals, imp_count, route_mount); + site_evidence->calls_emitted |= emit_xlang_refused_route( + ws->local_edge_buf, source_node, call, module_qn, rc->registry, rc->main_gbuf, + imp_keys, imp_vals, imp_count, route_mount); continue; } if (!target_node || source_node->id == target_node->id) { @@ -3390,19 +3416,19 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB (u[0] == '/' || strstr(u, "://") != NULL || (psvc == CBM_SVC_ASYNC && strlen(u) > PP_ESC_SPACE)); if (url_or_topic) { - emit_service_edge(ws->local_edge_buf, source_node, NULL, call, &res, module_qn, - rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - false, route_mount); + site_evidence->calls_emitted |= emit_service_edge( + ws->local_edge_buf, source_node, NULL, call, &res, module_qn, rc->registry, + rc->main_gbuf, imp_keys, imp_vals, imp_count, false, route_mount); ws->calls_resolved++; } } continue; } - call->coverage_candidate_qn = target_node->qualified_name; + site_evidence->candidate_qn = target_node->qualified_name; _rc_t0 = extract_now_ns(); - emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res, module_qn, - rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - drop_plain_call, route_mount); + site_evidence->calls_emitted |= emit_service_edge( + ws->local_edge_buf, source_node, target_node, call, &res, module_qn, rc->registry, + rc->main_gbuf, imp_keys, imp_vals, imp_count, drop_plain_call, route_mount); atomic_fetch_add_explicit(&rc->time_ns_rc_emit, extract_now_ns() - _rc_t0, memory_order_relaxed); ws->calls_resolved++; @@ -3896,8 +3922,9 @@ static CBMFileResult *resolve_load_result(resolve_ctx_t *rc, resolve_worker_stat } static void resolve_capture_coverage(const resolve_ctx_t *rc, const char *rel, - const CBMFileResult *result) { - cbm_pipeline_record_unresolved_calls(rc->pctx ? rc->pctx->pipeline : NULL, rel, result); + const CBMFileResult *result, const CBMCallEvidence *evidence) { + cbm_pipeline_record_unresolved_calls(rc->pctx ? rc->pctx->pipeline : NULL, rel, result, + evidence); } /* A C member access binds only through a field-owner row. The per-file walk @@ -3995,6 +4022,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { CBMLanguage lang = rc->files[file_idx].language; const char *rel = rc->files[file_idx].rel_path; + CBMCallEvidence *evidence = NULL; /* Skip cross-LSP for machine-generated files — they're huge (10k- * 70k lines for k8s protobuf/openapi), have low semantic value for @@ -4052,7 +4080,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { if (result->calls.count == 0 && result->usages.count == 0 && result->throws.count == 0 && result->rw.count == 0 && result->defs.count == 0 && result->impl_traits.count == 0 && !cross_lsp_eligible) { - resolve_capture_coverage(rc, rel, result); + resolve_capture_coverage(rc, rel, result, NULL); continue; } @@ -4181,9 +4209,12 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { /* Per-sub-phase wall-clock so we can attribute the dominant cost. */ uint64_t _ph_t0; + /* Cross-LSP may append synthetic calls; their positions are now final. */ + evidence = resolve_evidence_new(rc, result); /* ── CALLS resolution ──────────────────────────────────── */ _ph_t0 = extract_now_ns(); - resolve_file_calls(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, lang); + resolve_file_calls(rc, ws, result, rel, module_qn, imp_keys, imp_vals, imp_count, lang, + evidence); atomic_fetch_add_explicit(&rc->time_ns_calls, extract_now_ns() - _ph_t0, memory_order_relaxed); @@ -4211,9 +4242,10 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { memory_order_relaxed); resolve_file_cleanup: - /* The cross-file additions and actual emitted-site evidence live in - * this result, including when it was loaded from the spill store. */ - resolve_capture_coverage(rc, rel, result); + /* Capture the resolved file and side evidence before releasing either, + * including files loaded from the spill store. */ + resolve_capture_coverage(rc, rel, result, evidence); + cbm_free(CBM_MEM_CLASS_RESOLVE, evidence); cbm_registry_reach_cache_end(); cbm_registry_import_map_cache_end(); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index be3a1af70..34a579e13 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -927,28 +927,44 @@ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *row static void *unresolved_json_alloc(void *ctx, size_t size) { (void)ctx; - return cbm_alloc(CBM_MEM_CLASS_DUMP, size); + return cbm_alloc(CBM_MEM_CLASS_RESOLVE, size); } static void *unresolved_json_realloc(void *ctx, void *ptr, size_t old_size, size_t size) { (void)ctx; (void)old_size; - return cbm_realloc(CBM_MEM_CLASS_DUMP, ptr, size); + return cbm_realloc(CBM_MEM_CLASS_RESOLVE, ptr, size); } static void unresolved_json_free(void *ctx, void *ptr) { (void)ctx; - cbm_free(CBM_MEM_CLASS_DUMP, ptr); + cbm_free(CBM_MEM_CLASS_RESOLVE, ptr); } -enum { UNRESOLVED_KEY_OVERHEAD = 128, UNRESOLVED_INITIAL_ROWS = 64, UNRESOLVED_ROW_GROWTH = 2 }; +enum { + UNRESOLVED_KEY_OVERHEAD = 128, + UNRESOLVED_INITIAL_ROWS = 64, + UNRESOLVED_ROW_GROWTH = 2, + UNRESOLVED_MAX_SITES = 1000, + UNRESOLVED_MAX_JSON_BYTES = 128 * 1024, + UNRESOLVED_SITE_OVERHEAD = 256, + UNRESOLVED_TRUNCATION_RESERVE = 32, + UNRESOLVED_JSON_ARRAY_BYTES = 2, + UNRESOLVED_JSON_RAW_CHAR_BYTES = 1, + UNRESOLVED_JSON_ESCAPE_BYTES = 2, + UNRESOLVED_JSON_CONTROL_ESCAPE_BYTES = 6, + UNRESOLVED_JSON_CONTROL_BOUNDARY = 0x20, + UNRESOLVED_JSON_OVERSIZE = UNRESOLVED_MAX_JSON_BYTES + 1, + UNRESOLVED_NO_CARRIER = -1, + UNRESOLVED_AMBIGUOUS_CARRIER = -2 +}; static char *unresolved_site_key(const char *caller, const char *leaf, uint32_t start, uint32_t end, CBMSourceOrigin origin) { size_t cn = strlen(caller); size_t ln = strlen(leaf); size_t cap = cn + ln + UNRESOLVED_KEY_OVERHEAD; - char *key = cbm_alloc(CBM_MEM_CLASS_DUMP, cap); + char *key = cbm_alloc(CBM_MEM_CLASS_RESOLVE, cap); if (key) { (void)snprintf(key, cap, "%zu:%s|%zu:%s|%u:%u:%u", cn, caller, ln, leaf, start, end, (unsigned)origin); @@ -957,7 +973,7 @@ static char *unresolved_site_key(const char *caller, const char *leaf, uint32_t } typedef struct { - const CBMCall *call; + int call_index; } unresolved_carrier_t; typedef struct { @@ -965,18 +981,24 @@ typedef struct { yyjson_mut_val *sites; CBMHashTable *carriers; CBMHashTable *seen; + const CBMFileResult *result; + const CBMCallEvidence *evidence; + const CBMResolvedCall *selected[UNRESOLVED_MAX_SITES]; + int selected_count; + size_t json_bytes; + bool truncated; } unresolved_capture_t; static void unresolved_carrier_free(const char *key, void *value, void *ctx) { (void)ctx; - cbm_free(CBM_MEM_CLASS_DUMP, (void *)key); - cbm_free(CBM_MEM_CLASS_DUMP, value); + cbm_free(CBM_MEM_CLASS_RESOLVE, (void *)key); + cbm_free(CBM_MEM_CLASS_RESOLVE, value); } static void unresolved_key_free(const char *key, void *value, void *ctx) { (void)value; (void)ctx; - cbm_free(CBM_MEM_CLASS_DUMP, (void *)key); + cbm_free(CBM_MEM_CLASS_RESOLVE, (void *)key); } static void unresolved_capture_free(unresolved_capture_t *capture) { @@ -1007,52 +1029,129 @@ static bool unresolved_has_missing_calls(const CBMFileResult *result) { return false; } -static bool unresolved_add_carrier(CBMHashTable *carriers, const CBMCall *call) { - if (!call->callee_name || call->site_end_byte <= call->site_start_byte) { - return true; +static size_t unresolved_json_char_bytes(unsigned char ch) { + if (ch < UNRESOLVED_JSON_CONTROL_BOUNDARY) { + return UNRESOLVED_JSON_CONTROL_ESCAPE_BYTES; } - char *key = - unresolved_site_key("", cbm_lsp_bare_segment(call->callee_name), call->site_start_byte, - call->site_end_byte, call->source_origin); - if (!key) { - return false; + if (ch == '"' || ch == '\\') { + return UNRESOLVED_JSON_ESCAPE_BYTES; } - if (cbm_ht_has(carriers, key)) { - /* Ambiguous extractor occurrences must never erase a diagnostic. */ - unresolved_carrier_t *carrier = cbm_ht_get(carriers, key); - carrier->call = NULL; - cbm_free(CBM_MEM_CLASS_DUMP, key); - return true; - } - unresolved_carrier_t *carrier = cbm_alloc(CBM_MEM_CLASS_DUMP, sizeof(*carrier)); - if (!carrier) { - cbm_free(CBM_MEM_CLASS_DUMP, key); - return false; - } - carrier->call = call; - cbm_ht_set(carriers, key, carrier); - if (!cbm_ht_has(carriers, key)) { - cbm_free(CBM_MEM_CLASS_DUMP, key); - cbm_free(CBM_MEM_CLASS_DUMP, carrier); - return false; - } - return true; + return UNRESOLVED_JSON_RAW_CHAR_BYTES; } -static bool unresolved_build_carriers(CBMHashTable *carriers, const CBMFileResult *result) { - /* Caller names in the TS LSP walk can denote an outer factory. The - * extractor's exact occurrence is the authoritative enclosing function. - * Index by span AND leaf: two receiver calls on one line remain distinct. */ - for (int i = 0; i < result->calls.count; i++) { - if (!unresolved_add_carrier(carriers, &result->calls.items[i])) { +/* Stop at the budget even for one enormous string. Reserve six bytes for each + * escaped control character; quotes and backslashes use two bytes in JSON. */ +static size_t unresolved_string_json_bytes(const char *text) { + size_t bytes = 0; + for (const unsigned char *p = (const unsigned char *)text; p && *p; p++) { + size_t width = unresolved_json_char_bytes(*p); + if (bytes > UNRESOLVED_MAX_JSON_BYTES - width) { + return UNRESOLVED_JSON_OVERSIZE; + } + bytes += width; + } + return bytes; +} + +static size_t unresolved_site_json_bytes(const CBMResolvedCall *rc, const char *caller, + const char *leaf, const char *candidate) { + const char *fields[] = {caller, leaf, rc->reason ? rc->reason : "unresolved", candidate}; + size_t bytes = UNRESOLVED_SITE_OVERHEAD; + for (size_t i = 0; i < sizeof(fields) / sizeof(fields[0]); i++) { + size_t field_bytes = unresolved_string_json_bytes(fields[i]); + if (field_bytes > UNRESOLVED_MAX_JSON_BYTES - bytes) { + return UNRESOLVED_JSON_OVERSIZE; + } + bytes += field_bytes; + } + return bytes; +} + +/* Hash only the bounded diagnostic subset. Hashing every extracted call + * would retain unbounded side state. */ +static bool unresolved_select_diagnostics(unresolved_capture_t *capture) { + size_t selected_bytes = UNRESOLVED_JSON_ARRAY_BYTES + UNRESOLVED_TRUNCATION_RESERVE; + const CBMFileResult *result = capture->result; + for (int i = 0; i < result->resolved_calls.count; i++) { + const CBMResolvedCall *rc = &result->resolved_calls.items[i]; + if (!unresolved_call_is_missing(rc)) { + continue; + } + const char *leaf = cbm_lsp_bare_segment(rc->callee_qn); + size_t site_bytes = unresolved_site_json_bytes(rc, rc->caller_qn, leaf, NULL); + if (capture->selected_count == UNRESOLVED_MAX_SITES || + site_bytes > UNRESOLVED_MAX_JSON_BYTES - selected_bytes) { + capture->truncated = true; + break; + } + selected_bytes += site_bytes; + capture->selected[capture->selected_count++] = rc; + char *key = unresolved_site_key("", leaf, rc->site_start_byte, rc->site_end_byte, + rc->source_origin); + if (!key) { + return false; + } + if (cbm_ht_has(capture->carriers, key)) { + cbm_free(CBM_MEM_CLASS_RESOLVE, key); + continue; + } + unresolved_carrier_t *carrier = cbm_alloc(CBM_MEM_CLASS_RESOLVE, sizeof(*carrier)); + if (!carrier) { + cbm_free(CBM_MEM_CLASS_RESOLVE, key); + return false; + } + carrier->call_index = UNRESOLVED_NO_CARRIER; + cbm_ht_set(capture->carriers, key, carrier); + if (!cbm_ht_has(capture->carriers, key)) { + cbm_free(CBM_MEM_CLASS_RESOLVE, key); + cbm_free(CBM_MEM_CLASS_RESOLVE, carrier); return false; } } return true; } +static bool unresolved_build_carriers(unresolved_capture_t *capture) { + if (!unresolved_select_diagnostics(capture)) { + return false; + } + const CBMFileResult *result = capture->result; + /* Caller names in the TS LSP walk can denote an outer factory. Join on + * span, leaf and origin; the extractor owns the enclosing function name. */ + for (int i = 0; i < result->calls.count && capture->selected_count > 0; i++) { + const CBMCall *call = &result->calls.items[i]; + if (!call->callee_name || call->site_end_byte <= call->site_start_byte) { + continue; + } + const char *leaf = cbm_lsp_bare_segment(call->callee_name); + if (unresolved_string_json_bytes(leaf) > UNRESOLVED_MAX_JSON_BYTES) { + continue; /* No bounded diagnostic key can match this leaf. */ + } + char *key = unresolved_site_key("", leaf, call->site_start_byte, call->site_end_byte, + call->source_origin); + if (!key) { + return false; + } + unresolved_carrier_t *carrier = cbm_ht_get(capture->carriers, key); + cbm_free(CBM_MEM_CLASS_RESOLVE, key); + if (carrier) { + /* An ambiguous occurrence must never erase a diagnostic. */ + carrier->call_index = + carrier->call_index == UNRESOLVED_NO_CARRIER ? i : UNRESOLVED_AMBIGUOUS_CARRIER; + } + } + return true; +} + static bool unresolved_append_site(unresolved_capture_t *capture, const CBMResolvedCall *rc, - const CBMCall *call, const char *caller, const char *leaf) { + const CBMCall *call, const CBMCallEvidence *evidence, + const char *caller, const char *leaf) { + const char *candidate = evidence ? evidence->candidate_qn : NULL; + size_t site_bytes = unresolved_site_json_bytes(rc, caller, leaf, candidate); + if (site_bytes > UNRESOLVED_MAX_JSON_BYTES - capture->json_bytes) { + capture->truncated = true; + return true; + } yyjson_mut_doc *doc = capture->doc; yyjson_mut_val *site = yyjson_mut_obj(doc); bool ok = @@ -1065,9 +1164,10 @@ static bool unresolved_append_site(unresolved_capture_t *capture, const CBMResol if (ok && call && call->start_line > 0) { ok = yyjson_mut_obj_add_int(doc, site, "line", call->start_line); } - if (ok && call && call->coverage_candidate_qn) { - ok = yyjson_mut_obj_add_strcpy(doc, site, "candidate", call->coverage_candidate_qn); + if (ok && candidate) { + ok = yyjson_mut_obj_add_strcpy(doc, site, "candidate", candidate); } + capture->json_bytes += site_bytes; return ok && yyjson_mut_arr_add_val(capture->sites, site); } @@ -1079,38 +1179,52 @@ static bool unresolved_collect_site(unresolved_capture_t *capture, const CBMReso return false; } const unresolved_carrier_t *carrier = cbm_ht_get(capture->carriers, lookup); - const CBMCall *call = carrier ? carrier->call : NULL; - cbm_free(CBM_MEM_CLASS_DUMP, lookup); - /* This exact occurrence emitted a real CALLS edge, possibly via the - * registry after LSP failure. Never suppress a different occurrence. */ - if (call && call->coverage_calls_emitted) { + int call_index = carrier ? carrier->call_index : UNRESOLVED_NO_CARRIER; + const CBMCall *call = call_index >= 0 ? &capture->result->calls.items[call_index] : NULL; + const CBMCallEvidence *evidence = + call_index >= 0 && capture->evidence ? &capture->evidence[call_index] : NULL; + cbm_free(CBM_MEM_CLASS_RESOLVE, lookup); + /* Suppress only this exact occurrence's real CALLS edge, including dedup. + * HTTP/ASYNC/CONFIGURES/SPAWNS and a candidate alone never suppress it. */ + if (evidence && evidence->calls_emitted) { return true; } const char *caller = call && call->enclosing_func_qn ? call->enclosing_func_qn : rc->caller_qn; + /* Bound both the retained dedup key and copied JSON strings. */ + size_t site_bytes = + unresolved_site_json_bytes(rc, caller, leaf, evidence ? evidence->candidate_qn : NULL); + if (site_bytes > UNRESOLVED_MAX_JSON_BYTES - capture->json_bytes) { + capture->truncated = true; + return true; + } char *key = unresolved_site_key(caller, leaf, rc->site_start_byte, rc->site_end_byte, rc->source_origin); if (!key) { return false; } if (cbm_ht_has(capture->seen, key)) { - cbm_free(CBM_MEM_CLASS_DUMP, key); + cbm_free(CBM_MEM_CLASS_RESOLVE, key); return true; } cbm_ht_set(capture->seen, key, key); if (!cbm_ht_has(capture->seen, key)) { - cbm_free(CBM_MEM_CLASS_DUMP, key); + cbm_free(CBM_MEM_CLASS_RESOLVE, key); return false; } - return unresolved_append_site(capture, rc, call, caller, leaf); + return unresolved_append_site(capture, rc, call, evidence, caller, leaf); } -static bool unresolved_collect_sites(unresolved_capture_t *capture, const CBMFileResult *result) { - for (int i = 0; i < result->resolved_calls.count; i++) { - const CBMResolvedCall *rc = &result->resolved_calls.items[i]; - if (unresolved_call_is_missing(rc) && !unresolved_collect_site(capture, rc)) { +static bool unresolved_collect_sites(unresolved_capture_t *capture) { + for (int i = 0; i < capture->selected_count; i++) { + if (!unresolved_collect_site(capture, capture->selected[i])) { return false; } } + if (capture->truncated) { + yyjson_mut_val *marker = yyjson_mut_obj(capture->doc); + return marker && yyjson_mut_obj_add_bool(capture->doc, marker, "truncated", true) && + yyjson_mut_arr_add_val(capture->sites, marker); + } return true; } @@ -1122,8 +1236,8 @@ static bool unresolved_append_row(cbm_pipeline_t *p, const char *path, const cha if (p->unresolved_count == p->unresolved_capacity) { int capacity = p->unresolved_capacity ? p->unresolved_capacity * UNRESOLVED_ROW_GROWTH : UNRESOLVED_INITIAL_ROWS; - cbm_coverage_row_t *rows = - cbm_realloc(CBM_MEM_CLASS_DUMP, p->unresolved_rows, (size_t)capacity * sizeof(*rows)); + cbm_coverage_row_t *rows = cbm_realloc(CBM_MEM_CLASS_RESOLVE, p->unresolved_rows, + (size_t)capacity * sizeof(*rows)); if (!rows) { return false; } @@ -1136,7 +1250,8 @@ static bool unresolved_append_row(cbm_pipeline_t *p, const char *path, const cha } void cbm_pipeline_record_unresolved_calls(cbm_pipeline_t *p, const char *rel_path, - const CBMFileResult *result) { + const CBMFileResult *result, + const CBMCallEvidence *evidence) { if (!p || !rel_path || !result || p->unresolved_capture_failed) { return; } @@ -1147,24 +1262,32 @@ void cbm_pipeline_record_unresolved_calls(cbm_pipeline_t *p, const char *rel_pat .realloc = unresolved_json_realloc, .free = unresolved_json_free}; unresolved_capture_t capture = {.doc = yyjson_mut_doc_new(&allocator), - .carriers = cbm_ht_create(0), - .seen = cbm_ht_create(0)}; + .carriers = cbm_ht_create_in(CBM_MEM_CLASS_RESOLVE, 0), + .seen = cbm_ht_create_in(CBM_MEM_CLASS_RESOLVE, 0), + .result = result, + .evidence = evidence, + .json_bytes = UNRESOLVED_JSON_ARRAY_BYTES + + UNRESOLVED_TRUNCATION_RESERVE}; capture.sites = capture.doc ? yyjson_mut_arr(capture.doc) : NULL; bool ok = capture.doc && capture.carriers && capture.seen && capture.sites; if (capture.doc) { yyjson_mut_doc_set_root(capture.doc, capture.sites); } - ok = ok && unresolved_build_carriers(capture.carriers, result) && - unresolved_collect_sites(&capture, result); + ok = ok && unresolved_build_carriers(&capture) && unresolved_collect_sites(&capture); if (ok && yyjson_mut_arr_size(capture.sites) > 0) { char *detail = yyjson_mut_write_opts(capture.doc, 0, &allocator, NULL, NULL); - char *path = cbm_mem_strdup(CBM_MEM_CLASS_DUMP, rel_path); - cbm_mutex_lock(&p->unresolved_mutex); - ok = unresolved_append_row(p, path, detail); - cbm_mutex_unlock(&p->unresolved_mutex); + char *path = cbm_mem_strdup(CBM_MEM_CLASS_RESOLVE, rel_path); + /* The estimate reserves overhead and escaping. Enforce the serialized + * limit as well, before any detail is transferred to persistence. */ + ok = detail && strlen(detail) <= UNRESOLVED_MAX_JSON_BYTES; + if (ok) { + cbm_mutex_lock(&p->unresolved_mutex); + ok = unresolved_append_row(p, path, detail); + cbm_mutex_unlock(&p->unresolved_mutex); + } if (!ok) { - cbm_free(CBM_MEM_CLASS_DUMP, detail); - cbm_free(CBM_MEM_CLASS_DUMP, path); + cbm_free(CBM_MEM_CLASS_RESOLVE, detail); + cbm_free(CBM_MEM_CLASS_RESOLVE, path); } } if (!ok) { @@ -1232,10 +1355,10 @@ void cbm_pipeline_free(cbm_pipeline_t *p) { p->surface_rows = NULL; p->surface_row_count = 0; for (int i = 0; i < p->unresolved_count; i++) { - cbm_free(CBM_MEM_CLASS_DUMP, (char *)p->unresolved_rows[i].rel_path); - cbm_free(CBM_MEM_CLASS_DUMP, (char *)p->unresolved_rows[i].detail); + cbm_free(CBM_MEM_CLASS_RESOLVE, (char *)p->unresolved_rows[i].rel_path); + cbm_free(CBM_MEM_CLASS_RESOLVE, (char *)p->unresolved_rows[i].detail); } - cbm_free(CBM_MEM_CLASS_DUMP, p->unresolved_rows); + cbm_free(CBM_MEM_CLASS_RESOLVE, p->unresolved_rows); cbm_mutex_destroy(&p->unresolved_mutex); cbm_git_context_free(&p->git_ctx); /* gbuf, store, registry freed during/after run */ @@ -2422,9 +2545,6 @@ static int run_sequential_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, if (seq_cache) { for (int i = 0; i < file_count; i++) { if (seq_cache[i]) { - if (rc == 0) { - cbm_pipeline_record_unresolved_calls(p, files[i].rel_path, seq_cache[i]); - } cbm_free_result(seq_cache[i]); } } @@ -3456,6 +3576,11 @@ int cbm_pipeline_finalize_staged_generation(char *path, const char *destination, return finalize_staged_impl(path, destination, cancelled, healthy, NULL); } +static bool pipeline_unresolved_capture_complete(const cbm_pipeline_t *p, + bool coverage_rows_available) { + return coverage_rows_available && !p->unresolved_capture_failed; +} + /* Dump graph to SQLite and persist file hashes for incremental indexing. */ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *baseline_manifest, int baseline_count, struct timespec *t) { @@ -3517,11 +3642,11 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas p->file_errors_count + p->excluded_count + p->ignored_count + p->unresolved_count; cbm_coverage_row_t *cov = NULL; int cov_count = 0; - bool coverage_rows_available = cov_total == 0 && !p->unresolved_capture_failed; + bool coverage_rows_available = cov_total == 0; if (cov_total > 0) { cov = malloc((size_t)cov_total * sizeof(*cov)); if (cov) { - coverage_rows_available = !p->unresolved_capture_failed; + coverage_rows_available = true; for (int i = 0; i < p->unresolved_count; i++) { cov[cov_count++] = p->unresolved_rows[i]; } @@ -3563,6 +3688,8 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas .ignored_files_total = p->ignored_total, .coverage_version = CBM_SEMANTIC_INDEX_VERSION, .hash_records_complete = true, + .unresolved_calls_complete = + pipeline_unresolved_capture_complete(p, coverage_rows_available), }, .surface_rows = p->surface_rows, .surface_row_count = p->surface_row_count, diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index df6453c50..a6451f4b8 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1725,14 +1725,6 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed if (rc == 0) { rc = cbm_pipeline_pass_semantic(ctx, changed_files, ci); } - if (rc == 0 && cache) { - for (int i = 0; i < ci; i++) { - if (cache[i]) { - cbm_pipeline_record_unresolved_calls(ctx->pipeline, changed_files[i].rel_path, - cache[i]); - } - } - } if (rc == 0) { rc = cbm_pipeline_check_cancel(ctx); } @@ -2385,6 +2377,31 @@ done: return 1; } +static bool incremental_metadata_current(int meta_rc, const cbm_coverage_meta_t *meta, + const char *mode_name) { + return meta_rc == CBM_STORE_OK && meta->coverage_version == CBM_SEMANTIC_INDEX_VERSION && + meta->unresolved_calls_complete && meta->hash_records_complete && meta->index_mode && + strcmp(meta->index_mode, mode_name) == 0; +} + +static bool incremental_retained_capture_complete(cbm_store_t *store, const char *project) { + cbm_coverage_meta_t meta = {0}; + bool complete = cbm_store_coverage_meta_get(store, project, &meta) == CBM_STORE_OK && + meta.coverage_version >= CBM_UNRESOLVED_CALL_COVERAGE_VERSION && + meta.unresolved_calls_complete; + cbm_store_coverage_meta_clear(&meta); + return complete; +} + +static bool incremental_coverage_rows_available(int count, const cbm_coverage_row_t *rows) { + return count == 0 || rows != NULL; +} + +static bool incremental_capture_complete(bool rows_available, bool run_complete, + bool retained_complete) { + return rows_available && run_complete && retained_complete; +} + /* ── Delta-repair orchestration (closure route) ────────────────── * * The closure route's executor: clone the live generation, repair the @@ -2401,8 +2418,8 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char closure_plan_t *plan, cbm_file_info_t *changed_files, int ci, char **deleted, int deleted_count, cbm_file_hash_t *mode_skipped, int mode_skipped_count, cbm_coverage_row_t *old_cov, int old_cov_count, - cbm_store_t *route_store, const cbm_file_info_t *files, int file_count, - struct timespec t0) { + bool retained_unresolved_complete, cbm_store_t *route_store, + const cbm_file_info_t *files, int file_count, struct timespec t0) { struct timespec t; cbm_store_close(route_store); @@ -2705,10 +2722,10 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char &unresolved_complete); int cov_cap = old_cov_count + run_err_count + run_excluded_count + run_ignored_count + run_unresolved_count; - bool coverage_rows_available = cov_cap == 0 && unresolved_complete; + bool coverage_rows_available = cov_cap == 0; if (cov_cap > 0) { cov = (cbm_coverage_row_t *)malloc((size_t)cov_cap * sizeof(*cov)); - coverage_rows_available = cov != NULL && unresolved_complete; + coverage_rows_available = cov != NULL; } if (cov) { for (int i = 0; i < old_cov_count; i++) { @@ -2812,6 +2829,8 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char .ignored_files_total = run_ignored_total, .coverage_version = CBM_SEMANTIC_INDEX_VERSION, .hash_records_complete = true, + .unresolved_calls_complete = incremental_capture_complete( + coverage_rows_available, unresolved_complete, retained_unresolved_complete), }, .surface_rows = NULL, .surface_row_count = 0, @@ -2927,10 +2946,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_coverage_meta_t meta = {0}; int meta_rc = cbm_store_coverage_meta_get(store, project, &meta); const char *mode_name = incr_mode_name(cbm_pipeline_get_mode(p)); - bool metadata_current = meta_rc == CBM_STORE_OK && - meta.coverage_version == CBM_SEMANTIC_INDEX_VERSION && - meta.hash_records_complete && meta.index_mode && - strcmp(meta.index_mode, mode_name) == 0; + bool metadata_current = incremental_metadata_current(meta_rc, &meta, mode_name); bool exact = metadata_current && cbm_pipeline_semantic_manifests_equal(stored, stored_count, baseline_manifest, baseline_count); @@ -3030,6 +3046,11 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_store_free_file_hashes(stored, stored_count); + /* A subset repair cannot recover unresolved evidence omitted by an older + * attempt. Normal routing rebuilds such baselines; the legacy test route + * also preserves their incomplete signal when it retains unchanged files. */ + bool retained_unresolved_complete = incremental_retained_capture_complete(store, project); + /* Coverage rows (#963): the dump below rebuilds the DB file, wiping the * separate index_coverage table — capture the previous rows now (store * still open) so entries for files NOT re-extracted this run survive. */ @@ -3073,8 +3094,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * live database; its every failure falls back to a full rebuild. */ return run_closure_delta(p, db_path, project, baseline_manifest, baseline_count, &closure_plan, changed_files, ci, deleted, deleted_count, - mode_skipped, mode_skipped_count, old_cov, old_cov_count, store, - files, file_count, t0); + mode_skipped, mode_skipped_count, old_cov, old_cov_count, + retained_unresolved_complete, store, files, file_count, t0); } struct timespec t; @@ -3304,7 +3325,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil if (cov_cap > 0) { cov = (cbm_coverage_row_t *)malloc((size_t)cov_cap * sizeof(*cov)); } - bool coverage_rows_available = (cov_cap == 0 || cov != NULL) && unresolved_complete; + bool coverage_rows_available = incremental_coverage_rows_available(cov_cap, cov); if (cov) { CBMHashTable *changed_set = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64); for (int i = 0; i < ci; i++) { @@ -3392,6 +3413,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil .ignored_files_total = run_ignored_total, .coverage_version = CBM_SEMANTIC_INDEX_VERSION, .hash_records_complete = true, + .unresolved_calls_complete = incremental_capture_complete( + coverage_rows_available, unresolved_complete, retained_unresolved_complete), }; /* Publish surfaces: the surviving previous rows plus this run's fresh * ones (closure route). The legacy test route publishes none — its diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index f818bf307..95f26b687 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1196,7 +1196,7 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr /* Compatibility contract persisted in coverage metadata. Increment when a * graph/manifest semantic change makes prior exact-input indexes unsafe. */ -enum { CBM_SEMANTIC_INDEX_VERSION = CBM_UNRESOLVED_CALL_COVERAGE_VERSION }; +enum { CBM_SEMANTIC_INDEX_VERSION = 5 }; typedef struct { cbm_gbuf_t *gbuf; @@ -1281,8 +1281,15 @@ void cbm_pipeline_discard_stage(const char *stage_path); * Takes ownership; dump_and_persist_hashes writes them into the staging * store and cbm_pipeline_free releases them. Passing NULL/0 clears. */ void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *rows, int count); +/* One entry per call, owned by the current file's resolve pass. Candidate + * strings are borrowed from the graph and copied by coverage capture. */ +typedef struct { + const char *candidate_qn; + bool calls_emitted; +} CBMCallEvidence; void cbm_pipeline_record_unresolved_calls(cbm_pipeline_t *p, const char *rel_path, - const CBMFileResult *result); + const CBMFileResult *result, + const CBMCallEvidence *evidence); void cbm_pipeline_mark_unresolved_capture_failed(cbm_pipeline_t *p); void cbm_pipeline_get_unresolved_calls(cbm_pipeline_t *p, cbm_coverage_row_t **rows, int *count, bool *complete); diff --git a/src/store/store.c b/src/store/store.c index 996d71cc7..47ffdea85 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -347,6 +347,18 @@ static int init_schema(cbm_store_t *s) { ");" "CREATE INDEX IF NOT EXISTS idx_index_coverage_project_kind " "ON index_coverage(project, kind);" + /* Derived candidate lookup: inbound traces read only this index, not + * every file's diagnostic JSON. An empty candidate marks truncated or + * malformed evidence, whose omitted candidates cannot be ruled out. */ + "CREATE TABLE IF NOT EXISTS index_unresolved_candidates (" + " project TEXT NOT NULL," + " candidate TEXT NOT NULL," + " rel_path TEXT NOT NULL," + " kind TEXT NOT NULL DEFAULT 'unresolved_calls' CHECK(kind='unresolved_calls')," + " PRIMARY KEY(project, candidate, rel_path)," + " FOREIGN KEY(project, rel_path, kind) " + " REFERENCES index_coverage(project, rel_path, kind) ON DELETE CASCADE" + ");" /* One row per completed coverage persistence attempt. Kept separate * from projects so existing graph/artifact schema stays compatible and * a missing row unambiguously means coverage metadata is unavailable. */ @@ -359,7 +371,8 @@ static int init_schema(cbm_store_t *s) { " ignored_files_stored INTEGER NOT NULL DEFAULT 0," " ignored_files_total INTEGER NOT NULL DEFAULT 0," " coverage_version INTEGER NOT NULL DEFAULT 1," - " hash_records_complete INTEGER NOT NULL DEFAULT 0" + " hash_records_complete INTEGER NOT NULL DEFAULT 0," + " unresolved_calls_complete INTEGER NOT NULL DEFAULT 0" ");"; int rc = exec_sql(s, ddl); @@ -367,6 +380,33 @@ static int init_schema(cbm_store_t *s) { return rc; } + /* Existing writable stores retain their ordinary coverage metadata, but + * unresolved capture is untrusted until a full generation records it. */ + sqlite3_stmt *coverage_columns = NULL; + if (sqlite3_prepare_v2(s->db, "PRAGMA table_info(index_coverage_meta);", CBM_NOT_FOUND, + &coverage_columns, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage meta schema prepare"); + return CBM_STORE_ERR; + } + bool have_unresolved_complete = false; + int column_rc; + while ((column_rc = sqlite3_step(coverage_columns)) == SQLITE_ROW) { + const char *name = (const char *)sqlite3_column_text(coverage_columns, SKIP_ONE); + if (name && strcmp(name, "unresolved_calls_complete") == 0) { + have_unresolved_complete = true; + } + } + sqlite3_finalize(coverage_columns); + if (column_rc != SQLITE_DONE) { + store_set_error_sqlite(s, "coverage meta schema scan"); + return CBM_STORE_ERR; + } + if (!have_unresolved_complete && + exec_sql(s, "ALTER TABLE index_coverage_meta ADD COLUMN " + "unresolved_calls_complete INTEGER NOT NULL DEFAULT 0;") != CBM_STORE_OK) { + return CBM_STORE_ERR; + } + /* Schema-compat probe (#768): DBs created before the local_name_gen * discriminator still enforce UNIQUE(source_id,target_id,type) and lack * the column — the widened upsert in cbm_store_insert_edge can neither @@ -4208,6 +4248,114 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) { return CBM_STORE_OK; } +/* Materialize once at publication, after replacement and deleted-file pruning. + * json_each sees a safe marker array for malformed/legacy non-array payloads. + * The side table cascades with its authoritative coverage rows. */ +static int coverage_index_unresolved_candidates(cbm_store_t *s, const char *project) { + static const char sql[] = + "INSERT OR IGNORE INTO index_unresolved_candidates(project, candidate, rel_path) " + "SELECT c.project, CASE WHEN j.type != 'object' OR " + "json_extract(CASE WHEN j.type = 'object' THEN j.value ELSE '{}' END, '$.truncated') = 1 " + "THEN '' " + "ELSE json_extract(CASE WHEN j.type = 'object' THEN j.value ELSE '{}' END, '$.candidate') " + "END, c.rel_path " + "FROM index_coverage c, json_each(CASE " + "WHEN json_valid(c.detail) THEN CASE WHEN json_type(c.detail) = 'array' " + "THEN c.detail ELSE '[{\"truncated\":true}]' END " + "ELSE '[{\"truncated\":true}]' END) j " + "WHERE c.project = ?1 AND c.kind = 'unresolved_calls' AND " + "(j.type != 'object' OR json_extract(CASE WHEN j.type = 'object' THEN j.value ELSE '{}' " + "END, '$.truncated') = 1 OR " + "(json_type(CASE WHEN j.type = 'object' THEN j.value ELSE '{}' END, '$.candidate') = " + "'text' AND " + "length(json_extract(CASE WHEN j.type = 'object' THEN j.value ELSE '{}' END, " + "'$.candidate')) > 0));"; + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "unresolved candidate index prepare"); + return CBM_STORE_ERR; + } + bind_text(stmt, SKIP_ONE, project); + int rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) { + store_set_error_sqlite(s, "unresolved candidate index"); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + +/* The caller owns the coverage transaction and rolls it back on failure. */ +static int coverage_write_metadata(cbm_store_t *s, const char *project, + const cbm_coverage_meta_t *meta) { + enum { COVERAGE_META_INITIAL_VERSION = 1 }; + if (!meta) { + sqlite3_stmt *del_meta = NULL; + if (sqlite3_prepare_v2(s->db, "DELETE FROM index_coverage_meta WHERE project = ?1;", + CBM_NOT_FOUND, &del_meta, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage meta delete prepare"); + return CBM_STORE_ERR; + } + bind_text(del_meta, SKIP_ONE, project); + int meta_rc = sqlite3_step(del_meta); + sqlite3_finalize(del_meta); + if (meta_rc != SQLITE_DONE) { + store_set_error_sqlite(s, "coverage meta delete"); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; + } + char recorded_at[CBM_SZ_64]; + if (meta->recorded_at && meta->recorded_at[0]) { + (void)snprintf(recorded_at, sizeof(recorded_at), "%s", meta->recorded_at); + } else { + iso_now(recorded_at, sizeof(recorded_at)); + } + const char *generation = + meta->generation && meta->generation[0] ? meta->generation : recorded_at; + const char *index_mode = meta->index_mode && meta->index_mode[0] ? meta->index_mode : "unknown"; + const char *recording_status = meta->recording_status && meta->recording_status[0] + ? meta->recording_status + : "unavailable"; + int ignored_stored = meta->ignored_files_stored > 0 ? meta->ignored_files_stored : 0; + int ignored_total = meta->ignored_files_total > 0 ? meta->ignored_files_total : 0; + int coverage_version = + meta->coverage_version > 0 ? meta->coverage_version : COVERAGE_META_INITIAL_VERSION; + + sqlite3_stmt *up_meta = NULL; + if (sqlite3_prepare_v2(s->db, + "INSERT INTO index_coverage_meta " + "(project, generation, index_mode, recorded_at, recording_status, " + " ignored_files_stored, ignored_files_total, coverage_version, " + " hash_records_complete, unresolved_calls_complete) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) " + "ON CONFLICT(project) DO UPDATE SET generation=?2, index_mode=?3, " + "recorded_at=?4, recording_status=?5, ignored_files_stored=?6, " + "ignored_files_total=?7, coverage_version=?8, hash_records_complete=?9, " + "unresolved_calls_complete=?10;", + CBM_NOT_FOUND, &up_meta, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage meta upsert prepare"); + return CBM_STORE_ERR; + } + bind_text(up_meta, SKIP_ONE, project); + bind_text(up_meta, ST_COL_2, generation); + bind_text(up_meta, ST_COL_3, index_mode); + bind_text(up_meta, CBM_SZ_4, recorded_at); + bind_text(up_meta, CBM_SZ_5, recording_status); + sqlite3_bind_int(up_meta, ST_COL_6, ignored_stored); + sqlite3_bind_int(up_meta, ST_COL_7, ignored_total); + sqlite3_bind_int(up_meta, ST_COL_8, coverage_version); + sqlite3_bind_int(up_meta, ST_COL_9, meta->hash_records_complete); + sqlite3_bind_int(up_meta, ST_COL_10, meta->unresolved_calls_complete); + int meta_rc = sqlite3_step(up_meta); + sqlite3_finalize(up_meta); + if (meta_rc != SQLITE_DONE) { + store_set_error_sqlite(s, "coverage meta upsert"); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows, int count, const cbm_coverage_meta_t *meta) { @@ -4305,72 +4453,14 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project, (cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000; cov_t0 = cov_t1; - if (meta) { - char recorded_at[CBM_SZ_64]; - if (meta->recorded_at && meta->recorded_at[0]) { - snprintf(recorded_at, sizeof(recorded_at), "%s", meta->recorded_at); - } else { - iso_now(recorded_at, sizeof(recorded_at)); - } - const char *generation = - meta->generation && meta->generation[0] ? meta->generation : recorded_at; - const char *index_mode = - meta->index_mode && meta->index_mode[0] ? meta->index_mode : "unknown"; - const char *recording_status = meta->recording_status && meta->recording_status[0] - ? meta->recording_status - : "unavailable"; - int ignored_stored = meta->ignored_files_stored > 0 ? meta->ignored_files_stored : 0; - int ignored_total = meta->ignored_files_total > 0 ? meta->ignored_files_total : 0; - int coverage_version = meta->coverage_version > 0 ? meta->coverage_version : 1; + if (coverage_index_unresolved_candidates(s, project) != CBM_STORE_OK) { + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; + } - sqlite3_stmt *up_meta = NULL; - if (sqlite3_prepare_v2( - s->db, - "INSERT INTO index_coverage_meta " - "(project, generation, index_mode, recorded_at, recording_status, " - " ignored_files_stored, ignored_files_total, coverage_version, " - " hash_records_complete) " - "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) " - "ON CONFLICT(project) DO UPDATE SET generation=?2, index_mode=?3, " - "recorded_at=?4, recording_status=?5, ignored_files_stored=?6, " - "ignored_files_total=?7, coverage_version=?8, hash_records_complete=?9;", - CBM_NOT_FOUND, &up_meta, NULL) != SQLITE_OK) { - store_set_error_sqlite(s, "coverage meta upsert prepare"); - (void)exec_sql(s, "ROLLBACK;"); - return CBM_STORE_ERR; - } - bind_text(up_meta, SKIP_ONE, project); - bind_text(up_meta, ST_COL_2, generation); - bind_text(up_meta, ST_COL_3, index_mode); - bind_text(up_meta, CBM_SZ_4, recorded_at); - bind_text(up_meta, CBM_SZ_5, recording_status); - sqlite3_bind_int(up_meta, 6, ignored_stored); - sqlite3_bind_int(up_meta, 7, ignored_total); - sqlite3_bind_int(up_meta, 8, coverage_version); - sqlite3_bind_int(up_meta, 9, meta->hash_records_complete ? 1 : 0); - int meta_rc = sqlite3_step(up_meta); - sqlite3_finalize(up_meta); - if (meta_rc != SQLITE_DONE) { - store_set_error_sqlite(s, "coverage meta upsert"); - (void)exec_sql(s, "ROLLBACK;"); - return CBM_STORE_ERR; - } - } else { - sqlite3_stmt *del_meta = NULL; - if (sqlite3_prepare_v2(s->db, "DELETE FROM index_coverage_meta WHERE project = ?1;", - CBM_NOT_FOUND, &del_meta, NULL) != SQLITE_OK) { - store_set_error_sqlite(s, "coverage meta delete prepare"); - (void)exec_sql(s, "ROLLBACK;"); - return CBM_STORE_ERR; - } - bind_text(del_meta, SKIP_ONE, project); - int meta_rc = sqlite3_step(del_meta); - sqlite3_finalize(del_meta); - if (meta_rc != SQLITE_DONE) { - store_set_error_sqlite(s, "coverage meta delete"); - (void)exec_sql(s, "ROLLBACK;"); - return CBM_STORE_ERR; - } + if (coverage_write_metadata(s, project, meta) != CBM_STORE_OK) { + (void)exec_sql(s, "ROLLBACK;"); + return CBM_STORE_ERR; } /* Rebuild the derived miss-graph view from the now-authoritative table @@ -4492,12 +4582,41 @@ int cbm_store_coverage_get_scope(cbm_store_t *s, const char *project, const char return coverage_query_rows(s, project, scope, sql, out, count); } -int cbm_store_coverage_get_unresolved_calls(cbm_store_t *s, const char *project, - cbm_coverage_row_t **out, int *count) { +int cbm_store_coverage_get_unresolved_path(cbm_store_t *s, const char *project, + const char *rel_path, cbm_coverage_row_t **out, + int *count) { static const char sql[] = "SELECT rel_path, kind, detail FROM index_coverage " - "WHERE project = ?1 AND kind = ?2 " - "ORDER BY rel_path;"; - return coverage_query_rows(s, project, "unresolved_calls", sql, out, count); + "WHERE project = ?1 AND rel_path = ?2 " + "AND kind = 'unresolved_calls';"; + return coverage_query_rows(s, project, rel_path, sql, out, count); +} + +int cbm_store_coverage_has_unresolved_candidate(cbm_store_t *s, const char *project, + const char *candidate, bool *out) { + if (!out) { + return CBM_STORE_ERR; + } + *out = false; + if (!s || !s->db || !project || !candidate) { + return CBM_STORE_ERR; + } + static const char sql[] = "SELECT 1 FROM index_unresolved_candidates " + "WHERE project = ?1 AND candidate IN ('', ?2) LIMIT 1;"; + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "unresolved candidate lookup prepare"); + return CBM_STORE_ERR; + } + bind_text(stmt, SKIP_ONE, project); + bind_text(stmt, ST_COL_2, candidate); + int rc = sqlite3_step(stmt); + *out = rc == SQLITE_ROW; + sqlite3_finalize(stmt); + if (rc != SQLITE_ROW && rc != SQLITE_DONE) { + store_set_error_sqlite(s, "unresolved candidate lookup"); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; } void cbm_store_coverage_meta_clear(cbm_coverage_meta_t *meta) { @@ -4524,10 +4643,20 @@ int cbm_store_coverage_meta_get(cbm_store_t *s, const char *project, cbm_coverag if (sqlite3_prepare_v2(s->db, "SELECT project, generation, index_mode, recorded_at, recording_status, " "ignored_files_stored, ignored_files_total, coverage_version, " - "hash_records_complete FROM index_coverage_meta WHERE project = ?1;", + "hash_records_complete, unresolved_calls_complete " + "FROM index_coverage_meta WHERE project = ?1;", CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { - store_set_error_sqlite(s, "coverage meta get prepare"); - return CBM_STORE_ERR; + /* Query opens never migrate a legacy database. Preserve its ordinary + * report while the absent new signal remains explicitly incomplete. */ + if (sqlite3_prepare_v2( + s->db, + "SELECT project, generation, index_mode, recorded_at, recording_status, " + "ignored_files_stored, ignored_files_total, coverage_version, " + "hash_records_complete, 0 FROM index_coverage_meta WHERE project = ?1;", + CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "coverage meta get prepare"); + return CBM_STORE_ERR; + } } bind_text(stmt, SKIP_ONE, project); int rc = sqlite3_step(stmt); @@ -4541,6 +4670,7 @@ int cbm_store_coverage_meta_get(cbm_store_t *s, const char *project, cbm_coverag out->ignored_files_total = sqlite3_column_int(stmt, 6); out->coverage_version = sqlite3_column_int(stmt, 7); out->hash_records_complete = sqlite3_column_int(stmt, 8) != 0; + out->unresolved_calls_complete = sqlite3_column_int(stmt, ST_COL_9) != 0; sqlite3_finalize(stmt); if (!out->project || !out->generation || !out->index_mode || !out->recorded_at || !out->recording_status) { diff --git a/src/store/store.h b/src/store/store.h index de82075c7..959245c7f 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -723,6 +723,8 @@ enum { CBM_UNRESOLVED_CALL_COVERAGE_VERSION = 5 }; /* Metadata describing how completely one index run recorded the best-effort * coverage signal. `recording_status` is "complete", "truncated", or * "unavailable"; it is deliberately separate from hash_records_complete. + * unresolved_calls_complete describes only unresolved invocation capture; + * false keeps CALLS totals conservative without voiding other coverage. * Strings returned by cbm_store_coverage_meta_get are heap-owned. */ typedef struct { const char *project; @@ -734,6 +736,7 @@ typedef struct { int ignored_files_total; int coverage_version; bool hash_records_complete; + bool unresolved_calls_complete; } cbm_coverage_meta_t; /* Replace the project's coverage rows in one transaction, then prune rows for @@ -762,9 +765,14 @@ int cbm_store_coverage_get_path(cbm_store_t *s, const char *project, const char * ancestor that covers the scope. Prefix matching is segment-boundary safe. */ int cbm_store_coverage_get_scope(cbm_store_t *s, const char *project, const char *scope, cbm_coverage_row_t **out, int *count); -/* Fetch only unresolved invocation diagnostics for trace exactness checks. */ -int cbm_store_coverage_get_unresolved_calls(cbm_store_t *s, const char *project, - cbm_coverage_row_t **out, int *count); +/* Fetch bounded unresolved invocation diagnostics for one exact caller file. */ +int cbm_store_coverage_get_unresolved_path(cbm_store_t *s, const char *project, + const char *rel_path, cbm_coverage_row_t **out, + int *count); +/* Indexed candidate existence; truncated/malformed evidence also returns true + * because omitted candidates cannot prove an exact inbound total. */ +int cbm_store_coverage_has_unresolved_candidate(cbm_store_t *s, const char *project, + const char *candidate, bool *out); /* Fetch/free the metadata paired with the current coverage row set. */ int cbm_store_coverage_meta_get(cbm_store_t *s, const char *project, cbm_coverage_meta_t *out); diff --git a/src/store/store_content_digest.c b/src/store/store_content_digest.c index c9287adee..ff15aa33e 100644 --- a/src/store/store_content_digest.c +++ b/src/store/store_content_digest.c @@ -22,6 +22,8 @@ #include #include +enum { SC_SQL_TEXT_LENGTH = -1 }; + typedef struct { const char *name; const char *sql; /* ?1 = project; every column is hashed, rows in this order */ @@ -44,11 +46,15 @@ static const sc_query_t sc_queries[] = { "main.lsp_surface WHERE project=?1 ORDER BY 1,2,3,4,5;"}, {"index_coverage", "SELECT rel_path,kind,detail FROM main.index_coverage WHERE project=?1 " "ORDER BY 1,2,3;"}, - {"index_coverage_meta", "SELECT index_mode,recording_status,ignored_files_stored," - "ignored_files_total,coverage_version,hash_records_complete FROM " - "main.index_coverage_meta WHERE project=?1;"}, + {"index_coverage_meta", + "SELECT index_mode,recording_status,ignored_files_stored," + "ignored_files_total,coverage_version,hash_records_complete,unresolved_calls_complete FROM " + "main.index_coverage_meta WHERE project=?1;"}, {"project_summaries", "SELECT summary,source_hash FROM main.project_summaries WHERE " "project=?1;"}, + {"index_unresolved_candidates", "SELECT candidate,rel_path,kind FROM " + "main.index_unresolved_candidates WHERE project=?1 " + "ORDER BY 1,2,3;"}, }; /* What test selection reads: node identity and every edge by its endpoints @@ -84,7 +90,31 @@ static void sc_frame(cbm_sha256_ctx *ctx, unsigned char tag, const void *bytes, static bool sc_table(sqlite3 *db, const sc_query_t *q, const char *project, cbm_sha256_ctx *ctx) { sqlite3_stmt *st = NULL; if (sqlite3_prepare_v2(db, q->sql, -1, &st, NULL) != SQLITE_OK) { - return false; + sqlite3_finalize(st); + st = NULL; + if (strcmp(q->name, "index_unresolved_candidates") == 0) { + /* Omit the optional frame only for proven legacy absence. Errors + * reading a present candidate index must invalidate the digest. */ + sqlite3_stmt *schema = NULL; + bool absent = + sqlite3_prepare_v2( + db, + "SELECT 1 FROM main.sqlite_schema WHERE name='index_unresolved_candidates' " + "COLLATE NOCASE LIMIT 1;", + SC_SQL_TEXT_LENGTH, &schema, NULL) == SQLITE_OK && + sqlite3_step(schema) == SQLITE_DONE; + return sqlite3_finalize(schema) == SQLITE_OK && absent; + } + /* Read-only legacy databases retain their existing content digest. */ + if (strcmp(q->name, "index_coverage_meta") != 0 || + sqlite3_prepare_v2(db, + "SELECT index_mode,recording_status,ignored_files_stored," + "ignored_files_total,coverage_version,hash_records_complete FROM " + "main.index_coverage_meta WHERE project=?1;", + SC_SQL_TEXT_LENGTH, &st, NULL) != SQLITE_OK) { + sqlite3_finalize(st); + return false; + } } bool ok = sqlite3_bind_text(st, 1, project, -1, SQLITE_TRANSIENT) == SQLITE_OK; sc_frame(ctx, 'T', q->name, strlen(q->name)); diff --git a/src/store/store_graph_digest.c b/src/store/store_graph_digest.c index 9baac3c39..9cea8de80 100644 --- a/src/store/store_graph_digest.c +++ b/src/store/store_graph_digest.c @@ -16,9 +16,14 @@ enum { GD_ALLOW_INTEGER = 1 << GD_INTEGER, GD_ALLOW_TEXT = 1 << GD_TEXT, GD_ALLOW_BLOB = 1 << GD_BLOB, - GD_TABLE_COUNT = 9, - GD_MAX_COLUMNS = 9, + GD_TABLE_COUNT = 10, + GD_MAX_COLUMNS = 10, + GD_COVERAGE_META = 7, GD_STORE_META = 8, + GD_UNRESOLVED_CANDIDATES = 9, + GD_SCHEMA_DECLARED_TYPE = 2, + GD_SCHEMA_PRIMARY_KEY = 5, + GD_SCHEMA_HIDDEN = 6, GD_CHUNK_BYTES = 65536 }; @@ -87,9 +92,15 @@ static const gd_column_t gd_index_coverage_meta[] = {{"project", GD_T, 0, 1}, {"ignored_files_stored", GD_I, 0, 0}, {"ignored_files_total", GD_I, 0, 0}, {"coverage_version", GD_I, 0, 0}, - {"hash_records_complete", GD_I, 0, 0}}; + {"hash_records_complete", GD_I, 0, 0}, + {"unresolved_calls_complete", GD_I, 0, 0}}; static const gd_column_t gd_store_meta[] = {{"k", GD_T, 0, 1}, {"v", GD_T, 0, 0}}; +static const gd_column_t gd_unresolved_candidates[] = {{"project", GD_T, 0, 1}, + {"candidate", GD_T, 0, 2}, + {"rel_path", GD_T, 0, 3}, + {"kind", GD_T, 0, 0}}; + /* These descriptors and SQL strings are immutable; all other state is local. */ static const gd_table_t gd_tables[GD_TABLE_COUNT] = { {"projects", gd_projects, GD_COLUMNS(gd_projects), "PRAGMA main.table_xinfo('projects');", @@ -116,14 +127,28 @@ static const gd_table_t gd_tables[GD_TABLE_COUNT] = { "PRAGMA main.table_xinfo('index_coverage');", "SELECT project,rel_path,kind,detail FROM main.index_coverage " "ORDER BY CAST(project AS BLOB) ASC,CAST(rel_path AS BLOB) ASC,CAST(kind AS BLOB) ASC;"}, - {"index_coverage_meta", gd_index_coverage_meta, GD_COLUMNS(gd_index_coverage_meta), + {"index_coverage_meta", gd_index_coverage_meta, GD_COLUMNS(gd_index_coverage_meta) - 1, "PRAGMA main.table_xinfo('index_coverage_meta');", "SELECT project,generation,index_mode,recorded_at,recording_status,ignored_files_stored," "ignored_files_total,coverage_version,hash_records_complete FROM main.index_coverage_meta " "ORDER BY CAST(project AS BLOB) ASC;"}, {"store_meta", gd_store_meta, GD_COLUMNS(gd_store_meta), "PRAGMA main.table_xinfo('store_meta');", - "SELECT k,v FROM main.store_meta ORDER BY CAST(k AS BLOB) ASC;"}}; + "SELECT k,v FROM main.store_meta ORDER BY CAST(k AS BLOB) ASC;"}, + {"index_unresolved_candidates", gd_unresolved_candidates, GD_COLUMNS(gd_unresolved_candidates), + "PRAGMA main.table_xinfo('index_unresolved_candidates');", + "SELECT project,candidate,rel_path,kind FROM main.index_unresolved_candidates " + "ORDER BY CAST(project AS BLOB) ASC,CAST(candidate AS BLOB) ASC," + "CAST(rel_path AS BLOB) ASC,CAST(kind AS BLOB) ASC;"}}; + +/* Preserve the legacy canonical stream; newer metadata binds the independent + * capture completeness column as well. Both schemas are checked exactly. */ +static const gd_table_t gd_coverage_meta_current = { + "index_coverage_meta", gd_index_coverage_meta, GD_COLUMNS(gd_index_coverage_meta), + "PRAGMA main.table_xinfo('index_coverage_meta');", + "SELECT project,generation,index_mode,recorded_at,recording_status,ignored_files_stored," + "ignored_files_total,coverage_version,hash_records_complete,unresolved_calls_complete " + "FROM main.index_coverage_meta ORDER BY CAST(project AS BLOB) ASC;"}; #undef GD_B #undef GD_N @@ -142,8 +167,14 @@ typedef struct { uint64_t bytes; bool have_uid; bool have_counter; + bool have_unresolved_completeness; } gd_context_t; +static const gd_table_t *gd_table(const gd_context_t *g, int index) { + return index == GD_COVERAGE_META && g->have_unresolved_completeness ? &gd_coverage_meta_current + : &gd_tables[index]; +} + typedef struct { const unsigned char *data; size_t length; @@ -408,8 +439,30 @@ static bool gd_table_kinds(gd_context_t *g, bool present[GD_TABLE_COUNT]) { return rc == SQLITE_DONE && gd_finalize(g); } +static bool gd_column_schema(gd_context_t *g, const gd_column_t *expected) { + if (sqlite3_column_type(g->stmt, GD_SCHEMA_PRIMARY_KEY) != SQLITE_INTEGER || + sqlite3_column_type(g->stmt, GD_SCHEMA_HIDDEN) != SQLITE_INTEGER || + sqlite3_column_int64(g->stmt, GD_SCHEMA_PRIMARY_KEY) != expected->primary_key || + sqlite3_column_int64(g->stmt, GD_SCHEMA_HIDDEN) != expected->kind) { + return gd_fail(g, CBM_STORE_GRAPH_DIGEST_SCHEMA); + } + if (expected->primary_key && expected->types == GD_ALLOW_INTEGER) { + gd_text_t declared_type; + if (!gd_read_text(g, GD_SCHEMA_DECLARED_TYPE, &declared_type)) { + return false; + } + if (!gd_text_ascii_eq(declared_type, "INTEGER")) { + return gd_fail(g, CBM_STORE_GRAPH_DIGEST_SCHEMA); + } + } + return true; +} + static bool gd_columns(gd_context_t *g, const gd_table_t *table) { bool seen[GD_MAX_COLUMNS] = {false}; + bool coverage_meta = table == &gd_tables[GD_COVERAGE_META]; + int allowed_columns = + coverage_meta ? gd_coverage_meta_current.column_count : table->column_count; if (!gd_prepare(g, table->column_query)) { return false; } @@ -424,37 +477,30 @@ static bool gd_columns(gd_context_t *g, const gd_table_t *table) { return false; } int column = -1; - for (int i = 0; i < table->column_count; i++) { + for (int i = 0; i < allowed_columns; i++) { if (gd_text_eq(name, table->columns[i].name)) { column = i; break; } } - if (column < 0 || seen[column] || count == table->column_count || - sqlite3_column_type(g->stmt, 5) != SQLITE_INTEGER || - sqlite3_column_type(g->stmt, 6) != SQLITE_INTEGER) { + if (column < 0 || seen[column] || count == allowed_columns || + !gd_column_schema(g, &table->columns[column])) { return gd_fail(g, CBM_STORE_GRAPH_DIGEST_SCHEMA); } - const gd_column_t *expected = &table->columns[column]; - if (sqlite3_column_int64(g->stmt, 5) != expected->primary_key || - sqlite3_column_int64(g->stmt, 6) != expected->kind) { - return gd_fail(g, CBM_STORE_GRAPH_DIGEST_SCHEMA); - } - if (expected->primary_key && expected->types == GD_ALLOW_INTEGER) { - gd_text_t declared_type; - if (!gd_read_text(g, 2, &declared_type)) { - return false; - } - if (!gd_text_ascii_eq(declared_type, "INTEGER")) { - return gd_fail(g, CBM_STORE_GRAPH_DIGEST_SCHEMA); - } - } seen[column] = true; count++; } - if (rc != SQLITE_DONE || count != table->column_count) { + if (rc != SQLITE_DONE || count < table->column_count) { return gd_fail(g, CBM_STORE_GRAPH_DIGEST_SCHEMA); } + for (int i = 0; i < table->column_count; i++) { + if (!seen[i]) { + return gd_fail(g, CBM_STORE_GRAPH_DIGEST_SCHEMA); + } + } + if (coverage_meta) { + g->have_unresolved_completeness = seen[table->column_count]; + } return gd_finalize(g); } @@ -481,8 +527,21 @@ static bool gd_project(gd_context_t *g, const unsigned char *project, size_t len return rc == SQLITE_DONE && gd_finalize(g); } +static bool gd_validate_schema(gd_context_t *g, bool present[GD_TABLE_COUNT], + const unsigned char *project, size_t length) { + if (!gd_encoding(g) || !gd_table_kinds(g, present)) { + return false; + } + for (int i = 0; i < GD_TABLE_COUNT; i++) { + if (present[i] && !gd_columns(g, &gd_tables[i])) { + return false; + } + } + return gd_project(g, project, length); +} + static bool gd_table_header(gd_context_t *g, int index, bool present) { - const gd_table_t *table = &gd_tables[index]; + const gd_table_t *table = gd_table(g, index); if (!gd_u8(g, 0x20) || !gd_u32(g, (uint32_t)index) || !gd_literal(g, table->name) || !gd_u32(g, (uint32_t)table->column_count)) { return false; @@ -605,7 +664,7 @@ static bool gd_column_value(gd_context_t *g, int index, const gd_column_t *colum static bool gd_table_rows(gd_context_t *g, int index, bool present) { uint64_t rows = 0; - const gd_table_t *table = &gd_tables[index]; + const gd_table_t *table = gd_table(g, index); if (present) { if (!gd_prepare(g, table->row_query)) { return false; @@ -645,6 +704,19 @@ static bool gd_table_rows(gd_context_t *g, int index, bool present) { return gd_u8(g, 0x22) && gd_u64(g, rows); } +static bool gd_stream_tables(gd_context_t *g, const bool present[GD_TABLE_COUNT]) { + for (int i = 0; i < GD_TABLE_COUNT; i++) { + /* Legacy stores retain their byte-for-byte canonical stream. */ + if (i == GD_UNRESOLVED_CANDIDATES && !present[i]) { + continue; + } + if (!gd_table_header(g, i, present[i]) || !gd_table_rows(g, i, present[i])) { + return false; + } + } + return true; +} + static bool gd_project_argument(gd_context_t *g, const unsigned char *project, size_t length) { while (length) { size_t chunk = length > GD_CHUNK_BYTES ? GD_CHUNK_BYTES : length; @@ -694,15 +766,7 @@ cbm_store_graph_digest_status_t cbm_store_graph_digest( return g.status; } bool present[GD_TABLE_COUNT] = {false}; - if (!gd_encoding(&g) || !gd_table_kinds(&g, present)) { - goto done; - } - for (int i = 0; i < GD_TABLE_COUNT; i++) { - if (present[i] && !gd_columns(&g, &gd_tables[i])) { - goto done; - } - } - if (!gd_project(&g, project, project_len)) { + if (!gd_validate_schema(&g, present, project, project_len)) { goto done; } cbm_sha256_init(&g.sha); @@ -711,10 +775,8 @@ cbm_store_graph_digest_status_t cbm_store_graph_digest( !gd_u8(&g, 0x10) || !gd_value(&g, GD_TEXT, project, project_len)) { goto done; } - for (int i = 0; i < GD_TABLE_COUNT; i++) { - if (!gd_table_header(&g, i, present[i]) || !gd_table_rows(&g, i, present[i])) { - goto done; - } + if (!gd_stream_tables(&g, present)) { + goto done; } if (!gd_u8(&g, 0x7f) || !gd_u64(&g, g.rows) || !gd_check(&g)) { goto done; diff --git a/src/store/store_graph_digest.h b/src/store/store_graph_digest.h index 004ed7840..d7276fd33 100644 --- a/src/store/store_graph_digest.h +++ b/src/store/store_graph_digest.h @@ -33,7 +33,8 @@ typedef struct { } cbm_store_graph_digest_t; /* Borrow a live, exclusive D5 scope. Hash the version-1 canonical stream of - * all nine known tables, including their exact CREATE SQL and explicit project + * the nine legacy tables and the unresolved candidate index when present, + * including their exact CREATE SQL and explicit project * identity. The database must use UTF-8. No new handler or transaction is used. * Independent scopes/connections may run concurrently; no mutable state is * shared. The callback/connection restrictions of D5 continue to apply. diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 0f7a900ba..ab3e412fe 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -6937,6 +6937,7 @@ TEST(tool_trace_budget_never_slices_identifiers) { .recorded_at = "2026-09-24T00:00:00Z", .recording_status = "complete", .coverage_version = CBM_UNRESOLVED_CALL_COVERAGE_VERSION, + .unresolved_calls_complete = true, .hash_records_complete = true}; ASSERT_EQ(cbm_store_coverage_replace_ex(store, project, NULL, 0, &coverage_meta), CBM_STORE_OK); @@ -7289,6 +7290,7 @@ TEST(tool_trace_reports_engine_saturation_as_lower_bound) { .recorded_at = "2026-09-24T00:00:00Z", .recording_status = "complete", .coverage_version = CBM_UNRESOLVED_CALL_COVERAGE_VERSION, + .unresolved_calls_complete = true, .hash_records_complete = true}; ASSERT_EQ(cbm_store_coverage_replace_ex(store, project, NULL, 0, &coverage_meta), CBM_STORE_OK); cbm_node_t hub = {.project = project, @@ -7781,6 +7783,99 @@ TEST(tool_trace_path_evidence_is_opt_in_and_class_mapped) { PASS(); } +/* Unrelated file evidence must not poison a clean caller's exact outbound + * count. Inbound checks use their materialized candidate index, so corrupting + * an unrelated JSON payload after publication must not trigger a full scan. */ +TEST(tool_trace_path_reads_only_relevant_unresolved_evidence) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *store = cbm_mcp_server_store(srv); + const char *project = "bounded-trace"; + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_upsert_project(store, project, "/tmp/bounded-trace"), CBM_STORE_OK); + cbm_node_t root = {.project = project, + .label = "Function", + .name = "run", + .qualified_name = "bounded-trace.run", + .file_path = "clean.js", + .start_line = 1, + .end_line = 4}; + ASSERT_GT(cbm_store_upsert_node(store, &root), 0); + ASSERT_EQ(cbm_store_upsert_file_hash(store, project, "unrelated.js", "fixture", 0, 0), + CBM_STORE_OK); + cbm_coverage_row_t row = {.rel_path = "unrelated.js", + .kind = "unresolved_calls", + .detail = "[{\"caller\":\"bounded-trace.other\",\"leaf\":\"other\"," + "\"candidate\":\"bounded-trace.other\"}]"}; + cbm_coverage_meta_t meta = {.generation = "fixture", + .index_mode = "full", + .recording_status = "complete", + .coverage_version = CBM_UNRESOLVED_CALL_COVERAGE_VERSION, + .unresolved_calls_complete = true, + .hash_records_complete = true}; + ASSERT_EQ(cbm_store_coverage_replace_ex(store, project, &row, 1, &meta), CBM_STORE_OK); + ASSERT_EQ(cbm_store_exec(store, "UPDATE index_coverage SET detail='broken json' " + "WHERE project='bounded-trace';"), + CBM_STORE_OK); + const char *directions[] = {"outbound", "inbound"}; + const char *fields[] = {"callees_total_relation", "callers_total_relation"}; + for (int i = 0; i < 2; i++) { + char args[256], expected[80]; + snprintf(args, sizeof(args), + "{\"project\":\"%s\",\"function_name\":\"run\"," + "\"direction\":\"%s\",\"format\":\"json\"}", + project, directions[i]); + char *out = cbm_mcp_handle_tool(srv, "trace_path", args); + ASSERT_NOT_NULL(out); + snprintf(expected, sizeof(expected), "\"%s\":\"eq\"", fields[i]); + ASSERT(response_contains_json_fragment(out, expected)); + free(out); + } + cbm_mcp_server_free(srv); + PASS(); +} + +TEST(tool_trace_path_truncated_evidence_is_conservative) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *store = cbm_mcp_server_store(srv); + const char *project = "truncated-trace"; + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_upsert_project(store, project, "/tmp/truncated-trace"), CBM_STORE_OK); + cbm_node_t root = {.project = project, .label = "Function", .name = "run", + .qualified_name = "truncated-trace.run", .file_path = "clean.js", + .start_line = 1, .end_line = 4}; + ASSERT_GT(cbm_store_upsert_node(store, &root), 0); + ASSERT_EQ(cbm_store_upsert_file_hash(store, project, "unrelated.js", "fixture", 0, 0), CBM_STORE_OK); + ASSERT_EQ(cbm_store_upsert_file_hash(store, project, "clean.js", "fixture", 0, 0), CBM_STORE_OK); + cbm_coverage_meta_t meta = {.generation = "fixture", .index_mode = "full", + .recording_status = "complete", .coverage_version = CBM_UNRESOLVED_CALL_COVERAGE_VERSION, + .unresolved_calls_complete = true, .hash_records_complete = true}; + const char *details[] = {"[{\"truncated\":true}]", "[null]", "broken json"}; + const char *directions[] = {"outbound", "inbound"}; + const char *fields[] = {"callees_total_relation", "callers_total_relation"}; + for (size_t detail = 0; detail < sizeof(details) / sizeof(details[0]); detail++) { + for (int relevant = 0; relevant < 2; relevant++) { + cbm_coverage_row_t row = {.rel_path = relevant ? "clean.js" : "unrelated.js", + .kind = "unresolved_calls", .detail = details[detail]}; + ASSERT_EQ(cbm_store_coverage_replace_ex(store, project, &row, 1, &meta), CBM_STORE_OK); + for (int direction = 0; direction < 2; direction++) { + char args[256], expected[80]; + snprintf(args, sizeof(args), "{\"project\":\"%s\",\"function_name\":\"run\"," + "\"direction\":\"%s\",\"format\":\"json\"}", project, directions[direction]); + char *out = cbm_mcp_handle_tool(srv, "trace_path", args); + ASSERT_NOT_NULL(out); + snprintf(expected, sizeof(expected), "\"%s\":\"%s\"", fields[direction], + relevant || direction == 1 ? "unknown" : "eq"); + ASSERT(response_contains_json_fragment(out, expected)); + free(out); + } + } + } + cbm_mcp_server_free(srv); + PASS(); +} + TEST(tool_trace_path_marks_unresolved_call_totals_unknown) { cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); cbm_store_t *st = cbm_mcp_server_store(srv); @@ -7825,6 +7920,7 @@ TEST(tool_trace_path_marks_unresolved_call_totals_unknown) { .recorded_at = "2026-09-24T00:00:00Z", .recording_status = "complete", .coverage_version = CBM_UNRESOLVED_CALL_COVERAGE_VERSION, + .unresolved_calls_complete = true, .hash_records_complete = true}; ASSERT_EQ(cbm_store_coverage_replace_ex(st, proj, &row, 1, &meta), CBM_STORE_OK); @@ -7931,6 +8027,7 @@ TEST(tool_trace_path_outbound_requires_project_symbol) { .recorded_at = "2026-09-29T00:00:00Z", .recording_status = "complete", .coverage_version = CBM_UNRESOLVED_CALL_COVERAGE_VERSION, + .unresolved_calls_complete = true, .hash_records_complete = true}; const struct { const char *leaf; @@ -23282,7 +23379,44 @@ TEST(search_code_rejects_quote_in_file_pattern) { PASS(); } +TEST(tool_check_index_coverage_parse_gap_precedes_unresolved_calls) { + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *store = cbm_mcp_server_store(srv); + const char *project = "coverage-priority"; + cbm_mcp_server_set_project(srv, project); + ASSERT_EQ(cbm_store_upsert_project(store, project, "/tmp/coverage-priority"), CBM_STORE_OK); + ASSERT_EQ(cbm_store_upsert_file_hash(store, project, "mixed.js", "fixture", 0, 0), + CBM_STORE_OK); + const char *kinds[] = {"parse_partial", "parse_unusable", "oversized", "not_indexed_file"}; + const char *statuses[] = {"partial", "unusable", "skipped", "excluded"}; + for (size_t i = 0; i < sizeof(kinds) / sizeof(kinds[0]); i++) { + cbm_coverage_row_t rows[] = { + {.rel_path = "mixed.js", .kind = "unresolved_calls", .detail = "[]"}, + {.rel_path = "mixed.js", .kind = kinds[i], .detail = "1-2"}, + }; + ASSERT_EQ(cbm_store_coverage_replace(store, project, rows, 2), CBM_STORE_OK); + char *response = cbm_mcp_handle_tool( + srv, "check_index_coverage", + "{\"project\":\"coverage-priority\",\"paths\":[\"mixed.js\"],\"format\":\"json\"}"); + ASSERT_NOT_NULL(response); + char *inner = extract_text_content(response); + ASSERT_NOT_NULL(inner); + yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *path = yyjson_arr_get(yyjson_obj_get(yyjson_doc_get_root(doc), "paths"), 0); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(path, "status")), statuses[i]); + ASSERT_EQ(yyjson_arr_size(yyjson_obj_get(path, "coverage")), 2); + yyjson_doc_free(doc); + free(inner); + free(response); + } + cbm_mcp_server_free(srv); + PASS(); +} + SUITE(mcp) { + RUN_TEST(tool_check_index_coverage_parse_gap_precedes_unresolved_calls); RUN_TEST(index_repository_async_and_status_refused_without_daemon_issue2144); RUN_TEST(index_repository_schema_documents_async_polling_issue2144); RUN_TEST(tool_result_add_notice_keeps_payload_shape_issue2144); @@ -23474,6 +23608,8 @@ SUITE(mcp) { RUN_TEST(tool_trace_call_path_qn_fallback_frees_name_miss); RUN_TEST(trace_evidence_strategy_class_vocabulary_is_closed); RUN_TEST(tool_trace_path_evidence_is_opt_in_and_class_mapped); + RUN_TEST(tool_trace_path_reads_only_relevant_unresolved_evidence); + RUN_TEST(tool_trace_path_truncated_evidence_is_conservative); RUN_TEST(tool_trace_path_marks_unresolved_call_totals_unknown); RUN_TEST(tool_trace_path_outbound_requires_project_symbol); RUN_TEST(tool_index_unresolved_nested_functions_are_indexed); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 859f295f0..6d16917ed 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -857,6 +857,65 @@ static int pipeline_unresolved_review_case(int padding, bool spill) { PASS(); } +/* A noisy semantic resolver must retain an explicit bounded-coverage marker, + * including when a single reason would exceed the entire detail budget. */ +TEST(pipeline_unresolved_capture_bounds_count_and_json_bytes) { + char tmp[256] = "/tmp/cbm_unresolved_cap_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char db[512]; + snprintf(db, sizeof(db), "%s/index.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + CBMResolvedCall *sites = calloc(1001, sizeof(*sites)); + ASSERT_NOT_NULL(sites); + for (int i = 0; i < 1001; i++) { + sites[i] = (CBMResolvedCall){.kind = CBM_RESOLVED_INVOCATION, + .caller_qn = "cap.run", + .callee_qn = "missing", + .strategy = "lsp_unresolved", + .site_start_byte = (uint32_t)i * 2, + .site_end_byte = (uint32_t)i * 2 + 1}; + } + CBMFileResult result = {.resolved_calls = {.items = sites, .count = 1001}}; + cbm_pipeline_record_unresolved_calls(p, "many.js", &result, NULL); + cbm_coverage_row_t *rows = NULL; + int count = 0; + bool complete = false; + cbm_pipeline_get_unresolved_calls(p, &rows, &count, &complete); + ASSERT_TRUE(complete); + ASSERT_EQ(count, 1); + ASSERT_TRUE(strlen(rows[0].detail) <= 128 * 1024); + yyjson_doc *doc = yyjson_read(rows[0].detail, strlen(rows[0].detail), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *array = yyjson_doc_get_root(doc); + ASSERT_TRUE(yyjson_arr_size(array) <= 1001); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(yyjson_arr_get_last(array), "truncated"))); + yyjson_doc_free(doc); + + char *reason = malloc(200000); + ASSERT_NOT_NULL(reason); + memset(reason, '"', 199999); + reason[199999] = '\0'; + sites[0].reason = reason; + result.resolved_calls.count = 1; + cbm_pipeline_record_unresolved_calls(p, "huge.js", &result, NULL); + cbm_pipeline_get_unresolved_calls(p, &rows, &count, &complete); + ASSERT_TRUE(complete); + ASSERT_EQ(count, 2); + ASSERT_TRUE(strlen(rows[1].detail) <= 128 * 1024); + doc = yyjson_read(rows[1].detail, strlen(rows[1].detail), 0); + ASSERT_NOT_NULL(doc); + array = yyjson_doc_get_root(doc); + ASSERT_EQ(yyjson_arr_size(array), 1); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(yyjson_arr_get_last(array), "truncated"))); + yyjson_doc_free(doc); + free(reason); + free(sites); + cbm_pipeline_free(p); + rm_rf(tmp); + PASS(); +} + TEST(pipeline_unresolved_review_sequential) { return pipeline_unresolved_review_case(0, false); } @@ -5251,6 +5310,81 @@ TEST(pipeline_tsconfig_mutation_before_publication_preserves_previous_generation PASS(); } +static void fail_unresolved_capture_before_publication(void *userdata) { + cbm_pipeline_mark_unresolved_capture_failed(userdata); + cbm_pipeline_add_file_error(userdata, "generation.py", "1-1", "parse_partial"); +} + +TEST(pipeline_capture_failure_preserves_general_coverage_and_rebuilds) { + char tmp[256] = "/tmp/cbm_capture_metadata_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "generation.py", "def GeneralCoverage():\n return 1\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *first = cbm_pipeline_new(tmp, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(first); + cbm_pipeline_incremental_test_before_final_manifest_once( + fail_unresolved_capture_before_publication, first); + ASSERT_EQ(cbm_pipeline_run(first), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(first)); + cbm_pipeline_free(first); + cbm_store_t *store = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(store); + cbm_coverage_meta_t meta = {0}; + ASSERT_EQ(cbm_store_coverage_meta_get(store, project, &meta), CBM_STORE_OK); + ASSERT_STR_EQ(meta.recording_status, "complete"); + ASSERT_FALSE(meta.unresolved_calls_complete); + cbm_coverage_row_t *rows = NULL; + int row_count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(store, project, "generation.py", &rows, &row_count), + CBM_STORE_OK); + ASSERT_EQ(row_count, 1); + ASSERT_STR_EQ(rows[0].kind, "parse_partial"); + cbm_store_free_coverage(rows, row_count); + cbm_store_coverage_meta_clear(&meta); + cbm_store_close(store); + /* A partial repair cannot fill an omitted signal for unchanged files. */ + write_temp_file(tmp, "changed.py", "def NewCoverage():\n return 2\n"); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_incremental_test_force_legacy_partial_once(); + cbm_pipeline_t *partial = cbm_pipeline_new(tmp, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(partial); + ASSERT_EQ(cbm_pipeline_run(partial), 0); + ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), CBM_INCREMENTAL_ROUTE_LEGACY_PARTIAL); + cbm_pipeline_free(partial); + store = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_coverage_meta_get(store, project, &meta), CBM_STORE_OK); + ASSERT_STR_EQ(meta.recording_status, "complete"); + ASSERT_FALSE(meta.unresolved_calls_complete); + cbm_store_coverage_meta_clear(&meta); + rows = NULL; + row_count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(store, project, "generation.py", &rows, &row_count), + CBM_STORE_OK); + ASSERT_EQ(row_count, 1); + ASSERT_STR_EQ(rows[0].kind, "parse_partial"); + cbm_store_free_coverage(rows, row_count); + cbm_store_close(store); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *repair = cbm_pipeline_new(tmp, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(repair); + ASSERT_EQ(cbm_pipeline_run(repair), 0); + ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), CBM_INCREMENTAL_ROUTE_FORCED_FULL); + cbm_pipeline_free(repair); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *unchanged = cbm_pipeline_new(tmp, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(unchanged); + ASSERT_EQ(cbm_pipeline_run(unchanged), 0); + ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), CBM_INCREMENTAL_ROUTE_NOOP); + cbm_pipeline_free(unchanged); + cbm_pipeline_incremental_test_reset_faults(); + th_rmtree(tmp); + PASS(); +} + /* Metadata participates in exact-input compatibility. Old coverage schema or * an upgrade to a more comprehensive discovery/index mode must force a * complete replacement even when every semantic-input byte is unchanged; the @@ -18603,6 +18737,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_export_error_snapshot_on_artifact_failure); RUN_TEST(pipeline_records_unresolved_injected_call_sites); RUN_TEST(pipeline_unresolved_review_sequential); + RUN_TEST(pipeline_unresolved_capture_bounds_count_and_json_bytes); RUN_TEST(pipeline_unresolved_review_parallel); RUN_TEST(pipeline_unresolved_review_spill); RUN_TEST(pipeline_structure_edges); @@ -18990,6 +19125,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_source_addition_before_publication_preserves_previous_generation); RUN_TEST(pipeline_tsconfig_mutation_before_publication_preserves_previous_generation); RUN_TEST(pipeline_exact_inputs_migrate_coverage_metadata_and_index_mode); + RUN_TEST(pipeline_capture_failure_preserves_general_coverage_and_rebuilds); RUN_TEST(pipeline_existing_artifact_refreshes_after_default_forced_full_reindex); RUN_TEST(pipeline_full_cancel_after_predump_preserves_previous_generation); RUN_TEST(pipeline_full_cancel_after_destination_prepare_preserves_previous_generation); diff --git a/tests/test_store_bulk.c b/tests/test_store_bulk.c index 826cef87e..e7991b7ac 100644 --- a/tests/test_store_bulk.c +++ b/tests/test_store_bulk.c @@ -167,9 +167,265 @@ TEST(bulk_crash_recovery) { } #endif /* _WIN32 */ +TEST(coverage_meta_has_independent_unresolved_completeness) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + sqlite3_stmt *stmt = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(s), + "SELECT unresolved_calls_complete FROM index_coverage_meta;", -1, + &stmt, NULL), + SQLITE_OK); + sqlite3_finalize(stmt); + cbm_store_close(s); + PASS(); +} + +TEST(coverage_meta_legacy_reader_preserves_general_metadata) { + char path[256]; + make_temp_path(path, sizeof(path)); + cleanup_db(path); + cbm_store_t *writer = cbm_store_open_path(path); + ASSERT_NOT_NULL(writer); + ASSERT_EQ(cbm_store_upsert_project(writer, "legacy-coverage", "/tmp/legacy-coverage"), + CBM_STORE_OK); + ASSERT_EQ( + sqlite3_exec(cbm_store_get_db(writer), + "DROP TABLE index_coverage_meta;" + "CREATE TABLE index_coverage_meta(project TEXT PRIMARY KEY,generation TEXT," + "index_mode TEXT,recorded_at TEXT,recording_status TEXT," + "ignored_files_stored INTEGER,ignored_files_total INTEGER," + "coverage_version INTEGER,hash_records_complete INTEGER);" + "INSERT INTO index_coverage_meta VALUES('legacy-coverage','generation','full'," + "'recorded','truncated',1,2,5,1);", + NULL, NULL, NULL), + SQLITE_OK); + cbm_store_close(writer); + cbm_store_t *reader = cbm_store_open_path_query(path); + ASSERT_NOT_NULL(reader); + cbm_coverage_meta_t meta = {0}; + ASSERT_EQ(cbm_store_coverage_meta_get(reader, "legacy-coverage", &meta), CBM_STORE_OK); + ASSERT_STR_EQ(meta.recording_status, "truncated"); + ASSERT_EQ(meta.ignored_files_stored, 1); + ASSERT_EQ(meta.ignored_files_total, 2); + ASSERT_EQ(meta.coverage_version, 5); + ASSERT_TRUE(meta.hash_records_complete); + ASSERT_FALSE(meta.unresolved_calls_complete); + cbm_store_coverage_meta_clear(&meta); + cbm_store_close(reader); + writer = cbm_store_open_path(path); + ASSERT_NOT_NULL(writer); + sqlite3_stmt *stmt = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(writer), + "SELECT unresolved_calls_complete FROM index_coverage_meta;", -1, + &stmt, NULL), + SQLITE_OK); + ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int(stmt, 0), 0); + sqlite3_finalize(stmt); + ASSERT_EQ(cbm_store_coverage_meta_get(writer, "legacy-coverage", &meta), CBM_STORE_OK); + ASSERT_STR_EQ(meta.recording_status, "truncated"); + ASSERT_FALSE(meta.unresolved_calls_complete); + cbm_store_coverage_meta_clear(&meta); + cbm_store_close(writer); + cleanup_db(path); + PASS(); +} + +TEST(coverage_meta_completeness_round_trip_and_failed_replace) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + const char *project = "capture-meta"; + ASSERT_EQ(cbm_store_upsert_project(s, project, "/tmp/capture-meta"), CBM_STORE_OK); + ASSERT_EQ(cbm_store_upsert_file_hash(s, project, "gap.js", "fixture", 0, 0), CBM_STORE_OK); + cbm_coverage_row_t row = {.rel_path = "gap.js", .kind = "parse_partial", .detail = "2-3"}; + cbm_coverage_meta_t meta = {.generation = "baseline", + .index_mode = "full", + .recording_status = "complete", + .coverage_version = 5, + .hash_records_complete = true, + .unresolved_calls_complete = true}; + ASSERT_EQ(cbm_store_coverage_replace_ex(s, project, &row, 1, &meta), CBM_STORE_OK); + cbm_coverage_meta_t fetched = {0}; + ASSERT_EQ(cbm_store_coverage_meta_get(s, project, &fetched), CBM_STORE_OK); + ASSERT_TRUE(fetched.unresolved_calls_complete); + cbm_store_coverage_meta_clear(&fetched); + ASSERT_EQ( + sqlite3_exec(cbm_store_get_db(s), + "CREATE TRIGGER reject_capture_meta BEFORE UPDATE ON index_coverage_meta " + "BEGIN SELECT RAISE(ABORT, 'metadata fault'); END;", + NULL, NULL, NULL), + SQLITE_OK); + meta.generation = "failed"; + meta.unresolved_calls_complete = false; + ASSERT_EQ(cbm_store_coverage_replace_ex(s, project, NULL, 0, &meta), CBM_STORE_ERR); + ASSERT_EQ(cbm_store_coverage_meta_get(s, project, &fetched), CBM_STORE_OK); + ASSERT_STR_EQ(fetched.generation, "baseline"); + ASSERT_TRUE(fetched.unresolved_calls_complete); + cbm_store_coverage_meta_clear(&fetched); + cbm_coverage_row_t *rows = NULL; + int count = 0; + ASSERT_EQ(cbm_store_coverage_get(s, project, &rows, &count), CBM_STORE_OK); + ASSERT_EQ(count, 1); + ASSERT_STR_EQ(rows[0].kind, "parse_partial"); + cbm_store_free_coverage(rows, count); + ASSERT_EQ( + sqlite3_exec(cbm_store_get_db(s), "DROP TRIGGER reject_capture_meta;", NULL, NULL, NULL), + SQLITE_OK); + ASSERT_EQ(cbm_store_coverage_replace_ex(s, project, &row, 1, &meta), CBM_STORE_OK); + ASSERT_EQ(cbm_store_coverage_meta_get(s, project, &fetched), CBM_STORE_OK); + ASSERT_FALSE(fetched.unresolved_calls_complete); + ASSERT_STR_EQ(fetched.recording_status, "complete"); + cbm_store_coverage_meta_clear(&fetched); + ASSERT_EQ(cbm_store_coverage_replace_ex(s, project, &row, 1, NULL), CBM_STORE_OK); + ASSERT_EQ(cbm_store_coverage_meta_get(s, project, &fetched), CBM_STORE_NOT_FOUND); + cbm_store_close(s); + PASS(); +} + +typedef struct { + int statements; + int fullscan_steps; + int vm_steps; +} unresolved_query_cost_t; + +static int record_unresolved_query_cost(unsigned type, void *userdata, void *statement, + void *elapsed) { + (void)elapsed; + unresolved_query_cost_t *cost = userdata; + sqlite3_stmt *stmt = statement; + const char *sql = sqlite3_sql(stmt); + if (type == SQLITE_TRACE_PROFILE && sql && + (strstr(sql, "FROM index_unresolved_candidates") || + strstr(sql, "AND kind = 'unresolved_calls'"))) { + cost->statements++; + cost->fullscan_steps += sqlite3_stmt_status(stmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, 0); + cost->vm_steps += sqlite3_stmt_status(stmt, SQLITE_STMTSTATUS_VM_STEP, 0); + } + return 0; +} + +TEST(unresolved_coverage_queries_use_file_and_candidate_indexes) { + enum { UNRELATED_FILES = 2000 }; + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + const char *project = "bounded-lookups"; + ASSERT_EQ(cbm_store_upsert_project(s, project, "/tmp/bounded-lookups"), CBM_STORE_OK); + sqlite3 *db = cbm_store_get_db(s); + ASSERT_EQ(sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL), SQLITE_OK); + sqlite3_stmt *coverage = NULL; + sqlite3_stmt *candidate = NULL; + ASSERT_EQ(sqlite3_prepare_v2(db, + "INSERT INTO index_coverage(project,rel_path,kind,detail) " + "VALUES(?1,?2,'unresolved_calls',?3);", + -1, &coverage, NULL), + SQLITE_OK); + ASSERT_EQ( + sqlite3_prepare_v2(db, + "INSERT INTO index_unresolved_candidates(project,candidate,rel_path) " + "VALUES(?1,?2,?3);", + -1, &candidate, NULL), + SQLITE_OK); + for (int i = 0; i <= UNRELATED_FILES; i++) { + char path[64]; + snprintf(path, sizeof(path), "file-%04d.js", i); + ASSERT_EQ(sqlite3_bind_text(coverage, 1, project, -1, SQLITE_STATIC), SQLITE_OK); + ASSERT_EQ(sqlite3_bind_text(coverage, 2, path, -1, SQLITE_TRANSIENT), SQLITE_OK); + ASSERT_EQ(sqlite3_bind_text(coverage, 3, + i == 0 ? "[{\"candidate\":\"target.fn\"}]" + : "[{\"candidate\":\"other.fn\"}]", + -1, SQLITE_STATIC), + SQLITE_OK); + ASSERT_EQ(sqlite3_step(coverage), SQLITE_DONE); + ASSERT_EQ(sqlite3_reset(coverage), SQLITE_OK); + ASSERT_EQ(sqlite3_bind_text(candidate, 1, project, -1, SQLITE_STATIC), SQLITE_OK); + ASSERT_EQ( + sqlite3_bind_text(candidate, 2, i == 0 ? "target.fn" : "other.fn", -1, SQLITE_STATIC), + SQLITE_OK); + ASSERT_EQ(sqlite3_bind_text(candidate, 3, path, -1, SQLITE_TRANSIENT), SQLITE_OK); + ASSERT_EQ(sqlite3_step(candidate), SQLITE_DONE); + ASSERT_EQ(sqlite3_reset(candidate), SQLITE_OK); + } + sqlite3_finalize(coverage); + sqlite3_finalize(candidate); + ASSERT_EQ(sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL), SQLITE_OK); + unresolved_query_cost_t cost = {0}; + ASSERT_EQ(sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, record_unresolved_query_cost, &cost), + SQLITE_OK); + cbm_coverage_row_t *rows = NULL; + int count = 0; + ASSERT_EQ(cbm_store_coverage_get_unresolved_path(s, project, "file-0000.js", &rows, &count), + CBM_STORE_OK); + ASSERT_EQ(count, 1); + ASSERT_STR_EQ(rows[0].rel_path, "file-0000.js"); + cbm_store_free_coverage(rows, count); + bool found = false; + ASSERT_EQ(cbm_store_coverage_has_unresolved_candidate(s, project, "target.fn", &found), + CBM_STORE_OK); + ASSERT_TRUE(found); + ASSERT_EQ(cbm_store_coverage_has_unresolved_candidate(s, project, "absent.fn", &found), + CBM_STORE_OK); + ASSERT_FALSE(found); + ASSERT_EQ(sqlite3_trace_v2(db, 0, NULL, NULL), SQLITE_OK); + ASSERT_EQ(cost.statements, 3); + ASSERT_EQ(cost.fullscan_steps, 0); + ASSERT_TRUE(cost.vm_steps < 300); + cbm_store_close(s); + PASS(); +} + +TEST(unresolved_candidate_index_tracks_replacement_pruning_and_markers) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + const char *project = "candidate-lifecycle"; + ASSERT_EQ(cbm_store_upsert_project(s, project, "/tmp/candidate-lifecycle"), CBM_STORE_OK); + ASSERT_EQ(cbm_store_upsert_file_hash(s, project, "calls.js", "fixture", 0, 0), CBM_STORE_OK); + cbm_coverage_row_t row = {.rel_path = "calls.js", + .kind = "unresolved_calls", + .detail = "[{\"candidate\":\"old.fn\"}]"}; + ASSERT_EQ(cbm_store_coverage_replace(s, project, &row, 1), CBM_STORE_OK); + bool found = false; + ASSERT_EQ(cbm_store_coverage_has_unresolved_candidate(s, project, "old.fn", &found), + CBM_STORE_OK); + ASSERT_TRUE(found); + row.detail = "[{\"candidate\":\"new.fn\"}]"; + ASSERT_EQ(cbm_store_coverage_replace(s, project, &row, 1), CBM_STORE_OK); + ASSERT_EQ(cbm_store_coverage_has_unresolved_candidate(s, project, "old.fn", &found), + CBM_STORE_OK); + ASSERT_FALSE(found); + ASSERT_EQ(cbm_store_coverage_has_unresolved_candidate(s, project, "new.fn", &found), + CBM_STORE_OK); + ASSERT_TRUE(found); + const char *markers[] = {"[{\"truncated\":true}]", "invalid-json", "{}", "[1]"}; + for (size_t i = 0; i < sizeof(markers) / sizeof(markers[0]); i++) { + row.detail = markers[i]; + ASSERT_EQ(cbm_store_coverage_replace(s, project, &row, 1), CBM_STORE_OK); + ASSERT_EQ(cbm_store_coverage_has_unresolved_candidate(s, project, "absent.fn", &found), + CBM_STORE_OK); + ASSERT_TRUE(found); + } + ASSERT_EQ(cbm_store_delete_file_hash(s, project, "calls.js"), CBM_STORE_OK); + ASSERT_EQ(cbm_store_coverage_replace(s, project, &row, 1), CBM_STORE_OK); + ASSERT_EQ(cbm_store_coverage_has_unresolved_candidate(s, project, "absent.fn", &found), + CBM_STORE_OK); + ASSERT_FALSE(found); + ASSERT_EQ(cbm_store_upsert_file_hash(s, project, "calls.js", "fixture", 0, 0), CBM_STORE_OK); + ASSERT_EQ(cbm_store_coverage_replace(s, project, &row, 1), CBM_STORE_OK); + ASSERT_EQ(cbm_store_delete_project(s, project), CBM_STORE_OK); + ASSERT_EQ(cbm_store_coverage_has_unresolved_candidate(s, project, "absent.fn", &found), + CBM_STORE_OK); + ASSERT_FALSE(found); + cbm_store_close(s); + PASS(); +} + /* ── Suite ──────────────────────────────────────────────────────── */ SUITE(store_bulk) { + RUN_TEST(coverage_meta_has_independent_unresolved_completeness); + RUN_TEST(coverage_meta_legacy_reader_preserves_general_metadata); + RUN_TEST(coverage_meta_completeness_round_trip_and_failed_replace); + RUN_TEST(unresolved_coverage_queries_use_file_and_candidate_indexes); + RUN_TEST(unresolved_candidate_index_tracks_replacement_pruning_and_markers); RUN_TEST(bulk_pragma_wal_invariant); RUN_TEST(bulk_pragma_end_wal_invariant); #ifndef _WIN32 diff --git a/tests/test_store_graph_digest.c b/tests/test_store_graph_digest.c index abb32ba7c..489c0a813 100644 --- a/tests/test_store_graph_digest.c +++ b/tests/test_store_graph_digest.c @@ -55,9 +55,11 @@ static bool gd_int(sqlite3 *db, const char *sql, int64_t want) { return ok; } static bool gd_seed(sqlite3 *db, bool reverse, bool meta) { - static const char drop[] = "PRAGMA foreign_keys=OFF;" + static const char drop[] = + "PRAGMA foreign_keys=OFF;" "DROP TABLE IF EXISTS edges;DROP TABLE IF EXISTS nodes;DROP TABLE IF EXISTS file_hashes;" "DROP TABLE IF EXISTS project_summaries;DROP TABLE IF EXISTS lsp_surface;" + "DROP TABLE IF EXISTS index_unresolved_candidates;" "DROP TABLE IF EXISTS index_coverage;DROP TABLE IF EXISTS index_coverage_meta;" "DROP TABLE IF EXISTS projects;DROP TABLE IF EXISTS store_meta;"; if (!db || gd_sql(db,drop)!=SQLITE_OK) return false; @@ -119,6 +121,69 @@ static bool gd_cancel(void *opaque) { return c->stop || (c->armed && c->trip && c->calls>=c->trip); } +TEST(store_graph_digest_binds_unresolved_candidate_index) { + cbm_store_t *s = cbm_store_open_memory(); + cbm_coverage_row_t row = {.rel_path = "calls.js", + .kind = "unresolved_calls", + .detail = "[{\"candidate\":\"p.target\"}]"}; + bool setup = s && cbm_store_upsert_project(s, "p", "/tmp/digest-candidates") == CBM_STORE_OK && + cbm_store_upsert_file_hash(s, "p", "calls.js", "fixture", 0, 0) == CBM_STORE_OK && + cbm_store_coverage_replace(s, "p", &row, 1) == CBM_STORE_OK; + bool candidate_before = false, candidate_after = true; + cbm_store_graph_digest_t before = {0}, after = {0}; + unsigned char content_before[CBM_STORE_GRAPH_DIGEST_BYTES] = {0}, + content_after[CBM_STORE_GRAPH_DIGEST_BYTES] = {0}; + bool read_before = + setup && + cbm_store_coverage_has_unresolved_candidate(s, "p", "p.target", &candidate_before) == + CBM_STORE_OK && + gd_p(s, &before) && cbm_store_graph_content_digest(s, "p", content_before) == CBM_STORE_OK; + /* Deliberately bypass publication/generation advancement: inbound reads this table. */ + bool removed = read_before && + gd_sql(cbm_store_get_db(s), + "DELETE FROM index_unresolved_candidates WHERE project='p'") == SQLITE_OK; + bool read_after = + removed && + cbm_store_coverage_has_unresolved_candidate(s, "p", "p.target", &candidate_after) == + CBM_STORE_OK && + gd_p(s, &after) && cbm_store_graph_content_digest(s, "p", content_after) == CBM_STORE_OK; + cbm_store_close(s); + ASSERT_TRUE(setup); + ASSERT_TRUE(read_before); + ASSERT_TRUE(candidate_before); + ASSERT_TRUE(removed); + ASSERT_TRUE(read_after); + ASSERT_FALSE(candidate_after); + ASSERT_TRUE(gd_different(&before, &after)); + ASSERT_TRUE(memcmp(content_before, content_after, sizeof(content_before)) != 0); + PASS(); +} + +TEST(store_graph_digest_candidate_index_errors_are_not_legacy_absence) { + cbm_store_t *s = gd_fixture(false, false); + unsigned char legacy[CBM_STORE_GRAPH_DIGEST_BYTES] = {0}, + again[CBM_STORE_GRAPH_DIGEST_BYTES] = {0}; + bool old = s && cbm_store_graph_content_digest(s, "p", legacy) == CBM_STORE_OK; + bool malformed = + old && gd_sql(cbm_store_get_db(s), + "CREATE TABLE index_unresolved_candidates(project TEXT,unexpected TEXT)") == + SQLITE_OK; + int content_rc = malformed ? cbm_store_graph_content_digest(s, "p", again) : CBM_STORE_OK; + bool graph_rejected = malformed && gd_reject(s, CBM_STORE_GRAPH_DIGEST_SCHEMA); + bool absent = + graph_rejected && + gd_sql(cbm_store_get_db(s), "DROP TABLE index_unresolved_candidates") == SQLITE_OK && + cbm_store_graph_content_digest(s, "p", again) == CBM_STORE_OK; + cbm_store_close(s); + ASSERT_TRUE(old); + ASSERT_TRUE(malformed); + ASSERT_EQ(content_rc, CBM_STORE_ERR); + ASSERT_TRUE(graph_rejected); + ASSERT_TRUE(absent); + ASSERT_TRUE(memcmp(legacy, again, sizeof(legacy)) == 0); + PASS(); +} + TEST(store_graph_digest_independent_canonical_golden) { cbm_store_t *s=gd_fixture(false,false); cbm_store_graph_digest_t out={0}; bool setup=s && gd_int(cbm_store_get_db(s),"SELECT count(*) FROM nodes",2) && @@ -303,6 +368,26 @@ TEST(store_graph_digest_legacy_generation_and_live_schema_compatibility) { ASSERT_TRUE(seeded);ASSERT_TRUE(compatible);PASS(); } +/* Legacy streams stay stable, while the new independent capture signal must + * be bound: changing only that value changes the digest. */ +TEST(store_graph_digest_binds_unresolved_capture_completeness) { + cbm_store_t *store = gd_fixture(false, false); + ASSERT_NOT_NULL(store); + cbm_store_graph_digest_t incomplete = {0}, complete = {0}; + ASSERT_EQ(gd_sql(cbm_store_get_db(store), + "ALTER TABLE index_coverage_meta ADD COLUMN " + "unresolved_calls_complete INTEGER NOT NULL DEFAULT 0;"), + SQLITE_OK); + ASSERT_TRUE(gd_p(store, &incomplete)); + ASSERT_EQ(gd_sql(cbm_store_get_db(store), + "UPDATE index_coverage_meta SET unresolved_calls_complete=1;"), + SQLITE_OK); + ASSERT_TRUE(gd_p(store, &complete)); + cbm_store_close(store); + ASSERT_TRUE(gd_different(&incomplete, &complete)); + PASS(); +} + TEST(store_graph_digest_schema_and_runtime_types_fail_closed) { static const char *const bad[] = { "DROP TABLE file_hashes", "DROP TABLE project_summaries", "DROP TABLE lsp_surface", @@ -595,11 +680,14 @@ TEST(store_graph_digest_concurrent_isolation_with_held_callback_gates) { } SUITE(store_graph_digest) { + RUN_TEST(store_graph_digest_binds_unresolved_candidate_index); + RUN_TEST(store_graph_digest_candidate_index_errors_are_not_legacy_absence); RUN_TEST(store_graph_digest_independent_canonical_golden); RUN_TEST(store_graph_digest_logical_order_and_excluded_storage_are_stable); RUN_TEST(store_graph_digest_every_known_value_and_cross_project_edge_is_bound); RUN_TEST(store_graph_digest_raw_types_project_bytes_and_schema_sql_are_distinct); RUN_TEST(store_graph_digest_legacy_generation_and_live_schema_compatibility); + RUN_TEST(store_graph_digest_binds_unresolved_capture_completeness); RUN_TEST(store_graph_digest_schema_and_runtime_types_fail_closed); RUN_TEST(store_graph_digest_wal_pins_old_content_then_observes_next_scope); RUN_TEST(store_graph_digest_exact_budgets_zero_output_and_terminal_scope); From 66ff5b86a3f3219fcddc9b69f75a2036a96c8ae9 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Fri, 9 Oct 2026 12:12:49 -0500 Subject: [PATCH 076/217] test(language): cover the .d rule in the shared classifier tests lang_probe_bytes_follow_the_name, lang_classify_matches_every_content_rule and the unreadable-content test now include .d: a 4 KB probe, a cargo dep-info line that is not indexed and D source that stays D, through both cbm_language_classify and cbm_disambiguate_d. With lang_probe_d disabled, the first two fail, as does discover_d_dep_info_not_indexed_as_dlang. Signed-off-by: Nelson Spence --- tests/test_language.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_language.c b/tests/test_language.c index 0d73720e3..43388b3d6 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -1415,6 +1415,7 @@ TEST(lang_probe_bytes_follow_the_name) { ASSERT_EQ(cbm_language_probe_bytes("foo.inc"), 4096); ASSERT_EQ(cbm_language_probe_bytes("Form1.frm"), 4096); ASSERT_EQ(cbm_language_probe_bytes("App.res"), 4096); + ASSERT_EQ(cbm_language_probe_bytes("app.d"), 4096); ASSERT_EQ(cbm_language_probe_bytes("Widget.cfc"), 16384); ASSERT_EQ(cbm_language_probe_bytes("pom.xml"), 255); /* An unknown name falls back to a shebang probe of the first line. */ @@ -1456,6 +1457,9 @@ TEST(lang_classify_matches_every_content_rule) { {"p.frm", "#procedure foo\nLocal F = a;\n", CBM_LANG_FORM, cbm_disambiguate_frm}, {"q.res", "RSRC\0\0\0\1binary", CBM_LANG_COUNT, cbm_disambiguate_res}, {"r.res", "let x = 1\n", CBM_LANG_RESCRIPT, cbm_disambiguate_res}, + {"dep.d", "target/debug/deps/app-0a1b2c.d: src/main.rs src/lib.rs\n", CBM_LANG_COUNT, + cbm_disambiguate_d}, + {"app.d", "module app;\nimport std.stdio : writeln;\n", CBM_LANG_DLANG, cbm_disambiguate_d}, {"s.xml", "\n\n", CBM_LANG_OBJECTSCRIPT_EXPORT, NULL}, {"t.xml", "\n\n", CBM_LANG_XML, NULL}, @@ -1503,6 +1507,7 @@ TEST(lang_classify_unreadable_content_keeps_the_name_default) { ASSERT_EQ(cbm_language_classify("a.cfc", NULL, 0, false, false), CBM_LANG_CFSCRIPT); ASSERT_EQ(cbm_language_classify("a.frm", NULL, 0, false, false), CBM_LANG_FORM); ASSERT_EQ(cbm_language_classify("a.res", NULL, 0, false, false), CBM_LANG_RESCRIPT); + ASSERT_EQ(cbm_language_classify("a.d", NULL, 0, false, false), CBM_LANG_DLANG); ASSERT_EQ(cbm_language_classify("a.xml", NULL, 0, false, false), CBM_LANG_XML); ASSERT_EQ(cbm_language_classify("run-tests", NULL, 0, false, false), CBM_LANG_COUNT); ASSERT_EQ(cbm_language_classify("main.go", NULL, 0, false, false), CBM_LANG_GO); From 1c1d5422056e14ffd7ce86727d291b34f04da26d Mon Sep 17 00:00:00 2001 From: Scott Beck <296871+bline@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:16:12 -0600 Subject: [PATCH 077/217] fix(mcp): verify content for freshness and snippets; expose store generation (#2585) check_index_coverage judged freshness by mtime and size alone, although file_hashes records the SHA-256 of the indexed bytes. A same-size edit with the original mtime passed as metadata_match. Identical bytes rewritten with a new mtime stayed metadata_changed after re-indexing, because the no-op incremental route never rewrites the record. get_code_snippet sliced current disk bytes with indexed line numbers, so it could return another symbol's code under the requested name. No response carried an identity that changes on incremental re-index. - Freshness now compares the file's SHA-256 with the recorded hash. metadata_match additionally means the bytes match. content_changed (new) means same metadata but different bytes; content_match (new) means stale metadata but identical bytes. metadata_changed keeps its meaning. A per-path content_verified flag says whether the hash check ran. A path counts as current only when it did: without a usable hash or a readable file, the freshness label keeps its metadata value, but the path recommends read_source_and_reindex and is not exact-path verified. - get_code_snippet reads the file once, checks that buffer's hash, and slices the indexed range from the same buffer, so a concurrent write cannot produce a content_verified snippet of bytes that were never indexed. When the bytes are not the indexed bytes, it withholds the source and reports source_drift, freshness, and recommended_action "reindex_then_retry". Unlike a metadata gate, this also catches the same-size, preserved-mtime case. Unverifiable files keep the existing fallback, marked content_verified: false. - check_index_coverage metadata and get_code_snippet now carry store_generation ("ug", the identity query_graph cursors already use), which advances on every index run. It names the publication current when the response is built; it does not make the response one atomic read. metadata.generation is unchanged: it names the coverage-recording run. Tests (tests/test_mcp.c) reach each state through recorded hash rows instead of file times, so they behave the same on every platform. Without the fix they fail on the defects themselves: "metadata_match" != "content_changed" snippet source contains another function's code store_generation absent unverified metadata match recommended as current a write between check and slice is served as verified (test seam) The #1613 truncated-catalog test now records the file's real hash, as the indexer does, instead of an empty one. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Scott Beck <296871+bline@users.noreply.github.com> --- src/mcp/mcp.c | 267 +++++++++++++++++++++++++++++++++--- src/mcp/mcp_internal.h | 7 + tests/test_mcp.c | 298 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 555 insertions(+), 17 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 9466b7fa8..287cb9166 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -1816,6 +1816,8 @@ struct cbm_mcp_server { void *auto_index_count_test_context; cbm_mcp_snapshot_read_test_hook_fn snapshot_read_test_hook; void *snapshot_read_test_context; + cbm_mcp_snippet_capture_test_hook_fn snippet_capture_test_hook; + void *snippet_capture_test_context; #endif size_t search_output_limit_override; const char *search_scan_command_override; @@ -2182,6 +2184,16 @@ void cbm_mcp_server_set_snapshot_read_test_hook(cbm_mcp_server_t *srv, srv->snapshot_read_test_hook = hook; srv->snapshot_read_test_context = context; } + +void cbm_mcp_server_set_snippet_capture_test_hook(cbm_mcp_server_t *srv, + cbm_mcp_snippet_capture_test_hook_fn hook, + void *context) { + if (!srv) { + return; + } + srv->snippet_capture_test_hook = hook; + srv->snippet_capture_test_context = context; +} #endif /* ── Cache dir + project DB path helpers ───────────────────────── */ @@ -6789,10 +6801,161 @@ static coverage_path_result_t coverage_normalize_rel(const char *input, bool all return written > 0U || allow_root ? COVERAGE_PATH_OK : COVERAGE_PATH_INVALID; } -static const char *coverage_path_freshness(cbm_store_t *store, const char *project, - const char *root_path, const char *rel_path, - bool *outside) { +/* #2585: the store generation ("ug") advances on every + * index run, incremental ones included, and on every publication. It is read + * when the response is built and names the publication current then; it does + * not make the response one atomic read of that publication. + * metadata.generation names the coverage-recording run and stays fixed across + * incremental re-indexes. Empty when unreadable. */ +static void mcp_add_store_generation(yyjson_mut_doc *doc, yyjson_mut_val *obj, cbm_store_t *store) { + char generation[QUERY_GRAPH_GENERATION_CAP] = ""; + if (!store || cbm_store_generation(store, generation, sizeof(generation)) != CBM_STORE_OK) { + generation[0] = '\0'; + } + yyjson_mut_obj_add_strcpy(doc, obj, "store_generation", generation); +} + +/* #2585: file_hashes.sha256 is the SHA-256 of the raw bytes the indexer read + * (pipeline_incremental.c semantic_manifest_hash_file). Recomputing it is the + * only way to say the bytes on disk are the bytes that were indexed: mtime and + * size miss a same-size edit whose timestamp was preserved (cp -p, rsync -t, + * archive extraction), and go stale when identical bytes are rewritten. */ +static bool mcp_file_sha256_hex(const char *abs_path, char out[CBM_SHA256_HEX_LEN + 1]) { + FILE *file = cbm_fopen(abs_path, "rb"); + if (!file) { + return false; + } + cbm_sha256_ctx sha; + cbm_sha256_init(&sha); + unsigned char buf[CBM_SZ_4K]; + size_t len; + while ((len = fread(buf, 1, sizeof(buf), file)) > 0) { + cbm_sha256_update(&sha, buf, len); + } + bool valid = !ferror(file); + if (fclose(file) != 0) { + valid = false; + } + if (!valid) { + return false; + } + uint8_t digest[CBM_SHA256_DIGEST_LEN]; + cbm_sha256_final(&sha, digest); + static const char hex[] = "0123456789abcdef"; + for (int i = 0; i < CBM_SHA256_DIGEST_LEN; i++) { + out[i * 2] = hex[digest[i] >> 4]; + out[i * 2 + 1] = hex[digest[i] & 0x0f]; + } + out[CBM_SHA256_HEX_LEN] = '\0'; + return true; +} + +/* True when the recorded content hash is a well-formed SHA-256 the current + * bytes can be compared against. */ +static bool mcp_recorded_sha256(const char *sha) { + if (!sha || strlen(sha) != CBM_SHA256_HEX_LEN) { + return false; + } + for (int i = 0; i < CBM_SHA256_HEX_LEN; i++) { + if (!((sha[i] >= '0' && sha[i] <= '9') || (sha[i] >= 'a' && sha[i] <= 'f'))) { + return false; + } + } + return true; +} + +/* #2585: read a whole file into *bytes (released with cbm_sb_free), so that + * the bytes a response serves are the bytes whose hash was checked. */ +static bool mcp_read_file_bytes(const char *abs_path, cbm_sb_t *bytes) { + cbm_sb_init(bytes); + FILE *file = cbm_fopen(abs_path, "rb"); + if (!file) { + return false; + } + char buf[CBM_SZ_4K]; + size_t len; + while ((len = fread(buf, 1, sizeof(buf), file)) > 0) { + cbm_sb_append_n(bytes, buf, len); + } + bool valid = !ferror(file) && !bytes->oom; + if (fclose(file) != 0) { + valid = false; + } + if (!valid) { + cbm_sb_free(bytes); + } + return valid; +} + +/* Lines [start, end] of an in-memory file, with read_file_lines' semantics + * (text-mode line endings, NULL when the range is empty). */ +static char *mcp_slice_lines(cbm_sb_t *bytes, int start, int end) { + size_t begin = bytes->len; + size_t stop = bytes->len; + int lineno = 1; + for (size_t i = 0; i < bytes->len; i++) { + if (lineno >= start && begin == bytes->len) { + begin = i; + } + if (bytes->buf[i] == '\n') { + if (lineno >= end) { + stop = i + 1; + break; + } + lineno++; + } + } + if (begin >= stop) { + return NULL; + } + char saved = bytes->buf[stop]; + bytes->buf[stop] = '\0'; + char *slice = heap_strdup(bytes->buf + begin); + bytes->buf[stop] = saved; +#ifdef _WIN32 + /* read_file_lines opens in text mode. */ + if (slice) { + char *out = slice; + for (const char *in = slice; *in; in++) { + if (!(in[0] == '\r' && in[1] == '\n')) { + *out++ = *in; + } + } + *out = '\0'; + } +#endif + return slice; +} + +/* A path is current when its bytes were verified to be the indexed bytes, + * whatever its recorded metadata says. A metadata match whose content could not + * be verified is not: it is the verdict the hash check exists to replace. */ +static bool coverage_freshness_current(const char *freshness, bool content_verified) { + return content_verified && freshness && + (strcmp(freshness, "metadata_match") == 0 || strcmp(freshness, "content_match") == 0); +} + +/* Freshness of one tracked path. With a recorded content hash the verdict is + * by content: "metadata_match" (metadata and bytes agree), "content_changed" + * (metadata agrees, bytes differ), "content_match" (metadata is stale, bytes + * are identical) or "metadata_changed" (both differ). Without one it falls + * back to metadata alone and *content_verified stays false. + * + * When captured is non-NULL, the file is read once into it (released with + * cbm_sb_free) and the verdict is about exactly those bytes, so a caller + * serving them cannot race a concurrent write. *captured_ok stays false when + * the path never reached the file (missing, outside, not tracked, + * unreadable). */ +static const char *coverage_path_freshness_capture(cbm_store_t *store, const char *project, + const char *root_path, const char *rel_path, + bool *outside, bool *content_verified, + cbm_sb_t *captured, bool *captured_ok) { *outside = false; + *content_verified = false; + if (captured) { + cbm_sb_init(captured); + *captured_ok = false; + } if (!root_path || !root_path[0]) { return "unavailable"; } @@ -6825,8 +6988,35 @@ static const char *coverage_path_freshness(cbm_store_t *store, const char *proje return "unavailable"; } bool matches = hash.mtime_ns == info.mtime_ns && hash.size == info.size; + char current[CBM_SHA256_HEX_LEN + 1] = ""; + bool hashed = false; + if (captured) { + *captured_ok = mcp_read_file_bytes(abs_path, captured); + if (*captured_ok) { + cbm_sha256_hex(captured->buf ? captured->buf : "", captured->len, current); + hashed = true; + } + } else { + hashed = mcp_recorded_sha256(hash.sha256) && mcp_file_sha256_hex(abs_path, current); + } + bool verifiable = mcp_recorded_sha256(hash.sha256) && hashed; + bool same_bytes = verifiable && strcmp(current, hash.sha256) == 0; cbm_store_clear_file_hash(&hash); - return matches ? "metadata_match" : "metadata_changed"; + if (!verifiable) { + return matches ? "metadata_match" : "metadata_changed"; + } + *content_verified = true; + if (matches) { + return same_bytes ? "metadata_match" : "content_changed"; + } + return same_bytes ? "content_match" : "metadata_changed"; +} + +static const char *coverage_path_freshness(cbm_store_t *store, const char *project, + const char *root_path, const char *rel_path, + bool *outside, bool *content_verified) { + return coverage_path_freshness_capture(store, project, root_path, rel_path, outside, + content_verified, NULL, NULL); } /* Read an "start-end,start-end,...[,+]" string into a JSON ranges array. @@ -6971,8 +7161,9 @@ static const char *coverage_status(const cbm_coverage_row_t *rows, int count, return "no_recorded_issue"; } -static const char *coverage_recommended_action(const char *status, const char *freshness) { - if (!freshness || strcmp(freshness, "metadata_match") != 0) { +static const char *coverage_recommended_action(const char *status, const char *freshness, + bool content_verified) { + if (!coverage_freshness_current(freshness, content_verified)) { return "read_source_and_reindex"; } if (strcmp(status, "partial") == 0) { @@ -7064,6 +7255,7 @@ static char *handle_check_index_coverage(cbm_mcp_server_t *srv, const char *args yyjson_mut_obj_add_int(doc, meta_obj, "coverage_version", have_meta ? meta.coverage_version : 0); yyjson_mut_obj_add_bool(doc, meta_obj, "generation_matches", generation_matches); + mcp_add_store_generation(doc, meta_obj, store); yyjson_mut_obj_add_val(doc, root, "metadata", meta_obj); yyjson_mut_val *path_results = yyjson_mut_arr(doc); @@ -7103,18 +7295,22 @@ static char *handle_check_index_coverage(cbm_mcp_server_t *srv, const char *args yyjson_mut_obj_add_str(doc, item, "coverage_lookup", "error"); } bool outside = false; - const char *freshness = coverage_path_freshness( - store, project, have_project ? proj.root_path : NULL, rel, &outside); - bool exact_path_verified = - have_meta && meta.hash_records_complete && strcmp(freshness, "metadata_match") == 0; + bool content_verified = false; + const char *freshness = + coverage_path_freshness(store, project, have_project ? proj.root_path : NULL, rel, + &outside, &content_verified); + bool exact_path_verified = have_meta && meta.hash_records_complete && + coverage_freshness_current(freshness, content_verified); const char *status = outside ? "outside_project" : coverage_status(rows, row_count, rel, recording_status, generation_matches, lookup_ok, exact_path_verified); yyjson_mut_obj_add_strcpy(doc, item, "status", status); yyjson_mut_obj_add_strcpy(doc, item, "freshness", freshness); - yyjson_mut_obj_add_strcpy(doc, item, "recommended_action", - coverage_recommended_action(status, freshness)); + yyjson_mut_obj_add_bool(doc, item, "content_verified", content_verified); + yyjson_mut_obj_add_strcpy( + doc, item, "recommended_action", + coverage_recommended_action(status, freshness, content_verified)); yyjson_mut_val *coverage = yyjson_mut_arr(doc); for (int i = 0; i < row_count; i++) { coverage_add_row_json(doc, coverage, &rows[i], rel, diagnostics_full); @@ -12612,8 +12808,10 @@ static char *snippet_pdf_text(const cbm_node_t *node) { return out; } +/* #2585: when captured is non-NULL the lines come from those already-read + * bytes instead of a fresh read of the file. */ static char *resolve_snippet_source(const char *root_path, const char *file_path, int start, - int end, char **out_abs_path) { + int end, cbm_sb_t *captured, char **out_abs_path) { *out_abs_path = NULL; if (!root_path || !file_path) { return NULL; @@ -12624,7 +12822,8 @@ static char *resolve_snippet_source(const char *root_path, const char *file_path *out_abs_path = abs_path; if (cbm_path_within_root(root_path, abs_path)) { - return read_file_lines(abs_path, start, end); + return captured ? mcp_slice_lines(captured, start, end) + : read_file_lines(abs_path, start, end); } return NULL; } @@ -13109,15 +13308,44 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, if (!outline && end < original_end) { snippet_clipped = true; } + /* #2585: start/end are indexed line numbers. Slice the file only when its + * bytes are the indexed bytes; otherwise the same range can hold another + * symbol's code. The file is read once: the slice comes from the same + * bytes whose hash was checked, so a concurrent write cannot slip in + * between. Unverifiable files keep the previous behavior and say so + * through content_verified. */ + const char *source_freshness = NULL; + bool source_content_verified = false; + bool source_drift = false; + cbm_sb_t captured; + cbm_sb_init(&captured); + bool captured_ok = false; + if (!outline && !snippet_is_pdf(node->file_path) && root_path && node->file_path) { + bool outside = false; + source_freshness = coverage_path_freshness_capture( + srv->store, node->project, root_path, node->file_path, &outside, + &source_content_verified, &captured, &captured_ok); + source_drift = source_content_verified && + !coverage_freshness_current(source_freshness, source_content_verified); +#ifdef CBM_ENABLE_TEST_SEAMS + if (srv->snippet_capture_test_hook) { + srv->snippet_capture_test_hook(srv->snippet_capture_test_context, node->file_path); + } +#endif + } char *abs_path = NULL; char *source = NULL; - if (outline) { + if (outline || source_drift) { source = NULL; } else if (snippet_is_pdf(node->file_path)) { source = snippet_pdf_text(node); /* never the file's bytes */ } else { - source = resolve_snippet_source(root_path, node->file_path, start, end, &abs_path); + /* Without captured bytes (not tracked, unreadable) the file is read + * here, and the snippet is never content-verified. */ + source = resolve_snippet_source(root_path, node->file_path, start, end, + captured_ok ? &captured : NULL, &abs_path); } + cbm_sb_free(&captured); yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root_obj = yyjson_mut_obj(doc); @@ -13137,6 +13365,13 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, yyjson_mut_obj_add_str(doc, root_obj, "file_path", display_path); yyjson_mut_obj_add_int(doc, root_obj, "start_line", start); yyjson_mut_obj_add_int(doc, root_obj, "end_line", end); + mcp_add_store_generation(doc, root_obj, srv->store); + yyjson_mut_obj_add_bool(doc, root_obj, "content_verified", source_content_verified); + if (source_drift) { + yyjson_mut_obj_add_bool(doc, root_obj, "source_drift", true); + yyjson_mut_obj_add_strcpy(doc, root_obj, "freshness", source_freshness); + yyjson_mut_obj_add_str(doc, root_obj, "recommended_action", "reindex_then_retry"); + } if (snippet_clipped) { yyjson_mut_obj_add_bool(doc, root_obj, "source_truncated", true); yyjson_mut_obj_add_bool(doc, root_obj, "source_clipped", true); /* compatibility */ diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index ead1fca38..afbe380e5 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -102,6 +102,13 @@ void cbm_mcp_server_set_snapshot_read_test_hook(cbm_mcp_server_t *srv, cbm_mcp_snapshot_read_test_hook_fn hook, void *context); +/* Runs after get_code_snippet has read and verified a file's bytes and before + * it slices them, so a test can write the file at exactly that point (#2585). */ +typedef void (*cbm_mcp_snippet_capture_test_hook_fn)(void *context, const char *rel_path); +void cbm_mcp_server_set_snippet_capture_test_hook(cbm_mcp_server_t *srv, + cbm_mcp_snippet_capture_test_hook_fn hook, + void *context); + /* Filesystem PATH_MAX prevents a portable end-to-end fixture for multi-KiB * stored paths. This seam drives the normal search result ownership, * directory aggregation, and renderer pipeline with synthetic identities. */ diff --git a/tests/test_mcp.c b/tests/test_mcp.c index abd19c124..55b9dd163 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -10,6 +10,7 @@ #include "../src/foundation/log.h" #include "../src/foundation/platform.h" /* cbm_file_size */ #include "../src/foundation/mem.h" /* cbm_mem_set_budget_for_tests — over-budget seam */ +#include "../src/foundation/sha256.h" #include "../src/foundation/str_util.h" #include "../src/foundation/subprocess.h" #include "../src/foundation/workspace.h" @@ -6341,7 +6342,16 @@ TEST(tool_check_index_coverage_accepts_truncated_ignored_catalog_for_fresh_path_ * metadata_match contract below would fail. */ cbm_path_info_t path_info; ASSERT_EQ(cbm_path_info_utf8(source_path, &path_info), 0); - ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", path_info.mtime_ns, + /* The real content hash, as the indexer records it: a path counts as + * current only when its bytes are verified (#2585). */ + char content[4096]; + FILE *source = fopen(source_path, "rb"); + ASSERT_NOT_NULL(source); + size_t content_len = fread(content, 1, sizeof(content), source); + fclose(source); + char sha[CBM_SHA256_HEX_LEN + 1]; + cbm_sha256_hex(content, content_len, sha); + ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", sha, path_info.mtime_ns, path_info.size), CBM_STORE_OK); cbm_project_t project = {0}; @@ -6497,6 +6507,286 @@ TEST(tool_check_index_coverage_requires_source_when_file_metadata_changed) { PASS(); } +/* #2585 fixtures: record main.go's hash row with its real size and either its + * real or a shifted mtime, and either its real content hash or another + * file's. This reaches each freshness state without touching file times, which + * some platforms cannot set precisely. */ +static int record_main_go_hash(cbm_store_t *store, const char *tmp, bool real_content, + bool real_mtime) { + char path[512]; + snprintf(path, sizeof(path), "%s/project/main.go", tmp); + cbm_path_info_t info; + if (cbm_path_info_utf8(path, &info) != 0) { + return CBM_STORE_ERR; + } + FILE *fp = fopen(path, "rb"); + if (!fp) { + return CBM_STORE_ERR; + } + char bytes[4096]; + size_t len = fread(bytes, 1, sizeof(bytes), fp); + fclose(fp); + char sha[CBM_SHA256_HEX_LEN + 1]; + if (real_content) { + cbm_sha256_hex(bytes, len, sha); + } else { + cbm_sha256_hex("other bytes", strlen("other bytes"), sha); + } + int64_t mtime_ns = real_mtime ? info.mtime_ns : info.mtime_ns - (int64_t)CBM_NSEC_PER_SEC; + return cbm_store_upsert_file_hash(store, "test-project", "main.go", sha, mtime_ns, info.size); +} + +static yyjson_val *coverage_first_path(cbm_mcp_server_t *srv, yyjson_doc **doc, char **inner) { + char *response = cbm_mcp_handle_tool(srv, "check_index_coverage", + "{\"project\":\"test-project\",\"paths\":[\"main.go\"]," + "\"format\":\"json\"}"); + *inner = response ? extract_text_content(response) : NULL; + free(response); + *doc = *inner ? yyjson_read(*inner, strlen(*inner), 0) : NULL; + return *doc ? yyjson_arr_get(yyjson_obj_get(yyjson_doc_get_root(*doc), "paths"), 0) : NULL; +} + +/* #2585: freshness judged by mtime and size passed a same-size edit whose + * timestamp was preserved, and stayed stale after identical bytes were + * rewritten. With a recorded content hash it now reports the bytes. */ +TEST(tool_check_index_coverage_freshness_verifies_content_issue2585) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + cbm_store_t *store = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(store); + cbm_project_t project = {0}; + ASSERT_EQ(cbm_store_get_project(store, "test-project", &project), CBM_STORE_OK); + ASSERT_EQ(write_coverage_meta(store, project.indexed_at, "complete"), CBM_STORE_OK); + cbm_project_free_fields(&project); + + static const struct { + bool real_content; + bool real_mtime; + const char *freshness; + const char *action; + } cases[] = { + /* Same metadata, different bytes: previously metadata_match. */ + {false, true, "content_changed", "read_source_and_reindex"}, + {true, true, "metadata_match", "use_graph_with_best_effort_caveat"}, + /* Stale metadata, identical bytes: previously metadata_changed. */ + {true, false, "content_match", "use_graph_with_best_effort_caveat"}, + {false, false, "metadata_changed", "read_source_and_reindex"}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + ASSERT_EQ(record_main_go_hash(store, tmp, cases[i].real_content, cases[i].real_mtime), + CBM_STORE_OK); + yyjson_doc *doc = NULL; + char *inner = NULL; + yyjson_val *path = coverage_first_path(srv, &doc, &inner); + ASSERT_NOT_NULL(path); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(path, "freshness")), cases[i].freshness); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(path, "recommended_action")), cases[i].action); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(path, "content_verified"))); + yyjson_doc_free(doc); + free(inner); + } + + /* A row without a usable content hash keeps the metadata verdict and says + * the content was not verified. */ + ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "fixture", 0, 0), + CBM_STORE_OK); + yyjson_doc *doc = NULL; + char *inner = NULL; + yyjson_val *path = coverage_first_path(srv, &doc, &inner); + ASSERT_NOT_NULL(path); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(path, "freshness")), "metadata_changed"); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(path, "content_verified"))); + yyjson_doc_free(doc); + free(inner); + + /* Matching metadata alone is not evidence the bytes are current: without + * a usable hash, the path must not be recommended as if it were. */ + char main_go[512]; + snprintf(main_go, sizeof(main_go), "%s/project/main.go", tmp); + cbm_path_info_t info; + ASSERT_EQ(cbm_path_info_utf8(main_go, &info), 0); + ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "fixture", info.mtime_ns, + info.size), + CBM_STORE_OK); + path = coverage_first_path(srv, &doc, &inner); + ASSERT_NOT_NULL(path); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(path, "freshness")), "metadata_match"); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(path, "content_verified"))); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(path, "recommended_action")), + "read_source_and_reindex"); + yyjson_doc_free(doc); + free(inner); + + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + +static char *snippet_process_order(cbm_mcp_server_t *srv, yyjson_doc **doc) { + char *response = cbm_mcp_handle_tool( + srv, "get_code_snippet", + "{\"project\":\"test-project\"," + "\"qualified_name\":\"test-project.cmd.server.main.ProcessOrder\",\"format\":\"json\"}"); + char *inner = response ? extract_text_content(response) : NULL; + free(response); + *doc = inner ? yyjson_read(inner, strlen(inner), 0) : NULL; + return inner; +} + +/* #2585 (and #1750): the snippet's line range is the indexed range. When the + * file's bytes are not the indexed bytes, the same range can hold another + * symbol's code, so the source is withheld rather than misattributed. */ +TEST(tool_get_code_snippet_withholds_source_whose_content_drifted_issue2585) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + cbm_store_t *store = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(store); + + /* Same metadata, different bytes: mtime/size cannot see this, and the + * indexed range would be sliced from bytes that were never indexed. */ + ASSERT_EQ(record_main_go_hash(store, tmp, false, true), CBM_STORE_OK); + yyjson_doc *doc = NULL; + char *inner = snippet_process_order(srv, &doc); + ASSERT_NOT_NULL(doc); + yyjson_val *root = yyjson_doc_get_root(doc); + ASSERT_NULL(strstr(yyjson_get_str(yyjson_obj_get(root, "source")), "func")); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(root, "source_drift"))); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(root, "freshness")), "content_changed"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(root, "recommended_action")), "reindex_then_retry"); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(root, "content_verified"))); + yyjson_doc_free(doc); + free(inner); + + /* The indexed bytes, with current or stale metadata, are served. */ + for (int real_mtime = 1; real_mtime >= 0; real_mtime--) { + ASSERT_EQ(record_main_go_hash(store, tmp, true, real_mtime == 1), CBM_STORE_OK); + inner = snippet_process_order(srv, &doc); + ASSERT_NOT_NULL(doc); + root = yyjson_doc_get_root(doc); + ASSERT_NOT_NULL( + strstr(yyjson_get_str(yyjson_obj_get(root, "source")), "func ProcessOrder")); + ASSERT_NULL(yyjson_obj_get(root, "source_drift")); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(root, "content_verified"))); + yyjson_doc_free(doc); + free(inner); + } + + /* Without a usable hash the previous behavior holds, marked unverified. */ + ASSERT_EQ(cbm_store_upsert_file_hash(store, "test-project", "main.go", "", 0, 0), CBM_STORE_OK); + inner = snippet_process_order(srv, &doc); + ASSERT_NOT_NULL(doc); + root = yyjson_doc_get_root(doc); + ASSERT_NOT_NULL(strstr(yyjson_get_str(yyjson_obj_get(root, "source")), "func ProcessOrder")); + ASSERT_FALSE(yyjson_get_bool(yyjson_obj_get(root, "content_verified"))); + yyjson_doc_free(doc); + free(inner); + + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + +#ifdef CBM_ENABLE_TEST_SEAMS +/* Rewrites main.go so that the indexed ProcessOrder range holds other code. */ +static void snippet_capture_rewrite(void *context, const char *rel_path) { + (void)rel_path; + FILE *fp = fopen((const char *)context, "w"); + if (fp) { + fputs("package main\n\n// a\n// b\n// c\n// d\nfunc Intruder() {\n}\n", fp); + fclose(fp); + } +} + +/* #2585: the bytes get_code_snippet serves must be the bytes it verified. A + * write landing after the hash check and before the slice must not produce a + * content_verified snippet holding code that was never indexed. */ +TEST(tool_get_code_snippet_serves_the_bytes_it_verified_issue2585) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + cbm_store_t *store = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(store); + ASSERT_EQ(record_main_go_hash(store, tmp, true, true), CBM_STORE_OK); + char main_go[512]; + snprintf(main_go, sizeof(main_go), "%s/project/main.go", tmp); + + cbm_mcp_server_set_snippet_capture_test_hook(srv, snippet_capture_rewrite, main_go); + yyjson_doc *doc = NULL; + char *inner = snippet_process_order(srv, &doc); + cbm_mcp_server_set_snippet_capture_test_hook(srv, NULL, NULL); + ASSERT_NOT_NULL(doc); + yyjson_val *root = yyjson_doc_get_root(doc); + const char *source = yyjson_get_str(yyjson_obj_get(root, "source")); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(root, "content_verified"))); + ASSERT_NULL(strstr(source, "Intruder")); + ASSERT_NOT_NULL(strstr(source, "func ProcessOrder")); + yyjson_doc_free(doc); + free(inner); + + /* The write did land: the next call sees it and withholds the source. */ + inner = snippet_process_order(srv, &doc); + ASSERT_NOT_NULL(doc); + root = yyjson_doc_get_root(doc); + ASSERT_TRUE(yyjson_get_bool(yyjson_obj_get(root, "source_drift"))); + yyjson_doc_free(doc); + free(inner); + + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} +#endif + +/* #2585: metadata.generation names the coverage-recording run and does not + * change on incremental re-index, so it cannot tell two publications apart. + * store_generation does, and every response that carries it agrees. */ +TEST(tool_responses_carry_store_generation_issue2585) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + cbm_store_t *store = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(store); + ASSERT_EQ(write_coverage_meta(store, "coverage-run-1", "complete"), CBM_STORE_OK); + ASSERT_EQ(record_main_go_hash(store, tmp, true, true), CBM_STORE_OK); + + char before[96]; + ASSERT_EQ(cbm_store_generation(store, before, sizeof(before)), CBM_STORE_OK); + yyjson_doc *cdoc = NULL; + char *cinner = NULL; + ASSERT_NOT_NULL(coverage_first_path(srv, &cdoc, &cinner)); + yyjson_val *meta = yyjson_obj_get(yyjson_doc_get_root(cdoc), "metadata"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(meta, "store_generation")), before); + yyjson_doc *sdoc = NULL; + char *sinner = snippet_process_order(srv, &sdoc); + ASSERT_NOT_NULL(sdoc); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(sdoc), "store_generation")), + before); + yyjson_doc_free(cdoc); + free(cinner); + yyjson_doc_free(sdoc); + free(sinner); + + /* An index run advances the store generation, not the coverage one. */ + char project_dir[512]; + snprintf(project_dir, sizeof(project_dir), "%s/project", tmp); + ASSERT_EQ(cbm_store_upsert_project(store, "test-project", project_dir), CBM_STORE_OK); + char after[96]; + ASSERT_EQ(cbm_store_generation(store, after, sizeof(after)), CBM_STORE_OK); + ASSERT_STR_NEQ(after, before); + ASSERT_NOT_NULL(coverage_first_path(srv, &cdoc, &cinner)); + meta = yyjson_obj_get(yyjson_doc_get_root(cdoc), "metadata"); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(meta, "store_generation")), after); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(meta, "generation")), "coverage-run-1"); + yyjson_doc_free(cdoc); + free(cinner); + + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + TEST(tool_check_index_coverage_surfaces_lookup_errors) { char tmp[256]; cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); @@ -23929,6 +24219,12 @@ SUITE(mcp) { RUN_TEST(tool_check_index_coverage_rejects_stale_generation); RUN_TEST(tool_check_index_coverage_freshness_uses_indexer_mtime_source_issue1714); RUN_TEST(tool_check_index_coverage_requires_source_when_file_metadata_changed); + RUN_TEST(tool_check_index_coverage_freshness_verifies_content_issue2585); + RUN_TEST(tool_get_code_snippet_withholds_source_whose_content_drifted_issue2585); +#ifdef CBM_ENABLE_TEST_SEAMS + RUN_TEST(tool_get_code_snippet_serves_the_bytes_it_verified_issue2585); +#endif + RUN_TEST(tool_responses_carry_store_generation_issue2585); RUN_TEST(tool_check_index_coverage_surfaces_lookup_errors); RUN_TEST(tool_index_status_includes_git_metadata); From 9a180d06263c07c5afc5706e3965d2dbee73639e Mon Sep 17 00:00:00 2001 From: Scott Beck <296871+bline@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:26:06 -0600 Subject: [PATCH 078/217] fix(doc-links): report a path::member reference to a missing member (#2587) A Markdown reference `path::member` whose member the code file does not define became an exact MENTIONS edge to the file. A renamed or deleted definition therefore looked like a valid, coarser link instead of a stale reference, and nothing recorded that the member was not found. mdr_segment now says how a member lookup ended. When a code file does not define the member, the reference is an unresolved `missing` row, the same doc-rot reason a missing path gets. When the file defines the name more than once and no owner picks one, it is `ambiguous`. This matches the C resolver, which reports a missing member of an existing type as missing rather than falling back to the type. A member written on a non-code file (a document's heading) keeps the file edge, since those are not graph definitions. Line ranges without a member are unchanged. Test (tests/test_doc_links_md.c) indexes a fixture with a found member, a member reached through its owner, a missing member, a name defined twice, and a member of a Markdown file. Without the fix it fails on the defect: the section's MENTIONS edge to src/util/rounding.py exists (1, expected 0). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Scott Beck <296871+bline@users.noreply.github.com> --- src/pipeline/doc_links_md.c | 55 +++++++++++++++++++++++++----- tests/test_doc_links_md.c | 68 +++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/src/pipeline/doc_links_md.c b/src/pipeline/doc_links_md.c index 9e76d41a4..14d6db513 100644 --- a/src/pipeline/doc_links_md.c +++ b/src/pipeline/doc_links_md.c @@ -17,9 +17,12 @@ * * Unresolved: * missing the path is not in the graph, but its directory is: the - * document names something that is not there (doc rot) + * document names something that is not there (doc rot); or a + * code file's member that the file does not define * ambiguous a file name alone that is not next to the document but exists - * elsewhere in the repository: the document does not say which + * elsewhere in the repository: the document does not say which; + * or a member name the file defines more than once, with no + * owner to pick one * Not a reference to this repository at all (LOCAL: no edge, no row): a link * back into the same document; a path that is neither indexed nor of a type * the indexer reads (an image, an archive, an extensionless page); a path @@ -478,12 +481,24 @@ static void mdr_file_defs(const mdr_index_t *x, const char *path, int *lo, int * *hi = e; } +/* How a member lookup ended: found, or why not. */ +typedef enum { + MDR_MEMBER_FOUND = 0, + MDR_MEMBER_MISSING, /* no definition of that name in the file */ + MDR_MEMBER_AMBIGUOUS, /* several, and the owner does not pick one */ +} mdr_member_t; + /* The code definition of file `path` that the range or the member names, or * NULL. Read from the index's definitions by file, never from DEFINES edges: * an incremental run's graph holds the edges of the files it re-extracts - * only, while every node is there. */ + * only, while every node is there. *member_result (when not NULL) says how a + * member lookup ended. */ static const cbm_gbuf_node_t *mdr_segment(const mdr_index_t *x, const char *path, uint32_t first, - uint32_t last, const char *member) { + uint32_t last, const char *member, + mdr_member_t *member_result) { + if (member_result) { + *member_result = MDR_MEMBER_FOUND; + } int lo = 0; int hi = 0; mdr_file_defs(x, path, &lo, &hi); @@ -516,6 +531,9 @@ static const cbm_gbuf_node_t *mdr_segment(const mdr_index_t *x, const char *path size_t owner_len; mdr_member_names(member, &name, &name_len, &owner, &owner_len); if (!name) { + if (member_result) { + *member_result = MDR_MEMBER_MISSING; + } return NULL; } const cbm_gbuf_node_t *hit = NULL; @@ -537,7 +555,13 @@ static const cbm_gbuf_node_t *mdr_segment(const mdr_index_t *x, const char *path if (owned_hits == SKIP_ONE) { return owned; } - return owned_hits == 0 && hits == SKIP_ONE ? hit : NULL; + if (owned_hits == 0 && hits == SKIP_ONE) { + return hit; + } + if (member_result) { + *member_result = hits == 0 ? MDR_MEMBER_MISSING : MDR_MEMBER_AMBIGUOUS; + } + return NULL; } /* Is `path` of a file type the indexer reads? */ @@ -1151,9 +1175,21 @@ static void mdr_resolve_path(const mdr_index_t *x, const cbm_gbuf_t *graph, cons out->kind = CBM_DOCLINK_LOCAL; /* a link into the same document */ return; } - const cbm_gbuf_node_t *seg = (r->first > 0 || r->member) - ? mdr_segment(x, t, r->first, r->last, r->member) - : NULL; + mdr_member_t member_result = MDR_MEMBER_FOUND; + const cbm_gbuf_node_t *seg = + (r->first > 0 || r->member) + ? mdr_segment(x, t, r->first, r->last, r->member, &member_result) + : NULL; + /* A named member that a code file does not define is no coarser + * link to the file: it is a stale or unclear reference. A + * document's own headings are not definitions, so members of + * other files keep the file edge. */ + if (!seg && r->member && member_result != MDR_MEMBER_FOUND && mdr_code_file(t)) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = member_result == MDR_MEMBER_AMBIGUOUS ? CBM_DOCLINK_REASON_AMBIGUOUS + : CBM_DOCLINK_REASON_MISSING; + return; + } mdr_edge(out, seg ? seg : file, r->first, r->last); return; } @@ -1388,7 +1424,8 @@ static void mdr_resolve(const void *index, void *state, int run_file, const CBMD const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char *path, uint32_t first, uint32_t last, const char *member) { - return md_index ? mdr_segment((const mdr_index_t *)md_index, path, first, last, member) : NULL; + return md_index ? mdr_segment((const mdr_index_t *)md_index, path, first, last, member, NULL) + : NULL; } /* ── A region of a file's text (the field test's H8 bind_range) ─────── */ diff --git a/tests/test_doc_links_md.c b/tests/test_doc_links_md.c index 21e74f37e..11049e032 100644 --- a/tests/test_doc_links_md.c +++ b/tests/test_doc_links_md.c @@ -388,6 +388,73 @@ TEST(doc_links_md_resolve) { PASS(); } +/* #2587: a `path::member` reference whose member the code file does not + * define was an exact edge to the file, so a renamed or deleted definition + * looked like a valid, coarser link. It is a missing reference; a member the + * file defines more than once, with no owner to pick one, is ambiguous. */ +TEST(doc_links_md_member_not_defined_issue2587) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + th_write_file(TH_PATH(tmp, "docs/members.md"), + "# Members\n" + "\n" + "See `src/util/rounding.py::clamp` and " + "`src/util/rounding.py::no_such_function`.\n" + "Areas: `src/shapes.py::Square::area`, `src/shapes.py::area`.\n" + "Notes: `docs/other.md::Intro`.\n"); + th_write_file(TH_PATH(tmp, "docs/other.md"), "# Intro\n\nText.\n"); + th_write_file(TH_PATH(tmp, "src/util/rounding.py"), "def clamp(x, lo, hi):\n" + " return max(lo, min(hi, x))\n"); + th_write_file(TH_PATH(tmp, "src/shapes.py"), "class Circle:\n" + " def area(self):\n" + " return 3\n" + "\n" + "\n" + "class Square:\n" + " def area(self):\n" + " return 4\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/md.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + char props[512]; + char reason[64]; + + /* a member the file defines, alone or through its owner */ + ASSERT_EQ(md_edge(db, "Members", "rounding.clamp", props, sizeof(props)), 1); + ASSERT_EQ(md_edge(db, "Members", "Square.area", props, sizeof(props)), 1); + ASSERT_EQ(md_edge(db, "Members", "Circle.area", props, sizeof(props)), 0); + + /* a member the file does not define: no edge to the file, a missing row */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Members' AND t.label = 'File' " + "AND t.file_path = 'src/util/rounding.py'"), + 0); + dm_row(db, "docs/members.md", "src/util/rounding.py::no_such_function", reason, sizeof(reason), + NULL, 0); + ASSERT_STR_EQ(reason, "missing"); + + /* defined twice and no owner written: ambiguous, not the file */ + dm_row(db, "docs/members.md", "src/shapes.py::area", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "ambiguous"); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Members' AND t.label = 'File' " + "AND t.file_path = 'src/shapes.py'"), + 0); + + /* a document's headings are no definitions: the file edge stays */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Members' AND t.label = 'File' " + "AND t.file_path = 'docs/other.md'"), + 1); + + th_cleanup(tmp); + PASS(); +} + /* ── incremental == full ─────────────────────────────────────────── */ TEST(doc_links_md_incremental) { @@ -865,6 +932,7 @@ SUITE(doc_links_md) { RUN_TEST(doc_links_md_repeated_headings); RUN_TEST(doc_links_md_classify_span); RUN_TEST(doc_links_md_resolve); + RUN_TEST(doc_links_md_member_not_defined_issue2587); RUN_TEST(doc_links_md_incremental); RUN_TEST(doc_links_md_adr); RUN_TEST(doc_links_md_adr_label_choice); From 104dc21b045d5236a341c0967bef866a146fb0e0 Mon Sep 17 00:00:00 2001 From: lixsp11 <30531760+Lixsp11@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:28:45 +0900 Subject: [PATCH 079/217] fix(store): use 64-bit file offsets in the SQLite page writer sqlite_writer.c computed page offsets as (long)(page - 1) * 65536 and passed them to fseek()/ftell(). long is 32 bits on Windows, so once a database grows past 2 GiB the offset goes negative and fseek() fails; the result is ignored, so pages land wherever the stream happens to be. Past 4 GiB the offset wraps back to small positive values and the writer overwrites the start of the file. The staging database stops at exactly 4 GiB, is corrupt, and the run ends in persist_failed. Route every seek/tell in the writer through 64-bit wrappers (_fseeki64/_ftelli64 on Windows, fseeko/ftello elsewhere) and compute offsets with a new cbm_writer_page_offset() helper that widens before multiplying. Fixes #2399 Signed-off-by: lixsp11 <30531760+Lixsp11@users.noreply.github.com> --- internal/cbm/sqlite_writer.c | 37 ++++++++++++++++++++++-------------- internal/cbm/sqlite_writer.h | 4 ++++ tests/test_sqlite_writer.c | 13 +++++++++++++ 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/internal/cbm/sqlite_writer.c b/internal/cbm/sqlite_writer.c index f965da1ce..d684ff3a0 100644 --- a/internal/cbm/sqlite_writer.c +++ b/internal/cbm/sqlite_writer.c @@ -35,13 +35,22 @@ #include #include #define cbm_writer_getpid _getpid +/* long is 32 bits on Windows, so fseek/ftell cannot address past 2 GiB there. */ +#define cbm_writer_seek(fp, off, whence) _fseeki64((fp), (long long)(off), (whence)) +#define cbm_writer_tell(fp) ((int64_t)_ftelli64(fp)) #else #include #define cbm_writer_getpid getpid +#define cbm_writer_seek(fp, off, whence) fseeko((fp), (off_t)(off), (whence)) +#define cbm_writer_tell(fp) ((int64_t)ftello(fp)) #endif #define CBM_PAGE_SIZE 65536 +int64_t cbm_writer_page_offset(uint32_t page_num) { + return (int64_t)(page_num - SKIP_ONE) * CBM_PAGE_SIZE; +} + /* SQLite reserves the page containing the 1 GiB file offset (the "pending byte" * used for file locking on Windows). This page MUST be skipped during allocation * otherwise integrity_check reports "2nd reference to page N" because it marks @@ -601,8 +610,8 @@ static void pb_flush_leaf(PageBuilder *pb) { // Write page to file. Skip the pending byte page (SQLite reserved). pb->next_page = cbm_skip_pending_byte(pb->next_page); uint32_t page_num = pb->next_page; - long offset = (long)(page_num - SKIP_ONE) * CBM_PAGE_SIZE; - (void)fseek(pb->fp, offset, SEEK_SET); + int64_t offset = cbm_writer_page_offset(page_num); + (void)cbm_writer_seek(pb->fp, offset, SEEK_SET); (void)fwrite(pb->page, SKIP_ONE, CBM_PAGE_SIZE, pb->fp); // Record this leaf for interior page building @@ -707,7 +716,7 @@ static int write_interior_page(PageBuilder *pb, uint8_t *page, int cell_count, i page[HDR_FRAGBYTES_OFF] = 0; put_u32(page + HDR_RIGHTCHILD_OFF, right_child_page); - (void)fseek(pb->fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET); + (void)cbm_writer_seek(pb->fp, cbm_writer_page_offset(pnum), SEEK_SET); (void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, pb->fp); if (parent_count >= *parent_cap) { @@ -970,7 +979,7 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_ int data_len) { int per_page = CBM_PAGE_SIZE - BTREE_PTR_SIZE; uint32_t first_page = 0; - long prev_next_ptr_offset = -SKIP_ONE; + int64_t prev_next_ptr_offset = -SKIP_ONE; int offset = 0; while (offset < data_len) { @@ -983,7 +992,7 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_ if (prev_next_ptr_offset >= 0) { uint8_t ptr[BTREE_PTR_SIZE]; put_u32(ptr, pnum); - (void)fseek(fp, prev_next_ptr_offset, SEEK_SET); + (void)cbm_writer_seek(fp, prev_next_ptr_offset, SEEK_SET); (void)fwrite(ptr, SKIP_ONE, BTREE_PTR_SIZE, fp); } @@ -997,9 +1006,9 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_ put_u32(page, 0); // next-page pointer — 0 for now, backpatched on next iteration memcpy(page + BTREE_PTR_SIZE, data + offset, chunk); - long page_offset = (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE; + int64_t page_offset = cbm_writer_page_offset(pnum); prev_next_ptr_offset = page_offset; - (void)fseek(fp, page_offset, SEEK_SET); + (void)cbm_writer_seek(fp, page_offset, SEEK_SET); (void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp); offset += chunk; @@ -1276,7 +1285,7 @@ static uint32_t write_table_btree(FILE *fp, uint32_t *next_page, const uint8_t * put_u16(page + hdr + HDR_CELLCOUNT_OFF, 0); // 0 cells put_u16(page + hdr + HDR_CONTENT_OFF, (uint16_t)CBM_PAGE_SIZE); // content at end of page page[hdr + HDR_FRAGBYTES_OFF] = 0; // 0 fragmented bytes - (void)fseek(fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET); + (void)cbm_writer_seek(fp, cbm_writer_page_offset(pnum), SEEK_SET); (void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp); return pnum; } @@ -1326,7 +1335,7 @@ static uint32_t write_empty_index_leaf(FILE *fp, uint32_t *next_page) { put_u16(page + HDR_CELLCOUNT_OFF, 0); put_u16(page + HDR_CONTENT_OFF, (uint16_t)CBM_PAGE_SIZE); page[HDR_FRAGBYTES_OFF] = 0; - (void)fseek(fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET); + (void)cbm_writer_seek(fp, cbm_writer_page_offset(pnum), SEEK_SET); (void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp); return pnum; } @@ -2020,7 +2029,7 @@ static int write_master_page1(FILE *fp, MasterEntry *master, int master_count, u write_sqlite_file_header(page1, next_page - SKIP_ONE); - (void)fseek(fp, 0, SEEK_SET); + (void)cbm_writer_seek(fp, 0, SEEK_SET); (void)fwrite(page1, SKIP_ONE, CBM_PAGE_SIZE, fp); for (int i = 0; i < master_count; i++) { @@ -2034,12 +2043,12 @@ static int write_master_page1(FILE *fp, MasterEntry *master, int master_count, u /* Pad file to exact page boundary. */ static void pad_file_to_page_boundary(FILE *fp, uint32_t next_page) { - (void)fseek(fp, 0, SEEK_END); - long file_size = ftell(fp); - long expected_size = (long)(next_page - SKIP_ONE) * CBM_PAGE_SIZE; + (void)cbm_writer_seek(fp, 0, SEEK_END); + int64_t file_size = cbm_writer_tell(fp); + int64_t expected_size = cbm_writer_page_offset(next_page); if (file_size < expected_size) { uint8_t zero = 0; - (void)fseek(fp, expected_size - SKIP_ONE, SEEK_SET); + (void)cbm_writer_seek(fp, expected_size - SKIP_ONE, SEEK_SET); (void)fwrite(&zero, SKIP_ONE, SKIP_ONE, fp); } } diff --git a/internal/cbm/sqlite_writer.h b/internal/cbm/sqlite_writer.h index 1142e7d57..b92a13719 100644 --- a/internal/cbm/sqlite_writer.h +++ b/internal/cbm/sqlite_writer.h @@ -90,4 +90,8 @@ int cbm_writer_finalize(cbm_db_writer_t *w, const char *project, const char *roo CBMDumpEdge *edges, int edge_count, CBMDumpVector *vectors, int vector_count, CBMDumpTokenVec *token_vecs, int token_vec_count); +// Byte offset of 1-based page `page_num` in the output file. 64-bit on every +// platform, so databases larger than 2 GiB are addressed correctly on Windows. +int64_t cbm_writer_page_offset(uint32_t page_num); + #endif // CBM_SQLITE_WRITER_H diff --git a/tests/test_sqlite_writer.c b/tests/test_sqlite_writer.c index bacf06a95..cbeb2dfc3 100644 --- a/tests/test_sqlite_writer.c +++ b/tests/test_sqlite_writer.c @@ -996,6 +996,18 @@ TEST(sw_publish_preserves_live_reader) { /* ── Suite ─────────────────────────────────────────────────────── */ +TEST(sw_page_offsets_do_not_wrap_past_4gib) { + /* A 32-bit long offset wraps negative at 2 GiB and back to 0 at 4 GiB: on + * Windows a staging DB stopped at exactly 4 GiB with its first pages + * overwritten, and publish failed with persist_failed (#2399). */ + ASSERT_EQ(cbm_writer_page_offset(1), 0); + ASSERT_EQ(cbm_writer_page_offset(2), 65536); + ASSERT_EQ(cbm_writer_page_offset(32769), INT64_C(2147483648)); + ASSERT_EQ(cbm_writer_page_offset(65537), INT64_C(4294967296)); + ASSERT_EQ(cbm_writer_page_offset(UINT32_MAX), (int64_t)(UINT32_MAX - 1) * 65536); + PASS(); +} + TEST(sw_open_truncated_path_names_its_reason) { /* final_path is a fixed 4K buffer and make_writer_temp_path only fails by * truncation. Neither sets an errno of its own, so without naming one the @@ -1065,6 +1077,7 @@ SUITE(sqlite_writer) { RUN_TEST(sw_publish_failure_preserves_destination_sidecars); RUN_TEST(sw_publish_supports_non_ascii_path); RUN_TEST(sw_publish_preserves_live_reader); + RUN_TEST(sw_page_offsets_do_not_wrap_past_4gib); RUN_TEST(sw_open_truncated_path_names_its_reason); RUN_TEST(sw_publish_failure_reports_the_rename_not_the_cleanup); } From 19374f2d340df14c0273b1381c1ad2390b0157d9 Mon Sep 17 00:00:00 2001 From: harshitaajoshi Date: Mon, 28 Sep 2026 15:02:38 -0500 Subject: [PATCH 080/217] fix(mcp): apply search_graph label filter inside the BM25 candidate window (#2386) Signed-off-by: harshitaajoshi --- src/mcp/mcp.c | 68 ++++++++++++++++++--- tests/test_mcp.c | 150 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 7 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 03741e23a..60cef5334 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -3969,6 +3969,7 @@ enum { BM25_BIND_EXACT = 8, BM25_BIND_KIND = 9, BM25_BIND_NS = 10, + BM25_BIND_LABEL_WEIGHT = 11, BM25_SQL_AUTO_LEN = -1, /* Inner FTS5 candidate cap. SQLite can early-terminate a plain FTS5 query * (no JOIN/WHERE on outer table) of the form: @@ -3977,6 +3978,11 @@ enum { * and then joining/filtering/re-ranking those, we bound all work to O(N) where * N = BM25_INNER_LIMIT rather than the full match set size. */ BM25_INNER_LIMIT = 2000, + /* Longest label spliced into the MATCH pre-filter; real labels are short + * identifiers, so anything longer simply skips the pre-filter. */ + BM25_LABEL_MAX = 64, + /* Room for "() AND label : \"