From 2083b7c6befeab0ec823e1a6b53a12a52142e7d9 Mon Sep 17 00:00:00 2001 From: Ben Fairless Date: Tue, 18 Aug 2026 16:06:29 +0800 Subject: [PATCH 01/15] fix(extraction): reduce A::B.new callee to the constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Ruby `Const.new` call is rewritten to name the constant rather than the method, so the call links the class the way every other language's `new T()` does — the constructor body lives in `initialize`, so naming `new` links nothing. That rewrite only fired for bare `constant` receivers. A scope-resolved receiver fell through it and kept the whole call expression as its callee: `Admin::User.new`. That resolves to nothing, and it cannot join the Ruby LSP's constructor rows, whose callee is the class QN. Treat `scope_resolution` receivers the same way. The new extraction row pins both spellings and fails without this change, printing the 'Admin::User.new' callee that is the bug's signature. Refs #1701 Assisted-by: OpenCode:anthropic/claude-opus-5 Signed-off-by: Ben Fairless --- internal/cbm/extract_calls.c | 6 ++++- tests/test_extraction.c | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index c93e46a44..085868bdb 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -1668,11 +1668,15 @@ static char *extract_callee_name(CBMArena *a, TSNode node, const char *source, C // method is `new`. The constructor body lives in `initialize`, so a callee // of "new" never resolves. Redirect to the receiver type name so the call // links to the class/constructor like every other language's `new T()`. + // Scope-resolved receivers (`Admin::User.new`) get the same treatment — + // the callee carries the full constant path ("Admin::User"), whose bare + // leaf ("User") is what resolution joins on. if (lang == CBM_LANG_RUBY) { TSNode m = ts_node_child_by_field_name(node, TS_FIELD("method")); TSNode recv = ts_node_child_by_field_name(node, TS_FIELD("receiver")); if (!ts_node_is_null(m) && !ts_node_is_null(recv) && - strcmp(ts_node_type(recv), "constant") == 0) { + (strcmp(ts_node_type(recv), "constant") == 0 || + strcmp(ts_node_type(recv), "scope_resolution") == 0)) { char *mt = cbm_node_text(a, m, source); if (mt && strcmp(mt, "new") == 0) { char *rt = cbm_node_text(a, recv, source); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 447e7c104..9f5109f13 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -619,6 +619,57 @@ TEST(ruby_module) { PASS(); } +/* A Ruby `Const.new` call is rewritten to name the CONSTANT, not the method: + * the constructor body lives in `initialize`, so the call should link the class + * like every other language's `new T()`. The rewrite fired only for bare + * `constant` receivers, so a scope-resolved receiver kept the whole call text + * as its callee ("Admin::User.new"), which resolves to nothing and cannot join + * the Ruby LSP's constructor rows, whose callee is the class QN. + * + * Both spellings must land on the bare constant path; a retained ".new" suffix + * is the bug's signature. */ +TEST(ruby_constructor_callee_is_constant_path) { + CBMFileResult *r = extract("module Admin\n class User\n end\nend\n" + "class Widget\nend\n" + "class Maker\n" + " def build\n" + " Admin::User.new\n" + " Widget.new\n" + " end\n" + "end\n", + CBM_LANG_RUBY, "t", "maker.rb"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + int scoped = 0, bare = 0, kept_new = 0; + for (int i = 0; i < r->calls.count; i++) { + const char *callee = r->calls.items[i].callee_name; + if (strcmp(callee, "Admin::User") == 0) + scoped = 1; + if (strcmp(callee, "Widget") == 0) + bare = 1; + /* Either failure mode: the method name survived on its own, or the + * whole `Recv.new` expression did. */ + size_t n = strlen(callee); + if (strcmp(callee, "new") == 0 || (n >= 4 && strcmp(callee + n - 4, ".new") == 0)) + kept_new = 1; + } + if (!scoped || !bare || kept_new) { + printf(" ruby callees (%d):\n", r->calls.count); + for (int i = 0; i < r->calls.count; i++) + printf(" '%s'\n", r->calls.items[i].callee_name); + } + /* Scope-resolved receiver reduces to the constant path; its bare leaf is + * what resolution joins on. */ + ASSERT(scoped); + /* Bare receiver: pre-existing behaviour, guarded against regression. */ + ASSERT(bare); + ASSERT_FALSE(kept_new); + + cbm_free_result(r); + PASS(); +} + /* --- C# --- */ TEST(csharp_class) { CBMFileResult *r = extract("namespace App { public class Service { public void Run() {} public " @@ -5396,6 +5447,7 @@ SUITE(extraction) { RUN_TEST(php_function); RUN_TEST(ruby_class); RUN_TEST(ruby_module); + RUN_TEST(ruby_constructor_callee_is_constant_path); RUN_TEST(csharp_class); RUN_TEST(csharp_interface); RUN_TEST(swift_class); From 944b2293b44fee9328d2b54bd22f6c1b46e9a65f Mon Sep 17 00:00:00 2001 From: Ben Fairless Date: Wed, 19 Aug 2026 17:05:34 +0800 Subject: [PATCH 02/15] ci: retrigger after transient runner failures The previous run hit two transient infrastructure failures: - security/codeql-gate: gate poller timed out by ~4 seconds; CodeQL analysis itself completed successfully - test/test-unix (macos-15-intel): tests passed but the post-run artifact upload hit ENOTFOUND (network unavailability on the runner) No code changes. Assisted-by: Claude Code:claude-sonnet-4-6 Signed-off-by: Ben Fairless From ad786cd4661c98c16012d461b9443b3d7b3a197a Mon Sep 17 00:00:00 2001 From: Ben Fairless Date: Fri, 21 Aug 2026 10:05:00 +0800 Subject: [PATCH 03/15] fix(test): drop racy active_connections snapshot in connection-cap test daemon_runtime_connection_cap_covers_slow_hello_and_stopping_is_terminal failed on macos CI. The test's capacity_rejected condition sampled active_connections immediately after the overflow connect returned. On a loaded runner the server's HELLO timeout (5 s) can fire for the slow_hello raw connection during the overflow client_connect call (up to 2 s), dropping the count from 2 to 1 and falsifying the == 2 assertion. The capacity rejection message already proves the reason. Drop the point-in-time active_connections sample; keep active_clients == 1, which is stable because we still hold the accepted client open. Same class of race fixed in db91b88c for the forged-identity test. Assisted-by: Claude Code:claude-sonnet-4-6 Signed-off-by: Ben Fairless --- tests/test_daemon_runtime.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index 5f41d914e..34fda16d5 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -2677,10 +2677,15 @@ TEST(daemon_runtime_connection_cap_covers_slow_hello_and_stopping_is_terminal) { if (accepted) { overflow = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity, RUNTIME_TEST_TIMEOUT_MS, &overflow_result); + /* The "capacity" substring in the rejection message proves the reason. + * active_connections is NOT sampled here: on a loaded runner the + * server's HELLO timeout can fire for slow_hello between the raw + * connect and this read, making the count 1 instead of 2 — a + * scheduling race, not a product bug (see db91b88c for precedent). + * active_clients == 1 is safe because we still hold accepted. */ capacity_rejected = overflow == NULL && overflow_result.status == CBM_DAEMON_RUNTIME_CONNECT_REJECTED && strstr(overflow_result.message, "capacity") != NULL && - cbm_daemon_runtime_service_active_connections(fixture.service) == 2 && cbm_daemon_runtime_service_active_clients(fixture.service) == 1; cbm_daemon_ipc_connection_close(slow_hello); From 3db66f22ad7d5c435559869a34b6e14b1e54793b Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Mon, 28 Sep 2026 08:59:32 -0500 Subject: [PATCH 04/15] fix(pipeline): free trees and compact results in sequential definitions pass The sequential route (50 files or fewer, or CBM_WORKERS=1) cached each file's raw extraction result with its tree-sitter tree and uncompacted working arena for the rest of the run. Free the tree, compact the result and release the working arena exactly as the parallel worker already does; the cross-file LSP pass re-parses these files, as it always has on the parallel route. A 40-file repo of large Python files peaks at 246 MB instead of 962 MB; the graph is byte-identical. Signed-off-by: Nelson Spence --- src/pipeline/pass_definitions.c | 7 ++++ tests/test_parallel.c | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 7c7ed6884..b4d37fccf 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -770,6 +770,7 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t } free(local_cache); } + cbm_work_arena_release(); return CBM_NOT_FOUND; } @@ -868,6 +869,11 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t total_calls += result->calls.count; if (local_cache) { + /* The cache lives through every later pass: hold the result as the + * parallel extract does (pass_parallel.c), without its tree (the + * cross-file LSP re-parses) and compacted. */ + cbm_free_tree(result); + cbm_result_compact(result); local_cache[i] = result; } else { /* Cache unavailable: imports for this file can still only @@ -880,6 +886,7 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t cbm_free_result(result); } } + cbm_work_arena_release(); /* the working arena compaction kept between files */ /* Phase 2: now that all extraction results are cached and Module * nodes for every file are in the graph, walk the cache again to diff --git a/tests/test_parallel.c b/tests/test_parallel.c index 85df7d55d..0a285d21f 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -616,6 +616,65 @@ TEST(parallel_spill_mode_builds_the_same_graph) { PASS(); } +/* ── Sequential cache: held like the parallel one ─────────────────── */ + +/* The parallel extract caches each result without its parse tree and + * compacted (pass_parallel.c). The sequential definitions pass cached the raw + * result, tree and working arena included, and that cache lives through every + * later pass: about 5x the memory per file, for every file of the repo. */ +TEST(sequential_definitions_cache_drops_trees_and_compacts) { + if (ensure_parity_setup() != 0) + FAIL("setup failed"); + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + cbm_file_info_t *files = NULL; + int file_count = 0; + ASSERT_EQ(cbm_discover(g_par_tmpdir, &opts, &files, &file_count), 0); + ASSERT_GT(file_count, 0); + + cbm_gbuf_t *gbuf = cbm_gbuf_new("par-test", g_par_tmpdir); + cbm_registry_t *reg = cbm_registry_new(); + CBMFileResult **cache = (CBMFileResult **)calloc((size_t)file_count, sizeof(CBMFileResult *)); + ASSERT_NOT_NULL(cache); + atomic_int cancelled; + atomic_init(&cancelled, 0); + cbm_pipeline_ctx_t ctx = { + .project_name = "par-test", + .repo_path = g_par_tmpdir, + .gbuf = gbuf, + .registry = reg, + .cancelled = &cancelled, + .result_cache = cache, + }; + + cbm_init(); + cbm_work_arena_release(); /* start from a thread that keeps nothing */ + ASSERT_EQ(cbm_pipeline_pass_definitions(&ctx, files, file_count), 0); + ASSERT_FALSE(cbm_work_arena_keeping()); /* ...and end the pass that way */ + + int cached = 0; + for (int i = 0; i < file_count; i++) { + const CBMFileResult *r = cache[i]; + if (!r) { + continue; + } + cached++; + ASSERT_NULL(r->cached_tree); + /* One exact block: capacity == bytes used, no dead headroom. */ + ASSERT_EQ(r->arena.nblocks, 1); + ASSERT_EQ(cbm_arena_capacity(&r->arena), cbm_arena_total(&r->arena)); + } + ASSERT_EQ(cached, file_count); + + for (int i = 0; i < file_count; i++) { + cbm_free_result(cache[i]); + } + free(cache); + cbm_registry_free(reg); + cbm_gbuf_free(gbuf); + cbm_discover_free(files, file_count); + PASS(); +} + /* ── Empty file list ──────────────────────────────────────────────── */ TEST(parallel_empty_files) { @@ -4437,6 +4496,7 @@ SUITE(parallel) { RUN_TEST(parallel_semantic_fixture_expected_counts); RUN_TEST(parallel_total_edges); RUN_TEST(parallel_spill_mode_builds_the_same_graph); + RUN_TEST(sequential_definitions_cache_drops_trees_and_compacts); RUN_TEST(parallel_empty_files); RUN_TEST(parallel_args_json_no_overflow); From fd1b88374c0c21892c991c29909d22ed499c20fa Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Mon, 28 Sep 2026 08:59:32 -0500 Subject: [PATCH 05/15] fix(discover): keep make/cargo dep-info .d files out of the D grammar .d mapped to D unconditionally, so make-style dependency files written by cargo, rustc, gcc/clang -MD and CMake were parsed with the D grammar and walked at superlinear cost. Add cbm_disambiguate_d, a 4 KB first-line sniff in the shape of the existing .frm/.cls/.m disambiguators, that returns CBM_LANG_COUNT only for a make rule whose targets look like paths and keeps D on any doubt. A 200-file cargo dep-info corpus goes from 108.9 s to 1.27 s to index; 0 of 12,062 real D files are misclassified. Signed-off-by: Nelson Spence --- src/discover/discover.c | 5 +++ src/discover/discover.h | 6 +++ src/discover/language.c | 81 +++++++++++++++++++++++++++++++++++++++++ tests/test_discover.c | 48 ++++++++++++++++++++++++ tests/test_language.c | 51 ++++++++++++++++++++++++++ 5 files changed, 191 insertions(+) diff --git a/src/discover/discover.c b/src/discover/discover.c index 0e5ae5e4f..f3518abb8 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -803,6 +803,11 @@ static CBMLanguage detect_file_language(const char *entry_name, const char *abs_ if (dot && strcmp(dot, ".res") == 0) { lang = cbm_disambiguate_res(abs_path); } + /* Special: .d is shared by D and make-style dep-info (cargo, gcc -MD). + * Probe only while .d maps to D, so a user override to another language wins. */ + if (dot && strcmp(dot, ".d") == 0 && lang == CBM_LANG_DLANG) { + lang = cbm_disambiguate_d(abs_path); + } /* Special: ObjectScript Studio Export XML () is * detected by content; otherwise .xml stays XML. */ if (lang == CBM_LANG_XML) { diff --git a/src/discover/discover.h b/src/discover/discover.h index 89ccca4f3..24a08d5f4 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -57,6 +57,12 @@ CBMLanguage cbm_disambiguate_frm(const char *path); * CBM_LANG_RESCRIPT. On read failure, defaults to CBM_LANG_RESCRIPT. */ CBMLanguage cbm_disambiguate_res(const char *path); +/* Disambiguate .d files by reading first 4KB of content. + * Returns CBM_LANG_COUNT (not source) for a make-style dependency file written + * by rustc/cargo, gcc/clang -MD or CMake ("target ...: prereq ..."), otherwise + * CBM_LANG_DLANG. On read failure or any doubt, defaults to CBM_LANG_DLANG. */ +CBMLanguage cbm_disambiguate_d(const char *path); + /* Disambiguate .inc files by reading first 4KB of content. * Returns CBM_LANG_OBJECTSCRIPT_ROUTINE if it looks like an ObjectScript * include (a "ROUTINE " header), otherwise CBM_LANG_BITBAKE. diff --git a/src/discover/language.c b/src/discover/language.c index dcc41dafd..66de4f4df 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -1330,6 +1330,87 @@ CBMLanguage cbm_disambiguate_res(const char *path) { return memchr(buf, '\0', n) ? CBM_LANG_COUNT : CBM_LANG_RESCRIPT; } +/* A make rule's targets end at a ':' followed by whitespace or the end of the + * file, which skips drive-letter colons ("C:/", "C:\"). */ +static bool is_rule_colon(const char *p, bool whole_file) { + char next = p[SKIP_ONE]; + return *p == ':' && (next == ' ' || next == '\t' || next == '\r' || next == '\n' || + (next == '\0' && whole_file)); +} + +/* True if the line at p is a make rule "target...: prereq..." whose targets all + * look like paths (contain '/', '\' or '.'); a backslash-escaped space stays + * inside its target. D source fails this: "public:", "@safe:", "extern(C):" + * and "import a.b : c" all have a non-path word or D punctuation before the + * colon. */ +static bool is_dep_rule_line(const char *p, bool whole_file) { + bool in_target = false; + bool path_like = false; + bool any_target = false; + for (; *p && *p != '\n'; p++) { + if (is_rule_colon(p, whole_file)) { + return in_target ? path_like : any_target; + } + if (*p == ' ' || *p == '\t' || *p == '\r') { + if (in_target && !path_like) { + return false; + } + in_target = false; + path_like = false; + continue; + } + if (strchr("(){};=\"',", *p)) { + return false; + } + in_target = true; + any_target = true; + path_like = path_like || *p == '/' || *p == '\\' || *p == '.'; + if (*p == '\\' && p[SKIP_ONE] == ' ') { + p++; /* escaped space: part of this target */ + } + } + return false; +} + +/* Disambiguate .d files: shared by D source and make-style dependency files + * written by rustc/cargo (target//deps/-.d), gcc/clang -MD + * and CMake (.o.d). Those are "target: prereq ..." rules, often tens of + * KB on one line, which the D grammar parses slowly into nothing but a module + * node, so they are reported as unsupported (CBM_LANG_COUNT). Defaults to D on + * any doubt (preserves existing behaviour). */ +CBMLanguage cbm_disambiguate_d(const char *path) { + if (!path) { + return CBM_LANG_DLANG; + } + + FILE *f = cbm_fopen(path, "r"); + if (!f) { + return CBM_LANG_DLANG; + } + + char buf[CBM_SZ_4K + SKIP_ONE]; + size_t n = fread(buf, SKIP_ONE, CBM_SZ_4K, f); + buf[n] = '\0'; + (void)fclose(f); + /* A colon at the end of buf only ends the rule if nothing was cut off. */ + bool whole_file = n < CBM_SZ_4K && strlen(buf) == n; + + const char *p = buf; + if (strncmp(p, "\xEF\xBB\xBF", SLEN("\xEF\xBB\xBF")) == 0) { + p += SLEN("\xEF\xBB\xBF"); + } + while (*p && isspace((unsigned char)*p)) { + p++; + } + /* A shebang or a leading D comment (including the DUB single-file + * "/+ dub.sdl:" recipe) is D; dep-info never starts that way. */ + if (*p == '#' || + (p[0] == '/' && (p[SKIP_ONE] == '/' || p[SKIP_ONE] == '*' || p[SKIP_ONE] == '+'))) { + return CBM_LANG_DLANG; + } + return is_dep_rule_line(p, whole_file) ? CBM_LANG_COUNT : CBM_LANG_DLANG; +} + /* Disambiguate .cls files: shared by InterSystems ObjectScript UDL, Salesforce * Apex and Visual Basic 6 class modules (#721). ObjectScript class files begin * with a line of the form "Class ..."; VB6 class modules diff --git a/tests/test_discover.c b/tests/test_discover.c index 5e9bdd8af..033eb10c9 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -6,6 +6,7 @@ #include "test_framework.h" #include "test_helpers.h" #include "discover/discover.h" +#include "discover/userconfig.h" #include "foundation/platform.h" typedef struct { @@ -1990,6 +1991,49 @@ TEST(shebang_oversized_first_line_unindexed) { PASS(); } +/* ── .d: D source vs make/cargo dep-info ───────────────────────── */ + +/* rustc/cargo dep-info (and gcc -MD, CMake .o.d) files share .d with D source + * but are Makefile rules the D grammar parses slowly into nothing useful, so + * they must not be indexed as D. */ +static const char CARGO_DEP_INFO[] = "/home/u/proj/t/debug/deps/foo-abc123.d: src/lib.rs src/a.rs\n" + "\n" + "src/lib.rs:\n" + "src/a.rs:\n"; + +TEST(discover_d_dep_info_not_indexed_as_dlang) { + CBMLanguage dep; + CBMLanguage src; + ASSERT(shebang_probe("cbm_disc_dotd", "t/debug/deps/foo-abc123.d", CARGO_DEP_INFO, &dep)); + ASSERT(shebang_probe("cbm_disc_dotd", "app.d", "module app;\nimport std.stdio : writeln;\n", + &src)); + ASSERT_EQ(dep, CBM_LANG_COUNT); + ASSERT_EQ(src, CBM_LANG_DLANG); + PASS(); +} + +/* The probe runs only while .d maps to D, so a user override to another + * language wins. */ +TEST(discover_d_user_override_skips_dep_info_probe) { + char *cfg_dir = th_mktempdir("cbm_disc_dotd_uc"); + ASSERT(cfg_dir != NULL); + ASSERT_EQ(th_write_file(TH_PATH(cfg_dir, ".codebase-memory.json"), + "{\"extra_extensions\":{\".d\":\"makefile\"}}"), + 0); + cbm_userconfig_t *cfg = cbm_userconfig_load(cfg_dir); + th_cleanup(cfg_dir); + ASSERT_NOT_NULL(cfg); + + cbm_set_user_lang_config(cfg); + CBMLanguage lang; + bool ok = shebang_probe("cbm_disc_dotd_uc", "foo-abc123.d", CARGO_DEP_INFO, &lang); + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(cfg); + ASSERT(ok); + ASSERT_EQ(lang, CBM_LANG_MAKEFILE); + PASS(); +} + /* ── Suite ─────────────────────────────────────────────────────── */ SUITE(discover) { @@ -2082,6 +2126,10 @@ SUITE(discover) { RUN_TEST(shebang_embedded_nul_unindexed); RUN_TEST(shebang_oversized_first_line_unindexed); + /* .d: D source vs make/cargo dep-info */ + RUN_TEST(discover_d_dep_info_not_indexed_as_dlang); + RUN_TEST(discover_d_user_override_skips_dep_info_probe); + /* Integration tests (cross-platform) */ RUN_TEST(discover_simple); RUN_TEST(discover_wide_sibling_fanout_exceeds_initial_walk_stack); diff --git a/tests/test_language.c b/tests/test_language.c index dc2bcf7d3..d9d38429f 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -835,6 +835,55 @@ TEST(lang_cfc_default_on_read_fail) { PASS(); } +/* ── .d: D source vs make/cargo dep-info ───────────────────────── */ + +/* Write content to a temp .d file and return cbm_disambiguate_d() for it. A + * setup failure returns -1 so neither a D nor a COUNT expectation can pass + * vacuously. */ +static int disambiguate_d_content(const char *content) { + char path[256]; + snprintf(path, sizeof(path), "%s/test_lang_dotd.d", cbm_tmpdir()); + if (!write_probe_file(path, content)) { + return -1; + } + CBMLanguage lang = cbm_disambiguate_d(path); + remove(path); + return (int)lang; +} + +TEST(lang_d_dep_info_unsupported) { + /* cargo (absolute target), gcc -MD continuation, CMake object rule, Windows + * drive letter (not the rule colon), escaped space (one target, not "app"), + * and the GNU make manual's "foo.o foo.d : ..." (two targets, spaced colon). */ + ASSERT_EQ(disambiguate_d_content("/home/u/t/debug/deps/foo-abc123.d: src/lib.rs src/a.rs\n" + "\nsrc/lib.rs:\n"), + CBM_LANG_COUNT); + ASSERT_EQ(disambiguate_d_content("foo.o: foo.c /usr/include/stdio.h \\\n" + " /usr/include/features.h\n"), + CBM_LANG_COUNT); + ASSERT_EQ(disambiguate_d_content("CMakeFiles/x.dir/a.cpp.o: \\\n /src/a.cpp\n"), + CBM_LANG_COUNT); + ASSERT_EQ(disambiguate_d_content("C:/b/foo.o: C:/s/foo.c \\\r\n C:/s/foo.h\r\n"), + CBM_LANG_COUNT); + ASSERT_EQ(disambiguate_d_content("out/my\\ app: main.c\n"), CBM_LANG_COUNT); + ASSERT_EQ(disambiguate_d_content("foo.o foo.d : foo.c defs.h\n"), CBM_LANG_COUNT); + PASS(); +} + +TEST(lang_d_source_stays_dlang) { + ASSERT_EQ(disambiguate_d_content("module a;\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("import std.stdio : writeln;\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("@safe:\nvoid f() {}\n"), CBM_LANG_DLANG); + /* A leading comment is D, even DUB's "/+ dub.sdl:" single-file recipe. */ + ASSERT_EQ(disambiguate_d_content("/+ dub.sdl:\n name \"hello\"\n+/\nvoid main() {}\n"), + CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("enum E : ubyte {\n a,\n}\n"), CBM_LANG_DLANG); + /* Default on doubt: empty or unreadable. */ + ASSERT_EQ(disambiguate_d_content(""), CBM_LANG_DLANG); + ASSERT_EQ(cbm_disambiguate_d("/tmp/nonexistent_file_12345.d"), CBM_LANG_DLANG); + PASS(); +} + /* --- New languages (auto-generated) --- */ TEST(lang_ext_solidity) { ASSERT_EQ(cbm_language_for_extension(".sol"), CBM_LANG_SOLIDITY); @@ -1488,6 +1537,8 @@ SUITE(language) { RUN_TEST(lang_frm_form_stays_form); RUN_TEST(lang_res_binary_resource_unsupported); RUN_TEST(lang_res_rescript_stays_rescript); + RUN_TEST(lang_d_dep_info_unsupported); + RUN_TEST(lang_d_source_stays_dlang); /* Go test ports */ /* New languages */ From 9140e7dfbe2e4942428e017cab8efef1f9de7f84 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Thu, 1 Oct 2026 12:55:02 -0500 Subject: [PATCH 06/15] test(pipeline): dep-info .d files leave no File, Module or Folder nodes Run the full pipeline over a cargo dep-info file alone under its own directory, and a gcc -MD file beside real D source. The dep-info files get no File or Module node, their directory chain gets no Folder node, and the D file keeps its File, Module and Folder. With the .d probe disabled the test fails with three Folder nodes under the dep-info directory. Signed-off-by: Nelson Spence --- tests/test_pipeline.c | 66 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 03a5cad70..3b84a5d74 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -631,6 +631,71 @@ TEST(pipeline_structure_nodes) { PASS(); } +/* Number of `label` nodes whose file_path starts with `prefix`, or -1. */ +static int count_nodes_with_path_prefix(cbm_store_t *s, const char *project, const char *label, + const char *prefix) { + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_label(s, project, label, &nodes, &count) != CBM_STORE_OK) { + return -1; + } + int hits = 0; + for (int i = 0; i < count; i++) { + if (nodes[i].file_path && strncmp(nodes[i].file_path, prefix, strlen(prefix)) == 0) { + hits++; + } + } + cbm_store_free_nodes(nodes, count); + return hits; +} + +/* Make-style dep-info .d files are dropped at discovery, so the graph holds no + * File or Module node for them and no Folder for a directory that held only + * dep-info. Real D source beside a dep-info file keeps its nodes and folder. */ +TEST(pipeline_d_dep_info_leaves_no_nodes) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dotd_graph_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + write_temp_file(repo, "cargo-target/debug/deps/app-0123abcd.d", + "/r/cargo-target/debug/deps/app-0123abcd.d: src/main.rs\n\nsrc/main.rs:\n"); + write_temp_file(repo, "src/app.d", "module app;\n\nvoid main() {}\n"); + write_temp_file(repo, "src/app.o.d", "src/app.o: src/app.d\n"); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/test.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + const char *project = cbm_pipeline_project_name(p); + + int dep_folders = count_nodes_with_path_prefix(s, project, "Folder", "cargo-target"); + int dep_files = count_nodes_with_path_prefix(s, project, "File", "cargo-target"); + int dep_modules = count_nodes_with_path_prefix(s, project, "Module", "cargo-target"); + int obj_files = count_nodes_with_path_prefix(s, project, "File", "src/app.o.d"); + int obj_modules = count_nodes_with_path_prefix(s, project, "Module", "src/app.o.d"); + int src_folders = count_nodes_with_path_prefix(s, project, "Folder", "src"); + int src_files = count_nodes_with_path_prefix(s, project, "File", "src/app.d"); + int src_modules = count_nodes_with_path_prefix(s, project, "Module", "src/app.d"); + + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + + ASSERT_EQ(dep_folders, 0); + ASSERT_EQ(dep_files, 0); + ASSERT_EQ(dep_modules, 0); + ASSERT_EQ(obj_files, 0); + ASSERT_EQ(obj_modules, 0); + ASSERT_EQ(src_folders, 1); + ASSERT_EQ(src_files, 1); + ASSERT_EQ(src_modules, 1); + PASS(); +} + /* Issue #516: an ADR stored via manage_adr (project_summaries) must survive a * full re-index. A full re-index deletes the DB and rebuilds it from the graph * buffer, which writes an empty project_summaries table; the fix captures the @@ -16218,6 +16283,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_sql_dump_graph_matches_the_full_parse_issue1735); RUN_TEST(pipeline_spill_resolves_namespace_imports_like_memory); RUN_TEST(pipeline_structure_nodes); + RUN_TEST(pipeline_d_dep_info_leaves_no_nodes); RUN_TEST(pipeline_committed_counts_match_persisted); RUN_TEST(pipeline_adr_survives_full_reindex); RUN_TEST(pipeline_export_error_snapshot_on_artifact_failure); From ceb729d8a36b9a02778d86d0b209c7decbe58dd5 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Thu, 1 Oct 2026 14:09:50 -0500 Subject: [PATCH 07/15] fix(discover): keep D whose first colon follows an attribute or comment is_dep_rule_line took any '.' or '/' before the rule colon as a path, so valid D such as "@1.0:" (a float UDA) and "public/**/:" (a comment between tokens) was dropped as dep-info. A target holding '@' or a comment opener ("/*", "/+", "//") is now D. Over 12,062 real D files and 4,494 real files under deps/ dirs, no classification changes. A dep-info file whose target path contains '@' now stays D, as it does on main. Signed-off-by: Nelson Spence --- src/discover/language.c | 9 +++++++-- tests/test_language.c | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/discover/language.c b/src/discover/language.c index 66de4f4df..0d6d3faca 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -1342,7 +1342,9 @@ static bool is_rule_colon(const char *p, bool whole_file) { * look like paths (contain '/', '\' or '.'); a backslash-escaped space stays * inside its target. D source fails this: "public:", "@safe:", "extern(C):" * and "import a.b : c" all have a non-path word or D punctuation before the - * colon. */ + * colon. An attribute ('@', as in "@1.0:") or a comment between tokens + * ("public/+ +/:") is D even with a '.' or '/' before the colon, so a target + * holding either is never dep-info. */ static bool is_dep_rule_line(const char *p, bool whole_file) { bool in_target = false; bool path_like = false; @@ -1359,7 +1361,10 @@ static bool is_dep_rule_line(const char *p, bool whole_file) { path_like = false; continue; } - if (strchr("(){};=\"',", *p)) { + if (strchr("(){};=\"',@", *p)) { + return false; + } + if (*p == '/' && (p[SKIP_ONE] == '*' || p[SKIP_ONE] == '+' || p[SKIP_ONE] == '/')) { return false; } in_target = true; diff --git a/tests/test_language.c b/tests/test_language.c index d9d38429f..d93eb0ecb 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -878,6 +878,14 @@ TEST(lang_d_source_stays_dlang) { ASSERT_EQ(disambiguate_d_content("/+ dub.sdl:\n name \"hello\"\n+/\nvoid main() {}\n"), CBM_LANG_DLANG); ASSERT_EQ(disambiguate_d_content("enum E : ubyte {\n a,\n}\n"), CBM_LANG_DLANG); + /* A float UDA ("@ FloatLiteral") has a '.' before its colon, alone or glued + * to a keyword. */ + ASSERT_EQ(disambiguate_d_content("@1.0:\nvoid f() {}\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("nothrow@1.0:\nvoid f() {}\n"), CBM_LANG_DLANG); + /* A comment between tokens puts a '/' before the colon. */ + ASSERT_EQ(disambiguate_d_content("public/**/:\nvoid f() {}\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("public/+ +/:\nvoid f() {}\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("public//note: x\nvoid f() {}\n"), CBM_LANG_DLANG); /* Default on doubt: empty or unreadable. */ ASSERT_EQ(disambiguate_d_content(""), CBM_LANG_DLANG); ASSERT_EQ(cbm_disambiguate_d("/tmp/nonexistent_file_12345.d"), CBM_LANG_DLANG); From 006f2a991c6a8e0d141750d53959150147f876ad Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Thu, 1 Oct 2026 14:40:06 -0500 Subject: [PATCH 08/15] fix(discover): split dep-rule targets on every D whitespace is_dep_rule_line split targets only on space, tab and CR, so valid D such as "import\fstd.stdio : writeln;" kept the keyword and the dotted module name in one path-like target and was dropped as dep-info. Vertical tab, form feed and the UTF-8 separators U+2028/U+2029 now split targets too. Over 12,062 real D files and 4,494 real files under deps/ dirs, no classification changes. Signed-off-by: Nelson Spence --- src/discover/language.c | 18 +++++++++++++++++- tests/test_language.c | 10 ++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/discover/language.c b/src/discover/language.c index 0d6d3faca..c0f6f1894 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -1338,6 +1338,20 @@ static bool is_rule_colon(const char *p, bool whole_file) { (next == '\0' && whole_file)); } +/* Bytes of D whitespace at p, or 0: space, tab, vertical tab, form feed, CR + * and the UTF-8 line and paragraph separators U+2028/U+2029, all of which split + * D tokens. '\n' ends the line and is the caller's. */ +static size_t d_space_len(const char *p) { + if (*p == ' ' || *p == '\t' || *p == '\v' || *p == '\f' || *p == '\r') { + return SKIP_ONE; + } + if (strncmp(p, "\xE2\x80\xA8", SLEN("\xE2\x80\xA8")) == 0 || + strncmp(p, "\xE2\x80\xA9", SLEN("\xE2\x80\xA9")) == 0) { + return SLEN("\xE2\x80\xA8"); + } + return 0; +} + /* True if the line at p is a make rule "target...: prereq..." whose targets all * look like paths (contain '/', '\' or '.'); a backslash-escaped space stays * inside its target. D source fails this: "public:", "@safe:", "extern(C):" @@ -1353,12 +1367,14 @@ static bool is_dep_rule_line(const char *p, bool whole_file) { if (is_rule_colon(p, whole_file)) { return in_target ? path_like : any_target; } - if (*p == ' ' || *p == '\t' || *p == '\r') { + size_t space = d_space_len(p); + if (space) { if (in_target && !path_like) { return false; } in_target = false; path_like = false; + p += space - SKIP_ONE; continue; } if (strchr("(){};=\"',@", *p)) { diff --git a/tests/test_language.c b/tests/test_language.c index d93eb0ecb..9b3bdc0a9 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -886,6 +886,16 @@ TEST(lang_d_source_stays_dlang) { ASSERT_EQ(disambiguate_d_content("public/**/:\nvoid f() {}\n"), CBM_LANG_DLANG); ASSERT_EQ(disambiguate_d_content("public/+ +/:\nvoid f() {}\n"), CBM_LANG_DLANG); ASSERT_EQ(disambiguate_d_content("public//note: x\nvoid f() {}\n"), CBM_LANG_DLANG); + /* Every D whitespace splits tokens, not only space, tab and CR: form feed, + * vertical tab, U+2028 and U+2029. */ + ASSERT_EQ(disambiguate_d_content("import\fstd.stdio : writeln;\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("import\vstd.stdio : writeln;\n"), CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("import\xE2\x80\xA8" + "std.stdio : writeln;\n"), + CBM_LANG_DLANG); + ASSERT_EQ(disambiguate_d_content("import\xE2\x80\xA9" + "std.stdio : writeln;\n"), + CBM_LANG_DLANG); /* Default on doubt: empty or unreadable. */ ASSERT_EQ(disambiguate_d_content(""), CBM_LANG_DLANG); ASSERT_EQ(cbm_disambiguate_d("/tmp/nonexistent_file_12345.d"), CBM_LANG_DLANG); From 66ff5b86a3f3219fcddc9b69f75a2036a96c8ae9 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Fri, 9 Oct 2026 12:12:49 -0500 Subject: [PATCH 09/15] test(language): cover the .d rule in the shared classifier tests lang_probe_bytes_follow_the_name, lang_classify_matches_every_content_rule and the unreadable-content test now include .d: a 4 KB probe, a cargo dep-info line that is not indexed and D source that stays D, through both cbm_language_classify and cbm_disambiguate_d. With lang_probe_d disabled, the first two fail, as does discover_d_dep_info_not_indexed_as_dlang. Signed-off-by: Nelson Spence --- tests/test_language.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_language.c b/tests/test_language.c index 0d73720e3..43388b3d6 100644 --- a/tests/test_language.c +++ b/tests/test_language.c @@ -1415,6 +1415,7 @@ TEST(lang_probe_bytes_follow_the_name) { ASSERT_EQ(cbm_language_probe_bytes("foo.inc"), 4096); ASSERT_EQ(cbm_language_probe_bytes("Form1.frm"), 4096); ASSERT_EQ(cbm_language_probe_bytes("App.res"), 4096); + ASSERT_EQ(cbm_language_probe_bytes("app.d"), 4096); ASSERT_EQ(cbm_language_probe_bytes("Widget.cfc"), 16384); ASSERT_EQ(cbm_language_probe_bytes("pom.xml"), 255); /* An unknown name falls back to a shebang probe of the first line. */ @@ -1456,6 +1457,9 @@ TEST(lang_classify_matches_every_content_rule) { {"p.frm", "#procedure foo\nLocal F = a;\n", CBM_LANG_FORM, cbm_disambiguate_frm}, {"q.res", "RSRC\0\0\0\1binary", CBM_LANG_COUNT, cbm_disambiguate_res}, {"r.res", "let x = 1\n", CBM_LANG_RESCRIPT, cbm_disambiguate_res}, + {"dep.d", "target/debug/deps/app-0a1b2c.d: src/main.rs src/lib.rs\n", CBM_LANG_COUNT, + cbm_disambiguate_d}, + {"app.d", "module app;\nimport std.stdio : writeln;\n", CBM_LANG_DLANG, cbm_disambiguate_d}, {"s.xml", "\n\n", CBM_LANG_OBJECTSCRIPT_EXPORT, NULL}, {"t.xml", "\n\n", CBM_LANG_XML, NULL}, @@ -1503,6 +1507,7 @@ TEST(lang_classify_unreadable_content_keeps_the_name_default) { ASSERT_EQ(cbm_language_classify("a.cfc", NULL, 0, false, false), CBM_LANG_CFSCRIPT); ASSERT_EQ(cbm_language_classify("a.frm", NULL, 0, false, false), CBM_LANG_FORM); ASSERT_EQ(cbm_language_classify("a.res", NULL, 0, false, false), CBM_LANG_RESCRIPT); + ASSERT_EQ(cbm_language_classify("a.d", NULL, 0, false, false), CBM_LANG_DLANG); ASSERT_EQ(cbm_language_classify("a.xml", NULL, 0, false, false), CBM_LANG_XML); ASSERT_EQ(cbm_language_classify("run-tests", NULL, 0, false, false), CBM_LANG_COUNT); ASSERT_EQ(cbm_language_classify("main.go", NULL, 0, false, false), CBM_LANG_GO); From 9a180d06263c07c5afc5706e3965d2dbee73639e Mon Sep 17 00:00:00 2001 From: Scott Beck <296871+bline@users.noreply.github.com> Date: Sun, 11 Oct 2026 17:26:06 -0600 Subject: [PATCH 10/15] fix(doc-links): report a path::member reference to a missing member (#2587) A Markdown reference `path::member` whose member the code file does not define became an exact MENTIONS edge to the file. A renamed or deleted definition therefore looked like a valid, coarser link instead of a stale reference, and nothing recorded that the member was not found. mdr_segment now says how a member lookup ended. When a code file does not define the member, the reference is an unresolved `missing` row, the same doc-rot reason a missing path gets. When the file defines the name more than once and no owner picks one, it is `ambiguous`. This matches the C resolver, which reports a missing member of an existing type as missing rather than falling back to the type. A member written on a non-code file (a document's heading) keeps the file edge, since those are not graph definitions. Line ranges without a member are unchanged. Test (tests/test_doc_links_md.c) indexes a fixture with a found member, a member reached through its owner, a missing member, a name defined twice, and a member of a Markdown file. Without the fix it fails on the defect: the section's MENTIONS edge to src/util/rounding.py exists (1, expected 0). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Scott Beck <296871+bline@users.noreply.github.com> --- src/pipeline/doc_links_md.c | 55 +++++++++++++++++++++++++----- tests/test_doc_links_md.c | 68 +++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/src/pipeline/doc_links_md.c b/src/pipeline/doc_links_md.c index 9e76d41a4..14d6db513 100644 --- a/src/pipeline/doc_links_md.c +++ b/src/pipeline/doc_links_md.c @@ -17,9 +17,12 @@ * * Unresolved: * missing the path is not in the graph, but its directory is: the - * document names something that is not there (doc rot) + * document names something that is not there (doc rot); or a + * code file's member that the file does not define * ambiguous a file name alone that is not next to the document but exists - * elsewhere in the repository: the document does not say which + * elsewhere in the repository: the document does not say which; + * or a member name the file defines more than once, with no + * owner to pick one * Not a reference to this repository at all (LOCAL: no edge, no row): a link * back into the same document; a path that is neither indexed nor of a type * the indexer reads (an image, an archive, an extensionless page); a path @@ -478,12 +481,24 @@ static void mdr_file_defs(const mdr_index_t *x, const char *path, int *lo, int * *hi = e; } +/* How a member lookup ended: found, or why not. */ +typedef enum { + MDR_MEMBER_FOUND = 0, + MDR_MEMBER_MISSING, /* no definition of that name in the file */ + MDR_MEMBER_AMBIGUOUS, /* several, and the owner does not pick one */ +} mdr_member_t; + /* The code definition of file `path` that the range or the member names, or * NULL. Read from the index's definitions by file, never from DEFINES edges: * an incremental run's graph holds the edges of the files it re-extracts - * only, while every node is there. */ + * only, while every node is there. *member_result (when not NULL) says how a + * member lookup ended. */ static const cbm_gbuf_node_t *mdr_segment(const mdr_index_t *x, const char *path, uint32_t first, - uint32_t last, const char *member) { + uint32_t last, const char *member, + mdr_member_t *member_result) { + if (member_result) { + *member_result = MDR_MEMBER_FOUND; + } int lo = 0; int hi = 0; mdr_file_defs(x, path, &lo, &hi); @@ -516,6 +531,9 @@ static const cbm_gbuf_node_t *mdr_segment(const mdr_index_t *x, const char *path size_t owner_len; mdr_member_names(member, &name, &name_len, &owner, &owner_len); if (!name) { + if (member_result) { + *member_result = MDR_MEMBER_MISSING; + } return NULL; } const cbm_gbuf_node_t *hit = NULL; @@ -537,7 +555,13 @@ static const cbm_gbuf_node_t *mdr_segment(const mdr_index_t *x, const char *path if (owned_hits == SKIP_ONE) { return owned; } - return owned_hits == 0 && hits == SKIP_ONE ? hit : NULL; + if (owned_hits == 0 && hits == SKIP_ONE) { + return hit; + } + if (member_result) { + *member_result = hits == 0 ? MDR_MEMBER_MISSING : MDR_MEMBER_AMBIGUOUS; + } + return NULL; } /* Is `path` of a file type the indexer reads? */ @@ -1151,9 +1175,21 @@ static void mdr_resolve_path(const mdr_index_t *x, const cbm_gbuf_t *graph, cons out->kind = CBM_DOCLINK_LOCAL; /* a link into the same document */ return; } - const cbm_gbuf_node_t *seg = (r->first > 0 || r->member) - ? mdr_segment(x, t, r->first, r->last, r->member) - : NULL; + mdr_member_t member_result = MDR_MEMBER_FOUND; + const cbm_gbuf_node_t *seg = + (r->first > 0 || r->member) + ? mdr_segment(x, t, r->first, r->last, r->member, &member_result) + : NULL; + /* A named member that a code file does not define is no coarser + * link to the file: it is a stale or unclear reference. A + * document's own headings are not definitions, so members of + * other files keep the file edge. */ + if (!seg && r->member && member_result != MDR_MEMBER_FOUND && mdr_code_file(t)) { + out->kind = CBM_DOCLINK_UNRESOLVED; + out->reason = member_result == MDR_MEMBER_AMBIGUOUS ? CBM_DOCLINK_REASON_AMBIGUOUS + : CBM_DOCLINK_REASON_MISSING; + return; + } mdr_edge(out, seg ? seg : file, r->first, r->last); return; } @@ -1388,7 +1424,8 @@ static void mdr_resolve(const void *index, void *state, int run_file, const CBMD const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char *path, uint32_t first, uint32_t last, const char *member) { - return md_index ? mdr_segment((const mdr_index_t *)md_index, path, first, last, member) : NULL; + return md_index ? mdr_segment((const mdr_index_t *)md_index, path, first, last, member, NULL) + : NULL; } /* ── A region of a file's text (the field test's H8 bind_range) ─────── */ diff --git a/tests/test_doc_links_md.c b/tests/test_doc_links_md.c index 21e74f37e..11049e032 100644 --- a/tests/test_doc_links_md.c +++ b/tests/test_doc_links_md.c @@ -388,6 +388,73 @@ TEST(doc_links_md_resolve) { PASS(); } +/* #2587: a `path::member` reference whose member the code file does not + * define was an exact edge to the file, so a renamed or deleted definition + * looked like a valid, coarser link. It is a missing reference; a member the + * file defines more than once, with no owner to pick one, is ambiguous. */ +TEST(doc_links_md_member_not_defined_issue2587) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + th_write_file(TH_PATH(tmp, "docs/members.md"), + "# Members\n" + "\n" + "See `src/util/rounding.py::clamp` and " + "`src/util/rounding.py::no_such_function`.\n" + "Areas: `src/shapes.py::Square::area`, `src/shapes.py::area`.\n" + "Notes: `docs/other.md::Intro`.\n"); + th_write_file(TH_PATH(tmp, "docs/other.md"), "# Intro\n\nText.\n"); + th_write_file(TH_PATH(tmp, "src/util/rounding.py"), "def clamp(x, lo, hi):\n" + " return max(lo, min(hi, x))\n"); + th_write_file(TH_PATH(tmp, "src/shapes.py"), "class Circle:\n" + " def area(self):\n" + " return 3\n" + "\n" + "\n" + "class Square:\n" + " def area(self):\n" + " return 4\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/md.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + char props[512]; + char reason[64]; + + /* a member the file defines, alone or through its owner */ + ASSERT_EQ(md_edge(db, "Members", "rounding.clamp", props, sizeof(props)), 1); + ASSERT_EQ(md_edge(db, "Members", "Square.area", props, sizeof(props)), 1); + ASSERT_EQ(md_edge(db, "Members", "Circle.area", props, sizeof(props)), 0); + + /* a member the file does not define: no edge to the file, a missing row */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Members' AND t.label = 'File' " + "AND t.file_path = 'src/util/rounding.py'"), + 0); + dm_row(db, "docs/members.md", "src/util/rounding.py::no_such_function", reason, sizeof(reason), + NULL, 0); + ASSERT_STR_EQ(reason, "missing"); + + /* defined twice and no owner written: ambiguous, not the file */ + dm_row(db, "docs/members.md", "src/shapes.py::area", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "ambiguous"); + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Members' AND t.label = 'File' " + "AND t.file_path = 'src/shapes.py'"), + 0); + + /* a document's headings are no definitions: the file edge stays */ + ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%.Members' AND t.label = 'File' " + "AND t.file_path = 'docs/other.md'"), + 1); + + th_cleanup(tmp); + PASS(); +} + /* ── incremental == full ─────────────────────────────────────────── */ TEST(doc_links_md_incremental) { @@ -865,6 +932,7 @@ SUITE(doc_links_md) { RUN_TEST(doc_links_md_repeated_headings); RUN_TEST(doc_links_md_classify_span); RUN_TEST(doc_links_md_resolve); + RUN_TEST(doc_links_md_member_not_defined_issue2587); RUN_TEST(doc_links_md_incremental); RUN_TEST(doc_links_md_adr); RUN_TEST(doc_links_md_adr_label_choice); From 6ebd07816b5fce5dab2d49cbd15903b63f5a248a Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 20 Oct 2026 03:04:40 +0200 Subject: [PATCH 11/15] test(daemon): leave the connection-cap test out of this Ruby fix Restore tests/test_daemon_runtime.c to the PR's base, as asked in review: the dropped active_connections assertion is unrelated to the Ruby constructor callee and belongs in its own change. main has since reworked this test on its own (flaky-ledger item 12), so the merge of main that follows takes main's version. Signed-off-by: Martin Vogel --- tests/test_daemon_runtime.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index 34fda16d5..5f41d914e 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -2677,15 +2677,10 @@ TEST(daemon_runtime_connection_cap_covers_slow_hello_and_stopping_is_terminal) { if (accepted) { overflow = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity, RUNTIME_TEST_TIMEOUT_MS, &overflow_result); - /* The "capacity" substring in the rejection message proves the reason. - * active_connections is NOT sampled here: on a loaded runner the - * server's HELLO timeout can fire for slow_hello between the raw - * connect and this read, making the count 1 instead of 2 — a - * scheduling race, not a product bug (see db91b88c for precedent). - * active_clients == 1 is safe because we still hold accepted. */ capacity_rejected = overflow == NULL && overflow_result.status == CBM_DAEMON_RUNTIME_CONNECT_REJECTED && strstr(overflow_result.message, "capacity") != NULL && + cbm_daemon_runtime_service_active_connections(fixture.service) == 2 && cbm_daemon_runtime_service_active_clients(fixture.service) == 1; cbm_daemon_ipc_connection_close(slow_hello); From 5c9099f015bc65d67a9bfcac7156e50acc1b2f42 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 20 Oct 2026 19:29:47 +0200 Subject: [PATCH 12/15] fix(pipeline): keep PHP trees in the sequential cache, release on abort Two follow-ups to dropping parse trees from the sequential definitions cache, both for code main gained after this branch was cut: - php_import_kind_at_site (pass_pkgmap.c) reads a PHP call or usage site's syntax from the cached tree to tell a class site from a function or constant site. The sequential route retains no source, so without the tree every site was "unknown" and a same-named vendor `use function` or `use const` import vetoed the site's edge to the project class: `new Thing()`, `Thing $t` and `extends Thing` lost 2 CALLS and 3 USAGE edges on a three-file fixture. PHP results keep their tree; every other language still drops it. - The abort for configured test declarations (configured_failure) returned without cbm_work_arena_release, so the working arena that compaction hands back to the thread stayed held after the pass. Each new test fails without its change: ei_php_vendor_function_import_keeps_same_named_class_sites and sequential_definitions_configured_failure_releases_work_arena. The cache test now also requires PHP results to keep their tree. Signed-off-by: Martin Vogel --- src/pipeline/pass_definitions.c | 10 ++++- tests/test_edge_imports.c | 40 ++++++++++++++++++++ tests/test_parallel.c | 67 ++++++++++++++++++++++++++++++++- 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 4d544fc7f..a3ab79635 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -1136,8 +1136,13 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t if (local_cache) { /* The cache lives through every later pass: hold the result as the * parallel extract does (pass_parallel.c), without its tree (the - * cross-file LSP re-parses) and compacted. */ - cbm_free_tree(result); + * cross-file LSP re-parses) and compacted. A PHP result keeps its + * tree: the vendor-import check reads each call and usage site's + * syntax from it (php_import_kind_at_site, pass_pkgmap.c), and this + * route keeps no source to fall back on. */ + if (lang != CBM_LANG_PHP) { + cbm_free_tree(result); + } cbm_result_compact(result); local_cache[i] = result; } else { @@ -1214,5 +1219,6 @@ configured_failure: } cbm_free(CBM_MEM_CLASS_EXTRACT, local_cache); } + cbm_work_arena_release(); /* compaction keeps it between files: release on every exit */ return CBM_PIPELINE_ABORT_PRESERVE_DB; } diff --git a/tests/test_edge_imports.c b/tests/test_edge_imports.c index b5f1e7fbf..b38c8ae28 100644 --- a/tests/test_edge_imports.c +++ b/tests/test_edge_imports.c @@ -2009,6 +2009,45 @@ TEST(ei_php_vendor_types_never_bind_by_name_issue1186) { PASS(); } +/* A vendor `use function` import shares its local name with a project class the + * file also imports. `new Thing()`, a `Thing $t` parameter and `extends Thing` + * name the class, so the function import must not veto their edges. The check + * reads each site's syntax from the parse tree, which the sequential + * definitions pass (a repo of 50 files or fewer) keeps for PHP results. */ +TEST(ei_php_vendor_function_import_keeps_same_named_class_sites) { + static const EILangFile f[] = { + {"composer.json", "{\"autoload\":{\"psr-4\":{\"App\\\\\":\"app/\"}}}\n"}, + {"app/Models/Thing.php", "go();\n" + " }\n}\n"}}; + static const char *const uses = "app/Http/UsesThing.php"; + static const char *const thing = "app/Models/Thing.php"; + EILangProj lp; + cbm_store_t *store = ei_index_files(&lp, f, 3); + int calls = store ? ei_php_checked_file_edges(store, lp.project, uses, "CALLS", thing) : -1; + int usages = store ? ei_php_checked_file_edges(store, lp.project, uses, "USAGE", thing) : -1; + int inherits = + store ? ei_php_checked_file_edges(store, lp.project, uses, "INHERITS", thing) : -1; + ei_cleanup(&lp, store); + if (calls != 2 || usages != 2 || inherits != 1) { + fprintf(stderr, + " UsesThing.php -> Thing.php: CALLS %d (want 2), USAGE %d (want 2), " + "INHERITS %d (want 1)\n", + calls, usages, inherits); + } + ASSERT_EQ(calls, 2); /* new Thing() and $made->go() */ + ASSERT_EQ(usages, 2); /* extends Thing and the Thing $t parameter */ + ASSERT_EQ(inherits, 1); /* never vetoed: control */ + PASS(); +} + /* Build the metadata through the production writer, then challenge its * readback. A File node or an empty edge query alone is never completeness. */ static bool ei_php_seed_result(cbm_gbuf_t *gb, const char *project, const char *rel, @@ -2393,6 +2432,7 @@ SUITE(edge_imports) { RUN_TEST(ei_php_null_namespace_map_keeps_php_candidate_issue1186); RUN_TEST(ei_php_require_path_keeps_generic_resolution_issue1186); RUN_TEST(ei_php_vendor_types_never_bind_by_name_issue1186); + RUN_TEST(ei_php_vendor_function_import_keeps_same_named_class_sites); RUN_TEST(ei_php_binding_snapshot_requires_current_complete_evidence); RUN_TEST(ei_php_binding_snapshot_groups_kinds_and_invalidates_failed_rebuild); RUN_TEST(ei_php_psr4_missing_class_leaves_type_evidence_unknown); diff --git a/tests/test_parallel.c b/tests/test_parallel.c index 9dad1591c..9183f33b5 100644 --- a/tests/test_parallel.c +++ b/tests/test_parallel.c @@ -775,7 +775,8 @@ TEST(parallel_post_extract_projection_spills_before_resolve) { /* The parallel extract caches each result without its parse tree and * compacted (pass_parallel.c). The sequential definitions pass cached the raw * result, tree and working arena included, and that cache lives through every - * later pass: about 5x the memory per file, for every file of the repo. */ + * later pass: about 5x the memory per file, for every file of the repo. PHP + * keeps its tree: the vendor-import check reads its sites' syntax from it. */ TEST(sequential_definitions_cache_drops_trees_and_compacts) { if (ensure_parity_setup() != 0) FAIL("setup failed"); @@ -806,18 +807,25 @@ TEST(sequential_definitions_cache_drops_trees_and_compacts) { ASSERT_FALSE(cbm_work_arena_keeping()); /* ...and end the pass that way */ int cached = 0; + int php_trees = 0; for (int i = 0; i < file_count; i++) { const CBMFileResult *r = cache[i]; if (!r) { continue; } cached++; - ASSERT_NULL(r->cached_tree); + if (files[i].language == CBM_LANG_PHP) { + ASSERT_NOT_NULL(r->cached_tree); + php_trees++; + } else { + ASSERT_NULL(r->cached_tree); + } /* One exact block: capacity == bytes used, no dead headroom. */ ASSERT_EQ(r->arena.nblocks, 1); ASSERT_EQ(cbm_arena_capacity(&r->arena), cbm_arena_total(&r->arena)); } ASSERT_EQ(cached, file_count); + ASSERT_GT(php_trees, 0); /* app/Models.php and app/Services.php */ for (int i = 0; i < file_count; i++) { cbm_free_result(cache[i]); @@ -829,6 +837,60 @@ TEST(sequential_definitions_cache_drops_trees_and_compacts) { PASS(); } +/* A configured C test convention whose file cannot be extracted aborts the + * sequential pass (CBM_PIPELINE_ABORT_PRESERVE_DB). The Python file compacted + * before it handed its working arena back to this thread, so that exit has to + * release it like the others. */ +TEST(sequential_definitions_configured_failure_releases_work_arena) { + char dir[256]; + snprintf(dir, sizeof(dir), "/tmp/cbm_seqcf_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(dir)); + char py_path[320]; + char c_path[320]; + snprintf(py_path, sizeof(py_path), "%s/a.py", dir); + snprintf(c_path, sizeof(c_path), "%s/z.c", dir); + ASSERT_EQ(th_write_file(py_path, "def first():\n return 1\n"), 0); + ASSERT_EQ(th_write_file(c_path, "void second(void) {}\n"), 0); + cbm_file_info_t files[] = { + {.path = py_path, .rel_path = "a.py", .language = CBM_LANG_PYTHON}, + {.path = c_path, .rel_path = "z.c", .language = CBM_LANG_C}, + }; + static const char config[] = + "{\"test_impact\":{\"version\":1,\"tests\":{\"conventions\":[{\"language\":\"c\"," + "\"role\":\"case\",\"define_macro\":\"CHECK\",\"name_args\":[0]}]}}}"; + cbm_test_declarations_t *declarations = + cbm_test_declarations_parse(config, sizeof(config) - 1, false); + ASSERT_NOT_NULL(declarations); + + cbm_gbuf_t *gbuf = cbm_gbuf_new("seq-cf", dir); + cbm_registry_t *reg = cbm_registry_new(); + atomic_int cancelled; + atomic_init(&cancelled, 0); + cbm_pipeline_ctx_t ctx = { + .project_name = "seq-cf", + .repo_path = dir, + .gbuf = gbuf, + .registry = reg, + .cancelled = &cancelled, + .test_declarations = declarations, + .test_fault_mask = CBM_PIPELINE_TEST_FAULT_EXTRACT_NULL, /* z.c extracts to NULL */ + }; + + cbm_init(); + cbm_work_arena_release(); /* start from a thread that keeps nothing */ + int rc = cbm_pipeline_pass_definitions(&ctx, files, 2); + bool keeping = cbm_work_arena_keeping(); + cbm_work_arena_release(); + + cbm_registry_free(reg); + cbm_gbuf_free(gbuf); + cbm_test_declarations_free(declarations); + rm_rf(dir); + ASSERT_EQ(rc, CBM_PIPELINE_ABORT_PRESERVE_DB); + ASSERT_FALSE(keeping); + PASS(); +} + /* ── Empty file list ──────────────────────────────────────────────── */ TEST(parallel_empty_files) { @@ -5659,6 +5721,7 @@ SUITE(parallel) { RUN_TEST(parallel_spill_mode_builds_the_same_graph); RUN_TEST(parallel_post_extract_projection_spills_before_resolve); RUN_TEST(sequential_definitions_cache_drops_trees_and_compacts); + RUN_TEST(sequential_definitions_configured_failure_releases_work_arena); RUN_TEST(parallel_empty_files); RUN_TEST(parallel_args_json_no_overflow); RUN_TEST(parallel_wide_definition_keeps_fp_sp_bt_f14); From 1a7239f52bfb1f9b77a79d08fa91b0f06e714b8f Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 28 Oct 2026 00:43:12 +0100 Subject: [PATCH 13/15] fix(doc-links): a member the code file writes keeps its file edge (#2587) A `path::member` reference became a missing row whenever the member lookup found no definition node, but real documentation names members the file has without a node the lookup binds: a call form (`src/util.py::clamp()`), a C macro or a prototype in a header, a TypeScript namespace, a Python enum member, an annotated class attribute. Those lost their correct file edge and were counted as doc rot. A member now reads without a trailing call form, so `clamp()` binds `clamp`. When no definition matches, the resolver reads the code file and calls the member missing only when the file does not write its name at all (a whole identifier, case-sensitive); a member the file writes keeps the file edge it had before. A file that cannot be read whole makes no claim. Ambiguous members are unchanged. Test: doc_links_md_member_written_issue2587 indexes each shape under its own heading, and two members that stay missing (Color::BLUE, which the file writes only as LIGHT_BLUE and a lowercase "blue", and no_such_function). On the previous head all seven shapes are missing rows; without the call-form trim the call form is; without the identifier boundary, or with a case-insensitive match, BLUE takes the file edge. Signed-off-by: Martin Vogel --- src/pipeline/doc_links_md.c | 106 ++++++++++++++++++++++++++++++++---- tests/test_doc_links_md.c | 94 ++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 11 deletions(-) diff --git a/src/pipeline/doc_links_md.c b/src/pipeline/doc_links_md.c index 14d6db513..18c47b5cd 100644 --- a/src/pipeline/doc_links_md.c +++ b/src/pipeline/doc_links_md.c @@ -10,15 +10,17 @@ * document says (the field tests' rule for bare file names). * * Into the file: a line range picks the innermost code definition that holds - * the WHOLE range; a member (`path::Name`, `path::Owner::name`) picks the one - * definition of that name in the file. Otherwise the edge goes to the file, - * and the range stays on it ("target_lines"). Every edge is exact: the - * document wrote the path. + * the WHOLE range; a member (`path::Name`, `path::Owner::name`; a call form + * `path::name()` reads as `path::name`) picks the one definition of that name + * in the file. Otherwise the edge goes to the file, and the range stays on it + * ("target_lines"); so does a member the code file writes without a + * definition the graph binds (a macro, a prototype, an enum member, a class + * attribute). Every edge is exact: the document wrote the path. * * Unresolved: * missing the path is not in the graph, but its directory is: the * document names something that is not there (doc rot); or a - * code file's member that the file does not define + * member whose name the code file does not write at all * ambiguous a file name alone that is not next to the document but exists * elsewhere in the repository: the document does not say which; * or a member name the file defines more than once, with no @@ -36,11 +38,14 @@ #include "discover/discover.h" /* cbm_language_for_filename */ #include "doclink.h" #include "foundation/arena.h" +#include "foundation/compat_fs.h" /* cbm_fopen */ #include "foundation/constants.h" #include "foundation/hash_table.h" +#include "foundation/limits.h" /* cbm_max_file_bytes */ #include "foundation/mem_core.h" #include "graph_buffer/graph_buffer.h" +#include #include #include #include @@ -91,6 +96,7 @@ typedef struct { typedef struct { const char *project; size_t project_len; + const char *repo; /* the repository root (NULL: unknown; no file is read) */ const char **run_paths; /* run_file -> rel_path (NULL: not a Markdown file of this run) */ int run_count; /* base name -> number of File nodes with it (a file name alone that is @@ -288,6 +294,7 @@ static void *mdr_build(const cbm_doclink_build_in_t *in) { cbm_arena_init(&x->arena); x->project = in->ctx ? in->ctx->project_name : NULL; x->project_len = x->project ? strlen(x->project) : 0; + x->repo = in->ctx ? in->ctx->repo_path : NULL; x->run_count = in->run_file_count; x->run_paths = in->run_file_count > 0 ? (const char **)cbm_calloc(CBM_MEM_CLASS_OTHER, @@ -420,14 +427,31 @@ static bool mdr_code_label(const char *label) { return false; } +/* The length of the member text as a name: a call form (`name()`, + * `Owner.name(a, b)`) ends before its first '('. */ +static size_t mdr_member_len(const char *member) { + size_t n = strlen(member); + if (n > 0 && member[n - SKIP_ONE] == ')') { + const char *open = memchr(member, '(', n); + if (open) { + return (size_t)(open - member); + } + } + return n; +} + /* The member text's last segment (after "::" or "."), and the one before it. */ static void mdr_member_names(const char *member, const char **name, size_t *name_len, const char **owner, size_t *owner_len) { const char *segs[PAIR_LEN] = {NULL, NULL}; size_t lens[PAIR_LEN] = {0, 0}; const char *s = member; - while (*s) { + const char *end = member + mdr_member_len(member); + while (s < end) { size_t n = strcspn(s, ":."); + if (n > (size_t)(end - s)) { + n = (size_t)(end - s); + } if (n > 0) { segs[0] = segs[SKIP_ONE]; lens[0] = lens[SKIP_ONE]; @@ -435,7 +459,7 @@ static void mdr_member_names(const char *member, const char **name, size_t *name lens[SKIP_ONE] = n; } s += n; - while (*s == ':' || *s == '.') { + while (s < end && (*s == ':' || *s == '.')) { s++; } } @@ -564,6 +588,63 @@ static const cbm_gbuf_node_t *mdr_segment(const mdr_index_t *x, const char *path return NULL; } +/* A byte of an identifier: a letter, a digit, '_', and every byte of a + * multi-byte character (pipeline_incremental.c doc_row_ident_byte). */ +static bool mdr_ident_byte(unsigned char c) { + return isalnum(c) || c == '_' || c >= CBM_SZ_128; +} + +/* Does text[0, n) hold name[0, len) as a whole identifier (case-sensitive)? */ +static bool mdr_writes_ident(const char *text, size_t n, const char *name, size_t len) { + for (size_t i = 0; len > 0 && i + len <= n;) { + const char *hit = memchr(text + i, name[0], n - len - i + SKIP_ONE); + if (!hit) { + return false; + } + size_t at = (size_t)(hit - text); + if (memcmp(hit, name, len) == 0 && (at == 0 || !mdr_ident_byte((unsigned char)hit[-1])) && + (at + len == n || !mdr_ident_byte((unsigned char)hit[len]))) { + return true; + } + i = at + SKIP_ONE; + } + return false; +} + +/* Does the code file `path` write the member's name at all? A name the graph + * holds no definition for (a macro, a prototype, an enum member, a class + * attribute) is still there when the file writes it. Read from the file, + * never cached (resolvers run on worker threads). True -- no claim -- when + * the file cannot be read whole: a member is missing only when the file was + * read and does not write its name. */ +static bool mdr_file_writes(const mdr_index_t *x, const char *path, const char *member) { + const char *name; + const char *owner; + size_t name_len; + size_t owner_len; + mdr_member_names(member, &name, &name_len, &owner, &owner_len); + char abs_path[CBM_PATH_MAX]; + int w = name && x->repo ? snprintf(abs_path, sizeof(abs_path), "%s/%s", x->repo, path) : 0; + if (w <= 0 || (size_t)w >= sizeof(abs_path)) { + return true; + } + FILE *f = cbm_fopen(abs_path, "rb"); + if (!f) { + return true; + } + long size = fseek(f, 0, SEEK_END) == 0 ? ftell(f) : CBM_NOT_FOUND; + char *text = size >= 0 && size <= cbm_max_file_bytes() && fseek(f, 0, SEEK_SET) == 0 + ? (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + SKIP_ONE) + : NULL; + bool writes = true; + if (text && fread(text, SKIP_ONE, (size_t)size, f) == (size_t)size) { + writes = mdr_writes_ident(text, (size_t)size, name, name_len); + } + (void)fclose(f); + cbm_free(CBM_MEM_CLASS_OTHER, text); + return writes; +} + /* Is `path` of a file type the indexer reads? */ static bool mdr_indexable(const char *path) { return cbm_language_for_filename(mdr_base(path)) != CBM_LANG_COUNT; @@ -1181,10 +1262,13 @@ static void mdr_resolve_path(const mdr_index_t *x, const cbm_gbuf_t *graph, cons ? mdr_segment(x, t, r->first, r->last, r->member, &member_result) : NULL; /* A named member that a code file does not define is no coarser - * link to the file: it is a stale or unclear reference. A - * document's own headings are not definitions, so members of - * other files keep the file edge. */ - if (!seg && r->member && member_result != MDR_MEMBER_FOUND && mdr_code_file(t)) { + * link to the file: it is a stale or unclear reference. One the + * file writes without a definition the graph binds (a macro, a + * prototype, an enum member) is there, and keeps the file edge. + * A document's own headings are not definitions, so members of + * other files keep the file edge too. */ + if (!seg && r->member && member_result != MDR_MEMBER_FOUND && mdr_code_file(t) && + (member_result == MDR_MEMBER_AMBIGUOUS || !mdr_file_writes(x, t, r->member))) { out->kind = CBM_DOCLINK_UNRESOLVED; out->reason = member_result == MDR_MEMBER_AMBIGUOUS ? CBM_DOCLINK_REASON_AMBIGUOUS : CBM_DOCLINK_REASON_MISSING; diff --git a/tests/test_doc_links_md.c b/tests/test_doc_links_md.c index 11049e032..4c100356b 100644 --- a/tests/test_doc_links_md.c +++ b/tests/test_doc_links_md.c @@ -455,6 +455,99 @@ TEST(doc_links_md_member_not_defined_issue2587) { PASS(); } +/* MENTIONS edges from the section whose qualified name ends with `section` to + * the File node of `file`. */ +static int md_file_edges(const char *db, const char *section, const char *file) { + char sql[512]; + snprintf(sql, sizeof(sql), + "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id " + "JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' " + "AND s.qualified_name LIKE '%%.%s' AND t.label = 'File' AND t.file_path = '%s'", + section, file); + return dm_count(db, sql); +} + +/* #2587, the members a code file has although the graph defines no node for + * them: a call form, a macro, a prototype, a namespace, an enum member, a + * class attribute. Each is there -- the file writes it -- so it keeps its + * edge; only a name the file does not write at all is missing. */ +static const struct { + const char *section; + const char *raw; + const char *file; + bool missing; +} MD_WRITTEN[] = { + {"CallForm", "src/util.py::clamp()", NULL, false}, + {"Macro", "include/limits.h::MAX_ITEMS", "include/limits.h", false}, + {"FunctionMacro", "include/limits.h::SQUARE", "include/limits.h", false}, + {"Prototype", "include/limits.h::limits_init", "include/limits.h", false}, + {"Namespace", "src/ns.ts::Shapes", "src/ns.ts", false}, + {"EnumMember", "src/color.py::Color::RED", "src/color.py", false}, + {"Attribute", "src/model.py::Model::name", "src/model.py", false}, + /* not written: `blue` and LIGHT_BLUE are no BLUE */ + {"NotWritten", "src/color.py::Color::BLUE", "src/color.py", true}, + {"NotDefined", "src/util.py::no_such_function", "src/util.py", true}, +}; + +TEST(doc_links_md_member_written_issue2587) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char doc[2048] = ""; + size_t used = 0; + for (size_t i = 0; i < sizeof(MD_WRITTEN) / sizeof(MD_WRITTEN[0]); i++) { + used += (size_t)snprintf(doc + used, sizeof(doc) - used, "# %s\n\nSee `%s`.\n\n", + MD_WRITTEN[i].section, MD_WRITTEN[i].raw); + ASSERT(used < sizeof(doc)); + } + th_write_file(TH_PATH(tmp, "docs/written.md"), doc); + th_write_file(TH_PATH(tmp, "src/util.py"), "def clamp(x, lo, hi):\n" + " return max(lo, min(hi, x))\n"); + th_write_file(TH_PATH(tmp, "include/limits.h"), "#ifndef LIMITS_H\n" + "#define LIMITS_H\n" + "#define MAX_ITEMS 64\n" + "#define SQUARE(x) ((x) * (x))\n" + "int limits_init(void);\n" + "#endif\n"); + th_write_file(TH_PATH(tmp, "src/ns.ts"), "export namespace Shapes {\n" + " export const unit = 1;\n" + "}\n"); + th_write_file(TH_PATH(tmp, "src/color.py"), "from enum import Enum\n" + "\n" + "\n" + "class Color(Enum):\n" + " RED = 1\n" + " LIGHT_BLUE = 2 # blue\n"); + th_write_file(TH_PATH(tmp, "src/model.py"), "class Model:\n" + " name: str\n"); + char db[512]; + snprintf(db, sizeof(db), "%s/md.db", tmp); + ASSERT_EQ(dm_index(tmp, db, NULL), 0); + char props[512]; + char reason[64]; + + /* every reference, under its own heading: an edge, or a missing row */ + int wrong = 0; + for (size_t i = 0; i < sizeof(MD_WRITTEN) / sizeof(MD_WRITTEN[0]); i++) { + dm_row(db, "docs/written.md", MD_WRITTEN[i].raw, reason, sizeof(reason), NULL, 0); + int edges = MD_WRITTEN[i].file + ? md_file_edges(db, MD_WRITTEN[i].section, MD_WRITTEN[i].file) + : dm_mentions_from(db, MD_WRITTEN[i].section); + bool ok = MD_WRITTEN[i].missing ? strcmp(reason, "missing") == 0 && edges == 0 + : reason[0] == '\0' && edges >= 1; + if (!ok) { + printf(" %s: row '%s', %d edge(s)\n", MD_WRITTEN[i].raw, reason, edges); + wrong++; + } + } + ASSERT_EQ(wrong, 0); + /* the call form binds the function itself */ + ASSERT_EQ(md_edge(db, "CallForm", "util.clamp", props, sizeof(props)), 1); + + th_cleanup(tmp); + PASS(); +} + /* ── incremental == full ─────────────────────────────────────────── */ TEST(doc_links_md_incremental) { @@ -933,6 +1026,7 @@ SUITE(doc_links_md) { RUN_TEST(doc_links_md_classify_span); RUN_TEST(doc_links_md_resolve); RUN_TEST(doc_links_md_member_not_defined_issue2587); + RUN_TEST(doc_links_md_member_written_issue2587); RUN_TEST(doc_links_md_incremental); RUN_TEST(doc_links_md_adr); RUN_TEST(doc_links_md_adr_label_choice); From 69155bcf6c0e1cf9e7b0bded115f35d33e41ecff Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 28 Oct 2026 00:43:16 +0100 Subject: [PATCH 14/15] fix(incremental): re-resolve documents whose member rows name a changed file (#2587) An unresolved `path::member` row (missing or ambiguous) leads to its file by no edge, so an incremental run never re-resolved the document when that file changed. Two edits then left a row where a full index has an edge: a body edit that writes a missing member's name (no definition added), and a same-named pair of methods shrinking to one (the name stays, so no removed name reaches the rows). The closure now also takes the files whose missing or ambiguous rows with "::" name a changed or deleted file, matched by base name since a reference may be written relative to the document or to the root. When memory runs out, every such row's file is taken. Test: doc_links_md_member_missing_incremental requires incremental == full on the closure-repair route for both edits and for dropping the name again. Without the new closure rule the "member written" step diverges; with it limited to missing rows the "group shrinks" step does. Signed-off-by: Martin Vogel --- src/pipeline/pipeline_incremental.c | 64 ++++++++++++++++++++++++++++- tests/test_doc_links_md.c | 63 ++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 330d36d89..ec1afd9df 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1496,7 +1496,12 @@ static int surface_added_names(const char *stored_json, const char *fresh_json, * change to it is GLOBAL (it sets the global usings of a whole project); * - a REMOVED name can make another file's unresolved reference resolve (an * overload group shrinks) or change its reason: the files whose rows - * mention it re-resolve with the closure. */ + * mention it re-resolve with the closure; + * - an unresolved `path::member` row (missing, ambiguous) depends on the + * member's file as a whole, its text included (a member the file writes + * without a definition is no missing member, doc_links_md.c) and the size + * of a same-named group: the files whose such rows name a changed or + * deleted file, by its base name, re-resolve with the closure. */ /* Add s[0..n) to a heap-keyed name set. false when it cannot be recorded (too * long for a key, or out of memory): callers fail closed. */ @@ -1644,6 +1649,58 @@ static void doc_row_name_dependents(const cbm_doc_link_row_t *rows, int n, } } +/* Is an unresolved row's reason one a member lookup gives (doc_links_md.c)? */ +static bool doc_row_member_reason(const char *reason) { + return reason && (strcmp(reason, "missing") == 0 || strcmp(reason, "ambiguous") == 0); +} + +/* Files whose unresolved `path::member` rows name a member of one of `paths` + * (the changed and deleted files), and no edge leads from such a row to the + * file: a body edit can write or drop a missing member's name without + * changing a definition, and a same-named group can shrink to one without + * removing its name. Matched by the base name before "::", since a + * reference may be written relative to the document or to the root. When + * memory runs out every such row's file is added: re-resolving a file that did + * not need it costs time, keeping a stale row costs correctness. Keys are + * borrowed from `rows`. */ +static void doc_row_member_dependents(const cbm_doc_link_row_t *rows, int n, char *const *paths, + int npaths, CBMHashTable *out_paths) { + if (npaths <= 0) { + return; + } + CBMHashTable *bases = cbm_ht_create(CBM_SZ_64); /* NULL: fail closed, every member row */ + for (int i = 0; bases && i < npaths; i++) { + const char *slash = strrchr(paths[i], '/'); + const char *base = slash ? slash + SKIP_ONE : paths[i]; + cbm_ht_set(bases, base, (void *)base); /* borrowed from `paths` */ + } + for (int i = 0; i < n; i++) { + const char *raw = rows[i].raw; + const char *rel = rows[i].rel_path; + const char *sep = raw ? strstr(raw, "::") : NULL; + if (!sep || !rel || !rel[0] || !doc_row_member_reason(rows[i].reason) || + cbm_ht_get(out_paths, rel)) { + continue; + } + const char *base = sep; + while (base > raw && base[-1] != '/') { + base--; + } + char key[CBM_SZ_512]; + size_t bl = (size_t)(sep - base); + bool hit = !bases || bl >= sizeof(key); + if (!hit && bl > 0) { + memcpy(key, base, bl); + key[bl] = '\0'; + hit = cbm_ht_get(bases, key) != NULL; + } + if (hit) { + cbm_ht_set(out_paths, rel, (void *)rel); + } + } + cbm_ht_free(bases); +} + /* Doc-link carry-forward state of the legacy partial route: the previous * rows, the stored scopes of the files it does not re-extract, and the set * of re-extracted or deleted paths whose old rows it replaces. */ @@ -2652,8 +2709,11 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p } } /* Files whose unresolved doc-link references name something a changed or - * deleted file no longer declares re-resolve too. */ + * deleted file no longer declares re-resolve too, and so do the files + * whose unresolved `path::member` rows name a changed or deleted file. */ doc_row_name_dependents(plan->doc_rows, plan->doc_row_count, removed_names, row_deps); + doc_row_member_dependents(plan->doc_rows, plan->doc_row_count, changed_paths, + n_changed + n_deleted, row_deps); { row_dep_walk_t walk = {.closure = closure_set, .files = files_by_path, .added = 0}; cbm_ht_foreach(row_deps, row_dep_visitor, &walk); diff --git a/tests/test_doc_links_md.c b/tests/test_doc_links_md.c index 4c100356b..0eb9b000f 100644 --- a/tests/test_doc_links_md.c +++ b/tests/test_doc_links_md.c @@ -604,6 +604,68 @@ TEST(doc_links_md_incremental) { PASS(); } +/* #2587 incremental == full: whether a `path::member` row is missing depends + * on the member file's text, and the row leads to that file by no edge. A + * body edit that writes the name (no definition added) turns the row into a + * file edge, one that drops it turns the edge back into the row; a same-named + * group that shrinks to one turns an ambiguous row into an edge. */ +TEST(doc_links_md_member_missing_incremental) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_inc_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char repo[512]; + snprintf(repo, sizeof(repo), "%s/repo", tmp); + th_write_file(TH_PATH(repo, "docs/helpers.md"), "# Helpers\n\nSee `src/tools.py::helper2`.\n"); + th_write_file(TH_PATH(repo, "docs/areas.md"), "# Areas\n\nSee `src/shapes.py::area`.\n"); + th_write_file(TH_PATH(repo, "src/tools.py"), "def helper():\n" + " return 1\n"); + th_write_file(TH_PATH(repo, "src/shapes.py"), "class Circle:\n" + " def area(self):\n" + " return 3\n" + "\n" + "\n" + "class Square:\n" + " def area(self):\n" + " return 4\n"); + char db[512]; + char full_db[512]; + snprintf(db, sizeof(db), "%s/inc.db", tmp); + snprintf(full_db, sizeof(full_db), "%s/full.db", tmp); + ASSERT_EQ(dm_index(repo, db, NULL), 0); + char reason[64]; + dm_row(db, "docs/helpers.md", "src/tools.py::helper2", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "missing"); + dm_row(db, "docs/areas.md", "src/shapes.py::area", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "ambiguous"); + + /* the body writes the name, no definition added: the row becomes an edge */ + th_write_file(TH_PATH(repo, "src/tools.py"), "def helper():\n" + " return 1 # helper2 follows\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "member written", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + ASSERT_EQ(md_file_edges(db, "Helpers", "src/tools.py"), 1); + /* and drops it again: missing again */ + th_write_file(TH_PATH(repo, "src/tools.py"), "def helper():\n" + " return 1\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "member dropped", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), + 0); + dm_row(db, "docs/helpers.md", "src/tools.py::helper2", reason, sizeof(reason), NULL, 0); + ASSERT_STR_EQ(reason, "missing"); + /* one of two `area` methods goes, the name stays: the one left binds */ + th_write_file(TH_PATH(repo, "src/shapes.py"), "class Circle:\n" + " pass\n" + "\n" + "\n" + "class Square:\n" + " def area(self):\n" + " return 4\n"); + ASSERT_EQ(dm_step(repo, db, full_db, "group shrinks", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); + char props[512]; + ASSERT_EQ(md_edge(db, "Areas", "Square.area", props, sizeof(props)), 1); + th_cleanup(tmp); + PASS(); +} + /* ── ADRs ────────────────────────────────────────────────────────── */ static void md_write_adrs(const char *tmp, const char *first_status) { @@ -1028,6 +1090,7 @@ SUITE(doc_links_md) { RUN_TEST(doc_links_md_member_not_defined_issue2587); RUN_TEST(doc_links_md_member_written_issue2587); RUN_TEST(doc_links_md_incremental); + RUN_TEST(doc_links_md_member_missing_incremental); RUN_TEST(doc_links_md_adr); RUN_TEST(doc_links_md_adr_label_choice); RUN_TEST(doc_links_md_adr_reported); From 62965bc0f645df7edbf441ac0f8e61744fb0d7b6 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 28 Oct 2026 05:37:32 +0100 Subject: [PATCH 15/15] test(doc-links): exercise the ambiguous member row on a Ruby group (#2587) The "group shrinks" step of doc_links_md_member_missing_incremental removed one of two Python `area` methods and expected the closure-repair route. Since the Python namespace admission on main (merged into this branch with origin/main), a Python file whose surface changes always takes the full route (closure_decline reason=python_namespace_changed), so the step ran FORCED_FULL on every platform; the Windows leg was simply the first to report it. Nothing here depends on timing: the planner compares content hashes. The group now lives in a Ruby file. Ruby has no doc-link scope, so the lost method reports no removed name and the document re-resolves only through the ambiguous `path::member` rule, which this step exists to bind. (A Rust group was tried too: its scope reports the method as a removed name, so the step passed with the rule removed.) Test: the step takes closure repair with closure=2 (shapes.rb and docs/areas.md). With "ambiguous" dropped from doc_row_member_reason it stays on closure repair with closure=1 and keeps the stale ambiguous row where the full index has the Areas -> Square.area edge. Signed-off-by: Martin Vogel --- tests/test_doc_links_md.c | 40 +++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/tests/test_doc_links_md.c b/tests/test_doc_links_md.c index c2328397f..e93e3669c 100644 --- a/tests/test_doc_links_md.c +++ b/tests/test_doc_links_md.c @@ -608,7 +608,11 @@ TEST(doc_links_md_incremental) { * on the member file's text, and the row leads to that file by no edge. A * body edit that writes the name (no definition added) turns the row into a * file edge, one that drops it turns the edge back into the row; a same-named - * group that shrinks to one turns an ambiguous row into an edge. */ + * group that shrinks to one turns an ambiguous row into an edge. The group is + * Ruby: a Python file whose surface changes takes the full route (the Python + * namespace admission), and a language with a doc-link scope (Rust, Go, ...) + * reports the lost method as a removed name, so neither reaches the rule for + * ambiguous member rows. */ TEST(doc_links_md_member_missing_incremental) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_inc_XXXXXX"); @@ -616,17 +620,20 @@ TEST(doc_links_md_member_missing_incremental) { char repo[512]; snprintf(repo, sizeof(repo), "%s/repo", tmp); th_write_file(TH_PATH(repo, "docs/helpers.md"), "# Helpers\n\nSee `src/tools.py::helper2`.\n"); - th_write_file(TH_PATH(repo, "docs/areas.md"), "# Areas\n\nSee `src/shapes.py::area`.\n"); + th_write_file(TH_PATH(repo, "docs/areas.md"), "# Areas\n\nSee `src/shapes.rb::area`.\n"); th_write_file(TH_PATH(repo, "src/tools.py"), "def helper():\n" " return 1\n"); - th_write_file(TH_PATH(repo, "src/shapes.py"), "class Circle:\n" - " def area(self):\n" - " return 3\n" + th_write_file(TH_PATH(repo, "src/shapes.rb"), "class Circle\n" + " def area\n" + " 3\n" + " end\n" + "end\n" "\n" - "\n" - "class Square:\n" - " def area(self):\n" - " return 4\n"); + "class Square\n" + " def area\n" + " 4\n" + " end\n" + "end\n"); char db[512]; char full_db[512]; snprintf(db, sizeof(db), "%s/inc.db", tmp); @@ -635,7 +642,7 @@ TEST(doc_links_md_member_missing_incremental) { char reason[64]; dm_row(db, "docs/helpers.md", "src/tools.py::helper2", reason, sizeof(reason), NULL, 0); ASSERT_STR_EQ(reason, "missing"); - dm_row(db, "docs/areas.md", "src/shapes.py::area", reason, sizeof(reason), NULL, 0); + dm_row(db, "docs/areas.md", "src/shapes.rb::area", reason, sizeof(reason), NULL, 0); ASSERT_STR_EQ(reason, "ambiguous"); /* the body writes the name, no definition added: the row becomes an edge */ @@ -652,13 +659,14 @@ TEST(doc_links_md_member_missing_incremental) { dm_row(db, "docs/helpers.md", "src/tools.py::helper2", reason, sizeof(reason), NULL, 0); ASSERT_STR_EQ(reason, "missing"); /* one of two `area` methods goes, the name stays: the one left binds */ - th_write_file(TH_PATH(repo, "src/shapes.py"), "class Circle:\n" - " pass\n" + th_write_file(TH_PATH(repo, "src/shapes.rb"), "class Circle\n" + "end\n" "\n" - "\n" - "class Square:\n" - " def area(self):\n" - " return 4\n"); + "class Square\n" + " def area\n" + " 4\n" + " end\n" + "end\n"); ASSERT_EQ(dm_step(repo, db, full_db, "group shrinks", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0); char props[512]; ASSERT_EQ(md_edge(db, "Areas", "Square.area", props, sizeof(props)), 1);